diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index b4dedf5e942ad..187412be30fb1 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -49,8 +49,8 @@ sycl/source/stream.cpp @againull
 
 # Specialization constant
 sycl/include/CL/sycl/detail/sycl_fe_intrins.hpp @kbobrovs
-sycl/include/CL/sycl/detail/spec_constant_impl.hpp @kbobrovs
 sycl/include/CL/sycl/experimental/spec_constant.hpp @kbobrovs
+sycl/source/detail/spec_constant_impl.hpp @kbobrovs
 
 # Program manager
 sycl/source/detail/program_manager @kbobrovs
diff --git a/.github/workflows/clang-format.yml b/.github/workflows/clang-format.yml
index c1b4ea99066fd..3aeabb036ad58 100644
--- a/.github/workflows/clang-format.yml
+++ b/.github/workflows/clang-format.yml
@@ -14,11 +14,11 @@ jobs:
         fetch-depth: 2
 
     - name: Get clang-format first
-      run: sudo apt-get install -yqq clang-format-9
+      run: sudo apt-get install -yqq clang-format-10
 
     - name: Run clang-format for the patch
       run: |
-        git diff -U0 --no-color ${GITHUB_SHA}^1 ${GITHUB_SHA} -- | ./clang/tools/clang-format/clang-format-diff.py -p1 -binary clang-format-9 > ./clang-format.patch
+        git diff -U0 --no-color ${GITHUB_SHA}^1 ${GITHUB_SHA} -- | ./clang/tools/clang-format/clang-format-diff.py -p1 -binary clang-format-10 > ./clang-format.patch
 
     # Add patch with formatting fixes to CI job artifacts
     - uses: actions/upload-artifact@v1
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/NarrowingConversionsCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/NarrowingConversionsCheck.cpp
index aa860b30fe759..1837ccb6002f9 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/NarrowingConversionsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/NarrowingConversionsCheck.cpp
@@ -79,9 +79,8 @@ static QualType getUnqualifiedType(const Expr &E) {
 }
 
 static APValue getConstantExprValue(const ASTContext &Ctx, const Expr &E) {
-  llvm::APSInt IntegerConstant;
-  if (E.isIntegerConstantExpr(IntegerConstant, Ctx))
-    return APValue(IntegerConstant);
+  if (auto IntegerConstant = E.getIntegerConstantExpr(Ctx))
+    return APValue(*IntegerConstant);
   APValue Constant;
   if (Ctx.getLangOpts().CPlusPlus && E.isCXX11ConstantExpr(Ctx, &Constant))
     return Constant;
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsConstantArrayIndexCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsConstantArrayIndexCheck.cpp
index dd0bedd742a40..96b0bb0f9b02d 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsConstantArrayIndexCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsConstantArrayIndexCheck.cpp
@@ -65,9 +65,9 @@ void ProBoundsConstantArrayIndexCheck::check(
   if (IndexExpr->isValueDependent())
     return; // We check in the specialization.
 
-  llvm::APSInt Index;
-  if (!IndexExpr->isIntegerConstantExpr(Index, *Result.Context, nullptr,
-                                        /*isEvaluated=*/true)) {
+  Optional<llvm::APSInt> Index =
+      IndexExpr->getIntegerConstantExpr(*Result.Context);
+  if (!Index) {
     SourceRange BaseRange;
     if (const auto *ArraySubscriptE = dyn_cast<ArraySubscriptExpr>(Matched))
       BaseRange = ArraySubscriptE->getBase()->getSourceRange();
@@ -101,9 +101,9 @@ void ProBoundsConstantArrayIndexCheck::check(
   if (!StdArrayDecl)
     return;
 
-  if (Index.isSigned() && Index.isNegative()) {
+  if (Index->isSigned() && Index->isNegative()) {
     diag(Matched->getExprLoc(), "std::array<> index %0 is negative")
-        << Index.toString(10);
+        << Index->toString(10);
     return;
   }
 
@@ -118,11 +118,11 @@ void ProBoundsConstantArrayIndexCheck::check(
 
   // Get uint64_t values, because different bitwidths would lead to an assertion
   // in APInt::uge.
-  if (Index.getZExtValue() >= ArraySize.getZExtValue()) {
+  if (Index->getZExtValue() >= ArraySize.getZExtValue()) {
     diag(Matched->getExprLoc(),
          "std::array<> index %0 is past the end of the array "
          "(which contains %1 elements)")
-        << Index.toString(10) << ArraySize.toString(10, false);
+        << Index->toString(10) << ArraySize.toString(10, false);
   }
 }
 
diff --git a/clang-tools-extra/clang-tidy/misc/RedundantExpressionCheck.cpp b/clang-tools-extra/clang-tidy/misc/RedundantExpressionCheck.cpp
index aef513a527b54..b84e4d525d8cf 100644
--- a/clang-tools-extra/clang-tidy/misc/RedundantExpressionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/RedundantExpressionCheck.cpp
@@ -500,7 +500,13 @@ static bool retrieveIntegerConstantExpr(const MatchFinder::MatchResult &Result,
                                         const Expr *&ConstExpr) {
   std::string CstId = (Id + "-const").str();
   ConstExpr = Result.Nodes.getNodeAs<Expr>(CstId);
-  return ConstExpr && ConstExpr->isIntegerConstantExpr(Value, *Result.Context);
+  if (!ConstExpr)
+    return false;
+  Optional<llvm::APSInt> R = ConstExpr->getIntegerConstantExpr(*Result.Context);
+  if (!R)
+    return false;
+  Value = *R;
+  return true;
 }
 
 // Overloaded `retrieveIntegerConstantExpr` for compatibility.
@@ -673,7 +679,7 @@ static bool retrieveRelationalIntegerConstantExpr(
 
     if (const auto *Arg = OverloadedOperatorExpr->getArg(1)) {
       if (!Arg->isValueDependent() &&
-          !Arg->isIntegerConstantExpr(Value, *Result.Context))
+          !Arg->isIntegerConstantExpr(*Result.Context))
         return false;
     }
     Symbol = OverloadedOperatorExpr->getArg(0);
@@ -1265,21 +1271,23 @@ void RedundantExpressionCheck::check(const MatchFinder::MatchResult &Result) {
           "left-right-shift-confusion")) {
     const auto *ShiftingConst = Result.Nodes.getNodeAs<Expr>("shift-const");
     assert(ShiftingConst && "Expr* 'ShiftingConst' is nullptr!");
-    APSInt ShiftingValue;
+    Optional<llvm::APSInt> ShiftingValue =
+        ShiftingConst->getIntegerConstantExpr(*Result.Context);
 
-    if (!ShiftingConst->isIntegerConstantExpr(ShiftingValue, *Result.Context))
+    if (!ShiftingValue)
       return;
 
     const auto *AndConst = Result.Nodes.getNodeAs<Expr>("and-const");
     assert(AndConst && "Expr* 'AndCont' is nullptr!");
-    APSInt AndValue;
-    if (!AndConst->isIntegerConstantExpr(AndValue, *Result.Context))
+    Optional<llvm::APSInt> AndValue =
+        AndConst->getIntegerConstantExpr(*Result.Context);
+    if (!AndValue)
       return;
 
     // If ShiftingConst is shifted left with more bits than the position of the
     // leftmost 1 in the bit representation of AndValue, AndConstant is
     // ineffective.
-    if (AndValue.getActiveBits() > ShiftingValue)
+    if (AndValue->getActiveBits() > *ShiftingValue)
       return;
 
     auto Diag = diag(BinaryAndExpr->getOperatorLoc(),
diff --git a/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.cpp b/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.cpp
index 56d4cceb6002a..c20472c8de593 100644
--- a/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.cpp
@@ -438,11 +438,12 @@ static bool arrayMatchesBoundExpr(ASTContext *Context,
       Context->getAsConstantArrayType(ArrayType);
   if (!ConstType)
     return false;
-  llvm::APSInt ConditionSize;
-  if (!ConditionExpr->isIntegerConstantExpr(ConditionSize, *Context))
+  Optional<llvm::APSInt> ConditionSize =
+      ConditionExpr->getIntegerConstantExpr(*Context);
+  if (!ConditionSize)
     return false;
   llvm::APSInt ArraySize(ConstType->getSize());
-  return llvm::APSInt::isSameValue(ConditionSize, ArraySize);
+  return llvm::APSInt::isSameValue(*ConditionSize, ArraySize);
 }
 
 ForLoopIndexUseVisitor::ForLoopIndexUseVisitor(ASTContext *Context,
diff --git a/clang-tools-extra/clangd/FindTarget.cpp b/clang-tools-extra/clangd/FindTarget.cpp
index 627f40c854368..c6022b2463e85 100644
--- a/clang-tools-extra/clangd/FindTarget.cpp
+++ b/clang-tools-extra/clangd/FindTarget.cpp
@@ -59,24 +59,32 @@ nodeToString(const ast_type_traits::DynTypedNode &N) {
 }
 
 // Helper function for getMembersReferencedViaDependentName()
-// which takes a dependent type `T` and heuristically
+// which takes a possibly-dependent type `T` and heuristically
 // resolves it to a CXXRecordDecl in which we can try name lookup.
 CXXRecordDecl *resolveTypeToRecordDecl(const Type *T) {
   assert(T);
-  if (const auto *ICNT = T->getAs<InjectedClassNameType>()) {
+
+  if (const auto *RT = T->getAs<RecordType>())
+    return dyn_cast<CXXRecordDecl>(RT->getDecl());
+
+  if (const auto *ICNT = T->getAs<InjectedClassNameType>())
     T = ICNT->getInjectedSpecializationType().getTypePtrOrNull();
-  }
+  if (!T)
+    return nullptr;
+
   const auto *TST = T->getAs<TemplateSpecializationType>();
   if (!TST)
     return nullptr;
+
   const ClassTemplateDecl *TD = dyn_cast_or_null<ClassTemplateDecl>(
       TST->getTemplateName().getAsTemplateDecl());
   if (!TD)
     return nullptr;
+
   return TD->getTemplatedDecl();
 }
 
-// Given a dependent type and a member name, heuristically resolve the
+// Given a tag-decl type and a member name, heuristically resolve the
 // name to one or more declarations.
 // The current heuristic is simply to look up the name in the primary
 // template. This is a heuristic because the template could potentially
@@ -154,6 +162,10 @@ const Type *getPointeeType(const Type *T) {
   return FirstArg.getAsType().getTypePtrOrNull();
 }
 
+// Forward declaration, needed as this function is mutually recursive
+// with resolveDependentExprToDecls.
+const Type *resolveDependentExprToType(const Expr *E);
+
 // Try to heuristically resolve a dependent expression `E` to one
 // or more declarations that it likely references.
 std::vector<const NamedDecl *> resolveDependentExprToDecls(const Expr *E) {
@@ -163,6 +175,17 @@ std::vector<const NamedDecl *> resolveDependentExprToDecls(const Expr *E) {
     if (ME->isArrow()) {
       BaseType = getPointeeType(BaseType);
     }
+    if (!BaseType)
+      return {};
+    if (const auto *BT = BaseType->getAs<BuiltinType>()) {
+      // If BaseType is the type of a dependent expression, it's just
+      // represented as BultinType::Dependent which gives us no information. We
+      // can get further by analyzing the depedent expression.
+      Expr *Base = ME->isImplicitAccess() ? nullptr : ME->getBase();
+      if (Base && BT->getKind() == BuiltinType::Dependent) {
+        BaseType = resolveDependentExprToType(Base);
+      }
+    }
     return getMembersReferencedViaDependentName(
         BaseType, [ME](ASTContext &) { return ME->getMember(); },
         /*IsNonstaticMember=*/true);
@@ -173,9 +196,38 @@ std::vector<const NamedDecl *> resolveDependentExprToDecls(const Expr *E) {
         [RE](ASTContext &) { return RE->getDeclName(); },
         /*IsNonstaticMember=*/false);
   }
+  if (const auto *CE = dyn_cast<CallExpr>(E)) {
+    const auto *CalleeType = resolveDependentExprToType(CE->getCallee());
+    if (!CalleeType)
+      return {};
+    if (const auto *FnTypePtr = CalleeType->getAs<PointerType>())
+      CalleeType = FnTypePtr->getPointeeType().getTypePtr();
+    if (const FunctionType *FnType = CalleeType->getAs<FunctionType>()) {
+      if (const auto *D =
+              resolveTypeToRecordDecl(FnType->getReturnType().getTypePtr())) {
+        return {D};
+      }
+    }
+  }
+  if (const auto *ME = dyn_cast<MemberExpr>(E)) {
+    return {ME->getMemberDecl()};
+  }
   return {};
 }
 
+// Try to heuristically resolve the type of a dependent expression `E`.
+const Type *resolveDependentExprToType(const Expr *E) {
+  std::vector<const NamedDecl *> Decls = resolveDependentExprToDecls(E);
+  if (Decls.size() != 1) // Names an overload set -- just bail.
+    return nullptr;
+  if (const auto *TD = dyn_cast<TypeDecl>(Decls[0])) {
+    return TD->getTypeForDecl();
+  } else if (const auto *VD = dyn_cast<ValueDecl>(Decls[0])) {
+    return VD->getType().getTypePtrOrNull();
+  }
+  return nullptr;
+}
+
 const NamedDecl *getTemplatePattern(const NamedDecl *D) {
   if (const CXXRecordDecl *CRD = dyn_cast<CXXRecordDecl>(D)) {
     if (const auto *Result = CRD->getTemplateInstantiationPattern())
@@ -235,9 +287,8 @@ const NamedDecl *getTemplatePattern(const NamedDecl *D) {
 //    and both are lossy. We must know upfront what the caller ultimately wants.
 //
 // FIXME: improve common dependent scope using name lookup in primary templates.
-// e.g. template<typename T> int foo() { return std::vector<T>().size(); }
-// formally size() is unresolved, but the primary template is a good guess.
-// This affects:
+// We currently handle DependentScopeDeclRefExpr and
+// CXXDependentScopeMemberExpr, but some other constructs remain to be handled:
 //  - DependentTemplateSpecializationType,
 //  - DependentNameType
 //  - UnresolvedUsingValueDecl
@@ -302,6 +353,8 @@ struct TargetFinder {
       // Record the underlying decl instead, if allowed.
       D = USD->getTargetDecl();
       Flags |= Rel::Underlying; // continue with the underlying decl.
+    } else if (const auto *DG = dyn_cast<CXXDeductionGuideDecl>(D)) {
+      D = DG->getDeducedTemplate();
     }
 
     if (const Decl *Pat = getTemplatePattern(D)) {
@@ -659,6 +712,15 @@ llvm::SmallVector<ReferenceLoc, 2> refInDecl(const Decl *D) {
                                   /*IsDecl=*/true,
                                   {ND}});
     }
+
+    void VisitCXXDeductionGuideDecl(const CXXDeductionGuideDecl *DG) {
+      // The class template name in a deduction guide targets the class
+      // template.
+      Refs.push_back(ReferenceLoc{DG->getQualifierLoc(),
+                                  DG->getNameInfo().getLoc(),
+                                  /*IsDecl=*/false,
+                                  {DG->getDeducedTemplate()}});
+    }
   };
 
   Visitor V;
diff --git a/clang-tools-extra/clangd/Selection.cpp b/clang-tools-extra/clangd/Selection.cpp
index e94a3ca5a0c38..2e3c491d561e7 100644
--- a/clang-tools-extra/clangd/Selection.cpp
+++ b/clang-tools-extra/clangd/Selection.cpp
@@ -220,14 +220,26 @@ class SelectionTester {
         SelFirst, AllSpelledTokens.end(), [&](const syntax::Token &Tok) {
           return SM.getFileOffset(Tok.location()) < SelEnd;
         });
+    auto Sel = llvm::makeArrayRef(SelFirst, SelLimit);
+    // Find which of these are preprocessed to nothing and should be ignored.
+    std::vector<bool> PPIgnored(Sel.size(), false);
+    for (const syntax::TokenBuffer::Expansion &X :
+         Buf.expansionsOverlapping(Sel)) {
+      if (X.Expanded.empty()) {
+        for (const syntax::Token &Tok : X.Spelled) {
+          if (&Tok >= SelFirst && &Tok < SelLimit)
+            PPIgnored[&Tok - SelFirst] = true;
+        }
+      }
+    }
     // Precompute selectedness and offset for selected spelled tokens.
-    for (const syntax::Token *T = SelFirst; T < SelLimit; ++T) {
-      if (shouldIgnore(*T))
+    for (unsigned I = 0; I < Sel.size(); ++I) {
+      if (shouldIgnore(Sel[I]) || PPIgnored[I])
         continue;
       SpelledTokens.emplace_back();
       Tok &S = SpelledTokens.back();
-      S.Offset = SM.getFileOffset(T->location());
-      if (S.Offset >= SelBegin && S.Offset + T->length() <= SelEnd)
+      S.Offset = SM.getFileOffset(Sel[I].location());
+      if (S.Offset >= SelBegin && S.Offset + Sel[I].length() <= SelEnd)
         S.Selected = SelectionTree::Complete;
       else
         S.Selected = SelectionTree::Partial;
diff --git a/clang-tools-extra/clangd/URI.cpp b/clang-tools-extra/clangd/URI.cpp
index 2061a5601c58f..fad93143a30dd 100644
--- a/clang-tools-extra/clangd/URI.cpp
+++ b/clang-tools-extra/clangd/URI.cpp
@@ -26,6 +26,15 @@ inline llvm::Error make_string_error(const llvm::Twine &Message) {
                                              llvm::inconvertibleErrorCode());
 }
 
+bool isWindowsPath(llvm::StringRef Path) {
+  return Path.size() > 1 && llvm::isAlpha(Path[0]) && Path[1] == ':';
+}
+
+bool isNetworkPath(llvm::StringRef Path) {
+  return Path.size() > 2 && Path[0] == Path[1] &&
+         llvm::sys::path::is_separator(Path[0]);
+}
+
 /// This manages file paths in the file system. All paths in the scheme
 /// are absolute (with leading '/').
 /// Note that this scheme is hardcoded into the library and not registered in
@@ -33,28 +42,40 @@ inline llvm::Error make_string_error(const llvm::Twine &Message) {
 class FileSystemScheme : public URIScheme {
 public:
   llvm::Expected<std::string>
-  getAbsolutePath(llvm::StringRef /*Authority*/, llvm::StringRef Body,
+  getAbsolutePath(llvm::StringRef Authority, llvm::StringRef Body,
                   llvm::StringRef /*HintPath*/) const override {
     if (!Body.startswith("/"))
       return make_string_error("File scheme: expect body to be an absolute "
                                "path starting with '/': " +
                                Body);
-    // For Windows paths e.g. /X:
-    if (Body.size() > 2 && Body[0] == '/' && Body[2] == ':')
+    llvm::SmallString<128> Path;
+    if (!Authority.empty()) {
+      // Windows UNC paths e.g. file://server/share => \\server\share
+      ("//" + Authority).toVector(Path);
+    } else if (isWindowsPath(Body.substr(1))) {
+      // Windows paths e.g. file:///X:/path => X:\path
       Body.consume_front("/");
-    llvm::SmallVector<char, 16> Path(Body.begin(), Body.end());
+    }
+    Path.append(Body);
     llvm::sys::path::native(Path);
-    return std::string(Path.begin(), Path.end());
+    return std::string(Path);
   }
 
   llvm::Expected<URI>
   uriFromAbsolutePath(llvm::StringRef AbsolutePath) const override {
     std::string Body;
-    // For Windows paths e.g. X:
-    if (AbsolutePath.size() > 1 && AbsolutePath[1] == ':')
+    llvm::StringRef Authority;
+    llvm::StringRef Root = llvm::sys::path::root_name(AbsolutePath);
+    if (isNetworkPath(Root)) {
+      // Windows UNC paths e.g. \\server\share => file://server/share
+      Authority = Root.drop_front(2);
+      AbsolutePath.consume_front(Root);
+    } else if (isWindowsPath(Root)) {
+      // Windows paths e.g. X:\path => file:///X:/path
       Body = "/";
+    }
     Body += llvm::sys::path::convert_to_slash(AbsolutePath);
-    return URI("file", /*Authority=*/"", Body);
+    return URI("file", Authority, Body);
   }
 };
 
@@ -96,13 +117,13 @@ bool shouldEscape(unsigned char C) {
 void percentEncode(llvm::StringRef Content, std::string &Out) {
   std::string Result;
   for (unsigned char C : Content)
-    if (shouldEscape(C))
-    {
+    if (shouldEscape(C)) {
       Out.push_back('%');
       Out.push_back(llvm::hexdigit(C / 16));
       Out.push_back(llvm::hexdigit(C % 16));
-    } else
-    { Out.push_back(C); }
+    } else {
+      Out.push_back(C);
+    }
 }
 
 /// Decodes a string according to percent-encoding.
diff --git a/clang-tools-extra/clangd/XRefs.cpp b/clang-tools-extra/clangd/XRefs.cpp
index c208e953f2ab7..aeff7ebc32a26 100644
--- a/clang-tools-extra/clangd/XRefs.cpp
+++ b/clang-tools-extra/clangd/XRefs.cpp
@@ -1183,23 +1183,24 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const LocatedSymbol &S) {
 
 // FIXME(nridge): Reduce duplication between this function and declToSym().
 static llvm::Optional<TypeHierarchyItem>
-declToTypeHierarchyItem(ASTContext &Ctx, const NamedDecl &ND,
-                        const syntax::TokenBuffer &TB) {
+declToTypeHierarchyItem(ASTContext &Ctx, const NamedDecl &ND) {
   auto &SM = Ctx.getSourceManager();
   SourceLocation NameLoc = nameLocation(ND, Ctx.getSourceManager());
+  SourceLocation BeginLoc = SM.getSpellingLoc(SM.getFileLoc(ND.getBeginLoc()));
+  SourceLocation EndLoc = SM.getSpellingLoc(SM.getFileLoc(ND.getEndLoc()));
+  const auto DeclRange =
+      toHalfOpenFileRange(SM, Ctx.getLangOpts(), {BeginLoc, EndLoc});
+  if (!DeclRange)
+    return llvm::None;
   auto FilePath =
       getCanonicalPath(SM.getFileEntryForID(SM.getFileID(NameLoc)), SM);
   auto TUPath = getCanonicalPath(SM.getFileEntryForID(SM.getMainFileID()), SM);
   if (!FilePath || !TUPath)
     return llvm::None; // Not useful without a uri.
 
-  auto DeclToks = TB.spelledForExpanded(TB.expandedTokens(ND.getSourceRange()));
-  if (!DeclToks || DeclToks->empty())
-    return llvm::None;
-
-  auto NameToks = TB.spelledForExpanded(TB.expandedTokens(NameLoc));
-  if (!NameToks || NameToks->empty())
-    return llvm::None;
+  Position NameBegin = sourceLocToPosition(SM, NameLoc);
+  Position NameEnd = sourceLocToPosition(
+      SM, Lexer::getLocForEndOfToken(NameLoc, 0, SM, Ctx.getLangOpts()));
 
   index::SymbolInfo SymInfo = index::getSymbolInfo(&ND);
   // FIXME: this is not classifying constructors, destructors and operators
@@ -1210,12 +1211,9 @@ declToTypeHierarchyItem(ASTContext &Ctx, const NamedDecl &ND,
   THI.name = printName(Ctx, ND);
   THI.kind = SK;
   THI.deprecated = ND.isDeprecated();
-  THI.range = halfOpenToRange(
-      SM, syntax::Token::range(SM, DeclToks->front(), DeclToks->back())
-              .toCharRange(SM));
-  THI.selectionRange = halfOpenToRange(
-      SM, syntax::Token::range(SM, NameToks->front(), NameToks->back())
-              .toCharRange(SM));
+  THI.range = Range{sourceLocToPosition(SM, DeclRange->getBegin()),
+                    sourceLocToPosition(SM, DeclRange->getEnd())};
+  THI.selectionRange = Range{NameBegin, NameEnd};
   if (!THI.range.contains(THI.selectionRange)) {
     // 'selectionRange' must be contained in 'range', so in cases where clang
     // reports unrelated ranges we need to reconcile somehow.
@@ -1282,8 +1280,7 @@ using RecursionProtectionSet = llvm::SmallSet<const CXXRecordDecl *, 4>;
 
 static void fillSuperTypes(const CXXRecordDecl &CXXRD, ASTContext &ASTCtx,
                            std::vector<TypeHierarchyItem> &SuperTypes,
-                           RecursionProtectionSet &RPSet,
-                           const syntax::TokenBuffer &TB) {
+                           RecursionProtectionSet &RPSet) {
   // typeParents() will replace dependent template specializations
   // with their class template, so to avoid infinite recursion for
   // certain types of hierarchies, keep the templates encountered
@@ -1298,9 +1295,9 @@ static void fillSuperTypes(const CXXRecordDecl &CXXRD, ASTContext &ASTCtx,
 
   for (const CXXRecordDecl *ParentDecl : typeParents(&CXXRD)) {
     if (Optional<TypeHierarchyItem> ParentSym =
-            declToTypeHierarchyItem(ASTCtx, *ParentDecl, TB)) {
+            declToTypeHierarchyItem(ASTCtx, *ParentDecl)) {
       ParentSym->parents.emplace();
-      fillSuperTypes(*ParentDecl, ASTCtx, *ParentSym->parents, RPSet, TB);
+      fillSuperTypes(*ParentDecl, ASTCtx, *ParentSym->parents, RPSet);
       SuperTypes.emplace_back(std::move(*ParentSym));
     }
   }
@@ -1404,7 +1401,7 @@ getTypeHierarchy(ParsedAST &AST, Position Pos, int ResolveLevels,
     return llvm::None;
 
   Optional<TypeHierarchyItem> Result =
-      declToTypeHierarchyItem(AST.getASTContext(), *CXXRD, AST.getTokens());
+      declToTypeHierarchyItem(AST.getASTContext(), *CXXRD);
   if (!Result)
     return Result;
 
@@ -1413,8 +1410,7 @@ getTypeHierarchy(ParsedAST &AST, Position Pos, int ResolveLevels,
     Result->parents.emplace();
 
     RecursionProtectionSet RPSet;
-    fillSuperTypes(*CXXRD, AST.getASTContext(), *Result->parents, RPSet,
-                   AST.getTokens());
+    fillSuperTypes(*CXXRD, AST.getASTContext(), *Result->parents, RPSet);
   }
 
   if ((Direction == TypeHierarchyDirection::Children ||
diff --git a/clang-tools-extra/clangd/index/SymbolCollector.cpp b/clang-tools-extra/clangd/index/SymbolCollector.cpp
index b502dfb03aec3..6c11399c87b68 100644
--- a/clang-tools-extra/clangd/index/SymbolCollector.cpp
+++ b/clang-tools-extra/clangd/index/SymbolCollector.cpp
@@ -373,16 +373,12 @@ bool SymbolCollector::handleMacroOccurrence(const IdentifierInfo *Name,
                                             index::SymbolRoleSet Roles,
                                             SourceLocation Loc) {
   assert(PP.get());
-
-  const auto &SM = PP->getSourceManager();
-  auto DefLoc = MI->getDefinitionLoc();
-  auto SpellingLoc = SM.getSpellingLoc(Loc);
-  bool IsMainFileSymbol = SM.isInMainFile(SM.getExpansionLoc(DefLoc));
-
   // Builtin macros don't have useful locations and aren't needed in completion.
   if (MI->isBuiltinMacro())
     return true;
 
+  const auto &SM = PP->getSourceManager();
+  auto DefLoc = MI->getDefinitionLoc();
   // Also avoid storing predefined macros like __DBL_MIN__.
   if (SM.isWrittenInBuiltinFile(DefLoc))
     return true;
@@ -391,8 +387,13 @@ bool SymbolCollector::handleMacroOccurrence(const IdentifierInfo *Name,
   if (!ID)
     return true;
 
+  auto SpellingLoc = SM.getSpellingLoc(Loc);
+  bool IsMainFileOnly =
+      SM.isInMainFile(SM.getExpansionLoc(DefLoc)) &&
+      !isHeaderFile(SM.getFileEntryForID(SM.getMainFileID())->getName(),
+                    ASTCtx->getLangOpts());
   // Do not store references to main-file macros.
-  if ((static_cast<unsigned>(Opts.RefFilter) & Roles) && !IsMainFileSymbol &&
+  if ((static_cast<unsigned>(Opts.RefFilter) & Roles) && !IsMainFileOnly &&
       (Opts.RefsInHeaders || SM.getFileID(SpellingLoc) == SM.getMainFileID()))
     MacroRefs[*ID].push_back({Loc, Roles});
 
@@ -401,7 +402,7 @@ bool SymbolCollector::handleMacroOccurrence(const IdentifierInfo *Name,
     return true;
 
   // Skip main-file macros if we are not collecting them.
-  if (IsMainFileSymbol && !Opts.CollectMainFileSymbols)
+  if (IsMainFileOnly && !Opts.CollectMainFileSymbols)
     return false;
 
   // Mark the macro as referenced if this is a reference coming from the main
@@ -425,11 +426,12 @@ bool SymbolCollector::handleMacroOccurrence(const IdentifierInfo *Name,
   Symbol S;
   S.ID = std::move(*ID);
   S.Name = Name->getName();
-  if (!IsMainFileSymbol) {
+  if (!IsMainFileOnly) {
     S.Flags |= Symbol::IndexedForCodeCompletion;
     S.Flags |= Symbol::VisibleOutsideFile;
   }
   S.SymInfo = index::getSymbolInfoForMacro(*MI);
+  S.Origin = Opts.Origin;
   std::string FileURI;
   // FIXME: use the result to filter out symbols.
   shouldIndexFile(SM.getFileID(Loc));
diff --git a/clang-tools-extra/clangd/index/remote/Client.cpp b/clang-tools-extra/clangd/index/remote/Client.cpp
index b468831930765..35ce84068f406 100644
--- a/clang-tools-extra/clangd/index/remote/Client.cpp
+++ b/clang-tools-extra/clangd/index/remote/Client.cpp
@@ -29,16 +29,6 @@ class IndexClient : public clangd::SymbolIndex {
   using StreamingCall = std::unique_ptr<grpc::ClientReader<ReplyT>> (
       remote::SymbolIndex::Stub::*)(grpc::ClientContext *, const RequestT &);
 
-  template <typename ClangdRequestT, typename RequestT>
-  RequestT serializeRequest(ClangdRequestT Request) const {
-    return toProtobuf(Request);
-  }
-
-  template <>
-  FuzzyFindRequest serializeRequest(clangd::FuzzyFindRequest Request) const {
-    return toProtobuf(Request, ProjectRoot);
-  }
-
   template <typename RequestT, typename ReplyT, typename ClangdRequestT,
             typename CallbackT>
   bool streamRPC(ClangdRequestT Request,
@@ -46,24 +36,23 @@ class IndexClient : public clangd::SymbolIndex {
                  CallbackT Callback) const {
     bool FinalResult = false;
     trace::Span Tracer(RequestT::descriptor()->name());
-    const auto RPCRequest = serializeRequest<ClangdRequestT, RequestT>(Request);
+    const auto RPCRequest = ProtobufMarshaller->toProtobuf(Request);
     grpc::ClientContext Context;
     std::chrono::system_clock::time_point Deadline =
         std::chrono::system_clock::now() + DeadlineWaitingTime;
     Context.set_deadline(Deadline);
     auto Reader = (Stub.get()->*RPCCall)(&Context, RPCRequest);
-    llvm::BumpPtrAllocator Arena;
-    llvm::UniqueStringSaver Strings(Arena);
     ReplyT Reply;
     while (Reader->Read(&Reply)) {
       if (!Reply.has_stream_result()) {
         FinalResult = Reply.final_result();
         continue;
       }
-      auto Response =
-          fromProtobuf(Reply.stream_result(), &Strings, ProjectRoot);
-      if (!Response)
+      auto Response = ProtobufMarshaller->fromProtobuf(Reply.stream_result());
+      if (!Response) {
         elog("Received invalid {0}", ReplyT::descriptor()->name());
+        continue;
+      }
       Callback(*Response);
     }
     SPAN_ATTACH(Tracer, "status", Reader->Finish().ok());
@@ -74,7 +63,9 @@ class IndexClient : public clangd::SymbolIndex {
   IndexClient(
       std::shared_ptr<grpc::Channel> Channel, llvm::StringRef ProjectRoot,
       std::chrono::milliseconds DeadlineTime = std::chrono::milliseconds(1000))
-      : Stub(remote::SymbolIndex::NewStub(Channel)), ProjectRoot(ProjectRoot),
+      : Stub(remote::SymbolIndex::NewStub(Channel)),
+        ProtobufMarshaller(new Marshaller(/*RemoteIndexRoot=*/"",
+                                          /*LocalIndexRoot=*/ProjectRoot)),
         DeadlineWaitingTime(DeadlineTime) {}
 
   void lookup(const clangd::LookupRequest &Request,
@@ -105,7 +96,7 @@ class IndexClient : public clangd::SymbolIndex {
 
 private:
   std::unique_ptr<remote::SymbolIndex::Stub> Stub;
-  std::string ProjectRoot;
+  std::unique_ptr<Marshaller> ProtobufMarshaller;
   // Each request will be terminated if it takes too long.
   std::chrono::milliseconds DeadlineWaitingTime;
 };
diff --git a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
index db895bf039bab..b6c83c9740727 100644
--- a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
+++ b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
@@ -30,101 +30,28 @@ namespace clang {
 namespace clangd {
 namespace remote {
 
-namespace {
-
-clangd::SymbolLocation::Position fromProtobuf(const Position &Message) {
-  clangd::SymbolLocation::Position Result;
-  Result.setColumn(static_cast<uint32_t>(Message.column()));
-  Result.setLine(static_cast<uint32_t>(Message.line()));
-  return Result;
-}
-
-Position toProtobuf(const clangd::SymbolLocation::Position &Position) {
-  remote::Position Result;
-  Result.set_column(Position.column());
-  Result.set_line(Position.line());
-  return Result;
-}
-
-clang::index::SymbolInfo fromProtobuf(const SymbolInfo &Message) {
-  clang::index::SymbolInfo Result;
-  Result.Kind = static_cast<clang::index::SymbolKind>(Message.kind());
-  Result.SubKind = static_cast<clang::index::SymbolSubKind>(Message.subkind());
-  Result.Lang = static_cast<clang::index::SymbolLanguage>(Message.language());
-  Result.Properties =
-      static_cast<clang::index::SymbolPropertySet>(Message.properties());
-  return Result;
-}
-
-SymbolInfo toProtobuf(const clang::index::SymbolInfo &Info) {
-  SymbolInfo Result;
-  Result.set_kind(static_cast<uint32_t>(Info.Kind));
-  Result.set_subkind(static_cast<uint32_t>(Info.SubKind));
-  Result.set_language(static_cast<uint32_t>(Info.Lang));
-  Result.set_properties(static_cast<uint32_t>(Info.Properties));
-  return Result;
-}
-
-llvm::Optional<clangd::SymbolLocation>
-fromProtobuf(const SymbolLocation &Message, llvm::UniqueStringSaver *Strings,
-             llvm::StringRef IndexRoot) {
-  clangd::SymbolLocation Location;
-  auto URIString = relativePathToURI(Message.file_path(), IndexRoot);
-  if (!URIString)
-    return llvm::None;
-  Location.FileURI = Strings->save(*URIString).begin();
-  Location.Start = fromProtobuf(Message.start());
-  Location.End = fromProtobuf(Message.end());
-  return Location;
-}
-
-llvm::Optional<SymbolLocation>
-toProtobuf(const clangd::SymbolLocation &Location, llvm::StringRef IndexRoot) {
-  remote::SymbolLocation Result;
-  auto RelativePath = uriToRelativePath(Location.FileURI, IndexRoot);
-  if (!RelativePath)
-    return llvm::None;
-  *Result.mutable_file_path() = *RelativePath;
-  *Result.mutable_start() = toProtobuf(Location.Start);
-  *Result.mutable_end() = toProtobuf(Location.End);
-  return Result;
-}
-
-llvm::Optional<HeaderWithReferences>
-toProtobuf(const clangd::Symbol::IncludeHeaderWithReferences &IncludeHeader,
-           llvm::StringRef IndexRoot) {
-  HeaderWithReferences Result;
-  Result.set_references(IncludeHeader.References);
-  const std::string Header = IncludeHeader.IncludeHeader.str();
-  if (isLiteralInclude(Header)) {
-    Result.set_header(Header);
-    return Result;
+Marshaller::Marshaller(llvm::StringRef RemoteIndexRoot,
+                       llvm::StringRef LocalIndexRoot)
+    : Strings(Arena) {
+  if (!RemoteIndexRoot.empty()) {
+    assert(llvm::sys::path::is_absolute(RemoteIndexRoot));
+    assert(RemoteIndexRoot ==
+           llvm::sys::path::convert_to_slash(RemoteIndexRoot));
+    assert(RemoteIndexRoot.endswith(llvm::sys::path::get_separator()));
+    this->RemoteIndexRoot = RemoteIndexRoot.str();
   }
-  auto RelativePath = uriToRelativePath(Header, IndexRoot);
-  if (!RelativePath)
-    return llvm::None;
-  Result.set_header(*RelativePath);
-  return Result;
-}
-
-llvm::Optional<clangd::Symbol::IncludeHeaderWithReferences>
-fromProtobuf(const HeaderWithReferences &Message,
-             llvm::UniqueStringSaver *Strings, llvm::StringRef IndexRoot) {
-  std::string Header = Message.header();
-  if (Header.front() != '<' && Header.front() != '"') {
-    auto URIString = relativePathToURI(Header, IndexRoot);
-    if (!URIString)
-      return llvm::None;
-    Header = *URIString;
+  if (!LocalIndexRoot.empty()) {
+    assert(llvm::sys::path::is_absolute(LocalIndexRoot));
+    assert(LocalIndexRoot == llvm::sys::path::convert_to_slash(LocalIndexRoot));
+    assert(LocalIndexRoot.endswith(llvm::sys::path::get_separator()));
+    this->LocalIndexRoot = LocalIndexRoot.str();
   }
-  return clangd::Symbol::IncludeHeaderWithReferences{Strings->save(Header),
-                                                     Message.references()};
+  assert(!RemoteIndexRoot.empty() || !LocalIndexRoot.empty());
 }
 
-} // namespace
-
-clangd::FuzzyFindRequest fromProtobuf(const FuzzyFindRequest *Request,
-                                      llvm::StringRef IndexRoot) {
+clangd::FuzzyFindRequest
+Marshaller::fromProtobuf(const FuzzyFindRequest *Request) {
+  assert(RemoteIndexRoot);
   clangd::FuzzyFindRequest Result;
   Result.Query = Request->query();
   for (const auto &Scope : Request->scopes())
@@ -134,7 +61,7 @@ clangd::FuzzyFindRequest fromProtobuf(const FuzzyFindRequest *Request,
     Result.Limit = Request->limit();
   Result.RestrictForCodeCompletion = Request->restricted_for_code_completion();
   for (const auto &Path : Request->proximity_paths()) {
-    llvm::SmallString<256> LocalPath = llvm::StringRef(IndexRoot);
+    llvm::SmallString<256> LocalPath = llvm::StringRef(*RemoteIndexRoot);
     llvm::sys::path::append(LocalPath, Path);
     Result.ProximityPaths.push_back(std::string(LocalPath));
   }
@@ -143,34 +70,35 @@ clangd::FuzzyFindRequest fromProtobuf(const FuzzyFindRequest *Request,
   return Result;
 }
 
-llvm::Optional<clangd::Symbol> fromProtobuf(const Symbol &Message,
-                                            llvm::UniqueStringSaver *Strings,
-                                            llvm::StringRef IndexRoot) {
-  if (!Message.has_info() || !Message.has_definition() ||
-      !Message.has_canonical_declaration()) {
-    elog("Cannot convert Symbol from Protobuf: {0}",
-         Message.ShortDebugString());
+llvm::Optional<clangd::Symbol> Marshaller::fromProtobuf(const Symbol &Message) {
+  if (!Message.has_info() || !Message.has_canonical_declaration()) {
+    elog("Cannot convert Symbol from protobuf (missing info, definition or "
+         "declaration): {0}",
+         Message.DebugString());
     return llvm::None;
   }
   clangd::Symbol Result;
   auto ID = SymbolID::fromStr(Message.id());
   if (!ID) {
-    elog("Cannot parse SymbolID {0} given Protobuf: {1}", ID.takeError(),
-         Message.ShortDebugString());
+    elog("Cannot parse SymbolID {0} given protobuf: {1}", ID.takeError(),
+         Message.DebugString());
     return llvm::None;
   }
   Result.ID = *ID;
   Result.SymInfo = fromProtobuf(Message.info());
   Result.Name = Message.name();
   Result.Scope = Message.scope();
-  auto Definition = fromProtobuf(Message.definition(), Strings, IndexRoot);
-  if (!Definition)
-    return llvm::None;
-  Result.Definition = *Definition;
-  auto Declaration =
-      fromProtobuf(Message.canonical_declaration(), Strings, IndexRoot);
-  if (!Declaration)
+  if (Message.has_definition()) {
+    auto Definition = fromProtobuf(Message.definition());
+    if (Definition)
+      Result.Definition = *Definition;
+  }
+  auto Declaration = fromProtobuf(Message.canonical_declaration());
+  if (!Declaration) {
+    elog("Cannot convert Symbol from protobuf (invalid declaration): {0}",
+         Message.DebugString());
     return llvm::None;
+  }
   Result.CanonicalDeclaration = *Declaration;
   Result.References = Message.references();
   Result.Origin = static_cast<clangd::SymbolOrigin>(Message.origin());
@@ -181,39 +109,44 @@ llvm::Optional<clangd::Symbol> fromProtobuf(const Symbol &Message,
   Result.ReturnType = Message.return_type();
   Result.Type = Message.type();
   for (const auto &Header : Message.headers()) {
-    auto SerializedHeader = fromProtobuf(Header, Strings, IndexRoot);
+    auto SerializedHeader = fromProtobuf(Header);
     if (SerializedHeader)
       Result.IncludeHeaders.push_back(*SerializedHeader);
+    else
+      elog("Cannot convert HeaderWithIncludes from protobuf: {0}",
+           Header.DebugString());
   }
   Result.Flags = static_cast<clangd::Symbol::SymbolFlag>(Message.flags());
   return Result;
 }
 
-llvm::Optional<clangd::Ref> fromProtobuf(const Ref &Message,
-                                         llvm::UniqueStringSaver *Strings,
-                                         llvm::StringRef IndexRoot) {
+llvm::Optional<clangd::Ref> Marshaller::fromProtobuf(const Ref &Message) {
   if (!Message.has_location()) {
-    elog("Cannot convert Ref from Protobuf: {}", Message.ShortDebugString());
+    elog("Cannot convert Ref from protobuf (missing location): {0}",
+         Message.DebugString());
     return llvm::None;
   }
   clangd::Ref Result;
-  auto Location = fromProtobuf(Message.location(), Strings, IndexRoot);
-  if (!Location)
+  auto Location = fromProtobuf(Message.location());
+  if (!Location) {
+    elog("Cannot convert Ref from protobuf (invalid location): {0}",
+         Message.DebugString());
     return llvm::None;
+  }
   Result.Location = *Location;
   Result.Kind = static_cast<clangd::RefKind>(Message.kind());
   return Result;
 }
 
-LookupRequest toProtobuf(const clangd::LookupRequest &From) {
+LookupRequest Marshaller::toProtobuf(const clangd::LookupRequest &From) {
   LookupRequest RPCRequest;
   for (const auto &SymbolID : From.IDs)
     RPCRequest.add_ids(SymbolID.str());
   return RPCRequest;
 }
 
-FuzzyFindRequest toProtobuf(const clangd::FuzzyFindRequest &From,
-                            llvm::StringRef IndexRoot) {
+FuzzyFindRequest Marshaller::toProtobuf(const clangd::FuzzyFindRequest &From) {
+  assert(LocalIndexRoot);
   FuzzyFindRequest RPCRequest;
   RPCRequest.set_query(From.Query);
   for (const auto &Scope : From.Scopes)
@@ -224,7 +157,8 @@ FuzzyFindRequest toProtobuf(const clangd::FuzzyFindRequest &From,
   RPCRequest.set_restricted_for_code_completion(From.RestrictForCodeCompletion);
   for (const auto &Path : From.ProximityPaths) {
     llvm::SmallString<256> RelativePath = llvm::StringRef(Path);
-    if (llvm::sys::path::replace_path_prefix(RelativePath, IndexRoot, ""))
+    if (llvm::sys::path::replace_path_prefix(RelativePath, *LocalIndexRoot,
+                                             ""))
       RPCRequest.add_proximity_paths(llvm::sys::path::convert_to_slash(
           RelativePath, llvm::sys::path::Style::posix));
   }
@@ -233,7 +167,7 @@ FuzzyFindRequest toProtobuf(const clangd::FuzzyFindRequest &From,
   return RPCRequest;
 }
 
-RefsRequest toProtobuf(const clangd::RefsRequest &From) {
+RefsRequest Marshaller::toProtobuf(const clangd::RefsRequest &From) {
   RefsRequest RPCRequest;
   for (const auto &ID : From.IDs)
     RPCRequest.add_ids(ID.str());
@@ -243,18 +177,28 @@ RefsRequest toProtobuf(const clangd::RefsRequest &From) {
   return RPCRequest;
 }
 
-Symbol toProtobuf(const clangd::Symbol &From, llvm::StringRef IndexRoot) {
+llvm::Optional<Symbol> Marshaller::toProtobuf(const clangd::Symbol &From) {
   Symbol Result;
   Result.set_id(From.ID.str());
   *Result.mutable_info() = toProtobuf(From.SymInfo);
   Result.set_name(From.Name.str());
-  auto Definition = toProtobuf(From.Definition, IndexRoot);
-  if (Definition)
+  if (*From.Definition.FileURI) {
+    auto Definition = toProtobuf(From.Definition);
+    if (!Definition) {
+      elog("Can not convert Symbol to protobuf (invalid definition) {0}: {1}",
+           From, From.Definition);
+      return llvm::None;
+    }
     *Result.mutable_definition() = *Definition;
+  }
   Result.set_scope(From.Scope.str());
-  auto Declaration = toProtobuf(From.CanonicalDeclaration, IndexRoot);
-  if (Declaration)
-    *Result.mutable_canonical_declaration() = *Declaration;
+  auto Declaration = toProtobuf(From.CanonicalDeclaration);
+  if (!Declaration) {
+    elog("Can not convert Symbol to protobuf (invalid declaration) {0}: {1}",
+         From, From.CanonicalDeclaration);
+    return llvm::None;
+  }
+  *Result.mutable_canonical_declaration() = *Declaration;
   Result.set_references(From.References);
   Result.set_origin(static_cast<uint32_t>(From.Origin));
   Result.set_signature(From.Signature.str());
@@ -265,9 +209,12 @@ Symbol toProtobuf(const clangd::Symbol &From, llvm::StringRef IndexRoot) {
   Result.set_return_type(From.ReturnType.str());
   Result.set_type(From.Type.str());
   for (const auto &Header : From.IncludeHeaders) {
-    auto Serialized = toProtobuf(Header, IndexRoot);
-    if (!Serialized)
+    auto Serialized = toProtobuf(Header);
+    if (!Serialized) {
+      elog("Can not convert IncludeHeaderWithReferences to protobuf: {0}",
+           Header.IncludeHeader);
       continue;
+    }
     auto *NextHeader = Result.add_headers();
     *NextHeader = *Serialized;
   }
@@ -275,50 +222,38 @@ Symbol toProtobuf(const clangd::Symbol &From, llvm::StringRef IndexRoot) {
   return Result;
 }
 
-// FIXME(kirillbobyrev): A reference without location is invalid.
-// llvm::Optional<Ref> here and on the server side?
-Ref toProtobuf(const clangd::Ref &From, llvm::StringRef IndexRoot) {
+llvm::Optional<Ref> Marshaller::toProtobuf(const clangd::Ref &From) {
   Ref Result;
   Result.set_kind(static_cast<uint32_t>(From.Kind));
-  auto Location = toProtobuf(From.Location, IndexRoot);
-  if (Location)
-    *Result.mutable_location() = *Location;
+  auto Location = toProtobuf(From.Location);
+  if (!Location) {
+    elog("Can not convert Reference to protobuf (invalid location) {0}: {1}",
+         From, From.Location);
+    return llvm::None;
+  }
+  *Result.mutable_location() = *Location;
   return Result;
 }
 
-llvm::Optional<std::string> relativePathToURI(llvm::StringRef RelativePath,
-                                              llvm::StringRef IndexRoot) {
+llvm::Optional<std::string>
+Marshaller::relativePathToURI(llvm::StringRef RelativePath) {
+  assert(LocalIndexRoot);
   assert(RelativePath == llvm::sys::path::convert_to_slash(
                              RelativePath, llvm::sys::path::Style::posix));
-  assert(IndexRoot == llvm::sys::path::convert_to_slash(IndexRoot));
-  assert(IndexRoot.endswith(llvm::sys::path::get_separator()));
-  if (RelativePath.empty())
-    return std::string();
-  if (llvm::sys::path::is_absolute(RelativePath)) {
-    elog("Remote index client got absolute path from server: {0}",
-         RelativePath);
+  if (RelativePath.empty()) {
     return llvm::None;
   }
-  if (llvm::sys::path::is_relative(IndexRoot)) {
-    elog("Remote index client got a relative path as index root: {0}",
-         IndexRoot);
+  if (llvm::sys::path::is_absolute(RelativePath)) {
     return llvm::None;
   }
-  llvm::SmallString<256> FullPath = IndexRoot;
+  llvm::SmallString<256> FullPath = llvm::StringRef(*LocalIndexRoot);
   llvm::sys::path::append(FullPath, RelativePath);
   auto Result = URI::createFile(FullPath);
   return Result.toString();
 }
 
-llvm::Optional<std::string> uriToRelativePath(llvm::StringRef URI,
-                                              llvm::StringRef IndexRoot) {
-  assert(IndexRoot.endswith(llvm::sys::path::get_separator()));
-  assert(IndexRoot == llvm::sys::path::convert_to_slash(IndexRoot));
-  assert(!IndexRoot.empty());
-  if (llvm::sys::path::is_relative(IndexRoot)) {
-    elog("Index root {0} is not absolute path", IndexRoot);
-    return llvm::None;
-  }
+llvm::Optional<std::string> Marshaller::uriToRelativePath(llvm::StringRef URI) {
+  assert(RemoteIndexRoot);
   auto ParsedURI = URI::parse(URI);
   if (!ParsedURI) {
     elog("Remote index got bad URI from client {0}: {1}", URI,
@@ -326,14 +261,10 @@ llvm::Optional<std::string> uriToRelativePath(llvm::StringRef URI,
     return llvm::None;
   }
   if (ParsedURI->scheme() != "file") {
-    elog("Remote index got URI with scheme other than \"file\" {0}: {1}", URI,
-         ParsedURI->scheme());
     return llvm::None;
   }
   llvm::SmallString<256> Result = ParsedURI->body();
-  if (!llvm::sys::path::replace_path_prefix(Result, IndexRoot, "")) {
-    elog("Can not get relative path from the URI {0} given the index root {1}",
-         URI, IndexRoot);
+  if (!llvm::sys::path::replace_path_prefix(Result, *RemoteIndexRoot, "")) {
     return llvm::None;
   }
   // Make sure the result has UNIX slashes.
@@ -341,6 +272,94 @@ llvm::Optional<std::string> uriToRelativePath(llvm::StringRef URI,
                                            llvm::sys::path::Style::posix);
 }
 
+clangd::SymbolLocation::Position
+Marshaller::fromProtobuf(const Position &Message) {
+  clangd::SymbolLocation::Position Result;
+  Result.setColumn(static_cast<uint32_t>(Message.column()));
+  Result.setLine(static_cast<uint32_t>(Message.line()));
+  return Result;
+}
+
+Position
+Marshaller::toProtobuf(const clangd::SymbolLocation::Position &Position) {
+  remote::Position Result;
+  Result.set_column(Position.column());
+  Result.set_line(Position.line());
+  return Result;
+}
+
+clang::index::SymbolInfo Marshaller::fromProtobuf(const SymbolInfo &Message) {
+  clang::index::SymbolInfo Result;
+  Result.Kind = static_cast<clang::index::SymbolKind>(Message.kind());
+  Result.SubKind = static_cast<clang::index::SymbolSubKind>(Message.subkind());
+  Result.Lang = static_cast<clang::index::SymbolLanguage>(Message.language());
+  Result.Properties =
+      static_cast<clang::index::SymbolPropertySet>(Message.properties());
+  return Result;
+}
+
+SymbolInfo Marshaller::toProtobuf(const clang::index::SymbolInfo &Info) {
+  SymbolInfo Result;
+  Result.set_kind(static_cast<uint32_t>(Info.Kind));
+  Result.set_subkind(static_cast<uint32_t>(Info.SubKind));
+  Result.set_language(static_cast<uint32_t>(Info.Lang));
+  Result.set_properties(static_cast<uint32_t>(Info.Properties));
+  return Result;
+}
+
+llvm::Optional<clangd::SymbolLocation>
+Marshaller::fromProtobuf(const SymbolLocation &Message) {
+  clangd::SymbolLocation Location;
+  auto URIString = relativePathToURI(Message.file_path());
+  if (!URIString)
+    return llvm::None;
+  Location.FileURI = Strings.save(*URIString).begin();
+  Location.Start = fromProtobuf(Message.start());
+  Location.End = fromProtobuf(Message.end());
+  return Location;
+}
+
+llvm::Optional<SymbolLocation>
+Marshaller::toProtobuf(const clangd::SymbolLocation &Location) {
+  remote::SymbolLocation Result;
+  auto RelativePath = uriToRelativePath(Location.FileURI);
+  if (!RelativePath)
+    return llvm::None;
+  *Result.mutable_file_path() = *RelativePath;
+  *Result.mutable_start() = toProtobuf(Location.Start);
+  *Result.mutable_end() = toProtobuf(Location.End);
+  return Result;
+}
+
+llvm::Optional<HeaderWithReferences> Marshaller::toProtobuf(
+    const clangd::Symbol::IncludeHeaderWithReferences &IncludeHeader) {
+  HeaderWithReferences Result;
+  Result.set_references(IncludeHeader.References);
+  const std::string Header = IncludeHeader.IncludeHeader.str();
+  if (isLiteralInclude(Header)) {
+    Result.set_header(Header);
+    return Result;
+  }
+  auto RelativePath = uriToRelativePath(Header);
+  if (!RelativePath)
+    return llvm::None;
+  Result.set_header(*RelativePath);
+  return Result;
+}
+
+llvm::Optional<clangd::Symbol::IncludeHeaderWithReferences>
+Marshaller::fromProtobuf(const HeaderWithReferences &Message) {
+  std::string Header = Message.header();
+  if (Header.front() != '<' && Header.front() != '"') {
+    auto URIString = relativePathToURI(Header);
+    if (!URIString)
+      return llvm::None;
+    Header = *URIString;
+  }
+  return clangd::Symbol::IncludeHeaderWithReferences{Strings.save(Header),
+                                                     Message.references()};
+}
+
 } // namespace remote
 } // namespace clangd
 } // namespace clang
diff --git a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
index 86cc8fa272fc5..9129cff24db57 100644
--- a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
+++ b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
@@ -5,31 +5,6 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
-// Marshalling provides translation between native clangd types into the
-// Protobuf-generated classes. Most translations are 1-to-1 and wrap variables
-// into appropriate Protobuf types.
-//
-// A notable exception is URI translation. Because paths to files are different
-// on indexing machine and client machine
-// ("/remote/machine/projects/llvm-project/llvm/include/HelloWorld.h" versus
-// "/usr/local/username/llvm-project/llvm/include/HelloWorld.h"), they need to
-// be converted appropriately. Remote machine strips the prefix from the
-// absolute path and passes paths relative to the project root over the wire
-// ("include/HelloWorld.h" in this example). The indexed project root is passed
-// to the remote server. Client receives this relative path and constructs a URI
-// that points to the relevant file in the filesystem. The relative path is
-// appended to IndexRoot to construct the full path and build the final URI.
-//
-// Index root is the absolute path to the project and includes a trailing slash.
-// The relative path passed over the wire has unix slashes.
-//
-// toProtobuf() functions serialize native clangd types and strip IndexRoot from
-// the file paths specific to indexing machine. fromProtobuf() functions
-// deserialize clangd types and translate relative paths into machine-native
-// URIs.
-//
-//===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_REMOTE_MARSHALLING_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_REMOTE_MARSHALLING_H
@@ -43,33 +18,76 @@ namespace clang {
 namespace clangd {
 namespace remote {
 
-clangd::FuzzyFindRequest fromProtobuf(const FuzzyFindRequest *Request,
-                                      llvm::StringRef IndexRoot);
-llvm::Optional<clangd::Symbol> fromProtobuf(const Symbol &Message,
-                                            llvm::UniqueStringSaver *Strings,
-                                            llvm::StringRef IndexRoot);
-llvm::Optional<clangd::Ref> fromProtobuf(const Ref &Message,
-                                         llvm::UniqueStringSaver *Strings,
-                                         llvm::StringRef IndexRoot);
+// Marshalling provides an interface for translattion between native clangd
+// types into the Protobuf-generated classes. Most translations are 1-to-1 and
+// wrap variables into appropriate Protobuf types.
+//
+/// A notable exception is URI translation. Because paths to files are different
+/// on indexing machine and client machine
+/// ("/remote/machine/projects/llvm-project/llvm/include/HelloWorld.h" versus
+/// "/usr/local/username/llvm-project/llvm/include/HelloWorld.h"), they need to
+/// be converted appropriately. Remote machine strips the prefix
+/// (RemoteIndexRoot) from the absolute path and passes paths relative to the
+/// project root over the wire ("include/HelloWorld.h" in this example). The
+/// indexed project root is passed to the remote server. Client receives this
+/// relative path and constructs a URI that points to the relevant file in the
+/// filesystem. The relative path is appended to LocalIndexRoot to construct the
+/// full path and build the final URI.
+class Marshaller {
+public:
+  Marshaller() = delete;
+  Marshaller(llvm::StringRef RemoteIndexRoot, llvm::StringRef LocalIndexRoot);
+
+  clangd::FuzzyFindRequest fromProtobuf(const FuzzyFindRequest *Request);
+  llvm::Optional<clangd::Symbol> fromProtobuf(const Symbol &Message);
+  llvm::Optional<clangd::Ref> fromProtobuf(const Ref &Message);
+
+  /// toProtobuf() functions serialize native clangd types and strip IndexRoot
+  /// from the file paths specific to indexing machine. fromProtobuf() functions
+  /// deserialize clangd types and translate relative paths into machine-native
+  /// URIs.
+  LookupRequest toProtobuf(const clangd::LookupRequest &From);
+  FuzzyFindRequest toProtobuf(const clangd::FuzzyFindRequest &From);
+  RefsRequest toProtobuf(const clangd::RefsRequest &From);
+
+  llvm::Optional<Ref> toProtobuf(const clangd::Ref &From);
+  llvm::Optional<Symbol> toProtobuf(const clangd::Symbol &From);
 
-LookupRequest toProtobuf(const clangd::LookupRequest &From);
-FuzzyFindRequest toProtobuf(const clangd::FuzzyFindRequest &From,
-                            llvm::StringRef IndexRoot);
-RefsRequest toProtobuf(const clangd::RefsRequest &From);
+  /// Translates \p RelativePath into the absolute path and builds URI for the
+  /// user machine. This translation happens on the client side with the
+  /// \p RelativePath received from remote index server and \p IndexRoot is
+  /// provided by the client.
+  ///
+  /// The relative path passed over the wire has unix slashes.
+  llvm::Optional<std::string> relativePathToURI(llvm::StringRef RelativePath);
+  /// Translates a URI from the server's backing index to a relative path
+  /// suitable to send over the wire to the client.
+  llvm::Optional<std::string> uriToRelativePath(llvm::StringRef URI);
 
-Ref toProtobuf(const clangd::Ref &From, llvm::StringRef IndexRoot);
-Symbol toProtobuf(const clangd::Symbol &From, llvm::StringRef IndexRoot);
+private:
+  clangd::SymbolLocation::Position fromProtobuf(const Position &Message);
+  Position toProtobuf(const clangd::SymbolLocation::Position &Position);
+  clang::index::SymbolInfo fromProtobuf(const SymbolInfo &Message);
+  SymbolInfo toProtobuf(const clang::index::SymbolInfo &Info);
+  llvm::Optional<clangd::SymbolLocation>
+  fromProtobuf(const SymbolLocation &Message);
+  llvm::Optional<SymbolLocation>
+  toProtobuf(const clangd::SymbolLocation &Location);
+  llvm::Optional<HeaderWithReferences>
+  toProtobuf(const clangd::Symbol::IncludeHeaderWithReferences &IncludeHeader);
+  llvm::Optional<clangd::Symbol::IncludeHeaderWithReferences>
+  fromProtobuf(const HeaderWithReferences &Message);
 
-/// Translates \p RelativePath into the absolute path and builds URI for the
-/// user machine. This translation happens on the client side with the
-/// \p RelativePath received from remote index server and \p IndexRoot is
-/// provided by the client.
-llvm::Optional<std::string> relativePathToURI(llvm::StringRef RelativePath,
-                                              llvm::StringRef IndexRoot);
-/// Translates a URI from the server's backing index to a relative path suitable
-/// to send over the wire to the client.
-llvm::Optional<std::string> uriToRelativePath(llvm::StringRef URI,
-                                              llvm::StringRef IndexRoot);
+  /// RemoteIndexRoot and LocalIndexRoot are absolute paths to the project (on
+  /// remote and local machine respectively) and include a trailing slash. One
+  /// of them can be missing (if the machines are different they don't know each
+  /// other's specifics and will only do one-way translation), but both can not
+  /// be missing at the same time.
+  llvm::Optional<std::string> RemoteIndexRoot;
+  llvm::Optional<std::string> LocalIndexRoot;
+  llvm::BumpPtrAllocator Arena;
+  llvm::UniqueStringSaver Strings;
+};
 
 } // namespace remote
 } // namespace clangd
diff --git a/clang-tools-extra/clangd/index/remote/server/Server.cpp b/clang-tools-extra/clangd/index/remote/server/Server.cpp
index fecd72806cbc0..07b1c736b6725 100644
--- a/clang-tools-extra/clangd/index/remote/server/Server.cpp
+++ b/clang-tools-extra/clangd/index/remote/server/Server.cpp
@@ -50,7 +50,9 @@ class RemoteIndexServer final : public SymbolIndex::Service {
       : Index(std::move(Index)) {
     llvm::SmallString<256> NativePath = IndexRoot;
     llvm::sys::path::native(NativePath);
-    IndexedProjectRoot = std::string(NativePath);
+    ProtobufMarshaller = std::unique_ptr<Marshaller>(new Marshaller(
+        /*RemoteIndexRoot=*/llvm::StringRef(NativePath),
+        /*LocalIndexRoot=*/""));
   }
 
 private:
@@ -65,9 +67,11 @@ class RemoteIndexServer final : public SymbolIndex::Service {
       Req.IDs.insert(*SID);
     }
     Index->lookup(Req, [&](const clangd::Symbol &Sym) {
+      auto SerializedSymbol = ProtobufMarshaller->toProtobuf(Sym);
+      if (!SerializedSymbol)
+        return;
       LookupReply NextMessage;
-      *NextMessage.mutable_stream_result() =
-          toProtobuf(Sym, IndexedProjectRoot);
+      *NextMessage.mutable_stream_result() = *SerializedSymbol;
       Reply->Write(NextMessage);
     });
     LookupReply LastMessage;
@@ -79,11 +83,13 @@ class RemoteIndexServer final : public SymbolIndex::Service {
   grpc::Status FuzzyFind(grpc::ServerContext *Context,
                          const FuzzyFindRequest *Request,
                          grpc::ServerWriter<FuzzyFindReply> *Reply) override {
-    const auto Req = fromProtobuf(Request, IndexedProjectRoot);
+    const auto Req = ProtobufMarshaller->fromProtobuf(Request);
     bool HasMore = Index->fuzzyFind(Req, [&](const clangd::Symbol &Sym) {
+      auto SerializedSymbol = ProtobufMarshaller->toProtobuf(Sym);
+      if (!SerializedSymbol)
+        return;
       FuzzyFindReply NextMessage;
-      *NextMessage.mutable_stream_result() =
-          toProtobuf(Sym, IndexedProjectRoot);
+      *NextMessage.mutable_stream_result() = *SerializedSymbol;
       Reply->Write(NextMessage);
     });
     FuzzyFindReply LastMessage;
@@ -102,8 +108,11 @@ class RemoteIndexServer final : public SymbolIndex::Service {
       Req.IDs.insert(*SID);
     }
     bool HasMore = Index->refs(Req, [&](const clangd::Ref &Reference) {
+      auto SerializedRef = ProtobufMarshaller->toProtobuf(Reference);
+      if (!SerializedRef)
+        return;
       RefsReply NextMessage;
-      *NextMessage.mutable_stream_result() = toProtobuf(Reference, IndexRoot);
+      *NextMessage.mutable_stream_result() = *SerializedRef;
       Reply->Write(NextMessage);
     });
     RefsReply LastMessage;
@@ -113,7 +122,7 @@ class RemoteIndexServer final : public SymbolIndex::Service {
   }
 
   std::unique_ptr<clangd::SymbolIndex> Index;
-  std::string IndexedProjectRoot;
+  std::unique_ptr<Marshaller> ProtobufMarshaller;
 };
 
 void runServer(std::unique_ptr<clangd::SymbolIndex> Index,
diff --git a/clang-tools-extra/clangd/test/Inputs/background-index/compile_commands.json b/clang-tools-extra/clangd/test/Inputs/background-index/compile_commands.json.tmpl
similarity index 100%
rename from clang-tools-extra/clangd/test/Inputs/background-index/compile_commands.json
rename to clang-tools-extra/clangd/test/Inputs/background-index/compile_commands.json.tmpl
diff --git a/clang-tools-extra/clangd/test/Inputs/background-index/definition.jsonrpc b/clang-tools-extra/clangd/test/Inputs/background-index/definition.jsonrpc.tmpl
similarity index 100%
rename from clang-tools-extra/clangd/test/Inputs/background-index/definition.jsonrpc
rename to clang-tools-extra/clangd/test/Inputs/background-index/definition.jsonrpc.tmpl
diff --git a/clang-tools-extra/clangd/test/background-index.test b/clang-tools-extra/clangd/test/background-index.test
index 41184443e947a..1983f0957dccf 100644
--- a/clang-tools-extra/clangd/test/background-index.test
+++ b/clang-tools-extra/clangd/test/background-index.test
@@ -1,23 +1,23 @@
-# We need to splice paths into file:// URIs for this test.
-# UNSUPPORTED: windows-msvc
-
 # Use a copy of inputs, as we'll mutate it (as will the background index).
-# RUN: rm -rf %t
-# RUN: cp -r %S/Inputs/background-index %t
+# RUN: rm -rf %/t
+# RUN: cp -r %/S/Inputs/background-index %/t
 # Need to embed the correct temp path in the actual JSON-RPC requests.
-# RUN: sed -i -e "s|DIRECTORY|%t|" %t/definition.jsonrpc
-# RUN: sed -i -e "s|DIRECTORY|%t|" %t/compile_commands.json
+# RUN: sed -e "s|DIRECTORY|%/t|" %/t/definition.jsonrpc.tmpl > %/t/definition.jsonrpc.1
+# RUN: sed -e "s|DIRECTORY|%/t|" %/t/compile_commands.json.tmpl > %/t/compile_commands.json
+# On Windows, we need the URI in didOpen to look like "uri":"file:///C:/..."
+# (with the extra slash in the front), so we add it here.
+# RUN: sed -E -e 's|"file://([A-Z]):/|"file:///\1:/|g' %/t/definition.jsonrpc.1 > %/t/definition.jsonrpc
 
 # We're editing bar.cpp, which includes foo.h.
 # foo() is declared in foo.h and defined in foo.cpp.
 # The background index should allow us to go-to-definition on foo().
 # We should also see indexing progress notifications.
-# RUN: clangd -background-index -lit-test < %t/definition.jsonrpc | FileCheck %t/definition.jsonrpc --check-prefixes=CHECK,BUILD
+# RUN: clangd -background-index -lit-test < %/t/definition.jsonrpc | FileCheck %/t/definition.jsonrpc --check-prefixes=CHECK,BUILD
 
 # Test that the index is writing files in the expected location.
-# RUN: ls %t/.cache/clangd/index/foo.cpp.*.idx
-# RUN: ls %t/sub_dir/.cache/clangd/index/foo.h.*.idx
+# RUN: ls %/t/.cache/clangd/index/foo.cpp.*.idx
+# RUN: ls %/t/sub_dir/.cache/clangd/index/foo.h.*.idx
 
 # Test the index is read from disk: delete code and restart clangd.
-# RUN: rm %t/foo.cpp
-# RUN: clangd -background-index -lit-test < %t/definition.jsonrpc | FileCheck %t/definition.jsonrpc --check-prefixes=CHECK,USE
+# RUN: rm %/t/foo.cpp
+# RUN: clangd -background-index -lit-test < %/t/definition.jsonrpc | FileCheck %/t/definition.jsonrpc --check-prefixes=CHECK,USE
diff --git a/clang-tools-extra/clangd/test/did-change-configuration-params.test b/clang-tools-extra/clangd/test/did-change-configuration-params.test
index 4aef1011b3707..19d41d0812e23 100644
--- a/clang-tools-extra/clangd/test/did-change-configuration-params.test
+++ b/clang-tools-extra/clangd/test/did-change-configuration-params.test
@@ -1,5 +1,5 @@
 # RUN: clangd -compile_args_from=lsp -lit-test < %s 2> %t | FileCheck -strict-whitespace %s
-# RUN: cat %t | FileCheck --check-prefix=ERR %s
+# RUN: FileCheck --check-prefix=ERR --input-file=%t %s
 # UNSUPPORTED: windows-gnu,windows-msvc
 {"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"clangd","capabilities":{},"trace":"off"}}
 ---
diff --git a/clang-tools-extra/clangd/test/test-uri-windows.test b/clang-tools-extra/clangd/test/test-uri-windows.test
index 381c48fafc038..3f03b2985a70f 100644
--- a/clang-tools-extra/clangd/test/test-uri-windows.test
+++ b/clang-tools-extra/clangd/test/test-uri-windows.test
@@ -1,5 +1,5 @@
 # RUN: clangd -lit-test < %s | FileCheck -strict-whitespace %s
-# REQUIRES: windows-gnu || windows-msvc
+# UNSUPPORTED: !(windows-gnu || windows-msvc)
 # Test authority-less URI
 {"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"clangd","capabilities":{},"trace":"off"}}
 ---
diff --git a/clang-tools-extra/clangd/unittests/CMakeLists.txt b/clang-tools-extra/clangd/unittests/CMakeLists.txt
index c25e2b7f81037..8ede92c16f7ad 100644
--- a/clang-tools-extra/clangd/unittests/CMakeLists.txt
+++ b/clang-tools-extra/clangd/unittests/CMakeLists.txt
@@ -13,6 +13,10 @@ include_directories(
   ${CLANGD_BINARY_DIR}
   )
 
+if (CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
+  add_compile_options("-Wno-suggest-override")
+endif()
+
 if(CLANG_BUILT_STANDALONE)
   # LLVMTestingSupport library is needed for clangd tests.
   if (EXISTS ${LLVM_MAIN_SRC_DIR}/lib/Testing/Support
diff --git a/clang-tools-extra/clangd/unittests/FindTargetTests.cpp b/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
index bf5cc62411b74..f7af77e8b57be 100644
--- a/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
+++ b/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
@@ -376,6 +376,8 @@ TEST_F(TargetDeclTest, ClassTemplate) {
                {"template<> class Foo<int *>", Rel::TemplateInstantiation},
                {"template <typename T> class Foo<T *>", Rel::TemplatePattern});
 
+  Flags.push_back("-std=c++17"); // for CTAD tests
+
   Code = R"cpp(
     // Class template argument deduction
     template <typename T>
@@ -386,9 +388,20 @@ TEST_F(TargetDeclTest, ClassTemplate) {
       [[Test]] a(5);
     }
   )cpp";
-  Flags.push_back("-std=c++17");
   EXPECT_DECLS("DeducedTemplateSpecializationTypeLoc",
                {"struct Test", Rel::TemplatePattern});
+
+  Code = R"cpp(
+    // Deduction guide
+    template <typename T>
+    struct Test {
+      template <typename I>
+      Test(I, I);
+    };
+    template <typename I>
+    [[Test]](I, I) -> Test<typename I::type>;
+  )cpp";
+  EXPECT_DECLS("CXXDeductionGuideDecl", {"template <typename T> struct Test"});
 }
 
 TEST_F(TargetDeclTest, Concept) {
@@ -548,6 +561,74 @@ TEST_F(TargetDeclTest, OverloadExpr) {
   EXPECT_DECLS("UnresolvedMemberExpr", "void func(int *)", "void func(char *)");
 }
 
+TEST_F(TargetDeclTest, DependentExprs) {
+  Flags = {"-fno-delayed-template-parsing"};
+
+  // Heuristic resolution of method of dependent field
+  Code = R"cpp(
+        struct A { void foo() {} };
+        template <typename T>
+        struct B {
+          A a;
+          void bar() {
+            this->a.[[foo]]();
+          }
+        };
+      )cpp";
+  EXPECT_DECLS("CXXDependentScopeMemberExpr", "void foo()");
+
+  // Similar to above but base expression involves a function call.
+  Code = R"cpp(
+        struct A {
+          void foo() {}
+        };
+        struct B {
+          A getA();
+        };
+        template <typename T>
+        struct C {
+          B c;
+          void bar() {
+            this->c.getA().[[foo]]();
+          }
+        };
+      )cpp";
+  EXPECT_DECLS("CXXDependentScopeMemberExpr", "void foo()");
+
+  // Similar to above but uses a function pointer.
+  Code = R"cpp(
+        struct A {
+          void foo() {}
+        };
+        struct B {
+          using FPtr = A(*)();
+          FPtr fptr;
+        };
+        template <typename T>
+        struct C {
+          B c;
+          void bar() {
+            this->c.fptr().[[foo]]();
+          }
+        };
+      )cpp";
+  EXPECT_DECLS("CXXDependentScopeMemberExpr", "void foo()");
+
+  // Base expression involves a member access into this.
+  Code = R"cpp(
+        struct Bar {
+          int aaaa;
+        };
+        template <typename T> struct Foo {
+          Bar func(int);
+          void test() {
+            func(1).[[aaaa]];
+          }
+        };
+      )cpp";
+  EXPECT_DECLS("CXXDependentScopeMemberExpr", "int aaaa");
+}
+
 TEST_F(TargetDeclTest, ObjC) {
   Flags = {"-xobjective-c"};
   Code = R"cpp(
@@ -705,36 +786,37 @@ class FindExplicitReferencesTest : public ::testing::Test {
 
 TEST_F(FindExplicitReferencesTest, All) {
   std::pair</*Code*/ llvm::StringRef, /*References*/ llvm::StringRef> Cases[] =
-      {// Simple expressions.
-       {R"cpp(
+      {
+          // Simple expressions.
+          {R"cpp(
         int global;
         int func();
         void foo(int param) {
           $0^global = $1^param + $2^func();
         }
         )cpp",
-        "0: targets = {global}\n"
-        "1: targets = {param}\n"
-        "2: targets = {func}\n"},
-       {R"cpp(
+           "0: targets = {global}\n"
+           "1: targets = {param}\n"
+           "2: targets = {func}\n"},
+          {R"cpp(
         struct X { int a; };
         void foo(X x) {
           $0^x.$1^a = 10;
         }
         )cpp",
-        "0: targets = {x}\n"
-        "1: targets = {X::a}\n"},
-       {R"cpp(
+           "0: targets = {x}\n"
+           "1: targets = {X::a}\n"},
+          {R"cpp(
         // error-ok: testing with broken code
         int bar();
         int foo() {
           return $0^bar() + $1^bar(42);
         }
         )cpp",
-        "0: targets = {bar}\n"
-        "1: targets = {bar}\n"},
-       // Namespaces and aliases.
-       {R"cpp(
+           "0: targets = {bar}\n"
+           "1: targets = {bar}\n"},
+          // Namespaces and aliases.
+          {R"cpp(
           namespace ns {}
           namespace alias = ns;
           void foo() {
@@ -742,19 +824,19 @@ TEST_F(FindExplicitReferencesTest, All) {
             using namespace $1^alias;
           }
         )cpp",
-        "0: targets = {ns}\n"
-        "1: targets = {alias}\n"},
-       // Using declarations.
-       {R"cpp(
+           "0: targets = {ns}\n"
+           "1: targets = {alias}\n"},
+          // Using declarations.
+          {R"cpp(
           namespace ns { int global; }
           void foo() {
             using $0^ns::$1^global;
           }
         )cpp",
-        "0: targets = {ns}\n"
-        "1: targets = {ns::global}, qualifier = 'ns::'\n"},
-       // Simple types.
-       {R"cpp(
+           "0: targets = {ns}\n"
+           "1: targets = {ns::global}, qualifier = 'ns::'\n"},
+          // Simple types.
+          {R"cpp(
          struct Struct { int a; };
          using Typedef = int;
          void foo() {
@@ -763,13 +845,13 @@ TEST_F(FindExplicitReferencesTest, All) {
            static_cast<$4^Struct*>(0);
          }
        )cpp",
-        "0: targets = {Struct}\n"
-        "1: targets = {x}, decl\n"
-        "2: targets = {Typedef}\n"
-        "3: targets = {y}, decl\n"
-        "4: targets = {Struct}\n"},
-       // Name qualifiers.
-       {R"cpp(
+           "0: targets = {Struct}\n"
+           "1: targets = {x}, decl\n"
+           "2: targets = {Typedef}\n"
+           "3: targets = {y}, decl\n"
+           "4: targets = {Struct}\n"},
+          // Name qualifiers.
+          {R"cpp(
          namespace a { namespace b { struct S { typedef int type; }; } }
          void foo() {
            $0^a::$1^b::$2^S $3^x;
@@ -777,23 +859,23 @@ TEST_F(FindExplicitReferencesTest, All) {
            $6^S::$7^type $8^y;
          }
         )cpp",
-        "0: targets = {a}\n"
-        "1: targets = {a::b}, qualifier = 'a::'\n"
-        "2: targets = {a::b::S}, qualifier = 'a::b::'\n"
-        "3: targets = {x}, decl\n"
-        "4: targets = {a}\n"
-        "5: targets = {a::b}, qualifier = 'a::'\n"
-        "6: targets = {a::b::S}\n"
-        "7: targets = {a::b::S::type}, qualifier = 'struct S::'\n"
-        "8: targets = {y}, decl\n"},
-       {R"cpp(
+           "0: targets = {a}\n"
+           "1: targets = {a::b}, qualifier = 'a::'\n"
+           "2: targets = {a::b::S}, qualifier = 'a::b::'\n"
+           "3: targets = {x}, decl\n"
+           "4: targets = {a}\n"
+           "5: targets = {a::b}, qualifier = 'a::'\n"
+           "6: targets = {a::b::S}\n"
+           "7: targets = {a::b::S::type}, qualifier = 'struct S::'\n"
+           "8: targets = {y}, decl\n"},
+          {R"cpp(
          void foo() {
            $0^ten: // PRINT "HELLO WORLD!"
            goto $1^ten;
          }
        )cpp",
-       "0: targets = {ten}, decl\n"
-       "1: targets = {ten}\n"},
+        "0: targets = {ten}, decl\n"
+        "1: targets = {ten}\n"},
        // Simple templates.
        {R"cpp(
           template <class T> struct vector { using value_type = T; };
@@ -803,12 +885,12 @@ TEST_F(FindExplicitReferencesTest, All) {
             $2^vector<bool> $3^vb;
           }
         )cpp",
-        "0: targets = {vector<int>}\n"
-        "1: targets = {vi}, decl\n"
-        "2: targets = {vector<bool>}\n"
-        "3: targets = {vb}, decl\n"},
-       // Template type aliases.
-       {R"cpp(
+           "0: targets = {vector<int>}\n"
+           "1: targets = {vi}, decl\n"
+           "2: targets = {vector<bool>}\n"
+           "3: targets = {vb}, decl\n"},
+          // Template type aliases.
+          {R"cpp(
             template <class T> struct vector { using value_type = T; };
             template <> struct vector<bool> { using value_type = bool; };
             template <class T> using valias = vector<T>;
@@ -817,12 +899,12 @@ TEST_F(FindExplicitReferencesTest, All) {
               $2^valias<bool> $3^vb;
             }
           )cpp",
-        "0: targets = {valias}\n"
-        "1: targets = {vi}, decl\n"
-        "2: targets = {valias}\n"
-        "3: targets = {vb}, decl\n"},
-       // Injected class name.
-       {R"cpp(
+           "0: targets = {valias}\n"
+           "1: targets = {vi}, decl\n"
+           "2: targets = {valias}\n"
+           "3: targets = {vb}, decl\n"},
+          // Injected class name.
+          {R"cpp(
             namespace foo {
               template <typename $0^T>
               class $1^Bar {
@@ -831,13 +913,13 @@ TEST_F(FindExplicitReferencesTest, All) {
               };
             }
           )cpp",
-        "0: targets = {foo::Bar::T}, decl\n"
-        "1: targets = {foo::Bar}, decl\n"
-        "2: targets = {foo::Bar}\n"
-        "3: targets = {foo::Bar::f}, decl\n"
-        "4: targets = {foo::Bar}\n"},
-       // MemberExpr should know their using declaration.
-       {R"cpp(
+           "0: targets = {foo::Bar::T}, decl\n"
+           "1: targets = {foo::Bar}, decl\n"
+           "2: targets = {foo::Bar}\n"
+           "3: targets = {foo::Bar::f}, decl\n"
+           "4: targets = {foo::Bar}\n"},
+          // MemberExpr should know their using declaration.
+          {R"cpp(
             struct X { void func(int); };
             struct Y : X {
               using X::func;
@@ -846,10 +928,10 @@ TEST_F(FindExplicitReferencesTest, All) {
               $0^y.$1^func(1);
             }
         )cpp",
-        "0: targets = {y}\n"
-        "1: targets = {Y::func}\n"},
-       // DeclRefExpr should know their using declaration.
-       {R"cpp(
+           "0: targets = {y}\n"
+           "1: targets = {Y::func}\n"},
+          // DeclRefExpr should know their using declaration.
+          {R"cpp(
             namespace ns { void bar(int); }
             using ns::bar;
 
@@ -857,9 +939,9 @@ TEST_F(FindExplicitReferencesTest, All) {
               $0^bar(10);
             }
         )cpp",
-        "0: targets = {bar}\n"},
-       // References from a macro.
-       {R"cpp(
+           "0: targets = {bar}\n"},
+          // References from a macro.
+          {R"cpp(
             #define FOO a
             #define BAR b
 
@@ -867,10 +949,10 @@ TEST_F(FindExplicitReferencesTest, All) {
               $0^FOO+$1^BAR;
             }
         )cpp",
-        "0: targets = {a}\n"
-        "1: targets = {b}\n"},
-       // No references from implicit nodes.
-       {R"cpp(
+           "0: targets = {a}\n"
+           "1: targets = {b}\n"},
+          // No references from implicit nodes.
+          {R"cpp(
             struct vector {
               int *begin();
               int *end();
@@ -882,15 +964,15 @@ TEST_F(FindExplicitReferencesTest, All) {
               }
             }
         )cpp",
-        "0: targets = {x}, decl\n"
-        "1: targets = {vector}\n"
-        "2: targets = {x}\n"},
+           "0: targets = {x}, decl\n"
+           "1: targets = {vector}\n"
+           "2: targets = {x}\n"},
 // Handle UnresolvedLookupExpr.
 // FIXME
 // This case fails when expensive checks are enabled.
 // Seems like the order of ns1::func and ns2::func isn't defined.
 #ifndef EXPENSIVE_CHECKS
-       {R"cpp(
+          {R"cpp(
             namespace ns1 { void func(char*); }
             namespace ns2 { void func(int*); }
             using namespace ns1;
@@ -901,11 +983,11 @@ TEST_F(FindExplicitReferencesTest, All) {
               $0^func($1^t);
             }
         )cpp",
-        "0: targets = {ns1::func, ns2::func}\n"
-        "1: targets = {t}\n"},
+           "0: targets = {ns1::func, ns2::func}\n"
+           "1: targets = {t}\n"},
 #endif
-       // Handle UnresolvedMemberExpr.
-       {R"cpp(
+          // Handle UnresolvedMemberExpr.
+          {R"cpp(
             struct X {
               void func(char*);
               void func(int*);
@@ -916,11 +998,11 @@ TEST_F(FindExplicitReferencesTest, All) {
               $0^x.$1^func($2^t);
             }
         )cpp",
-        "0: targets = {x}\n"
-        "1: targets = {X::func, X::func}\n"
-        "2: targets = {t}\n"},
-       // Handle DependentScopeDeclRefExpr.
-       {R"cpp(
+           "0: targets = {x}\n"
+           "1: targets = {X::func, X::func}\n"
+           "2: targets = {t}\n"},
+          // Handle DependentScopeDeclRefExpr.
+          {R"cpp(
             template <class T>
             struct S {
               static int value;
@@ -931,11 +1013,11 @@ TEST_F(FindExplicitReferencesTest, All) {
               $0^S<$1^T>::$2^value;
             }
        )cpp",
-        "0: targets = {S}\n"
-        "1: targets = {T}\n"
-        "2: targets = {S::value}, qualifier = 'S<T>::'\n"},
-       // Handle CXXDependentScopeMemberExpr.
-       {R"cpp(
+           "0: targets = {S}\n"
+           "1: targets = {T}\n"
+           "2: targets = {S::value}, qualifier = 'S<T>::'\n"},
+          // Handle CXXDependentScopeMemberExpr.
+          {R"cpp(
             template <class T>
             struct S {
               int value;
@@ -946,10 +1028,10 @@ TEST_F(FindExplicitReferencesTest, All) {
               $0^t.$1^value;
             }
        )cpp",
-        "0: targets = {t}\n"
-        "1: targets = {S::value}\n"},
-       // Type template parameters.
-       {R"cpp(
+           "0: targets = {t}\n"
+           "1: targets = {S::value}\n"},
+          // Type template parameters.
+          {R"cpp(
             template <class T>
             void foo() {
               static_cast<$0^T>(0);
@@ -957,21 +1039,21 @@ TEST_F(FindExplicitReferencesTest, All) {
               $2^T $3^t;
             }
         )cpp",
-        "0: targets = {T}\n"
-        "1: targets = {T}\n"
-        "2: targets = {T}\n"
-        "3: targets = {t}, decl\n"},
-       // Non-type template parameters.
-       {R"cpp(
+           "0: targets = {T}\n"
+           "1: targets = {T}\n"
+           "2: targets = {T}\n"
+           "3: targets = {t}, decl\n"},
+          // Non-type template parameters.
+          {R"cpp(
             template <int I>
             void foo() {
               int $0^x = $1^I;
             }
         )cpp",
-        "0: targets = {x}, decl\n"
-        "1: targets = {I}\n"},
-       // Template template parameters.
-       {R"cpp(
+           "0: targets = {x}, decl\n"
+           "1: targets = {I}\n"},
+          // Template template parameters.
+          {R"cpp(
             template <class T> struct vector {};
 
             template <template<class> class TT, template<class> class ...TP>
@@ -982,16 +1064,16 @@ TEST_F(FindExplicitReferencesTest, All) {
               $6^foo<$7^TP...>();
             }
         )cpp",
-        "0: targets = {TT}\n"
-        "1: targets = {x}, decl\n"
-        "2: targets = {foo}\n"
-        "3: targets = {TT}\n"
-        "4: targets = {foo}\n"
-        "5: targets = {vector}\n"
-        "6: targets = {foo}\n"
-        "7: targets = {TP}\n"},
-       // Non-type template parameters with declarations.
-       {R"cpp(
+           "0: targets = {TT}\n"
+           "1: targets = {x}, decl\n"
+           "2: targets = {foo}\n"
+           "3: targets = {TT}\n"
+           "4: targets = {foo}\n"
+           "5: targets = {vector}\n"
+           "6: targets = {foo}\n"
+           "7: targets = {TP}\n"},
+          // Non-type template parameters with declarations.
+          {R"cpp(
             int func();
             template <int(*)()> struct wrapper {};
 
@@ -1001,12 +1083,12 @@ TEST_F(FindExplicitReferencesTest, All) {
               $3^FuncParam();
             }
         )cpp",
-        "0: targets = {wrapper<&func>}\n"
-        "1: targets = {func}\n"
-        "2: targets = {w}, decl\n"
-        "3: targets = {FuncParam}\n"},
-       // declaration references.
-       {R"cpp(
+           "0: targets = {wrapper<&func>}\n"
+           "1: targets = {func}\n"
+           "2: targets = {w}, decl\n"
+           "3: targets = {FuncParam}\n"},
+          // declaration references.
+          {R"cpp(
              namespace ns {}
              class S {};
              void foo() {
@@ -1018,19 +1100,19 @@ TEST_F(FindExplicitReferencesTest, All) {
                namespace $9^NS = $10^ns;
              }
            )cpp",
-        "0: targets = {Foo}, decl\n"
-        "1: targets = {foo()::Foo::Foo}, decl\n"
-        "2: targets = {Foo}\n"
-        "3: targets = {foo()::Foo::field}, decl\n"
-        "4: targets = {Var}, decl\n"
-        "5: targets = {E}, decl\n"
-        "6: targets = {foo()::ABC}, decl\n"
-        "7: targets = {INT}, decl\n"
-        "8: targets = {INT2}, decl\n"
-        "9: targets = {NS}, decl\n"
-        "10: targets = {ns}\n"},
-       // User-defined conversion operator.
-       {R"cpp(
+           "0: targets = {Foo}, decl\n"
+           "1: targets = {foo()::Foo::Foo}, decl\n"
+           "2: targets = {Foo}\n"
+           "3: targets = {foo()::Foo::field}, decl\n"
+           "4: targets = {Var}, decl\n"
+           "5: targets = {E}, decl\n"
+           "6: targets = {foo()::ABC}, decl\n"
+           "7: targets = {INT}, decl\n"
+           "8: targets = {INT2}, decl\n"
+           "9: targets = {NS}, decl\n"
+           "10: targets = {ns}\n"},
+          // User-defined conversion operator.
+          {R"cpp(
             void foo() {
                class $0^Bar {};
                class $1^Foo {
@@ -1043,18 +1125,18 @@ TEST_F(FindExplicitReferencesTest, All) {
                $7^f.$8^operator $9^Bar();
             }
         )cpp",
-        "0: targets = {Bar}, decl\n"
-        "1: targets = {Foo}, decl\n"
-        "2: targets = {foo()::Foo::operator Bar}, decl\n"
-        "3: targets = {Bar}\n"
-        "4: targets = {Bar}\n"
-        "5: targets = {Foo}\n"
-        "6: targets = {f}, decl\n"
-        "7: targets = {f}\n"
-        "8: targets = {foo()::Foo::operator Bar}\n"
-        "9: targets = {Bar}\n"},
-       // Destructor.
-       {R"cpp(
+           "0: targets = {Bar}, decl\n"
+           "1: targets = {Foo}, decl\n"
+           "2: targets = {foo()::Foo::operator Bar}, decl\n"
+           "3: targets = {Bar}\n"
+           "4: targets = {Bar}\n"
+           "5: targets = {Foo}\n"
+           "6: targets = {f}, decl\n"
+           "7: targets = {f}\n"
+           "8: targets = {foo()::Foo::operator Bar}\n"
+           "9: targets = {Bar}\n"},
+          // Destructor.
+          {R"cpp(
              void foo() {
                class $0^Foo {
                public:
@@ -1069,18 +1151,18 @@ TEST_F(FindExplicitReferencesTest, All) {
                $6^f.~ /*...*/ $7^Foo();
              }
            )cpp",
-        "0: targets = {Foo}, decl\n"
-        // FIXME: It's better to target destructor's FunctionDecl instead of
-        // the type itself (similar to constructor).
-        "1: targets = {Foo}\n"
-        "2: targets = {foo()::Foo::destructMe}, decl\n"
-        "3: targets = {Foo}\n"
-        "4: targets = {Foo}\n"
-        "5: targets = {f}, decl\n"
-        "6: targets = {f}\n"
-        "7: targets = {Foo}\n"},
-       // cxx constructor initializer.
-       {R"cpp(
+           "0: targets = {Foo}, decl\n"
+           // FIXME: It's better to target destructor's FunctionDecl instead of
+           // the type itself (similar to constructor).
+           "1: targets = {Foo}\n"
+           "2: targets = {foo()::Foo::destructMe}, decl\n"
+           "3: targets = {Foo}\n"
+           "4: targets = {Foo}\n"
+           "5: targets = {f}, decl\n"
+           "6: targets = {f}\n"
+           "7: targets = {Foo}\n"},
+          // cxx constructor initializer.
+          {R"cpp(
              class Base {};
              void foo() {
                // member initializer
@@ -1100,34 +1182,34 @@ TEST_F(FindExplicitReferencesTest, All) {
                };
              }
            )cpp",
-        "0: targets = {X}, decl\n"
-        "1: targets = {foo()::X::abc}, decl\n"
-        "2: targets = {foo()::X::X}, decl\n"
-        "3: targets = {foo()::X::abc}\n"
-        "4: targets = {Derived}, decl\n"
-        "5: targets = {Base}\n"
-        "6: targets = {Base}\n"
-        "7: targets = {foo()::Derived::B}, decl\n"
-        "8: targets = {foo()::Derived::Derived}, decl\n"
-        "9: targets = {Base}\n"
-        "10: targets = {Foo}, decl\n"
-        "11: targets = {foo()::Foo::Foo}, decl\n"
-        "12: targets = {foo()::Foo::Foo}, decl\n"
-        "13: targets = {Foo}\n"},
-       // Anonymous entities should not be reported.
-       {
-           R"cpp(
+           "0: targets = {X}, decl\n"
+           "1: targets = {foo()::X::abc}, decl\n"
+           "2: targets = {foo()::X::X}, decl\n"
+           "3: targets = {foo()::X::abc}\n"
+           "4: targets = {Derived}, decl\n"
+           "5: targets = {Base}\n"
+           "6: targets = {Base}\n"
+           "7: targets = {foo()::Derived::B}, decl\n"
+           "8: targets = {foo()::Derived::Derived}, decl\n"
+           "9: targets = {Base}\n"
+           "10: targets = {Foo}, decl\n"
+           "11: targets = {foo()::Foo::Foo}, decl\n"
+           "12: targets = {foo()::Foo::Foo}, decl\n"
+           "13: targets = {Foo}\n"},
+          // Anonymous entities should not be reported.
+          {
+              R"cpp(
              void foo() {
               class {} $0^x;
               int (*$1^fptr)(int $2^a, int) = nullptr;
              }
            )cpp",
-           "0: targets = {x}, decl\n"
-           "1: targets = {fptr}, decl\n"
-           "2: targets = {a}, decl\n"},
-       // Namespace aliases should be handled properly.
-       {
-           R"cpp(
+              "0: targets = {x}, decl\n"
+              "1: targets = {fptr}, decl\n"
+              "2: targets = {a}, decl\n"},
+          // Namespace aliases should be handled properly.
+          {
+              R"cpp(
                 namespace ns { struct Type {}; }
                 namespace alias = ns;
                 namespace rec_alias = alias;
@@ -1138,28 +1220,28 @@ TEST_F(FindExplicitReferencesTest, All) {
                   $6^rec_alias::$7^Type $8^c;
                 }
            )cpp",
-           "0: targets = {ns}\n"
-           "1: targets = {ns::Type}, qualifier = 'ns::'\n"
-           "2: targets = {a}, decl\n"
-           "3: targets = {alias}\n"
-           "4: targets = {ns::Type}, qualifier = 'alias::'\n"
-           "5: targets = {b}, decl\n"
-           "6: targets = {rec_alias}\n"
-           "7: targets = {ns::Type}, qualifier = 'rec_alias::'\n"
-           "8: targets = {c}, decl\n"},
-       // Handle SizeOfPackExpr.
-       {
-           R"cpp(
+              "0: targets = {ns}\n"
+              "1: targets = {ns::Type}, qualifier = 'ns::'\n"
+              "2: targets = {a}, decl\n"
+              "3: targets = {alias}\n"
+              "4: targets = {ns::Type}, qualifier = 'alias::'\n"
+              "5: targets = {b}, decl\n"
+              "6: targets = {rec_alias}\n"
+              "7: targets = {ns::Type}, qualifier = 'rec_alias::'\n"
+              "8: targets = {c}, decl\n"},
+          // Handle SizeOfPackExpr.
+          {
+              R"cpp(
                 template <typename... E>
                 void foo() {
                   constexpr int $0^size = sizeof...($1^E);
                 };
             )cpp",
-           "0: targets = {size}, decl\n"
-           "1: targets = {E}\n"},
-       // Class template argument deduction
-       {
-           R"cpp(
+              "0: targets = {size}, decl\n"
+              "1: targets = {E}\n"},
+          // Class template argument deduction
+          {
+              R"cpp(
                 template <typename T>
                 struct Test {
                 Test(T);
@@ -1168,51 +1250,51 @@ TEST_F(FindExplicitReferencesTest, All) {
                 $0^Test $1^a(5);
               }
             )cpp",
-           "0: targets = {Test}\n"
-           "1: targets = {a}, decl\n"},
-       // Templates
-       {R"cpp(
+              "0: targets = {Test}\n"
+              "1: targets = {a}, decl\n"},
+          // Templates
+          {R"cpp(
             namespace foo {
               template <typename $0^T>
               class $1^Bar {};
             }
           )cpp",
-        "0: targets = {foo::Bar::T}, decl\n"
-        "1: targets = {foo::Bar}, decl\n"},
-       // Templates
-       {R"cpp(
+           "0: targets = {foo::Bar::T}, decl\n"
+           "1: targets = {foo::Bar}, decl\n"},
+          // Templates
+          {R"cpp(
             namespace foo {
               template <typename $0^T>
               void $1^func();
             }
           )cpp",
-        "0: targets = {T}, decl\n"
-        "1: targets = {foo::func}, decl\n"},
-       // Templates
-       {R"cpp(
+           "0: targets = {T}, decl\n"
+           "1: targets = {foo::func}, decl\n"},
+          // Templates
+          {R"cpp(
             namespace foo {
               template <typename $0^T>
               $1^T $2^x;
             }
           )cpp",
-        "0: targets = {foo::T}, decl\n"
-        "1: targets = {foo::T}\n"
-        "2: targets = {foo::x}, decl\n"},
-       // Templates
-       {R"cpp(
+           "0: targets = {foo::T}, decl\n"
+           "1: targets = {foo::T}\n"
+           "2: targets = {foo::x}, decl\n"},
+          // Templates
+          {R"cpp(
             template<typename T> class vector {};
             namespace foo {
               template <typename $0^T>
               using $1^V = $2^vector<$3^T>;
             }
           )cpp",
-        "0: targets = {foo::T}, decl\n"
-        "1: targets = {foo::V}, decl\n"
-        "2: targets = {vector}\n"
-        "3: targets = {foo::T}\n"},
-       // Concept
-       {
-           R"cpp(
+           "0: targets = {foo::T}, decl\n"
+           "1: targets = {foo::V}, decl\n"
+           "2: targets = {vector}\n"
+           "3: targets = {foo::T}\n"},
+          // Concept
+          {
+              R"cpp(
               template <typename T>
               concept Drawable = requires (T t) { t.draw(); };
 
@@ -1223,17 +1305,17 @@ TEST_F(FindExplicitReferencesTest, All) {
                 }
               }
           )cpp",
-           "0: targets = {T}, decl\n"
-           "1: targets = {Drawable}\n"
-           "2: targets = {T}\n"
-           "3: targets = {foo::bar}, decl\n"
-           "4: targets = {T}\n"
-           "5: targets = {t}, decl\n"
-           "6: targets = {t}\n"
-           "7: targets = {}\n"},
-       // Objective-C: properties
-       {
-           R"cpp(
+              "0: targets = {T}, decl\n"
+              "1: targets = {Drawable}\n"
+              "2: targets = {T}\n"
+              "3: targets = {foo::bar}, decl\n"
+              "4: targets = {T}\n"
+              "5: targets = {t}, decl\n"
+              "6: targets = {t}\n"
+              "7: targets = {}\n"},
+          // Objective-C: properties
+          {
+              R"cpp(
             @interface I {}
             @property(retain) I* x;
             @property(retain) I* y;
@@ -1243,12 +1325,12 @@ TEST_F(FindExplicitReferencesTest, All) {
               $0^f.$1^x.$2^y = 0;
             }
           )cpp",
-           "0: targets = {f}\n"
-           "1: targets = {I::x}\n"
-           "2: targets = {I::y}\n"},
-       // Objective-C: implicit properties
-       {
-           R"cpp(
+              "0: targets = {f}\n"
+              "1: targets = {I::x}\n"
+              "2: targets = {I::y}\n"},
+          // Objective-C: implicit properties
+          {
+              R"cpp(
             @interface I {}
             -(I*)x;
             -(void)setY:(I*)y;
@@ -1258,11 +1340,11 @@ TEST_F(FindExplicitReferencesTest, All) {
               $0^f.$1^x.$2^y = 0;
             }
           )cpp",
-           "0: targets = {f}\n"
-           "1: targets = {I::x}\n"
-           "2: targets = {I::setY:}\n"},
-       // Designated initializers.
-       {R"cpp(
+              "0: targets = {f}\n"
+              "1: targets = {I::x}\n"
+              "2: targets = {I::setY:}\n"},
+          // Designated initializers.
+          {R"cpp(
             void foo() {
               struct $0^Foo {
                 int $1^Bar;
@@ -1270,12 +1352,12 @@ TEST_F(FindExplicitReferencesTest, All) {
               $2^Foo $3^f { .$4^Bar = 42 };
             }
         )cpp",
-        "0: targets = {Foo}, decl\n"
-        "1: targets = {foo()::Foo::Bar}, decl\n"
-        "2: targets = {Foo}\n"
-        "3: targets = {f}, decl\n"
-        "4: targets = {foo()::Foo::Bar}\n"},
-       {R"cpp(
+           "0: targets = {Foo}, decl\n"
+           "1: targets = {foo()::Foo::Bar}, decl\n"
+           "2: targets = {Foo}\n"
+           "3: targets = {f}, decl\n"
+           "4: targets = {foo()::Foo::Bar}\n"},
+          {R"cpp(
             void foo() {
               struct $0^Baz {
                 int $1^Field;
@@ -1295,7 +1377,7 @@ TEST_F(FindExplicitReferencesTest, All) {
         "6: targets = {bar}, decl\n"
         "7: targets = {foo()::Bar::Foo}\n"
         "8: targets = {foo()::Baz::Field}\n"},
-      {R"cpp(
+       {R"cpp(
            template<typename T>
            void crash(T);
            template<typename T>
@@ -1305,10 +1387,9 @@ TEST_F(FindExplicitReferencesTest, All) {
         )cpp",
         "0: targets = {crash}\n"
         "1: targets = {}\n"
-        "2: targets = {T}\n"
-      },
-      // unknown template name should not crash.
-      {R"cpp(
+        "2: targets = {T}\n"},
+       // unknown template name should not crash.
+       {R"cpp(
         template <template <typename> typename T>
         struct Base {};
         namespace foo {
@@ -1316,12 +1397,34 @@ TEST_F(FindExplicitReferencesTest, All) {
         struct $1^Derive : $2^Base<$3^T::template $4^Unknown> {};
         }
       )cpp",
-      "0: targets = {foo::Derive::T}, decl\n"
-      "1: targets = {foo::Derive}, decl\n"
-      "2: targets = {Base}\n"
-      "3: targets = {foo::Derive::T}\n"
-      "4: targets = {}, qualifier = 'T::'\n"},
-    };
+        "0: targets = {foo::Derive::T}, decl\n"
+        "1: targets = {foo::Derive}, decl\n"
+        "2: targets = {Base}\n"
+        "3: targets = {foo::Derive::T}\n"
+        "4: targets = {}, qualifier = 'T::'\n"},
+       // deduction guide
+       {R"cpp(
+          namespace foo {
+            template <typename $0^T>
+            struct $1^Test {
+              template <typename $2^I>
+              $3^Test($4^I);
+            };
+            template <typename $5^I>
+            $6^Test($7^I) -> $8^Test<typename $9^I::$10^type>;
+          }
+        )cpp",
+        "0: targets = {T}, decl\n"
+        "1: targets = {foo::Test}, decl\n"
+        "2: targets = {I}, decl\n"
+        "3: targets = {foo::Test::Test<T>}, decl\n"
+        "4: targets = {I}\n"
+        "5: targets = {I}, decl\n"
+        "6: targets = {foo::Test}\n"
+        "7: targets = {I}\n"
+        "8: targets = {foo::Test}\n"
+        "9: targets = {I}\n"
+        "10: targets = {}, qualifier = 'I::'\n"}};
 
   for (const auto &C : Cases) {
     llvm::StringRef ExpectedCode = C.first;
diff --git a/clang-tools-extra/clangd/unittests/HoverTests.cpp b/clang-tools-extra/clangd/unittests/HoverTests.cpp
index 19ab6d63947b7..636e5d99be522 100644
--- a/clang-tools-extra/clangd/unittests/HoverTests.cpp
+++ b/clang-tools-extra/clangd/unittests/HoverTests.cpp
@@ -957,6 +957,16 @@ TEST(Hover, NoHover) {
           template <typename T> void foo() {
             (void)[[size^of]](T);
           })cpp",
+      R"cpp(// should not crash on invalid semantic form of init-list-expr.
+            /*error-ok*/
+            struct Foo {
+              int xyz = 0;
+            };
+            class Bar {};
+            constexpr Foo s = ^{
+              .xyz = Bar(),
+            };
+          )cpp",
       // literals
       "auto x = t^rue;",
       "auto x = '^A';",
diff --git a/clang-tools-extra/clangd/unittests/SelectionTests.cpp b/clang-tools-extra/clangd/unittests/SelectionTests.cpp
index 051580ba6e49b..6a9f587a77688 100644
--- a/clang-tools-extra/clangd/unittests/SelectionTests.cpp
+++ b/clang-tools-extra/clangd/unittests/SelectionTests.cpp
@@ -177,11 +177,29 @@ TEST(SelectionTest, CommonAncestor) {
       {
           R"cpp(
             void foo();
-            #define CALL_FUNCTION(X) X^()^
+            #^define CALL_FUNCTION(X) X(^)
             void bar() { CALL_FUNCTION(foo); }
           )cpp",
           nullptr,
       },
+      {
+          R"cpp(
+            void foo();
+            #define CALL_FUNCTION(X) X()
+            void bar() { CALL_FUNCTION(foo^)^; }
+          )cpp",
+          nullptr,
+      },
+      {
+          R"cpp(
+            namespace ns {
+            #if 0
+            void fo^o() {}
+            #endif
+            }
+          )cpp",
+          nullptr,
+      },
       {
           R"cpp(
             struct S { S(const char*); };
@@ -388,7 +406,8 @@ TEST(SelectionTest, CommonAncestor) {
         void test(S2 s2) {
           s2[[-^>]]f();
         }
-      )cpp", "DeclRefExpr"} // DeclRefExpr to the "operator->" method.
+      )cpp",
+       "DeclRefExpr"} // DeclRefExpr to the "operator->" method.
   };
   for (const Case &C : Cases) {
     trace::TestTracer Tracer;
@@ -538,7 +557,7 @@ TEST(SelectionTest, IncludedFile) {
   auto AST = TU.build();
   auto T = makeSelectionTree(Case, AST);
 
-  EXPECT_EQ("WhileStmt", T.commonAncestor()->kind());
+  EXPECT_EQ(nullptr, T.commonAncestor());
 }
 
 TEST(SelectionTest, MacroArgExpansion) {
diff --git a/clang-tools-extra/clangd/unittests/SymbolCollectorTests.cpp b/clang-tools-extra/clangd/unittests/SymbolCollectorTests.cpp
index d90f99ff9f8ca..9e4f75b5cca3f 100644
--- a/clang-tools-extra/clangd/unittests/SymbolCollectorTests.cpp
+++ b/clang-tools-extra/clangd/unittests/SymbolCollectorTests.cpp
@@ -1402,6 +1402,9 @@ TEST_F(SymbolCollectorTest, Origin) {
   runSymbolCollector("class Foo {};", /*Main=*/"");
   EXPECT_THAT(Symbols, UnorderedElementsAre(
                            Field(&Symbol::Origin, SymbolOrigin::Static)));
+  runSymbolCollector("#define FOO", /*Main=*/"");
+  EXPECT_THAT(Symbols, UnorderedElementsAre(
+                           Field(&Symbol::Origin, SymbolOrigin::Static)));
 }
 
 TEST_F(SymbolCollectorTest, CollectMacros) {
@@ -1504,6 +1507,13 @@ TEST_F(SymbolCollectorTest, BadUTF8) {
   EXPECT_THAT(Refs, Contains(Pair(findSymbol(Symbols, "PUNCT").ID, _)));
 }
 
+TEST_F(SymbolCollectorTest, MacrosInHeaders) {
+  CollectorOpts.CollectMacro = true;
+  TestFileName = testPath("test.h");
+  runSymbolCollector("", "#define X");
+  EXPECT_THAT(Symbols,
+              UnorderedElementsAre(AllOf(QName("X"), ForCodeCompletion(true))));
+}
 } // namespace
 } // namespace clangd
 } // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/TypeHierarchyTests.cpp b/clang-tools-extra/clangd/unittests/TypeHierarchyTests.cpp
index 73e124d79b07d..8e615e215fc52 100644
--- a/clang-tools-extra/clangd/unittests/TypeHierarchyTests.cpp
+++ b/clang-tools-extra/clangd/unittests/TypeHierarchyTests.cpp
@@ -523,6 +523,33 @@ TEST(TypeHierarchy, DeriveFromTemplate) {
                                    WithKind(SymbolKind::Struct), Children()))));
 }
 
+TEST(TypeHierarchy, Preamble) {
+  Annotations SourceAnnotations(R"cpp(
+struct Ch^ild : Parent {
+  int b;
+};)cpp");
+
+  Annotations HeaderInPreambleAnnotations(R"cpp(
+struct [[Parent]] {
+  int a;
+};)cpp");
+
+  TestTU TU = TestTU::withCode(SourceAnnotations.code());
+  TU.HeaderCode = HeaderInPreambleAnnotations.code().str();
+  auto AST = TU.build();
+
+  llvm::Optional<TypeHierarchyItem> Result = getTypeHierarchy(
+      AST, SourceAnnotations.point(), 1, TypeHierarchyDirection::Parents);
+
+  ASSERT_TRUE(Result);
+  EXPECT_THAT(
+      *Result,
+      AllOf(WithName("Child"),
+            Parents(AllOf(WithName("Parent"),
+                          SelectionRangeIs(HeaderInPreambleAnnotations.range()),
+                          Parents()))));
+}
+
 SymbolID findSymbolIDByName(SymbolIndex *Index, llvm::StringRef Name,
                             llvm::StringRef TemplateArgs = "") {
   SymbolID Result;
diff --git a/clang-tools-extra/clangd/unittests/URITests.cpp b/clang-tools-extra/clangd/unittests/URITests.cpp
index 52ca7b4447cd8..cd734cfe0837a 100644
--- a/clang-tools-extra/clangd/unittests/URITests.cpp
+++ b/clang-tools-extra/clangd/unittests/URITests.cpp
@@ -76,6 +76,16 @@ TEST(URITest, Create) {
 #endif
 }
 
+TEST(URITest, CreateUNC) {
+#ifdef _WIN32
+  EXPECT_THAT(createOrDie("\\\\test.org\\x\\y\\z"), "file://test.org/x/y/z");
+  EXPECT_THAT(createOrDie("\\\\10.0.0.1\\x\\y\\z"), "file://10.0.0.1/x/y/z");
+#else
+  EXPECT_THAT(createOrDie("//test.org/x/y/z"), "file://test.org/x/y/z");
+  EXPECT_THAT(createOrDie("//10.0.0.1/x/y/z"), "file://10.0.0.1/x/y/z");
+#endif
+}
+
 TEST(URITest, FailedCreate) {
   EXPECT_ERROR(URI::create("/x/y/z", "no"));
   // Path has to be absolute.
@@ -127,15 +137,32 @@ TEST(URITest, Resolve) {
   EXPECT_THAT(resolveOrDie(parseOrDie("file:///c:/x/y/z")), "c:\\x\\y\\z");
 #else
   EXPECT_EQ(resolveOrDie(parseOrDie("file:/a/b/c")), "/a/b/c");
-  EXPECT_EQ(resolveOrDie(parseOrDie("file://auth/a/b/c")), "/a/b/c");
+  EXPECT_EQ(resolveOrDie(parseOrDie("file://auth/a/b/c")), "//auth/a/b/c");
   EXPECT_THAT(resolveOrDie(parseOrDie("file://au%3dth/%28x%29/y/%20z")),
-              "/(x)/y/ z");
+              "//au=th/(x)/y/ z");
   EXPECT_THAT(resolveOrDie(parseOrDie("file:///c:/x/y/z")), "c:/x/y/z");
 #endif
   EXPECT_EQ(resolveOrDie(parseOrDie("unittest:///a"), testPath("x")),
             testPath("a"));
 }
 
+TEST(URITest, ResolveUNC) {
+#ifdef _WIN32
+  EXPECT_THAT(resolveOrDie(parseOrDie("file://example.com/x/y/z")),
+              "\\\\example.com\\x\\y\\z");
+  EXPECT_THAT(resolveOrDie(parseOrDie("file://127.0.0.1/x/y/z")),
+              "\\\\127.0.0.1\\x\\y\\z");
+  // Ensure non-traditional file URI still resolves to correct UNC path.
+  EXPECT_THAT(resolveOrDie(parseOrDie("file:////127.0.0.1/x/y/z")),
+              "\\\\127.0.0.1\\x\\y\\z");
+#else
+  EXPECT_THAT(resolveOrDie(parseOrDie("file://example.com/x/y/z")),
+              "//example.com/x/y/z");
+  EXPECT_THAT(resolveOrDie(parseOrDie("file://127.0.0.1/x/y/z")),
+              "//127.0.0.1/x/y/z");
+#endif
+}
+
 std::string resolvePathOrDie(llvm::StringRef AbsPath,
                              llvm::StringRef HintPath = "") {
   auto Path = URI::resolvePath(AbsPath, HintPath);
diff --git a/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp b/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
index a14ff13150d37..7db7c03d61c9b 100644
--- a/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
+++ b/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
@@ -13,6 +13,7 @@
 #include "index/Serialization.h"
 #include "index/Symbol.h"
 #include "index/SymbolID.h"
+#include "index/SymbolLocation.h"
 #include "index/remote/marshalling/Marshalling.h"
 #include "clang/Index/IndexSymbol.h"
 #include "llvm/ADT/SmallString.h"
@@ -39,29 +40,35 @@ const char *testPathURI(llvm::StringRef Path,
 TEST(RemoteMarshallingTest, URITranslation) {
   llvm::BumpPtrAllocator Arena;
   llvm::UniqueStringSaver Strings(Arena);
+  Marshaller ProtobufMarshaller(
+      testPath("remote/machine/projects/llvm-project/"),
+      testPath("home/my-projects/llvm-project/"));
   clangd::Ref Original;
   Original.Location.FileURI =
       testPathURI("remote/machine/projects/llvm-project/clang-tools-extra/"
                   "clangd/unittests/remote/MarshallingTests.cpp",
                   Strings);
-  auto Serialized =
-      toProtobuf(Original, testPath("remote/machine/projects/llvm-project/"));
-  EXPECT_EQ(Serialized.location().file_path(),
+  auto Serialized = ProtobufMarshaller.toProtobuf(Original);
+  EXPECT_TRUE(Serialized);
+  EXPECT_EQ(Serialized->location().file_path(),
             "clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp");
-  const std::string LocalIndexPrefix = testPath("local/machine/project/");
-  auto Deserialized = fromProtobuf(Serialized, &Strings,
-                                   testPath("home/my-projects/llvm-project/"));
+  auto Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
   EXPECT_TRUE(Deserialized);
-  EXPECT_EQ(Deserialized->Location.FileURI,
-            testPathURI("home/my-projects/llvm-project/clang-tools-extra/"
-                        "clangd/unittests/remote/MarshallingTests.cpp",
-                        Strings));
+  EXPECT_STREQ(Deserialized->Location.FileURI,
+               testPathURI("home/my-projects/llvm-project/clang-tools-extra/"
+                           "clangd/unittests/remote/MarshallingTests.cpp",
+                           Strings));
+
+  // Can't have empty paths.
+  *Serialized->mutable_location()->mutable_file_path() = std::string();
+  Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
+  EXPECT_FALSE(Deserialized);
 
   clangd::Ref WithInvalidURI;
-  // Invalid URI results in empty path.
+  // Invalid URI results in serialization failure.
   WithInvalidURI.Location.FileURI = "This is not a URI";
-  Serialized = toProtobuf(WithInvalidURI, testPath("home/"));
-  EXPECT_EQ(Serialized.location().file_path(), "");
+  Serialized = ProtobufMarshaller.toProtobuf(WithInvalidURI);
+  EXPECT_FALSE(Serialized);
 
   // Can not use URIs with scheme different from "file".
   auto UnittestURI =
@@ -69,15 +76,15 @@ TEST(RemoteMarshallingTest, URITranslation) {
   EXPECT_TRUE(bool(UnittestURI));
   WithInvalidURI.Location.FileURI =
       Strings.save(UnittestURI->toString()).begin();
-  Serialized = toProtobuf(WithInvalidURI, testPath("project/lib/"));
-  EXPECT_EQ(Serialized.location().file_path(), "");
+  Serialized = ProtobufMarshaller.toProtobuf(WithInvalidURI);
+  EXPECT_FALSE(Serialized);
 
+  // Paths transmitted over the wire can not be absolute, they have to be
+  // relative.
   Ref WithAbsolutePath;
   *WithAbsolutePath.mutable_location()->mutable_file_path() =
       "/usr/local/user/home/HelloWorld.cpp";
-  Deserialized = fromProtobuf(WithAbsolutePath, &Strings, LocalIndexPrefix);
-  // Paths transmitted over the wire can not be absolute, they have to be
-  // relative.
+  Deserialized = ProtobufMarshaller.fromProtobuf(WithAbsolutePath);
   EXPECT_FALSE(Deserialized);
 }
 
@@ -128,48 +135,63 @@ TEST(RemoteMarshallingTest, SymbolSerialization) {
 
   Sym.Flags = clangd::Symbol::SymbolFlag::IndexedForCodeCompletion;
 
+  Marshaller ProtobufMarshaller(testPath("home/"), testPath("home/"));
+
   // Check that symbols are exactly the same if the path to indexed project is
   // the same on indexing machine and the client.
-  auto Serialized = toProtobuf(Sym, testPath("home/"));
-  auto Deserialized = fromProtobuf(Serialized, &Strings, testPath("home/"));
+  auto Serialized = ProtobufMarshaller.toProtobuf(Sym);
+  EXPECT_TRUE(Serialized);
+  auto Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
   EXPECT_TRUE(Deserialized);
   EXPECT_EQ(toYAML(Sym), toYAML(*Deserialized));
   // Serialized paths are relative and have UNIX slashes.
-  EXPECT_EQ(convert_to_slash(Serialized.definition().file_path(),
+  EXPECT_EQ(convert_to_slash(Serialized->definition().file_path(),
                              llvm::sys::path::Style::posix),
-            Serialized.definition().file_path());
+            Serialized->definition().file_path());
   EXPECT_TRUE(
-      llvm::sys::path::is_relative(Serialized.definition().file_path()));
+      llvm::sys::path::is_relative(Serialized->definition().file_path()));
+
+  // Missing definition is OK.
+  Sym.Definition = clangd::SymbolLocation();
+  Serialized = ProtobufMarshaller.toProtobuf(Sym);
+  EXPECT_TRUE(Serialized);
+  Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
+  EXPECT_TRUE(Deserialized);
+
+  // Relative path is absolute.
+  *Serialized->mutable_canonical_declaration()->mutable_file_path() =
+      convert_to_slash("/path/to/Declaration.h");
+  Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
+  EXPECT_FALSE(Deserialized);
 
   // Fail with an invalid URI.
   Location.FileURI = "Not A URI";
   Sym.Definition = Location;
-  Serialized = toProtobuf(Sym, testPath("home/"));
-  Deserialized = fromProtobuf(Serialized, &Strings, testPath("home/"));
-  EXPECT_FALSE(Deserialized);
+  Serialized = ProtobufMarshaller.toProtobuf(Sym);
+  EXPECT_FALSE(Serialized);
 
   // Schemes other than "file" can not be used.
   auto UnittestURI = URI::create(testPath("home/SomePath.h"), "unittest");
   EXPECT_TRUE(bool(UnittestURI));
   Location.FileURI = Strings.save(UnittestURI->toString()).begin();
   Sym.Definition = Location;
-  Serialized = toProtobuf(Sym, testPath("home/"));
-  Deserialized = fromProtobuf(Serialized, &Strings, testPath("home/"));
-  EXPECT_FALSE(Deserialized);
+  Serialized = ProtobufMarshaller.toProtobuf(Sym);
+  EXPECT_FALSE(Serialized);
 
   // Passing root that is not prefix of the original file path.
   Location.FileURI = testPathURI("home/File.h", Strings);
   Sym.Definition = Location;
   // Check that the symbol is valid and passing the correct path works.
-  Serialized = toProtobuf(Sym, testPath("home/"));
-  Deserialized = fromProtobuf(Serialized, &Strings, testPath("home/"));
+  Serialized = ProtobufMarshaller.toProtobuf(Sym);
+  EXPECT_TRUE(Serialized);
+  Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
   EXPECT_TRUE(Deserialized);
-  EXPECT_EQ(Deserialized->Definition.FileURI,
-            testPathURI("home/File.h", Strings));
+  EXPECT_STREQ(Deserialized->Definition.FileURI,
+               testPathURI("home/File.h", Strings));
   // Fail with a wrong root.
-  Serialized = toProtobuf(Sym, testPath("nothome/"));
-  Deserialized = fromProtobuf(Serialized, &Strings, testPath("home/"));
-  EXPECT_FALSE(Deserialized);
+  Marshaller WrongMarshaller(testPath("nothome/"), testPath("home/"));
+  Serialized = WrongMarshaller.toProtobuf(Sym);
+  EXPECT_FALSE(Serialized);
 }
 
 TEST(RemoteMarshallingTest, RefSerialization) {
@@ -188,9 +210,12 @@ TEST(RemoteMarshallingTest, RefSerialization) {
       "llvm-project/llvm/clang-tools-extra/clangd/Protocol.h", Strings);
   Ref.Location = Location;
 
-  auto Serialized = toProtobuf(Ref, testPath("llvm-project/"));
-  auto Deserialized =
-      fromProtobuf(Serialized, &Strings, testPath("llvm-project/"));
+  Marshaller ProtobufMarshaller(testPath("llvm-project/"),
+                                testPath("llvm-project/"));
+
+  auto Serialized = ProtobufMarshaller.toProtobuf(Ref);
+  EXPECT_TRUE(Serialized);
+  auto Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
   EXPECT_TRUE(Deserialized);
   EXPECT_EQ(toYAML(Ref), toYAML(*Deserialized));
 }
@@ -242,10 +267,13 @@ TEST(RemoteMarshallingTest, IncludeHeaderURIs) {
                     InvalidHeaders.end());
   Sym.IncludeHeaders = AllHeaders;
 
-  auto Serialized = toProtobuf(Sym, convert_to_slash("/"));
-  EXPECT_EQ(static_cast<size_t>(Serialized.headers_size()),
+  Marshaller ProtobufMarshaller(convert_to_slash("/"), convert_to_slash("/"));
+
+  auto Serialized = ProtobufMarshaller.toProtobuf(Sym);
+  EXPECT_EQ(static_cast<size_t>(Serialized->headers_size()),
             ValidHeaders.size());
-  auto Deserialized = fromProtobuf(Serialized, &Strings, convert_to_slash("/"));
+  EXPECT_TRUE(Serialized);
+  auto Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
   EXPECT_TRUE(Deserialized);
 
   Sym.IncludeHeaders = ValidHeaders;
@@ -254,38 +282,41 @@ TEST(RemoteMarshallingTest, IncludeHeaderURIs) {
 
 TEST(RemoteMarshallingTest, FuzzyFindRequestSerialization) {
   clangd::FuzzyFindRequest Request;
-  Request.ProximityPaths = {testPath("remote/Header.h"),
-                            testPath("remote/subdir/OtherHeader.h"),
-                            testPath("notremote/File.h"), "Not a Path."};
-  auto Serialized = toProtobuf(Request, testPath("remote/"));
+  Request.ProximityPaths = {testPath("local/Header.h"),
+                            testPath("local/subdir/OtherHeader.h"),
+                            testPath("remote/File.h"), "Not a Path."};
+  Marshaller ProtobufMarshaller(testPath("remote/"), testPath("local/"));
+  auto Serialized = ProtobufMarshaller.toProtobuf(Request);
   EXPECT_EQ(Serialized.proximity_paths_size(), 2);
-  auto Deserialized = fromProtobuf(&Serialized, testPath("home/"));
+  auto Deserialized = ProtobufMarshaller.fromProtobuf(&Serialized);
   EXPECT_THAT(Deserialized.ProximityPaths,
-              testing::ElementsAre(testPath("home/Header.h"),
-                                   testPath("home/subdir/OtherHeader.h")));
+              testing::ElementsAre(testPath("remote/Header.h"),
+                                   testPath("remote/subdir/OtherHeader.h")));
 }
 
 TEST(RemoteMarshallingTest, RelativePathToURITranslation) {
-  EXPECT_TRUE(relativePathToURI("lib/File.cpp", testPath("home/project/")));
+  Marshaller ProtobufMarshaller(/*RemoteIndexRoot=*/"",
+                                /*LocalIndexRoot=*/testPath("home/project/"));
+  EXPECT_TRUE(ProtobufMarshaller.relativePathToURI("lib/File.cpp"));
   // RelativePath can not be absolute.
-  EXPECT_FALSE(relativePathToURI("/lib/File.cpp", testPath("home/project/")));
-  // IndexRoot has to be absolute path.
-  EXPECT_FALSE(relativePathToURI("lib/File.cpp", "home/project/"));
+  EXPECT_FALSE(ProtobufMarshaller.relativePathToURI("/lib/File.cpp"));
+  // RelativePath can not be empty.
+  EXPECT_FALSE(ProtobufMarshaller.relativePathToURI(std::string()));
 }
 
 TEST(RemoteMarshallingTest, URIToRelativePathTranslation) {
   llvm::BumpPtrAllocator Arena;
   llvm::UniqueStringSaver Strings(Arena);
-  EXPECT_TRUE(
-      uriToRelativePath(testPathURI("home/project/lib/File.cpp", Strings),
-                        testPath("home/project/")));
-  // IndexRoot has to be absolute path.
-  EXPECT_FALSE(uriToRelativePath(
-      testPathURI("home/project/lib/File.cpp", Strings), "home/project/"));
-  // IndexRoot has to be be a prefix of the file path.
-  EXPECT_FALSE(
-      uriToRelativePath(testPathURI("home/project/lib/File.cpp", Strings),
-                        testPath("home/other/project/")));
+  Marshaller ProtobufMarshaller(/*RemoteIndexRoot=*/testPath("remote/project/"),
+                                /*LocalIndexRoot=*/"");
+  EXPECT_TRUE(ProtobufMarshaller.uriToRelativePath(
+      testPathURI("remote/project/lib/File.cpp", Strings)));
+  // RemoteIndexRoot has to be be a prefix of the file path.
+  Marshaller WrongMarshaller(
+      /*RemoteIndexRoot=*/testPath("remote/other/project/"),
+      /*LocalIndexRoot=*/"");
+  EXPECT_FALSE(WrongMarshaller.uriToRelativePath(
+      testPathURI("remote/project/lib/File.cpp", Strings)));
 }
 
 } // namespace
diff --git a/clang-tools-extra/unittests/CMakeLists.txt b/clang-tools-extra/unittests/CMakeLists.txt
index 086a68e638307..72abe0fa6d0c1 100644
--- a/clang-tools-extra/unittests/CMakeLists.txt
+++ b/clang-tools-extra/unittests/CMakeLists.txt
@@ -5,6 +5,10 @@ function(add_extra_unittest test_dirname)
   add_unittest(ExtraToolsUnitTests ${test_dirname} ${ARGN})
 endfunction()
 
+if (CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
+  add_compile_options("-Wno-suggest-override")
+endif()
+
 add_subdirectory(clang-apply-replacements)
 add_subdirectory(clang-change-namespace)
 add_subdirectory(clang-doc)
diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt
index a29e29f8ce5c9..5542250cb278a 100644
--- a/clang/CMakeLists.txt
+++ b/clang/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.4.3)
+cmake_minimum_required(VERSION 3.13.4)
 
 if(POLICY CMP0075)
   cmake_policy(SET CMP0075 NEW)
@@ -114,6 +114,10 @@ if( CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR )
   option(CLANG_ENABLE_BOOTSTRAP "Generate the clang bootstrap target" OFF)
   option(LLVM_ENABLE_LIBXML2 "Use libxml2 if available." ON)
 
+  if(LLVM_ENABLE_ZLIB)
+    find_package(ZLIB)
+  endif()
+
   include(AddLLVM)
   include(TableGen)
   include(HandleLLVMOptions)
diff --git a/clang/docs/ClangFormatStyleOptions.rst b/clang/docs/ClangFormatStyleOptions.rst
index 6647b117ac596..c35718b51248c 100644
--- a/clang/docs/ClangFormatStyleOptions.rst
+++ b/clang/docs/ClangFormatStyleOptions.rst
@@ -794,6 +794,43 @@ the configuration (without a prefix: ``Auto``).
            int aaaaaaaaaaaaaaaaaaaa,
            int aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa) {}
 
+**BitFieldColonSpacing** (``BitFieldColonSpacingStyle``)
+  The BitFieldColonSpacingStyle to use for bitfields.
+
+  Possible values:
+
+  * ``BFCS_Both`` (in configuration: ``Both``)
+    Add one space on each side of the ``:``
+
+    .. code-block:: c++
+
+      unsigned bf : 2;
+
+  * ``BFCS_None`` (in configuration: ``None``)
+    Add no space around the ``:`` (except when needed for
+    ``AlignConsecutiveBitFields``).
+
+    .. code-block:: c++
+
+      unsigned bf:2;
+
+  * ``BFCS_Before`` (in configuration: ``Before``)
+    Add space before the ``:`` only
+
+    .. code-block:: c++
+
+      unsigned bf :2;
+
+  * ``BFCS_After`` (in configuration: ``After``)
+    Add space after the ``:`` only (space may be added before if
+    needed for ``AlignConsecutiveBitFields``).
+
+    .. code-block:: c++
+
+      unsigned bf: 2;
+
+
+
 **BraceWrapping** (``BraceWrappingFlags``)
   Control of individual brace wrapping cases.
 
diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst
index 06ecc186c7dc2..e12bc2ce96333 100644
--- a/clang/docs/LanguageExtensions.rst
+++ b/clang/docs/LanguageExtensions.rst
@@ -1719,6 +1719,9 @@ This extension also works in C++ mode, as far as that goes, but does not apply
 to the C++ ``std::complex``.  (In C++11, list initialization allows the same
 syntax to be used with ``std::complex`` with the same meaning.)
 
+For GCC compatibility, ``__builtin_complex(re, im)`` can also be used to
+construct a complex number from the given real and imaginary components.
+
 Builtin Functions
 =================
 
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 10ead604239c9..9274081c4d62c 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -63,6 +63,32 @@ New Compiler Flags
 
 - ...
 
+- -fpch-codegen and -fpch-debuginfo generate shared code and/or debuginfo
+  for contents of a precompiled header in a separate object file. This object
+  file needs to be linked in, but its contents do not need to be generated
+  for other objects using the precompiled header. This should usually save
+  compile time. If not using clang-cl, the separate object file needs to
+  be created explicitly from the precompiled header.
+  Example of use:
+
+  .. code-block:: console
+
+    $ clang++ -x c++-header header.h -o header.pch -fpch-codegen -fpch-debuginfo
+    $ clang++ -c header.pch -o shared.o
+    $ clang++ -c source.cpp -o source.o -include-pch header.pch
+    $ clang++ -o binary source.o shared.o
+
+  - Using -fpch-instantiate-templates when generating the precompiled header
+    usually increases the amount of code/debuginfo that can be shared.
+  - In some cases, especially when building with optimizations enabled, using
+    -fpch-codegen may generate so much code in the shared object that compiling
+    it may be a net loss in build time.
+  - Since headers may bring in private symbols of other libraries, it may be
+    sometimes necessary to discard unused symbols (such as by adding
+    -Wl,--gc-sections on ELF platforms to the linking command, and possibly
+    adding -fdata-sections -ffunction-sections to the command generating
+    the shared object).
+
 Deprecated Compiler Flags
 -------------------------
 
@@ -149,7 +175,33 @@ AST Matchers
 clang-format
 ------------
 
-- ...
+- Option ``BitFieldColonSpacing`` has been added that decides how
+  space should be added around identifier, colon and bit-width in
+  bitfield definitions.
+
+  .. code-block:: c++
+
+    // Both (default)
+    struct F {
+      unsigned dscp : 6;
+      unsigned ecn  : 2; // AlignConsecutiveBitFields=true
+    };
+    // None
+    struct F {
+      unsigned dscp:6;
+      unsigned ecn :2;
+    };
+    // Before
+    struct F {
+      unsigned dscp :6;
+      unsigned ecn  :2;
+    };
+    // After
+    struct F {
+      unsigned dscp: 6;
+      unsigned ecn : 2;
+    };
+
 
 libclang
 --------
diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h
index e8c514c204a55..f6720c5492a58 100644
--- a/clang/include/clang/AST/ASTContext.h
+++ b/clang/include/clang/AST/ASTContext.h
@@ -2091,6 +2091,10 @@ class ASTContext : public RefCountedBase<ASTContext> {
     return getTypeSizeInCharsIfKnown(QualType(Ty, 0));
   }
 
+  /// Returns the bitwidth of \p T, an SVE type attributed with
+  /// 'arm_sve_vector_bits'. Should only be called if T->isVLST().
+  unsigned getBitwidthForAttributedSveType(const Type *T) const;
+
   /// Return the ABI-specified alignment of a (complete) type \p T, in
   /// bits.
   unsigned getTypeAlign(QualType T) const { return getTypeInfo(T).Align; }
diff --git a/clang/include/clang/AST/DeclarationName.h b/clang/include/clang/AST/DeclarationName.h
index 82f6868e3a7e1..a037e8b197bc3 100644
--- a/clang/include/clang/AST/DeclarationName.h
+++ b/clang/include/clang/AST/DeclarationName.h
@@ -857,6 +857,16 @@ struct DenseMapInfo<clang::DeclarationName> {
   }
 };
 
+template <> struct PointerLikeTypeTraits<clang::DeclarationName> {
+  static inline void *getAsVoidPointer(clang::DeclarationName P) {
+    return P.getAsOpaquePtr();
+  }
+  static inline clang::DeclarationName getFromVoidPointer(void *P) {
+    return clang::DeclarationName::getFromOpaquePtr(P);
+  }
+  static constexpr int NumLowBitsAvailable = 0;
+};
+
 } // namespace llvm
 
 // The definition of AssumedTemplateStorage is factored out of TemplateName to
diff --git a/clang/include/clang/AST/DependenceFlags.h b/clang/include/clang/AST/DependenceFlags.h
index 14a7ffaecb2b0..ca96b65574bdd 100644
--- a/clang/include/clang/AST/DependenceFlags.h
+++ b/clang/include/clang/AST/DependenceFlags.h
@@ -41,6 +41,7 @@ struct ExprDependenceScope {
     TypeInstantiation = Type | Instantiation,
     ValueInstantiation = Value | Instantiation,
     TypeValueInstantiation = Type | Value | Instantiation,
+    ErrorDependent = Error | ValueInstantiation,
 
     LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/Error)
   };
diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h
index c13b971192850..96db7bc3be290 100644
--- a/clang/include/clang/AST/Expr.h
+++ b/clang/include/clang/AST/Expr.h
@@ -522,16 +522,15 @@ class Expr : public ValueStmt {
   /// semantically correspond to a bool.
   bool isKnownToHaveBooleanValue(bool Semantic = true) const;
 
-  /// isIntegerConstantExpr - Return true if this expression is a valid integer
-  /// constant expression, and, if so, return its value in Result.  If not a
-  /// valid i-c-e, return false and fill in Loc (if specified) with the location
-  /// of the invalid expression.
+  /// isIntegerConstantExpr - Return the value if this expression is a valid
+  /// integer constant expression.  If not a valid i-c-e, return None and fill
+  /// in Loc (if specified) with the location of the invalid expression.
   ///
   /// Note: This does not perform the implicit conversions required by C++11
   /// [expr.const]p5.
-  bool isIntegerConstantExpr(llvm::APSInt &Result, const ASTContext &Ctx,
-                             SourceLocation *Loc = nullptr,
-                             bool isEvaluated = true) const;
+  Optional<llvm::APSInt> getIntegerConstantExpr(const ASTContext &Ctx,
+                                                SourceLocation *Loc = nullptr,
+                                                bool isEvaluated = true) const;
   bool isIntegerConstantExpr(const ASTContext &Ctx,
                              SourceLocation *Loc = nullptr) const;
 
@@ -2273,12 +2272,12 @@ class UnaryOperator final
   /// Is FPFeatures in Trailing Storage?
   bool hasStoredFPFeatures() const { return UnaryOperatorBits.HasFPFeatures; }
 
-protected:
-  /// Get FPFeatures from trailing storage
+  /// Get FPFeatures from trailing storage.
   FPOptionsOverride getStoredFPFeatures() const {
     return getTrailingFPFeatures();
   }
 
+protected:
   /// Set FPFeatures in trailing storage, used only by Serialization
   void setStoredFPFeatures(FPOptionsOverride F) { getTrailingFPFeatures() = F; }
 
@@ -2788,6 +2787,8 @@ class CallExpr : public Expr {
   //
   // * An array of getNumArgs() "Stmt *" for the argument expressions.
   //
+  // * An optional of type FPOptionsOverride.
+  //
   // Note that we store the offset in bytes from the this pointer to the start
   // of the trailing objects. It would be perfectly possible to compute it
   // based on the dynamic kind of the CallExpr. However 1.) we have plenty of
@@ -2809,6 +2810,15 @@ class CallExpr : public Expr {
   /// this pointer to the trailing objects.
   static unsigned offsetToTrailingObjects(StmtClass SC);
 
+  unsigned getSizeOfTrailingStmts() const {
+    return (1 + getNumPreArgs() + getNumArgs()) * sizeof(Stmt *);
+  }
+
+  size_t getOffsetOfTrailingFPFeatures() const {
+    assert(hasStoredFPFeatures());
+    return CallExprBits.OffsetToTrailingObjects + getSizeOfTrailingStmts();
+  }
+
 public:
   enum class ADLCallKind : bool { NotADL, UsesADL };
   static constexpr ADLCallKind NotADL = ADLCallKind::NotADL;
@@ -2819,16 +2829,19 @@ class CallExpr : public Expr {
   /// allocated for the trailing objects.
   CallExpr(StmtClass SC, Expr *Fn, ArrayRef<Expr *> PreArgs,
            ArrayRef<Expr *> Args, QualType Ty, ExprValueKind VK,
-           SourceLocation RParenLoc, unsigned MinNumArgs, ADLCallKind UsesADL);
+           SourceLocation RParenLoc, FPOptionsOverride FPFeatures,
+           unsigned MinNumArgs, ADLCallKind UsesADL);
 
   /// Build an empty call expression, for deserialization.
   CallExpr(StmtClass SC, unsigned NumPreArgs, unsigned NumArgs,
-           EmptyShell Empty);
+           bool hasFPFeatures, EmptyShell Empty);
 
   /// Return the size in bytes needed for the trailing objects.
   /// Used by the derived classes to allocate the right amount of storage.
-  static unsigned sizeOfTrailingObjects(unsigned NumPreArgs, unsigned NumArgs) {
-    return (1 + NumPreArgs + NumArgs) * sizeof(Stmt *);
+  static unsigned sizeOfTrailingObjects(unsigned NumPreArgs, unsigned NumArgs,
+                                        bool HasFPFeatures) {
+    return (1 + NumPreArgs + NumArgs) * sizeof(Stmt *) +
+           HasFPFeatures * sizeof(FPOptionsOverride);
   }
 
   Stmt *getPreArg(unsigned I) {
@@ -2846,22 +2859,43 @@ class CallExpr : public Expr {
 
   unsigned getNumPreArgs() const { return CallExprBits.NumPreArgs; }
 
+  /// Return a pointer to the trailing FPOptions
+  FPOptionsOverride *getTrailingFPFeatures() {
+    assert(hasStoredFPFeatures());
+    return reinterpret_cast<FPOptionsOverride *>(
+        reinterpret_cast<char *>(this) + CallExprBits.OffsetToTrailingObjects +
+        getSizeOfTrailingStmts());
+  }
+  const FPOptionsOverride *getTrailingFPFeatures() const {
+    assert(hasStoredFPFeatures());
+    return reinterpret_cast<const FPOptionsOverride *>(
+        reinterpret_cast<const char *>(this) +
+        CallExprBits.OffsetToTrailingObjects + getSizeOfTrailingStmts());
+  }
+
 public:
-  /// Create a call expression. Fn is the callee expression, Args is the
-  /// argument array, Ty is the type of the call expression (which is *not*
-  /// the return type in general), VK is the value kind of the call expression
-  /// (lvalue, rvalue, ...), and RParenLoc is the location of the right
-  /// parenthese in the call expression. MinNumArgs specifies the minimum
-  /// number of arguments. The actual number of arguments will be the greater
-  /// of Args.size() and MinNumArgs. This is used in a few places to allocate
-  /// enough storage for the default arguments. UsesADL specifies whether the
-  /// callee was found through argument-dependent lookup.
+  /// Create a call expression.
+  /// \param Fn     The callee expression,
+  /// \param Args   The argument array,
+  /// \param Ty     The type of the call expression (which is *not* the return
+  ///               type in general),
+  /// \param VK     The value kind of the call expression (lvalue, rvalue, ...),
+  /// \param RParenLoc  The location of the right parenthesis in the call
+  ///                   expression.
+  /// \param FPFeatures Floating-point features associated with the call,
+  /// \param MinNumArgs Specifies the minimum number of arguments. The actual
+  ///                   number of arguments will be the greater of Args.size()
+  ///                   and MinNumArgs. This is used in a few places to allocate
+  ///                   enough storage for the default arguments.
+  /// \param UsesADL    Specifies whether the callee was found through
+  ///                   argument-dependent lookup.
   ///
   /// Note that you can use CreateTemporary if you need a temporary call
   /// expression on the stack.
   static CallExpr *Create(const ASTContext &Ctx, Expr *Fn,
                           ArrayRef<Expr *> Args, QualType Ty, ExprValueKind VK,
-                          SourceLocation RParenLoc, unsigned MinNumArgs = 0,
+                          SourceLocation RParenLoc,
+                          FPOptionsOverride FPFeatures, unsigned MinNumArgs = 0,
                           ADLCallKind UsesADL = NotADL);
 
   /// Create a temporary call expression with no arguments in the memory
@@ -2878,7 +2912,7 @@ class CallExpr : public Expr {
 
   /// Create an empty call expression, for deserialization.
   static CallExpr *CreateEmpty(const ASTContext &Ctx, unsigned NumArgs,
-                               EmptyShell Empty);
+                               bool HasFPFeatures, EmptyShell Empty);
 
   Expr *getCallee() { return cast<Expr>(getTrailingStmts()[FN]); }
   const Expr *getCallee() const { return cast<Expr>(getTrailingStmts()[FN]); }
@@ -2892,6 +2926,8 @@ class CallExpr : public Expr {
   }
   bool usesADL() const { return getADLCallKind() == UsesADL; }
 
+  bool hasStoredFPFeatures() const { return CallExprBits.HasFPFeatures; }
+
   Decl *getCalleeDecl() { return getCallee()->getReferencedDeclOfCallee(); }
   const Decl *getCalleeDecl() const {
     return getCallee()->getReferencedDeclOfCallee();
@@ -2984,6 +3020,31 @@ class CallExpr : public Expr {
   /// this function call.
   unsigned getNumCommas() const { return getNumArgs() ? getNumArgs() - 1 : 0; }
 
+  /// Get FPOptionsOverride from trailing storage.
+  FPOptionsOverride getStoredFPFeatures() const {
+    assert(hasStoredFPFeatures());
+    return *getTrailingFPFeatures();
+  }
+  /// Set FPOptionsOverride in trailing storage. Used only by Serialization.
+  void setStoredFPFeatures(FPOptionsOverride F) {
+    assert(hasStoredFPFeatures());
+    *getTrailingFPFeatures() = F;
+  }
+
+  // Get the FP features status of this operator. Only meaningful for
+  // operations on floating point types.
+  FPOptions getFPFeaturesInEffect(const LangOptions &LO) const {
+    if (hasStoredFPFeatures())
+      return getStoredFPFeatures().applyOverrides(LO);
+    return FPOptions::defaultWithoutTrailingStorage(LO);
+  }
+
+  FPOptionsOverride getFPFeatures() const {
+    if (hasStoredFPFeatures())
+      return getStoredFPFeatures();
+    return FPOptionsOverride();
+  }
+
   /// getBuiltinCallee - If this is a call to a builtin, return the builtin ID
   /// of the callee. If not, return 0.
   unsigned getBuiltinCallee() const;
@@ -4690,6 +4751,13 @@ class InitListExpr : public Expr {
       setDependence(getDependence() | expr->getDependence());
   }
 
+  /// Mark the semantic form of the InitListExpr as error when the semantic
+  /// analysis fails.
+  void markError() {
+    assert(isSemanticForm());
+    setDependence(getDependence() | ExprDependence::ErrorDependent);
+  }
+
   /// Reserve space for some number of initializers.
   void reserveInits(const ASTContext &C, unsigned NumInits);
 
diff --git a/clang/include/clang/AST/ExprCXX.h b/clang/include/clang/AST/ExprCXX.h
index 6f0b68479b9d7..3f272c96a2d87 100644
--- a/clang/include/clang/AST/ExprCXX.h
+++ b/clang/include/clang/AST/ExprCXX.h
@@ -84,7 +84,6 @@ class CXXOperatorCallExpr final : public CallExpr {
   friend class ASTStmtWriter;
 
   SourceRange Range;
-  FPOptionsOverride Overrides;
 
   // CXXOperatorCallExpr has some trailing objects belonging
   // to CallExpr. See CallExpr for the details.
@@ -96,7 +95,7 @@ class CXXOperatorCallExpr final : public CallExpr {
                       SourceLocation OperatorLoc, FPOptionsOverride FPFeatures,
                       ADLCallKind UsesADL);
 
-  CXXOperatorCallExpr(unsigned NumArgs, EmptyShell Empty);
+  CXXOperatorCallExpr(unsigned NumArgs, bool HasFPFeatures, EmptyShell Empty);
 
 public:
   static CXXOperatorCallExpr *
@@ -106,7 +105,8 @@ class CXXOperatorCallExpr final : public CallExpr {
          ADLCallKind UsesADL = NotADL);
 
   static CXXOperatorCallExpr *CreateEmpty(const ASTContext &Ctx,
-                                          unsigned NumArgs, EmptyShell Empty);
+                                          unsigned NumArgs, bool HasFPFeatures,
+                                          EmptyShell Empty);
 
   /// Returns the kind of overloaded operator that this expression refers to.
   OverloadedOperatorKind getOperator() const {
@@ -164,11 +164,6 @@ class CXXOperatorCallExpr final : public CallExpr {
   static bool classof(const Stmt *T) {
     return T->getStmtClass() == CXXOperatorCallExprClass;
   }
-
-  // Set the FPFeatures status of this operator. Only meaningful for
-  // operations on floating point types.
-  void setFPFeatures(FPOptionsOverride F) { Overrides = F; }
-  FPOptionsOverride getFPFeatures() const { return Overrides; }
 };
 
 /// Represents a call to a member function that
@@ -184,18 +179,20 @@ class CXXMemberCallExpr final : public CallExpr {
   // to CallExpr. See CallExpr for the details.
 
   CXXMemberCallExpr(Expr *Fn, ArrayRef<Expr *> Args, QualType Ty,
-                    ExprValueKind VK, SourceLocation RP, unsigned MinNumArgs);
+                    ExprValueKind VK, SourceLocation RP,
+                    FPOptionsOverride FPOptions, unsigned MinNumArgs);
 
-  CXXMemberCallExpr(unsigned NumArgs, EmptyShell Empty);
+  CXXMemberCallExpr(unsigned NumArgs, bool HasFPFeatures, EmptyShell Empty);
 
 public:
   static CXXMemberCallExpr *Create(const ASTContext &Ctx, Expr *Fn,
                                    ArrayRef<Expr *> Args, QualType Ty,
                                    ExprValueKind VK, SourceLocation RP,
+                                   FPOptionsOverride FPFeatures,
                                    unsigned MinNumArgs = 0);
 
   static CXXMemberCallExpr *CreateEmpty(const ASTContext &Ctx, unsigned NumArgs,
-                                        EmptyShell Empty);
+                                        bool HasFPFeatures, EmptyShell Empty);
 
   /// Retrieve the implicit object argument for the member call.
   ///
@@ -242,18 +239,21 @@ class CUDAKernelCallExpr final : public CallExpr {
 
   CUDAKernelCallExpr(Expr *Fn, CallExpr *Config, ArrayRef<Expr *> Args,
                      QualType Ty, ExprValueKind VK, SourceLocation RP,
-                     unsigned MinNumArgs);
+                     FPOptionsOverride FPFeatures, unsigned MinNumArgs);
 
-  CUDAKernelCallExpr(unsigned NumArgs, EmptyShell Empty);
+  CUDAKernelCallExpr(unsigned NumArgs, bool HasFPFeatures, EmptyShell Empty);
 
 public:
   static CUDAKernelCallExpr *Create(const ASTContext &Ctx, Expr *Fn,
                                     CallExpr *Config, ArrayRef<Expr *> Args,
                                     QualType Ty, ExprValueKind VK,
-                                    SourceLocation RP, unsigned MinNumArgs = 0);
+                                    SourceLocation RP,
+                                    FPOptionsOverride FPFeatures,
+                                    unsigned MinNumArgs = 0);
 
   static CUDAKernelCallExpr *CreateEmpty(const ASTContext &Ctx,
-                                         unsigned NumArgs, EmptyShell Empty);
+                                         unsigned NumArgs, bool HasFPFeatures,
+                                         EmptyShell Empty);
 
   const CallExpr *getConfig() const {
     return cast_or_null<CallExpr>(getPreArg(CONFIG));
@@ -619,18 +619,20 @@ class UserDefinedLiteral final : public CallExpr {
 
   UserDefinedLiteral(Expr *Fn, ArrayRef<Expr *> Args, QualType Ty,
                      ExprValueKind VK, SourceLocation LitEndLoc,
-                     SourceLocation SuffixLoc);
+                     SourceLocation SuffixLoc, FPOptionsOverride FPFeatures);
 
-  UserDefinedLiteral(unsigned NumArgs, EmptyShell Empty);
+  UserDefinedLiteral(unsigned NumArgs, bool HasFPFeatures, EmptyShell Empty);
 
 public:
   static UserDefinedLiteral *Create(const ASTContext &Ctx, Expr *Fn,
                                     ArrayRef<Expr *> Args, QualType Ty,
                                     ExprValueKind VK, SourceLocation LitEndLoc,
-                                    SourceLocation SuffixLoc);
+                                    SourceLocation SuffixLoc,
+                                    FPOptionsOverride FPFeatures);
 
   static UserDefinedLiteral *CreateEmpty(const ASTContext &Ctx,
-                                         unsigned NumArgs, EmptyShell Empty);
+                                         unsigned NumArgs, bool HasFPOptions,
+                                         EmptyShell Empty);
 
   /// The kind of literal operator which is invoked.
   enum LiteralOperatorKind {
diff --git a/clang/include/clang/AST/ExternalASTSource.h b/clang/include/clang/AST/ExternalASTSource.h
index def877b918169..caae0770931b3 100644
--- a/clang/include/clang/AST/ExternalASTSource.h
+++ b/clang/include/clang/AST/ExternalASTSource.h
@@ -161,10 +161,6 @@ class ExternalASTSource : public RefCountedBase<ExternalASTSource> {
   /// Retrieve the module that corresponds to the given module ID.
   virtual Module *getModule(unsigned ID) { return nullptr; }
 
-  /// Determine whether D comes from a PCH which was built with a corresponding
-  /// object file.
-  virtual bool DeclIsFromPCHWithObjectFile(const Decl *D) { return false; }
-
   /// Return a descriptor for the corresponding module, if one exists.
   virtual llvm::Optional<ASTSourceDescriptor> getSourceDescriptor(unsigned ID);
 
diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 291eeb942b071..c649502f765b4 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -5393,7 +5393,7 @@ class OMPMapClause final : public OMPMappableExprListClause<OMPMapClause>,
   /// Map-type-modifiers for the 'map' clause.
   OpenMPMapModifierKind MapTypeModifiers[NumberOfOMPMapClauseModifiers] = {
       OMPC_MAP_MODIFIER_unknown, OMPC_MAP_MODIFIER_unknown,
-      OMPC_MAP_MODIFIER_unknown};
+      OMPC_MAP_MODIFIER_unknown, OMPC_MAP_MODIFIER_unknown};
 
   /// Location of map-type-modifiers for the 'map' clause.
   SourceLocation MapTypeModifiersLoc[NumberOfOMPMapClauseModifiers];
diff --git a/clang/include/clang/AST/Redeclarable.h b/clang/include/clang/AST/Redeclarable.h
index 0975773dd2cb5..87252337a0f4b 100644
--- a/clang/include/clang/AST/Redeclarable.h
+++ b/clang/include/clang/AST/Redeclarable.h
@@ -370,6 +370,7 @@ template <typename decl_type> class CanonicalDeclPtr {
 
 private:
   friend struct llvm::DenseMapInfo<CanonicalDeclPtr<decl_type>>;
+  friend struct llvm::PointerLikeTypeTraits<CanonicalDeclPtr<decl_type>>;
 
   decl_type *Ptr = nullptr;
 };
@@ -407,6 +408,20 @@ struct DenseMapInfo<clang::CanonicalDeclPtr<decl_type>> {
   }
 };
 
+template <typename decl_type>
+struct PointerLikeTypeTraits<clang::CanonicalDeclPtr<decl_type>> {
+  static inline void *getAsVoidPointer(clang::CanonicalDeclPtr<decl_type> P) {
+    return P.Ptr;
+  }
+  static inline clang::CanonicalDeclPtr<decl_type> getFromVoidPointer(void *P) {
+    clang::CanonicalDeclPtr<decl_type> C;
+    C.Ptr = PointerLikeTypeTraits<decl_type *>::getFromVoidPtr(P);
+    return C;
+  }
+  static constexpr int NumLowBitsAvailable =
+      PointerLikeTypeTraits<decl_type *>::NumLowBitsAvailable;
+};
+
 } // namespace llvm
 
 #endif // LLVM_CLANG_AST_REDECLARABLE_H
diff --git a/clang/include/clang/AST/Stmt.h b/clang/include/clang/AST/Stmt.h
index d3fad58fcf596..13f265223ac11 100644
--- a/clang/include/clang/AST/Stmt.h
+++ b/clang/include/clang/AST/Stmt.h
@@ -464,8 +464,11 @@ class alignas(void *) Stmt {
     /// True if the callee of the call expression was found using ADL.
     unsigned UsesADL : 1;
 
+    /// True if the call expression has some floating-point features.
+    unsigned HasFPFeatures : 1;
+
     /// Padding used to align OffsetToTrailingObjects to a byte multiple.
-    unsigned : 24 - 2 - NumExprBits;
+    unsigned : 24 - 3 - NumExprBits;
 
     /// The offset in bytes from the this pointer to the start of the
     /// trailing objects belonging to CallExpr. Intentionally byte sized
diff --git a/clang/include/clang/AST/TextNodeDumper.h b/clang/include/clang/AST/TextNodeDumper.h
index b4cfb5a380d17..8e8084aec3c18 100644
--- a/clang/include/clang/AST/TextNodeDumper.h
+++ b/clang/include/clang/AST/TextNodeDumper.h
@@ -155,6 +155,7 @@ class TextNodeDumper
   const comments::CommandTraits *Traits = nullptr;
 
   const char *getCommandName(unsigned CommandID);
+  void printFPOptions(FPOptionsOverride FPO);
 
   void dumpAPValueChildren(const APValue &Value, QualType Ty,
                            const APValue &(*IdxToChildFun)(const APValue &,
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index fe88753c0063d..bc9559776d4e6 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -1938,6 +1938,9 @@ class alignas(8) Type : public ExtQualsTypeCommonBase {
   bool isSizelessType() const;
   bool isSizelessBuiltinType() const;
 
+  /// Determines if this is a vector-length-specific type (VLST), i.e. a
+  /// sizeless type with the 'arm_sve_vector_bits' attribute applied.
+  bool isVLST() const;
   /// Determines if this is a sizeless type supported by the
   /// 'arm_sve_vector_bits' type attribute, which can be applied to a single
   /// SVE vector or predicate, excluding tuple types such as svint32x4_t.
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index cfa980f14b67a..73dec5e69c61a 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -1288,7 +1288,9 @@ def LoopUnrollHint : InheritableAttr {
 }
 
 def IntelReqdSubGroupSize: InheritableAttr {
-  let Spellings = [GNU<"intel_reqd_sub_group_size">, CXX11<"cl", "intel_reqd_sub_group_size">];
+  let Spellings = [GNU<"intel_reqd_sub_group_size">,
+                   CXX11<"cl", "intel_reqd_sub_group_size">,
+                   CXX11<"intel", "reqd_sub_group_size">];
   let Args = [ExprArgument<"SubGroupSize">];
   let Subjects = SubjectList<[Function, CXXMethod], ErrorDiag>;
   let Documentation = [IntelReqdSubGroupSizeDocs];
@@ -2024,8 +2026,10 @@ def NeonVectorType : TypeAttr {
 
 def ArmSveVectorBits : TypeAttr {
   let Spellings = [GNU<"arm_sve_vector_bits">];
-  let Args = [IntArgument<"NumBits">];
+  let Subjects = SubjectList<[TypedefName], ErrorDiag>;
+  let Args = [UnsignedArgument<"NumBits">];
   let Documentation = [ArmSveVectorBitsDocs];
+  let PragmaAttributeSupport = 0;
 }
 
 def ArmMveStrictPolymorphism : TypeAttr, TargetSpecificAttr<TargetARM> {
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 0812e3476d5d5..467b96793263f 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -3472,9 +3472,10 @@ code. See `cl_intel_required_subgroup_size
 for details.
 
 SYCL documentation:
-The [[cl::intel_reqd_sub_group_size(n)]] attribute indicates that the kernel
-must be compiled and executed with a sub-group of size n. The value of n must be
-set to a sub-group size supported by the device, or device compilation will fail.
+The [[cl::intel_reqd_sub_group_size(n)]] and [[intel::reqd_sub_group_size(n)]]
+attribute indicates that the kernel must be compiled and executed with a
+sub-group of size n. The value of n must be set to a sub-group size supported
+by the device, or device compilation will fail.
 
 In addition to device functions, the required sub-group size attribute may also
 be specified in the definition of a named functor object and lambda functions,
@@ -3495,6 +3496,19 @@ as in the examples below:
        /* kernel code */
   });
 
+  class Functor
+  {
+      [[intel::reqd_sub_group_size(16)]] void operator()(item<1> item)
+      {
+          /* kernel code */
+      }
+  }
+
+  kernel<class kernel_name>(
+  []() [[intel::reqd_sub_group_size(n)]] {
+       /* kernel code */
+  });
+
 See Sub-groups for NDRange Parallelism proposal in sycl/doc/extensions/sub_group_ndrange/sub_group_ndrange.md
   }];
 }
diff --git a/clang/include/clang/Basic/Builtins.def b/clang/include/clang/Basic/Builtins.def
index 219b6c7d0a239..2a291ce1fe2c9 100644
--- a/clang/include/clang/Basic/Builtins.def
+++ b/clang/include/clang/Basic/Builtins.def
@@ -395,6 +395,9 @@ BUILTIN(__builtin_ctanh, "XdXd", "Fne")
 BUILTIN(__builtin_ctanhf, "XfXf", "Fne")
 BUILTIN(__builtin_ctanhl, "XLdXLd", "Fne")
 
+// GCC-compatible C99 CMPLX implementation.
+BUILTIN(__builtin_complex, "v.", "nct")
+
 // FP Comparisons.
 BUILTIN(__builtin_isgreater     , "i.", "Fnct")
 BUILTIN(__builtin_isgreaterequal, "i.", "Fnct")
diff --git a/clang/include/clang/Basic/BuiltinsPPC.def b/clang/include/clang/Basic/BuiltinsPPC.def
index 5d445c253a855..5aed03f74f654 100644
--- a/clang/include/clang/Basic/BuiltinsPPC.def
+++ b/clang/include/clang/Basic/BuiltinsPPC.def
@@ -329,12 +329,12 @@ BUILTIN(__builtin_altivec_vinswlx, "V4UiV4UiUiUi", "")
 BUILTIN(__builtin_altivec_vinswrx, "V4UiV4UiUiUi", "")
 BUILTIN(__builtin_altivec_vinsdlx, "V2ULLiV2ULLiULLiULLi", "")
 BUILTIN(__builtin_altivec_vinsdrx, "V2ULLiV2ULLiULLiULLi", "")
-BUILTIN(__builtin_altivec_vinsbvlx, "V16UcV16UcULLiV16Uc", "")
-BUILTIN(__builtin_altivec_vinsbvrx, "V16UcV16UcULLiV16Uc", "")
-BUILTIN(__builtin_altivec_vinshvlx, "V8UsV8UsULLiV8Us", "")
-BUILTIN(__builtin_altivec_vinshvrx, "V8UsV8UsULLiV8Us", "")
-BUILTIN(__builtin_altivec_vinswvlx, "V4UiV4UiULLiV4Ui", "")
-BUILTIN(__builtin_altivec_vinswvrx, "V4UiV4UiULLiV4Ui", "")
+BUILTIN(__builtin_altivec_vinsbvlx, "V16UcV16UcUiV16Uc", "")
+BUILTIN(__builtin_altivec_vinsbvrx, "V16UcV16UcUiV16Uc", "")
+BUILTIN(__builtin_altivec_vinshvlx, "V8UsV8UsUiV8Us", "")
+BUILTIN(__builtin_altivec_vinshvrx, "V8UsV8UsUiV8Us", "")
+BUILTIN(__builtin_altivec_vinswvlx, "V4UiV4UiUiV4Ui", "")
+BUILTIN(__builtin_altivec_vinswvrx, "V4UiV4UiUiV4Ui", "")
 
 // VSX built-ins.
 
@@ -467,7 +467,7 @@ BUILTIN(__builtin_vsx_xxsldwi, "v.", "t")
 
 BUILTIN(__builtin_vsx_xxeval, "V2ULLiV2ULLiV2ULLiV2ULLiIi", "")
 
-BUILTIN(__builtin_vsx_xvtlsbb, "iV16Ucb", "")
+BUILTIN(__builtin_vsx_xvtlsbb, "iV16UcUi", "")
 
 // P10 Vector Permute Extended built-in.
 BUILTIN(__builtin_vsx_xxpermx, "V16UcV16UcV16UcV16UcIi", "")
diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index ccfa91cb7f2b0..b931512afd338 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -480,6 +480,9 @@ def warn_drv_msvc_not_found : Warning<
   "try running Clang from a developer command prompt">,
   InGroup<DiagGroup<"msvc-not-found">>;
 
+def warn_drv_use_ld_non_word : Warning<
+  "'-fuse-ld=' taking a path is deprecated. Use '--ld-path=' instead">;
+
 def warn_drv_fine_grained_bitfield_accesses_ignored : Warning<
   "option '-ffine-grained-bitfield-accesses' cannot be enabled together with a sanitizer; flag ignored">,
   InGroup<OptionIgnored>;
diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index 3ed6518d23aca..389a0eede5c85 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -1250,7 +1250,8 @@ def err_omp_decl_in_declare_simd_variant : Error<
 def err_omp_unknown_map_type : Error<
   "incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'">;
 def err_omp_unknown_map_type_modifier : Error<
-  "incorrect map type modifier, expected 'always', 'close', or 'mapper'">;
+  "incorrect map type modifier, expected 'always', 'close', "
+  "%select{or 'mapper'|'mapper', or 'present'}0">;
 def err_omp_map_type_missing : Error<
   "missing map type">;
 def err_omp_map_type_modifier_missing : Error<
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 9c4fc3ab93b59..62926c9edaf16 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -2853,7 +2853,8 @@ def err_attribute_bad_sve_vector_size : Error<
   "invalid SVE vector size '%0', must match value set by "
   "'-msve-vector-bits' ('%1')">;
 def err_attribute_arm_feature_sve_bits_unsupported : Error<
-  "%0 is not supported when '-msve-vector-bits=<bits>' is not specified">;
+  "%0 is only supported when '-msve-vector-bits=<bits>' is specified with a "
+  "value of 128, 256, 512, 1024 or 2048.">;
 def err_attribute_requires_positive_integer : Error<
   "%0 attribute requires a %select{positive|non-negative}1 "
   "integral compile time constant expression">;
@@ -7326,6 +7327,8 @@ let CategoryName = "Lambda Issue" in {
     "here">;
   def note_var_explicitly_captured_here : Note<"variable %0 is"
     "%select{| explicitly}1 captured here">;
+  def err_implicit_this_capture : Error<
+    "implicit capture of 'this' is not allowed for kernel functions">;
 
   // C++14 lambda init-captures.
   def warn_cxx11_compat_init_capture : Warning<
@@ -8188,6 +8191,10 @@ def warn_cast_pointer_from_sel : Warning<
 def warn_function_def_in_objc_container : Warning<
   "function definition inside an Objective-C container is deprecated">,
   InGroup<FunctionDefInObjCContainer>;
+def err_typecheck_call_requires_real_fp : Error<
+  "argument type %0 is not a real floating point type">;
+def err_typecheck_call_different_arg_types : Error<
+  "arguments are of different types%diff{ ($ vs $)|}0,1">;
 
 def warn_cast_calling_conv : Warning<
   "cast between incompatible calling conventions '%0' and '%1'; "
@@ -10977,6 +10984,9 @@ def warn_sycl_implicit_decl
               "declaration for a kernel type name; your program may not "
               "be portable">,
       InGroup<SyclStrict>, DefaultIgnore;
+def warn_sycl_restrict_recursion
+    : Warning<"SYCL kernel cannot call a recursive function">,
+      InGroup<SyclStrict>, DefaultError;
 def err_ivdep_duplicate_arg : Error<
   "duplicate argument to 'ivdep'. attribute requires one or both of a safelen "
   "and array">;
@@ -10988,6 +10998,11 @@ def err_ivdep_declrefexpr_arg : Error<
 def warn_ivdep_redundant : Warning <"ignoring redundant Intel FPGA loop "
   "attribute 'ivdep': safelen %select{INF|%1}0 >= safelen %select{INF|%3}2">,
   InGroup<IgnoredAttributes>;
+def warn_attribute_spelling_deprecated : Warning<
+  "attribute %0 is deprecated">,
+  InGroup<DeprecatedAttributes>;
+def note_spelling_suggestion : Note<
+  "did you mean to use %0 instead?">;
 
 // errors of expect.with.probability
 def err_probability_not_constant_float : Error<
diff --git a/clang/include/clang/Basic/FPOptions.def b/clang/include/clang/Basic/FPOptions.def
index 6b6789b8ecc87..1733689775d4b 100644
--- a/clang/include/clang/Basic/FPOptions.def
+++ b/clang/include/clang/Basic/FPOptions.def
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 // This file defines the Floating Point language options. Users of this file
-//  must define the FPOPT macro to make use of this information.
+//  must define the OPTION macro to make use of this information.
 #ifndef OPTION
 #  error Define the OPTION macro to handle floating point language options
 #endif
diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h
index a66f2ac349cfa..9a2c17ba8ab7a 100644
--- a/clang/include/clang/Basic/LangOptions.h
+++ b/clang/include/clang/Basic/LangOptions.h
@@ -382,6 +382,8 @@ class FPOptions {
 
   using RoundingMode = llvm::RoundingMode;
 
+  static constexpr unsigned StorageBitSize = 8 * sizeof(storage_type);
+
   // Define a fake option named "First" so that we have a PREVIOUS even for the
   // real first option.
   static constexpr storage_type FirstShift = 0, FirstWidth = 0;
@@ -393,6 +395,12 @@ class FPOptions {
                                              << NAME##Shift;
 #include "clang/Basic/FPOptions.def"
 
+  static constexpr storage_type TotalWidth = 0
+#define OPTION(NAME, TYPE, WIDTH, PREVIOUS) +WIDTH
+#include "clang/Basic/FPOptions.def"
+      ;
+  static_assert(TotalWidth <= StorageBitSize, "Too short type for FPOptions");
+
 private:
   storage_type Value;
 
@@ -410,8 +418,8 @@ class FPOptions {
     setFPContractMode(LO.getDefaultFPContractMode());
     setRoundingMode(LO.getFPRoundingMode());
     setFPExceptionMode(LO.getFPExceptionMode());
-    setAllowFEnvAccess(LangOptions::FPM_Off),
-        setAllowFPReassociate(LO.AllowFPReassoc);
+    setAllowFEnvAccess(LangOptions::FPM_Off);
+    setAllowFPReassociate(LO.AllowFPReassoc);
     setNoHonorNaNs(LO.NoHonorNaNs);
     setNoHonorInfs(LO.NoHonorInfs);
     setNoSignedZero(LO.NoSignedZero);
@@ -461,18 +469,45 @@ class FPOptions {
   LLVM_DUMP_METHOD void dump();
 };
 
-/// The FPOptions override type is value of the new FPOptions
-///  plus a mask showing which fields are actually set in it:
+/// Represents difference between two FPOptions values.
+///
+/// The effect of language constructs changing the set of floating point options
+/// is usually a change of some FP properties while leaving others intact. This
+/// class describes such changes by keeping information about what FP options
+/// are overridden.
+///
+/// The integral set of FP options, described by the class FPOptions, may be
+/// represented as a default FP option set, defined by language standard and
+/// command line options, with the overrides introduced by pragmas.
+///
+/// The is implemented as a value of the new FPOptions plus a mask showing which
+/// fields are actually set in it.
 class FPOptionsOverride {
   FPOptions Options;
   FPOptions::storage_type OverrideMask = 0;
 
 public:
   using RoundingMode = llvm::RoundingMode;
+
+  /// The type suitable for storing values of FPOptionsOverride. Must be twice
+  /// as wide as bit size of FPOption.
+  using storage_type = uint32_t;
+  static_assert(sizeof(storage_type) >= 2 * sizeof(FPOptions::storage_type),
+                "Too short type for FPOptionsOverride");
+
+  /// Bit mask selecting bits of OverrideMask in serialized representation of
+  /// FPOptionsOverride.
+  static constexpr storage_type OverrideMaskBits =
+      (static_cast<storage_type>(1) << FPOptions::StorageBitSize) - 1;
+
   FPOptionsOverride() {}
+  FPOptionsOverride(FPOptions::storage_type Value, FPOptions::storage_type Mask)
+      : Options(Value), OverrideMask(Mask) {}
+  FPOptionsOverride(const LangOptions &LO)
+      : Options(LO), OverrideMask(OverrideMaskBits) {}
 
   // Used for serializing.
-  explicit FPOptionsOverride(unsigned I) { getFromOpaqueInt(I); }
+  explicit FPOptionsOverride(storage_type I) { getFromOpaqueInt(I); }
 
   bool requiresTrailingStorage() const { return OverrideMask != 0; }
 
@@ -503,19 +538,24 @@ class FPOptionsOverride {
       setAllowFPContractAcrossStatement();
   }
 
-  unsigned getAsOpaqueInt() const {
-    return Options.getAsOpaqueInt() << 16 | OverrideMask;
+  storage_type getAsOpaqueInt() const {
+    return (static_cast<storage_type>(Options.getAsOpaqueInt())
+            << FPOptions::StorageBitSize) |
+           OverrideMask;
   }
-  void getFromOpaqueInt(unsigned I) {
-    OverrideMask = I & 0xffff;
-    Options.getFromOpaqueInt(I >> 16);
+  void getFromOpaqueInt(storage_type I) {
+    OverrideMask = I & OverrideMaskBits;
+    Options.getFromOpaqueInt(I >> FPOptions::StorageBitSize);
   }
 
-  FPOptions applyOverrides(const LangOptions &LO) {
-    FPOptions Base(LO);
-    FPOptions result((Base.getAsOpaqueInt() & ~OverrideMask) |
+  FPOptions applyOverrides(FPOptions Base) {
+    FPOptions Result((Base.getAsOpaqueInt() & ~OverrideMask) |
                      (Options.getAsOpaqueInt() & OverrideMask));
-    return result;
+    return Result;
+  }
+
+  FPOptions applyOverrides(const LangOptions &LO) {
+    return applyOverrides(FPOptions(LO));
   }
 
   bool operator==(FPOptionsOverride other) const {
diff --git a/clang/include/clang/Basic/Module.h b/clang/include/clang/Basic/Module.h
index 6b932a9a84d04..94dd215379661 100644
--- a/clang/include/clang/Basic/Module.h
+++ b/clang/include/clang/Basic/Module.h
@@ -457,8 +457,12 @@ class Module {
   /// Determine whether this module is a submodule.
   bool isSubModule() const { return Parent != nullptr; }
 
-  /// Determine whether this module is a submodule of the given other
-  /// module.
+  /// Check if this module is a (possibly transitive) submodule of \p Other.
+  ///
+  /// The 'A is a submodule of B' relation is a partial order based on the
+  /// the parent-child relationship between individual modules.
+  ///
+  /// Returns \c false if \p Other is \c nullptr.
   bool isSubModuleOf(const Module *Other) const;
 
   /// Determine whether this module is a part of a framework,
diff --git a/clang/include/clang/Basic/OpenMPKinds.def b/clang/include/clang/Basic/OpenMPKinds.def
index 9f2bf1abc2875..275c4fcdabb25 100644
--- a/clang/include/clang/Basic/OpenMPKinds.def
+++ b/clang/include/clang/Basic/OpenMPKinds.def
@@ -124,6 +124,7 @@ OPENMP_MAP_KIND(release)
 OPENMP_MAP_MODIFIER_KIND(always)
 OPENMP_MAP_MODIFIER_KIND(close)
 OPENMP_MAP_MODIFIER_KIND(mapper)
+OPENMP_MAP_MODIFIER_KIND(present)
 
 // Modifiers for 'to' clause.
 OPENMP_TO_MODIFIER_KIND(mapper)
diff --git a/clang/include/clang/Basic/OpenMPKinds.h b/clang/include/clang/Basic/OpenMPKinds.h
index 08aaf2d43bfdc..dc6198f93f9d8 100644
--- a/clang/include/clang/Basic/OpenMPKinds.h
+++ b/clang/include/clang/Basic/OpenMPKinds.h
@@ -170,7 +170,8 @@ enum OpenMPReductionClauseModifier {
   OMPC_REDUCTION_unknown,
 };
 
-unsigned getOpenMPSimpleClauseType(OpenMPClauseKind Kind, llvm::StringRef Str);
+unsigned getOpenMPSimpleClauseType(OpenMPClauseKind Kind, llvm::StringRef Str,
+                                   unsigned OpenMPVersion);
 const char *getOpenMPSimpleClauseTypeName(OpenMPClauseKind Kind, unsigned Type);
 
 /// Checks if the specified directive is a directive with an associated
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index b4e7cbf987c9c..4bbea6b0cf19c 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1459,6 +1459,10 @@ def fpch_instantiate_templates:
 def fno_pch_instantiate_templates:
   Flag <["-"], "fno-pch-instantiate-templates">,
   Group<f_Group>, Flags<[CC1Option]>;
+defm pch_codegen: OptInFFlag<"pch-codegen", "Generate ", "Do not generate ",
+  "code for uses of this PCH that assumes an explicit object file will be built for the PCH">;
+defm pch_debuginfo: OptInFFlag<"pch-debuginfo", "Generate ", "Do not generate ",
+  "debug info for types in an object file built from this PCH and do not generate them elsewhere">;
 
 def fmodules : Flag <["-"], "fmodules">, Group<f_Group>,
   Flags<[DriverOption, CC1Option]>,
@@ -2406,8 +2410,9 @@ foreach i = {8-15,18} in
 
 def msve_vector_bits_EQ : Joined<["-"], "msve-vector-bits=">,
   Group<m_aarch64_Features_Group>, Flags<[DriverOption,CC1Option]>,
-  HelpText<"Set the size of fixed-length SVE vectors in bits.">,
-  Values<"128,256,512,1024,2048">;
+  HelpText<"Specify the size in bits of an SVE vector register. Defaults to the"
+           " vector length agnostic value of \"scalable\". (AArch64 only)">,
+  Values<"128,256,512,1024,2048,scalable">;
 
 def msign_return_address_EQ : Joined<["-"], "msign-return-address=">,
   Flags<[CC1Option]>, Group<m_Group>, Values<"none,all,non-leaf">,
@@ -3341,6 +3346,7 @@ defm : BooleanFFlag<"keep-inline-functions">, Group<clang_ignored_gcc_optimizati
 def fprofile_dir : Joined<["-"], "fprofile-dir=">, Group<f_Group>;
 
 def fuse_ld_EQ : Joined<["-"], "fuse-ld=">, Group<f_Group>, Flags<[CoreOption]>;
+def ld_path_EQ : Joined<["--"], "ld-path=">;
 
 defm align_labels : BooleanFFlag<"align-labels">, Group<clang_ignored_gcc_optimization_f_Group>;
 def falign_labels_EQ : Joined<["-"], "falign-labels=">, Group<clang_ignored_gcc_optimization_f_Group>;
@@ -4757,6 +4763,7 @@ def _SLASH_FI : CLJoinedOrSeparate<"FI">,
 def _SLASH_Fe : CLJoined<"Fe">,
   HelpText<"Set output executable file name">,
   MetaVarName<"<file or dir/>">;
+def _SLASH_Fe_COLON : CLJoined<"Fe:">, Alias<_SLASH_Fe>;
 def _SLASH_Fi : CLCompileJoined<"Fi">,
   HelpText<"Set preprocess output file name (with /P)">,
   MetaVarName<"<file>">;
diff --git a/clang/include/clang/Driver/SanitizerArgs.h b/clang/include/clang/Driver/SanitizerArgs.h
index 934dab808e823..563d6c3ff9de2 100644
--- a/clang/include/clang/Driver/SanitizerArgs.h
+++ b/clang/include/clang/Driver/SanitizerArgs.h
@@ -74,6 +74,7 @@ class SanitizerArgs {
            !Sanitizers.has(SanitizerKind::Address) &&
            !Sanitizers.has(SanitizerKind::HWAddress);
   }
+  bool needsFuzzerInterceptors() const;
   bool needsUbsanRt() const;
   bool requiresMinimalRuntime() const { return MinimalRuntime; }
   bool needsDfsanRt() const { return Sanitizers.has(SanitizerKind::DataFlow); }
diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h
old mode 100755
new mode 100644
index 7201c11f1158e..269eab971a2cb
--- a/clang/include/clang/Format/Format.h
+++ b/clang/include/clang/Format/Format.h
@@ -2233,6 +2233,34 @@ struct FormatStyle {
   /// \endcode
   bool SpaceBeforeSquareBrackets;
 
+  /// Styles for adding spacing around ``:`` in bitfield definitions.
+  enum BitFieldColonSpacingStyle {
+    /// Add one space on each side of the ``:``
+    /// \code
+    ///   unsigned bf : 2;
+    /// \endcode
+    BFCS_Both,
+    /// Add no space around the ``:`` (except when needed for
+    /// ``AlignConsecutiveBitFields``).
+    /// \code
+    ///   unsigned bf:2;
+    /// \endcode
+    BFCS_None,
+    /// Add space before the ``:`` only
+    /// \code
+    ///   unsigned bf :2;
+    /// \endcode
+    BFCS_Before,
+    /// Add space after the ``:`` only (space may be added before if
+    /// needed for ``AlignConsecutiveBitFields``).
+    /// \code
+    ///   unsigned bf: 2;
+    /// \endcode
+    BFCS_After
+  };
+  /// The BitFieldColonSpacingStyle to use for bitfields.
+  BitFieldColonSpacingStyle BitFieldColonSpacing;
+
   /// Supported language standards for parsing and formatting C++ constructs.
   /// \code
   ///    Latest:                                vector<set<int>>
@@ -2409,6 +2437,7 @@ struct FormatStyle {
            SpacesInParentheses == R.SpacesInParentheses &&
            SpacesInSquareBrackets == R.SpacesInSquareBrackets &&
            SpaceBeforeSquareBrackets == R.SpaceBeforeSquareBrackets &&
+           BitFieldColonSpacing == R.BitFieldColonSpacing &&
            Standard == R.Standard && TabWidth == R.TabWidth &&
            StatementMacros == R.StatementMacros && UseTab == R.UseTab &&
            UseCRLF == R.UseCRLF && TypenameMacros == R.TypenameMacros;
diff --git a/clang/include/clang/Sema/MultiplexExternalSemaSource.h b/clang/include/clang/Sema/MultiplexExternalSemaSource.h
index e94dd5d468711..b54a6283d6408 100644
--- a/clang/include/clang/Sema/MultiplexExternalSemaSource.h
+++ b/clang/include/clang/Sema/MultiplexExternalSemaSource.h
@@ -153,8 +153,6 @@ class MultiplexExternalSemaSource : public ExternalSemaSource {
   /// Retrieve the module that corresponds to the given module ID.
   Module *getModule(unsigned ID) override;
 
-  bool DeclIsFromPCHWithObjectFile(const Decl *D) override;
-
   /// Perform layout on the given record.
   ///
   /// This routine allows the external AST source to provide an specific
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 3c7396a5fd5d8..e8b8f2859cc6b 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -730,7 +730,7 @@ class Sema final {
   PragmaStack<StringLiteral *> CodeSegStack;
 
   // This stack tracks the current state of Sema.CurFPFeatures.
-  PragmaStack<unsigned> FpPragmaStack;
+  PragmaStack<FPOptionsOverride::storage_type> FpPragmaStack;
   FPOptionsOverride CurFPFeatureOverrides() {
     FPOptionsOverride result;
     if (!FpPragmaStack.hasValue()) {
@@ -1548,12 +1548,12 @@ class Sema final {
       S.CurFPFeatures = OldFPFeaturesState;
       S.FpPragmaStack.CurrentValue = OldOverrides;
     }
-    unsigned getOverrides() { return OldOverrides; }
+    FPOptionsOverride::storage_type getOverrides() { return OldOverrides; }
 
   private:
     Sema& S;
     FPOptions OldFPFeaturesState;
-    unsigned OldOverrides;
+    FPOptionsOverride::storage_type OldOverrides;
   };
 
   void addImplicitTypedef(StringRef Name, QualType T);
@@ -2151,7 +2151,10 @@ class Sema final {
   bool RequireCompleteSizedType(SourceLocation Loc, QualType T, unsigned DiagID,
                                 const Ts &... Args) {
     SizelessTypeDiagnoser<Ts...> Diagnoser(DiagID, Args...);
-    return RequireCompleteType(Loc, T, CompleteTypeKind::Normal, Diagnoser);
+    CompleteTypeKind Kind = CompleteTypeKind::Normal;
+    if (T->isVLST())
+      Kind = CompleteTypeKind::AcceptSizeless;
+    return RequireCompleteType(Loc, T, Kind, Diagnoser);
   }
 
   void completeExprArrayBound(Expr *E);
@@ -2169,7 +2172,10 @@ class Sema final {
   bool RequireCompleteSizedExprType(Expr *E, unsigned DiagID,
                                     const Ts &... Args) {
     SizelessTypeDiagnoser<Ts...> Diagnoser(DiagID, Args...);
-    return RequireCompleteExprType(E, CompleteTypeKind::Normal, Diagnoser);
+    CompleteTypeKind Kind = CompleteTypeKind::Normal;
+    if (E->getType()->isVLST())
+      Kind = CompleteTypeKind::AcceptSizeless;
+    return RequireCompleteExprType(E, Kind, Diagnoser);
   }
 
   bool RequireLiteralType(SourceLocation Loc, QualType T,
@@ -12345,6 +12351,7 @@ class Sema final {
   bool SemaBuiltinVAStartARMMicrosoft(CallExpr *Call);
   bool SemaBuiltinUnorderedCompare(CallExpr *TheCall);
   bool SemaBuiltinFPClassification(CallExpr *TheCall, unsigned NumArgs);
+  bool SemaBuiltinComplex(CallExpr *TheCall);
   bool SemaBuiltinVSX(CallExpr *TheCall);
   bool SemaBuiltinOSLogFormat(CallExpr *TheCall);
 
@@ -12742,6 +12749,7 @@ class Sema final {
   void checkSYCLDeviceVarDecl(VarDecl *Var);
   void ConstructOpenCLKernel(FunctionDecl *KernelCallerFunc, MangleContext &MC);
   void MarkDevice();
+  void MarkSyclSimd();
 
   /// Creates a DeviceDiagBuilder that emits the diagnostic if the current
   /// context is "used as device code".
@@ -12865,16 +12873,16 @@ FPGALoopAttrT *Sema::BuildSYCLIntelFPGALoopAttr(const AttributeCommonInfo &A,
     return nullptr;
 
   if (E && !E->isInstantiationDependent()) {
-    llvm::APSInt ArgVal(32);
+    Optional<llvm::APSInt> ArgVal = E->getIntegerConstantExpr(getASTContext());
 
-    if (!E->isIntegerConstantExpr(ArgVal, getASTContext())) {
+    if (!ArgVal) {
       Diag(E->getExprLoc(), diag::err_attribute_argument_type)
           << A.getAttrName() << AANT_ArgumentIntegerConstant
           << E->getSourceRange();
       return nullptr;
     }
 
-    int Val = ArgVal.getSExtValue();
+    int Val = ArgVal->getSExtValue();
 
     if (A.getParsedKind() == ParsedAttr::AT_SYCLIntelFPGAII ||
         A.getParsedKind() == ParsedAttr::AT_SYCLIntelFPGALoopCoalesce) {
diff --git a/clang/include/clang/Sema/SemaInternal.h b/clang/include/clang/Sema/SemaInternal.h
index cdaf7b70a92f0..842eec099540c 100644
--- a/clang/include/clang/Sema/SemaInternal.h
+++ b/clang/include/clang/Sema/SemaInternal.h
@@ -168,6 +168,11 @@ class TypoCorrectionConsumer : public VisibleDeclConsumer {
     return TC;
   }
 
+  /// In the case of deeply invalid expressions, `getNextCorrection()` will
+  /// never be called since the transform never makes progress. If we don't
+  /// detect this we risk trying to correct typos forever.
+  bool hasMadeAnyCorrectionProgress() const { return CurrentTCIndex != 0; }
+
   /// Reset the consumer's position in the stream of viable corrections
   /// (i.e. getNextCorrection() will return each of the previously returned
   /// corrections in order before returning any new corrections).
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index a80366f0ee045..e883eb2f1749f 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -858,10 +858,10 @@ class ASTReader
   SourceLocation PointersToMembersPragmaLocation;
 
   /// The pragma float_control state.
-  Optional<unsigned> FpPragmaCurrentValue;
+  Optional<FPOptionsOverride::storage_type> FpPragmaCurrentValue;
   SourceLocation FpPragmaCurrentLocation;
   struct FpPragmaStackEntry {
-    unsigned Value;
+    FPOptionsOverride::storage_type Value;
     SourceLocation Location;
     SourceLocation PushLocation;
     StringRef SlotLabel;
@@ -2085,8 +2085,6 @@ class ASTReader
   /// Note: overrides method in ExternalASTSource
   Module *getModule(unsigned ID) override;
 
-  bool DeclIsFromPCHWithObjectFile(const Decl *D) override;
-
   /// Retrieve the module file with a given local ID within the specified
   /// ModuleFile.
   ModuleFile *getLocalModuleFile(ModuleFile &M, unsigned ID);
diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h
index cec29da69372f..598e612107024 100644
--- a/clang/include/clang/Serialization/ModuleFile.h
+++ b/clang/include/clang/Serialization/ModuleFile.h
@@ -155,9 +155,6 @@ class ModuleFile {
   /// Whether timestamps are included in this module file.
   bool HasTimestamps = false;
 
-  /// Whether the PCH has a corresponding object file.
-  bool PCHHasObjectFile = false;
-
   /// Whether the top-level module has been read from the AST file.
   bool DidReadTopLevelSubmodule = false;
 
diff --git a/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporterVisitors.h b/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporterVisitors.h
index 365b1ff1bfe3e..1aff2ca34a701 100644
--- a/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporterVisitors.h
+++ b/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporterVisitors.h
@@ -373,7 +373,7 @@ class SuppressInlineDefensiveChecksVisitor final : public BugReporterVisitor {
 class FalsePositiveRefutationBRVisitor final : public BugReporterVisitor {
 private:
   /// Holds the constraints in a given path
-  ConstraintRangeTy Constraints;
+  ConstraintMap Constraints;
 
 public:
   FalsePositiveRefutationBRVisitor();
@@ -390,7 +390,6 @@ class FalsePositiveRefutationBRVisitor final : public BugReporterVisitor {
                       bool OverwriteConstraintsOnExistingSyms);
 };
 
-
 /// The visitor detects NoteTags and displays the event notes they contain.
 class TagVisitor : public BugReporterVisitor {
 public:
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/RangedConstraintManager.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/RangedConstraintManager.h
index a42eebd7d4e81..bc5d5f57cd687 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/RangedConstraintManager.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/RangedConstraintManager.h
@@ -126,6 +126,8 @@ class RangeSet {
   RangeSet Intersect(BasicValueFactory &BV, Factory &F,
                      const RangeSet &Other) const;
   RangeSet Negate(BasicValueFactory &BV, Factory &F) const;
+  RangeSet Delete(BasicValueFactory &BV, Factory &F,
+                  const llvm::APSInt &Point) const;
 
   void print(raw_ostream &os) const;
 
@@ -134,15 +136,8 @@ class RangeSet {
   }
 };
 
-class ConstraintRange {};
-using ConstraintRangeTy = llvm::ImmutableMap<SymbolRef, RangeSet>;
-
-template <>
-struct ProgramStateTrait<ConstraintRange>
-  : public ProgramStatePartialTrait<ConstraintRangeTy> {
-  static void *GDMIndex();
-};
-
+using ConstraintMap = llvm::ImmutableMap<SymbolRef, RangeSet>;
+ConstraintMap getConstraintMap(ProgramStateRef State);
 
 class RangedConstraintManager : public SimpleConstraintManager {
 public:
@@ -169,8 +164,8 @@ class RangedConstraintManager : public SimpleConstraintManager {
 protected:
   /// Assume a constraint between a symbolic expression and a concrete integer.
   virtual ProgramStateRef assumeSymRel(ProgramStateRef State, SymbolRef Sym,
-                               BinaryOperator::Opcode op,
-                               const llvm::APSInt &Int);
+                                       BinaryOperator::Opcode op,
+                                       const llvm::APSInt &Int);
 
   //===------------------------------------------------------------------===//
   // Interface that subclasses must implement.
@@ -218,8 +213,9 @@ class RangedConstraintManager : public SimpleConstraintManager {
   static void computeAdjustment(SymbolRef &Sym, llvm::APSInt &Adjustment);
 };
 
-} // end GR namespace
+} // namespace ento
+} // namespace clang
 
-} // end clang namespace
+REGISTER_FACTORY_WITH_PROGRAMSTATE(ConstraintMap)
 
 #endif
diff --git a/clang/include/clang/Tooling/Syntax/Tokens.h b/clang/include/clang/Tooling/Syntax/Tokens.h
index a7f9369ddfff7..6e253abd6b3fe 100644
--- a/clang/include/clang/Tooling/Syntax/Tokens.h
+++ b/clang/include/clang/Tooling/Syntax/Tokens.h
@@ -275,6 +275,10 @@ class TokenBuffer {
   /// macro expands to.
   llvm::Optional<Expansion>
   expansionStartingAt(const syntax::Token *Spelled) const;
+  /// Returns all expansions (partially) expanded from the specified tokens.
+  /// This is the expansions whose Spelled range intersects \p Spelled.
+  std::vector<Expansion>
+  expansionsOverlapping(llvm::ArrayRef<syntax::Token> Spelled) const;
 
   /// Lexed tokens of a file before preprocessing. E.g. for the following input
   ///     #define DECL(name) int name = 10
@@ -352,6 +356,12 @@ class TokenBuffer {
   mappingStartingBeforeSpelled(const MarkedFile &F,
                                const syntax::Token *Spelled);
 
+  /// Convert a private Mapping to a public Expansion.
+  Expansion makeExpansion(const MarkedFile &, const Mapping &) const;
+  /// Returns the file that the Spelled tokens are taken from.
+  /// Asserts that they are non-empty, from a tracked file, and in-bounds.
+  const MarkedFile &fileForSpelled(llvm::ArrayRef<syntax::Token> Spelled) const;
+
   /// Token stream produced after preprocessing, conceputally this captures the
   /// same stream as 'clang -E' (excluding the preprocessor directives like
   /// #file, etc.).
diff --git a/clang/include/clang/Tooling/Transformer/RewriteRule.h b/clang/include/clang/Tooling/Transformer/RewriteRule.h
index d9e68717d5c8f..1be5727364608 100644
--- a/clang/include/clang/Tooling/Transformer/RewriteRule.h
+++ b/clang/include/clang/Tooling/Transformer/RewriteRule.h
@@ -47,6 +47,8 @@ using EditGenerator = MatchConsumer<llvm::SmallVector<Edit, 1>>;
 
 using TextGenerator = std::shared_ptr<MatchComputation<std::string>>;
 
+using AnyGenerator = MatchConsumer<llvm::Any>;
+
 // Description of a source-code edit, expressed in terms of an AST node.
 // Includes: an ID for the (bound) node, a selector for source related to the
 // node, a replacement and, optionally, an explanation for the edit.
@@ -87,7 +89,12 @@ struct ASTEdit {
   RangeSelector TargetRange;
   TextGenerator Replacement;
   TextGenerator Note;
-  llvm::Any Metadata;
+  // Not all transformations will want or need to attach metadata and therefore
+  // should not be requierd to do so.
+  AnyGenerator Metadata = [](const ast_matchers::MatchFinder::MatchResult &)
+      -> llvm::Expected<llvm::Any> {
+    return llvm::Expected<llvm::Any>(llvm::Any());
+  };
 };
 
 /// Lifts a list of `ASTEdit`s into an `EditGenerator`.
@@ -261,9 +268,27 @@ inline ASTEdit insertAfter(RangeSelector S, TextGenerator Replacement) {
 /// Removes the source selected by \p S.
 ASTEdit remove(RangeSelector S);
 
-inline ASTEdit withMetadata(ASTEdit edit, llvm::Any Metadata) {
-  edit.Metadata = std::move(Metadata);
-  return edit;
+// FIXME: If `Metadata` returns an `llvm::Expected<T>` the `AnyGenerator` will
+// construct an `llvm::Expected<llvm::Any>` where no error is present but the
+// `llvm::Any` holds the error. This is unlikely but potentially surprising.
+// Perhaps the `llvm::Expected` should be unwrapped, or perhaps this should be a
+// compile-time error. No solution here is perfect.
+//
+// Note: This function template accepts any type callable with a MatchResult
+// rather than a `std::function` because the return-type needs to be deduced. If
+// it accepted a `std::function<R(MatchResult)>`, lambdas or other callable types
+// would not be able to deduce `R`, and users would be forced to specify
+// explicitly the type they intended to return by wrapping the lambda at the
+// call-site.
+template <typename Callable>
+inline ASTEdit withMetadata(ASTEdit Edit, Callable Metadata) {
+  Edit.Metadata =
+      [Gen = std::move(Metadata)](
+          const ast_matchers::MatchFinder::MatchResult &R) -> llvm::Any {
+    return Gen(R);
+  };
+
+  return Edit;
 }
 
 /// The following three functions are a low-level part of the RewriteRule
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 3b036a608c736..886fdd8c6eb6c 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -1876,6 +1876,50 @@ TypeInfo ASTContext::getTypeInfo(const Type *T) const {
   return TI;
 }
 
+static unsigned getSveVectorWidth(const Type *T) {
+  // Get the vector size from the 'arm_sve_vector_bits' attribute via the
+  // AttributedTypeLoc associated with the typedef decl.
+  if (const auto *TT = T->getAs<TypedefType>()) {
+    const TypedefNameDecl *Typedef = TT->getDecl();
+    TypeSourceInfo *TInfo = Typedef->getTypeSourceInfo();
+    TypeLoc TL = TInfo->getTypeLoc();
+    if (AttributedTypeLoc ATL = TL.getAs<AttributedTypeLoc>())
+      if (const auto *Attr = ATL.getAttrAs<ArmSveVectorBitsAttr>())
+        return Attr->getNumBits();
+  }
+
+  llvm_unreachable("bad 'arm_sve_vector_bits' attribute!");
+}
+
+static unsigned getSvePredWidth(const ASTContext &Context, const Type *T) {
+  return getSveVectorWidth(T) / Context.getCharWidth();
+}
+
+unsigned ASTContext::getBitwidthForAttributedSveType(const Type *T) const {
+  assert(T->isVLST() &&
+         "getBitwidthForAttributedSveType called for non-attributed type!");
+
+  switch (T->castAs<BuiltinType>()->getKind()) {
+  default:
+    llvm_unreachable("unknown builtin type!");
+  case BuiltinType::SveInt8:
+  case BuiltinType::SveInt16:
+  case BuiltinType::SveInt32:
+  case BuiltinType::SveInt64:
+  case BuiltinType::SveUint8:
+  case BuiltinType::SveUint16:
+  case BuiltinType::SveUint32:
+  case BuiltinType::SveUint64:
+  case BuiltinType::SveFloat16:
+  case BuiltinType::SveFloat32:
+  case BuiltinType::SveFloat64:
+  case BuiltinType::SveBFloat16:
+    return getSveVectorWidth(T);
+  case BuiltinType::SveBool:
+    return getSvePredWidth(*this, T);
+  }
+}
+
 /// getTypeInfoImpl - Return the size of the specified type, in bits.  This
 /// method does not work on incomplete types.
 ///
@@ -2287,7 +2331,10 @@ TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const {
       Align = Info.Align;
       AlignIsRequired = Info.AlignIsRequired;
     }
-    Width = Info.Width;
+    if (T->isVLST())
+      Width = getBitwidthForAttributedSveType(T);
+    else
+      Width = Info.Width;
     break;
   }
 
@@ -9496,17 +9543,15 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS,
           const ConstantArrayType* CAT)
           -> std::pair<bool,llvm::APInt> {
         if (VAT) {
-          llvm::APSInt TheInt;
+          Optional<llvm::APSInt> TheInt;
           Expr *E = VAT->getSizeExpr();
-          if (E && E->isIntegerConstantExpr(TheInt, *this))
-            return std::make_pair(true, TheInt);
-          else
-            return std::make_pair(false, TheInt);
-        } else if (CAT) {
-            return std::make_pair(true, CAT->getSize());
-        } else {
-            return std::make_pair(false, llvm::APInt());
+          if (E && (TheInt = E->getIntegerConstantExpr(*this)))
+            return std::make_pair(true, *TheInt);
+          return std::make_pair(false, llvm::APSInt());
         }
+        if (CAT)
+          return std::make_pair(true, CAT->getSize());
+        return std::make_pair(false, llvm::APInt());
       };
 
       bool HaveLSize, HaveRSize;
@@ -10457,37 +10502,6 @@ bool ASTContext::DeclMustBeEmitted(const Decl *D) {
   else
     return false;
 
-  if (D->isFromASTFile() && !LangOpts.BuildingPCHWithObjectFile) {
-    assert(getExternalSource() && "It's from an AST file; must have a source.");
-    // On Windows, PCH files are built together with an object file. If this
-    // declaration comes from such a PCH and DeclMustBeEmitted would return
-    // true, it would have returned true and the decl would have been emitted
-    // into that object file, so it doesn't need to be emitted here.
-    // Note that decls are still emitted if they're referenced, as usual;
-    // DeclMustBeEmitted is used to decide whether a decl must be emitted even
-    // if it's not referenced.
-    //
-    // Explicit template instantiation definitions are tricky. If there was an
-    // explicit template instantiation decl in the PCH before, it will look like
-    // the definition comes from there, even if that was just the declaration.
-    // (Explicit instantiation defs of variable templates always get emitted.)
-    bool IsExpInstDef =
-        isa<FunctionDecl>(D) &&
-        cast<FunctionDecl>(D)->getTemplateSpecializationKind() ==
-            TSK_ExplicitInstantiationDefinition;
-
-    // Implicit member function definitions, such as operator= might not be
-    // marked as template specializations, since they're not coming from a
-    // template but synthesized directly on the class.
-    IsExpInstDef |=
-        isa<CXXMethodDecl>(D) &&
-        cast<CXXMethodDecl>(D)->getParent()->getTemplateSpecializationKind() ==
-            TSK_ExplicitInstantiationDefinition;
-
-    if (getExternalSource()->DeclIsFromPCHWithObjectFile(D) && !IsExpInstDef)
-      return false;
-  }
-
   // If this is a member of a class template, we do not need to emit it.
   if (D->getDeclContext()->isDependentContext())
     return false;
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index 61df87604f205..6623900aff28d 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -7344,7 +7344,8 @@ ExpectedStmt ASTNodeImporter::VisitCXXMemberCallExpr(CXXMemberCallExpr *E) {
     return std::move(Err);
 
   return CXXMemberCallExpr::Create(Importer.getToContext(), ToCallee, ToArgs,
-                                   ToType, E->getValueKind(), ToRParenLoc);
+                                   ToType, E->getValueKind(), ToRParenLoc,
+                                   E->getFPFeatures());
 }
 
 ExpectedStmt ASTNodeImporter::VisitCXXThisExpr(CXXThisExpr *E) {
@@ -7654,8 +7655,8 @@ ExpectedStmt ASTNodeImporter::VisitCallExpr(CallExpr *E) {
   }
 
   return CallExpr::Create(Importer.getToContext(), ToCallee, ToArgs, ToType,
-                          E->getValueKind(), ToRParenLoc, /*MinNumArgs=*/0,
-                          E->getADLCallKind());
+                          E->getValueKind(), ToRParenLoc, E->getFPFeatures(),
+                          /*MinNumArgs=*/0, E->getADLCallKind());
 }
 
 ExpectedStmt ASTNodeImporter::VisitLambdaExpr(LambdaExpr *E) {
diff --git a/clang/lib/AST/ComputeDependence.cpp b/clang/lib/AST/ComputeDependence.cpp
index 2333993dbeb40..c3a2090266621 100644
--- a/clang/lib/AST/ComputeDependence.cpp
+++ b/clang/lib/AST/ComputeDependence.cpp
@@ -502,7 +502,7 @@ ExprDependence clang::computeDependence(RecoveryExpr *E) {
   //     dependent type), or the type is known and dependent, or it has
   //     type-dependent subexpressions.
   auto D = toExprDependence(E->getType()->getDependence()) |
-           ExprDependence::ValueInstantiation | ExprDependence::Error;
+           ExprDependence::ErrorDependent;
   // FIXME: remove the type-dependent bit from subexpressions, if the
   // RecoveryExpr has a non-dependent type.
   for (auto *S : E->subExpressions())
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index 399e7e13c4459..29f96674e1747 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -1303,8 +1303,8 @@ OverloadedOperatorKind UnaryOperator::getOverloadedOperator(Opcode Opc) {
 
 CallExpr::CallExpr(StmtClass SC, Expr *Fn, ArrayRef<Expr *> PreArgs,
                    ArrayRef<Expr *> Args, QualType Ty, ExprValueKind VK,
-                   SourceLocation RParenLoc, unsigned MinNumArgs,
-                   ADLCallKind UsesADL)
+                   SourceLocation RParenLoc, FPOptionsOverride FPFeatures,
+                   unsigned MinNumArgs, ADLCallKind UsesADL)
     : Expr(SC, Ty, VK, OK_Ordinary), RParenLoc(RParenLoc) {
   NumArgs = std::max<unsigned>(Args.size(), MinNumArgs);
   unsigned NumPreArgs = PreArgs.size();
@@ -1327,10 +1327,14 @@ CallExpr::CallExpr(StmtClass SC, Expr *Fn, ArrayRef<Expr *> PreArgs,
     setArg(I, nullptr);
 
   setDependence(computeDependence(this, PreArgs));
+
+  CallExprBits.HasFPFeatures = FPFeatures.requiresTrailingStorage();
+  if (hasStoredFPFeatures())
+    setStoredFPFeatures(FPFeatures);
 }
 
 CallExpr::CallExpr(StmtClass SC, unsigned NumPreArgs, unsigned NumArgs,
-                   EmptyShell Empty)
+                   bool HasFPFeatures, EmptyShell Empty)
     : Expr(SC, Empty), NumArgs(NumArgs) {
   CallExprBits.NumPreArgs = NumPreArgs;
   assert((NumPreArgs == getNumPreArgs()) && "NumPreArgs overflow!");
@@ -1339,19 +1343,21 @@ CallExpr::CallExpr(StmtClass SC, unsigned NumPreArgs, unsigned NumArgs,
   CallExprBits.OffsetToTrailingObjects = OffsetToTrailingObjects;
   assert((CallExprBits.OffsetToTrailingObjects == OffsetToTrailingObjects) &&
          "OffsetToTrailingObjects overflow!");
+  CallExprBits.HasFPFeatures = HasFPFeatures;
 }
 
 CallExpr *CallExpr::Create(const ASTContext &Ctx, Expr *Fn,
                            ArrayRef<Expr *> Args, QualType Ty, ExprValueKind VK,
-                           SourceLocation RParenLoc, unsigned MinNumArgs,
+                           SourceLocation RParenLoc,
+                           FPOptionsOverride FPFeatures, unsigned MinNumArgs,
                            ADLCallKind UsesADL) {
   unsigned NumArgs = std::max<unsigned>(Args.size(), MinNumArgs);
-  unsigned SizeOfTrailingObjects =
-      CallExpr::sizeOfTrailingObjects(/*NumPreArgs=*/0, NumArgs);
+  unsigned SizeOfTrailingObjects = CallExpr::sizeOfTrailingObjects(
+      /*NumPreArgs=*/0, NumArgs, FPFeatures.requiresTrailingStorage());
   void *Mem =
       Ctx.Allocate(sizeof(CallExpr) + SizeOfTrailingObjects, alignof(CallExpr));
   return new (Mem) CallExpr(CallExprClass, Fn, /*PreArgs=*/{}, Args, Ty, VK,
-                            RParenLoc, MinNumArgs, UsesADL);
+                            RParenLoc, FPFeatures, MinNumArgs, UsesADL);
 }
 
 CallExpr *CallExpr::CreateTemporary(void *Mem, Expr *Fn, QualType Ty,
@@ -1360,17 +1366,18 @@ CallExpr *CallExpr::CreateTemporary(void *Mem, Expr *Fn, QualType Ty,
   assert(!(reinterpret_cast<uintptr_t>(Mem) % alignof(CallExpr)) &&
          "Misaligned memory in CallExpr::CreateTemporary!");
   return new (Mem) CallExpr(CallExprClass, Fn, /*PreArgs=*/{}, /*Args=*/{}, Ty,
-                            VK, RParenLoc,
+                            VK, RParenLoc, FPOptionsOverride(),
                             /*MinNumArgs=*/0, UsesADL);
 }
 
 CallExpr *CallExpr::CreateEmpty(const ASTContext &Ctx, unsigned NumArgs,
-                                EmptyShell Empty) {
+                                bool HasFPFeatures, EmptyShell Empty) {
   unsigned SizeOfTrailingObjects =
-      CallExpr::sizeOfTrailingObjects(/*NumPreArgs=*/0, NumArgs);
+      CallExpr::sizeOfTrailingObjects(/*NumPreArgs=*/0, NumArgs, HasFPFeatures);
   void *Mem =
       Ctx.Allocate(sizeof(CallExpr) + SizeOfTrailingObjects, alignof(CallExpr));
-  return new (Mem) CallExpr(CallExprClass, /*NumPreArgs=*/0, NumArgs, Empty);
+  return new (Mem)
+      CallExpr(CallExprClass, /*NumPreArgs=*/0, NumArgs, HasFPFeatures, Empty);
 }
 
 unsigned CallExpr::offsetToTrailingObjects(StmtClass SC) {
@@ -3740,6 +3747,9 @@ Expr::isNullPointerConstant(ASTContext &Ctx,
                             NullPointerConstantValueDependence NPC) const {
   if (isValueDependent() &&
       (!Ctx.getLangOpts().CPlusPlus11 || Ctx.getLangOpts().MSVCCompat)) {
+    // Error-dependent expr should never be a null pointer.
+    if (containsErrors())
+      return NPCK_NotNull;
     switch (NPC) {
     case NPC_NeverValueDependent:
       llvm_unreachable("Unexpected value dependent expression!");
diff --git a/clang/lib/AST/ExprCXX.cpp b/clang/lib/AST/ExprCXX.cpp
index 5d99f61c579ff..3d61496f30e2a 100644
--- a/clang/lib/AST/ExprCXX.cpp
+++ b/clang/lib/AST/ExprCXX.cpp
@@ -528,17 +528,18 @@ CXXOperatorCallExpr::CXXOperatorCallExpr(OverloadedOperatorKind OpKind,
                                          FPOptionsOverride FPFeatures,
                                          ADLCallKind UsesADL)
     : CallExpr(CXXOperatorCallExprClass, Fn, /*PreArgs=*/{}, Args, Ty, VK,
-               OperatorLoc, /*MinNumArgs=*/0, UsesADL) {
+               OperatorLoc, FPFeatures, /*MinNumArgs=*/0, UsesADL) {
   CXXOperatorCallExprBits.OperatorKind = OpKind;
   assert(
       (CXXOperatorCallExprBits.OperatorKind == static_cast<unsigned>(OpKind)) &&
       "OperatorKind overflow!");
   Range = getSourceRangeImpl();
-  Overrides = FPFeatures;
 }
 
-CXXOperatorCallExpr::CXXOperatorCallExpr(unsigned NumArgs, EmptyShell Empty)
-    : CallExpr(CXXOperatorCallExprClass, /*NumPreArgs=*/0, NumArgs, Empty) {}
+CXXOperatorCallExpr::CXXOperatorCallExpr(unsigned NumArgs, bool HasFPFeatures,
+                                         EmptyShell Empty)
+    : CallExpr(CXXOperatorCallExprClass, /*NumPreArgs=*/0, NumArgs,
+               HasFPFeatures, Empty) {}
 
 CXXOperatorCallExpr *
 CXXOperatorCallExpr::Create(const ASTContext &Ctx,
@@ -548,8 +549,8 @@ CXXOperatorCallExpr::Create(const ASTContext &Ctx,
                             FPOptionsOverride FPFeatures, ADLCallKind UsesADL) {
   // Allocate storage for the trailing objects of CallExpr.
   unsigned NumArgs = Args.size();
-  unsigned SizeOfTrailingObjects =
-      CallExpr::sizeOfTrailingObjects(/*NumPreArgs=*/0, NumArgs);
+  unsigned SizeOfTrailingObjects = CallExpr::sizeOfTrailingObjects(
+      /*NumPreArgs=*/0, NumArgs, FPFeatures.requiresTrailingStorage());
   void *Mem = Ctx.Allocate(sizeof(CXXOperatorCallExpr) + SizeOfTrailingObjects,
                            alignof(CXXOperatorCallExpr));
   return new (Mem) CXXOperatorCallExpr(OpKind, Fn, Args, Ty, VK, OperatorLoc,
@@ -558,13 +559,14 @@ CXXOperatorCallExpr::Create(const ASTContext &Ctx,
 
 CXXOperatorCallExpr *CXXOperatorCallExpr::CreateEmpty(const ASTContext &Ctx,
                                                       unsigned NumArgs,
+                                                      bool HasFPFeatures,
                                                       EmptyShell Empty) {
   // Allocate storage for the trailing objects of CallExpr.
   unsigned SizeOfTrailingObjects =
-      CallExpr::sizeOfTrailingObjects(/*NumPreArgs=*/0, NumArgs);
+      CallExpr::sizeOfTrailingObjects(/*NumPreArgs=*/0, NumArgs, HasFPFeatures);
   void *Mem = Ctx.Allocate(sizeof(CXXOperatorCallExpr) + SizeOfTrailingObjects,
                            alignof(CXXOperatorCallExpr));
-  return new (Mem) CXXOperatorCallExpr(NumArgs, Empty);
+  return new (Mem) CXXOperatorCallExpr(NumArgs, HasFPFeatures, Empty);
 }
 
 SourceRange CXXOperatorCallExpr::getSourceRangeImpl() const {
@@ -593,36 +595,43 @@ SourceRange CXXOperatorCallExpr::getSourceRangeImpl() const {
 
 CXXMemberCallExpr::CXXMemberCallExpr(Expr *Fn, ArrayRef<Expr *> Args,
                                      QualType Ty, ExprValueKind VK,
-                                     SourceLocation RP, unsigned MinNumArgs)
+                                     SourceLocation RP,
+                                     FPOptionsOverride FPOptions,
+                                     unsigned MinNumArgs)
     : CallExpr(CXXMemberCallExprClass, Fn, /*PreArgs=*/{}, Args, Ty, VK, RP,
-               MinNumArgs, NotADL) {}
+               FPOptions, MinNumArgs, NotADL) {}
 
-CXXMemberCallExpr::CXXMemberCallExpr(unsigned NumArgs, EmptyShell Empty)
-    : CallExpr(CXXMemberCallExprClass, /*NumPreArgs=*/0, NumArgs, Empty) {}
+CXXMemberCallExpr::CXXMemberCallExpr(unsigned NumArgs, bool HasFPFeatures,
+                                     EmptyShell Empty)
+    : CallExpr(CXXMemberCallExprClass, /*NumPreArgs=*/0, NumArgs, HasFPFeatures,
+               Empty) {}
 
 CXXMemberCallExpr *CXXMemberCallExpr::Create(const ASTContext &Ctx, Expr *Fn,
                                              ArrayRef<Expr *> Args, QualType Ty,
                                              ExprValueKind VK,
                                              SourceLocation RP,
+                                             FPOptionsOverride FPFeatures,
                                              unsigned MinNumArgs) {
   // Allocate storage for the trailing objects of CallExpr.
   unsigned NumArgs = std::max<unsigned>(Args.size(), MinNumArgs);
-  unsigned SizeOfTrailingObjects =
-      CallExpr::sizeOfTrailingObjects(/*NumPreArgs=*/0, NumArgs);
+  unsigned SizeOfTrailingObjects = CallExpr::sizeOfTrailingObjects(
+      /*NumPreArgs=*/0, NumArgs, FPFeatures.requiresTrailingStorage());
   void *Mem = Ctx.Allocate(sizeof(CXXMemberCallExpr) + SizeOfTrailingObjects,
                            alignof(CXXMemberCallExpr));
-  return new (Mem) CXXMemberCallExpr(Fn, Args, Ty, VK, RP, MinNumArgs);
+  return new (Mem)
+      CXXMemberCallExpr(Fn, Args, Ty, VK, RP, FPFeatures, MinNumArgs);
 }
 
 CXXMemberCallExpr *CXXMemberCallExpr::CreateEmpty(const ASTContext &Ctx,
                                                   unsigned NumArgs,
+                                                  bool HasFPFeatures,
                                                   EmptyShell Empty) {
   // Allocate storage for the trailing objects of CallExpr.
   unsigned SizeOfTrailingObjects =
-      CallExpr::sizeOfTrailingObjects(/*NumPreArgs=*/0, NumArgs);
+      CallExpr::sizeOfTrailingObjects(/*NumPreArgs=*/0, NumArgs, HasFPFeatures);
   void *Mem = Ctx.Allocate(sizeof(CXXMemberCallExpr) + SizeOfTrailingObjects,
                            alignof(CXXMemberCallExpr));
-  return new (Mem) CXXMemberCallExpr(NumArgs, Empty);
+  return new (Mem) CXXMemberCallExpr(NumArgs, HasFPFeatures, Empty);
 }
 
 Expr *CXXMemberCallExpr::getImplicitObjectArgument() const {
@@ -846,37 +855,43 @@ SourceLocation CXXFunctionalCastExpr::getEndLoc() const {
 UserDefinedLiteral::UserDefinedLiteral(Expr *Fn, ArrayRef<Expr *> Args,
                                        QualType Ty, ExprValueKind VK,
                                        SourceLocation LitEndLoc,
-                                       SourceLocation SuffixLoc)
+                                       SourceLocation SuffixLoc,
+                                       FPOptionsOverride FPFeatures)
     : CallExpr(UserDefinedLiteralClass, Fn, /*PreArgs=*/{}, Args, Ty, VK,
-               LitEndLoc, /*MinNumArgs=*/0, NotADL),
+               LitEndLoc, FPFeatures, /*MinNumArgs=*/0, NotADL),
       UDSuffixLoc(SuffixLoc) {}
 
-UserDefinedLiteral::UserDefinedLiteral(unsigned NumArgs, EmptyShell Empty)
-    : CallExpr(UserDefinedLiteralClass, /*NumPreArgs=*/0, NumArgs, Empty) {}
+UserDefinedLiteral::UserDefinedLiteral(unsigned NumArgs, bool HasFPFeatures,
+                                       EmptyShell Empty)
+    : CallExpr(UserDefinedLiteralClass, /*NumPreArgs=*/0, NumArgs,
+               HasFPFeatures, Empty) {}
 
 UserDefinedLiteral *UserDefinedLiteral::Create(const ASTContext &Ctx, Expr *Fn,
                                                ArrayRef<Expr *> Args,
                                                QualType Ty, ExprValueKind VK,
                                                SourceLocation LitEndLoc,
-                                               SourceLocation SuffixLoc) {
+                                               SourceLocation SuffixLoc,
+                                               FPOptionsOverride FPFeatures) {
   // Allocate storage for the trailing objects of CallExpr.
   unsigned NumArgs = Args.size();
-  unsigned SizeOfTrailingObjects =
-      CallExpr::sizeOfTrailingObjects(/*NumPreArgs=*/0, NumArgs);
+  unsigned SizeOfTrailingObjects = CallExpr::sizeOfTrailingObjects(
+      /*NumPreArgs=*/0, NumArgs, FPFeatures.requiresTrailingStorage());
   void *Mem = Ctx.Allocate(sizeof(UserDefinedLiteral) + SizeOfTrailingObjects,
                            alignof(UserDefinedLiteral));
-  return new (Mem) UserDefinedLiteral(Fn, Args, Ty, VK, LitEndLoc, SuffixLoc);
+  return new (Mem)
+      UserDefinedLiteral(Fn, Args, Ty, VK, LitEndLoc, SuffixLoc, FPFeatures);
 }
 
 UserDefinedLiteral *UserDefinedLiteral::CreateEmpty(const ASTContext &Ctx,
                                                     unsigned NumArgs,
+                                                    bool HasFPOptions,
                                                     EmptyShell Empty) {
   // Allocate storage for the trailing objects of CallExpr.
   unsigned SizeOfTrailingObjects =
-      CallExpr::sizeOfTrailingObjects(/*NumPreArgs=*/0, NumArgs);
+      CallExpr::sizeOfTrailingObjects(/*NumPreArgs=*/0, NumArgs, HasFPOptions);
   void *Mem = Ctx.Allocate(sizeof(UserDefinedLiteral) + SizeOfTrailingObjects,
                            alignof(UserDefinedLiteral));
-  return new (Mem) UserDefinedLiteral(NumArgs, Empty);
+  return new (Mem) UserDefinedLiteral(NumArgs, HasFPOptions, Empty);
 }
 
 UserDefinedLiteral::LiteralOperatorKind
@@ -1643,34 +1658,39 @@ TypeTraitExpr *TypeTraitExpr::CreateDeserialized(const ASTContext &C,
 CUDAKernelCallExpr::CUDAKernelCallExpr(Expr *Fn, CallExpr *Config,
                                        ArrayRef<Expr *> Args, QualType Ty,
                                        ExprValueKind VK, SourceLocation RP,
+                                       FPOptionsOverride FPFeatures,
                                        unsigned MinNumArgs)
     : CallExpr(CUDAKernelCallExprClass, Fn, /*PreArgs=*/Config, Args, Ty, VK,
-               RP, MinNumArgs, NotADL) {}
+               RP, FPFeatures, MinNumArgs, NotADL) {}
 
-CUDAKernelCallExpr::CUDAKernelCallExpr(unsigned NumArgs, EmptyShell Empty)
+CUDAKernelCallExpr::CUDAKernelCallExpr(unsigned NumArgs, bool HasFPFeatures,
+                                       EmptyShell Empty)
     : CallExpr(CUDAKernelCallExprClass, /*NumPreArgs=*/END_PREARG, NumArgs,
-               Empty) {}
+               HasFPFeatures, Empty) {}
 
 CUDAKernelCallExpr *
 CUDAKernelCallExpr::Create(const ASTContext &Ctx, Expr *Fn, CallExpr *Config,
                            ArrayRef<Expr *> Args, QualType Ty, ExprValueKind VK,
-                           SourceLocation RP, unsigned MinNumArgs) {
+                           SourceLocation RP, FPOptionsOverride FPFeatures,
+                           unsigned MinNumArgs) {
   // Allocate storage for the trailing objects of CallExpr.
   unsigned NumArgs = std::max<unsigned>(Args.size(), MinNumArgs);
-  unsigned SizeOfTrailingObjects =
-      CallExpr::sizeOfTrailingObjects(/*NumPreArgs=*/END_PREARG, NumArgs);
+  unsigned SizeOfTrailingObjects = CallExpr::sizeOfTrailingObjects(
+      /*NumPreArgs=*/END_PREARG, NumArgs, FPFeatures.requiresTrailingStorage());
   void *Mem = Ctx.Allocate(sizeof(CUDAKernelCallExpr) + SizeOfTrailingObjects,
                            alignof(CUDAKernelCallExpr));
-  return new (Mem) CUDAKernelCallExpr(Fn, Config, Args, Ty, VK, RP, MinNumArgs);
+  return new (Mem)
+      CUDAKernelCallExpr(Fn, Config, Args, Ty, VK, RP, FPFeatures, MinNumArgs);
 }
 
 CUDAKernelCallExpr *CUDAKernelCallExpr::CreateEmpty(const ASTContext &Ctx,
                                                     unsigned NumArgs,
+                                                    bool HasFPFeatures,
                                                     EmptyShell Empty) {
   // Allocate storage for the trailing objects of CallExpr.
-  unsigned SizeOfTrailingObjects =
-      CallExpr::sizeOfTrailingObjects(/*NumPreArgs=*/END_PREARG, NumArgs);
+  unsigned SizeOfTrailingObjects = CallExpr::sizeOfTrailingObjects(
+      /*NumPreArgs=*/END_PREARG, NumArgs, HasFPFeatures);
   void *Mem = Ctx.Allocate(sizeof(CUDAKernelCallExpr) + SizeOfTrailingObjects,
                            alignof(CUDAKernelCallExpr));
-  return new (Mem) CUDAKernelCallExpr(NumArgs, Empty);
+  return new (Mem) CUDAKernelCallExpr(NumArgs, HasFPFeatures, Empty);
 }
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 1e313647aa2e5..8c40a887b6f4b 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -13286,6 +13286,7 @@ class ComplexExprEvaluator
   bool VisitBinaryOperator(const BinaryOperator *E);
   bool VisitUnaryOperator(const UnaryOperator *E);
   bool VisitInitListExpr(const InitListExpr *E);
+  bool VisitCallExpr(const CallExpr *E);
 };
 } // end anonymous namespace
 
@@ -13754,6 +13755,23 @@ bool ComplexExprEvaluator::VisitInitListExpr(const InitListExpr *E) {
   return ExprEvaluatorBaseTy::VisitInitListExpr(E);
 }
 
+bool ComplexExprEvaluator::VisitCallExpr(const CallExpr *E) {
+  switch (E->getBuiltinCallee()) {
+  case Builtin::BI__builtin_complex:
+    Result.makeComplexFloat();
+    if (!EvaluateFloat(E->getArg(0), Result.FloatReal, Info))
+      return false;
+    if (!EvaluateFloat(E->getArg(1), Result.FloatImag, Info))
+      return false;
+    return true;
+
+  default:
+    break;
+  }
+
+  return ExprEvaluatorBaseTy::VisitCallExpr(E);
+}
+
 //===----------------------------------------------------------------------===//
 // Atomic expression evaluation, essentially just handling the NonAtomicToAtomic
 // implicit conversion.
@@ -14897,16 +14915,22 @@ bool Expr::isIntegerConstantExpr(const ASTContext &Ctx,
   return true;
 }
 
-bool Expr::isIntegerConstantExpr(llvm::APSInt &Value, const ASTContext &Ctx,
-                                 SourceLocation *Loc, bool isEvaluated) const {
+Optional<llvm::APSInt> Expr::getIntegerConstantExpr(const ASTContext &Ctx,
+                                                    SourceLocation *Loc,
+                                                    bool isEvaluated) const {
   assert(!isValueDependent() &&
          "Expression evaluator can't be called on a dependent expression.");
 
-  if (Ctx.getLangOpts().CPlusPlus11)
-    return EvaluateCPlusPlus11IntegralConstantExpr(Ctx, this, &Value, Loc);
+  APSInt Value;
+
+  if (Ctx.getLangOpts().CPlusPlus11) {
+    if (EvaluateCPlusPlus11IntegralConstantExpr(Ctx, this, &Value, Loc))
+      return Value;
+    return None;
+  }
 
   if (!isIntegerConstantExpr(Ctx, Loc))
-    return false;
+    return None;
 
   // The only possible side-effects here are due to UB discovered in the
   // evaluation (for instance, INT_MAX + 1). In such a case, we are still
@@ -14920,8 +14944,7 @@ bool Expr::isIntegerConstantExpr(llvm::APSInt &Value, const ASTContext &Ctx,
   if (!::EvaluateAsInt(this, ExprResult, Ctx, SE_AllowSideEffects, Info))
     llvm_unreachable("ICE cannot be evaluated!");
 
-  Value = ExprResult.Val.getInt();
-  return true;
+  return ExprResult.Val.getInt();
 }
 
 bool Expr::isCXX98IntegralConstantExpr(const ASTContext &Ctx) const {
diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp
index 1e9121c14c47d..0f01164f2f199 100644
--- a/clang/lib/AST/MicrosoftMangle.cpp
+++ b/clang/lib/AST/MicrosoftMangle.cpp
@@ -1372,9 +1372,9 @@ void MicrosoftCXXNameMangler::mangleIntegerLiteral(const llvm::APSInt &Value,
 
 void MicrosoftCXXNameMangler::mangleExpression(const Expr *E) {
   // See if this is a constant expression.
-  llvm::APSInt Value;
-  if (E->isIntegerConstantExpr(Value, Context.getASTContext())) {
-    mangleIntegerLiteral(Value, E->getType()->isBooleanType());
+  if (Optional<llvm::APSInt> Value =
+          E->getIntegerConstantExpr(Context.getASTContext())) {
+    mangleIntegerLiteral(*Value, E->getType()->isBooleanType());
     return;
   }
 
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index a0b0dca553907..6933c57425520 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -2127,21 +2127,21 @@ void OMPTraitInfo::getAsVariantMatchInfo(ASTContext &ASTCtx,
                    TraitProperty::user_condition_unknown &&
                "Ill-formed user condition, expected unknown trait property!");
 
-        llvm::APSInt CondVal;
-        if (Selector.ScoreOrCondition->isIntegerConstantExpr(CondVal, ASTCtx))
-          VMI.addTrait(CondVal.isNullValue()
-                          ? TraitProperty::user_condition_false
-                          : TraitProperty::user_condition_true);
+        if (Optional<APSInt> CondVal =
+                Selector.ScoreOrCondition->getIntegerConstantExpr(ASTCtx))
+          VMI.addTrait(CondVal->isNullValue()
+                           ? TraitProperty::user_condition_false
+                           : TraitProperty::user_condition_true);
         else
           VMI.addTrait(TraitProperty::user_condition_false);
         continue;
       }
 
-      llvm::APSInt Score;
+      Optional<llvm::APSInt> Score;
       llvm::APInt *ScorePtr = nullptr;
       if (Selector.ScoreOrCondition) {
-        if (Selector.ScoreOrCondition->isIntegerConstantExpr(Score, ASTCtx))
-          ScorePtr = &Score;
+        if ((Score = Selector.ScoreOrCondition->getIntegerConstantExpr(ASTCtx)))
+          ScorePtr = &*Score;
         else
           VMI.addTrait(TraitProperty::user_condition_false);
       }
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index 5b0a0ac392c00..91b984820cd22 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -710,6 +710,13 @@ const char *TextNodeDumper::getCommandName(unsigned CommandID) {
   return "<not a builtin command>";
 }
 
+void TextNodeDumper::printFPOptions(FPOptionsOverride FPO) {
+#define OPTION(NAME, TYPE, WIDTH, PREVIOUS)                                    \
+  if (FPO.has##NAME##Override())                                               \
+    OS << " " #NAME "=" << FPO.get##NAME##Override();
+#include "clang/Basic/FPOptions.def"
+}
+
 void TextNodeDumper::visitTextComment(const comments::TextComment *C,
                                       const comments::FullComment *) {
   OS << " Text=\"" << C->getText() << "\"";
@@ -937,6 +944,8 @@ void TextNodeDumper::VisitConstantExpr(const ConstantExpr *Node) {
 void TextNodeDumper::VisitCallExpr(const CallExpr *Node) {
   if (Node->usesADL())
     OS << " adl";
+  if (Node->hasStoredFPFeatures())
+    printFPOptions(Node->getFPFeatures());
 }
 
 void TextNodeDumper::VisitCXXOperatorCallExpr(const CXXOperatorCallExpr *Node) {
@@ -1053,6 +1062,8 @@ void TextNodeDumper::VisitUnaryOperator(const UnaryOperator *Node) {
      << UnaryOperator::getOpcodeStr(Node->getOpcode()) << "'";
   if (!Node->canOverflow())
     OS << " cannot overflow";
+  if (Node->hasStoredFPFeatures())
+    printFPOptions(Node->getStoredFPFeatures());
 }
 
 void TextNodeDumper::VisitUnaryExprOrTypeTraitExpr(
@@ -1081,6 +1092,8 @@ void TextNodeDumper::VisitExtVectorElementExpr(
 
 void TextNodeDumper::VisitBinaryOperator(const BinaryOperator *Node) {
   OS << " '" << BinaryOperator::getOpcodeStr(Node->getOpcode()) << "'";
+  if (Node->hasStoredFPFeatures())
+    printFPOptions(Node->getStoredFPFeatures());
 }
 
 void TextNodeDumper::VisitCompoundAssignOperator(
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index baf335ab9e546..6edf367a4fa4d 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -2318,6 +2318,13 @@ bool Type::isVLSTBuiltinType() const {
   return false;
 }
 
+bool Type::isVLST() const {
+  if (!isVLSTBuiltinType())
+    return false;
+
+  return hasAttr(attr::ArmSveVectorBits);
+}
+
 bool QualType::isPODType(const ASTContext &Context) const {
   // C++11 has a more relaxed definition of POD.
   if (Context.getLangOpts().CPlusPlus11)
diff --git a/clang/lib/Analysis/BodyFarm.cpp b/clang/lib/Analysis/BodyFarm.cpp
index f9f0553d28f05..ad2dc8ec7a60a 100644
--- a/clang/lib/Analysis/BodyFarm.cpp
+++ b/clang/lib/Analysis/BodyFarm.cpp
@@ -267,7 +267,7 @@ static CallExpr *create_call_once_funcptr_call(ASTContext &C, ASTMaker M,
   }
 
   return CallExpr::Create(C, SubExpr, CallArgs, C.VoidTy, VK_RValue,
-                          SourceLocation());
+                          SourceLocation(), FPOptionsOverride());
 }
 
 static CallExpr *create_call_once_lambda_call(ASTContext &C, ASTMaker M,
@@ -514,7 +514,7 @@ static Stmt *create_dispatch_once(ASTContext &C, const FunctionDecl *D) {
       /*Args=*/None,
       /*QualType=*/C.VoidTy,
       /*ExprValueType=*/VK_RValue,
-      /*SourceLocation=*/SourceLocation());
+      /*SourceLocation=*/SourceLocation(), FPOptionsOverride());
 
   // (2) Create the assignment to the predicate.
   Expr *DoneValue =
@@ -578,8 +578,8 @@ static Stmt *create_dispatch_sync(ASTContext &C, const FunctionDecl *D) {
   ASTMaker M(C);
   DeclRefExpr *DR = M.makeDeclRefExpr(PV);
   ImplicitCastExpr *ICE = M.makeLvalueToRvalue(DR, Ty);
-  CallExpr *CE =
-      CallExpr::Create(C, ICE, None, C.VoidTy, VK_RValue, SourceLocation());
+  CallExpr *CE = CallExpr::Create(C, ICE, None, C.VoidTy, VK_RValue,
+                                  SourceLocation(), FPOptionsOverride());
   return CE;
 }
 
diff --git a/clang/lib/Basic/Module.cpp b/clang/lib/Basic/Module.cpp
index b3daaa3a44422..b25248de6832d 100644
--- a/clang/lib/Basic/Module.cpp
+++ b/clang/lib/Basic/Module.cpp
@@ -173,14 +173,10 @@ bool Module::isAvailable(const LangOptions &LangOpts, const TargetInfo &Target,
 }
 
 bool Module::isSubModuleOf(const Module *Other) const {
-  const Module *This = this;
-  do {
-    if (This == Other)
+  for (auto *Parent = this; Parent; Parent = Parent->Parent) {
+    if (Parent == Other)
       return true;
-
-    This = This->Parent;
-  } while (This);
-
+  }
   return false;
 }
 
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index cae61ad4f2e3a..4807702e896e8 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -20,8 +20,8 @@
 using namespace clang;
 using namespace llvm::omp;
 
-unsigned clang::getOpenMPSimpleClauseType(OpenMPClauseKind Kind,
-                                          StringRef Str) {
+unsigned clang::getOpenMPSimpleClauseType(OpenMPClauseKind Kind, StringRef Str,
+                                          unsigned OpenMPVersion) {
   switch (Kind) {
   case OMPC_default:
     return llvm::StringSwitch<unsigned>(Str)
@@ -51,14 +51,18 @@ unsigned clang::getOpenMPSimpleClauseType(OpenMPClauseKind Kind,
 #define OPENMP_LINEAR_KIND(Name) .Case(#Name, OMPC_LINEAR_##Name)
 #include "clang/Basic/OpenMPKinds.def"
         .Default(OMPC_LINEAR_unknown);
-  case OMPC_map:
-    return llvm::StringSwitch<unsigned>(Str)
+  case OMPC_map: {
+    unsigned Type = llvm::StringSwitch<unsigned>(Str)
 #define OPENMP_MAP_KIND(Name)                                                  \
   .Case(#Name, static_cast<unsigned>(OMPC_MAP_##Name))
 #define OPENMP_MAP_MODIFIER_KIND(Name)                                         \
   .Case(#Name, static_cast<unsigned>(OMPC_MAP_MODIFIER_##Name))
 #include "clang/Basic/OpenMPKinds.def"
         .Default(OMPC_MAP_unknown);
+    if (OpenMPVersion < 51 && Type == OMPC_MAP_MODIFIER_present)
+      return OMPC_MAP_MODIFIER_unknown;
+    return Type;
+  }
   case OMPC_to:
     return llvm::StringSwitch<unsigned>(Str)
 #define OPENMP_TO_MODIFIER_KIND(Name)                                          \
diff --git a/clang/lib/Basic/Targets/SystemZ.h b/clang/lib/Basic/Targets/SystemZ.h
index d7869e3754a82..39fdcf90d0c86 100644
--- a/clang/lib/Basic/Targets/SystemZ.h
+++ b/clang/lib/Basic/Targets/SystemZ.h
@@ -157,6 +157,10 @@ class LLVM_LIBRARY_VISIBILITY SystemZTargetInfo : public TargetInfo {
   const char *getLongDoubleMangling() const override { return "g"; }
 
   bool hasExtIntType() const override { return true; }
+
+  int getEHDataRegisterNumber(unsigned RegNo) const override {
+    return RegNo < 4 ? 6 + RegNo : -1;
+  }
 };
 } // namespace targets
 } // namespace clang
diff --git a/clang/lib/Basic/Targets/X86.h b/clang/lib/Basic/Targets/X86.h
index b84795989c062..d99358d6f8066 100644
--- a/clang/lib/Basic/Targets/X86.h
+++ b/clang/lib/Basic/Targets/X86.h
@@ -360,7 +360,10 @@ class LLVM_LIBRARY_VISIBILITY X86_32TargetInfo : public X86TargetInfo {
     LongDoubleWidth = 96;
     LongDoubleAlign = 32;
     SuitableAlign = 128;
-    resetDataLayout("e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-f64:32:64-"
+    resetDataLayout(Triple.isOSBinFormatMachO() ?
+                    "e-m:o-p:32:32-p270:32:32-p271:32:32-p272:64:64-f64:32:64-"
+                    "f80:32-n8:16:32-S128" :
+                    "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-f64:32:64-"
                     "f80:32-n8:16:32-S128");
     SizeType = UnsignedInt;
     PtrDiffType = SignedInt;
diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index 59421c234442a..4bc48c9cd5287 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -27,6 +27,7 @@
 #include "llvm/CodeGen/RegAllocRegistry.h"
 #include "llvm/CodeGen/SchedulerRegistry.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
+#include "llvm/GenXIntrinsics/GenXSPIRVWriterAdaptor.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/IRPrintingPasses.h"
 #include "llvm/IR/LegacyPassManager.h"
@@ -40,6 +41,7 @@
 #include "llvm/Passes/PassBuilder.h"
 #include "llvm/Passes/PassPlugin.h"
 #include "llvm/Passes/StandardInstrumentations.h"
+#include "llvm/SYCLLowerIR/LowerESIMD.h"
 #include "llvm/Support/BuryPointer.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -74,6 +76,7 @@
 #include "llvm/Transforms/ObjCARC.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Scalar/GVN.h"
+#include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h"
 #include "llvm/Transforms/Utils.h"
 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
@@ -784,6 +787,25 @@ void EmitAssemblyHelper::CreatePasses(legacy::PassManager &MPM,
 
   PMBuilder.populateFunctionPassManager(FPM);
   PMBuilder.populateModulePassManager(MPM);
+
+  // Customize the tail of the module passes list for the ESIMD extension.
+  if (LangOpts.SYCLIsDevice && LangOpts.SYCLExplicitSIMD &&
+      CodeGenOpts.OptimizationLevel != 0) {
+    MPM.add(createESIMDLowerVecArgPass());
+    MPM.add(createESIMDLowerLoadStorePass());
+    MPM.add(createSROAPass());
+    MPM.add(createEarlyCSEPass(true));
+    MPM.add(createInstructionCombiningPass());
+    MPM.add(createDeadCodeEliminationPass());
+    MPM.add(createFunctionInliningPass(
+        CodeGenOpts.OptimizationLevel, CodeGenOpts.OptimizeSize,
+        (!CodeGenOpts.SampleProfileFile.empty() &&
+         CodeGenOpts.PrepareForThinLTO)));
+    MPM.add(createSROAPass());
+    MPM.add(createEarlyCSEPass(true));
+    MPM.add(createInstructionCombiningPass());
+    MPM.add(createDeadCodeEliminationPass());
+  }
 }
 
 static void setCommandLineOpts(const CodeGenOptions &CodeGenOpts) {
@@ -878,6 +900,11 @@ void EmitAssemblyHelper::EmitAssembly(BackendAction Action,
   PerFunctionPasses.add(
       createTargetTransformInfoWrapperPass(getTargetIRAnalysis()));
 
+  // ESIMD extension always requires lowering of certain IR constructs, such as
+  // ESIMD C++ intrinsics, as the last FE step.
+  if (LangOpts.SYCLIsDevice && LangOpts.SYCLExplicitSIMD)
+    PerModulePasses.add(createSYCLLowerESIMDPass());
+
   CreatePasses(PerModulePasses, PerFunctionPasses);
 
   legacy::PassManager CodeGenPasses;
@@ -890,6 +917,9 @@ void EmitAssemblyHelper::EmitAssembly(BackendAction Action,
   if (LangOpts.SYCLIsDevice && CodeGenOpts.DisableLLVMPasses)
     PerModulePasses.add(createDeadCodeEliminationPass());
 
+  if (LangOpts.SYCLIsDevice && LangOpts.SYCLExplicitSIMD)
+    PerModulePasses.add(createGenXSPIRVWriterAdaptorPass());
+
   switch (Action) {
   case Backend_EmitNothing:
     break;
@@ -1157,7 +1187,7 @@ void EmitAssemblyHelper::EmitAssemblyWithNewPassManager(
   PTO.Coroutines = LangOpts.Coroutines;
 
   PassInstrumentationCallbacks PIC;
-  StandardInstrumentations SI;
+  StandardInstrumentations SI(CodeGenOpts.DebugPassManager);
   SI.registerCallbacks(PIC);
   PassBuilder PB(TM.get(), PTO, PGOOpt, &PIC);
 
@@ -1376,6 +1406,13 @@ void EmitAssemblyHelper::EmitAssemblyWithNewPassManager(
     }
 
     if (CodeGenOpts.OptimizationLevel == 0) {
+      // FIXME: the backends do not handle matrix intrinsics currently. Make
+      // sure they are also lowered in O0. A lightweight version of the pass
+      // should run in the backend pipeline on demand.
+      if (LangOpts.MatrixTypes)
+        MPM.addPass(
+            createModuleToFunctionPassAdaptor(LowerMatrixIntrinsicsPass()));
+
       addCoroutinePassesAtO0(MPM, LangOpts, CodeGenOpts);
       addSanitizersAtO0(MPM, TargetTriple, LangOpts, CodeGenOpts);
     }
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 5bf2cc0beea85..4800c9adcd38d 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -1978,6 +1978,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     Value *Result = Builder.CreateSelect(CmpResult, NegOp, ArgValue, "abs");
     return RValue::get(Result);
   }
+  case Builtin::BI__builtin_complex: {
+    Value *Real = EmitScalarExpr(E->getArg(0));
+    Value *Imag = EmitScalarExpr(E->getArg(1));
+    return RValue::getComplex({Real, Imag});
+  }
   case Builtin::BI__builtin_conj:
   case Builtin::BI__builtin_conjf:
   case Builtin::BI__builtin_conjl:
@@ -4425,11 +4430,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
       } else {
         // If this is required to be a constant, constant fold it so that we
         // know that the generated intrinsic gets a ConstantInt.
-        llvm::APSInt Result;
-        bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result,getContext());
-        assert(IsConst && "Constant arg isn't actually constant?");
-        (void)IsConst;
-        ArgValue = llvm::ConstantInt::get(getLLVMContext(), Result);
+        ArgValue = llvm::ConstantInt::get(
+            getLLVMContext(),
+            *E->getArg(i)->getIntegerConstantExpr(getContext()));
       }
 
       // If the intrinsic arg type is different from the builtin arg type
@@ -5602,13 +5605,14 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr(
     SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1,
     llvm::Triple::ArchType Arch) {
   // Get the last argument, which specifies the vector type.
-  llvm::APSInt NeonTypeConst;
   const Expr *Arg = E->getArg(E->getNumArgs() - 1);
-  if (!Arg->isIntegerConstantExpr(NeonTypeConst, getContext()))
+  Optional<llvm::APSInt> NeonTypeConst =
+      Arg->getIntegerConstantExpr(getContext());
+  if (!NeonTypeConst)
     return nullptr;
 
   // Determine the type of this overloaded NEON intrinsic.
-  NeonTypeFlags Type(NeonTypeConst.getZExtValue());
+  NeonTypeFlags Type(NeonTypeConst->getZExtValue());
   bool Usgn = Type.isUnsigned();
   bool Quad = Type.isQuad();
   const bool HasLegalHalfType = getTarget().hasLegalHalfType();
@@ -6904,10 +6908,9 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     } else {
       // If this is required to be a constant, constant fold it so that we know
       // that the generated intrinsic gets a ConstantInt.
-      llvm::APSInt Result;
-      bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext());
-      assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst;
-      Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
+      Ops.push_back(llvm::ConstantInt::get(
+          getLLVMContext(),
+          *E->getArg(i)->getIntegerConstantExpr(getContext())));
     }
   }
 
@@ -7118,9 +7121,9 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
 
   // Get the last argument, which specifies the vector type.
   assert(HasExtraArg);
-  llvm::APSInt Result;
   const Expr *Arg = E->getArg(E->getNumArgs()-1);
-  if (!Arg->isIntegerConstantExpr(Result, getContext()))
+  Optional<llvm::APSInt> Result = Arg->getIntegerConstantExpr(getContext());
+  if (!Result)
     return nullptr;
 
   if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f ||
@@ -7133,7 +7136,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
       Ty = DoubleTy;
 
     // Determine whether this is an unsigned conversion or not.
-    bool usgn = Result.getZExtValue() == 1;
+    bool usgn = Result->getZExtValue() == 1;
     unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
 
     // Call the appropriate intrinsic.
@@ -7142,7 +7145,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   }
 
   // Determine the type of this overloaded NEON intrinsic.
-  NeonTypeFlags Type(Result.getZExtValue());
+  NeonTypeFlags Type = Result->getZExtValue();
   bool usgn = Type.isUnsigned();
   bool rightShift = false;
 
@@ -7286,11 +7289,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
 
 template<typename Integer>
 static Integer GetIntegerConstantValue(const Expr *E, ASTContext &Context) {
-  llvm::APSInt IntVal;
-  bool IsConst = E->isIntegerConstantExpr(IntVal, Context);
-  assert(IsConst && "Sema should have checked this was a constant");
-  (void)IsConst;
-  return IntVal.getExtValue();
+  return E->getIntegerConstantExpr(Context)->getExtValue();
 }
 
 static llvm::Value *SignOrZeroExtend(CGBuilderTy &Builder, llvm::Value *V,
@@ -7563,13 +7562,13 @@ static Value *EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID
   assert(E->getNumArgs() >= 3);
 
   // Get the last argument, which specifies the vector type.
-  llvm::APSInt Result;
   const Expr *Arg = E->getArg(E->getNumArgs() - 1);
-  if (!Arg->isIntegerConstantExpr(Result, CGF.getContext()))
+  Optional<llvm::APSInt> Result = Arg->getIntegerConstantExpr(CGF.getContext());
+  if (!Result)
     return nullptr;
 
   // Determine the type of this overloaded NEON intrinsic.
-  NeonTypeFlags Type(Result.getZExtValue());
+  NeonTypeFlags Type = Result->getZExtValue();
   llvm::VectorType *Ty = GetNeonType(&CGF, Type);
   if (!Ty)
     return nullptr;
@@ -8246,15 +8245,15 @@ Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID,
     else {
       // If this is required to be a constant, constant fold it so that we know
       // that the generated intrinsic gets a ConstantInt.
-      llvm::APSInt Result;
-      if (!E->getArg(i)->isIntegerConstantExpr(Result, getContext()))
-        llvm_unreachable("Expected argument to be a constant");
+      Optional<llvm::APSInt> Result =
+          E->getArg(i)->getIntegerConstantExpr(getContext());
+      assert(Result && "Expected argument to be a constant");
 
       // Immediates for SVE llvm intrinsics are always 32bit.  We can safely
       // truncate because the immediate has been range checked and no valid
       // immediate requires more than a handful of bits.
-      Result = Result.extOrTrunc(32);
-      Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
+      *Result = Result->extOrTrunc(32);
+      Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), *Result));
     }
   }
 
@@ -8958,11 +8957,9 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
     } else {
       // If this is required to be a constant, constant fold it so that we know
       // that the generated intrinsic gets a ConstantInt.
-      llvm::APSInt Result;
-      bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext());
-      assert(IsConst && "Constant arg isn't actually constant?");
-      (void)IsConst;
-      Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
+      Ops.push_back(llvm::ConstantInt::get(
+          getLLVMContext(),
+          *E->getArg(i)->getIntegerConstantExpr(getContext())));
     }
   }
 
@@ -8977,12 +8974,11 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
     return Result;
   }
 
-  llvm::APSInt Result;
   const Expr *Arg = E->getArg(E->getNumArgs()-1);
   NeonTypeFlags Type(0);
-  if (Arg->isIntegerConstantExpr(Result, getContext()))
+  if (Optional<llvm::APSInt> Result = Arg->getIntegerConstantExpr(getContext()))
     // Determine the type of this overloaded NEON intrinsic.
-    Type = NeonTypeFlags(Result.getZExtValue());
+    Type = NeonTypeFlags(Result->getZExtValue());
 
   bool usgn = Type.isUnsigned();
   bool quad = Type.isQuad();
@@ -11813,10 +11809,8 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
 
     // If this is required to be a constant, constant fold it so that we know
     // that the generated intrinsic gets a ConstantInt.
-    llvm::APSInt Result;
-    bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext());
-    assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst;
-    Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
+    Ops.push_back(llvm::ConstantInt::get(
+        getLLVMContext(), *E->getArg(i)->getIntegerConstantExpr(getContext())));
   }
 
   // These exist so that the builtin that takes an immediate can be bounds
@@ -15095,11 +15089,8 @@ Value *CodeGenFunction::EmitSystemZBuiltinExpr(unsigned BuiltinID,
     llvm::Type *ResultType = ConvertType(E->getType());
     Value *X = EmitScalarExpr(E->getArg(0));
     // Constant-fold the M4 and M5 mask arguments.
-    llvm::APSInt M4, M5;
-    bool IsConstM4 = E->getArg(1)->isIntegerConstantExpr(M4, getContext());
-    bool IsConstM5 = E->getArg(2)->isIntegerConstantExpr(M5, getContext());
-    assert(IsConstM4 && IsConstM5 && "Constant arg isn't actually constant?");
-    (void)IsConstM4; (void)IsConstM5;
+    llvm::APSInt M4 = *E->getArg(1)->getIntegerConstantExpr(getContext());
+    llvm::APSInt M5 = *E->getArg(2)->getIntegerConstantExpr(getContext());
     // Check whether this instance can be represented via a LLVM standard
     // intrinsic.  We only support some combinations of M4 and M5.
     Intrinsic::ID ID = Intrinsic::not_intrinsic;
@@ -15154,10 +15145,7 @@ Value *CodeGenFunction::EmitSystemZBuiltinExpr(unsigned BuiltinID,
     Value *X = EmitScalarExpr(E->getArg(0));
     Value *Y = EmitScalarExpr(E->getArg(1));
     // Constant-fold the M4 mask argument.
-    llvm::APSInt M4;
-    bool IsConstM4 = E->getArg(2)->isIntegerConstantExpr(M4, getContext());
-    assert(IsConstM4 && "Constant arg isn't actually constant?");
-    (void)IsConstM4;
+    llvm::APSInt M4 = *E->getArg(2)->getIntegerConstantExpr(getContext());
     // Check whether this instance can be represented via a LLVM standard
     // intrinsic.  We only support some values of M4.
     Intrinsic::ID ID = Intrinsic::not_intrinsic;
@@ -15191,10 +15179,7 @@ Value *CodeGenFunction::EmitSystemZBuiltinExpr(unsigned BuiltinID,
     Value *X = EmitScalarExpr(E->getArg(0));
     Value *Y = EmitScalarExpr(E->getArg(1));
     // Constant-fold the M4 mask argument.
-    llvm::APSInt M4;
-    bool IsConstM4 = E->getArg(2)->isIntegerConstantExpr(M4, getContext());
-    assert(IsConstM4 && "Constant arg isn't actually constant?");
-    (void)IsConstM4;
+    llvm::APSInt M4 = *E->getArg(2)->getIntegerConstantExpr(getContext());
     // Check whether this instance can be represented via a LLVM standard
     // intrinsic.  We only support some values of M4.
     Intrinsic::ID ID = Intrinsic::not_intrinsic;
@@ -15861,10 +15846,11 @@ CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E) {
     Address Dst = EmitPointerWithAlignment(E->getArg(0));
     Value *Src = EmitScalarExpr(E->getArg(1));
     Value *Ldm = EmitScalarExpr(E->getArg(2));
-    llvm::APSInt isColMajorArg;
-    if (!E->getArg(3)->isIntegerConstantExpr(isColMajorArg, getContext()))
+    Optional<llvm::APSInt> isColMajorArg =
+        E->getArg(3)->getIntegerConstantExpr(getContext());
+    if (!isColMajorArg)
       return nullptr;
-    bool isColMajor = isColMajorArg.getSExtValue();
+    bool isColMajor = isColMajorArg->getSExtValue();
     NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
     unsigned IID = isColMajor ? II.IID_col : II.IID_row;
     if (IID == 0)
@@ -15905,10 +15891,11 @@ CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E) {
     Value *Dst = EmitScalarExpr(E->getArg(0));
     Address Src = EmitPointerWithAlignment(E->getArg(1));
     Value *Ldm = EmitScalarExpr(E->getArg(2));
-    llvm::APSInt isColMajorArg;
-    if (!E->getArg(3)->isIntegerConstantExpr(isColMajorArg, getContext()))
+    Optional<llvm::APSInt> isColMajorArg =
+        E->getArg(3)->getIntegerConstantExpr(getContext());
+    if (!isColMajorArg)
       return nullptr;
-    bool isColMajor = isColMajorArg.getSExtValue();
+    bool isColMajor = isColMajorArg->getSExtValue();
     NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
     unsigned IID = isColMajor ? II.IID_col : II.IID_row;
     if (IID == 0)
@@ -15955,16 +15942,20 @@ CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E) {
     Address SrcA = EmitPointerWithAlignment(E->getArg(1));
     Address SrcB = EmitPointerWithAlignment(E->getArg(2));
     Address SrcC = EmitPointerWithAlignment(E->getArg(3));
-    llvm::APSInt LayoutArg;
-    if (!E->getArg(4)->isIntegerConstantExpr(LayoutArg, getContext()))
+    Optional<llvm::APSInt> LayoutArg =
+        E->getArg(4)->getIntegerConstantExpr(getContext());
+    if (!LayoutArg)
       return nullptr;
-    int Layout = LayoutArg.getSExtValue();
+    int Layout = LayoutArg->getSExtValue();
     if (Layout < 0 || Layout > 3)
       return nullptr;
     llvm::APSInt SatfArg;
     if (BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1)
       SatfArg = 0;  // .b1 does not have satf argument.
-    else if (!E->getArg(5)->isIntegerConstantExpr(SatfArg, getContext()))
+    else if (Optional<llvm::APSInt> OptSatfArg =
+                 E->getArg(5)->getIntegerConstantExpr(getContext()))
+      SatfArg = *OptSatfArg;
+    else
       return nullptr;
     bool Satf = SatfArg.getSExtValue();
     NVPTXMmaInfo MI = getNVPTXMmaInfo(BuiltinID);
@@ -16293,9 +16284,8 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
   case WebAssembly::BI__builtin_wasm_extract_lane_i64x2:
   case WebAssembly::BI__builtin_wasm_extract_lane_f32x4:
   case WebAssembly::BI__builtin_wasm_extract_lane_f64x2: {
-    llvm::APSInt LaneConst;
-    if (!E->getArg(1)->isIntegerConstantExpr(LaneConst, getContext()))
-      llvm_unreachable("Constant arg isn't actually constant?");
+    llvm::APSInt LaneConst =
+        *E->getArg(1)->getIntegerConstantExpr(getContext());
     Value *Vec = EmitScalarExpr(E->getArg(0));
     Value *Lane = llvm::ConstantInt::get(getLLVMContext(), LaneConst);
     Value *Extract = Builder.CreateExtractElement(Vec, Lane);
@@ -16321,9 +16311,8 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
   case WebAssembly::BI__builtin_wasm_replace_lane_i64x2:
   case WebAssembly::BI__builtin_wasm_replace_lane_f32x4:
   case WebAssembly::BI__builtin_wasm_replace_lane_f64x2: {
-    llvm::APSInt LaneConst;
-    if (!E->getArg(1)->isIntegerConstantExpr(LaneConst, getContext()))
-      llvm_unreachable("Constant arg isn't actually constant?");
+    llvm::APSInt LaneConst =
+        *E->getArg(1)->getIntegerConstantExpr(getContext());
     Value *Vec = EmitScalarExpr(E->getArg(0));
     Value *Lane = llvm::ConstantInt::get(getLLVMContext(), LaneConst);
     Value *Val = EmitScalarExpr(E->getArg(2));
@@ -16585,10 +16574,10 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
     Ops[OpIdx++] = EmitScalarExpr(E->getArg(0));
     Ops[OpIdx++] = EmitScalarExpr(E->getArg(1));
     while (OpIdx < 18) {
-      llvm::APSInt LaneConst;
-      if (!E->getArg(OpIdx)->isIntegerConstantExpr(LaneConst, getContext()))
-        llvm_unreachable("Constant arg isn't actually constant?");
-      Ops[OpIdx++] = llvm::ConstantInt::get(getLLVMContext(), LaneConst);
+      Optional<llvm::APSInt> LaneConst =
+          E->getArg(OpIdx)->getIntegerConstantExpr(getContext());
+      assert(LaneConst && "Constant arg isn't actually constant?");
+      Ops[OpIdx++] = llvm::ConstantInt::get(getLLVMContext(), *LaneConst);
     }
     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_shuffle);
     return Builder.CreateCall(Callee, Ops);
@@ -16893,15 +16882,15 @@ RValue CodeGenFunction::EmitIntelFPGAMemBuiltin(const CallExpr *E) {
   SmallString<256> AnnotStr;
   llvm::raw_svector_ostream Out(AnnotStr);
 
-  llvm::APSInt Params;
-  bool IsConst = E->getArg(1)->isIntegerConstantExpr(Params, getContext());
-  assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst;
-  Out << "{params:" << Params.toString(10) << "}";
+  Optional<llvm::APSInt> Params =
+      E->getArg(1)->getIntegerConstantExpr(getContext());
+  assert(Params.hasValue() && "Constant arg isn't actually constant?");
+  Out << "{params:" << Params->toString(10) << "}";
 
-  llvm::APSInt CacheSize;
-  IsConst = E->getArg(2)->isIntegerConstantExpr(CacheSize, getContext());
-  assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst;
-  Out << "{cache-size:" << CacheSize.toString(10) << "}";
+  Optional<llvm::APSInt> CacheSize =
+      E->getArg(2)->getIntegerConstantExpr(getContext());
+  assert(CacheSize.hasValue() && "Constant arg isn't actually constant?");
+  Out << "{cache-size:" << CacheSize->toString(10) << "}";
 
   llvm::Value *Ann = EmitAnnotationCall(F, PtrVal, AnnotStr, SourceLocation());
 
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index f1355c1ec21f9..2a33c8e42293b 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -4971,10 +4971,21 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
 
   // 4. Finish the call.
 
+  // SYCL does not support C++ exceptions or termination in device code, so all
+  // functions have to return.
+  bool SyclSkipNoReturn = false;
+  if (getLangOpts().SYCLIsDevice && CI->doesNotReturn()) {
+    if (auto *F = CI->getCalledFunction())
+      F->removeFnAttr(llvm::Attribute::NoReturn);
+    CI->removeAttribute(llvm::AttributeList::FunctionIndex,
+                        llvm::Attribute::NoReturn);
+    SyclSkipNoReturn = true;
+  }
+
   // If the call doesn't return for non-sycl devices, finish the basic block and
   // clear the insertion point; this allows the rest of IRGen to discard
   // unreachable code.
-  if (CI->doesNotReturn() && !getLangOpts().SYCLIsDevice) {
+  if (!SyclSkipNoReturn && CI->doesNotReturn()) {
     if (UnusedReturnSizePtr)
       PopCleanupBlock();
 
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index d6a55c6a6b495..96abefd384f2c 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -3904,15 +3904,17 @@ LValue CodeGenFunction::EmitOMPArraySectionExpr(const OMPArraySectionExpr *E,
     llvm::APSInt ConstLength;
     if (Length) {
       // Idx = LowerBound + Length - 1;
-      if (Length->isIntegerConstantExpr(ConstLength, C)) {
-        ConstLength = ConstLength.zextOrTrunc(PointerWidthInBits);
+      if (Optional<llvm::APSInt> CL = Length->getIntegerConstantExpr(C)) {
+        ConstLength = CL->zextOrTrunc(PointerWidthInBits);
         Length = nullptr;
       }
       auto *LowerBound = E->getLowerBound();
       llvm::APSInt ConstLowerBound(PointerWidthInBits, /*isUnsigned=*/false);
-      if (LowerBound && LowerBound->isIntegerConstantExpr(ConstLowerBound, C)) {
-        ConstLowerBound = ConstLowerBound.zextOrTrunc(PointerWidthInBits);
-        LowerBound = nullptr;
+      if (LowerBound) {
+        if (Optional<llvm::APSInt> LB = LowerBound->getIntegerConstantExpr(C)) {
+          ConstLowerBound = LB->zextOrTrunc(PointerWidthInBits);
+          LowerBound = nullptr;
+        }
       }
       if (!Length)
         --ConstLength;
@@ -3949,8 +3951,10 @@ LValue CodeGenFunction::EmitOMPArraySectionExpr(const OMPArraySectionExpr *E,
                              : BaseTy;
       if (auto *VAT = C.getAsVariableArrayType(ArrayTy)) {
         Length = VAT->getSizeExpr();
-        if (Length->isIntegerConstantExpr(ConstLength, C))
+        if (Optional<llvm::APSInt> L = Length->getIntegerConstantExpr(C)) {
+          ConstLength = *L;
           Length = nullptr;
+        }
       } else {
         auto *CAT = C.getAsConstantArrayType(ArrayTy);
         ConstLength = CAT->getSize();
diff --git a/clang/lib/CodeGen/CGLoopInfo.cpp b/clang/lib/CodeGen/CGLoopInfo.cpp
index da06b613411b7..e670c947e03dc 100644
--- a/clang/lib/CodeGen/CGLoopInfo.cpp
+++ b/clang/lib/CodeGen/CGLoopInfo.cpp
@@ -996,36 +996,24 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
       addSYCLIVDepInfo(Header->getContext(), IntelFPGAIVDep->getSafelenValue(),
                        IntelFPGAIVDep->getArrayDecl());
 
-    if (IntelFPGAII) {
-      llvm::APSInt ArgVal(32);
-      bool IsValid =
-          IntelFPGAII->getIntervalExpr()->isIntegerConstantExpr(ArgVal, Ctx);
-      assert(IsValid && "Not an integer constant expression");
-      (void)IsValid;
-      setSYCLIInterval(ArgVal.getSExtValue());
-    }
+    if (IntelFPGAII)
+      setSYCLIInterval(IntelFPGAII->getIntervalExpr()
+                           ->getIntegerConstantExpr(Ctx)
+                           ->getSExtValue());
 
     if (IntelFPGAMaxConcurrency) {
-      llvm::APSInt ArgVal(32);
-      bool IsValid =
-          IntelFPGAMaxConcurrency->getNThreadsExpr()->isIntegerConstantExpr(
-              ArgVal, Ctx);
-      assert(IsValid && "Not an integer constant expression");
-      (void)IsValid;
       setSYCLMaxConcurrencyEnable();
-      setSYCLMaxConcurrencyNThreads(ArgVal.getSExtValue());
+      setSYCLMaxConcurrencyNThreads(IntelFPGAMaxConcurrency->getNThreadsExpr()
+                                        ->getIntegerConstantExpr(Ctx)
+                                        ->getSExtValue());
     }
 
     if (IntelFPGALoopCoalesce) {
-      llvm::APSInt ArgVal(32);
-      if (auto *LCE = IntelFPGALoopCoalesce->getNExpr()) {
-        bool IsValid = LCE->isIntegerConstantExpr(ArgVal, Ctx);
-        assert(IsValid && "Not an integer constant expression");
-        (void)IsValid;
-        setSYCLLoopCoalesceNLevels(ArgVal.getSExtValue());
-      } else {
+      if (auto *LCE = IntelFPGALoopCoalesce->getNExpr())
+        setSYCLLoopCoalesceNLevels(
+            LCE->getIntegerConstantExpr(Ctx)->getSExtValue());
+      else
         setSYCLLoopCoalesceEnable();
-      }
     }
 
     if (IntelFPGADisableLoopPipelining) {
@@ -1033,25 +1021,18 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
     }
 
     if (IntelFPGAMaxInterleaving) {
-      llvm::APSInt ArgVal(32);
-      bool IsValid =
-          IntelFPGAMaxInterleaving->getNExpr()->isIntegerConstantExpr(ArgVal,
-                                                                      Ctx);
-      assert(IsValid && "Not an integer constant expression");
-      (void)IsValid;
       setSYCLMaxInterleavingEnable();
-      setSYCLMaxInterleavingNInvocations(ArgVal.getSExtValue());
+      setSYCLMaxInterleavingNInvocations(IntelFPGAMaxInterleaving->getNExpr()
+                                             ->getIntegerConstantExpr(Ctx)
+                                             ->getSExtValue());
     }
 
     if (IntelFPGASpeculatedIterations) {
-      llvm::APSInt ArgVal(32);
-      bool IsValid =
-          IntelFPGASpeculatedIterations->getNExpr()->isIntegerConstantExpr(
-              ArgVal, Ctx);
-      assert(IsValid && "Not an integer constant expression");
-      (void)IsValid;
       setSYCLSpeculatedIterationsEnable();
-      setSYCLSpeculatedIterationsNIterations(ArgVal.getSExtValue());
+      setSYCLSpeculatedIterationsNIterations(
+          IntelFPGASpeculatedIterations->getNExpr()
+              ->getIntegerConstantExpr(Ctx)
+              ->getSExtValue());
     }
   }
 
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index f6d36bd84385f..f16f2ed89de89 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -7043,6 +7043,9 @@ class MappableExprsHandler {
     /// Close is a hint to the runtime to allocate memory close to
     /// the target device.
     OMP_MAP_CLOSE = 0x400,
+    /// 0x800 is reserved for compatibility with XLC.
+    /// Produce a runtime error if the data is not already allocated.
+    OMP_MAP_PRESENT = 0x1000,
     /// The 16 MSBs of the flags indicate whether the entry is member of some
     /// struct/class.
     OMP_MAP_MEMBER_OF = 0xffff000000000000,
@@ -7287,6 +7290,9 @@ class MappableExprsHandler {
     if (llvm::find(MapModifiers, OMPC_MAP_MODIFIER_close)
         != MapModifiers.end())
       Bits |= OMP_MAP_CLOSE;
+    if (llvm::find(MapModifiers, OMPC_MAP_MODIFIER_present)
+        != MapModifiers.end())
+      Bits |= OMP_MAP_PRESENT;
     return Bits;
   }
 
@@ -7379,6 +7385,9 @@ class MappableExprsHandler {
     //
     // map(p[1:24])
     // p, &p[1], 24*sizeof(float), TARGET_PARAM | TO | FROM
+    // for data directives
+    // p, p, sizeof(float*), TARGET_PARAM | TO | FROM
+    // p, &p[1], 24*sizeof(float), PTR_AND_OBJ | TO | FROM
     //
     // map(s)
     // &s, &s, sizeof(S2), TARGET_PARAM | TO | FROM
@@ -7557,9 +7566,15 @@ class MappableExprsHandler {
       if (Ty->isAnyPointerType() && std::next(I) != CE) {
         BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());
 
-        // We do not need to generate individual map information for the
-        // pointer, it can be associated with the combined storage.
-        ++I;
+        // For non-data directives, we do not need to generate individual map
+        // information for the pointer, it can be associated with the combined
+        // storage.
+        if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
+            !CurDir.is<const OMPExecutableDirective *>() ||
+            !isOpenMPTargetDataManagementDirective(
+                CurDir.get<const OMPExecutableDirective *>()
+                    ->getDirectiveKind()))
+          ++I;
       }
     }
 
@@ -7633,6 +7648,7 @@ class MappableExprsHandler {
                 isa<MemberExpr>(Next->getAssociatedExpression()) ||
                 isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) ||
                 isa<OMPArraySectionExpr>(Next->getAssociatedExpression()) ||
+                isa<OMPArrayShapingExpr>(Next->getAssociatedExpression()) ||
                 isa<UnaryOperator>(Next->getAssociatedExpression()) ||
                 isa<BinaryOperator>(Next->getAssociatedExpression())) &&
                "Unexpected expression");
@@ -7960,6 +7976,13 @@ class MappableExprsHandler {
     CombinedInfo.Sizes.push_back(Size);
     // Map type is always TARGET_PARAM
     CombinedInfo.Types.push_back(OMP_MAP_TARGET_PARAM);
+    // If any element has the present modifier, then make sure the runtime
+    // doesn't attempt to allocate the struct.
+    if (CurTypes.end() !=
+        llvm::find_if(CurTypes, [](OpenMPOffloadMappingFlags Type) {
+          return Type & OMP_MAP_PRESENT;
+        }))
+      CombinedInfo.Types.back() |= OMP_MAP_PRESENT;
     // Remove TARGET_PARAM flag from the first element
     (*CurTypes.begin()) &= ~OMP_MAP_TARGET_PARAM;
 
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
index 92eca33ee97d6..1cd89c540f478 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
@@ -20,6 +20,7 @@
 #include "clang/AST/StmtVisitor.h"
 #include "clang/Basic/Cuda.h"
 #include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/Frontend/OpenMP/OMPGridValues.h"
 #include "llvm/IR/IntrinsicsNVPTX.h"
 
 using namespace clang;
@@ -195,11 +196,9 @@ class ExecutionRuntimeModesRAII {
 /// code.  For all practical purposes this is fine because the configuration
 /// is the same for all known NVPTX architectures.
 enum MachineConfiguration : unsigned {
-  WarpSize = 32,
-  /// Number of bits required to represent a lane identifier, which is
-  /// computed as log_2(WarpSize).
-  LaneIDBits = 5,
-  LaneIDMask = WarpSize - 1,
+  /// See "llvm/Frontend/OpenMP/OMPGridValues.h" for various related target
+  /// specific Grid Values like GV_Warp_Size, GV_Warp_Size_Log2,
+  /// and GV_Warp_Size_Log2_Mask.
 
   /// Global memory alignment for performance.
   GlobalMemoryAlignment = 128,
@@ -431,6 +430,7 @@ class CheckVarsEscapingDeclContext final
     assert(!GlobalizedRD &&
            "Record for globalized variables is built already.");
     ArrayRef<const ValueDecl *> EscapedDeclsForParallel, EscapedDeclsForTeams;
+    unsigned WarpSize = CGF.getTarget().getGridValue(llvm::omp::GV_Warp_Size);
     if (IsInTTDRegion)
       EscapedDeclsForTeams = EscapedDecls.getArrayRef();
     else
@@ -634,6 +634,8 @@ static llvm::Value *getNVPTXThreadID(CodeGenFunction &CGF) {
 /// on the NVPTX device, to generate more efficient code.
 static llvm::Value *getNVPTXWarpID(CodeGenFunction &CGF) {
   CGBuilderTy &Bld = CGF.Builder;
+  unsigned LaneIDBits =
+      CGF.getTarget().getGridValue(llvm::omp::GV_Warp_Size_Log2);
   return Bld.CreateAShr(getNVPTXThreadID(CGF), LaneIDBits, "nvptx_warp_id");
 }
 
@@ -642,6 +644,8 @@ static llvm::Value *getNVPTXWarpID(CodeGenFunction &CGF) {
 /// on the NVPTX device, to generate more efficient code.
 static llvm::Value *getNVPTXLaneID(CodeGenFunction &CGF) {
   CGBuilderTy &Bld = CGF.Builder;
+  unsigned LaneIDMask = CGF.getContext().getTargetInfo().getGridValue(
+      llvm::omp::GV_Warp_Size_Log2_Mask);
   return Bld.CreateAnd(getNVPTXThreadID(CGF), Bld.getInt32(LaneIDMask),
                        "nvptx_lane_id");
 }
@@ -2058,6 +2062,7 @@ llvm::Function *CGOpenMPRuntimeGPU::emitTeamsOutlinedFunction(
   const RecordDecl *GlobalizedRD = nullptr;
   llvm::SmallVector<const ValueDecl *, 4> LastPrivatesReductions;
   llvm::SmallDenseMap<const ValueDecl *, const FieldDecl *> MappedDeclsFields;
+  unsigned WarpSize = CGM.getTarget().getGridValue(llvm::omp::GV_Warp_Size);
   // Globalize team reductions variable unconditionally in all modes.
   if (getExecutionMode() != CGOpenMPRuntimeGPU::EM_SPMD)
     getTeamsReductionVars(CGM.getContext(), D, LastPrivatesReductions);
@@ -3233,6 +3238,7 @@ static llvm::Value *emitInterWarpCopyFunction(CodeGenModule &CGM,
       "__openmp_nvptx_data_transfer_temporary_storage";
   llvm::GlobalVariable *TransferMedium =
       M.getGlobalVariable(TransferMediumName);
+  unsigned WarpSize = CGF.getTarget().getGridValue(llvm::omp::GV_Warp_Size);
   if (!TransferMedium) {
     auto *Ty = llvm::ArrayType::get(CGM.Int32Ty, WarpSize);
     unsigned SharedAddressSpace = C.getTargetAddressSpace(LangAS::cuda_shared);
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.h b/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.h
index 40679f9143b96..6dab79e6e20ae 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.h
@@ -26,7 +26,7 @@ class CGOpenMPRuntimeNVPTX : public CGOpenMPRuntimeGPU {
 
 public:
   explicit CGOpenMPRuntimeNVPTX(CodeGenModule &CGM);
-  llvm::Value *getGPUWarpSize(CodeGenFunction &CGF);
+  llvm::Value *getGPUWarpSize(CodeGenFunction &CGF) override;
 };
 
 } // CodeGen namespace.
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 45ee10d829df0..126d78c41e2f5 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -1977,12 +1977,16 @@ static llvm::MDNode *getAsmSrcLocInfo(const StringLiteral *Str,
 }
 
 static void UpdateAsmCallInst(llvm::CallBase &Result, bool HasSideEffect,
-                              bool ReadOnly, bool ReadNone, const AsmStmt &S,
+                              bool ReadOnly, bool ReadNone, bool NoMerge,
+                              const AsmStmt &S,
                               const std::vector<llvm::Type *> &ResultRegTypes,
                               CodeGenFunction &CGF,
                               std::vector<llvm::Value *> &RegResults) {
   Result.addAttribute(llvm::AttributeList::FunctionIndex,
                       llvm::Attribute::NoUnwind);
+  if (NoMerge)
+    Result.addAttribute(llvm::AttributeList::FunctionIndex,
+                        llvm::Attribute::NoMerge);
   // Attach readnone and readonly attributes.
   if (!HasSideEffect) {
     if (ReadNone)
@@ -2357,12 +2361,14 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
         Builder.CreateCallBr(IA, Fallthrough, Transfer, Args);
     EmitBlock(Fallthrough);
     UpdateAsmCallInst(cast<llvm::CallBase>(*Result), HasSideEffect, ReadOnly,
-                      ReadNone, S, ResultRegTypes, *this, RegResults);
+                      ReadNone, InNoMergeAttributedStmt, S, ResultRegTypes,
+                      *this, RegResults);
   } else {
     llvm::CallInst *Result =
         Builder.CreateCall(IA, Args, getBundlesForFunclet(IA));
     UpdateAsmCallInst(cast<llvm::CallBase>(*Result), HasSideEffect, ReadOnly,
-                      ReadNone, S, ResultRegTypes, *this, RegResults);
+                      ReadNone, InNoMergeAttributedStmt, S, ResultRegTypes,
+                      *this, RegResults);
   }
 
   assert(RegResults.size() == ResultRegTypes.size());
diff --git a/clang/lib/CodeGen/CMakeLists.txt b/clang/lib/CodeGen/CMakeLists.txt
index 46e019be63287..a32addd7a898f 100644
--- a/clang/lib/CodeGen/CMakeLists.txt
+++ b/clang/lib/CodeGen/CMakeLists.txt
@@ -31,6 +31,13 @@ if (MSVC)
   set_source_files_properties(CodeGenModule.cpp PROPERTIES COMPILE_FLAGS /bigobj)
 endif()
 
+get_property(LLVMGenXIntrinsics_SOURCE_DIR GLOBAL PROPERTY LLVMGenXIntrinsics_SOURCE_PROP)
+get_property(LLVMGenXIntrinsics_BINARY_DIR GLOBAL PROPERTY LLVMGenXIntrinsics_BINARY_PROP)
+
+include_directories(
+  ${LLVMGenXIntrinsics_SOURCE_DIR}/GenXIntrinsics/include
+  ${LLVMGenXIntrinsics_BINARY_DIR}/GenXIntrinsics/include)
+
 add_clang_library(clangCodeGen
   BackendUtil.cpp
   CGAtomic.cpp
@@ -91,8 +98,14 @@ add_clang_library(clangCodeGen
   TargetInfo.cpp
   VarBypassDetector.cpp
 
+  ADDITIONAL_HEADER_DIRS
+  ${LLVMGenXIntrinsics_SOURCE_DIR}/GenXIntrinsics/include
+  ${LLVMGenXIntrinsics_BINARY_DIR}/GenXIntrinsics/include
+
   DEPENDS
+  ${codegen_deps}
   intrinsics_gen
+  LLVMGenXIntrinsics
 
   LINK_LIBS
   clangAnalysis
@@ -102,4 +115,5 @@ add_clang_library(clangCodeGen
   clangFrontend
   clangLex
   clangSerialization
+  LLVMGenXIntrinsics
   )
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index ea9dee22378b0..43ba631031afc 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -609,14 +609,12 @@ void CodeGenFunction::EmitOpenCLKernelMetadata(const FunctionDecl *FD,
 
   if (const IntelReqdSubGroupSizeAttr *A =
           FD->getAttr<IntelReqdSubGroupSizeAttr>()) {
-    llvm::APSInt ArgVal(32);
     llvm::LLVMContext &Context = getLLVMContext();
-    bool IsValid = A->getSubGroupSize()->isIntegerConstantExpr(
-        ArgVal, FD->getASTContext());
-    assert(IsValid && "Not an integer constant expression");
-    (void)IsValid;
-    llvm::Metadata *AttrMDArgs[] = {
-        llvm::ConstantAsMetadata::get(Builder.getInt32(ArgVal.getSExtValue()))};
+    Optional<llvm::APSInt> ArgVal =
+        A->getSubGroupSize()->getIntegerConstantExpr(FD->getASTContext());
+    assert(ArgVal.hasValue() && "Not an integer constant expression");
+    llvm::Metadata *AttrMDArgs[] = {llvm::ConstantAsMetadata::get(
+        Builder.getInt32(ArgVal->getSExtValue()))};
     Fn->setMetadata("intel_reqd_sub_group_size",
                     llvm::MDNode::get(Context, AttrMDArgs));
   }
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 3d633e35fdcd1..43f0db10f49b9 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -641,12 +641,12 @@ void CodeGenModule::Release() {
     SPIRVerMD->addOperand(llvm::MDNode::get(Ctx, SPIRVerElts));
     // We are trying to look like OpenCL C++ for SPIR-V translator.
     // 4 - OpenCL_CPP, 100000 - OpenCL C++ version 1.0
-    // 6 - ESIMD, if any kernel or function is an explicit SIMD one
+    // 0 - ESIMD, if any kernel or function is an explicit SIMD one
     int Lang = llvm::any_of(TheModule,
                             [](const auto &F) {
                               return F.getMetadata("sycl_explicit_simd");
                             })
-                   ? 6
+                   ? 0
                    : 4;
 
     llvm::Metadata *SPIRVSourceElts[] = {
@@ -3523,16 +3523,14 @@ static void maybeEmitPipeStorageMetadata(const VarDecl *D,
     return;
 
   if (auto *IOAttr = D->getAttr<SYCLIntelPipeIOAttr>()) {
-    llvm::APSInt ID(32);
+    Optional<llvm::APSInt> ID =
+        IOAttr->getID()->getIntegerConstantExpr(D->getASTContext());
     llvm::LLVMContext &Context = CGM.getLLVMContext();
-    bool IsValid =
-        IOAttr->getID()->isIntegerConstantExpr(ID, D->getASTContext());
-    assert(IsValid && "Not an integer constant expression");
-    (void)IsValid;
+    assert(bool(ID) && "Not an integer constant expression");
 
     llvm::Metadata *AttrMDArgs[] = {
         llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
-            llvm::Type::getInt32Ty(Context), ID.getSExtValue()))};
+            llvm::Type::getInt32Ty(Context), ID->getSExtValue()))};
     GV->setMetadata(IOAttr->getSpelling(),
                     llvm::MDNode::get(Context, AttrMDArgs));
   }
@@ -3889,13 +3887,13 @@ LangAS CodeGenModule::GetGlobalVarAddressSpace(const VarDecl *D) {
   }
 
   if (LangOpts.SYCLIsDevice) {
+    if (!D)
+      return LangAS::opencl_global;
     auto *Scope = D->getAttr<SYCLScopeAttr>();
-
     if (Scope && Scope->isWorkGroup())
       return LangAS::opencl_local;
-    if (!D || D->getType().getAddressSpace() == LangAS::Default) {
+    if (D->getType().getAddressSpace() == LangAS::Default)
       return LangAS::opencl_global;
-    }
   }
 
   if (LangOpts.CUDA && LangOpts.CUDAIsDevice) {
diff --git a/clang/lib/Driver/SanitizerArgs.cpp b/clang/lib/Driver/SanitizerArgs.cpp
index bcc9ffc7ff8f6..e4fda752c041d 100644
--- a/clang/lib/Driver/SanitizerArgs.cpp
+++ b/clang/lib/Driver/SanitizerArgs.cpp
@@ -240,6 +240,10 @@ static SanitizerMask parseSanitizeTrapArgs(const Driver &D,
   return TrappingKinds;
 }
 
+bool SanitizerArgs::needsFuzzerInterceptors() const {
+  return needsFuzzer() && !needsAsanRt() && !needsTsanRt() && !needsMsanRt();
+}
+
 bool SanitizerArgs::needsUbsanRt() const {
   // All of these include ubsan.
   if (needsAsanRt() || needsMsanRt() || needsHwasanRt() || needsTsanRt() ||
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 30a64b1de34a9..3021ca245ad68 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -605,18 +605,42 @@ std::string ToolChain::GetProgramPath(const char *Name) const {
 }
 
 std::string ToolChain::GetLinkerPath() const {
+  // Get -fuse-ld= first to prevent -Wunused-command-line-argument. -fuse-ld= is
+  // considered as the linker flavor, e.g. "bfd", "gold", or "lld".
   const Arg* A = Args.getLastArg(options::OPT_fuse_ld_EQ);
   StringRef UseLinker = A ? A->getValue() : CLANG_DEFAULT_LINKER;
 
+  // --ld-path= takes precedence over -fuse-ld= and specifies the executable
+  // name. -B, COMPILER_PATH and PATH and consulted if the value does not
+  // contain a path component separator.
+  if (const Arg *A = Args.getLastArg(options::OPT_ld_path_EQ)) {
+    std::string Path(A->getValue());
+    if (!Path.empty()) {
+      if (llvm::sys::path::parent_path(Path).empty())
+        Path = GetProgramPath(A->getValue());
+      if (llvm::sys::fs::can_execute(Path))
+        return std::string(Path);
+    }
+    getDriver().Diag(diag::err_drv_invalid_linker_name) << A->getAsString(Args);
+    return GetProgramPath(getDefaultLinker());
+  }
+  // If we're passed -fuse-ld= with no argument, or with the argument ld,
+  // then use whatever the default system linker is.
+  if (UseLinker.empty() || UseLinker == "ld")
+    return GetProgramPath(getDefaultLinker());
+
+  // Extending -fuse-ld= to an absolute or relative path is unexpected. Checking
+  // for the linker flavor is brittle. In addition, prepending "ld." or "ld64."
+  // to a relative path is surprising. This is more complex due to priorities
+  // among -B, COMPILER_PATH and PATH. --ld-path= should be used instead.
+  if (UseLinker.find('/') != StringRef::npos)
+    getDriver().Diag(diag::warn_drv_use_ld_non_word);
+
   if (llvm::sys::path::is_absolute(UseLinker)) {
     // If we're passed what looks like an absolute path, don't attempt to
     // second-guess that.
     if (llvm::sys::fs::can_execute(UseLinker))
       return std::string(UseLinker);
-  } else if (UseLinker.empty() || UseLinker == "ld") {
-    // If we're passed -fuse-ld= with no argument, or with the argument ld,
-    // then use whatever the default system linker is.
-    return GetProgramPath(getDefaultLinker());
   } else {
     llvm::SmallString<8> LinkerName;
     if (Triple.isOSDarwin())
diff --git a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
index 428b72a489041..43959f5abe437 100644
--- a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
@@ -370,8 +370,8 @@ void aarch64::getAArch64TargetFeatures(const Driver &D,
     V8_6Pos = Features.insert(std::next(V8_6Pos), {"+i8mm", "+bf16"});
 
   bool HasSve = llvm::is_contained(Features, "+sve");
-  // -msve_vector_bits=<bits> flag is valid only if SVE is enabled.
-  if (Arg *A = Args.getLastArg(options::OPT_msve_vector_bits_EQ))
+  // -msve-vector-bits=<bits> flag is valid only if SVE is enabled.
+  if (Args.hasArg(options::OPT_msve_vector_bits_EQ))
     if (!HasSve)
       D.Diag(diag::err_drv_invalid_sve_vector_bits);
 
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index af00b1a3b92be..4d1b12e337dbb 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -1763,15 +1763,15 @@ void Clang::AddAArch64TargetArgs(const ArgList &Args,
   if (Arg *A = Args.getLastArg(options::OPT_msve_vector_bits_EQ)) {
     StringRef Val = A->getValue();
     const Driver &D = getToolChain().getDriver();
-    if (!Val.equals("128") && !Val.equals("256") && !Val.equals("512") &&
-        !Val.equals("1024") && !Val.equals("2048")) {
+    if (Val.equals("128") || Val.equals("256") || Val.equals("512") ||
+        Val.equals("1024") || Val.equals("2048"))
+      CmdArgs.push_back(
+          Args.MakeArgString(llvm::Twine("-msve-vector-bits=") + Val));
+    // Silently drop requests for vector-length agnostic code as it's implied.
+    else if (!Val.equals("scalable"))
       // Handle the unsupported values passed to msve-vector-bits.
       D.Diag(diag::err_drv_unsupported_option_argument)
           << A->getOption().getName() << Val;
-    } else if (A->getOption().matches(options::OPT_msve_vector_bits_EQ)) {
-      CmdArgs.push_back(
-          Args.MakeArgString(llvm::Twine("-msve-vector-bits=") + Val));
-    }
   }
 }
 
@@ -5774,6 +5774,12 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (Args.hasFlag(options::OPT_fpch_instantiate_templates,
                    options::OPT_fno_pch_instantiate_templates, false))
     CmdArgs.push_back("-fpch-instantiate-templates");
+  if (Args.hasFlag(options::OPT_fpch_codegen, options::OPT_fno_pch_codegen,
+                   false))
+    CmdArgs.push_back("-fmodules-codegen");
+  if (Args.hasFlag(options::OPT_fpch_debuginfo, options::OPT_fno_pch_debuginfo,
+                   false))
+    CmdArgs.push_back("-fmodules-debuginfo");
 
   Args.AddLastArg(CmdArgs, options::OPT_fexperimental_new_pass_manager,
                   options::OPT_fno_experimental_new_pass_manager);
@@ -7658,6 +7664,8 @@ void SPIRVTranslator::ConstructJob(Compilation &C, const JobAction &JA,
   if (getToolChain().getTriple().isSYCLDeviceEnvironment()) {
     TranslatorArgs.push_back("-spirv-max-version=1.1");
     std::string ExtArg("-spirv-ext=+all");
+    if (C.getArgs().hasArg(options::OPT_fsycl_esimd))
+      TranslatorArgs.push_back("-spirv-allow-unknown-intrinsics");
     // Disable SPV_INTEL_usm_storage_classes by default since it adds new
     // storage classes that represent global_device and global_host address
     // spaces, which are not supported for all targets. With the extension
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 1c612d36269d9..01780924fb06a 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -792,6 +792,9 @@ bool tools::addSanitizerRuntimes(const ToolChain &TC, const ArgList &Args,
       !Args.hasArg(options::OPT_shared)) {
 
     addSanitizerRuntime(TC, Args, CmdArgs, "fuzzer", false, true);
+    if (SanArgs.needsFuzzerInterceptors())
+      addSanitizerRuntime(TC, Args, CmdArgs, "fuzzer_interceptors", false,
+                          true);
     if (!Args.hasArg(clang::driver::options::OPT_nostdlibxx))
       TC.AddCXXStdlibLibArgs(Args, CmdArgs);
   }
diff --git a/clang/lib/Driver/ToolChains/Cuda.cpp b/clang/lib/Driver/ToolChains/Cuda.cpp
index 7ce542dd798a3..e5e2547299861 100644
--- a/clang/lib/Driver/ToolChains/Cuda.cpp
+++ b/clang/lib/Driver/ToolChains/Cuda.cpp
@@ -259,13 +259,13 @@ void CudaInstallationDetector::AddCudaIncludeArgs(
 void CudaInstallationDetector::CheckCudaVersionSupportsArch(
     CudaArch Arch) const {
   if (Arch == CudaArch::UNKNOWN || Version == CudaVersion::UNKNOWN ||
-      ArchsWithBadVersion.count(Arch) > 0)
+      ArchsWithBadVersion[(int)Arch])
     return;
 
   auto MinVersion = MinVersionForCudaArch(Arch);
   auto MaxVersion = MaxVersionForCudaArch(Arch);
   if (Version < MinVersion || Version > MaxVersion) {
-    ArchsWithBadVersion.insert(Arch);
+    ArchsWithBadVersion[(int)Arch] = true;
     D.Diag(diag::err_drv_cuda_version_unsupported)
         << CudaArchToString(Arch) << CudaVersionToString(MinVersion)
         << CudaVersionToString(MaxVersion) << InstallPath
diff --git a/clang/lib/Driver/ToolChains/Cuda.h b/clang/lib/Driver/ToolChains/Cuda.h
index 3e51b7254b163..47a4dcc1c2307 100644
--- a/clang/lib/Driver/ToolChains/Cuda.h
+++ b/clang/lib/Driver/ToolChains/Cuda.h
@@ -16,9 +16,9 @@
 #include "clang/Driver/Tool.h"
 #include "clang/Driver/ToolChain.h"
 #include "llvm/ADT/Optional.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/VersionTuple.h"
+#include <bitset>
 #include <set>
 #include <vector>
 
@@ -42,7 +42,7 @@ class CudaInstallationDetector {
 
   // CUDA architectures for which we have raised an error in
   // CheckCudaVersionSupportsArch.
-  mutable llvm::SmallSet<CudaArch, 4> ArchsWithBadVersion;
+  mutable std::bitset<(int)CudaArch::LAST> ArchsWithBadVersion;
 
 public:
   CudaInstallationDetector(const Driver &D, const llvm::Triple &HostTriple,
diff --git a/clang/lib/Driver/ToolChains/ROCm.h b/clang/lib/Driver/ToolChains/ROCm.h
index 962c72fedfe07..27c7d8b0ee543 100644
--- a/clang/lib/Driver/ToolChains/ROCm.h
+++ b/clang/lib/Driver/ToolChains/ROCm.h
@@ -13,7 +13,6 @@
 #include "clang/Basic/LLVM.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Options.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/Triple.h"
@@ -103,10 +102,6 @@ class RocmInstallationDetector {
            CorrectlyRoundedSqrt.isValid();
   }
 
-  // GPU architectures for which we have raised an error in
-  // CheckRocmVersionSupportsArch.
-  mutable llvm::SmallSet<CudaArch, 4> ArchsWithBadVersion;
-
   void scanLibDevicePath(llvm::StringRef Path);
   void ParseHIPVersionFile(llvm::StringRef V);
   SmallVector<Candidate, 4> getInstallationPathCandidates();
@@ -124,12 +119,6 @@ class RocmInstallationDetector {
                                   bool DAZ, bool FiniteOnly, bool UnsafeMathOpt,
                                   bool FastRelaxedMath, bool CorrectSqrt) const;
 
-  /// Emit an error if Version does not support the given Arch.
-  ///
-  /// If either Version or Arch is unknown, does not emit an error.  Emits at
-  /// most one error per Arch.
-  void CheckRocmVersionSupportsArch(CudaArch Arch) const;
-
   /// Check whether we detected a valid HIP runtime.
   bool hasHIPRuntime() const { return HasHIPRuntime; }
 
diff --git a/clang/lib/Driver/ToolChains/SYCL.cpp b/clang/lib/Driver/ToolChains/SYCL.cpp
index 80aa75bcae16c..f3b58dd3d50f6 100644
--- a/clang/lib/Driver/ToolChains/SYCL.cpp
+++ b/clang/lib/Driver/ToolChains/SYCL.cpp
@@ -42,6 +42,8 @@ const char *SYCL::Linker::constructLLVMSpirvCommand(Compilation &C,
   } else {
     CmdArgs.push_back("-spirv-max-version=1.1");
     CmdArgs.push_back("-spirv-ext=+all");
+    if (C.getArgs().hasArg(options::OPT_fsycl_esimd))
+      CmdArgs.push_back("-spirv-allow-unknown-intrinsics");
     CmdArgs.push_back("-o");
     CmdArgs.push_back(Output.getFilename());
   }
diff --git a/clang/lib/Driver/Types.cpp b/clang/lib/Driver/Types.cpp
index 8a0ba93f9b841..47959d7d0f8c9 100644
--- a/clang/lib/Driver/Types.cpp
+++ b/clang/lib/Driver/Types.cpp
@@ -141,7 +141,7 @@ bool types::isAcceptedByClang(ID Id) {
   case TY_CXXHeader: case TY_PP_CXXHeader:
   case TY_ObjCXXHeader: case TY_PP_ObjCXXHeader:
   case TY_CXXModule: case TY_PP_CXXModule:
-  case TY_AST: case TY_ModuleFile:
+  case TY_AST: case TY_ModuleFile: case TY_PCH:
   case TY_LLVM_IR: case TY_LLVM_BC:
   case TY_SPIRV:
     return true;
diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index 0d277a6464af2..3966f0a38639b 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -366,6 +366,17 @@ struct ScalarEnumerationTraits<FormatStyle::SpaceBeforeParensOptions> {
   }
 };
 
+template <>
+struct ScalarEnumerationTraits<FormatStyle::BitFieldColonSpacingStyle> {
+  static void enumeration(IO &IO,
+                          FormatStyle::BitFieldColonSpacingStyle &Value) {
+    IO.enumCase(Value, "Both", FormatStyle::BFCS_Both);
+    IO.enumCase(Value, "None", FormatStyle::BFCS_None);
+    IO.enumCase(Value, "Before", FormatStyle::BFCS_Before);
+    IO.enumCase(Value, "After", FormatStyle::BFCS_After);
+  }
+};
+
 template <> struct MappingTraits<FormatStyle> {
   static void mapping(IO &IO, FormatStyle &Style) {
     // When reading, read the language first, we need it for getPredefinedStyle.
@@ -593,6 +604,7 @@ template <> struct MappingTraits<FormatStyle> {
     IO.mapOptional("SpacesInSquareBrackets", Style.SpacesInSquareBrackets);
     IO.mapOptional("SpaceBeforeSquareBrackets",
                    Style.SpaceBeforeSquareBrackets);
+    IO.mapOptional("BitFieldColonSpacing", Style.BitFieldColonSpacing);
     IO.mapOptional("Standard", Style.Standard);
     IO.mapOptional("StatementMacros", Style.StatementMacros);
     IO.mapOptional("TabWidth", Style.TabWidth);
@@ -918,6 +930,7 @@ FormatStyle getLLVMStyle(FormatStyle::LanguageKind Language) {
   LLVMStyle.SpaceBeforeAssignmentOperators = true;
   LLVMStyle.SpaceBeforeCpp11BracedList = false;
   LLVMStyle.SpaceBeforeSquareBrackets = false;
+  LLVMStyle.BitFieldColonSpacing = FormatStyle::BFCS_Both;
   LLVMStyle.SpacesInAngles = false;
   LLVMStyle.SpacesInConditionalStatement = false;
 
diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index 7f8e351265127..0ab09b4a1218a 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -3251,6 +3251,9 @@ bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line,
   if (Right.is(TT_RangeBasedForLoopColon) &&
       !Style.SpaceBeforeRangeBasedForLoopColon)
     return false;
+  if (Left.is(TT_BitFieldColon))
+    return Style.BitFieldColonSpacing == FormatStyle::BFCS_Both ||
+           Style.BitFieldColonSpacing == FormatStyle::BFCS_After;
   if (Right.is(tok::colon)) {
     if (Line.First->isOneOf(tok::kw_case, tok::kw_default) ||
         !Right.getNextNonComment() || Right.getNextNonComment()->is(tok::semi))
@@ -3267,6 +3270,9 @@ bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line,
       return false;
     if (Right.is(TT_CSharpNamedArgumentColon))
       return false;
+    if (Right.is(TT_BitFieldColon))
+      return Style.BitFieldColonSpacing == FormatStyle::BFCS_Both ||
+             Style.BitFieldColonSpacing == FormatStyle::BFCS_Before;
     return true;
   }
   if (Left.is(TT_UnaryOperator)) {
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index 0a04ab5c940c9..b64c7105a142f 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -816,10 +816,14 @@ static bool ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
           Args.getLastArg(OPT_emit_llvm_uselists, OPT_no_emit_llvm_uselists))
     Opts.EmitLLVMUseLists = A->getOption().getID() == OPT_emit_llvm_uselists;
 
+  // ESIMD GPU Back-end requires optimized IR
+  bool IsSyclESIMD = Args.hasFlag(options::OPT_fsycl_esimd,
+                                  options::OPT_fno_sycl_esimd, false);
+
   Opts.DisableLLVMPasses =
       Args.hasArg(OPT_disable_llvm_passes) ||
       (Args.hasArg(OPT_fsycl_is_device) && Triple.isSPIR() &&
-       !Args.hasArg(OPT_fsycl_enable_optimizations));
+       !Args.hasArg(OPT_fsycl_enable_optimizations) && !IsSyclESIMD);
   Opts.DisableLifetimeMarkers = Args.hasArg(OPT_disable_lifetimemarkers);
 
   const llvm::Triple::ArchType DebugEntryValueArchs[] = {
@@ -2028,8 +2032,9 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
     // FIXME: Supporting '<lang>-header-cpp-output' would be useful.
     bool Preprocessed = XValue.consume_back("-cpp-output");
     bool ModuleMap = XValue.consume_back("-module-map");
-    IsHeaderFile =
-        !Preprocessed && !ModuleMap && XValue.consume_back("-header");
+    IsHeaderFile = !Preprocessed && !ModuleMap &&
+                   XValue != "precompiled-header" &&
+                   XValue.consume_back("-header");
 
     // Principal languages.
     DashX = llvm::StringSwitch<InputKind>(XValue)
@@ -2056,7 +2061,7 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
       DashX = llvm::StringSwitch<InputKind>(XValue)
                   .Case("cpp-output", InputKind(Language::C).getPreprocessed())
                   .Case("assembler-with-cpp", Language::Asm)
-                  .Cases("ast", "pcm",
+                  .Cases("ast", "pcm", "precompiled-header",
                          InputKind(Language::Unknown, InputKind::Precompiled))
                   .Case("ir", Language::LLVM_IR)
                   .Default(Language::Unknown);
diff --git a/clang/lib/Frontend/Rewrite/RewriteModernObjC.cpp b/clang/lib/Frontend/Rewrite/RewriteModernObjC.cpp
index e122b10e76d37..8c41e71ef0187 100644
--- a/clang/lib/Frontend/Rewrite/RewriteModernObjC.cpp
+++ b/clang/lib/Frontend/Rewrite/RewriteModernObjC.cpp
@@ -2109,8 +2109,9 @@ RewriteModernObjC::SynthesizeCallToFunctionDecl(FunctionDecl *FD,
                              DRE, nullptr, VK_RValue);
 
   const auto *FT = msgSendType->castAs<FunctionType>();
-  CallExpr *Exp = CallExpr::Create(
-      *Context, ICE, Args, FT->getCallResultType(*Context), VK_RValue, EndLoc);
+  CallExpr *Exp =
+      CallExpr::Create(*Context, ICE, Args, FT->getCallResultType(*Context),
+                       VK_RValue, EndLoc, FPOptionsOverride());
   return Exp;
 }
 
@@ -2692,7 +2693,7 @@ Stmt *RewriteModernObjC::RewriteObjCBoxedExpr(ObjCBoxedExpr *Exp) {
 
   auto *FT = msgSendType->castAs<FunctionType>();
   CallExpr *CE = CallExpr::Create(*Context, PE, MsgExprs, FT->getReturnType(),
-                                  VK_RValue, EndLoc);
+                                  VK_RValue, EndLoc, FPOptionsOverride());
   ReplaceStmt(Exp, CE);
   return CE;
 }
@@ -2732,7 +2733,7 @@ Stmt *RewriteModernObjC::RewriteObjCArrayLiteralExpr(ObjCArrayLiteral *Exp) {
     InitExprs.push_back(Exp->getElement(i));
   Expr *NSArrayCallExpr =
       CallExpr::Create(*Context, NSArrayDRE, InitExprs, NSArrayFType, VK_LValue,
-                       SourceLocation());
+                       SourceLocation(), FPOptionsOverride());
 
   FieldDecl *ARRFD = FieldDecl::Create(*Context, nullptr, SourceLocation(),
                                     SourceLocation(),
@@ -2813,7 +2814,7 @@ Stmt *RewriteModernObjC::RewriteObjCArrayLiteralExpr(ObjCArrayLiteral *Exp) {
 
   const FunctionType *FT = msgSendType->castAs<FunctionType>();
   CallExpr *CE = CallExpr::Create(*Context, PE, MsgExprs, FT->getReturnType(),
-                                  VK_RValue, EndLoc);
+                                  VK_RValue, EndLoc, FPOptionsOverride());
   ReplaceStmt(Exp, CE);
   return CE;
 }
@@ -2861,7 +2862,7 @@ Stmt *RewriteModernObjC::RewriteObjCDictionaryLiteralExpr(ObjCDictionaryLiteral
   // (const id [])objects
   Expr *NSValueCallExpr =
       CallExpr::Create(*Context, NSDictDRE, ValueExprs, NSDictFType, VK_LValue,
-                       SourceLocation());
+                       SourceLocation(), FPOptionsOverride());
 
   FieldDecl *ARRFD = FieldDecl::Create(*Context, nullptr, SourceLocation(),
                                        SourceLocation(),
@@ -2879,8 +2880,9 @@ Stmt *RewriteModernObjC::RewriteObjCDictionaryLiteralExpr(ObjCDictionaryLiteral
                              CK_BitCast,
                              DictLiteralValueME);
   // (const id <NSCopying> [])keys
-  Expr *NSKeyCallExpr = CallExpr::Create(
-      *Context, NSDictDRE, KeyExprs, NSDictFType, VK_LValue, SourceLocation());
+  Expr *NSKeyCallExpr =
+      CallExpr::Create(*Context, NSDictDRE, KeyExprs, NSDictFType, VK_LValue,
+                       SourceLocation(), FPOptionsOverride());
 
   MemberExpr *DictLiteralKeyME =
       MemberExpr::CreateImplicit(*Context, NSKeyCallExpr, false, ARRFD,
@@ -2964,7 +2966,7 @@ Stmt *RewriteModernObjC::RewriteObjCDictionaryLiteralExpr(ObjCDictionaryLiteral
 
   const FunctionType *FT = msgSendType->castAs<FunctionType>();
   CallExpr *CE = CallExpr::Create(*Context, PE, MsgExprs, FT->getReturnType(),
-                                  VK_RValue, EndLoc);
+                                  VK_RValue, EndLoc, FPOptionsOverride());
   ReplaceStmt(Exp, CE);
   return CE;
 }
@@ -3175,8 +3177,9 @@ Expr *RewriteModernObjC::SynthMsgSendStretCallExpr(FunctionDecl *MsgSendStretFla
                            ID, FuncType, nullptr, SC_Extern, false, false);
   DeclRefExpr *DRE = new (Context)
       DeclRefExpr(*Context, FD, false, castType, VK_RValue, SourceLocation());
-  CallExpr *STCE = CallExpr::Create(*Context, DRE, MsgExprs, castType,
-                                    VK_LValue, SourceLocation());
+  CallExpr *STCE =
+      CallExpr::Create(*Context, DRE, MsgExprs, castType, VK_LValue,
+                       SourceLocation(), FPOptionsOverride());
 
   FieldDecl *FieldD = FieldDecl::Create(*Context, nullptr, SourceLocation(),
                                     SourceLocation(),
@@ -3276,8 +3279,9 @@ Stmt *RewriteModernObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
       DeclRefExpr *DRE = new (Context)
           DeclRefExpr(*Context, SuperConstructorFunctionDecl, false, superType,
                       VK_LValue, SourceLocation());
-      SuperRep = CallExpr::Create(*Context, DRE, InitExprs, superType,
-                                  VK_LValue, SourceLocation());
+      SuperRep =
+          CallExpr::Create(*Context, DRE, InitExprs, superType, VK_LValue,
+                           SourceLocation(), FPOptionsOverride());
       // The code for super is a little tricky to prevent collision with
       // the structure definition in the header. The rewriter has it's own
       // internal definition (__rw_objc_super) that is uses. This is why
@@ -3371,8 +3375,9 @@ Stmt *RewriteModernObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
       DeclRefExpr *DRE = new (Context)
           DeclRefExpr(*Context, SuperConstructorFunctionDecl, false, superType,
                       VK_LValue, SourceLocation());
-      SuperRep = CallExpr::Create(*Context, DRE, InitExprs, superType,
-                                  VK_LValue, SourceLocation());
+      SuperRep =
+          CallExpr::Create(*Context, DRE, InitExprs, superType, VK_LValue,
+                           SourceLocation(), FPOptionsOverride());
       // The code for super is a little tricky to prevent collision with
       // the structure definition in the header. The rewriter has it's own
       // internal definition (__rw_objc_super) that is uses. This is why
@@ -3537,7 +3542,7 @@ Stmt *RewriteModernObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
 
   const FunctionType *FT = msgSendType->castAs<FunctionType>();
   CallExpr *CE = CallExpr::Create(*Context, PE, MsgExprs, FT->getReturnType(),
-                                  VK_RValue, EndLoc);
+                                  VK_RValue, EndLoc, FPOptionsOverride());
   Stmt *ReplacingStmt = CE;
   if (MsgSendStretFlavor) {
     // We have the method which returns a struct/union. Must also generate
@@ -4647,8 +4652,9 @@ Stmt *RewriteModernObjC::SynthesizeBlockCall(CallExpr *Exp, const Expr *BlockExp
        E = Exp->arg_end(); I != E; ++I) {
     BlkExprs.push_back(*I);
   }
-  CallExpr *CE = CallExpr::Create(*Context, PE, BlkExprs, Exp->getType(),
-                                  VK_RValue, SourceLocation());
+  CallExpr *CE =
+      CallExpr::Create(*Context, PE, BlkExprs, Exp->getType(), VK_RValue,
+                       SourceLocation(), FPOptionsOverride());
   return CE;
 }
 
@@ -5391,7 +5397,7 @@ Stmt *RewriteModernObjC::SynthBlockInitExpr(BlockExpr *Exp,
     InitExprs.push_back(FlagExp);
   }
   NewRep = CallExpr::Create(*Context, DRE, InitExprs, FType, VK_LValue,
-                            SourceLocation());
+                            SourceLocation(), FPOptionsOverride());
 
   if (GlobalBlockExpr) {
     assert (!GlobalConstructionExp &&
diff --git a/clang/lib/Frontend/Rewrite/RewriteObjC.cpp b/clang/lib/Frontend/Rewrite/RewriteObjC.cpp
index 3f320dc57aa69..4ecd6e95de10e 100644
--- a/clang/lib/Frontend/Rewrite/RewriteObjC.cpp
+++ b/clang/lib/Frontend/Rewrite/RewriteObjC.cpp
@@ -2027,8 +2027,9 @@ RewriteObjC::SynthesizeCallToFunctionDecl(FunctionDecl *FD,
 
   const auto *FT = msgSendType->castAs<FunctionType>();
 
-  CallExpr *Exp = CallExpr::Create(
-      *Context, ICE, Args, FT->getCallResultType(*Context), VK_RValue, EndLoc);
+  CallExpr *Exp =
+      CallExpr::Create(*Context, ICE, Args, FT->getCallResultType(*Context),
+                       VK_RValue, EndLoc, FPOptionsOverride());
   return Exp;
 }
 
@@ -2614,8 +2615,9 @@ CallExpr *RewriteObjC::SynthMsgSendStretCallExpr(FunctionDecl *MsgSendStretFlavo
   ParenExpr *PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(), cast);
 
   const auto *FT = msgSendType->castAs<FunctionType>();
-  CallExpr *STCE = CallExpr::Create(*Context, PE, MsgExprs, FT->getReturnType(),
-                                    VK_RValue, SourceLocation());
+  CallExpr *STCE =
+      CallExpr::Create(*Context, PE, MsgExprs, FT->getReturnType(), VK_RValue,
+                       SourceLocation(), FPOptionsOverride());
   return STCE;
 }
 
@@ -2707,8 +2709,9 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
       DeclRefExpr *DRE = new (Context)
           DeclRefExpr(*Context, SuperConstructorFunctionDecl, false, superType,
                       VK_LValue, SourceLocation());
-      SuperRep = CallExpr::Create(*Context, DRE, InitExprs, superType,
-                                  VK_LValue, SourceLocation());
+      SuperRep =
+          CallExpr::Create(*Context, DRE, InitExprs, superType, VK_LValue,
+                           SourceLocation(), FPOptionsOverride());
       // The code for super is a little tricky to prevent collision with
       // the structure definition in the header. The rewriter has it's own
       // internal definition (__rw_objc_super) that is uses. This is why
@@ -2802,8 +2805,9 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
       DeclRefExpr *DRE = new (Context)
           DeclRefExpr(*Context, SuperConstructorFunctionDecl, false, superType,
                       VK_LValue, SourceLocation());
-      SuperRep = CallExpr::Create(*Context, DRE, InitExprs, superType,
-                                  VK_LValue, SourceLocation());
+      SuperRep =
+          CallExpr::Create(*Context, DRE, InitExprs, superType, VK_LValue,
+                           SourceLocation(), FPOptionsOverride());
       // The code for super is a little tricky to prevent collision with
       // the structure definition in the header. The rewriter has it's own
       // internal definition (__rw_objc_super) that is uses. This is why
@@ -2968,7 +2972,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
 
   const auto *FT = msgSendType->castAs<FunctionType>();
   CallExpr *CE = CallExpr::Create(*Context, PE, MsgExprs, FT->getReturnType(),
-                                  VK_RValue, EndLoc);
+                                  VK_RValue, EndLoc, FPOptionsOverride());
   Stmt *ReplacingStmt = CE;
   if (MsgSendStretFlavor) {
     // We have the method which returns a struct/union. Must also generate
@@ -3817,8 +3821,9 @@ Stmt *RewriteObjC::SynthesizeBlockCall(CallExpr *Exp, const Expr *BlockExp) {
        E = Exp->arg_end(); I != E; ++I) {
     BlkExprs.push_back(*I);
   }
-  CallExpr *CE = CallExpr::Create(*Context, PE, BlkExprs, Exp->getType(),
-                                  VK_RValue, SourceLocation());
+  CallExpr *CE =
+      CallExpr::Create(*Context, PE, BlkExprs, Exp->getType(), VK_RValue,
+                       SourceLocation(), FPOptionsOverride());
   return CE;
 }
 
@@ -4530,7 +4535,7 @@ Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
     InitExprs.push_back(FlagExp);
   }
   NewRep = CallExpr::Create(*Context, DRE, InitExprs, FType, VK_LValue,
-                            SourceLocation());
+                            SourceLocation(), FPOptionsOverride());
   NewRep = UnaryOperator::Create(
       const_cast<ASTContext &>(*Context), NewRep, UO_AddrOf,
       Context->getPointerType(NewRep->getType()), VK_RValue, OK_Ordinary,
diff --git a/clang/lib/Headers/__clang_hip_libdevice_declares.h b/clang/lib/Headers/__clang_hip_libdevice_declares.h
index e1cd49a39c65d..7110404434405 100644
--- a/clang/lib/Headers/__clang_hip_libdevice_declares.h
+++ b/clang/lib/Headers/__clang_hip_libdevice_declares.h
@@ -78,6 +78,7 @@ __device__ __attribute__((const)) float __ocml_len4_f32(float, float, float,
 __device__ __attribute__((pure)) float __ocml_ncdf_f32(float);
 __device__ __attribute__((pure)) float __ocml_ncdfinv_f32(float);
 __device__ __attribute__((pure)) float __ocml_pow_f32(float, float);
+__device__ __attribute__((pure)) float __ocml_pown_f32(float, int);
 __device__ __attribute__((pure)) float __ocml_rcbrt_f32(float);
 __device__ __attribute__((const)) float __ocml_remainder_f32(float, float);
 __device__ float __ocml_remquo_f32(float, float,
@@ -205,6 +206,7 @@ __device__ __attribute__((const)) double __ocml_len4_f64(double, double, double,
 __device__ __attribute__((pure)) double __ocml_ncdf_f64(double);
 __device__ __attribute__((pure)) double __ocml_ncdfinv_f64(double);
 __device__ __attribute__((pure)) double __ocml_pow_f64(double, double);
+__device__ __attribute__((pure)) double __ocml_pown_f64(double, int);
 __device__ __attribute__((pure)) double __ocml_rcbrt_f64(double);
 __device__ __attribute__((const)) double __ocml_remainder_f64(double, double);
 __device__ double __ocml_remquo_f64(double, double,
@@ -290,6 +292,7 @@ __device__ __attribute__((const)) _Float16 __ocml_rsqrt_f16(_Float16);
 __device__ _Float16 __ocml_sin_f16(_Float16);
 __device__ __attribute__((const)) _Float16 __ocml_sqrt_f16(_Float16);
 __device__ __attribute__((const)) _Float16 __ocml_trunc_f16(_Float16);
+__device__ __attribute__((pure)) _Float16 __ocml_pown_f16(_Float16, int);
 
 typedef _Float16 __2f16 __attribute__((ext_vector_type(2)));
 typedef short __2i16 __attribute__((ext_vector_type(2)));
@@ -320,6 +323,7 @@ __device__ __attribute__((const)) __2f16 __ocml_rsqrt_2f16(__2f16);
 __device__ __2f16 __ocml_sin_2f16(__2f16);
 __device__ __attribute__((const)) __2f16 __ocml_sqrt_2f16(__2f16);
 __device__ __attribute__((const)) __2f16 __ocml_trunc_2f16(__2f16);
+__device__ __attribute__((const)) __2f16 __ocml_pown_2f16(__2f16, __2i16);
 
 } // extern "C"
 
diff --git a/clang/lib/Headers/__clang_hip_math.h b/clang/lib/Headers/__clang_hip_math.h
index cf7014b9aefef..47d3c17175591 100644
--- a/clang/lib/Headers/__clang_hip_math.h
+++ b/clang/lib/Headers/__clang_hip_math.h
@@ -294,6 +294,8 @@ normf(int __dim,
 __DEVICE__
 inline float powf(float __x, float __y) { return __ocml_pow_f32(__x, __y); }
 __DEVICE__
+inline float powif(float __x, int __y) { return __ocml_pown_f32(__x, __y); }
+__DEVICE__
 inline float rcbrtf(float __x) { return __ocml_rcbrt_f32(__x); }
 __DEVICE__
 inline float remainderf(float __x, float __y) {
@@ -759,6 +761,8 @@ inline double normcdfinv(double __x) { return __ocml_ncdfinv_f64(__x); }
 __DEVICE__
 inline double pow(double __x, double __y) { return __ocml_pow_f64(__x, __y); }
 __DEVICE__
+inline double powi(double __x, int __y) { return __ocml_pown_f64(__x, __y); }
+__DEVICE__
 inline double rcbrt(double __x) { return __ocml_rcbrt_f64(__x); }
 __DEVICE__
 inline double remainder(double __x, double __y) {
@@ -1134,6 +1138,7 @@ __DEF_FUN1(double, trunc);
   __DEVICE__                                                                   \
   inline float __func(float __x, int __y) { return __func##f(__x, __y); }
 __DEF_FLOAT_FUN2I(scalbn)
+__DEF_FLOAT_FUN2I(ldexp)
 
 template <class T> __DEVICE__ inline T min(T __arg1, T __arg2) {
   return (__arg1 < __arg2) ? __arg1 : __arg2;
@@ -1173,6 +1178,17 @@ __host__ inline static int max(int __arg1, int __arg2) {
   return std::max(__arg1, __arg2);
 }
 
+__DEVICE__
+inline float pow(float __base, int __iexp) { return powif(__base, __iexp); }
+
+__DEVICE__
+inline double pow(double __base, int __iexp) { return powi(__base, __iexp); }
+
+__DEVICE__
+inline _Float16 pow(_Float16 __base, int __iexp) {
+  return __ocml_pown_f16(__base, __iexp);
+}
+
 #pragma pop_macro("__DEF_FUN1")
 #pragma pop_macro("__DEF_FUN2")
 #pragma pop_macro("__DEF_FUNI")
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 5223755c8fdf1..79744a7519e04 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -1473,7 +1473,8 @@ parseOpenMPSimpleClause(Parser &P, OpenMPClauseKind Kind) {
     return llvm::None;
 
   unsigned Type = getOpenMPSimpleClauseType(
-      Kind, Tok.isAnnotation() ? "" : P.getPreprocessor().getSpelling(Tok));
+      Kind, Tok.isAnnotation() ? "" : P.getPreprocessor().getSpelling(Tok),
+      P.getLangOpts().OpenMP);
   SourceLocation TypeLoc = Tok.getLocation();
   if (Tok.isNot(tok::r_paren) && Tok.isNot(tok::comma) &&
       Tok.isNot(tok::annot_pragma_openmp_end))
@@ -2871,7 +2872,8 @@ OMPClause *Parser::ParseOpenMPSingleExprWithArgClause(OpenMPDirectiveKind DKind,
     Arg[Modifier2] = OMPC_SCHEDULE_MODIFIER_unknown;
     Arg[ScheduleKind] = OMPC_SCHEDULE_unknown;
     unsigned KindModifier = getOpenMPSimpleClauseType(
-        Kind, Tok.isAnnotation() ? "" : PP.getSpelling(Tok));
+        Kind, Tok.isAnnotation() ? "" : PP.getSpelling(Tok),
+        getLangOpts().OpenMP);
     if (KindModifier > OMPC_SCHEDULE_unknown) {
       // Parse 'modifier'
       Arg[Modifier1] = KindModifier;
@@ -2883,7 +2885,8 @@ OMPClause *Parser::ParseOpenMPSingleExprWithArgClause(OpenMPDirectiveKind DKind,
         // Parse ',' 'modifier'
         ConsumeAnyToken();
         KindModifier = getOpenMPSimpleClauseType(
-            Kind, Tok.isAnnotation() ? "" : PP.getSpelling(Tok));
+            Kind, Tok.isAnnotation() ? "" : PP.getSpelling(Tok),
+            getLangOpts().OpenMP);
         Arg[Modifier2] = KindModifier > OMPC_SCHEDULE_unknown
                              ? KindModifier
                              : (unsigned)OMPC_SCHEDULE_unknown;
@@ -2898,7 +2901,8 @@ OMPClause *Parser::ParseOpenMPSingleExprWithArgClause(OpenMPDirectiveKind DKind,
       else
         Diag(Tok, diag::warn_pragma_expected_colon) << "schedule modifier";
       KindModifier = getOpenMPSimpleClauseType(
-          Kind, Tok.isAnnotation() ? "" : PP.getSpelling(Tok));
+          Kind, Tok.isAnnotation() ? "" : PP.getSpelling(Tok),
+          getLangOpts().OpenMP);
     }
     Arg[ScheduleKind] = KindModifier;
     KLoc[ScheduleKind] = Tok.getLocation();
@@ -2912,7 +2916,8 @@ OMPClause *Parser::ParseOpenMPSingleExprWithArgClause(OpenMPDirectiveKind DKind,
       DelimLoc = ConsumeAnyToken();
   } else if (Kind == OMPC_dist_schedule) {
     Arg.push_back(getOpenMPSimpleClauseType(
-        Kind, Tok.isAnnotation() ? "" : PP.getSpelling(Tok)));
+        Kind, Tok.isAnnotation() ? "" : PP.getSpelling(Tok),
+        getLangOpts().OpenMP));
     KLoc.push_back(Tok.getLocation());
     if (Tok.isNot(tok::r_paren) && Tok.isNot(tok::comma) &&
         Tok.isNot(tok::annot_pragma_openmp_end))
@@ -2922,7 +2927,8 @@ OMPClause *Parser::ParseOpenMPSingleExprWithArgClause(OpenMPDirectiveKind DKind,
   } else if (Kind == OMPC_defaultmap) {
     // Get a defaultmap modifier
     unsigned Modifier = getOpenMPSimpleClauseType(
-        Kind, Tok.isAnnotation() ? "" : PP.getSpelling(Tok));
+        Kind, Tok.isAnnotation() ? "" : PP.getSpelling(Tok),
+        getLangOpts().OpenMP);
     // Set defaultmap modifier to unknown if it is either scalar, aggregate, or
     // pointer
     if (Modifier < OMPC_DEFAULTMAP_MODIFIER_unknown)
@@ -2940,7 +2946,8 @@ OMPClause *Parser::ParseOpenMPSingleExprWithArgClause(OpenMPDirectiveKind DKind,
         Diag(Tok, diag::warn_pragma_expected_colon) << "defaultmap modifier";
       // Get a defaultmap kind
       Arg.push_back(getOpenMPSimpleClauseType(
-          Kind, Tok.isAnnotation() ? "" : PP.getSpelling(Tok)));
+          Kind, Tok.isAnnotation() ? "" : PP.getSpelling(Tok),
+          getLangOpts().OpenMP));
       KLoc.push_back(Tok.getLocation());
       if (Tok.isNot(tok::r_paren) && Tok.isNot(tok::comma) &&
           Tok.isNot(tok::annot_pragma_openmp_end))
@@ -2955,7 +2962,8 @@ OMPClause *Parser::ParseOpenMPSingleExprWithArgClause(OpenMPDirectiveKind DKind,
         NextToken().is(tok::colon)) {
       // Parse optional <device modifier> ':'
       Arg.push_back(getOpenMPSimpleClauseType(
-          Kind, Tok.isAnnotation() ? "" : PP.getSpelling(Tok)));
+          Kind, Tok.isAnnotation() ? "" : PP.getSpelling(Tok),
+          getLangOpts().OpenMP));
       KLoc.push_back(Tok.getLocation());
       ConsumeAnyToken();
       // Parse ':'
@@ -3057,14 +3065,16 @@ static bool ParseReductionId(Parser &P, CXXScopeSpec &ReductionIdScopeSpec,
 }
 
 /// Checks if the token is a valid map-type-modifier.
+/// FIXME: It will return an OpenMPMapClauseKind if that's what it parses.
 static OpenMPMapModifierKind isMapModifier(Parser &P) {
   Token Tok = P.getCurToken();
   if (!Tok.is(tok::identifier))
     return OMPC_MAP_MODIFIER_unknown;
 
   Preprocessor &PP = P.getPreprocessor();
-  OpenMPMapModifierKind TypeModifier = static_cast<OpenMPMapModifierKind>(
-      getOpenMPSimpleClauseType(OMPC_map, PP.getSpelling(Tok)));
+  OpenMPMapModifierKind TypeModifier =
+      static_cast<OpenMPMapModifierKind>(getOpenMPSimpleClauseType(
+          OMPC_map, PP.getSpelling(Tok), P.getLangOpts().OpenMP));
   return TypeModifier;
 }
 
@@ -3099,12 +3109,14 @@ bool Parser::parseMapperModifier(OpenMPVarListDataTy &Data) {
 
 /// Parse map-type-modifiers in map clause.
 /// map([ [map-type-modifier[,] [map-type-modifier[,] ...] map-type : ] list)
-/// where, map-type-modifier ::= always | close | mapper(mapper-identifier)
+/// where, map-type-modifier ::= always | close | mapper(mapper-identifier) |
+/// present
 bool Parser::parseMapTypeModifiers(OpenMPVarListDataTy &Data) {
   while (getCurToken().isNot(tok::colon)) {
     OpenMPMapModifierKind TypeModifier = isMapModifier(*this);
     if (TypeModifier == OMPC_MAP_MODIFIER_always ||
-        TypeModifier == OMPC_MAP_MODIFIER_close) {
+        TypeModifier == OMPC_MAP_MODIFIER_close ||
+        TypeModifier == OMPC_MAP_MODIFIER_present) {
       Data.MapTypeModifiers.push_back(TypeModifier);
       Data.MapTypeModifiersLoc.push_back(Tok.getLocation());
       ConsumeToken();
@@ -3126,7 +3138,8 @@ bool Parser::parseMapTypeModifiers(OpenMPVarListDataTy &Data) {
       // Potential map-type token as it is followed by a colon.
       if (PP.LookAhead(0).is(tok::colon))
         return false;
-      Diag(Tok, diag::err_omp_unknown_map_type_modifier);
+      Diag(Tok, diag::err_omp_unknown_map_type_modifier)
+          << (getLangOpts().OpenMP >= 51 ? 1 : 0);
       ConsumeToken();
     }
     if (getCurToken().is(tok::comma))
@@ -3136,14 +3149,16 @@ bool Parser::parseMapTypeModifiers(OpenMPVarListDataTy &Data) {
 }
 
 /// Checks if the token is a valid map-type.
+/// FIXME: It will return an OpenMPMapModifierKind if that's what it parses.
 static OpenMPMapClauseKind isMapType(Parser &P) {
   Token Tok = P.getCurToken();
   // The map-type token can be either an identifier or the C++ delete keyword.
   if (!Tok.isOneOf(tok::identifier, tok::kw_delete))
     return OMPC_MAP_unknown;
   Preprocessor &PP = P.getPreprocessor();
-  OpenMPMapClauseKind MapType = static_cast<OpenMPMapClauseKind>(
-      getOpenMPSimpleClauseType(OMPC_map, PP.getSpelling(Tok)));
+  OpenMPMapClauseKind MapType =
+      static_cast<OpenMPMapClauseKind>(getOpenMPSimpleClauseType(
+          OMPC_map, PP.getSpelling(Tok), P.getLangOpts().OpenMP));
   return MapType;
 }
 
@@ -3297,7 +3312,8 @@ bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
         (Tok.is(tok::identifier) || Tok.is(tok::kw_default)) &&
         NextToken().is(tok::comma)) {
       // Parse optional reduction modifier.
-      Data.ExtraModifier = getOpenMPSimpleClauseType(Kind, PP.getSpelling(Tok));
+      Data.ExtraModifier = getOpenMPSimpleClauseType(Kind, PP.getSpelling(Tok),
+                                                     getLangOpts().OpenMP);
       Data.ExtraModifierLoc = Tok.getLocation();
       ConsumeToken();
       assert(Tok.is(tok::comma) && "Expected comma.");
@@ -3342,7 +3358,8 @@ bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
     // Handle dependency type for depend clause.
     ColonProtectionRAIIObject ColonRAII(*this);
     Data.ExtraModifier = getOpenMPSimpleClauseType(
-        Kind, Tok.is(tok::identifier) ? PP.getSpelling(Tok) : "");
+        Kind, Tok.is(tok::identifier) ? PP.getSpelling(Tok) : "",
+        getLangOpts().OpenMP);
     Data.ExtraModifierLoc = Tok.getLocation();
     if (Data.ExtraModifier == OMPC_DEPEND_unknown) {
       SkipUntil(tok::colon, tok::r_paren, tok::annot_pragma_openmp_end,
@@ -3367,7 +3384,8 @@ bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
     // Try to parse modifier if any.
     Data.ExtraModifier = OMPC_LINEAR_val;
     if (Tok.is(tok::identifier) && PP.LookAhead(0).is(tok::l_paren)) {
-      Data.ExtraModifier = getOpenMPSimpleClauseType(Kind, PP.getSpelling(Tok));
+      Data.ExtraModifier = getOpenMPSimpleClauseType(Kind, PP.getSpelling(Tok),
+                                                     getLangOpts().OpenMP);
       Data.ExtraModifierLoc = ConsumeToken();
       LinearT.consumeOpen();
       NeedRParenForLinear = true;
@@ -3380,7 +3398,8 @@ bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
     if ((getLangOpts().OpenMP >= 50 && !isOpenMPDistributeDirective(DKind) &&
          !isOpenMPTaskLoopDirective(DKind)) &&
         Tok.is(tok::identifier) && PP.LookAhead(0).is(tok::colon)) {
-      Data.ExtraModifier = getOpenMPSimpleClauseType(Kind, PP.getSpelling(Tok));
+      Data.ExtraModifier = getOpenMPSimpleClauseType(Kind, PP.getSpelling(Tok),
+                                                     getLangOpts().OpenMP);
       Data.ExtraModifierLoc = Tok.getLocation();
       ConsumeToken();
       assert(Tok.is(tok::colon) && "Expected colon.");
@@ -3425,13 +3444,15 @@ bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
     if (Tok.is(tok::identifier)) {
       bool IsMapperModifier = false;
       if (Kind == OMPC_to) {
-        auto Modifier = static_cast<OpenMPToModifierKind>(
-            getOpenMPSimpleClauseType(Kind, PP.getSpelling(Tok)));
+        auto Modifier =
+            static_cast<OpenMPToModifierKind>(getOpenMPSimpleClauseType(
+                Kind, PP.getSpelling(Tok), getLangOpts().OpenMP));
         if (Modifier == OMPC_TO_MODIFIER_mapper)
           IsMapperModifier = true;
       } else {
-        auto Modifier = static_cast<OpenMPFromModifierKind>(
-            getOpenMPSimpleClauseType(Kind, PP.getSpelling(Tok)));
+        auto Modifier =
+            static_cast<OpenMPFromModifierKind>(getOpenMPSimpleClauseType(
+                Kind, PP.getSpelling(Tok), getLangOpts().OpenMP));
         if (Modifier == OMPC_FROM_MODIFIER_mapper)
           IsMapperModifier = true;
       }
diff --git a/clang/lib/Sema/DeclSpec.cpp b/clang/lib/Sema/DeclSpec.cpp
index f4c30c90ad271..f553b5ca60793 100644
--- a/clang/lib/Sema/DeclSpec.cpp
+++ b/clang/lib/Sema/DeclSpec.cpp
@@ -1279,6 +1279,7 @@ void DeclSpec::Finish(Sema &S, const PrintingPolicy &Policy) {
         S.Diag(TSTLoc, diag::ext_integer_complex);
     } else if (TypeSpecType != TST_float && TypeSpecType != TST_double &&
                TypeSpecType != TST_float128) {
+      // FIXME: _Float16, __fp16?
       S.Diag(TSCLoc, diag::err_invalid_complex_spec)
         << getSpecifierName((TST)TypeSpecType, Policy);
       TypeSpecComplex = TSC_unspecified;
diff --git a/clang/lib/Sema/MultiplexExternalSemaSource.cpp b/clang/lib/Sema/MultiplexExternalSemaSource.cpp
index 80333e63127e4..252008cda15d2 100644
--- a/clang/lib/Sema/MultiplexExternalSemaSource.cpp
+++ b/clang/lib/Sema/MultiplexExternalSemaSource.cpp
@@ -172,13 +172,6 @@ Module *MultiplexExternalSemaSource::getModule(unsigned ID) {
   return nullptr;
 }
 
-bool MultiplexExternalSemaSource::DeclIsFromPCHWithObjectFile(const Decl *D) {
-  for (auto *S : Sources)
-    if (S->DeclIsFromPCHWithObjectFile(D))
-      return true;
-  return false;
-}
-
 bool MultiplexExternalSemaSource::layoutRecordType(const RecordDecl *Record,
                                                    uint64_t &Size,
                                                    uint64_t &Alignment,
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index 2eeca2a0bd50c..0f351dfc9cc8e 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -42,7 +42,7 @@
 #include "clang/Sema/TemplateInstCallback.h"
 #include "clang/Sema/TypoCorrection.h"
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/Support/TimeProfiler.h"
 
 using namespace clang;
@@ -567,8 +567,10 @@ ExprResult Sema::ImpCastExprToType(Expr *E, QualType Ty,
   if (VK == VK_RValue && !E->isRValue()) {
     switch (Kind) {
     default:
-      llvm_unreachable("can't implicitly cast lvalue to rvalue with this cast "
-                       "kind");
+      llvm_unreachable(("can't implicitly cast lvalue to rvalue with this cast "
+                        "kind: " +
+                        std::string(CastExpr::getCastKindName(Kind)))
+                           .c_str());
     case CK_Dependent:
     case CK_LValueToRValue:
     case CK_ArrayToPointerDecay:
@@ -990,6 +992,8 @@ void Sema::ActOnEndOfTranslationUnitFragment(TUFragmentKind Kind) {
       SyclIntHeader->emit(getLangOpts().SYCLIntHeader);
     MarkDevice();
   }
+  if (getLangOpts().SYCLExplicitSIMD)
+    MarkSyclSimd();
 
   emitDeferredDiags();
 
@@ -1525,7 +1529,7 @@ class DeferredDiagnosticsEmitter
   typedef UsedDeclVisitor<DeferredDiagnosticsEmitter> Inherited;
 
   // Whether the function is already in the current use-path.
-  llvm::SmallSet<CanonicalDeclPtr<Decl>, 4> InUsePath;
+  llvm::SmallPtrSet<CanonicalDeclPtr<Decl>, 4> InUsePath;
 
   // The current use-path.
   llvm::SmallVector<CanonicalDeclPtr<FunctionDecl>, 4> UsePath;
@@ -1534,7 +1538,7 @@ class DeferredDiagnosticsEmitter
   // case not in OpenMP device context. Done[1] is for the case in OpenMP
   // device context. We need two sets because diagnostics emission may be
   // different depending on whether it is in OpenMP device context.
-  llvm::SmallSet<CanonicalDeclPtr<Decl>, 4> DoneMap[2];
+  llvm::SmallPtrSet<CanonicalDeclPtr<Decl>, 4> DoneMap[2];
 
   // Emission state of the root node of the current use graph.
   bool ShouldEmitRootNode;
diff --git a/clang/lib/Sema/SemaAttr.cpp b/clang/lib/Sema/SemaAttr.cpp
index b354e810974c4..f9785e4bea5e2 100644
--- a/clang/lib/Sema/SemaAttr.cpp
+++ b/clang/lib/Sema/SemaAttr.cpp
@@ -300,20 +300,18 @@ void Sema::ActOnPragmaPack(SourceLocation PragmaLoc, PragmaMsStackAction Action,
   // If specified then alignment must be a "small" power of two.
   unsigned AlignmentVal = 0;
   if (Alignment) {
-    llvm::APSInt Val;
+    Optional<llvm::APSInt> Val;
 
     // pack(0) is like pack(), which just works out since that is what
     // we use 0 for in PackAttr.
-    if (Alignment->isTypeDependent() ||
-        Alignment->isValueDependent() ||
-        !Alignment->isIntegerConstantExpr(Val, Context) ||
-        !(Val == 0 || Val.isPowerOf2()) ||
-        Val.getZExtValue() > 16) {
+    if (Alignment->isTypeDependent() || Alignment->isValueDependent() ||
+        !(Val = Alignment->getIntegerConstantExpr(Context)) ||
+        !(*Val == 0 || Val->isPowerOf2()) || Val->getZExtValue() > 16) {
       Diag(PragmaLoc, diag::warn_pragma_pack_invalid_alignment);
       return; // Ignore
     }
 
-    AlignmentVal = (unsigned) Val.getZExtValue();
+    AlignmentVal = (unsigned)Val->getZExtValue();
   }
   if (Action == Sema::PSK_Show) {
     // Show the current alignment, making sure to show the right value
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index b1f258fcfff17..fa383bcd89a16 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -1582,6 +1582,10 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
     if (checkArgCount(*this, TheCall, 1)) return true;
     TheCall->setType(Context.IntTy);
     break;
+  case Builtin::BI__builtin_complex:
+    if (SemaBuiltinComplex(TheCall))
+      return ExprError();
+    break;
   case Builtin::BI__builtin_constant_p: {
     if (checkArgCount(*this, TheCall, 1)) return true;
     ExprResult Arg = DefaultFunctionArrayLvalueConversion(TheCall->getArg(0));
@@ -2309,10 +2313,7 @@ bool Sema::CheckARMCoprocessorImmediate(const TargetInfo &TI,
   if (CoprocArg->isTypeDependent() || CoprocArg->isValueDependent())
     return false;
 
-  llvm::APSInt CoprocNoAP;
-  bool IsICE = CoprocArg->isIntegerConstantExpr(CoprocNoAP, Context);
-  (void)IsICE;
-  assert(IsICE && "Coprocossor immediate is not a constant expression");
+  llvm::APSInt CoprocNoAP = *CoprocArg->getIntegerConstantExpr(Context);
   int64_t CoprocNo = CoprocNoAP.getExtValue();
   assert(CoprocNo >= 0 && "Coprocessor immediate must be non-negative");
 
@@ -2596,9 +2597,8 @@ bool Sema::CheckBPFBuiltinFunctionCall(unsigned BuiltinID,
   Expr *Arg;
   if (BuiltinID == BPF::BI__builtin_btf_type_id) {
     // The second argument needs to be a constant int
-    llvm::APSInt Value;
     Arg = TheCall->getArg(1);
-    if (!Arg->isIntegerConstantExpr(Value, Context)) {
+    if (!Arg->isIntegerConstantExpr(Context)) {
       Diag(Arg->getBeginLoc(), diag::err_btf_type_id_not_const)
           << 2 << Arg->getSourceRange();
       return true;
@@ -2624,8 +2624,7 @@ bool Sema::CheckBPFBuiltinFunctionCall(unsigned BuiltinID,
 
   // The second argument needs to be a constant int
   Arg = TheCall->getArg(1);
-  llvm::APSInt Value;
-  if (!Arg->isIntegerConstantExpr(Value, Context)) {
+  if (!Arg->isIntegerConstantExpr(Context)) {
     Diag(Arg->getBeginLoc(), diag::err_preserve_field_info_not_const)
         << 2 << Arg->getSourceRange();
     return true;
@@ -3223,11 +3222,10 @@ bool Sema::CheckSystemZBuiltinFunctionCall(unsigned BuiltinID,
                                            CallExpr *TheCall) {
   if (BuiltinID == SystemZ::BI__builtin_tabort) {
     Expr *Arg = TheCall->getArg(0);
-    llvm::APSInt AbortCode(32);
-    if (Arg->isIntegerConstantExpr(AbortCode, Context) &&
-        AbortCode.getSExtValue() >= 0 && AbortCode.getSExtValue() < 256)
-      return Diag(Arg->getBeginLoc(), diag::err_systemz_invalid_tabort_code)
-             << Arg->getSourceRange();
+    if (Optional<llvm::APSInt> AbortCode = Arg->getIntegerConstantExpr(Context))
+      if (AbortCode->getSExtValue() >= 0 && AbortCode->getSExtValue() < 256)
+        return Diag(Arg->getBeginLoc(), diag::err_systemz_invalid_tabort_code)
+               << Arg->getSourceRange();
   }
 
   // For intrinsics which take an immediate value as part of the instruction,
@@ -5059,21 +5057,21 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange,
   }
 
   if (SubExprs.size() >= 2 && Form != Init) {
-    llvm::APSInt Result(32);
-    if (SubExprs[1]->isIntegerConstantExpr(Result, Context) &&
-        !isValidOrderingForOp(Result.getSExtValue(), Op))
-      Diag(SubExprs[1]->getBeginLoc(),
-           diag::warn_atomic_op_has_invalid_memory_order)
-          << SubExprs[1]->getSourceRange();
+    if (Optional<llvm::APSInt> Result =
+            SubExprs[1]->getIntegerConstantExpr(Context))
+      if (!isValidOrderingForOp(Result->getSExtValue(), Op))
+        Diag(SubExprs[1]->getBeginLoc(),
+             diag::warn_atomic_op_has_invalid_memory_order)
+            << SubExprs[1]->getSourceRange();
   }
 
   if (auto ScopeModel = AtomicExpr::getScopeModel(Op)) {
     auto *Scope = Args[Args.size() - 1];
-    llvm::APSInt Result(32);
-    if (Scope->isIntegerConstantExpr(Result, Context) &&
-        !ScopeModel->isValid(Result.getZExtValue())) {
-      Diag(Scope->getBeginLoc(), diag::err_atomic_op_has_invalid_synch_scope)
-          << Scope->getSourceRange();
+    if (Optional<llvm::APSInt> Result =
+            Scope->getIntegerConstantExpr(Context)) {
+      if (!ScopeModel->isValid(Result->getZExtValue()))
+        Diag(Scope->getBeginLoc(), diag::err_atomic_op_has_invalid_synch_scope)
+            << Scope->getSourceRange();
     }
     SubExprs.push_back(Scope);
   }
@@ -5928,6 +5926,61 @@ bool Sema::SemaBuiltinFPClassification(CallExpr *TheCall, unsigned NumArgs) {
   return false;
 }
 
+/// Perform semantic analysis for a call to __builtin_complex.
+bool Sema::SemaBuiltinComplex(CallExpr *TheCall) {
+  if (checkArgCount(*this, TheCall, 2))
+    return true;
+
+  bool Dependent = false;
+  for (unsigned I = 0; I != 2; ++I) {
+    Expr *Arg = TheCall->getArg(I);
+    QualType T = Arg->getType();
+    if (T->isDependentType()) {
+      Dependent = true;
+      continue;
+    }
+
+    // Despite supporting _Complex int, GCC requires a real floating point type
+    // for the operands of __builtin_complex.
+    if (!T->isRealFloatingType()) {
+      return Diag(Arg->getBeginLoc(), diag::err_typecheck_call_requires_real_fp)
+             << Arg->getType() << Arg->getSourceRange();
+    }
+
+    ExprResult Converted = DefaultLvalueConversion(Arg);
+    if (Converted.isInvalid())
+      return true;
+    TheCall->setArg(I, Converted.get());
+  }
+
+  if (Dependent) {
+    TheCall->setType(Context.DependentTy);
+    return false;
+  }
+
+  Expr *Real = TheCall->getArg(0);
+  Expr *Imag = TheCall->getArg(1);
+  if (!Context.hasSameType(Real->getType(), Imag->getType())) {
+    return Diag(Real->getBeginLoc(),
+                diag::err_typecheck_call_different_arg_types)
+           << Real->getType() << Imag->getType()
+           << Real->getSourceRange() << Imag->getSourceRange();
+  }
+
+  // We don't allow _Complex _Float16 nor _Complex __fp16 as type specifiers;
+  // don't allow this builtin to form those types either.
+  // FIXME: Should we allow these types?
+  if (Real->getType()->isFloat16Type())
+    return Diag(TheCall->getBeginLoc(), diag::err_invalid_complex_spec)
+           << "_Float16";
+  if (Real->getType()->isHalfType())
+    return Diag(TheCall->getBeginLoc(), diag::err_invalid_complex_spec)
+           << "half";
+
+  TheCall->setType(Context.getComplexType(Real->getType()));
+  return false;
+}
+
 // Customized Sema Checking for VSX builtins that have the following signature:
 // vector [...] builtinName(vector [...], vector [...], const int);
 // Which takes the same type of vectors (any legal vector type) for the first
@@ -5950,8 +6003,7 @@ bool Sema::SemaBuiltinVSX(CallExpr *TheCall) {
            << TheCall->getSourceRange();
 
   // Check the third argument is a compile time constant
-  llvm::APSInt Value;
-  if(!TheCall->getArg(2)->isIntegerConstantExpr(Value, Context))
+  if (!TheCall->getArg(2)->isIntegerConstantExpr(Context))
     return Diag(TheCall->getBeginLoc(),
                 diag::err_vsx_builtin_nonconstant_argument)
            << 3 /* argument index */ << TheCall->getDirectCallee()
@@ -6046,17 +6098,18 @@ ExprResult Sema::SemaBuiltinShuffleVector(CallExpr *TheCall) {
         TheCall->getArg(i)->isValueDependent())
       continue;
 
-    llvm::APSInt Result(32);
-    if (!TheCall->getArg(i)->isIntegerConstantExpr(Result, Context))
+    Optional<llvm::APSInt> Result;
+    if (!(Result = TheCall->getArg(i)->getIntegerConstantExpr(Context)))
       return ExprError(Diag(TheCall->getBeginLoc(),
                             diag::err_shufflevector_nonconstant_argument)
                        << TheCall->getArg(i)->getSourceRange());
 
     // Allow -1 which will be translated to undef in the IR.
-    if (Result.isSigned() && Result.isAllOnesValue())
+    if (Result->isSigned() && Result->isAllOnesValue())
       continue;
 
-    if (Result.getActiveBits() > 64 || Result.getZExtValue() >= numElements*2)
+    if (Result->getActiveBits() > 64 ||
+        Result->getZExtValue() >= numElements * 2)
       return ExprError(Diag(TheCall->getBeginLoc(),
                             diag::err_shufflevector_argument_too_large)
                        << TheCall->getArg(i)->getSourceRange());
@@ -6303,10 +6356,11 @@ bool Sema::SemaBuiltinConstantArg(CallExpr *TheCall, int ArgNum,
 
   if (Arg->isTypeDependent() || Arg->isValueDependent()) return false;
 
-  if (!Arg->isIntegerConstantExpr(Result, Context))
+  Optional<llvm::APSInt> R;
+  if (!(R = Arg->getIntegerConstantExpr(Context)))
     return Diag(TheCall->getBeginLoc(), diag::err_constant_integer_arg_type)
            << FDecl->getDeclName() << Arg->getSourceRange();
-
+  Result = *R;
   return false;
 }
 
@@ -10466,14 +10520,15 @@ static IntRange GetExprRange(ASTContext &C, const Expr *E, unsigned MaxWidth,
 
       // If the shift amount is a positive constant, drop the width by
       // that much.
-      llvm::APSInt shift;
-      if (BO->getRHS()->isIntegerConstantExpr(shift, C) &&
-          shift.isNonNegative()) {
-        unsigned zext = shift.getZExtValue();
-        if (zext >= L.Width)
-          L.Width = (L.NonNegative ? 0 : 1);
-        else
-          L.Width -= zext;
+      if (Optional<llvm::APSInt> shift =
+              BO->getRHS()->getIntegerConstantExpr(C)) {
+        if (shift->isNonNegative()) {
+          unsigned zext = shift->getZExtValue();
+          if (zext >= L.Width)
+            L.Width = (L.NonNegative ? 0 : 1);
+          else
+            L.Width -= zext;
+        }
       }
 
       return L;
@@ -10497,9 +10552,9 @@ static IntRange GetExprRange(ASTContext &C, const Expr *E, unsigned MaxWidth,
       IntRange L = GetExprRange(C, BO->getLHS(), opWidth, InConstantContext);
 
       // If the divisor is constant, use that.
-      llvm::APSInt divisor;
-      if (BO->getRHS()->isIntegerConstantExpr(divisor, C)) {
-        unsigned log2 = divisor.logBase2(); // floor(log_2(divisor))
+      if (Optional<llvm::APSInt> divisor =
+              BO->getRHS()->getIntegerConstantExpr(C)) {
+        unsigned log2 = divisor->logBase2(); // floor(log_2(divisor))
         if (log2 >= L.Width)
           L.Width = (L.NonNegative ? 0 : 1);
         else
@@ -10931,23 +10986,20 @@ static void AnalyzeComparison(Sema &S, BinaryOperator *E) {
   Expr *RHS = E->getRHS();
 
   if (T->isIntegralType(S.Context)) {
-    llvm::APSInt RHSValue;
-    llvm::APSInt LHSValue;
-
-    bool IsRHSIntegralLiteral = RHS->isIntegerConstantExpr(RHSValue, S.Context);
-    bool IsLHSIntegralLiteral = LHS->isIntegerConstantExpr(LHSValue, S.Context);
+    Optional<llvm::APSInt> RHSValue = RHS->getIntegerConstantExpr(S.Context);
+    Optional<llvm::APSInt> LHSValue = LHS->getIntegerConstantExpr(S.Context);
 
     // We don't care about expressions whose result is a constant.
-    if (IsRHSIntegralLiteral && IsLHSIntegralLiteral)
+    if (RHSValue && LHSValue)
       return AnalyzeImpConvsInComparison(S, E);
 
     // We only care about expressions where just one side is literal
-    if (IsRHSIntegralLiteral ^ IsLHSIntegralLiteral) {
+    if ((bool)RHSValue ^ (bool)LHSValue) {
       // Is the constant on the RHS or LHS?
-      const bool RhsConstant = IsRHSIntegralLiteral;
+      const bool RhsConstant = (bool)RHSValue;
       Expr *Const = RhsConstant ? RHS : LHS;
       Expr *Other = RhsConstant ? LHS : RHS;
-      const llvm::APSInt &Value = RhsConstant ? RHSValue : LHSValue;
+      const llvm::APSInt &Value = RhsConstant ? *RHSValue : *LHSValue;
 
       // Check whether an integer constant comparison results in a value
       // of 'true' or 'false'.
@@ -11905,8 +11957,8 @@ static void CheckImplicitConversion(Sema &S, Expr *E, QualType T,
     if (SourcePrecision > 0 && TargetPrecision > 0 &&
         SourcePrecision > TargetPrecision) {
 
-      llvm::APSInt SourceInt;
-      if (E->isIntegerConstantExpr(SourceInt, S.Context)) {
+      if (Optional<llvm::APSInt> SourceInt =
+              E->getIntegerConstantExpr(S.Context)) {
         // If the source integer is a constant, convert it to the target
         // floating point type. Issue a warning if the value changes
         // during the whole conversion.
@@ -11914,11 +11966,11 @@ static void CheckImplicitConversion(Sema &S, Expr *E, QualType T,
             S.Context.getFloatTypeSemantics(QualType(TargetBT, 0)));
         llvm::APFloat::opStatus ConversionStatus =
             TargetFloatValue.convertFromAPInt(
-                SourceInt, SourceBT->isSignedInteger(),
+                *SourceInt, SourceBT->isSignedInteger(),
                 llvm::APFloat::rmNearestTiesToEven);
 
         if (ConversionStatus != llvm::APFloat::opOK) {
-          std::string PrettySourceValue = SourceInt.toString(10);
+          std::string PrettySourceValue = SourceInt->toString(10);
           SmallString<32> PrettyTargetValue;
           TargetFloatValue.toString(PrettyTargetValue, TargetPrecision);
 
@@ -13641,10 +13693,9 @@ getAlignmentAndOffsetFromBinAddOrSub(const Expr *PtrE, const Expr *IntE,
   if (!P)
     return llvm::None;
 
-  llvm::APSInt IdxRes;
   CharUnits EltSize = Ctx.getTypeSizeInChars(PointeeType);
-  if (IntE->isIntegerConstantExpr(IdxRes, Ctx)) {
-    CharUnits Offset = EltSize * IdxRes.getExtValue();
+  if (Optional<llvm::APSInt> IdxRes = IntE->getIntegerConstantExpr(Ctx)) {
+    CharUnits Offset = EltSize * IdxRes->getExtValue();
     if (IsSub)
       Offset = -Offset;
     return std::make_pair(P->first, P->second + Offset);
@@ -14269,9 +14320,10 @@ namespace {
           return;
         if (Expr *RHS = BinOp->getRHS()) {
           RHS = RHS->IgnoreParenCasts();
-          llvm::APSInt Value;
+          Optional<llvm::APSInt> Value;
           VarWillBeReased =
-            (RHS && RHS->isIntegerConstantExpr(Value, Context) && Value == 0);
+              (RHS && (Value = RHS->getIntegerConstantExpr(Context)) &&
+               *Value == 0);
         }
       }
     }
@@ -15498,14 +15550,15 @@ ExprResult Sema::SemaBuiltinMatrixTranspose(CallExpr *TheCall,
 // Get and verify the matrix dimensions.
 static llvm::Optional<unsigned>
 getAndVerifyMatrixDimension(Expr *Expr, StringRef Name, Sema &S) {
-  llvm::APSInt Value(64);
   SourceLocation ErrorPos;
-  if (!Expr->isIntegerConstantExpr(Value, S.Context, &ErrorPos)) {
+  Optional<llvm::APSInt> Value =
+      Expr->getIntegerConstantExpr(S.Context, &ErrorPos);
+  if (!Value) {
     S.Diag(Expr->getBeginLoc(), diag::err_builtin_matrix_scalar_unsigned_arg)
         << Name;
     return {};
   }
-  uint64_t Dim = Value.getZExtValue();
+  uint64_t Dim = Value->getZExtValue();
   if (!ConstantMatrixType::isDimensionValid(Dim)) {
     S.Diag(Expr->getBeginLoc(), diag::err_builtin_matrix_invalid_dimension)
         << Name << ConstantMatrixType::getMaxElementsPerDimension();
@@ -15609,13 +15662,15 @@ ExprResult Sema::SemaBuiltinMatrixColumnMajorLoad(CallExpr *TheCall,
   StrideExpr = StrideConv.get();
   TheCall->setArg(3, StrideExpr);
 
-  llvm::APSInt Value(64);
-  if (MaybeRows && StrideExpr->isIntegerConstantExpr(Value, Context)) {
-    uint64_t Stride = Value.getZExtValue();
-    if (Stride < *MaybeRows) {
-      Diag(StrideExpr->getBeginLoc(),
-           diag::err_builtin_matrix_stride_too_small);
-      ArgError = true;
+  if (MaybeRows) {
+    if (Optional<llvm::APSInt> Value =
+            StrideExpr->getIntegerConstantExpr(Context)) {
+      uint64_t Stride = Value->getZExtValue();
+      if (Stride < *MaybeRows) {
+        Diag(StrideExpr->getBeginLoc(),
+             diag::err_builtin_matrix_stride_too_small);
+        ArgError = true;
+      }
     }
   }
 
@@ -15706,13 +15761,15 @@ ExprResult Sema::SemaBuiltinMatrixColumnMajorStore(CallExpr *TheCall,
   }
 
   // Check stride argument.
-  llvm::APSInt Value(64);
-  if (MatrixTy && StrideExpr->isIntegerConstantExpr(Value, Context)) {
-    uint64_t Stride = Value.getZExtValue();
-    if (Stride < MatrixTy->getNumRows()) {
-      Diag(StrideExpr->getBeginLoc(),
-           diag::err_builtin_matrix_stride_too_small);
-      ArgError = true;
+  if (MatrixTy) {
+    if (Optional<llvm::APSInt> Value =
+            StrideExpr->getIntegerConstantExpr(Context)) {
+      uint64_t Stride = Value->getZExtValue();
+      if (Stride < MatrixTy->getNumRows()) {
+        Diag(StrideExpr->getBeginLoc(),
+             diag::err_builtin_matrix_stride_too_small);
+        ArgError = true;
+      }
     }
   }
 
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 03eae7bb0a740..73150e03fea5f 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -8010,7 +8010,7 @@ void Sema::CheckVariableDeclarationType(VarDecl *NewVD) {
     return;
   }
 
-  if (!NewVD->hasLocalStorage() && T->isSizelessType()) {
+  if (!NewVD->hasLocalStorage() && T->isSizelessType() && !T->isVLST()) {
     Diag(NewVD->getLocation(), diag::err_sizeless_nonlocal) << T;
     NewVD->setInvalidDecl();
     return;
@@ -11069,8 +11069,14 @@ bool Sema::CheckForConstantInitializer(Expr *Init, QualType DclT) {
   // except that the aforementioned are allowed in unevaluated
   // expressions.  Everything else falls under the
   // "may accept other forms of constant expressions" exception.
-  // (We never end up here for C++, so the constant expression
-  // rules there don't matter.)
+  //
+  // Regular C++ code will not end up here (exceptions: language extensions,
+  // OpenCL C++ etc), so the constant expression rules there don't matter.
+  if (Init->isValueDependent()) {
+    assert(Init->containsErrors() &&
+           "Dependent code should only occur in error-recovery path.");
+    return true;
+  }
   const Expr *Culprit;
   if (Init->isConstantInitializer(Context, false, &Culprit))
     return false;
@@ -12042,7 +12048,7 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
     // Try to correct any TypoExprs in the initialization arguments.
     for (size_t Idx = 0; Idx < Args.size(); ++Idx) {
       ExprResult Res = CorrectDelayedTyposInExpr(
-          Args[Idx], VDecl, /*RecoverUncorrectedTypos=*/false,
+          Args[Idx], VDecl, /*RecoverUncorrectedTypos=*/true,
           [this, Entity, Kind](Expr *E) {
             InitializationSequence Init(*this, Entity, Kind, MultiExprArg(E));
             return Init.Failed() ? ExprError() : E;
@@ -13161,20 +13167,20 @@ void Sema::FinalizeDeclaration(Decl *ThisDecl) {
     if (!MagicValueExpr) {
       continue;
     }
-    llvm::APSInt MagicValueInt;
-    if (!MagicValueExpr->isIntegerConstantExpr(MagicValueInt, Context)) {
+    Optional<llvm::APSInt> MagicValueInt;
+    if (!(MagicValueInt = MagicValueExpr->getIntegerConstantExpr(Context))) {
       Diag(I->getRange().getBegin(),
            diag::err_type_tag_for_datatype_not_ice)
         << LangOpts.CPlusPlus << MagicValueExpr->getSourceRange();
       continue;
     }
-    if (MagicValueInt.getActiveBits() > 64) {
+    if (MagicValueInt->getActiveBits() > 64) {
       Diag(I->getRange().getBegin(),
            diag::err_type_tag_for_datatype_too_large)
         << LangOpts.CPlusPlus << MagicValueExpr->getSourceRange();
       continue;
     }
-    uint64_t MagicValue = MagicValueInt.getZExtValue();
+    uint64_t MagicValue = MagicValueInt->getZExtValue();
     RegisterTypeTagForDatatype(I->getArgumentKind(),
                                MagicValue,
                                I->getMatchingCType(),
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index b5170932ae739..054f880600407 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -240,9 +240,9 @@ template <typename AttrInfo>
 static bool checkUInt32Argument(Sema &S, const AttrInfo &AI, const Expr *Expr,
                                 uint32_t &Val, unsigned Idx = UINT_MAX,
                                 bool StrictlyUnsigned = false) {
-  llvm::APSInt I(32);
+  Optional<llvm::APSInt> I = llvm::APSInt(32);
   if (Expr->isTypeDependent() || Expr->isValueDependent() ||
-      !Expr->isIntegerConstantExpr(I, S.Context)) {
+      !(I = Expr->getIntegerConstantExpr(S.Context))) {
     if (Idx != UINT_MAX)
       S.Diag(getAttrLoc(AI), diag::err_attribute_argument_n_type)
           << &AI << Idx << AANT_ArgumentIntegerConstant
@@ -253,19 +253,19 @@ static bool checkUInt32Argument(Sema &S, const AttrInfo &AI, const Expr *Expr,
     return false;
   }
 
-  if (!I.isIntN(32)) {
+  if (!I->isIntN(32)) {
     S.Diag(Expr->getExprLoc(), diag::err_ice_too_large)
-        << I.toString(10, false) << 32 << /* Unsigned */ 1;
+        << I->toString(10, false) << 32 << /* Unsigned */ 1;
     return false;
   }
 
-  if (StrictlyUnsigned && I.isSigned() && I.isNegative()) {
+  if (StrictlyUnsigned && I->isSigned() && I->isNegative()) {
     S.Diag(getAttrLoc(AI), diag::err_attribute_requires_positive_integer)
         << &AI << /*non-negative*/ 1;
     return false;
   }
 
-  Val = (uint32_t)I.getZExtValue();
+  Val = (uint32_t)I->getZExtValue();
   return true;
 }
 
@@ -332,16 +332,16 @@ static bool checkFunctionOrMethodParameterIndex(
   unsigned NumParams =
       (HP ? getFunctionOrMethodNumParams(D) : 0) + HasImplicitThisParam;
 
-  llvm::APSInt IdxInt;
+  Optional<llvm::APSInt> IdxInt;
   if (IdxExpr->isTypeDependent() || IdxExpr->isValueDependent() ||
-      !IdxExpr->isIntegerConstantExpr(IdxInt, S.Context)) {
+      !(IdxInt = IdxExpr->getIntegerConstantExpr(S.Context))) {
     S.Diag(getAttrLoc(AI), diag::err_attribute_argument_n_type)
         << &AI << AttrArgNum << AANT_ArgumentIntegerConstant
         << IdxExpr->getSourceRange();
     return false;
   }
 
-  unsigned IdxSource = IdxInt.getLimitedValue(UINT_MAX);
+  unsigned IdxSource = IdxInt->getLimitedValue(UINT_MAX);
   if (IdxSource < 1 || (!IV && IdxSource > NumParams)) {
     S.Diag(getAttrLoc(AI), diag::err_attribute_argument_out_of_bounds)
         << &AI << AttrArgNum << IdxExpr->getSourceRange();
@@ -1605,8 +1605,8 @@ void Sema::AddAssumeAlignedAttr(Decl *D, const AttributeCommonInfo &CI, Expr *E,
   }
 
   if (!E->isValueDependent()) {
-    llvm::APSInt I(64);
-    if (!E->isIntegerConstantExpr(I, Context)) {
+    Optional<llvm::APSInt> I = llvm::APSInt(64);
+    if (!(I = E->getIntegerConstantExpr(Context))) {
       if (OE)
         Diag(AttrLoc, diag::err_attribute_argument_n_type)
           << &TmpAttr << 1 << AANT_ArgumentIntegerConstant
@@ -1618,27 +1618,22 @@ void Sema::AddAssumeAlignedAttr(Decl *D, const AttributeCommonInfo &CI, Expr *E,
       return;
     }
 
-    if (!I.isPowerOf2()) {
+    if (!I->isPowerOf2()) {
       Diag(AttrLoc, diag::err_alignment_not_power_of_two)
         << E->getSourceRange();
       return;
     }
 
-    if (I > Sema::MaximumAlignment)
+    if (*I > Sema::MaximumAlignment)
       Diag(CI.getLoc(), diag::warn_assume_aligned_too_great)
           << CI.getRange() << Sema::MaximumAlignment;
   }
 
-  if (OE) {
-    if (!OE->isValueDependent()) {
-      llvm::APSInt I(64);
-      if (!OE->isIntegerConstantExpr(I, Context)) {
-        Diag(AttrLoc, diag::err_attribute_argument_n_type)
-          << &TmpAttr << 2 << AANT_ArgumentIntegerConstant
-          << OE->getSourceRange();
-        return;
-      }
-    }
+  if (OE && !OE->isValueDependent() && !OE->isIntegerConstantExpr(Context)) {
+    Diag(AttrLoc, diag::err_attribute_argument_n_type)
+        << &TmpAttr << 2 << AANT_ArgumentIntegerConstant
+        << OE->getSourceRange();
+    return;
   }
 
   D->addAttr(::new (Context) AssumeAlignedAttr(Context, CI, E, OE));
@@ -2732,36 +2727,36 @@ static void handleSentinelAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   unsigned sentinel = (unsigned)SentinelAttr::DefaultSentinel;
   if (AL.getNumArgs() > 0) {
     Expr *E = AL.getArgAsExpr(0);
-    llvm::APSInt Idx(32);
+    Optional<llvm::APSInt> Idx = llvm::APSInt(32);
     if (E->isTypeDependent() || E->isValueDependent() ||
-        !E->isIntegerConstantExpr(Idx, S.Context)) {
+        !(Idx = E->getIntegerConstantExpr(S.Context))) {
       S.Diag(AL.getLoc(), diag::err_attribute_argument_n_type)
           << AL << 1 << AANT_ArgumentIntegerConstant << E->getSourceRange();
       return;
     }
 
-    if (Idx.isSigned() && Idx.isNegative()) {
+    if (Idx->isSigned() && Idx->isNegative()) {
       S.Diag(AL.getLoc(), diag::err_attribute_sentinel_less_than_zero)
         << E->getSourceRange();
       return;
     }
 
-    sentinel = Idx.getZExtValue();
+    sentinel = Idx->getZExtValue();
   }
 
   unsigned nullPos = (unsigned)SentinelAttr::DefaultNullPos;
   if (AL.getNumArgs() > 1) {
     Expr *E = AL.getArgAsExpr(1);
-    llvm::APSInt Idx(32);
+    Optional<llvm::APSInt> Idx = llvm::APSInt(32);
     if (E->isTypeDependent() || E->isValueDependent() ||
-        !E->isIntegerConstantExpr(Idx, S.Context)) {
+        !(Idx = E->getIntegerConstantExpr(S.Context))) {
       S.Diag(AL.getLoc(), diag::err_attribute_argument_n_type)
           << AL << 2 << AANT_ArgumentIntegerConstant << E->getSourceRange();
       return;
     }
-    nullPos = Idx.getZExtValue();
+    nullPos = Idx->getZExtValue();
 
-    if ((Idx.isSigned() && Idx.isNegative()) || nullPos > 1) {
+    if ((Idx->isSigned() && Idx->isNegative()) || nullPos > 1) {
       // FIXME: This error message could be improved, it would be nice
       // to say what the bounds actually are.
       S.Diag(AL.getLoc(), diag::err_attribute_sentinel_not_zero_or_one)
@@ -2987,14 +2982,14 @@ void Sema::addIntelReqdSubGroupSizeAttr(Decl *D,
     return;
 
   if (!E->isInstantiationDependent()) {
-    llvm::APSInt ArgVal(32);
-    if (!E->isIntegerConstantExpr(ArgVal, getASTContext())) {
+    Optional<llvm::APSInt> ArgVal = E->getIntegerConstantExpr(getASTContext());
+    if (!ArgVal) {
       Diag(E->getExprLoc(), diag::err_attribute_argument_type)
           << Attr.getAttrName() << AANT_ArgumentIntegerConstant
           << E->getSourceRange();
       return;
     }
-    int32_t ArgInt = ArgVal.getSExtValue();
+    int32_t ArgInt = ArgVal->getSExtValue();
     if (ArgInt <= 0) {
       Diag(E->getExprLoc(), diag::err_attribute_requires_positive_integer)
           << Attr.getAttrName() << /*positive*/ 0;
@@ -3014,6 +3009,13 @@ static void handleSubGroupSize(Sema &S, Decl *D, const ParsedAttr &AL) {
   if (D->getAttr<IntelReqdSubGroupSizeAttr>())
     S.Diag(AL.getLoc(), diag::warn_duplicate_attribute) << AL;
 
+  if (AL.getAttributeSpellingListIndex() ==
+      IntelReqdSubGroupSizeAttr::CXX11_cl_intel_reqd_sub_group_size) {
+    S.Diag(AL.getLoc(), diag::warn_attribute_spelling_deprecated) << AL;
+    S.Diag(AL.getLoc(), diag::note_spelling_suggestion)
+        << "'intel::reqd_sub_group_size'";
+  }
+
   S.addIntelReqdSubGroupSizeAttr(D, AL, E);
 }
 
@@ -5049,19 +5051,19 @@ static Expr *makeLaunchBoundsArgExpr(Sema &S, Expr *E,
   if (E->isValueDependent())
     return E;
 
-  llvm::APSInt I(64);
-  if (!E->isIntegerConstantExpr(I, S.Context)) {
+  Optional<llvm::APSInt> I = llvm::APSInt(64);
+  if (!(I = E->getIntegerConstantExpr(S.Context))) {
     S.Diag(E->getExprLoc(), diag::err_attribute_argument_n_type)
         << &AL << Idx << AANT_ArgumentIntegerConstant << E->getSourceRange();
     return nullptr;
   }
   // Make sure we can fit it in 32 bits.
-  if (!I.isIntN(32)) {
-    S.Diag(E->getExprLoc(), diag::err_ice_too_large) << I.toString(10, false)
-                                                     << 32 << /* Unsigned */ 1;
+  if (!I->isIntN(32)) {
+    S.Diag(E->getExprLoc(), diag::err_ice_too_large)
+        << I->toString(10, false) << 32 << /* Unsigned */ 1;
     return nullptr;
   }
-  if (I < 0)
+  if (*I < 0)
     S.Diag(E->getExprLoc(), diag::warn_attribute_argument_n_negative)
         << &AL << Idx << E->getSourceRange();
 
@@ -5554,14 +5556,14 @@ void Sema::addSYCLIntelPipeIOAttr(Decl *D, const AttributeCommonInfo &Attr,
   }
 
   if (!E->isInstantiationDependent()) {
-    llvm::APSInt ArgVal(32);
-    if (!E->isIntegerConstantExpr(ArgVal, getASTContext())) {
+    Optional<llvm::APSInt> ArgVal = E->getIntegerConstantExpr(getASTContext());
+    if (!ArgVal) {
       Diag(E->getExprLoc(), diag::err_attribute_argument_type)
           << Attr.getAttrName() << AANT_ArgumentIntegerConstant
           << E->getSourceRange();
       return;
     }
-    int32_t ArgInt = ArgVal.getSExtValue();
+    int32_t ArgInt = ArgVal->getSExtValue();
     if (ArgInt < 0) {
       Diag(E->getExprLoc(), diag::err_attribute_requires_positive_integer)
           << Attr.getAttrName() << /*non-negative*/ 1;
@@ -6291,18 +6293,18 @@ static void handleMSP430InterruptAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   }
 
   Expr *NumParamsExpr = static_cast<Expr *>(AL.getArgAsExpr(0));
-  llvm::APSInt NumParams(32);
-  if (!NumParamsExpr->isIntegerConstantExpr(NumParams, S.Context)) {
+  Optional<llvm::APSInt> NumParams = llvm::APSInt(32);
+  if (!(NumParams = NumParamsExpr->getIntegerConstantExpr(S.Context))) {
     S.Diag(AL.getLoc(), diag::err_attribute_argument_type)
         << AL << AANT_ArgumentIntegerConstant
         << NumParamsExpr->getSourceRange();
     return;
   }
   // The argument should be in range 0..63.
-  unsigned Num = NumParams.getLimitedValue(255);
+  unsigned Num = NumParams->getLimitedValue(255);
   if (Num > 63) {
     S.Diag(AL.getLoc(), diag::err_attribute_argument_out_of_bounds)
-        << AL << (int)NumParams.getSExtValue()
+        << AL << (int)NumParams->getSExtValue()
         << NumParamsExpr->getSourceRange();
     return;
   }
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 5db96698c2a06..97f74fa436a7a 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -4761,12 +4761,13 @@ ExprResult Sema::CreateBuiltinMatrixSubscriptExpr(Expr *Base, Expr *RowIdx,
       return nullptr;
     }
 
-    llvm::APSInt Idx;
-    if (IndexExpr->isIntegerConstantExpr(Idx, Context) &&
-        (Idx < 0 || Idx >= Dim)) {
-      Diag(IndexExpr->getBeginLoc(), diag::err_matrix_index_outside_range)
-          << IsColumnIdx << Dim;
-      return nullptr;
+    if (Optional<llvm::APSInt> Idx =
+            IndexExpr->getIntegerConstantExpr(Context)) {
+      if ((*Idx < 0 || *Idx >= Dim)) {
+        Diag(IndexExpr->getBeginLoc(), diag::err_matrix_index_outside_range)
+            << IsColumnIdx << Dim;
+        return nullptr;
+      }
     }
 
     ExprResult ConvExpr =
@@ -5182,12 +5183,11 @@ ExprResult Sema::ActOnOMPIteratorExpr(Scope *S, SourceLocation IteratorKwLoc,
         IsCorrect = false;
         continue;
       }
-      llvm::APSInt Result;
-      bool IsConstant = Step->isIntegerConstantExpr(Result, Context);
+      Optional<llvm::APSInt> Result = Step->getIntegerConstantExpr(Context);
       // OpenMP 5.0, 2.1.6 Iterators, Restrictions
       // If the step expression of a range-specification equals zero, the
       // behavior is unspecified.
-      if (IsConstant && Result.isNullValue()) {
+      if (Result && Result->isNullValue()) {
         Diag(Step->getExprLoc(), diag::err_omp_iterator_step_constant_zero)
             << Step << Step->getSourceRange();
         IsCorrect = false;
@@ -6347,7 +6347,7 @@ ExprResult Sema::BuildCallExpr(Scope *Scope, Expr *Fn, SourceLocation LParenLoc,
       }
 
       return CallExpr::Create(Context, Fn, /*Args=*/{}, Context.VoidTy,
-                              VK_RValue, RParenLoc);
+                              VK_RValue, RParenLoc, CurFPFeatureOverrides());
     }
     if (Fn->getType() == Context.PseudoObjectTy) {
       ExprResult result = CheckPlaceholderExpr(Fn);
@@ -6361,7 +6361,7 @@ ExprResult Sema::BuildCallExpr(Scope *Scope, Expr *Fn, SourceLocation LParenLoc,
       if (ExecConfig) {
         return CUDAKernelCallExpr::Create(
             Context, Fn, cast<CallExpr>(ExecConfig), ArgExprs,
-            Context.DependentTy, VK_RValue, RParenLoc);
+            Context.DependentTy, VK_RValue, RParenLoc, CurFPFeatureOverrides());
       } else {
 
         tryImplicitlyCaptureThisIfImplicitMemberFunctionAccessWithDependentArgs(
@@ -6369,7 +6369,7 @@ ExprResult Sema::BuildCallExpr(Scope *Scope, Expr *Fn, SourceLocation LParenLoc,
             Fn->getBeginLoc());
 
         return CallExpr::Create(Context, Fn, ArgExprs, Context.DependentTy,
-                                VK_RValue, RParenLoc);
+                                VK_RValue, RParenLoc, CurFPFeatureOverrides());
       }
     }
 
@@ -6398,7 +6398,7 @@ ExprResult Sema::BuildCallExpr(Scope *Scope, Expr *Fn, SourceLocation LParenLoc,
     if (!find.HasFormOfMemberPointer) {
       if (Expr::hasAnyTypeDependentArguments(ArgExprs))
         return CallExpr::Create(Context, Fn, ArgExprs, Context.DependentTy,
-                                VK_RValue, RParenLoc);
+                                VK_RValue, RParenLoc, CurFPFeatureOverrides());
       OverloadExpr *ovl = find.Expression;
       if (UnresolvedLookupExpr *ULE = dyn_cast<UnresolvedLookupExpr>(ovl))
         return BuildOverloadedCallExpr(
@@ -6589,12 +6589,13 @@ ExprResult Sema::BuildResolvedCallExpr(Expr *Fn, NamedDecl *NDecl,
   if (Config) {
     assert(UsesADL == ADLCallKind::NotADL &&
            "CUDAKernelCallExpr should not use ADL");
-    TheCall =
-        CUDAKernelCallExpr::Create(Context, Fn, cast<CallExpr>(Config), Args,
-                                   ResultTy, VK_RValue, RParenLoc, NumParams);
+    TheCall = CUDAKernelCallExpr::Create(Context, Fn, cast<CallExpr>(Config),
+                                         Args, ResultTy, VK_RValue, RParenLoc,
+                                         CurFPFeatureOverrides(), NumParams);
   } else {
-    TheCall = CallExpr::Create(Context, Fn, Args, ResultTy, VK_RValue,
-                               RParenLoc, NumParams, UsesADL);
+    TheCall =
+        CallExpr::Create(Context, Fn, Args, ResultTy, VK_RValue, RParenLoc,
+                         CurFPFeatureOverrides(), NumParams, UsesADL);
   }
 
   if (!getLangOpts().CPlusPlus) {
@@ -6621,10 +6622,11 @@ ExprResult Sema::BuildResolvedCallExpr(Expr *Fn, NamedDecl *NDecl,
       if (Config)
         TheCall = CUDAKernelCallExpr::Create(
             Context, Fn, cast<CallExpr>(Config), Args, ResultTy, VK_RValue,
-            RParenLoc, NumParams);
+            RParenLoc, CurFPFeatureOverrides(), NumParams);
       else
-        TheCall = CallExpr::Create(Context, Fn, Args, ResultTy, VK_RValue,
-                                   RParenLoc, NumParams, UsesADL);
+        TheCall =
+            CallExpr::Create(Context, Fn, Args, ResultTy, VK_RValue, RParenLoc,
+                             CurFPFeatureOverrides(), NumParams, UsesADL);
     }
     // We can now handle the nulled arguments for the default arguments.
     TheCall->setNumArgsUnsafe(std::max<unsigned>(Args.size(), NumParams));
@@ -8131,6 +8133,15 @@ QualType Sema::CheckConditionalOperands(ExprResult &Cond, ExprResult &LHS,
   // If both operands have arithmetic type, do the usual arithmetic conversions
   // to find a common type: C99 6.5.15p3,5.
   if (LHSTy->isArithmeticType() && RHSTy->isArithmeticType()) {
+    // Disallow invalid arithmetic conversions, such as those between ExtInts of
+    // different sizes, or between ExtInts and other types.
+    if (ResTy.isNull() && (LHSTy->isExtIntType() || RHSTy->isExtIntType())) {
+      Diag(QuestionLoc, diag::err_typecheck_cond_incompatible_operands)
+          << LHSTy << RHSTy << LHS.get()->getSourceRange()
+          << RHS.get()->getSourceRange();
+      return QualType();
+    }
+
     LHS = ImpCastExprToType(LHS.get(), ResTy, PrepareScalarCast(LHS, ResTy));
     RHS = ImpCastExprToType(RHS.get(), ResTy, PrepareScalarCast(RHS, ResTy));
 
@@ -18485,9 +18496,12 @@ Sema::ConditionResult Sema::ActOnCondition(Scope *S, SourceLocation Loc,
     Cond = CheckSwitchCondition(Loc, SubExpr);
     break;
   }
-  if (Cond.isInvalid())
-    return ConditionError();
-
+  if (Cond.isInvalid()) {
+    Cond = CreateRecoveryExpr(SubExpr->getBeginLoc(), SubExpr->getEndLoc(),
+                              {SubExpr});
+    if (!Cond.get())
+      return ConditionError();
+  }
   // FIXME: FullExprArg doesn't have an invalid bit, so check nullness instead.
   FullExprArg FullExpr = MakeFullExpr(Cond.get(), Loc);
   if (!FullExpr.get())
@@ -19101,7 +19115,8 @@ ExprResult Sema::CheckPlaceholderExpr(Expr *E) {
                               CK_BuiltinFnToFnPtr)
                 .get();
         return CallExpr::Create(Context, E, /*Args=*/{}, Context.IntTy,
-                                VK_RValue, SourceLocation());
+                                VK_RValue, SourceLocation(),
+                                FPOptionsOverride());
       }
     }
 
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index 3d9a3a767ef45..aec24a17877fd 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -172,7 +172,7 @@ ParsedType Sema::getDestructorName(SourceLocation TildeLoc,
   bool Failed = false;
 
   llvm::SmallVector<NamedDecl*, 8> FoundDecls;
-  llvm::SmallSet<CanonicalDeclPtr<Decl>, 8> FoundDeclSet;
+  llvm::SmallPtrSet<CanonicalDeclPtr<Decl>, 8> FoundDeclSet;
 
   // If we have an object type, it's because we are in a
   // pseudo-destructor-expression or a member access expression, and
@@ -1394,6 +1394,9 @@ Sema::ActOnCXXTypeConstructExpr(ParsedType TypeRep,
   if (!Result.isInvalid() && Result.get()->isInstantiationDependent() &&
       !Result.get()->isTypeDependent())
     Result = CorrectDelayedTyposInExpr(Result.get());
+  else if (Result.isInvalid())
+    Result = CreateRecoveryExpr(TInfo->getTypeLoc().getBeginLoc(),
+                                RParenOrBraceLoc, exprs, Ty);
   return Result;
 }
 
@@ -2078,29 +2081,29 @@ Sema::BuildCXXNew(SourceRange Range, bool UseGlobal,
     // per CWG1464. Otherwise, if it's not a constant, we must have an
     // unparenthesized array type.
     if (!(*ArraySize)->isValueDependent()) {
-      llvm::APSInt Value;
       // We've already performed any required implicit conversion to integer or
       // unscoped enumeration type.
       // FIXME: Per CWG1464, we are required to check the value prior to
       // converting to size_t. This will never find a negative array size in
       // C++14 onwards, because Value is always unsigned here!
-      if ((*ArraySize)->isIntegerConstantExpr(Value, Context)) {
-        if (Value.isSigned() && Value.isNegative()) {
+      if (Optional<llvm::APSInt> Value =
+              (*ArraySize)->getIntegerConstantExpr(Context)) {
+        if (Value->isSigned() && Value->isNegative()) {
           return ExprError(Diag((*ArraySize)->getBeginLoc(),
                                 diag::err_typecheck_negative_array_size)
                            << (*ArraySize)->getSourceRange());
         }
 
         if (!AllocType->isDependentType()) {
-          unsigned ActiveSizeBits =
-            ConstantArrayType::getNumAddressingBits(Context, AllocType, Value);
+          unsigned ActiveSizeBits = ConstantArrayType::getNumAddressingBits(
+              Context, AllocType, *Value);
           if (ActiveSizeBits > ConstantArrayType::getMaxSizeBits(Context))
             return ExprError(
                 Diag((*ArraySize)->getBeginLoc(), diag::err_array_too_large)
-                << Value.toString(10) << (*ArraySize)->getSourceRange());
+                << Value->toString(10) << (*ArraySize)->getSourceRange());
         }
 
-        KnownArraySize = Value.getZExtValue();
+        KnownArraySize = Value->getZExtValue();
       } else if (TypeIdParens.isValid()) {
         // Can't have dynamic array size when the type-id is in parentheses.
         Diag((*ArraySize)->getBeginLoc(), diag::ext_new_paren_array_nonconst)
@@ -7625,7 +7628,8 @@ ExprResult Sema::BuildCXXMemberCallExpr(Expr *E, NamedDecl *FoundDecl,
   ResultType = ResultType.getNonLValueExprType(Context);
 
   CXXMemberCallExpr *CE = CXXMemberCallExpr::Create(
-      Context, ME, /*Args=*/{}, ResultType, VK, Exp.get()->getEndLoc());
+      Context, ME, /*Args=*/{}, ResultType, VK, Exp.get()->getEndLoc(),
+      CurFPFeatureOverrides());
 
   if (CheckFunctionCall(Method, CE,
                         Method->getType()->castAs<FunctionProtoType>()))
@@ -7986,19 +7990,26 @@ class TransformTypos : public TreeTransform<TransformTypos> {
     }
   }
 
-  /// If corrections for the first TypoExpr have been exhausted for a
-  /// given combination of the other TypoExprs, retry those corrections against
-  /// the next combination of substitutions for the other TypoExprs by advancing
-  /// to the next potential correction of the second TypoExpr. For the second
-  /// and subsequent TypoExprs, if its stream of corrections has been exhausted,
-  /// the stream is reset and the next TypoExpr's stream is advanced by one (a
-  /// TypoExpr's correction stream is advanced by removing the TypoExpr from the
-  /// TransformCache). Returns true if there is still any untried combinations
-  /// of corrections.
+  /// Try to advance the typo correction state of the first unfinished TypoExpr.
+  /// We allow advancement of the correction stream by removing it from the
+  /// TransformCache which allows `TransformTypoExpr` to advance during the
+  /// next transformation attempt.
+  ///
+  /// Any substitution attempts for the previous TypoExprs (which must have been
+  /// finished) will need to be retried since it's possible that they will now
+  /// be invalid given the latest advancement.
+  ///
+  /// We need to be sure that we're making progress - it's possible that the
+  /// tree is so malformed that the transform never makes it to the
+  /// `TransformTypoExpr`.
+  ///
+  /// Returns true if there are any untried correction combinations.
   bool CheckAndAdvanceTypoExprCorrectionStreams() {
     for (auto TE : TypoExprs) {
       auto &State = SemaRef.getTypoExprState(TE);
       TransformCache.erase(TE);
+      if (!State.Consumer->hasMadeAnyCorrectionProgress())
+        return false;
       if (!State.Consumer->finished())
         return true;
       State.Consumer->resetCorrectionStream();
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index 0459aeae8ea57..82f91c91c9884 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -962,6 +962,8 @@ InitListChecker::InitListChecker(Sema &S, const InitializedEntity &Entity,
       FillInEmptyInitializations(Entity, FullyStructuredList,
                                  RequiresSecondPass, nullptr, 0);
   }
+  if (hadError && FullyStructuredList)
+    FullyStructuredList->markError();
 }
 
 int InitListChecker::numArrayElements(QualType DeclType) {
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 8bf605e5e76b8..51609e37e20ca 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -6027,8 +6027,7 @@ Sema::checkOpenMPDeclareVariantFunction(Sema::DeclGroupPtrTy DG,
   // Deal with non-constant score and user condition expressions.
   auto HandleNonConstantScoresAndConditions = [this](Expr *&E,
                                                      bool IsScore) -> bool {
-    llvm::APSInt Result;
-    if (!E || E->isIntegerConstantExpr(Result, Context))
+    if (!E || E->isIntegerConstantExpr(Context))
       return false;
 
     if (IsScore) {
@@ -6514,14 +6513,14 @@ bool OpenMPIterationSpaceChecker::setStep(Expr *NewStep, bool Subtract) {
     //  loop. If test-expr is of form b relational-op var and relational-op is
     //  > or >= then incr-expr must cause var to increase on each iteration of
     //  the loop.
-    llvm::APSInt Result;
-    bool IsConstant = NewStep->isIntegerConstantExpr(Result, SemaRef.Context);
+    Optional<llvm::APSInt> Result =
+        NewStep->getIntegerConstantExpr(SemaRef.Context);
     bool IsUnsigned = !NewStep->getType()->hasSignedIntegerRepresentation();
     bool IsConstNeg =
-        IsConstant && Result.isSigned() && (Subtract != Result.isNegative());
+        Result && Result->isSigned() && (Subtract != Result->isNegative());
     bool IsConstPos =
-        IsConstant && Result.isSigned() && (Subtract == Result.isNegative());
-    bool IsConstZero = IsConstant && !Result.getBoolValue();
+        Result && Result->isSigned() && (Subtract == Result->isNegative());
+    bool IsConstZero = Result && !Result->getBoolValue();
 
     // != with increment is treated as <; != with decrement is treated as >
     if (!TestIsLessOp.hasValue())
@@ -6969,9 +6968,16 @@ calculateNumIters(Sema &SemaRef, Scope *S, SourceLocation DefaultLoc,
   ExprResult NewStep = tryBuildCapture(SemaRef, Step, Captures);
   if (!NewStep.isUsable())
     return nullptr;
-  llvm::APSInt LRes, URes, SRes;
-  bool IsLowerConst = Lower->isIntegerConstantExpr(LRes, SemaRef.Context);
-  bool IsStepConst = Step->isIntegerConstantExpr(SRes, SemaRef.Context);
+  llvm::APSInt LRes, SRes;
+  bool IsLowerConst = false, IsStepConst = false;
+  if (Optional<llvm::APSInt> Res = Lower->getIntegerConstantExpr(SemaRef.Context)) {
+    LRes = *Res;
+    IsLowerConst = true;
+  }
+  if (Optional<llvm::APSInt> Res = Step->getIntegerConstantExpr(SemaRef.Context)) {
+    SRes = *Res;
+    IsStepConst = true;
+  }
   bool NoNeedToConvert = IsLowerConst && !RoundToStep &&
                          ((!TestIsStrictOp && LRes.isNonNegative()) ||
                           (TestIsStrictOp && LRes.isStrictlyPositive()));
@@ -7004,7 +7010,12 @@ calculateNumIters(Sema &SemaRef, Scope *S, SourceLocation DefaultLoc,
     }
     NeedToReorganize = NoNeedToConvert;
   }
-  bool IsUpperConst = Upper->isIntegerConstantExpr(URes, SemaRef.Context);
+  llvm::APSInt URes;
+  bool IsUpperConst = false;
+  if (Optional<llvm::APSInt> Res = Upper->getIntegerConstantExpr(SemaRef.Context)) {
+    URes = *Res;
+    IsUpperConst = true;
+  }
   if (NoNeedToConvert && IsLowerConst && IsUpperConst &&
       (!RoundToStep || IsStepConst)) {
     unsigned BW = LRes.getBitWidth() > URes.getBitWidth() ? LRes.getBitWidth()
@@ -7952,9 +7963,9 @@ static ExprResult widenIterationCount(unsigned Bits, Expr *E, Sema &SemaRef) {
 static bool fitsInto(unsigned Bits, bool Signed, const Expr *E, Sema &SemaRef) {
   if (E == nullptr)
     return false;
-  llvm::APSInt Result;
-  if (E->isIntegerConstantExpr(Result, SemaRef.Context))
-    return Signed ? Result.isSignedIntN(Bits) : Result.isIntN(Bits);
+  if (Optional<llvm::APSInt> Result =
+          E->getIntegerConstantExpr(SemaRef.Context))
+    return Signed ? Result->isSignedIntN(Bits) : Result->isIntN(Bits);
   return false;
 }
 
@@ -8227,9 +8238,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
 
   // Calculate the last iteration number beforehand instead of doing this on
   // each iteration. Do not do this if the number of iterations may be kfold-ed.
-  llvm::APSInt Result;
-  bool IsConstant =
-      LastIteration.get()->isIntegerConstantExpr(Result, SemaRef.Context);
+  bool IsConstant = LastIteration.get()->isIntegerConstantExpr(SemaRef.Context);
   ExprResult CalcLastIteration;
   if (!IsConstant) {
     ExprResult SaveRef =
@@ -12620,15 +12629,16 @@ isNonNegativeIntegerValue(Expr *&ValExpr, Sema &SemaRef, OpenMPClauseKind CKind,
 
     ValExpr = Value.get();
     // The expression must evaluate to a non-negative integer value.
-    llvm::APSInt Result;
-    if (ValExpr->isIntegerConstantExpr(Result, SemaRef.Context) &&
-        Result.isSigned() &&
-        !((!StrictlyPositive && Result.isNonNegative()) ||
-          (StrictlyPositive && Result.isStrictlyPositive()))) {
-      SemaRef.Diag(Loc, diag::err_omp_negative_expression_in_clause)
-          << getOpenMPClauseName(CKind) << (StrictlyPositive ? 1 : 0)
-          << ValExpr->getSourceRange();
-      return false;
+    if (Optional<llvm::APSInt> Result =
+            ValExpr->getIntegerConstantExpr(SemaRef.Context)) {
+      if (Result->isSigned() &&
+          !((!StrictlyPositive && Result->isNonNegative()) ||
+            (StrictlyPositive && Result->isStrictlyPositive()))) {
+        SemaRef.Diag(Loc, diag::err_omp_negative_expression_in_clause)
+            << getOpenMPClauseName(CKind) << (StrictlyPositive ? 1 : 0)
+            << ValExpr->getSourceRange();
+        return false;
+      }
     }
     if (!BuildCapture)
       return true;
@@ -13263,9 +13273,9 @@ OMPClause *Sema::ActOnOpenMPScheduleClause(
       // OpenMP [2.7.1, Restrictions]
       //  chunk_size must be a loop invariant integer expression with a positive
       //  value.
-      llvm::APSInt Result;
-      if (ValExpr->isIntegerConstantExpr(Result, Context)) {
-        if (Result.isSigned() && !Result.isStrictlyPositive()) {
+      if (Optional<llvm::APSInt> Result =
+              ValExpr->getIntegerConstantExpr(Context)) {
+        if (Result->isSigned() && !Result->isStrictlyPositive()) {
           Diag(ChunkSizeLoc, diag::err_omp_negative_expression_in_clause)
               << "schedule" << 1 << ChunkSize->getSourceRange();
           return nullptr;
@@ -15311,7 +15321,8 @@ static bool actOnOMPReductionKindClause(
           S.DefaultLvalueConversion(DeclareReductionRef.get()).get());
       Expr *Args[] = {LHS.get(), RHS.get()};
       ReductionOp =
-          CallExpr::Create(Context, OVE, Args, Context.VoidTy, VK_RValue, ELoc);
+          CallExpr::Create(Context, OVE, Args, Context.VoidTy, VK_RValue, ELoc,
+                           S.CurFPFeatureOverrides());
     } else {
       ReductionOp = S.BuildBinOp(
           Stack->getCurScope(), ReductionId.getBeginLoc(), BOK, LHSDRE, RHSDRE);
@@ -15742,12 +15753,12 @@ OMPClause *Sema::ActOnOpenMPLinearClause(
 
     // Warn about zero linear step (it would be probably better specified as
     // making corresponding variables 'const').
-    llvm::APSInt Result;
-    bool IsConstant = StepExpr->isIntegerConstantExpr(Result, Context);
-    if (IsConstant && !Result.isNegative() && !Result.isStrictlyPositive())
-      Diag(StepLoc, diag::warn_omp_linear_step_zero) << Vars[0]
-                                                     << (Vars.size() > 1);
-    if (!IsConstant && CalcStep.isUsable()) {
+    if (Optional<llvm::APSInt> Result =
+            StepExpr->getIntegerConstantExpr(Context)) {
+      if (!Result->isNegative() && !Result->isStrictlyPositive())
+        Diag(StepLoc, diag::warn_omp_linear_step_zero)
+            << Vars[0] << (Vars.size() > 1);
+    } else if (CalcStep.isUsable()) {
       // Calculate the step beforehand instead of doing this on each iteration.
       // (This is not used if the number of iterations may be kfold-ed).
       CalcStepExpr = CalcStep.get();
@@ -17484,6 +17495,7 @@ static void checkMappableExpressionList(
                           /*CurrentRegionOnly=*/true, CurComponents, CKind))
       break;
     if (CKind == OMPC_map &&
+        (SemaRef.getLangOpts().OpenMP <= 45 || StartLoc.isValid()) &&
         checkMapConflicts(SemaRef, DSAS, CurDeclaration, SimpleExpr,
                           /*CurrentRegionOnly=*/false, CurComponents, CKind))
       break;
@@ -17629,9 +17641,9 @@ OMPClause *Sema::ActOnOpenMPMapClause(
     OpenMPMapClauseKind MapType, bool IsMapTypeImplicit, SourceLocation MapLoc,
     SourceLocation ColonLoc, ArrayRef<Expr *> VarList,
     const OMPVarListLocTy &Locs, ArrayRef<Expr *> UnresolvedMappers) {
-  OpenMPMapModifierKind Modifiers[] = {OMPC_MAP_MODIFIER_unknown,
-                                       OMPC_MAP_MODIFIER_unknown,
-                                       OMPC_MAP_MODIFIER_unknown};
+  OpenMPMapModifierKind Modifiers[] = {
+      OMPC_MAP_MODIFIER_unknown, OMPC_MAP_MODIFIER_unknown,
+      OMPC_MAP_MODIFIER_unknown, OMPC_MAP_MODIFIER_unknown};
   SourceLocation ModifiersLoc[NumberOfOMPMapClauseModifiers];
 
   // Process map-type-modifiers, flag errors for duplicate modifiers.
@@ -18279,9 +18291,9 @@ OMPClause *Sema::ActOnOpenMPDistScheduleClause(
       // OpenMP [2.7.1, Restrictions]
       //  chunk_size must be a loop invariant integer expression with a positive
       //  value.
-      llvm::APSInt Result;
-      if (ValExpr->isIntegerConstantExpr(Result, Context)) {
-        if (Result.isSigned() && !Result.isStrictlyPositive()) {
+      if (Optional<llvm::APSInt> Result =
+              ValExpr->getIntegerConstantExpr(Context)) {
+        if (Result->isSigned() && !Result->isStrictlyPositive()) {
           Diag(ChunkSizeLoc, diag::err_omp_negative_expression_in_clause)
               << "dist_schedule" << ChunkSize->getSourceRange();
           return nullptr;
@@ -19028,7 +19040,7 @@ OMPClause *Sema::ActOnOpenMPUsesAllocatorClause(
           [](const UsesAllocatorsData &D) { return D.AllocatorTraits; }) &&
       !findOMPAlloctraitT(*this, StartLoc, DSAStack))
     return nullptr;
-  llvm::SmallSet<CanonicalDeclPtr<Decl>, 4> PredefinedAllocators;
+  llvm::SmallPtrSet<CanonicalDeclPtr<Decl>, 4> PredefinedAllocators;
   for (int I = 0; I < OMPAllocateDeclAttr::OMPUserDefinedMemAlloc; ++I) {
     auto AllocatorKind = static_cast<OMPAllocateDeclAttr::AllocatorTypeTy>(I);
     StringRef Allocator =
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 8635397f48067..5b4e7a2fdafa5 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -346,7 +346,6 @@ NarrowingKind StandardConversionSequence::getNarrowingKind(
                ToType->isRealFloatingType()) {
       if (IgnoreFloatToIntegralConversion)
         return NK_Not_Narrowing;
-      llvm::APSInt IntConstantValue;
       const Expr *Initializer = IgnoreNarrowingConversion(Ctx, Converted);
       assert(Initializer && "Unknown conversion expression");
 
@@ -354,19 +353,20 @@ NarrowingKind StandardConversionSequence::getNarrowingKind(
       if (Initializer->isValueDependent())
         return NK_Dependent_Narrowing;
 
-      if (Initializer->isIntegerConstantExpr(IntConstantValue, Ctx)) {
+      if (Optional<llvm::APSInt> IntConstantValue =
+              Initializer->getIntegerConstantExpr(Ctx)) {
         // Convert the integer to the floating type.
         llvm::APFloat Result(Ctx.getFloatTypeSemantics(ToType));
-        Result.convertFromAPInt(IntConstantValue, IntConstantValue.isSigned(),
+        Result.convertFromAPInt(*IntConstantValue, IntConstantValue->isSigned(),
                                 llvm::APFloat::rmNearestTiesToEven);
         // And back.
-        llvm::APSInt ConvertedValue = IntConstantValue;
+        llvm::APSInt ConvertedValue = *IntConstantValue;
         bool ignored;
         Result.convertToInteger(ConvertedValue,
                                 llvm::APFloat::rmTowardZero, &ignored);
         // If the resulting value is different, this was a narrowing conversion.
-        if (IntConstantValue != ConvertedValue) {
-          ConstantValue = APValue(IntConstantValue);
+        if (*IntConstantValue != ConvertedValue) {
+          ConstantValue = APValue(*IntConstantValue);
           ConstantType = Initializer->getType();
           return NK_Constant_Narrowing;
         }
@@ -430,17 +430,18 @@ NarrowingKind StandardConversionSequence::getNarrowingKind(
         (FromWidth == ToWidth && FromSigned != ToSigned) ||
         (FromSigned && !ToSigned)) {
       // Not all values of FromType can be represented in ToType.
-      llvm::APSInt InitializerValue;
       const Expr *Initializer = IgnoreNarrowingConversion(Ctx, Converted);
 
       // If it's value-dependent, we can't tell whether it's narrowing.
       if (Initializer->isValueDependent())
         return NK_Dependent_Narrowing;
 
-      if (!Initializer->isIntegerConstantExpr(InitializerValue, Ctx)) {
+      Optional<llvm::APSInt> OptInitializerValue;
+      if (!(OptInitializerValue = Initializer->getIntegerConstantExpr(Ctx))) {
         // Such conversions on variables are always narrowing.
         return NK_Variable_Narrowing;
       }
+      llvm::APSInt &InitializerValue = *OptInitializerValue;
       bool Narrowing = false;
       if (FromWidth < ToWidth) {
         // Negative -> unsigned is narrowing. Otherwise, more bits is never
@@ -2183,21 +2184,22 @@ bool Sema::IsIntegralPromotion(Expr *From, QualType FromType, QualType ToType) {
   // compatibility.
   if (From) {
     if (FieldDecl *MemberDecl = From->getSourceBitField()) {
-      llvm::APSInt BitWidth;
+      Optional<llvm::APSInt> BitWidth;
       if (FromType->isIntegralType(Context) &&
-          MemberDecl->getBitWidth()->isIntegerConstantExpr(BitWidth, Context)) {
-        llvm::APSInt ToSize(BitWidth.getBitWidth(), BitWidth.isUnsigned());
+          (BitWidth =
+               MemberDecl->getBitWidth()->getIntegerConstantExpr(Context))) {
+        llvm::APSInt ToSize(BitWidth->getBitWidth(), BitWidth->isUnsigned());
         ToSize = Context.getTypeSize(ToType);
 
         // Are we promoting to an int from a bitfield that fits in an int?
-        if (BitWidth < ToSize ||
-            (FromType->isSignedIntegerType() && BitWidth <= ToSize)) {
+        if (*BitWidth < ToSize ||
+            (FromType->isSignedIntegerType() && *BitWidth <= ToSize)) {
           return To->getKind() == BuiltinType::Int;
         }
 
         // Are we promoting to an unsigned int from an unsigned bitfield
         // that fits into an unsigned int?
-        if (FromType->isUnsignedIntegerType() && BitWidth <= ToSize) {
+        if (FromType->isUnsignedIntegerType() && *BitWidth <= ToSize) {
           return To->getKind() == BuiltinType::UInt;
         }
 
@@ -12795,8 +12797,9 @@ bool Sema::buildOverloadedCallSet(Scope *S, Expr *Fn,
       // then create a type dependent CallExpr. The goal is to postpone name
       // lookup to instantiation time to be able to search into type dependent
       // base classes.
-      CallExpr *CE = CallExpr::Create(Context, Fn, Args, Context.DependentTy,
-                                      VK_RValue, RParenLoc);
+      CallExpr *CE =
+          CallExpr::Create(Context, Fn, Args, Context.DependentTy, VK_RValue,
+                           RParenLoc, CurFPFeatureOverrides());
       CE->markDependentForPostponedNameLookup();
       *Result = CE;
       return true;
@@ -13976,9 +13979,9 @@ Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
         << (qualsString.find(' ') == std::string::npos ? 1 : 2);
     }
 
-    CXXMemberCallExpr *call =
-        CXXMemberCallExpr::Create(Context, MemExprE, Args, resultType,
-                                  valueKind, RParenLoc, proto->getNumParams());
+    CXXMemberCallExpr *call = CXXMemberCallExpr::Create(
+        Context, MemExprE, Args, resultType, valueKind, RParenLoc,
+        CurFPFeatureOverrides(), proto->getNumParams());
 
     if (CheckCallReturnType(proto->getReturnType(), op->getRHS()->getBeginLoc(),
                             call, nullptr))
@@ -13995,7 +13998,7 @@ Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
 
   if (isa<CXXPseudoDestructorExpr>(NakedMemExpr))
     return CallExpr::Create(Context, MemExprE, Args, Context.VoidTy, VK_RValue,
-                            RParenLoc);
+                            RParenLoc, CurFPFeatureOverrides());
 
   UnbridgedCastsSet UnbridgedCasts;
   if (checkArgPlaceholdersForOverload(*this, Args, UnbridgedCasts))
@@ -14133,9 +14136,9 @@ Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
 
   assert(Method && "Member call to something that isn't a method?");
   const auto *Proto = Method->getType()->castAs<FunctionProtoType>();
-  CXXMemberCallExpr *TheCall =
-      CXXMemberCallExpr::Create(Context, MemExprE, Args, ResultType, VK,
-                                RParenLoc, Proto->getNumParams());
+  CXXMemberCallExpr *TheCall = CXXMemberCallExpr::Create(
+      Context, MemExprE, Args, ResultType, VK, RParenLoc,
+      CurFPFeatureOverrides(), Proto->getNumParams());
 
   // Check for a valid return type.
   if (CheckCallReturnType(Method->getReturnType(), MemExpr->getMemberLoc(),
@@ -14678,7 +14681,7 @@ ExprResult Sema::BuildLiteralOperatorCall(LookupResult &R,
 
   UserDefinedLiteral *UDL = UserDefinedLiteral::Create(
       Context, Fn.get(), llvm::makeArrayRef(ConvArgs, Args.size()), ResultTy,
-      VK, LitEndLoc, UDSuffixLoc);
+      VK, LitEndLoc, UDSuffixLoc, CurFPFeatureOverrides());
 
   if (CheckCallReturnType(FD->getReturnType(), UDSuffixLoc, UDL, FD))
     return ExprError();
diff --git a/clang/lib/Sema/SemaSYCL.cpp b/clang/lib/Sema/SemaSYCL.cpp
index c176f6e639f51..ad5a3768846c7 100644
--- a/clang/lib/Sema/SemaSYCL.cpp
+++ b/clang/lib/Sema/SemaSYCL.cpp
@@ -314,11 +314,7 @@ static void reportConflictingAttrs(Sema &S, FunctionDecl *F, const Attr *A1,
 
 /// Returns the signed constant integer value represented by given expression
 static int64_t getIntExprValue(const Expr *E, ASTContext &Ctx) {
-  llvm::APSInt Val(32);
-  bool IsValid = E->isIntegerConstantExpr(Val, Ctx);
-  assert(IsValid && "expression must be constant integer");
-  (void)IsValid;
-  return Val.getSExtValue();
+  return E->getIntegerConstantExpr(Ctx)->getSExtValue();
 }
 
 class MarkDeviceFunction : public RecursiveASTVisitor<MarkDeviceFunction> {
@@ -354,8 +350,7 @@ class MarkDeviceFunction : public RecursiveASTVisitor<MarkDeviceFunction> {
       // all functions used by kernel have already been parsed and have
       // definitions.
       if (RecursiveSet.count(Callee) && !ConstexprDepth) {
-        SemaRef.Diag(e->getExprLoc(), diag::err_sycl_restrict)
-            << Sema::KernelCallRecursiveFunction;
+        SemaRef.Diag(e->getExprLoc(), diag::warn_sycl_restrict_recursion);
         SemaRef.Diag(Callee->getSourceRange().getBegin(),
                      diag::note_sycl_recursive_function_declared_here)
             << Sema::KernelCallRecursiveFunction;
@@ -515,52 +510,25 @@ class MarkDeviceFunction : public RecursiveASTVisitor<MarkDeviceFunction> {
 
       if (auto *A = FD->getAttr<IntelReqdSubGroupSizeAttr>())
         Attrs.insert(A);
+
       if (auto *A = FD->getAttr<ReqdWorkGroupSizeAttr>())
         Attrs.insert(A);
-      // Allow the following kernel attributes only on lambda functions and
-      // function objects that are called directly from a kernel (i.e. the one
-      // passed to the parallel_for function). For all other cases,
-      // emit a warning and ignore.
-      if (auto *A = FD->getAttr<SYCLIntelKernelArgsRestrictAttr>()) {
-        if (ParentFD == SYCLKernel) {
-          Attrs.insert(A);
-        } else {
-          SemaRef.Diag(A->getLocation(), diag::warn_attribute_ignored) << A;
-          FD->dropAttr<SYCLIntelKernelArgsRestrictAttr>();
-        }
-      }
-      if (auto *A = FD->getAttr<SYCLIntelNumSimdWorkItemsAttr>()) {
-        if (ParentFD == SYCLKernel) {
-          Attrs.insert(A);
-        } else {
-          SemaRef.Diag(A->getLocation(), diag::warn_attribute_ignored) << A;
-          FD->dropAttr<SYCLIntelNumSimdWorkItemsAttr>();
-        }
-      }
-      if (auto *A = FD->getAttr<SYCLIntelMaxWorkGroupSizeAttr>()) {
-        if (ParentFD == SYCLKernel) {
-          Attrs.insert(A);
-        } else {
-          SemaRef.Diag(A->getLocation(), diag::warn_attribute_ignored) << A;
-          FD->dropAttr<SYCLIntelMaxWorkGroupSizeAttr>();
-        }
-      }
-      if (auto *A = FD->getAttr<SYCLIntelMaxGlobalWorkDimAttr>()) {
-        if (ParentFD == SYCLKernel) {
-          Attrs.insert(A);
-        } else {
-          SemaRef.Diag(A->getLocation(), diag::warn_attribute_ignored) << A;
-          FD->dropAttr<SYCLIntelMaxGlobalWorkDimAttr>();
-        }
-      }
-      if (auto *A = FD->getAttr<SYCLIntelNoGlobalWorkOffsetAttr>()) {
-        if (ParentFD == SYCLKernel) {
-          Attrs.insert(A);
-        } else {
-          SemaRef.Diag(A->getLocation(), diag::warn_attribute_ignored) << A;
-          FD->dropAttr<SYCLIntelNoGlobalWorkOffsetAttr>();
-        }
-      }
+
+      if (auto *A = FD->getAttr<SYCLIntelKernelArgsRestrictAttr>())
+        Attrs.insert(A);
+
+      if (auto *A = FD->getAttr<SYCLIntelNumSimdWorkItemsAttr>())
+        Attrs.insert(A);
+
+      if (auto *A = FD->getAttr<SYCLIntelMaxWorkGroupSizeAttr>())
+        Attrs.insert(A);
+
+      if (auto *A = FD->getAttr<SYCLIntelMaxGlobalWorkDimAttr>())
+        Attrs.insert(A);
+
+      if (auto *A = FD->getAttr<SYCLIntelNoGlobalWorkOffsetAttr>())
+        Attrs.insert(A);
+
       if (auto *A = FD->getAttr<SYCLSimdAttr>())
         Attrs.insert(A);
       // Propagate the explicit SIMD attribute through call graph - it is used
@@ -729,7 +697,7 @@ getKernelInvocationKind(FunctionDecl *KernelCallerFunc) {
       .Default(InvokeUnknown);
 }
 
-static CXXRecordDecl *getKernelObjectType(FunctionDecl *Caller) {
+static const CXXRecordDecl *getKernelObjectType(FunctionDecl *Caller) {
   return (*Caller->param_begin())->getType()->getAsCXXRecordDecl();
 }
 
@@ -878,12 +846,12 @@ class KernelObjVisitor {
   }
 
   template <typename ParentTy, typename... Handlers>
-  void VisitRecord(CXXRecordDecl *Owner, ParentTy &Parent,
-                   CXXRecordDecl *Wrapper, Handlers &... handlers);
+  void VisitRecord(const CXXRecordDecl *Owner, ParentTy &Parent,
+                   const CXXRecordDecl *Wrapper, Handlers &... handlers);
 
   template <typename... Handlers>
-  void VisitRecordHelper(CXXRecordDecl *Owner,
-                         clang::CXXRecordDecl::base_class_range Range,
+  void VisitRecordHelper(const CXXRecordDecl *Owner,
+                         clang::CXXRecordDecl::base_class_const_range Range,
                          Handlers &... handlers) {
     for (const auto &Base : Range) {
       (void)std::initializer_list<int>{
@@ -906,15 +874,15 @@ class KernelObjVisitor {
   }
 
   template <typename... Handlers>
-  void VisitRecordHelper(CXXRecordDecl *Owner,
-                         clang::RecordDecl::field_range Range,
+  void VisitRecordHelper(const CXXRecordDecl *Owner,
+                         RecordDecl::field_range Range,
                          Handlers &... handlers) {
     VisitRecordFields(Owner, handlers...);
   }
 
   // FIXME: Can this be refactored/handled some other way?
   template <typename ParentTy, typename... Handlers>
-  void VisitStreamRecord(CXXRecordDecl *Owner, ParentTy &Parent,
+  void VisitStreamRecord(const CXXRecordDecl *Owner, ParentTy &Parent,
                          CXXRecordDecl *Wrapper, Handlers &... handlers) {
     (void)std::initializer_list<int>{
         (handlers.enterStruct(Owner, Parent), 0)...};
@@ -933,14 +901,15 @@ class KernelObjVisitor {
   }
 
   template <typename... Handlers>
-  void VisitRecordBases(CXXRecordDecl *KernelFunctor, Handlers &... handlers) {
+  void VisitRecordBases(const CXXRecordDecl *KernelFunctor,
+                        Handlers &... handlers) {
     VisitRecordHelper(KernelFunctor, KernelFunctor->bases(), handlers...);
   }
 
   // A visitor function that dispatches to functions as defined in
   // SyclKernelFieldHandler for the purposes of kernel generation.
   template <typename... Handlers>
-  void VisitRecordFields(CXXRecordDecl *Owner, Handlers &... handlers) {
+  void VisitRecordFields(const CXXRecordDecl *Owner, Handlers &... handlers) {
 
     for (const auto Field : Owner->fields()) {
       (void)std::initializer_list<int>{
@@ -987,8 +956,8 @@ class KernelObjVisitor {
 // type (which doesn't exist in cases where it is a FieldDecl in the
 // 'root'), and Wrapper is the current struct being unwrapped.
 template <typename ParentTy, typename... Handlers>
-void KernelObjVisitor::VisitRecord(CXXRecordDecl *Owner, ParentTy &Parent,
-                                   CXXRecordDecl *Wrapper,
+void KernelObjVisitor::VisitRecord(const CXXRecordDecl *Owner, ParentTy &Parent,
+                                   const CXXRecordDecl *Wrapper,
                                    Handlers &... handlers) {
   (void)std::initializer_list<int>{(handlers.enterStruct(Owner, Parent), 0)...};
   VisitRecordHelper(Wrapper, Wrapper->bases(), handlers...);
@@ -1357,7 +1326,7 @@ class SyclKernelBodyCreator : public SyclKernelFieldHandler {
   llvm::SmallVector<Expr *, 16> InitExprs;
   VarDecl *KernelObjClone;
   InitializedEntity VarEntity;
-  CXXRecordDecl *KernelObj;
+  const CXXRecordDecl *KernelObj;
   llvm::SmallVector<Expr *, 16> MemberExprBases;
   uint64_t ArrayIndex;
   FunctionDecl *KernelCallerFunc;
@@ -1514,7 +1483,8 @@ class SyclKernelBodyCreator : public SyclKernelFieldHandler {
     // [kernel_obj or wrapper object].accessor.__init(_ValueType*,
     // range<int>, range<int>, id<int>)
     CXXMemberCallExpr *Call = CXXMemberCallExpr::Create(
-        SemaRef.Context, MethodME, ParamStmts, ResultTy, VK, SourceLocation());
+        SemaRef.Context, MethodME, ParamStmts, ResultTy, VK, SourceLocation(),
+        FPOptionsOverride());
     if (MethodName == FinalizeMethodName)
       FinalizeStmts.push_back(Call);
     else
@@ -1524,7 +1494,7 @@ class SyclKernelBodyCreator : public SyclKernelFieldHandler {
   // FIXME Avoid creation of kernel obj clone.
   // See https://github.com/intel/llvm/issues/1544 for details.
   static VarDecl *createKernelObjClone(ASTContext &Ctx, DeclContext *DC,
-                                       CXXRecordDecl *KernelObj) {
+                                       const CXXRecordDecl *KernelObj) {
     TypeSourceInfo *TSInfo =
         KernelObj->isLambda() ? KernelObj->getLambdaTypeInfo() : nullptr;
     VarDecl *VD = VarDecl::Create(
@@ -1572,7 +1542,7 @@ class SyclKernelBodyCreator : public SyclKernelFieldHandler {
 
 public:
   SyclKernelBodyCreator(Sema &S, SyclKernelDeclCreator &DC,
-                        CXXRecordDecl *KernelObj,
+                        const CXXRecordDecl *KernelObj,
                         FunctionDecl *KernelCallerFunc)
       : SyclKernelFieldHandler(S), DeclCreator(DC),
         KernelObjClone(createKernelObjClone(S.getASTContext(),
@@ -1934,7 +1904,7 @@ class SyclKernelIntHeaderCreator : public SyclKernelFieldHandler {
 void Sema::ConstructOpenCLKernel(FunctionDecl *KernelCallerFunc,
                                  MangleContext &MC) {
   // The first argument to the KernelCallerFunc is the lambda object.
-  CXXRecordDecl *KernelObj = getKernelObjectType(KernelCallerFunc);
+  const CXXRecordDecl *KernelObj = getKernelObjectType(KernelCallerFunc);
   assert(KernelObj && "invalid kernel caller");
 
   // Calculate both names, since Integration headers need both.
@@ -1943,6 +1913,11 @@ void Sema::ConstructOpenCLKernel(FunctionDecl *KernelCallerFunc,
       constructKernelName(*this, KernelCallerFunc, MC);
   StringRef KernelName(getLangOpts().SYCLUnnamedLambda ? StableName
                                                        : CalculatedName);
+  if (KernelObj->isLambda()) {
+    for (const LambdaCapture &LC : KernelObj->captures())
+      if (LC.capturesThis() && LC.isImplicit())
+        Diag(LC.getLocation(), diag::err_implicit_this_capture);
+  }
   SyclKernelFieldChecker checker(*this);
   SyclKernelDeclCreator kernel_decl(
       *this, checker, KernelName, KernelObj->getLocation(),
@@ -1963,6 +1938,27 @@ void Sema::ConstructOpenCLKernel(FunctionDecl *KernelCallerFunc,
   ConstructingOpenCLKernel = false;
 }
 
+// This function marks all the callees of explicit SIMD kernel
+// with !sycl_explicit_simd. We want to have different semantics
+// for functions that are called from SYCL and E-SIMD contexts.
+// Later, functions marked with !sycl_explicit_simd will be cloned
+// to maintain two different semantics.
+void Sema::MarkSyclSimd() {
+  for (Decl *D : syclDeviceDecls())
+    if (auto SYCLKernel = dyn_cast<FunctionDecl>(D))
+      if (SYCLKernel->hasAttr<SYCLSimdAttr>()) {
+        MarkDeviceFunction Marker(*this);
+        Marker.SYCLCG.addToCallGraph(getASTContext().getTranslationUnitDecl());
+        llvm::SmallPtrSet<FunctionDecl *, 10> VisitedSet;
+        Marker.CollectKernelSet(SYCLKernel, SYCLKernel, VisitedSet);
+        for (const auto &elt : Marker.KernelSet) {
+          if (FunctionDecl *Def = elt->getDefinition())
+            if (!Def->hasAttr<SYCLSimdAttr>())
+              Def->addAttr(SYCLSimdAttr::CreateImplicit(getASTContext()));
+        }
+      }
+}
+
 void Sema::MarkDevice(void) {
   // Create the call graph so we can detect recursion and check the validity
   // of new operator overrides. Add the kernel function itself in case
@@ -1973,7 +1969,8 @@ void Sema::MarkDevice(void) {
   // Iterate through SYCL_EXTERNAL functions and add them to the device decls.
   for (const auto &entry : *Marker.SYCLCG.getRoot()) {
     if (auto *FD = dyn_cast<FunctionDecl>(entry.Callee->getDecl())) {
-      if (FD->hasAttr<SYCLDeviceAttr>() && !FD->hasAttr<SYCLKernelAttr>())
+      if (FD->hasAttr<SYCLDeviceAttr>() && !FD->hasAttr<SYCLKernelAttr>() &&
+          FD->hasBody())
         addSyclDeviceDecl(FD);
     }
   }
@@ -2027,6 +2024,38 @@ void Sema::MarkDevice(void) {
               Diag(Attr->getLocation(), diag::note_conflicting_attribute);
               SYCLKernel->setInvalidDecl();
             }
+          } else if (auto *Existing =
+                         SYCLKernel->getAttr<SYCLIntelMaxWorkGroupSizeAttr>()) {
+            if (Existing->getXDim() < Attr->getXDim() ||
+                Existing->getYDim() < Attr->getYDim() ||
+                Existing->getZDim() < Attr->getZDim()) {
+              Diag(SYCLKernel->getLocation(),
+                   diag::err_conflicting_sycl_kernel_attributes);
+              Diag(Existing->getLocation(), diag::note_conflicting_attribute);
+              Diag(Attr->getLocation(), diag::note_conflicting_attribute);
+              SYCLKernel->setInvalidDecl();
+            } else {
+              SYCLKernel->addAttr(A);
+            }
+          } else {
+            SYCLKernel->addAttr(A);
+          }
+          break;
+        }
+        case attr::Kind::SYCLIntelMaxWorkGroupSize: {
+          auto *Attr = cast<SYCLIntelMaxWorkGroupSizeAttr>(A);
+          if (auto *Existing = SYCLKernel->getAttr<ReqdWorkGroupSizeAttr>()) {
+            if (Existing->getXDim() > Attr->getXDim() ||
+                Existing->getYDim() > Attr->getYDim() ||
+                Existing->getZDim() > Attr->getZDim()) {
+              Diag(SYCLKernel->getLocation(),
+                   diag::err_conflicting_sycl_kernel_attributes);
+              Diag(Existing->getLocation(), diag::note_conflicting_attribute);
+              Diag(Attr->getLocation(), diag::note_conflicting_attribute);
+              SYCLKernel->setInvalidDecl();
+            } else {
+              SYCLKernel->addAttr(A);
+            }
           } else {
             SYCLKernel->addAttr(A);
           }
@@ -2035,10 +2064,9 @@ void Sema::MarkDevice(void) {
         case attr::Kind::SYCLIntelKernelArgsRestrict:
         case attr::Kind::SYCLIntelNumSimdWorkItems:
         case attr::Kind::SYCLIntelMaxGlobalWorkDim:
-        case attr::Kind::SYCLIntelMaxWorkGroupSize:
         case attr::Kind::SYCLIntelNoGlobalWorkOffset:
         case attr::Kind::SYCLSimd: {
-          if ((A->getKind() == attr::Kind::SYCLSimd) &&
+          if ((A->getKind() == attr::Kind::SYCLSimd) && KernelBody &&
               !KernelBody->getAttr<SYCLSimdAttr>()) {
             // Usual kernel can't call ESIMD functions.
             Diag(KernelBody->getLocation(),
diff --git a/clang/lib/Sema/SemaStmtAttr.cpp b/clang/lib/Sema/SemaStmtAttr.cpp
index c4f33bd8ffd90..a0b6550136f92 100644
--- a/clang/lib/Sema/SemaStmtAttr.cpp
+++ b/clang/lib/Sema/SemaStmtAttr.cpp
@@ -139,12 +139,11 @@ static IVDepExprResult HandleFPGAIVDepAttrExpr(Sema &S, Expr *E,
   if (E->isInstantiationDependent())
     return IVDepExprResult::Dependent;
 
-  llvm::APSInt ArgVal;
-
-  if (E->isIntegerConstantExpr(ArgVal, S.getASTContext())) {
-    if (checkSYCLIntelFPGAIVDepSafeLen(S, ArgVal, E))
+  Optional<llvm::APSInt> ArgVal = E->getIntegerConstantExpr(S.getASTContext());
+  if (ArgVal) {
+    if (checkSYCLIntelFPGAIVDepSafeLen(S, *ArgVal, E))
       return IVDepExprResult::Invalid;
-    SafelenValue = ArgVal.getZExtValue();
+    SafelenValue = ArgVal->getZExtValue();
     return IVDepExprResult::SafeLen;
   }
 
@@ -387,6 +386,7 @@ class CallExprFinder : public ConstEvaluatedExprVisitor<CallExprFinder> {
   bool foundCallExpr() { return FoundCallExpr; }
 
   void VisitCallExpr(const CallExpr *E) { FoundCallExpr = true; }
+  void VisitAsmStmt(const AsmStmt *S) { FoundCallExpr = true; }
 
   void Visit(const Stmt *St) {
     if (!St)
@@ -624,20 +624,19 @@ static bool CheckLoopUnrollAttrExpr(Sema &S, Expr *E,
                                     const AttributeCommonInfo &A,
                                     unsigned *UnrollFactor = nullptr) {
   if (E && !E->isInstantiationDependent()) {
-    llvm::APSInt ArgVal(32);
-
-    if (!E->isIntegerConstantExpr(ArgVal, S.Context))
+    Optional<llvm::APSInt> ArgVal = E->getIntegerConstantExpr(S.Context);
+    if (!ArgVal)
       return S.Diag(E->getExprLoc(), diag::err_attribute_argument_type)
              << A.getAttrName() << AANT_ArgumentIntegerConstant
              << E->getSourceRange();
 
-    if (ArgVal.isNonPositive())
+    if (ArgVal->isNonPositive())
       return S.Diag(E->getExprLoc(),
                     diag::err_attribute_requires_positive_integer)
              << A.getAttrName() << /* positive */ 0;
 
     if (UnrollFactor)
-      *UnrollFactor = ArgVal.getZExtValue();
+      *UnrollFactor = ArgVal->getZExtValue();
   }
   return false;
 }
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 5392be57a3aa2..52062e9a5039c 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -2100,24 +2100,23 @@ DeduceTemplateArgumentsByTypeMatch(Sema &S,
             const auto *ArgConstMatrix = dyn_cast<ConstantMatrixType>(Arg);
             const auto *ArgDepMatrix = dyn_cast<DependentSizedMatrixType>(Arg);
             if (!ParamExpr->isValueDependent()) {
-              llvm::APSInt ParamConst(
-                  S.Context.getTypeSize(S.Context.getSizeType()));
-              if (!ParamExpr->isIntegerConstantExpr(ParamConst, S.Context))
+              Optional<llvm::APSInt> ParamConst =
+                  ParamExpr->getIntegerConstantExpr(S.Context);
+              if (!ParamConst)
                 return Sema::TDK_NonDeducedMismatch;
 
               if (ArgConstMatrix) {
-                if ((ArgConstMatrix->*GetArgDimension)() == ParamConst)
+                if ((ArgConstMatrix->*GetArgDimension)() == *ParamConst)
                   return Sema::TDK_Success;
                 return Sema::TDK_NonDeducedMismatch;
               }
 
               Expr *ArgExpr = (ArgDepMatrix->*GetArgDimensionExpr)();
-              llvm::APSInt ArgConst(
-                  S.Context.getTypeSize(S.Context.getSizeType()));
-              if (!ArgExpr->isValueDependent() &&
-                  ArgExpr->isIntegerConstantExpr(ArgConst, S.Context) &&
-                  ArgConst == ParamConst)
-                return Sema::TDK_Success;
+              if (!ArgExpr->isValueDependent())
+                if (Optional<llvm::APSInt> ArgConst =
+                        ArgExpr->getIntegerConstantExpr(S.Context))
+                  if (*ArgConst == *ParamConst)
+                    return Sema::TDK_Success;
               return Sema::TDK_NonDeducedMismatch;
             }
 
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 7fdebc5045eca..9ce5ec66dd814 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -37,6 +37,7 @@
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/Support/ErrorHandling.h"
+#include <bitset>
 
 using namespace clang;
 
@@ -2312,7 +2313,7 @@ QualType Sema::BuildArrayType(QualType T, ArrayType::ArraySizeModifier ASM,
       return QualType();
   }
 
-  if (T->isSizelessType()) {
+  if (T->isSizelessType() && !T->isVLST()) {
     Diag(Loc, diag::err_array_incomplete_or_sizeless_type) << 1 << T;
     return QualType();
   }
@@ -2487,8 +2488,8 @@ QualType Sema::BuildVectorType(QualType CurType, Expr *SizeExpr,
     return Context.getDependentVectorType(CurType, SizeExpr, AttrLoc,
                                                VectorType::GenericVector);
 
-  llvm::APSInt VecSize(32);
-  if (!SizeExpr->isIntegerConstantExpr(VecSize, Context)) {
+  Optional<llvm::APSInt> VecSize = SizeExpr->getIntegerConstantExpr(Context);
+  if (!VecSize) {
     Diag(AttrLoc, diag::err_attribute_argument_type)
         << "vector_size" << AANT_ArgumentIntegerConstant
         << SizeExpr->getSourceRange();
@@ -2500,13 +2501,13 @@ QualType Sema::BuildVectorType(QualType CurType, Expr *SizeExpr,
                                                VectorType::GenericVector);
 
   // vecSize is specified in bytes - convert to bits.
-  if (!VecSize.isIntN(61)) {
+  if (!VecSize->isIntN(61)) {
     // Bit size will overflow uint64.
     Diag(AttrLoc, diag::err_attribute_size_too_large)
         << SizeExpr->getSourceRange() << "vector";
     return QualType();
   }
-  uint64_t VectorSizeBits = VecSize.getZExtValue() * 8;
+  uint64_t VectorSizeBits = VecSize->getZExtValue() * 8;
   unsigned TypeSize = static_cast<unsigned>(Context.getTypeSize(CurType));
 
   if (VectorSizeBits == 0) {
@@ -2551,22 +2552,22 @@ QualType Sema::BuildExtVectorType(QualType T, Expr *ArraySize,
   }
 
   if (!ArraySize->isTypeDependent() && !ArraySize->isValueDependent()) {
-    llvm::APSInt vecSize(32);
-    if (!ArraySize->isIntegerConstantExpr(vecSize, Context)) {
+    Optional<llvm::APSInt> vecSize = ArraySize->getIntegerConstantExpr(Context);
+    if (!vecSize) {
       Diag(AttrLoc, diag::err_attribute_argument_type)
         << "ext_vector_type" << AANT_ArgumentIntegerConstant
         << ArraySize->getSourceRange();
       return QualType();
     }
 
-    if (!vecSize.isIntN(32)) {
+    if (!vecSize->isIntN(32)) {
       Diag(AttrLoc, diag::err_attribute_size_too_large)
           << ArraySize->getSourceRange() << "vector";
       return QualType();
     }
     // Unlike gcc's vector_size attribute, the size is specified as the
     // number of elements, not the number of bytes.
-    unsigned vectorSize = static_cast<unsigned>(vecSize.getZExtValue());
+    unsigned vectorSize = static_cast<unsigned>(vecSize->getZExtValue());
 
     if (vectorSize == 0) {
       Diag(AttrLoc, diag::err_attribute_zero_size)
@@ -2597,18 +2598,15 @@ QualType Sema::BuildMatrixType(QualType ElementTy, Expr *NumRows, Expr *NumCols,
     return Context.getDependentSizedMatrixType(ElementTy, NumRows, NumCols,
                                                AttrLoc);
 
-  // Both row and column values can only be 20 bit wide currently.
-  llvm::APSInt ValueRows(32), ValueColumns(32);
-
-  bool const RowsIsInteger = NumRows->isIntegerConstantExpr(ValueRows, Context);
-  bool const ColumnsIsInteger =
-      NumCols->isIntegerConstantExpr(ValueColumns, Context);
+  Optional<llvm::APSInt> ValueRows = NumRows->getIntegerConstantExpr(Context);
+  Optional<llvm::APSInt> ValueColumns =
+      NumCols->getIntegerConstantExpr(Context);
 
   auto const RowRange = NumRows->getSourceRange();
   auto const ColRange = NumCols->getSourceRange();
 
   // Both are row and column expressions are invalid.
-  if (!RowsIsInteger && !ColumnsIsInteger) {
+  if (!ValueRows && !ValueColumns) {
     Diag(AttrLoc, diag::err_attribute_argument_type)
         << "matrix_type" << AANT_ArgumentIntegerConstant << RowRange
         << ColRange;
@@ -2616,22 +2614,22 @@ QualType Sema::BuildMatrixType(QualType ElementTy, Expr *NumRows, Expr *NumCols,
   }
 
   // Only the row expression is invalid.
-  if (!RowsIsInteger) {
+  if (!ValueRows) {
     Diag(AttrLoc, diag::err_attribute_argument_type)
         << "matrix_type" << AANT_ArgumentIntegerConstant << RowRange;
     return QualType();
   }
 
   // Only the column expression is invalid.
-  if (!ColumnsIsInteger) {
+  if (!ValueColumns) {
     Diag(AttrLoc, diag::err_attribute_argument_type)
         << "matrix_type" << AANT_ArgumentIntegerConstant << ColRange;
     return QualType();
   }
 
   // Check the matrix dimensions.
-  unsigned MatrixRows = static_cast<unsigned>(ValueRows.getZExtValue());
-  unsigned MatrixColumns = static_cast<unsigned>(ValueColumns.getZExtValue());
+  unsigned MatrixRows = static_cast<unsigned>(ValueRows->getZExtValue());
+  unsigned MatrixColumns = static_cast<unsigned>(ValueColumns->getZExtValue());
   if (MatrixRows == 0 && MatrixColumns == 0) {
     Diag(AttrLoc, diag::err_attribute_zero_size)
         << "matrix" << RowRange << ColRange;
@@ -6276,13 +6274,15 @@ static bool BuildAddressSpaceIndex(Sema &S, LangAS &ASIdx,
                                    const Expr *AddrSpace,
                                    SourceLocation AttrLoc) {
   if (!AddrSpace->isValueDependent()) {
-    llvm::APSInt addrSpace(32);
-    if (!AddrSpace->isIntegerConstantExpr(addrSpace, S.Context)) {
+    Optional<llvm::APSInt> OptAddrSpace =
+        AddrSpace->getIntegerConstantExpr(S.Context);
+    if (!OptAddrSpace) {
       S.Diag(AttrLoc, diag::err_attribute_argument_type)
           << "'address_space'" << AANT_ArgumentIntegerConstant
           << AddrSpace->getSourceRange();
       return false;
     }
+    llvm::APSInt &addrSpace = *OptAddrSpace;
 
     // Bounds checking.
     if (addrSpace.isSigned()) {
@@ -6940,32 +6940,32 @@ static bool handleMSPointerTypeQualifierAttr(TypeProcessingState &State,
     break;
   }
 
-  llvm::SmallSet<attr::Kind, 2> Attrs;
+  std::bitset<attr::LastAttr> Attrs;
   attr::Kind NewAttrKind = A->getKind();
   QualType Desugared = Type;
   const AttributedType *AT = dyn_cast<AttributedType>(Type);
   while (AT) {
-    Attrs.insert(AT->getAttrKind());
+    Attrs[AT->getAttrKind()] = true;
     Desugared = AT->getModifiedType();
     AT = dyn_cast<AttributedType>(Desugared);
   }
 
   // You cannot specify duplicate type attributes, so if the attribute has
   // already been applied, flag it.
-  if (Attrs.count(NewAttrKind)) {
+  if (Attrs[NewAttrKind]) {
     S.Diag(PAttr.getLoc(), diag::warn_duplicate_attribute_exact) << PAttr;
     return true;
   }
-  Attrs.insert(NewAttrKind);
+  Attrs[NewAttrKind] = true;
 
   // You cannot have both __sptr and __uptr on the same type, nor can you
   // have __ptr32 and __ptr64.
-  if (Attrs.count(attr::Ptr32) && Attrs.count(attr::Ptr64)) {
+  if (Attrs[attr::Ptr32] && Attrs[attr::Ptr64]) {
     S.Diag(PAttr.getLoc(), diag::err_attributes_are_not_compatible)
         << "'__ptr32'"
         << "'__ptr64'";
     return true;
-  } else if (Attrs.count(attr::SPtr) && Attrs.count(attr::UPtr)) {
+  } else if (Attrs[attr::SPtr] && Attrs[attr::UPtr]) {
     S.Diag(PAttr.getLoc(), diag::err_attributes_are_not_compatible)
         << "'__sptr'"
         << "'__uptr'";
@@ -6991,12 +6991,12 @@ static bool handleMSPointerTypeQualifierAttr(TypeProcessingState &State,
   LangAS ASIdx = LangAS::Default;
   uint64_t PtrWidth = S.Context.getTargetInfo().getPointerWidth(0);
   if (PtrWidth == 32) {
-    if (Attrs.count(attr::Ptr64))
+    if (Attrs[attr::Ptr64])
       ASIdx = LangAS::ptr64;
-    else if (Attrs.count(attr::UPtr))
+    else if (Attrs[attr::UPtr])
       ASIdx = LangAS::ptr32_uptr;
-  } else if (PtrWidth == 64 && Attrs.count(attr::Ptr32)) {
-    if (Attrs.count(attr::UPtr))
+  } else if (PtrWidth == 64 && Attrs[attr::Ptr32]) {
+    if (Attrs[attr::UPtr])
       ASIdx = LangAS::ptr32_uptr;
     else
       ASIdx = LangAS::ptr32_sptr;
@@ -7772,14 +7772,17 @@ static bool isPermittedNeonBaseType(QualType &Ty,
 static bool verifyValidIntegerConstantExpr(Sema &S, const ParsedAttr &Attr,
                                            llvm::APSInt &Result) {
   const auto *AttrExpr = Attr.getArgAsExpr(0);
-  if (AttrExpr->isTypeDependent() || AttrExpr->isValueDependent() ||
-      !AttrExpr->isIntegerConstantExpr(Result, S.Context)) {
-    S.Diag(Attr.getLoc(), diag::err_attribute_argument_type)
-        << Attr << AANT_ArgumentIntegerConstant << AttrExpr->getSourceRange();
-    Attr.setInvalid();
-    return false;
+  if (!AttrExpr->isTypeDependent() && !AttrExpr->isValueDependent()) {
+    if (Optional<llvm::APSInt> Res =
+            AttrExpr->getIntegerConstantExpr(S.Context)) {
+      Result = *Res;
+      return true;
+    }
   }
-  return true;
+  S.Diag(Attr.getLoc(), diag::err_attribute_argument_type)
+      << Attr << AANT_ArgumentIntegerConstant << AttrExpr->getSourceRange();
+  Attr.setInvalid();
+  return false;
 }
 
 /// HandleNeonVectorTypeAttr - The "neon_vector_type" and
@@ -7834,10 +7837,14 @@ static void HandleNeonVectorTypeAttr(QualType &CurType, const ParsedAttr &Attr,
 /// HandleArmSveVectorBitsTypeAttr - The "arm_sve_vector_bits" attribute is
 /// used to create fixed-length versions of sizeless SVE types defined by
 /// the ACLE, such as svint32_t and svbool_t.
-static void HandleArmSveVectorBitsTypeAttr(QualType &CurType,
-                                           const ParsedAttr &Attr, Sema &S) {
+static void HandleArmSveVectorBitsTypeAttr(TypeProcessingState &State,
+                                           QualType &CurType,
+                                           ParsedAttr &Attr) {
+  Sema &S = State.getSema();
+  ASTContext &Ctx = S.Context;
+
   // Target must have SVE.
-  if (!S.Context.getTargetInfo().hasFeature("sve")) {
+  if (!Ctx.getTargetInfo().hasFeature("sve")) {
     S.Diag(Attr.getLoc(), diag::err_attribute_unsupported) << Attr;
     Attr.setInvalid();
     return;
@@ -7881,6 +7888,9 @@ static void HandleArmSveVectorBitsTypeAttr(QualType &CurType,
     Attr.setInvalid();
     return;
   }
+
+  auto *A = ::new (Ctx) ArmSveVectorBitsAttr(Ctx, Attr, VecSize);
+  CurType = State.getAttributedType(A, CurType, CurType);
 }
 
 static void HandleArmMveStrictPolymorphismAttr(TypeProcessingState &State,
@@ -8152,7 +8162,7 @@ static void processTypeAttrs(TypeProcessingState &state, QualType &type,
       attr.setUsedAsTypeAttr();
       break;
     case ParsedAttr::AT_ArmSveVectorBits:
-      HandleArmSveVectorBitsTypeAttr(type, attr, state.getSema());
+      HandleArmSveVectorBitsTypeAttr(state, type, attr);
       attr.setUsedAsTypeAttr();
       break;
     case ParsedAttr::AT_ArmMveStrictPolymorphism: {
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 7627868ea0cb8..641ae24de2c7f 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -3504,7 +3504,8 @@ class TreeTransform {
     // Build the CallExpr
     ExprResult TheCall = CallExpr::Create(
         SemaRef.Context, Callee, SubExprs, Builtin->getCallResultType(),
-        Expr::getValueKindForType(Builtin->getReturnType()), RParenLoc);
+        Expr::getValueKindForType(Builtin->getReturnType()), RParenLoc,
+        FPOptionsOverride());
 
     // Type-check the __builtin_shufflevector expression.
     return SemaRef.SemaBuiltinShuffleVector(cast<CallExpr>(TheCall.get()));
@@ -10473,6 +10474,15 @@ TreeTransform<Derived>::TransformCallExpr(CallExpr *E) {
   // FIXME: Wrong source location information for the '('.
   SourceLocation FakeLParenLoc
     = ((Expr *)Callee.get())->getSourceRange().getBegin();
+
+  Sema::FPFeaturesStateRAII FPFeaturesState(getSema());
+  if (E->hasStoredFPFeatures()) {
+    FPOptionsOverride NewOverrides = E->getFPFeatures();
+    getSema().CurFPFeatures =
+        NewOverrides.applyOverrides(getSema().getLangOpts());
+    getSema().FpPragmaStack.CurrentValue = NewOverrides.getAsOpaqueInt();
+  }
+
   return getDerived().RebuildCallExpr(Callee.get(), FakeLParenLoc,
                                       Args,
                                       E->getRParenLoc());
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 7f91806392a34..9f8d724e237ec 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -2747,7 +2747,7 @@ ASTReader::ReadControlBlock(ModuleFile &F,
         return VersionMismatch;
       }
 
-      bool hasErrors = Record[7];
+      bool hasErrors = Record[6];
       if (hasErrors && !DisableValidation && !AllowASTWithCompilerErrors) {
         Diag(diag::err_pch_with_compiler_errors);
         return HadErrors;
@@ -2765,8 +2765,6 @@ ASTReader::ReadControlBlock(ModuleFile &F,
 
       F.HasTimestamps = Record[5];
 
-      F.PCHHasObjectFile = Record[6];
-
       const std::string &CurBranch = getClangFullRepositoryVersion();
       StringRef ASTBranch = Blob;
       if (StringRef(CurBranch) != ASTBranch && !DisableValidation) {
@@ -8597,11 +8595,6 @@ Module *ASTReader::getModule(unsigned ID) {
   return getSubmodule(ID);
 }
 
-bool ASTReader::DeclIsFromPCHWithObjectFile(const Decl *D) {
-  ModuleFile *MF = getOwningModuleFile(D);
-  return MF && MF->PCHHasObjectFile;
-}
-
 ModuleFile *ASTReader::getLocalModuleFile(ModuleFile &F, unsigned ID) {
   if (ID & 1) {
     // It's a module, look it up by submodule ID.
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index eef4ab16ec15e..0b87161ddeea3 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -504,10 +504,9 @@ uint64_t ASTDeclReader::GetCurrentCursorOffset() {
 
 void ASTDeclReader::ReadFunctionDefinition(FunctionDecl *FD) {
   if (Record.readInt()) {
-    Reader.DefinitionSource[FD] = Loc.F->Kind == ModuleKind::MK_MainFile;
-    if (Reader.getContext().getLangOpts().BuildingPCHWithObjectFile &&
-        Reader.DeclIsFromPCHWithObjectFile(FD))
-      Reader.DefinitionSource[FD] = true;
+    Reader.DefinitionSource[FD] =
+        Loc.F->Kind == ModuleKind::MK_MainFile ||
+        Reader.getContext().getLangOpts().BuildingPCHWithObjectFile;
   }
   if (auto *CD = dyn_cast<CXXConstructorDecl>(FD)) {
     CD->setNumCtorInitializers(Record.readInt());
@@ -1436,10 +1435,9 @@ ASTDeclReader::RedeclarableResult ASTDeclReader::VisitVarDeclImpl(VarDecl *VD) {
   }
 
   if (VD->getStorageDuration() == SD_Static && Record.readInt()) {
-    Reader.DefinitionSource[VD] = Loc.F->Kind == ModuleKind::MK_MainFile;
-    if (Reader.getContext().getLangOpts().BuildingPCHWithObjectFile &&
-        Reader.DeclIsFromPCHWithObjectFile(VD))
-      Reader.DefinitionSource[VD] = true;
+    Reader.DefinitionSource[VD] =
+        Loc.F->Kind == ModuleKind::MK_MainFile ||
+        Reader.getContext().getLangOpts().BuildingPCHWithObjectFile;
   }
 
   enum VarKind {
@@ -1700,10 +1698,9 @@ void ASTDeclReader::ReadCXXDefinitionData(
   Data.HasODRHash = true;
 
   if (Record.readInt()) {
-    Reader.DefinitionSource[D] = Loc.F->Kind == ModuleKind::MK_MainFile;
-    if (Reader.getContext().getLangOpts().BuildingPCHWithObjectFile &&
-        Reader.DeclIsFromPCHWithObjectFile(D))
-      Reader.DefinitionSource[D] = true;
+    Reader.DefinitionSource[D] = 
+        Loc.F->Kind == ModuleKind::MK_MainFile ||
+        Reader.getContext().getLangOpts().BuildingPCHWithObjectFile;
   }
 
   Data.NumBases = Record.readInt();
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index a40c5499a6d77..8a7beed229c87 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -991,12 +991,15 @@ void ASTStmtReader::VisitOMPIteratorExpr(OMPIteratorExpr *E) {
 void ASTStmtReader::VisitCallExpr(CallExpr *E) {
   VisitExpr(E);
   unsigned NumArgs = Record.readInt();
+  bool HasFPFeatures = Record.readInt();
   assert((NumArgs == E->getNumArgs()) && "Wrong NumArgs!");
   E->setRParenLoc(readSourceLocation());
   E->setCallee(Record.readSubExpr());
   for (unsigned I = 0; I != NumArgs; ++I)
     E->setArg(I, Record.readSubExpr());
   E->setADLCallKind(static_cast<CallExpr::ADLCallKind>(Record.readInt()));
+  if (HasFPFeatures)
+    E->setStoredFPFeatures(FPOptionsOverride(Record.readInt()));
 }
 
 void ASTStmtReader::VisitCXXMemberCallExpr(CXXMemberCallExpr *E) {
@@ -1662,7 +1665,6 @@ void ASTStmtReader::VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
   VisitCallExpr(E);
   E->CXXOperatorCallExprBits.OperatorKind = Record.readInt();
   E->Range = Record.readSourceRange();
-  E->setFPFeatures(FPOptionsOverride(Record.readInt()));
 }
 
 void ASTStmtReader::VisitCXXRewrittenBinaryOperator(
@@ -2977,7 +2979,8 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) {
 
     case EXPR_CALL:
       S = CallExpr::CreateEmpty(
-          Context, /*NumArgs=*/Record[ASTStmtReader::NumExprFields], Empty);
+          Context, /*NumArgs=*/Record[ASTStmtReader::NumExprFields],
+          /*HasFPFeatures=*/Record[ASTStmtReader::NumExprFields + 1], Empty);
       break;
 
     case EXPR_RECOVERY:
@@ -3585,12 +3588,14 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) {
 
     case EXPR_CXX_OPERATOR_CALL:
       S = CXXOperatorCallExpr::CreateEmpty(
-          Context, /*NumArgs=*/Record[ASTStmtReader::NumExprFields], Empty);
+          Context, /*NumArgs=*/Record[ASTStmtReader::NumExprFields],
+          /*HasFPFeatures=*/Record[ASTStmtReader::NumExprFields + 1], Empty);
       break;
 
     case EXPR_CXX_MEMBER_CALL:
       S = CXXMemberCallExpr::CreateEmpty(
-          Context, /*NumArgs=*/Record[ASTStmtReader::NumExprFields], Empty);
+          Context, /*NumArgs=*/Record[ASTStmtReader::NumExprFields],
+          /*HasFPFeatures=*/Record[ASTStmtReader::NumExprFields + 1], Empty);
       break;
 
     case EXPR_CXX_REWRITTEN_BINARY_OPERATOR:
@@ -3648,7 +3653,8 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) {
 
     case EXPR_USER_DEFINED_LITERAL:
       S = UserDefinedLiteral::CreateEmpty(
-          Context, /*NumArgs=*/Record[ASTStmtReader::NumExprFields], Empty);
+          Context, /*NumArgs=*/Record[ASTStmtReader::NumExprFields],
+          /*HasFPFeatures=*/Record[ASTStmtReader::NumExprFields + 1], Empty);
       break;
 
     case EXPR_CXX_STD_INITIALIZER_LIST:
@@ -3831,7 +3837,8 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) {
 
     case EXPR_CUDA_KERNEL_CALL:
       S = CUDAKernelCallExpr::CreateEmpty(
-          Context, /*NumArgs=*/Record[ASTStmtReader::NumExprFields], Empty);
+          Context, /*NumArgs=*/Record[ASTStmtReader::NumExprFields],
+          /*HasFPFeatures=*/Record[ASTStmtReader::NumExprFields + 1], Empty);
       break;
 
     case EXPR_ASTYPE:
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 2345a12caeb2e..9ea90b2a0212a 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -81,7 +81,7 @@
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
-#include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
@@ -1119,7 +1119,6 @@ void ASTWriter::WriteControlBlock(Preprocessor &PP, ASTContext &Context,
   MetadataAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 16)); // Clang min.
   MetadataAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // Relocatable
   MetadataAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // Timestamps
-  MetadataAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // PCHHasObjectFile
   MetadataAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // Errors
   MetadataAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // SVN branch/tag
   unsigned MetadataAbbrevCode = Stream.EmitAbbrev(std::move(MetadataAbbrev));
@@ -1134,7 +1133,6 @@ void ASTWriter::WriteControlBlock(Preprocessor &PP, ASTContext &Context,
         CLANG_VERSION_MINOR,
         !isysroot.empty(),
         IncludeTimestamps,
-        Context.getLangOpts().BuildingPCHWithObjectFile,
         ASTHasCompilerErrors};
     Stream.EmitRecordWithBlob(MetadataAbbrevCode, Record,
                               getClangFullRepositoryVersion());
@@ -3690,7 +3688,7 @@ ASTWriter::GenerateNameLookupTable(const DeclContext *ConstDC,
 
   // We also build up small sets of the constructor and conversion function
   // names which are visible.
-  llvm::SmallSet<DeclarationName, 8> ConstructorNameSet, ConversionNameSet;
+  llvm::SmallPtrSet<DeclarationName, 8> ConstructorNameSet, ConversionNameSet;
 
   for (auto &Lookup : *DC->buildLookup()) {
     auto &Name = Lookup.first;
diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp
index eecdf89c791ad..ec05ab772d933 100644
--- a/clang/lib/Serialization/ASTWriterDecl.cpp
+++ b/clang/lib/Serialization/ASTWriterDecl.cpp
@@ -1031,8 +1031,10 @@ void ASTDeclWriter::VisitVarDecl(VarDecl *D) {
       // that module interface unit, not by its users. (Inline variables are
       // still emitted in module users.)
       ModulesCodegen =
-          (Writer.WritingModule->Kind == Module::ModuleInterfaceUnit &&
-           Writer.Context->GetGVALinkageForVariable(D) == GVA_StrongExternal);
+          (Writer.WritingModule->Kind == Module::ModuleInterfaceUnit ||
+           (D->hasAttr<DLLExportAttr>() &&
+            Writer.Context->getLangOpts().BuildingPCHWithObjectFile)) &&
+          Writer.Context->GetGVALinkageForVariable(D) == GVA_StrongExternal;
     }
     Record.push_back(ModulesCodegen);
     if (ModulesCodegen)
@@ -2469,7 +2471,10 @@ void ASTRecordWriter::AddFunctionDefinition(const FunctionDecl *FD) {
       Linkage = Writer->Context->GetGVALinkageForFunction(FD);
       ModulesCodegen = *Linkage == GVA_StrongExternal;
     }
-    if (Writer->Context->getLangOpts().ModulesCodegen) {
+    if (Writer->Context->getLangOpts().ModulesCodegen ||
+        (FD->hasAttr<DLLExportAttr>() &&
+         Writer->Context->getLangOpts().BuildingPCHWithObjectFile)) {
+
       // Under -fmodules-codegen, codegen is performed for all non-internal,
       // non-always_inline functions, unless they are available elsewhere.
       if (!FD->hasAttr<AlwaysInlineAttr>()) {
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index 0767b3a24bf2f..d2e5d6e44d781 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -848,12 +848,15 @@ void ASTStmtWriter::VisitOMPIteratorExpr(OMPIteratorExpr *E) {
 void ASTStmtWriter::VisitCallExpr(CallExpr *E) {
   VisitExpr(E);
   Record.push_back(E->getNumArgs());
+  Record.push_back(E->hasStoredFPFeatures());
   Record.AddSourceLocation(E->getRParenLoc());
   Record.AddStmt(E->getCallee());
   for (CallExpr::arg_iterator Arg = E->arg_begin(), ArgEnd = E->arg_end();
        Arg != ArgEnd; ++Arg)
     Record.AddStmt(*Arg);
   Record.push_back(static_cast<unsigned>(E->getADLCallKind()));
+  if (E->hasStoredFPFeatures())
+    Record.push_back(E->getFPFeatures().getAsOpaqueInt());
   Code = serialization::EXPR_CALL;
 }
 
@@ -1550,7 +1553,6 @@ void ASTStmtWriter::VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
   VisitCallExpr(E);
   Record.push_back(E->getOperator());
   Record.AddSourceRange(E->Range);
-  Record.push_back(E->getFPFeatures().getAsOpaqueInt());
   Code = serialization::EXPR_CXX_OPERATOR_CALL;
 }
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/ObjCMissingSuperCallChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ObjCMissingSuperCallChecker.cpp
index 24e2a4dea922e..35a600f2d7b86 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ObjCMissingSuperCallChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ObjCMissingSuperCallChecker.cpp
@@ -21,7 +21,7 @@
 #include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
 #include "clang/StaticAnalyzer/Core/Checker.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h"
-#include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -74,7 +74,7 @@ class ObjCSuperCallChecker : public Checker<
   void initializeSelectors(ASTContext &Ctx) const;
   void fillSelectors(ASTContext &Ctx, ArrayRef<SelectorDescriptor> Sel,
                      StringRef ClassName) const;
-  mutable llvm::StringMap<llvm::SmallSet<Selector, 16> > SelectorsForClass;
+  mutable llvm::StringMap<llvm::SmallPtrSet<Selector, 16>> SelectorsForClass;
   mutable bool IsInitialized;
 };
 
@@ -100,7 +100,8 @@ bool ObjCSuperCallChecker::isCheckableClass(const ObjCImplementationDecl *D,
 void ObjCSuperCallChecker::fillSelectors(ASTContext &Ctx,
                                          ArrayRef<SelectorDescriptor> Sel,
                                          StringRef ClassName) const {
-  llvm::SmallSet<Selector, 16> &ClassSelectors = SelectorsForClass[ClassName];
+  llvm::SmallPtrSet<Selector, 16> &ClassSelectors =
+      SelectorsForClass[ClassName];
   // Fill the Selectors SmallSet with all selectors we want to check.
   for (ArrayRef<SelectorDescriptor>::iterator I = Sel.begin(), E = Sel.end();
        I != E; ++I) {
diff --git a/clang/lib/StaticAnalyzer/Checkers/SmartPtrModeling.cpp b/clang/lib/StaticAnalyzer/Checkers/SmartPtrModeling.cpp
index bcc7d4103c1c6..c36e89c3e3a90 100644
--- a/clang/lib/StaticAnalyzer/Checkers/SmartPtrModeling.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/SmartPtrModeling.cpp
@@ -30,7 +30,8 @@ using namespace clang;
 using namespace ento;
 
 namespace {
-class SmartPtrModeling : public Checker<eval::Call, check::DeadSymbols> {
+class SmartPtrModeling
+    : public Checker<eval::Call, check::DeadSymbols, check::RegionChanges> {
 
   bool isNullAfterMoveMethod(const CallEvent &Call) const;
 
@@ -40,6 +41,12 @@ class SmartPtrModeling : public Checker<eval::Call, check::DeadSymbols> {
   bool evalCall(const CallEvent &Call, CheckerContext &C) const;
   void checkPreCall(const CallEvent &Call, CheckerContext &C) const;
   void checkDeadSymbols(SymbolReaper &SymReaper, CheckerContext &C) const;
+  ProgramStateRef
+  checkRegionChanges(ProgramStateRef State,
+                     const InvalidatedSymbols *Invalidated,
+                     ArrayRef<const MemRegion *> ExplicitRegions,
+                     ArrayRef<const MemRegion *> Regions,
+                     const LocationContext *LCtx, const CallEvent *Call) const;
 
 private:
   ProgramStateRef updateTrackedRegion(const CallEvent &Call, CheckerContext &C,
@@ -87,6 +94,31 @@ bool isNullSmartPtr(const ProgramStateRef State, const MemRegion *ThisRegion) {
 } // namespace ento
 } // namespace clang
 
+// If a region is removed all of the subregions need to be removed too.
+static TrackedRegionMapTy
+removeTrackedSubregions(TrackedRegionMapTy RegionMap,
+                        TrackedRegionMapTy::Factory &RegionMapFactory,
+                        const MemRegion *Region) {
+  if (!Region)
+    return RegionMap;
+  for (const auto &E : RegionMap) {
+    if (E.first->isSubRegionOf(Region))
+      RegionMap = RegionMapFactory.remove(RegionMap, E.first);
+  }
+  return RegionMap;
+}
+
+static ProgramStateRef updateSwappedRegion(ProgramStateRef State,
+                                           const MemRegion *Region,
+                                           const SVal *RegionInnerPointerVal) {
+  if (RegionInnerPointerVal) {
+    State = State->set<TrackedRegionMap>(Region, *RegionInnerPointerVal);
+  } else {
+    State = State->remove<TrackedRegionMap>(Region);
+  }
+  return State;
+}
+
 bool SmartPtrModeling::isNullAfterMoveMethod(const CallEvent &Call) const {
   // TODO: Update CallDescription to support anonymous calls?
   // TODO: Handle other methods, such as .get() or .release().
@@ -108,7 +140,8 @@ bool SmartPtrModeling::evalCall(const CallEvent &Call,
         cast<CXXInstanceCall>(&Call)->getCXXThisVal().getAsRegion();
 
     if (!move::isMovedFrom(State, ThisR)) {
-      // TODO: Model this case as well. At least, avoid invalidation of globals.
+      // TODO: Model this case as well. At least, avoid invalidation of
+      // globals.
       return false;
     }
 
@@ -158,6 +191,20 @@ void SmartPtrModeling::checkDeadSymbols(SymbolReaper &SymReaper,
   C.addTransition(State);
 }
 
+ProgramStateRef SmartPtrModeling::checkRegionChanges(
+    ProgramStateRef State, const InvalidatedSymbols *Invalidated,
+    ArrayRef<const MemRegion *> ExplicitRegions,
+    ArrayRef<const MemRegion *> Regions, const LocationContext *LCtx,
+    const CallEvent *Call) const {
+  TrackedRegionMapTy RegionMap = State->get<TrackedRegionMap>();
+  TrackedRegionMapTy::Factory &RegionMapFactory =
+      State->get_context<TrackedRegionMap>();
+  for (const auto *Region : Regions)
+    RegionMap = removeTrackedSubregions(RegionMap, RegionMapFactory,
+                                        Region->getBaseRegion());
+  return State->set<TrackedRegionMap>(RegionMap);
+}
+
 void SmartPtrModeling::handleReset(const CallEvent &Call,
                                    CheckerContext &C) const {
   const auto *IC = dyn_cast<CXXInstanceCall>(&Call);
@@ -169,7 +216,7 @@ void SmartPtrModeling::handleReset(const CallEvent &Call,
     return;
   auto State = updateTrackedRegion(Call, C, ThisValRegion);
   C.addTransition(State);
-  // TODO: Make sure to ivalidate the the region in the Store if we don't have
+  // TODO: Make sure to ivalidate the region in the Store if we don't have
   // time to model all methods.
 }
 
@@ -197,7 +244,30 @@ void SmartPtrModeling::handleRelease(const CallEvent &Call,
 
 void SmartPtrModeling::handleSwap(const CallEvent &Call,
                                   CheckerContext &C) const {
-  // TODO: Add support to handle swap method.
+  // To model unique_ptr::swap() method.
+  const auto *IC = dyn_cast<CXXInstanceCall>(&Call);
+  if (!IC)
+    return;
+
+  const MemRegion *ThisRegion = IC->getCXXThisVal().getAsRegion();
+  if (!ThisRegion)
+    return;
+
+  const auto *ArgRegion = Call.getArgSVal(0).getAsRegion();
+  if (!ArgRegion)
+    return;
+
+  auto State = C.getState();
+  const auto *ThisRegionInnerPointerVal =
+      State->get<TrackedRegionMap>(ThisRegion);
+  const auto *ArgRegionInnerPointerVal =
+      State->get<TrackedRegionMap>(ArgRegion);
+
+  // Swap the tracked region values.
+  State = updateSwappedRegion(State, ThisRegion, ArgRegionInnerPointerVal);
+  State = updateSwappedRegion(State, ArgRegion, ThisRegionInnerPointerVal);
+
+  C.addTransition(State);
 }
 
 ProgramStateRef
diff --git a/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp
index 8b575f4f4759c..e437c33bd81ad 100644
--- a/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp
@@ -302,6 +302,13 @@ class StdLibraryFunctionsChecker
       Tmp.Op = BinaryOperator::negateComparisonOp(Op);
       return std::make_shared<BufferSizeConstraint>(Tmp);
     }
+
+    bool checkSpecificValidity(const FunctionDecl *FD) const override {
+      const bool ValidArg = getArgType(FD, ArgN)->isPointerType();
+      assert(ValidArg &&
+             "This constraint should be applied only on a pointer type");
+      return ValidArg;
+    }
   };
 
   /// The complete list of constraints that defines a single branch.
@@ -318,8 +325,8 @@ class StdLibraryFunctionsChecker
   // concessive signature, meaning there may be irrelevant types in the
   // signature which we do not check against a function with concrete types.
   struct Signature {
-    const ArgTypes ArgTys;
-    const QualType RetTy;
+    ArgTypes ArgTys;
+    QualType RetTy;
     Signature(ArgTypes ArgTys, QualType RetTy) : ArgTys(ArgTys), RetTy(RetTy) {
       assertRetTypeSuitableForSignature(RetTy);
       for (size_t I = 0, E = ArgTys.size(); I != E; ++I) {
@@ -327,6 +334,7 @@ class StdLibraryFunctionsChecker
         assertArgTypeSuitableForSignature(ArgTy);
       }
     }
+
     bool matches(const FunctionDecl *FD) const;
 
   private:
@@ -380,7 +388,7 @@ class StdLibraryFunctionsChecker
   ///   rules for the given parameter's type, those rules are checked once the
   ///   signature is matched.
   class Summary {
-    const Signature Sign;
+    Optional<Signature> Sign;
     const InvalidationKind InvalidationKd;
     Cases CaseConstraints;
     ConstraintSet ArgConstraints;
@@ -391,7 +399,14 @@ class StdLibraryFunctionsChecker
 
   public:
     Summary(ArgTypes ArgTys, QualType RetTy, InvalidationKind InvalidationKd)
-        : Sign(ArgTys, RetTy), InvalidationKd(InvalidationKd) {}
+        : Sign(Signature(ArgTys, RetTy)), InvalidationKd(InvalidationKd) {}
+
+    Summary(InvalidationKind InvalidationKd) : InvalidationKd(InvalidationKd) {}
+
+    Summary &setSignature(const Signature &S) {
+      Sign = S;
+      return *this;
+    }
 
     Summary &Case(ConstraintSet&& CS) {
       CaseConstraints.push_back(std::move(CS));
@@ -413,7 +428,9 @@ class StdLibraryFunctionsChecker
     // Returns true if the summary should be applied to the given function.
     // And if yes then store the function declaration.
     bool matchesAndSet(const FunctionDecl *FD) {
-      bool Result = Sign.matches(FD) && validateByConstraints(FD);
+      assert(Sign &&
+             "Signature must be set before comparing to a FunctionDecl");
+      bool Result = Sign->matches(FD) && validateByConstraints(FD);
       if (Result) {
         assert(!this->FD && "FD must not be set more than once");
         this->FD = FD;
@@ -761,6 +778,10 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
   BasicValueFactory &BVF = SVB.getBasicValueFactory();
   const ASTContext &ACtx = BVF.getContext();
 
+  auto getRestrictTy = [&ACtx](QualType Ty) {
+    return ACtx.getLangOpts().C99 ? ACtx.getRestrictType(Ty) : Ty;
+  };
+
   // These types are useful for writing specifications quickly,
   // New specifications should probably introduce more types.
   // Some types are hard to obtain from the AST, eg. "ssize_t".
@@ -779,28 +800,18 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
   const QualType IntPtrTy = ACtx.getPointerType(IntTy); // int *
   const QualType UnsignedIntPtrTy =
       ACtx.getPointerType(UnsignedIntTy); // unsigned int *
-  const QualType VoidPtrRestrictTy =
-      ACtx.getLangOpts().C99 ? ACtx.getRestrictType(VoidPtrTy) // void *restrict
-                             : VoidPtrTy;
+  const QualType VoidPtrRestrictTy = getRestrictTy(VoidPtrTy);
   const QualType ConstVoidPtrTy =
       ACtx.getPointerType(ACtx.VoidTy.withConst()); // const void *
   const QualType CharPtrTy = ACtx.getPointerType(ACtx.CharTy); // char *
-  const QualType CharPtrRestrictTy =
-      ACtx.getLangOpts().C99 ? ACtx.getRestrictType(CharPtrTy) // char *restrict
-                             : CharPtrTy;
+  const QualType CharPtrRestrictTy = getRestrictTy(CharPtrTy);
   const QualType ConstCharPtrTy =
       ACtx.getPointerType(ACtx.CharTy.withConst()); // const char *
-  const QualType ConstCharPtrRestrictTy =
-      ACtx.getLangOpts().C99
-          ? ACtx.getRestrictType(ConstCharPtrTy) // const char *restrict
-          : ConstCharPtrTy;
+  const QualType ConstCharPtrRestrictTy = getRestrictTy(ConstCharPtrTy);
   const QualType Wchar_tPtrTy = ACtx.getPointerType(ACtx.WCharTy); // wchar_t *
   const QualType ConstWchar_tPtrTy =
       ACtx.getPointerType(ACtx.WCharTy.withConst()); // const wchar_t *
-  const QualType ConstVoidPtrRestrictTy =
-      ACtx.getLangOpts().C99
-          ? ACtx.getRestrictType(ConstVoidPtrTy) // const void *restrict
-          : ConstVoidPtrTy;
+  const QualType ConstVoidPtrRestrictTy = getRestrictTy(ConstVoidPtrTy);
 
   const RangeInt IntMax = BVF.getMaxValue(IntTy).getLimitedValue();
   const RangeInt UnsignedIntMax =
@@ -840,11 +851,13 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
     // Add a summary to a FunctionDecl found by lookup. The lookup is performed
     // by the given Name, and in the global scope. The summary will be attached
     // to the found FunctionDecl only if the signatures match.
-    void operator()(StringRef Name, Summary S) {
+    //
+    // Returns true if the summary has been added, false otherwise.
+    bool operator()(StringRef Name, Summary S) {
       IdentifierInfo &II = ACtx.Idents.get(Name);
       auto LookupRes = ACtx.getTranslationUnitDecl()->lookup(&II);
       if (LookupRes.size() == 0)
-        return;
+        return false;
       for (Decl *D : LookupRes) {
         if (auto *FD = dyn_cast<FunctionDecl>(D)) {
           if (S.matchesAndSet(FD)) {
@@ -856,10 +869,14 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
               FD->print(llvm::errs());
               llvm::errs() << "\n";
             }
-            return;
+            return true;
           }
         }
       }
+      return false;
+    }
+    bool operator()(StringRef Name, Signature Sign, Summary Sum) {
+      return operator()(Name, Sum.setSignature(Sign));
     }
     // Add several summaries for the given name.
     void operator()(StringRef Name, const std::vector<Summary> &Summaries) {
@@ -927,8 +944,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
     // FILE *
     FilePtrTy = ACtx.getPointerType(*FileTy);
     // FILE *restrict
-    FilePtrRestrictTy =
-        ACtx.getLangOpts().C99 ? ACtx.getRestrictType(*FilePtrTy) : *FilePtrTy;
+    FilePtrRestrictTy = getRestrictTy(*FilePtrTy);
   }
 
   using RetType = QualType;
@@ -1434,9 +1450,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
     Optional<QualType> StructStatPtrTy, StructStatPtrRestrictTy;
     if (StructStatTy) {
       StructStatPtrTy = ACtx.getPointerType(*StructStatTy);
-      StructStatPtrRestrictTy = ACtx.getLangOpts().C99
-                                    ? ACtx.getRestrictType(*StructStatPtrTy)
-                                    : *StructStatPtrTy;
+      StructStatPtrRestrictTy = getRestrictTy(*StructStatPtrTy);
     }
 
     if (StructStatPtrTy)
@@ -1702,6 +1716,276 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
             .ArgConstraint(ArgumentCondition(0, WithinRange, Range(0, IntMax)))
             .ArgConstraint(NotNull(ArgNo(1)))
             .ArgConstraint(NotNull(ArgNo(2))));
+
+    Optional<QualType> StructSockaddrTy = lookupType("sockaddr", ACtx);
+    Optional<QualType> StructSockaddrPtrTy, ConstStructSockaddrPtrTy,
+        StructSockaddrPtrRestrictTy, ConstStructSockaddrPtrRestrictTy;
+    if (StructSockaddrTy) {
+      StructSockaddrPtrTy = ACtx.getPointerType(*StructSockaddrTy);
+      ConstStructSockaddrPtrTy =
+          ACtx.getPointerType(StructSockaddrTy->withConst());
+      StructSockaddrPtrRestrictTy = getRestrictTy(*StructSockaddrPtrTy);
+      ConstStructSockaddrPtrRestrictTy =
+          getRestrictTy(*ConstStructSockaddrPtrTy);
+    }
+    Optional<QualType> Socklen_tTy = lookupType("socklen_t", ACtx);
+    Optional<QualType> Socklen_tPtrTy, Socklen_tPtrRestrictTy;
+    Optional<RangeInt> Socklen_tMax;
+    if (Socklen_tTy) {
+      Socklen_tMax = BVF.getMaxValue(*Socklen_tTy).getLimitedValue();
+      Socklen_tPtrTy = ACtx.getPointerType(*Socklen_tTy);
+      Socklen_tPtrRestrictTy = getRestrictTy(*Socklen_tPtrTy);
+    }
+
+    // In 'socket.h' of some libc implementations with C99, sockaddr parameter
+    // is a transparent union of the underlying sockaddr_ family of pointers
+    // instead of being a pointer to struct sockaddr. In these cases, the
+    // standardized signature will not match, thus we try to match with another
+    // signature that has the joker Irrelevant type. We also remove those
+    // constraints which require pointer types for the sockaddr param.
+    if (StructSockaddrPtrRestrictTy && Socklen_tPtrRestrictTy) {
+      auto Accept = Summary(NoEvalCall)
+                        .ArgConstraint(ArgumentCondition(0, WithinRange,
+                                                         Range(0, IntMax)));
+      if (!addToFunctionSummaryMap(
+              "accept",
+              // int accept(int socket, struct sockaddr *restrict address,
+              //            socklen_t *restrict address_len);
+              Signature(ArgTypes{IntTy, *StructSockaddrPtrRestrictTy,
+                                 *Socklen_tPtrRestrictTy},
+                        RetType{IntTy}),
+              Accept))
+        addToFunctionSummaryMap(
+            "accept",
+            Signature(ArgTypes{IntTy, Irrelevant, *Socklen_tPtrRestrictTy},
+                      RetType{IntTy}),
+            Accept);
+
+      // int bind(int socket, const struct sockaddr *address, socklen_t
+      //          address_len);
+      if (!addToFunctionSummaryMap(
+              "bind",
+              Summary(ArgTypes{IntTy, *ConstStructSockaddrPtrTy, *Socklen_tTy},
+                      RetType{IntTy}, NoEvalCall)
+                  .ArgConstraint(
+                      ArgumentCondition(0, WithinRange, Range(0, IntMax)))
+                  .ArgConstraint(NotNull(ArgNo(1)))
+                  .ArgConstraint(
+                      BufferSize(/*Buffer=*/ArgNo(1), /*BufSize=*/ArgNo(2)))
+                  .ArgConstraint(ArgumentCondition(2, WithinRange,
+                                                   Range(0, *Socklen_tMax)))))
+        // Do not add constraints on sockaddr.
+        addToFunctionSummaryMap(
+            "bind", Summary(ArgTypes{IntTy, Irrelevant, *Socklen_tTy},
+                            RetType{IntTy}, NoEvalCall)
+                        .ArgConstraint(
+                            ArgumentCondition(0, WithinRange, Range(0, IntMax)))
+                        .ArgConstraint(ArgumentCondition(
+                            2, WithinRange, Range(0, *Socklen_tMax))));
+
+      // int getpeername(int socket, struct sockaddr *restrict address,
+      //                 socklen_t *restrict address_len);
+      if (!addToFunctionSummaryMap(
+              "getpeername",
+              Summary(ArgTypes{IntTy, *StructSockaddrPtrRestrictTy,
+                               *Socklen_tPtrRestrictTy},
+                      RetType{IntTy}, NoEvalCall)
+                  .ArgConstraint(
+                      ArgumentCondition(0, WithinRange, Range(0, IntMax)))
+                  .ArgConstraint(NotNull(ArgNo(1)))
+                  .ArgConstraint(NotNull(ArgNo(2)))))
+        addToFunctionSummaryMap(
+            "getpeername",
+            Summary(ArgTypes{IntTy, Irrelevant, *Socklen_tPtrRestrictTy},
+                    RetType{IntTy}, NoEvalCall)
+                .ArgConstraint(
+                    ArgumentCondition(0, WithinRange, Range(0, IntMax))));
+
+      // int getsockname(int socket, struct sockaddr *restrict address,
+      //                 socklen_t *restrict address_len);
+      if (!addToFunctionSummaryMap(
+              "getsockname",
+              Summary(ArgTypes{IntTy, *StructSockaddrPtrRestrictTy,
+                               *Socklen_tPtrRestrictTy},
+                      RetType{IntTy}, NoEvalCall)
+                  .ArgConstraint(
+                      ArgumentCondition(0, WithinRange, Range(0, IntMax)))
+                  .ArgConstraint(NotNull(ArgNo(1)))
+                  .ArgConstraint(NotNull(ArgNo(2)))))
+        addToFunctionSummaryMap(
+            "getsockname",
+            Summary(ArgTypes{IntTy, Irrelevant, *Socklen_tPtrRestrictTy},
+                    RetType{IntTy}, NoEvalCall)
+                .ArgConstraint(
+                    ArgumentCondition(0, WithinRange, Range(0, IntMax))));
+
+      // int connect(int socket, const struct sockaddr *address, socklen_t
+      //             address_len);
+      if (!addToFunctionSummaryMap(
+              "connect",
+              Summary(ArgTypes{IntTy, *ConstStructSockaddrPtrTy, *Socklen_tTy},
+                      RetType{IntTy}, NoEvalCall)
+                  .ArgConstraint(
+                      ArgumentCondition(0, WithinRange, Range(0, IntMax)))
+                  .ArgConstraint(NotNull(ArgNo(1)))))
+        addToFunctionSummaryMap(
+            "connect", Summary(ArgTypes{IntTy, Irrelevant, *Socklen_tTy},
+                               RetType{IntTy}, NoEvalCall)
+                           .ArgConstraint(ArgumentCondition(0, WithinRange,
+                                                            Range(0, IntMax))));
+
+      auto Recvfrom = Summary(NoEvalCall)
+                          .ArgConstraint(ArgumentCondition(0, WithinRange,
+                                                           Range(0, IntMax)))
+                          .ArgConstraint(BufferSize(/*Buffer=*/ArgNo(1),
+                                                    /*BufSize=*/ArgNo(2)));
+      if (Ssize_tTy &&
+          !addToFunctionSummaryMap(
+              "recvfrom",
+              // ssize_t recvfrom(int socket, void *restrict buffer,
+              //                  size_t length,
+              //                  int flags, struct sockaddr *restrict address,
+              //                  socklen_t *restrict address_len);
+              Signature(ArgTypes{IntTy, VoidPtrRestrictTy, SizeTy, IntTy,
+                                 *StructSockaddrPtrRestrictTy,
+                                 *Socklen_tPtrRestrictTy},
+                        RetType{*Ssize_tTy}),
+              Recvfrom))
+        addToFunctionSummaryMap(
+            "recvfrom",
+            Signature(ArgTypes{IntTy, VoidPtrRestrictTy, SizeTy, IntTy,
+                               Irrelevant, *Socklen_tPtrRestrictTy},
+                      RetType{*Ssize_tTy}),
+            Recvfrom);
+
+      auto Sendto = Summary(NoEvalCall)
+                        .ArgConstraint(
+                            ArgumentCondition(0, WithinRange, Range(0, IntMax)))
+                        .ArgConstraint(BufferSize(/*Buffer=*/ArgNo(1),
+                                                  /*BufSize=*/ArgNo(2)));
+      if (Ssize_tTy &&
+          !addToFunctionSummaryMap(
+              "sendto",
+              // ssize_t sendto(int socket, const void *message, size_t length,
+              //                int flags, const struct sockaddr *dest_addr,
+              //                socklen_t dest_len);
+              Signature(ArgTypes{IntTy, ConstVoidPtrTy, SizeTy, IntTy,
+                                 *ConstStructSockaddrPtrTy, *Socklen_tTy},
+                        RetType{*Ssize_tTy}),
+              Sendto))
+        addToFunctionSummaryMap(
+            "sendto",
+            Signature(ArgTypes{IntTy, ConstVoidPtrTy, SizeTy, IntTy, Irrelevant,
+                               *Socklen_tTy},
+                      RetType{*Ssize_tTy}),
+            Sendto);
+    }
+
+    // int listen(int sockfd, int backlog);
+    addToFunctionSummaryMap(
+        "listen", Summary(ArgTypes{IntTy, IntTy}, RetType{IntTy}, NoEvalCall)
+                      .ArgConstraint(
+                          ArgumentCondition(0, WithinRange, Range(0, IntMax))));
+
+    if (Ssize_tTy)
+      // ssize_t recv(int sockfd, void *buf, size_t len, int flags);
+      addToFunctionSummaryMap(
+          "recv", Summary(ArgTypes{IntTy, VoidPtrTy, SizeTy, IntTy},
+                          RetType{*Ssize_tTy}, NoEvalCall)
+                      .ArgConstraint(
+                          ArgumentCondition(0, WithinRange, Range(0, IntMax)))
+                      .ArgConstraint(BufferSize(/*Buffer=*/ArgNo(1),
+                                                /*BufSize=*/ArgNo(2))));
+
+    Optional<QualType> StructMsghdrTy = lookupType("msghdr", ACtx);
+    Optional<QualType> StructMsghdrPtrTy, ConstStructMsghdrPtrTy;
+    if (StructMsghdrTy) {
+      StructMsghdrPtrTy = ACtx.getPointerType(*StructMsghdrTy);
+      ConstStructMsghdrPtrTy = ACtx.getPointerType(StructMsghdrTy->withConst());
+    }
+
+    if (Ssize_tTy && StructMsghdrPtrTy)
+      // ssize_t recvmsg(int sockfd, struct msghdr *msg, int flags);
+      addToFunctionSummaryMap(
+          "recvmsg", Summary(ArgTypes{IntTy, *StructMsghdrPtrTy, IntTy},
+                             RetType{*Ssize_tTy}, NoEvalCall)
+                         .ArgConstraint(ArgumentCondition(0, WithinRange,
+                                                          Range(0, IntMax))));
+
+    if (Ssize_tTy && ConstStructMsghdrPtrTy)
+      // ssize_t sendmsg(int sockfd, const struct msghdr *msg, int flags);
+      addToFunctionSummaryMap(
+          "sendmsg", Summary(ArgTypes{IntTy, *ConstStructMsghdrPtrTy, IntTy},
+                             RetType{*Ssize_tTy}, NoEvalCall)
+                         .ArgConstraint(ArgumentCondition(0, WithinRange,
+                                                          Range(0, IntMax))));
+
+    if (Socklen_tTy)
+      // int setsockopt(int socket, int level, int option_name,
+      //                const void *option_value, socklen_t option_len);
+      addToFunctionSummaryMap(
+          "setsockopt",
+          Summary(ArgTypes{IntTy, IntTy, IntTy, ConstVoidPtrTy, *Socklen_tTy},
+                  RetType{IntTy}, NoEvalCall)
+              .ArgConstraint(NotNull(ArgNo(3)))
+              .ArgConstraint(
+                  BufferSize(/*Buffer=*/ArgNo(3), /*BufSize=*/ArgNo(4)))
+              .ArgConstraint(
+                  ArgumentCondition(4, WithinRange, Range(0, *Socklen_tMax))));
+
+    if (Socklen_tPtrRestrictTy)
+      // int getsockopt(int socket, int level, int option_name,
+      //                void *restrict option_value,
+      //                socklen_t *restrict option_len);
+      addToFunctionSummaryMap(
+          "getsockopt", Summary(ArgTypes{IntTy, IntTy, IntTy, VoidPtrRestrictTy,
+                                         *Socklen_tPtrRestrictTy},
+                                RetType{IntTy}, NoEvalCall)
+                            .ArgConstraint(NotNull(ArgNo(3)))
+                            .ArgConstraint(NotNull(ArgNo(4))));
+
+    if (Ssize_tTy)
+      // ssize_t send(int sockfd, const void *buf, size_t len, int flags);
+      addToFunctionSummaryMap(
+          "send", Summary(ArgTypes{IntTy, ConstVoidPtrTy, SizeTy, IntTy},
+                          RetType{*Ssize_tTy}, NoEvalCall)
+                      .ArgConstraint(
+                          ArgumentCondition(0, WithinRange, Range(0, IntMax)))
+                      .ArgConstraint(BufferSize(/*Buffer=*/ArgNo(1),
+                                                /*BufSize=*/ArgNo(2))));
+
+    // int socketpair(int domain, int type, int protocol, int sv[2]);
+    addToFunctionSummaryMap("socketpair",
+                            Summary(ArgTypes{IntTy, IntTy, IntTy, IntPtrTy},
+                                    RetType{IntTy}, NoEvalCall)
+                                .ArgConstraint(NotNull(ArgNo(3))));
+
+    if (ConstStructSockaddrPtrRestrictTy && Socklen_tTy)
+      // int getnameinfo(const struct sockaddr *restrict sa, socklen_t salen,
+      //                 char *restrict node, socklen_t nodelen,
+      //                 char *restrict service,
+      //                 socklen_t servicelen, int flags);
+      //
+      // This is defined in netdb.h. And contrary to 'socket.h', the sockaddr
+      // parameter is never handled as a transparent union in netdb.h
+      addToFunctionSummaryMap(
+          "getnameinfo",
+          Summary(ArgTypes{*ConstStructSockaddrPtrRestrictTy, *Socklen_tTy,
+                           CharPtrRestrictTy, *Socklen_tTy, CharPtrRestrictTy,
+                           *Socklen_tTy, IntTy},
+                  RetType{IntTy}, NoEvalCall)
+              .ArgConstraint(
+                  BufferSize(/*Buffer=*/ArgNo(0), /*BufSize=*/ArgNo(1)))
+              .ArgConstraint(
+                  ArgumentCondition(1, WithinRange, Range(0, *Socklen_tMax)))
+              .ArgConstraint(
+                  BufferSize(/*Buffer=*/ArgNo(2), /*BufSize=*/ArgNo(3)))
+              .ArgConstraint(
+                  ArgumentCondition(3, WithinRange, Range(0, *Socklen_tMax)))
+              .ArgConstraint(
+                  BufferSize(/*Buffer=*/ArgNo(4), /*BufSize=*/ArgNo(5)))
+              .ArgConstraint(
+                  ArgumentCondition(5, WithinRange, Range(0, *Socklen_tMax))));
   }
 
   // Functions for testing.
diff --git a/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
index f6abbe4f8f03b..6b176b3c4e2b2 100644
--- a/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
@@ -204,7 +204,8 @@ class StreamChecker : public Checker<check::PreCall, eval::Call,
   BugType BT_IllegalWhence{this, "Illegal whence argument",
                            "Stream handling error"};
   BugType BT_StreamEof{this, "Stream already in EOF", "Stream handling error"};
-  BugType BT_ResourceLeak{this, "Resource leak", "Stream handling error"};
+  BugType BT_ResourceLeak{this, "Resource leak", "Stream handling error",
+                          /*SuppressOnSink =*/true};
 
 public:
   void checkPreCall(const CallEvent &Call, CheckerContext &C) const;
@@ -337,6 +338,12 @@ class StreamChecker : public Checker<check::PreCall, eval::Call,
   /// to ensure uniform handling.
   void reportFEofWarning(CheckerContext &C, ProgramStateRef State) const;
 
+  /// Emit resource leak warnings for the given symbols.
+  /// Createn a non-fatal error node for these, and returns it (if any warnings
+  /// were generated). Return value is non-null.
+  ExplodedNode *reportLeaks(const SmallVector<SymbolRef, 2> &LeakedSyms,
+                            CheckerContext &C, ExplodedNode *Pred) const;
+
   /// Find the description data of the function called by a call event.
   /// Returns nullptr if no function is recognized.
   const FnDescription *lookupFn(const CallEvent &Call) const {
@@ -956,28 +963,14 @@ void StreamChecker::reportFEofWarning(CheckerContext &C,
   C.addTransition(State);
 }
 
-void StreamChecker::checkDeadSymbols(SymbolReaper &SymReaper,
-                                     CheckerContext &C) const {
-  ProgramStateRef State = C.getState();
-
-  // TODO: Clean up the state.
-  const StreamMapTy &Map = State->get<StreamMap>();
-  for (const auto &I : Map) {
-    SymbolRef Sym = I.first;
-    const StreamState &SS = I.second;
-    if (!SymReaper.isDead(Sym) || !SS.isOpened())
-      continue;
-
-    ExplodedNode *N = C.generateErrorNode();
-    if (!N)
-      continue;
-
-    // Do not warn for non-closed stream at program exit.
-    ExplodedNode *Pred = C.getPredecessor();
-    if (Pred && Pred->getCFGBlock() &&
-        Pred->getCFGBlock()->hasNoReturnElement())
-      continue;
+ExplodedNode *
+StreamChecker::reportLeaks(const SmallVector<SymbolRef, 2> &LeakedSyms,
+                           CheckerContext &C, ExplodedNode *Pred) const {
+  ExplodedNode *Err = C.generateNonFatalErrorNode(C.getState(), Pred);
+  if (!Err)
+    return Pred;
 
+  for (SymbolRef LeakSym : LeakedSyms) {
     // Resource leaks can result in multiple warning that describe the same kind
     // of programming error:
     //  void f() {
@@ -989,8 +982,7 @@ void StreamChecker::checkDeadSymbols(SymbolReaper &SymReaper,
     // from a different kinds of errors), the reduction in redundant reports
     // makes this a worthwhile heuristic.
     // FIXME: Add a checker option to turn this uniqueing feature off.
-
-    const ExplodedNode *StreamOpenNode = getAcquisitionSite(N, Sym, C);
+    const ExplodedNode *StreamOpenNode = getAcquisitionSite(Err, LeakSym, C);
     assert(StreamOpenNode && "Could not find place of stream opening.");
     PathDiagnosticLocation LocUsedForUniqueing =
         PathDiagnosticLocation::createBegin(
@@ -1000,12 +992,38 @@ void StreamChecker::checkDeadSymbols(SymbolReaper &SymReaper,
     std::unique_ptr<PathSensitiveBugReport> R =
         std::make_unique<PathSensitiveBugReport>(
             BT_ResourceLeak,
-            "Opened stream never closed. Potential resource leak.", N,
+            "Opened stream never closed. Potential resource leak.", Err,
             LocUsedForUniqueing,
             StreamOpenNode->getLocationContext()->getDecl());
-    R->markInteresting(Sym);
+    R->markInteresting(LeakSym);
     C.emitReport(std::move(R));
   }
+
+  return Err;
+}
+
+void StreamChecker::checkDeadSymbols(SymbolReaper &SymReaper,
+                                     CheckerContext &C) const {
+  ProgramStateRef State = C.getState();
+
+  llvm::SmallVector<SymbolRef, 2> LeakedSyms;
+
+  const StreamMapTy &Map = State->get<StreamMap>();
+  for (const auto &I : Map) {
+    SymbolRef Sym = I.first;
+    const StreamState &SS = I.second;
+    if (!SymReaper.isDead(Sym))
+      continue;
+    if (SS.isOpened())
+      LeakedSyms.push_back(Sym);
+    State = State->remove<StreamMap>(Sym);
+  }
+
+  ExplodedNode *N = C.getPredecessor();
+  if (!LeakedSyms.empty())
+    N = reportLeaks(LeakedSyms, C, N);
+
+  C.addTransition(State, N);
 }
 
 ProgramStateRef StreamChecker::checkPointerEscape(
diff --git a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
index ef4d38ff498ff..bc72f4f8c1e30 100644
--- a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
+++ b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
@@ -2813,7 +2813,7 @@ UndefOrNullArgVisitor::VisitNode(const ExplodedNode *N, BugReporterContext &BRC,
 //===----------------------------------------------------------------------===//
 
 FalsePositiveRefutationBRVisitor::FalsePositiveRefutationBRVisitor()
-    : Constraints(ConstraintRangeTy::Factory().getEmptyMap()) {}
+    : Constraints(ConstraintMap::Factory().getEmptyMap()) {}
 
 void FalsePositiveRefutationBRVisitor::finalizeVisitor(
     BugReporterContext &BRC, const ExplodedNode *EndPathNode,
@@ -2855,9 +2855,8 @@ void FalsePositiveRefutationBRVisitor::finalizeVisitor(
 void FalsePositiveRefutationBRVisitor::addConstraints(
     const ExplodedNode *N, bool OverwriteConstraintsOnExistingSyms) {
   // Collect new constraints
-  const ConstraintRangeTy &NewCs = N->getState()->get<ConstraintRange>();
-  ConstraintRangeTy::Factory &CF =
-      N->getState()->get_context<ConstraintRange>();
+  ConstraintMap NewCs = getConstraintMap(N->getState());
+  ConstraintMap::Factory &CF = N->getState()->get_context<ConstraintMap>();
 
   // Add constraints if we don't have them yet
   for (auto const &C : NewCs) {
diff --git a/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp b/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp
index cb6f61e86ae3b..32766d796add6 100644
--- a/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp
+++ b/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp
@@ -89,7 +89,7 @@ class OperatorRelationsTable {
   }
 
   TriStateKind getCmpOpState(BinaryOperatorKind CurrentOP,
-                         BinaryOperatorKind QueriedOP) const {
+                             BinaryOperatorKind QueriedOP) const {
     return CmpOpTable[getIndexFromOp(CurrentOP)][getIndexFromOp(QueriedOP)];
   }
 
@@ -364,6 +364,18 @@ RangeSet RangeSet::Negate(BasicValueFactory &BV, Factory &F) const {
   return newRanges;
 }
 
+RangeSet RangeSet::Delete(BasicValueFactory &BV, Factory &F,
+                          const llvm::APSInt &Point) const {
+  llvm::APSInt Upper = Point;
+  llvm::APSInt Lower = Point;
+
+  ++Upper;
+  --Lower;
+
+  // Notice that the lower bound is greater than the upper bound.
+  return Intersect(BV, F, Upper, Lower);
+}
+
 void RangeSet::print(raw_ostream &os) const {
   bool isFirst = true;
   os << "{ ";
@@ -379,7 +391,315 @@ void RangeSet::print(raw_ostream &os) const {
   os << " }";
 }
 
+REGISTER_SET_FACTORY_WITH_PROGRAMSTATE(SymbolSet, SymbolRef)
+
+namespace {
+class EquivalenceClass;
+} // end anonymous namespace
+
+REGISTER_MAP_WITH_PROGRAMSTATE(ClassMap, SymbolRef, EquivalenceClass)
+REGISTER_MAP_WITH_PROGRAMSTATE(ClassMembers, EquivalenceClass, SymbolSet)
+REGISTER_MAP_WITH_PROGRAMSTATE(ConstraintRange, EquivalenceClass, RangeSet)
+
+REGISTER_SET_FACTORY_WITH_PROGRAMSTATE(ClassSet, EquivalenceClass)
+REGISTER_MAP_WITH_PROGRAMSTATE(DisequalityMap, EquivalenceClass, ClassSet)
+
 namespace {
+/// This class encapsulates a set of symbols equal to each other.
+///
+/// The main idea of the approach requiring such classes is in narrowing
+/// and sharing constraints between symbols within the class.  Also we can
+/// conclude that there is no practical need in storing constraints for
+/// every member of the class separately.
+///
+/// Main terminology:
+///
+///   * "Equivalence class" is an object of this class, which can be efficiently
+///     compared to other classes.  It represents the whole class without
+///     storing the actual in it.  The members of the class however can be
+///     retrieved from the state.
+///
+///   * "Class members" are the symbols corresponding to the class.  This means
+///     that A == B for every member symbols A and B from the class.  Members of
+///     each class are stored in the state.
+///
+///   * "Trivial class" is a class that has and ever had only one same symbol.
+///
+///   * "Merge operation" merges two classes into one.  It is the main operation
+///     to produce non-trivial classes.
+///     If, at some point, we can assume that two symbols from two distinct
+///     classes are equal, we can merge these classes.
+class EquivalenceClass : public llvm::FoldingSetNode {
+public:
+  /// Find equivalence class for the given symbol in the given state.
+  LLVM_NODISCARD static inline EquivalenceClass find(ProgramStateRef State,
+                                                     SymbolRef Sym);
+
+  /// Merge classes for the given symbols and return a new state.
+  LLVM_NODISCARD static inline ProgramStateRef
+  merge(BasicValueFactory &BV, RangeSet::Factory &F, ProgramStateRef State,
+        SymbolRef First, SymbolRef Second);
+  // Merge this class with the given class and return a new state.
+  LLVM_NODISCARD inline ProgramStateRef merge(BasicValueFactory &BV,
+                                              RangeSet::Factory &F,
+                                              ProgramStateRef State,
+                                              EquivalenceClass Other);
+
+  /// Return a set of class members for the given state.
+  LLVM_NODISCARD inline SymbolSet getClassMembers(ProgramStateRef State);
+  /// Return true if the current class is trivial in the given state.
+  LLVM_NODISCARD inline bool isTrivial(ProgramStateRef State);
+  /// Return true if the current class is trivial and its only member is dead.
+  LLVM_NODISCARD inline bool isTriviallyDead(ProgramStateRef State,
+                                             SymbolReaper &Reaper);
+
+  LLVM_NODISCARD static inline ProgramStateRef
+  markDisequal(BasicValueFactory &BV, RangeSet::Factory &F,
+               ProgramStateRef State, SymbolRef First, SymbolRef Second);
+  LLVM_NODISCARD static inline ProgramStateRef
+  markDisequal(BasicValueFactory &BV, RangeSet::Factory &F,
+               ProgramStateRef State, EquivalenceClass First,
+               EquivalenceClass Second);
+  LLVM_NODISCARD inline ProgramStateRef
+  markDisequal(BasicValueFactory &BV, RangeSet::Factory &F,
+               ProgramStateRef State, EquivalenceClass Other) const;
+  LLVM_NODISCARD static inline ClassSet
+  getDisequalClasses(ProgramStateRef State, SymbolRef Sym);
+  LLVM_NODISCARD inline ClassSet
+  getDisequalClasses(ProgramStateRef State) const;
+  LLVM_NODISCARD inline ClassSet
+  getDisequalClasses(DisequalityMapTy Map, ClassSet::Factory &Factory) const;
+
+  LLVM_NODISCARD static inline Optional<bool>
+  areEqual(ProgramStateRef State, SymbolRef First, SymbolRef Second);
+
+  /// Check equivalence data for consistency.
+  LLVM_NODISCARD LLVM_ATTRIBUTE_UNUSED static bool
+  isClassDataConsistent(ProgramStateRef State);
+
+  LLVM_NODISCARD QualType getType() const {
+    return getRepresentativeSymbol()->getType();
+  }
+
+  EquivalenceClass() = delete;
+  EquivalenceClass(const EquivalenceClass &) = default;
+  EquivalenceClass &operator=(const EquivalenceClass &) = delete;
+  EquivalenceClass(EquivalenceClass &&) = default;
+  EquivalenceClass &operator=(EquivalenceClass &&) = delete;
+
+  bool operator==(const EquivalenceClass &Other) const {
+    return ID == Other.ID;
+  }
+  bool operator<(const EquivalenceClass &Other) const { return ID < Other.ID; }
+  bool operator!=(const EquivalenceClass &Other) const {
+    return !operator==(Other);
+  }
+
+  static void Profile(llvm::FoldingSetNodeID &ID, uintptr_t CID) {
+    ID.AddInteger(CID);
+  }
+
+  void Profile(llvm::FoldingSetNodeID &ID) const { Profile(ID, this->ID); }
+
+private:
+  /* implicit */ EquivalenceClass(SymbolRef Sym)
+      : ID(reinterpret_cast<uintptr_t>(Sym)) {}
+
+  /// This function is intended to be used ONLY within the class.
+  /// The fact that ID is a pointer to a symbol is an implementation detail
+  /// and should stay that way.
+  /// In the current implementation, we use it to retrieve the only member
+  /// of the trivial class.
+  SymbolRef getRepresentativeSymbol() const {
+    return reinterpret_cast<SymbolRef>(ID);
+  }
+  static inline SymbolSet::Factory &getMembersFactory(ProgramStateRef State);
+
+  inline ProgramStateRef mergeImpl(BasicValueFactory &BV, RangeSet::Factory &F,
+                                   ProgramStateRef State, SymbolSet Members,
+                                   EquivalenceClass Other,
+                                   SymbolSet OtherMembers);
+  static inline void
+  addToDisequalityInfo(DisequalityMapTy &Info, ConstraintRangeTy &Constraints,
+                       BasicValueFactory &BV, RangeSet::Factory &F,
+                       ProgramStateRef State, EquivalenceClass First,
+                       EquivalenceClass Second);
+
+  /// This is a unique identifier of the class.
+  uintptr_t ID;
+};
+
+//===----------------------------------------------------------------------===//
+//                             Constraint functions
+//===----------------------------------------------------------------------===//
+
+LLVM_NODISCARD inline const RangeSet *getConstraint(ProgramStateRef State,
+                                                    EquivalenceClass Class) {
+  return State->get<ConstraintRange>(Class);
+}
+
+LLVM_NODISCARD inline const RangeSet *getConstraint(ProgramStateRef State,
+                                                    SymbolRef Sym) {
+  return getConstraint(State, EquivalenceClass::find(State, Sym));
+}
+
+//===----------------------------------------------------------------------===//
+//                       Equality/diseqiality abstraction
+//===----------------------------------------------------------------------===//
+
+/// A small helper structure representing symbolic equality.
+///
+/// Equality check can have different forms (like a == b or a - b) and this
+/// class encapsulates those away if the only thing the user wants to check -
+/// whether it's equality/diseqiality or not and have an easy access to the
+/// compared symbols.
+struct EqualityInfo {
+public:
+  SymbolRef Left, Right;
+  // true for equality and false for disequality.
+  bool IsEquality = true;
+
+  void invert() { IsEquality = !IsEquality; }
+  /// Extract equality information from the given symbol and the constants.
+  ///
+  /// This function assumes the following expression Sym + Adjustment != Int.
+  /// It is a default because the most widespread case of the equality check
+  /// is (A == B) + 0 != 0.
+  static Optional<EqualityInfo> extract(SymbolRef Sym, const llvm::APSInt &Int,
+                                        const llvm::APSInt &Adjustment) {
+    // As of now, the only equality form supported is Sym + 0 != 0.
+    if (!Int.isNullValue() || !Adjustment.isNullValue())
+      return llvm::None;
+
+    return extract(Sym);
+  }
+  /// Extract equality information from the given symbol.
+  static Optional<EqualityInfo> extract(SymbolRef Sym) {
+    return EqualityExtractor().Visit(Sym);
+  }
+
+private:
+  class EqualityExtractor
+      : public SymExprVisitor<EqualityExtractor, Optional<EqualityInfo>> {
+  public:
+    Optional<EqualityInfo> VisitSymSymExpr(const SymSymExpr *Sym) const {
+      switch (Sym->getOpcode()) {
+      case BO_Sub:
+        // This case is: A - B != 0 -> disequality check.
+        return EqualityInfo{Sym->getLHS(), Sym->getRHS(), false};
+      case BO_EQ:
+        // This case is: A == B != 0 -> equality check.
+        return EqualityInfo{Sym->getLHS(), Sym->getRHS(), true};
+      case BO_NE:
+        // This case is: A != B != 0 -> diseqiality check.
+        return EqualityInfo{Sym->getLHS(), Sym->getRHS(), false};
+      default:
+        return llvm::None;
+      }
+    }
+  };
+};
+
+//===----------------------------------------------------------------------===//
+//                            Intersection functions
+//===----------------------------------------------------------------------===//
+
+template <class SecondTy, class... RestTy>
+LLVM_NODISCARD inline RangeSet intersect(BasicValueFactory &BV,
+                                         RangeSet::Factory &F, RangeSet Head,
+                                         SecondTy Second, RestTy... Tail);
+
+template <class... RangeTy> struct IntersectionTraits;
+
+template <class... TailTy> struct IntersectionTraits<RangeSet, TailTy...> {
+  // Found RangeSet, no need to check any further
+  using Type = RangeSet;
+};
+
+template <> struct IntersectionTraits<> {
+  // We ran out of types, and we didn't find any RangeSet, so the result should
+  // be optional.
+  using Type = Optional<RangeSet>;
+};
+
+template <class OptionalOrPointer, class... TailTy>
+struct IntersectionTraits<OptionalOrPointer, TailTy...> {
+  // If current type is Optional or a raw pointer, we should keep looking.
+  using Type = typename IntersectionTraits<TailTy...>::Type;
+};
+
+template <class EndTy>
+LLVM_NODISCARD inline EndTy intersect(BasicValueFactory &BV,
+                                      RangeSet::Factory &F, EndTy End) {
+  // If the list contains only RangeSet or Optional<RangeSet>, simply return
+  // that range set.
+  return End;
+}
+
+LLVM_NODISCARD LLVM_ATTRIBUTE_UNUSED inline Optional<RangeSet>
+intersect(BasicValueFactory &BV, RangeSet::Factory &F, const RangeSet *End) {
+  // This is an extraneous conversion from a raw pointer into Optional<RangeSet>
+  if (End) {
+    return *End;
+  }
+  return llvm::None;
+}
+
+template <class... RestTy>
+LLVM_NODISCARD inline RangeSet intersect(BasicValueFactory &BV,
+                                         RangeSet::Factory &F, RangeSet Head,
+                                         RangeSet Second, RestTy... Tail) {
+  // Here we call either the <RangeSet,RangeSet,...> or <RangeSet,...> version
+  // of the function and can be sure that the result is RangeSet.
+  return intersect(BV, F, Head.Intersect(BV, F, Second), Tail...);
+}
+
+template <class SecondTy, class... RestTy>
+LLVM_NODISCARD inline RangeSet intersect(BasicValueFactory &BV,
+                                         RangeSet::Factory &F, RangeSet Head,
+                                         SecondTy Second, RestTy... Tail) {
+  if (Second) {
+    // Here we call the <RangeSet,RangeSet,...> version of the function...
+    return intersect(BV, F, Head, *Second, Tail...);
+  }
+  // ...and here it is either <RangeSet,RangeSet,...> or <RangeSet,...>, which
+  // means that the result is definitely RangeSet.
+  return intersect(BV, F, Head, Tail...);
+}
+
+/// Main generic intersect function.
+/// It intersects all of the given range sets.  If some of the given arguments
+/// don't hold a range set (nullptr or llvm::None), the function will skip them.
+///
+/// Available representations for the arguments are:
+///   * RangeSet
+///   * Optional<RangeSet>
+///   * RangeSet *
+/// Pointer to a RangeSet is automatically assumed to be nullable and will get
+/// checked as well as the optional version.  If this behaviour is undesired,
+/// please dereference the pointer in the call.
+///
+/// Return type depends on the arguments' types.  If we can be sure in compile
+/// time that there will be a range set as a result, the returning type is
+/// simply RangeSet, in other cases we have to back off to Optional<RangeSet>.
+///
+/// Please, prefer optional range sets to raw pointers.  If the last argument is
+/// a raw pointer and all previous arguments are None, it will cost one
+/// additional check to convert RangeSet * into Optional<RangeSet>.
+template <class HeadTy, class SecondTy, class... RestTy>
+LLVM_NODISCARD inline
+    typename IntersectionTraits<HeadTy, SecondTy, RestTy...>::Type
+    intersect(BasicValueFactory &BV, RangeSet::Factory &F, HeadTy Head,
+              SecondTy Second, RestTy... Tail) {
+  if (Head) {
+    return intersect(BV, F, *Head, Second, Tail...);
+  }
+  return intersect(BV, F, Second, Tail...);
+}
+
+//===----------------------------------------------------------------------===//
+//                           Symbolic reasoning logic
+//===----------------------------------------------------------------------===//
 
 /// A little component aggregating all of the reasoning we have about
 /// the ranges of symbolic expressions.
@@ -389,10 +709,11 @@ namespace {
 class SymbolicRangeInferrer
     : public SymExprVisitor<SymbolicRangeInferrer, RangeSet> {
 public:
+  template <class SourceType>
   static RangeSet inferRange(BasicValueFactory &BV, RangeSet::Factory &F,
-                             ProgramStateRef State, SymbolRef Sym) {
+                             ProgramStateRef State, SourceType Origin) {
     SymbolicRangeInferrer Inferrer(BV, F, State);
-    return Inferrer.infer(Sym);
+    return Inferrer.infer(Origin);
   }
 
   RangeSet VisitSymExpr(SymbolRef Sym) {
@@ -442,37 +763,35 @@ class SymbolicRangeInferrer
   }
 
   RangeSet infer(SymbolRef Sym) {
-    const RangeSet *AssociatedRange = State->get<ConstraintRange>(Sym);
-
-    // If Sym is a difference of symbols A - B, then maybe we have range set
-    // stored for B - A.
-    const RangeSet *RangeAssociatedWithNegatedSym =
-        getRangeForMinusSymbol(State, Sym);
-
-    // If we have range set stored for both A - B and B - A then calculate the
-    // effective range set by intersecting the range set for A - B and the
-    // negated range set of B - A.
-    if (AssociatedRange && RangeAssociatedWithNegatedSym)
-      return AssociatedRange->Intersect(
-          ValueFactory, RangeFactory,
-          RangeAssociatedWithNegatedSym->Negate(ValueFactory, RangeFactory));
-
-    if (AssociatedRange)
-      return *AssociatedRange;
-
-    if (RangeAssociatedWithNegatedSym)
-      return RangeAssociatedWithNegatedSym->Negate(ValueFactory, RangeFactory);
+    if (Optional<RangeSet> ConstraintBasedRange = intersect(
+            ValueFactory, RangeFactory, getConstraint(State, Sym),
+            // If Sym is a difference of symbols A - B, then maybe we have range
+            // set stored for B - A.
+            //
+            // If we have range set stored for both A - B and B - A then
+            // calculate the effective range set by intersecting the range set
+            // for A - B and the negated range set of B - A.
+            getRangeForNegatedSub(Sym), getRangeForEqualities(Sym))) {
+      return *ConstraintBasedRange;
+    }
 
     // If Sym is a comparison expression (except <=>),
     // find any other comparisons with the same operands.
     // See function description.
-    const RangeSet CmpRangeSet = getRangeForComparisonSymbol(State, Sym);
-    if (!CmpRangeSet.isEmpty())
-      return CmpRangeSet;
+    if (Optional<RangeSet> CmpRangeSet = getRangeForComparisonSymbol(Sym)) {
+      return *CmpRangeSet;
+    }
 
     return Visit(Sym);
   }
 
+  RangeSet infer(EquivalenceClass Class) {
+    if (const RangeSet *AssociatedConstraint = getConstraint(State, Class))
+      return *AssociatedConstraint;
+
+    return infer(Class.getType());
+  }
+
   /// Infer range information solely from the type.
   RangeSet infer(QualType T) {
     // Lazily generate a new RangeSet representing all possible values for the
@@ -621,8 +940,7 @@ class SymbolicRangeInferrer
   /// Return a range set subtracting zero from \p Domain.
   RangeSet assumeNonZero(RangeSet Domain, QualType T) {
     APSIntType IntType = ValueFactory.getAPSIntType(T);
-    return Domain.Intersect(ValueFactory, RangeFactory,
-                            ++IntType.getZeroValue(), --IntType.getZeroValue());
+    return Domain.Delete(ValueFactory, RangeFactory, IntType.getZeroValue());
   }
 
   // FIXME: Once SValBuilder supports unary minus, we should use SValBuilder to
@@ -630,23 +948,26 @@ class SymbolicRangeInferrer
   //        symbol manually. This will allow us to support finding ranges of not
   //        only negated SymSymExpr-type expressions, but also of other, simpler
   //        expressions which we currently do not know how to negate.
-  const RangeSet *getRangeForMinusSymbol(ProgramStateRef State, SymbolRef Sym) {
+  Optional<RangeSet> getRangeForNegatedSub(SymbolRef Sym) {
     if (const SymSymExpr *SSE = dyn_cast<SymSymExpr>(Sym)) {
       if (SSE->getOpcode() == BO_Sub) {
         QualType T = Sym->getType();
+
+        // Do not negate unsigned ranges
+        if (!T->isUnsignedIntegerOrEnumerationType() &&
+            !T->isSignedIntegerOrEnumerationType())
+          return llvm::None;
+
         SymbolManager &SymMgr = State->getSymbolManager();
-        SymbolRef negSym =
+        SymbolRef NegatedSym =
             SymMgr.getSymSymExpr(SSE->getRHS(), BO_Sub, SSE->getLHS(), T);
 
-        if (const RangeSet *negV = State->get<ConstraintRange>(negSym)) {
-          // Unsigned range set cannot be negated, unless it is [0, 0].
-          if (T->isUnsignedIntegerOrEnumerationType() ||
-              T->isSignedIntegerOrEnumerationType())
-            return negV;
+        if (const RangeSet *NegatedRange = getConstraint(State, NegatedSym)) {
+          return NegatedRange->Negate(ValueFactory, RangeFactory);
         }
       }
     }
-    return nullptr;
+    return llvm::None;
   }
 
   // Returns ranges only for binary comparison operators (except <=>)
@@ -659,18 +980,16 @@ class SymbolicRangeInferrer
   // It covers all possible combinations (see CmpOpTable description).
   // Note that `x` and `y` can also stand for subexpressions,
   // not only for actual symbols.
-  RangeSet getRangeForComparisonSymbol(ProgramStateRef State, SymbolRef Sym) {
-    const RangeSet EmptyRangeSet = RangeFactory.getEmptySet();
-
-    auto SSE = dyn_cast<SymSymExpr>(Sym);
+  Optional<RangeSet> getRangeForComparisonSymbol(SymbolRef Sym) {
+    const auto *SSE = dyn_cast<SymSymExpr>(Sym);
     if (!SSE)
-      return EmptyRangeSet;
+      return llvm::None;
 
     BinaryOperatorKind CurrentOP = SSE->getOpcode();
 
     // We currently do not support <=> (C++20).
     if (!BinaryOperator::isComparisonOp(CurrentOP) || (CurrentOP == BO_Cmp))
-      return EmptyRangeSet;
+      return llvm::None;
 
     static const OperatorRelationsTable CmpOpTable{};
 
@@ -679,10 +998,6 @@ class SymbolicRangeInferrer
     QualType T = SSE->getType();
 
     SymbolManager &SymMgr = State->getSymbolManager();
-    const llvm::APSInt &Zero = ValueFactory.getValue(0, T);
-    const llvm::APSInt &One = ValueFactory.getValue(1, T);
-    const RangeSet TrueRangeSet(RangeFactory, One, One);
-    const RangeSet FalseRangeSet(RangeFactory, Zero, Zero);
 
     int UnknownStates = 0;
 
@@ -693,7 +1008,7 @@ class SymbolicRangeInferrer
       // Let's find an expression e.g. (x < y).
       BinaryOperatorKind QueriedOP = OperatorRelationsTable::getOpFromIndex(i);
       const SymSymExpr *SymSym = SymMgr.getSymSymExpr(LHS, QueriedOP, RHS, T);
-      const RangeSet *QueriedRangeSet = State->get<ConstraintRange>(SymSym);
+      const RangeSet *QueriedRangeSet = getConstraint(State, SymSym);
 
       // If ranges were not previously found,
       // try to find a reversed expression (y > x).
@@ -701,7 +1016,7 @@ class SymbolicRangeInferrer
         const BinaryOperatorKind ROP =
             BinaryOperator::reverseComparisonOp(QueriedOP);
         SymSym = SymMgr.getSymSymExpr(RHS, ROP, LHS, T);
-        QueriedRangeSet = State->get<ConstraintRange>(SymSym);
+        QueriedRangeSet = getConstraint(State, SymSym);
       }
 
       if (!QueriedRangeSet || QueriedRangeSet->isEmpty())
@@ -732,11 +1047,38 @@ class SymbolicRangeInferrer
           continue;
       }
 
-      return (BranchState == OperatorRelationsTable::True) ? TrueRangeSet
-                                                           : FalseRangeSet;
+      return (BranchState == OperatorRelationsTable::True) ? getTrueRange(T)
+                                                           : getFalseRange(T);
     }
 
-    return EmptyRangeSet;
+    return llvm::None;
+  }
+
+  Optional<RangeSet> getRangeForEqualities(SymbolRef Sym) {
+    Optional<EqualityInfo> Equality = EqualityInfo::extract(Sym);
+
+    if (!Equality)
+      return llvm::None;
+
+    if (Optional<bool> AreEqual = EquivalenceClass::areEqual(
+            State, Equality->Left, Equality->Right)) {
+      if (*AreEqual == Equality->IsEquality) {
+        return getTrueRange(Sym->getType());
+      }
+      return getFalseRange(Sym->getType());
+    }
+
+    return llvm::None;
+  }
+
+  RangeSet getTrueRange(QualType T) {
+    RangeSet TypeRange = infer(T);
+    return assumeNonZero(TypeRange, T);
+  }
+
+  RangeSet getFalseRange(QualType T) {
+    const llvm::APSInt &Zero = ValueFactory.getValue(0, T);
+    return RangeSet(RangeFactory, Zero);
   }
 
   BasicValueFactory &ValueFactory;
@@ -744,6 +1086,10 @@ class SymbolicRangeInferrer
   ProgramStateRef State;
 };
 
+//===----------------------------------------------------------------------===//
+//               Range-based reasoning about symbolic operations
+//===----------------------------------------------------------------------===//
+
 template <>
 RangeSet SymbolicRangeInferrer::VisitBinaryOperator<BO_Or>(Range LHS, Range RHS,
                                                            QualType T) {
@@ -904,6 +1250,10 @@ RangeSet SymbolicRangeInferrer::VisitBinaryOperator<BO_Rem>(Range LHS,
   return {RangeFactory, ValueFactory.getValue(Min), ValueFactory.getValue(Max)};
 }
 
+//===----------------------------------------------------------------------===//
+//                  Constraint manager implementation details
+//===----------------------------------------------------------------------===//
+
 class RangeConstraintManager : public RangedConstraintManager {
 public:
   RangeConstraintManager(ExprEngine *EE, SValBuilder &SVB)
@@ -915,7 +1265,11 @@ class RangeConstraintManager : public RangedConstraintManager {
 
   bool haveEqualConstraints(ProgramStateRef S1,
                             ProgramStateRef S2) const override {
-    return S1->get<ConstraintRange>() == S2->get<ConstraintRange>();
+    // NOTE: ClassMembers are as simple as back pointers for ClassMap,
+    //       so comparing constraint ranges and class maps should be
+    //       sufficient.
+    return S1->get<ConstraintRange>() == S2->get<ConstraintRange>() &&
+           S1->get<ClassMap>() == S2->get<ClassMap>();
   }
 
   bool canReasonAbout(SVal X) const override;
@@ -971,6 +1325,7 @@ class RangeConstraintManager : public RangedConstraintManager {
   RangeSet::Factory F;
 
   RangeSet getRange(ProgramStateRef State, SymbolRef Sym);
+  RangeSet getRange(ProgramStateRef State, EquivalenceClass Class);
 
   RangeSet getSymLTRange(ProgramStateRef St, SymbolRef Sym,
                          const llvm::APSInt &Int,
@@ -987,6 +1342,79 @@ class RangeConstraintManager : public RangedConstraintManager {
   RangeSet getSymGERange(ProgramStateRef St, SymbolRef Sym,
                          const llvm::APSInt &Int,
                          const llvm::APSInt &Adjustment);
+
+  //===------------------------------------------------------------------===//
+  // Equality tracking implementation
+  //===------------------------------------------------------------------===//
+
+  ProgramStateRef trackEQ(ProgramStateRef State, SymbolRef Sym,
+                          const llvm::APSInt &Int,
+                          const llvm::APSInt &Adjustment) {
+    if (auto Equality = EqualityInfo::extract(Sym, Int, Adjustment)) {
+      // Extract function assumes that we gave it Sym + Adjustment != Int,
+      // so the result should be opposite.
+      Equality->invert();
+      return track(State, *Equality);
+    }
+
+    return State;
+  }
+
+  ProgramStateRef trackNE(ProgramStateRef State, SymbolRef Sym,
+                          const llvm::APSInt &Int,
+                          const llvm::APSInt &Adjustment) {
+    if (auto Equality = EqualityInfo::extract(Sym, Int, Adjustment)) {
+      return track(State, *Equality);
+    }
+
+    return State;
+  }
+
+  ProgramStateRef track(ProgramStateRef State, EqualityInfo ToTrack) {
+    if (ToTrack.IsEquality) {
+      return trackEquality(State, ToTrack.Left, ToTrack.Right);
+    }
+    return trackDisequality(State, ToTrack.Left, ToTrack.Right);
+  }
+
+  ProgramStateRef trackDisequality(ProgramStateRef State, SymbolRef LHS,
+                                   SymbolRef RHS) {
+    return EquivalenceClass::markDisequal(getBasicVals(), F, State, LHS, RHS);
+  }
+
+  ProgramStateRef trackEquality(ProgramStateRef State, SymbolRef LHS,
+                                SymbolRef RHS) {
+    return EquivalenceClass::merge(getBasicVals(), F, State, LHS, RHS);
+  }
+
+  LLVM_NODISCARD inline ProgramStateRef setConstraint(ProgramStateRef State,
+                                                      EquivalenceClass Class,
+                                                      RangeSet Constraint) {
+    ConstraintRangeTy Constraints = State->get<ConstraintRange>();
+    ConstraintRangeTy::Factory &CF = State->get_context<ConstraintRange>();
+
+    // Add new constraint.
+    Constraints = CF.add(Constraints, Class, Constraint);
+
+    // There is a chance that we might need to update constraints for the
+    // classes that are known to be disequal to Class.
+    //
+    // In order for this to be even possible, the new constraint should
+    // be simply a constant because we can't reason about range disequalities.
+    if (const llvm::APSInt *Point = Constraint.getConcreteValue())
+      for (EquivalenceClass DisequalClass : Class.getDisequalClasses(State)) {
+        RangeSet UpdatedConstraint =
+            getRange(State, DisequalClass).Delete(getBasicVals(), F, *Point);
+        Constraints = CF.add(Constraints, DisequalClass, UpdatedConstraint);
+      }
+
+    return State->set<ConstraintRange>(Constraints);
+  }
+
+  LLVM_NODISCARD inline ProgramStateRef
+  setConstraint(ProgramStateRef State, SymbolRef Sym, RangeSet Constraint) {
+    return setConstraint(State, EquivalenceClass::find(State, Sym), Constraint);
+  }
 };
 
 } // end anonymous namespace
@@ -997,6 +1425,372 @@ ento::CreateRangeConstraintManager(ProgramStateManager &StMgr,
   return std::make_unique<RangeConstraintManager>(Eng, StMgr.getSValBuilder());
 }
 
+ConstraintMap ento::getConstraintMap(ProgramStateRef State) {
+  ConstraintMap::Factory &F = State->get_context<ConstraintMap>();
+  ConstraintMap Result = F.getEmptyMap();
+
+  ConstraintRangeTy Constraints = State->get<ConstraintRange>();
+  for (std::pair<EquivalenceClass, RangeSet> ClassConstraint : Constraints) {
+    EquivalenceClass Class = ClassConstraint.first;
+    SymbolSet ClassMembers = Class.getClassMembers(State);
+    assert(!ClassMembers.isEmpty() &&
+           "Class must always have at least one member!");
+
+    SymbolRef Representative = *ClassMembers.begin();
+    Result = F.add(Result, Representative, ClassConstraint.second);
+  }
+
+  return Result;
+}
+
+//===----------------------------------------------------------------------===//
+//                     EqualityClass implementation details
+//===----------------------------------------------------------------------===//
+
+inline EquivalenceClass EquivalenceClass::find(ProgramStateRef State,
+                                               SymbolRef Sym) {
+  // We store far from all Symbol -> Class mappings
+  if (const EquivalenceClass *NontrivialClass = State->get<ClassMap>(Sym))
+    return *NontrivialClass;
+
+  // This is a trivial class of Sym.
+  return Sym;
+}
+
+inline ProgramStateRef EquivalenceClass::merge(BasicValueFactory &BV,
+                                               RangeSet::Factory &F,
+                                               ProgramStateRef State,
+                                               SymbolRef First,
+                                               SymbolRef Second) {
+  EquivalenceClass FirstClass = find(State, First);
+  EquivalenceClass SecondClass = find(State, Second);
+
+  return FirstClass.merge(BV, F, State, SecondClass);
+}
+
+inline ProgramStateRef EquivalenceClass::merge(BasicValueFactory &BV,
+                                               RangeSet::Factory &F,
+                                               ProgramStateRef State,
+                                               EquivalenceClass Other) {
+  // It is already the same class.
+  if (*this == Other)
+    return State;
+
+  // FIXME: As of now, we support only equivalence classes of the same type.
+  //        This limitation is connected to the lack of explicit casts in
+  //        our symbolic expression model.
+  //
+  //        That means that for `int x` and `char y` we don't distinguish
+  //        between these two very different cases:
+  //          * `x == y`
+  //          * `(char)x == y`
+  //
+  //        The moment we introduce symbolic casts, this restriction can be
+  //        lifted.
+  if (getType() != Other.getType())
+    return State;
+
+  SymbolSet Members = getClassMembers(State);
+  SymbolSet OtherMembers = Other.getClassMembers(State);
+
+  // We estimate the size of the class by the height of tree containing
+  // its members.  Merging is not a trivial operation, so it's easier to
+  // merge the smaller class into the bigger one.
+  if (Members.getHeight() >= OtherMembers.getHeight()) {
+    return mergeImpl(BV, F, State, Members, Other, OtherMembers);
+  } else {
+    return Other.mergeImpl(BV, F, State, OtherMembers, *this, Members);
+  }
+}
+
+inline ProgramStateRef
+EquivalenceClass::mergeImpl(BasicValueFactory &ValueFactory,
+                            RangeSet::Factory &RangeFactory,
+                            ProgramStateRef State, SymbolSet MyMembers,
+                            EquivalenceClass Other, SymbolSet OtherMembers) {
+  // Essentially what we try to recreate here is some kind of union-find
+  // data structure.  It does have certain limitations due to persistence
+  // and the need to remove elements from classes.
+  //
+  // In this setting, EquialityClass object is the representative of the class
+  // or the parent element.  ClassMap is a mapping of class members to their
+  // parent. Unlike the union-find structure, they all point directly to the
+  // class representative because we don't have an opportunity to actually do
+  // path compression when dealing with immutability.  This means that we
+  // compress paths every time we do merges.  It also means that we lose
+  // the main amortized complexity benefit from the original data structure.
+  ConstraintRangeTy Constraints = State->get<ConstraintRange>();
+  ConstraintRangeTy::Factory &CRF = State->get_context<ConstraintRange>();
+
+  // 1. If the merged classes have any constraints associated with them, we
+  //    need to transfer them to the class we have left.
+  //
+  // Intersection here makes perfect sense because both of these constraints
+  // must hold for the whole new class.
+  if (Optional<RangeSet> NewClassConstraint =
+          intersect(ValueFactory, RangeFactory, getConstraint(State, *this),
+                    getConstraint(State, Other))) {
+    // NOTE: Essentially, NewClassConstraint should NEVER be infeasible because
+    //       range inferrer shouldn't generate ranges incompatible with
+    //       equivalence classes. However, at the moment, due to imperfections
+    //       in the solver, it is possible and the merge function can also
+    //       return infeasible states aka null states.
+    if (NewClassConstraint->isEmpty())
+      // Infeasible state
+      return nullptr;
+
+    // No need in tracking constraints of a now-dissolved class.
+    Constraints = CRF.remove(Constraints, Other);
+    // Assign new constraints for this class.
+    Constraints = CRF.add(Constraints, *this, *NewClassConstraint);
+
+    State = State->set<ConstraintRange>(Constraints);
+  }
+
+  // 2. Get ALL equivalence-related maps
+  ClassMapTy Classes = State->get<ClassMap>();
+  ClassMapTy::Factory &CMF = State->get_context<ClassMap>();
+
+  ClassMembersTy Members = State->get<ClassMembers>();
+  ClassMembersTy::Factory &MF = State->get_context<ClassMembers>();
+
+  DisequalityMapTy DisequalityInfo = State->get<DisequalityMap>();
+  DisequalityMapTy::Factory &DF = State->get_context<DisequalityMap>();
+
+  ClassSet::Factory &CF = State->get_context<ClassSet>();
+  SymbolSet::Factory &F = getMembersFactory(State);
+
+  // 2. Merge members of the Other class into the current class.
+  SymbolSet NewClassMembers = MyMembers;
+  for (SymbolRef Sym : OtherMembers) {
+    NewClassMembers = F.add(NewClassMembers, Sym);
+    // *this is now the class for all these new symbols.
+    Classes = CMF.add(Classes, Sym, *this);
+  }
+
+  // 3. Adjust member mapping.
+  //
+  // No need in tracking members of a now-dissolved class.
+  Members = MF.remove(Members, Other);
+  // Now only the current class is mapped to all the symbols.
+  Members = MF.add(Members, *this, NewClassMembers);
+
+  // 4. Update disequality relations
+  ClassSet DisequalToOther = Other.getDisequalClasses(DisequalityInfo, CF);
+  if (!DisequalToOther.isEmpty()) {
+    ClassSet DisequalToThis = getDisequalClasses(DisequalityInfo, CF);
+    DisequalityInfo = DF.remove(DisequalityInfo, Other);
+
+    for (EquivalenceClass DisequalClass : DisequalToOther) {
+      DisequalToThis = CF.add(DisequalToThis, DisequalClass);
+
+      // Disequality is a symmetric relation meaning that if
+      // DisequalToOther not null then the set for DisequalClass is not
+      // empty and has at least Other.
+      ClassSet OriginalSetLinkedToOther =
+          *DisequalityInfo.lookup(DisequalClass);
+
+      // Other will be eliminated and we should replace it with the bigger
+      // united class.
+      ClassSet NewSet = CF.remove(OriginalSetLinkedToOther, Other);
+      NewSet = CF.add(NewSet, *this);
+
+      DisequalityInfo = DF.add(DisequalityInfo, DisequalClass, NewSet);
+    }
+
+    DisequalityInfo = DF.add(DisequalityInfo, *this, DisequalToThis);
+    State = State->set<DisequalityMap>(DisequalityInfo);
+  }
+
+  // 5. Update the state
+  State = State->set<ClassMap>(Classes);
+  State = State->set<ClassMembers>(Members);
+
+  return State;
+}
+
+inline SymbolSet::Factory &
+EquivalenceClass::getMembersFactory(ProgramStateRef State) {
+  return State->get_context<SymbolSet>();
+}
+
+SymbolSet EquivalenceClass::getClassMembers(ProgramStateRef State) {
+  if (const SymbolSet *Members = State->get<ClassMembers>(*this))
+    return *Members;
+
+  // This class is trivial, so we need to construct a set
+  // with just that one symbol from the class.
+  SymbolSet::Factory &F = getMembersFactory(State);
+  return F.add(F.getEmptySet(), getRepresentativeSymbol());
+}
+
+bool EquivalenceClass::isTrivial(ProgramStateRef State) {
+  return State->get<ClassMembers>(*this) == nullptr;
+}
+
+bool EquivalenceClass::isTriviallyDead(ProgramStateRef State,
+                                       SymbolReaper &Reaper) {
+  return isTrivial(State) && Reaper.isDead(getRepresentativeSymbol());
+}
+
+inline ProgramStateRef EquivalenceClass::markDisequal(BasicValueFactory &VF,
+                                                      RangeSet::Factory &RF,
+                                                      ProgramStateRef State,
+                                                      SymbolRef First,
+                                                      SymbolRef Second) {
+  return markDisequal(VF, RF, State, find(State, First), find(State, Second));
+}
+
+inline ProgramStateRef EquivalenceClass::markDisequal(BasicValueFactory &VF,
+                                                      RangeSet::Factory &RF,
+                                                      ProgramStateRef State,
+                                                      EquivalenceClass First,
+                                                      EquivalenceClass Second) {
+  return First.markDisequal(VF, RF, State, Second);
+}
+
+inline ProgramStateRef
+EquivalenceClass::markDisequal(BasicValueFactory &VF, RangeSet::Factory &RF,
+                               ProgramStateRef State,
+                               EquivalenceClass Other) const {
+  // If we know that two classes are equal, we can only produce an infeasible
+  // state.
+  if (*this == Other) {
+    return nullptr;
+  }
+
+  DisequalityMapTy DisequalityInfo = State->get<DisequalityMap>();
+  ConstraintRangeTy Constraints = State->get<ConstraintRange>();
+
+  // Disequality is a symmetric relation, so if we mark A as disequal to B,
+  // we should also mark B as disequalt to A.
+  addToDisequalityInfo(DisequalityInfo, Constraints, VF, RF, State, *this,
+                       Other);
+  addToDisequalityInfo(DisequalityInfo, Constraints, VF, RF, State, Other,
+                       *this);
+
+  State = State->set<DisequalityMap>(DisequalityInfo);
+  State = State->set<ConstraintRange>(Constraints);
+
+  return State;
+}
+
+inline void EquivalenceClass::addToDisequalityInfo(
+    DisequalityMapTy &Info, ConstraintRangeTy &Constraints,
+    BasicValueFactory &VF, RangeSet::Factory &RF, ProgramStateRef State,
+    EquivalenceClass First, EquivalenceClass Second) {
+
+  // 1. Get all of the required factories.
+  DisequalityMapTy::Factory &F = State->get_context<DisequalityMap>();
+  ClassSet::Factory &CF = State->get_context<ClassSet>();
+  ConstraintRangeTy::Factory &CRF = State->get_context<ConstraintRange>();
+
+  // 2. Add Second to the set of classes disequal to First.
+  const ClassSet *CurrentSet = Info.lookup(First);
+  ClassSet NewSet = CurrentSet ? *CurrentSet : CF.getEmptySet();
+  NewSet = CF.add(NewSet, Second);
+
+  Info = F.add(Info, First, NewSet);
+
+  // 3. If Second is known to be a constant, we can delete this point
+  //    from the constraint asociated with First.
+  //
+  //    So, if Second == 10, it means that First != 10.
+  //    At the same time, the same logic does not apply to ranges.
+  if (const RangeSet *SecondConstraint = Constraints.lookup(Second))
+    if (const llvm::APSInt *Point = SecondConstraint->getConcreteValue()) {
+
+      RangeSet FirstConstraint = SymbolicRangeInferrer::inferRange(
+          VF, RF, State, First.getRepresentativeSymbol());
+
+      FirstConstraint = FirstConstraint.Delete(VF, RF, *Point);
+      Constraints = CRF.add(Constraints, First, FirstConstraint);
+    }
+}
+
+inline Optional<bool> EquivalenceClass::areEqual(ProgramStateRef State,
+                                                 SymbolRef FirstSym,
+                                                 SymbolRef SecondSym) {
+  EquivalenceClass First = find(State, FirstSym);
+  EquivalenceClass Second = find(State, SecondSym);
+
+  // The same equivalence class => symbols are equal.
+  if (First == Second)
+    return true;
+
+  // Let's check if we know anything about these two classes being not equal to
+  // each other.
+  ClassSet DisequalToFirst = First.getDisequalClasses(State);
+  if (DisequalToFirst.contains(Second))
+    return false;
+
+  // It is not clear.
+  return llvm::None;
+}
+
+inline ClassSet EquivalenceClass::getDisequalClasses(ProgramStateRef State,
+                                                     SymbolRef Sym) {
+  return find(State, Sym).getDisequalClasses(State);
+}
+
+inline ClassSet
+EquivalenceClass::getDisequalClasses(ProgramStateRef State) const {
+  return getDisequalClasses(State->get<DisequalityMap>(),
+                            State->get_context<ClassSet>());
+}
+
+inline ClassSet
+EquivalenceClass::getDisequalClasses(DisequalityMapTy Map,
+                                     ClassSet::Factory &Factory) const {
+  if (const ClassSet *DisequalClasses = Map.lookup(*this))
+    return *DisequalClasses;
+
+  return Factory.getEmptySet();
+}
+
+bool EquivalenceClass::isClassDataConsistent(ProgramStateRef State) {
+  ClassMembersTy Members = State->get<ClassMembers>();
+
+  for (std::pair<EquivalenceClass, SymbolSet> ClassMembersPair : Members) {
+    for (SymbolRef Member : ClassMembersPair.second) {
+      // Every member of the class should have a mapping back to the class.
+      if (find(State, Member) == ClassMembersPair.first) {
+        continue;
+      }
+
+      return false;
+    }
+  }
+
+  DisequalityMapTy Disequalities = State->get<DisequalityMap>();
+  for (std::pair<EquivalenceClass, ClassSet> DisequalityInfo : Disequalities) {
+    EquivalenceClass Class = DisequalityInfo.first;
+    ClassSet DisequalClasses = DisequalityInfo.second;
+
+    // There is no use in keeping empty sets in the map.
+    if (DisequalClasses.isEmpty())
+      return false;
+
+    // Disequality is symmetrical, i.e. for every Class A and B that A != B,
+    // B != A should also be true.
+    for (EquivalenceClass DisequalClass : DisequalClasses) {
+      const ClassSet *DisequalToDisequalClasses =
+          Disequalities.lookup(DisequalClass);
+
+      // It should be a set of at least one element: Class
+      if (!DisequalToDisequalClasses ||
+          !DisequalToDisequalClasses->contains(Class))
+        return false;
+    }
+  }
+
+  return true;
+}
+
+//===----------------------------------------------------------------------===//
+//                    RangeConstraintManager implementation
+//===----------------------------------------------------------------------===//
+
 bool RangeConstraintManager::canReasonAbout(SVal X) const {
   Optional<nonloc::SymbolVal> SymVal = X.getAs<nonloc::SymbolVal>();
   if (SymVal && SymVal->isExpression()) {
@@ -1045,7 +1839,7 @@ bool RangeConstraintManager::canReasonAbout(SVal X) const {
 
 ConditionTruthVal RangeConstraintManager::checkNull(ProgramStateRef State,
                                                     SymbolRef Sym) {
-  const RangeSet *Ranges = State->get<ConstraintRange>(Sym);
+  const RangeSet *Ranges = getConstraint(State, Sym);
 
   // If we don't have any information about this symbol, it's underconstrained.
   if (!Ranges)
@@ -1069,28 +1863,148 @@ ConditionTruthVal RangeConstraintManager::checkNull(ProgramStateRef State,
 
 const llvm::APSInt *RangeConstraintManager::getSymVal(ProgramStateRef St,
                                                       SymbolRef Sym) const {
-  const ConstraintRangeTy::data_type *T = St->get<ConstraintRange>(Sym);
+  const RangeSet *T = getConstraint(St, Sym);
   return T ? T->getConcreteValue() : nullptr;
 }
 
+//===----------------------------------------------------------------------===//
+//                Remove dead symbols from existing constraints
+//===----------------------------------------------------------------------===//
+
 /// Scan all symbols referenced by the constraints. If the symbol is not alive
 /// as marked in LSymbols, mark it as dead in DSymbols.
 ProgramStateRef
 RangeConstraintManager::removeDeadBindings(ProgramStateRef State,
                                            SymbolReaper &SymReaper) {
-  bool Changed = false;
-  ConstraintRangeTy CR = State->get<ConstraintRange>();
-  ConstraintRangeTy::Factory &CRFactory = State->get_context<ConstraintRange>();
+  ClassMembersTy ClassMembersMap = State->get<ClassMembers>();
+  ClassMembersTy NewClassMembersMap = ClassMembersMap;
+  ClassMembersTy::Factory &EMFactory = State->get_context<ClassMembers>();
+  SymbolSet::Factory &SetFactory = State->get_context<SymbolSet>();
+
+  ConstraintRangeTy Constraints = State->get<ConstraintRange>();
+  ConstraintRangeTy NewConstraints = Constraints;
+  ConstraintRangeTy::Factory &ConstraintFactory =
+      State->get_context<ConstraintRange>();
+
+  ClassMapTy Map = State->get<ClassMap>();
+  ClassMapTy NewMap = Map;
+  ClassMapTy::Factory &ClassFactory = State->get_context<ClassMap>();
+
+  DisequalityMapTy Disequalities = State->get<DisequalityMap>();
+  DisequalityMapTy::Factory &DisequalityFactory =
+      State->get_context<DisequalityMap>();
+  ClassSet::Factory &ClassSetFactory = State->get_context<ClassSet>();
+
+  bool ClassMapChanged = false;
+  bool MembersMapChanged = false;
+  bool ConstraintMapChanged = false;
+  bool DisequalitiesChanged = false;
+
+  auto removeDeadClass = [&](EquivalenceClass Class) {
+    // Remove associated constraint ranges.
+    Constraints = ConstraintFactory.remove(Constraints, Class);
+    ConstraintMapChanged = true;
+
+    // Update disequality information to not hold any information on the
+    // removed class.
+    ClassSet DisequalClasses =
+        Class.getDisequalClasses(Disequalities, ClassSetFactory);
+    if (!DisequalClasses.isEmpty()) {
+      for (EquivalenceClass DisequalClass : DisequalClasses) {
+        ClassSet DisequalToDisequalSet =
+            DisequalClass.getDisequalClasses(Disequalities, ClassSetFactory);
+        // DisequalToDisequalSet is guaranteed to be non-empty for consistent
+        // disequality info.
+        assert(!DisequalToDisequalSet.isEmpty());
+        ClassSet NewSet = ClassSetFactory.remove(DisequalToDisequalSet, Class);
+
+        // No need in keeping an empty set.
+        if (NewSet.isEmpty()) {
+          Disequalities =
+              DisequalityFactory.remove(Disequalities, DisequalClass);
+        } else {
+          Disequalities =
+              DisequalityFactory.add(Disequalities, DisequalClass, NewSet);
+        }
+      }
+      // Remove the data for the class
+      Disequalities = DisequalityFactory.remove(Disequalities, Class);
+      DisequalitiesChanged = true;
+    }
+  };
+
+  // 1. Let's see if dead symbols are trivial and have associated constraints.
+  for (std::pair<EquivalenceClass, RangeSet> ClassConstraintPair :
+       Constraints) {
+    EquivalenceClass Class = ClassConstraintPair.first;
+    if (Class.isTriviallyDead(State, SymReaper)) {
+      // If this class is trivial, we can remove its constraints right away.
+      removeDeadClass(Class);
+    }
+  }
+
+  // 2. We don't need to track classes for dead symbols.
+  for (std::pair<SymbolRef, EquivalenceClass> SymbolClassPair : Map) {
+    SymbolRef Sym = SymbolClassPair.first;
 
-  for (ConstraintRangeTy::iterator I = CR.begin(), E = CR.end(); I != E; ++I) {
-    SymbolRef Sym = I.getKey();
     if (SymReaper.isDead(Sym)) {
-      Changed = true;
-      CR = CRFactory.remove(CR, Sym);
+      ClassMapChanged = true;
+      NewMap = ClassFactory.remove(NewMap, Sym);
     }
   }
 
-  return Changed ? State->set<ConstraintRange>(CR) : State;
+  // 3. Remove dead members from classes and remove dead non-trivial classes
+  //    and their constraints.
+  for (std::pair<EquivalenceClass, SymbolSet> ClassMembersPair :
+       ClassMembersMap) {
+    EquivalenceClass Class = ClassMembersPair.first;
+    SymbolSet LiveMembers = ClassMembersPair.second;
+    bool MembersChanged = false;
+
+    for (SymbolRef Member : ClassMembersPair.second) {
+      if (SymReaper.isDead(Member)) {
+        MembersChanged = true;
+        LiveMembers = SetFactory.remove(LiveMembers, Member);
+      }
+    }
+
+    // Check if the class changed.
+    if (!MembersChanged)
+      continue;
+
+    MembersMapChanged = true;
+
+    if (LiveMembers.isEmpty()) {
+      // The class is dead now, we need to wipe it out of the members map...
+      NewClassMembersMap = EMFactory.remove(NewClassMembersMap, Class);
+
+      // ...and remove all of its constraints.
+      removeDeadClass(Class);
+    } else {
+      // We need to change the members associated with the class.
+      NewClassMembersMap =
+          EMFactory.add(NewClassMembersMap, Class, LiveMembers);
+    }
+  }
+
+  // 4. Update the state with new maps.
+  //
+  // Here we try to be humble and update a map only if it really changed.
+  if (ClassMapChanged)
+    State = State->set<ClassMap>(NewMap);
+
+  if (MembersMapChanged)
+    State = State->set<ClassMembers>(NewClassMembersMap);
+
+  if (ConstraintMapChanged)
+    State = State->set<ConstraintRange>(Constraints);
+
+  if (DisequalitiesChanged)
+    State = State->set<DisequalityMap>(Disequalities);
+
+  assert(EquivalenceClass::isClassDataConsistent(State));
+
+  return State;
 }
 
 RangeSet RangeConstraintManager::getRange(ProgramStateRef State,
@@ -1098,6 +2012,11 @@ RangeSet RangeConstraintManager::getRange(ProgramStateRef State,
   return SymbolicRangeInferrer::inferRange(getBasicVals(), F, State, Sym);
 }
 
+RangeSet RangeConstraintManager::getRange(ProgramStateRef State,
+                                          EquivalenceClass Class) {
+  return SymbolicRangeInferrer::inferRange(getBasicVals(), F, State, Class);
+}
+
 //===------------------------------------------------------------------------===
 // assumeSymX methods: protected interface for RangeConstraintManager.
 //===------------------------------------------------------------------------===/
@@ -1119,15 +2038,16 @@ RangeConstraintManager::assumeSymNE(ProgramStateRef St, SymbolRef Sym,
   if (AdjustmentType.testInRange(Int, true) != APSIntType::RTR_Within)
     return St;
 
-  llvm::APSInt Lower = AdjustmentType.convert(Int) - Adjustment;
-  llvm::APSInt Upper = Lower;
-  --Lower;
-  ++Upper;
+  llvm::APSInt Point = AdjustmentType.convert(Int) - Adjustment;
 
-  // [Int-Adjustment+1, Int-Adjustment-1]
-  // Notice that the lower bound is greater than the upper bound.
-  RangeSet New = getRange(St, Sym).Intersect(getBasicVals(), F, Upper, Lower);
-  return New.isEmpty() ? nullptr : St->set<ConstraintRange>(Sym, New);
+  RangeSet New = getRange(St, Sym).Delete(getBasicVals(), F, Point);
+
+  if (New.isEmpty())
+    // this is infeasible assumption
+    return nullptr;
+
+  ProgramStateRef NewState = setConstraint(St, Sym, New);
+  return trackNE(NewState, Sym, Int, Adjustment);
 }
 
 ProgramStateRef
@@ -1142,7 +2062,13 @@ RangeConstraintManager::assumeSymEQ(ProgramStateRef St, SymbolRef Sym,
   // [Int-Adjustment, Int-Adjustment]
   llvm::APSInt AdjInt = AdjustmentType.convert(Int) - Adjustment;
   RangeSet New = getRange(St, Sym).Intersect(getBasicVals(), F, AdjInt, AdjInt);
-  return New.isEmpty() ? nullptr : St->set<ConstraintRange>(Sym, New);
+
+  if (New.isEmpty())
+    // this is infeasible assumption
+    return nullptr;
+
+  ProgramStateRef NewState = setConstraint(St, Sym, New);
+  return trackEQ(NewState, Sym, Int, Adjustment);
 }
 
 RangeSet RangeConstraintManager::getSymLTRange(ProgramStateRef St,
@@ -1178,7 +2104,7 @@ RangeConstraintManager::assumeSymLT(ProgramStateRef St, SymbolRef Sym,
                                     const llvm::APSInt &Int,
                                     const llvm::APSInt &Adjustment) {
   RangeSet New = getSymLTRange(St, Sym, Int, Adjustment);
-  return New.isEmpty() ? nullptr : St->set<ConstraintRange>(Sym, New);
+  return New.isEmpty() ? nullptr : setConstraint(St, Sym, New);
 }
 
 RangeSet RangeConstraintManager::getSymGTRange(ProgramStateRef St,
@@ -1214,7 +2140,7 @@ RangeConstraintManager::assumeSymGT(ProgramStateRef St, SymbolRef Sym,
                                     const llvm::APSInt &Int,
                                     const llvm::APSInt &Adjustment) {
   RangeSet New = getSymGTRange(St, Sym, Int, Adjustment);
-  return New.isEmpty() ? nullptr : St->set<ConstraintRange>(Sym, New);
+  return New.isEmpty() ? nullptr : setConstraint(St, Sym, New);
 }
 
 RangeSet RangeConstraintManager::getSymGERange(ProgramStateRef St,
@@ -1250,13 +2176,13 @@ RangeConstraintManager::assumeSymGE(ProgramStateRef St, SymbolRef Sym,
                                     const llvm::APSInt &Int,
                                     const llvm::APSInt &Adjustment) {
   RangeSet New = getSymGERange(St, Sym, Int, Adjustment);
-  return New.isEmpty() ? nullptr : St->set<ConstraintRange>(Sym, New);
+  return New.isEmpty() ? nullptr : setConstraint(St, Sym, New);
 }
 
-RangeSet RangeConstraintManager::getSymLERange(
-      llvm::function_ref<RangeSet()> RS,
-      const llvm::APSInt &Int,
-      const llvm::APSInt &Adjustment) {
+RangeSet
+RangeConstraintManager::getSymLERange(llvm::function_ref<RangeSet()> RS,
+                                      const llvm::APSInt &Int,
+                                      const llvm::APSInt &Adjustment) {
   // Before we do any real work, see if the value can even show up.
   APSIntType AdjustmentType(Adjustment);
   switch (AdjustmentType.testInRange(Int, true)) {
@@ -1293,7 +2219,7 @@ RangeConstraintManager::assumeSymLE(ProgramStateRef St, SymbolRef Sym,
                                     const llvm::APSInt &Int,
                                     const llvm::APSInt &Adjustment) {
   RangeSet New = getSymLERange(St, Sym, Int, Adjustment);
-  return New.isEmpty() ? nullptr : St->set<ConstraintRange>(Sym, New);
+  return New.isEmpty() ? nullptr : setConstraint(St, Sym, New);
 }
 
 ProgramStateRef RangeConstraintManager::assumeSymWithinInclusiveRange(
@@ -1303,7 +2229,7 @@ ProgramStateRef RangeConstraintManager::assumeSymWithinInclusiveRange(
   if (New.isEmpty())
     return nullptr;
   RangeSet Out = getSymLERange([&] { return New; }, To, Adjustment);
-  return Out.isEmpty() ? nullptr : State->set<ConstraintRange>(Sym, Out);
+  return Out.isEmpty() ? nullptr : setConstraint(State, Sym, Out);
 }
 
 ProgramStateRef RangeConstraintManager::assumeSymOutsideInclusiveRange(
@@ -1312,7 +2238,7 @@ ProgramStateRef RangeConstraintManager::assumeSymOutsideInclusiveRange(
   RangeSet RangeLT = getSymLTRange(State, Sym, From, Adjustment);
   RangeSet RangeGT = getSymGTRange(State, Sym, To, Adjustment);
   RangeSet New(RangeLT.addRange(F, RangeGT));
-  return New.isEmpty() ? nullptr : State->set<ConstraintRange>(Sym, New);
+  return New.isEmpty() ? nullptr : setConstraint(State, Sym, New);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1332,17 +2258,25 @@ void RangeConstraintManager::printJson(raw_ostream &Out, ProgramStateRef State,
 
   ++Space;
   Out << '[' << NL;
-  for (ConstraintRangeTy::iterator I = Constraints.begin();
-       I != Constraints.end(); ++I) {
-    Indent(Out, Space, IsDot)
-        << "{ \"symbol\": \"" << I.getKey() << "\", \"range\": \"";
-    I.getData().print(Out);
-    Out << "\" }";
-
-    if (std::next(I) != Constraints.end())
-      Out << ',';
-    Out << NL;
+  bool First = true;
+  for (std::pair<EquivalenceClass, RangeSet> P : Constraints) {
+    SymbolSet ClassMembers = P.first.getClassMembers(State);
+
+    // We can print the same constraint for every class member.
+    for (SymbolRef ClassMember : ClassMembers) {
+      if (First) {
+        First = false;
+      } else {
+        Out << ',';
+        Out << NL;
+      }
+      Indent(Out, Space, IsDot)
+          << "{ \"symbol\": \"" << ClassMember << "\", \"range\": \"";
+      P.second.print(Out);
+      Out << "\" }";
+    }
   }
+  Out << NL;
 
   --Space;
   Indent(Out, Space, IsDot) << "]," << NL;
diff --git a/clang/lib/StaticAnalyzer/Core/RangedConstraintManager.cpp b/clang/lib/StaticAnalyzer/Core/RangedConstraintManager.cpp
index 4748c106eb55d..e7a03e6ed582e 100644
--- a/clang/lib/StaticAnalyzer/Core/RangedConstraintManager.cpp
+++ b/clang/lib/StaticAnalyzer/Core/RangedConstraintManager.cpp
@@ -40,19 +40,20 @@ ProgramStateRef RangedConstraintManager::assumeSym(ProgramStateRef State,
     }
 
   } else if (const SymSymExpr *SSE = dyn_cast<SymSymExpr>(Sym)) {
-    // Translate "a != b" to "(b - a) != 0".
-    // We invert the order of the operands as a heuristic for how loop
-    // conditions are usually written ("begin != end") as compared to length
-    // calculations ("end - begin"). The more correct thing to do would be to
-    // canonicalize "a - b" and "b - a", which would allow us to treat
-    // "a != b" and "b != a" the same.
-    SymbolManager &SymMgr = getSymbolManager();
     BinaryOperator::Opcode Op = SSE->getOpcode();
     assert(BinaryOperator::isComparisonOp(Op));
 
-    // For now, we only support comparing pointers.
+    // We convert equality operations for pointers only.
     if (Loc::isLocType(SSE->getLHS()->getType()) &&
         Loc::isLocType(SSE->getRHS()->getType())) {
+      // Translate "a != b" to "(b - a) != 0".
+      // We invert the order of the operands as a heuristic for how loop
+      // conditions are usually written ("begin != end") as compared to length
+      // calculations ("end - begin"). The more correct thing to do would be to
+      // canonicalize "a - b" and "b - a", which would allow us to treat
+      // "a != b" and "b != a" the same.
+
+      SymbolManager &SymMgr = getSymbolManager();
       QualType DiffTy = SymMgr.getContext().getPointerDiffType();
       SymbolRef Subtraction =
           SymMgr.getSymSymExpr(SSE->getRHS(), BO_Sub, SSE->getLHS(), DiffTy);
@@ -63,6 +64,25 @@ ProgramStateRef RangedConstraintManager::assumeSym(ProgramStateRef State,
         Op = BinaryOperator::negateComparisonOp(Op);
       return assumeSymRel(State, Subtraction, Op, Zero);
     }
+
+    if (BinaryOperator::isEqualityOp(Op)) {
+      SymbolManager &SymMgr = getSymbolManager();
+
+      QualType ExprType = SSE->getType();
+      SymbolRef CanonicalEquality =
+          SymMgr.getSymSymExpr(SSE->getLHS(), BO_EQ, SSE->getRHS(), ExprType);
+
+      bool WasEqual = SSE->getOpcode() == BO_EQ;
+      bool IsExpectedEqual = WasEqual == Assumption;
+
+      const llvm::APSInt &Zero = getBasicVals().getValue(0, ExprType);
+
+      if (IsExpectedEqual) {
+        return assumeSymNE(State, CanonicalEquality, Zero, Zero);
+      }
+
+      return assumeSymEQ(State, CanonicalEquality, Zero, Zero);
+    }
   }
 
   // If we get here, there's nothing else we can do but treat the symbol as
@@ -199,11 +219,6 @@ void RangedConstraintManager::computeAdjustment(SymbolRef &Sym,
   }
 }
 
-void *ProgramStateTrait<ConstraintRange>::GDMIndex() {
-  static int Index;
-  return &Index;
-}
-
 } // end of namespace ento
 
 } // end of namespace clang
diff --git a/clang/lib/Tooling/Refactoring/RefactoringActions.cpp b/clang/lib/Tooling/Refactoring/RefactoringActions.cpp
index 7ac723f67c047..bf98941f568b3 100644
--- a/clang/lib/Tooling/Refactoring/RefactoringActions.cpp
+++ b/clang/lib/Tooling/Refactoring/RefactoringActions.cpp
@@ -18,8 +18,8 @@ namespace {
 
 class DeclNameOption final : public OptionalRefactoringOption<std::string> {
 public:
-  StringRef getName() const { return "name"; }
-  StringRef getDescription() const {
+  StringRef getName() const override { return "name"; }
+  StringRef getDescription() const override {
     return "Name of the extracted declaration";
   }
 };
diff --git a/clang/lib/Tooling/Syntax/Tokens.cpp b/clang/lib/Tooling/Syntax/Tokens.cpp
index c6b904822b8b5..a83cc2d528615 100644
--- a/clang/lib/Tooling/Syntax/Tokens.cpp
+++ b/clang/lib/Tooling/Syntax/Tokens.cpp
@@ -249,22 +249,7 @@ llvm::SmallVector<llvm::ArrayRef<syntax::Token>, 1>
 TokenBuffer::expandedForSpelled(llvm::ArrayRef<syntax::Token> Spelled) const {
   if (Spelled.empty())
     return {};
-  assert(Spelled.front().location().isFileID());
-
-  auto FID = sourceManager().getFileID(Spelled.front().location());
-  auto It = Files.find(FID);
-  assert(It != Files.end());
-
-  const MarkedFile &File = It->second;
-  // `Spelled` must be a subrange of `File.SpelledTokens`.
-  assert(File.SpelledTokens.data() <= Spelled.data());
-  assert(&Spelled.back() <=
-         File.SpelledTokens.data() + File.SpelledTokens.size());
-#ifndef NDEBUG
-  auto T1 = Spelled.back().location();
-  auto T2 = File.SpelledTokens.back().location();
-  assert(T1 == T2 || sourceManager().isBeforeInTranslationUnit(T1, T2));
-#endif
+  const auto &File = fileForSpelled(Spelled);
 
   auto *FrontMapping = mappingStartingBeforeSpelled(File, &Spelled.front());
   unsigned SpelledFrontI = &Spelled.front() - File.SpelledTokens.data();
@@ -395,16 +380,39 @@ TokenBuffer::spelledForExpanded(llvm::ArrayRef<syntax::Token> Expanded) const {
                   : LastSpelled + 1);
 }
 
+TokenBuffer::Expansion TokenBuffer::makeExpansion(const MarkedFile &F,
+                                                  const Mapping &M) const {
+  Expansion E;
+  E.Spelled = llvm::makeArrayRef(F.SpelledTokens.data() + M.BeginSpelled,
+                                 F.SpelledTokens.data() + M.EndSpelled);
+  E.Expanded = llvm::makeArrayRef(ExpandedTokens.data() + M.BeginExpanded,
+                                  ExpandedTokens.data() + M.EndExpanded);
+  return E;
+}
+
+const TokenBuffer::MarkedFile &
+TokenBuffer::fileForSpelled(llvm::ArrayRef<syntax::Token> Spelled) const {
+  assert(!Spelled.empty());
+  assert(Spelled.front().location().isFileID() && "not a spelled token");
+  auto FileIt = Files.find(SourceMgr->getFileID(Spelled.front().location()));
+  assert(FileIt != Files.end() && "file not tracked by token buffer");
+  const auto &File = FileIt->second;
+  assert(File.SpelledTokens.data() <= Spelled.data() &&
+         Spelled.end() <=
+             (File.SpelledTokens.data() + File.SpelledTokens.size()) &&
+         "Tokens not in spelled range");
+#ifndef NDEBUG
+  auto T1 = Spelled.back().location();
+  auto T2 = File.SpelledTokens.back().location();
+  assert(T1 == T2 || sourceManager().isBeforeInTranslationUnit(T1, T2));
+#endif
+  return File;
+}
+
 llvm::Optional<TokenBuffer::Expansion>
 TokenBuffer::expansionStartingAt(const syntax::Token *Spelled) const {
   assert(Spelled);
-  assert(Spelled->location().isFileID() && "not a spelled token");
-  auto FileIt = Files.find(SourceMgr->getFileID(Spelled->location()));
-  assert(FileIt != Files.end() && "file not tracked by token buffer");
-
-  auto &File = FileIt->second;
-  assert(File.SpelledTokens.data() <= Spelled &&
-         Spelled < (File.SpelledTokens.data() + File.SpelledTokens.size()));
+  const auto &File = fileForSpelled(*Spelled);
 
   unsigned SpelledIndex = Spelled - File.SpelledTokens.data();
   auto M = llvm::partition_point(File.Mappings, [&](const Mapping &M) {
@@ -412,14 +420,27 @@ TokenBuffer::expansionStartingAt(const syntax::Token *Spelled) const {
   });
   if (M == File.Mappings.end() || M->BeginSpelled != SpelledIndex)
     return llvm::None;
+  return makeExpansion(File, *M);
+}
 
-  Expansion E;
-  E.Spelled = llvm::makeArrayRef(File.SpelledTokens.data() + M->BeginSpelled,
-                                 File.SpelledTokens.data() + M->EndSpelled);
-  E.Expanded = llvm::makeArrayRef(ExpandedTokens.data() + M->BeginExpanded,
-                                  ExpandedTokens.data() + M->EndExpanded);
-  return E;
+std::vector<TokenBuffer::Expansion> TokenBuffer::expansionsOverlapping(
+    llvm::ArrayRef<syntax::Token> Spelled) const {
+  if (Spelled.empty())
+    return {};
+  const auto &File = fileForSpelled(Spelled);
+
+  // Find the first overlapping range, and then copy until we stop overlapping.
+  unsigned SpelledBeginIndex = Spelled.begin() - File.SpelledTokens.data();
+  unsigned SpelledEndIndex = Spelled.end() - File.SpelledTokens.data();
+  auto M = llvm::partition_point(File.Mappings, [&](const Mapping &M) {
+    return M.EndSpelled <= SpelledBeginIndex;
+  });
+  std::vector<TokenBuffer::Expansion> Expansions;
+  for (; M != File.Mappings.end() && M->BeginSpelled < SpelledEndIndex; ++M)
+    Expansions.push_back(makeExpansion(File, *M));
+  return Expansions;
 }
+
 llvm::ArrayRef<syntax::Token>
 syntax::spelledTokensTouching(SourceLocation Loc,
                               llvm::ArrayRef<syntax::Token> Tokens) {
diff --git a/clang/lib/Tooling/Transformer/RewriteRule.cpp b/clang/lib/Tooling/Transformer/RewriteRule.cpp
index 995bec03cd669..a212a868c81d8 100644
--- a/clang/lib/Tooling/Transformer/RewriteRule.cpp
+++ b/clang/lib/Tooling/Transformer/RewriteRule.cpp
@@ -44,10 +44,13 @@ translateEdits(const MatchResult &Result, ArrayRef<ASTEdit> ASTEdits) {
     auto Replacement = E.Replacement->eval(Result);
     if (!Replacement)
       return Replacement.takeError();
+    auto Metadata = E.Metadata(Result);
+    if (!Metadata)
+      return Metadata.takeError();
     transformer::Edit T;
     T.Range = *EditRange;
     T.Replacement = std::move(*Replacement);
-    T.Metadata = E.Metadata;
+    T.Metadata = std::move(*Metadata);
     Edits.push_back(std::move(T));
   }
   return Edits;
diff --git a/clang/test/.clang-format b/clang/test/.clang-format
index a6176c2e00131..f7fb083d1055d 100644
--- a/clang/test/.clang-format
+++ b/clang/test/.clang-format
@@ -1,5 +1,3 @@
 BasedOnStyle: LLVM
 ColumnLimit: 0
-AlignTrailingComments: false
-CommentPragmas: "(^ ?CHECK|^ ?expected-)"
 AlwaysBreakTemplateDeclarations: No
diff --git a/clang/test/AST/ast-dump-fpfeatures.cpp b/clang/test/AST/ast-dump-fpfeatures.cpp
new file mode 100644
index 0000000000000..796b0a0283828
--- /dev/null
+++ b/clang/test/AST/ast-dump-fpfeatures.cpp
@@ -0,0 +1,37 @@
+// Test without serialization:
+// RUN: %clang_cc1 -fsyntax-only -triple x86_64-pc-linux -std=c++11 -ast-dump %s \
+// RUN: | FileCheck --strict-whitespace %s
+
+// Test with serialization:
+// RUN: %clang_cc1 -triple x86_64-pc-linux -emit-pch -o %t %s
+// RUN: %clang_cc1 -x c++ -triple x86_64-pc-linux -include-pch %t -ast-dump-all /dev/null \
+// RUN: | sed -e "s/ <undeserialized declarations>//" -e "s/ imported//" \
+// RUN: | FileCheck --strict-whitespace %s
+
+float func_01(float x);
+
+template <typename T>
+T func_02(T x) {
+#pragma STDC FP_CONTRACT ON
+  return func_01(x);
+}
+
+float func_03(float x) {
+#pragma STDC FP_CONTRACT OFF
+  return func_02(x);
+}
+
+// CHECK:      FunctionTemplateDecl {{.*}} func_02
+// CHECK:        FunctionDecl {{.*}} func_02 'float (float)'
+// CHECK-NEXT:     TemplateArgument type 'float'
+// CHECK-NEXT:       BuiltinType {{.*}} 'float'
+// CHECK-NEXT:     ParmVarDecl {{.*}} x 'float'
+// CHECK-NEXT:     CompoundStmt
+// CHECK-NEXT:       ReturnStmt
+// CHECK-NEXT:         CallExpr {{.*}} FPContractMode=1
+
+// CHECK:      FunctionDecl {{.*}} func_03 'float (float)'
+// CHECK-NEXT:   ParmVarDecl {{.*}} x 'float'
+// CHECK-NEXT:     CompoundStmt
+// CHECK-NEXT:       ReturnStmt
+// CHECK-NEXT:         CallExpr {{.*}} FPContractMode=0
\ No newline at end of file
diff --git a/clang/test/AST/ast-dump-recovery.c b/clang/test/AST/ast-dump-recovery.c
new file mode 100644
index 0000000000000..f3a33fdac49b9
--- /dev/null
+++ b/clang/test/AST/ast-dump-recovery.c
@@ -0,0 +1,52 @@
+// RUN: not %clang_cc1 -triple x86_64-unknown-unknown -frecovery-ast -fno-recovery-ast-type -ast-dump %s | FileCheck -strict-whitespace %s
+
+int some_func(int);
+
+// CHECK:     VarDecl {{.*}} unmatch_arg_call 'int' cinit
+// CHECK-NEXT: `-RecoveryExpr {{.*}} contains-errors
+// CHECK-NEXT:   `-DeclRefExpr {{.*}} 'some_func'
+int unmatch_arg_call = some_func();
+
+const int a = 1;
+
+// CHECK:     VarDecl {{.*}} postfix_inc
+// CHECK-NEXT: `-RecoveryExpr {{.*}} contains-errors
+// CHECK-NEXT:   `-DeclRefExpr {{.*}} 'a'
+int postfix_inc = a++;
+
+// CHECK:     VarDecl {{.*}} unary_address
+// CHECK-NEXT:`-RecoveryExpr {{.*}} contains-errors
+// CHECK-NEXT:  `-ParenExpr {{.*}}
+// CHECK-NEXT:    `-BinaryOperator {{.*}} '+'
+// CHECK-NEXT:      |-ImplicitCastExpr
+// CHECK-NEXT:      | `-DeclRefExpr {{.*}} 'a'
+// CHECK-NEXT:      `-IntegerLiteral {{.*}} 'int'
+int unary_address = &(a + 1);
+
+// CHECK:       VarDecl {{.*}} ternary 'int' cinit
+// CHECK-NEXT:  `-RecoveryExpr {{.*}}
+// CHECK-NEXT:    |-DeclRefExpr {{.*}} 'a'
+// CHECK-NEXT:    |-TypoExpr {{.*}}
+// CHECK-NEXT:    `-DeclRefExpr {{.*}} 'a'
+// FIXME: The TypoExpr should never be print, and should be downgraded to
+// RecoveryExpr -- typo correction is performed too early in C-only codepath,
+// which makes no correction when clang finishes the full expr (Sema::Sema::ActOnFinishFullExpr).
+// this will be fixed when we support dependent mechanism and delayed typo correction for C.
+int ternary = a ? undef : a;
+
+void test1() {
+  // CHECK:     `-RecoveryExpr {{.*}} contains-errors
+  // CHECK-NEXT:  `-DeclRefExpr {{.*}} 'a' 'const int'
+  static int foo = a++; // verify no crash on local static var decl.
+}
+
+void test2() {
+  int* ptr;
+  // FIXME: the top-level expr should be a binary operator.
+  // CHECK:      ImplicitCastExpr {{.*}} contains-errors <LValueToRValue>
+  // CHECK-NEXT: `-RecoveryExpr {{.*}} contains-errors lvalue
+  // CHECK-NEXT:   |-DeclRefExpr {{.*}} 'ptr' 'int *'
+  // CHECK-NEXT:   `-RecoveryExpr {{.*}}
+  // CHECK-NEXT:     `-DeclRefExpr {{.*}} 'some_func'
+  ptr = some_func(); // should not crash
+}
diff --git a/clang/test/AST/ast-dump-recovery.cpp b/clang/test/AST/ast-dump-recovery.cpp
index 740864a26481b..fd7c923b7e51e 100644
--- a/clang/test/AST/ast-dump-recovery.cpp
+++ b/clang/test/AST/ast-dump-recovery.cpp
@@ -24,8 +24,11 @@ void test_invalid_call(int s) {
   // CHECK-NEXT:   |-UnresolvedLookupExpr {{.*}} 'some_func'
   // CHECK-NEXT:   `-RecoveryExpr {{.*}} contains-errors
   s = some_func(undef1);
-  // CHECK: `-VarDecl {{.*}} invalid var 'int'
-  // FIXME: preserve the broken call.
+
+  // CHECK:     VarDecl {{.*}} var 'int'
+  // CHECK-NEXT: `-CallExpr {{.*}} '<dependent type>' contains-errors
+  // CHECK-NEXT:   |-UnresolvedLookupExpr {{.*}} 'some_func'
+  // CHECK-NEXT:   `-RecoveryExpr {{.*}} contains-errors
   int var = some_func(undef1);
 }
 
@@ -157,11 +160,14 @@ void InvalidInitalizer(int x) {
   // CHECK-NEXT: `-RecoveryExpr {{.*}} contains-errors
   // CHECK-NEXT:  `-InitListExpr
   Bar b2 = {1};
-  // FIXME: preserve the invalid initializer.
-  // CHECK: `-VarDecl {{.*}} b3 'Bar'
+  // CHECK:     `-VarDecl {{.*}} b3 'Bar'
+  // CHECK-NEXT:  `-RecoveryExpr {{.*}} 'Bar' contains-errors
+  // CHECK-NEXT:    `-DeclRefExpr {{.*}} 'x' 'int'
   Bar b3 = Bar(x);
-  // FIXME: preserve the invalid initializer.
-  // CHECK: `-VarDecl {{.*}} b4 'Bar'
+  // CHECK:     `-VarDecl {{.*}} b4 'Bar'
+  // CHECK-NEXT:  `-RecoveryExpr {{.*}} 'Bar' contains-errors
+  // CHECK-NEXT:    `-InitListExpr {{.*}} 'void'
+  // CHECK-NEXT:      `-DeclRefExpr {{.*}} 'x' 'int'
   Bar b4 = Bar{x};
   // CHECK:     `-VarDecl {{.*}} b5 'Bar'
   // CHECK-NEXT: `-CXXUnresolvedConstructExpr {{.*}} 'Bar' contains-errors 'Bar'
@@ -174,6 +180,16 @@ void InvalidInitalizer(int x) {
   // CHECK-NEXT:   `-RecoveryExpr {{.*}} contains-errors
   // CHECK-NEXT:     `-UnresolvedLookupExpr {{.*}} 'invalid'
   Bar b6 = Bar{invalid()};
+
+  // CHECK:     RecoveryExpr {{.*}} 'Bar' contains-errors
+  // CHECK-NEXT:  `-IntegerLiteral {{.*}} 'int' 1
+  Bar(1);
+
+  // CHECK:     `-VarDecl {{.*}} var1
+  // CHECK-NEXT: `-BinaryOperator {{.*}} '<dependent type>' contains-errors
+  // CHECK-NEXT:   |-RecoveryExpr {{.*}} '<dependent type>' contains-errors
+  // CHECK-NEXT:   `-IntegerLiteral {{.*}} 'int' 1
+  int var1 = undef + 1;
 }
 void InitializerForAuto() {
   // CHECK:     `-VarDecl {{.*}} invalid a 'auto'
@@ -222,3 +238,30 @@ void ValueCategory() {
   // CHECK:  RecoveryExpr {{.*}} 'int' contains-errors xvalue
   xvalue(); // call to a function (rvalue reference return type) yields an xvalue.
 }
+
+void InvalidCondition() {
+  // CHECK:      IfStmt {{.*}}
+  // CHECK-NEXT: |-RecoveryExpr {{.*}} <col:7, col:15> '<dependent type>' contains-errors
+  // CHECK-NEXT: | `-UnresolvedLookupExpr {{.*}} <col:7>
+  if (invalid()) {}
+
+  // CHECK:      WhileStmt {{.*}}
+  // CHECK-NEXT: |-RecoveryExpr {{.*}} <col:10, col:18> '<dependent type>' contains-errors
+  // CHECK-NEXT: | `-UnresolvedLookupExpr {{.*}} <col:10>
+  while (invalid()) {}
+
+  // CHECK:      SwitchStmt {{.*}}
+  // CHECK-NEXT: |-RecoveryExpr {{.*}} '<dependent type>' contains-errors
+  // CHECK-NEXT: | `-UnresolvedLookupExpr {{.*}} <col:10>
+  switch(invalid()) {
+    case 1:
+      break;
+  }
+  // FIXME: figure out why the type of ConditionalOperator is not int.
+  // CHECK:      ConditionalOperator {{.*}} '<dependent type>' contains-errors
+  // CHECK-NEXT: |-RecoveryExpr {{.*}} '<dependent type>' contains-errors
+  // CHECK-NEXT: | `-UnresolvedLookupExpr {{.*}}
+  // CHECK-NEXT: |-IntegerLiteral {{.*}} 'int' 1
+  // CHECK-NEXT: `-IntegerLiteral {{.*}} 'int' 2
+  invalid() ? 1 : 2;
+}
diff --git a/clang/test/Analysis/Inputs/system-header-simulator-cxx.h b/clang/test/Analysis/Inputs/system-header-simulator-cxx.h
index 1dee3294d7323..9010ce2bb9b6c 100644
--- a/clang/test/Analysis/Inputs/system-header-simulator-cxx.h
+++ b/clang/test/Analysis/Inputs/system-header-simulator-cxx.h
@@ -953,24 +953,31 @@ next(ForwardIterator it,
 
 #if __cplusplus >= 201103L
 namespace std {
-  template <typename T> // TODO: Implement the stub for deleter.
-  class unique_ptr {
-  public:
-    unique_ptr() {}
-    unique_ptr(T *) {}
-    unique_ptr(const unique_ptr &) = delete;
-    unique_ptr(unique_ptr &&);
-
-    T *get() const;
-    T *release() const;
-    void reset(T *p = nullptr) const;
-    void swap(unique_ptr<T> &p) const;
-
-    typename std::add_lvalue_reference<T>::type operator*() const;
-    T *operator->() const;
-    operator bool() const;
-  };
+template <typename T> // TODO: Implement the stub for deleter.
+class unique_ptr {
+public:
+  unique_ptr() noexcept {}
+  unique_ptr(T *) noexcept {}
+  unique_ptr(const unique_ptr &) noexcept = delete;
+  unique_ptr(unique_ptr &&) noexcept;
+
+  T *get() const noexcept;
+  T *release() noexcept;
+  void reset(T *p = nullptr) noexcept;
+  void swap(unique_ptr<T> &p) noexcept;
+
+  typename std::add_lvalue_reference<T>::type operator*() const;
+  T *operator->() const noexcept;
+  operator bool() const noexcept;
+  unique_ptr<T> &operator=(unique_ptr<T> &&p) noexcept;
+};
+
+// TODO :: Once the deleter parameter is added update with additional template parameter.
+template <typename T>
+void swap(unique_ptr<T> &x, unique_ptr<T> &y) noexcept {
+  x.swap(y);
 }
+} // namespace std
 #endif
 
 #ifdef TEST_INLINABLE_ALLOCATORS
diff --git a/clang/test/Analysis/equality_tracking.c b/clang/test/Analysis/equality_tracking.c
new file mode 100644
index 0000000000000..3103e8516dcdf
--- /dev/null
+++ b/clang/test/Analysis/equality_tracking.c
@@ -0,0 +1,187 @@
+// RUN: %clang_analyze_cc1 -verify %s \
+// RUN:   -analyzer-checker=core,debug.ExprInspection \
+// RUN:   -analyzer-config eagerly-assume=false
+
+#define NULL (void *)0
+
+#define UCHAR_MAX (unsigned char)(~0U)
+#define CHAR_MAX (char)(UCHAR_MAX & (UCHAR_MAX >> 1))
+#define CHAR_MIN (char)(UCHAR_MAX & ~(UCHAR_MAX >> 1))
+
+void clang_analyzer_eval(int);
+void clang_analyzer_warnIfReached();
+
+int getInt();
+
+void zeroImpliesEquality(int a, int b) {
+  clang_analyzer_eval((a - b) == 0); // expected-warning{{UNKNOWN}}
+  if ((a - b) == 0) {
+    clang_analyzer_eval(b != a);    // expected-warning{{FALSE}}
+    clang_analyzer_eval(b == a);    // expected-warning{{TRUE}}
+    clang_analyzer_eval(!(a != b)); // expected-warning{{TRUE}}
+    clang_analyzer_eval(!(b == a)); // expected-warning{{FALSE}}
+    return;
+  }
+  clang_analyzer_eval((a - b) == 0); // expected-warning{{FALSE}}
+  clang_analyzer_eval(b == a);       // expected-warning{{FALSE}}
+  clang_analyzer_eval(b != a);       // expected-warning{{TRUE}}
+}
+
+void zeroImpliesReversedEqual(int a, int b) {
+  clang_analyzer_eval((b - a) == 0); // expected-warning{{UNKNOWN}}
+  if ((b - a) == 0) {
+    clang_analyzer_eval(b != a); // expected-warning{{FALSE}}
+    clang_analyzer_eval(b == a); // expected-warning{{TRUE}}
+    return;
+  }
+  clang_analyzer_eval((b - a) == 0); // expected-warning{{FALSE}}
+  clang_analyzer_eval(b == a);       // expected-warning{{FALSE}}
+  clang_analyzer_eval(b != a);       // expected-warning{{TRUE}}
+}
+
+void canonicalEqual(int a, int b) {
+  clang_analyzer_eval(a == b); // expected-warning{{UNKNOWN}}
+  if (a == b) {
+    clang_analyzer_eval(b == a); // expected-warning{{TRUE}}
+    return;
+  }
+  clang_analyzer_eval(a == b); // expected-warning{{FALSE}}
+  clang_analyzer_eval(b == a); // expected-warning{{FALSE}}
+}
+
+void test(int a, int b, int c, int d) {
+  if (a == b && c == d) {
+    if (a == 0 && b == d) {
+      clang_analyzer_eval(c == 0); // expected-warning{{TRUE}}
+    }
+    c = 10;
+    if (b == d) {
+      clang_analyzer_eval(c == 10); // expected-warning{{TRUE}}
+      clang_analyzer_eval(d == 10); // expected-warning{{UNKNOWN}}
+                                    // expected-warning@-1{{FALSE}}
+      clang_analyzer_eval(b == a);  // expected-warning{{TRUE}}
+      clang_analyzer_eval(a == d);  // expected-warning{{TRUE}}
+
+      b = getInt();
+      clang_analyzer_eval(a == d); // expected-warning{{TRUE}}
+      clang_analyzer_eval(a == b); // expected-warning{{UNKNOWN}}
+    }
+  }
+
+  if (a != b && b == c) {
+    if (c == 42) {
+      clang_analyzer_eval(b == 42); // expected-warning{{TRUE}}
+      clang_analyzer_eval(a != 42); // expected-warning{{TRUE}}
+    }
+  }
+}
+
+void testIntersection(int a, int b, int c) {
+  if (a < 42 && b > 15 && c >= 25 && c <= 30) {
+    if (a != b)
+      return;
+
+    clang_analyzer_eval(a > 15);  // expected-warning{{TRUE}}
+    clang_analyzer_eval(b < 42);  // expected-warning{{TRUE}}
+    clang_analyzer_eval(a <= 30); // expected-warning{{UNKNOWN}}
+
+    if (c == b) {
+      // For all equal symbols, we should track the minimal common range.
+      //
+      // Also, it should be noted that c is dead at this point, but the
+      // constraint initially associated with c is still around.
+      clang_analyzer_eval(a >= 25 && a <= 30); // expected-warning{{TRUE}}
+      clang_analyzer_eval(b >= 25 && b <= 30); // expected-warning{{TRUE}}
+    }
+  }
+}
+
+void testPromotion(int a, char b) {
+  if (b > 10) {
+    if (a == b) {
+      // FIXME: support transferring char ranges onto equal int symbols
+      //        when char is promoted to int
+      clang_analyzer_eval(a > 10);        // expected-warning{{UNKNOWN}}
+      clang_analyzer_eval(a <= CHAR_MAX); // expected-warning{{UNKNOWN}}
+    }
+  }
+}
+
+void testPromotionOnlyTypes(int a, char b) {
+  if (a == b) {
+    // FIXME: support transferring char ranges onto equal int symbols
+    //        when char is promoted to int
+    clang_analyzer_eval(a <= CHAR_MAX); // expected-warning{{UNKNOWN}}
+  }
+}
+
+void testDowncast(int a, unsigned char b) {
+  if (a <= -10) {
+    if ((unsigned char)a == b) {
+      // Even though ranges for a and b do not intersect,
+      // ranges for (unsigned char)a and b do.
+      clang_analyzer_warnIfReached(); // expected-warning{{REACHABLE}}
+    }
+    if (a == b) {
+      // FIXME: This case on the other hand is different, it shouldn't be
+      //        reachable.  However, the corrent symbolic information available
+      //        to the solver doesn't allow it to distinguish this expression
+      //        from the previous one.
+      clang_analyzer_warnIfReached(); // expected-warning{{REACHABLE}}
+    }
+  }
+}
+
+void testPointers(int *a, int *b, int *c, int *d) {
+  if (a == b && c == d) {
+    if (a == NULL && b == d) {
+      clang_analyzer_eval(c == NULL); // expected-warning{{TRUE}}
+    }
+  }
+
+  if (a != b && b == c) {
+    if (c == NULL) {
+      clang_analyzer_eval(a != NULL); // expected-warning{{TRUE}}
+    }
+  }
+}
+
+void testDisequalitiesAfter(int a, int b, int c) {
+  if (a >= 10 && b <= 42) {
+    if (a == b && c == 15 && c != a) {
+      clang_analyzer_eval(b != c);  // expected-warning{{TRUE}}
+      clang_analyzer_eval(a != 15); // expected-warning{{TRUE}}
+      clang_analyzer_eval(b != 15); // expected-warning{{TRUE}}
+      clang_analyzer_eval(b >= 10); // expected-warning{{TRUE}}
+      clang_analyzer_eval(a <= 42); // expected-warning{{TRUE}}
+    }
+  }
+}
+
+void testDisequalitiesBefore(int a, int b, int c) {
+  if (a >= 10 && b <= 42 && c == 15) {
+    if (a == b && c != a) {
+      clang_analyzer_eval(b != c);  // expected-warning{{TRUE}}
+      clang_analyzer_eval(a != 15); // expected-warning{{TRUE}}
+      clang_analyzer_eval(b != 15); // expected-warning{{TRUE}}
+      clang_analyzer_eval(b >= 10); // expected-warning{{TRUE}}
+      clang_analyzer_eval(a <= 42); // expected-warning{{TRUE}}
+    }
+  }
+}
+
+void avoidInfeasibleConstraintsForClasses(int a, int b) {
+  if (a >= 0 && a <= 10 && b >= 20 && b <= 50) {
+    if ((b - a) == 0) {
+      clang_analyzer_warnIfReached(); // no warning
+    }
+    if (a == b) {
+      clang_analyzer_warnIfReached(); // no warning
+    }
+    if (a != b) {
+      clang_analyzer_warnIfReached(); // expected-warning{{REACHABLE}}
+    } else {
+      clang_analyzer_warnIfReached(); // no warning
+    }
+  }
+}
diff --git a/clang/test/Analysis/mutually_exclusive_null_fp.cpp b/clang/test/Analysis/mutually_exclusive_null_fp.cpp
new file mode 100644
index 0000000000000..ed3f2d09857c2
--- /dev/null
+++ b/clang/test/Analysis/mutually_exclusive_null_fp.cpp
@@ -0,0 +1,26 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=core -verify %s
+
+// rdar://problem/56586853
+// expected-no-diagnostics
+
+struct Data {
+  int x;
+  Data *data;
+};
+
+int compare(Data &a, Data &b) {
+  Data *aData = a.data;
+  Data *bData = b.data;
+
+  // Covers the cases where both pointers are null as well as both pointing to the same buffer.
+  if (aData == bData)
+    return 0;
+
+  if (aData && !bData)
+    return 1;
+
+  if (!aData && bData)
+    return -1;
+
+  return compare(*aData, *bData); // no-warning
+}
diff --git a/clang/test/Analysis/smart-ptr.cpp b/clang/test/Analysis/smart-ptr.cpp
index 4ab7e2bbd3bfc..168682ba758f6 100644
--- a/clang/test/Analysis/smart-ptr.cpp
+++ b/clang/test/Analysis/smart-ptr.cpp
@@ -67,14 +67,14 @@ void derefAfterRelease() {
   std::unique_ptr<A> P(new A());
   P.release();
   clang_analyzer_numTimesReached(); // expected-warning {{1}}
-  P->foo(); // expected-warning {{Dereference of null smart pointer [alpha.cplusplus.SmartPtr]}}
+  P->foo();                         // expected-warning {{Dereference of null smart pointer [alpha.cplusplus.SmartPtr]}}
 }
 
 void derefAfterReset() {
   std::unique_ptr<A> P(new A());
   P.reset();
   clang_analyzer_numTimesReached(); // expected-warning {{1}}
-  P->foo(); // expected-warning {{Dereference of null smart pointer [alpha.cplusplus.SmartPtr]}}
+  P->foo();                         // expected-warning {{Dereference of null smart pointer [alpha.cplusplus.SmartPtr]}}
 }
 
 void derefAfterResetWithNull() {
@@ -101,3 +101,130 @@ void derefOnReleasedNullRawPtr() {
   A *AP = P.release();
   AP->foo(); // expected-warning {{Called C++ object pointer is null [core.CallAndMessage]}}
 }
+
+void pass_smart_ptr_by_ref(std::unique_ptr<A> &a);
+void pass_smart_ptr_by_const_ref(const std::unique_ptr<A> &a);
+void pass_smart_ptr_by_rvalue_ref(std::unique_ptr<A> &&a);
+void pass_smart_ptr_by_const_rvalue_ref(const std::unique_ptr<A> &&a);
+void pass_smart_ptr_by_ptr(std::unique_ptr<A> *a);
+void pass_smart_ptr_by_const_ptr(const std::unique_ptr<A> *a);
+
+void regioninvalidationTest() {
+  {
+    std::unique_ptr<A> P;
+    pass_smart_ptr_by_ref(P);
+    P->foo(); // no-warning
+  }
+  {
+    std::unique_ptr<A> P;
+    pass_smart_ptr_by_const_ref(P);
+    P->foo(); // expected-warning {{Dereference of null smart pointer [alpha.cplusplus.SmartPtr]}}
+  }
+  {
+    std::unique_ptr<A> P;
+    pass_smart_ptr_by_rvalue_ref(std::move(P));
+    P->foo(); // no-warning
+  }
+  {
+    std::unique_ptr<A> P;
+    pass_smart_ptr_by_const_rvalue_ref(std::move(P));
+    P->foo(); // expected-warning {{Dereference of null smart pointer [alpha.cplusplus.SmartPtr]}}
+  }
+  {
+    std::unique_ptr<A> P;
+    pass_smart_ptr_by_ptr(&P);
+    P->foo();
+  }
+  {
+    std::unique_ptr<A> P;
+    pass_smart_ptr_by_const_ptr(&P);
+    P->foo(); // expected-warning {{Dereference of null smart pointer [alpha.cplusplus.SmartPtr]}}
+  }
+}
+
+struct StructWithSmartPtr {
+  std::unique_ptr<A> P;
+};
+
+void pass_struct_with_smart_ptr_by_ref(StructWithSmartPtr &a);
+void pass_struct_with_smart_ptr_by_const_ref(const StructWithSmartPtr &a);
+void pass_struct_with_smart_ptr_by_rvalue_ref(StructWithSmartPtr &&a);
+void pass_struct_with_smart_ptr_by_const_rvalue_ref(const StructWithSmartPtr &&a);
+void pass_struct_with_smart_ptr_by_ptr(StructWithSmartPtr *a);
+void pass_struct_with_smart_ptr_by_const_ptr(const StructWithSmartPtr *a);
+
+void regioninvalidationTestWithinStruct() {
+  {
+    StructWithSmartPtr S;
+    pass_struct_with_smart_ptr_by_ref(S);
+    S.P->foo(); // no-warning
+  }
+  {
+    StructWithSmartPtr S;
+    pass_struct_with_smart_ptr_by_const_ref(S);
+    S.P->foo(); // expected-warning {{Dereference of null smart pointer [alpha.cplusplus.SmartPtr]}}
+  }
+  {
+    StructWithSmartPtr S;
+    pass_struct_with_smart_ptr_by_rvalue_ref(std::move(S));
+    S.P->foo(); // no-warning
+  }
+  {
+    StructWithSmartPtr S;
+    pass_struct_with_smart_ptr_by_const_rvalue_ref(std::move(S));
+    S.P->foo(); // expected-warning {{Dereference of null smart pointer [alpha.cplusplus.SmartPtr]}}
+  }
+  {
+    StructWithSmartPtr S;
+    pass_struct_with_smart_ptr_by_ptr(&S);
+    S.P->foo();
+  }
+  {
+    StructWithSmartPtr S;
+    pass_struct_with_smart_ptr_by_const_ptr(&S);
+    S.P->foo(); // expected-warning {{Dereference of null smart pointer [alpha.cplusplus.SmartPtr]}}
+  }
+}
+
+void derefAfterAssignment() {
+  {
+    std::unique_ptr<A> P(new A());
+    std::unique_ptr<A> Q;
+    Q = std::move(P);
+    Q->foo(); // no-warning
+  }
+  {
+    std::unique_ptr<A> P;
+    std::unique_ptr<A> Q;
+    Q = std::move(P);
+    // TODO: Fix test with expecting warning after '=' operator overloading modeling.
+    Q->foo(); // no-warning
+  }
+}
+
+void derefOnSwappedNullPtr() {
+  std::unique_ptr<A> P(new A());
+  std::unique_ptr<A> PNull;
+  P.swap(PNull);
+  PNull->foo(); // No warning.
+  (*P).foo();   // expected-warning {{Dereference of null smart pointer [alpha.cplusplus.SmartPtr]}}
+}
+
+void derefOnStdSwappedNullPtr() {
+  std::unique_ptr<A> P;
+  std::unique_ptr<A> PNull;
+  std::swap(P, PNull);
+  PNull->foo(); // expected-warning {{Dereference of null smart pointer [alpha.cplusplus.SmartPtr]}}
+  P->foo();     // expected-warning {{Dereference of null smart pointer [alpha.cplusplus.SmartPtr]}}
+}
+
+void derefOnSwappedValidPtr() {
+  std::unique_ptr<A> P(new A());
+  std::unique_ptr<A> PValid(new A());
+  P.swap(PValid);
+  (*P).foo();    // No warning.
+  PValid->foo(); // No warning.
+  std::swap(P, PValid);
+  P->foo();      // No warning.
+  PValid->foo(); // No warning.
+}
diff --git a/clang/test/Analysis/std-c-library-functions-POSIX-socket-sockaddr.cpp b/clang/test/Analysis/std-c-library-functions-POSIX-socket-sockaddr.cpp
new file mode 100644
index 0000000000000..a35bb5cdc3a04
--- /dev/null
+++ b/clang/test/Analysis/std-c-library-functions-POSIX-socket-sockaddr.cpp
@@ -0,0 +1,37 @@
+// RUN: %clang_analyze_cc1 %s \
+// RUN:   -analyzer-checker=core \
+// RUN:   -analyzer-checker=apiModeling.StdCLibraryFunctions \
+// RUN:   -analyzer-config apiModeling.StdCLibraryFunctions:ModelPOSIX=true \
+// RUN:   -analyzer-config apiModeling.StdCLibraryFunctions:DisplayLoadedSummaries=true \
+// RUN:   -analyzer-checker=debug.ExprInspection \
+// RUN:   -analyzer-config eagerly-assume=false \
+// RUN:   -triple i686-unknown-linux 2>&1 | FileCheck %s
+
+// We test here that functions from socket.h are added when sockaddr is not a
+// transparent union of other sockaddr_ pointers. This is the case in C++.
+
+// CHECK: Loaded summary for: int accept(int socket, struct sockaddr *address, socklen_t *address_len)
+// CHECK: Loaded summary for: int bind(int socket, const struct sockaddr *address, socklen_t address_len)
+// CHECK: Loaded summary for: int getpeername(int socket, struct sockaddr *address, socklen_t *address_len)
+// CHECK: Loaded summary for: int getsockname(int socket, struct sockaddr *address, socklen_t *address_len)
+// CHECK: Loaded summary for: int connect(int socket, const struct sockaddr *address, socklen_t address_len)
+// CHECK: Loaded summary for: ssize_t recvfrom(int socket, void *buffer, size_t length, int flags, struct sockaddr *address, socklen_t *address_len)
+// CHECK: Loaded summary for: ssize_t sendto(int socket, const void *message, size_t length, int flags, const struct sockaddr *dest_addr, socklen_t dest_len)
+
+struct sockaddr;
+using socklen_t = unsigned;
+int accept(int socket, struct sockaddr *address, socklen_t *address_len);
+int bind(int socket, const struct sockaddr *address, socklen_t address_len);
+int getpeername(int socket, struct sockaddr *address, socklen_t *address_len);
+int getsockname(int socket, struct sockaddr *address, socklen_t *address_len);
+int connect(int socket, const struct sockaddr *address, socklen_t address_len);
+typedef decltype(sizeof(int)) size_t;
+typedef size_t ssize_t;
+ssize_t recvfrom(int socket, void *buffer, size_t length, int flags, struct sockaddr *address, socklen_t *address_len);
+ssize_t sendto(int socket, const void *message, size_t length, int flags, const struct sockaddr *dest_addr, socklen_t dest_len);
+
+// Must have at least one call expression to initialize the summary map.
+int bar(void);
+void foo() {
+  bar();
+}
diff --git a/clang/test/Analysis/std-c-library-functions-POSIX.c b/clang/test/Analysis/std-c-library-functions-POSIX.c
index e6f5b19baa00e..3638ad100240a 100644
--- a/clang/test/Analysis/std-c-library-functions-POSIX.c
+++ b/clang/test/Analysis/std-c-library-functions-POSIX.c
@@ -79,6 +79,22 @@
 // CHECK: Loaded summary for: int execv(const char *path, char *const argv[])
 // CHECK: Loaded summary for: int execvp(const char *file, char *const argv[])
 // CHECK: Loaded summary for: int getopt(int argc, char *const argv[], const char *optstring)
+// CHECK: Loaded summary for: int accept(int socket, __SOCKADDR_ARG address, socklen_t *restrict address_len)
+// CHECK: Loaded summary for: int bind(int socket, __CONST_SOCKADDR_ARG address, socklen_t address_len)
+// CHECK: Loaded summary for: int getpeername(int socket, __SOCKADDR_ARG address, socklen_t *restrict address_len)
+// CHECK: Loaded summary for: int getsockname(int socket, __SOCKADDR_ARG address, socklen_t *restrict address_len)
+// CHECK: Loaded summary for: int connect(int socket, __CONST_SOCKADDR_ARG address, socklen_t address_len)
+// CHECK: Loaded summary for: ssize_t recvfrom(int socket, void *restrict buffer, size_t length, int flags, __SOCKADDR_ARG address, socklen_t *restrict address_len)
+// CHECK: Loaded summary for: ssize_t sendto(int socket, const void *message, size_t length, int flags, __CONST_SOCKADDR_ARG dest_addr, socklen_t dest_len)
+// CHECK: Loaded summary for: int listen(int sockfd, int backlog)
+// CHECK: Loaded summary for: ssize_t recv(int sockfd, void *buf, size_t len, int flags)
+// CHECK: Loaded summary for: ssize_t recvmsg(int sockfd, struct msghdr *msg, int flags)
+// CHECK: Loaded summary for: ssize_t sendmsg(int sockfd, const struct msghdr *msg, int flags)
+// CHECK: Loaded summary for: int setsockopt(int socket, int level, int option_name, const void *option_value, socklen_t option_len)
+// CHECK: Loaded summary for: int getsockopt(int socket, int level, int option_name, void *restrict option_value, socklen_t *restrict option_len)
+// CHECK: Loaded summary for: ssize_t send(int sockfd, const void *buf, size_t len, int flags)
+// CHECK: Loaded summary for: int socketpair(int domain, int type, int protocol, int sv[2])
+// CHECK: Loaded summary for: int getnameinfo(const struct sockaddr *restrict sa, socklen_t salen, char *restrict node, socklen_t nodelen, char *restrict service, socklen_t servicelen, int flags)
 
 long a64l(const char *str64);
 char *l64a(long value);
@@ -171,6 +187,46 @@ int execv(const char *path, char *const argv[]);
 int execvp(const char *file, char *const argv[]);
 int getopt(int argc, char *const argv[], const char *optstring);
 
+// In some libc implementations, sockaddr parameter is a transparent
+// union of the underlying sockaddr_ pointers instead of being a
+// pointer to struct sockaddr.
+// We match that with the joker Irrelevant type.
+struct sockaddr;
+struct sockaddr_at;
+#define __SOCKADDR_ALLTYPES    \
+  __SOCKADDR_ONETYPE(sockaddr) \
+  __SOCKADDR_ONETYPE(sockaddr_at)
+#define __SOCKADDR_ONETYPE(type) struct type *__restrict __##type##__;
+typedef union {
+  __SOCKADDR_ALLTYPES
+} __SOCKADDR_ARG __attribute__((__transparent_union__));
+#undef __SOCKADDR_ONETYPE
+#define __SOCKADDR_ONETYPE(type) const struct type *__restrict __##type##__;
+typedef union {
+  __SOCKADDR_ALLTYPES
+} __CONST_SOCKADDR_ARG __attribute__((__transparent_union__));
+#undef __SOCKADDR_ONETYPE
+typedef unsigned socklen_t;
+
+int accept(int socket, __SOCKADDR_ARG address, socklen_t *restrict address_len);
+int bind(int socket, __CONST_SOCKADDR_ARG address, socklen_t address_len);
+int getpeername(int socket, __SOCKADDR_ARG address, socklen_t *restrict address_len);
+int getsockname(int socket, __SOCKADDR_ARG address, socklen_t *restrict address_len);
+int connect(int socket, __CONST_SOCKADDR_ARG address, socklen_t address_len);
+ssize_t recvfrom(int socket, void *restrict buffer, size_t length, int flags, __SOCKADDR_ARG address, socklen_t *restrict address_len);
+ssize_t sendto(int socket, const void *message, size_t length, int flags, __CONST_SOCKADDR_ARG dest_addr, socklen_t dest_len);
+
+int listen(int sockfd, int backlog);
+ssize_t recv(int sockfd, void *buf, size_t len, int flags);
+struct msghdr;
+ssize_t recvmsg(int sockfd, struct msghdr *msg, int flags);
+ssize_t sendmsg(int sockfd, const struct msghdr *msg, int flags);
+int setsockopt(int socket, int level, int option_name, const void *option_value, socklen_t option_len);
+int getsockopt(int socket, int level, int option_name, void *restrict option_value, socklen_t *restrict option_len);
+ssize_t send(int sockfd, const void *buf, size_t len, int flags);
+int socketpair(int domain, int type, int protocol, int sv[2]);
+int getnameinfo(const struct sockaddr *restrict sa, socklen_t salen, char *restrict node, socklen_t nodelen, char *restrict service, socklen_t servicelen, int flags);
+
 // Must have at least one call expression to initialize the summary map.
 int bar(void);
 void foo() {
diff --git a/clang/test/Analysis/stream-note.c b/clang/test/Analysis/stream-note.c
index 08927e8902be4..71a5ba2315d9c 100644
--- a/clang/test/Analysis/stream-note.c
+++ b/clang/test/Analysis/stream-note.c
@@ -46,3 +46,34 @@ void check_note_freopen() {
 }
 // expected-warning@-1 {{Opened stream never closed. Potential resource leak}}
 // expected-note@-2 {{Opened stream never closed. Potential resource leak}}
+
+void check_note_leak_2(int c) {
+  FILE *F1 = fopen("foo1.c", "r"); // expected-note {{Stream opened here}}
+  if (!F1)
+    // expected-note@-1 {{'F1' is non-null}}
+    // expected-note@-2 {{Taking false branch}}
+    // expected-note@-3 {{'F1' is non-null}}
+    // expected-note@-4 {{Taking false branch}}
+    return;
+  FILE *F2 = fopen("foo2.c", "r"); // expected-note {{Stream opened here}}
+  if (!F2) {
+    // expected-note@-1 {{'F2' is non-null}}
+    // expected-note@-2 {{Taking false branch}}
+    // expected-note@-3 {{'F2' is non-null}}
+    // expected-note@-4 {{Taking false branch}}
+    fclose(F1);
+    return;
+  }
+  if (c)
+    // expected-note@-1 {{Assuming 'c' is not equal to 0}}
+    // expected-note@-2 {{Taking true branch}}
+    // expected-note@-3 {{Assuming 'c' is not equal to 0}}
+    // expected-note@-4 {{Taking true branch}}
+    return;
+  // expected-warning@-1 {{Opened stream never closed. Potential resource leak}}
+  // expected-note@-2 {{Opened stream never closed. Potential resource leak}}
+  // expected-warning@-3 {{Opened stream never closed. Potential resource leak}}
+  // expected-note@-4 {{Opened stream never closed. Potential resource leak}}
+  fclose(F1);
+  fclose(F2);
+}
diff --git a/clang/test/Analysis/stream.c b/clang/test/Analysis/stream.c
index cd5d28ae84554..c57f3159fcc7d 100644
--- a/clang/test/Analysis/stream.c
+++ b/clang/test/Analysis/stream.c
@@ -261,7 +261,9 @@ void check_leak_noreturn_2() {
   if (!F1)
     return;
   if (Test == 1) {
-    return; // expected-warning {{Opened stream never closed. Potential resource leak}}
+    return; // no warning
   }
   rewind(F1);
-} // no warning
+} // expected-warning {{Opened stream never closed. Potential resource leak}}
+// FIXME: This warning should be placed at the `return` above.
+// See https://reviews.llvm.org/D83120 about details.
diff --git a/clang/test/CMakeLists.txt b/clang/test/CMakeLists.txt
index f331543e5a928..8792ddfef91fa 100644
--- a/clang/test/CMakeLists.txt
+++ b/clang/test/CMakeLists.txt
@@ -9,15 +9,6 @@ endif ()
 
 string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} CLANG_TOOLS_DIR ${LLVM_RUNTIME_OUTPUT_INTDIR})
 
-if(CLANG_BUILT_STANDALONE)
-  # Set HAVE_LIBZ according to recorded LLVM_ENABLE_ZLIB value. This
-  # value is forced to 0 if zlib was not found, so it is fine to use it
-  # instead of HAVE_LIBZ (not recorded).
-  if(LLVM_ENABLE_ZLIB)
-    set(HAVE_LIBZ 1)
-  endif()
-endif()
-
 llvm_canonicalize_cmake_booleans(
   CLANG_BUILD_EXAMPLES
   CLANG_ENABLE_ARCMT
@@ -25,7 +16,7 @@ llvm_canonicalize_cmake_booleans(
   CLANG_SPAWN_CC1
   ENABLE_BACKTRACES
   ENABLE_EXPERIMENTAL_NEW_PASS_MANAGER
-  HAVE_LIBZ
+  LLVM_ENABLE_ZLIB
   LLVM_ENABLE_PER_TARGET_RUNTIME_DIR
   LLVM_ENABLE_PLUGINS
   LLVM_ENABLE_THREADS)
diff --git a/clang/test/CodeGen/aarch64-bf16-ldst-intrinsics.c b/clang/test/CodeGen/aarch64-bf16-ldst-intrinsics.c
index 2ccfb0ca34c3f..6772c37c9723e 100644
--- a/clang/test/CodeGen/aarch64-bf16-ldst-intrinsics.c
+++ b/clang/test/CodeGen/aarch64-bf16-ldst-intrinsics.c
@@ -3,6 +3,8 @@
 // RUN: %clang_cc1 -triple armv8.6a-arm-none-eabi -target-feature +neon -target-feature +bf16 -mfloat-abi hard \
 // RUN:  -O2 -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK32
 
+// REQUIRES: arm-registered-target,aarch64-registered-target
+
 #include "arm_neon.h"
 
 bfloat16x4_t test_vld1_bf16(bfloat16_t const *ptr) {
diff --git a/clang/test/CodeGen/asm-cet.S b/clang/test/CodeGen/asm-cet.S
index 3644ed7d96492..02a1f0a3efdec 100644
--- a/clang/test/CodeGen/asm-cet.S
+++ b/clang/test/CodeGen/asm-cet.S
@@ -1,11 +1,11 @@
 // REQUIRES: x86-registered-target
-// RUN: %clang --target=x86_64-pc-linux -fcf-protection  -include cet.h -c %s -o - | llvm-readelf -n | FileCheck %s
-// RUN: %clang --target=x86_64-pc-linux -include cet.h -c %s -o - | llvm-readelf -S | FileCheck %s --check-prefixes=NOCET
+// RUN: %clang --target=x86_64-pc-linux -fcf-protection  -include cet.h -c %s -o - | llvm-readelf -n - | FileCheck %s
+// RUN: %clang --target=x86_64-pc-linux -include cet.h -c %s -o - | llvm-readelf -S - | FileCheck %s --check-prefixes=NOCET
 // RUN: %clang --target=x86_64-pc-linux -include cet.h -S %s -o - | FileCheck %s --check-prefixes=NOENDBR
 // RUN: %clang --target=x86_64-pc-linux -fcf-protection  -include cet.h -S %s -o - | FileCheck %s --check-prefixes=ENDBR64
 
-// RUN: %clang --target=i386-pc-linux -fcf-protection  -include cet.h -c %s -o - | llvm-readelf -n | FileCheck %s
-// RUN: %clang --target=i386-pc-linux -include cet.h -c %s -o - | llvm-readelf -S | FileCheck %s --check-prefixes=NOCET
+// RUN: %clang --target=i386-pc-linux -fcf-protection  -include cet.h -c %s -o - | llvm-readelf -n - | FileCheck %s
+// RUN: %clang --target=i386-pc-linux -include cet.h -c %s -o - | llvm-readelf -S - | FileCheck %s --check-prefixes=NOCET
 // RUN: %clang --target=i386-pc-linux -include cet.h -S %s -o - | FileCheck %s --check-prefixes=NOENDBR
 // RUN: %clang --target=i386-pc-linux -fcf-protection  -include cet.h -S %s -o - | FileCheck %s --check-prefixes=ENDBR32
 
diff --git a/clang/test/CodeGen/attr-nomerge.cpp b/clang/test/CodeGen/attr-nomerge.cpp
index 284ce78279661..3405ea737df1f 100644
--- a/clang/test/CodeGen/attr-nomerge.cpp
+++ b/clang/test/CodeGen/attr-nomerge.cpp
@@ -10,6 +10,7 @@ void foo(int i) {
   [[clang::nomerge]] f(bar(), bar());
   [[clang::nomerge]] [] { bar(); bar(); }(); // nomerge only applies to the anonymous function call
   [[clang::nomerge]] for (bar(); bar(); bar()) {}
+  [[clang::nomerge]] { asm("nop"); }
   bar();
 }
 // CHECK: call zeroext i1 @_Z3barv() #[[NOMERGEATTR:[0-9]+]]
@@ -22,5 +23,7 @@ void foo(int i) {
 // CHECK: call zeroext i1 @_Z3barv() #[[NOMERGEATTR]]
 // CHECK: call zeroext i1 @_Z3barv() #[[NOMERGEATTR]]
 // CHECK: call zeroext i1 @_Z3barv() #[[NOMERGEATTR]]
+// CHECK: call void asm {{.*}} #[[NOMERGEATTR2:[0-9]+]]
 // CHECK: call zeroext i1 @_Z3barv()
 // CHECK: attributes #[[NOMERGEATTR]] = { nomerge }
+// CHECK: attributes #[[NOMERGEATTR2]] = { nomerge nounwind }
diff --git a/clang/test/CodeGen/builtin-complex.c b/clang/test/CodeGen/builtin-complex.c
new file mode 100644
index 0000000000000..cf97b18d87471
--- /dev/null
+++ b/clang/test/CodeGen/builtin-complex.c
@@ -0,0 +1,22 @@
+// RUN: %clang_cc1 -triple x86_64-linux -w -S -o - -emit-llvm -DT=float %s | FileCheck %s --check-prefixes=CHECK,CHECK-FLOAT
+// RUN: %clang_cc1 -triple x86_64-linux -w -S -o - -emit-llvm -DT=double %s | FileCheck %s --check-prefixes=CHECK,CHECK-DOUBLE
+// RUN: %clang_cc1 -triple x86_64-linux -w -S -o - -emit-llvm -DT="long double" %s | FileCheck %s --check-prefixes=CHECK,CHECK-FP80
+// RUN: %clang_cc1 -triple x86_64-linux -w -S -o - -emit-llvm -DT=__float128 %s | FileCheck %s --check-prefixes=CHECK,CHECK-FP128
+// FIXME: If we start to support _Complex __fp16 or _Complex _Float16, add tests for them too.
+
+// CHECK-FLOAT: @global = global { [[T:float]], [[T]] } { [[T]] 1.0{{.*}}, [[T]] 2.0{{.*}} }
+// CHECK-DOUBLE: @global = global { [[T:double]], [[T]] } { [[T]] 1.0{{.*}}, [[T]] 2.0{{.*}} }
+// CHECK-FP80: @global = global { [[T:x86_fp80]], [[T]] } { [[T]] 0xK3FFF8000000000000000, [[T]] 0xK40008000000000000000 }
+// CHECK-FP128: @global = global { [[T:fp128]], [[T]] } { [[T]] 0xL00000000000000003FFF000000000000, [[T]] 0xL00000000000000004000000000000000 }
+_Complex T global = __builtin_complex(1.0, 2.0);
+
+// CHECK-LABEL: @test
+_Complex T test(T a, T b) {
+  return __builtin_complex(a, b);
+  // CHECK: %[[A:.*]] = load [[T]], [[T]]* %a.addr,
+  // CHECK: %[[B:.*]] = load [[T]], [[T]]* %b.addr,
+  // CHECK: %[[RET_RE:.*]] = getelementptr inbounds { [[T]], [[T]] }, { [[T]], [[T]] }* %[[RET:[^,]*]], i32 0, i32 0
+  // CHECK: %[[RET_IM:.*]] = getelementptr inbounds { [[T]], [[T]] }, { [[T]], [[T]] }* %[[RET]], i32 0, i32 1
+  // CHECK: store [[T]] %[[A]], [[T]]* %[[RET_RE]],
+  // CHECK: store [[T]] %[[B]], [[T]]* %[[RET_IM]],
+}
diff --git a/clang/test/CodeGen/builtins-ppc-p10vector.c b/clang/test/CodeGen/builtins-ppc-p10vector.c
index 4e804fbafb301..6f38ac77ee242 100644
--- a/clang/test/CodeGen/builtins-ppc-p10vector.c
+++ b/clang/test/CodeGen/builtins-ppc-p10vector.c
@@ -434,25 +434,25 @@ vector unsigned long long test_vec_insertl_ul(void) {
 }
 
 vector unsigned char test_vec_insertl_ucv(void) {
-  // CHECK-BE: @llvm.ppc.altivec.vinsbvlx(<16 x i8> %{{.+}}, i64 %{{.+}}, <16 x i8>
+  // CHECK-BE: @llvm.ppc.altivec.vinsbvlx(<16 x i8> %{{.+}}, i32 %{{.+}}, <16 x i8>
   // CHECK-BE-NEXT: ret <16 x i8>
-  // CHECK-LE: @llvm.ppc.altivec.vinsbvrx(<16 x i8> %{{.+}}, i64 %{{.+}}, <16 x i8>
+  // CHECK-LE: @llvm.ppc.altivec.vinsbvrx(<16 x i8> %{{.+}}, i32 %{{.+}}, <16 x i8>
   // CHECK-LE-NEXT: ret <16 x i8>
   return vec_insertl(vuca, vucb, uia);
 }
 
 vector unsigned short test_vec_insertl_usv(void) {
-  // CHECK-BE: @llvm.ppc.altivec.vinshvlx(<8 x i16> %{{.+}}, i64 %{{.+}}, <8 x i16>
+  // CHECK-BE: @llvm.ppc.altivec.vinshvlx(<8 x i16> %{{.+}}, i32 %{{.+}}, <8 x i16>
   // CHECK-BE-NEXT: ret <8 x i16>
-  // CHECK-LE: @llvm.ppc.altivec.vinshvrx(<8 x i16> %{{.+}}, i64 %{{.+}}, <8 x i16>
+  // CHECK-LE: @llvm.ppc.altivec.vinshvrx(<8 x i16> %{{.+}}, i32 %{{.+}}, <8 x i16>
   // CHECK-LE-NEXT: ret <8 x i16>
   return vec_insertl(vusa, vusb, uia);
 }
 
 vector unsigned int test_vec_insertl_uiv(void) {
-  // CHECK-BE: @llvm.ppc.altivec.vinswvlx(<4 x i32> %{{.+}}, i64 %{{.+}}, <4 x i32>
+  // CHECK-BE: @llvm.ppc.altivec.vinswvlx(<4 x i32> %{{.+}}, i32 %{{.+}}, <4 x i32>
   // CHECK-BE-NEXT: ret <4 x i32>
-  // CHECK-LE: @llvm.ppc.altivec.vinswvrx(<4 x i32> %{{.+}}, i64 %{{.+}}, <4 x i32>
+  // CHECK-LE: @llvm.ppc.altivec.vinswvrx(<4 x i32> %{{.+}}, i32 %{{.+}}, <4 x i32>
   // CHECK-LE-NEXT: ret <4 x i32>
   return vec_insertl(vuia, vuib, uia);
 }
@@ -490,25 +490,25 @@ vector unsigned long long test_vec_inserth_ul(void) {
 }
 
 vector unsigned char test_vec_inserth_ucv(void) {
-  // CHECK-BE: @llvm.ppc.altivec.vinsbvrx(<16 x i8> %{{.+}}, i64 %{{.+}}, <16 x i8>
+  // CHECK-BE: @llvm.ppc.altivec.vinsbvrx(<16 x i8> %{{.+}}, i32 %{{.+}}, <16 x i8>
   // CHECK-BE-NEXT: ret <16 x i8>
-  // CHECK-LE: @llvm.ppc.altivec.vinsbvlx(<16 x i8> %{{.+}}, i64 %{{.+}}, <16 x i8>
+  // CHECK-LE: @llvm.ppc.altivec.vinsbvlx(<16 x i8> %{{.+}}, i32 %{{.+}}, <16 x i8>
   // CHECK-LE-NEXT: ret <16 x i8>
   return vec_inserth(vuca, vucb, uia);
 }
 
 vector unsigned short test_vec_inserth_usv(void) {
-  // CHECK-BE: @llvm.ppc.altivec.vinshvrx(<8 x i16> %{{.+}}, i64 %{{.+}}, <8 x i16>
+  // CHECK-BE: @llvm.ppc.altivec.vinshvrx(<8 x i16> %{{.+}}, i32 %{{.+}}, <8 x i16>
   // CHECK-BE-NEXT: ret <8 x i16>
-  // CHECK-LE: @llvm.ppc.altivec.vinshvlx(<8 x i16> %{{.+}}, i64 %{{.+}}, <8 x i16>
+  // CHECK-LE: @llvm.ppc.altivec.vinshvlx(<8 x i16> %{{.+}}, i32 %{{.+}}, <8 x i16>
   // CHECK-LE-NEXT: ret <8 x i16>
   return vec_inserth(vusa, vusb, uia);
 }
 
 vector unsigned int test_vec_inserth_uiv(void) {
-  // CHECK-BE: @llvm.ppc.altivec.vinswvrx(<4 x i32> %{{.+}}, i64 %{{.+}}, <4 x i32>
+  // CHECK-BE: @llvm.ppc.altivec.vinswvrx(<4 x i32> %{{.+}}, i32 %{{.+}}, <4 x i32>
   // CHECK-BE-NEXT: ret <4 x i32>
-  // CHECK-LE: @llvm.ppc.altivec.vinswvlx(<4 x i32> %{{.+}}, i64 %{{.+}}, <4 x i32>
+  // CHECK-LE: @llvm.ppc.altivec.vinswvlx(<4 x i32> %{{.+}}, i32 %{{.+}}, <4 x i32>
   // CHECK-LE-NEXT: ret <4 x i32>
   return vec_inserth(vuia, vuib, uia);
 }
@@ -583,13 +583,13 @@ vector float test_vec_vec_splati_ins_f(void) {
 }
 
 int test_vec_test_lsbb_all_ones(void) {
-  // CHECK: @llvm.ppc.vsx.xvtlsbb(<16 x i8> %{{.+}}, i1 true
+  // CHECK: @llvm.ppc.vsx.xvtlsbb(<16 x i8> %{{.+}}, i32 1
   // CHECK-NEXT: ret i32
   return vec_test_lsbb_all_ones(vuca);
 }
 
 int test_vec_test_lsbb_all_zeros(void) {
-  // CHECK: @llvm.ppc.vsx.xvtlsbb(<16 x i8> %{{.+}}, i1 false
+  // CHECK: @llvm.ppc.vsx.xvtlsbb(<16 x i8> %{{.+}}, i32 0
   // CHECK-NEXT: ret i32
   return vec_test_lsbb_all_zeros(vuca);
 }
diff --git a/clang/test/CodeGen/builtins-systemz.c b/clang/test/CodeGen/builtins-systemz.c
index f5de7009acba0..bf8a995383a22 100644
--- a/clang/test/CodeGen/builtins-systemz.c
+++ b/clang/test/CodeGen/builtins-systemz.c
@@ -142,3 +142,10 @@ void test_htmxl1(void) {
   result = __TM_failure_code (tdb);
 }
 
+void test_eh_return_data_regno() {
+  volatile int res;
+  res = __builtin_eh_return_data_regno(0); // CHECK: store volatile i32 6
+  res = __builtin_eh_return_data_regno(1); // CHECK: store volatile i32 7
+  res = __builtin_eh_return_data_regno(2); // CHECK: store volatile i32 8
+  res = __builtin_eh_return_data_regno(3); // CHECK: store volatile i32 9
+}
diff --git a/clang/test/CodeGen/complex-builtints.c b/clang/test/CodeGen/complex-builtins-3.c
similarity index 100%
rename from clang/test/CodeGen/complex-builtints.c
rename to clang/test/CodeGen/complex-builtins-3.c
diff --git a/clang/test/CodeGen/matrix-lowering-opt-levels.c b/clang/test/CodeGen/matrix-lowering-opt-levels.c
new file mode 100644
index 0000000000000..73a9458b8bf5d
--- /dev/null
+++ b/clang/test/CodeGen/matrix-lowering-opt-levels.c
@@ -0,0 +1,28 @@
+// RUN: %clang -O0 -fenable-matrix -S -emit-llvm %s -o - | FileCheck  %s
+// RUN: %clang -O1 -fenable-matrix -S -emit-llvm %s -o - | FileCheck  %s
+// RUN: %clang -O2 -fenable-matrix -S -emit-llvm %s -o - | FileCheck  %s
+// RUN: %clang -O3 -fenable-matrix -S -emit-llvm %s -o - | FileCheck  %s
+// RUN: %clang -Ofast -fenable-matrix -S -emit-llvm %s -o - | FileCheck  %s
+// RUN: %clang -Os -fenable-matrix -S -emit-llvm %s -o - | FileCheck  %s
+// RUN: %clang -Oz -fenable-matrix -S -emit-llvm %s -o - | FileCheck  %s
+
+// RUN: %clang -O0 -fenable-matrix -fexperimental-new-pass-manager -S -emit-llvm %s -o - | FileCheck  %s
+// RUN: %clang -O1 -fenable-matrix -fexperimental-new-pass-manager -S -emit-llvm %s -o - | FileCheck  %s
+// RUN: %clang -O2 -fenable-matrix -fexperimental-new-pass-manager -S -emit-llvm %s -o - | FileCheck  %s
+// RUN: %clang -O3 -fenable-matrix -fexperimental-new-pass-manager -S -emit-llvm %s -o - | FileCheck  %s
+// RUN: %clang -Ofast -fenable-matrix -fexperimental-new-pass-manager -S -emit-llvm %s -o - | FileCheck  %s
+// RUN: %clang -Os -fenable-matrix -fexperimental-new-pass-manager -S -emit-llvm %s -o - | FileCheck  %s
+// RUN: %clang -Oz -fenable-matrix -fexperimental-new-pass-manager -S -emit-llvm %s -o - | FileCheck  %s
+
+// Smoke test that the matrix intrinsics are lowered at any optimisation level.
+
+typedef float m4x4_t __attribute__((matrix_type(4, 4)));
+
+m4x4_t f(m4x4_t a, m4x4_t b, m4x4_t c) {
+  //
+  // CHECK-LABEL: f(
+  // CHECK-NOT:     @llvm.matrix
+  // CHECK:       }
+  //
+  return a + b * c;
+}
diff --git a/clang/test/CodeGen/pch-dllexport.cpp b/clang/test/CodeGen/pch-dllexport.cpp
index 034995fdffc4f..72f344c77be1d 100644
--- a/clang/test/CodeGen/pch-dllexport.cpp
+++ b/clang/test/CodeGen/pch-dllexport.cpp
@@ -3,13 +3,20 @@
 // RUN: %clang_cc1 -triple i686-pc-win32 -fms-extensions -emit-obj -emit-llvm -include-pch %t -o - %s | FileCheck -check-prefix=PCH %s
 
 // Build PCH with object file, then use it.
-// RUN: %clang_cc1 -triple i686-pc-win32 -fms-extensions -emit-pch -building-pch-with-obj -o %t %s
-// RUN: %clang_cc1 -triple i686-pc-win32 -fms-extensions -emit-obj -emit-llvm -include-pch %t -building-pch-with-obj -o - %s | FileCheck -check-prefix=OBJ %s
-// RUN: %clang_cc1 -triple i686-pc-win32 -fms-extensions -emit-obj -emit-llvm -include-pch %t -o - %s | FileCheck -check-prefix=PCHWITHOBJ %s
+// RUN: %clang_cc1 -triple i686-pc-win32 -O1 -fms-extensions -emit-pch -building-pch-with-obj -o %t %s
+// RUN: %clang_cc1 -triple i686-pc-win32 -O1 -disable-llvm-optzns -fms-extensions -emit-obj -emit-llvm -include-pch %t -building-pch-with-obj -o - %s | FileCheck -check-prefix=OBJ %s
+// RUN: %clang_cc1 -triple i686-pc-win32 -O1 -disable-llvm-optzns -fms-extensions -emit-obj -emit-llvm -include-pch %t -o - %s | FileCheck -check-prefix=PCHWITHOBJ -check-prefix=PCHWITHOBJ-O1 %s
 
 // Check for vars separately to avoid having to reorder the check statements.
+// RUN: %clang_cc1 -triple i686-pc-win32 -O1 -disable-llvm-optzns -fms-extensions -emit-obj -emit-llvm -include-pch %t -o - %s | FileCheck -check-prefix=PCHWITHOBJVARS %s
+
+// Test the PCHWITHOBJ at -O0 where available_externally definitions are not
+// provided:
+// RUN: %clang_cc1 -triple i686-pc-win32 -fms-extensions -emit-pch -building-pch-with-obj -o %t %s
+// RUN: %clang_cc1 -triple i686-pc-win32 -fms-extensions -emit-obj -emit-llvm -include-pch %t -o - %s | FileCheck -check-prefix=PCHWITHOBJ -check-prefix=PCHWITHOBJ-O0 %s
 // RUN: %clang_cc1 -triple i686-pc-win32 -fms-extensions -emit-obj -emit-llvm -include-pch %t -o - %s | FileCheck -check-prefix=PCHWITHOBJVARS %s
 
+
 #ifndef IN_HEADER
 #define IN_HEADER
 
@@ -23,7 +30,8 @@ inline void __declspec(dllexport) foo() {}
 inline void __declspec(dllexport) baz() {}
 // OBJ: define weak_odr dso_local dllexport void @"?baz@@YAXXZ"
 // PCH: define weak_odr dso_local dllexport void @"?baz@@YAXXZ"
-// PCHWITHOBJ: define weak_odr dso_local dllexport void @"?baz@@YAXXZ"
+// PCHWITHOBJ-O1: define available_externally dso_local void @"?baz@@YAXXZ"
+// PCHWITHOBJ-O0-NOT: define {{.*}}"?baz@@YAXXZ"
 
 
 struct __declspec(dllexport) S {
diff --git a/clang/test/CodeGen/target-data.c b/clang/test/CodeGen/target-data.c
index 004659539a9f1..db0d8d1cc222d 100644
--- a/clang/test/CodeGen/target-data.c
+++ b/clang/test/CodeGen/target-data.c
@@ -14,6 +14,10 @@
 // RUN:     FileCheck --check-prefix=I686-CYGWIN %s
 // I686-CYGWIN: target datalayout = "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:32-n8:16:32-a:0:32-S32"
 
+// RUN: %clang_cc1 -triple i686-pc-macho -emit-llvm -o - %s | \
+// RUN:     FileCheck --check-prefix=I686-MACHO %s
+// I686-MACHO: target datalayout = "e-m:o-p:32:32-p270:32:32-p271:32:32-p272:64:64-f64:32:64-f80:32-n8:16:32-S128"
+
 // RUN: %clang_cc1 -triple x86_64-unknown-unknown -emit-llvm -o - %s | \
 // RUN:     FileCheck --check-prefix=X86_64 %s
 // X86_64: target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/clang/test/CodeGen/thinlto-distributed-newpm.ll b/clang/test/CodeGen/thinlto-distributed-newpm.ll
index 013da3c9af4cb..6269fd677ca82 100644
--- a/clang/test/CodeGen/thinlto-distributed-newpm.ll
+++ b/clang/test/CodeGen/thinlto-distributed-newpm.ll
@@ -62,13 +62,13 @@
 ; CHECK-O: Running pass: InstCombinePass on main
 ; CHECK-O: Running analysis: TargetLibraryAnalysis on main
 ; CHECK-O: Running analysis: OptimizationRemarkEmitterAnalysis on main
+; CHECK-O: Running analysis: TargetIRAnalysis on main
 ; CHECK-O: Running analysis: AAManager on main
 ; CHECK-O: Running analysis: BasicAA on main
 ; CHECK-O: Running analysis: ScopedNoAliasAA on main
 ; CHECK-O: Running analysis: TypeBasedAA on main
 ; CHECK-O: Running analysis: OuterAnalysisManagerProxy
 ; CHECK-O: Running pass: SimplifyCFGPass on main
-; CHECK-O: Running analysis: TargetIRAnalysis on main
 ; CHECK-O: Finished {{.*}}Function pass manager run.
 ; CHECK-O: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
 ; CHECK-O: Running analysis: GlobalsAA
diff --git a/clang/test/CodeGenCXX/Inputs/profile-remap_entry.proftext b/clang/test/CodeGenCXX/Inputs/profile-remap_entry.proftext
new file mode 100644
index 0000000000000..65bc6ff46711b
--- /dev/null
+++ b/clang/test/CodeGenCXX/Inputs/profile-remap_entry.proftext
@@ -0,0 +1,8 @@
+:ir
+:entry_first
+_ZN3Foo8functionENS_1XE
+29667547796
+2
+100
+90
+
diff --git a/clang/test/CodeGenCXX/profile-remap.cpp b/clang/test/CodeGenCXX/profile-remap.cpp
index 0fc7c7b40033e..b519e55d21122 100644
--- a/clang/test/CodeGenCXX/profile-remap.cpp
+++ b/clang/test/CodeGenCXX/profile-remap.cpp
@@ -3,6 +3,8 @@
 // RUN: %clang_cc1 -triple x86_64-linux-gnu -fprofile-sample-use=%S/Inputs/profile-remap.samples -fprofile-remapping-file=%S/Inputs/profile-remap.map -fexperimental-new-pass-manager -O2 %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SAMPLES
 // RUN: llvm-profdata merge -output %T.profdata %S/Inputs/profile-remap.proftext
 // RUN: %clang_cc1 -triple x86_64-linux-gnu -fprofile-instrument-use-path=%T.profdata -fprofile-remapping-file=%S/Inputs/profile-remap.map -fexperimental-new-pass-manager -O2 %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK,CHECK-INSTR
+// RUN: llvm-profdata merge -output %T.profdata %S/Inputs/profile-remap_entry.proftext
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -fprofile-instrument-use-path=%T.profdata -fprofile-remapping-file=%S/Inputs/profile-remap.map -fexperimental-new-pass-manager -O2 %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK,CHECK-INSTR
 
 namespace Foo {
   struct X {};
diff --git a/clang/test/CodeGenOpenCL/convergent.cl b/clang/test/CodeGenOpenCL/convergent.cl
index 9471f93a42dc2..43d728b896575 100644
--- a/clang/test/CodeGenOpenCL/convergent.cl
+++ b/clang/test/CodeGenOpenCL/convergent.cl
@@ -70,10 +70,9 @@ void test_merge_if(int a) {
 // CHECK-NOT: call spir_func void @g()
 // CHECK: br label %[[if_end]]
 // CHECK: [[if_end]]:
-// FIXME: SimplifyCFG is being stupid inserting this Phi. It is not supposed to be here.
-// CHECK:  %[[tobool_not_pr:.+]] = phi i1
+// CHECK-NOT: phi i1
 // CHECK:  tail call spir_func void @convfun() #[[attr4:.+]]
-// CHECK:  br i1 %[[tobool_not_pr]], label %[[if_end3:.+]], label %[[if_then2:.+]]
+// CHECK:  br i1 %[[tobool]], label %[[if_end3:.+]], label %[[if_then2:.+]]
 // CHECK: [[if_then2]]:
 // CHECK: tail call spir_func void @g()
 // CHECK:  br label %[[if_end3:.+]]
diff --git a/clang/test/CodeGenSYCL/esimd-private-global.cpp b/clang/test/CodeGenSYCL/esimd-private-global.cpp
index d5c475b5656af..b0e80eb34ee78 100644
--- a/clang/test/CodeGenSYCL/esimd-private-global.cpp
+++ b/clang/test/CodeGenSYCL/esimd-private-global.cpp
@@ -9,6 +9,6 @@ __attribute__((opencl_private)) __attribute__((register_num(17))) int vc;
 
 SYCL_EXTERNAL void init_vc(int x) {
   vc = x;
-  // CHECK: store i32 %0, i32* @vc
+  // CHECK: store i32 %{{[0-9a-zA-Z_]+}}, i32* @vc
 }
-// CHECK: attributes #0 = { "genx_byte_offset"="17" "genx_volatile" }
+// CHECK: attributes #0 = {{.*"VCByteOffset"="17".*"VCVolatile"}}
diff --git a/clang/test/CodeGenSYCL/esimd_metadata1.cpp b/clang/test/CodeGenSYCL/esimd_metadata1.cpp
index 717cb5c85fc16..554f508ef4f32 100644
--- a/clang/test/CodeGenSYCL/esimd_metadata1.cpp
+++ b/clang/test/CodeGenSYCL/esimd_metadata1.cpp
@@ -19,6 +19,6 @@ void bar() {
 }
 
 // CHECK: !spirv.Source = !{[[LANG:![0-9]+]]}
-// CHECK: [[LANG]] = !{i32 6, i32 100000}
+// CHECK: [[LANG]] = !{i32 0, i32 {{[0-9]+}}}
 // CHECK: ![[EMPTY]] = !{}
 // CHECK: ![[REQD_SIZE]] = !{i32 1}
diff --git a/clang/test/CodeGenSYCL/esimd_metadata2.cpp b/clang/test/CodeGenSYCL/esimd_metadata2.cpp
new file mode 100644
index 0000000000000..c14b0af75f3c1
--- /dev/null
+++ b/clang/test/CodeGenSYCL/esimd_metadata2.cpp
@@ -0,0 +1,39 @@
+// RUN: %clang_cc1 -disable-llvm-passes -triple spir64-unknown-unknown-sycldevice -fsycl -fsycl-is-device -fsycl-explicit-simd -S -emit-llvm %s -o - | FileCheck %s --check-prefixes CHECK,CHECK-ESIMD
+
+// In ESIMD mode:
+// 1. Attribute !intel_reqd_sub_group_size !1 is added
+//    for kernels with !sycl_explicit_simd
+// 2. Attribute !sycl_explicit_simd is propagated to all the
+//    callees of ESIMD kernel.
+
+__attribute__((sycl_device)) void shared_func_decl();
+__attribute__((sycl_device)) void shared_func() { shared_func_decl(); }
+
+__attribute__((sycl_device)) __attribute__((sycl_explicit_simd)) void esimd_func() { shared_func(); }
+
+// CHECK-ESIMD-DAG: define spir_kernel void @{{.*}}kernel_cm() #{{[0-9]+}} !sycl_explicit_simd !{{[0-9]+}} {{.*}} !intel_reqd_sub_group_size ![[SGSIZE1:[0-9]+]] {{.*}}{
+// CHECK-ESIMD-DAG: define spir_func void @{{.*}}esimd_funcv() #{{[0-9]+}} !sycl_explicit_simd !{{[0-9]+}} {
+// CHECK-ESIMD-DAG: define spir_func void @{{.*}}shared_funcv() #{{[0-9]+}} !sycl_explicit_simd !{{[0-9]+}} {
+// CHECK-ESIMD-DAG: define linkonce_odr spir_func void @_ZN12ESIMDFunctorclEv({{.*}}) #{{[0-9]+}} {{.*}} !sycl_explicit_simd !{{[0-9]+}} {
+// CHECK-ESIMD-DAG: declare spir_func void @{{.*}}shared_func_declv() #{{[0-9]+}}
+
+class ESIMDFunctor {
+public:
+  void operator()() __attribute__((sycl_explicit_simd)) {
+    esimd_func();
+  }
+};
+
+template <typename name, typename Func>
+__attribute__((sycl_kernel)) void kernel(Func kernelFunc) {
+  kernelFunc();
+}
+
+void bar() {
+  ESIMDFunctor cmf;
+  kernel<class kernel_cm>(cmf);
+}
+
+// CHECK: !spirv.Source = !{[[LANG:![0-9]+]]}
+// CHECK: [[LANG]] = !{i32 0, i32 {{[0-9]+}}}
+// CHECK-ESIMD: ![[SGSIZE1]] = !{i32 1}
diff --git a/clang/test/CodeGenSYCL/esimd_metadata3.cpp b/clang/test/CodeGenSYCL/esimd_metadata3.cpp
new file mode 100644
index 0000000000000..02885b1b478db
--- /dev/null
+++ b/clang/test/CodeGenSYCL/esimd_metadata3.cpp
@@ -0,0 +1,7 @@
+// RUN: %clang_cc1 -disable-llvm-passes -triple spir64-unknown-unknown-sycldevice -fsycl -fsycl-is-device -fsycl-explicit-simd -S -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK-ESIMD
+
+__attribute__((sycl_device)) void funcWithSpirvIntrin() {}
+__attribute__((sycl_device)) __attribute__((sycl_explicit_simd)) void standaloneCmFunc() { funcWithSpirvIntrin(); }
+
+// CHECK-ESIMD-DAG: define spir_func void @{{.*}}funcWithSpirvIntrinv() #{{[0-9]+}} !sycl_explicit_simd !{{[0-9]+}} {
+// CHECK-ESIMD-DAG: define spir_func void @{{.*}}standaloneCmFuncv() #{{[0-9]+}} !sycl_explicit_simd !{{[0-9]+}} {
diff --git a/clang/test/CodeGenSYCL/remove-ur-inst.cpp b/clang/test/CodeGenSYCL/remove-ur-inst.cpp
index 67fe27cf0c22b..7866aff7f07a3 100644
--- a/clang/test/CodeGenSYCL/remove-ur-inst.cpp
+++ b/clang/test/CodeGenSYCL/remove-ur-inst.cpp
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -fsycl -fsycl-is-device -triple spir64-unknown-unknown-sycldevice -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -fsycl -fsycl-is-device -fsycl-enable-optimizations -triple spir64-unknown-unknown-sycldevice -emit-llvm %s -o - | FileCheck %s
 
 SYCL_EXTERNAL void doesNotReturn() throw() __attribute__((__noreturn__));
 
@@ -11,6 +12,7 @@ int main() {
   kernel<class test>([]() {
     doesNotReturn();
     // CHECK-NOT: unreachable
+    // CHECK-NOT: noreturn
   });
   return 0;
-}
\ No newline at end of file
+}
diff --git a/clang/test/Driver/Inputs/basic_freebsd64_tree/usr/bin/ld.bfd b/clang/test/Driver/Inputs/basic_freebsd64_tree/usr/bin/ld.bfd
new file mode 100755
index 0000000000000..b23e55619b2ff
--- /dev/null
+++ b/clang/test/Driver/Inputs/basic_freebsd64_tree/usr/bin/ld.bfd
@@ -0,0 +1 @@
+#!/bin/true
diff --git a/clang/test/Driver/aarch64-sve-vector-bits.c b/clang/test/Driver/aarch64-sve-vector-bits.c
index c3d0d05bb9b6d..ffe82f113fd17 100644
--- a/clang/test/Driver/aarch64-sve-vector-bits.c
+++ b/clang/test/Driver/aarch64-sve-vector-bits.c
@@ -12,12 +12,15 @@
 // RUN:  -msve-vector-bits=1024 2>&1 | FileCheck --check-prefix=CHECK-1024 %s
 // RUN: %clang -c %s -### -target aarch64-none-linux-gnu -march=armv8-a+sve \
 // RUN:  -msve-vector-bits=2048 2>&1 | FileCheck --check-prefix=CHECK-2048 %s
+// RUN: %clang -c %s -### -target aarch64-none-linux-gnu -march=armv8-a+sve \
+// RUN:  -msve-vector-bits=scalable 2>&1 | FileCheck --check-prefix=CHECK-SCALABLE %s
 
 // CHECK-128: "-msve-vector-bits=128"
 // CHECK-256: "-msve-vector-bits=256"
 // CHECK-512: "-msve-vector-bits=512"
 // CHECK-1024: "-msve-vector-bits=1024"
 // CHECK-2048: "-msve-vector-bits=2048"
+// CHECK-SCALABLE-NOT: "-msve-vector-bits=
 
 // Bail out if -msve-vector-bits is specified without SVE enabled
 // -----------------------------------------------------------------------------
@@ -47,11 +50,13 @@
 // -----------------------------------------------------------------------------
 // RUN: not %clang -c %s -o /dev/null -target aarch64-none-linux-gnu \
 // RUN:  -march=armv8-a+sve 2>&1 | FileCheck --check-prefix=CHECK-NO-FLAG-ERROR %s
+// RUN: not %clang -c %s -o /dev/null -target aarch64-none-linux-gnu \
+// RUN:  -march=armv8-a+sve -msve-vector-bits=scalable 2>&1 | FileCheck --check-prefix=CHECK-NO-FLAG-ERROR %s
 
 typedef __SVInt32_t svint32_t;
 typedef svint32_t noflag __attribute__((arm_sve_vector_bits(256)));
 
-// CHECK-NO-FLAG-ERROR: error: 'arm_sve_vector_bits' is not supported when '-msve-vector-bits=<bits>' is not specified
+// CHECK-NO-FLAG-ERROR: error: 'arm_sve_vector_bits' is only supported when '-msve-vector-bits=<bits>' is specified with a value of 128, 256, 512, 1024 or 2048
 
 // Error if attribute vector size != -msve-vector-bits
 // -----------------------------------------------------------------------------
diff --git a/clang/test/Driver/cl-outputs.c b/clang/test/Driver/cl-outputs.c
index 6e105e46850dd..75ff77e15910b 100644
--- a/clang/test/Driver/cl-outputs.c
+++ b/clang/test/Driver/cl-outputs.c
@@ -99,6 +99,7 @@
 // DEFAULTDLL: "-implib:cl-outputs.lib"
 
 // RUN: %clang_cl /Fefoo -### -- %s 2>&1 | FileCheck -check-prefix=FeNOEXT %s
+// RUN: %clang_cl /Fe:foo -### -- %s 2>&1 | FileCheck -check-prefix=FeNOEXT %s
 // FeNOEXT: "-out:foo.exe"
 
 // RUN: %clang_cl /Fe -### -- %s 2>&1 | FileCheck -check-prefix=FeEMPTY %s
@@ -111,6 +112,7 @@
 // FeNOEXTDLL: "-implib:foo.lib"
 
 // RUN: %clang_cl /Fefoo.ext -### -- %s 2>&1 | FileCheck -check-prefix=FeEXT %s
+// RUN: %clang_cl /Fe:foo.ext -### -- %s 2>&1 | FileCheck -check-prefix=FeEXT %s
 // FeEXT: "-out:foo.ext"
 
 // RUN: %clang_cl /LD /Fefoo.ext -### -- %s 2>&1 | FileCheck -check-prefix=FeEXTDLL %s
diff --git a/clang/test/Driver/fuse-ld.c b/clang/test/Driver/fuse-ld.c
index f2ca9fb36194e..678913dd84c36 100644
--- a/clang/test/Driver/fuse-ld.c
+++ b/clang/test/Driver/fuse-ld.c
@@ -2,6 +2,7 @@
 // RUN:     -fuse-ld=/usr/local/bin/or1k-linux-ld 2>&1 \
 // RUN:     -target x86_64-unknown-linux \
 // RUN:   | FileCheck %s --check-prefix=CHECK-ABSOLUTE-LD
+// CHECK-ABSOLUTE-LD: warning: '-fuse-ld=' taking a path is deprecated. Use '--ld-path=' instead
 // CHECK-ABSOLUTE-LD: /usr/local/bin/or1k-linux-ld
 
 
diff --git a/clang/test/Driver/ld-path.c b/clang/test/Driver/ld-path.c
new file mode 100644
index 0000000000000..a982ea3739a18
--- /dev/null
+++ b/clang/test/Driver/ld-path.c
@@ -0,0 +1,66 @@
+/// This tests uses the PATH environment variable.
+// UNSUPPORTED: system-windows
+
+// RUN: cd %S
+
+/// If --ld-path= specifies a word (without /), -B and COMPILER_PATH are
+/// consulted to locate the linker.
+// RUN: %clang %s -### -B %S/Inputs/basic_freebsd_tree/usr/bin --ld-path=ld.bfd \
+// RUN:   --target=x86_64-unknown-freebsd --sysroot=%S/Inputs/basic_freebsd_tree 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=BFD
+// RUN: env COMPILER_PATH=%S/Inputs/basic_freebsd_tree/usr/bin %clang %s -### --ld-path=ld.bfd \
+// RUN:   --target=x86_64-unknown-freebsd --sysroot=%S/Inputs/basic_freebsd_tree 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=BFD
+/// Then PATH is consulted.
+// RUN: env PATH=%S/Inputs/basic_freebsd_tree/usr/bin %clang %s -### --ld-path=ld.bfd \
+// RUN:   --target=x86_64-unknown-freebsd --sysroot=%S/Inputs/basic_freebsd_tree 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=BFD
+
+// BFD: Inputs/basic_freebsd_tree/usr/bin/ld.bfd"
+
+// RUN: env PATH=%S/Inputs/basic_freebsd_tree/usr/bin %clang %s -### --ld-path=ld.gold \
+// RUN:   --target=x86_64-unknown-freebsd --sysroot=%S/Inputs/basic_freebsd_tree 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=GOLD
+
+// GOLD: Inputs/basic_freebsd_tree/usr/bin/ld.gold"
+
+// RUN: env COMPILER_PATH= PATH=%S/Inputs/basic_freebsd_tree/usr/bin %clang %s -### --ld-path=not_exist \
+// RUN:   --target=x86_64-unknown-freebsd --sysroot=%S/Inputs/basic_freebsd_tree 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=NOT_EXIST
+
+// NOT_EXIST: error: invalid linker name in argument '--ld-path=not_exist'
+
+// RUN: %clang %s -### --ld-path= \
+// RUN:   --target=x86_64-unknown-freebsd --sysroot=%S/Inputs/basic_freebsd_tree 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=EMPTY
+
+// EMPTY: error: invalid linker name in argument '--ld-path='
+
+/// If --ld-path= contains a slash, PATH is not consulted.
+// RUN: env COMPILER_PATH=%S/Inputs/basic_freebsd_tree/usr/bin %clang %s -### --ld-path=./ld.bfd \
+// RUN:   --target=x86_64-unknown-freebsd --sysroot=%S/Inputs/basic_freebsd_tree 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=NO_BFD
+
+// NO_BFD: error: invalid linker name in argument '--ld-path=./ld.bfd'
+
+/// --ld-path can specify an absolute path.
+// RUN: %clang %s -### --ld-path=%S/Inputs/basic_freebsd_tree/usr/bin/ld.bfd \
+// RUN:   --target=x86_64-unknown-freebsd --sysroot=%S/Inputs/basic_freebsd_tree 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=BFD
+
+// RUN: %clang %s -### --ld-path=Inputs/basic_freebsd_tree/usr/bin/ld.bfd \
+// RUN:   --target=x86_64-unknown-freebsd --sysroot=%S/Inputs/basic_freebsd_tree 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=BFD
+
+/// --ld-path= and -fuse-ld= can be used together. --ld-path= takes precedence.
+/// -fuse-ld= can be used to specify the linker flavor.
+// RUN: %clang %s -### -Werror --ld-path=%S/Inputs/basic_freebsd_tree/usr/bin/ld.bfd -fuse-ld=gold \
+// RUN:   --target=x86_64-unknown-freebsd --sysroot=%S/Inputs/basic_freebsd_tree 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=BFD --implicit-check-not=error:
+
+/// --ld-path= respects -working-directory.
+// RUN: %clang %s -### --ld-path=usr/bin/ld.bfd -working-directory=%S/Inputs/basic_freebsd_tree \
+// RUN:   --target=x86_64-unknown-freebsd --sysroot=%S/Inputs/basic_freebsd_tree 2>&1 | \
+// RUN:   FileCheck %s --check-prefix=USR_BIN_BFD
+
+// USR_BIN_BFD: "usr/bin/ld.bfd"
diff --git a/clang/test/Driver/pch-codegen.cpp b/clang/test/Driver/pch-codegen.cpp
new file mode 100644
index 0000000000000..f16f6b51bf409
--- /dev/null
+++ b/clang/test/Driver/pch-codegen.cpp
@@ -0,0 +1,38 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+
+// Create PCH without codegen.
+// RUN: %clang -x c++-header %S/../Modules/Inputs/codegen-flags/foo.h -o %t/foo-cg.pch -### 2>&1 | FileCheck %s -check-prefix=CHECK-PCH-CREATE
+// CHECK-PCH-CREATE: -emit-pch
+// CHECK-PCH-CREATE-NOT: -fmodules-codegen
+// CHECK-PCH-CREATE-NOT: -fmodules-debuginfo
+
+// Create PCH with -fpch-codegen.
+// RUN: %clang -x c++-header -fpch-codegen %S/../Modules/Inputs/codegen-flags/foo.h -o %t/foo-cg.pch -### 2>&1 | FileCheck %s -check-prefix=CHECK-PCH-CODEGEN-CREATE
+// CHECK-PCH-CODEGEN-CREATE: -emit-pch
+// CHECK-PCH-CODEGEN-CREATE: -fmodules-codegen
+// CHECK-PCH-CODEGEN-CREATE: "-x" "c++-header"
+// CHECK-PCH-CODEGEN-CREATE-NOT: -fmodules-debuginfo
+
+// Create PCH with -fpch-debuginfo.
+// RUN: %clang -x c++-header -fpch-debuginfo %S/../Modules/Inputs/codegen-flags/foo.h -g -o %t/foo-di.pch -### 2>&1 | FileCheck %s -check-prefix=CHECK-PCH-DEBUGINFO-CREATE
+// CHECK-PCH-DEBUGINFO-CREATE: -emit-pch
+// CHECK-PCH-DEBUGINFO-CREATE: -fmodules-debuginfo
+// CHECK-PCH-DEBUGINFO-CREATE: "-x" "c++-header"
+// CHECK-PCH-DEBUGINFO-CREATE-NOT: -fmodules-codegen
+
+// Create PCH's object file for -fpch-codegen.
+// RUN: touch %t/foo-cg.pch
+// RUN: %clang -c -fintegrated-as %t/foo-cg.pch -o %t/foo-cg.o -### 2>&1 | FileCheck %s -check-prefix=CHECK-PCH-CODEGEN-OBJ
+// CHECK-PCH-CODEGEN-OBJ: -emit-obj
+// CHECK-PCH-CODEGEN-OBJ: "-main-file-name" "foo-cg.pch"
+// CHECK-PCH-CODEGEN-OBJ: "-o" "{{.*}}foo-cg.o"
+// CHECK-PCH-CODEGEN-OBJ: "-x" "precompiled-header"
+
+// Create PCH's object file for -fpch-debuginfo.
+// RUN: touch %t/foo-di.pch
+// RUN: %clang -c -fintegrated-as %t/foo-di.pch -g -o %t/foo-di.o -### 2>&1 | FileCheck %s -check-prefix=CHECK-PCH-DEBUGINFO-OBJ
+// CHECK-PCH-DEBUGINFO-OBJ: -emit-obj
+// CHECK-PCH-DEBUGINFO-OBJ: "-main-file-name" "foo-di.pch"
+// CHECK-PCH-DEBUGINFO-OBJ: "-o" "{{.*}}foo-di.o"
+// CHECK-PCH-DEBUGINFO-OBJ: "-x" "precompiled-header"
diff --git a/clang/test/Driver/sycl-offload.c b/clang/test/Driver/sycl-offload.c
index a297c7d71979b..1b29a0d7277a6 100644
--- a/clang/test/Driver/sycl-offload.c
+++ b/clang/test/Driver/sycl-offload.c
@@ -593,6 +593,10 @@
 // CHECK-LINK-SYCL-DEBUG: "{{.*}}link{{(.exe)?}}"
 // CHECK-LINK-SYCL-DEBUG: "-defaultlib:sycld.lib"
 
+/// Check "-spirv-allow-unknown-intrinsics" option is emitted for llvm-spirv tool for esimd mode
+// RUN: %clangxx %s -fsycl -fsycl-explicit-simd -### 2>&1 | FileCheck %s --check-prefix=CHK-FSYCL-ESIMD
+// CHK-FSYCL-ESIMD: llvm-spirv{{.*}}-spirv-allow-unknown-intrinsics
+
 /// ###########################################################################
 
 /// Check -Xsycl-target-backend triggers error when multiple triples are used.
diff --git a/clang/test/InterfaceStubs/func.ifs b/clang/test/InterfaceStubs/func.ifs
index 9de5213de9d6c..01459dc845076 100644
--- a/clang/test/InterfaceStubs/func.ifs
+++ b/clang/test/InterfaceStubs/func.ifs
@@ -1,7 +1,7 @@
 # RUN: %clang -emit-interface-stubs -o - %s %S/object.ifs -emit-merged-ifs | \
 # RUN: FileCheck %s --check-prefixes=CHECK-IFS
 
-# RUN: %clang -emit-interface-stubs -o - %s %S/object.ifs | llvm-readelf --all | \
+# RUN: %clang -emit-interface-stubs -o - %s %S/object.ifs | llvm-readelf --all - | \
 # RUN: FileCheck %s --check-prefixes=CHECK-ELF
 
 # RUN: %clang -emit-interface-stubs -o - %s %s -emit-merged-ifs | \
diff --git a/clang/test/InterfaceStubs/object.ifs b/clang/test/InterfaceStubs/object.ifs
index 3afdf4e65eef7..5c0cdbbac64eb 100644
--- a/clang/test/InterfaceStubs/object.ifs
+++ b/clang/test/InterfaceStubs/object.ifs
@@ -1,7 +1,7 @@
 # RUN: %clang -emit-interface-stubs -o - -emit-merged-ifs %s | \
 # RUN: FileCheck %s --check-prefixes=CHECK-IFS
 
-# RUN: %clang -emit-interface-stubs -o - %s | llvm-readelf --all | \
+# RUN: %clang -emit-interface-stubs -o - %s | llvm-readelf --all - | \
 # RUN: FileCheck %s --check-prefixes=CHECK-ELF
 
 # CHECK-IFS: --- !experimental-ifs-v2
diff --git a/clang/test/Misc/warning-flags.c b/clang/test/Misc/warning-flags.c
index e4f9069b88c86..168f2a3551bab 100644
--- a/clang/test/Misc/warning-flags.c
+++ b/clang/test/Misc/warning-flags.c
@@ -18,7 +18,7 @@ This test serves two purposes:
 
 The list of warnings below should NEVER grow.  It should gradually shrink to 0.
 
-CHECK: Warnings without flags (68):
+CHECK: Warnings without flags (69):
 
 CHECK-NEXT:   ext_expected_semi_decl_list
 CHECK-NEXT:   ext_explicit_specialization_storage_class
@@ -47,6 +47,7 @@ CHECK-NEXT:   warn_double_const_requires_fp64
 CHECK-NEXT:   warn_drv_assuming_mfloat_abi_is
 CHECK-NEXT:   warn_drv_clang_unsupported
 CHECK-NEXT:   warn_drv_pch_not_first_include
+CHECK-NEXT:   warn_drv_use_ld_non_word
 CHECK-NEXT:   warn_dup_category_def
 CHECK-NEXT:   warn_enum_value_overflow
 CHECK-NEXT:   warn_expected_qualified_after_typename
diff --git a/clang/test/OpenMP/target_data_ast_print.cpp b/clang/test/OpenMP/target_data_ast_print.cpp
index fcd6e928655c2..6c5eae2d33fe7 100644
--- a/clang/test/OpenMP/target_data_ast_print.cpp
+++ b/clang/test/OpenMP/target_data_ast_print.cpp
@@ -5,6 +5,14 @@
 // RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=50 -ast-print %s | FileCheck %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s
+
+// RUN: %clang_cc1 -DOMP51 -verify -fopenmp -fopenmp-version=51 -ast-print %s | FileCheck -check-prefixes=CHECK,OMP51 %s
+// RUN: %clang_cc1 -DOMP51 -fopenmp -fopenmp-version=51 -x c++ -std=c++11 -emit-pch -o %t %s
+// RUN: %clang_cc1 -DOMP51 -fopenmp -fopenmp-version=51 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck -check-prefixes=CHECK,OMP51 %s
+
+// RUN: %clang_cc1 -DOMP51 -verify -fopenmp-simd -fopenmp-version=51 -ast-print %s | FileCheck -check-prefixes=CHECK,OMP51 %s
+// RUN: %clang_cc1 -DOMP51 -fopenmp-simd -fopenmp-version=51 -x c++ -std=c++11 -emit-pch -o %t %s
+// RUN: %clang_cc1 -DOMP51 -fopenmp-simd -fopenmp-version=51 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck -check-prefixes=CHECK,OMP51 %s
 // expected-no-diagnostics
 
 #ifndef HEADER
@@ -43,6 +51,11 @@ T tmain(T argc, T *argv) {
 #pragma omp target data map(close,alloc: e)
   foo();
 
+#ifdef OMP51
+#pragma omp target data map(present,alloc: e)
+  foo();
+#endif
+
 // nesting a target region
 #pragma omp target data map(e)
 {
@@ -50,6 +63,10 @@ T tmain(T argc, T *argv) {
     foo();
   #pragma omp target map(close, alloc: e)
     foo();
+#ifdef OMP51
+  #pragma omp target map(present, alloc: e)
+    foo();
+#endif
 }
 
   return 0;
@@ -75,12 +92,16 @@ T tmain(T argc, T *argv) {
 // CHECK-NEXT: foo();
 // CHECK-NEXT: #pragma omp target data map(close,alloc: e)
 // CHECK-NEXT: foo();
+// OMP51-NEXT: #pragma omp target data map(present,alloc: e)
+// OMP51-NEXT: foo();
 // CHECK-NEXT: #pragma omp target data map(tofrom: e)
 // CHECK-NEXT: {
 // CHECK-NEXT: #pragma omp target map(always,alloc: e)
 // CHECK-NEXT: foo();
 // CHECK-NEXT: #pragma omp target map(close,alloc: e)
 // CHECK-NEXT: foo();
+// OMP51-NEXT: #pragma omp target map(present,alloc: e)
+// OMP51-NEXT: foo();
 // CHECK: template<> int tmain<int, 5>(int argc, int *argv) {
 // CHECK-NEXT: int i, j, b, c, d, e, x[20];
 // CHECK-NEXT: #pragma omp target data map(to: c)
@@ -101,12 +122,16 @@ T tmain(T argc, T *argv) {
 // CHECK-NEXT: foo();
 // CHECK-NEXT: #pragma omp target data map(close,alloc: e)
 // CHECK-NEXT: foo();
+// OMP51-NEXT: #pragma omp target data map(present,alloc: e)
+// OMP51-NEXT: foo();
 // CHECK-NEXT: #pragma omp target data map(tofrom: e)
 // CHECK-NEXT: {
 // CHECK-NEXT: #pragma omp target map(always,alloc: e)
 // CHECK-NEXT: foo();
 // CHECK-NEXT: #pragma omp target map(close,alloc: e)
 // CHECK-NEXT: foo();
+// OMP51-NEXT: #pragma omp target map(present,alloc: e)
+// OMP51-NEXT: foo();
 // CHECK: template<> char tmain<char, 1>(char argc, char *argv) {
 // CHECK-NEXT: char i, j, b, c, d, e, x[20];
 // CHECK-NEXT: #pragma omp target data map(to: c)
@@ -127,12 +152,16 @@ T tmain(T argc, T *argv) {
 // CHECK-NEXT: foo();
 // CHECK-NEXT: #pragma omp target data map(close,alloc: e)
 // CHECK-NEXT: foo();
+// OMP51-NEXT: #pragma omp target data map(present,alloc: e)
+// OMP51-NEXT: foo();
 // CHECK-NEXT: #pragma omp target data map(tofrom: e)
 // CHECK-NEXT: {
 // CHECK-NEXT: #pragma omp target map(always,alloc: e)
 // CHECK-NEXT: foo();
 // CHECK-NEXT: #pragma omp target map(close,alloc: e)
 // CHECK-NEXT: foo();
+// OMP51-NEXT: #pragma omp target map(present,alloc: e)
+// OMP51-NEXT: foo();
 
 int main (int argc, char **argv) {
   int b = argc, c, d, e, f, g, x[20];
@@ -185,6 +214,13 @@ int main (int argc, char **argv) {
   foo();
 // CHECK-NEXT: foo();
 
+// OMP51-NEXT: #pragma omp target data map(present,alloc: e)
+// OMP51-NEXT: foo();
+#ifdef OMP51
+#pragma omp target data map(present,alloc: e)
+  foo();
+#endif
+
 // nesting a target region
 #pragma omp target data map(e)
 // CHECK-NEXT: #pragma omp target data map(tofrom: e)
@@ -197,6 +233,10 @@ int main (int argc, char **argv) {
 #pragma omp target map(close, alloc: e)
 // CHECK-NEXT: #pragma omp target map(close,alloc: e)
   foo();
+// CHECK-NEXT: foo();
+#pragma omp target map(always, alloc: e)
+// CHECK-NEXT: #pragma omp target map(always,alloc: e)
+  foo();
 }
 
   return tmain<int, 5>(argc, &argc) + tmain<char, 1>(argv[0][0], argv[0]);
diff --git a/clang/test/OpenMP/target_data_codegen.cpp b/clang/test/OpenMP/target_data_codegen.cpp
index 274b3e16b2f69..80e674c0b49d9 100644
--- a/clang/test/OpenMP/target_data_codegen.cpp
+++ b/clang/test/OpenMP/target_data_codegen.cpp
@@ -225,6 +225,99 @@ void foo(int arg) {
   // CK1-DAG: [[GEPS]] = getelementptr inbounds {{.+}}[[S]]
   #pragma omp target data map(always close, to: lb)
   {++arg;}
+
+}
+#endif
+///==========================================================================///
+// RUN: %clang_cc1 -DCK1A -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK1A --check-prefix CK1A-64
+// RUN: %clang_cc1 -DCK1A -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1A --check-prefix CK1A-64
+// RUN: %clang_cc1 -DCK1A -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK1A --check-prefix CK1A-32
+// RUN: %clang_cc1 -DCK1A -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1A --check-prefix CK1A-32
+
+// RUN: %clang_cc1 -DCK1A -verify -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK1A -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK1A -verify -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK1A -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
+#ifdef CK1A
+
+// CK1A: [[ST:%.+]] = type { i32, double* }
+template <typename T>
+struct ST {
+  T a;
+  double *b;
+};
+
+ST<int> gb;
+double gc[100];
+
+// PRESENT=0x1000 | TARGET_PARAM=0x20 | TO=0x1 = 0x1021
+// CK1A: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 [[#0x1021]]]
+
+// PRESENT=0x1000 | CLOSE=0x400 | TARGET_PARAM=0x20 | ALWAYS=0x4 | TO=0x1 = 0x1425
+// CK1A: [[MTYPE01:@.+]] = {{.+}}constant [1 x i64] [i64 [[#0x1425]]]
+
+// CK1A-LABEL: _Z3fooi
+void foo(int arg) {
+  int la;
+  float lb[arg];
+
+  // Region 00
+  // CK1A-DAG: call void @__tgt_target_data_begin_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i[[sz:32|64]]* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}})
+  // CK1A-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
+  // CK1A-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
+  // CK1A-DAG: [[GEPS]] = getelementptr inbounds {{.+}}[[S:%[^,]+]]
+
+  // CK1A-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK1A-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK1A-DAG: [[S0:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 0
+  // CK1A-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to float**
+  // CK1A-DAG: [[CP0:%.+]] = bitcast i8** [[P0]] to float**
+  // CK1A-DAG: store float* [[VAR0:%.+]], float** [[CBP0]]
+  // CK1A-DAG: store float* [[VAR0]], float** [[CP0]]
+  // CK1A-DAG: store i[[sz]] [[CSVAL0:%[^,]+]], i[[sz]]* [[S0]]
+  // CK1A-64-DAG: [[CSVAL0]] = mul nuw i64 %{{[^,]+}}, 4
+  // CK1A-32-DAG: [[CSVAL0]] = sext i32 [[CSVAL032:%.+]] to i64
+  // CK1A-32-DAG: [[CSVAL032]] = mul nuw i32 %{{[^,]+}}, 4
+  // CK1A: %{{.+}} = add nsw i32 %{{[^,]+}}, 1
+
+  // CK1A-DAG: call void @__tgt_target_data_end_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i[[sz]]* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}})
+  // CK1A-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP]]
+  // CK1A-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P]]
+  // CK1A-DAG: [[GEPS]] = getelementptr inbounds {{.+}}[[S]]
+  #pragma omp target data map(present, to: lb)
+  {++arg;}
+
+  // Region 01
+  // CK1A-DAG: call void @__tgt_target_data_begin_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i[[sz]]* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE01]]{{.+}})
+  // CK1A-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
+  // CK1A-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
+  // CK1A-DAG: [[GEPS]] = getelementptr inbounds {{.+}}[[S:%[^,]+]]
+
+  // CK1A-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK1A-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK1A-DAG: [[S0:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 0
+  // CK1A-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to float**
+  // CK1A-DAG: [[CP0:%.+]] = bitcast i8** [[P0]] to float**
+  // CK1A-DAG: store float* [[VAR0:%.+]], float** [[CBP0]]
+  // CK1A-DAG: store float* [[VAR0]], float** [[CP0]]
+  // CK1A-DAG: store i[[sz]] [[CSVAL0:%[^,]+]], i[[sz]]* [[S0]]
+  // CK1A-64-DAG: [[CSVAL0]] = mul nuw i64 %{{[^,]+}}, 4
+  // CK1A-32-DAG: [[CSVAL0]] = sext i32 [[CSVAL032:%.+]] to i64
+  // CK1A-32-DAG: [[CSVAL032]] = mul nuw i32 %{{[^,]+}}, 4
+  // CK1A: %{{.+}} = add nsw i32 %{{[^,]+}}, 1
+
+  // CK1A-DAG: call void @__tgt_target_data_end_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i[[sz]]* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE01]]{{.+}})
+  // CK1A-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP]]
+  // CK1A-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P]]
+  // CK1A-DAG: [[GEPS]] = getelementptr inbounds {{.+}}[[S]]
+  #pragma omp target data map(always close present, to: lb)
+  {++arg;}
+
 }
 #endif
 ///==========================================================================///
@@ -462,7 +555,7 @@ struct S2 {
 
 void test_close_modifier(int arg) {
   S2 *ps;
-  // CK5: private unnamed_addr constant [5 x i64] [i64 1059, i64 32, {{.*}}, i64 16, i64 1043]
+  // CK5: private unnamed_addr constant [6 x i64] [i64 1059, i64 32, i64 562949953422339, i64 562949953421328, i64 16, i64 1043]
   #pragma omp target data map(close,tofrom: arg, ps->ps->ps->ps->s)
   {
     ++(arg);
@@ -510,4 +603,101 @@ void test_device_ptr_addr(int arg) {
   { ++arg, ++(*p); }
 }
 #endif
+///==========================================================================///
+// RUN: %clang_cc1 -DCK8 -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK8 --check-prefix CK8-64
+// RUN: %clang_cc1 -DCK8 -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK8 --check-prefix CK8-64
+// RUN: %clang_cc1 -DCK8 -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK8 --check-prefix CK8-32
+// RUN: %clang_cc1 -DCK8 -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK8 --check-prefix CK8-32
+
+// RUN: %clang_cc1 -DCK8 -verify -fopenmp-simd -fopenmp-version=51 -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -DCK8 -fopenmp-simd -fopenmp-version=51 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -DCK8 -verify -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -DCK8 -fopenmp-simd -fopenmp-version=51 -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// SIMD-ONLY2-NOT: {{__kmpc|__tgt}}#ifdef CK8
+#ifdef CK8
+struct S1 {
+  int i;
+};
+struct S2 {
+  S1 s;
+  struct S2 *ps;
+};
+
+void test_present_modifier(int arg) {
+  S2 *ps1;
+  S2 *ps2;
+
+  // Make sure the struct picks up present even if another element of the struct
+  // doesn't have present.
+
+  // CK8: private unnamed_addr constant [15 x i64]
+
+  // ps1
+  //
+  // PRESENT=0x1000 | TARGET_PARAM=0x20 = 0x1020
+  // MEMBER_OF_1=0x1000000000000 | FROM=0x2 | TO=0x1 = 0x1000000000003
+  // MEMBER_OF_1=0x1000000000000 | PTR_AND_OBJ=0x10 | FROM=0x2 | TO=0x1 = 0x1000000000013
+  // MEMBER_OF_1=0x1000000000000 | PRESENT=0x1000 | FROM=0x2 | TO=0x1 = 0x1000000001003
+  // MEMBER_OF_1=0x1000000000000 | PRESENT=0x1000 | PTR_AND_OBJ=0x10 = 0x1000000001010
+  // PRESENT=0x1000 | PTR_AND_OBJ=0x10 = 0x1010
+  // PRESENT=0x1000 | PTR_AND_OBJ=0x10 | FROM=0x2 | TO=0x1 = 0x1013
+  //
+  // CK8-SAME: {{^}} [i64 [[#0x1020]], i64 [[#0x1000000000003]],
+  // CK8-SAME: {{^}} i64 [[#0x1000000000013]], i64 [[#0x1000000001003]],
+  // CK8-SAME: {{^}} i64 [[#0x1000000001010]], i64 [[#0x1010]], i64 [[#0x1013]],
+
+  // arg
+  //
+  // PRESENT=0x1000 | TARGET_PARAM=0x20 | FROM=0x2 | TO=0x1 = 0x1023
+  //
+  // CK8-SAME: {{^}} i64 [[#0x1023]],
+
+  // ps2
+  //
+  // PRESENT=0x1000 | TARGET_PARAM=0x20 = 0x1020
+  // MEMBER_OF_9=0x9000000000000 | PRESENT=0x1000 | FROM=0x2 | TO=0x1 = 0x9000000001003
+  // MEMBER_OF_9=0x9000000000000 | PRESENT=0x1000 | PTR_AND_OBJ=0x10 | FROM=0x2 | TO=0x1 = 0x9000000001013
+  // MEMBER_OF_9=0x9000000000000 | FROM=0x2 | TO=0x1 = 0x9000000000003
+  // MEMBER_OF_9=0x9000000000000 | PTR_AND_OBJ=0x10 = 0x9000000000010
+  // PTR_AND_OBJ=0x10 = 0x10
+  // PTR_AND_OBJ=0x10 | FROM=0x2 | TO=0x1 = 0x13
+  //
+  // CK8-SAME: {{^}} i64 [[#0x1020]], i64 [[#0x9000000001003]],
+  // CK8-SAME: {{^}} i64 [[#0x9000000001013]], i64 [[#0x9000000000003]],
+  // CK8-SAME: {{^}} i64 [[#0x9000000000010]], i64 [[#0x10]], i64 [[#0x13]]]
+  #pragma omp target data map(tofrom: ps1->s) \
+                          map(present,tofrom: arg, ps1->ps->ps->ps->s, ps2->s) \
+                          map(tofrom: ps2->ps->ps->ps->s)
+  {
+    ++(arg);
+  }
+}
+#endif
+///==========================================================================///
+// RUN: %clang_cc1 -DCK9 -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK9 --check-prefix CK9-64
+// RUN: %clang_cc1 -DCK9 -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK9 --check-prefix CK9-64
+// RUN: %clang_cc1 -DCK9 -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK9 --check-prefix CK9-32
+// RUN: %clang_cc1 -DCK9 -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK9 --check-prefix CK9-32
+
+// RUN: %clang_cc1 -DCK9 -verify -fopenmp-simd -fopenmp-version=51 -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -DCK9 -fopenmp-simd -fopenmp-version=51 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -DCK9 -verify -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -DCK9 -fopenmp-simd -fopenmp-version=51 -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// SIMD-ONLY2-NOT: {{__kmpc|__tgt}}
+#ifdef CK9
+void test_present_modifier(int arg) {
+  // PRESENT=0x1000 | TARGET_PARAM=0x20 | FROM=0x2 | TO=0x1 = 0x1023
+  // CK9: private unnamed_addr constant [1 x i64] [i64 [[#0x1023]]]
+  #pragma omp target data map(present,tofrom: arg)
+  {++arg;}
+}
+#endif
 #endif
diff --git a/clang/test/OpenMP/target_data_use_device_ptr_codegen.cpp b/clang/test/OpenMP/target_data_use_device_ptr_codegen.cpp
index a3d8043b6b4e9..fe6ea01b43c91 100644
--- a/clang/test/OpenMP/target_data_use_device_ptr_codegen.cpp
+++ b/clang/test/OpenMP/target_data_use_device_ptr_codegen.cpp
@@ -22,18 +22,18 @@
 double *g;
 
 // CK1: @g = global double*
-// CK1: [[MTYPE00:@.+]] = {{.*}}constant [1 x i64] [i64 99]
-// CK1: [[MTYPE01:@.+]] = {{.*}}constant [1 x i64] [i64 99]
-// CK1: [[MTYPE03:@.+]] = {{.*}}constant [1 x i64] [i64 99]
-// CK1: [[MTYPE04:@.+]] = {{.*}}constant [1 x i64] [i64 99]
-// CK1: [[MTYPE05:@.+]] = {{.*}}constant [1 x i64] [i64 99]
-// CK1: [[MTYPE06:@.+]] = {{.*}}constant [1 x i64] [i64 99]
-// CK1: [[MTYPE07:@.+]] = {{.*}}constant [1 x i64] [i64 99]
-// CK1: [[MTYPE08:@.+]] = {{.*}}constant [2 x i64] [{{i64 35, i64 99|i64 99, i64 35}}]
-// CK1: [[MTYPE09:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 99]
-// CK1: [[MTYPE10:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 99]
-// CK1: [[MTYPE11:@.+]] = {{.*}}constant [2 x i64] [i64 96, i64 35]
-// CK1: [[MTYPE12:@.+]] = {{.*}}constant [2 x i64] [i64 96, i64 35]
+// CK1: [[MTYPE00:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
+// CK1: [[MTYPE01:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
+// CK1: [[MTYPE03:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
+// CK1: [[MTYPE04:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
+// CK1: [[MTYPE05:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
+// CK1: [[MTYPE06:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
+// CK1: [[MTYPE07:@.+]] = {{.*}}constant [2 x i64] [i64 99, i64 19]
+// CK1: [[MTYPE08:@.+]] = {{.*}}constant [4 x i64] [i64 99, i64 19, i64 35, i64 19]
+// CK1: [[MTYPE09:@.+]] = {{.*}}constant [4 x i64] [i64 99, i64 19, i64 99, i64 19]
+// CK1: [[MTYPE10:@.+]] = {{.*}}constant [4 x i64] [i64 99, i64 19, i64 99, i64 19]
+// CK1: [[MTYPE11:@.+]] = {{.*}}constant [3 x i64] [i64 96, i64 35, i64 19]
+// CK1: [[MTYPE12:@.+]] = {{.*}}constant [3 x i64] [i64 96, i64 35, i64 19]
 
 // CK1-LABEL: @_Z3foo
 template<typename T>
@@ -42,7 +42,7 @@ void foo(float *&lr, T *&tr) {
   T *t;
 
   // CK1:     [[T:%.+]] = load double*, double** [[DECL:@g]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to double**
   // CK1:     store double* [[T]], double** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE00]]
@@ -61,7 +61,7 @@ void foo(float *&lr, T *&tr) {
   ++g;
 
   // CK1:     [[T1:%.+]] = load float*, float** [[DECL:%.+]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to float**
   // CK1:     store float* [[T1]], float** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE01]]
@@ -92,7 +92,7 @@ void foo(float *&lr, T *&tr) {
   ++l;
 
   // CK1:     [[T1:%.+]] = load float*, float** [[DECL:%.+]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to float**
   // CK1:     store float* [[T1]], float** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE03]]
@@ -115,7 +115,7 @@ void foo(float *&lr, T *&tr) {
 
   // CK1:     [[BTHEN]]:
   // CK1:     [[T1:%.+]] = load float*, float** [[DECL:%.+]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to float**
   // CK1:     store float* [[T1]], float** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE04]]
@@ -152,7 +152,7 @@ void foo(float *&lr, T *&tr) {
 
   // CK1:     [[T2:%.+]] = load float**, float*** [[DECL:%.+]],
   // CK1:     [[T1:%.+]] = load float*, float** [[T2]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to float**
   // CK1:     store float* [[T1]], float** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE05]]
@@ -174,7 +174,7 @@ void foo(float *&lr, T *&tr) {
   ++lr;
 
   // CK1:     [[T1:%.+]] = load i32*, i32** [[DECL:%.+]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to i32**
   // CK1:     store i32* [[T1]], i32** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE06]]
@@ -194,7 +194,7 @@ void foo(float *&lr, T *&tr) {
 
   // CK1:     [[T2:%.+]] = load i32**, i32*** [[DECL:%.+]],
   // CK1:     [[T1:%.+]] = load i32*, i32** [[T2]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [1 x i8*], [1 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to i32**
   // CK1:     store i32* [[T1]], i32** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE07]]
@@ -216,7 +216,7 @@ void foo(float *&lr, T *&tr) {
   ++tr;
 
   // CK1:     [[T1:%.+]] = load float*, float** [[DECL:%.+]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [4 x i8*], [4 x i8*]* %{{.+}}, i32 0, i32
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to float**
   // CK1:     store float* [[T1]], float** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE08]]
@@ -280,7 +280,7 @@ void foo(float *&lr, T *&tr) {
   ++l; ++t;
 
   // CK1:     [[T1:%.+]] = load i32*, i32** [[DECL:%.+]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [3 x i8*], [3 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to i32**
   // CK1:     store i32* [[T1]], i32** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE11]]
@@ -300,7 +300,7 @@ void foo(float *&lr, T *&tr) {
 
   // CK1:     [[T2:%.+]] = load i32**, i32*** [[DECL:%.+]],
   // CK1:     [[T1:%.+]] = load i32*, i32** [[T2]],
-  // CK1:     [[BP:%.+]] = getelementptr inbounds [2 x i8*], [2 x i8*]* %{{.+}}, i32 0, i32 0
+  // CK1:     [[BP:%.+]] = getelementptr inbounds [3 x i8*], [3 x i8*]* %{{.+}}, i32 0, i32 0
   // CK1:     [[CBP:%.+]] = bitcast i8** [[BP]] to i32**
   // CK1:     store i32* [[T1]], i32** [[CBP]],
   // CK1:     call void @__tgt_target_data_begin{{.+}}[[MTYPE12]]
@@ -348,7 +348,7 @@ void bar(float *&a, int *&b) {
 // CK2: [[ST:%.+]] = type { double*, double** }
 // CK2: [[MTYPE00:@.+]] = {{.*}}constant [2 x i64] [i64 32, i64 281474976710739]
 // CK2: [[MTYPE01:@.+]] = {{.*}}constant [2 x i64] [i64 32, i64 281474976710739]
-// CK2: [[MTYPE02:@.+]] = {{.*}}constant [3 x i64] [i64 35, i64 32, i64 562949953421392]
+// CK2: [[MTYPE02:@.+]] = {{.*}}constant [4 x i64] [i64 35, i64 19, i64 32, i64 844424930132048]
 // CK2: [[MTYPE03:@.+]] = {{.*}}constant [3 x i64] [i64 32, i64 281474976710739, i64 281474976710736]
 
 template <typename T>
@@ -404,7 +404,7 @@ struct ST {
     // CK2:     getelementptr inbounds double, double* [[TTTT]], i32 1
     b++;
 
-    // CK2:     [[BP:%.+]] = getelementptr inbounds [3 x i8*], [3 x i8*]* %{{.+}}, i32 0, i32 2
+    // CK2:     [[BP:%.+]] = getelementptr inbounds [4 x i8*], [4 x i8*]* %{{.+}}, i32 0, i32 3
     // CK2:     [[CBP:%.+]] = bitcast i8** [[BP]] to double***
     // CK2:     store double** [[RVAL:%.+]], double*** [[CBP]],
     // CK2:     call void @__tgt_target_data_begin{{.+}}[[MTYPE02]]
diff --git a/clang/test/OpenMP/target_enter_data_codegen.cpp b/clang/test/OpenMP/target_enter_data_codegen.cpp
index 541ea57b17e5b..1cd028a870470 100644
--- a/clang/test/OpenMP/target_enter_data_codegen.cpp
+++ b/clang/test/OpenMP/target_enter_data_codegen.cpp
@@ -203,6 +203,92 @@ void foo(int arg) {
 }
 #endif
 ///==========================================================================///
+// RUN: %clang_cc1 -DCK1A -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK1A --check-prefix CK1A-64
+// RUN: %clang_cc1 -DCK1A -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1A --check-prefix CK1A-64
+// RUN: %clang_cc1 -DCK1A -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK1A --check-prefix CK1A-32
+// RUN: %clang_cc1 -DCK1A -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1A --check-prefix CK1A-32
+
+// RUN: %clang_cc1 -DCK1A -verify -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK1A -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK1A -verify -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK1A -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
+#ifdef CK1A
+
+// CK1A: [[ST:%.+]] = type { i32, double* }
+template <typename T>
+struct ST {
+  T a;
+  double *b;
+};
+
+ST<int> gb;
+double gc[100];
+
+// PRESENT=0x1000 | TARGET_PARAM=0x20 | TO=0x1 = 0x1021
+// CK1A: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 [[#0x1021]]]
+
+// PRESENT=0x1000 | CLOSE=0x400 | TARGET_PARAM=0x20 | ALWAYS=0x4 | TO=0x1 = 0x1425
+// CK1A: [[MTYPE01:@.+]] = {{.+}}constant [1 x i64] [i64 [[#0x1425]]]
+
+// CK1A-LABEL: _Z3fooi
+void foo(int arg) {
+  int la;
+  float lb[arg];
+
+  // Region 00
+  // CK1A-DAG: call void @__tgt_target_data_begin_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i[[sz:32|64]]* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}})
+  // CK1A-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
+  // CK1A-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
+  // CK1A-DAG: [[GEPS]] = getelementptr inbounds {{.+}}[[S:%[^,]+]]
+
+  // CK1A-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK1A-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK1A-DAG: [[S0:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 0
+  // CK1A-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to float**
+  // CK1A-DAG: [[CP0:%.+]] = bitcast i8** [[P0]] to float**
+  // CK1A-DAG: store float* [[VAR0:%.+]], float** [[CBP0]]
+  // CK1A-DAG: store float* [[VAR0]], float** [[CP0]]
+  // CK1A-DAG: store i[[sz]] [[CSVAL0:%[^,]+]], i[[sz]]* [[S0]]
+  // CK1A-64-DAG: [[CSVAL0]] = mul nuw i64 %{{[^,]+}}, 4
+  // CK1A-32-DAG: [[CSVAL0]] = sext i32 [[CSVAL032:%.+]] to i64
+  // CK1A-32-DAG: [[CSVAL032]] = mul nuw i32 %{{[^,]+}}, 4
+  // CK1A: %{{.+}} = add nsw i32 %{{[^,]+}}, 1
+  // CK1A-NOT: __tgt_target_data_end
+  #pragma omp target enter data map(present, to: lb)
+  {++arg;}
+
+  // CK1A: %{{.+}} = add nsw i32 %{{[^,]+}}, 1
+  {++arg;}
+
+  // Region 01
+  // CK1A-DAG: call void @__tgt_target_data_begin_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i[[sz]]* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE01]]{{.+}})
+  // CK1A-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
+  // CK1A-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
+  // CK1A-DAG: [[GEPS]] = getelementptr inbounds {{.+}}[[S:%[^,]+]]
+
+  // CK1A-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK1A-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK1A-DAG: [[S0:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 0
+  // CK1A-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to float**
+  // CK1A-DAG: [[CP0:%.+]] = bitcast i8** [[P0]] to float**
+  // CK1A-DAG: store float* [[VAR0:%.+]], float** [[CBP0]]
+  // CK1A-DAG: store float* [[VAR0]], float** [[CP0]]
+  // CK1A-DAG: store i[[sz]] [[CSVAL0:%[^,]+]], i[[sz]]* [[S0]]
+  // CK1A-64-DAG: [[CSVAL0]] = mul nuw i64 %{{[^,]+}}, 4
+  // CK1A-32-DAG: [[CSVAL0]] = sext i32 [[CSVAL032:%.+]] to i64
+  // CK1A-32-DAG: [[CSVAL032]] = mul nuw i32 %{{[^,]+}}, 4
+  // CK1A: %{{.+}} = add nsw i32 %{{[^,]+}}, 1
+  // CK1A-NOT: __tgt_target_data_end
+  #pragma omp target enter data map(always close present, to: lb)
+  {++arg;}
+}
+#endif
+///==========================================================================///
 // RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64
 // RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK2 --check-prefix CK2-64
diff --git a/clang/test/OpenMP/target_map_codegen.cpp b/clang/test/OpenMP/target_map_codegen.cpp
index 2eab004eeff2e..e63f19ad73c0a 100644
--- a/clang/test/OpenMP/target_map_codegen.cpp
+++ b/clang/test/OpenMP/target_map_codegen.cpp
@@ -14,6 +14,27 @@
 // RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK1 --check-prefix CK1-32
 
+// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY0 %s
+
+// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK1 --check-prefix CK1-64
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK1 --check-prefix CK1-64
+// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK1 --check-prefix CK1-32
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK1 --check-prefix CK1-32
+
+// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK1 --check-prefix CK1-64
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK1 --check-prefix CK1-64
+// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK1 --check-prefix CK1-32
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK1 --check-prefix CK1-32
+
 // RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY0 %s
@@ -21,6 +42,7 @@
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
+
 #ifdef CK1
 
 class B {
@@ -89,6 +111,20 @@ void implicit_maps_integer (int a){
 // RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK2 --check-prefix CK2-32
 
+// RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK2 --check-prefix CK2-64
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK2 --check-prefix CK2-64
+// RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK2 --check-prefix CK2-32
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK2 --check-prefix CK2-32
+
+// RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK2 --check-prefix CK2-64
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK2 --check-prefix CK2-64
+// RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK2 --check-prefix CK2-32
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK2 --check-prefix CK2-32
+
 // RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY1 %s
@@ -179,6 +215,20 @@ void implicit_maps_reference (int a, int *b){
 // RUN: %clang_cc1 -DCK3 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK3 --check-prefix CK3-32
 
+// RUN: %clang_cc1 -DCK3 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK3 --check-prefix CK3-64
+// RUN: %clang_cc1 -DCK3 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK3 --check-prefix CK3-64
+// RUN: %clang_cc1 -DCK3 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK3 --check-prefix CK3-32
+// RUN: %clang_cc1 -DCK3 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK3 --check-prefix CK3-32
+
+// RUN: %clang_cc1 -DCK3 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK3 --check-prefix CK3-64
+// RUN: %clang_cc1 -DCK3 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK3 --check-prefix CK3-64
+// RUN: %clang_cc1 -DCK3 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK3 --check-prefix CK3-32
+// RUN: %clang_cc1 -DCK3 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK3 --check-prefix CK3-32
+
 // RUN: %clang_cc1 -DCK3 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY2 %s
 // RUN: %clang_cc1 -DCK3 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY2 %s
@@ -233,6 +283,20 @@ void implicit_maps_parameter (int a){
 // RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK4 --check-prefix CK4-32
 
+// RUN: %clang_cc1 -DCK4 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK4 --check-prefix CK4-64
+// RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK4 --check-prefix CK4-64
+// RUN: %clang_cc1 -DCK4 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK4 --check-prefix CK4-32
+// RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK4 --check-prefix CK4-32
+
+// RUN: %clang_cc1 -DCK4 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK4 --check-prefix CK4-64
+// RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK4 --check-prefix CK4-64
+// RUN: %clang_cc1 -DCK4 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK4 --check-prefix CK4-32
+// RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK4 --check-prefix CK4-32
+
 // RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY3 %s
 // RUN: %clang_cc1 -DCK4 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY3 %s
@@ -299,6 +363,20 @@ void implicit_maps_nested_integer (int a){
 // RUN: %clang_cc1 -DCK5 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK5 --check-prefix CK5-32
 
+// RUN: %clang_cc1 -DCK5 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK5 --check-prefix CK5-64
+// RUN: %clang_cc1 -DCK5 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK5 --check-prefix CK5-64
+// RUN: %clang_cc1 -DCK5 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK5 --check-prefix CK5-32
+// RUN: %clang_cc1 -DCK5 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK5 --check-prefix CK5-32
+
+// RUN: %clang_cc1 -DCK5 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK5 --check-prefix CK5-64
+// RUN: %clang_cc1 -DCK5 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK5 --check-prefix CK5-64
+// RUN: %clang_cc1 -DCK5 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK5 --check-prefix CK5-32
+// RUN: %clang_cc1 -DCK5 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK5 --check-prefix CK5-32
+
 // RUN: %clang_cc1 -DCK5 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY4 %s
 // RUN: %clang_cc1 -DCK5 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY4 %s
@@ -353,6 +431,13 @@ void implicit_maps_nested_integer_and_enum (int a){
 
 #endif
 ///==========================================================================///
+// RUN: %clang_cc1 -DCK6 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK6 --check-prefix CK6-64
+// RUN: %clang_cc1 -DCK6 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK6 --check-prefix CK6-64
+// RUN: %clang_cc1 -DCK6 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK6 --check-prefix CK6-32
+// RUN: %clang_cc1 -DCK6 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK6 --check-prefix CK6-32
+
 // RUN: %clang_cc1 -DCK6 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK6 --check-prefix CK6-64
 // RUN: %clang_cc1 -DCK6 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK6 --check-prefix CK6-64
@@ -360,6 +445,13 @@ void implicit_maps_nested_integer_and_enum (int a){
 // RUN: %clang_cc1 -DCK6 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK6 --check-prefix CK6-32
 
+// RUN: %clang_cc1 -DCK6 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK6 --check-prefix CK6-64
+// RUN: %clang_cc1 -DCK6 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK6 --check-prefix CK6-64
+// RUN: %clang_cc1 -DCK6 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK6 --check-prefix CK6-32
+// RUN: %clang_cc1 -DCK6 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK6 --check-prefix CK6-32
+
 // RUN: %clang_cc1 -DCK6 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY5 %s
 // RUN: %clang_cc1 -DCK6 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY5 %s
@@ -368,7 +460,7 @@ void implicit_maps_nested_integer_and_enum (int a){
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY5 %s
 // SIMD-ONLY5-NOT: {{__kmpc|__tgt}}
 #ifdef CK6
-// CK6-LABEL: @.__omp_offloading_{{.*}}implicit_maps_host_global{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
+// CK6-DAG: @.__omp_offloading_{{.*}}implicit_maps_host_global{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
 
 // CK6-DAG: [[GBL:@Gi]] = global i32 0
 // CK6-DAG: [[SIZES:@.+]] = {{.+}}constant [1 x i64] [i64 4]
@@ -416,6 +508,20 @@ void implicit_maps_host_global (int a){
 // RUN: %clang_cc1 -DCK7 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK7  --check-prefix CK7-32
 
+// RUN: %clang_cc1 -DCK7 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK7 --check-prefix CK7-64
+// RUN: %clang_cc1 -DCK7 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK7  --check-prefix CK7-64
+// RUN: %clang_cc1 -DCK7 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK7  --check-prefix CK7-32
+// RUN: %clang_cc1 -DCK7 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK7  --check-prefix CK7-32
+
+// RUN: %clang_cc1 -DCK7 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK7 --check-prefix CK7-64
+// RUN: %clang_cc1 -DCK7 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK7  --check-prefix CK7-64
+// RUN: %clang_cc1 -DCK7 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK7  --check-prefix CK7-32
+// RUN: %clang_cc1 -DCK7 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK7  --check-prefix CK7-32
+
 // RUN: %clang_cc1 -DCK7 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY6 %s
 // RUN: %clang_cc1 -DCK7 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY6 %s
@@ -488,6 +594,20 @@ void implicit_maps_double (int a){
 // RUN: %clang_cc1 -DCK8 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK8
 
+// RUN: %clang_cc1 -DCK8 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK8
+// RUN: %clang_cc1 -DCK8 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK8
+// RUN: %clang_cc1 -DCK8 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK8
+// RUN: %clang_cc1 -DCK8 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK8
+
+// RUN: %clang_cc1 -DCK8 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK8
+// RUN: %clang_cc1 -DCK8 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK8
+// RUN: %clang_cc1 -DCK8 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK8
+// RUN: %clang_cc1 -DCK8 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK8
+
 // RUN: %clang_cc1 -DCK8 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY7 %s
 // RUN: %clang_cc1 -DCK8 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY7 %s
@@ -542,6 +662,20 @@ void implicit_maps_float (int a){
 // RUN: %clang_cc1 -DCK9 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK9
 
+// RUN: %clang_cc1 -DCK9 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK9
+// RUN: %clang_cc1 -DCK9 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK9
+// RUN: %clang_cc1 -DCK9 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK9
+// RUN: %clang_cc1 -DCK9 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK9
+
+// RUN: %clang_cc1 -DCK9 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK9
+// RUN: %clang_cc1 -DCK9 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK9
+// RUN: %clang_cc1 -DCK9 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK9
+// RUN: %clang_cc1 -DCK9 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK9
+
 // RUN: %clang_cc1 -DCK9 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // RUN: %clang_cc1 -DCK9 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
@@ -593,6 +727,20 @@ void implicit_maps_array (int a){
 // RUN: %clang_cc1 -DCK10 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK10
 
+// RUN: %clang_cc1 -DCK10 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK10
+// RUN: %clang_cc1 -DCK10 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK10
+// RUN: %clang_cc1 -DCK10 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK10
+// RUN: %clang_cc1 -DCK10 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK10
+
+// RUN: %clang_cc1 -DCK10 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK10
+// RUN: %clang_cc1 -DCK10 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK10
+// RUN: %clang_cc1 -DCK10 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK10
+// RUN: %clang_cc1 -DCK10 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK10
+
 // RUN: %clang_cc1 -DCK10 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY9 %s
 // RUN: %clang_cc1 -DCK10 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY9 %s
@@ -695,6 +843,20 @@ void implicit_maps_double_complex (int a, int *b){
 // RUN: %clang_cc1 -DCK12 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK12 --check-prefix CK12-32
 
+// RUN: %clang_cc1 -DCK12 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK12 --check-prefix CK12-64
+// RUN: %clang_cc1 -DCK12 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK12 --check-prefix CK12-64
+// RUN: %clang_cc1 -DCK12 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK12 --check-prefix CK12-32
+// RUN: %clang_cc1 -DCK12 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK12 --check-prefix CK12-32
+
+// RUN: %clang_cc1 -DCK12 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK12 --check-prefix CK12-64
+// RUN: %clang_cc1 -DCK12 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK12 --check-prefix CK12-64
+// RUN: %clang_cc1 -DCK12 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK12 --check-prefix CK12-32
+// RUN: %clang_cc1 -DCK12 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK12 --check-prefix CK12-32
+
 // RUN: %clang_cc1 -DCK12 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY11 %s
 // RUN: %clang_cc1 -DCK12 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY11 %s
@@ -766,6 +928,20 @@ void implicit_maps_float_complex (int a){
 // RUN: %clang_cc1 -DCK13 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
 
+// RUN: %clang_cc1 -DCK13 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK13
+// RUN: %clang_cc1 -DCK13 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
+// RUN: %clang_cc1 -DCK13 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
+// RUN: %clang_cc1 -DCK13 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
+
+// RUN: %clang_cc1 -DCK13 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK13
+// RUN: %clang_cc1 -DCK13 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
+// RUN: %clang_cc1 -DCK13 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
+// RUN: %clang_cc1 -DCK13 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
+
 // RUN: %clang_cc1 -DCK13 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY12 %s
 // RUN: %clang_cc1 -DCK13 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY12 %s
@@ -848,6 +1024,20 @@ void implicit_maps_variable_length_array (int a){
 // RUN: %clang_cc1 -DCK14 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK14 --check-prefix CK14-32
 
+// RUN: %clang_cc1 -DCK14 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK14 --check-prefix CK14-64
+// RUN: %clang_cc1 -DCK14 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK14 --check-prefix CK14-64
+// RUN: %clang_cc1 -DCK14 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK14 --check-prefix CK14-32
+// RUN: %clang_cc1 -DCK14 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK14 --check-prefix CK14-32
+
+// RUN: %clang_cc1 -DCK14 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK14 --check-prefix CK14-64
+// RUN: %clang_cc1 -DCK14 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK14 --check-prefix CK14-64
+// RUN: %clang_cc1 -DCK14 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK14 --check-prefix CK14-32
+// RUN: %clang_cc1 -DCK14 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK14 --check-prefix CK14-32
+
 // RUN: %clang_cc1 -DCK14 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY13 %s
 // RUN: %clang_cc1 -DCK14 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY13 %s
@@ -957,6 +1147,20 @@ void implicit_maps_class (int a){
 // RUN: %clang_cc1 -DCK15 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK15 --check-prefix CK15-32
 
+// RUN: %clang_cc1 -DCK15 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK15 --check-prefix CK15-64
+// RUN: %clang_cc1 -DCK15 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK15 --check-prefix CK15-64
+// RUN: %clang_cc1 -DCK15 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK15 --check-prefix CK15-32
+// RUN: %clang_cc1 -DCK15 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK15 --check-prefix CK15-32
+
+// RUN: %clang_cc1 -DCK15 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK15 --check-prefix CK15-64
+// RUN: %clang_cc1 -DCK15 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK15 --check-prefix CK15-64
+// RUN: %clang_cc1 -DCK15 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK15 --check-prefix CK15-32
+// RUN: %clang_cc1 -DCK15 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK15 --check-prefix CK15-32
+
 // RUN: %clang_cc1 -DCK15 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY14 %s
 // RUN: %clang_cc1 -DCK15 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY14 %s
@@ -1136,6 +1340,20 @@ void implicit_maps_templated_class (int a){
 // RUN: %clang_cc1 -DCK16 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK16 --check-prefix CK16-32
 
+// RUN: %clang_cc1 -DCK16 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK16 --check-prefix CK16-64
+// RUN: %clang_cc1 -DCK16 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK16 --check-prefix CK16-64
+// RUN: %clang_cc1 -DCK16 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK16 --check-prefix CK16-32
+// RUN: %clang_cc1 -DCK16 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK16 --check-prefix CK16-32
+
+// RUN: %clang_cc1 -DCK16 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK16 --check-prefix CK16-64
+// RUN: %clang_cc1 -DCK16 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK16 --check-prefix CK16-64
+// RUN: %clang_cc1 -DCK16 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK16 --check-prefix CK16-32
+// RUN: %clang_cc1 -DCK16 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK16 --check-prefix CK16-32
+
 // RUN: %clang_cc1 -DCK16 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY15 %s
 // RUN: %clang_cc1 -DCK16 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY15 %s
@@ -1197,6 +1415,20 @@ void implicit_maps_templated_function (int a){
 // RUN: %clang_cc1 -DCK17 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK17
 
+// RUN: %clang_cc1 -DCK17 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK17
+// RUN: %clang_cc1 -DCK17 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK17
+// RUN: %clang_cc1 -DCK17 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK17
+// RUN: %clang_cc1 -DCK17 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK17
+
+// RUN: %clang_cc1 -DCK17 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK17
+// RUN: %clang_cc1 -DCK17 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK17
+// RUN: %clang_cc1 -DCK17 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK17
+// RUN: %clang_cc1 -DCK17 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK17
+
 // RUN: %clang_cc1 -DCK17 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
 // RUN: %clang_cc1 -DCK17 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
@@ -1254,6 +1486,20 @@ void implicit_maps_struct (int a){
 // RUN: %clang_cc1 -DCK18 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK18 --check-prefix CK18-32
 
+// RUN: %clang_cc1 -DCK18 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK18 --check-prefix CK18-64
+// RUN: %clang_cc1 -DCK18 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK18 --check-prefix CK18-64
+// RUN: %clang_cc1 -DCK18 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK18 --check-prefix CK18-32
+// RUN: %clang_cc1 -DCK18 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK18 --check-prefix CK18-32
+
+// RUN: %clang_cc1 -DCK18 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK18 --check-prefix CK18-64
+// RUN: %clang_cc1 -DCK18 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK18 --check-prefix CK18-64
+// RUN: %clang_cc1 -DCK18 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK18 --check-prefix CK18-32
+// RUN: %clang_cc1 -DCK18 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK18 --check-prefix CK18-32
+
 // RUN: %clang_cc1 -DCK18 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY17 %s
 // RUN: %clang_cc1 -DCK18 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY17 %s
@@ -1314,6 +1560,20 @@ void implicit_maps_template_type_capture (int a){
 // RUN: %clang_cc1 -DUSE -DCK19 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -DUSE -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK19,CK19-32,CK19-USE
 
+// RUN: %clang_cc1 -DUSE -DCK19 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefixes=CK19,CK19-64,CK19-USE
+// RUN: %clang_cc1 -DUSE -DCK19 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -DUSE -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK19,CK19-64,CK19-USE
+// RUN: %clang_cc1 -DUSE -DCK19 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK19,CK19-32,CK19-USE
+// RUN: %clang_cc1 -DUSE -DCK19 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -DUSE -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK19,CK19-32,CK19-USE
+
+// RUN: %clang_cc1 -DUSE -DCK19 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefixes=CK19,CK19-64,CK19-USE
+// RUN: %clang_cc1 -DUSE -DCK19 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -DUSE -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK19,CK19-64,CK19-USE
+// RUN: %clang_cc1 -DUSE -DCK19 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK19,CK19-32,CK19-USE
+// RUN: %clang_cc1 -DUSE -DCK19 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -DUSE -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK19,CK19-32,CK19-USE
+
 // RUN: %clang_cc1 -DUSE -DCK19 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
 // RUN: %clang_cc1 -DUSE -DCK19 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -DUSE -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
@@ -1328,6 +1588,20 @@ void implicit_maps_template_type_capture (int a){
 // RUN: %clang_cc1 -DCK19 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK19,CK19-32,CK19-NOUSE
 
+// RUN: %clang_cc1 -DCK19 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefixes=CK19,CK19-64,CK19-NOUSE
+// RUN: %clang_cc1 -DCK19 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK19,CK19-64,CK19-NOUSE
+// RUN: %clang_cc1 -DCK19 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK19,CK19-32,CK19-NOUSE
+// RUN: %clang_cc1 -DCK19 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK19,CK19-32,CK19-NOUSE
+
+// RUN: %clang_cc1 -DCK19 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefixes=CK19,CK19-64,CK19-NOUSE
+// RUN: %clang_cc1 -DCK19 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK19,CK19-64,CK19-NOUSE
+// RUN: %clang_cc1 -DCK19 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK19,CK19-32,CK19-NOUSE
+// RUN: %clang_cc1 -DCK19 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK19,CK19-32,CK19-NOUSE
+
 // RUN: %clang_cc1 -DCK19 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
 // RUN: %clang_cc1 -DCK19 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
@@ -3121,6 +3395,20 @@ void explicit_maps_single (int ii){
 // RUN: %clang_cc1 -DCK20 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK20 --check-prefix CK20-32
 
+// RUN: %clang_cc1 -DCK20 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK20 --check-prefix CK20-64
+// RUN: %clang_cc1 -DCK20 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK20 --check-prefix CK20-64
+// RUN: %clang_cc1 -DCK20 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK20 --check-prefix CK20-32
+// RUN: %clang_cc1 -DCK20 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK20 --check-prefix CK20-32
+
+// RUN: %clang_cc1 -DCK20 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK20 --check-prefix CK20-64
+// RUN: %clang_cc1 -DCK20 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK20 --check-prefix CK20-64
+// RUN: %clang_cc1 -DCK20 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK20 --check-prefix CK20-32
+// RUN: %clang_cc1 -DCK20 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK20 --check-prefix CK20-32
+
 // RUN: %clang_cc1 -DCK20 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY19 %s
 // RUN: %clang_cc1 -DCK20 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY19 %s
@@ -3249,6 +3537,20 @@ void explicit_maps_references_and_function_args (int a, float b, int (&c)[10], f
 // RUN: %clang_cc1 -DUSE -DCK21 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -DUSE -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK21,CK21-32,CK21-USE
 
+// RUN: %clang_cc1 -DUSE -DCK21 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefixes=CK21,CK21-64,CK21-USE
+// RUN: %clang_cc1 -DUSE -DCK21 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -DUSE -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK21,CK21-64,CK21-USE
+// RUN: %clang_cc1 -DUSE -DCK21 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK21,CK21-32,CK21-USE
+// RUN: %clang_cc1 -DUSE -DCK21 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -DUSE -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK21,CK21-32,CK21-USE
+
+// RUN: %clang_cc1 -DUSE -DCK21 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefixes=CK21,CK21-64,CK21-USE
+// RUN: %clang_cc1 -DUSE -DCK21 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -DUSE -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK21,CK21-64,CK21-USE
+// RUN: %clang_cc1 -DUSE -DCK21 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK21,CK21-32,CK21-USE
+// RUN: %clang_cc1 -DUSE -DCK21 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -DUSE -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK21,CK21-32,CK21-USE
+
 // RUN: %clang_cc1 -DUSE -DCK21 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY20 %s
 // RUN: %clang_cc1 -DUSE -DCK21 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -DUSE -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY20 %s
@@ -3263,6 +3565,20 @@ void explicit_maps_references_and_function_args (int a, float b, int (&c)[10], f
 // RUN: %clang_cc1 -DCK21 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK21,CK21-32,CK21-NOUSE
 
+// RUN: %clang_cc1 -DCK21 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefixes=CK21,CK21-64,CK21-NOUSE
+// RUN: %clang_cc1 -DCK21 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK21,CK21-64,CK21-NOUSE
+// RUN: %clang_cc1 -DCK21 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK21,CK21-32,CK21-NOUSE
+// RUN: %clang_cc1 -DCK21 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK21,CK21-32,CK21-NOUSE
+
+// RUN: %clang_cc1 -DCK21 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefixes=CK21,CK21-64,CK21-NOUSE
+// RUN: %clang_cc1 -DCK21 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK21,CK21-64,CK21-NOUSE
+// RUN: %clang_cc1 -DCK21 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK21,CK21-32,CK21-NOUSE
+// RUN: %clang_cc1 -DCK21 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK21,CK21-32,CK21-NOUSE
+
 // RUN: %clang_cc1 -DCK21 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY20 %s
 // RUN: %clang_cc1 -DCK21 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY20 %s
@@ -3514,6 +3830,20 @@ int explicit_maps_template_args_and_members(int a){
 // RUN: %clang_cc1 -DCK22 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK22 --check-prefix CK22-32
 
+// RUN: %clang_cc1 -DCK22 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK22 --check-prefix CK22-64
+// RUN: %clang_cc1 -DCK22 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK22 --check-prefix CK22-64
+// RUN: %clang_cc1 -DCK22 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK22 --check-prefix CK22-32
+// RUN: %clang_cc1 -DCK22 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK22 --check-prefix CK22-32
+
+// RUN: %clang_cc1 -DCK22 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK22 --check-prefix CK22-64
+// RUN: %clang_cc1 -DCK22 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK22 --check-prefix CK22-64
+// RUN: %clang_cc1 -DCK22 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK22 --check-prefix CK22-32
+// RUN: %clang_cc1 -DCK22 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK22 --check-prefix CK22-32
+
 // RUN: %clang_cc1 -DCK22 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY21 %s
 // RUN: %clang_cc1 -DCK22 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY21 %s
@@ -3884,6 +4214,20 @@ int explicit_maps_globals(void){
 // RUN: %clang_cc1 -std=c++11 -DCK23 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -std=c++11 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK23 --check-prefix CK23-32
 
+// RUN: %clang_cc1 -std=c++11 -DCK23 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK23 --check-prefix CK23-64
+// RUN: %clang_cc1 -std=c++11 -DCK23 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -std=c++11 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK23 --check-prefix CK23-64
+// RUN: %clang_cc1 -std=c++11 -DCK23 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK23 --check-prefix CK23-32
+// RUN: %clang_cc1 -std=c++11 -DCK23 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -std=c++11 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK23 --check-prefix CK23-32
+
+// RUN: %clang_cc1 -std=c++11 -DCK23 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK23 --check-prefix CK23-64
+// RUN: %clang_cc1 -std=c++11 -DCK23 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -std=c++11 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK23 --check-prefix CK23-64
+// RUN: %clang_cc1 -std=c++11 -DCK23 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK23 --check-prefix CK23-32
+// RUN: %clang_cc1 -std=c++11 -DCK23 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -std=c++11 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK23 --check-prefix CK23-32
+
 // RUN: %clang_cc1 -std=c++11 -DCK23 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY22 %s
 // RUN: %clang_cc1 -std=c++11 -DCK23 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -std=c++11 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY22 %s
@@ -4065,6 +4409,20 @@ int explicit_maps_inside_captured(int a){
 // RUN: %clang_cc1 -DCK24 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK24 --check-prefix CK24-32
 
+// RUN: %clang_cc1 -DCK24 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK24 --check-prefix CK24-64
+// RUN: %clang_cc1 -DCK24 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK24 --check-prefix CK24-64
+// RUN: %clang_cc1 -DCK24 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK24 --check-prefix CK24-32
+// RUN: %clang_cc1 -DCK24 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK24 --check-prefix CK24-32
+
+// RUN: %clang_cc1 -DCK24 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK24 --check-prefix CK24-64
+// RUN: %clang_cc1 -DCK24 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK24 --check-prefix CK24-64
+// RUN: %clang_cc1 -DCK24 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK24 --check-prefix CK24-32
+// RUN: %clang_cc1 -DCK24 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK24 --check-prefix CK24-32
+
 // RUN: %clang_cc1 -DCK24 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY23 %s
 // RUN: %clang_cc1 -DCK24 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY23 %s
@@ -4680,6 +5038,20 @@ int explicit_maps_struct_fields(int a){
 // RUN: %clang_cc1 -DCK25 -std=c++11 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -std=c++11 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK25 --check-prefix CK25-32
 
+// RUN: %clang_cc1 -DCK25 -std=c++11 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK25 --check-prefix CK25-64
+// RUN: %clang_cc1 -DCK25 -std=c++11 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -std=c++11 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK25 --check-prefix CK25-64
+// RUN: %clang_cc1 -DCK25 -std=c++11 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK25 --check-prefix CK25-32
+// RUN: %clang_cc1 -DCK25 -std=c++11 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -std=c++11 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK25 --check-prefix CK25-32
+
+// RUN: %clang_cc1 -DCK25 -std=c++11 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK25 --check-prefix CK25-64
+// RUN: %clang_cc1 -DCK25 -std=c++11 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -std=c++11 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK25 --check-prefix CK25-64
+// RUN: %clang_cc1 -DCK25 -std=c++11 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK25 --check-prefix CK25-32
+// RUN: %clang_cc1 -DCK25 -std=c++11 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -std=c++11 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK25 --check-prefix CK25-32
+
 // RUN: %clang_cc1 -DCK25 -std=c++11 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY24 %s
 // RUN: %clang_cc1 -DCK25 -std=c++11 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -std=c++11 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY24 %s
@@ -4795,6 +5167,20 @@ int explicit_maps_with_inner_lambda(int a){
 // RUN: %clang_cc1 -DCK26 -std=c++11 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -std=c++11 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK26 --check-prefix CK26-32
 
+// RUN: %clang_cc1 -DCK26 -std=c++11 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK26 --check-prefix CK26-64
+// RUN: %clang_cc1 -DCK26 -std=c++11 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -std=c++11 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK26 --check-prefix CK26-64
+// RUN: %clang_cc1 -DCK26 -std=c++11 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK26 --check-prefix CK26-32
+// RUN: %clang_cc1 -DCK26 -std=c++11 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -std=c++11 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK26 --check-prefix CK26-32
+
+// RUN: %clang_cc1 -DCK26 -std=c++11 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK26 --check-prefix CK26-64
+// RUN: %clang_cc1 -DCK26 -std=c++11 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -std=c++11 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK26 --check-prefix CK26-64
+// RUN: %clang_cc1 -DCK26 -std=c++11 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK26 --check-prefix CK26-32
+// RUN: %clang_cc1 -DCK26 -std=c++11 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -std=c++11 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK26 --check-prefix CK26-32
+
 // RUN: %clang_cc1 -DCK26 -std=c++11 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY25 %s
 // RUN: %clang_cc1 -DCK26 -std=c++11 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -std=c++11 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY25 %s
@@ -4999,6 +5385,20 @@ int explicit_maps_with_private_class_members(){
 // RUN: %clang_cc1 -DCK27 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK27 --check-prefix CK27-32
 
+// RUN: %clang_cc1 -DCK27 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK27 --check-prefix CK27-64
+// RUN: %clang_cc1 -DCK27 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK27 --check-prefix CK27-64
+// RUN: %clang_cc1 -DCK27 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK27 --check-prefix CK27-32
+// RUN: %clang_cc1 -DCK27 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK27 --check-prefix CK27-32
+
+// RUN: %clang_cc1 -DCK27 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK27 --check-prefix CK27-64
+// RUN: %clang_cc1 -DCK27 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK27 --check-prefix CK27-64
+// RUN: %clang_cc1 -DCK27 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK27 --check-prefix CK27-32
+// RUN: %clang_cc1 -DCK27 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK27 --check-prefix CK27-32
+
 // RUN: %clang_cc1 -DCK27 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY26 %s
 // RUN: %clang_cc1 -DCK27 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY26 %s
@@ -5228,6 +5628,20 @@ void zero_size_section_and_private_maps (int ii){
 // RUN: %clang_cc1 -DCK28 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK28 --check-prefix CK28-32
 
+// RUN: %clang_cc1 -DCK28 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK28 --check-prefix CK28-64
+// RUN: %clang_cc1 -DCK28 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK28 --check-prefix CK28-64
+// RUN: %clang_cc1 -DCK28 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK28 --check-prefix CK28-32
+// RUN: %clang_cc1 -DCK28 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK28 --check-prefix CK28-32
+
+// RUN: %clang_cc1 -DCK28 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK28 --check-prefix CK28-64
+// RUN: %clang_cc1 -DCK28 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK28 --check-prefix CK28-64
+// RUN: %clang_cc1 -DCK28 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK28 --check-prefix CK28-32
+// RUN: %clang_cc1 -DCK28 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK28 --check-prefix CK28-32
+
 // RUN: %clang_cc1 -DCK28 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY27 %s
 // RUN: %clang_cc1 -DCK28 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY27 %s
@@ -5301,6 +5715,20 @@ void explicit_maps_pointer_references (int *p){
 // RUN: %clang_cc1 -DCK29 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK29 --check-prefix CK29-32
 
+// RUN: %clang_cc1 -DCK29 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK29 --check-prefix CK29-64
+// RUN: %clang_cc1 -DCK29 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK29 --check-prefix CK29-64
+// RUN: %clang_cc1 -DCK29 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK29 --check-prefix CK29-32
+// RUN: %clang_cc1 -DCK29 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK29 --check-prefix CK29-32
+
+// RUN: %clang_cc1 -DCK29 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK29 --check-prefix CK29-64
+// RUN: %clang_cc1 -DCK29 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK29 --check-prefix CK29-64
+// RUN: %clang_cc1 -DCK29 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK29 --check-prefix CK29-32
+// RUN: %clang_cc1 -DCK29 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK29 --check-prefix CK29-32
+
 // RUN: %clang_cc1 -DCK29 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY28 %s
 // RUN: %clang_cc1 -DCK29 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY28 %s
@@ -5490,6 +5918,20 @@ void explicit_maps_member_pointer_references(SSA *sap) {
 // RUN: %clang_cc1 -DCK30 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK30 --check-prefix CK30-32
 
+// RUN: %clang_cc1 -DCK30 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK30 --check-prefix CK30-64
+// RUN: %clang_cc1 -DCK30 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK30 --check-prefix CK30-64
+// RUN: %clang_cc1 -DCK30 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK30 --check-prefix CK30-32
+// RUN: %clang_cc1 -DCK30 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK30 --check-prefix CK30-32
+
+// RUN: %clang_cc1 -DCK30 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK30 --check-prefix CK30-64
+// RUN: %clang_cc1 -DCK30 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK30 --check-prefix CK30-64
+// RUN: %clang_cc1 -DCK30 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK30 --check-prefix CK30-32
+// RUN: %clang_cc1 -DCK30 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK30 --check-prefix CK30-32
+
 // RUN: %clang_cc1 -DCK30 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY30 %s
 // RUN: %clang_cc1 -DCK30 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY30 %s
@@ -5639,6 +6081,20 @@ void map_with_deep_copy() {
 // RUN: %clang_cc1 -DCK31 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK31 --check-prefix CK31-32
 
+// RUN: %clang_cc1 -DCK31 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK31 --check-prefix CK31-64
+// RUN: %clang_cc1 -DCK31 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK31 --check-prefix CK31-64
+// RUN: %clang_cc1 -DCK31 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK31 --check-prefix CK31-32
+// RUN: %clang_cc1 -DCK31 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK31 --check-prefix CK31-32
+
+// RUN: %clang_cc1 -DCK31 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK31 --check-prefix CK31-64
+// RUN: %clang_cc1 -DCK31 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK31 --check-prefix CK31-64
+// RUN: %clang_cc1 -DCK31 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK31 --check-prefix CK31-32
+// RUN: %clang_cc1 -DCK31 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK31 --check-prefix CK31-32
+
 // RUN: %clang_cc1 -DCK31 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
 // RUN: %clang_cc1 -DCK31 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
@@ -5702,6 +6158,304 @@ void explicit_maps_single (int ii){
 // CK31: define {{.+}}[[CALL00]]
 // CK31: define {{.+}}[[CALL01]]
 
+#endif
+///==========================================================================///
+// RUN: %clang_cc1 -DUSE -DCK31A -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefixes=CK31A,CK31A-64,CK31A-USE
+// RUN: %clang_cc1 -DUSE -DCK31A -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -DUSE -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK31A,CK31A-64,CK31A-USE
+// RUN: %clang_cc1 -DUSE -DCK31A -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK31A,CK31A-32,CK31A-USE
+// RUN: %clang_cc1 -DUSE -DCK31A -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -DUSE -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK31A,CK31A-32,CK31A-USE
+
+// RUN: %clang_cc1 -DUSE -DCK31A -verify -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -DUSE -DCK31A -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -DUSE -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -DUSE -DCK31A -verify -fopenmp-version=51 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -DUSE -DCK31A -fopenmp-version=51 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -DUSE -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+
+// RUN: %clang_cc1 -DCK31A -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefixes=CK31A,CK31A-64,CK31A-NOUSE
+// RUN: %clang_cc1 -DCK31A -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK31A,CK31A-64,CK31A-NOUSE
+// RUN: %clang_cc1 -DCK31A -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK31A,CK31A-32,CK31A-NOUSE
+// RUN: %clang_cc1 -DCK31A -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK31A,CK31A-32,CK31A-NOUSE
+
+// RUN: %clang_cc1 -DCK31A -verify -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -DCK31A -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -DCK31A -verify -fopenmp-version=51 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -DCK31A -fopenmp-version=51 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+// SIMD-ONLY18-NOT: {{__kmpc|__tgt}}
+#ifdef CK31A
+
+// CK31A: [[ST:%.+]] = type { i32, i32 }
+struct ST {
+  int i;
+  int j;
+};
+
+// CK31A-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
+//
+// PRESENT=0x1000 | TARGET_PARAM=0x20 = 0x1020
+// CK31A: [[MTYPE00:@.+]] = private {{.*}}constant [7 x i64] [i64 [[#0x1020]],
+//
+// MEMBER_OF_1=0x1000000000000 | FROM=0x2 | TO=0x1 = 0x1000000000003
+// MEMBER_OF_1=0x1000000000000 | PRESENT=0x1000 | FROM=0x2 | TO=0x1 = 0x1000000001003
+// PRESENT=0x1000 | TARGET_PARAM=0x20 | FROM=0x2 | TO=0x1 = 0x1023
+// CK31A-SAME: {{^}} i64 [[#0x1000000000003]], i64 [[#0x1000000001003]], i64 [[#0x1023]],
+//
+// PRESENT=0x1000 | TARGET_PARAM=0x20 = 0x1020
+// MEMBER_OF_5=0x5000000000000 | PRESENT=0x1000 | FROM=0x2 | TO=0x1 = 0x5000000001003
+// MEMBER_OF_5=0x5000000000000 | FROM=0x2 | TO=0x1 = 0x5000000000003
+// CK31A-SAME: {{^}} i64 [[#0x1020]], i64 [[#0x5000000001003]], i64 [[#0x5000000000003]]]
+
+// CK31A-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
+// CK31A: [[SIZE01:@.+]] = private {{.*}}constant [1 x i[[Z:64|32]]] [i[[Z:64|32]] 4]
+//
+// PRESENT=0x1000 | CLOSE=0x400 | TARGET_PARAM=0x20 | ALWAYS=0x4 | FROM=0x2 | TO=0x1 = 0x1427
+// CK31A: [[MTYPE01:@.+]] = private {{.*}}constant [1 x i64] [i64 [[#0x1427]]]
+
+// CK31A-LABEL: explicit_maps_single{{.*}}(
+void explicit_maps_single (int ii){
+  // CK31A: alloca i32
+
+  // Map of a scalar.
+  // CK31A: [[A:%.+]] = alloca i32
+  int a = ii;
+
+  // CK31A: [[ST1:%.+]] = alloca [[ST]]
+  // CK31A: [[ST2:%.+]] = alloca [[ST]]
+  struct ST st1;
+  struct ST st2;
+
+  // Make sure the struct picks up present even if another element of the struct
+  // doesn't have present.
+  // Region 00
+  // CK31A: [[ST1_I:%.+]] = getelementptr inbounds [[ST]], [[ST]]* [[ST1]], i{{.+}} 0, i{{.+}} 0
+  // CK31A: [[ST1_J:%.+]] = getelementptr inbounds [[ST]], [[ST]]* [[ST1]], i{{.+}} 0, i{{.+}} 1
+  // CK31A: [[ST2_I:%.+]] = getelementptr inbounds [[ST]], [[ST]]* [[ST2]], i{{.+}} 0, i{{.+}} 0
+  // CK31A: [[ST2_J:%.+]] = getelementptr inbounds [[ST]], [[ST]]* [[ST2]], i{{.+}} 0, i{{.+}} 1
+  // CK31A-DAG: call i32 @__tgt_target_mapper(i64 {{[^,]+}}, i8* {{[^,]+}}, i32 7, i8** [[GEPBP:%[0-9]+]], i8** [[GEPP:%[0-9]+]], i64* [[GEPS:%.+]], i64* getelementptr {{.+}}[7 x i{{.+}}]* [[MTYPE00]]{{.+}})
+  // CK31A-DAG: [[GEPS]] = getelementptr inbounds [7 x i64], [7 x i64]* [[S:%.+]], i{{.+}} 0, i{{.+}} 0
+  // CK31A-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
+  // CK31A-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
+
+  // st1
+  // CK31A-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK31A-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK31A-DAG: [[S0:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 0
+  // CK31A-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to [[ST]]**
+  // CK31A-DAG: [[CP0:%.+]] = bitcast i8** [[P0]] to i32**
+  // CK31A-DAG: store [[ST]]* [[ST1]], [[ST]]** [[CBP0]]
+  // CK31A-DAG: store i32* [[ST1_I]], i32** [[CP0]]
+  // CK31A-DAG: store i64 %{{.+}}, i64* [[S0]]
+
+  // st1.i
+  // CK31A-DAG: [[BP1:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK31A-DAG: [[P1:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK31A-DAG: [[S1:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 1
+  // CK31A-DAG: [[CBP1:%.+]] = bitcast i8** [[BP1]] to [[ST]]**
+  // CK31A-DAG: [[CP1:%.+]] = bitcast i8** [[P1]] to i32**
+  // CK31A-DAG: store [[ST]]* [[ST1]], [[ST]]** [[CBP1]]
+  // CK31A-DAG: store i32* [[ST1_I]], i32** [[CP1]]
+  // CK31A-DAG: store i64 4, i64* [[S1]]
+
+  // st1.j
+  // CK31A-DAG: [[BP2:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
+  // CK31A-DAG: [[P2:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
+  // CK31A-DAG: [[S2:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 2
+  // CK31A-DAG: [[CBP2:%.+]] = bitcast i8** [[BP2]] to [[ST]]**
+  // CK31A-DAG: [[CP2:%.+]] = bitcast i8** [[P2]] to i32**
+  // CK31A-DAG: store [[ST]]* [[ST1]], [[ST]]** [[CBP2]]
+  // CK31A-DAG: store i32* [[ST1_J]], i32** [[CP2]]
+  // CK31A-DAG: store i64 4, i64* [[S2]]
+
+  // a
+  // CK31A-DAG: [[BP3:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 3
+  // CK31A-DAG: [[P3:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 3
+  // CK31A-DAG: [[S3:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 3
+  // CK31A-DAG: [[CBP3:%.+]] = bitcast i8** [[BP3]] to i32**
+  // CK31A-DAG: [[CP3:%.+]] = bitcast i8** [[P3]] to i32**
+  // CK31A-DAG: store i32* [[A]], i32** [[CBP3]]
+  // CK31A-DAG: store i32* [[A]], i32** [[CP3]]
+  // CK31A-DAG: store i64 4, i64* [[S3]]
+
+  // st2
+  // CK31A-DAG: [[BP4:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 4
+  // CK31A-DAG: [[P4:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 4
+  // CK31A-DAG: [[S4:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 4
+  // CK31A-DAG: [[CBP4:%.+]] = bitcast i8** [[BP4]] to [[ST]]**
+  // CK31A-DAG: [[CP4:%.+]] = bitcast i8** [[P4]] to i32**
+  // CK31A-DAG: store [[ST]]* [[ST2]], [[ST]]** [[CBP4]]
+  // CK31A-DAG: store i32* [[ST2_I]], i32** [[CP4]]
+  // CK31A-DAG: store i64 %{{.+}}, i64* [[S4]]
+
+  // st2.i
+  // CK31A-DAG: [[BP5:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 5
+  // CK31A-DAG: [[P5:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 5
+  // CK31A-DAG: [[S5:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 5
+  // CK31A-DAG: [[CBP5:%.+]] = bitcast i8** [[BP5]] to [[ST]]**
+  // CK31A-DAG: [[CP5:%.+]] = bitcast i8** [[P5]] to i32**
+  // CK31A-DAG: store [[ST]]* [[ST2]], [[ST]]** [[CBP5]]
+  // CK31A-DAG: store i32* [[ST2_I]], i32** [[CP5]]
+  // CK31A-DAG: store i64 4, i64* [[S5]]
+
+  // st2.j
+  // CK31A-DAG: [[BP6:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 6
+  // CK31A-DAG: [[P6:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 6
+  // CK31A-DAG: [[S6:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 6
+  // CK31A-DAG: [[CBP6:%.+]] = bitcast i8** [[BP6]] to [[ST]]**
+  // CK31A-DAG: [[CP6:%.+]] = bitcast i8** [[P6]] to i32**
+  // CK31A-DAG: store [[ST]]* [[ST2]], [[ST]]** [[CBP6]]
+  // CK31A-DAG: store i32* [[ST2_J]], i32** [[CP6]]
+  // CK31A-DAG: store i64 4, i64* [[S6]]
+
+  // CK31A-USE: call void [[CALL00:@.+]]([[ST]]* [[ST1]], i32* [[A]], [[ST]]* [[ST2]])
+  // CK31A-NOUSE: call void [[CALL00:@.+]]()
+  #pragma omp target map(tofrom: st1.i) map(present, tofrom: a, st1.j, st2.i) map(tofrom: st2.j)
+  {
+#ifdef USE
+    st1.i++;
+    a++;
+    st1.j++;
+    st2.i++;
+    st2.j++;
+#endif
+  }
+
+  // Always Close Present.
+  // Region 01
+  // CK31A-DAG: call i32 @__tgt_target_mapper(i64 {{[^,]+}}, i8* {{[^,]+}}, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[SIZE01]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE01]]{{.+}})
+  // CK31A-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
+  // CK31A-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
+
+  // CK31A-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK31A-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK31A-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to i32**
+  // CK31A-DAG: [[CP0:%.+]] = bitcast i8** [[P0]] to i32**
+  // CK31A-DAG: store i32* [[VAR0:%.+]], i32** [[CBP0]]
+  // CK31A-DAG: store i32* [[VAR0]], i32** [[CP0]]
+
+  // CK31A-USE: call void [[CALL01:@.+]](i32* {{[^,]+}})
+  // CK31A-NOUSE: call void [[CALL01:@.+]]()
+  #pragma omp target map(always close present tofrom: a)
+  {
+#ifdef USE
+    a++;
+#endif
+  }
+}
+// CK31A: define {{.+}}[[CALL00]]
+// CK31A: define {{.+}}[[CALL01]]
+
+#endif
+///==========================================================================///
+// RUN: %clang_cc1 -DUSE -DCK31B -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefixes=CK31B,CK31B-64,CK31B-USE
+// RUN: %clang_cc1 -DUSE -DCK31B -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -DUSE -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK31B,CK31B-64,CK31B-USE
+// RUN: %clang_cc1 -DUSE -DCK31B -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK31B,CK31B-32,CK31B-USE
+// RUN: %clang_cc1 -DUSE -DCK31B -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -DUSE -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK31B,CK31B-32,CK31B-USE
+
+// RUN: %clang_cc1 -DUSE -DCK31B -verify -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -DUSE -DCK31B -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -DUSE -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -DUSE -DCK31B -verify -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -DUSE -DCK31B -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -DUSE -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+
+// RUN: %clang_cc1 -DCK31B -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefixes=CK31B,CK31B-64,CK31B-NOUSE
+// RUN: %clang_cc1 -DCK31B -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK31B,CK31B-64,CK31B-NOUSE
+// RUN: %clang_cc1 -DCK31B -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK31B,CK31B-32,CK31B-NOUSE
+// RUN: %clang_cc1 -DCK31B -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefixes=CK31B,CK31B-32,CK31B-NOUSE
+
+// RUN: %clang_cc1 -DCK31B -verify -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -DCK31B -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -DCK31B -verify -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -DCK31B -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+// SIMD-ONLY18-NOT: {{__kmpc|__tgt}}
+#ifdef CK31B
+
+// CK31B: [[ST:%.+]] = type { i32, i32 }
+
+// PRESENT=0x1000 | TARGET_PARAM=0x20 = 0x1020
+// MEMBER_OF_1=0x1000000000000 | FROM=0x2 | TO=0x1 = 0x1000000000003
+// MEMBER_OF_1=0x1000000000000 | PRESENT=0x1000 | FROM=0x2 | TO=0x1 = 0x1000000001003
+
+// CK31B-LABEL: @.__omp_offloading_{{.*}}test_present_members{{.*}}_l{{[0-9]+}}.region_id = weak constant i8 0
+// CK31B: [[MTYPE00:@.+]] = private {{.*}}constant [3 x i64] [i64 [[#0x1020]],
+// CK31B-SAME: {{^}} i64 [[#0x1000000000003]], i64 [[#0x1000000001003]]]
+
+struct ST {
+  int i;
+  int j;
+  // CK31B-LABEL: define {{.*}}test_present_members{{.*}}(
+  void test_present_members() {
+    // Make sure the struct picks up present even if another element of the
+    // struct doesn't have present.
+    // Region 00
+    // CK31B: [[I:%.+]] = getelementptr inbounds [[ST]], [[ST]]* [[THIS:%.+]], i{{.+}} 0, i{{.+}} 0
+    // CK31B: [[J:%.+]] = getelementptr inbounds [[ST]], [[ST]]* [[THIS]], i{{.+}} 0, i{{.+}} 1
+    // CK31B-DAG: call i32 @__tgt_target_mapper(i64 {{[^,]+}}, i8* {{[^,]+}}, i32 3, i8** [[GEPBP:%[0-9]+]], i8** [[GEPP:%[0-9]+]], i64* [[GEPS:%.+]], i64* getelementptr {{.+}}[3 x i{{.+}}]* [[MTYPE00]]{{.+}})
+    // CK31B-DAG: [[GEPS]] = getelementptr inbounds [3 x i64], [3 x i64]* [[S:%.+]], i{{.+}} 0, i{{.+}} 0
+    // CK31B-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
+    // CK31B-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
+
+    // this
+    // CK31B-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+    // CK31B-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+    // CK31B-DAG: [[S0:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 0
+    // CK31B-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to [[ST]]**
+    // CK31B-DAG: [[CP0:%.+]] = bitcast i8** [[P0]] to i32**
+    // CK31B-DAG: store [[ST]]* [[THIS]], [[ST]]** [[CBP0]]
+    // CK31B-DAG: store i32* [[I]], i32** [[CP0]]
+    // CK31B-DAG: store i64 %{{.+}}, i64* [[S0]]
+
+    // i
+    // CK31B-DAG: [[BP1:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+    // CK31B-DAG: [[P1:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+    // CK31B-DAG: [[S1:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 1
+    // CK31B-DAG: [[CBP1:%.+]] = bitcast i8** [[BP1]] to [[ST]]**
+    // CK31B-DAG: [[CP1:%.+]] = bitcast i8** [[P1]] to i32**
+    // CK31B-DAG: store [[ST]]* [[THIS]], [[ST]]** [[CBP1]]
+    // CK31B-DAG: store i32* [[I]], i32** [[CP1]]
+    // CK31B-DAG: store i64 4, i64* [[S1]]
+
+    // j
+    // CK31B-DAG: [[BP2:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
+    // CK31B-DAG: [[P2:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
+    // CK31B-DAG: [[S2:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 2
+    // CK31B-DAG: [[CBP2:%.+]] = bitcast i8** [[BP2]] to [[ST]]**
+    // CK31B-DAG: [[CP2:%.+]] = bitcast i8** [[P2]] to i32**
+    // CK31B-DAG: store [[ST]]* [[THIS]], [[ST]]** [[CBP2]]
+    // CK31B-DAG: store i32* [[J]], i32** [[CP2]]
+    // CK31B-DAG: store i64 4, i64* [[S2]]
+
+    // CK31B-USE: call void [[CALL00:@.+]]([[ST]]* [[THIS]])
+    // CK31B-NOUSE: call void [[CALL00:@.+]]()
+    #pragma omp target map(tofrom: i) map(present, tofrom: j)
+    {
+#ifdef USE
+      i++;
+      j++;
+#endif
+    }
+  }
+};
+
+void test() {
+  ST s;
+  s.test_present_members();
+}
+
+// CK31B: define {{.+}}[[CALL00]]
+
 #endif
 ///==========================================================================///
 // RUN: %clang_cc1 -DCK32 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK32 --check-prefix CK32-64
diff --git a/clang/test/OpenMP/target_map_messages.cpp b/clang/test/OpenMP/target_map_messages.cpp
index 833f509cd7f0e..1d962f8f091fa 100644
--- a/clang/test/OpenMP/target_map_messages.cpp
+++ b/clang/test/OpenMP/target_map_messages.cpp
@@ -1,13 +1,15 @@
-// RUN: %clang_cc1 -verify=expected,le50 -fopenmp -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,le45 -fopenmp -fopenmp-version=40 -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,le45 -fopenmp -fopenmp-version=45 -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,le50 -fopenmp -fopenmp-version=50 -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51 -fopenmp -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,lt50,lt51 -fopenmp -fopenmp-version=40 -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,lt50,lt51 -fopenmp -fopenmp-version=45 -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51 -fopenmp -fopenmp-version=50 -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,ge51 -fopenmp -fopenmp-version=51 -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
 // RUN: %clang_cc1 -DCCODE -verify -fopenmp -ferror-limit 200 -x c %s -Wno-openmp -Wuninitialized
 
-// RUN: %clang_cc1 -verify=expected,le50 -fopenmp-simd -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,le45 -fopenmp-simd -fopenmp-version=40 -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,le45 -fopenmp-simd -fopenmp-version=45 -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,le50 -fopenmp-simd -fopenmp-version=50 -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51 -fopenmp-simd -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,lt50,lt51 -fopenmp-simd -fopenmp-version=40 -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,lt50,lt51 -fopenmp-simd -fopenmp-version=45 -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51 -fopenmp-simd -fopenmp-version=50 -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,ge51 -fopenmp-simd -fopenmp-version=51 -ferror-limit 200 %s -Wno-openmp-target -Wuninitialized
 // RUN: %clang_cc1 -DCCODE -verify -fopenmp-simd -ferror-limit 200 -x c %s -Wno-openmp-mapping -Wuninitialized
 #ifdef CCODE
 void foo(int arg) {
@@ -63,9 +65,9 @@ struct SA {
     {}
     #pragma omp target map(arg[2:2],a,d) // expected-error {{subscripted value is not an array or pointer}}
     {}
-    #pragma omp target map(arg,a*2) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}} le50-error {{expected addressable lvalue in 'map' clause}}
+    #pragma omp target map(arg,a*2) // lt50-error {{expected expression containing only member accesses and/or array sections based on named variables}} ge50-error {{expected addressable lvalue in 'map' clause}}
     {}
-    #pragma omp target map(arg,(c+1)[2]) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+    #pragma omp target map(arg,(c+1)[2]) // lt50-error {{expected expression containing only member accesses and/or array sections based on named variables}}
     {}
     #pragma omp target map(arg,a[:2],d) // expected-error {{subscripted value is not an array or pointer}}
     {}
@@ -116,9 +118,38 @@ struct SA {
     {}
     #pragma omp target map(close)   // expected-error {{use of undeclared identifier 'close'}}
     {}
+    // lt51-error@+1 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+    #pragma omp target map(present, tofrom: c,f)
+    {}
+    // lt51-error@+1 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+    #pragma omp target map(present, tofrom: c[1:2],f)
+    {}
+    // lt51-error@+1 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+    #pragma omp target map(present, tofrom: c,f[1:2])
+    {}
+    // expected-error@+2 {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
+    // lt51-error@+1 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+    #pragma omp target map(present, tofrom: c[:],f)
+    {}
+    // expected-error@+2 {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
+    // lt51-error@+1 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+    #pragma omp target map(present, tofrom: c,f[:])
+    {}
+    // expected-error@+1 {{use of undeclared identifier 'present'}}
+    #pragma omp target map(present)
+    {}
     #pragma omp target map(close, close, tofrom: a)   // expected-error {{same map type modifier has been specified more than once}}
     {}
-    #pragma omp target map(always, close, always, close, tofrom: a)   // expected-error {{same map type modifier has been specified more than once}} expected-error {{same map type modifier has been specified more than once}}
+    #pragma omp target map(always, close, always, close, tofrom: a)   // expected-error 2 {{same map type modifier has been specified more than once}}
+    {}
+    // ge51-error@+2 {{same map type modifier has been specified more than once}}
+    // lt51-error@+1 2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+    #pragma omp target map(present, present, tofrom: a)
+    {}
+    // expected-error@+3 2 {{same map type modifier has been specified more than once}}
+    // ge51-error@+2 1 {{same map type modifier has been specified more than once}}
+    // lt51-error@+1 2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+    #pragma omp target map(always, close, present, always, close, present, tofrom: a)
     {}
     #pragma omp target map( , tofrom: a)   // expected-error {{missing map type modifier}}
     {}
@@ -126,21 +157,33 @@ struct SA {
     {}
     #pragma omp target map( , , : a)   // expected-error {{missing map type modifier}} expected-error {{missing map type modifier}} expected-error {{missing map type}}
     {}
-    #pragma omp target map( d, f, bf: a)   // expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}}
+    // ge51-error@+3 2 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+    // lt51-error@+2 2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+    // expected-error@+1 {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}}
+    #pragma omp target map( d, f, bf: a)
     {}
-    #pragma omp target map( , f, : a)   // expected-error {{missing map type modifier}} expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{missing map type}}
+    // expected-error@+4 {{missing map type modifier}}
+    // ge51-error@+3 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+    // lt51-error@+2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+    // expected-error@+1 {{missing map type}}
+    #pragma omp target map( , f, : a)
     {}
     #pragma omp target map(always close: a)   // expected-error {{missing map type}}
     {}
     #pragma omp target map(always close bf: a)   // expected-error {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}}
     {}
-    #pragma omp target map(always tofrom close: a)   // expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{missing map type}}
+    // ge51-error@+3 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+    // lt51-error@+2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+    // expected-error@+1 {{missing map type}}
+    #pragma omp target map(always tofrom close: a)
     {}
-    #pragma omp target map(tofrom from: a)   // expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+    // ge51-error@+2 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+    // lt51-error@+1 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+    #pragma omp target map(tofrom from: a)
     {}
     #pragma omp target map(close bf: a)   // expected-error {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}}
     {}
-    #pragma omp target map(([b[I]][bf])f)  // le45-error {{expected ',' or ']' in lambda capture list}} le45-error {{expected ')'}} le45-note {{to match this '('}}
+    #pragma omp target map(([b[I]][bf])f)  // lt50-error {{expected ',' or ']' in lambda capture list}} lt50-error {{expected ')'}} lt50-note {{to match this '('}}
     {}
     return;
   }
@@ -313,7 +356,7 @@ void SAclient(int arg) {
   {}
 #pragma omp target map(r.S.Arr[:12])
   {}
-#pragma omp target map(r.S.foo() [:12]) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}} le50-error {{expected addressable lvalue in 'map' clause}}
+#pragma omp target map(r.S.foo() [:12]) // lt50-error {{expected expression containing only member accesses and/or array sections based on named variables}} ge50-error {{expected addressable lvalue in 'map' clause}}
   {}
 #pragma omp target map(r.C, r.D)
   {}
@@ -347,7 +390,7 @@ void SAclient(int arg) {
   {}
 #pragma omp target map(r.S.Ptr[:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
   {}
-#pragma omp target map((p + 1)->A) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target map((p + 1)->A) // lt50-error {{expected expression containing only member accesses and/or array sections based on named variables}}
   {}
 #pragma omp target map(u.B)        // expected-error {{mapping of union members is not allowed}}
   {}
@@ -467,7 +510,7 @@ T tmain(T argc) {
   T *k = &j;
   T x;
   T y;
-  T to, tofrom, always, close;
+  T to, tofrom, always, close, present;
   const T (&l)[5] = da;
 #pragma omp target map // expected-error {{expected '(' after 'map'}}
   {}
@@ -493,7 +536,7 @@ T tmain(T argc) {
   foo();
 #pragma omp target map(T) // expected-error {{'T' does not refer to a value}}
   foo();
-#pragma omp target map(I) // le45-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} le50-error 2 {{expected addressable lvalue in 'map' clause}}
+#pragma omp target map(I) // lt50-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} ge50-error 2 {{expected addressable lvalue in 'map' clause}}
   foo();
 #pragma omp target map(S2::S2s)
   foo();
@@ -510,7 +553,7 @@ T tmain(T argc) {
 #pragma omp target map(to, x)
   foo();
 #pragma omp target data map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
-#pragma omp target data map(tofrom: argc > 0 ? x : y) // le45-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} le50-error 2 {{expected addressable lvalue in 'map' clause}}
+#pragma omp target data map(tofrom: argc > 0 ? x : y) // lt50-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} ge50-error 2 {{expected addressable lvalue in 'map' clause}}
 #pragma omp target data map(argc)
 #pragma omp target data map(S1) // expected-error {{'S1' does not refer to a value}}
 #pragma omp target data map(a, b, c, d, f) // expected-error {{incomplete type 'S1' where a complete type is required}} warn-warning 2 {{Type 'const S2' is not trivially copyable and not guaranteed to be mapped correctly}} warn-warning 2 {{Type 'const S3' is not trivially copyable and not guaranteed to be mapped correctly}}
@@ -540,17 +583,37 @@ T tmain(T argc) {
 
 #pragma omp target data map(always, tofrom: x)
 #pragma omp target data map(always: x) // expected-error {{missing map type}}
-#pragma omp target data map(tofrom, always: x) // expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{missing map type}}
+// ge51-error@+3 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+// lt51-error@+2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+// expected-error@+1 {{missing map type}}
+#pragma omp target data map(tofrom, always: x)
 #pragma omp target data map(always, tofrom: always, tofrom, x)
 #pragma omp target map(tofrom j) // expected-error {{expected ',' or ')' in 'map' clause}}
   foo();
 
 #pragma omp target data map(close, tofrom: x)
 #pragma omp target data map(close: x) // expected-error {{missing map type}}
-#pragma omp target data map(tofrom, close: x) // expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{missing map type}}
+// ge51-error@+3 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+// lt51-error@+2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+// expected-error@+1 {{missing map type}}
+#pragma omp target data map(tofrom, close: x)
 #pragma omp target data map(close, tofrom: close, tofrom, x)
   foo();
 
+// lt51-error@+1 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+#pragma omp target data map(present, tofrom: x)
+// ge51-error@+2 {{missing map type}}
+// lt51-error@+1 {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}}
+#pragma omp target data map(present: x)
+// ge51-error@+4 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+// lt51-error@+3 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+// ge51-error@+2 {{missing map type}}
+// lt51-error@+1 {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}}
+#pragma omp target data map(tofrom, present: x)
+// lt51-error@+1 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+#pragma omp target data map(present, tofrom: present, tofrom, x)
+  foo();
+
   T marr[10][10], iarr[5];
 #pragma omp target data map(marr[10][0:2:2]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
   {}
@@ -600,11 +663,12 @@ int main(int argc, char **argv) {
   S6<int> m;
   int x;
   int y;
-  int to, tofrom, always, close;
+  int to, tofrom, always, close, present;
   const int (&l)[5] = da;
   SC1 s;
   SC1 *p;
-#pragma omp target data map // expected-error {{expected '(' after 'map'}} le45-error {{expected at least one 'map' or 'use_device_ptr' clause for '#pragma omp target data'}} le50-error {{expected at least one 'map', 'use_device_ptr', or 'use_device_addr' clause for '#pragma omp target data'}}
+  int Arr[10];
+#pragma omp target data map // expected-error {{expected '(' after 'map'}} lt50-error {{expected at least one 'map' or 'use_device_ptr' clause for '#pragma omp target data'}} ge50-error {{expected at least one 'map', 'use_device_ptr', or 'use_device_addr' clause for '#pragma omp target data'}}
 #pragma omp target data map( // expected-error {{expected ')'}} expected-note {{to match this '('}} expected-error {{expected expression}}
 #pragma omp target data map() // expected-error {{expected expression}}
 #pragma omp target data map(alloc) // expected-error {{use of undeclared identifier 'alloc'}}
@@ -623,7 +687,7 @@ int main(int argc, char **argv) {
 #pragma omp target map(to, x)
   foo();
 #pragma omp target data map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
-#pragma omp target data map(tofrom: argc > 0 ? argv[1] : argv[2]) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}} le50-error {{expected addressable lvalue in 'map' clause}}
+#pragma omp target data map(tofrom: argc > 0 ? argv[1] : argv[2]) // lt50-error {{expected expression containing only member accesses and/or array sections based on named variables}} ge50-error {{expected addressable lvalue in 'map' clause}}
 #pragma omp target data map(argc)
 #pragma omp target data map(S1) // expected-error {{'S1' does not refer to a value}}
 #pragma omp target data map(a, b, c, d, f) // expected-error {{incomplete type 'S1' where a complete type is required}} warn-warning {{Type 'const S2' is not trivially copyable and not guaranteed to be mapped correctly}} warn-warning {{Type 'const S3' is not trivially copyable and not guaranteed to be mapped correctly}}
@@ -654,13 +718,30 @@ int main(int argc, char **argv) {
 
 #pragma omp target data map(always, tofrom: x)
 #pragma omp target data map(always: x) // expected-error {{missing map type}}
-#pragma omp target data map(tofrom, always: x) // expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{missing map type}}
+// ge51-error@+3 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+// lt51-error@+2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+// expected-error@+1 {{missing map type}}
+#pragma omp target data map(tofrom, always: x)
 #pragma omp target data map(always, tofrom: always, tofrom, x)
 #pragma omp target map(tofrom j) // expected-error {{expected ',' or ')' in 'map' clause}}
   foo();
 #pragma omp target data map(close, tofrom: x)
 #pragma omp target data map(close: x) // expected-error {{missing map type}}
-#pragma omp target data map(tofrom, close: x) // expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{missing map type}}
+// ge51-error@+3 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+// lt51-error@+2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+// expected-error@+1 {{missing map type}}
+#pragma omp target data map(tofrom, close: x)
+  foo();
+// lt51-error@+1 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+#pragma omp target data map(present, tofrom: x)
+// ge51-error@+2 {{missing map type}}
+// lt51-error@+1 {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}}
+#pragma omp target data map(present: x)
+// ge51-error@+4 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+// lt51-error@+3 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+// ge51-error@+2 {{missing map type}}
+// lt51-error@+1 {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}}
+#pragma omp target data map(tofrom, present: x)
   foo();
 #pragma omp target private(j) map(j) // expected-error {{private variable cannot be in a map clause in '#pragma omp target' directive}}  expected-note {{defined as private}}
   {}
@@ -717,21 +798,21 @@ int main(int argc, char **argv) {
 
   int **BB, *offset, *a;
 
-#pragma omp target map(**(BB+*offset)) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target map(**(BB+*offset)) // lt50-error {{expected expression containing only member accesses and/or array sections based on named variables}}
   {}
-#pragma omp target map(**(BB+y)) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target map(**(BB+y)) // lt50-error {{expected expression containing only member accesses and/or array sections based on named variables}}
   {}
-#pragma omp target map(*(a+*offset)) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target map(*(a+*offset)) // lt50-error {{expected expression containing only member accesses and/or array sections based on named variables}}
   {}
-#pragma omp target map(**(*offset+BB)) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target map(**(*offset+BB)) // lt50-error {{expected expression containing only member accesses and/or array sections based on named variables}}
   {}
-#pragma omp target map(**(y+BB)) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target map(**(y+BB)) // lt50-error {{expected expression containing only member accesses and/or array sections based on named variables}}
   {}
-#pragma omp target map(*(*offset+a)) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target map(*(*offset+a)) // lt50-error {{expected expression containing only member accesses and/or array sections based on named variables}}
   {}
-#pragma omp target map(**(*offset+BB+*a)) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target map(**(*offset+BB+*a)) // lt50-error {{expected expression containing only member accesses and/or array sections based on named variables}}
   {}
-#pragma omp target map(**(*(*(&offset))+BB+*a)) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target map(**(*(*(&offset))+BB+*a)) // lt50-error {{expected expression containing only member accesses and/or array sections based on named variables}}
   {}
 #pragma omp target map(*(a+(a))) // expected-error {{invalid operands to binary expression ('int *' and 'int *')}}
   {}
@@ -750,6 +831,12 @@ int main(int argc, char **argv) {
 #pragma omp target map(iarr[:2:d]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
   {}
 
+#pragma omp target data map(Arr[0:4]) // lt50-note {{used here}}
+  {
+#pragma omp target
+    Arr[0] = 2; // lt50-error {{original storage of expression in data environment is shared but data environment do not fully contain mapped expression storage}}
+  }
+
   return tmain<int, 3>(argc)+tmain<from, 4>(argc); // expected-note {{in instantiation of function template specialization 'tmain<int, 3>' requested here}} expected-note {{in instantiation of function template specialization 'tmain<int, 4>' requested here}}
 }
 #endif
diff --git a/clang/test/OpenMP/target_parallel_for_map_messages.cpp b/clang/test/OpenMP/target_parallel_for_map_messages.cpp
index b44c330e2260d..665ecfa926aa9 100644
--- a/clang/test/OpenMP/target_parallel_for_map_messages.cpp
+++ b/clang/test/OpenMP/target_parallel_for_map_messages.cpp
@@ -1,8 +1,10 @@
-// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,lt50,lt51 -fopenmp -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51 -fopenmp -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,ge51 -fopenmp -fopenmp-version=51 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
-// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,lt50,lt51 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,ge51 -fopenmp-simd -fopenmp-version=51 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
 void foo() {
 }
@@ -105,7 +107,9 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for map(T) // expected-error {{'T' does not refer to a value}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target parallel for map(I) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
+// ge50-error@+2 2 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+1 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target parallel for map(I)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for map(S2::S2s)
   for (i = 0; i < argc; ++i) foo();
@@ -123,8 +127,10 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
+// ge50-error@+3 2 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+2 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
 #pragma omp target parallel for map(tofrom \
-                                    : argc > 0 ? x : y) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
+                                    : argc > 0 ? x : y)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for map(argc)
   for (i = 0; i < argc; ++i) foo();
@@ -175,7 +181,10 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for map(always: x) // expected-error {{missing map type}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target parallel for map(tofrom, always: x) // expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{missing map type}}
+// ge51-error@+3 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+// lt51-error@+2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+// expected-error@+1 {{missing map type}}
+#pragma omp target parallel for map(tofrom, always: x)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for map(always, tofrom: always, tofrom, x)
   for (i = 0; i < argc; ++i) foo();
@@ -233,8 +242,10 @@ int main(int argc, char **argv) {
 #pragma omp target parallel for map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   for (i = 0; i < argc; ++i)
     foo();
+// ge50-error@+3 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+2 {{expected expression containing only member accesses and/or array sections based on named variables}}
 #pragma omp target parallel for map(tofrom \
-                                    : argc > 0 ? argv[1] : argv[2]) // omp4-error {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error {{expected addressable lvalue in 'map' clause}}
+                                    : argc > 0 ? argv[1] : argv[2])
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for map(argc)
   for (i = 0; i < argc; ++i) foo();
@@ -287,7 +298,10 @@ int main(int argc, char **argv) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for map(always: x) // expected-error {{missing map type}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target parallel for map(tofrom, always: x) // expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{missing map type}}
+// ge51-error@+3 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+// lt51-error@+2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+// expected-error@+1 {{missing map type}}
+#pragma omp target parallel for map(tofrom, always: x)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for map(always, tofrom: always, tofrom, x)
   for (i = 0; i < argc; ++i) foo();
diff --git a/clang/test/OpenMP/target_parallel_for_simd_map_messages.cpp b/clang/test/OpenMP/target_parallel_for_simd_map_messages.cpp
index d4c0b30f57664..0272cd7c75779 100644
--- a/clang/test/OpenMP/target_parallel_for_simd_map_messages.cpp
+++ b/clang/test/OpenMP/target_parallel_for_simd_map_messages.cpp
@@ -1,8 +1,10 @@
-// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 %s -Wno-openmp-mapping -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,lt50,lt51 -fopenmp -fopenmp-version=45 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51 -fopenmp -fopenmp-version=50 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,ge51 -fopenmp -fopenmp-version=51 %s -Wno-openmp-mapping -Wuninitialized
 
-// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 %s -Wno-openmp-mapping -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,lt50,lt51 -fopenmp-simd -fopenmp-version=45 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51 -fopenmp-simd -fopenmp-version=50 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,ge51 -fopenmp-simd -fopenmp-version=51 %s -Wno-openmp-mapping -Wuninitialized
 
 void foo() {
 }
@@ -105,7 +107,9 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for simd map(T) // expected-error {{'T' does not refer to a value}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target parallel for simd map(I) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
+// ge50-error@+2 2 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+1 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target parallel for simd map(I)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for simd map(S2::S2s)
   for (i = 0; i < argc; ++i) foo();
@@ -123,8 +127,10 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for simd map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
+// ge50-error@+3 2 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+2 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
 #pragma omp target parallel for simd map(tofrom \
-                                         : argc > 0 ? x : y) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
+                                         : argc > 0 ? x : y)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for simd map(argc)
   for (i = 0; i < argc; ++i) foo();
@@ -175,7 +181,10 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for simd map(always: x) // expected-error {{missing map type}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target parallel for simd map(tofrom, always: x) // expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{missing map type}}
+// ge51-error@+3 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+// lt51-error@+2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+// expected-error@+1 {{missing map type}}
+#pragma omp target parallel for simd map(tofrom, always: x)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for simd map(always, tofrom: always, tofrom, x)
   for (i = 0; i < argc; ++i) foo();
@@ -233,8 +242,10 @@ int main(int argc, char **argv) {
 #pragma omp target parallel for simd map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   for (i = 0; i < argc; ++i)
     foo();
+// ge50-error@+3 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+2 {{expected expression containing only member accesses and/or array sections based on named variables}}
 #pragma omp target parallel for simd map(tofrom \
-                                         : argc > 0 ? argv[1] : argv[2]) // omp4-error {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error {{expected addressable lvalue in 'map' clause}}
+                                         : argc > 0 ? argv[1] : argv[2])
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for simd map(argc)
   for (i = 0; i < argc; ++i) foo();
@@ -287,7 +298,10 @@ int main(int argc, char **argv) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for simd map(always: x) // expected-error {{missing map type}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target parallel for simd map(tofrom, always: x) // expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{missing map type}}
+// ge51-error@+3 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+// lt51-error@+2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+// expected-error@+1 {{missing map type}}
+#pragma omp target parallel for simd map(tofrom, always: x)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target parallel for simd map(always, tofrom: always, tofrom, x)
   for (i = 0; i < argc; ++i) foo();
diff --git a/clang/test/OpenMP/target_parallel_map_messages.cpp b/clang/test/OpenMP/target_parallel_map_messages.cpp
index 6e080fda9226b..fe5330f2daec0 100644
--- a/clang/test/OpenMP/target_parallel_map_messages.cpp
+++ b/clang/test/OpenMP/target_parallel_map_messages.cpp
@@ -1,8 +1,10 @@
-// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,lt50,lt51 -fopenmp -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51 -fopenmp -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,ge51 -fopenmp -fopenmp-version=51 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
-// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,lt50,lt51 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,ge51 -fopenmp-simd -fopenmp-version=51 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
 void foo() {
 }
@@ -105,7 +107,9 @@ T tmain(T argc) {
   foo();
 #pragma omp target parallel map(T) // expected-error {{'T' does not refer to a value}}
   foo();
-#pragma omp target parallel map(I) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
+// ge50-error@+2 2 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+1 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target parallel map(I)
   foo();
 #pragma omp target parallel map(S2::S2s)
   foo();
@@ -123,8 +127,10 @@ T tmain(T argc) {
   foo();
 #pragma omp target parallel map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   foo();
+// ge50-error@+3 2 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+2 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
 #pragma omp target parallel map(tofrom \
-                                : argc > 0 ? x : y) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
+                                : argc > 0 ? x : y)
   foo();
 #pragma omp target parallel map(argc)
   foo();
@@ -175,7 +181,10 @@ T tmain(T argc) {
   foo();
 #pragma omp target parallel map(always: x) // expected-error {{missing map type}}
   foo();
-#pragma omp target parallel map(tofrom, always: x) // expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{missing map type}}
+// ge51-error@+3 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+// lt51-error@+2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+// expected-error@+1 {{missing map type}}
+#pragma omp target parallel map(tofrom, always: x)
   foo();
 #pragma omp target parallel map(always, tofrom: always, tofrom, x)
   foo();
@@ -231,8 +240,10 @@ int main(int argc, char **argv) {
   foo();
 #pragma omp target parallel map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   foo();
+// ge50-error@+3 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+2 {{expected expression containing only member accesses and/or array sections based on named variables}}
 #pragma omp target parallel map(tofrom \
-                                : argc > 0 ? argv[1] : argv[2]) // omp4-error {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error {{expected addressable lvalue in 'map' clause}}
+                                : argc > 0 ? argv[1] : argv[2])
   foo();
 #pragma omp target parallel map(argc)
   foo();
@@ -285,7 +296,10 @@ int main(int argc, char **argv) {
   foo();
 #pragma omp target parallel map(always: x) // expected-error {{missing map type}}
   foo();
-#pragma omp target parallel map(tofrom, always: x) // expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{missing map type}}
+// ge51-error@+3 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+// lt51-error@+2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+// expected-error@+1 {{missing map type}}
+#pragma omp target parallel map(tofrom, always: x)
   foo();
 #pragma omp target parallel map(always, tofrom: always, tofrom, x)
   foo();
diff --git a/clang/test/OpenMP/target_simd_map_messages.cpp b/clang/test/OpenMP/target_simd_map_messages.cpp
index 57929e276260e..4a0522725800b 100644
--- a/clang/test/OpenMP/target_simd_map_messages.cpp
+++ b/clang/test/OpenMP/target_simd_map_messages.cpp
@@ -1,8 +1,10 @@
-// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 %s -Wno-openmp-mapping -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,lt50,lt51 -fopenmp -fopenmp-version=45 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51 -fopenmp -fopenmp-version=50 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,ge51 -fopenmp -fopenmp-version=51 %s -Wno-openmp-mapping -Wuninitialized
 
-// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 %s -Wno-openmp-mapping -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,lt50,lt51 -fopenmp-simd -fopenmp-version=45 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51 -fopenmp-simd -fopenmp-version=50 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,ge51 -fopenmp-simd -fopenmp-version=51 %s -Wno-openmp-mapping -Wuninitialized
 
 void foo() {
 }
@@ -99,7 +101,9 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target simd map(T) // expected-error {{'T' does not refer to a value}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target simd map(I) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
+// ge50-error@+2 2 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+1 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target simd map(I)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target simd map(S2::S2s)
   for (i = 0; i < argc; ++i) foo();
@@ -117,8 +121,10 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target simd map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
+// ge50-error@+3 2 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+2 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
 #pragma omp target simd map(tofrom \
-                            : argc > 0 ? x : y) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
+                            : argc > 0 ? x : y)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target simd map(argc)
   for (i = 0; i < argc; ++i) foo();
@@ -169,7 +175,10 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target simd map(always: x) // expected-error {{missing map type}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target simd map(tofrom, always: x) // expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{missing map type}}
+// ge51-error@+3 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+// lt51-error@+2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+// expected-error@+1 {{missing map type}}
+#pragma omp target simd map(tofrom, always: x)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target simd map(always, tofrom: always, tofrom, x)
   for (i = 0; i < argc; ++i) foo();
@@ -221,8 +230,10 @@ int main(int argc, char **argv) {
 #pragma omp target simd map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   for (i = 0; i < argc; ++i)
     foo();
+// ge50-error@+3 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+2 {{expected expression containing only member accesses and/or array sections based on named variables}}
 #pragma omp target simd map(tofrom \
-                            : argc > 0 ? argv[1] : argv[2]) // omp4-error {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error {{expected addressable lvalue in 'map' clause}}
+                            : argc > 0 ? argv[1] : argv[2])
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target simd map(argc)
   for (i = 0; i < argc; ++i) foo();
@@ -275,7 +286,10 @@ int main(int argc, char **argv) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target simd map(always: x) // expected-error {{missing map type}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target simd map(tofrom, always: x) // expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{missing map type}}
+// ge51-error@+3 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+// lt51-error@+2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+// expected-error@+1 {{missing map type}}
+#pragma omp target simd map(tofrom, always: x)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target simd map(always, tofrom: always, tofrom, x)
   for (i = 0; i < argc; ++i) foo();
diff --git a/clang/test/OpenMP/target_teams_distribute_map_messages.cpp b/clang/test/OpenMP/target_teams_distribute_map_messages.cpp
index 38855680c1fe5..30dd2fded1b33 100644
--- a/clang/test/OpenMP/target_teams_distribute_map_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_map_messages.cpp
@@ -1,8 +1,10 @@
-// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,lt50,lt51 -fopenmp -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51 -fopenmp -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,ge51 -fopenmp -fopenmp-version=51 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
-// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,lt50,lt51 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,ge51 -fopenmp-simd -fopenmp-version=51 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
 void foo() {
 }
@@ -105,7 +107,9 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute map(T) // expected-error {{'T' does not refer to a value}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute map(I) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
+// ge50-error@+2 2 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+1 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target teams distribute map(I)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute map(S2::S2s)
   for (i = 0; i < argc; ++i) foo();
@@ -123,8 +127,10 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
+// ge50-error@+3 2 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+2 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
 #pragma omp target teams distribute map(tofrom \
-                                        : argc > 0 ? x : y) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
+                                        : argc > 0 ? x : y)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute map(argc)
   for (i = 0; i < argc; ++i) foo();
@@ -175,7 +181,10 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute map(always: x) // expected-error {{missing map type}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute map(tofrom, always: x) // expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{missing map type}}
+// ge51-error@+3 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+// lt51-error@+2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+// expected-error@+1 {{missing map type}}
+#pragma omp target teams distribute map(tofrom, always: x)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute map(always, tofrom: always, tofrom, x)
   for (i = 0; i < argc; ++i) foo();
@@ -233,8 +242,10 @@ int main(int argc, char **argv) {
 #pragma omp target teams distribute map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   for (i = 0; i < argc; ++i)
     foo();
+// ge50-error@+3 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+2 {{expected expression containing only member accesses and/or array sections based on named variables}}
 #pragma omp target teams distribute map(tofrom \
-                                        : argc > 0 ? argv[1] : argv[2]) // omp4-error {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error {{expected addressable lvalue in 'map' clause}}
+                                        : argc > 0 ? argv[1] : argv[2])
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute map(argc)
   for (i = 0; i < argc; ++i) foo();
@@ -287,7 +298,10 @@ int main(int argc, char **argv) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute map(always: x) // expected-error {{missing map type}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute map(tofrom, always: x) // expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{missing map type}}
+// ge51-error@+3 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+// lt51-error@+2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+// expected-error@+1 {{missing map type}}
+#pragma omp target teams distribute map(tofrom, always: x)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute map(always, tofrom: always, tofrom, x)
   for (i = 0; i < argc; ++i) foo();
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_map_messages.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_map_messages.cpp
index a1af4600b59a4..d82d9f65cb5b0 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_map_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_map_messages.cpp
@@ -1,7 +1,8 @@
-// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,lt50,lt51 -fopenmp -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51 -fopenmp -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,ge51 -fopenmp -fopenmp-version=51 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
-// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,lt50,lt51 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
 void foo() {
 }
@@ -104,7 +105,9 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for map(T) // expected-error {{'T' does not refer to a value}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for map(I) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
+// ge50-error@+2 2 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+1 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target teams distribute parallel for map(I)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for map(S2::S2s)
   for (i = 0; i < argc; ++i) foo();
@@ -122,8 +125,10 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
+// ge50-error@+3 2 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+2 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
 #pragma omp target teams distribute parallel for map(tofrom \
-                                                     : argc > 0 ? x : y) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
+                                                     : argc > 0 ? x : y)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for map(argc)
   for (i = 0; i < argc; ++i) foo();
@@ -174,7 +179,10 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for map(always: x) // expected-error {{missing map type}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for map(tofrom, always: x) // expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{missing map type}}
+// ge51-error@+3 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+// lt51-error@+2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+// expected-error@+1 {{missing map type}}
+#pragma omp target teams distribute parallel for map(tofrom, always: x)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for map(always, tofrom: always, tofrom, x)
   for (i = 0; i < argc; ++i) foo();
@@ -231,8 +239,10 @@ int main(int argc, char **argv) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
+// ge50-error@+3 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+2 {{expected expression containing only member accesses and/or array sections based on named variables}}
 #pragma omp target teams distribute parallel for map(tofrom \
-                                                     : argc > 0 ? argv[1] : argv[2]) // omp4-error {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error {{expected addressable lvalue in 'map' clause}}
+                                                     : argc > 0 ? argv[1] : argv[2])
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for map(argc)
   for (i = 0; i < argc; ++i) foo();
@@ -285,7 +295,10 @@ int main(int argc, char **argv) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for map(always: x) // expected-error {{missing map type}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for map(tofrom, always: x) // expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{missing map type}}
+// ge51-error@+3 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+// lt51-error@+2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+// expected-error@+1 {{missing map type}}
+#pragma omp target teams distribute parallel for map(tofrom, always: x)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for map(always, tofrom: always, tofrom, x)
   for (i = 0; i < argc; ++i) foo();
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_map_messages.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_map_messages.cpp
index b53a506bf234b..713103872016f 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_map_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_map_messages.cpp
@@ -1,8 +1,10 @@
-// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,lt50,lt51 -fopenmp -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51 -fopenmp -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,ge51 -fopenmp -fopenmp-version=51 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
-// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,lt50,lt51 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,ge51 -fopenmp-simd -fopenmp-version=51 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
 void foo() {
 }
@@ -105,7 +107,9 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for simd map(T) // expected-error {{'T' does not refer to a value}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for simd map(I) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
+// ge50-error@+2 2 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+1 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target teams distribute parallel for simd map(I)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for simd map(S2::S2s)
   for (i = 0; i < argc; ++i) foo();
@@ -123,8 +127,10 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for simd map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
+// ge50-error@+3 2 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+2 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
 #pragma omp target teams distribute parallel for simd map(tofrom \
-                                                          : argc > 0 ? x : y) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
+                                                          : argc > 0 ? x : y)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for simd map(argc)
   for (i = 0; i < argc; ++i) foo();
@@ -175,7 +181,10 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for simd map(always: x) // expected-error {{missing map type}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for simd map(tofrom, always: x) // expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{missing map type}}
+// ge51-error@+3 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+// lt51-error@+2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+// expected-error@+1 {{missing map type}}
+#pragma omp target teams distribute parallel for simd map(tofrom, always: x)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for simd map(always, tofrom: always, tofrom, x)
   for (i = 0; i < argc; ++i) foo();
@@ -232,8 +241,10 @@ int main(int argc, char **argv) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for simd map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
+// ge50-error@+3 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+2 {{expected expression containing only member accesses and/or array sections based on named variables}}
 #pragma omp target teams distribute parallel for simd map(tofrom \
-                                                          : argc > 0 ? argv[1] : argv[2]) // omp4-error {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error {{expected addressable lvalue in 'map' clause}}
+                                                          : argc > 0 ? argv[1] : argv[2])
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for simd map(argc)
   for (i = 0; i < argc; ++i) foo();
@@ -286,7 +297,10 @@ int main(int argc, char **argv) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for simd map(always: x) // expected-error {{missing map type}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute parallel for simd map(tofrom, always: x) // expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{missing map type}}
+// ge51-error@+3 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+// lt51-error@+2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+// expected-error@+1 {{missing map type}}
+#pragma omp target teams distribute parallel for simd map(tofrom, always: x)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute parallel for simd map(always, tofrom: always, tofrom, x)
   for (i = 0; i < argc; ++i) foo();
diff --git a/clang/test/OpenMP/target_teams_distribute_simd_map_messages.cpp b/clang/test/OpenMP/target_teams_distribute_simd_map_messages.cpp
index 3bf612b7637f0..0c47134494daa 100644
--- a/clang/test/OpenMP/target_teams_distribute_simd_map_messages.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_simd_map_messages.cpp
@@ -1,8 +1,10 @@
-// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,lt50,lt51 -fopenmp -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51 -fopenmp -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,ge51 -fopenmp -fopenmp-version=51 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
-// RUN: %clang_cc1 -verify=expected,omp4 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,omp5 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,lt50,lt51 -fopenmp-simd -fopenmp-version=45 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,ge51 -fopenmp-simd -fopenmp-version=51 -ferror-limit 100 %s -Wno-openmp-mapping -Wuninitialized
 
 void foo() {
 }
@@ -105,7 +107,9 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute simd map(T) // expected-error {{'T' does not refer to a value}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute simd map(I) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
+// ge50-error@+2 2 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+1 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target teams distribute simd map(I)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute simd map(S2::S2s)
   for (i = 0; i < argc; ++i) foo();
@@ -123,8 +127,10 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute simd map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   for (i = 0; i < argc; ++i) foo();
+// ge50-error@+3 2 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+2 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
 #pragma omp target teams distribute simd map(tofrom \
-                                             : argc > 0 ? x : y) // omp4-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error 2 {{expected addressable lvalue in 'map' clause}}
+                                             : argc > 0 ? x : y)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute simd map(argc)
   for (i = 0; i < argc; ++i) foo();
@@ -175,7 +181,10 @@ T tmain(T argc) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute simd map(always: x) // expected-error {{missing map type}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute simd map(tofrom, always: x) // expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{missing map type}}
+// ge51-error@+3 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+// lt51-error@+2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+// expected-error@+1 {{missing map type}}
+#pragma omp target teams distribute simd map(tofrom, always: x)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute simd map(always, tofrom: always, tofrom, x)
   for (i = 0; i < argc; ++i) foo();
@@ -233,8 +242,10 @@ int main(int argc, char **argv) {
 #pragma omp target teams distribute simd map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
   for (i = 0; i < argc; ++i)
     foo();
+// ge50-error@+3 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+2 {{expected expression containing only member accesses and/or array sections based on named variables}}
 #pragma omp target teams distribute simd map(tofrom \
-                                             : argc > 0 ? argv[1] : argv[2]) // omp4-error {{expected expression containing only member accesses and/or array sections based on named variables}} omp5-error {{expected addressable lvalue in 'map' clause}}
+                                             : argc > 0 ? argv[1] : argv[2])
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute simd map(argc)
   for (i = 0; i < argc; ++i) foo();
@@ -287,7 +298,10 @@ int main(int argc, char **argv) {
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute simd map(always: x) // expected-error {{missing map type}}
   for (i = 0; i < argc; ++i) foo();
-#pragma omp target teams distribute simd map(tofrom, always: x) // expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{missing map type}}
+// ge51-error@+3 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+// lt51-error@+2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+// expected-error@+1 {{missing map type}}
+#pragma omp target teams distribute simd map(tofrom, always: x)
   for (i = 0; i < argc; ++i) foo();
 #pragma omp target teams distribute simd map(always, tofrom: always, tofrom, x)
   for (i = 0; i < argc; ++i) foo();
diff --git a/clang/test/OpenMP/target_teams_map_messages.cpp b/clang/test/OpenMP/target_teams_map_messages.cpp
index 7828099d3e41e..95a79c591d7d7 100644
--- a/clang/test/OpenMP/target_teams_map_messages.cpp
+++ b/clang/test/OpenMP/target_teams_map_messages.cpp
@@ -1,10 +1,11 @@
-// RUN: %clang_cc1 -verify=expected,le50 -fopenmp -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,le45 -fopenmp-version=40 -fopenmp -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,le45 -fopenmp-version=45 -fopenmp -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,le50 -fopenmp-version=50 -fopenmp -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized
-
-// RUN: %clang_cc1 -verify=expected,le50 -fopenmp-simd -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized
-// RUN: %clang_cc1 -DCCODE -verify=expected,le45 -fopenmp -ferror-limit 200 -x c %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge45,ge50,lt51 -fopenmp -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,lt45,lt50,lt51 -fopenmp-version=40 -fopenmp -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge45,lt50,lt51 -fopenmp-version=45 -fopenmp -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge45,ge50,lt51 -fopenmp-version=50 -fopenmp -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge45,ge50,ge51 -fopenmp-version=51 -fopenmp -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized
+
+// RUN: %clang_cc1 -verify=expected,ge45,ge50,lt51 -fopenmp-simd -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -DCCODE -verify=expected,ge45,ge50,lt51 -fopenmp -ferror-limit 200 -x c %s -Wno-openmp-mapping -Wuninitialized
 #ifdef CCODE
 void foo(int arg) {
   const int n = 0;
@@ -44,9 +45,12 @@ struct SA {
     {}
     #pragma omp target teams map(arg[2:2],a,d) // expected-error {{subscripted value is not an array or pointer}}
     {}
-    #pragma omp target teams map(arg,a*2) // le50-error {{expected addressable lvalue in 'map' clause}} le45-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+    // ge50-error@+2 {{expected addressable lvalue in 'map' clause}}
+    // lt50-error@+1 {{expected expression containing only member accesses and/or array sections based on named variables}}
+    #pragma omp target teams map(arg,a*2)
     {}
-    #pragma omp target teams map(arg,(c+1)[2]) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+    // lt50-error@+1 {{expected expression containing only member accesses and/or array sections based on named variables}}
+    #pragma omp target teams map(arg,(c+1)[2])
     {}
     #pragma omp target teams map(arg,a[:2],d) // expected-error {{subscripted value is not an array or pointer}}
     {}
@@ -252,7 +256,9 @@ void SAclient(int arg) {
   {}
   #pragma omp target teams map(r.S.Arr[:12])
   {}
-  #pragma omp target teams map(r.S.foo()[:12]) // le50-error {{expected addressable lvalue in 'map' clause}} le45-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+  // ge50-error@+2 {{expected addressable lvalue in 'map' clause}}
+  // lt50-error@+1 {{expected expression containing only member accesses and/or array sections based on named variables}}
+  #pragma omp target teams map(r.S.foo()[:12])
   {}
   #pragma omp target teams map(r.C, r.D)
   {}
@@ -286,7 +292,8 @@ void SAclient(int arg) {
   {}
   #pragma omp target teams map(r.S.Ptr[:])  // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
   {}
-  #pragma omp target teams map((p+1)->A) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+  // lt50-error@+1 {{expected expression containing only member accesses and/or array sections based on named variables}}
+  #pragma omp target teams map((p+1)->A)
   {}
   #pragma omp target teams map(u.B)  // expected-error {{mapping of union members is not allowed}}
   {}
@@ -416,7 +423,9 @@ T tmain(T argc) {
   foo();
 #pragma omp target teams map(T) // expected-error {{'T' does not refer to a value}}
   foo();
-#pragma omp target teams map(I) // le50-error 2 {{expected addressable lvalue in 'map' clause}} le45-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
+// ge50-error@+2 2 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+1 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target teams map(I)
   foo();
 #pragma omp target teams map(S2::S2s)
   foo();
@@ -434,7 +443,9 @@ T tmain(T argc) {
   foo();
 
 #pragma omp target data map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
-#pragma omp target data map(tofrom: argc > 0 ? x : y) // le50-error 2 {{expected addressable lvalue in 'map' clause}} le45-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
+// ge50-error@+2 2 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+1 2 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target data map(tofrom: argc > 0 ? x : y)
 
 #pragma omp target data map(argc)
 #pragma omp target data map(S1) // expected-error {{'S1' does not refer to a value}}
@@ -468,7 +479,10 @@ T tmain(T argc) {
 
 #pragma omp target data map(always, tofrom: x)
 #pragma omp target data map(always: x) // expected-error {{missing map type}}
-#pragma omp target data map(tofrom, always: x) // expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{missing map type}}
+// ge51-error@+3 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+// lt51-error@+2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+// expected-error@+1 {{missing map type}}
+#pragma omp target data map(tofrom, always: x)
 #pragma omp target data map(always, tofrom: always, tofrom, x)
 #pragma omp target teams map(tofrom j) // expected-error {{expected ',' or ')' in 'map' clause}}
   foo();
@@ -488,7 +502,10 @@ int main(int argc, char **argv) {
   int y;
   int to, tofrom, always;
   const int (&l)[5] = da;
-#pragma omp target data map // expected-error {{expected '(' after 'map'}} le45-error {{expected at least one 'map' or 'use_device_ptr' clause for '#pragma omp target data'}} le50-error {{expected at least one 'map', 'use_device_ptr', or 'use_device_addr' clause for '#pragma omp target data'}}
+// expected-error@+3 {{expected '(' after 'map'}}
+// ge50-error@+2 {{expected at least one 'map', 'use_device_ptr', or 'use_device_addr' clause for '#pragma omp target data'}}
+// lt50-error@+1 {{expected at least one 'map' or 'use_device_ptr' clause for '#pragma omp target data'}}
+#pragma omp target data map
 #pragma omp target data map( // expected-error {{expected ')'}} expected-note {{to match this '('}} expected-error {{expected expression}}
 #pragma omp target data map() // expected-error {{expected expression}}
 #pragma omp target data map(alloc) // expected-error {{use of undeclared identifier 'alloc'}}
@@ -509,7 +526,9 @@ int main(int argc, char **argv) {
   foo();
 
 #pragma omp target data map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
-#pragma omp target data map(tofrom: argc > 0 ? argv[1] : argv[2]) // le50-error {{expected addressable lvalue in 'map' clause}} le45-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+// ge50-error@+2 {{expected addressable lvalue in 'map' clause}}
+// lt50-error@+1 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target data map(tofrom: argc > 0 ? argv[1] : argv[2])
 #pragma omp target data map(argc)
 #pragma omp target data map(S1) // expected-error {{'S1' does not refer to a value}}
 #pragma omp target data map(a, b, c, d, f) // expected-error {{incomplete type 'S1' where a complete type is required}}
@@ -543,15 +562,22 @@ int main(int argc, char **argv) {
 
 #pragma omp target data map(always, tofrom: x)
 #pragma omp target data map(always: x) // expected-error {{missing map type}}
-#pragma omp target data map(tofrom, always: x) // expected-error {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}} expected-error {{missing map type}}
+// ge51-error@+3 {{incorrect map type modifier, expected 'always', 'close', 'mapper', or 'present'}}
+// lt51-error@+2 {{incorrect map type modifier, expected 'always', 'close', or 'mapper'}}
+// expected-error@+1 {{missing map type}}
+#pragma omp target data map(tofrom, always: x)
 #pragma omp target data map(always, tofrom: always, tofrom, x)
 #pragma omp target teams map(tofrom j) // expected-error {{expected ',' or ')' in 'map' clause}}
   foo();
 
-#pragma omp target teams private(j) map(j) // le45-error {{private variable cannot be in a map clause in '#pragma omp target teams' directive}}  le45-note {{defined as private}}
+// lt50-error@+2 {{private variable cannot be in a map clause in '#pragma omp target teams' directive}}
+// lt50-note@+1 {{defined as private}}
+#pragma omp target teams private(j) map(j)
   {}
 
-#pragma omp target teams firstprivate(j) map(j) // le45-error {{firstprivate variable cannot be in a map clause in '#pragma omp target teams' directive}} le45-note {{defined as firstprivate}}
+// lt50-error@+2 {{firstprivate variable cannot be in a map clause in '#pragma omp target teams' directive}}
+// lt50-note@+1 {{defined as firstprivate}}
+#pragma omp target teams firstprivate(j) map(j)
   {}
 
 #pragma omp target teams map(m)
@@ -559,21 +585,29 @@ int main(int argc, char **argv) {
 
   int **BB, *offset, *a;
 
-#pragma omp target teams map(**(BB+*offset)) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+// lt50-error@+1 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target teams map(**(BB+*offset))
   {}
-#pragma omp target teams map(**(BB+y)) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+// lt50-error@+1 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target teams map(**(BB+y))
   {}
-#pragma omp target teams map(*(a+*offset)) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+// lt50-error@+1 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target teams map(*(a+*offset))
   {}
-#pragma omp target teams map(**(*offset+BB)) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+// lt50-error@+1 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target teams map(**(*offset+BB))
   {}
-#pragma omp target teams map(**(y+BB)) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+// lt50-error@+1 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target teams map(**(y+BB))
   {}
-#pragma omp target teams map(*(*offset+a)) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+// lt50-error@+1 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target teams map(*(*offset+a))
   {}
-#pragma omp target teams map(**(*offset+BB+*a)) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+// lt50-error@+1 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target teams map(**(*offset+BB+*a))
   {}
-#pragma omp target teams map(**(*(*(&offset))+BB+*a)) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+// lt50-error@+1 {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target teams map(**(*(*(&offset))+BB+*a))
   {}
 #pragma omp target teams map(*(a+(a))) // expected-error {{invalid operands to binary expression ('int *' and 'int *')}}
   {}
diff --git a/clang/test/OpenMP/target_update_codegen.cpp b/clang/test/OpenMP/target_update_codegen.cpp
index 5d569f18ce6b6..637b580d754f2 100644
--- a/clang/test/OpenMP/target_update_codegen.cpp
+++ b/clang/test/OpenMP/target_update_codegen.cpp
@@ -310,23 +310,22 @@ void device_side_scan(int arg) {
 
 #ifdef CK5
 
-// CK5: [[SIZE00:@.+]] = {{.+}}constant [1 x i[[sz:64|32]]] [i{{64|32}} 4]
-// CK5: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 33]
+// CK5: [[SIZE00:@.+]] = {{.+}}constant [2 x i[[sz:64|32]]] [i{{64|32}} {{8|4}}, i{{64|32}} 4]
+// CK5: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
 
 // CK5-LABEL: lvalue
 void lvalue(int *B, int l, int e) {
 
-  // CK5-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK5-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK5-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK5-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK5-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
-  // CK5-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
-  // CK5-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32**
+  // CK5-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK5-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK5-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32***
   // CK5-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to i32**
-  // CK5-DAG: store i32* [[B_VAL:%.+]], i32** [[BPC0]]
+  // CK5-DAG: store i32** [[B_ADDR:%.+]], i32*** [[BPC0]]
   // CK5-DAG: store i32* [[B_VAL_2:%.+]], i32** [[PC0]]
-  // CK5-DAG: [[B_VAL]] = load i32*, i32** [[B_ADDR:%.+]]
   // CK5-DAG: [[B_VAL_2]] = load i32*, i32** [[B_ADDR]]
   #pragma omp target update to(*B)
   *B += e;
@@ -352,27 +351,28 @@ void lvalue(int *B, int l, int e) {
 
 #ifdef CK6
 
-// CK6: [[SIZE00:@.+]] = {{.+}}constant [1 x i[[sz:64|32]]] [i{{64|32}} 4]
-// CK6: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 33]
+// CK6: [[SIZE00:@.+]] = {{.+}}constant [2 x i[[sz:64|32]]] [i{{64|32}} {{8|4}}, i{{64|32}} 4]
+// CK6: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
 
 // CK6-LABEL: lvalue
 void lvalue(int *B, int l, int e) {
 
-  // CK6-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK6-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK6-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK6-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK6-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
-  // CK6-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
-  // CK6-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32**
+  // CK6-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK6-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK6-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32***
   // CK6-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to i32**
-  // CK6-DAG: store i32* [[ZERO:%.+]], i32** [[BPC0]]
+  // CK6-DAG: store i32** [[B_ADDR:%.+]], i32*** [[BPC0]]
   // CK6-DAG: store i32* [[ADD_PTR:%.+]], i32** [[PC0]]
   // CK6-64-DAG: [[ADD_PTR]] = getelementptr inbounds i32, i32* [[ONE:%.+]], i{{32|64}} [[IDX_EXT:%.+]]
   // CK6-32-DAG: [[ADD_PTR]] = getelementptr inbounds i32, i32* [[ONE:%.+]], i{{32|64}} [[L_VAL:%.+]]
   // CK6-64-DAG: [[IDX_EXT]] = sext i32 [[L_VAL:%.+]] to i64
   // CK6-DAG: [[L_VAL]] = load i32, i32* [[L_ADDR:%.+]]
   // CK6-DAG: store i32 {{.+}}, i32* [[L_ADDR]]
+  // CK6-DAG: [[ONE]] = load i32*, i32** [[B_ADDR]]
   #pragma omp target update to(*(B+l))
   *(B+l) += e;
   #pragma omp target update from(*(B+l))
@@ -397,23 +397,22 @@ void lvalue(int *B, int l, int e) {
 
 #ifdef CK7
 
-// CK7: [[SIZE00:@.+]] = {{.+}}constant [1 x i[[sz:64|32]]] [i{{64|32}} 4]
-// CK7: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 33]
+// CK7: [[SIZE00:@.+]] = {{.+}}constant [2 x i[[sz:64|32]]] [i{{64|32}} {{8|4}}, i{{64|32}} 4]
+// CK7: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
 
 // CK7-LABEL: lvalue
 void lvalue(int *B, int l, int e) {
 
-  // CK7-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK7-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK7-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK7-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK7-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
-  // CK7-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
-  // CK7-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32**
+  // CK7-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK7-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK7-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32***
   // CK7-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to i32**
-  // CK7-DAG: store i32* [[B_VAL:%.+]], i32** [[BPC0]]
+  // CK7-DAG: store i32** [[B_ADDR:%.+]], i32*** [[BPC0]]
   // CK7-DAG: store i32* [[ARRAY_IDX:%.+]], i32** [[PC0]]
-  // CK7-DAG: [[B_VAL]] = load i32*, i32** [[B_ADDR:%.+]]
   // CK7-DAG: [[ARRAY_IDX]] = getelementptr inbounds i32, i32* [[ADD_PTR:%.+]], i{{32|64}} [[IDX_PROM:%.+]]
   // CK7-64-DAG: [[ADD_PTR]] = getelementptr inbounds i32, i32* [[ONE:%.+]], i64 [[IDX_EXT:%.+]]
   // CK7-32-DAG: [[ADD_PTR]] = getelementptr inbounds i32, i32* [[B_VAL_2:%.+]], i32 [[L_VAL:%.+]]
@@ -447,19 +446,18 @@ void lvalue(int *B, int l, int e) {
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 #ifdef CK8
-// CK8-64: [[SIZE00:@.+]] = {{.+}}constant [2 x i[[sz:64|32]]] [i{{64|32}} 8, i64 4]
-// CK8-32: [[SIZE00:@.+]] = {{.+}}constant [2 x i[[sz:64|32]]] [i{{64|32}} 4, i64 4]
-// CK8: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
+// CK8: [[SIZE00:@.+]] = {{.+}}constant [3 x i[[sz:64|32]]] [i{{64|32}} {{8|4}}, i{{64|32}} {{8|4}}, i{{64|32}} 4]
+// CK8: [[MTYPE00:@.+]] = {{.+}}constant [3 x i64] [i64 33, i64 16, i64 17]
 
 // CK8-LABEL: lvalue
 void lvalue(int **B, int l, int e) {
 
-  // CK8-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}], [2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}, [2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK8-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 3, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}], [3 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}, [3 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK8-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK8-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK8-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK8-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK8-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
+  // CK8-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
   // CK8-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32***
   // CK8-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to i32**
   // CK8-DAG: store i32** [[ARRAY_IDX_1:%.+]], i32*** [[BPC0]]
@@ -503,20 +501,19 @@ struct S {
   double *p;
 };
 
-// CK9: [[SIZE00:@.+]] = {{.+}}constant [2 x i[[sz:64|32]]] [i{{64|32}} 32, i64 281474976710673]
-// CK9: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 32, i64 281474976710674]
+// CK9: [[MTYPE00:@.+]] = {{.+}}constant [3 x i64] [i64 32, i64 281474976710657, i64 281474976710673]
 
 // CK9-LABEL: lvalue
 void lvalue(struct S *s, int l, int e) {
 
-  // CK9-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}, [2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK9-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 3, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}, [3 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK9-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK9-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK9-DAG: [[GSIZE]] = getelementptr inbounds {{.+}}[[SIZE:%[^,]+]]
   //
-  // CK9-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK9-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK9-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 1
+  // CK9-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
+  // CK9-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
+  // CK9-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 2
   // CK9-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to double***
   // CK9-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to double**
   // CK9-DAG: store double** [[P:%.+]], double*** [[BPC0]]
@@ -554,19 +551,19 @@ struct S {
   double *p;
 };
 
-// CK10: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 32, i64 281474976710674]
+// CK10: [[MTYPE00:@.+]] = {{.+}}constant [3 x i64] [i64 32, i64 281474976710657, i64 281474976710673]
 
 // CK10-LABEL: lvalue
 void lvalue(struct S *s, int l, int e) {
 
-  // CK10-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}, [2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK10-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 3, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}, [3 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK10-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK10-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK10-DAG: [[GSIZE]] = getelementptr inbounds {{.+}}[[SIZE:%[^,]+]]
   //
-  // CK10-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK10-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK10-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 1
+  // CK10-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
+  // CK10-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
+  // CK10-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 2
   // CK10-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to double***
   // CK10-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to double**
   // CK10-DAG: store double** [[P_VAL:%.+]], double*** [[BPC0]]
@@ -604,19 +601,19 @@ void lvalue(struct S *s, int l, int e) {
 struct S {
   double *p;
 };
-// CK11: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 32, i64 281474976710674]
+// CK11: [[MTYPE00:@.+]] = {{.+}}constant [3 x i64] [i64 32, i64 281474976710657, i64 281474976710673]
 
 // CK11-LABEL: lvalue
 void lvalue(struct S *s, int l, int e) {
 
-  // CK11-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}, [2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK11-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 3, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}, [3 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK11-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK11-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK11-DAG: [[GSIZE]] = getelementptr inbounds {{.+}}[[SIZE:%[^,]+]]
   //
-  // CK11-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK11-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK11-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 1
+  // CK11-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
+  // CK11-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
+  // CK11-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 2
   // CK11-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to double***
   // CK11-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to double**
   // CK11-DAG: store double** [[P:%.+]], double*** [[BPC0]]
@@ -656,40 +653,41 @@ struct S {
   double *p;
   struct S *sp;
 };
-// CK12: [[MTYPE00:@.+]] = {{.+}}constant [3 x i64] [i64 32, i64 281474976710672, i64 17]
+// CK12: [[MTYPE00:@.+]] = {{.+}}constant [4 x i64] [i64 32, i64 281474976710657, i64 281474976710672, i64 17]
 
 // CK12-LABEL: lvalue
 void lvalue(struct S *s, int l, int e) {
 
-  // CK12-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 3, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}, [3 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK12-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 4, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i{{.+}} [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[4 x i{{.+}}, [4 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK12-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK12-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK12-DAG: [[GSIZE]] = getelementptr inbounds {{.+}}[[SIZE:%[^,]+]]
   //
-  // CK12-DAG: [[BP2:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
-  // CK12-DAG: [[P2:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
-  // CK12-DAG: [[SIZE2:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 2
+  // CK12-DAG: [[BP2:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 3
+  // CK12-DAG: [[P2:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 3
+  // CK12-DAG: [[SIZE2:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 3
   // CK12-DAG: [[BPC2:%.+]] = bitcast i8** [[BP2]] to double***
   // CK12-DAG: [[PC2:%.+]] = bitcast i8** [[P2]] to double**
   // CK12-DAG: store double** [[P_VAL:%.+]], double*** [[BPC2]]
   // CK12-DAG: store double* [[SIX:%.+]], double** [[PC2]]
   // CK12-DAG: store i{{.+}} 8, i{{.+}}* [[SIZE2]]
-  // CK12-DAG: [[BP1:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK12-DAG: [[P1:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK12-DAG: [[SIZE1:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 1
+  // CK12-DAG: [[BP1:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
+  // CK12-DAG: [[P1:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
+  // CK12-DAG: [[SIZE1:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 2
   // CK12-DAG: [[BPC1:%.+]] = bitcast i8** [[BP1]] to [[STRUCT_S:%.+]]***
   // CK12-DAG: [[PC1:%.+]] = bitcast i8** [[P1]] to double***
   // CK12-DAG: store [[STRUCT_S]]** [[SP:%.+]], [[STRUCT_S]]*** [[BPC1]]
   // CK12-DAG: store double** [[P_VAL:%.+]], double*** [[PC1]]
   // CK12-DAG: store i{{.+}} {{4|8}}, i{{.+}}* [[SIZE1]]
-  // CK12-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
-  // CK12-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
-  // CK12-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 0
+  // CK12-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK12-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK12-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 1
   // CK12-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to [[STRUCT_S:%.+]]**
   // CK12-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to [[STRUCT_S]]***
-  // CK12-DAG: store [[STRUCT_S]]** [[SP:%.+]], [[STRUCT_S]]*** [[S_VAL:%.+]]
+  // CK12-DAG: store [[STRUCT_S]]** [[S:%.+]], [[STRUCT_S]]*** [[S_VAL:%.+]]
   // CK12-DAG: store i{{.+}} {{.+}}, i{{.+}}* [[SIZE0]]
   // CK12-DAG: [[SP]] = getelementptr inbounds [[STRUCT_S]], [[STRUCT_S]]* [[ONE:%.+]], i32 0, i32 1
+  // CK12-DAG: [[ONE]] = load %struct.S*, %struct.S** [[S]],
   #pragma omp target update to(*(s->sp->p))
     *(s->sp->p) = e;
   #pragma omp target update from(*(s->sp->p))
@@ -713,23 +711,22 @@ void lvalue(struct S *s, int l, int e) {
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK13
 
-// CK13: [[SIZE00:@.+]] = {{.+}}constant [1 x i64] [i64 4]
-// CK13: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 33]
+// CK13: [[SIZE00:@.+]] = {{.+}}constant [2 x i64] [i64 {{8|4}}, i64 4]
+// CK13: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
 
 // CK13-LABEL: lvalue
 void lvalue(int **BB, int a, int b) {
 
-  // CK13-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK13-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK13-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK13-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK13-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
-  // CK13-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
-  // CK13-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32***
+  // CK13-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK13-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK13-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to i32****
   // CK13-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to i32**
-  // CK13-DAG: store i32** [[ZERO:%.+]], i32*** [[BPC0]]
+  // CK13-DAG: store i32*** [[BB_ADDR:%.+]], i32**** [[BPC0]]
   // CK13-DAG: store i32* [[ADD_PTR_2:%.+]], i32** [[PC0]]
-  // CK13-DAG: [[ZERO]] = load i32**, i32*** [[BB_ADDR:%.+]]
   // CK13-64-DAG: [[ADD_PTR_2]] = getelementptr inbounds i32, i32* [[RESULT:%.+]], i64 [[IDX_EXT_1:%.+]]
   // CK13-32-DAG: [[ADD_PTR_2]] = getelementptr inbounds i32, i32* [[RESULT:%.+]], i32 [[B_ADDR:%.+]]
   // CK13-64-DAG: [[IDX_EXT_1]] = sext i32 [[B_ADDR:%.+]]
@@ -834,7 +831,7 @@ void lvalue_member(SSA *sap) {
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK15
 
-// CK15: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 32, i64 281474976710673]
+// CK15: [[MTYPE00:@.+]] = {{.+}}constant [3 x i64] [i64 32, i64 281474976710657, i64 281474976710673]
 
 struct SSA {
   double *p;
@@ -845,31 +842,39 @@ struct SSA {
 //CK-15-LABEL: lvalue_member
 void lvalue_member(SSA *sap) {
 
-  // CK15-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK15-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 3, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GSIZE:%.+]], {{.+}}getelementptr {{.+}}[3 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK15-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK15-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK15-DAG: [[GSIZE]] = getelementptr inbounds {{.+}}[[SIZE:%[^,]+]]
 
-  // CK15-DAG: [[BP1:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
-  // CK15-DAG: [[P1:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
-  // CK15-DAG: [[SIZE1:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 1
-  // CK15-DAG: [[BPC1:%.+]] = bitcast i8** [[BP1]] to double***
-  // CK15-DAG: [[PC1:%.+]] = bitcast i8** [[P1]] to double**
-  // CK15-DAG: store double** [[P_VAL:%.+]], double*** [[BPC1]]
-  // CK15-DAG: store double* [[ADD_PTR:%.+]], double** [[PC1]]
-  // CK15-DAG: store i64 8, i64* [[SIZE1]]
+  // CK15-DAG: [[BP2:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 2
+  // CK15-DAG: [[P2:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 2
+  // CK15-DAG: [[SIZE2:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 2
+  // CK15-DAG: [[BPC2:%.+]] = bitcast i8** [[BP2]] to double***
+  // CK15-DAG: [[PC2:%.+]] = bitcast i8** [[P2]] to double**
+  // CK15-DAG: store double** [[P_VAL:%.+]], double*** [[BPC2]]
+  // CK15-DAG: store double* [[ADD_PTR:%.+]], double** [[PC2]]
+  // CK15-DAG: store i64 8, i64* [[SIZE2]]
   // CK15-DAG: [[ADD_PTR]] = getelementptr inbounds double, double* [[THREE:%.+]], i{{.+}} 3
   // CK15-DAG: [[THREE]] = load double*, double** [[P_VAL_1:%.+]]
   // CK15-DAG: [[P_VAL]] = getelementptr inbounds [[SSA:%.+]], [[SSA:%.+]]* [[THIS:%.+]], i32 0, i32 0
+  // CK15-DAG: [[BP1:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK15-DAG: [[P1:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK15-DAG: [[SIZE1:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 1
+  // CK15-DAG: [[BPC1:%.+]] = bitcast i8** [[BP1]] to [[SSA]]**
+  // CK15-DAG: [[PC1:%.+]] = bitcast i8** [[P1]] to [[SSA]]***
+  // CK15-DAG: store [[SSA]]* [[SAP_VAL:%.+]], [[SSA]]** [[BPC1]],
+  // CK15-DAG: store [[SSA]]** [[SAP_ADDR:%.+]], [[SSA]]*** [[PC1]]
+  // CK15-DAG: store i{{.+}} {{8|4}}, i{{.+}}* [[SIZE1]]
+  // CK15-DAG: [[SAP_VAL]] = load [[SSA]]*, [[SSA]]** [[SAP_ADDR]],
   // CK15-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
   // CK15-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
   // CK15-DAG: [[SIZE0:%.+]] = getelementptr inbounds {{.+}}[[SIZE]], i{{.+}} 0, i{{.+}} 0
-  // CK15-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to [[SSA]]**
+  // CK15-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to [[SSA]]***
+  // CK15-DAG: store [[SSA]]** [[SAP_ADDR]], [[SSA]]*** [[BPC0]],
   // CK15-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to double***
-  // CK15-DAG: store [[SSA]]* [[SAP_VAL:%.+]], [[SSA]]** [[BPC0]],
-  // CK15-DAG: store double** [[P_VAL]], double*** [[PC0]]
+  // CK15-DAG: store double** [[P_VAL]], double*** [[PC0]],
   // CK15-DAG: store i{{.+}} [[COMPUTE_SIZE:%.+]], i{{.+}}* [[SIZE0]]
-  // CK15-DAG: [[SAP_VAL]] = load [[SSA]]*, [[SSA]]** [[SAP_ADDR:%.+]],
   // CK15-DAG: [[COMPUTE_SIZE]] = sdiv exact i64 [[NINE:%.+]], ptrtoint (i8* getelementptr (i8, i8* null, i32 1) to i64)
   // CK15-DAG: [[NINE]] = sub i{{.+}} [[SEVEN:%.+]], [[EIGHT:%.+]]
   // CK15-DAG: [[SEVEN]] = ptrtoint i8* [[SIX:%.+]] to i64
@@ -899,27 +904,26 @@ void lvalue_member(SSA *sap) {
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK16
 
-// CK16: [[SIZE00:@.+]] = {{.+}}constant [1 x i64] [i64 4]
-// CK16: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 33]
+// CK16: [[SIZE00:@.+]] = {{.+}}constant [2 x i64] [i64 {{8|4}}, i64 4]
+// CK16: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
 
 //CK16-LABEL: lvalue_find_base
 void lvalue_find_base(float *f, int *i) {
 
-  // CK16-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK16-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK16-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK16-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK16-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
-  // CK16-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
-  // CK16-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float**
+  // CK16-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK16-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK16-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float***
   // CK16-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float**
-  // CK16-DAG: store float* [[ZERO:%.+]], float** [[BPC0]]
+  // CK16-DAG: store float** [[F_ADDR:%.+]], float*** [[BPC0]]
   // CK16-DAG: store float* [[ADD_PTR:%.+]], float** [[PC0]]
   // CK16-32-DAG: [[ADD_PTR]] = getelementptr inbounds float, float* [[THREE:%.+]], i32 [[I:%.+]]
   // CK16-64-DAG: [[ADD_PTR]] = getelementptr inbounds float, float* [[THREE:%.+]], i64 [[IDX_EXT:%.+]]
-  // CK16-DAG: [[THREE]] = load float*, float** [[F_ADDR:%.+]],
+  // CK16-DAG: [[THREE]] = load float*, float** [[F_ADDR]],
   // CK16-64-DAG: [[IDX_EXT]] = sext i32 [[I:%.+]] to i64
-  // CK16-DAG: [[ZERO]] = load float*, float** [[F_ADDR:%.+]]
 
   #pragma omp target update to(*(*i+f))
   *(*i+f) = 1.0;
@@ -944,8 +948,8 @@ void lvalue_find_base(float *f, int *i) {
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK17
 
-// CK17: [[SIZE00:@.+]] = {{.+}}constant [1 x i64] [i64 4]
-// CK17: [[MTYPE00:@.+]] = {{.+}}constant [1 x i64] [i64 33]
+// CK17: [[SIZE00:@.+]] = {{.+}}constant [2 x i64] [i64 {{4|8}}, i64 4]
+// CK17: [[MTYPE00:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 17]
 
 struct SSA {
   int i;
@@ -955,17 +959,16 @@ struct SSA {
 //CK17-LABEL: lvalue_find_base
 void lvalue_find_base(float **f, SSA *sa) {
 
-  // CK17-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
+  // CK17-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[SIZE00]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE00]]{{.+}}, i8** null)
   // CK17-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK17-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
 
-  // CK17-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
-  // CK17-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
-  // CK17-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float***
+  // CK17-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK17-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK17-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float****
   // CK17-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float**
-  // CK17-DAG: store float** [[ZERO:%.+]], float*** [[BPC0]],
+  // CK17-DAG: store float*** [[F_ADDR:%.+]], float**** [[BPC0]],
   // CK17-DAG: store float* [[ADD_PTR_4:%.+]], float** [[PC0]],
-  // CK17-DAG: [[ZERO]] = load float**, float*** [[F_ADDR:%.+]],
   // CK17-64-DAG: [[ADD_PTR_4]] = getelementptr inbounds float, float* [[SEVEN:%.+]], i64 [[IDX_EXT_3:%.+]]
   // CK17-64-DAG: [[IDX_EXT_3]] = sext i32 [[I_VAL:%.+]] to i64
   // CK17-32-DAG: [[ADD_PTR_4]] = getelementptr inbounds float, float* [[SEVEN:%.+]], i32 [[I_VAL:%.+]]
@@ -1002,13 +1005,13 @@ void lvalue_find_base(float **f, SSA *sa) {
 // SIMD-ONLY18-NOT: {{__kmpc|__tgt}}
 #ifdef CK18
 
-// CK18-DAG: [[MTYPE_TO:@.+]] = {{.+}}constant [1 x i64] [i64 33]
-// CK18-DAG: [[MTYPE_FROM:@.+]] = {{.+}}constant [1 x i64] [i64 34]
+// CK18-DAG: [[MTYPE_TO:@.+]] = {{.+}}constant [2 x i64] [i64 33, i64 16]
+// CK18-DAG: [[MTYPE_FROM:@.+]] = {{.+}}constant [2 x i64] [i64 34, i64 16]
 
 //CK18-LABEL: array_shaping
 void array_shaping(float *f, int sa) {
 
-  // CK18-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE_TO]]{{.+}}, i8** null)
+  // CK18-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE_TO]]{{.+}}, i8** null)
   // CK18-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK18-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK18-DAG: [[GEPS]] = getelementptr inbounds {{.+}}[[S:%[^,]+]]
@@ -1018,14 +1021,25 @@ void array_shaping(float *f, int sa) {
   // CK18-DAG: [[S0:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 0
 
   // CK18-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float**
-  // CK18-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float**
+  // CK18-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float***
 
   // CK18-DAG: store float* [[F1:%.+]], float** [[BPC0]],
-  // CK18-DAG: store float* [[F2:%.+]], float** [[PC0]],
-  // CK18-DAG: store i64 [[SIZE:%.+]], i64* [[S0]],
+  // CK18-DAG: store float** [[F_ADDR:%.+]], float*** [[PC0]],
+  // CK18-DAG: store i64 {{8|4}}, i64* [[S0]],
+  // CK18-DAG: [[F1]] = load float*, float** [[F_ADDR]],
 
-  // CK18-DAG: [[F1]] = load float*, float** [[F_ADDR:%.+]],
+  // CK18-DAG: [[BP1:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK18-DAG: [[P1:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK18-DAG: [[S1:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 1
+
+  // CK18-DAG: [[BPC1:%.+]] = bitcast i8** [[BP1]] to float***
+  // CK18-DAG: [[PC1:%.+]] = bitcast i8** [[P1]] to float**
+
+  // CK18-DAG: store float** [[F_ADDR]], float*** [[BPC1]],
+  // CK18-DAG: store float* [[F2:%.+]], float** [[PC1]],
+  // CK18-DAG: store i64 [[SIZE:%.+]], i64* [[S1]],
   // CK18-DAG: [[F2]] = load float*, float** [[F_ADDR]],
+
   // CK18-64-DAG: [[SIZE]] = mul nuw i64 [[SZ1:%.+]], 4
   // CK18-64-DAG: [[SZ1]] = mul nuw i64 12, %{{.+}}
   // CK18-32-DAG: [[SIZE]] = sext i32 [[SZ1:%.+]] to i64
@@ -1033,7 +1047,7 @@ void array_shaping(float *f, int sa) {
   // CK18-32-DAG: [[SZ2]] = mul nuw i32 12, %{{.+}}
   #pragma omp target update to(([3][sa][4])f)
   sa = 1;
-  // CK18-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE_FROM]]{{.+}}, i8** null)
+  // CK18-DAG: call void @__tgt_target_data_update_mapper(i64 -1, i32 2, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[2 x i{{.+}}]* [[MTYPE_FROM]]{{.+}}, i8** null)
   // CK18-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
   // CK18-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
   // CK18-DAG: [[GEPS]] = getelementptr inbounds {{.+}}[[S:%[^,]+]]
@@ -1043,14 +1057,25 @@ void array_shaping(float *f, int sa) {
   // CK18-DAG: [[S0:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 0
 
   // CK18-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float**
-  // CK18-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float**
+  // CK18-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float***
 
   // CK18-DAG: store float* [[F1:%.+]], float** [[BPC0]],
-  // CK18-DAG: store float* [[F2:%.+]], float** [[PC0]],
-  // CK18-DAG: store i64 [[SIZE:%.+]], i64* [[S0]],
+  // CK18-DAG: store float** [[F_ADDR:%.+]], float*** [[PC0]],
+  // CK18-DAG: store i64 {{8|4}}, i64* [[S0]],
+  // CK18-DAG: [[F1]] = load float*, float** [[F_ADDR]],
 
-  // CK18-DAG: [[F1]] = load float*, float** [[F_ADDR:%.+]],
+  // CK18-DAG: [[BP1:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 1
+  // CK18-DAG: [[P1:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 1
+  // CK18-DAG: [[S1:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 1
+
+  // CK18-DAG: [[BPC1:%.+]] = bitcast i8** [[BP1]] to float***
+  // CK18-DAG: [[PC1:%.+]] = bitcast i8** [[P1]] to float**
+
+  // CK18-DAG: store float** [[F_ADDR]], float*** [[BPC1]],
+  // CK18-DAG: store float* [[F2:%.+]], float** [[PC1]],
+  // CK18-DAG: store i64 [[SIZE:%.+]], i64* [[S1]],
   // CK18-DAG: [[F2]] = load float*, float** [[F_ADDR]],
+
   // CK18-64-DAG: [[SIZE]] = mul nuw i64 [[SZ1:%.+]], 5
   // CK18-64-DAG: [[SZ1]] = mul nuw i64 4, %{{.+}}
   // CK18-32-DAG: [[SIZE]] = sext i32 [[SZ1:%.+]] to i64
diff --git a/clang/test/PCH/codegen.cpp b/clang/test/PCH/codegen.cpp
index 49ed30aeaf054..9d817316bc8ae 100644
--- a/clang/test/PCH/codegen.cpp
+++ b/clang/test/PCH/codegen.cpp
@@ -9,8 +9,8 @@
 // RUN: %clang_cc1 -triple=x86_64-linux-gnu -fmodules-codegen -x c++-header -building-pch-with-obj -emit-pch %S/../Modules/Inputs/codegen-flags/foo.h -o %t/foo-cg.pch
 // RUN: %clang_cc1 -triple=x86_64-linux-gnu -fmodules-debuginfo -x c++-header -building-pch-with-obj -emit-pch %S/../Modules/Inputs/codegen-flags/foo.h -o %t/foo-di.pch
 
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -debug-info-kind=limited -o - %s -include-pch %t/foo-cg.pch -building-pch-with-obj -fmodules-codegen | FileCheck --check-prefix=CG %s
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -debug-info-kind=limited -o - %s -include-pch %t/foo-di.pch -building-pch-with-obj -fmodules-debuginfo | FileCheck --check-prefix=DI %s
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -debug-info-kind=limited -o - -x precompiled-header %t/foo-cg.pch | FileCheck --check-prefix=CG %s
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -debug-info-kind=limited -o - -x precompiled-header %t/foo-di.pch | FileCheck --check-prefix=DI %s
 
 // RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -debug-info-kind=limited -o - -include-pch %t/foo-cg.pch %S/../Modules/Inputs/codegen-flags/use.cpp | FileCheck --check-prefix=CG-USE %s
 // RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -debug-info-kind=limited -o - -include-pch %t/foo-di.pch %S/../Modules/Inputs/codegen-flags/use.cpp | FileCheck --check-prefix=DI-USE %s
@@ -20,8 +20,8 @@
 // RUN: %clang_cc1 -triple=x86_64-linux-gnu -fmodules-codegen -x c++-header -building-pch-with-obj -emit-pch -fpch-instantiate-templates %S/../Modules/Inputs/codegen-flags/foo.h -o %t/foo-cg.pch
 // RUN: %clang_cc1 -triple=x86_64-linux-gnu -fmodules-debuginfo -x c++-header -building-pch-with-obj -emit-pch -fpch-instantiate-templates %S/../Modules/Inputs/codegen-flags/foo.h -o %t/foo-di.pch
 
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -debug-info-kind=limited -o - %s -include-pch %t/foo-cg.pch -building-pch-with-obj -fmodules-codegen | FileCheck --check-prefix=CG %s
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -debug-info-kind=limited -o - %s -include-pch %t/foo-di.pch -building-pch-with-obj -fmodules-debuginfo | FileCheck --check-prefix=DI %s
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -debug-info-kind=limited -o - -x precompiled-header %t/foo-cg.pch | FileCheck --check-prefix=CG %s
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -debug-info-kind=limited -o - -x precompiled-header %t/foo-di.pch | FileCheck --check-prefix=DI %s
 
 // RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -debug-info-kind=limited -o - -include-pch %t/foo-cg.pch %S/../Modules/Inputs/codegen-flags/use.cpp | FileCheck --check-prefix=CG-USE %s
 // RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -debug-info-kind=limited -o - -include-pch %t/foo-di.pch %S/../Modules/Inputs/codegen-flags/use.cpp | FileCheck --check-prefix=DI-USE %s
diff --git a/clang/test/Profile/Inputs/gcc-flag-compatibility_IR_entry.proftext b/clang/test/Profile/Inputs/gcc-flag-compatibility_IR_entry.proftext
new file mode 100644
index 0000000000000..4903f56c419e0
--- /dev/null
+++ b/clang/test/Profile/Inputs/gcc-flag-compatibility_IR_entry.proftext
@@ -0,0 +1,12 @@
+# IR level Instrumentation Flag
+:ir
+:entry_first
+main
+# Func Hash:
+34137660316
+# Num Counters:
+2
+# Counter Values:
+1
+100
+
diff --git a/clang/test/Profile/gcc-flag-compatibility.c b/clang/test/Profile/gcc-flag-compatibility.c
index e3594b0ebc71c..743b7ff555554 100644
--- a/clang/test/Profile/gcc-flag-compatibility.c
+++ b/clang/test/Profile/gcc-flag-compatibility.c
@@ -49,6 +49,10 @@
 // RUN: llvm-profdata merge %S/Inputs/gcc-flag-compatibility_IR.proftext -o %t.dir/some/path/file.prof
 // RUN: %clang %s -o - -emit-llvm -S -fprofile-use=%t.dir/some/path/file.prof -fno-experimental-new-pass-manager | FileCheck -check-prefix=PROFILE-USE-IR %s
 // RUN: %clang %s -o - -emit-llvm -S -fprofile-use=%t.dir/some/path/file.prof -fexperimental-new-pass-manager | FileCheck -check-prefix=PROFILE-USE-IR %s
+//
+// RUN: llvm-profdata merge %S/Inputs/gcc-flag-compatibility_IR_entry.proftext -o %t.dir/some/path/file.prof
+// RUN: %clang %s -o - -emit-llvm -S -fprofile-use=%t.dir/some/path/file.prof -fno-experimental-new-pass-manager | FileCheck -check-prefix=PROFILE-USE-IR %s
+// RUN: %clang %s -o - -emit-llvm -S -fprofile-use=%t.dir/some/path/file.prof -fexperimental-new-pass-manager | FileCheck -check-prefix=PROFILE-USE-IR %s
 
 // PROFILE-USE-IR: = !{!"branch_weights", i32 100, i32 1}
 
diff --git a/clang/test/Sema/Float16.c b/clang/test/Sema/Float16.c
index bdfb01702c371..872bd732bd418 100644
--- a/clang/test/Sema/Float16.c
+++ b/clang/test/Sema/Float16.c
@@ -3,9 +3,16 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -triple armv7a-linux-gnu %s -DHAVE
 // RUN: %clang_cc1 -fsyntax-only -verify -triple aarch64-linux-gnu %s -DHAVE
 
-#ifdef HAVE
-// expected-no-diagnostics
-#else
+#ifndef HAVE
 // expected-error@+2{{_Float16 is not supported on this target}}
 #endif // HAVE
 _Float16 f;
+
+#ifdef HAVE
+// FIXME: Should this be valid?
+_Complex _Float16 a; // expected-error {{'_Complex _Float16' is invalid}}
+void builtin_complex() {
+  _Float16 a = 0;
+  (void)__builtin_complex(a, a); // expected-error {{'_Complex _Float16' is invalid}}
+}
+#endif
diff --git a/clang/test/Sema/attr-arm-sve-vector-bits.c b/clang/test/Sema/attr-arm-sve-vector-bits.c
index 48ca7d8fa8125..30ae4c66f72e0 100644
--- a/clang/test/Sema/attr-arm-sve-vector-bits.c
+++ b/clang/test/Sema/attr-arm-sve-vector-bits.c
@@ -60,3 +60,168 @@ typedef int badtype2 __attribute__((arm_sve_vector_bits(N)));           // expec
 typedef float badtype3 __attribute__((arm_sve_vector_bits(N)));         // expected-error {{'arm_sve_vector_bits' attribute applied to non-SVE type 'float'}}
 typedef svint8x2_t badtype4 __attribute__((arm_sve_vector_bits(N)));    // expected-error {{'arm_sve_vector_bits' attribute applied to non-SVE type 'svint8x2_t' (aka '__clang_svint8x2_t')}}
 typedef svfloat32x3_t badtype5 __attribute__((arm_sve_vector_bits(N))); // expected-error {{'arm_sve_vector_bits' attribute applied to non-SVE type 'svfloat32x3_t' (aka '__clang_svfloat32x3_t')}}
+
+// Attribute only applies to typedefs.
+svint8_t non_typedef_type __attribute__((arm_sve_vector_bits(N)));  // expected-error {{'arm_sve_vector_bits' attribute only applies to typedefs}}
+
+// Test that we can define non-local fixed-length SVE types (unsupported for
+// sizeless types).
+fixed_int8_t global_int8;
+fixed_bfloat16_t global_bfloat16;
+fixed_bool_t global_bool;
+
+extern fixed_int8_t extern_int8;
+extern fixed_bfloat16_t extern_bfloat16;
+extern fixed_bool_t extern_bool;
+
+static fixed_int8_t static_int8;
+static fixed_bfloat16_t static_bfloat16;
+static fixed_bool_t static_bool;
+
+fixed_int8_t *global_int8_ptr;
+extern fixed_int8_t *extern_int8_ptr;
+static fixed_int8_t *static_int8_ptr;
+__thread fixed_int8_t thread_int8;
+
+typedef fixed_int8_t int8_typedef;
+typedef fixed_int8_t *int8_ptr_typedef;
+
+// Test sized expressions
+int sizeof_int8 = sizeof(global_int8);
+int sizeof_int8_var = sizeof(*global_int8_ptr);
+int sizeof_int8_var_ptr = sizeof(global_int8_ptr);
+
+extern fixed_int8_t *extern_int8_ptr;
+
+int alignof_int8 = __alignof__(extern_int8);
+int alignof_int8_var = __alignof__(*extern_int8_ptr);
+int alignof_int8_var_ptr = __alignof__(extern_int8_ptr);
+
+void f(int c) {
+  fixed_int8_t fs8;
+  svint8_t ss8;
+
+  void *sel __attribute__((unused));
+  sel = c ? ss8 : fs8; // expected-error {{incompatible operand types ('svint8_t' (aka '__SVInt8_t') and 'fixed_int8_t' (aka '__SVInt8_t'))}}
+  sel = c ? fs8 : ss8; // expected-error {{incompatible operand types ('fixed_int8_t' (aka '__SVInt8_t') and 'svint8_t' (aka '__SVInt8_t'))}}
+}
+
+// --------------------------------------------------------------------------//
+// Sizeof
+
+#define VECTOR_SIZE ((N / 8))
+#define PRED_SIZE ((N / 64))
+
+_Static_assert(sizeof(fixed_int8_t) == VECTOR_SIZE, "");
+
+_Static_assert(sizeof(fixed_int16_t) == VECTOR_SIZE, "");
+_Static_assert(sizeof(fixed_int32_t) == VECTOR_SIZE, "");
+_Static_assert(sizeof(fixed_int64_t) == VECTOR_SIZE, "");
+
+_Static_assert(sizeof(fixed_uint8_t) == VECTOR_SIZE, "");
+_Static_assert(sizeof(fixed_uint16_t) == VECTOR_SIZE, "");
+_Static_assert(sizeof(fixed_uint32_t) == VECTOR_SIZE, "");
+_Static_assert(sizeof(fixed_uint64_t) == VECTOR_SIZE, "");
+
+_Static_assert(sizeof(fixed_float16_t) == VECTOR_SIZE, "");
+_Static_assert(sizeof(fixed_float32_t) == VECTOR_SIZE, "");
+_Static_assert(sizeof(fixed_float64_t) == VECTOR_SIZE, "");
+
+_Static_assert(sizeof(fixed_bfloat16_t) == VECTOR_SIZE, "");
+
+_Static_assert(sizeof(fixed_bool_t) == PRED_SIZE, "");
+
+// --------------------------------------------------------------------------//
+// Alignof
+
+#define VECTOR_ALIGN 16
+#define PRED_ALIGN 2
+
+_Static_assert(__alignof__(fixed_int8_t) == VECTOR_ALIGN, "");
+_Static_assert(__alignof__(fixed_int16_t) == VECTOR_ALIGN, "");
+_Static_assert(__alignof__(fixed_int32_t) == VECTOR_ALIGN, "");
+_Static_assert(__alignof__(fixed_int64_t) == VECTOR_ALIGN, "");
+
+_Static_assert(__alignof__(fixed_uint8_t) == VECTOR_ALIGN, "");
+_Static_assert(__alignof__(fixed_uint16_t) == VECTOR_ALIGN, "");
+_Static_assert(__alignof__(fixed_uint32_t) == VECTOR_ALIGN, "");
+_Static_assert(__alignof__(fixed_uint64_t) == VECTOR_ALIGN, "");
+
+_Static_assert(__alignof__(fixed_float16_t) == VECTOR_ALIGN, "");
+_Static_assert(__alignof__(fixed_float32_t) == VECTOR_ALIGN, "");
+_Static_assert(__alignof__(fixed_float64_t) == VECTOR_ALIGN, "");
+
+_Static_assert(__alignof__(fixed_bfloat16_t) == VECTOR_ALIGN, "");
+
+_Static_assert(__alignof__(fixed_bool_t) == PRED_ALIGN, "");
+
+// --------------------------------------------------------------------------//
+// Structs
+
+struct struct_int64 { fixed_int64_t x, y[5]; };
+struct struct_float64 { fixed_float64_t x, y[5]; };
+struct struct_bfloat16 { fixed_bfloat16_t x, y[5]; };
+struct struct_bool { fixed_bool_t x, y[5]; };
+
+// --------------------------------------------------------------------------//
+// Unions
+union union_int64 { fixed_int64_t x, y[5]; };
+union union_float64 { fixed_float64_t x, y[5]; };
+union union_bfloat16 { fixed_bfloat16_t x, y[5]; };
+union union_bool { fixed_bool_t x, y[5]; };
+
+// --------------------------------------------------------------------------//
+// Implicit casts
+
+#define TEST_CAST(TYPE)                                          \
+  sv##TYPE##_t to_sv##TYPE##_t(fixed_##TYPE##_t x) { return x; } \
+  fixed_##TYPE##_t from_sv##TYPE##_t(sv##TYPE##_t x) { return x; }
+
+TEST_CAST(int8)
+TEST_CAST(int16)
+TEST_CAST(int32)
+TEST_CAST(int64)
+TEST_CAST(uint8)
+TEST_CAST(uint16)
+TEST_CAST(uint32)
+TEST_CAST(uint64)
+TEST_CAST(float16)
+TEST_CAST(float32)
+TEST_CAST(float64)
+TEST_CAST(bfloat16)
+TEST_CAST(bool)
+
+// Test the implicit conversion only applies to valid types
+fixed_int8_t to_fixed_int8_t__from_svuint8_t(svuint8_t x) { return x; } // expected-error {{returning 'svuint8_t' (aka '__SVUint8_t') from a function with incompatible result type 'fixed_int8_t' (aka '__SVInt8_t')}}
+fixed_bool_t to_fixed_bool_t__from_svint32_t(svint32_t x) { return x; } // expected-error {{returning 'svint32_t' (aka '__SVInt32_t') from a function with incompatible result type 'fixed_bool_t' (aka '__SVBool_t')}}
+
+// Test the implicit conversion only applies to fixed-length types
+typedef signed int vSInt32 __attribute__((__vector_size__(16)));
+svint32_t to_svint32_t_from_gnut(vSInt32 x) { return x; } // expected-error {{returning 'vSInt32' (vector of 4 'int' values) from a function with incompatible result type 'svint32_t' (aka '__SVInt32_t')}}
+
+vSInt32 to_gnut_from_svint32_t(svint32_t x) { return x; } // expected-error {{returning 'svint32_t' (aka '__SVInt32_t') from a function with incompatible result type 'vSInt32' (vector of 4 'int' values)}}
+
+// --------------------------------------------------------------------------//
+// Test the scalable and fixed-length types can be used interchangeably
+
+svint32_t __attribute__((overloadable)) svfunc(svint32_t op1, svint32_t op2);
+svfloat64_t __attribute__((overloadable)) svfunc(svfloat64_t op1, svfloat64_t op2);
+svbool_t __attribute__((overloadable)) svfunc(svbool_t op1, svbool_t op2);
+
+#define TEST_CALL(TYPE)                                              \
+  fixed_##TYPE##_t                                                   \
+      call_##TYPE##_ff(fixed_##TYPE##_t op1, fixed_##TYPE##_t op2) { \
+    return svfunc(op1, op2);                                         \
+  }                                                                  \
+  fixed_##TYPE##_t                                                   \
+      call_##TYPE##_fs(fixed_##TYPE##_t op1, sv##TYPE##_t op2) {     \
+    return svfunc(op1, op2);                                         \
+  }                                                                  \
+  fixed_##TYPE##_t                                                   \
+      call_##TYPE##_sf(sv##TYPE##_t op1, fixed_##TYPE##_t op2) {     \
+    return svfunc(op1, op2);                                         \
+  }
+
+TEST_CALL(int32)
+TEST_CALL(float64)
+TEST_CALL(bool)
diff --git a/clang/test/Sema/builtins.c b/clang/test/Sema/builtins.c
index 90c033e47cd17..4b445724f712a 100644
--- a/clang/test/Sema/builtins.c
+++ b/clang/test/Sema/builtins.c
@@ -356,3 +356,22 @@ int test_cxx_builtin() {
   // expected-error@+1 {{use of unknown builtin '__builtin_is_constant_evaluated'}}
   return __builtin_is_constant_evaluated();
 }
+
+void test_builtin_complex() {
+  __builtin_complex(); // expected-error {{too few}}
+  __builtin_complex(1); // expected-error {{too few}}
+  __builtin_complex(1, 2, 3); // expected-error {{too many}}
+
+  _Static_assert(_Generic(__builtin_complex(1.0f, 2.0f), _Complex float: 1, default: 0), "");
+  _Static_assert(_Generic(__builtin_complex(1.0, 2.0), _Complex double: 1, default: 0), "");
+  _Static_assert(_Generic(__builtin_complex(1.0l, 2.0l), _Complex long double: 1, default: 0), "");
+
+  __builtin_complex(1, 2); // expected-error {{argument type 'int' is not a real floating point type}}
+  __builtin_complex(1, 2.0); // expected-error {{argument type 'int' is not a real floating point type}}
+  __builtin_complex(1.0, 2); // expected-error {{argument type 'int' is not a real floating point type}}
+
+  __builtin_complex(1.0, 2.0f); // expected-error {{arguments are of different types ('double' vs 'float')}}
+  __builtin_complex(1.0f, 2.0); // expected-error {{arguments are of different types ('float' vs 'double')}}
+}
+
+_Complex double builtin_complex_static_init = __builtin_complex(1.0, 2.0);
diff --git a/clang/test/Sema/ext-int.c b/clang/test/Sema/ext-int.c
new file mode 100644
index 0000000000000..6996942a204bc
--- /dev/null
+++ b/clang/test/Sema/ext-int.c
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s -Wimplicit-int-conversion -triple x86_64-gnu-linux
+
+typedef _ExtInt(31) EI31;
+
+void Ternary(_ExtInt(30) s30, EI31 s31a, _ExtInt(31) s31b,
+             _ExtInt(32) s32, int b) {
+  b ? s30 : s31a; // expected-error{{incompatible operand types}}
+  b ? s31a : s30; // expected-error{{incompatible operand types}}
+  b ? s32 : 0; // expected-error{{incompatible operand types}}
+  (void)(b ? s31a : s31b);
+  (void)(s30 ? s31a : s31b);
+}
diff --git a/clang/test/Sema/fp16-sema.c b/clang/test/Sema/fp16-sema.c
index e678f9a829e46..b0f6348a52a43 100644
--- a/clang/test/Sema/fp16-sema.c
+++ b/clang/test/Sema/fp16-sema.c
@@ -28,3 +28,9 @@ extern __fp16 *(*gf1) (void);
 typedef __fp16 (*tf1) (void); // expected-error {{function return value cannot have __fp16 type; did you forget * ?}}
 typedef __fp16 *(*tg1) (void);
 
+void testComplex() {
+  // FIXME: Should these be valid?
+  _Complex __fp16 a; // expected-error {{'_Complex half' is invalid}}
+  __fp16 b;
+  a = __builtin_complex(b, b); // expected-error {{'_Complex half' is invalid}}
+}
diff --git a/clang/test/Sema/typo-correction-no-hang.cpp b/clang/test/Sema/typo-correction-no-hang.cpp
new file mode 100644
index 0000000000000..3c591645be25c
--- /dev/null
+++ b/clang/test/Sema/typo-correction-no-hang.cpp
@@ -0,0 +1,40 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s
+
+// From `test/Sema/typo-correction.c` but for C++ since the behavior varies
+// between the two languages.
+struct rdar38642201 {
+  int fieldName;
+};
+
+void rdar38642201_callee(int x, int y);
+void rdar38642201_caller() {
+  struct rdar38642201 structVar;
+  rdar38642201_callee(
+      structVar1.fieldName1.member1,  //expected-error{{use of undeclared identifier 'structVar1'}}
+      structVar2.fieldName2.member2); //expected-error{{use of undeclared identifier 'structVar2'}}
+}
+
+// Similar reproducer.
+class A {
+public:
+  int minut() const = delete;
+  int hour() const = delete;
+
+  int longit() const; //expected-note{{'longit' declared here}}
+  int latit() const;
+};
+
+class B {
+public:
+  A depar() const { return A(); }
+};
+
+int Foo(const B &b) {
+  return b.deparT().hours() * 60 + //expected-error{{no member named 'deparT' in 'B'}}
+         b.deparT().minutes();     //expected-error{{no member named 'deparT' in 'B'}}
+}
+
+int Bar(const B &b) {
+  return b.depar().longitude() + //expected-error{{no member named 'longitude' in 'A'; did you mean 'longit'?}}
+         b.depar().latitude();   //expected-error{{no member named 'latitude' in 'A'}}
+}
diff --git a/clang/test/Sema/typo-correction-recursive.cpp b/clang/test/Sema/typo-correction-recursive.cpp
index 48bd3b80c5994..b39beb5493f65 100644
--- a/clang/test/Sema/typo-correction-recursive.cpp
+++ b/clang/test/Sema/typo-correction-recursive.cpp
@@ -118,3 +118,15 @@ int testDeepAmbiguity() {
       asDeepASItGet().
       functionE();
 }
+
+struct Dog {
+  int age;  //expected-note{{'age' declared here}}
+  int size; //expected-note{{'size' declared here}}
+};
+
+int from_dog_years(int DogYears, int DogSize);
+int get_dog_years() {
+  struct Dog doggo;
+  return from_dog_years(doggo.agee,   //expected-error{{no member named 'agee' in 'Dog'; did you mean 'age'}}
+                        doggo.sizee); //expected-error{{no member named 'sizee' in 'Dog'; did you mean 'size'}}
+}
diff --git a/clang/test/SemaCXX/builtins.cpp b/clang/test/SemaCXX/builtins.cpp
index d08e673c04b77..80f75c8232c20 100644
--- a/clang/test/SemaCXX/builtins.cpp
+++ b/clang/test/SemaCXX/builtins.cpp
@@ -144,3 +144,12 @@ void test_noexcept(int *i) {
 }
 #undef TEST_TYPE
 } // end namespace test_launder
+
+template<typename T> void test_builtin_complex(T v, double d) {
+  (void)__builtin_complex(v, d); // expected-error {{different types}} expected-error {{not a real floating}}
+  (void)__builtin_complex(d, v); // expected-error {{different types}} expected-error {{not a real floating}}
+  (void)__builtin_complex(v, v); // expected-error {{not a real floating}}
+}
+template void test_builtin_complex(double, double);
+template void test_builtin_complex(float, double); // expected-note {{instantiation of}}
+template void test_builtin_complex(int, double); // expected-note {{instantiation of}}
diff --git a/clang/test/SemaCXX/constant-expression-cxx11.cpp b/clang/test/SemaCXX/constant-expression-cxx11.cpp
index eac0256c4fb21..14d36543cb202 100644
--- a/clang/test/SemaCXX/constant-expression-cxx11.cpp
+++ b/clang/test/SemaCXX/constant-expression-cxx11.cpp
@@ -2043,14 +2043,11 @@ namespace BadDefaultInit {
         X<A().k>::n; // expected-note {{in evaluation of exception specification for 'BadDefaultInit::A::A' needed here}}
   };
 
-  // FIXME: The "constexpr constructor must initialize all members" diagnostic
-  // here is bogus (we discard the k(k) initializer because the parameter 'k'
-  // has been marked invalid).
   struct B {
-    constexpr B( // expected-warning {{initialize all members}}
+    constexpr B(
         int k = X<B().k>::n) : // expected-error {{default argument to function 'B' that is declared later}} expected-note {{here}}
       k(k) {}
-    int k; // expected-note {{not initialized}}
+    int k;
   };
 }
 
diff --git a/clang/test/SemaCXX/cxx0x-initializer-constructor.cpp b/clang/test/SemaCXX/cxx0x-initializer-constructor.cpp
index 513c670d392d5..92e3676954a21 100644
--- a/clang/test/SemaCXX/cxx0x-initializer-constructor.cpp
+++ b/clang/test/SemaCXX/cxx0x-initializer-constructor.cpp
@@ -163,7 +163,7 @@ namespace objects {
     // (for the second phase, no constructor is viable)
     G g1{1, 2, 3}; // expected-error {{no matching constructor}}
     (void) new G{1, 2, 3}; // expected-error {{no matching constructor}}
-    (void) G{1, 2, 3} // expected-error {{no matching constructor}}
+    (void) G{1, 2, 3}; // expected-error {{no matching constructor}}
 
     // valid (T deduced to <>).
     G g2({1, 2, 3});
diff --git a/clang/test/SemaCXX/cxx1z-copy-omission.cpp b/clang/test/SemaCXX/cxx1z-copy-omission.cpp
index eceac810e72a5..a850cf6143cd4 100644
--- a/clang/test/SemaCXX/cxx1z-copy-omission.cpp
+++ b/clang/test/SemaCXX/cxx1z-copy-omission.cpp
@@ -106,7 +106,7 @@ void test_expressions(bool b) {
   sizeof(Indestructible{}); // expected-error {{deleted}}
   sizeof(make_indestructible()); // expected-error {{deleted}}
   sizeof(make_incomplete()); // expected-error {{incomplete}}
-  typeid(Indestructible{}); // expected-error {{deleted}}
+  typeid(Indestructible{}); // expected-error {{deleted}} expected-error {{you need to include <typeinfo>}}
   typeid(make_indestructible()); // expected-error {{deleted}} \
                                  // expected-error {{need to include <typeinfo>}}
   typeid(make_incomplete()); // expected-error {{incomplete}} \
diff --git a/clang/test/SemaCXX/ext-int.cpp b/clang/test/SemaCXX/ext-int.cpp
index 14f11a6bb9614..0f2a3b89be1f1 100644
--- a/clang/test/SemaCXX/ext-int.cpp
+++ b/clang/test/SemaCXX/ext-int.cpp
@@ -275,3 +275,12 @@ void ImplicitCasts(_ExtInt(31) s31, _ExtInt(33) s33, int i) {
   // expected-warning@+1{{implicit conversion loses integer precision}}
   i = s33;
 }
+
+void Ternary(_ExtInt(30) s30, _ExtInt(31) s31a, _ExtInt(31) s31b,
+             _ExtInt(32) s32, bool b) {
+  b ? s30 : s31a; // expected-error{{incompatible operand types}}
+  b ? s31a : s30; // expected-error{{incompatible operand types}}
+  b ? s32 : (int)0; // expected-error{{incompatible operand types}}
+  (void)(b ? s31a : s31b);
+  (void)(s30 ? s31a : s31b);
+}
diff --git a/clang/test/SemaCXX/typo-correction-delayed.cpp b/clang/test/SemaCXX/typo-correction-delayed.cpp
index 610d439713814..66d19daf66fd3 100644
--- a/clang/test/SemaCXX/typo-correction-delayed.cpp
+++ b/clang/test/SemaCXX/typo-correction-delayed.cpp
@@ -149,7 +149,8 @@ void test() {
 }
 
 namespace PR21905 {
-int (*a) () = (void)Z;  // expected-error-re {{use of undeclared identifier 'Z'{{$}}}}
+int (*a)() = (void)Z; // expected-error-re {{use of undeclared identifier 'Z'{{$}}}} \
+                      // expected-error {{cannot initialize a variable of type 'int (*)()' with an rvalue of type 'void'}}
 }
 
 namespace PR21947 {
diff --git a/clang/test/SemaOpenCL/recovery-expr.cl b/clang/test/SemaOpenCL/recovery-expr.cl
new file mode 100644
index 0000000000000..902b10dbb5789
--- /dev/null
+++ b/clang/test/SemaOpenCL/recovery-expr.cl
@@ -0,0 +1,7 @@
+// RUN: %clang_cc1 %s -verify -pedantic -fsyntax-only -cl-std=clc++ -frecovery-ast
+
+void kernel nocrash() {
+  constant int L1 = 0;
+
+  private int *constant L2 = L1++; // expected-error {{read-only variable is not assignable}}
+}
diff --git a/clang/test/SemaSYCL/check-notdirect-attribute-propagation.cpp b/clang/test/SemaSYCL/check-notdirect-attribute-propagation.cpp
new file mode 100644
index 0000000000000..3dd98f12b0120
--- /dev/null
+++ b/clang/test/SemaSYCL/check-notdirect-attribute-propagation.cpp
@@ -0,0 +1,54 @@
+// RUN: %clang_cc1 %s -fsyntax-only -fsycl -fsycl-is-device -triple spir64 -verify
+// RUN: %clang_cc1 %s -fsyntax-only -fsycl -fsycl-is-device -triple spir64 -DTRIGGER_ERROR -verify
+// RUN: %clang_cc1 %s -fsyntax-only -ast-dump -fsycl -fsycl-is-device -triple spir64 | FileCheck %s
+
+#ifndef TRIGGER_ERROR
+[[intelfpga::no_global_work_offset]] void not_direct_one() {} // expected-no-diagnostics
+
+[[intel::reqd_sub_group_size(1)]] void func_one() {
+  not_direct_one();
+}
+
+#else
+[[cl::reqd_work_group_size(2, 2, 2)]] void not_direct_two() {} // expected-note {{conflicting attribute is here}}
+
+[[intelfpga::max_work_group_size(1, 1, 1)]] // expected-note {{conflicting attribute is here}}
+void
+func_two() {
+  not_direct_two();
+}
+
+[[cl::reqd_work_group_size(4, 4, 4)]] // expected-note 2 {{conflicting attribute is here}}
+void
+func_three() {
+  not_direct_two();
+}
+#endif
+
+template <typename Name, typename Type>
+[[clang::sycl_kernel]] void __my_kernel__(Type bar) {
+  bar();
+#ifndef TRIGGER_ERROR
+  func_one();
+#else
+  func_two();
+  func_three();
+#endif
+}
+
+template <typename Name, typename Type>
+void parallel_for(Type lambda) {
+  __my_kernel__<Name>(lambda);
+}
+
+void invoke_foo2() {
+#ifndef TRIGGER_ERROR
+  // CHECK-LABEL:  FunctionDecl {{.*}} invoke_foo2 'void ()'
+  // CHECK:        `-FunctionDecl {{.*}}KernelName 'void ()'
+  // CHECK:        -IntelReqdSubGroupSizeAttr {{.*}}
+  // CHECK:        `-SYCLIntelNoGlobalWorkOffsetAttr {{.*}} Enabled
+  parallel_for<class KernelName>([]() {});
+#else
+  parallel_for<class KernelName>([]() {}); // expected-error 2 {{conflicting attributes applied to a SYCL kernel or SYCL_EXTERNAL function}}
+#endif
+}
diff --git a/clang/test/SemaSYCL/intel-max-global-work-dim.cpp b/clang/test/SemaSYCL/intel-max-global-work-dim.cpp
index 85db992f46cca..d391924773f1e 100644
--- a/clang/test/SemaSYCL/intel-max-global-work-dim.cpp
+++ b/clang/test/SemaSYCL/intel-max-global-work-dim.cpp
@@ -20,8 +20,7 @@ void foo() {
 
 #else // __SYCL_DEVICE_ONLY__
 
-[[intelfpga::max_global_work_dim(2)]] // expected-warning{{'max_global_work_dim' attribute ignored}}
-void func_ignore() {}
+[[intelfpga::max_global_work_dim(2)]] void func_do_not_ignore() {}
 
 struct FuncObj {
   [[intelfpga::max_global_work_dim(1)]]
@@ -68,9 +67,9 @@ int main() {
       []() [[intelfpga::max_global_work_dim(2)]] {});
 
   // CHECK-LABEL: FunctionDecl {{.*}}test_kernel3
-  // CHECK-NOT:   SYCLIntelMaxGlobalWorkDimAttr {{.*}}
+  // CHECK:       SYCLIntelMaxGlobalWorkDimAttr {{.*}}
   kernel<class test_kernel3>(
-      []() {func_ignore();});
+      []() { func_do_not_ignore(); });
 
   kernel<class test_kernel4>(
       TRIFuncObjGood1());
diff --git a/clang/test/SemaSYCL/intel-max-work-group-size.cpp b/clang/test/SemaSYCL/intel-max-work-group-size.cpp
index 441c1fea7df7e..f5b2a67599e8b 100644
--- a/clang/test/SemaSYCL/intel-max-work-group-size.cpp
+++ b/clang/test/SemaSYCL/intel-max-work-group-size.cpp
@@ -20,8 +20,7 @@ void foo() {
 
 #else // __SYCL_DEVICE_ONLY__
 
-[[intelfpga::max_work_group_size(2, 2, 2)]] // expected-warning{{'max_work_group_size' attribute ignored}}
-void func_ignore() {}
+[[intelfpga::max_work_group_size(2, 2, 2)]] void func_do_not_ignore() {}
 
 struct FuncObj {
   [[intelfpga::max_work_group_size(4, 4, 4)]]
@@ -53,9 +52,9 @@ int main() {
       []() [[intelfpga::max_work_group_size(8, 8, 8)]] {});
 
   // CHECK-LABEL: FunctionDecl {{.*}}test_kernel3
-  // CHECK-NOT:   SYCLIntelMaxWorkGroupSizeAttr {{.*}}
+  // CHECK:       SYCLIntelMaxWorkGroupSizeAttr {{.*}}
   kernel<class test_kernel3>(
-      []() {func_ignore();});
+      []() { func_do_not_ignore(); });
 
 #ifdef TRIGGER_ERROR
   [[intelfpga::max_work_group_size(1, 1, 1)]] int Var = 0; // expected-error{{'max_work_group_size' attribute only applies to functions}}
diff --git a/clang/test/SemaSYCL/intel-restrict.cpp b/clang/test/SemaSYCL/intel-restrict.cpp
index 810e7b0b3a5d3..b998b261f4320 100644
--- a/clang/test/SemaSYCL/intel-restrict.cpp
+++ b/clang/test/SemaSYCL/intel-restrict.cpp
@@ -1,8 +1,7 @@
 // RUN: %clang_cc1 %s -fsyntax-only -fsycl -fsycl-is-device -triple spir64 -DCHECKDIAG -verify
 // RUN: %clang_cc1 %s -fsyntax-only -ast-dump -fsycl -fsycl-is-device -triple spir64 | FileCheck %s
 
-[[intel::kernel_args_restrict]] // expected-warning{{'kernel_args_restrict' attribute ignored}}
-void func_ignore() {}
+[[intel::kernel_args_restrict]] void func_do_not_ignore() {}
 
 struct FuncObj {
   [[intel::kernel_args_restrict]]
@@ -29,7 +28,7 @@ int main() {
       []() [[intel::kernel_args_restrict]] {});
 
   // CHECK-LABEL: FunctionDecl {{.*}}test_kernel3
-  // CHECK-NOT:   SYCLIntelKernelArgsRestrictAttr
+  // CHECK:       SYCLIntelKernelArgsRestrictAttr
   kernel<class test_kernel3>(
-      []() {func_ignore();});
+      []() { func_do_not_ignore(); });
 }
diff --git a/clang/test/SemaSYCL/lambda_implicit_capture_this.cpp b/clang/test/SemaSYCL/lambda_implicit_capture_this.cpp
new file mode 100644
index 0000000000000..331eb1b64ae3e
--- /dev/null
+++ b/clang/test/SemaSYCL/lambda_implicit_capture_this.cpp
@@ -0,0 +1,26 @@
+// RUN: %clang_cc1 -I %S/Inputs -fsycl -fsycl-is-device -fsyntax-only -verify %s
+
+template <typename name, typename Func>
+__attribute__((sycl_kernel)) void kernel(Func kernelFunc) {
+  kernelFunc();
+}
+
+class Class {
+public:
+  Class() : member(1) {}
+  void function();
+  int member;
+};
+
+void Class::function() {
+  kernel<class kernel_wrapper>(
+      [=]() {
+        int acc[1] = {5};
+        acc[0] *= member; // expected-error{{implicit capture of 'this' is not allowed for kernel functions}}
+      });
+}
+
+int main(int argc, char *argv[]) {
+  Class c;
+  c.function();
+}
diff --git a/clang/test/SemaSYCL/num_simd_work_items.cpp b/clang/test/SemaSYCL/num_simd_work_items.cpp
index 4eaf95080e275..638539917a283 100644
--- a/clang/test/SemaSYCL/num_simd_work_items.cpp
+++ b/clang/test/SemaSYCL/num_simd_work_items.cpp
@@ -20,8 +20,7 @@ void foo() {
 
 #else // __SYCL_DEVICE_ONLY__
 
-[[intelfpga::num_simd_work_items(2)]] // expected-warning{{'num_simd_work_items' attribute ignored}}
-void func_ignore() {}
+[[intelfpga::num_simd_work_items(2)]] void func_do_not_ignore() {}
 
 struct FuncObj {
   [[intelfpga::num_simd_work_items(42)]]
@@ -45,9 +44,9 @@ int main() {
       []() [[intelfpga::num_simd_work_items(8)]] {});
 
   // CHECK-LABEL: FunctionDecl {{.*}}test_kernel3
-  // CHECK-NOT:   SYCLIntelNumSimdWorkItemsAttr {{.*}} 2
+  // CHECK:       SYCLIntelNumSimdWorkItemsAttr {{.*}} 2
   kernel<class test_kernel3>(
-      []() {func_ignore();});
+      []() { func_do_not_ignore(); });
 
 #ifdef TRIGGER_ERROR
   [[intelfpga::num_simd_work_items(0)]] int Var = 0; // expected-error{{'num_simd_work_items' attribute only applies to functions}}
diff --git a/clang/test/SemaSYCL/reqd-sub-group-size-device.cpp b/clang/test/SemaSYCL/reqd-sub-group-size-device.cpp
index aa61f8a9663c9..ffa4b176207a0 100644
--- a/clang/test/SemaSYCL/reqd-sub-group-size-device.cpp
+++ b/clang/test/SemaSYCL/reqd-sub-group-size-device.cpp
@@ -1,22 +1,29 @@
 // RUN: %clang_cc1 -fsycl -fsycl-is-device -fsyntax-only -verify -DTRIGGER_ERROR %s
 // RUN: %clang_cc1 -fsycl -fsycl-is-device -ast-dump %s | FileCheck %s
 
-[[cl::intel_reqd_sub_group_size(4)]] void foo() {} // expected-note {{conflicting attribute is here}}
+[[intel::reqd_sub_group_size(4)]] void foo() {} // expected-note {{conflicting attribute is here}}
 // expected-note@-1 {{conflicting attribute is here}}
-[[cl::intel_reqd_sub_group_size(32)]] void baz() {} // expected-note {{conflicting attribute is here}}
+[[intel::reqd_sub_group_size(32)]] void baz() {} // expected-note {{conflicting attribute is here}}
 
 class Functor16 {
 public:
+  // expected-warning@+2 {{attribute 'intel_reqd_sub_group_size' is deprecated}}
+  // expected-note@+1 {{did you mean to use 'intel::reqd_sub_group_size' instead?}}
   [[cl::intel_reqd_sub_group_size(16)]] void operator()() {}
 };
 
 class Functor8 { // expected-error {{conflicting attributes applied to a SYCL kernel}}
 public:
-  [[cl::intel_reqd_sub_group_size(8)]] void operator()() { // expected-note {{conflicting attribute is here}}
+  [[intel::reqd_sub_group_size(8)]] void operator()() { // expected-note {{conflicting attribute is here}}
     foo();
   }
 };
 
+class Functor4 {
+public:
+  [[intel::reqd_sub_group_size(12)]] void operator()() {}
+};
+
 class Functor {
 public:
   void operator()() {
@@ -46,24 +53,31 @@ void bar() {
   });
 #endif
 
-  kernel<class kernel_name5>([]() [[cl::intel_reqd_sub_group_size(2)]] { });
-  kernel<class kernel_name6>([]() [[cl::intel_reqd_sub_group_size(4)]] { foo(); });
+  kernel<class kernel_name5>([]() [[intel::reqd_sub_group_size(2)]]{});
+  kernel<class kernel_name6>([]() [[intel::reqd_sub_group_size(4)]] { foo(); });
+  // expected-warning@+2 {{attribute 'intel_reqd_sub_group_size' is deprecated}}
+  // expected-note@+1 {{did you mean to use 'intel::reqd_sub_group_size' instead?}}
+  kernel<class kernel_name7>([]() [[cl::intel_reqd_sub_group_size(6)]]{});
+
+  Functor4 f4;
+  kernel<class kernel_name8>(f4);
 }
 
-[[cl::intel_reqd_sub_group_size(16)]] SYCL_EXTERNAL void B();
-[[cl::intel_reqd_sub_group_size(16)]] void A() {
+[[intel::reqd_sub_group_size(16)]] SYCL_EXTERNAL void B();
+[[intel::reqd_sub_group_size(16)]] void A() {
 }
-[[cl::intel_reqd_sub_group_size(16)]] SYCL_EXTERNAL void B() {
+
+[[intel::reqd_sub_group_size(16)]] SYCL_EXTERNAL void B() {
   A();
 }
 
 #ifdef TRIGGER_ERROR
 // expected-note@+1 {{conflicting attribute is here}}
-[[cl::intel_reqd_sub_group_size(2)]] void sg_size2() {}
+[[intel::reqd_sub_group_size(2)]] void sg_size2() {}
 
 // expected-note@+2 {{conflicting attribute is here}}
 // expected-error@+1 {{conflicting attributes applied to a SYCL kernel}}
-[[cl::intel_reqd_sub_group_size(4)]] __attribute__((sycl_device)) void sg_size4() {
+[[intel::reqd_sub_group_size(4)]] __attribute__((sycl_device)) void sg_size4() {
   sg_size2();
 }
 #endif
@@ -77,3 +91,9 @@ void bar() {
 // CHECK: FunctionDecl {{.*}} {{.*}}kernel_name5
 // CHECK: IntelReqdSubGroupSizeAttr {{.*}}
 // CHECK-NEXT: IntegerLiteral{{.*}}2{{$}}
+// CHECK: FunctionDecl {{.*}} {{.*}}kernel_name7
+// CHECK: IntelReqdSubGroupSizeAttr {{.*}}
+// CHECK-NEXT: IntegerLiteral{{.*}}6{{$}}
+// CHECK: FunctionDecl {{.*}} {{.*}}kernel_name8
+// CHECK: IntelReqdSubGroupSizeAttr {{.*}}
+// CHECK-NEXT: IntegerLiteral{{.*}}12{{$}}
diff --git a/clang/test/SemaSYCL/restrict-recursion3.cpp b/clang/test/SemaSYCL/restrict-recursion3.cpp
index 7370a92600bd6..1a4a671904169 100644
--- a/clang/test/SemaSYCL/restrict-recursion3.cpp
+++ b/clang/test/SemaSYCL/restrict-recursion3.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fsycl -fsycl-is-device -fcxx-exceptions -Wno-return-type -Wno-sycl-strict -verify -fsyntax-only -std=c++17 %s
+// RUN: %clang_cc1 -fsycl -fsycl-is-device -fcxx-exceptions -Wno-return-type -Wno-error=sycl-strict -verify -fsyntax-only -std=c++17 %s
 
 // This recursive function is not called from sycl kernel,
 // so it should not be diagnosed.
@@ -16,6 +16,7 @@ using myFuncDef = int(int, int);
 
 typedef __typeof__(sizeof(int)) size_t;
 
+// expected-warning@+1 {{SYCL 1.2.1 specification does not allow 'sycl_device' attribute applied to a function with a raw pointer return type}}
 SYCL_EXTERNAL
 void *operator new(size_t);
 
@@ -34,7 +35,7 @@ template <typename name, typename Func>
 __attribute__((sycl_kernel)) void kernel_single_task2(Func kernelFunc) {
   // expected-note@+1 {{called by 'kernel_single_task2}}
   kernelFunc();
-  // expected-error@+1 2{{SYCL kernel cannot call a recursive function}}
+  // expected-warning@+1 2{{SYCL kernel cannot call a recursive function}}
   kernel_single_task2<name, Func>(kernelFunc);
 }
 
diff --git a/clang/test/SemaSYCL/restrict-recursion4.cpp b/clang/test/SemaSYCL/restrict-recursion4.cpp
index fdc94d969715c..c755f0eaec63a 100644
--- a/clang/test/SemaSYCL/restrict-recursion4.cpp
+++ b/clang/test/SemaSYCL/restrict-recursion4.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fsycl -fsycl-is-device -fcxx-exceptions -Wno-return-type -Wno-sycl-strict -verify -fsyntax-only -std=c++17 %s
+// RUN: %clang_cc1 -fsycl -fsycl-is-device -fcxx-exceptions -Wno-return-type -Wno-error=sycl-strict -verify -fsyntax-only -std=c++17 %s
 
 // This recursive function is not called from sycl kernel,
 // so it should not be diagnosed.
@@ -10,7 +10,7 @@ int fib(int n) {
 
 // expected-note@+1 2{{function implemented using recursion declared here}}
 void kernel2(void) {
-  // expected-error@+1 {{SYCL kernel cannot call a recursive function}}
+  // expected-warning@+1 {{SYCL kernel cannot call a recursive function}}
   kernel2();
 }
 
@@ -18,13 +18,14 @@ using myFuncDef = int(int, int);
 
 typedef __typeof__(sizeof(int)) size_t;
 
+// expected-warning@+1 {{SYCL 1.2.1 specification does not allow 'sycl_device' attribute applied to a function with a raw pointer return type}}
 SYCL_EXTERNAL
 void *operator new(size_t);
 
 void usage2(myFuncDef functionPtr) {
   // expected-error@+1 {{SYCL kernel cannot allocate storage}}
   int *ip = new int;
-  // expected-error@+1 {{SYCL kernel cannot call a recursive function}}
+  // expected-warning@+1 {{SYCL kernel cannot call a recursive function}}
   kernel2();
 }
 
diff --git a/clang/test/SemaSYCL/sycl-device-reqd-sub-group-size-template.cpp b/clang/test/SemaSYCL/sycl-device-reqd-sub-group-size-template.cpp
index 73b2a6b56e3b4..4c4a376cf4591 100644
--- a/clang/test/SemaSYCL/sycl-device-reqd-sub-group-size-template.cpp
+++ b/clang/test/SemaSYCL/sycl-device-reqd-sub-group-size-template.cpp
@@ -1,12 +1,12 @@
 // RUN: %clang_cc1 -fsycl -fsycl-is-device -fsyntax-only -ast-dump -verify -pedantic %s | FileCheck %s
 
-// Test that checkes template parameter support for 'intel_reqd_sub_group_size' attribute on sycl device.
+// Test that checkes template parameter support for 'reqd_sub_group_size' attribute on sycl device.
 
 template <int SIZE>
 class KernelFunctor {
 public:
-  //expected-error@+1{{'intel_reqd_sub_group_size' attribute requires a positive integral compile time constant expression}}
-  [[cl::intel_reqd_sub_group_size(SIZE)]] void operator()() {}
+  // expected-error@+1{{'reqd_sub_group_size' attribute requires a positive integral compile time constant expression}}
+  [[intel::reqd_sub_group_size(SIZE)]] void operator()() {}
 };
 
 int main() {
diff --git a/clang/test/SemaSYCL/sycl-esimd.cpp b/clang/test/SemaSYCL/sycl-esimd.cpp
index a336f101a3e9b..f235394b9971c 100644
--- a/clang/test/SemaSYCL/sycl-esimd.cpp
+++ b/clang/test/SemaSYCL/sycl-esimd.cpp
@@ -17,7 +17,7 @@ void kernel0(F f) __attribute__((sycl_kernel)) {
 }
 
 // expected-note@+1{{conflicting attribute is here}}
-[[cl::intel_reqd_sub_group_size(2)]] void g0() {}
+[[intel::reqd_sub_group_size(2)]] void g0() {}
 
 void test0() {
   // expected-error@+2{{conflicting attributes applied to a SYCL kernel}}
diff --git a/clang/test/SemaTemplate/instantiate-expr-3.cpp b/clang/test/SemaTemplate/instantiate-expr-3.cpp
index 142e4ebcedc68..689fd22ea4484 100644
--- a/clang/test/SemaTemplate/instantiate-expr-3.cpp
+++ b/clang/test/SemaTemplate/instantiate-expr-3.cpp
@@ -65,7 +65,7 @@ struct StatementExpr0 {
   void f(T t) {
     (void)({
         if (t) // expected-error{{contextually convertible}}
-          t = t + 17;
+          t = t + 17; // expected-error {{invalid operands to binary expression ('N1::X' and 'int')}}
         t + 12; // expected-error{{invalid operands}}
       });
   }
diff --git a/clang/test/lit.site.cfg.py.in b/clang/test/lit.site.cfg.py.in
index d9b5b2f2592e5..286ea06d798c1 100644
--- a/clang/test/lit.site.cfg.py.in
+++ b/clang/test/lit.site.cfg.py.in
@@ -16,7 +16,7 @@ config.host_triple = "@LLVM_HOST_TRIPLE@"
 config.target_triple = "@TARGET_TRIPLE@"
 config.host_cxx = "@CMAKE_CXX_COMPILER@"
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
-config.have_zlib = @HAVE_LIBZ@
+config.have_zlib = @LLVM_ENABLE_ZLIB@
 config.clang_arcmt = @CLANG_ENABLE_ARCMT@
 config.clang_default_cxx_stdlib = "@CLANG_DEFAULT_CXX_STDLIB@"
 config.clang_staticanalyzer = @CLANG_ENABLE_STATIC_ANALYZER@
diff --git a/clang/test/utils/update_cc_test_checks/Inputs/check-attributes.cpp b/clang/test/utils/update_cc_test_checks/Inputs/check-attributes.cpp
new file mode 100644
index 0000000000000..9a80ab603c7ed
--- /dev/null
+++ b/clang/test/utils/update_cc_test_checks/Inputs/check-attributes.cpp
@@ -0,0 +1,15 @@
+// RUN: %clang_cc1 -triple=x86_64-unknown-linux-gnu -emit-llvm -o - %s | FileCheck %s
+struct RT {
+  char A;
+  int B[10][20];
+  char C;
+};
+struct ST {
+  int X;
+  double Y;
+  struct RT Z;
+};
+
+int *foo(struct ST *s) {
+  return &s[1].Z.B[5][13];
+}
diff --git a/clang/test/utils/update_cc_test_checks/Inputs/check-attributes.cpp.funcattrs.expected b/clang/test/utils/update_cc_test_checks/Inputs/check-attributes.cpp.funcattrs.expected
new file mode 100644
index 0000000000000..d24ad04778986
--- /dev/null
+++ b/clang/test/utils/update_cc_test_checks/Inputs/check-attributes.cpp.funcattrs.expected
@@ -0,0 +1,29 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --check-attributes
+// RUN: %clang_cc1 -triple=x86_64-unknown-linux-gnu -emit-llvm -o - %s | FileCheck %s
+struct RT {
+  char A;
+  int B[10][20];
+  char C;
+};
+struct ST {
+  int X;
+  double Y;
+  struct RT Z;
+};
+
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: @_Z3fooP2ST(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[S_ADDR:%.*]] = alloca %struct.ST*, align 8
+// CHECK-NEXT:    store %struct.ST* [[S:%.*]], %struct.ST** [[S_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load %struct.ST*, %struct.ST** [[S_ADDR]], align 8
+// CHECK-NEXT:    [[PTRIDX:%.*]] = getelementptr inbounds [[STRUCT_ST:%.*]], %struct.ST* [[TMP0]], i64 1
+// CHECK-NEXT:    [[Z:%.*]] = getelementptr inbounds [[STRUCT_ST]], %struct.ST* [[PTRIDX]], i32 0, i32 2
+// CHECK-NEXT:    [[B:%.*]] = getelementptr inbounds [[STRUCT_RT:%.*]], %struct.RT* [[Z]], i32 0, i32 1
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [10 x [20 x i32]], [10 x [20 x i32]]* [[B]], i64 0, i64 5
+// CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [20 x i32], [20 x i32]* [[ARRAYIDX]], i64 0, i64 13
+// CHECK-NEXT:    ret i32* [[ARRAYIDX1]]
+//
+int *foo(struct ST *s) {
+  return &s[1].Z.B[5][13];
+}
diff --git a/clang/test/utils/update_cc_test_checks/Inputs/check-attributes.cpp.plain.expected b/clang/test/utils/update_cc_test_checks/Inputs/check-attributes.cpp.plain.expected
new file mode 100644
index 0000000000000..b02f2390ede22
--- /dev/null
+++ b/clang/test/utils/update_cc_test_checks/Inputs/check-attributes.cpp.plain.expected
@@ -0,0 +1,28 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -triple=x86_64-unknown-linux-gnu -emit-llvm -o - %s | FileCheck %s
+struct RT {
+  char A;
+  int B[10][20];
+  char C;
+};
+struct ST {
+  int X;
+  double Y;
+  struct RT Z;
+};
+
+// CHECK-LABEL: @_Z3fooP2ST(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[S_ADDR:%.*]] = alloca %struct.ST*, align 8
+// CHECK-NEXT:    store %struct.ST* [[S:%.*]], %struct.ST** [[S_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load %struct.ST*, %struct.ST** [[S_ADDR]], align 8
+// CHECK-NEXT:    [[PTRIDX:%.*]] = getelementptr inbounds [[STRUCT_ST:%.*]], %struct.ST* [[TMP0]], i64 1
+// CHECK-NEXT:    [[Z:%.*]] = getelementptr inbounds [[STRUCT_ST]], %struct.ST* [[PTRIDX]], i32 0, i32 2
+// CHECK-NEXT:    [[B:%.*]] = getelementptr inbounds [[STRUCT_RT:%.*]], %struct.RT* [[Z]], i32 0, i32 1
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [10 x [20 x i32]], [10 x [20 x i32]]* [[B]], i64 0, i64 5
+// CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [20 x i32], [20 x i32]* [[ARRAYIDX]], i64 0, i64 13
+// CHECK-NEXT:    ret i32* [[ARRAYIDX1]]
+//
+int *foo(struct ST *s) {
+  return &s[1].Z.B[5][13];
+}
diff --git a/clang/test/utils/update_cc_test_checks/check_attrs.test b/clang/test/utils/update_cc_test_checks/check_attrs.test
new file mode 100644
index 0000000000000..658baf6530ee1
--- /dev/null
+++ b/clang/test/utils/update_cc_test_checks/check_attrs.test
@@ -0,0 +1,9 @@
+## Test that CHECK lines are generated as expected without --check-attributes
+# RUN: cp %S/Inputs/check-attributes.cpp %t.cpp && %update_cc_test_checks %t.cpp
+# RUN: diff -u %S/Inputs/check-attributes.cpp.plain.expected %t.cpp
+## Test with --check-attributes flag.
+# RUN: cp %S/Inputs/check-attributes.cpp %t.cpp && %update_cc_test_checks %t.cpp --check-attributes
+# RUN: diff -u %S/Inputs/check-attributes.cpp.funcattrs.expected %t.cpp
+## Check that re-running update_cc_test_checks doesn't change the output
+# RUN: %update_cc_test_checks %t.cpp
+# RUN: diff -u %S/Inputs/check-attributes.cpp.funcattrs.expected %t.cpp
diff --git a/clang/tools/clang-extdef-mapping/ClangExtDefMapGen.cpp b/clang/tools/clang-extdef-mapping/ClangExtDefMapGen.cpp
index dabcd6aa8f153..b91bf798aa7f9 100644
--- a/clang/tools/clang-extdef-mapping/ClangExtDefMapGen.cpp
+++ b/clang/tools/clang-extdef-mapping/ClangExtDefMapGen.cpp
@@ -104,7 +104,7 @@ void MapExtDefNamesConsumer::addIfInMain(const DeclaratorDecl *DD,
 class MapExtDefNamesAction : public ASTFrontendAction {
 protected:
   std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
-                                                 llvm::StringRef) {
+                                                 llvm::StringRef) override {
     return std::make_unique<MapExtDefNamesConsumer>(CI.getASTContext());
   }
 };
diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
index 91659e76219ab..91d2dc23d8610 100644
--- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -179,12 +179,12 @@ class SingleCommandCompilationDatabase : public tooling::CompilationDatabase {
   SingleCommandCompilationDatabase(tooling::CompileCommand Cmd)
       : Command(std::move(Cmd)) {}
 
-  virtual std::vector<tooling::CompileCommand>
-  getCompileCommands(StringRef FilePath) const {
+  std::vector<tooling::CompileCommand>
+  getCompileCommands(StringRef FilePath) const override {
     return {Command};
   }
 
-  virtual std::vector<tooling::CompileCommand> getAllCompileCommands() const {
+  std::vector<tooling::CompileCommand> getAllCompileCommands() const override {
     return {Command};
   }
 
diff --git a/clang/tools/libclang/CMakeLists.txt b/clang/tools/libclang/CMakeLists.txt
index 9b34682cc49bb..a4077140acee8 100644
--- a/clang/tools/libclang/CMakeLists.txt
+++ b/clang/tools/libclang/CMakeLists.txt
@@ -68,7 +68,12 @@ endif ()
 
 if (HAVE_LIBDL)
   list(APPEND LIBS ${CMAKE_DL_LIBS})
-endif()
+elseif (CLANG_BUILT_STANDALONE)
+  find_library(DL_LIBRARY_PATH dl)
+  if (DL_LIBRARY_PATH)
+    list(APPEND LIBS dl)
+  endif ()
+endif ()
 
 option(LIBCLANG_BUILD_STATIC
   "Build libclang as a static library (in addition to a shared one)" OFF)
diff --git a/clang/tools/scan-build-py/tests/functional/exec/CMakeLists.txt b/clang/tools/scan-build-py/tests/functional/exec/CMakeLists.txt
index 42ee1d11db828..007ad4530d6dc 100644
--- a/clang/tools/scan-build-py/tests/functional/exec/CMakeLists.txt
+++ b/clang/tools/scan-build-py/tests/functional/exec/CMakeLists.txt
@@ -1,6 +1,6 @@
 project(exec C)
 
-cmake_minimum_required(VERSION 3.4.3)
+cmake_minimum_required(VERSION 3.13.4)
 
 include(CheckCCompilerFlag)
 check_c_compiler_flag("-std=c99" C99_SUPPORTED)
diff --git a/clang/unittests/AST/ASTImporterFixtures.h b/clang/unittests/AST/ASTImporterFixtures.h
index 4fbceef393844..a63a98b7654c9 100644
--- a/clang/unittests/AST/ASTImporterFixtures.h
+++ b/clang/unittests/AST/ASTImporterFixtures.h
@@ -24,6 +24,7 @@
 #include "llvm/Support/ErrorHandling.h"
 
 #include "DeclMatcher.h"
+#include "MatchVerifier.h"
 
 #include <sstream>
 
@@ -202,6 +203,229 @@ class ASTImporterOptionSpecificTestBase
   std::vector<std::string> getExtraArgs() const override { return GetParam(); }
 };
 
+// Base class for those tests which use the family of `testImport` functions.
+class TestImportBase
+    : public CompilerOptionSpecificTest,
+      public ::testing::WithParamInterface<std::vector<std::string>> {
+
+  template <typename NodeType>
+  llvm::Expected<NodeType> importNode(ASTUnit *From, ASTUnit *To,
+                                      ASTImporter &Importer, NodeType Node) {
+    ASTContext &ToCtx = To->getASTContext();
+
+    // Add 'From' file to virtual file system so importer can 'find' it
+    // while importing SourceLocations. It is safe to add same file multiple
+    // times - it just isn't replaced.
+    StringRef FromFileName = From->getMainFileName();
+    createVirtualFileIfNeeded(To, FromFileName,
+                              From->getBufferForFile(FromFileName));
+
+    auto Imported = Importer.Import(Node);
+
+    if (Imported) {
+      // This should dump source locations and assert if some source locations
+      // were not imported.
+      SmallString<1024> ImportChecker;
+      llvm::raw_svector_ostream ToNothing(ImportChecker);
+      ToCtx.getTranslationUnitDecl()->print(ToNothing);
+
+      // This traverses the AST to catch certain bugs like poorly or not
+      // implemented subtrees.
+      (*Imported)->dump(ToNothing);
+    }
+
+    return Imported;
+  }
+
+  template <typename NodeType>
+  testing::AssertionResult
+  testImport(const std::string &FromCode,
+             const std::vector<std::string> &FromArgs,
+             const std::string &ToCode, const std::vector<std::string> &ToArgs,
+             MatchVerifier<NodeType> &Verifier,
+             const internal::BindableMatcher<NodeType> &SearchMatcher,
+             const internal::BindableMatcher<NodeType> &VerificationMatcher) {
+    const char *const InputFileName = "input.cc";
+    const char *const OutputFileName = "output.cc";
+
+    std::unique_ptr<ASTUnit> FromAST = tooling::buildASTFromCodeWithArgs(
+                                 FromCode, FromArgs, InputFileName),
+                             ToAST = tooling::buildASTFromCodeWithArgs(
+                                 ToCode, ToArgs, OutputFileName);
+
+    ASTContext &FromCtx = FromAST->getASTContext(),
+               &ToCtx = ToAST->getASTContext();
+
+    ASTImporter Importer(ToCtx, ToAST->getFileManager(), FromCtx,
+                         FromAST->getFileManager(), false);
+
+    auto FoundNodes = match(SearchMatcher, FromCtx);
+    if (FoundNodes.size() != 1)
+      return testing::AssertionFailure()
+             << "Multiple potential nodes were found!";
+
+    auto ToImport = selectFirst<NodeType>(DeclToImportID, FoundNodes);
+    if (!ToImport)
+      return testing::AssertionFailure() << "Node type mismatch!";
+
+    // Sanity check: the node being imported should match in the same way as
+    // the result node.
+    internal::BindableMatcher<NodeType> WrapperMatcher(VerificationMatcher);
+    EXPECT_TRUE(Verifier.match(ToImport, WrapperMatcher));
+
+    auto Imported = importNode(FromAST.get(), ToAST.get(), Importer, ToImport);
+    if (!Imported) {
+      std::string ErrorText;
+      handleAllErrors(
+          Imported.takeError(),
+          [&ErrorText](const ImportError &Err) { ErrorText = Err.message(); });
+      return testing::AssertionFailure()
+             << "Import failed, error: \"" << ErrorText << "\"!";
+    }
+
+    return Verifier.match(*Imported, WrapperMatcher);
+  }
+
+  template <typename NodeType>
+  testing::AssertionResult
+  testImport(const std::string &FromCode,
+             const std::vector<std::string> &FromArgs,
+             const std::string &ToCode, const std::vector<std::string> &ToArgs,
+             MatchVerifier<NodeType> &Verifier,
+             const internal::BindableMatcher<NodeType> &VerificationMatcher) {
+    return testImport(
+        FromCode, FromArgs, ToCode, ToArgs, Verifier,
+        translationUnitDecl(
+            has(namedDecl(hasName(DeclToImportID)).bind(DeclToImportID))),
+        VerificationMatcher);
+  }
+
+protected:
+  std::vector<std::string> getExtraArgs() const override { return GetParam(); }
+
+public:
+  /// Test how AST node named "declToImport" located in the translation unit
+  /// of "FromCode" virtual file is imported to "ToCode" virtual file.
+  /// The verification is done by running AMatcher over the imported node.
+  template <typename NodeType, typename MatcherType>
+  void testImport(const std::string &FromCode, TestLanguage FromLang,
+                  const std::string &ToCode, TestLanguage ToLang,
+                  MatchVerifier<NodeType> &Verifier,
+                  const MatcherType &AMatcher) {
+    std::vector<std::string> FromArgs = getCommandLineArgsForLanguage(FromLang);
+    std::vector<std::string> ToArgs = getCommandLineArgsForLanguage(ToLang);
+    EXPECT_TRUE(
+        testImport(FromCode, FromArgs, ToCode, ToArgs, Verifier, AMatcher));
+  }
+
+  struct ImportAction {
+    StringRef FromFilename;
+    StringRef ToFilename;
+    // FIXME: Generalize this to support other node kinds.
+    internal::BindableMatcher<Decl> ImportPredicate;
+
+    ImportAction(StringRef FromFilename, StringRef ToFilename,
+                 DeclarationMatcher ImportPredicate)
+        : FromFilename(FromFilename), ToFilename(ToFilename),
+          ImportPredicate(ImportPredicate) {}
+
+    ImportAction(StringRef FromFilename, StringRef ToFilename,
+                 const std::string &DeclName)
+        : FromFilename(FromFilename), ToFilename(ToFilename),
+          ImportPredicate(namedDecl(hasName(DeclName))) {}
+  };
+
+  using SingleASTUnit = std::unique_ptr<ASTUnit>;
+  using AllASTUnits = llvm::StringMap<SingleASTUnit>;
+
+  struct CodeEntry {
+    std::string CodeSample;
+    TestLanguage Lang;
+  };
+
+  using CodeFiles = llvm::StringMap<CodeEntry>;
+
+  /// Builds an ASTUnit for one potential compile options set.
+  SingleASTUnit createASTUnit(StringRef FileName, const CodeEntry &CE) const {
+    std::vector<std::string> Args = getCommandLineArgsForLanguage(CE.Lang);
+    auto AST = tooling::buildASTFromCodeWithArgs(CE.CodeSample, Args, FileName);
+    EXPECT_TRUE(AST.get());
+    return AST;
+  }
+
+  /// Test an arbitrary sequence of imports for a set of given in-memory files.
+  /// The verification is done by running VerificationMatcher against a
+  /// specified AST node inside of one of given files.
+  /// \param CodeSamples Map whose key is the file name and the value is the
+  /// file content.
+  /// \param ImportActions Sequence of imports. Each import in sequence
+  /// specifies "from file" and "to file" and a matcher that is used for
+  /// searching a declaration for import in "from file".
+  /// \param FileForFinalCheck Name of virtual file for which the final check is
+  /// applied.
+  /// \param FinalSelectPredicate Matcher that specifies the AST node in the
+  /// FileForFinalCheck for which the verification will be done.
+  /// \param VerificationMatcher Matcher that will be used for verification
+  /// after all imports in sequence are done.
+  void testImportSequence(const CodeFiles &CodeSamples,
+                          const std::vector<ImportAction> &ImportActions,
+                          StringRef FileForFinalCheck,
+                          internal::BindableMatcher<Decl> FinalSelectPredicate,
+                          internal::BindableMatcher<Decl> VerificationMatcher) {
+    AllASTUnits AllASTs;
+    using ImporterKey = std::pair<const ASTUnit *, const ASTUnit *>;
+    llvm::DenseMap<ImporterKey, std::unique_ptr<ASTImporter>> Importers;
+
+    auto GenASTsIfNeeded = [this, &AllASTs, &CodeSamples](StringRef Filename) {
+      if (!AllASTs.count(Filename)) {
+        auto Found = CodeSamples.find(Filename);
+        assert(Found != CodeSamples.end() && "Wrong file for import!");
+        AllASTs[Filename] = createASTUnit(Filename, Found->getValue());
+      }
+    };
+
+    for (const ImportAction &Action : ImportActions) {
+      StringRef FromFile = Action.FromFilename, ToFile = Action.ToFilename;
+      GenASTsIfNeeded(FromFile);
+      GenASTsIfNeeded(ToFile);
+
+      ASTUnit *From = AllASTs[FromFile].get();
+      ASTUnit *To = AllASTs[ToFile].get();
+
+      // Create a new importer if needed.
+      std::unique_ptr<ASTImporter> &ImporterRef = Importers[{From, To}];
+      if (!ImporterRef)
+        ImporterRef.reset(new ASTImporter(
+            To->getASTContext(), To->getFileManager(), From->getASTContext(),
+            From->getFileManager(), false));
+
+      // Find the declaration and import it.
+      auto FoundDecl = match(Action.ImportPredicate.bind(DeclToImportID),
+                             From->getASTContext());
+      EXPECT_TRUE(FoundDecl.size() == 1);
+      const Decl *ToImport = selectFirst<Decl>(DeclToImportID, FoundDecl);
+      auto Imported = importNode(From, To, *ImporterRef, ToImport);
+      EXPECT_TRUE(static_cast<bool>(Imported));
+      if (!Imported)
+        llvm::consumeError(Imported.takeError());
+    }
+
+    // Find the declaration and import it.
+    auto FoundDecl = match(FinalSelectPredicate.bind(DeclToVerifyID),
+                           AllASTs[FileForFinalCheck]->getASTContext());
+    EXPECT_TRUE(FoundDecl.size() == 1);
+    const Decl *ToVerify = selectFirst<Decl>(DeclToVerifyID, FoundDecl);
+    MatchVerifier<Decl> Verifier;
+    EXPECT_TRUE(Verifier.match(
+        ToVerify, internal::BindableMatcher<Decl>(VerificationMatcher)));
+  }
+};
+
+template <typename T> RecordDecl *getRecordDecl(T *D) {
+  auto *ET = cast<ElaboratedType>(D->getType().getTypePtr());
+  return cast<RecordType>(ET->getNamedType().getTypePtr())->getDecl();
+}
+
 template <class T>
 ::testing::AssertionResult isSuccess(llvm::Expected<T> &ValOrErr) {
   if (ValOrErr)
diff --git a/clang/unittests/AST/ASTImporterTest.cpp b/clang/unittests/AST/ASTImporterTest.cpp
index f92ea8de16516..9cab6cca0dc42 100644
--- a/clang/unittests/AST/ASTImporterTest.cpp
+++ b/clang/unittests/AST/ASTImporterTest.cpp
@@ -18,7 +18,6 @@
 #include "gtest/gtest.h"
 
 #include "ASTImporterFixtures.h"
-#include "MatchVerifier.h"
 
 namespace clang {
 namespace ast_matchers {
@@ -27,230 +26,6 @@ using internal::Matcher;
 using internal::BindableMatcher;
 using llvm::StringMap;
 
-// Base class for those tests which use the family of `testImport` functions.
-class TestImportBase
-    : public CompilerOptionSpecificTest,
-      public ::testing::WithParamInterface<std::vector<std::string>> {
-
-  template <typename NodeType>
-  llvm::Expected<NodeType> importNode(ASTUnit *From, ASTUnit *To,
-                                      ASTImporter &Importer, NodeType Node) {
-    ASTContext &ToCtx = To->getASTContext();
-
-    // Add 'From' file to virtual file system so importer can 'find' it
-    // while importing SourceLocations. It is safe to add same file multiple
-    // times - it just isn't replaced.
-    StringRef FromFileName = From->getMainFileName();
-    createVirtualFileIfNeeded(To, FromFileName,
-                              From->getBufferForFile(FromFileName));
-
-    auto Imported = Importer.Import(Node);
-
-    if (Imported) {
-      // This should dump source locations and assert if some source locations
-      // were not imported.
-      SmallString<1024> ImportChecker;
-      llvm::raw_svector_ostream ToNothing(ImportChecker);
-      ToCtx.getTranslationUnitDecl()->print(ToNothing);
-
-      // This traverses the AST to catch certain bugs like poorly or not
-      // implemented subtrees.
-      (*Imported)->dump(ToNothing);
-    }
-
-    return Imported;
-  }
-
-  template <typename NodeType>
-  testing::AssertionResult
-  testImport(const std::string &FromCode,
-             const std::vector<std::string> &FromArgs,
-             const std::string &ToCode, const std::vector<std::string> &ToArgs,
-             MatchVerifier<NodeType> &Verifier,
-             const BindableMatcher<NodeType> &SearchMatcher,
-             const BindableMatcher<NodeType> &VerificationMatcher) {
-    const char *const InputFileName = "input.cc";
-    const char *const OutputFileName = "output.cc";
-
-    std::unique_ptr<ASTUnit> FromAST = tooling::buildASTFromCodeWithArgs(
-                                 FromCode, FromArgs, InputFileName),
-                             ToAST = tooling::buildASTFromCodeWithArgs(
-                                 ToCode, ToArgs, OutputFileName);
-
-    ASTContext &FromCtx = FromAST->getASTContext(),
-               &ToCtx = ToAST->getASTContext();
-
-    ASTImporter Importer(ToCtx, ToAST->getFileManager(), FromCtx,
-                         FromAST->getFileManager(), false);
-
-    auto FoundNodes = match(SearchMatcher, FromCtx);
-    if (FoundNodes.size() != 1)
-      return testing::AssertionFailure()
-             << "Multiple potential nodes were found!";
-
-    auto ToImport = selectFirst<NodeType>(DeclToImportID, FoundNodes);
-    if (!ToImport)
-      return testing::AssertionFailure() << "Node type mismatch!";
-
-    // Sanity check: the node being imported should match in the same way as
-    // the result node.
-    BindableMatcher<NodeType> WrapperMatcher(VerificationMatcher);
-    EXPECT_TRUE(Verifier.match(ToImport, WrapperMatcher));
-
-    auto Imported = importNode(FromAST.get(), ToAST.get(), Importer, ToImport);
-    if (!Imported) {
-      std::string ErrorText;
-      handleAllErrors(
-          Imported.takeError(),
-          [&ErrorText](const ImportError &Err) { ErrorText = Err.message(); });
-      return testing::AssertionFailure()
-             << "Import failed, error: \"" << ErrorText << "\"!";
-    }
-
-    return Verifier.match(*Imported, WrapperMatcher);
-  }
-
-  template <typename NodeType>
-  testing::AssertionResult
-  testImport(const std::string &FromCode,
-             const std::vector<std::string> &FromArgs,
-             const std::string &ToCode, const std::vector<std::string> &ToArgs,
-             MatchVerifier<NodeType> &Verifier,
-             const BindableMatcher<NodeType> &VerificationMatcher) {
-    return testImport(
-        FromCode, FromArgs, ToCode, ToArgs, Verifier,
-        translationUnitDecl(
-            has(namedDecl(hasName(DeclToImportID)).bind(DeclToImportID))),
-        VerificationMatcher);
-  }
-
-protected:
-  std::vector<std::string> getExtraArgs() const override { return GetParam(); }
-
-public:
-
-  /// Test how AST node named "declToImport" located in the translation unit
-  /// of "FromCode" virtual file is imported to "ToCode" virtual file.
-  /// The verification is done by running AMatcher over the imported node.
-  template <typename NodeType, typename MatcherType>
-  void testImport(const std::string &FromCode, TestLanguage FromLang,
-                  const std::string &ToCode, TestLanguage ToLang,
-                  MatchVerifier<NodeType> &Verifier,
-                  const MatcherType &AMatcher) {
-    std::vector<std::string> FromArgs = getCommandLineArgsForLanguage(FromLang);
-    std::vector<std::string> ToArgs = getCommandLineArgsForLanguage(ToLang);
-    EXPECT_TRUE(
-        testImport(FromCode, FromArgs, ToCode, ToArgs, Verifier, AMatcher));
-  }
-
-  struct ImportAction {
-    StringRef FromFilename;
-    StringRef ToFilename;
-    // FIXME: Generalize this to support other node kinds.
-    BindableMatcher<Decl> ImportPredicate;
-
-    ImportAction(StringRef FromFilename, StringRef ToFilename,
-                 DeclarationMatcher ImportPredicate)
-        : FromFilename(FromFilename), ToFilename(ToFilename),
-          ImportPredicate(ImportPredicate) {}
-
-    ImportAction(StringRef FromFilename, StringRef ToFilename,
-                 const std::string &DeclName)
-        : FromFilename(FromFilename), ToFilename(ToFilename),
-          ImportPredicate(namedDecl(hasName(DeclName))) {}
-  };
-
-  using SingleASTUnit = std::unique_ptr<ASTUnit>;
-  using AllASTUnits = StringMap<SingleASTUnit>;
-
-  struct CodeEntry {
-    std::string CodeSample;
-    TestLanguage Lang;
-  };
-
-  using CodeFiles = StringMap<CodeEntry>;
-
-  /// Builds an ASTUnit for one potential compile options set.
-  SingleASTUnit createASTUnit(StringRef FileName, const CodeEntry &CE) const {
-    std::vector<std::string> Args = getCommandLineArgsForLanguage(CE.Lang);
-    auto AST = tooling::buildASTFromCodeWithArgs(CE.CodeSample, Args, FileName);
-    EXPECT_TRUE(AST.get());
-    return AST;
-  }
-
-  /// Test an arbitrary sequence of imports for a set of given in-memory files.
-  /// The verification is done by running VerificationMatcher against a
-  /// specified AST node inside of one of given files.
-  /// \param CodeSamples Map whose key is the file name and the value is the
-  /// file content.
-  /// \param ImportActions Sequence of imports. Each import in sequence
-  /// specifies "from file" and "to file" and a matcher that is used for
-  /// searching a declaration for import in "from file".
-  /// \param FileForFinalCheck Name of virtual file for which the final check is
-  /// applied.
-  /// \param FinalSelectPredicate Matcher that specifies the AST node in the
-  /// FileForFinalCheck for which the verification will be done.
-  /// \param VerificationMatcher Matcher that will be used for verification
-  /// after all imports in sequence are done.
-  void testImportSequence(const CodeFiles &CodeSamples,
-                          const std::vector<ImportAction> &ImportActions,
-                          StringRef FileForFinalCheck,
-                          BindableMatcher<Decl> FinalSelectPredicate,
-                          BindableMatcher<Decl> VerificationMatcher) {
-    AllASTUnits AllASTs;
-    using ImporterKey = std::pair<const ASTUnit *, const ASTUnit *>;
-    llvm::DenseMap<ImporterKey, std::unique_ptr<ASTImporter>> Importers;
-
-    auto GenASTsIfNeeded = [this, &AllASTs, &CodeSamples](StringRef Filename) {
-      if (!AllASTs.count(Filename)) {
-        auto Found = CodeSamples.find(Filename);
-        assert(Found != CodeSamples.end() && "Wrong file for import!");
-        AllASTs[Filename] = createASTUnit(Filename, Found->getValue());
-      }
-    };
-
-    for (const ImportAction &Action : ImportActions) {
-      StringRef FromFile = Action.FromFilename, ToFile = Action.ToFilename;
-      GenASTsIfNeeded(FromFile);
-      GenASTsIfNeeded(ToFile);
-
-      ASTUnit *From = AllASTs[FromFile].get();
-      ASTUnit *To = AllASTs[ToFile].get();
-
-      // Create a new importer if needed.
-      std::unique_ptr<ASTImporter> &ImporterRef = Importers[{From, To}];
-      if (!ImporterRef)
-        ImporterRef.reset(new ASTImporter(
-            To->getASTContext(), To->getFileManager(), From->getASTContext(),
-            From->getFileManager(), false));
-
-      // Find the declaration and import it.
-      auto FoundDecl = match(Action.ImportPredicate.bind(DeclToImportID),
-                             From->getASTContext());
-      EXPECT_TRUE(FoundDecl.size() == 1);
-      const Decl *ToImport = selectFirst<Decl>(DeclToImportID, FoundDecl);
-      auto Imported = importNode(From, To, *ImporterRef, ToImport);
-      EXPECT_TRUE(static_cast<bool>(Imported));
-      if (!Imported)
-        llvm::consumeError(Imported.takeError());
-    }
-
-    // Find the declaration and import it.
-    auto FoundDecl = match(FinalSelectPredicate.bind(DeclToVerifyID),
-                           AllASTs[FileForFinalCheck]->getASTContext());
-    EXPECT_TRUE(FoundDecl.size() == 1);
-    const Decl *ToVerify = selectFirst<Decl>(DeclToVerifyID, FoundDecl);
-    MatchVerifier<Decl> Verifier;
-    EXPECT_TRUE(
-        Verifier.match(ToVerify, BindableMatcher<Decl>(VerificationMatcher)));
-  }
-};
-
-template <typename T> RecordDecl *getRecordDecl(T *D) {
-  auto *ET = cast<ElaboratedType>(D->getType().getTypePtr());
-  return cast<RecordType>(ET->getNamedType().getTypePtr())->getDecl();
-}
-
 static const RecordDecl *getRecordDeclOfFriend(FriendDecl *FD) {
   QualType Ty = FD->getFriendType()->getType().getCanonicalType();
   return cast<RecordType>(Ty)->getDecl();
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
index 36e92c632c03a..4bd50191e1630 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
@@ -3219,13 +3219,13 @@ TEST_P(ASTMatchersTest, IsArray) {
 }
 
 TEST_P(ASTMatchersTest, HasArraySize) {
-  if (GetParam().Language != Lang_CXX03) {
-    // FIXME: Fix this test to work in all C++ language modes.
+  if (!GetParam().isCXX()) {
     return;
   }
 
   EXPECT_TRUE(matches("struct MyClass {}; MyClass *p1 = new MyClass[10];",
-                      cxxNewExpr(hasArraySize(integerLiteral(equals(10))))));
+                      cxxNewExpr(hasArraySize(
+                          ignoringParenImpCasts(integerLiteral(equals(10)))))));
 }
 
 TEST_P(ASTMatchersTest, HasDefinition_MatchesStructDefinition) {
diff --git a/clang/unittests/CMakeLists.txt b/clang/unittests/CMakeLists.txt
index 4c222e24599f0..9a52b9fb02620 100644
--- a/clang/unittests/CMakeLists.txt
+++ b/clang/unittests/CMakeLists.txt
@@ -1,6 +1,10 @@
 add_custom_target(ClangUnitTests)
 set_target_properties(ClangUnitTests PROPERTIES FOLDER "Clang tests")
 
+if (CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
+  add_compile_options("-Wno-suggest-override")
+endif()
+
 if(CLANG_BUILT_STANDALONE)
   # LLVMTestingSupport library is needed for some of the unittests.
   if (EXISTS ${LLVM_MAIN_SRC_DIR}/lib/Testing/Support
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index 6ac3ffbffd1c8..83dd5bbba911d 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -2165,6 +2165,33 @@ TEST_F(FormatTest, FormatsBitfields) {
                "  uchar : 8;\n"
                "  uchar other;\n"
                "};");
+  FormatStyle Style = getLLVMStyle();
+  Style.BitFieldColonSpacing = FormatStyle::BFCS_None;
+  verifyFormat("struct Bitfields {\n"
+               "  unsigned sClass:8;\n"
+               "  unsigned ValueKind:2;\n"
+               "  uchar other;\n"
+               "};",
+               Style);
+  verifyFormat("struct A {\n"
+               "  int aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa:1,\n"
+               "      bbbbbbbbbbbbbbbbbbbbbbbbb:2;\n"
+               "};",
+               Style);
+  Style.BitFieldColonSpacing = FormatStyle::BFCS_Before;
+  verifyFormat("struct Bitfields {\n"
+               "  unsigned sClass :8;\n"
+               "  unsigned ValueKind :2;\n"
+               "  uchar other;\n"
+               "};",
+               Style);
+  Style.BitFieldColonSpacing = FormatStyle::BFCS_After;
+  verifyFormat("struct Bitfields {\n"
+               "  unsigned sClass: 8;\n"
+               "  unsigned ValueKind: 2;\n"
+               "  uchar other;\n"
+               "};",
+               Style);
 }
 
 TEST_F(FormatTest, FormatsNamespaces) {
@@ -12156,6 +12183,21 @@ TEST_F(FormatTest, AlignConsecutiveBitFields) {
                "int       oneTwoThree : 23 = 0;",
                Alignment);
 
+  Alignment.BitFieldColonSpacing = FormatStyle::BFCS_None;
+  verifyFormat("int const a          :5;\n"
+               "int       oneTwoThree:23;",
+               Alignment);
+
+  Alignment.BitFieldColonSpacing = FormatStyle::BFCS_Before;
+  verifyFormat("int const a           :5;\n"
+               "int       oneTwoThree :23;",
+               Alignment);
+
+  Alignment.BitFieldColonSpacing = FormatStyle::BFCS_After;
+  verifyFormat("int const a          : 5;\n"
+               "int       oneTwoThree: 23;",
+               Alignment);
+
   // Known limitations: ':' is only recognized as a bitfield colon when
   // followed by a number.
   /*
@@ -14004,6 +14046,16 @@ TEST_F(FormatTest, ParsesConfiguration) {
   CHECK_PARSE("IndentExternBlock: false", IndentExternBlock,
               FormatStyle::IEBS_NoIndent);
 
+  Style.BitFieldColonSpacing = FormatStyle::BFCS_None;
+  CHECK_PARSE("BitFieldColonSpacing: Both", BitFieldColonSpacing,
+              FormatStyle::BFCS_Both);
+  CHECK_PARSE("BitFieldColonSpacing: None", BitFieldColonSpacing,
+              FormatStyle::BFCS_None);
+  CHECK_PARSE("BitFieldColonSpacing: Before", BitFieldColonSpacing,
+              FormatStyle::BFCS_Before);
+  CHECK_PARSE("BitFieldColonSpacing: After", BitFieldColonSpacing,
+              FormatStyle::BFCS_After);
+
   // FIXME: This is required because parsing a configuration simply overwrites
   // the first N elements of the list instead of resetting it.
   Style.ForEachMacros.clear();
diff --git a/clang/unittests/Tooling/Syntax/TokensTest.cpp b/clang/unittests/Tooling/Syntax/TokensTest.cpp
index 453ab608e6598..7d5943bf2520d 100644
--- a/clang/unittests/Tooling/Syntax/TokensTest.cpp
+++ b/clang/unittests/Tooling/Syntax/TokensTest.cpp
@@ -53,6 +53,7 @@ using namespace clang;
 using namespace clang::syntax;
 
 using llvm::ValueIs;
+using ::testing::_;
 using ::testing::AllOf;
 using ::testing::Contains;
 using ::testing::ElementsAre;
@@ -755,7 +756,7 @@ TEST_F(TokenBufferTest, ExpandedTokensForRange) {
   EXPECT_THAT(Buffer.expandedTokens(SourceRange()), testing::IsEmpty());
 }
 
-TEST_F(TokenBufferTest, ExpansionStartingAt) {
+TEST_F(TokenBufferTest, ExpansionsOverlapping) {
   // Object-like macro expansions.
   recordTokens(R"cpp(
     #define FOO 3+4
@@ -763,17 +764,25 @@ TEST_F(TokenBufferTest, ExpansionStartingAt) {
     int b = FOO 2;
   )cpp");
 
-  llvm::ArrayRef<syntax::Token> Foo1 = findSpelled("FOO 1").drop_back();
+  llvm::ArrayRef<syntax::Token> Foo1 = findSpelled("FOO 1");
   EXPECT_THAT(
       Buffer.expansionStartingAt(Foo1.data()),
-      ValueIs(IsExpansion(SameRange(Foo1),
+      ValueIs(IsExpansion(SameRange(Foo1.drop_back()),
                           SameRange(findExpanded("3 + 4 1").drop_back()))));
+  EXPECT_THAT(
+      Buffer.expansionsOverlapping(Foo1),
+      ElementsAre(IsExpansion(SameRange(Foo1.drop_back()),
+                              SameRange(findExpanded("3 + 4 1").drop_back()))));
 
-  llvm::ArrayRef<syntax::Token> Foo2 = findSpelled("FOO 2").drop_back();
+  llvm::ArrayRef<syntax::Token> Foo2 = findSpelled("FOO 2");
   EXPECT_THAT(
       Buffer.expansionStartingAt(Foo2.data()),
-      ValueIs(IsExpansion(SameRange(Foo2),
+      ValueIs(IsExpansion(SameRange(Foo2.drop_back()),
                           SameRange(findExpanded("3 + 4 2").drop_back()))));
+  EXPECT_THAT(Buffer.expansionsOverlapping(
+                  llvm::makeArrayRef(Foo1.begin(), Foo2.end())),
+              ElementsAre(IsExpansion(SameRange(Foo1.drop_back()), _),
+                          IsExpansion(SameRange(Foo2.drop_back()), _)));
 
   // Function-like macro expansions.
   recordTokens(R"cpp(
@@ -798,6 +807,11 @@ TEST_F(TokenBufferTest, ExpansionStartingAt) {
   for (const auto &T : ID2.drop_front())
     EXPECT_EQ(Buffer.expansionStartingAt(&T), llvm::None);
 
+  EXPECT_THAT(Buffer.expansionsOverlapping(llvm::makeArrayRef(
+                  findSpelled("1 + 2").data(), findSpelled("4").data())),
+              ElementsAre(IsExpansion(SameRange(ID1), _),
+                          IsExpansion(SameRange(ID2), _)));
+
   // PP directives.
   recordTokens(R"cpp(
 #define FOO 1
@@ -823,6 +837,11 @@ int b = 1;
   // Only the first spelled token should be found.
   for (const auto &T : PragmaOnce.drop_front())
     EXPECT_EQ(Buffer.expansionStartingAt(&T), llvm::None);
+
+  EXPECT_THAT(
+      Buffer.expansionsOverlapping(findSpelled("FOO ; # pragma")),
+      ElementsAre(IsExpansion(SameRange(findSpelled("FOO ;").drop_back()), _),
+                  IsExpansion(SameRange(PragmaOnce), _)));
 }
 
 TEST_F(TokenBufferTest, TokensToFileRange) {
diff --git a/clang/unittests/Tooling/TransformerTest.cpp b/clang/unittests/Tooling/TransformerTest.cpp
index 7d6b632937487..59b334b0ea5a4 100644
--- a/clang/unittests/Tooling/TransformerTest.cpp
+++ b/clang/unittests/Tooling/TransformerTest.cpp
@@ -440,6 +440,12 @@ TEST_F(TransformerTest, RemoveEdit) {
 }
 
 TEST_F(TransformerTest, WithMetadata) {
+  auto makeMetadata = [](const MatchFinder::MatchResult &R) -> llvm::Any {
+    int N =
+        R.Nodes.getNodeAs<IntegerLiteral>("int")->getValue().getLimitedValue();
+    return N;
+  };
+
   std::string Input = R"cc(
     int f() {
       int x = 5;
@@ -448,8 +454,11 @@ TEST_F(TransformerTest, WithMetadata) {
   )cc";
 
   Transformer T(
-      makeRule(declStmt().bind("decl"),
-               withMetadata(remove(statement(std::string("decl"))), 17)),
+      makeRule(
+          declStmt(containsDeclaration(0, varDecl(hasInitializer(
+                                              integerLiteral().bind("int")))))
+              .bind("decl"),
+          withMetadata(remove(statement(std::string("decl"))), makeMetadata)),
       consumer());
   T.registerMatchers(&MatchFinder);
   auto Factory = newFrontendActionFactory(&MatchFinder);
@@ -459,7 +468,7 @@ TEST_F(TransformerTest, WithMetadata) {
   ASSERT_EQ(Changes.size(), 1u);
   const llvm::Any &Metadata = Changes[0].getMetadata();
   ASSERT_TRUE(llvm::any_isa<int>(Metadata));
-  EXPECT_THAT(llvm::any_cast<int>(Metadata), 17);
+  EXPECT_THAT(llvm::any_cast<int>(Metadata), 5);
 }
 
 TEST_F(TransformerTest, MultiChange) {
diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html
index 65f196ad59d0f..c7369525c36fb 100755
--- a/clang/www/cxx_dr_status.html
+++ b/clang/www/cxx_dr_status.html
@@ -1504,7 +1504,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://wg21.link/cwg244">244</a></td>
     <td>CD1</td>
     <td>Destructor lookup</td>
-    <td class="unreleased" align="center">Clang 11</td>
+    <td class="full" align="center">Clang 11</td>
   </tr>
   <tr id="245">
     <td><a href="https://wg21.link/cwg245">245</a></td>
@@ -2789,7 +2789,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://wg21.link/cwg458">458</a></td>
     <td>C++11</td>
     <td>Hiding of member template parameters by other members</td>
-    <td class="unreleased" align="center">Clang 11</td>
+    <td class="full" align="center">Clang 11</td>
   </tr>
   <tr class="open" id="459">
     <td><a href="https://wg21.link/cwg459">459</a></td>
@@ -3967,7 +3967,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://wg21.link/cwg654">654</a></td>
     <td>CD1</td>
     <td>Conversions to and from <TT>nullptr_t</TT></td>
-    <td class="unreleased" align="center">Superseded by <a href="#1423">1423</a></td>
+    <td class="full" align="center">Superseded by <a href="#1423">1423</a></td>
   </tr>
   <tr id="655">
     <td><a href="https://wg21.link/cwg655">655</a></td>
@@ -8353,7 +8353,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://wg21.link/cwg1423">1423</a></td>
     <td>CD3</td>
     <td>Convertibility of <TT>nullptr</TT> to <TT>bool</TT></td>
-    <td class="unreleased" align="center">Clang 11</td>
+    <td class="full" align="center">Clang 11</td>
   </tr>
   <tr id="1424">
     <td><a href="https://wg21.link/cwg1424">1424</a></td>
@@ -8899,7 +8899,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://wg21.link/cwg1514">1514</a></td>
     <td>C++14</td>
     <td>Ambiguity between enumeration definition and zero-length bit-field</td>
-    <td class="unreleased" align="center">Clang 11</td>
+    <td class="full" align="center">Clang 11</td>
   </tr>
   <tr id="1515">
     <td><a href="https://wg21.link/cwg1515">1515</a></td>
@@ -10333,7 +10333,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://wg21.link/cwg1753">1753</a></td>
     <td>CD4</td>
     <td><I>decltype-specifier</I> in <I>nested-name-specifier</I> of destructor</td>
-    <td class="unreleased" align="center">Clang 11</td>
+    <td class="full" align="center">Clang 11</td>
   </tr>
   <tr id="1754">
     <td><a href="https://wg21.link/cwg1754">1754</a></td>
@@ -11611,7 +11611,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://wg21.link/cwg1966">1966</a></td>
     <td>CD4</td>
     <td>Colon following enumeration <I>elaborated-type-specifier</I></td>
-    <td class="unreleased" align="center">Clang 11</td>
+    <td class="full" align="center">Clang 11</td>
   </tr>
   <tr id="1967">
     <td><a href="https://wg21.link/cwg1967">1967</a></td>
@@ -11971,7 +11971,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://wg21.link/cwg2026">2026</a></td>
     <td>CD4</td>
     <td>Zero-initialization and <TT>constexpr</TT></td>
-    <td class="unreleased" align="center">Clang 11</td>
+    <td class="full" align="center">Clang 11</td>
   </tr>
   <tr id="2027">
     <td><a href="https://wg21.link/cwg2027">2027</a></td>
@@ -12307,7 +12307,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://wg21.link/cwg2082">2082</a></td>
     <td>CD4</td>
     <td>Referring to parameters in unevaluated operands of default arguments</td>
-    <td class="unreleased" align="center">Clang 11</td>
+    <td class="full" align="center">Clang 11</td>
   </tr>
   <tr id="2083">
     <td><a href="https://wg21.link/cwg2083">2083</a></td>
@@ -12757,7 +12757,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://wg21.link/cwg2157">2157</a></td>
     <td>CD4</td>
     <td>Further disambiguation of enumeration <I>elaborated-type-specifier</I></td>
-    <td class="unreleased" align="center">Clang 11</td>
+    <td class="full" align="center">Clang 11</td>
   </tr>
   <tr class="open" id="2158">
     <td><a href="https://wg21.link/cwg2158">2158</a></td>
@@ -13213,7 +13213,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://wg21.link/cwg2233">2233</a></td>
     <td>DRWP</td>
     <td>Function parameter packs following default arguments</td>
-    <td class="unreleased" align="center">Clang 11</td>
+    <td class="full" align="center">Clang 11</td>
   </tr>
   <tr id="2234">
     <td><a href="https://wg21.link/cwg2234">2234</a></td>
@@ -13891,7 +13891,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://wg21.link/cwg2346">2346</a></td>
     <td>DRWP</td>
     <td>Local variables in default arguments</td>
-    <td class="unreleased" align="center">Clang 11</td>
+    <td class="full" align="center">Clang 11</td>
   </tr>
   <tr id="2347">
     <td><a href="https://wg21.link/cwg2347">2347</a></td>
@@ -14499,6 +14499,24 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td>Unintended description of abbreviated function templates</td>
     <td class="none" align="center">Unknown</td>
   </tr>
+  <tr class="open" id="2448">
+    <td><a href="https://wg21.link/cwg2448">2448</a></td>
+    <td>open</td>
+    <td>Cv-qualification of arithmetic types and deprecation of volatile</td>
+    <td align="center">Not resolved</td>
+  </tr>
+  <tr class="open" id="2449">
+    <td><a href="https://wg21.link/cwg2449">2449</a></td>
+    <td>open</td>
+    <td>Thunks as an implementation technique for pointers to virtual functions</td>
+    <td align="center">Not resolved</td>
+  </tr>
+  <tr class="open" id="2450">
+    <td><a href="https://wg21.link/cwg2450">2450</a></td>
+    <td>open</td>
+    <td><I>braced-init-list</I> as a <I>template-argument</I></td>
+    <td align="center">Not resolved</td>
+  </tr>
 </table>
 
 </div>
diff --git a/clang/www/make_cxx_dr_status b/clang/www/make_cxx_dr_status
index cbb0dcc675b2b..3a9b18f59cfd5 100755
--- a/clang/www/make_cxx_dr_status
+++ b/clang/www/make_cxx_dr_status
@@ -93,7 +93,7 @@ print >> out_file, '''\
     <th>Available in Clang?</th>
   </tr>'''
 
-latest_release = 10
+latest_release = 11
 
 def availability(issue):
   status = status_map.get(issue, 'unknown')
diff --git a/compiler-rt/CMakeLists.txt b/compiler-rt/CMakeLists.txt
index fa62814b635d3..cfbd07a40e15f 100644
--- a/compiler-rt/CMakeLists.txt
+++ b/compiler-rt/CMakeLists.txt
@@ -3,7 +3,7 @@
 # An important constraint of the build is that it only produces libraries
 # based on the ability of the host toolchain to target various platforms.
 
-cmake_minimum_required(VERSION 3.4.3)
+cmake_minimum_required(VERSION 3.13.4)
 
 if(POLICY CMP0075)
   cmake_policy(SET CMP0075 NEW)
diff --git a/compiler-rt/cmake/Modules/AddCompilerRT.cmake b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
index dab55707338a4..efb6608182705 100644
--- a/compiler-rt/cmake/Modules/AddCompilerRT.cmake
+++ b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
@@ -403,6 +403,7 @@ set(COMPILER_RT_GMOCK_CFLAGS
 
 append_list_if(COMPILER_RT_DEBUG -DSANITIZER_DEBUG=1 COMPILER_RT_UNITTEST_CFLAGS)
 append_list_if(COMPILER_RT_HAS_WCOVERED_SWITCH_DEFAULT_FLAG -Wno-covered-switch-default COMPILER_RT_UNITTEST_CFLAGS)
+append_list_if(COMPILER_RT_HAS_WSUGGEST_OVERRIDE_FLAG -Wno-suggest-override COMPILER_RT_UNITTEST_CFLAGS)
 
 if(MSVC)
   # gtest use a lot of stuff marked as deprecated on Windows.
diff --git a/compiler-rt/cmake/Modules/CustomLibcxx/CMakeLists.txt b/compiler-rt/cmake/Modules/CustomLibcxx/CMakeLists.txt
index e61c222587e65..26d17ce6f14a7 100644
--- a/compiler-rt/cmake/Modules/CustomLibcxx/CMakeLists.txt
+++ b/compiler-rt/cmake/Modules/CustomLibcxx/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.4.3)
+cmake_minimum_required(VERSION 3.13.4)
 project(custom-libcxx C CXX)
 
 # Build static libcxxabi.
diff --git a/compiler-rt/cmake/config-ix.cmake b/compiler-rt/cmake/config-ix.cmake
index 2edc1dabd90d3..0a27910ed4943 100644
--- a/compiler-rt/cmake/config-ix.cmake
+++ b/compiler-rt/cmake/config-ix.cmake
@@ -106,6 +106,7 @@ check_cxx_compiler_flag("-Werror -Wnon-virtual-dtor"   COMPILER_RT_HAS_WNON_VIRT
 check_cxx_compiler_flag("-Werror -Wvariadic-macros"    COMPILER_RT_HAS_WVARIADIC_MACROS_FLAG)
 check_cxx_compiler_flag("-Werror -Wunused-parameter"   COMPILER_RT_HAS_WUNUSED_PARAMETER_FLAG)
 check_cxx_compiler_flag("-Werror -Wcovered-switch-default" COMPILER_RT_HAS_WCOVERED_SWITCH_DEFAULT_FLAG)
+check_cxx_compiler_flag("-Werror -Wsuggest-override"   COMPILER_RT_HAS_WSUGGEST_OVERRIDE_FLAG)
 check_cxx_compiler_flag(-Wno-pedantic COMPILER_RT_HAS_WNO_PEDANTIC)
 
 check_cxx_compiler_flag(/W4 COMPILER_RT_HAS_W4_FLAG)
@@ -121,7 +122,8 @@ check_cxx_compiler_flag(/wd4800 COMPILER_RT_HAS_WD4800_FLAG)
 check_symbol_exists(__func__ "" COMPILER_RT_HAS_FUNC_SYMBOL)
 
 # Includes.
-check_include_files("sys/auxv.h" COMPILER_RT_HAS_AUXV)
+check_cxx_compiler_flag(-nostdinc++ COMPILER_RT_HAS_NOSTDINCXX_FLAG)
+check_include_files("sys/auxv.h"    COMPILER_RT_HAS_AUXV)
 
 # Libraries.
 check_library_exists(dl dlopen "" COMPILER_RT_HAS_LIBDL)
@@ -608,7 +610,8 @@ else()
   set(CAN_SYMBOLIZE 1)
 endif()
 
-find_program(GOLD_EXECUTABLE NAMES ${LLVM_DEFAULT_TARGET_TRIPLE}-ld.gold ld.gold ${LLVM_DEFAULT_TARGET_TRIPLE}-ld ld DOC "The gold linker")
+find_program(GNU_LD_EXECUTABLE NAMES ${LLVM_DEFAULT_TARGET_TRIPLE}-ld.bfd ld.bfd DOC "GNU ld")
+find_program(GOLD_EXECUTABLE NAMES ${LLVM_DEFAULT_TARGET_TRIPLE}-ld.gold ld.gold DOC "GNU gold")
 
 if(COMPILER_RT_SUPPORTED_ARCH)
   list(REMOVE_DUPLICATES COMPILER_RT_SUPPORTED_ARCH)
diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc
index 6d0ffb12294b0..a6913527e67f0 100644
--- a/compiler-rt/include/profile/InstrProfData.inc
+++ b/compiler-rt/include/profile/InstrProfData.inc
@@ -157,8 +157,6 @@ VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA
 #ifndef VALUE_RANGE_PROF
 VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
 #else /* VALUE_RANGE_PROF */
-/* FIXME: This is to be removed after switching to the new memop value
- * profiling. */
 VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) \
                       INSTR_PROF_COMMA
 VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeStart, Type::getInt64Ty(Ctx)) \
@@ -755,14 +753,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
 #define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
 #define INSTR_PROF_VALUE_PROF_FUNC_STR \
         INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
-/* FIXME: This is to be removed after switching to the new memop value
- * profiling. */
 #define INSTR_PROF_VALUE_RANGE_PROF_FUNC __llvm_profile_instrument_range
 #define INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR \
         INSTR_PROF_QUOTE(INSTR_PROF_VALUE_RANGE_PROF_FUNC)
-#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
-#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR                                   \
-  INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
 
 /* InstrProfile per-function control data alignment.  */
 #define INSTR_PROF_DATA_ALIGNMENT 8
@@ -790,121 +783,3 @@ typedef struct InstrProfValueData {
 #endif
 
 #undef COVMAP_V2_OR_V3
-
-#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
-
-#ifdef __cplusplus
-#define INSTR_PROF_INLINE inline
-#else
-#define INSTR_PROF_INLINE
-#endif
-
-/* The value range buckets (22 buckets) for the memop size value profiling looks
- * like:
- *
- *   [0, 0]
- *   [1, 1]
- *   [2, 2]
- *   [3, 3]
- *   [4, 4]
- *   [5, 5]
- *   [6, 6]
- *   [7, 7]
- *   [8, 8]
- *   [9, 15]
- *   [16, 16]
- *   [17, 31]
- *   [32, 32]
- *   [33, 63]
- *   [64, 64]
- *   [65, 127]
- *   [128, 128]
- *   [129, 255]
- *   [256, 256]
- *   [257, 511]
- *   [512, 512]
- *   [513, UINT64_MAX]
- *
- * Each range has a 'representative value' which is the lower end value of the
- * range and used to store in the runtime profile data records and the VP
- * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127].
- */
-
-/*
- * Clz and Popcount. This code was copied from
- * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and
- * llvm/include/llvm/Support/MathExtras.h.
- */
-#if defined(_MSC_VER) && !defined(__clang__)
-
-#include <intrin.h>
-INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
-int InstProfClzll(unsigned long long X) {
-  unsigned long LeadZeroIdx = 0;
-#if !defined(_M_ARM64) && !defined(_M_X64)
-  // Scan the high 32 bits.
-  if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32)))
-    return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset
-                                                      // from the MSB.
-  // Scan the low 32 bits.
-  if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X)))
-    return (int)(63 - LeadZeroIdx);
-#else
-  if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx;
-#endif
-  return 64;
-}
-INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
-int InstProfPopcountll(unsigned long long X) {
-  // This code originates from https://reviews.llvm.org/rG30626254510f.
-  unsigned long long v = X;
-  v = v - ((v >> 1) & 0x5555555555555555ULL);
-  v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
-  v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
-  return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56);
-}
-
-#else
-
-INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
-int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); }
-INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
-int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); }
-
-#endif  /* defined(_MSC_VER) && !defined(__clang__) */
-
-/* Map an (observed) memop size value to the representative value of its range.
- * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */
-INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t
-InstrProfGetRangeRepValue(uint64_t Value) {
-  if (Value <= 8)
-    // The first ranges are individually tracked. Use the value as is.
-    return Value;
-  else if (Value >= 513)
-    // The last range is mapped to its lowest value.
-    return 513;
-  else if (InstProfPopcountll(Value) == 1)
-    // If it's a power of two, use it as is.
-    return Value;
-  else
-    // Otherwise, take to the previous power of two + 1.
-    return (1 << (64 - InstProfClzll(Value) - 1)) + 1;
-}
-
-/* Return true if the range that an (observed) memop size value belongs to has
- * only a single value in the range.  For example, 0 -> true, 8 -> true, 10 ->
- * false, 64 -> true, 100 -> false, 513 -> false. */
-INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned
-InstrProfIsSingleValRange(uint64_t Value) {
-  if (Value <= 8)
-    // The first ranges are individually tracked.
-    return 1;
-  else if (InstProfPopcountll(Value) == 1)
-    // If it's a power of two, there's only one value.
-    return 1;
-  else
-    // Otherwise, there's more than one value in the range.
-    return 0;
-}
-
-#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */
diff --git a/compiler-rt/lib/asan/asan_fuchsia.cpp b/compiler-rt/lib/asan/asan_fuchsia.cpp
index 64f6dcbcefeb2..ec15abfa0fe89 100644
--- a/compiler-rt/lib/asan/asan_fuchsia.cpp
+++ b/compiler-rt/lib/asan/asan_fuchsia.cpp
@@ -198,6 +198,10 @@ bool HandleDlopenInit() {
   return false;
 }
 
+void FlushUnneededASanShadowMemory(uptr p, uptr size) {
+  __sanitizer_fill_shadow(p, size, 0, 0);
+}
+
 }  // namespace __asan
 
 // These are declared (in extern "C") by <zircon/sanitizer.h>.
diff --git a/compiler-rt/lib/asan/asan_linux.cpp b/compiler-rt/lib/asan/asan_linux.cpp
index aa93bbd79d132..fb1a442b3d437 100644
--- a/compiler-rt/lib/asan/asan_linux.cpp
+++ b/compiler-rt/lib/asan/asan_linux.cpp
@@ -114,6 +114,12 @@ void AsanApplyToGlobals(globals_op_fptr op, const void *needle) {
   UNIMPLEMENTED();
 }
 
+void FlushUnneededASanShadowMemory(uptr p, uptr size) {
+  // Since asan's mapping is compacting, the shadow chunk may be
+  // not page-aligned, so we only flush the page-aligned portion.
+  ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size));
+}
+
 #if SANITIZER_ANDROID
 // FIXME: should we do anything for Android?
 void AsanCheckDynamicRTPrereqs() {}
diff --git a/compiler-rt/lib/asan/asan_mac.cpp b/compiler-rt/lib/asan/asan_mac.cpp
index 3182aacb0b5e9..c6950547f089f 100644
--- a/compiler-rt/lib/asan/asan_mac.cpp
+++ b/compiler-rt/lib/asan/asan_mac.cpp
@@ -89,6 +89,12 @@ void AsanApplyToGlobals(globals_op_fptr op, const void *needle) {
   op(globals, size / sizeof(__asan_global));
 }
 
+void FlushUnneededASanShadowMemory(uptr p, uptr size) {
+  // Since asan's mapping is compacting, the shadow chunk may be
+  // not page-aligned, so we only flush the page-aligned portion.
+  ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size));
+}
+
 void ReadContextStack(void *context, uptr *stack, uptr *ssize) {
   UNIMPLEMENTED();
 }
diff --git a/compiler-rt/lib/asan/asan_poisoning.cpp b/compiler-rt/lib/asan/asan_poisoning.cpp
index f3fbe684e2cbc..44f872ef61909 100644
--- a/compiler-rt/lib/asan/asan_poisoning.cpp
+++ b/compiler-rt/lib/asan/asan_poisoning.cpp
@@ -62,12 +62,6 @@ struct ShadowSegmentEndpoint {
   }
 };
 
-void FlushUnneededASanShadowMemory(uptr p, uptr size) {
-  // Since asan's mapping is compacting, the shadow chunk may be
-  // not page-aligned, so we only flush the page-aligned portion.
-  ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size));
-}
-
 void AsanPoisonOrUnpoisonIntraObjectRedzone(uptr ptr, uptr size, bool poison) {
   uptr end = ptr + size;
   if (Verbosity()) {
diff --git a/compiler-rt/lib/asan/asan_rtems.cpp b/compiler-rt/lib/asan/asan_rtems.cpp
index 2e5b2f0a3b210..ea0b4ad9db681 100644
--- a/compiler-rt/lib/asan/asan_rtems.cpp
+++ b/compiler-rt/lib/asan/asan_rtems.cpp
@@ -50,6 +50,12 @@ void AsanApplyToGlobals(globals_op_fptr op, const void *needle) {
   UNIMPLEMENTED();
 }
 
+void FlushUnneededASanShadowMemory(uptr p, uptr size) {
+  // Since asan's mapping is compacting, the shadow chunk may be
+  // not page-aligned, so we only flush the page-aligned portion.
+  ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size));
+}
+
 void AsanCheckDynamicRTPrereqs() {}
 void AsanCheckIncompatibleRT() {}
 void InitializeAsanInterceptors() {}
diff --git a/compiler-rt/lib/asan/asan_win.cpp b/compiler-rt/lib/asan/asan_win.cpp
index fe635c2d5b6b4..8044ae16ff9bd 100644
--- a/compiler-rt/lib/asan/asan_win.cpp
+++ b/compiler-rt/lib/asan/asan_win.cpp
@@ -191,6 +191,12 @@ void AsanApplyToGlobals(globals_op_fptr op, const void *needle) {
   UNIMPLEMENTED();
 }
 
+void FlushUnneededASanShadowMemory(uptr p, uptr size) {
+  // Since asan's mapping is compacting, the shadow chunk may be
+  // not page-aligned, so we only flush the page-aligned portion.
+  ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size));
+}
+
 // ---------------------- TSD ---------------- {{{
 static bool tsd_key_inited = false;
 
diff --git a/compiler-rt/lib/asan/scripts/asan_device_setup b/compiler-rt/lib/asan/scripts/asan_device_setup
index 041bf92e62533..27fedea3f863a 100755
--- a/compiler-rt/lib/asan/scripts/asan_device_setup
+++ b/compiler-rt/lib/asan/scripts/asan_device_setup
@@ -330,7 +330,7 @@ function generate_zygote_wrapper { # from, to
 ASAN_OPTIONS=$ASAN_OPTIONS \\
 ASAN_ACTIVATION_OPTIONS=include_if_exists=/data/local/tmp/asan.options.%b \\
 LD_PRELOAD=$_ld_preload \\
-exec $_to \$@
+exec $_to "\$@"
 
 EOF
 }
diff --git a/compiler-rt/lib/builtins/CMakeLists.txt b/compiler-rt/lib/builtins/CMakeLists.txt
index 058bfc815a1a6..99aed905d7596 100644
--- a/compiler-rt/lib/builtins/CMakeLists.txt
+++ b/compiler-rt/lib/builtins/CMakeLists.txt
@@ -3,7 +3,7 @@
 # architecture-specific code in various subdirectories.
 
 if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
-  cmake_minimum_required(VERSION 3.4.3)
+  cmake_minimum_required(VERSION 3.13.4)
 
   project(CompilerRTBuiltins C ASM)
   set(COMPILER_RT_STANDALONE_BUILD TRUE)
diff --git a/compiler-rt/lib/dfsan/dfsan.cpp b/compiler-rt/lib/dfsan/dfsan.cpp
index 0e2fb9f5f3347..105989c93ab10 100644
--- a/compiler-rt/lib/dfsan/dfsan.cpp
+++ b/compiler-rt/lib/dfsan/dfsan.cpp
@@ -158,6 +158,12 @@ static void dfsan_check_label(dfsan_label label) {
   }
 }
 
+static void ReportUnsupportedFast16(const char *func) {
+  Report("FATAL: DataFlowSanitizer: %s is unsupported in fast16labels mode\n",
+         func);
+  Die();
+}
+
 // Resolves the union of two unequal labels.  Nonequality is a precondition for
 // this function (the instrumentation pass inlines the equality test).
 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
@@ -253,8 +259,10 @@ dfsan_union(dfsan_label l1, dfsan_label l2) {
 
 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
 dfsan_label dfsan_create_label(const char *desc, void *userdata) {
+  if (flags().fast16labels)
+    ReportUnsupportedFast16("dfsan_create_label");
   dfsan_label label =
-    atomic_fetch_add(&__dfsan_last_label, 1, memory_order_relaxed) + 1;
+      atomic_fetch_add(&__dfsan_last_label, 1, memory_order_relaxed) + 1;
   dfsan_check_label(label);
   __dfsan_label_info[label].l1 = __dfsan_label_info[label].l2 = 0;
   __dfsan_label_info[label].desc = desc;
@@ -311,11 +319,15 @@ dfsan_read_label(const void *addr, uptr size) {
 
 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
 const struct dfsan_label_info *dfsan_get_label_info(dfsan_label label) {
+  if (flags().fast16labels)
+    ReportUnsupportedFast16("dfsan_get_label_info");
   return &__dfsan_label_info[label];
 }
 
 extern "C" SANITIZER_INTERFACE_ATTRIBUTE int
 dfsan_has_label(dfsan_label label, dfsan_label elem) {
+  if (flags().fast16labels)
+    return label & elem;
   if (label == elem)
     return true;
   const dfsan_label_info *info = dfsan_get_label_info(label);
@@ -328,6 +340,8 @@ dfsan_has_label(dfsan_label label, dfsan_label elem) {
 
 extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_label
 dfsan_has_label_with_desc(dfsan_label label, const char *desc) {
+  if (flags().fast16labels)
+    ReportUnsupportedFast16("dfsan_has_label_with_desc");
   const dfsan_label_info *info = dfsan_get_label_info(label);
   if (info->l1 != 0) {
     return dfsan_has_label_with_desc(info->l1, desc) ||
@@ -347,9 +361,11 @@ dfsan_get_label_count(void) {
 
 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
 dfsan_dump_labels(int fd) {
+  if (flags().fast16labels)
+    return;
+
   dfsan_label last_label =
       atomic_load(&__dfsan_last_label, memory_order_relaxed);
-
   for (uptr l = 1; l <= last_label; ++l) {
     char buf[64];
     internal_snprintf(buf, sizeof(buf), "%u %u %u ", l,
diff --git a/compiler-rt/lib/fuzzer/CMakeLists.txt b/compiler-rt/lib/fuzzer/CMakeLists.txt
index b5be6b89452e9..b98c27145687b 100644
--- a/compiler-rt/lib/fuzzer/CMakeLists.txt
+++ b/compiler-rt/lib/fuzzer/CMakeLists.txt
@@ -46,6 +46,8 @@ set(LIBFUZZER_HEADERS
   FuzzerUtil.h
   FuzzerValueBitMap.h)
 
+include_directories(../../include)
+
 CHECK_CXX_SOURCE_COMPILES("
   static thread_local int blah;
   int main() {
@@ -99,6 +101,13 @@ add_compiler_rt_object_libraries(RTfuzzer_main
   CFLAGS ${LIBFUZZER_CFLAGS}
   DEPS ${LIBFUZZER_DEPS})
 
+add_compiler_rt_object_libraries(RTfuzzer_interceptors
+  OS ${FUZZER_SUPPORTED_OS}
+  ARCHS ${FUZZER_SUPPORTED_ARCH}
+  SOURCES FuzzerInterceptors.cpp
+  CFLAGS ${LIBFUZZER_CFLAGS}
+  DEPS ${LIBFUZZER_DEPS})
+
 add_compiler_rt_runtime(clang_rt.fuzzer
   STATIC
   OS ${FUZZER_SUPPORTED_OS}
@@ -115,6 +124,14 @@ add_compiler_rt_runtime(clang_rt.fuzzer_no_main
   CFLAGS ${LIBFUZZER_CFLAGS}
   PARENT_TARGET fuzzer)
 
+add_compiler_rt_runtime(clang_rt.fuzzer_interceptors
+  STATIC
+  OS ${FUZZER_SUPPORTED_OS}
+  ARCHS ${FUZZER_SUPPORTED_ARCH}
+  OBJECT_LIBS RTfuzzer_interceptors
+  CFLAGS ${LIBFUZZER_CFLAGS}
+  PARENT_TARGET fuzzer)
+
 if(OS_NAME MATCHES "Linux|Fuchsia" AND
    COMPILER_RT_LIBCXX_PATH AND
    COMPILER_RT_LIBCXXABI_PATH)
@@ -148,7 +165,10 @@ if(OS_NAME MATCHES "Linux|Fuchsia" AND
     add_dependencies(RTfuzzer.${arch} libcxx_fuzzer_${arch}-build)
     target_compile_options(RTfuzzer_main.${arch} PRIVATE -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1)
     add_dependencies(RTfuzzer_main.${arch} libcxx_fuzzer_${arch}-build)
+    target_compile_options(RTfuzzer_interceptors.${arch} PRIVATE -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1)
+    add_dependencies(RTfuzzer_interceptors.${arch} libcxx_fuzzer_${arch}-build)
     partially_link_libcxx(fuzzer_no_main ${LIBCXX_${arch}_PREFIX} ${arch})
+    partially_link_libcxx(fuzzer_interceptors ${LIBCXX_${arch}_PREFIX} ${arch})
     partially_link_libcxx(fuzzer ${LIBCXX_${arch}_PREFIX} ${arch})
   endforeach()
 endif()
diff --git a/compiler-rt/lib/fuzzer/FuzzerInterceptors.cpp b/compiler-rt/lib/fuzzer/FuzzerInterceptors.cpp
new file mode 100644
index 0000000000000..a1a64780de34d
--- /dev/null
+++ b/compiler-rt/lib/fuzzer/FuzzerInterceptors.cpp
@@ -0,0 +1,235 @@
+//===-- FuzzerInterceptors.cpp --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Intercept certain libc functions to aid fuzzing.
+// Linked only when other RTs that define their own interceptors are not linked.
+//===----------------------------------------------------------------------===//
+
+#include "FuzzerPlatform.h"
+
+#if LIBFUZZER_LINUX
+
+#define GET_CALLER_PC() __builtin_return_address(0)
+
+#define PTR_TO_REAL(x) real_##x
+#define REAL(x) __interception::PTR_TO_REAL(x)
+#define FUNC_TYPE(x) x##_type
+#define DEFINE_REAL(ret_type, func, ...)                                       \
+  typedef ret_type (*FUNC_TYPE(func))(__VA_ARGS__);                            \
+  namespace __interception {                                                   \
+  FUNC_TYPE(func) PTR_TO_REAL(func);                                           \
+  }
+
+#include <cassert>
+#include <cstdint>
+#include <dlfcn.h> // for dlsym()
+#include <sanitizer/common_interface_defs.h>
+
+static void *getFuncAddr(const char *name, uintptr_t wrapper_addr) {
+  void *addr = dlsym(RTLD_NEXT, name);
+  if (!addr) {
+    // If the lookup using RTLD_NEXT failed, the sanitizer runtime library is
+    // later in the library search order than the DSO that we are trying to
+    // intercept, which means that we cannot intercept this function. We still
+    // want the address of the real definition, though, so look it up using
+    // RTLD_DEFAULT.
+    addr = dlsym(RTLD_DEFAULT, name);
+
+    // In case `name' is not loaded, dlsym ends up finding the actual wrapper.
+    // We don't want to intercept the wrapper and have it point to itself.
+    if (reinterpret_cast<uintptr_t>(addr) == wrapper_addr)
+      addr = nullptr;
+  }
+  return addr;
+}
+
+static int FuzzerInited = 0;
+static bool FuzzerInitIsRunning;
+
+static void fuzzerInit();
+
+static void ensureFuzzerInited() {
+  assert(!FuzzerInitIsRunning);
+  if (!FuzzerInited) {
+    fuzzerInit();
+  }
+}
+
+static int internal_strcmp_strncmp(const char *s1, const char *s2, bool strncmp,
+                                   size_t n) {
+  size_t i = 0;
+  while (true) {
+    if (strncmp) {
+      if (i == n)
+        break;
+      i++;
+    }
+    unsigned c1 = *s1;
+    unsigned c2 = *s2;
+    if (c1 != c2)
+      return (c1 < c2) ? -1 : 1;
+    if (c1 == 0)
+      break;
+    s1++;
+    s2++;
+  }
+  return 0;
+}
+
+static int internal_strncmp(const char *s1, const char *s2, size_t n) {
+  return internal_strcmp_strncmp(s1, s2, true, n);
+}
+
+static int internal_strcmp(const char *s1, const char *s2) {
+  return internal_strcmp_strncmp(s1, s2, false, 0);
+}
+
+static int internal_memcmp(const void *s1, const void *s2, size_t n) {
+  const uint8_t *t1 = static_cast<const uint8_t *>(s1);
+  const uint8_t *t2 = static_cast<const uint8_t *>(s2);
+  for (size_t i = 0; i < n; ++i, ++t1, ++t2)
+    if (*t1 != *t2)
+      return *t1 < *t2 ? -1 : 1;
+  return 0;
+}
+
+static size_t internal_strlen(const char *s) {
+  size_t i = 0;
+  while (s[i])
+    i++;
+  return i;
+}
+
+static char *internal_strstr(const char *haystack, const char *needle) {
+  // This is O(N^2), but we are not using it in hot places.
+  size_t len1 = internal_strlen(haystack);
+  size_t len2 = internal_strlen(needle);
+  if (len1 < len2)
+    return nullptr;
+  for (size_t pos = 0; pos <= len1 - len2; pos++) {
+    if (internal_memcmp(haystack + pos, needle, len2) == 0)
+      return const_cast<char *>(haystack) + pos;
+  }
+  return nullptr;
+}
+
+extern "C" {
+
+DEFINE_REAL(int, bcmp, const void *, const void *, size_t)
+DEFINE_REAL(int, memcmp, const void *, const void *, size_t)
+DEFINE_REAL(int, strncmp, const char *, const char *, size_t)
+DEFINE_REAL(int, strcmp, const char *, const char *)
+DEFINE_REAL(int, strncasecmp, const char *, const char *, size_t)
+DEFINE_REAL(int, strcasecmp, const char *, const char *)
+DEFINE_REAL(char *, strstr, const char *, const char *)
+DEFINE_REAL(char *, strcasestr, const char *, const char *)
+DEFINE_REAL(void *, memmem, const void *, size_t, const void *, size_t)
+
+ATTRIBUTE_INTERFACE int bcmp(const char *s1, const char *s2, size_t n) {
+  if (!FuzzerInited)
+    return internal_memcmp(s1, s2, n);
+  int result = REAL(bcmp)(s1, s2, n);
+  __sanitizer_weak_hook_memcmp(GET_CALLER_PC(), s1, s2, n, result);
+  return result;
+}
+
+ATTRIBUTE_INTERFACE int memcmp(const void *s1, const void *s2, size_t n) {
+  if (!FuzzerInited)
+    return internal_memcmp(s1, s2, n);
+  int result = REAL(memcmp)(s1, s2, n);
+  __sanitizer_weak_hook_memcmp(GET_CALLER_PC(), s1, s2, n, result);
+  return result;
+}
+
+ATTRIBUTE_INTERFACE int strncmp(const char *s1, const char *s2, size_t n) {
+  if (!FuzzerInited)
+    return internal_strncmp(s1, s2, n);
+  int result = REAL(strncmp)(s1, s2, n);
+  __sanitizer_weak_hook_strncmp(GET_CALLER_PC(), s1, s2, n, result);
+  return result;
+}
+
+ATTRIBUTE_INTERFACE int strcmp(const char *s1, const char *s2) {
+  if (!FuzzerInited)
+    return internal_strcmp(s1, s2);
+  int result = REAL(strcmp)(s1, s2);
+  __sanitizer_weak_hook_strcmp(GET_CALLER_PC(), s1, s2, result);
+  return result;
+}
+
+ATTRIBUTE_INTERFACE int strncasecmp(const char *s1, const char *s2, size_t n) {
+  ensureFuzzerInited();
+  int result = REAL(strncasecmp)(s1, s2, n);
+  __sanitizer_weak_hook_strncasecmp(GET_CALLER_PC(), s1, s2, n, result);
+  return result;
+}
+
+ATTRIBUTE_INTERFACE int strcasecmp(const char *s1, const char *s2) {
+  ensureFuzzerInited();
+  int result = REAL(strcasecmp)(s1, s2);
+  __sanitizer_weak_hook_strcasecmp(GET_CALLER_PC(), s1, s2, result);
+  return result;
+}
+
+ATTRIBUTE_INTERFACE char *strstr(const char *s1, const char *s2) {
+  if (!FuzzerInited)
+    return internal_strstr(s1, s2);
+  char *result = REAL(strstr)(s1, s2);
+  __sanitizer_weak_hook_strstr(GET_CALLER_PC(), s1, s2, result);
+  return result;
+}
+
+ATTRIBUTE_INTERFACE char *strcasestr(const char *s1, const char *s2) {
+  ensureFuzzerInited();
+  char *result = REAL(strcasestr)(s1, s2);
+  __sanitizer_weak_hook_strcasestr(GET_CALLER_PC(), s1, s2, result);
+  return result;
+}
+
+ATTRIBUTE_INTERFACE
+void *memmem(const void *s1, size_t len1, const void *s2, size_t len2) {
+  ensureFuzzerInited();
+  void *result = REAL(memmem)(s1, len1, s2, len2);
+  __sanitizer_weak_hook_memmem(GET_CALLER_PC(), s1, len1, s2, len2, result);
+  return result;
+}
+
+__attribute__((section(".preinit_array"),
+               used)) static void (*__local_fuzzer_preinit)(void) = fuzzerInit;
+
+} // extern "C"
+
+static void fuzzerInit() {
+  assert(!FuzzerInitIsRunning);
+  if (FuzzerInited)
+    return;
+  FuzzerInitIsRunning = true;
+
+  REAL(bcmp) = reinterpret_cast<memcmp_type>(
+      getFuncAddr("bcmp", reinterpret_cast<uintptr_t>(&bcmp)));
+  REAL(memcmp) = reinterpret_cast<memcmp_type>(
+      getFuncAddr("memcmp", reinterpret_cast<uintptr_t>(&memcmp)));
+  REAL(strncmp) = reinterpret_cast<strncmp_type>(
+      getFuncAddr("strncmp", reinterpret_cast<uintptr_t>(&strncmp)));
+  REAL(strcmp) = reinterpret_cast<strcmp_type>(
+      getFuncAddr("strcmp", reinterpret_cast<uintptr_t>(&strcmp)));
+  REAL(strncasecmp) = reinterpret_cast<strncasecmp_type>(
+      getFuncAddr("strncasecmp", reinterpret_cast<uintptr_t>(&strncasecmp)));
+  REAL(strcasecmp) = reinterpret_cast<strcasecmp_type>(
+      getFuncAddr("strcasecmp", reinterpret_cast<uintptr_t>(&strcasecmp)));
+  REAL(strstr) = reinterpret_cast<strstr_type>(
+      getFuncAddr("strstr", reinterpret_cast<uintptr_t>(&strstr)));
+  REAL(strcasestr) = reinterpret_cast<strcasestr_type>(
+      getFuncAddr("strcasestr", reinterpret_cast<uintptr_t>(&strcasestr)));
+  REAL(memmem) = reinterpret_cast<memmem_type>(
+      getFuncAddr("memmem", reinterpret_cast<uintptr_t>(&memmem)));
+
+  FuzzerInitIsRunning = false;
+  FuzzerInited = 1;
+}
+
+#endif
diff --git a/compiler-rt/lib/gwp_asan/CMakeLists.txt b/compiler-rt/lib/gwp_asan/CMakeLists.txt
index cb550bd121470..2fc789c96b137 100644
--- a/compiler-rt/lib/gwp_asan/CMakeLists.txt
+++ b/compiler-rt/lib/gwp_asan/CMakeLists.txt
@@ -31,10 +31,8 @@ set(GWP_ASAN_HEADERS
 # allocators. Some supporting allocators (e.g. scudo standalone) cannot use any
 # parts of the C++ standard library.
 set(GWP_ASAN_CFLAGS ${SANITIZER_COMMON_CFLAGS} -fno-rtti -fno-exceptions
-    -nostdinc++ -pthread)
+    -nostdinc++ -pthread -fno-omit-frame-pointer)
 append_list_if(COMPILER_RT_HAS_FPIC_FLAG -fPIC GWP_ASAN_CFLAGS)
-append_list_if(COMPILER_RT_HAS_OMIT_FRAME_POINTER_FLAG -fno-omit-frame-pointer
-               GWP_ASAN_CFLAGS)
 
 # Remove -stdlib= which is unused when passing -nostdinc++.
 string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
diff --git a/compiler-rt/lib/profile/CMakeLists.txt b/compiler-rt/lib/profile/CMakeLists.txt
index 63532b72ff82e..5ff0e10182b4d 100644
--- a/compiler-rt/lib/profile/CMakeLists.txt
+++ b/compiler-rt/lib/profile/CMakeLists.txt
@@ -97,7 +97,7 @@ if(COMPILER_RT_TARGET_HAS_ATOMICS)
  set(EXTRA_FLAGS
      ${EXTRA_FLAGS}
      -DCOMPILER_RT_HAS_ATOMICS=1)
-endif() 
+endif()
 
 if(COMPILER_RT_TARGET_HAS_FCNTL_LCK)
  set(EXTRA_FLAGS
@@ -111,6 +111,9 @@ if(COMPILER_RT_TARGET_HAS_UNAME)
      -DCOMPILER_RT_HAS_UNAME=1)
 endif()
 
+# We don't use the C++ Standard Library here, so avoid including it by mistake.
+append_list_if(COMPILER_RT_HAS_NOSTDINCXX_FLAG -nostdinc++ EXTRA_FLAGS)
+
 # This appears to be a C-only warning banning the use of locals in aggregate
 # initializers. All other compilers accept this, though.
 # nonstandard extension used : 'identifier' : cannot be initialized using address of automatic variable
diff --git a/compiler-rt/lib/profile/InstrProfilingValue.c b/compiler-rt/lib/profile/InstrProfilingValue.c
index 76e1d3fa11b80..29b9e628a9c95 100644
--- a/compiler-rt/lib/profile/InstrProfilingValue.c
+++ b/compiler-rt/lib/profile/InstrProfilingValue.c
@@ -17,14 +17,13 @@
 
 #define INSTR_PROF_VALUE_PROF_DATA
 #define INSTR_PROF_COMMON_API_IMPL
-#define INSTR_PROF_VALUE_PROF_MEMOP_API
 #include "profile/InstrProfData.inc"
 
 static int hasStaticCounters = 1;
 static int OutOfNodesWarnings = 0;
 static int hasNonDefaultValsPerSite = 0;
 #define INSTR_PROF_MAX_VP_WARNS 10
-#define INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE 24
+#define INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE 16
 #define INSTR_PROF_VNODE_POOL_SIZE 1024
 
 #ifndef _MSC_VER
@@ -94,6 +93,8 @@ static int allocateValueProfileCounters(__llvm_profile_data *Data) {
   for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
     NumVSites += Data->NumValueSites[VKI];
 
+  if (NumVSites == 0)
+    return 0;
   ValueProfNode **Mem =
       (ValueProfNode **)calloc(NumVSites, sizeof(ValueProfNode *));
   if (!Mem)
@@ -251,8 +252,6 @@ __llvm_profile_instrument_target_value(uint64_t TargetValue, void *Data,
  * The range for large values is optional. The default value of INT64_MIN
  * indicates it is not specified.
  */
-/* FIXME: This is to be removed after switching to the new memop value
- * profiling. */
 COMPILER_RT_VISIBILITY void __llvm_profile_instrument_range(
     uint64_t TargetValue, void *Data, uint32_t CounterIndex,
     int64_t PreciseRangeStart, int64_t PreciseRangeLast, int64_t LargeValue) {
@@ -266,18 +265,6 @@ COMPILER_RT_VISIBILITY void __llvm_profile_instrument_range(
   __llvm_profile_instrument_target(TargetValue, Data, CounterIndex);
 }
 
-/*
- * The target values are partitioned into multiple ranges. The range spec is
- * defined in InstrProfData.inc.
- */
-COMPILER_RT_VISIBILITY void
-__llvm_profile_instrument_memop(uint64_t TargetValue, void *Data,
-                                uint32_t CounterIndex) {
-  // Map the target value to the representative value of its range.
-  uint64_t RepValue = InstrProfGetRangeRepValue(TargetValue);
-  __llvm_profile_instrument_target(RepValue, Data, CounterIndex);
-}
-
 /*
  * A wrapper struct that represents value profile runtime data.
  * Like InstrProfRecord class which is used by profiling host tools,
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp
index ec77b9cbfee8c..3157b35ffaf80 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp
@@ -137,8 +137,14 @@ static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) {
 
 #endif  // SANITIZER_GO || defined(SANITIZER_USE_MALLOC)
 
+namespace {
 const u64 kBlockMagic = 0x6A6CB03ABCEBC041ull;
 
+struct BlockHeader {
+  u64 magic;
+};
+}  // namespace
+
 static void NORETURN ReportInternalAllocatorOutOfMemory(uptr requested_size) {
   SetAllocatorOutOfMemory();
   Report("FATAL: %s: internal allocator is out of memory trying to allocate "
@@ -147,27 +153,28 @@ static void NORETURN ReportInternalAllocatorOutOfMemory(uptr requested_size) {
 }
 
 void *InternalAlloc(uptr size, InternalAllocatorCache *cache, uptr alignment) {
-  if (size + sizeof(u64) < size)
+  uptr s = size + sizeof(BlockHeader);
+  if (s < size)
     return nullptr;
-  void *p = RawInternalAlloc(size + sizeof(u64), cache, alignment);
+  BlockHeader *p = (BlockHeader *)RawInternalAlloc(s, cache, alignment);
   if (UNLIKELY(!p))
-    ReportInternalAllocatorOutOfMemory(size + sizeof(u64));
-  ((u64*)p)[0] = kBlockMagic;
-  return (char*)p + sizeof(u64);
+    ReportInternalAllocatorOutOfMemory(s);
+  p->magic = kBlockMagic;
+  return p + 1;
 }
 
 void *InternalRealloc(void *addr, uptr size, InternalAllocatorCache *cache) {
   if (!addr)
     return InternalAlloc(size, cache);
-  if (size + sizeof(u64) < size)
+  uptr s = size + sizeof(BlockHeader);
+  if (s < size)
     return nullptr;
-  addr = (char*)addr - sizeof(u64);
-  size = size + sizeof(u64);
-  CHECK_EQ(kBlockMagic, ((u64*)addr)[0]);
-  void *p = RawInternalRealloc(addr, size, cache);
+  BlockHeader *p = (BlockHeader *)addr - 1;
+  CHECK_EQ(kBlockMagic, p->magic);
+  p = (BlockHeader *)RawInternalRealloc(p, s, cache);
   if (UNLIKELY(!p))
-    ReportInternalAllocatorOutOfMemory(size);
-  return (char*)p + sizeof(u64);
+    ReportInternalAllocatorOutOfMemory(s);
+  return p + 1;
 }
 
 void *InternalReallocArray(void *addr, uptr count, uptr size,
@@ -198,10 +205,10 @@ void *InternalCalloc(uptr count, uptr size, InternalAllocatorCache *cache) {
 void InternalFree(void *addr, InternalAllocatorCache *cache) {
   if (!addr)
     return;
-  addr = (char*)addr - sizeof(u64);
-  CHECK_EQ(kBlockMagic, ((u64*)addr)[0]);
-  ((u64*)addr)[0] = 0;
-  RawInternalFree(addr, cache);
+  BlockHeader *p = (BlockHeader *)addr - 1;
+  CHECK_EQ(kBlockMagic, p->magic);
+  p->magic = 0;
+  RawInternalFree(p, cache);
 }
 
 // LowLevelAllocator
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
index 0fdaf00e67c17..974e89ab3657c 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
@@ -5836,6 +5836,79 @@ INTERCEPTOR(int, xdr_string, __sanitizer_XDR *xdrs, char **p,
 #define INIT_XDR
 #endif  // SANITIZER_INTERCEPT_XDR
 
+#if SANITIZER_INTERCEPT_XDRREC
+typedef int (*xdrrec_cb)(char*, char*, int);
+struct XdrRecWrapper {
+  char *handle;
+  xdrrec_cb rd, wr;
+};
+typedef AddrHashMap<XdrRecWrapper *, 11> XdrRecWrapMap;
+static XdrRecWrapMap *xdrrec_wrap_map;
+
+static int xdrrec_wr_wrap(char *handle, char *buf, int count) {
+  COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
+  COMMON_INTERCEPTOR_INITIALIZE_RANGE(buf, count);
+  XdrRecWrapper *wrap = (XdrRecWrapper *)handle;
+  return wrap->wr(wrap->handle, buf, count);
+}
+
+static int xdrrec_rd_wrap(char *handle, char *buf, int count) {
+  COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
+  XdrRecWrapper *wrap = (XdrRecWrapper *)handle;
+  return wrap->rd(wrap->handle, buf, count);
+}
+
+// This doesn't apply to the solaris version as it has a different function
+// signature.
+INTERCEPTOR(void, xdrrec_create, __sanitizer_XDR *xdr, unsigned sndsize,
+            unsigned rcvsize, char *handle, int (*rd)(char*, char*, int),
+            int (*wr)(char*, char*, int)) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, xdrrec_create, xdr, sndsize, rcvsize,
+                           handle, rd, wr);
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, &xdr->x_op, sizeof xdr->x_op);
+  
+  // We can't allocate a wrapper on the stack, as the handle is used outside
+  // this stack frame. So we put it on the heap, and keep track of it with
+  // the HashMap (keyed by x_private). When we later need to xdr_destroy,
+  // we can index the map, free the wrapper, and then clean the map entry.
+  XdrRecWrapper *wrap_data =
+      (XdrRecWrapper *)InternalAlloc(sizeof(XdrRecWrapper));
+  wrap_data->handle = handle;
+  wrap_data->rd = rd;
+  wrap_data->wr = wr;
+  if (wr)
+    wr = xdrrec_wr_wrap;
+  if (rd)
+    rd = xdrrec_rd_wrap;
+  handle = (char *)wrap_data;
+
+  REAL(xdrrec_create)(xdr, sndsize, rcvsize, handle, rd, wr);
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, xdr, sizeof *xdr);
+
+  XdrRecWrapMap::Handle wrap(xdrrec_wrap_map, xdr->x_private, false, true);
+  *wrap = wrap_data;
+}
+
+// We have to intercept this to be able to free wrapper memory;
+// otherwise it's not necessary.
+INTERCEPTOR(void, xdr_destroy, __sanitizer_XDR *xdr) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, xdr_destroy, xdr);
+
+  XdrRecWrapMap::Handle wrap(xdrrec_wrap_map, xdr->x_private, true);
+  InternalFree(*wrap);
+  REAL(xdr_destroy)(xdr);
+}
+#define INIT_XDRREC_LINUX \
+  static u64 xdrrec_wrap_mem[sizeof(XdrRecWrapMap) / sizeof(u64) + 1]; \
+  xdrrec_wrap_map = new ((void *)&xdrrec_wrap_mem) XdrRecWrapMap(); \
+  COMMON_INTERCEPT_FUNCTION(xdrrec_create); \
+  COMMON_INTERCEPT_FUNCTION(xdr_destroy);
+#else
+#define INIT_XDRREC_LINUX
+#endif
+
 #if SANITIZER_INTERCEPT_TSEARCH
 INTERCEPTOR(void *, tsearch, void *key, void **rootp,
             int (*compar)(const void *, const void *)) {
@@ -10094,6 +10167,7 @@ static void InitializeCommonInterceptors() {
   INIT_BZERO;
   INIT_FTIME;
   INIT_XDR;
+  INIT_XDRREC_LINUX;
   INIT_TSEARCH;
   INIT_LIBIO_INTERNALS;
   INIT_FOPEN;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
index 04b61d6daae78..6d174d8335c9e 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
@@ -440,6 +440,7 @@
 #define SANITIZER_INTERCEPT_FTIME \
   (!SI_FREEBSD && !SI_NETBSD && !SI_OPENBSD && SI_POSIX)
 #define SANITIZER_INTERCEPT_XDR SI_LINUX_NOT_ANDROID || SI_SOLARIS
+#define SANITIZER_INTERCEPT_XDRREC SI_LINUX_NOT_ANDROID
 #define SANITIZER_INTERCEPT_TSEARCH \
   (SI_LINUX_NOT_ANDROID || SI_MAC || SI_NETBSD || SI_OPENBSD || SI_SOLARIS)
 #define SANITIZER_INTERCEPT_LIBIO_INTERNALS SI_LINUX_NOT_ANDROID
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.cpp
index 6ec1a1bdd114c..565b31f68aaee 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_solaris.cpp
@@ -202,7 +202,8 @@ CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_name);
 CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phdr);
 CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phnum);
 
-CHECK_TYPE_SIZE(glob_t);
+// There are additional fields we are not interested in.
+COMPILER_CHECK(sizeof(__sanitizer_glob_t) <= sizeof(glob_t));
 CHECK_SIZE_AND_OFFSET(glob_t, gl_pathc);
 CHECK_SIZE_AND_OFFSET(glob_t, gl_pathv);
 CHECK_SIZE_AND_OFFSET(glob_t, gl_offs);
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp
index f920172c06d63..41a66405405ef 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp
@@ -44,6 +44,12 @@
 #define MAP_NORESERVE 0
 #endif
 
+#if SANITIZER_SOLARIS
+// Illumos' declaration of madvie cannot be made visible if _XOPEN_SOURCE
+// is defined as g++ does on Solaris.
+extern "C" int madvise(void *, size_t, int);
+#endif
+
 typedef void (*sa_sigaction_t)(int, siginfo_t *, void *);
 
 namespace __sanitizer {
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cpp
index 8793423a60178..4063ec8deaa12 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cpp
@@ -35,7 +35,8 @@ bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
   char *last = data_.proc_self_maps.data + data_.proc_self_maps.len;
   if (data_.current >= last) return false;
 
-  prxmap_t *xmapentry = (prxmap_t*)data_.current;
+  prxmap_t *xmapentry =
+      const_cast<prxmap_t *>(reinterpret_cast<const prxmap_t *>(data_.current));
 
   segment->start = (uptr)xmapentry->pr_vaddr;
   segment->end = (uptr)(xmapentry->pr_vaddr + xmapentry->pr_size);
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp
index 651d5056dd9d5..bd72c0ae00cbe 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp
@@ -89,14 +89,14 @@ class SuspendedThreadsListLinux : public SuspendedThreadsList {
  public:
   SuspendedThreadsListLinux() { thread_ids_.reserve(1024); }
 
-  tid_t GetThreadID(uptr index) const;
-  uptr ThreadCount() const;
+  tid_t GetThreadID(uptr index) const override;
+  uptr ThreadCount() const override;
   bool ContainsTid(tid_t thread_id) const;
   void Append(tid_t tid);
 
   PtraceRegistersStatus GetRegistersAndSP(uptr index, uptr *buffer,
-                                          uptr *sp) const;
-  uptr RegisterCount() const;
+                                          uptr *sp) const override;
+  uptr RegisterCount() const override;
 
  private:
   InternalMmapVector<tid_t> thread_ids_;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.cpp
index 29cbf62acd5cc..f0f150871d3a7 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.cpp
@@ -33,8 +33,15 @@ bool DlAddrSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
   int result = dladdr((const void *)addr, &info);
   if (!result) return false;
 
-  CHECK(addr >= reinterpret_cast<uptr>(info.dli_saddr));
-  stack->info.function_offset = addr - reinterpret_cast<uptr>(info.dli_saddr);
+  // Compute offset if possible. `dladdr()` doesn't always ensure that `addr >=
+  // sym_addr` so only compute the offset when this holds. Failure to find the
+  // function offset is not treated as a failure because it might still be
+  // possible to get the symbol name.
+  uptr sym_addr = reinterpret_cast<uptr>(info.dli_saddr);
+  if (addr >= sym_addr) {
+    stack->info.function_offset = addr - sym_addr;
+  }
+
   const char *demangled = DemangleSwiftAndCXX(info.dli_sname);
   if (!demangled) return false;
   stack->info.function = internal_strdup(demangled);
@@ -219,10 +226,10 @@ bool AtosSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
       start_address = reinterpret_cast<uptr>(info.dli_saddr);
   }
 
-  // Only assig to `function_offset` if we were able to get the function's
-  // start address.
-  if (start_address != AddressInfo::kUnknown) {
-    CHECK(addr >= start_address);
+  // Only assign to `function_offset` if we were able to get the function's
+  // start address and we got a sensible `start_address` (dladdr doesn't always
+  // ensure that `addr >= sym_addr`).
+  if (start_address != AddressInfo::kUnknown && addr >= start_address) {
     stack->info.function_offset = addr - start_address;
   }
   return true;
diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h
index ae085befc4f15..582178ee36b40 100644
--- a/compiler-rt/lib/scudo/standalone/combined.h
+++ b/compiler-rt/lib/scudo/standalone/combined.h
@@ -41,7 +41,7 @@ extern "C" size_t android_unsafe_frame_pointer_chase(scudo::uptr *buf,
 
 namespace scudo {
 
-enum class Option { ReleaseInterval };
+enum class Option { ReleaseInterval, MemtagTuning };
 
 template <class Params, void (*PostInitCallback)(void) = EmptyCallback>
 class Allocator {
@@ -154,6 +154,7 @@ class Allocator {
     Options.DeallocTypeMismatch = getFlags()->dealloc_type_mismatch;
     Options.DeleteSizeMismatch = getFlags()->delete_size_mismatch;
     Options.TrackAllocationStacks = false;
+    Options.UseOddEvenTags = true;
     Options.QuarantineMaxChunkSize =
         static_cast<u32>(getFlags()->quarantine_max_chunk_size);
 
@@ -196,6 +197,10 @@ class Allocator {
 #endif // GWP_ASAN_HOOKS
   }
 
+  ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) {
+    TSDRegistry.initThreadMaybe(this, MinimalInit);
+  }
+
   void reset() { memset(this, 0, sizeof(*this)); }
 
   void unmapTestOnly() {
@@ -247,6 +252,19 @@ class Allocator {
 #endif
   }
 
+  uptr computeOddEvenMaskForPointerMaybe(uptr Ptr, uptr Size) {
+    if (!Options.UseOddEvenTags)
+      return 0;
+
+    // If a chunk's tag is odd, we want the tags of the surrounding blocks to be
+    // even, and vice versa. Blocks are laid out Size bytes apart, and adding
+    // Size to Ptr will flip the least significant set bit of Size in Ptr, so
+    // that bit will have the pattern 010101... for consecutive blocks, which we
+    // can use to determine which tag mask to use.
+    return (Ptr & (1ULL << getLeastSignificantSetBitIndex(Size))) ? 0xaaaa
+                                                                  : 0x5555;
+  }
+
   NOINLINE void *allocate(uptr Size, Chunk::Origin Origin,
                           uptr Alignment = MinAlignment,
                           bool ZeroContents = false) {
@@ -346,7 +364,8 @@ class Allocator {
       if (UNLIKELY(useMemoryTagging())) {
         uptr PrevUserPtr;
         Chunk::UnpackedHeader Header;
-        const uptr BlockEnd = BlockUptr + PrimaryT::getSizeByClassId(ClassId);
+        const uptr BlockSize = PrimaryT::getSizeByClassId(ClassId);
+        const uptr BlockEnd = BlockUptr + BlockSize;
         // If possible, try to reuse the UAF tag that was set by deallocate().
         // For simplicity, only reuse tags if we have the same start address as
         // the previous allocation. This handles the majority of cases since
@@ -396,7 +415,9 @@ class Allocator {
             memset(TaggedPtr, 0, archMemoryTagGranuleSize());
           }
         } else {
-          TaggedPtr = prepareTaggedChunk(Ptr, Size, BlockEnd);
+          const uptr OddEvenMask =
+              computeOddEvenMaskForPointerMaybe(BlockUptr, BlockSize);
+          TaggedPtr = prepareTaggedChunk(Ptr, Size, OddEvenMask, BlockEnd);
         }
         storeAllocationStackMaybe(Ptr);
       } else if (UNLIKELY(FillContents != NoFill)) {
@@ -676,6 +697,23 @@ class Allocator {
       Secondary.setReleaseToOsIntervalMs(static_cast<s32>(Value));
       return true;
     }
+    if (O == Option::MemtagTuning) {
+      // Enabling odd/even tags involves a tradeoff between use-after-free
+      // detection and buffer overflow detection. Odd/even tags make it more
+      // likely for buffer overflows to be detected by increasing the size of
+      // the guaranteed "red zone" around the allocation, but on the other hand
+      // use-after-free is less likely to be detected because the tag space for
+      // any particular chunk is cut in half. Therefore we use this tuning
+      // setting to control whether odd/even tags are enabled.
+      if (Value == M_MEMTAG_TUNING_BUFFER_OVERFLOW) {
+        Options.UseOddEvenTags = true;
+        return true;
+      }
+      if (Value == M_MEMTAG_TUNING_UAF) {
+        Options.UseOddEvenTags = false;
+        return true;
+      }
+    }
     return false;
   }
 
@@ -917,6 +955,7 @@ class Allocator {
     u8 DeallocTypeMismatch : 1; // dealloc_type_mismatch
     u8 DeleteSizeMismatch : 1;  // delete_size_mismatch
     u8 TrackAllocationStacks : 1;
+    u8 UseOddEvenTags : 1;
     u32 QuarantineMaxChunkSize; // quarantine_max_chunk_size
   } Options;
 
@@ -977,19 +1016,19 @@ class Allocator {
            reinterpret_cast<uptr>(Ptr) - SizeOrUnusedBytes;
   }
 
-  ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) {
-    TSDRegistry.initThreadMaybe(this, MinimalInit);
-  }
-
   void quarantineOrDeallocateChunk(void *Ptr, Chunk::UnpackedHeader *Header,
                                    uptr Size) {
     Chunk::UnpackedHeader NewHeader = *Header;
     if (UNLIKELY(NewHeader.ClassId && useMemoryTagging())) {
       u8 PrevTag = extractTag(loadTag(reinterpret_cast<uptr>(Ptr)));
       uptr TaggedBegin, TaggedEnd;
+      const uptr OddEvenMask = computeOddEvenMaskForPointerMaybe(
+          reinterpret_cast<uptr>(getBlockBegin(Ptr, &NewHeader)),
+          SizeClassMap::getSizeByClassId(NewHeader.ClassId));
       // Exclude the previous tag so that immediate use after free is detected
       // 100% of the time.
-      setRandomTag(Ptr, Size, 1UL << PrevTag, &TaggedBegin, &TaggedEnd);
+      setRandomTag(Ptr, Size, OddEvenMask | (1UL << PrevTag), &TaggedBegin,
+                   &TaggedEnd);
       storeDeallocationStackMaybe(Ptr, PrevTag);
     }
     // If the quarantine is disabled, the actual size of a chunk is 0 or larger
diff --git a/compiler-rt/lib/scudo/standalone/include/scudo/interface.h b/compiler-rt/lib/scudo/standalone/include/scudo/interface.h
index d30fb6514a14a..27c1684596c44 100644
--- a/compiler-rt/lib/scudo/standalone/include/scudo/interface.h
+++ b/compiler-rt/lib/scudo/standalone/include/scudo/interface.h
@@ -10,6 +10,7 @@
 #define SCUDO_INTERFACE_H_
 
 #include <stddef.h>
+#include <stdint.h>
 
 extern "C" {
 
@@ -105,6 +106,29 @@ size_t __scudo_get_stack_depot_size();
 const char *__scudo_get_region_info_addr();
 size_t __scudo_get_region_info_size();
 
+#ifndef M_DECAY_TIME
+#define M_DECAY_TIME -100
+#endif
+
+#ifndef M_PURGE
+#define M_PURGE -101
+#endif
+
+// Tune the allocator's choice of memory tags to make it more likely that
+// a certain class of memory errors will be detected. The value argument should
+// be one of the enumerators of the scudo_memtag_tuning enum below.
+#ifndef M_MEMTAG_TUNING
+#define M_MEMTAG_TUNING -102
+#endif
+
+enum scudo_memtag_tuning {
+  // Tune for buffer overflows.
+  M_MEMTAG_TUNING_BUFFER_OVERFLOW,
+
+  // Tune for use-after-free.
+  M_MEMTAG_TUNING_UAF,
+};
+
 } // extern "C"
 
 #endif // SCUDO_INTERFACE_H_
diff --git a/compiler-rt/lib/scudo/standalone/memtag.h b/compiler-rt/lib/scudo/standalone/memtag.h
index 6f347f4694e82..519c827e68286 100644
--- a/compiler-rt/lib/scudo/standalone/memtag.h
+++ b/compiler-rt/lib/scudo/standalone/memtag.h
@@ -126,7 +126,8 @@ inline void setRandomTag(void *Ptr, uptr Size, uptr ExcludeMask,
       : "memory");
 }
 
-inline void *prepareTaggedChunk(void *Ptr, uptr Size, uptr BlockEnd) {
+inline void *prepareTaggedChunk(void *Ptr, uptr Size, uptr ExcludeMask,
+                                uptr BlockEnd) {
   // Prepare the granule before the chunk to store the chunk header by setting
   // its tag to 0. Normally its tag will already be 0, but in the case where a
   // chunk holding a low alignment allocation is reused for a higher alignment
@@ -138,7 +139,7 @@ inline void *prepareTaggedChunk(void *Ptr, uptr Size, uptr BlockEnd) {
                        : "memory");
 
   uptr TaggedBegin, TaggedEnd;
-  setRandomTag(Ptr, Size, 0, &TaggedBegin, &TaggedEnd);
+  setRandomTag(Ptr, Size, ExcludeMask, &TaggedBegin, &TaggedEnd);
 
   // Finally, set the tag of the granule past the end of the allocation to 0,
   // to catch linear overflows even if a previous larger allocation used the
@@ -235,9 +236,11 @@ inline void setRandomTag(void *Ptr, uptr Size, uptr ExcludeMask,
   UNREACHABLE("memory tagging not supported");
 }
 
-inline void *prepareTaggedChunk(void *Ptr, uptr Size, uptr BlockEnd) {
+inline void *prepareTaggedChunk(void *Ptr, uptr Size, uptr ExcludeMask,
+                                uptr BlockEnd) {
   (void)Ptr;
   (void)Size;
+  (void)ExcludeMask;
   (void)BlockEnd;
   UNREACHABLE("memory tagging not supported");
 }
diff --git a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
index 7e04afb90bb1f..b5035d05aabb0 100644
--- a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
@@ -67,15 +67,17 @@ void checkMemoryTaggingMaybe(AllocatorT *Allocator, void *P, scudo::uptr Size,
       "");
 }
 
+template <typename Config> struct TestAllocator : scudo::Allocator<Config> {
+  TestAllocator() {
+    this->reset();
+    this->initThreadMaybe();
+  }
+  ~TestAllocator() { this->unmapTestOnly(); }
+};
+
 template <class Config> static void testAllocator() {
-  using AllocatorT = scudo::Allocator<Config>;
-  auto Deleter = [](AllocatorT *A) {
-    A->unmapTestOnly();
-    delete A;
-  };
-  std::unique_ptr<AllocatorT, decltype(Deleter)> Allocator(new AllocatorT,
-                                                           Deleter);
-  Allocator->reset();
+  using AllocatorT = TestAllocator<Config>;
+  auto Allocator = std::make_unique<AllocatorT>();
 
   EXPECT_FALSE(Allocator->isOwned(&Mutex));
   EXPECT_FALSE(Allocator->isOwned(&Allocator));
@@ -348,14 +350,8 @@ template <typename AllocatorT> static void stressAllocator(AllocatorT *A) {
 }
 
 template <class Config> static void testAllocatorThreaded() {
-  using AllocatorT = scudo::Allocator<Config>;
-  auto Deleter = [](AllocatorT *A) {
-    A->unmapTestOnly();
-    delete A;
-  };
-  std::unique_ptr<AllocatorT, decltype(Deleter)> Allocator(new AllocatorT,
-                                                           Deleter);
-  Allocator->reset();
+  using AllocatorT = TestAllocator<Config>;
+  auto Allocator = std::make_unique<AllocatorT>();
   std::thread Threads[32];
   for (scudo::uptr I = 0; I < ARRAY_SIZE(Threads); I++)
     Threads[I] = std::thread(stressAllocator<AllocatorT>, Allocator.get());
@@ -401,14 +397,8 @@ struct DeathConfig {
 };
 
 TEST(ScudoCombinedTest, DeathCombined) {
-  using AllocatorT = scudo::Allocator<DeathConfig>;
-  auto Deleter = [](AllocatorT *A) {
-    A->unmapTestOnly();
-    delete A;
-  };
-  std::unique_ptr<AllocatorT, decltype(Deleter)> Allocator(new AllocatorT,
-                                                           Deleter);
-  Allocator->reset();
+  using AllocatorT = TestAllocator<DeathConfig>;
+  auto Allocator = std::make_unique<AllocatorT>();
 
   const scudo::uptr Size = 1000U;
   void *P = Allocator->allocate(Size, Origin);
@@ -442,14 +432,8 @@ TEST(ScudoCombinedTest, DeathCombined) {
 // Ensure that releaseToOS can be called prior to any other allocator
 // operation without issue.
 TEST(ScudoCombinedTest, ReleaseToOS) {
-  using AllocatorT = scudo::Allocator<DeathConfig>;
-  auto Deleter = [](AllocatorT *A) {
-    A->unmapTestOnly();
-    delete A;
-  };
-  std::unique_ptr<AllocatorT, decltype(Deleter)> Allocator(new AllocatorT,
-                                                           Deleter);
-  Allocator->reset();
+  using AllocatorT = TestAllocator<DeathConfig>;
+  auto Allocator = std::make_unique<AllocatorT>();
 
   Allocator->releaseToOS();
 }
@@ -457,14 +441,8 @@ TEST(ScudoCombinedTest, ReleaseToOS) {
 // Verify that when a region gets full, the allocator will still manage to
 // fulfill the allocation through a larger size class.
 TEST(ScudoCombinedTest, FullRegion) {
-  using AllocatorT = scudo::Allocator<DeathConfig>;
-  auto Deleter = [](AllocatorT *A) {
-    A->unmapTestOnly();
-    delete A;
-  };
-  std::unique_ptr<AllocatorT, decltype(Deleter)> Allocator(new AllocatorT,
-                                                           Deleter);
-  Allocator->reset();
+  using AllocatorT = TestAllocator<DeathConfig>;
+  auto Allocator = std::make_unique<AllocatorT>();
 
   std::vector<void *> V;
   scudo::uptr FailedAllocationsCount = 0;
@@ -491,3 +469,42 @@ TEST(ScudoCombinedTest, FullRegion) {
   }
   EXPECT_EQ(FailedAllocationsCount, 0U);
 }
+
+TEST(ScudoCombinedTest, OddEven) {
+  using AllocatorT = TestAllocator<scudo::AndroidConfig>;
+  using SizeClassMap = AllocatorT::PrimaryT::SizeClassMap;
+  auto Allocator = std::make_unique<AllocatorT>();
+
+  if (!Allocator->useMemoryTagging())
+    return;
+
+  auto CheckOddEven = [](scudo::uptr P1, scudo::uptr P2) {
+    scudo::uptr Tag1 = scudo::extractTag(scudo::loadTag(P1));
+    scudo::uptr Tag2 = scudo::extractTag(scudo::loadTag(P2));
+    EXPECT_NE(Tag1 % 2, Tag2 % 2);
+  };
+
+  for (scudo::uptr ClassId = 1U; ClassId <= SizeClassMap::LargestClassId;
+       ClassId++) {
+    const scudo::uptr Size = SizeClassMap::getSizeByClassId(ClassId);
+
+    std::set<scudo::uptr> Ptrs;
+    bool Found = false;
+    for (unsigned I = 0; I != 65536; ++I) {
+      scudo::uptr P = scudo::untagPointer(reinterpret_cast<scudo::uptr>(
+          Allocator->allocate(Size - scudo::Chunk::getHeaderSize(), Origin)));
+      if (Ptrs.count(P - Size)) {
+        Found = true;
+        CheckOddEven(P, P - Size);
+        break;
+      }
+      if (Ptrs.count(P + Size)) {
+        Found = true;
+        CheckOddEven(P, P + Size);
+        break;
+      }
+      Ptrs.insert(P);
+    }
+    EXPECT_TRUE(Found);
+  }
+}
diff --git a/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp b/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp
index b41908cf47814..4b851a7af181f 100644
--- a/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "scudo/interface.h"
 #include "tests/scudo_unit_test.h"
 
 #include <errno.h>
@@ -188,14 +189,6 @@ TEST(ScudoWrappersCTest, Realloc) {
   }
 }
 
-#ifndef M_DECAY_TIME
-#define M_DECAY_TIME -100
-#endif
-
-#ifndef M_PURGE
-#define M_PURGE -101
-#endif
-
 #if !SCUDO_FUCHSIA
 TEST(ScudoWrappersCTest, MallOpt) {
   errno = 0;
diff --git a/compiler-rt/lib/scudo/standalone/wrappers_c.h b/compiler-rt/lib/scudo/standalone/wrappers_c.h
index 33a0c53cec035..6d0cecdc4b41d 100644
--- a/compiler-rt/lib/scudo/standalone/wrappers_c.h
+++ b/compiler-rt/lib/scudo/standalone/wrappers_c.h
@@ -41,12 +41,4 @@ struct __scudo_mallinfo {
 #define SCUDO_MALLINFO __scudo_mallinfo
 #endif
 
-#ifndef M_DECAY_TIME
-#define M_DECAY_TIME -100
-#endif
-
-#ifndef M_PURGE
-#define M_PURGE -101
-#endif
-
 #endif // SCUDO_WRAPPERS_C_H_
diff --git a/compiler-rt/lib/scudo/standalone/wrappers_c.inc b/compiler-rt/lib/scudo/standalone/wrappers_c.inc
index 4396dfc50d1df..da5a170569d7c 100644
--- a/compiler-rt/lib/scudo/standalone/wrappers_c.inc
+++ b/compiler-rt/lib/scudo/standalone/wrappers_c.inc
@@ -173,6 +173,9 @@ INTERFACE WEAK int SCUDO_PREFIX(mallopt)(int param, UNUSED int value) {
   } else if (param == M_PURGE) {
     SCUDO_ALLOCATOR.releaseToOS();
     return 1;
+  } else if (param == M_MEMTAG_TUNING) {
+    return SCUDO_ALLOCATOR.setOption(scudo::Option::MemtagTuning,
+                                     static_cast<scudo::sptr>(value));
   }
   return 0;
 }
diff --git a/compiler-rt/test/asan/TestCases/Darwin/symbolizer-function-offset-dladdr.cpp b/compiler-rt/test/asan/TestCases/Darwin/symbolizer-function-offset-dladdr.cpp
new file mode 100644
index 0000000000000..bbf0ddf1c7ef6
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/Darwin/symbolizer-function-offset-dladdr.cpp
@@ -0,0 +1,44 @@
+// FIXME(dliew): Duplicated from `test/sanitizer_common/TestCases/Darwin/symbolizer-function-offset-dladdr.cpp`.
+// This case can be dropped once sanitizer_common tests work on iOS devices (rdar://problem/47333049).
+
+// NOTE: `detect_leaks=0` is necessary because with LSan enabled the dladdr
+// symbolizer actually leaks memory because the call to
+// `__sanitizer::DemangleCXXABI` leaks memory which LSan detects
+// (rdar://problem/42868950).
+
+// RUN: %clangxx_asan %s -O0 -o %t
+// RUN: %env_asan_opts=detect_leaks=0,verbosity=2,external_symbolizer_path=,stack_trace_format='"function_name_%f___function_offset_%q"' %run %t > %t.output 2>&1
+// RUN: FileCheck -input-file=%t.output %s
+#include <sanitizer/common_interface_defs.h>
+#include <stdio.h>
+
+void baz() {
+  printf("Do stuff in baz\n");
+  __sanitizer_print_stack_trace();
+}
+
+void bar() {
+  printf("Do stuff in bar\n");
+  baz();
+}
+
+void foo() {
+  printf("Do stuff in foo\n");
+  bar();
+}
+
+int main() {
+  printf("Do stuff in main\n");
+  foo();
+  return 0;
+}
+
+// CHECK: External symbolizer is explicitly disabled
+// CHECK: Using dladdr symbolizer
+
+// These `function_offset` patterns are designed to disallow `0x0` which is the
+// value printed for `kUnknown`.
+// CHECK: function_name_baz{{(\(\))?}}___function_offset_0x{{0*[1-9a-f][0-9a-f]*$}}
+// CHECK: function_name_bar{{(\(\))?}}___function_offset_0x{{0*[1-9a-f][0-9a-f]*$}}
+// CHECK: function_name_foo{{(\(\))?}}___function_offset_0x{{0*[1-9a-f][0-9a-f]*$}}
+// CHECK: function_name_main{{(\(\))?}}___function_offset_0x{{0*[1-9a-f][0-9a-f]*$}}
diff --git a/compiler-rt/test/dfsan/fast16labels.c b/compiler-rt/test/dfsan/fast16labels.c
index a9b7160395404..269d1e2efc0af 100644
--- a/compiler-rt/test/dfsan/fast16labels.c
+++ b/compiler-rt/test/dfsan/fast16labels.c
@@ -1,4 +1,11 @@
-// RUN: %clang_dfsan %s -o %t && DFSAN_OPTIONS=fast16labels=1 %run %t
+// RUN: %clang_dfsan %s -o %t
+// RUN: DFSAN_OPTIONS=fast16labels=1 %run %t
+// RUN: DFSAN_OPTIONS=fast16labels=1 not %run %t dfsan_create_label 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CREATE-LABEL
+// RUN: DFSAN_OPTIONS=fast16labels=1 not %run %t dfsan_get_label_info 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=GET-LABEL-INFO
+// RUN: DFSAN_OPTIONS=fast16labels=1 not %run %t dfsan_has_label_with_desc \
+// RUN:   2>&1 | FileCheck %s --check-prefix=HAS-LABEL-WITH-DESC
 //
 // Tests DFSAN_OPTIONS=fast16labels=1
 //
@@ -6,12 +13,26 @@
 
 #include <assert.h>
 #include <stdio.h>
+#include <string.h>
 
 int foo(int a, int b) {
   return a + b;
 }
 
-int main() {
+int main(int argc, char *argv[]) {
+  // Death tests for unsupported API usage.
+  const char *command = (argc < 2) ? "" : argv[1];
+  // CREATE-LABEL: FATAL: DataFlowSanitizer: dfsan_create_label is unsupported
+  if (strcmp(command, "dfsan_create_label") == 0)
+    dfsan_create_label("", NULL);
+  // GET-LABEL-INFO: FATAL: DataFlowSanitizer: dfsan_get_label_info is unsupported
+  if (strcmp(command, "dfsan_get_label_info") == 0)
+    dfsan_get_label_info(1);
+  // HAS-LABEL-WITH-DESC: FATAL: DataFlowSanitizer: dfsan_has_label_with_desc is unsupported
+  if (strcmp(command, "dfsan_has_label_with_desc") == 0)
+    dfsan_has_label_with_desc(1, "");
+
+  // Supported usage.
   int a = 10;
   int b = 20;
   dfsan_set_label(8, &a, sizeof(a));
@@ -22,4 +43,7 @@ int main() {
   dfsan_label l = dfsan_get_label(c);
   printf("C: 0x%x\n", l);
   assert(l == 520);  // OR of the other two labels.
+  assert(dfsan_has_label(l, 8));
+  assert(dfsan_has_label(l, 512));
+  assert(!dfsan_has_label(l, 1));
 }
diff --git a/compiler-rt/test/fuzzer/CustomAllocator.cpp b/compiler-rt/test/fuzzer/CustomAllocator.cpp
new file mode 100644
index 0000000000000..ebfa5901ea7a6
--- /dev/null
+++ b/compiler-rt/test/fuzzer/CustomAllocator.cpp
@@ -0,0 +1,46 @@
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Test whether calling certain libFuzzer's interceptors inside allocators
+// does not cause an assertion failure.
+#include <assert.h>
+#include <cstddef>
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <malloc.h>
+
+static const char *buf1 = "aaaa";
+static const char *buf2 = "bbbb";
+
+static void callFuzzerInterceptors(const char *prefix) {
+  int memcmp_result = memcmp(buf1, buf2, 4);
+  if (memcmp_result != 0) {
+    fprintf(stderr, "%s-MEMCMP\n", prefix);
+  }
+  int strncmp_result = strncmp(buf1, buf2, 4);
+  if (strncmp_result != 0) {
+    fprintf(stderr, "%s-STRNCMP\n", prefix);
+  }
+  int strcmp_result = strcmp(buf1, buf2);
+  if (strcmp_result != 0) {
+    fprintf(stderr, "%s-STRCMP\n", prefix);
+  }
+  const char *strstr_result = strstr(buf1, buf2);
+  if (strstr_result == nullptr) {
+    fprintf(stderr, "%s-STRSTR\n", prefix);
+  }
+}
+
+extern "C" void *__libc_calloc(size_t, size_t);
+
+extern "C" void *calloc(size_t n, size_t elem_size) {
+  static bool CalledOnce = false;
+  if (!CalledOnce) {
+    callFuzzerInterceptors("CALLOC");
+    CalledOnce = true;
+  }
+  return __libc_calloc(n, elem_size);
+}
diff --git a/compiler-rt/test/fuzzer/MemcmpTest.cpp b/compiler-rt/test/fuzzer/MemcmpTest.cpp
index 060c5b9b11f9c..09f56ff78dad1 100644
--- a/compiler-rt/test/fuzzer/MemcmpTest.cpp
+++ b/compiler-rt/test/fuzzer/MemcmpTest.cpp
@@ -8,13 +8,17 @@
 #include <cstdlib>
 #include <cstring>
 
+#ifndef MEMCMP
+# define MEMCMP memcmp
+#endif
+
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   // TODO: check other sizes.
-  if (Size >= 8 && memcmp(Data, "01234567", 8) == 0) {
-    if (Size >= 12 && memcmp(Data + 8, "ABCD", 4) == 0) {
-      if (Size >= 14 && memcmp(Data + 12, "XY", 2) == 0) {
-        if (Size >= 17 && memcmp(Data + 14, "KLM", 3) == 0) {
-          if (Size >= 27 && memcmp(Data + 17, "ABCDE-GHIJ", 10) == 0){
+  if (Size >= 8 && MEMCMP(Data, "01234567", 8) == 0) {
+    if (Size >= 12 && MEMCMP(Data + 8, "ABCD", 4) == 0) {
+      if (Size >= 14 && MEMCMP(Data + 12, "XY", 2) == 0) {
+        if (Size >= 17 && MEMCMP(Data + 14, "KLM", 3) == 0) {
+          if (Size >= 27 && MEMCMP(Data + 17, "ABCDE-GHIJ", 10) == 0){
             fprintf(stderr, "BINGO %zd\n", Size);
             for (size_t i = 0; i < Size; i++) {
               uint8_t C = Data[i];
diff --git a/compiler-rt/test/fuzzer/bcmp.test b/compiler-rt/test/fuzzer/bcmp.test
new file mode 100644
index 0000000000000..37ee6bedd4ee1
--- /dev/null
+++ b/compiler-rt/test/fuzzer/bcmp.test
@@ -0,0 +1,4 @@
+UNSUPPORTED: freebsd
+RUN: %cpp_compiler -DMEMCMP=bcmp %S/MemcmpTest.cpp -o %t
+RUN: not %run %t -seed=1 -runs=10000000 2>&1 | FileCheck %s
+CHECK: BINGO
diff --git a/compiler-rt/test/fuzzer/custom-allocator.test b/compiler-rt/test/fuzzer/custom-allocator.test
new file mode 100644
index 0000000000000..a1f41c5f867dc
--- /dev/null
+++ b/compiler-rt/test/fuzzer/custom-allocator.test
@@ -0,0 +1,8 @@
+UNSUPPORTED: darwin, freebsd, windows
+RUN: %cpp_compiler -fno-sanitize=address -fno-builtin %S/CustomAllocator.cpp %S/EmptyTest.cpp -o %t-NoAsanCustomAllocatorTest
+RUN: %run %t-NoAsanCustomAllocatorTest -runs=1 2>&1 | FileCheck %s
+
+CHECK: CALLOC-MEMCMP
+CHECK-NEXT: CALLOC-STRNCMP
+CHECK-NEXT: CALLOC-STRCMP
+CHECK-NEXT: CALLOC-STRSTR
diff --git a/compiler-rt/test/fuzzer/memcmp64.test b/compiler-rt/test/fuzzer/memcmp64.test
index 24d14bf73bbf4..feb81d1e4fb28 100644
--- a/compiler-rt/test/fuzzer/memcmp64.test
+++ b/compiler-rt/test/fuzzer/memcmp64.test
@@ -1,4 +1,8 @@
 UNSUPPORTED: freebsd
 RUN: %cpp_compiler %S/Memcmp64BytesTest.cpp -o %t-Memcmp64BytesTest
 RUN: not %run %t-Memcmp64BytesTest        -seed=1 -runs=1000000   2>&1 | FileCheck %s
+
+RUN: %cpp_compiler -fno-sanitize=address -fno-builtin-memcmp %S/Memcmp64BytesTest.cpp -o %t-NoAsanMemcmp64BytesTest
+RUN: not %run %t-NoAsanMemcmp64BytesTest  -seed=1 -runs=1000000   2>&1 | FileCheck %s
+
 CHECK: BINGO
diff --git a/compiler-rt/test/fuzzer/noasan-memcmp.test b/compiler-rt/test/fuzzer/noasan-memcmp.test
new file mode 100644
index 0000000000000..c90755c53a900
--- /dev/null
+++ b/compiler-rt/test/fuzzer/noasan-memcmp.test
@@ -0,0 +1,9 @@
+UNSUPPORTED: darwin, freebsd, windows
+
+RUN: %cpp_compiler -fno-sanitize=address -fno-builtin-memcmp %S/MemcmpTest.cpp -o %t-NoAsanMemcmpTest
+RUN: not %run %t-NoAsanMemcmpTest -seed=1 -runs=10000000   2>&1 | FileCheck %s
+
+RUN: %cpp_compiler -fno-sanitize=address -fno-builtin-calloc -fno-builtin-memcmp %S/CustomAllocator.cpp %S/MemcmpTest.cpp -o %t-NoAsanCustomAllocatorMemcmpTest
+RUN: not %run %t-NoAsanCustomAllocatorMemcmpTest -seed=1 -runs=10000000   2>&1 | FileCheck %s
+
+CHECK: BINGO
diff --git a/compiler-rt/test/fuzzer/noasan-strcmp.test b/compiler-rt/test/fuzzer/noasan-strcmp.test
new file mode 100644
index 0000000000000..76b7c5de7c7bb
--- /dev/null
+++ b/compiler-rt/test/fuzzer/noasan-strcmp.test
@@ -0,0 +1,9 @@
+UNSUPPORTED: darwin, freebsd, windows
+
+RUN: %cpp_compiler -fno-sanitize=address -fno-builtin-strcmp %S/StrcmpTest.cpp -o %t-NoAsanStrcmpTest
+RUN: not %run %t-NoAsanStrcmpTest -seed=1 -runs=2000000   2>&1 | FileCheck %s
+
+RUN: %cpp_compiler -fno-sanitize=address -fno-builtin-calloc -fno-builtin-strcmp %S/CustomAllocator.cpp %S/StrcmpTest.cpp -o %t-NoAsanCustomAllocatorStrcmpTest
+RUN: not %run %t-NoAsanCustomAllocatorStrcmpTest -seed=1 -runs=2000000   2>&1 | FileCheck %s
+
+CHECK: BINGO
diff --git a/compiler-rt/test/fuzzer/noasan-strncmp.test b/compiler-rt/test/fuzzer/noasan-strncmp.test
new file mode 100644
index 0000000000000..705781ec39586
--- /dev/null
+++ b/compiler-rt/test/fuzzer/noasan-strncmp.test
@@ -0,0 +1,9 @@
+UNSUPPORTED: darwin, freebsd, windows
+
+RUN: %cpp_compiler -fno-sanitize=address -fno-builtin-strncmp %S/StrncmpTest.cpp -o %t-NoAsanStrncmpTest
+RUN: not %run %t-NoAsanStrncmpTest -seed=2 -runs=10000000   2>&1 | FileCheck %s
+
+RUN: %cpp_compiler -fno-sanitize=address -fno-builtin-calloc -fno-builtin-strncmp %S/CustomAllocator.cpp %S/StrncmpTest.cpp -o %t-NoAsanCustomAllocatorStrncmpTest
+RUN: not %run %t-NoAsanCustomAllocatorStrncmpTest -seed=2 -runs=10000000   2>&1 | FileCheck %s
+
+CHECK: BINGO
diff --git a/compiler-rt/test/fuzzer/noasan-strstr.test b/compiler-rt/test/fuzzer/noasan-strstr.test
new file mode 100644
index 0000000000000..f06e903149bd9
--- /dev/null
+++ b/compiler-rt/test/fuzzer/noasan-strstr.test
@@ -0,0 +1,9 @@
+UNSUPPORTED: darwin, freebsd, windows
+
+RUN: %cpp_compiler -fno-sanitize=address -fno-builtin-strstr %S/StrstrTest.cpp -o %t-NoAsanStrstrTest
+RUN: not %run %t-NoAsanStrstrTest -seed=1 -runs=2000000   2>&1 | FileCheck %s
+
+RUN: %cpp_compiler -fno-sanitize=address -fno-builtin-calloc -fno-builtin-strstr %S/CustomAllocator.cpp %S/StrstrTest.cpp -o %t-NoAsanCustomAllocatorStrstrTest
+RUN: not %run %t-NoAsanCustomAllocatorStrstrTest -seed=1 -runs=2000000   2>&1 | FileCheck %s
+
+CHECK: BINGO
diff --git a/compiler-rt/test/fuzzer/strcmp.test b/compiler-rt/test/fuzzer/strcmp.test
index bd917bba6b698..14c22635fb162 100644
--- a/compiler-rt/test/fuzzer/strcmp.test
+++ b/compiler-rt/test/fuzzer/strcmp.test
@@ -2,4 +2,3 @@ UNSUPPORTED: freebsd
 RUN: %cpp_compiler %S/StrcmpTest.cpp -o %t-StrcmpTest
 RUN: not %run %t-StrcmpTest               -seed=1 -runs=2000000   2>&1 | FileCheck %s
 CHECK: BINGO
-
diff --git a/compiler-rt/test/fuzzer/strncmp.test b/compiler-rt/test/fuzzer/strncmp.test
index 50189445b102c..fb820b7d20e87 100644
--- a/compiler-rt/test/fuzzer/strncmp.test
+++ b/compiler-rt/test/fuzzer/strncmp.test
@@ -2,4 +2,3 @@ UNSUPPORTED: freebsd
 RUN: %cpp_compiler %S/StrncmpTest.cpp -o %t-StrncmpTest
 RUN: not %run %t-StrncmpTest              -seed=2 -runs=10000000   2>&1 | FileCheck %s
 CHECK: BINGO
-
diff --git a/compiler-rt/test/fuzzer/strstr.test b/compiler-rt/test/fuzzer/strstr.test
index f1fb210b47c7f..60261e3f1ff3d 100644
--- a/compiler-rt/test/fuzzer/strstr.test
+++ b/compiler-rt/test/fuzzer/strstr.test
@@ -2,4 +2,3 @@ UNSUPPORTED: freebsd
 RUN: %cpp_compiler %S/StrstrTest.cpp -o %t-StrstrTest
 RUN: not %run %t-StrstrTest               -seed=1 -runs=2000000   2>&1 | FileCheck %s
 CHECK: BINGO
-
diff --git a/compiler-rt/test/lit.common.cfg.py b/compiler-rt/test/lit.common.cfg.py
index 7c98c387c8705..fdc28a4637ff7 100644
--- a/compiler-rt/test/lit.common.cfg.py
+++ b/compiler-rt/test/lit.common.cfg.py
@@ -413,19 +413,18 @@ def get_apple_min_deploy_target_flag_aligned_with_osx(version):
 def is_darwin_lto_supported():
   return os.path.exists(os.path.join(config.llvm_shlib_dir, 'libLTO.dylib'))
 
-def is_linux_lto_supported():
-  if config.use_lld:
-    return True
-
+def is_binutils_lto_supported():
   if not os.path.exists(os.path.join(config.llvm_shlib_dir, 'LLVMgold.so')):
     return False
 
-  ld_cmd = subprocess.Popen([config.gold_executable, '--help'], stdout = subprocess.PIPE, env={'LANG': 'C'})
-  ld_out = ld_cmd.stdout.read().decode()
-  ld_cmd.wait()
-
-  if not '-plugin' in ld_out:
-    return False
+  # We require both ld.bfd and ld.gold exist and support plugins. They are in
+  # the same repository 'binutils-gdb' and usually built together.
+  for exe in (config.gnu_ld_executable, config.gold_executable):
+    ld_cmd = subprocess.Popen([exe, '--help'], stdout=subprocess.PIPE, env={'LANG': 'C'})
+    ld_out = ld_cmd.stdout.read().decode()
+    ld_cmd.wait()
+    if not '-plugin' in ld_out:
+      return False
 
   return True
 
@@ -436,13 +435,20 @@ def is_windows_lto_supported():
   config.lto_supported = True
   config.lto_launch = ["env", "DYLD_LIBRARY_PATH=" + config.llvm_shlib_dir]
   config.lto_flags = []
-elif config.host_os in ['Linux', 'FreeBSD', 'NetBSD'] and is_linux_lto_supported():
-  config.lto_supported = True
-  config.lto_launch = []
+elif config.host_os in ['Linux', 'FreeBSD', 'NetBSD']:
+  config.lto_supported = False
   if config.use_lld:
-    config.lto_flags = ["-fuse-ld=lld"]
-  else:
-    config.lto_flags = ["-fuse-ld=gold"]
+    config.lto_supported = True
+  if is_binutils_lto_supported():
+    config.available_features.add('binutils_lto')
+    config.lto_supported = True
+
+  if config.lto_supported:
+    config.lto_launch = []
+    if config.use_lld:
+      config.lto_flags = ["-fuse-ld=lld"]
+    else:
+      config.lto_flags = ["-fuse-ld=gold"]
 elif config.host_os == 'Windows' and is_windows_lto_supported():
   config.lto_supported = True
   config.lto_launch = []
diff --git a/compiler-rt/test/lit.common.configured.in b/compiler-rt/test/lit.common.configured.in
index 4a3e268c8a6f9..000bf9b984709 100644
--- a/compiler-rt/test/lit.common.configured.in
+++ b/compiler-rt/test/lit.common.configured.in
@@ -19,6 +19,7 @@ set_default("compiler_rt_obj_root", "@COMPILER_RT_BINARY_DIR@")
 set_default("enable_per_target_runtime_dir", @LLVM_ENABLE_PER_TARGET_RUNTIME_DIR_PYBOOL@)
 set_default("llvm_tools_dir", "@LLVM_TOOLS_DIR@")
 set_default("llvm_shlib_dir", "@LLVM_LIBRARY_OUTPUT_INTDIR@")
+set_default("gnu_ld_executable", "@GNU_LD_EXECUTABLE@")
 set_default("gold_executable", "@GOLD_EXECUTABLE@")
 set_default("clang", "@COMPILER_RT_RESOLVED_TEST_COMPILER@")
 set_default("compiler_id", "@COMPILER_RT_TEST_COMPILER_ID@")
@@ -56,7 +57,7 @@ elif config.android:
 else:
   set_default("target_suffix", "-%s" % config.target_arch)
 
-set_default("have_zlib", "@HAVE_LIBZ@")
+set_default("have_zlib", "@LLVM_ENABLE_ZLIB@")
 set_default("libcxx_used", "@LLVM_LIBCXX_USED@")
 
 # LLVM tools dir can be passed in lit parameters, so try to
diff --git a/compiler-rt/test/msan/libatomic.c b/compiler-rt/test/msan/libatomic.c
new file mode 100644
index 0000000000000..5d8aa9ab6a60f
--- /dev/null
+++ b/compiler-rt/test/msan/libatomic.c
@@ -0,0 +1,41 @@
+// RUN: %clangxx_msan -fsanitize-memory-track-origins=2 -latomic -DTEST_STORE -O0 %s -o %t && %run %t 2>&1
+// RUN: %clangxx_msan -fsanitize-memory-track-origins=0 -latomic -DTEST_LOAD -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK
+// RUN: %clangxx_msan -fsanitize-memory-track-origins=2 -latomic -DTEST_LOAD -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-SHADOW
+
+// PPC has no libatomic
+// UNSUPPORTED: powerpc64-target-arch
+// UNSUPPORTED: powerpc64le-target-arch
+
+#include <sanitizer/msan_interface.h>
+#include <stdatomic.h>
+
+typedef struct __attribute((packed)) {
+  uint8_t val[3];
+} i24;
+
+void copy(i24 *dst, i24 *src);
+
+int main() {
+  i24 uninit;
+  i24 init = {0};
+
+  __msan_check_mem_is_initialized(&init, 3);
+  copy(&init, &uninit);
+  __msan_check_mem_is_initialized(&init, 3);
+}
+
+void copy(i24 *dst, i24 *src) {
+#ifdef TEST_LOAD
+  __atomic_load(src, dst, __ATOMIC_RELAXED);
+
+  // CHECK: MemorySanitizer: use-of-uninitialized-value
+  // CHECK: #0 {{0x[a-f0-9]+}} in main{{.*}}libatomic.c:[[@LINE-8]]
+
+  // CHECK-SHADOW: Uninitialized value was stored to memory at
+  // CHECK-SHADOW: #0 {{0x[a-f0-9]+}} in copy{{.*}}libatomic.c:[[@LINE-6]]
+#endif
+#ifdef TEST_STORE
+  // Store always writes a clean shadow
+  __atomic_store(src, dst, __ATOMIC_RELAXED);
+#endif
+}
diff --git a/compiler-rt/test/profile/Posix/gcov-fork.c b/compiler-rt/test/profile/Posix/gcov-fork.c
index 022ce716a8dcb..b89eb64922f0c 100644
--- a/compiler-rt/test/profile/Posix/gcov-fork.c
+++ b/compiler-rt/test/profile/Posix/gcov-fork.c
@@ -8,6 +8,7 @@
 // RUN: rm -f gcov-fork.gcda && %run %t
 // RUN: llvm-cov gcov -t gcov-fork.gcda | FileCheck %s
 
+#include <sys/wait.h>
 #include <unistd.h>
 
 void func1() {}                    // CHECK:      1: [[#@LINE]]:void func1()
diff --git a/compiler-rt/test/profile/instrprof-lto-pgogen.c b/compiler-rt/test/profile/instrprof-lto-pgogen.c
index 99870c70bef37..3538f269f510d 100644
--- a/compiler-rt/test/profile/instrprof-lto-pgogen.c
+++ b/compiler-rt/test/profile/instrprof-lto-pgogen.c
@@ -1,13 +1,13 @@
-// REQUIRES: lto
-// XFAIL: msvc
+// REQUIRES: binutils_lto
 
-// RUN: %clang_pgogen=%t.profraw -flto %s -o %t
+// RUN: %clang_pgogen=%t.profraw -fuse-ld=bfd -flto %s -o %t
 // RUN: %run %t
 // RUN: llvm-profdata merge %t.profraw -o %t.profdata
 // RUN: llvm-profdata show %t.profdata | FileCheck %s
 
-// Testing a bug that happens when trying to generate IR
-// profile with BFD linker + LTO plugin
+/// Test that we work around https://sourceware.org/bugzilla/show_bug.cgi?id=26262
+/// (as of GNU ld 2.35) which happens when trying to generate IR profile with
+/// BFD linker + LLVMgold.so
 
 // CHECK: Instrumentation level: IR
 int main() { return 0; }
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/xdrrec.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/xdrrec.cpp
new file mode 100644
index 0000000000000..f39bed3da3763
--- /dev/null
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/xdrrec.cpp
@@ -0,0 +1,27 @@
+// RUN: %clangxx -O0 %s -o %t && %run %t | FileCheck %s
+// REQUIRES: !android
+#include <cassert>
+#include <rpc/xdr.h>
+
+int print_msg(char *handle, char *buf, int len) {
+  if (len > 0) {
+    for (size_t i = 0; i < len; i++) {
+      printf("%02x ", (uint8_t)buf[i]);
+    }
+    printf("\n");
+  }
+  return len;
+}
+
+int main() {
+  XDR xdrs;
+  xdrs.x_op = XDR_ENCODE;
+
+  xdrrec_create(&xdrs, 0, 0, nullptr, nullptr, print_msg);
+  unsigned foo = 42;
+  assert(xdr_u_int(&xdrs, &foo));
+  assert(xdrrec_endofrecord(&xdrs, /*sendnow*/ true));
+  xdr_destroy(&xdrs);
+}
+
+// CHECK: 00 00 00 2a
diff --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index 13e675f1096e5..9dd6281d410bf 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.9.0)
+cmake_minimum_required(VERSION 3.13.4)
 
 # RPATH settings on macOS do not affect INSTALL_NAME.
 if (POLICY CMP0068)
diff --git a/flang/include/flang/Lower/OpenACC.h b/flang/include/flang/Lower/OpenACC.h
new file mode 100644
index 0000000000000..a3e6f177f91de
--- /dev/null
+++ b/flang/include/flang/Lower/OpenACC.h
@@ -0,0 +1,35 @@
+//===-- Lower/OpenACC.h -- lower OpenACC directives -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_LOWER_OPENACC_H
+#define FORTRAN_LOWER_OPENACC_H
+
+namespace Fortran {
+namespace parser {
+struct OpenACCConstruct;
+} // namespace parser
+
+namespace lower {
+
+class AbstractConverter;
+
+namespace pft {
+struct Evaluation;
+} // namespace pft
+
+void genOpenACCConstruct(AbstractConverter &, pft::Evaluation &,
+                         const parser::OpenACCConstruct &);
+
+} // namespace lower
+} // namespace Fortran
+
+#endif // FORTRAN_LOWER_OPENACC_H
diff --git a/flang/include/flang/Lower/PFTBuilder.h b/flang/include/flang/Lower/PFTBuilder.h
index 1201470de704c..3230d36e0e9a8 100644
--- a/flang/include/flang/Lower/PFTBuilder.h
+++ b/flang/include/flang/Lower/PFTBuilder.h
@@ -131,8 +131,8 @@ using Constructs =
                parser::ForallConstruct>;
 
 using Directives =
-    std::tuple<parser::CompilerDirective, parser::OpenMPConstruct,
-               parser::OmpEndLoopDirective>;
+    std::tuple<parser::CompilerDirective, parser::OpenACCConstruct,
+               parser::OpenMPConstruct, parser::OmpEndLoopDirective>;
 
 template <typename A>
 static constexpr bool isActionStmt{common::HasMember<A, ActionStmts>};
diff --git a/flang/lib/Evaluate/constant.cpp b/flang/lib/Evaluate/constant.cpp
index ecdcc0ab8e3e6..e7497630f5969 100644
--- a/flang/lib/Evaluate/constant.cpp
+++ b/flang/lib/Evaluate/constant.cpp
@@ -244,19 +244,26 @@ std::size_t Constant<Type<TypeCategory::Character, KIND>>::CopyFrom(
     std::size_t count, ConstantSubscripts &resultSubscripts,
     const std::vector<int> *dimOrder) {
   CHECK(length_ == source.length_);
-  std::size_t copied{0};
-  std::size_t elementBytes{length_ * sizeof(decltype(values_[0]))};
-  ConstantSubscripts sourceSubscripts{source.lbounds()};
-  while (copied < count) {
-    auto *dest{&values_.at(SubscriptsToOffset(resultSubscripts) * length_)};
-    const auto *src{&source.values_.at(
-        source.SubscriptsToOffset(sourceSubscripts) * length_)};
-    std::memcpy(dest, src, elementBytes);
-    copied++;
-    source.IncrementSubscripts(sourceSubscripts);
-    IncrementSubscripts(resultSubscripts, dimOrder);
+  if (length_ == 0) {
+    // It's possible that the array of strings consists of all empty strings.
+    // If so, constant folding will result in a string that's completely empty
+    // and the length_ will be zero, and there's nothing to do.
+    return count;
+  } else {
+    std::size_t copied{0};
+    std::size_t elementBytes{length_ * sizeof(decltype(values_[0]))};
+    ConstantSubscripts sourceSubscripts{source.lbounds()};
+    while (copied < count) {
+      auto *dest{&values_.at(SubscriptsToOffset(resultSubscripts) * length_)};
+      const auto *src{&source.values_.at(
+          source.SubscriptsToOffset(sourceSubscripts) * length_)};
+      std::memcpy(dest, src, elementBytes);
+      copied++;
+      source.IncrementSubscripts(sourceSubscripts);
+      IncrementSubscripts(resultSubscripts, dimOrder);
+    }
+    return copied;
   }
-  return copied;
 }
 
 // Constant<SomeDerived> specialization
diff --git a/flang/lib/Lower/CMakeLists.txt b/flang/lib/Lower/CMakeLists.txt
index 975065c9ed7de..b28e0a2994549 100644
--- a/flang/lib/Lower/CMakeLists.txt
+++ b/flang/lib/Lower/CMakeLists.txt
@@ -13,6 +13,7 @@ add_flang_library(FortranLower
   IntrinsicCall.cpp
   IO.cpp
   Mangler.cpp
+  OpenACC.cpp
   OpenMP.cpp
   PFTBuilder.cpp
 
diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
new file mode 100644
index 0000000000000..7202d4ec03199
--- /dev/null
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -0,0 +1,52 @@
+//===-- OpenMP.cpp -- OpenACC directive lowering --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Lower/OpenACC.h"
+#include "flang/Lower/Bridge.h"
+#include "flang/Lower/FIRBuilder.h"
+#include "flang/Lower/PFTBuilder.h"
+#include "flang/Parser/parse-tree.h"
+#include "llvm/Frontend/OpenACC/ACC.h.inc"
+
+#define TODO() llvm_unreachable("not yet implemented")
+
+void Fortran::lower::genOpenACCConstruct(
+    Fortran::lower::AbstractConverter &absConv,
+    Fortran::lower::pft::Evaluation &eval,
+    const Fortran::parser::OpenACCConstruct &accConstruct) {
+
+  std::visit(
+      common::visitors{
+          [&](const Fortran::parser::OpenACCBlockConstruct &blockConstruct) {
+            TODO();
+          },
+          [&](const Fortran::parser::OpenACCCombinedConstruct
+                  &combinedConstruct) { TODO(); },
+          [&](const Fortran::parser::OpenACCLoopConstruct &loopConstruct) {
+            TODO();
+          },
+          [&](const Fortran::parser::OpenACCStandaloneConstruct
+                  &standaloneConstruct) { TODO(); },
+          [&](const Fortran::parser::OpenACCRoutineConstruct
+                  &routineConstruct) { TODO(); },
+          [&](const Fortran::parser::OpenACCCacheConstruct &cacheConstruct) {
+            TODO();
+          },
+          [&](const Fortran::parser::OpenACCWaitConstruct &waitConstruct) {
+            TODO();
+          },
+          [&](const Fortran::parser::OpenACCAtomicConstruct &atomicConstruct) {
+            TODO();
+          },
+      },
+      accConstruct.u);
+}
diff --git a/flang/lib/Lower/OpenMP.cpp b/flang/lib/Lower/OpenMP.cpp
index e839c14a2a5d1..bc7cd27df320e 100644
--- a/flang/lib/Lower/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP.cpp
@@ -39,7 +39,9 @@ static void genOMP(Fortran::lower::AbstractConverter &absConv,
         absConv.getCurrentLocation());
     break;
   case llvm::omp::Directive::OMPD_taskyield:
-    TODO();
+    absConv.getFirOpBuilder().create<mlir::omp::TaskyieldOp>(
+        absConv.getCurrentLocation());
+    break;
   case llvm::omp::Directive::OMPD_target_enter_data:
     TODO();
   case llvm::omp::Directive::OMPD_target_exit_data:
diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index 07ddbddaa08da..c81d6cb508464 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -246,6 +246,7 @@ void Prescanner::NextLine() {
 }
 
 void Prescanner::LabelField(TokenSequence &token, int outCol) {
+  bool badLabel{false};
   for (; *at_ != '\n' && column_ <= 6; ++at_) {
     if (*at_ == '\t') {
       ++at_;
@@ -255,6 +256,11 @@ void Prescanner::LabelField(TokenSequence &token, int outCol) {
     if (*at_ != ' ' &&
         !(*at_ == '0' && column_ == 6)) { // '0' in column 6 becomes space
       EmitChar(token, *at_);
+      if (!IsDecimalDigit(*at_) && !badLabel) {
+        Say(GetProvenance(at_),
+            "Character in fixed-form label field must be a digit"_en_US);
+        badLabel = true;
+      }
       ++outCol;
     }
     ++column_;
@@ -262,17 +268,11 @@ void Prescanner::LabelField(TokenSequence &token, int outCol) {
   if (outCol > 1) {
     token.CloseToken();
   }
-  if (outCol < 7) {
-    if (outCol == 1) {
-      token.Put("      ", 6, sixSpaceProvenance_.start());
-    } else {
-      for (; outCol < 7; ++outCol) {
-        token.PutNextTokenChar(' ', spaceProvenance_);
-      }
-      token.CloseToken();
-    }
-  }
   SkipToNextSignificantCharacter();
+  if (IsDecimalDigit(*at_)) {
+    Say(GetProvenance(at_),
+        "Label digit is not in fixed-form label field"_en_US);
+  }
 }
 
 void Prescanner::SkipToEndOfLine() {
@@ -887,10 +887,8 @@ const char *Prescanner::FixedFormContinuationLine(bool mightNeedSpace) {
         return nextLine_ + 6;
       }
     }
-    if (delimiterNesting_ > 0) {
-      if (!IsFixedFormCommentChar(col1)) {
-        return nextLine_;
-      }
+    if (IsImplicitContinuation()) {
+      return nextLine_;
     }
   }
   return nullptr; // not a continuation line
@@ -927,7 +925,7 @@ const char *Prescanner::FreeFormContinuationLine(bool ampersand) {
       return p + 1;
     } else if (*p == '!' || *p == '\n' || *p == '#') {
       return nullptr;
-    } else if (ampersand || delimiterNesting_ > 0) {
+    } else if (ampersand || IsImplicitContinuation()) {
       if (p > nextLine_) {
         --p;
       } else {
@@ -981,6 +979,14 @@ bool Prescanner::FreeFormContinuation() {
   return false;
 }
 
+// Implicit line continuation allows a preprocessor macro call with
+// arguments to span multiple lines.
+bool Prescanner::IsImplicitContinuation() const {
+  return !inPreprocessorDirective_ && !inCharLiteral_ &&
+      delimiterNesting_ > 0 && nextLine_ < limit_ &&
+      ClassifyLine(nextLine_).kind == LineClassification::Kind::Source;
+}
+
 bool Prescanner::Continuation(bool mightNeedFixedFormSpace) {
   if (*at_ == '\n' || *at_ == '&') {
     if (inFixedForm_) {
diff --git a/flang/lib/Parser/prescan.h b/flang/lib/Parser/prescan.h
index 3a08a90c3679f..6e492a73597b3 100644
--- a/flang/lib/Parser/prescan.h
+++ b/flang/lib/Parser/prescan.h
@@ -166,6 +166,7 @@ class Prescanner {
   const char *IsPreprocessorDirectiveLine(const char *) const;
   const char *FixedFormContinuationLine(bool mightNeedSpace);
   const char *FreeFormContinuationLine(bool ampersand);
+  bool IsImplicitContinuation() const;
   bool FixedFormContinuation(bool mightNeedSpace);
   bool FreeFormContinuation();
   bool Continuation(bool mightNeedFixedFormSpace);
@@ -220,8 +221,6 @@ class Prescanner {
       cooked_.allSources().CompilerInsertionProvenance(' ')};
   const Provenance backslashProvenance_{
       cooked_.allSources().CompilerInsertionProvenance('\\')};
-  const ProvenanceRange sixSpaceProvenance_{
-      cooked_.allSources().AddCompilerInsertion("      "s)};
 
   // To avoid probing the set of active compiler directive sentinel strings
   // on every comment line, they're checked first with a cheap Bloom filter.
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index d857d36ed05d9..725dfcd9ced83 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -44,7 +44,8 @@ bool OmpStructureChecker::HasInvalidWorksharingNesting(
 void OmpStructureChecker::CheckAllowed(llvm::omp::Clause type) {
   if (!GetContext().allowedClauses.test(type) &&
       !GetContext().allowedOnceClauses.test(type) &&
-      !GetContext().allowedExclusiveClauses.test(type)) {
+      !GetContext().allowedExclusiveClauses.test(type) &&
+      !GetContext().requiredClauses.test(type)) {
     context_.Say(GetContext().clauseSource,
         "%s clause is not allowed on the %s directive"_err_en_US,
         parser::ToUpperCaseLetters(llvm::omp::getOpenMPClauseName(type).str()),
diff --git a/flang/lib/Semantics/expression.cpp b/flang/lib/Semantics/expression.cpp
index 4cb43a91ef09d..e78c2d20edcf7 100644
--- a/flang/lib/Semantics/expression.cpp
+++ b/flang/lib/Semantics/expression.cpp
@@ -123,6 +123,7 @@ class ArgumentAnalyzer {
   }
   void Analyze(const parser::Variable &);
   void Analyze(const parser::ActualArgSpec &, bool isSubroutine);
+  void ConvertBOZ(std::size_t i, std::optional<DynamicType> otherType);
 
   bool IsIntrinsicRelational(RelationalOperator) const;
   bool IsIntrinsicLogical() const;
@@ -141,6 +142,7 @@ class ArgumentAnalyzer {
   // Find and return a user-defined assignment
   std::optional<ProcedureRef> TryDefinedAssignment();
   std::optional<ProcedureRef> GetDefinedAssignmentProc();
+  std::optional<DynamicType> GetType(std::size_t) const;
   void Dump(llvm::raw_ostream &);
 
 private:
@@ -153,7 +155,6 @@ class ArgumentAnalyzer {
   void AddAssignmentConversion(
       const DynamicType &lhsType, const DynamicType &rhsType);
   bool OkLogicalIntegerAssignment(TypeCategory lhs, TypeCategory rhs);
-  std::optional<DynamicType> GetType(std::size_t) const;
   int GetRank(std::size_t) const;
   bool IsBOZLiteral(std::size_t i) const {
     return std::holds_alternative<BOZLiteralConstant>(GetExpr(i).u);
@@ -2337,12 +2338,16 @@ MaybeExpr RelationHelper(ExpressionAnalyzer &context, RelationalOperator opr,
   analyzer.Analyze(std::get<1>(x.t));
   if (analyzer.fatalErrors()) {
     return std::nullopt;
-  } else if (analyzer.IsIntrinsicRelational(opr)) {
-    return AsMaybeExpr(Relate(context.GetContextualMessages(), opr,
-        analyzer.MoveExpr(0), analyzer.MoveExpr(1)));
   } else {
-    return analyzer.TryDefinedOp(opr,
-        "Operands of %s must have comparable types; have %s and %s"_err_en_US);
+    analyzer.ConvertBOZ(0, analyzer.GetType(1));
+    analyzer.ConvertBOZ(1, analyzer.GetType(0));
+    if (analyzer.IsIntrinsicRelational(opr)) {
+      return AsMaybeExpr(Relate(context.GetContextualMessages(), opr,
+          analyzer.MoveExpr(0), analyzer.MoveExpr(1)));
+    } else {
+      return analyzer.TryDefinedOp(opr,
+          "Operands of %s must have comparable types; have %s and %s"_err_en_US);
+    }
   }
 }
 
@@ -3014,6 +3019,29 @@ int ArgumentAnalyzer::GetRank(std::size_t i) const {
   return i < actuals_.size() ? actuals_[i].value().Rank() : 0;
 }
 
+// If the argument at index i is a BOZ literal, convert its type to match the
+// otherType.  It it's REAL convert to REAL, otherwise convert to INTEGER.
+// Note that IBM supports comparing BOZ literals to CHARACTER operands.  That
+// is not currently supported.
+void ArgumentAnalyzer::ConvertBOZ(
+    std::size_t i, std::optional<DynamicType> otherType) {
+  if (IsBOZLiteral(i)) {
+    Expr<SomeType> &&argExpr{MoveExpr(i)};
+    auto *boz{std::get_if<BOZLiteralConstant>(&argExpr.u)};
+    if (otherType && otherType->category() == TypeCategory::Real) {
+      MaybeExpr realExpr{ConvertToKind<TypeCategory::Real>(
+          context_.context().GetDefaultKind(TypeCategory::Real),
+          std::move(*boz))};
+      actuals_[i] = std::move(*realExpr);
+    } else {
+      MaybeExpr intExpr{ConvertToKind<TypeCategory::Integer>(
+          context_.context().GetDefaultKind(TypeCategory::Integer),
+          std::move(*boz))};
+      actuals_[i] = std::move(*intExpr);
+    }
+  }
+}
+
 // Report error resolving opr when there is a user-defined one available
 void ArgumentAnalyzer::SayNoMatch(const std::string &opr, bool isAssignment) {
   std::string type0{TypeAsFortran(0)};
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 73d111ca3c093..7189b48482654 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -1438,6 +1438,7 @@ class ResolveNamesVisitor : public virtual ScopeHandler,
   void PreSpecificationConstruct(const parser::SpecificationConstruct &);
   void CreateGeneric(const parser::GenericSpec &);
   void FinishSpecificationPart(const std::list<parser::DeclarationConstruct> &);
+  void AnalyzeStmtFunctionStmt(const parser::StmtFunctionStmt &);
   void CheckImports();
   void CheckImport(const SourceName &, const SourceName &);
   void HandleCall(Symbol::Flag, const parser::Call &);
@@ -6101,23 +6102,11 @@ void ResolveNamesVisitor::FinishSpecificationPart(
     }
   }
   currScope().InstantiateDerivedTypes(context());
-  // Analyze the bodies of statement functions now that the symbol in this
-  // specification part have been fully declared and implicitly typed.
   for (const auto &decl : decls) {
     if (const auto *statement{std::get_if<
             parser::Statement<common::Indirection<parser::StmtFunctionStmt>>>(
             &decl.u)}) {
-      const parser::StmtFunctionStmt &stmtFunc{statement->statement.value()};
-      if (Symbol * symbol{std::get<parser::Name>(stmtFunc.t).symbol}) {
-        if (auto *details{symbol->detailsIf<SubprogramDetails>()}) {
-          if (auto expr{AnalyzeExpr(context(),
-                  std::get<parser::Scalar<parser::Expr>>(stmtFunc.t))}) {
-            details->set_stmtFunction(std::move(*expr));
-          } else {
-            context().SetError(*symbol);
-          }
-        }
-      }
+      AnalyzeStmtFunctionStmt(statement->statement.value());
     }
   }
   // TODO: what about instantiations in BLOCK?
@@ -6126,6 +6115,33 @@ void ResolveNamesVisitor::FinishSpecificationPart(
   CheckEquivalenceSets();
 }
 
+// Analyze the bodies of statement functions now that the symbols in this
+// specification part have been fully declared and implicitly typed.
+void ResolveNamesVisitor::AnalyzeStmtFunctionStmt(
+    const parser::StmtFunctionStmt &stmtFunc) {
+  Symbol *symbol{std::get<parser::Name>(stmtFunc.t).symbol};
+  if (!symbol || !symbol->has<SubprogramDetails>()) {
+    return;
+  }
+  auto &details{symbol->get<SubprogramDetails>()};
+  auto expr{AnalyzeExpr(
+      context(), std::get<parser::Scalar<parser::Expr>>(stmtFunc.t))};
+  if (!expr) {
+    context().SetError(*symbol);
+    return;
+  }
+  if (auto type{evaluate::DynamicType::From(*symbol)}) {
+    auto converted{ConvertToType(*type, std::move(*expr))};
+    if (!converted) {
+      context().SetError(*symbol);
+      return;
+    }
+    details.set_stmtFunction(std::move(*converted));
+  } else {
+    details.set_stmtFunction(std::move(*expr));
+  }
+}
+
 void ResolveNamesVisitor::CheckImports() {
   auto &scope{currScope()};
   switch (scope.GetImportKind()) {
diff --git a/flang/lib/Semantics/symbol.cpp b/flang/lib/Semantics/symbol.cpp
index a7e2696518ffe..e0d80ec6d1c8b 100644
--- a/flang/lib/Semantics/symbol.cpp
+++ b/flang/lib/Semantics/symbol.cpp
@@ -111,6 +111,9 @@ llvm::raw_ostream &operator<<(
     }
   }
   os << (sep == '(' ? "()" : ")");
+  if (x.stmtFunction_) {
+    os << " -> " << x.stmtFunction_->AsFortran();
+  }
   return os;
 }
 
diff --git a/flang/runtime/edit-input.cpp b/flang/runtime/edit-input.cpp
index dd708c7bfb913..ebbe41b49b2ce 100644
--- a/flang/runtime/edit-input.cpp
+++ b/flang/runtime/edit-input.cpp
@@ -177,8 +177,8 @@ static int ScanRealInput(char *buffer, int bufferSize, IoStatementState &io,
           continue;
         }
       }
-      if (ch == '0' && got == start) {
-        // omit leading zeroes
+      if (ch == '0' && got == start && !decimalPoint) {
+        // omit leading zeroes before the decimal
       } else if (ch >= '0' && ch <= '9') {
         if (got < bufferSize) {
           buffer[got++] = ch;
@@ -248,7 +248,7 @@ bool EditCommonRealInput(IoStatementState &io, const DataEdit &edit, void *n) {
   int exponent{0};
   int got{ScanRealInput(buffer, maxDigits + 2, io, edit, exponent)};
   if (got >= maxDigits + 2) {
-    io.GetIoErrorHandler().Crash("EditRealInput: buffer was too small");
+    io.GetIoErrorHandler().Crash("EditCommonRealInput: buffer was too small");
     return false;
   }
   if (got == 0) {
@@ -277,6 +277,8 @@ template <int binaryPrecision>
 bool EditRealInput(IoStatementState &io, const DataEdit &edit, void *n) {
   switch (edit.descriptor) {
   case DataEdit::ListDirected:
+  case DataEdit::ListDirectedRealPart:
+  case DataEdit::ListDirectedImaginaryPart:
   case 'F':
   case 'E': // incl. EN, ES, & EX
   case 'D':
diff --git a/flang/runtime/environment.cpp b/flang/runtime/environment.cpp
index e05baa3323cd4..2b62e8a729a5d 100644
--- a/flang/runtime/environment.cpp
+++ b/flang/runtime/environment.cpp
@@ -7,13 +7,35 @@
 //===----------------------------------------------------------------------===//
 
 #include "environment.h"
+#include "tools.h"
 #include <cstdio>
 #include <cstdlib>
+#include <cstring>
 #include <limits>
 
 namespace Fortran::runtime {
+
 ExecutionEnvironment executionEnvironment;
 
+std::optional<Convert> GetConvertFromString(const char *x, std::size_t n) {
+  static const char *keywords[]{
+      "UNKNOWN", "NATIVE", "LITTLE_ENDIAN", "BIG_ENDIAN", "SWAP", nullptr};
+  switch (IdentifyValue(x, n, keywords)) {
+  case 0:
+    return Convert::Unknown;
+  case 1:
+    return Convert::Native;
+  case 2:
+    return Convert::LittleEndian;
+  case 3:
+    return Convert::BigEndian;
+  case 4:
+    return Convert::Swap;
+  default:
+    return std::nullopt;
+  }
+}
+
 void ExecutionEnvironment::Configure(
     int ac, const char *av[], const char *env[]) {
   argc = ac;
@@ -22,6 +44,7 @@ void ExecutionEnvironment::Configure(
   listDirectedOutputLineLengthLimit = 79; // PGI default
   defaultOutputRoundingMode =
       decimal::FortranRounding::RoundNearest; // RP(==RN)
+  conversion = Convert::Unknown;
 
   if (auto *x{std::getenv("FORT_FMT_RECL")}) {
     char *end;
@@ -34,6 +57,15 @@ void ExecutionEnvironment::Configure(
     }
   }
 
+  if (auto *x{std::getenv("FORT_CONVERT")}) {
+    if (auto convert{GetConvertFromString(x, std::strlen(x))}) {
+      conversion = *convert;
+    } else {
+      std::fprintf(
+          stderr, "Fortran runtime: FORT_CONVERT=%s is invalid; ignored\n", x);
+    }
+  }
+
   // TODO: Set RP/ROUND='PROCESSOR_DEFINED' from environment
 }
 } // namespace Fortran::runtime
diff --git a/flang/runtime/environment.h b/flang/runtime/environment.h
index 0d9d318915cd9..c04c3133f02a0 100644
--- a/flang/runtime/environment.h
+++ b/flang/runtime/environment.h
@@ -10,8 +10,23 @@
 #define FORTRAN_RUNTIME_ENVIRONMENT_H_
 
 #include "flang/Decimal/decimal.h"
+#include <optional>
 
 namespace Fortran::runtime {
+
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+constexpr bool isHostLittleEndian{false};
+#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+constexpr bool isHostLittleEndian{true};
+#else
+#error host endianness is not known
+#endif
+
+// External unformatted I/O data conversions
+enum class Convert { Unknown, Native, LittleEndian, BigEndian, Swap };
+
+std::optional<Convert> GetConvertFromString(const char *, std::size_t);
+
 struct ExecutionEnvironment {
   void Configure(int argc, const char *argv[], const char *envp[]);
 
@@ -20,6 +35,7 @@ struct ExecutionEnvironment {
   const char **envp;
   int listDirectedOutputLineLengthLimit;
   enum decimal::FortranRounding defaultOutputRoundingMode;
+  Convert conversion;
 };
 extern ExecutionEnvironment executionEnvironment;
 } // namespace Fortran::runtime
diff --git a/flang/runtime/format-implementation.h b/flang/runtime/format-implementation.h
index ce0e08b99005e..a4453cd172ea5 100644
--- a/flang/runtime/format-implementation.h
+++ b/flang/runtime/format-implementation.h
@@ -225,6 +225,7 @@ int FormatControl<CONTEXT>::CueUpNextDataEdit(Context &context, bool stop) {
   while (true) {
     std::optional<int> repeat;
     bool unlimited{false};
+    auto maybeReversionPoint{offset_};
     CharType ch{GetNextChar(context)};
     while (ch == ',' || ch == ':') {
       // Skip commas, and don't complain if they're missing; the format
@@ -254,6 +255,7 @@ int FormatControl<CONTEXT>::CueUpNextDataEdit(Context &context, bool stop) {
         return 0;
       }
       stack_[height_].start = offset_ - 1; // the '('
+      RUNTIME_CHECK(context, format_[stack_[height_].start] == '(');
       if (unlimited || height_ == 0) {
         stack_[height_].remaining = Iteration::unlimited;
         unlimitedLoopCheck = offset_ - 1;
@@ -265,6 +267,12 @@ int FormatControl<CONTEXT>::CueUpNextDataEdit(Context &context, bool stop) {
       } else {
         stack_[height_].remaining = 0;
       }
+      if (height_ == 1) {
+        // Subtle point (F'2018 13.4 para 9): tha last parenthesized group
+        // at height 1 becomes the restart point after control reaches the
+        // end of the format, including its repeat count.
+        stack_[0].start = maybeReversionPoint - 1;
+      }
       ++height_;
     } else if (height_ == 0) {
       context.SignalError(IostatErrorInFormat, "FORMAT lacks initial '('");
@@ -276,14 +284,15 @@ int FormatControl<CONTEXT>::CueUpNextDataEdit(Context &context, bool stop) {
         }
         context.AdvanceRecord(); // implied / before rightmost )
       }
+      auto restart{stack_[height_ - 1].start + 1};
       if (stack_[height_ - 1].remaining == Iteration::unlimited) {
-        offset_ = stack_[height_ - 1].start + 1;
+        offset_ = restart;
         if (offset_ == unlimitedLoopCheck) {
           context.SignalError(IostatErrorInFormat,
               "Unlimited repetition in FORMAT lacks data edit descriptors");
         }
       } else if (stack_[height_ - 1].remaining-- > 0) {
-        offset_ = stack_[height_ - 1].start + 1;
+        offset_ = restart;
       } else {
         --height_;
       }
@@ -396,7 +405,7 @@ DataEdit FormatControl<CONTEXT>::GetNextDataEdit(
     ++height_;
   }
   edit.repeat = 1;
-  if (height_ > 1) {
+  if (height_ > 1) { // Subtle: stack_[0].start doesn't necessarily point to '('
     int start{stack_[height_ - 1].start};
     if (format_[start] != '(') {
       if (stack_[height_ - 1].remaining > maxRepeat) {
diff --git a/flang/runtime/format.cpp b/flang/runtime/format.cpp
index 2c259b1f7b83b..65ed12447bb58 100644
--- a/flang/runtime/format.cpp
+++ b/flang/runtime/format.cpp
@@ -15,7 +15,8 @@ DataEdit DefaultFormatControlCallbacks::GetNextDataEdit(int) {
         "non-formatted I/O statement");
   return {};
 }
-bool DefaultFormatControlCallbacks::Emit(const char *, std::size_t) {
+bool DefaultFormatControlCallbacks::Emit(
+    const char *, std::size_t, std::size_t) {
   Crash("DefaultFormatControlCallbacks::Emit(char) called for non-output I/O "
         "statement");
   return {};
diff --git a/flang/runtime/format.h b/flang/runtime/format.h
index 07eff97a70484..7d33e0ffbd147 100644
--- a/flang/runtime/format.h
+++ b/flang/runtime/format.h
@@ -63,7 +63,7 @@ struct DataEdit {
 struct DefaultFormatControlCallbacks : public IoErrorHandler {
   using IoErrorHandler::IoErrorHandler;
   DataEdit GetNextDataEdit(int = 1);
-  bool Emit(const char *, std::size_t);
+  bool Emit(const char *, std::size_t, std::size_t elementBytes = 0);
   bool Emit(const char16_t *, std::size_t);
   bool Emit(const char32_t *, std::size_t);
   std::optional<char32_t> GetCurrentChar();
diff --git a/flang/runtime/io-api.cpp b/flang/runtime/io-api.cpp
index 2f077e1f9ff8f..708090ac68561 100644
--- a/flang/runtime/io-api.cpp
+++ b/flang/runtime/io-api.cpp
@@ -614,6 +614,24 @@ bool IONAME(SetAsynchronous)(
   }
 }
 
+bool IONAME(SetConvert)(
+    Cookie cookie, const char *keyword, std::size_t length) {
+  IoStatementState &io{*cookie};
+  auto *open{io.get_if<OpenStatementState>()};
+  if (!open) {
+    io.GetIoErrorHandler().Crash(
+        "SetConvert() called when not in an OPEN statement");
+  }
+  if (auto convert{GetConvertFromString(keyword, length)}) {
+    open->set_convert(*convert);
+    return true;
+  } else {
+    open->SignalError(IostatErrorInKeyword, "Invalid CONVERT='%.*s'",
+        static_cast<int>(length), keyword);
+    return false;
+  }
+}
+
 bool IONAME(SetEncoding)(
     Cookie cookie, const char *keyword, std::size_t length) {
   IoStatementState &io{*cookie};
@@ -818,21 +836,27 @@ bool IONAME(OutputDescriptor)(Cookie cookie, const Descriptor &) {
   io.GetIoErrorHandler().Crash("OutputDescriptor: not yet implemented"); // TODO
 }
 
-bool IONAME(OutputUnformattedBlock)(
-    Cookie cookie, const char *x, std::size_t length) {
+bool IONAME(InputDescriptor)(Cookie cookie, const Descriptor &) {
+  IoStatementState &io{*cookie};
+  io.GetIoErrorHandler().Crash("InputDescriptor: not yet implemented"); // TODO
+}
+
+bool IONAME(OutputUnformattedBlock)(Cookie cookie, const char *x,
+    std::size_t length, std::size_t elementBytes) {
   IoStatementState &io{*cookie};
   if (auto *unf{io.get_if<UnformattedIoStatementState<Direction::Output>>()}) {
-    return unf->Emit(x, length);
+    return unf->Emit(x, length, elementBytes);
   }
   io.GetIoErrorHandler().Crash("OutputUnformattedBlock() called for an I/O "
                                "statement that is not unformatted output");
   return false;
 }
 
-bool IONAME(InputUnformattedBlock)(Cookie cookie, char *x, std::size_t length) {
+bool IONAME(InputUnformattedBlock)(
+    Cookie cookie, char *x, std::size_t length, std::size_t elementBytes) {
   IoStatementState &io{*cookie};
   if (auto *unf{io.get_if<UnformattedIoStatementState<Direction::Input>>()}) {
-    return unf->Receive(x, length);
+    return unf->Receive(x, length, elementBytes);
   }
   io.GetIoErrorHandler().Crash("InputUnformattedBlock() called for an I/O "
                                "statement that is not unformatted output");
@@ -868,86 +892,101 @@ bool IONAME(InputInteger)(Cookie cookie, std::int64_t &n, int kind) {
   return false;
 }
 
-bool IONAME(OutputReal32)(Cookie cookie, float x) {
+template <int PREC, typename REAL>
+static bool OutputReal(Cookie cookie, REAL x) {
   IoStatementState &io{*cookie};
   if (!io.get_if<OutputStatementState>()) {
     io.GetIoErrorHandler().Crash(
-        "OutputReal32() called for a non-output I/O statement");
+        "OutputReal() called for a non-output I/O statement");
     return false;
   }
   if (auto edit{io.GetNextDataEdit()}) {
-    return RealOutputEditing<24>{io, x}.Edit(*edit);
+    return RealOutputEditing<PREC>{io, x}.Edit(*edit);
   }
   return false;
 }
 
-bool IONAME(InputReal32)(Cookie cookie, float &x) {
-  IoStatementState &io{*cookie};
-  if (!io.get_if<InputStatementState>()) {
-    io.GetIoErrorHandler().Crash(
-        "InputReal32() called for a non-input I/O statement");
-    return false;
-  }
-  if (auto edit{io.GetNextDataEdit()}) {
-    if (edit->descriptor == DataEdit::ListDirectedNullValue) {
-      return true;
-    }
-    return EditRealInput<24>(io, *edit, reinterpret_cast<void *>(&x));
-  }
-  return false;
+bool IONAME(OutputReal32)(Cookie cookie, float x) {
+  return OutputReal<24, float>(cookie, x);
 }
 
 bool IONAME(OutputReal64)(Cookie cookie, double x) {
-  IoStatementState &io{*cookie};
-  if (!io.get_if<OutputStatementState>()) {
-    io.GetIoErrorHandler().Crash(
-        "OutputReal64() called for a non-output I/O statement");
-    return false;
-  }
-  if (auto edit{io.GetNextDataEdit()}) {
-    return RealOutputEditing<53>{io, x}.Edit(*edit);
-  }
-  return false;
+  return OutputReal<53, double>(cookie, x);
 }
 
-bool IONAME(InputReal64)(Cookie cookie, double &x) {
+template <int PREC, typename REAL>
+static bool InputReal(Cookie cookie, REAL &x) {
   IoStatementState &io{*cookie};
   if (!io.get_if<InputStatementState>()) {
     io.GetIoErrorHandler().Crash(
-        "InputReal64() called for a non-input I/O statement");
+        "InputReal() called for a non-input I/O statement");
     return false;
   }
   if (auto edit{io.GetNextDataEdit()}) {
     if (edit->descriptor == DataEdit::ListDirectedNullValue) {
       return true;
     }
-    return EditRealInput<53>(io, *edit, reinterpret_cast<void *>(&x));
+    return EditRealInput<PREC>(io, *edit, reinterpret_cast<void *>(&x));
   }
   return false;
 }
 
-bool IONAME(OutputComplex32)(Cookie cookie, float r, float z) {
+bool IONAME(InputReal32)(Cookie cookie, float &x) {
+  return InputReal<24, float>(cookie, x);
+}
+
+bool IONAME(InputReal64)(Cookie cookie, double &x) {
+  return InputReal<53, double>(cookie, x);
+}
+
+template <int PREC, typename REAL>
+static bool OutputComplex(Cookie cookie, REAL r, REAL z) {
   IoStatementState &io{*cookie};
   if (io.get_if<ListDirectedStatementState<Direction::Output>>()) {
     DataEdit real, imaginary;
     real.descriptor = DataEdit::ListDirectedRealPart;
     imaginary.descriptor = DataEdit::ListDirectedImaginaryPart;
-    return RealOutputEditing<24>{io, r}.Edit(real) &&
-        RealOutputEditing<24>{io, z}.Edit(imaginary);
+    return RealOutputEditing<PREC>{io, r}.Edit(real) &&
+        RealOutputEditing<PREC>{io, z}.Edit(imaginary);
   }
-  return IONAME(OutputReal32)(cookie, r) && IONAME(OutputReal32)(cookie, z);
+  return OutputReal<PREC, REAL>(cookie, r) && OutputReal<PREC, REAL>(cookie, z);
+}
+
+bool IONAME(OutputComplex32)(Cookie cookie, float r, float z) {
+  return OutputComplex<24, float>(cookie, r, z);
 }
 
 bool IONAME(OutputComplex64)(Cookie cookie, double r, double z) {
+  return OutputComplex<53, double>(cookie, r, z);
+}
+
+template <int PREC, typename REAL>
+static bool InputComplex(Cookie cookie, REAL x[2]) {
   IoStatementState &io{*cookie};
-  if (io.get_if<ListDirectedStatementState<Direction::Output>>()) {
-    DataEdit real, imaginary;
-    real.descriptor = DataEdit::ListDirectedRealPart;
-    imaginary.descriptor = DataEdit::ListDirectedImaginaryPart;
-    return RealOutputEditing<53>{io, r}.Edit(real) &&
-        RealOutputEditing<53>{io, z}.Edit(imaginary);
+  if (!io.get_if<InputStatementState>()) {
+    io.GetIoErrorHandler().Crash(
+        "InputComplex() called for a non-input I/O statement");
+    return false;
+  }
+  for (int j{0}; j < 2; ++j) {
+    if (auto edit{io.GetNextDataEdit()}) {
+      if (edit->descriptor == DataEdit::ListDirectedNullValue) {
+        return true;
+      }
+      if (!EditRealInput<PREC>(io, *edit, reinterpret_cast<void *>(&x[j]))) {
+        return false;
+      }
+    }
   }
-  return IONAME(OutputReal64)(cookie, r) && IONAME(OutputReal64)(cookie, z);
+  return true;
+}
+
+bool IONAME(InputComplex32)(Cookie cookie, float x[2]) {
+  return InputComplex<24, float>(cookie, x);
+}
+
+bool IONAME(InputComplex64)(Cookie cookie, double x[2]) {
+  return InputComplex<53, double>(cookie, x);
 }
 
 bool IONAME(OutputAscii)(Cookie cookie, const char *x, std::size_t length) {
diff --git a/flang/runtime/io-api.h b/flang/runtime/io-api.h
index 6b394dbdec022..f6ebc63e3f3d2 100644
--- a/flang/runtime/io-api.h
+++ b/flang/runtime/io-api.h
@@ -211,8 +211,10 @@ bool IONAME(SetSign)(Cookie, const char *, std::size_t);
 // and avoid the following items when they might crash.
 bool IONAME(OutputDescriptor)(Cookie, const Descriptor &);
 bool IONAME(InputDescriptor)(Cookie, const Descriptor &);
-bool IONAME(OutputUnformattedBlock)(Cookie, const char *, std::size_t);
-bool IONAME(InputUnformattedBlock)(Cookie, char *, std::size_t);
+bool IONAME(OutputUnformattedBlock)(
+    Cookie, const char *, std::size_t, std::size_t elementBytes);
+bool IONAME(InputUnformattedBlock)(
+    Cookie, char *, std::size_t, std::size_t elementBytes);
 bool IONAME(OutputInteger64)(Cookie, std::int64_t);
 bool IONAME(InputInteger)(Cookie, std::int64_t &, int kind = 8);
 bool IONAME(OutputReal32)(Cookie, float);
@@ -220,7 +222,9 @@ bool IONAME(InputReal32)(Cookie, float &);
 bool IONAME(OutputReal64)(Cookie, double);
 bool IONAME(InputReal64)(Cookie, double &);
 bool IONAME(OutputComplex32)(Cookie, float, float);
+bool IONAME(InputComplex32)(Cookie, float[2]);
 bool IONAME(OutputComplex64)(Cookie, double, double);
+bool IONAME(InputComplex64)(Cookie, double[2]);
 bool IONAME(OutputAscii)(Cookie, const char *, std::size_t);
 bool IONAME(InputAscii)(Cookie, char *, std::size_t);
 bool IONAME(OutputLogical)(Cookie, bool);
@@ -236,6 +240,8 @@ bool IONAME(SetAccess)(Cookie, const char *, std::size_t);
 bool IONAME(SetAction)(Cookie, const char *, std::size_t);
 // ASYNCHRONOUS=YES, NO
 bool IONAME(SetAsynchronous)(Cookie, const char *, std::size_t);
+// CONVERT=NATIVE, LITTLE_ENDIAN, BIG_ENDIAN, or SWAP
+bool IONAME(SetConvert)(Cookie, const char *, std::size_t);
 // ENCODING=UTF-8, DEFAULT
 bool IONAME(SetEncoding)(Cookie, const char *, std::size_t);
 // FORM=FORMATTED, UNFORMATTED
diff --git a/flang/runtime/io-stmt.cpp b/flang/runtime/io-stmt.cpp
index 9e89e0c28816d..b8e7781f235b0 100644
--- a/flang/runtime/io-stmt.cpp
+++ b/flang/runtime/io-stmt.cpp
@@ -38,7 +38,7 @@ InternalIoStatementState<DIR, CHAR>::InternalIoStatementState(
 
 template <Direction DIR, typename CHAR>
 bool InternalIoStatementState<DIR, CHAR>::Emit(
-    const CharType *data, std::size_t chars) {
+    const CharType *data, std::size_t chars, std::size_t /*elementBytes*/) {
   if constexpr (DIR == Direction::Input) {
     Crash("InternalIoStatementState<Direction::Input>::Emit() called");
     return false;
@@ -167,7 +167,7 @@ int OpenStatementState::EndIoStatement() {
                 "than 'OLD'");
   }
   unit().OpenUnit(status_.value_or(OpenStatus::Unknown), action_, position_,
-      std::move(path_), pathLength_, *this);
+      std::move(path_), pathLength_, convert_, *this);
   return ExternalIoStatementBase::EndIoStatement();
 }
 
@@ -195,11 +195,12 @@ template <Direction DIR> int ExternalIoStatementState<DIR>::EndIoStatement() {
 }
 
 template <Direction DIR>
-bool ExternalIoStatementState<DIR>::Emit(const char *data, std::size_t chars) {
+bool ExternalIoStatementState<DIR>::Emit(
+    const char *data, std::size_t bytes, std::size_t elementBytes) {
   if constexpr (DIR == Direction::Input) {
     Crash("ExternalIoStatementState::Emit(char) called for input statement");
   }
-  return unit().Emit(data, chars * sizeof(*data), *this);
+  return unit().Emit(data, bytes, elementBytes, *this);
 }
 
 template <Direction DIR>
@@ -210,8 +211,8 @@ bool ExternalIoStatementState<DIR>::Emit(
         "ExternalIoStatementState::Emit(char16_t) called for input statement");
   }
   // TODO: UTF-8 encoding
-  return unit().Emit(
-      reinterpret_cast<const char *>(data), chars * sizeof(*data), *this);
+  return unit().Emit(reinterpret_cast<const char *>(data), chars * sizeof *data,
+      static_cast<int>(sizeof *data), *this);
 }
 
 template <Direction DIR>
@@ -222,8 +223,8 @@ bool ExternalIoStatementState<DIR>::Emit(
         "ExternalIoStatementState::Emit(char32_t) called for input statement");
   }
   // TODO: UTF-8 encoding
-  return unit().Emit(
-      reinterpret_cast<const char *>(data), chars * sizeof(*data), *this);
+  return unit().Emit(reinterpret_cast<const char *>(data), chars * sizeof *data,
+      static_cast<int>(sizeof *data), *this);
 }
 
 template <Direction DIR>
@@ -277,8 +278,10 @@ std::optional<DataEdit> IoStatementState::GetNextDataEdit(int n) {
       [&](auto &x) { return x.get().GetNextDataEdit(*this, n); }, u_);
 }
 
-bool IoStatementState::Emit(const char *data, std::size_t n) {
-  return std::visit([=](auto &x) { return x.get().Emit(data, n); }, u_);
+bool IoStatementState::Emit(
+    const char *data, std::size_t n, std::size_t elementBytes) {
+  return std::visit(
+      [=](auto &x) { return x.get().Emit(data, n, elementBytes); }, u_);
 }
 
 std::optional<char32_t> IoStatementState::GetCurrentChar() {
@@ -506,6 +509,7 @@ ListDirectedStatementState<Direction::Input>::GetNextDataEdit(
   } else if (realPart_) {
     realPart_ = false;
     imaginaryPart_ = true;
+    edit.descriptor = DataEdit::ListDirectedImaginaryPart;
   }
   if (!ch) {
     return std::nullopt;
@@ -571,17 +575,29 @@ ListDirectedStatementState<Direction::Input>::GetNextDataEdit(
   if (!imaginaryPart_ && ch && *ch == '(') {
     realPart_ = true;
     io.HandleRelativePosition(1);
+    edit.descriptor = DataEdit::ListDirectedRealPart;
   }
   return edit;
 }
 
 template <Direction DIR>
-bool UnformattedIoStatementState<DIR>::Receive(char *data, std::size_t bytes) {
+bool UnformattedIoStatementState<DIR>::Receive(
+    char *data, std::size_t bytes, std::size_t elementBytes) {
   if constexpr (DIR == Direction::Output) {
     this->Crash(
         "UnformattedIoStatementState::Receive() called for output statement");
   }
-  return this->unit().Receive(data, bytes, *this);
+  return this->unit().Receive(data, bytes, elementBytes, *this);
+}
+
+template <Direction DIR>
+bool UnformattedIoStatementState<DIR>::Emit(
+    const char *data, std::size_t bytes, std::size_t elementBytes) {
+  if constexpr (DIR == Direction::Input) {
+    this->Crash(
+        "UnformattedIoStatementState::Emit() called for input statement");
+  }
+  return ExternalIoStatementState<DIR>::Emit(data, bytes, elementBytes);
 }
 
 template <Direction DIR>
diff --git a/flang/runtime/io-stmt.h b/flang/runtime/io-stmt.h
index da58769ef1143..755e5946ff3b9 100644
--- a/flang/runtime/io-stmt.h
+++ b/flang/runtime/io-stmt.h
@@ -49,7 +49,7 @@ class IoStatementState {
   // This design avoids virtual member functions and function pointers,
   // which may not have good support in some runtime environments.
   std::optional<DataEdit> GetNextDataEdit(int = 1);
-  bool Emit(const char *, std::size_t);
+  bool Emit(const char *, std::size_t, std::size_t elementBytes = 0);
   std::optional<char32_t> GetCurrentChar(); // vacant after end of record
   bool AdvanceRecord(int = 1);
   void BackspaceRecord();
@@ -159,7 +159,8 @@ class InternalIoStatementState : public IoStatementBase,
   InternalIoStatementState(
       const Descriptor &, const char *sourceFile = nullptr, int sourceLine = 0);
   int EndIoStatement();
-  bool Emit(const CharType *, std::size_t chars /* not bytes */);
+  bool Emit(const CharType *, std::size_t chars /* not necessarily bytes */,
+      std::size_t elementBytes = 0);
   std::optional<char32_t> GetCurrentChar();
   bool AdvanceRecord(int = 1);
   void BackspaceRecord();
@@ -238,7 +239,7 @@ class ExternalIoStatementState : public ExternalIoStatementBase,
 public:
   using ExternalIoStatementBase::ExternalIoStatementBase;
   int EndIoStatement();
-  bool Emit(const char *, std::size_t);
+  bool Emit(const char *, std::size_t, std::size_t elementBytes = 0);
   bool Emit(const char16_t *, std::size_t chars /* not bytes */);
   bool Emit(const char32_t *, std::size_t chars /* not bytes */);
   std::optional<char32_t> GetCurrentChar();
@@ -283,7 +284,8 @@ template <Direction DIR>
 class UnformattedIoStatementState : public ExternalIoStatementState<DIR> {
 public:
   using ExternalIoStatementState<DIR>::ExternalIoStatementState;
-  bool Receive(char *, std::size_t);
+  bool Receive(char *, std::size_t, std::size_t elementBytes = 0);
+  bool Emit(const char *, std::size_t, std::size_t elementBytes = 0);
   int EndIoStatement();
 };
 
@@ -298,6 +300,7 @@ class OpenStatementState : public ExternalIoStatementBase {
   void set_path(const char *, std::size_t, int kind); // FILE=
   void set_position(Position position) { position_ = position; } // POSITION=
   void set_action(Action action) { action_ = action; } // ACTION=
+  void set_convert(Convert convert) { convert_ = convert; } // CONVERT=
   int EndIoStatement();
 
 private:
@@ -305,6 +308,7 @@ class OpenStatementState : public ExternalIoStatementBase {
   std::optional<OpenStatus> status_;
   Position position_{Position::AsIs};
   std::optional<Action> action_;
+  Convert convert_{Convert::Native};
   OwningPtr<char> path_;
   std::size_t pathLength_;
 };
diff --git a/flang/runtime/main.cpp b/flang/runtime/main.cpp
index 5de2e6434abb1..d01a8bffc7f2a 100644
--- a/flang/runtime/main.cpp
+++ b/flang/runtime/main.cpp
@@ -33,4 +33,14 @@ void RTNAME(ProgramStart)(int argc, const char *argv[], const char *envp[]) {
   ConfigureFloatingPoint();
   // I/O is initialized on demand so that it works for non-Fortran main().
 }
+
+void RTNAME(ByteswapOption)() {
+  if (Fortran::runtime::executionEnvironment.conversion ==
+      Fortran::runtime::Convert::Unknown) {
+    // The environment variable overrides the command-line option;
+    // either of them take precedence over explicit OPEN(CONVERT=) specifiers.
+    Fortran::runtime::executionEnvironment.conversion =
+        Fortran::runtime::Convert::Swap;
+  }
+}
 }
diff --git a/flang/runtime/main.h b/flang/runtime/main.h
index 31ef8f8f4446e..a69eead6bb96c 100644
--- a/flang/runtime/main.h
+++ b/flang/runtime/main.h
@@ -14,6 +14,7 @@
 
 FORTRAN_EXTERN_C_BEGIN
 void RTNAME(ProgramStart)(int, const char *[], const char *[]);
+void RTNAME(ByteswapOption)(); // -byteswapio
 FORTRAN_EXTERN_C_END
 
 #endif // FORTRAN_RUNTIME_MAIN_H_
diff --git a/flang/runtime/unit.cpp b/flang/runtime/unit.cpp
index c6af53e6ec223..a4c69df8d6a9a 100644
--- a/flang/runtime/unit.cpp
+++ b/flang/runtime/unit.cpp
@@ -7,10 +7,12 @@
 //===----------------------------------------------------------------------===//
 
 #include "unit.h"
+#include "environment.h"
 #include "io-error.h"
 #include "lock.h"
 #include "unit-map.h"
 #include <cstdio>
+#include <utility>
 
 namespace Fortran::runtime::io {
 
@@ -65,7 +67,7 @@ ExternalFileUnit &ExternalFileUnit::LookUpOrCreateAnonymous(
     result.OpenUnit(
         dir == Direction::Input ? OpenStatus::Old : OpenStatus::Replace,
         Action::ReadWrite, Position::Rewind, std::move(path),
-        std::strlen(path.get()), handler);
+        std::strlen(path.get()), Convert::Native, handler);
     result.isUnformatted = isUnformatted;
   }
   return result;
@@ -90,7 +92,13 @@ int ExternalFileUnit::NewUnit(const Terminator &terminator) {
 
 void ExternalFileUnit::OpenUnit(OpenStatus status, std::optional<Action> action,
     Position position, OwningPtr<char> &&newPath, std::size_t newPathLength,
-    IoErrorHandler &handler) {
+    Convert convert, IoErrorHandler &handler) {
+  if (executionEnvironment.conversion != Convert::Unknown) {
+    convert = executionEnvironment.conversion;
+  }
+  swapEndianness_ = convert == Convert::Swap ||
+      (convert == Convert::LittleEndian && !isHostLittleEndian) ||
+      (convert == Convert::BigEndian && isHostLittleEndian);
   if (IsOpen()) {
     if (status == OpenStatus::Old &&
         (!newPath.get() ||
@@ -213,8 +221,20 @@ void ExternalFileUnit::FlushAll(IoErrorHandler &handler) {
   }
 }
 
-bool ExternalFileUnit::Emit(
-    const char *data, std::size_t bytes, IoErrorHandler &handler) {
+static void SwapEndianness(
+    char *data, std::size_t bytes, std::size_t elementBytes) {
+  if (elementBytes > 1) {
+    auto half{elementBytes >> 1};
+    for (std::size_t j{0}; j + elementBytes <= bytes; j += elementBytes) {
+      for (std::size_t k{0}; k < half; ++k) {
+        std::swap(data[j + k], data[j + elementBytes - 1 - k]);
+      }
+    }
+  }
+}
+
+bool ExternalFileUnit::Emit(const char *data, std::size_t bytes,
+    std::size_t elementBytes, IoErrorHandler &handler) {
   auto furthestAfter{std::max(furthestPositionInRecord,
       positionInRecord + static_cast<std::int64_t>(bytes))};
   if (furthestAfter > recordLength.value_or(furthestAfter)) {
@@ -230,14 +250,18 @@ bool ExternalFileUnit::Emit(
     std::memset(Frame() + recordOffsetInFrame_ + furthestPositionInRecord, ' ',
         positionInRecord - furthestPositionInRecord);
   }
-  std::memcpy(Frame() + recordOffsetInFrame_ + positionInRecord, data, bytes);
+  char *to{Frame() + recordOffsetInFrame_ + positionInRecord};
+  std::memcpy(to, data, bytes);
+  if (swapEndianness_) {
+    SwapEndianness(to, bytes, elementBytes);
+  }
   positionInRecord += bytes;
   furthestPositionInRecord = furthestAfter;
   return true;
 }
 
-bool ExternalFileUnit::Receive(
-    char *data, std::size_t bytes, IoErrorHandler &handler) {
+bool ExternalFileUnit::Receive(char *data, std::size_t bytes,
+    std::size_t elementBytes, IoErrorHandler &handler) {
   RUNTIME_CHECK(handler, direction_ == Direction::Input);
   auto furthestAfter{std::max(furthestPositionInRecord,
       positionInRecord + static_cast<std::int64_t>(bytes))};
@@ -252,6 +276,9 @@ bool ExternalFileUnit::Receive(
   auto got{ReadFrame(frameOffsetInFile_, need, handler)};
   if (got >= need) {
     std::memcpy(data, Frame() + recordOffsetInFrame_ + positionInRecord, bytes);
+    if (swapEndianness_) {
+      SwapEndianness(data, bytes, elementBytes);
+    }
     positionInRecord += bytes;
     furthestPositionInRecord = furthestAfter;
     return true;
@@ -365,7 +392,7 @@ bool ExternalFileUnit::AdvanceRecord(IoErrorHandler &handler) {
         }
       } else {
         positionInRecord = furthestPositionInRecord;
-        ok &= Emit("\n", 1, handler); // TODO: Windows CR+LF
+        ok &= Emit("\n", 1, 1, handler); // TODO: Windows CR+LF
       }
     }
     frameOffsetInFile_ +=
diff --git a/flang/runtime/unit.h b/flang/runtime/unit.h
index d2d2dce035f14..f94e4229cd4c8 100644
--- a/flang/runtime/unit.h
+++ b/flang/runtime/unit.h
@@ -49,7 +49,8 @@ class ExternalFileUnit : public ConnectionState,
   static void FlushAll(IoErrorHandler &);
 
   void OpenUnit(OpenStatus, std::optional<Action>, Position,
-      OwningPtr<char> &&path, std::size_t pathLength, IoErrorHandler &);
+      OwningPtr<char> &&path, std::size_t pathLength, Convert,
+      IoErrorHandler &);
   void CloseUnit(CloseStatus, IoErrorHandler &);
   void DestroyClosed();
 
@@ -67,8 +68,9 @@ class ExternalFileUnit : public ConnectionState,
     return *io_;
   }
 
-  bool Emit(const char *, std::size_t, IoErrorHandler &);
-  bool Receive(char *, std::size_t, IoErrorHandler &);
+  bool Emit(
+      const char *, std::size_t, std::size_t elementBytes, IoErrorHandler &);
+  bool Receive(char *, std::size_t, std::size_t elementBytes, IoErrorHandler &);
   std::optional<char32_t> GetCurrentChar(IoErrorHandler &);
   void SetLeftTabLimit();
   void BeginReadingRecord(IoErrorHandler &);
@@ -122,6 +124,8 @@ class ExternalFileUnit : public ConnectionState,
   // manage the frame and the current record therein separately.
   std::int64_t frameOffsetInFile_{0};
   std::size_t recordOffsetInFrame_{0}; // of currentRecordNumber
+
+  bool swapEndianness_{false};
 };
 
 } // namespace Fortran::runtime::io
diff --git a/flang/test/Lower/pre-fir-tree05.f90 b/flang/test/Lower/pre-fir-tree05.f90
new file mode 100644
index 0000000000000..f635785e3274b
--- /dev/null
+++ b/flang/test/Lower/pre-fir-tree05.f90
@@ -0,0 +1,33 @@
+! RUN: %f18 -fdebug-pre-fir-tree -fparse-only -fopenacc %s | FileCheck %s
+
+! Test structure of the Pre-FIR tree with OpenACC construct
+
+! CHECK: Subroutine foo
+subroutine foo()
+  ! CHECK-NEXT: <<OpenACCConstruct>>
+  ! CHECK-NEXT: <<OpenACCConstruct>>
+  !$acc parallel
+  !$acc loop
+  ! CHECK-NEXT: <<DoConstruct>>
+  ! CHECK-NEXT: NonLabelDoStmt
+  do i=1,5
+    ! CHECK-NEXT: PrintStmt
+    print *, "hey"
+    ! CHECK-NEXT: <<DoConstruct>>
+    ! CHECK-NEXT: NonLabelDoStmt
+    do j=1,5
+      ! CHECK-NEXT: PrintStmt
+      print *, "hello", i, j
+    ! CHECK-NEXT: EndDoStmt
+    ! CHECK-NEXT: <<End DoConstruct>>
+    end do
+  ! CHECK-NEXT: EndDoStmt
+  ! CHECK-NEXT: <<End DoConstruct>>
+  end do
+  !$acc end parallel
+  ! CHECK-NEXT: <<End OpenACCConstruct>>
+  ! CHECK-NEXT: <<End OpenACCConstruct>>
+  ! CHECK-NEXT: ContinueStmt
+end subroutine
+! CHECK-NEXT: EndSubroutine foo
+
diff --git a/flang/test/Parser/badlabel.f b/flang/test/Parser/badlabel.f
new file mode 100644
index 0000000000000..28008bd809d95
--- /dev/null
+++ b/flang/test/Parser/badlabel.f
@@ -0,0 +1,14 @@
+! RUN: %f18 -E %s 2>&1 | FileCheck %s
+! CHECK: Label digit is not in fixed-form label field
+      1 continue
+! CHECK: Label digit is not in fixed-form label field
+ 1    2 continue
+! CHECK-NOT: Label is not in fixed-form label field
+      con
+     3 tinue
+! CHECK: Character in fixed-form label field must be a digit
+end
+! CHECK: 1continue
+! CHECK: 12continue
+! CHECK: continue
+! CHECK: end
diff --git a/flang/test/Parser/continuation-in-if.f b/flang/test/Parser/continuation-in-if.f
new file mode 100644
index 0000000000000..928170eb6cd51
--- /dev/null
+++ b/flang/test/Parser/continuation-in-if.f
@@ -0,0 +1,9 @@
+! RUN: %f18 -funparse %s 2>&1 | FileCheck %s
+! CHECK: CALL foo("N","N")
+#ifdef transpose
+      call foo('T',
+#else
+      call foo('N',
+#endif
+     $     'N')
+      end
diff --git a/flang/test/Semantics/modfile25.f90 b/flang/test/Semantics/modfile25.f90
index 850298d54e4dd..76dc61a54b6ce 100644
--- a/flang/test/Semantics/modfile25.f90
+++ b/flang/test/Semantics/modfile25.f90
@@ -22,6 +22,9 @@ subroutine subr(x,n1,n2)
     real, intent(in) :: x(:,:)
     integer, intent(in) :: n1(3), n2(:)
     real, allocatable :: a(:,:,:)
+    ! the following fail if we don't handle empty strings
+    Character(0) :: ch1(1,2,3) = Reshape([('',n=1,1*2*3)],[1,2,3])
+    Character(0) :: ch2(3) = reshape(['','',''], [3])
     a = reshape(x,n1)
     a = reshape(x,n2(10:30:9)) ! fails if we can't figure out triplet shape
   end subroutine
diff --git a/flang/test/Semantics/resolve63.f90 b/flang/test/Semantics/resolve63.f90
index 50e1fd0ebdbc6..bd4e1d14e195a 100644
--- a/flang/test/Semantics/resolve63.f90
+++ b/flang/test/Semantics/resolve63.f90
@@ -44,12 +44,32 @@ logical function not_r(x)
   type(t) :: x, y
   real :: r
   logical :: l
+  integer :: iVar
+  complex :: cvar
+  character :: charVar
 contains
   subroutine test_relational()
     l = x == y  !OK
     l = x .eq. y  !OK
+    l = x .eq. y  !OK
+    l = iVar == z'fe' !OK
+    l = z'fe' == iVar !OK
+    l = r == z'fe' !OK
+    l = z'fe' == r !OK
+    l = cVar == z'fe' !OK
+    l = z'fe' == cVar !OK
+    !ERROR: No intrinsic or user-defined OPERATOR(==) matches operand types CHARACTER(KIND=1) and INTEGER(4)
+    l = charVar == z'fe'
+    !ERROR: No intrinsic or user-defined OPERATOR(==) matches operand types INTEGER(4) and CHARACTER(KIND=1)
+    l = z'fe' == charVar
+    !ERROR: No intrinsic or user-defined OPERATOR(==) matches operand types LOGICAL(4) and INTEGER(4)
+    l = l == z'fe' !OK
+    !ERROR: No intrinsic or user-defined OPERATOR(==) matches operand types INTEGER(4) and LOGICAL(4)
+    l = z'fe' == l !OK
     !ERROR: No intrinsic or user-defined OPERATOR(==) matches operand types TYPE(t) and REAL(4)
     l = x == r
+
+    lVar = z'a' == b'1010' !OK
   end
   subroutine test_numeric()
     l = x + r  !OK
diff --git a/flang/tools/f18/f18.cpp b/flang/tools/f18/f18.cpp
index 574a37074e523..bcafb0d53cc7b 100644
--- a/flang/tools/f18/f18.cpp
+++ b/flang/tools/f18/f18.cpp
@@ -88,6 +88,7 @@ struct DriverOptions {
   bool forcedForm{false}; // -Mfixed or -Mfree appeared
   bool warnOnNonstandardUsage{false}; // -Mstandard
   bool warningsAreErrors{false}; // -Werror
+  bool byteswapio{false}; // -byteswapio
   Fortran::parser::Encoding encoding{Fortran::parser::Encoding::UTF_8};
   bool parseOnly{false};
   bool dumpProvenance{false};
@@ -585,6 +586,8 @@ int main(int argc, char *const argv[]) {
       driver.getDefinitionArgs = {arguments[0], arguments[1], arguments[2]};
     } else if (arg == "-fget-symbols-sources") {
       driver.getSymbolsSources = true;
+    } else if (arg == "-byteswapio") {
+      driver.byteswapio = true; // TODO: Pass to lowering, generate call
     } else if (arg == "-help" || arg == "--help" || arg == "-?") {
       llvm::errs()
           << "f18 options:\n"
diff --git a/flang/unittests/CMakeLists.txt b/flang/unittests/CMakeLists.txt
index d53d155f2f2b5..21da59f3afcbf 100644
--- a/flang/unittests/CMakeLists.txt
+++ b/flang/unittests/CMakeLists.txt
@@ -5,6 +5,10 @@ function(add_flang_unittest test_dirname)
   add_unittest(FlangUnitTests ${test_dirname} ${ARGN})
 endfunction()
 
+if (CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
+  add_compile_options("-Wno-suggest-override")
+endif()
+
 add_subdirectory(Optimizer)
 add_subdirectory(Decimal)
 add_subdirectory(Evaluate)
diff --git a/flang/unittests/Lower/OpenMPLoweringTest.cpp b/flang/unittests/Lower/OpenMPLoweringTest.cpp
index 185d07105a192..fd580d71fab3c 100644
--- a/flang/unittests/Lower/OpenMPLoweringTest.cpp
+++ b/flang/unittests/Lower/OpenMPLoweringTest.cpp
@@ -56,4 +56,19 @@ TEST_F(OpenMPLoweringTest, TaskWait) {
   EXPECT_EQ(succeeded(taskWaitOp.verify()), true);
 }
 
+TEST_F(OpenMPLoweringTest, TaskYield) {
+  // Construct a dummy parse tree node for `!OMP taskyield`.
+  struct Fortran::parser::OmpSimpleStandaloneDirective taskYieldDirective(
+      llvm::omp::Directive::OMPD_taskyield);
+
+  // Check and lower the `!OMP taskyield` node to `TaskYieldOp` operation of
+  // OpenMPDialect.
+  EXPECT_EQ(taskYieldDirective.v, llvm::omp::Directive::OMPD_taskyield);
+  auto taskYieldOp = mlirOpBuilder->create<mlir::omp::TaskyieldOp>(
+      mlirOpBuilder->getUnknownLoc());
+
+  EXPECT_EQ(taskYieldOp.getOperationName(), "omp.taskyield");
+  EXPECT_EQ(succeeded(taskYieldOp.verify()), true);
+}
+
 // main() from gtest_main
diff --git a/flang/unittests/Runtime/external-io.cpp b/flang/unittests/Runtime/external-io.cpp
index 63b910b239568..6806ce94ae1e7 100644
--- a/flang/unittests/Runtime/external-io.cpp
+++ b/flang/unittests/Runtime/external-io.cpp
@@ -35,7 +35,7 @@ void TestDirectUnformatted() {
     IONAME(SetRec)(io, j) || (Fail() << "SetRec(" << j << ')', 0);
     buffer = j;
     IONAME(OutputUnformattedBlock)
-    (io, reinterpret_cast<const char *>(&buffer), recl) ||
+    (io, reinterpret_cast<const char *>(&buffer), recl, recl) ||
         (Fail() << "OutputUnformattedBlock()", 0);
     IONAME(EndIoStatement)
     (io) == IostatOk ||
@@ -47,7 +47,7 @@ void TestDirectUnformatted() {
     IONAME(SetRec)
     (io, j) || (Fail() << "SetRec(" << j << ')', 0);
     IONAME(InputUnformattedBlock)
-    (io, reinterpret_cast<char *>(&buffer), recl) ||
+    (io, reinterpret_cast<char *>(&buffer), recl, recl) ||
         (Fail() << "InputUnformattedBlock()", 0);
     IONAME(EndIoStatement)
     (io) == IostatOk ||
@@ -65,6 +65,72 @@ void TestDirectUnformatted() {
   llvm::errs() << "end TestDirectUnformatted()\n";
 }
 
+void TestDirectUnformattedSwapped() {
+  llvm::errs() << "begin TestDirectUnformattedSwapped()\n";
+  // OPEN(NEWUNIT=unit,ACCESS='DIRECT',ACTION='READWRITE',&
+  //   FORM='UNFORMATTED',RECL=8,STATUS='SCRATCH',CONVERT='NATIVE')
+  auto io{IONAME(BeginOpenNewUnit)(__FILE__, __LINE__)};
+  IONAME(SetAccess)(io, "DIRECT", 6) || (Fail() << "SetAccess(DIRECT)", 0);
+  IONAME(SetAction)
+  (io, "READWRITE", 9) || (Fail() << "SetAction(READWRITE)", 0);
+  IONAME(SetForm)
+  (io, "UNFORMATTED", 11) || (Fail() << "SetForm(UNFORMATTED)", 0);
+  IONAME(SetConvert)
+  (io, "NATIVE", 6) || (Fail() << "SetConvert(NATIVE)", 0);
+  std::int64_t buffer;
+  static constexpr std::size_t recl{sizeof buffer};
+  IONAME(SetRecl)(io, recl) || (Fail() << "SetRecl()", 0);
+  IONAME(SetStatus)(io, "SCRATCH", 7) || (Fail() << "SetStatus(SCRATCH)", 0);
+  int unit{-1};
+  IONAME(GetNewUnit)(io, unit) || (Fail() << "GetNewUnit()", 0);
+  llvm::errs() << "unit=" << unit << '\n';
+  IONAME(EndIoStatement)
+  (io) == IostatOk || (Fail() << "EndIoStatement() for OpenNewUnit", 0);
+  static constexpr int records{10};
+  for (int j{1}; j <= records; ++j) {
+    // WRITE(UNIT=unit,REC=j) j
+    io = IONAME(BeginUnformattedOutput)(unit, __FILE__, __LINE__);
+    IONAME(SetRec)(io, j) || (Fail() << "SetRec(" << j << ')', 0);
+    buffer = j;
+    IONAME(OutputUnformattedBlock)
+    (io, reinterpret_cast<const char *>(&buffer), recl, recl) ||
+        (Fail() << "OutputUnformattedBlock()", 0);
+    IONAME(EndIoStatement)
+    (io) == IostatOk ||
+        (Fail() << "EndIoStatement() for OutputUnformattedBlock", 0);
+  }
+  // OPEN(UNIT=unit,STATUS='OLD',CONVERT='SWAP')
+  io = IONAME(BeginOpenUnit)(unit, __FILE__, __LINE__);
+  IONAME(SetStatus)(io, "OLD", 3) || (Fail() << "SetStatus(OLD)", 0);
+  IONAME(SetConvert)
+  (io, "SWAP", 4) || (Fail() << "SetConvert(SWAP)", 0);
+  IONAME(EndIoStatement)
+  (io) == IostatOk || (Fail() << "EndIoStatement() for OpenUnit", 0);
+  for (int j{records}; j >= 1; --j) {
+    // READ(UNIT=unit,REC=j) n
+    io = IONAME(BeginUnformattedInput)(unit, __FILE__, __LINE__);
+    IONAME(SetRec)
+    (io, j) || (Fail() << "SetRec(" << j << ')', 0);
+    IONAME(InputUnformattedBlock)
+    (io, reinterpret_cast<char *>(&buffer), recl, recl) ||
+        (Fail() << "InputUnformattedBlock()", 0);
+    IONAME(EndIoStatement)
+    (io) == IostatOk ||
+        (Fail() << "EndIoStatement() for InputUnformattedBlock", 0);
+    if (buffer >> 56 != j) {
+      Fail() << "Read back " << (buffer >> 56)
+             << " from direct unformatted record " << j << ", expected " << j
+             << '\n';
+    }
+  }
+  // CLOSE(UNIT=unit,STATUS='DELETE')
+  io = IONAME(BeginClose)(unit, __FILE__, __LINE__);
+  IONAME(SetStatus)(io, "DELETE", 6) || (Fail() << "SetStatus(DELETE)", 0);
+  IONAME(EndIoStatement)
+  (io) == IostatOk || (Fail() << "EndIoStatement() for Close", 0);
+  llvm::errs() << "end TestDirectUnformatted()\n";
+}
+
 void TestSequentialFixedUnformatted() {
   llvm::errs() << "begin TestSequentialFixedUnformatted()\n";
   // OPEN(NEWUNIT=unit,ACCESS='SEQUENTIAL',ACTION='READWRITE',&
@@ -91,7 +157,7 @@ void TestSequentialFixedUnformatted() {
     io = IONAME(BeginUnformattedOutput)(unit, __FILE__, __LINE__);
     buffer = j;
     IONAME(OutputUnformattedBlock)
-    (io, reinterpret_cast<const char *>(&buffer), recl) ||
+    (io, reinterpret_cast<const char *>(&buffer), recl, recl) ||
         (Fail() << "OutputUnformattedBlock()", 0);
     IONAME(EndIoStatement)
     (io) == IostatOk ||
@@ -105,7 +171,7 @@ void TestSequentialFixedUnformatted() {
     // DO J=1,RECORDS; READ(UNIT=unit) n; check n; END DO
     io = IONAME(BeginUnformattedInput)(unit, __FILE__, __LINE__);
     IONAME(InputUnformattedBlock)
-    (io, reinterpret_cast<char *>(&buffer), recl) ||
+    (io, reinterpret_cast<char *>(&buffer), recl, recl) ||
         (Fail() << "InputUnformattedBlock()", 0);
     IONAME(EndIoStatement)
     (io) == IostatOk ||
@@ -125,7 +191,7 @@ void TestSequentialFixedUnformatted() {
     // READ(UNIT=unit) n
     io = IONAME(BeginUnformattedInput)(unit, __FILE__, __LINE__);
     IONAME(InputUnformattedBlock)
-    (io, reinterpret_cast<char *>(&buffer), recl) ||
+    (io, reinterpret_cast<char *>(&buffer), recl, recl) ||
         (Fail() << "InputUnformattedBlock()", 0);
     IONAME(EndIoStatement)
     (io) == IostatOk ||
@@ -175,7 +241,8 @@ void TestSequentialVariableUnformatted() {
     // DO J=1,RECORDS; WRITE(UNIT=unit) BUFFER(0:j); END DO
     io = IONAME(BeginUnformattedOutput)(unit, __FILE__, __LINE__);
     IONAME(OutputUnformattedBlock)
-    (io, reinterpret_cast<const char *>(&buffer), j * sizeof *buffer) ||
+    (io, reinterpret_cast<const char *>(&buffer), j * sizeof *buffer,
+        sizeof *buffer) ||
         (Fail() << "OutputUnformattedBlock()", 0);
     IONAME(EndIoStatement)
     (io) == IostatOk ||
@@ -189,7 +256,8 @@ void TestSequentialVariableUnformatted() {
     // DO J=1,RECORDS; READ(UNIT=unit) n; check n; END DO
     io = IONAME(BeginUnformattedInput)(unit, __FILE__, __LINE__);
     IONAME(InputUnformattedBlock)
-    (io, reinterpret_cast<char *>(&buffer), j * sizeof *buffer) ||
+    (io, reinterpret_cast<char *>(&buffer), j * sizeof *buffer,
+        sizeof *buffer) ||
         (Fail() << "InputUnformattedBlock()", 0);
     IONAME(EndIoStatement)
     (io) == IostatOk ||
@@ -211,7 +279,8 @@ void TestSequentialVariableUnformatted() {
     // READ(unit=unit) n; check
     io = IONAME(BeginUnformattedInput)(unit, __FILE__, __LINE__);
     IONAME(InputUnformattedBlock)
-    (io, reinterpret_cast<char *>(&buffer), j * sizeof *buffer) ||
+    (io, reinterpret_cast<char *>(&buffer), j * sizeof *buffer,
+        sizeof *buffer) ||
         (Fail() << "InputUnformattedBlock()", 0);
     IONAME(EndIoStatement)
     (io) == IostatOk ||
@@ -390,6 +459,7 @@ void TestStreamUnformatted() {
 int main() {
   StartTests();
   TestDirectUnformatted();
+  TestDirectUnformattedSwapped();
   TestSequentialFixedUnformatted();
   TestSequentialVariableUnformatted();
   TestDirectFormatted();
diff --git a/flang/unittests/Runtime/hello.cpp b/flang/unittests/Runtime/hello.cpp
index ff806523eb5f0..f6db4a8e47dc8 100644
--- a/flang/unittests/Runtime/hello.cpp
+++ b/flang/unittests/Runtime/hello.cpp
@@ -38,16 +38,16 @@ static void hello() {
 }
 
 static void multiline() {
-  char buffer[4][32];
+  char buffer[5][32];
   StaticDescriptor<1> staticDescriptor[2];
   Descriptor &whole{staticDescriptor[0].descriptor()};
-  SubscriptValue extent[]{4};
+  SubscriptValue extent[]{5};
   whole.Establish(TypeCode{CFI_type_char}, sizeof buffer[0], &buffer, 1, extent,
       CFI_attribute_pointer);
   whole.Dump();
   whole.Check();
   Descriptor &section{staticDescriptor[1].descriptor()};
-  SubscriptValue lowers[]{0}, uppers[]{3}, strides[]{1};
+  SubscriptValue lowers[]{0}, uppers[]{4}, strides[]{1};
   section.Establish(whole.type(), whole.ElementBytes(), nullptr, 1, extent,
       CFI_attribute_pointer);
   if (auto error{
@@ -57,12 +57,16 @@ static void multiline() {
   }
   section.Dump();
   section.Check();
-  const char *format{"('?abcde,',T1,'>',T9,A,TL12,A,TR25,'<'//G0,25X,'done')"};
+  const char *format{
+      "('?abcde,',T1,'>',T9,A,TL12,A,TR25,'<'//G0,17X,'abcd',1(2I4))"};
   auto cookie{IONAME(BeginInternalArrayFormattedOutput)(
       section, format, std::strlen(format))};
   IONAME(OutputAscii)(cookie, "WORLD", 5);
   IONAME(OutputAscii)(cookie, "HELLO", 5);
   IONAME(OutputInteger64)(cookie, 789);
+  for (int j{666}; j <= 999; j += 111) {
+    IONAME(OutputInteger64)(cookie, j);
+  }
   if (auto status{IONAME(EndIoStatement)(cookie)}) {
     Fail() << "multiline: '" << format << "' failed, status "
            << static_cast<int>(status) << '\n';
@@ -70,12 +74,50 @@ static void multiline() {
     test(format,
         ">HELLO, WORLD                  <"
         "                                "
-        "789                         done"
+        "789                 abcd 666 777"
+        " 888 999                        "
         "                                ",
         std::string{buffer[0], sizeof buffer});
   }
 }
 
+static void listInputTest() {
+  static const char input[]{",1*,(5.,6..)"};
+  auto cookie{IONAME(BeginInternalListInput)(input, sizeof input - 1)};
+  float z[6];
+  for (int j{0}; j < 6; ++j) {
+    z[j] = -(j + 1);
+  }
+  for (int j{0}; j < 6; j += 2) {
+    if (!IONAME(InputComplex32)(cookie, &z[j])) {
+      Fail() << "InputComplex32 failed\n";
+    }
+  }
+  auto status{IONAME(EndIoStatement)(cookie)};
+  if (status) {
+    Fail() << "Failed complex list-directed input, status "
+           << static_cast<int>(status) << '\n';
+  } else {
+    char output[33];
+    output[32] = '\0';
+    cookie = IONAME(BeginInternalListOutput)(output, 32);
+    for (int j{0}; j < 6; j += 2) {
+      if (!IONAME(OutputComplex32)(cookie, z[j], z[j + 1])) {
+        Fail() << "OutputComplex32 failed\n";
+      }
+    }
+    status = IONAME(EndIoStatement)(cookie);
+    static const char expect[33]{" (-1.,-2.) (-3.,-4.) (5.,6.)    "};
+    if (status) {
+      Fail() << "Failed complex list-directed output, status "
+             << static_cast<int>(status) << '\n';
+    } else if (std::strncmp(output, expect, 33) != 0) {
+      Fail() << "Failed complex list-directed output, expected '" << expect
+             << "', but got '" << output << "'\n";
+    }
+  }
+}
+
 static void realTest(const char *format, double x, const char *expect) {
   char buffer[800];
   auto cookie{IONAME(BeginInternalFormattedOutput)(
@@ -420,11 +462,16 @@ int main() {
   realInTest("(F18.0)", "                 0", 0x0);
   realInTest("(F18.0)", "                  ", 0x0);
   realInTest("(F18.0)", "                -0", 0x8000000000000000);
+  realInTest("(F18.0)", "                01", 0x3ff0000000000000);
   realInTest("(F18.0)", "                 1", 0x3ff0000000000000);
   realInTest("(F18.0)", "              125.", 0x405f400000000000);
   realInTest("(F18.0)", "              12.5", 0x4029000000000000);
   realInTest("(F18.0)", "              1.25", 0x3ff4000000000000);
+  realInTest("(F18.0)", "             01.25", 0x3ff4000000000000);
   realInTest("(F18.0)", "              .125", 0x3fc0000000000000);
+  realInTest("(F18.0)", "             0.125", 0x3fc0000000000000);
+  realInTest("(F18.0)", "             .0625", 0x3fb0000000000000);
+  realInTest("(F18.0)", "            0.0625", 0x3fb0000000000000);
   realInTest("(F18.0)", "               125", 0x405f400000000000);
   realInTest("(F18.1)", "               125", 0x4029000000000000);
   realInTest("(F18.2)", "               125", 0x3ff4000000000000);
@@ -434,5 +481,7 @@ int main() {
   realInTest("(BZ,F18.0)", "              125 ", 0x4093880000000000); // 1250
   realInTest("(DC,F18.0)", "              12,5", 0x4029000000000000);
 
+  listInputTest();
+
   return EndTests();
 }
diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index 5d607b82892ff..6a1ff3bd64a9b 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -11,6 +11,8 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strlen
     libc.src.string.memchr
     libc.src.string.strchr
+    libc.src.string.strstr
+    libc.src.string.strnlen
 )
 
 set(TARGET_LIBM_ENTRYPOINTS
@@ -30,6 +32,12 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.floor
     libc.src.math.floorf
     libc.src.math.floorl
+    libc.src.math.fmax
+    libc.src.math.fmaxf
+    libc.src.math.fmaxl
+    libc.src.math.fmin
+    libc.src.math.fminf
+    libc.src.math.fminl
     libc.src.math.frexp
     libc.src.math.frexpf
     libc.src.math.frexpl
diff --git a/libc/config/linux/api.td b/libc/config/linux/api.td
index 9b9ed1b49298d..1ec1a024f85d0 100644
--- a/libc/config/linux/api.td
+++ b/libc/config/linux/api.td
@@ -163,6 +163,12 @@ def MathAPI : PublicAPI<"math.h"> {
    "floor",
    "floorf",
    "floorl",
+   "fmax",
+   "fmaxf",
+   "fmaxl",
+   "fmin",
+   "fminf",
+   "fminl",
    "frexp",
    "frexpf",
    "frexpl",
@@ -210,6 +216,7 @@ def StringAPI : PublicAPI<"string.h"> {
     "strtok",
     "strerror",
     "strlen",
+    "strnlen"
   ];
 
   let TypeDeclarations = [
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index fc62fbb880b41..b20f58c451847 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -29,6 +29,8 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strcmp
     libc.src.string.memchr
     libc.src.string.strchr
+    libc.src.string.strstr
+    libc.src.string.strnlen
 
     # sys/mman.h entrypoints
     libc.src.sys.mman.mmap
@@ -63,6 +65,12 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.floor
     libc.src.math.floorf
     libc.src.math.floorl
+    libc.src.math.fmin
+    libc.src.math.fminf
+    libc.src.math.fminl
+    libc.src.math.fmax
+    libc.src.math.fmaxf
+    libc.src.math.fmaxl
     libc.src.math.frexp
     libc.src.math.frexpf
     libc.src.math.frexpl
diff --git a/libc/spec/posix.td b/libc/spec/posix.td
index 7085e81a6b80a..732b6a6be2500 100644
--- a/libc/spec/posix.td
+++ b/libc/spec/posix.td
@@ -11,6 +11,8 @@ def RestrictStructSigactionPtr : RestrictedPtrType<StructSigaction>;
 def ConstRestrictStructSigactionPtr : ConstType<RestrictStructSigactionPtr>;
 
 def POSIX : StandardSpec<"POSIX"> {
+  PtrType CharPtr = PtrType<CharType>;
+  ConstType ConstCharPtr = ConstType<CharPtr>;
   NamedType OffTType = NamedType<"off_t">;
   NamedType SSizeTType = NamedType<"ssize_t">;
 
@@ -203,11 +205,30 @@ def POSIX : StandardSpec<"POSIX"> {
         >,
     ]
   >;
+  
+  HeaderSpec String = HeaderSpec<
+    "string.h",
+    [
+        Macro<"NULL">,
+    ],
+    [
+        SizeTType,
+    ],
+    [], // Enumerations
+    [
+        FunctionSpec<
+            "strnlen",
+             RetValSpec<SizeTType>,
+             [ArgSpec<ConstCharPtr>, ArgSpec<SizeTType>]
+        >,
+    ]
+  >;
 
   let Headers = [
     Errno,
     SysMMan,
     Signal,
     UniStd,
+    String
   ];
 }
diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td
index 2af79373c30ce..6a11b002d8742 100644
--- a/libc/spec/stdc.td
+++ b/libc/spec/stdc.td
@@ -205,6 +205,14 @@ def StdC : StandardSpec<"stdc"> {
           FunctionSpec<"floorf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
           FunctionSpec<"floorl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>]>,
 
+          FunctionSpec<"fmin", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
+          FunctionSpec<"fminf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
+          FunctionSpec<"fminl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
+
+          FunctionSpec<"fmax", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
+          FunctionSpec<"fmaxf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
+          FunctionSpec<"fmaxl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
+
           FunctionSpec<"frexp", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<IntPtr>]>,
           FunctionSpec<"frexpf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<IntPtr>]>,
           FunctionSpec<"frexpl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<IntPtr>]>,
diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt
index aa8680383b7ab..da18aeba9a2a5 100644
--- a/libc/src/math/CMakeLists.txt
+++ b/libc/src/math/CMakeLists.txt
@@ -413,3 +413,75 @@ add_entrypoint_object(
   COMPILE_OPTIONS
     -O2
 )
+
+add_entrypoint_object(
+  fmin
+  SRCS
+    fmin.cpp
+  HDRS
+    fmin.h
+  DEPENDS
+    libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fminf
+  SRCS
+    fminf.cpp
+  HDRS
+    fminf.h
+  DEPENDS
+    libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fminl
+  SRCS
+    fminl.cpp
+  HDRS
+    fminl.h
+  DEPENDS
+    libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fmax
+  SRCS
+    fmax.cpp
+  HDRS
+    fmax.h
+  DEPENDS
+    libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fmaxf
+  SRCS
+    fmaxf.cpp
+  HDRS
+    fmaxf.h
+  DEPENDS
+    libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fmaxl
+  SRCS
+    fmaxl.cpp
+  HDRS
+    fmaxl.h
+  DEPENDS
+    libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
+)
diff --git a/libc/src/math/fmax.cpp b/libc/src/math/fmax.cpp
new file mode 100644
index 0000000000000..ba5d93388998e
--- /dev/null
+++ b/libc/src/math/fmax.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of fmax function -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/common.h"
+#include "utils/FPUtil/BasicOperations.h"
+
+namespace __llvm_libc {
+
+double LLVM_LIBC_ENTRYPOINT(fmax)(double x, double y) {
+  return fputil::fmax(x, y);
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/math/fmax.h b/libc/src/math/fmax.h
new file mode 100644
index 0000000000000..9f057983d28bb
--- /dev/null
+++ b/libc/src/math/fmax.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for fmax --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAX_H
+#define LLVM_LIBC_SRC_MATH_FMAX_H
+
+namespace __llvm_libc {
+
+double fmax(double x, double y);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_MATH_FMAX_H
diff --git a/libc/src/math/fmaxf.cpp b/libc/src/math/fmaxf.cpp
new file mode 100644
index 0000000000000..55629040eba66
--- /dev/null
+++ b/libc/src/math/fmaxf.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of fmaxf function ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/common.h"
+#include "utils/FPUtil/BasicOperations.h"
+
+namespace __llvm_libc {
+
+float LLVM_LIBC_ENTRYPOINT(fmaxf)(float x, float y) {
+  return fputil::fmax(x, y);
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/math/fmaxf.h b/libc/src/math/fmaxf.h
new file mode 100644
index 0000000000000..e37df5cf9565d
--- /dev/null
+++ b/libc/src/math/fmaxf.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for fmaxf -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXF_H
+#define LLVM_LIBC_SRC_MATH_FMAXF_H
+
+namespace __llvm_libc {
+
+float fmaxf(float x, float y);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXF_H
diff --git a/libc/src/math/fmaxl.cpp b/libc/src/math/fmaxl.cpp
new file mode 100644
index 0000000000000..c944187f26b7a
--- /dev/null
+++ b/libc/src/math/fmaxl.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of fmaxl function ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/common.h"
+#include "utils/FPUtil/BasicOperations.h"
+
+namespace __llvm_libc {
+
+long double LLVM_LIBC_ENTRYPOINT(fmaxl)(long double x, long double y) {
+  return fputil::fmax(x, y);
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/math/fmaxl.h b/libc/src/math/fmaxl.h
new file mode 100644
index 0000000000000..41d80ba4aa52c
--- /dev/null
+++ b/libc/src/math/fmaxl.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for fmaxl -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXL_H
+#define LLVM_LIBC_SRC_MATH_FMAXL_H
+
+namespace __llvm_libc {
+
+long double fmaxl(long double x, long double y);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXL_H
diff --git a/libc/src/math/fmin.cpp b/libc/src/math/fmin.cpp
new file mode 100644
index 0000000000000..ef5efad598115
--- /dev/null
+++ b/libc/src/math/fmin.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of fmin function -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/common.h"
+#include "utils/FPUtil/BasicOperations.h"
+
+namespace __llvm_libc {
+
+double LLVM_LIBC_ENTRYPOINT(fmin)(double x, double y) {
+  return fputil::fmin(x, y);
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/math/fmin.h b/libc/src/math/fmin.h
new file mode 100644
index 0000000000000..52334ee8e4560
--- /dev/null
+++ b/libc/src/math/fmin.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for fmin --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMIN_H
+#define LLVM_LIBC_SRC_MATH_FMIN_H
+
+namespace __llvm_libc {
+
+double fmin(double x, double y);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_MATH_FMIN_H
diff --git a/libc/src/math/fminf.cpp b/libc/src/math/fminf.cpp
new file mode 100644
index 0000000000000..97ad399f4a2bc
--- /dev/null
+++ b/libc/src/math/fminf.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of fminf function ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/common.h"
+#include "utils/FPUtil/BasicOperations.h"
+
+namespace __llvm_libc {
+
+float LLVM_LIBC_ENTRYPOINT(fminf)(float x, float y) {
+  return fputil::fmin(x, y);
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/math/fminf.h b/libc/src/math/fminf.h
new file mode 100644
index 0000000000000..62dad57d852aa
--- /dev/null
+++ b/libc/src/math/fminf.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for fminf -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINF_H
+#define LLVM_LIBC_SRC_MATH_FMINF_H
+
+namespace __llvm_libc {
+
+float fminf(float x, float y);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_MATH_FMINF_H
diff --git a/libc/src/math/fminl.cpp b/libc/src/math/fminl.cpp
new file mode 100644
index 0000000000000..e66f0d72bfc33
--- /dev/null
+++ b/libc/src/math/fminl.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of fminl function ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/common.h"
+#include "utils/FPUtil/BasicOperations.h"
+
+namespace __llvm_libc {
+
+long double LLVM_LIBC_ENTRYPOINT(fminl)(long double x, long double y) {
+  return fputil::fmin(x, y);
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/math/fminl.h b/libc/src/math/fminl.h
new file mode 100644
index 0000000000000..c19505c018283
--- /dev/null
+++ b/libc/src/math/fminl.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for fminl -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINL_H
+#define LLVM_LIBC_SRC_MATH_FMINL_H
+
+namespace __llvm_libc {
+
+long double fminl(long double x, long double y);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_MATH_FMINL_H
diff --git a/libc/src/string/CMakeLists.txt b/libc/src/string/CMakeLists.txt
index c8e1e4dfa39f9..8bd7c1c045cf4 100644
--- a/libc/src/string/CMakeLists.txt
+++ b/libc/src/string/CMakeLists.txt
@@ -60,6 +60,24 @@ add_entrypoint_object(
     strchr.h
 )
 
+add_entrypoint_object(
+  strstr
+  SRCS
+    strstr.cpp
+  HDRS
+    strstr.h
+)
+
+add_entrypoint_object(
+  strnlen
+  SRCS
+    strnlen.cpp
+  HDRS
+    strnlen.h
+  DEPENDS
+    .memchr
+)
+
 # Helper to define a function with multiple implementations
 # - Computes flags to satisfy required/rejected features and arch,
 # - Declares an entry point,
diff --git a/libc/src/string/strnlen.cpp b/libc/src/string/strnlen.cpp
new file mode 100644
index 0000000000000..17dd6e171504a
--- /dev/null
+++ b/libc/src/string/strnlen.cpp
@@ -0,0 +1,23 @@
+//===-- Implementation of strnlen------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/string/strnlen.h"
+
+#include "src/__support/common.h"
+#include "src/string/memchr.h"
+#include <stddef.h>
+
+namespace __llvm_libc {
+
+size_t LLVM_LIBC_ENTRYPOINT(strnlen)(const char *src, size_t n) {
+  const char *temp =
+      reinterpret_cast<char *>(__llvm_libc::memchr(src, '\0', n));
+  return temp ? temp - src : n;
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/string/strnlen.h b/libc/src/string/strnlen.h
new file mode 100644
index 0000000000000..2d2ee9703d83d
--- /dev/null
+++ b/libc/src/string/strnlen.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for strnlen ------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STRING_STRNLEN_H
+#define LLVM_LIBC_SRC_STRING_STRNLEN_H
+
+#include <stddef.h>
+
+namespace __llvm_libc {
+
+size_t strnlen(const char *src, size_t n);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_STRING_STRNLEN_H
diff --git a/libc/src/string/strstr.cpp b/libc/src/string/strstr.cpp
new file mode 100644
index 0000000000000..a598f6610621f
--- /dev/null
+++ b/libc/src/string/strstr.cpp
@@ -0,0 +1,29 @@
+//===-- Implementation of strstr ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/string/strstr.h"
+
+#include "src/__support/common.h"
+#include <stddef.h>
+
+namespace __llvm_libc {
+
+// TODO: This is a simple brute force implementation. This can be
+// improved upon using well known string matching algorithms.
+char *LLVM_LIBC_ENTRYPOINT(strstr)(const char *haystack, const char *needle) {
+  for (size_t i = 0; haystack[i]; ++i) {
+    size_t j;
+    for (j = 0; haystack[i + j] && haystack[i + j] == needle[j]; ++j)
+      ;
+    if (!needle[j])
+      return const_cast<char *>(haystack + i);
+  }
+  return nullptr;
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/string/strstr.h b/libc/src/string/strstr.h
new file mode 100644
index 0000000000000..463ffb7494c9e
--- /dev/null
+++ b/libc/src/string/strstr.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for strstr ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STRING_STRSTR_H
+#define LLVM_LIBC_SRC_STRING_STRSTR_H
+
+namespace __llvm_libc {
+
+char *strstr(const char *haystack, const char *needle);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_STRING_STRSTR_H
diff --git a/libc/test/src/math/CMakeLists.txt b/libc/test/src/math/CMakeLists.txt
index 6c6cb9036e53c..3bd40f6b32f14 100644
--- a/libc/test/src/math/CMakeLists.txt
+++ b/libc/test/src/math/CMakeLists.txt
@@ -437,3 +437,75 @@ add_math_unittest(
     libc.src.math.modfl
     libc.utils.FPUtil.fputil
 )
+
+add_math_unittest(
+  fminf_test
+  SUITE
+    libc_math_unittests
+  SRCS
+    fminf_test.cpp
+  DEPENDS
+    libc.include.math
+    libc.src.math.fminf
+    libc.utils.FPUtil.fputil
+)
+
+add_math_unittest(
+  fmin_test
+  SUITE
+    libc_math_unittests
+  SRCS
+    fmin_test.cpp
+  DEPENDS
+    libc.include.math
+    libc.src.math.fmin
+    libc.utils.FPUtil.fputil
+)
+
+add_math_unittest(
+  fminl_test
+  SUITE
+    libc_math_unittests
+  SRCS
+    fminl_test.cpp
+  DEPENDS
+    libc.include.math
+    libc.src.math.fminl
+    libc.utils.FPUtil.fputil
+)
+
+add_math_unittest(
+  fmaxf_test
+  SUITE
+    libc_math_unittests
+  SRCS
+    fmaxf_test.cpp
+  DEPENDS
+    libc.include.math
+    libc.src.math.fmaxf
+    libc.utils.FPUtil.fputil
+)
+
+add_math_unittest(
+  fmax_test
+  SUITE
+    libc_math_unittests
+  SRCS
+    fmax_test.cpp
+  DEPENDS
+    libc.include.math
+    libc.src.math.fmax
+    libc.utils.FPUtil.fputil
+)
+
+add_math_unittest(
+  fmaxl_test
+  SUITE
+    libc_math_unittests
+  SRCS
+    fmaxl_test.cpp
+  DEPENDS
+    libc.include.math
+    libc.src.math.fmaxl
+    libc.utils.FPUtil.fputil
+)
diff --git a/libc/test/src/math/fmax_test.cpp b/libc/test/src/math/fmax_test.cpp
new file mode 100644
index 0000000000000..8d1c6c2763821
--- /dev/null
+++ b/libc/test/src/math/fmax_test.cpp
@@ -0,0 +1,73 @@
+//===-- Unittests for fmax -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#include "include/math.h"
+#include "src/math/fmax.h"
+#include "utils/FPUtil/FPBits.h"
+#include "utils/UnitTest/Test.h"
+
+using FPBits = __llvm_libc::fputil::FPBits<double>;
+
+double nan = FPBits::buildNaN(1);
+double inf = FPBits::inf();
+double negInf = FPBits::negInf();
+
+TEST(FmaxTest, NaNArg) {
+  EXPECT_EQ(inf, __llvm_libc::fmax(nan, inf));
+  EXPECT_EQ(negInf, __llvm_libc::fmax(negInf, nan));
+  EXPECT_EQ(0.0, __llvm_libc::fmax(nan, 0.0));
+  EXPECT_EQ(-0.0, __llvm_libc::fmax(-0.0, nan));
+  EXPECT_EQ(-1.2345, __llvm_libc::fmax(nan, -1.2345));
+  EXPECT_EQ(1.2345, __llvm_libc::fmax(1.2345, nan));
+  EXPECT_NE(isnan(__llvm_libc::fmax(nan, nan)), 0);
+}
+
+TEST(FmaxTest, InfArg) {
+  EXPECT_EQ(inf, __llvm_libc::fmax(negInf, inf));
+  EXPECT_EQ(inf, __llvm_libc::fmax(inf, 0.0));
+  EXPECT_EQ(inf, __llvm_libc::fmax(-0.0, inf));
+  EXPECT_EQ(inf, __llvm_libc::fmax(inf, 1.2345));
+  EXPECT_EQ(inf, __llvm_libc::fmax(-1.2345, inf));
+}
+
+TEST(FmaxTest, NegInfArg) {
+  EXPECT_EQ(inf, __llvm_libc::fmax(inf, negInf));
+  EXPECT_EQ(0.0, __llvm_libc::fmax(negInf, 0.0));
+  EXPECT_EQ(-0.0, __llvm_libc::fmax(-0.0, negInf));
+  EXPECT_EQ(-1.2345, __llvm_libc::fmax(negInf, -1.2345));
+  EXPECT_EQ(1.2345, __llvm_libc::fmax(1.2345, negInf));
+}
+
+TEST(FmaxTest, BothZero) {
+  EXPECT_EQ(0.0, __llvm_libc::fmax(0.0, 0.0));
+  EXPECT_EQ(0.0, __llvm_libc::fmax(-0.0, 0.0));
+  EXPECT_EQ(0.0, __llvm_libc::fmax(0.0, -0.0));
+  EXPECT_EQ(-0.0, __llvm_libc::fmax(-0.0, -0.0));
+}
+
+TEST(FmaxTest, InDoubleRange) {
+  using UIntType = FPBits::UIntType;
+  constexpr UIntType count = 10000001;
+  constexpr UIntType step = UIntType(-1) / count;
+  for (UIntType i = 0, v = 0, w = UIntType(-1); i <= count;
+       ++i, v += step, w -= step) {
+    double x = FPBits(v), y = FPBits(w);
+    if (isnan(x) || isinf(x))
+      continue;
+    if (isnan(y) || isinf(y))
+      continue;
+    if ((x == 0) && (y == 0))
+      continue;
+
+    if (x > y) {
+      ASSERT_EQ(x, __llvm_libc::fmax(x, y));
+    } else {
+      ASSERT_EQ(y, __llvm_libc::fmax(x, y));
+    }
+  }
+}
diff --git a/libc/test/src/math/fmaxf_test.cpp b/libc/test/src/math/fmaxf_test.cpp
new file mode 100644
index 0000000000000..fe9eaad171bd2
--- /dev/null
+++ b/libc/test/src/math/fmaxf_test.cpp
@@ -0,0 +1,73 @@
+//===-- Unittests for fmaxf -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#include "include/math.h"
+#include "src/math/fmaxf.h"
+#include "utils/FPUtil/FPBits.h"
+#include "utils/UnitTest/Test.h"
+
+using FPBits = __llvm_libc::fputil::FPBits<float>;
+
+float nan = FPBits::buildNaN(1);
+float inf = FPBits::inf();
+float negInf = FPBits::negInf();
+
+TEST(FmaxfTest, NaNArg) {
+  EXPECT_EQ(inf, __llvm_libc::fmaxf(nan, inf));
+  EXPECT_EQ(negInf, __llvm_libc::fmaxf(negInf, nan));
+  EXPECT_EQ(0.0f, __llvm_libc::fmaxf(nan, 0.0f));
+  EXPECT_EQ(-0.0f, __llvm_libc::fmaxf(-0.0f, nan));
+  EXPECT_EQ(-1.2345f, __llvm_libc::fmaxf(nan, -1.2345f));
+  EXPECT_EQ(1.2345f, __llvm_libc::fmaxf(1.2345f, nan));
+  EXPECT_NE(isnan(__llvm_libc::fmaxf(nan, nan)), 0);
+}
+
+TEST(FmaxfTest, InfArg) {
+  EXPECT_EQ(inf, __llvm_libc::fmaxf(negInf, inf));
+  EXPECT_EQ(inf, __llvm_libc::fmaxf(inf, 0.0f));
+  EXPECT_EQ(inf, __llvm_libc::fmaxf(-0.0f, inf));
+  EXPECT_EQ(inf, __llvm_libc::fmaxf(inf, 1.2345f));
+  EXPECT_EQ(inf, __llvm_libc::fmaxf(-1.2345f, inf));
+}
+
+TEST(FmaxfTest, NegInfArg) {
+  EXPECT_EQ(inf, __llvm_libc::fmaxf(inf, negInf));
+  EXPECT_EQ(0.0f, __llvm_libc::fmaxf(negInf, 0.0f));
+  EXPECT_EQ(-0.0f, __llvm_libc::fmaxf(-0.0f, negInf));
+  EXPECT_EQ(-1.2345f, __llvm_libc::fmaxf(negInf, -1.2345f));
+  EXPECT_EQ(1.2345f, __llvm_libc::fmaxf(1.2345f, negInf));
+}
+
+TEST(FmaxfTest, BothZero) {
+  EXPECT_EQ(0.0f, __llvm_libc::fmaxf(0.0f, 0.0f));
+  EXPECT_EQ(0.0f, __llvm_libc::fmaxf(-0.0f, 0.0f));
+  EXPECT_EQ(0.0f, __llvm_libc::fmaxf(0.0f, -0.0f));
+  EXPECT_EQ(-0.0f, __llvm_libc::fmaxf(-0.0f, -0.0f));
+}
+
+TEST(FmaxfTest, InFloatRange) {
+  using UIntType = FPBits::UIntType;
+  constexpr UIntType count = 10000001;
+  constexpr UIntType step = UIntType(-1) / count;
+  for (UIntType i = 0, v = 0, w = UIntType(-1); i <= count;
+       ++i, v += step, w -= step) {
+    float x = FPBits(v), y = FPBits(w);
+    if (isnan(x) || isinf(x))
+      continue;
+    if (isnan(y) || isinf(y))
+      continue;
+    if ((x == 0) && (y == 0))
+      continue;
+
+    if (x > y) {
+      ASSERT_EQ(x, __llvm_libc::fmaxf(x, y));
+    } else {
+      ASSERT_EQ(y, __llvm_libc::fmaxf(x, y));
+    }
+  }
+}
diff --git a/libc/test/src/math/fmaxl_test.cpp b/libc/test/src/math/fmaxl_test.cpp
new file mode 100644
index 0000000000000..9c7aa21582ebc
--- /dev/null
+++ b/libc/test/src/math/fmaxl_test.cpp
@@ -0,0 +1,73 @@
+//===-- Unittests for fmaxl -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#include "include/math.h"
+#include "src/math/fmaxl.h"
+#include "utils/FPUtil/FPBits.h"
+#include "utils/UnitTest/Test.h"
+
+using FPBits = __llvm_libc::fputil::FPBits<long double>;
+
+long double nan = FPBits::buildNaN(1);
+long double inf = FPBits::inf();
+long double negInf = FPBits::negInf();
+
+TEST(FmaxlTest, NaNArg) {
+  EXPECT_EQ(inf, __llvm_libc::fmaxl(nan, inf));
+  EXPECT_EQ(negInf, __llvm_libc::fmaxl(negInf, nan));
+  EXPECT_EQ(0.0L, __llvm_libc::fmaxl(nan, 0.0L));
+  EXPECT_EQ(-0.0L, __llvm_libc::fmaxl(-0.0L, nan));
+  EXPECT_EQ(-1.2345L, __llvm_libc::fmaxl(nan, -1.2345L));
+  EXPECT_EQ(1.2345L, __llvm_libc::fmaxl(1.2345L, nan));
+  EXPECT_NE(isnan(__llvm_libc::fmaxl(nan, nan)), 0);
+}
+
+TEST(FmaxlTest, InfArg) {
+  EXPECT_EQ(inf, __llvm_libc::fmaxl(negInf, inf));
+  EXPECT_EQ(inf, __llvm_libc::fmaxl(inf, 0.0L));
+  EXPECT_EQ(inf, __llvm_libc::fmaxl(-0.0L, inf));
+  EXPECT_EQ(inf, __llvm_libc::fmaxl(inf, 1.2345L));
+  EXPECT_EQ(inf, __llvm_libc::fmaxl(-1.2345L, inf));
+}
+
+TEST(FmaxlTest, NegInfArg) {
+  EXPECT_EQ(inf, __llvm_libc::fmaxl(inf, negInf));
+  EXPECT_EQ(0.0L, __llvm_libc::fmaxl(negInf, 0.0L));
+  EXPECT_EQ(-0.0L, __llvm_libc::fmaxl(-0.0L, negInf));
+  EXPECT_EQ(-1.2345L, __llvm_libc::fmaxl(negInf, -1.2345L));
+  EXPECT_EQ(1.2345L, __llvm_libc::fmaxl(1.2345L, negInf));
+}
+
+TEST(FmaxlTest, BothZero) {
+  EXPECT_EQ(0.0L, __llvm_libc::fmaxl(0.0L, 0.0L));
+  EXPECT_EQ(0.0L, __llvm_libc::fmaxl(-0.0L, 0.0L));
+  EXPECT_EQ(0.0L, __llvm_libc::fmaxl(0.0L, -0.0L));
+  EXPECT_EQ(-0.0L, __llvm_libc::fmaxl(-0.0L, -0.0L));
+}
+
+TEST(FmaxlTest, InLongDoubleRange) {
+  using UIntType = FPBits::UIntType;
+  constexpr UIntType count = 10000001;
+  constexpr UIntType step = UIntType(-1) / count;
+  for (UIntType i = 0, v = 0, w = UIntType(-1); i <= count;
+       ++i, v += step, w -= step) {
+    long double x = FPBits(v), y = FPBits(w);
+    if (isnan(x) || isinf(x))
+      continue;
+    if (isnan(y) || isinf(y))
+      continue;
+    if ((x == 0) && (y == 0))
+      continue;
+
+    if (x > y) {
+      ASSERT_EQ(x, __llvm_libc::fmaxl(x, y));
+    } else {
+      ASSERT_EQ(y, __llvm_libc::fmaxl(x, y));
+    }
+  }
+}
diff --git a/libc/test/src/math/fmin_test.cpp b/libc/test/src/math/fmin_test.cpp
new file mode 100644
index 0000000000000..7fd1e8af1e510
--- /dev/null
+++ b/libc/test/src/math/fmin_test.cpp
@@ -0,0 +1,73 @@
+//===-- Unittests for fmin -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#include "include/math.h"
+#include "src/math/fmin.h"
+#include "utils/FPUtil/FPBits.h"
+#include "utils/UnitTest/Test.h"
+
+using FPBits = __llvm_libc::fputil::FPBits<double>;
+
+double nan = static_cast<double>(FPBits::buildNaN(1));
+double inf = static_cast<double>(FPBits::inf());
+double negInf = static_cast<double>(FPBits::negInf());
+
+TEST(FminTest, NaNArg) {
+  EXPECT_EQ(inf, __llvm_libc::fmin(nan, inf));
+  EXPECT_EQ(negInf, __llvm_libc::fmin(negInf, nan));
+  EXPECT_EQ(0.0, __llvm_libc::fmin(nan, 0.0));
+  EXPECT_EQ(-0.0, __llvm_libc::fmin(-0.0, nan));
+  EXPECT_EQ(-1.2345, __llvm_libc::fmin(nan, -1.2345));
+  EXPECT_EQ(1.2345, __llvm_libc::fmin(1.2345, nan));
+  EXPECT_NE(isnan(__llvm_libc::fmin(nan, nan)), 0);
+}
+
+TEST(FminTest, InfArg) {
+  EXPECT_EQ(negInf, __llvm_libc::fmin(negInf, inf));
+  EXPECT_EQ(0.0, __llvm_libc::fmin(inf, 0.0));
+  EXPECT_EQ(-0.0, __llvm_libc::fmin(-0.0, inf));
+  EXPECT_EQ(1.2345, __llvm_libc::fmin(inf, 1.2345));
+  EXPECT_EQ(-1.2345, __llvm_libc::fmin(-1.2345, inf));
+}
+
+TEST(FminTest, NegInfArg) {
+  EXPECT_EQ(negInf, __llvm_libc::fmin(inf, negInf));
+  EXPECT_EQ(negInf, __llvm_libc::fmin(negInf, 0.0));
+  EXPECT_EQ(negInf, __llvm_libc::fmin(-0.0, negInf));
+  EXPECT_EQ(negInf, __llvm_libc::fmin(negInf, -1.2345));
+  EXPECT_EQ(negInf, __llvm_libc::fmin(1.2345, negInf));
+}
+
+TEST(FminTest, BothZero) {
+  EXPECT_EQ(0.0, __llvm_libc::fmin(0.0, 0.0));
+  EXPECT_EQ(-0.0, __llvm_libc::fmin(-0.0, 0.0));
+  EXPECT_EQ(-0.0, __llvm_libc::fmin(0.0, -0.0));
+  EXPECT_EQ(-0.0, __llvm_libc::fmin(-0.0, -0.0));
+}
+
+TEST(FminTest, InFloatRange) {
+  using UIntType = FPBits::UIntType;
+  constexpr UIntType count = 10000000;
+  constexpr UIntType step = UIntType(-1) / count;
+  for (UIntType i = 0, v = 0, w = UIntType(-1); i <= count;
+       ++i, v += step, w -= step) {
+    double x = FPBits(v), y = FPBits(w);
+    if (isnan(x) || isinf(x))
+      continue;
+    if (isnan(y) || isinf(y))
+      continue;
+    if ((x == 0) && (y == 0))
+      continue;
+
+    if (x < y) {
+      ASSERT_EQ(x, __llvm_libc::fmin(x, y));
+    } else {
+      ASSERT_EQ(y, __llvm_libc::fmin(x, y));
+    }
+  }
+}
diff --git a/libc/test/src/math/fminf_test.cpp b/libc/test/src/math/fminf_test.cpp
new file mode 100644
index 0000000000000..dea4e7ccf5246
--- /dev/null
+++ b/libc/test/src/math/fminf_test.cpp
@@ -0,0 +1,73 @@
+//===-- Unittests for fminf ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#include "include/math.h"
+#include "src/math/fminf.h"
+#include "utils/FPUtil/FPBits.h"
+#include "utils/UnitTest/Test.h"
+
+using FPBits = __llvm_libc::fputil::FPBits<float>;
+
+float nan = static_cast<float>(FPBits::buildNaN(1));
+float inf = static_cast<float>(FPBits::inf());
+float negInf = static_cast<float>(FPBits::negInf());
+
+TEST(FminfTest, NaNArg) {
+  EXPECT_EQ(inf, __llvm_libc::fminf(nan, inf));
+  EXPECT_EQ(negInf, __llvm_libc::fminf(negInf, nan));
+  EXPECT_EQ(0.0f, __llvm_libc::fminf(nan, 0.0f));
+  EXPECT_EQ(-0.0f, __llvm_libc::fminf(-0.0f, nan));
+  EXPECT_EQ(-1.2345f, __llvm_libc::fminf(nan, -1.2345f));
+  EXPECT_EQ(1.2345f, __llvm_libc::fminf(1.2345f, nan));
+  EXPECT_NE(isnan(__llvm_libc::fminf(nan, nan)), 0);
+}
+
+TEST(FminfTest, InfArg) {
+  EXPECT_EQ(negInf, __llvm_libc::fminf(negInf, inf));
+  EXPECT_EQ(0.0f, __llvm_libc::fminf(inf, 0.0f));
+  EXPECT_EQ(-0.0f, __llvm_libc::fminf(-0.0f, inf));
+  EXPECT_EQ(1.2345f, __llvm_libc::fminf(inf, 1.2345f));
+  EXPECT_EQ(-1.2345f, __llvm_libc::fminf(-1.2345f, inf));
+}
+
+TEST(FminfTest, NegInfArg) {
+  EXPECT_EQ(negInf, __llvm_libc::fminf(inf, negInf));
+  EXPECT_EQ(negInf, __llvm_libc::fminf(negInf, 0.0f));
+  EXPECT_EQ(negInf, __llvm_libc::fminf(-0.0f, negInf));
+  EXPECT_EQ(negInf, __llvm_libc::fminf(negInf, -1.2345f));
+  EXPECT_EQ(negInf, __llvm_libc::fminf(1.2345f, negInf));
+}
+
+TEST(FminfTest, BothZero) {
+  EXPECT_EQ(0.0f, __llvm_libc::fminf(0.0f, 0.0f));
+  EXPECT_EQ(-0.0f, __llvm_libc::fminf(-0.0f, 0.0f));
+  EXPECT_EQ(-0.0f, __llvm_libc::fminf(0.0f, -0.0f));
+  EXPECT_EQ(-0.0f, __llvm_libc::fminf(-0.0f, -0.0f));
+}
+
+TEST(FminfTest, InFloatRange) {
+  using UIntType = FPBits::UIntType;
+  constexpr UIntType count = 10000000;
+  constexpr UIntType step = UIntType(-1) / count;
+  for (UIntType i = 0, v = 0, w = UIntType(-1); i <= count;
+       ++i, v += step, w -= step) {
+    float x = FPBits(v), y = FPBits(w);
+    if (isnan(x) || isinf(x))
+      continue;
+    if (isnan(y) || isinf(y))
+      continue;
+    if ((x == 0) && (y == 0))
+      continue;
+
+    if (x < y) {
+      ASSERT_EQ(x, __llvm_libc::fminf(x, y));
+    } else {
+      ASSERT_EQ(y, __llvm_libc::fminf(x, y));
+    }
+  }
+}
diff --git a/libc/test/src/math/fminl_test.cpp b/libc/test/src/math/fminl_test.cpp
new file mode 100644
index 0000000000000..3eabb55b0cab2
--- /dev/null
+++ b/libc/test/src/math/fminl_test.cpp
@@ -0,0 +1,73 @@
+//===-- Unittests for fmin -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#include "include/math.h"
+#include "src/math/fminl.h"
+#include "utils/FPUtil/FPBits.h"
+#include "utils/UnitTest/Test.h"
+
+using FPBits = __llvm_libc::fputil::FPBits<long double>;
+
+long double nan = static_cast<long double>(FPBits::buildNaN(1));
+long double inf = static_cast<long double>(FPBits::inf());
+long double negInf = static_cast<long double>(FPBits::negInf());
+
+TEST(FminlTest, NaNArg) {
+  EXPECT_EQ(inf, __llvm_libc::fminl(nan, inf));
+  EXPECT_EQ(negInf, __llvm_libc::fminl(negInf, nan));
+  EXPECT_EQ(0.0L, __llvm_libc::fminl(nan, 0.0L));
+  EXPECT_EQ(-0.0L, __llvm_libc::fminl(-0.0L, nan));
+  EXPECT_EQ(-1.2345L, __llvm_libc::fminl(nan, -1.2345L));
+  EXPECT_EQ(1.2345L, __llvm_libc::fminl(1.2345L, nan));
+  EXPECT_NE(isnan(__llvm_libc::fminl(nan, nan)), 0);
+}
+
+TEST(FminlTest, InfArg) {
+  EXPECT_EQ(negInf, __llvm_libc::fminl(negInf, inf));
+  EXPECT_EQ(0.0L, __llvm_libc::fminl(inf, 0.0L));
+  EXPECT_EQ(-0.0L, __llvm_libc::fminl(-0.0L, inf));
+  EXPECT_EQ(1.2345L, __llvm_libc::fminl(inf, 1.2345L));
+  EXPECT_EQ(-1.2345L, __llvm_libc::fminl(-1.2345L, inf));
+}
+
+TEST(FminlTest, NegInfArg) {
+  EXPECT_EQ(negInf, __llvm_libc::fminl(inf, negInf));
+  EXPECT_EQ(negInf, __llvm_libc::fminl(negInf, 0.0L));
+  EXPECT_EQ(negInf, __llvm_libc::fminl(-0.0L, negInf));
+  EXPECT_EQ(negInf, __llvm_libc::fminl(negInf, -1.2345L));
+  EXPECT_EQ(negInf, __llvm_libc::fminl(1.2345L, negInf));
+}
+
+TEST(FminlTest, BothZero) {
+  EXPECT_EQ(0.0L, __llvm_libc::fminl(0.0L, 0.0L));
+  EXPECT_EQ(-0.0L, __llvm_libc::fminl(-0.0L, 0.0L));
+  EXPECT_EQ(-0.0L, __llvm_libc::fminl(0.0L, -0.0L));
+  EXPECT_EQ(-0.0L, __llvm_libc::fminl(-0.0L, -0.0L));
+}
+
+TEST(FminlTest, InLongDoubleRange) {
+  using UIntType = FPBits::UIntType;
+  constexpr UIntType count = 10000000;
+  constexpr UIntType step = UIntType(-1) / count;
+  for (UIntType i = 0, v = 0, w = UIntType(-1); i <= count;
+       ++i, v += step, w -= step) {
+    long double x = FPBits(v), y = FPBits(w);
+    if (isnan(x) || isinf(x))
+      continue;
+    if (isnan(y) || isinf(y))
+      continue;
+    if ((x == 0) && (y == 0))
+      continue;
+
+    if (x < y) {
+      ASSERT_EQ(x, __llvm_libc::fminl(x, y));
+    } else {
+      ASSERT_EQ(y, __llvm_libc::fminl(x, y));
+    }
+  }
+}
diff --git a/libc/test/src/string/CMakeLists.txt b/libc/test/src/string/CMakeLists.txt
index be4836818212d..be43cc912b5a3 100644
--- a/libc/test/src/string/CMakeLists.txt
+++ b/libc/test/src/string/CMakeLists.txt
@@ -62,6 +62,26 @@ add_libc_unittest(
     libc.src.string.strchr
 )
 
+add_libc_unittest(
+  strstr_test
+  SUITE
+    libc_string_unittests
+  SRCS
+    strstr_test.cpp
+  DEPENDS
+    libc.src.string.strstr
+)
+
+add_libc_unittest(
+  strnlen_test
+  SUITE
+    libc_string_unittests
+  SRCS
+    strnlen_test.cpp
+  DEPENDS
+    libc.src.string.strnlen
+)
+
 # Tests all implementations that can run on the host.
 function(add_libc_multi_impl_test name)
   get_property(fq_implementations GLOBAL PROPERTY ${name}_implementations)
diff --git a/libc/test/src/string/strnlen_test.cpp b/libc/test/src/string/strnlen_test.cpp
new file mode 100644
index 0000000000000..9d8616bc8bd7d
--- /dev/null
+++ b/libc/test/src/string/strnlen_test.cpp
@@ -0,0 +1,46 @@
+//===-- Unittests for strnlen----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/string/strnlen.h"
+#include "utils/UnitTest/Test.h"
+#include <stddef.h>
+
+TEST(StrNLenTest, EmptyString) {
+  const char *empty = "";
+  ASSERT_EQ(static_cast<size_t>(0), __llvm_libc::strnlen(empty, 0));
+  // If N is greater than string length, this should still return 0.
+  ASSERT_EQ(static_cast<size_t>(0), __llvm_libc::strnlen(empty, 1));
+}
+
+TEST(StrNLenTest, OneCharacterString) {
+  const char *single = "X";
+  ASSERT_EQ(static_cast<size_t>(1), __llvm_libc::strnlen(single, 1));
+  // If N is zero, this should return 0.
+  ASSERT_EQ(static_cast<size_t>(0), __llvm_libc::strnlen(single, 0));
+  // If N is greater than string length, this should still return 1.
+  ASSERT_EQ(static_cast<size_t>(1), __llvm_libc::strnlen(single, 2));
+}
+
+TEST(StrNLenTest, ManyCharacterString) {
+  const char *many = "123456789";
+  ASSERT_EQ(static_cast<size_t>(9), __llvm_libc::strnlen(many, 9));
+  // If N is smaller than the string length, it should return N.
+  ASSERT_EQ(static_cast<size_t>(3), __llvm_libc::strnlen(many, 3));
+  // If N is zero, this should return 0.
+  ASSERT_EQ(static_cast<size_t>(0), __llvm_libc::strnlen(many, 0));
+  // If N is greater than the string length, this should still return 9.
+  ASSERT_EQ(static_cast<size_t>(9), __llvm_libc::strnlen(many, 42));
+}
+
+TEST(StrNLenTest, CharactersAfterNullTerminatorShouldNotBeIncluded) {
+  const char str[5] = {'a', 'b', 'c', '\0', 'd'};
+  ASSERT_EQ(static_cast<size_t>(3), __llvm_libc::strnlen(str, 3));
+  // This should only read up to the null terminator.
+  ASSERT_EQ(static_cast<size_t>(3), __llvm_libc::strnlen(str, 4));
+  ASSERT_EQ(static_cast<size_t>(3), __llvm_libc::strnlen(str, 5));
+}
diff --git a/libc/test/src/string/strstr_test.cpp b/libc/test/src/string/strstr_test.cpp
new file mode 100644
index 0000000000000..f4edbb2cca594
--- /dev/null
+++ b/libc/test/src/string/strstr_test.cpp
@@ -0,0 +1,114 @@
+//===-- Unittests for strstr ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/string/strstr.h"
+#include "utils/UnitTest/Test.h"
+
+TEST(StrStrTest, NeedleNotInHaystack) {
+  const char *haystack = "12345";
+  const char *needle = "a";
+  ASSERT_STREQ(__llvm_libc::strstr(haystack, needle), nullptr);
+}
+
+TEST(StrStrTest, NeedleIsEmptyString) {
+  const char *haystack = "12345";
+  const char *needle = "";
+  ASSERT_STREQ(__llvm_libc::strstr(haystack, needle), haystack);
+}
+
+TEST(StrStrTest, HaystackIsEmptyString) {
+  const char *haystack = "";
+  const char *needle = "12345";
+  ASSERT_STREQ(__llvm_libc::strstr(haystack, needle), nullptr);
+}
+
+TEST(StrStrTest, HaystackAndNeedleAreEmptyStrings) {
+  const char *haystack = "";
+  const char *needle = "";
+  ASSERT_STREQ(__llvm_libc::strstr(haystack, needle), "");
+}
+
+TEST(StrStrTest, HaystackAndNeedleAreSingleCharacters) {
+  const char *haystack = "a";
+  // Same characer returns that character.
+  ASSERT_STREQ(__llvm_libc::strstr(haystack, /*needle=*/"a"), "a");
+  // Different character returns nullptr.
+  ASSERT_STREQ(__llvm_libc::strstr(haystack, /*needle=*/"b"), nullptr);
+}
+
+TEST(StrStrTest, NeedleEqualToHaystack) {
+  const char *haystack = "12345";
+  const char *needle = "12345";
+  ASSERT_STREQ(__llvm_libc::strstr(haystack, needle), "12345");
+}
+
+TEST(StrStrTest, NeedleSmallerThanHaystack) {
+  const char *haystack = "12345";
+  const char *needle = "345";
+  ASSERT_STREQ(__llvm_libc::strstr(haystack, needle), "345");
+}
+
+TEST(StrStrTest, NeedleLargerThanHaystack) {
+  const char *haystack = "123";
+  const char *needle = "12345";
+  ASSERT_STREQ(__llvm_libc::strstr(haystack, needle), nullptr);
+}
+
+TEST(StrStrTest, NeedleAtBeginning) {
+  const char *haystack = "12345";
+  const char *needle = "12";
+  ASSERT_STREQ(__llvm_libc::strstr(haystack, needle), "12345");
+}
+
+TEST(StrStrTest, NeedleInMiddle) {
+  const char *haystack = "abcdefghi";
+  const char *needle = "def";
+  ASSERT_STREQ(__llvm_libc::strstr(haystack, needle), "defghi");
+}
+
+TEST(StrStrTest, NeedleDirectlyBeforeNullTerminator) {
+  const char *haystack = "abcdefghi";
+  const char *needle = "ghi";
+  ASSERT_STREQ(__llvm_libc::strstr(haystack, needle), "ghi");
+}
+
+TEST(StrStrTest, NeedlePastNullTerminator) {
+  const char haystack[5] = {'1', '2', '\0', '3', '4'};
+  // Shouldn't find anything after the null terminator.
+  ASSERT_STREQ(__llvm_libc::strstr(haystack, /*needle=*/"3"), nullptr);
+  ASSERT_STREQ(__llvm_libc::strstr(haystack, /*needle=*/"4"), nullptr);
+}
+
+TEST(StrStrTest, PartialNeedle) {
+  const char *haystack = "la_ap_lap";
+  const char *needle = "lap";
+  // Shouldn't find la or ap.
+  ASSERT_STREQ(__llvm_libc::strstr(haystack, needle), "lap");
+}
+
+TEST(StrStrTest, MisspelledNeedle) {
+  const char *haystack = "atalloftwocities...wait, tale";
+  const char *needle = "tale";
+  ASSERT_STREQ(__llvm_libc::strstr(haystack, needle), "tale");
+}
+
+TEST(StrStrTest, AnagramNeedle) {
+  const char *haystack = "dgo_ogd_god_odg_gdo_dog";
+  const char *needle = "dog";
+  ASSERT_STREQ(__llvm_libc::strstr(haystack, needle), "dog");
+}
+
+TEST(StrStrTest, MorphedNeedle) {
+  // Changes a single letter in the needle to mismatch with the haystack.
+  const char *haystack = "once upon a time";
+  ASSERT_STREQ(__llvm_libc::strstr(haystack, "time"), "time");
+  ASSERT_STREQ(__llvm_libc::strstr(haystack, "lime"), nullptr);
+  ASSERT_STREQ(__llvm_libc::strstr(haystack, "tome"), nullptr);
+  ASSERT_STREQ(__llvm_libc::strstr(haystack, "tire"), nullptr);
+  ASSERT_STREQ(__llvm_libc::strstr(haystack, "timo"), nullptr);
+}
diff --git a/libc/utils/CPP/TypeTraits.h b/libc/utils/CPP/TypeTraits.h
index 9b121c03f8b98..1f2c85af3a1cb 100644
--- a/libc/utils/CPP/TypeTraits.h
+++ b/libc/utils/CPP/TypeTraits.h
@@ -49,7 +49,8 @@ template <typename Type> struct IsIntegral {
       IsSameV<unsigned short, TypeNoCV> || IsSameV<int, TypeNoCV> ||
       IsSameV<unsigned int, TypeNoCV> || IsSameV<long, TypeNoCV> ||
       IsSameV<unsigned long, TypeNoCV> || IsSameV<long long, TypeNoCV> ||
-      IsSameV<unsigned long long, TypeNoCV> || IsSameV<bool, TypeNoCV>;
+      IsSameV<unsigned long long, TypeNoCV> || IsSameV<bool, TypeNoCV> ||
+      IsSameV<__uint128_t, TypeNoCV>;
 };
 
 template <typename T> struct IsPointerTypeNoCV : public FalseValue {};
@@ -65,6 +66,11 @@ template <typename Type> struct IsFloatingPointType {
                                 IsSame<long double, TypeNoCV>::Value;
 };
 
+template <typename Type> struct IsArithmetic {
+  static constexpr bool Value =
+      IsIntegral<Type>::Value || IsFloatingPointType<Type>::Value;
+};
+
 } // namespace cpp
 } // namespace __llvm_libc
 
diff --git a/libc/utils/FPUtil/BasicOperations.h b/libc/utils/FPUtil/BasicOperations.h
index 70c0bb557cce6..78856926af4bf 100644
--- a/libc/utils/FPUtil/BasicOperations.h
+++ b/libc/utils/FPUtil/BasicOperations.h
@@ -24,6 +24,44 @@ static inline T abs(T x) {
   return T(bits);
 }
 
+template <typename T,
+          cpp::EnableIfType<cpp::IsFloatingPointType<T>::Value, int> = 0>
+static inline T fmin(T x, T y) {
+  FPBits<T> bitx(x), bity(y);
+
+  if (bitx.isNaN()) {
+    return y;
+  } else if (bity.isNaN()) {
+    return x;
+  } else if (bitx.sign != bity.sign) {
+    // To make sure that fmin(+0, -0) == -0 == fmin(-0, +0), whenever x and
+    // y has different signs and both are not NaNs, we return the number
+    // with negative sign.
+    return (bitx.sign ? x : y);
+  } else {
+    return (x < y ? x : y);
+  }
+}
+
+template <typename T,
+          cpp::EnableIfType<cpp::IsFloatingPointType<T>::Value, int> = 0>
+static inline T fmax(T x, T y) {
+  FPBits<T> bitx(x), bity(y);
+
+  if (bitx.isNaN()) {
+    return y;
+  } else if (bity.isNaN()) {
+    return x;
+  } else if (bitx.sign != bity.sign) {
+    // To make sure that fmax(+0, -0) == +0 == fmax(-0, +0), whenever x and
+    // y has different signs and both are not NaNs, we return the number
+    // with positive sign.
+    return (bitx.sign ? y : x);
+  } else {
+    return (x > y ? x : y);
+  }
+}
+
 } // namespace fputil
 } // namespace __llvm_libc
 
diff --git a/libc/utils/FPUtil/CMakeLists.txt b/libc/utils/FPUtil/CMakeLists.txt
index 682db93859a96..e6113723592d1 100644
--- a/libc/utils/FPUtil/CMakeLists.txt
+++ b/libc/utils/FPUtil/CMakeLists.txt
@@ -14,6 +14,7 @@ add_header_library(
     FloatOperations.h
     FloatProperties.h
     FPBits.h
+    BasicOperations.h
     ManipulationFunctions.h
     NearestIntegerOperations.h
   DEPENDS
diff --git a/libc/utils/UnitTest/Test.cpp b/libc/utils/UnitTest/Test.cpp
index b2b70c8ad30a7..84730ad31cca8 100644
--- a/libc/utils/UnitTest/Test.cpp
+++ b/libc/utils/UnitTest/Test.cpp
@@ -8,7 +8,9 @@
 
 #include "Test.h"
 
+#include "utils/FPUtil/FPBits.h"
 #include "utils/testutils/ExecuteFunction.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -32,77 +34,143 @@ class RunContext {
 
 namespace internal {
 
+// Display the first N hexadecimal digits of an integer in upper case.
+template <typename T>
+cpp::EnableIfType<cpp::IsIntegral<T>::Value, std::string>
+uintToHex(T X, size_t Length = sizeof(T) * 2) {
+  std::string s(Length, '0');
+
+  for (auto it = s.rbegin(), end = s.rend(); it != end; ++it, X >>= 4) {
+    unsigned char Mod = static_cast<unsigned char>(X) & 15;
+    *it = llvm::hexdigit(Mod, true);
+  }
+
+  return s;
+}
+
+// When the value is not floating-point type, just display it as normal.
+template <typename ValType>
+cpp::EnableIfType<!cpp::IsFloatingPointType<ValType>::Value, std::string>
+describeValue(ValType Value) {
+  return std::to_string(Value);
+}
+
+template <> std::string describeValue<llvm::StringRef>(llvm::StringRef Value) {
+  return std::string(Value);
+}
+
+// When the value is __uint128_t, also show its hexadecimal digits.
+// Using template to force exact match, prevent ambiguous promotion.
+template <> std::string describeValue<__uint128_t>(__uint128_t Value) {
+  return "0x" + uintToHex(Value);
+}
+
+// When the value is a floating point type, also show its sign | exponent |
+// mantissa.
+template <typename ValType>
+cpp::EnableIfType<cpp::IsFloatingPointType<ValType>::Value, std::string>
+describeValue(ValType Value) {
+  fputil::FPBits<ValType> Bits(Value);
+
+  if (Bits.isNaN()) {
+    return "(NaN)";
+  } else if (Bits.isInf()) {
+    return Bits.sign ? "(-Infinity)" : "(+Infinity)";
+  } else {
+    constexpr int ExponentWidthInHex =
+        (fputil::ExponentWidth<ValType>::value - 1) / 4 + 1;
+    constexpr int MantissaWidthInHex =
+        (fputil::MantissaWidth<ValType>::value - 1) / 4 + 1;
+
+    return std::string("Sign: ") + (Bits.sign ? '1' : '0') + ", Exponent: 0x" +
+           uintToHex<uint16_t>(Bits.exponent, ExponentWidthInHex) +
+           ", Mantissa: 0x" +
+           uintToHex<typename fputil::FPBits<ValType>::UIntType>(
+               Bits.mantissa, MantissaWidthInHex);
+  }
+}
+
+template <typename ValType>
+void explainDifference(ValType LHS, ValType RHS, const char *LHSStr,
+                       const char *RHSStr, const char *File, unsigned long Line,
+                       llvm::StringRef OpString) {
+  size_t OffsetLength = OpString.size() > 2 ? OpString.size() - 2 : 0;
+  std::string Offset(OffsetLength, ' ');
+
+  llvm::outs() << File << ":" << Line << ": FAILURE\n"
+               << Offset << "Expected: " << LHSStr << '\n'
+               << Offset << "Which is: " << describeValue(LHS) << '\n'
+               << "To be " << OpString << ": " << RHSStr << '\n'
+               << Offset << "Which is: " << describeValue(RHS) << '\n';
+}
+
+template <typename ValType>
+cpp::EnableIfType<!cpp::IsFloatingPointType<ValType>::Value, bool>
+testEQ(ValType LHS, ValType RHS) {
+  return LHS == RHS;
+}
+
+// For floating points, we consider all NaNs are equal, and +0.0 is not equal to
+// -0.0.
+template <typename ValType>
+cpp::EnableIfType<cpp::IsFloatingPointType<ValType>::Value, bool>
+testEQ(ValType LHS, ValType RHS) {
+  fputil::FPBits<ValType> LHSBits(LHS), RHSBits(RHS);
+
+  return (LHSBits.isNaN() && RHSBits.isNaN()) ||
+         (LHSBits.bitsAsUInt() == RHSBits.bitsAsUInt());
+}
+
 template <typename ValType>
 bool test(RunContext &Ctx, TestCondition Cond, ValType LHS, ValType RHS,
           const char *LHSStr, const char *RHSStr, const char *File,
           unsigned long Line) {
+  auto ExplainDifference = [=](llvm::StringRef OpString) {
+    explainDifference(LHS, RHS, LHSStr, RHSStr, File, Line, OpString);
+  };
+
   switch (Cond) {
   case Cond_EQ:
-    if (LHS == RHS)
+    if (testEQ(LHS, RHS))
       return true;
 
     Ctx.markFail();
-    llvm::outs() << File << ":" << Line << ": FAILURE\n"
-                 << "      Expected: " << LHSStr << '\n'
-                 << "      Which is: " << LHS << '\n'
-                 << "To be equal to: " << RHSStr << '\n'
-                 << "      Which is: " << RHS << '\n';
-
+    ExplainDifference("equal to");
     return false;
   case Cond_NE:
-    if (LHS != RHS)
+    if (!testEQ(LHS, RHS))
       return true;
 
     Ctx.markFail();
-    llvm::outs() << File << ":" << Line << ": FAILURE\n"
-                 << "          Expected: " << LHSStr << '\n'
-                 << "          Which is: " << LHS << '\n'
-                 << "To be not equal to: " << RHSStr << '\n'
-                 << "          Which is: " << RHS << '\n';
+    ExplainDifference("not equal to");
     return false;
   case Cond_LT:
     if (LHS < RHS)
       return true;
 
     Ctx.markFail();
-    llvm::outs() << File << ":" << Line << ": FAILURE\n"
-                 << "       Expected: " << LHSStr << '\n'
-                 << "       Which is: " << LHS << '\n'
-                 << "To be less than: " << RHSStr << '\n'
-                 << "       Which is: " << RHS << '\n';
+    ExplainDifference("less than");
     return false;
   case Cond_LE:
     if (LHS <= RHS)
       return true;
 
     Ctx.markFail();
-    llvm::outs() << File << ":" << Line << ": FAILURE\n"
-                 << "                   Expected: " << LHSStr << '\n'
-                 << "                   Which is: " << LHS << '\n'
-                 << "To be less than or equal to: " << RHSStr << '\n'
-                 << "                   Which is: " << RHS << '\n';
+    ExplainDifference("less than or equal to");
     return false;
   case Cond_GT:
     if (LHS > RHS)
       return true;
 
     Ctx.markFail();
-    llvm::outs() << File << ":" << Line << ": FAILURE\n"
-                 << "          Expected: " << LHSStr << '\n'
-                 << "          Which is: " << LHS << '\n'
-                 << "To be greater than: " << RHSStr << '\n'
-                 << "          Which is: " << RHS << '\n';
+    ExplainDifference("greater than");
     return false;
   case Cond_GE:
     if (LHS >= RHS)
       return true;
 
     Ctx.markFail();
-    llvm::outs() << File << ":" << Line << ": FAILURE\n"
-                 << "                      Expected: " << LHSStr << '\n'
-                 << "                      Which is: " << LHS << '\n'
-                 << "To be greater than or equal to: " << RHSStr << '\n'
-                 << "                      Which is: " << RHS << '\n';
+    ExplainDifference("greater than or equal to");
     return false;
   default:
     Ctx.markFail();
@@ -218,6 +286,26 @@ template bool Test::test<unsigned long long, 0>(
     unsigned long long RHS, const char *LHSStr, const char *RHSStr,
     const char *File, unsigned long Line);
 
+template bool Test::test<__uint128_t, 0>(RunContext &Ctx, TestCondition Cond,
+                                         __uint128_t LHS, __uint128_t RHS,
+                                         const char *LHSStr, const char *RHSStr,
+                                         const char *File, unsigned long Line);
+
+template bool Test::test<float, 0>(RunContext &Ctx, TestCondition Cond,
+                                   float LHS, float RHS, const char *LHSStr,
+                                   const char *RHSStr, const char *File,
+                                   unsigned long Line);
+
+template bool Test::test<double, 0>(RunContext &Ctx, TestCondition Cond,
+                                    double LHS, double RHS, const char *LHSStr,
+                                    const char *RHSStr, const char *File,
+                                    unsigned long Line);
+
+template bool Test::test<long double, 0>(RunContext &Ctx, TestCondition Cond,
+                                         long double LHS, long double RHS,
+                                         const char *LHSStr, const char *RHSStr,
+                                         const char *File, unsigned long Line);
+
 bool Test::testStrEq(RunContext &Ctx, const char *LHS, const char *RHS,
                      const char *LHSStr, const char *RHSStr, const char *File,
                      unsigned long Line) {
diff --git a/libc/utils/UnitTest/Test.h b/libc/utils/UnitTest/Test.h
index fbed1d6a6606f..ec92bd2187528 100644
--- a/libc/utils/UnitTest/Test.h
+++ b/libc/utils/UnitTest/Test.h
@@ -78,7 +78,7 @@ class Test {
   // |Cond| on mismatched |LHS| and |RHS| types can potentially succeed because
   // of type promotion.
   template <typename ValType,
-            cpp::EnableIfType<cpp::IsIntegral<ValType>::Value, ValType> = 0>
+            cpp::EnableIfType<cpp::IsArithmetic<ValType>::Value, int> = 0>
   static bool test(RunContext &Ctx, TestCondition Cond, ValType LHS,
                    ValType RHS, const char *LHSStr, const char *RHSStr,
                    const char *File, unsigned long Line) {
diff --git a/libclc/CMakeLists.txt b/libclc/CMakeLists.txt
index b1f73e0896cb9..e2c737f11a7aa 100644
--- a/libclc/CMakeLists.txt
+++ b/libclc/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required( VERSION 3.9.2 )
+cmake_minimum_required(VERSION 3.13.4)
 
 add_custom_target(libspirv-builtins COMMENT "Build libspirv builtins")
 add_custom_target(libclc-builtins COMMENT "Build libclc builtins")
diff --git a/libcxx/CMakeLists.txt b/libcxx/CMakeLists.txt
index caf655d6799aa..fd877f1adaeb4 100644
--- a/libcxx/CMakeLists.txt
+++ b/libcxx/CMakeLists.txt
@@ -8,7 +8,7 @@ endif()
 #===============================================================================
 # Setup Project
 #===============================================================================
-cmake_minimum_required(VERSION 3.4.3)
+cmake_minimum_required(VERSION 3.13.4)
 
 if(POLICY CMP0042)
   cmake_policy(SET CMP0042 NEW) # Set MACOSX_RPATH=YES by default
@@ -519,13 +519,18 @@ remove_flags(-Wno-pedantic -pedantic-errors -pedantic)
 # Required flags ==============================================================
 function(cxx_add_basic_build_flags target)
 
-  # Require C++14 for all targets. C++14 is needed to ensure constant
-  # initialization for certain globals (ex global memory resources).
+  # Require C++17 for all targets. C++17 is needed to use aligned allocation
+  # in the dylib.
   set_target_properties(${target} PROPERTIES
-    CXX_STANDARD 14
+    CXX_STANDARD 17
     CXX_STANDARD_REQUIRED YES
     CXX_EXTENSIONS NO)
 
+  # When building the dylib, don't warn for unavailable aligned allocation
+  # functions based on the deployment target -- they are always available
+  # because they are provided by the dylib itself.
+  target_add_compile_flags_if_supported(${target} PRIVATE -faligned-allocation)
+
   # On all systems the system c++ standard library headers need to be excluded.
   # MSVC only has -X, which disables all default includes; including the crt.
   # Thus, we do nothing and hope we don't accidentally include any of the C++
@@ -578,6 +583,7 @@ function(cxx_add_warning_flags target)
     target_add_compile_flags_if_supported(${target} PRIVATE
       -Wno-user-defined-literals
       -Wno-covered-switch-default
+      -Wno-suggest-override
       -Wno-ignored-attributes # FIXME: Caused by _LIBCPP_NODEBUG_TYPE not being supported on older clangs
     )
     if (LIBCXX_TARGETING_CLANG_CL)
@@ -602,7 +608,8 @@ function(cxx_add_warning_flags target)
     target_add_compile_flags_if_supported(${target} PRIVATE
       -Wno-literal-suffix
       -Wno-c++14-compat
-      -Wno-noexcept-type)
+      -Wno-noexcept-type
+      -Wno-suggest-override)
   endif()
   if (LIBCXX_ENABLE_WERROR)
     target_add_compile_flags_if_supported(${target} PRIVATE -Werror)
diff --git a/libcxx/docs/DesignDocs/UniquePtrTrivialAbi.rst b/libcxx/docs/DesignDocs/UniquePtrTrivialAbi.rst
new file mode 100644
index 0000000000000..a0f260a44e815
--- /dev/null
+++ b/libcxx/docs/DesignDocs/UniquePtrTrivialAbi.rst
@@ -0,0 +1,149 @@
+=============================================
+Enable std::unique_ptr [[clang::trivial_abi]]
+=============================================
+
+Background
+==========
+
+Consider the follow snippets
+
+
+.. code-block:: cpp
+
+    void raw_func(Foo* raw_arg) { ... }
+    void smart_func(std::unique_ptr<Foo> smart_arg) { ... }
+
+    Foo* raw_ptr_retval() { ... }
+    std::unique_ptr<Foo*> smart_ptr_retval() { ... }
+
+
+
+The argument ``raw_arg`` could be passed in a register but ``smart_arg`` could not, due to current
+implementation.
+
+Specifically, in the ``smart_arg`` case, the caller secretly constructs a temporary ``std::unique_ptr``
+in its stack-frame, and then passes a pointer to it to the callee in a hidden parameter.
+Similarly, the return value from ``smart_ptr_retval`` is secretly allocated in the caller and
+passed as a secret reference to the callee.
+
+
+Goal
+===================
+
+``std::unique_ptr`` is passed directly in a register.
+
+Design
+======
+
+* Annotate the two definitions of ``std::unique_ptr``  with ``clang::trivial_abi`` attribute.
+* Put the attribuate behind a flag because this change has potential compilation and runtime breakages.
+
+
+This comes with some side effects:
+
+* ``std::unique_ptr`` parameters will now be destroyed by callees, rather than callers.
+  It is worth noting that destruction by callee is not unique to the use of trivial_abi attribute.
+  In most Microsoft's ABIs, arguments are always destroyed by the callee.
+
+  Consequently, this may change the destruction order for function parameters to an order that is non-conforming to the standard.
+  For example:
+
+
+  .. code-block:: cpp
+
+    struct A { ~A(); };
+    struct B { ~B(); };
+    struct C { C(A, unique_ptr<B>, A) {} };
+    C c{{}, make_unique<B>, {}};
+
+
+  In a conforming implementation, the destruction order for C::C's parameters is required to be ``~A(), ~B(), ~A()`` but with this mode enabled, we'll instead see ``~B(), ~A(), ~A()``.
+
+* Reduced code-size.
+
+
+Performance impact
+------------------
+
+Google has measured performance improvements of up to 1.6% on some large server macrobenchmarks, and a small reduction in binary sizes.
+
+This also affects null pointer optimization
+
+Clang's optimizer can now figure out when a `std::unique_ptr` is known to contain *non*-null.
+(Actually, this has been a *missed* optimization all along.)
+
+
+.. code-block:: cpp
+
+    struct Foo {
+      ~Foo();
+    };
+    std::unique_ptr<Foo> make_foo();
+    void do_nothing(const Foo&)
+
+    void bar() {
+      auto x = make_foo();
+      do_nothing(*x);
+    }
+
+
+With this change, ``~Foo()`` will be called even if ``make_foo`` returns ``unique_ptr<Foo>(nullptr)``.
+The compiler can now assume that ``x.get()`` cannot be null by the end of ``bar()``, because
+the deference of ``x`` would be UB if it were ``nullptr``. (This dereference would not have caused
+a segfault, because no load is generated for dereferencing a pointer to a reference. This can be detected with ``-fsanitize=null``).
+
+
+Potential breakages
+-------------------
+
+The following breakages were discovered by enabling this change and fixing the resulting issues in a large code base.
+
+- Compilation failures
+
+ - Function definitions now require complete type ``T`` for parameters with type ``std::unique_ptr<T>``. The following code will no longer compile.
+
+   .. code-block:: cpp
+
+       class Foo;
+       void func(std::unique_ptr<Foo> arg) { /* never use `arg` directly */ }
+
+ - Fix: Remove forward-declaration of ``Foo`` and include its proper header.
+
+- Runtime Failures
+
+ - Lifetime of ``std::unique_ptr<>`` arguments end earlier (at the end of the callee's body, rather than at the end of the full expression containing the call).
+
+   .. code-block:: cpp
+
+     util::Status run_worker(std::unique_ptr<Foo>);
+     void func() {
+        std::unique_ptr<Foo> smart_foo = ...;
+        Foo* owned_foo = smart_foo.get();
+        // Currently, the following would "work" because the argument to run_worker() is deleted at the end of func()
+        // With the new calling convention, it will be deleted at the end of run_worker(),
+        // making this an access to freed memory.
+        owned_foo->Bar(run_worker(std::move(smart_foo)));
+                  ^
+                 // <<<Crash expected here
+     }
+
+ - Lifetime of local *returned* ``std::unique_ptr<>`` ends earlier.
+
+   Spot the bug:
+
+    .. code-block:: cpp
+
+     std::unique_ptr<Foo> create_and_subscribe(Bar* subscriber) {
+       auto foo = std::make_unique<Foo>();
+       subscriber->sub([&foo] { foo->do_thing();} );
+       return foo;
+     }
+
+   One could point out this is an obvious stack-use-after return bug.
+   With the current calling convention, running this code with ASAN enabled, however, would not yield any "issue".
+   So is this a bug in ASAN? (Spoiler: No)
+
+   This currently would "work" only because the storage for ``foo`` is in the caller's stackframe.
+   In other words, ``&foo`` in callee and ``&foo`` in the caller are the same address.
+
+ASAN can be used to detect both of these.
diff --git a/libcxx/docs/index.rst b/libcxx/docs/index.rst
index c60a72d497244..75d5b8226b4c8 100644
--- a/libcxx/docs/index.rst
+++ b/libcxx/docs/index.rst
@@ -164,6 +164,7 @@ Design Documents
    DesignDocs/FileTimeType
    DesignDocs/FeatureTestMacros
    DesignDocs/ExtendedCXX03Support
+   DesignDocs/UniquePtrTrivialAbi
 
 * `<atomic> design <http://libcxx.llvm.org/atomic_design.html>`_
 * `<type_traits> design <http://libcxx.llvm.org/type_traits_design.html>`_
diff --git a/libcxx/include/__config b/libcxx/include/__config
index 3b019a0493b55..ebdd64ed4292d 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -105,6 +105,10 @@
 // Re-worked external template instantiations for std::string with a focus on
 // performance and fast-path inlining.
 #  define _LIBCPP_ABI_STRING_OPTIMIZED_EXTERNAL_INSTANTIATION
+// Enable clang::trivial_abi on std::unique_ptr.
+#  define _LIBCPP_ABI_ENABLE_UNIQUE_PTR_TRIVIAL_ABI
+// Enable clang::trivial_abi on std::shared_ptr and std::weak_ptr
+#  define _LIBCPP_ABI_ENABLE_SHARED_PTR_TRIVIAL_ABI
 #elif _LIBCPP_ABI_VERSION == 1
 #  if !defined(_LIBCPP_OBJECT_FORMAT_COFF)
 // Enable compiling copies of now inline methods into the dylib to support
diff --git a/libcxx/include/memory b/libcxx/include/memory
index 1f9f36c5bbbb0..e38e80568eb98 100644
--- a/libcxx/include/memory
+++ b/libcxx/include/memory
@@ -338,7 +338,7 @@ public:
     pointer release() noexcept;
     void reset(pointer p = pointer()) noexcept;
     void reset(nullptr_t) noexcept;
-    template <class U> void reset(U) = delete;
+  template <class U> void reset(U) = delete;
     void swap(unique_ptr& u) noexcept;
 };
 
@@ -2316,8 +2316,14 @@ struct __unique_ptr_deleter_sfinae<_Deleter&> {
   typedef false_type __enable_rval_overload;
 };
 
+#if defined(_LIBCPP_ABI_ENABLE_UNIQUE_PTR_TRIVIAL_ABI)
+#  define _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI __attribute__((trivial_abi))
+#else
+#  define _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI
+#endif
+
 template <class _Tp, class _Dp = default_delete<_Tp> >
-class _LIBCPP_TEMPLATE_VIS unique_ptr {
+class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr {
 public:
   typedef _Tp element_type;
   typedef _Dp deleter_type;
@@ -2525,7 +2531,7 @@ public:
 
 
 template <class _Tp, class _Dp>
-class _LIBCPP_TEMPLATE_VIS unique_ptr<_Tp[], _Dp> {
+class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr<_Tp[], _Dp> {
 public:
   typedef _Tp element_type;
   typedef _Dp deleter_type;
@@ -3537,8 +3543,14 @@ struct __compatible_with
     : is_convertible<_Tp*, _Up*> {};
 #endif // _LIBCPP_STD_VER > 14
 
+#if defined(_LIBCPP_ABI_ENABLE_SHARED_PTR_TRIVIAL_ABI)
+#  define _LIBCPP_SHARED_PTR_TRIVIAL_ABI __attribute__((trivial_abi))
+#else
+#  define _LIBCPP_SHARED_PTR_TRIVIAL_ABI
+#endif
+
 template<class _Tp>
-class _LIBCPP_TEMPLATE_VIS shared_ptr
+class _LIBCPP_SHARED_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS shared_ptr
 {
 public:
 #if _LIBCPP_STD_VER > 14
@@ -4526,7 +4538,7 @@ get_deleter(const shared_ptr<_Tp>& __p) _NOEXCEPT
 #endif  // _LIBCPP_NO_RTTI
 
 template<class _Tp>
-class _LIBCPP_TEMPLATE_VIS weak_ptr
+class _LIBCPP_SHARED_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS weak_ptr
 {
 public:
     typedef _Tp element_type;
diff --git a/libcxx/include/variant b/libcxx/include/variant
index 03557239a69e6..33d5dc7dbeb12 100644
--- a/libcxx/include/variant
+++ b/libcxx/include/variant
@@ -227,7 +227,10 @@ public:
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if _LIBCPP_STD_VER > 14
+// TODO: GCC 5 lies about its support for C++17 (it says it supports it but it
+//       really doesn't). That breaks variant, which uses some C++17 features.
+//       Remove this once we drop support for GCC 5.
+#if _LIBCPP_STD_VER > 14 && !(defined(_LIBCPP_COMPILER_GCC) && _GNUC_VER_NEW < 6000)
 
 _LIBCPP_NORETURN
 inline _LIBCPP_INLINE_VISIBILITY
diff --git a/libcxx/src/barrier.cpp b/libcxx/src/barrier.cpp
index c5e33cbba3bd0..9ee476993b813 100644
--- a/libcxx/src/barrier.cpp
+++ b/libcxx/src/barrier.cpp
@@ -26,21 +26,15 @@ class __barrier_algorithm_base {
         } __tickets[64];
     };
 
-    ptrdiff_t&         __expected;
-    unique_ptr<char[]> __state_allocation;
-    __state_t*         __state;
+    ptrdiff_t&              __expected;
+    unique_ptr<__state_t[]> __state;
 
     _LIBCPP_HIDDEN
     __barrier_algorithm_base(ptrdiff_t& __expected)
         : __expected(__expected)
     {
         size_t const __count = (__expected + 1) >> 1;
-        size_t const __size = sizeof(__state_t) * __count;
-        size_t __allocation_size = __size + alignof(__state_t);
-        __state_allocation = unique_ptr<char[]>(new char[__allocation_size]);
-        void* __allocation = __state_allocation.get();
-        void* const __state_ = align(alignof(__state_t), __size, __allocation, __allocation_size);
-        __state = new (__state_) __barrier_algorithm_base::__state_t[__count];
+        __state = unique_ptr<__state_t[]>(new __state_t[__count]);
     }
     _LIBCPP_HIDDEN
     bool __arrive(__barrier_phase_t __old_phase)
diff --git a/libcxx/test/libcxx/memory/trivial_abi/shared_ptr_arg.pass.cpp b/libcxx/test/libcxx/memory/trivial_abi/shared_ptr_arg.pass.cpp
new file mode 100644
index 0000000000000..fdf4a172bab28
--- /dev/null
+++ b/libcxx/test/libcxx/memory/trivial_abi/shared_ptr_arg.pass.cpp
@@ -0,0 +1,52 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <memory>
+
+// Test shared_ptr<T> with trivial_abi as parameter type.
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ABI_ENABLE_SHARED_PTR_TRIVIAL_ABI
+
+// There were assertion failures in both parse and codegen, which are fixed in clang 11.
+// UNSUPPORTED: gcc, clang-4, clang-5, clang-6, clang-7, clang-8, clang-9, clang-10
+// UNSUPPORTED: c++03
+
+#include <memory>
+#include <cassert>
+
+__attribute__((noinline)) void call_something() { asm volatile(""); }
+
+struct Node {
+  int* shared_val;
+
+  explicit Node(int* ptr) : shared_val(ptr) {}
+  ~Node() { ++(*shared_val); }
+};
+
+__attribute__((noinline)) bool get_val(std::shared_ptr<Node> /*unused*/) {
+  call_something();
+  return true;
+}
+
+__attribute__((noinline)) void expect_1(int* shared, bool /*unused*/) {
+  assert(*shared == 1);
+}
+
+int main(int, char**) {
+  int shared = 0;
+
+  // Without trivial-abi, the shared_ptr is deleted at the end of this
+  // statement; expect_1 will see shared == 0 because it's not incremented (in
+  // ~Node()) until expect_1 returns.
+  //
+  // With trivial-abi, expect_1 will see shared == 1 because shared_val is
+  // incremented before get_val returns.
+  expect_1(&shared, get_val(std::make_shared<Node>(&shared)));
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/memory/trivial_abi/unique_ptr_arg.pass.cpp b/libcxx/test/libcxx/memory/trivial_abi/unique_ptr_arg.pass.cpp
new file mode 100644
index 0000000000000..e7336ddcea713
--- /dev/null
+++ b/libcxx/test/libcxx/memory/trivial_abi/unique_ptr_arg.pass.cpp
@@ -0,0 +1,53 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <memory>
+
+// Test unique_ptr<T> with trivial_abi as parameter type.
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ABI_ENABLE_UNIQUE_PTR_TRIVIAL_ABI
+
+// There were assertion failures in both parse and codegen, which are fixed in clang 11.
+// UNSUPPORTED: gcc, clang-4, clang-5, clang-6, clang-7, clang-8, clang-9, clang-10
+
+#include <memory>
+#include <cassert>
+
+__attribute__((noinline)) void call_something() { asm volatile(""); }
+
+struct Node {
+  int* shared_val;
+
+  explicit Node(int* ptr) : shared_val(ptr) {}
+  ~Node() { ++(*shared_val); }
+};
+
+__attribute__((noinline)) bool get_val(std::unique_ptr<Node> /*unused*/) {
+  call_something();
+  return true;
+}
+
+__attribute__((noinline)) void expect_1(int* shared, bool /*unused*/) {
+  assert(*shared == 1);
+}
+
+int main(int, char**) {
+  int shared = 0;
+
+  // Without trivial-abi, the unique_ptr is deleted at the end of this
+  // statement; expect_1 will see shared == 0 because it's not incremented (in
+  // ~Node()) until expect_1 returns.
+  //
+  // With trivial-abi, expect_1 will see shared == 1 because shared_val is
+  // incremented before get_val returns.
+  expect_1(&shared, get_val(std::unique_ptr<Node>(new Node(&shared))));
+
+  // Check that the shared-value is still 1.
+  expect_1(&shared, true);
+  return 0;
+}
diff --git a/libcxx/test/libcxx/memory/trivial_abi/unique_ptr_array.pass.cpp b/libcxx/test/libcxx/memory/trivial_abi/unique_ptr_array.pass.cpp
new file mode 100644
index 0000000000000..4e4b521131a2f
--- /dev/null
+++ b/libcxx/test/libcxx/memory/trivial_abi/unique_ptr_array.pass.cpp
@@ -0,0 +1,56 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <memory>
+
+// Test unique_ptr<T[]> with trivial_abi as parameter type.
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ABI_ENABLE_UNIQUE_PTR_TRIVIAL_ABI
+
+// There were assertion failures in both parse and codegen, which are fixed in clang 11.
+// UNSUPPORTED: gcc, clang-4, clang-5, clang-6, clang-7, clang-8, clang-9, clang-10
+// UNSUPPORTED: c++03
+
+#include <memory>
+#include <cassert>
+
+__attribute__((noinline)) void call_something() { asm volatile(""); }
+
+struct Node {
+  int* shared_val;
+
+  explicit Node(int* ptr) : shared_val(ptr) {}
+  ~Node() { ++(*shared_val); }
+};
+
+__attribute__((noinline)) bool get_val(std::unique_ptr<Node[]> /*unused*/) {
+  call_something();
+  return true;
+}
+
+__attribute__((noinline)) void expect_3(int* shared, bool /*unused*/) {
+  assert(*shared == 3);
+}
+
+int main(int, char**) {
+  int shared = 0;
+
+  // Without trivial-abi, the unique_ptr is deleted at the end of this
+  // statement, expect_3 will see shared == 0 because it's not incremented (in
+  // ~Node()) until the end of this statement.
+  //
+  // With trivial-abi, shared_val is incremented 3 times before get_val returns
+  // because ~Node() was called 3 times.
+  expect_3(&shared, get_val(std::unique_ptr<Node[]>(new Node[3]{
+                        Node(&shared), Node(&shared), Node(&shared)})));
+
+  // Check that shared_value is still 3 (ie., ~Node() isn't called again by the end of the full-expression above)
+  expect_3(&shared, true);
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/memory/trivial_abi/unique_ptr_destruction_order.pass.cpp b/libcxx/test/libcxx/memory/trivial_abi/unique_ptr_destruction_order.pass.cpp
new file mode 100644
index 0000000000000..980349866a987
--- /dev/null
+++ b/libcxx/test/libcxx/memory/trivial_abi/unique_ptr_destruction_order.pass.cpp
@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <memory>
+
+// Test arguments destruction order involving unique_ptr<T> with trivial_abi.
+// Note: Unlike other tests in this directory, this is the only test that
+// exhibits a difference between the two modes in Microsft ABI.
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ABI_ENABLE_UNIQUE_PTR_TRIVIAL_ABI
+
+// There were assertion failures in both parse and codegen, which are fixed in clang 11.
+// UNSUPPORTED: gcc, clang-4, clang-5, clang-6, clang-7, clang-8, clang-9, clang-10
+
+#include <memory>
+#include <cassert>
+
+__attribute__((noinline)) void call_something() { asm volatile(""); }
+
+struct Base {
+  char* shared_buff;
+  int* cur_idx;
+  const char id;
+
+  explicit Base(char* buf, int* idx, char ch)
+      : shared_buff(buf), cur_idx(idx), id(ch) {}
+  ~Base() { shared_buff[(*cur_idx)++] = id; }
+};
+
+struct A : Base {
+  explicit A(char* buf, int* idx) : Base(buf, idx, 'A') {}
+};
+
+struct B : Base {
+  explicit B(char* buf, int* idx) : Base(buf, idx, 'B') {}
+};
+
+struct C : Base {
+  explicit C(char* buf, int* idx) : Base(buf, idx, 'C') {}
+};
+
+__attribute__((noinline)) void func(A /*unused*/, std::unique_ptr<B> /*unused*/,
+                                    C /*unused*/) {
+  call_something();
+}
+
+int main(int, char**) {
+  char shared_buf[3] = {'0', '0', '0'};
+  int cur_idx = 0;
+
+  func(A(shared_buf, &cur_idx), std::unique_ptr<B>(new B(shared_buf, &cur_idx)),
+       C(shared_buf, &cur_idx));
+
+  // With trivial_abi, the std::unique_ptr<B> arg is always destructed first.
+  assert(shared_buf[0] == 'B');
+  return 0;
+}
diff --git a/libcxx/test/libcxx/memory/trivial_abi/unique_ptr_ret.pass.cpp b/libcxx/test/libcxx/memory/trivial_abi/unique_ptr_ret.pass.cpp
new file mode 100644
index 0000000000000..73fe18becc3a9
--- /dev/null
+++ b/libcxx/test/libcxx/memory/trivial_abi/unique_ptr_ret.pass.cpp
@@ -0,0 +1,52 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <memory>
+
+// Test unique_ptr<T> with trivial_abi as return-type.
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ABI_ENABLE_UNIQUE_PTR_TRIVIAL_ABI
+
+// There were assertion failures in both parse and codegen, which are fixed in clang 11.
+// UNSUPPORTED: gcc, clang-4, clang-5, clang-6, clang-7, clang-8, clang-9, clang-10
+
+#include <memory>
+#include <cassert>
+
+__attribute__((noinline)) void call_something() { asm volatile(""); }
+
+struct Node {
+  explicit Node() {}
+  ~Node() {}
+};
+
+__attribute__((noinline)) std::unique_ptr<Node> make_val(void** local_addr) {
+  call_something();
+
+  auto ret = std::unique_ptr<Node>(new Node);
+
+  // Capture the local address of ret.
+  *local_addr = &ret;
+
+  return ret;
+}
+
+int main(int, char**) {
+  void* local_addr = nullptr;
+  auto ret = make_val(&local_addr);
+  assert(local_addr != nullptr);
+
+  // Without trivial_abi, &ret == local_addr because the return value
+  // is allocated here in main's stackframe.
+  //
+  // With trivial_abi, local_addr is the address of a local variable in
+  // make_val, and hence different from &ret.
+  assert((void*)&ret != local_addr);
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/memory/trivial_abi/weak_ptr_ret.pass.cpp b/libcxx/test/libcxx/memory/trivial_abi/weak_ptr_ret.pass.cpp
new file mode 100644
index 0000000000000..e69c94506f2af
--- /dev/null
+++ b/libcxx/test/libcxx/memory/trivial_abi/weak_ptr_ret.pass.cpp
@@ -0,0 +1,58 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <memory>
+
+// Test weak_ptr<T> with trivial_abi as return-type.
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ABI_ENABLE_SHARED_PTR_TRIVIAL_ABI
+
+// There were assertion failures in both parse and codegen, which are fixed in clang 11.
+// UNSUPPORTED: gcc, clang-4, clang-5, clang-6, clang-7, clang-8, clang-9, clang-10
+
+#include <memory>
+#include <cassert>
+
+__attribute__((noinline)) void call_something() { asm volatile(""); }
+
+struct Node {
+  explicit Node() {}
+  ~Node() {}
+};
+
+__attribute__((noinline)) std::weak_ptr<Node>
+make_val(std::shared_ptr<Node>& sptr, void** local_addr) {
+  call_something();
+
+  std::weak_ptr<Node> ret;
+  ret = sptr;
+
+  // Capture the local address of ret.
+  *local_addr = &ret;
+
+  return ret;
+}
+
+int main(int, char**) {
+  void* local_addr = nullptr;
+  auto sptr = std::make_shared<Node>();
+  std::weak_ptr<Node> ret = make_val(sptr, &local_addr);
+  assert(local_addr != nullptr);
+
+  // Without trivial_abi, &ret == local_addr because the return value
+  // is allocated here in main's stackframe.
+  //
+  // With trivial_abi, local_addr is the address of a local variable in
+  // make_val, and hence different from &ret.
+#ifndef __arm__
+  // On ARM32, structs larger than 4 bytes cannot be returned in registers.
+  // Thus, weak_ptr will be passed indrectly even if it is trivial.
+  assert((void*)&ret != local_addr);
+#endif
+  return 0;
+}
diff --git a/libcxx/utils/ci/runtimes/CMakeLists.txt b/libcxx/utils/ci/runtimes/CMakeLists.txt
index 43ebf9e4a196c..f46ef63b809c7 100644
--- a/libcxx/utils/ci/runtimes/CMakeLists.txt
+++ b/libcxx/utils/ci/runtimes/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.4.3)
+cmake_minimum_required(VERSION 3.13.4)
 
 if(POLICY CMP0068)
   cmake_policy(SET CMP0068 NEW)
diff --git a/libcxx/utils/libcxx/test/dsl.py b/libcxx/utils/libcxx/test/dsl.py
index 2c54921844b2a..cd500e132946e 100644
--- a/libcxx/utils/libcxx/test/dsl.py
+++ b/libcxx/utils/libcxx/test/dsl.py
@@ -52,13 +52,14 @@ def _executeScriptInternal(test, commands):
     res = ('', '', 127, None)
   return res
 
-def _makeConfigTest(config):
+def _makeConfigTest(config, testPrefix=None):
   sourceRoot = os.path.join(config.test_exec_root, '__config_src__')
   execRoot = os.path.join(config.test_exec_root, '__config_exec__')
   suite = lit.Test.TestSuite('__config__', sourceRoot, execRoot, config)
   if not os.path.exists(sourceRoot):
     os.makedirs(sourceRoot)
-  tmp = tempfile.NamedTemporaryFile(dir=sourceRoot, delete=False, suffix='.cpp')
+  tmp = tempfile.NamedTemporaryFile(dir=sourceRoot, delete=False, suffix='.cpp',
+                                    prefix=testPrefix)
   tmp.close()
   pathInSuite = [os.path.relpath(tmp.name, sourceRoot)]
   class TestWrapper(lit.Test.Test):
@@ -82,7 +83,7 @@ def sourceBuilds(config, source):
     _executeScriptInternal(test, ['rm %t.exe'])
     return exitCode == 0
 
-def programOutput(config, program, args=[]):
+def programOutput(config, program, args=[], testPrefix=None):
   """
   Compiles a program for the test target, run it on the test target and return
   the output.
@@ -91,7 +92,7 @@ def programOutput(config, program, args=[]):
   execution of the program is done through the %{exec} substitution, which means
   that the program may be run on a remote host depending on what %{exec} does.
   """
-  with _makeConfigTest(config) as test:
+  with _makeConfigTest(config, testPrefix=testPrefix) as test:
     with open(test.getSourcePath(), 'w') as source:
       source.write(program)
     try:
@@ -142,7 +143,8 @@ def hasLocale(config, locale):
       else                                      return 1;
     }
   """
-  return programOutput(config, program, args=[pipes.quote(locale)]) != None
+  return programOutput(config, program, args=[pipes.quote(locale)],
+                       testPrefix="check_locale_" + locale) is not None
 
 def compilerMacros(config, flags=''):
   """
diff --git a/libcxx/utils/libcxx/test/features.py b/libcxx/utils/libcxx/test/features.py
index 6a16ca851d3f9..dfdc0bae7a29a 100644
--- a/libcxx/utils/libcxx/test/features.py
+++ b/libcxx/utils/libcxx/test/features.py
@@ -104,10 +104,10 @@
   'cs_CZ.ISO8859-2': ['cs_CZ.ISO8859-2', 'Czech_Czech Republic.1250']
 }
 for locale, alts in locales.items():
-  DEFAULT_FEATURES += [
-    Feature(name='locale.{}'.format(locale),
-            when=lambda cfg: any(hasLocale(cfg, alt) for alt in alts))
-  ]
+  # Note: Using alts directly in the lambda body here will bind it to the value at the
+  # end of the loop. Assigning it to a default argument works around this issue.
+  DEFAULT_FEATURES.append(Feature(name='locale.{}'.format(locale),
+                                  when=lambda cfg, alts=alts: any(hasLocale(cfg, alt) for alt in alts)))
 
 
 # Add features representing the platform name: darwin, linux, windows, etc...
diff --git a/libcxx/utils/merge_archives.py b/libcxx/utils/merge_archives.py
index cc96cb2aa50ce..d751526359bac 100755
--- a/libcxx/utils/merge_archives.py
+++ b/libcxx/utils/merge_archives.py
@@ -93,7 +93,7 @@ def main():
     parser.add_argument(
         '-L', dest='search_paths',
         help='Paths to search for the libraries along', action='append',
-        nargs=1)
+        nargs=1, default=[])
     parser.add_argument(
         '--ar', dest='ar_exe', required=False,
         help='The ar executable to use, finds \'ar\' in the path if not given',
diff --git a/libcxx/utils/run.py b/libcxx/utils/run.py
index e88fff1282482..cdfa2387b22d5 100755
--- a/libcxx/utils/run.py
+++ b/libcxx/utils/run.py
@@ -23,12 +23,9 @@ def main():
     parser.add_argument('--execdir', type=str, required=True)
     parser.add_argument('--codesign_identity', type=str, required=False, default=None)
     parser.add_argument('--env', type=str, nargs='*', required=False, default=dict())
-    (args, remaining) = parser.parse_known_args(sys.argv[1:])
-
-    if len(remaining) < 2:
-        sys.stderr.write('Missing actual commands to run')
-        exit(1)
-    commandLine = remaining[1:] # Skip the '--'
+    parser.add_argument("command", nargs=argparse.ONE_OR_MORE)
+    args = parser.parse_args()
+    commandLine = args.command
 
     # Do any necessary codesigning.
     if args.codesign_identity:
diff --git a/libcxx/utils/ssh.py b/libcxx/utils/ssh.py
index f74ed785bbbfc..876e35460dc7d 100755
--- a/libcxx/utils/ssh.py
+++ b/libcxx/utils/ssh.py
@@ -29,13 +29,9 @@ def main():
     parser.add_argument('--execdir', type=str, required=True)
     parser.add_argument('--codesign_identity', type=str, required=False, default=None)
     parser.add_argument('--env', type=str, nargs='*', required=False, default=dict())
-    (args, remaining) = parser.parse_known_args(sys.argv[1:])
-
-    if len(remaining) < 2:
-        sys.stderr.write('Missing actual commands to run')
-        return 1
-
-    commandLine = remaining[1:] # Skip the '--'
+    parser.add_argument("command", nargs=argparse.ONE_OR_MORE)
+    args = parser.parse_args()
+    commandLine = args.command
 
     ssh = lambda command: ['ssh', '-oBatchMode=yes', args.host, command]
     scp = lambda src, dst: ['scp', '-q', '-oBatchMode=yes', src, '{}:{}'.format(args.host, dst)]
diff --git a/libcxxabi/CMakeLists.txt b/libcxxabi/CMakeLists.txt
index e4e20d950b890..96a1c625222a8 100644
--- a/libcxxabi/CMakeLists.txt
+++ b/libcxxabi/CMakeLists.txt
@@ -8,7 +8,7 @@ endif()
 # Setup Project
 #===============================================================================
 
-cmake_minimum_required(VERSION 3.4.3)
+cmake_minimum_required(VERSION 3.13.4)
 
 if(POLICY CMP0042)
   cmake_policy(SET CMP0042 NEW) # Set MACOSX_RPATH=YES by default
@@ -283,6 +283,8 @@ add_compile_flags_if_supported(-Wunused-variable)
 add_compile_flags_if_supported(-Wwrite-strings)
 add_compile_flags_if_supported(-Wundef)
 
+add_compile_flags_if_supported(-Wno-suggest-override)
+
 if (LIBCXXABI_ENABLE_WERROR)
   add_compile_flags_if_supported(-Werror)
   add_compile_flags_if_supported(-WX)
diff --git a/libcxxabi/src/CMakeLists.txt b/libcxxabi/src/CMakeLists.txt
index 0503569115286..85e3fda034f7c 100644
--- a/libcxxabi/src/CMakeLists.txt
+++ b/libcxxabi/src/CMakeLists.txt
@@ -177,7 +177,7 @@ if (LIBCXXABI_ENABLE_SHARED)
                           CXX_EXTENSIONS
                             OFF
                           CXX_STANDARD
-                            11
+                            17
                           CXX_STANDARD_REQUIRED
                             ON
                           COMPILE_FLAGS
@@ -234,7 +234,7 @@ if (LIBCXXABI_ENABLE_STATIC)
                           CXX_EXTENSIONS
                             OFF
                           CXX_STANDARD
-                            11
+                            17
                           CXX_STANDARD_REQUIRED
                             ON
                           COMPILE_FLAGS
diff --git a/libcxxabi/src/private_typeinfo.cpp b/libcxxabi/src/private_typeinfo.cpp
index 55a90b3ae1d4d..3e8bdae32e418 100644
--- a/libcxxabi/src/private_typeinfo.cpp
+++ b/libcxxabi/src/private_typeinfo.cpp
@@ -641,6 +641,7 @@ __dynamic_cast(const void *static_ptr, const __class_type_info *static_type,
         {
             // We get here only if there is some kind of visibility problem
             //   in client code.
+            static_assert(std::atomic<size_t>::is_always_lock_free, "");
             static std::atomic<size_t> error_count(0);
             size_t error_count_snapshot = error_count.fetch_add(1, std::memory_order_relaxed);
             if ((error_count_snapshot & (error_count_snapshot-1)) == 0)
@@ -667,6 +668,7 @@ __dynamic_cast(const void *static_ptr, const __class_type_info *static_type,
         if (info.path_dst_ptr_to_static_ptr == unknown &&
             info.path_dynamic_ptr_to_static_ptr == unknown)
         {
+            static_assert(std::atomic<size_t>::is_always_lock_free, "");
             static std::atomic<size_t> error_count(0);
             size_t error_count_snapshot = error_count.fetch_add(1, std::memory_order_relaxed);
             if ((error_count_snapshot & (error_count_snapshot-1)) == 0)
diff --git a/libunwind/CMakeLists.txt b/libunwind/CMakeLists.txt
index b50550dc376ee..c5b536532f3c2 100644
--- a/libunwind/CMakeLists.txt
+++ b/libunwind/CMakeLists.txt
@@ -2,7 +2,7 @@
 # Setup Project
 #===============================================================================
 
-cmake_minimum_required(VERSION 3.4.3)
+cmake_minimum_required(VERSION 3.13.4)
 
 if (POLICY CMP0042)
   cmake_policy(SET CMP0042 NEW) # Set MACOSX_RPATH=YES by default
diff --git a/lld/CMakeLists.txt b/lld/CMakeLists.txt
index 5090c935e75ad..bcfc2c6270b3f 100644
--- a/lld/CMakeLists.txt
+++ b/lld/CMakeLists.txt
@@ -1,7 +1,7 @@
 # Check if lld is built as a standalone project.
 if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
   project(lld)
-  cmake_minimum_required(VERSION 3.4.3)
+  cmake_minimum_required(VERSION 3.13.4)
 
   set(CMAKE_INCLUDE_CURRENT_DIR ON)
   set(LLD_BUILT_STANDALONE TRUE)
@@ -51,6 +51,10 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
   set(LLVM_RUNTIME_OUTPUT_INTDIR ${CMAKE_BINARY_DIR}/${CMAKE_CFG_INTDIR}/bin)
   find_program(LLVM_TABLEGEN_EXE "llvm-tblgen" ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH)
 
+  if(LLVM_ENABLE_ZLIB)
+    find_package(ZLIB)
+  endif()
+
   include(AddLLVM)
   include(TableGen)
   include(HandleLLVMOptions)
diff --git a/lld/COFF/SymbolTable.cpp b/lld/COFF/SymbolTable.cpp
index d4d2a159a639d..173e32f628ef3 100644
--- a/lld/COFF/SymbolTable.cpp
+++ b/lld/COFF/SymbolTable.cpp
@@ -136,12 +136,16 @@ getFileLine(const SectionChunk *c, uint32_t addr) {
 // of all references to that symbol from that file. If no debug information is
 // available, returns just the name of the file, else one string per actual
 // reference as described in the debug info.
-std::vector<std::string> getSymbolLocations(ObjFile *file, uint32_t symIndex) {
+// Returns up to maxStrings string descriptions, along with the total number of
+// locations found.
+static std::pair<std::vector<std::string>, size_t>
+getSymbolLocations(ObjFile *file, uint32_t symIndex, size_t maxStrings) {
   struct Location {
     Symbol *sym;
     std::pair<StringRef, uint32_t> fileLine;
   };
   std::vector<Location> locations;
+  size_t numLocations = 0;
 
   for (Chunk *c : file->getChunks()) {
     auto *sc = dyn_cast<SectionChunk>(c);
@@ -150,6 +154,10 @@ std::vector<std::string> getSymbolLocations(ObjFile *file, uint32_t symIndex) {
     for (const coff_relocation &r : sc->getRelocs()) {
       if (r.SymbolTableIndex != symIndex)
         continue;
+      numLocations++;
+      if (locations.size() >= maxStrings)
+        continue;
+
       Optional<std::pair<StringRef, uint32_t>> fileLine =
           getFileLine(sc, r.VirtualAddress);
       Symbol *sym = getSymbol(sc, r.VirtualAddress);
@@ -160,8 +168,12 @@ std::vector<std::string> getSymbolLocations(ObjFile *file, uint32_t symIndex) {
     }
   }
 
-  if (locations.empty())
-    return std::vector<std::string>({"\n>>> referenced by " + toString(file)});
+  if (maxStrings == 0)
+    return std::make_pair(std::vector<std::string>(), numLocations);
+
+  if (numLocations == 0)
+    return std::make_pair(
+        std::vector<std::string>{"\n>>> referenced by " + toString(file)}, 1);
 
   std::vector<std::string> symbolLocations(locations.size());
   size_t i = 0;
@@ -175,17 +187,26 @@ std::vector<std::string> getSymbolLocations(ObjFile *file, uint32_t symIndex) {
     if (loc.sym)
       os << ":(" << toString(*loc.sym) << ')';
   }
-  return symbolLocations;
+  return std::make_pair(symbolLocations, numLocations);
+}
+
+std::vector<std::string> getSymbolLocations(ObjFile *file, uint32_t symIndex) {
+  return getSymbolLocations(file, symIndex, SIZE_MAX).first;
 }
 
-std::vector<std::string> getSymbolLocations(InputFile *file,
-                                            uint32_t symIndex) {
+static std::pair<std::vector<std::string>, size_t>
+getSymbolLocations(InputFile *file, uint32_t symIndex, size_t maxStrings) {
   if (auto *o = dyn_cast<ObjFile>(file))
-    return getSymbolLocations(o, symIndex);
-  if (auto *b = dyn_cast<BitcodeFile>(file))
-    return getSymbolLocations(b);
+    return getSymbolLocations(o, symIndex, maxStrings);
+  if (auto *b = dyn_cast<BitcodeFile>(file)) {
+    std::vector<std::string> symbolLocations = getSymbolLocations(b);
+    size_t numLocations = symbolLocations.size();
+    if (symbolLocations.size() > maxStrings)
+      symbolLocations.resize(maxStrings);
+    return std::make_pair(symbolLocations, numLocations);
+  }
   llvm_unreachable("unsupported file type passed to getSymbolLocations");
-  return {};
+  return std::make_pair(std::vector<std::string>(), (size_t)0);
 }
 
 // For an undefined symbol, stores all files referencing it and the index of
@@ -205,20 +226,21 @@ static void reportUndefinedSymbol(const UndefinedDiag &undefDiag) {
   os << "undefined symbol: " << toString(*undefDiag.sym);
 
   const size_t maxUndefReferences = 3;
-  size_t i = 0, numRefs = 0;
+  size_t numDisplayedRefs = 0, numRefs = 0;
   for (const UndefinedDiag::File &ref : undefDiag.files) {
-    std::vector<std::string> symbolLocations =
-        getSymbolLocations(ref.file, ref.symIndex);
-    numRefs += symbolLocations.size();
+    std::vector<std::string> symbolLocations;
+    size_t totalLocations = 0;
+    std::tie(symbolLocations, totalLocations) = getSymbolLocations(
+        ref.file, ref.symIndex, maxUndefReferences - numDisplayedRefs);
+
+    numRefs += totalLocations;
+    numDisplayedRefs += symbolLocations.size();
     for (const std::string &s : symbolLocations) {
-      if (i >= maxUndefReferences)
-        break;
       os << s;
-      i++;
     }
   }
-  if (i < numRefs)
-    os << "\n>>> referenced " << numRefs - i << " more times";
+  if (numDisplayedRefs < numRefs)
+    os << "\n>>> referenced " << numRefs - numDisplayedRefs << " more times";
   errorOrWarn(os.str());
 }
 
diff --git a/lld/ELF/Arch/PPC64.cpp b/lld/ELF/Arch/PPC64.cpp
index 71c568088fb9f..da0b510d96a10 100644
--- a/lld/ELF/Arch/PPC64.cpp
+++ b/lld/ELF/Arch/PPC64.cpp
@@ -106,6 +106,11 @@ bool elf::isPPC64SmallCodeModelTocReloc(RelType type) {
   return type == R_PPC64_TOC16 || type == R_PPC64_TOC16_DS;
 }
 
+void elf::writePrefixedInstruction(uint8_t *loc, uint64_t insn) {
+  insn = config->isLE ? insn << 32 | insn >> 32 : insn;
+  write64(loc, insn);
+}
+
 static bool addOptional(StringRef name, uint64_t value,
                         std::vector<Defined *> &defined) {
   Symbol *sym = symtab->find(name);
@@ -376,15 +381,6 @@ static uint32_t readFromHalf16(const uint8_t *loc) {
   return read32(config->isLE ? loc : loc - 2);
 }
 
-// The prefixed instruction is always a 4 byte prefix followed by a 4 byte
-// instruction. Therefore, the prefix is always in lower memory than the
-// instruction (regardless of endianness).
-// As a result, we need to shift the pieces around on little endian machines.
-static void writePrefixedInstruction(uint8_t *loc, uint64_t insn) {
-  insn = config->isLE ? insn << 32 | insn >> 32 : insn;
-  write64(loc, insn);
-}
-
 static uint64_t readPrefixedInstruction(const uint8_t *loc) {
   uint64_t fullInstr = read64(loc);
   return config->isLE ? (fullInstr << 32 | fullInstr >> 32) : fullInstr;
@@ -1048,17 +1044,15 @@ bool PPC64::needsThunk(RelExpr expr, RelType type, const InputFile *file,
   if (s.isInPlt())
     return true;
 
-  // FIXME: Remove the fatal error once the call protocol is implemented.
-  if (type == R_PPC64_REL24_NOTOC && (s.stOther >> 5) > 1)
-    fatal("unimplemented feature: local function call with the reltype"
-          " R_PPC64_REL24_NOTOC and the callee needs toc-pointer setup");
-
   // This check looks at the st_other bits of the callee with relocation
   // R_PPC64_REL14 or R_PPC64_REL24. If the value is 1, then the callee
   // clobbers the TOC and we need an R2 save stub.
   if (type != R_PPC64_REL24_NOTOC && (s.stOther >> 5) == 1)
     return true;
 
+  if (type == R_PPC64_REL24_NOTOC && (s.stOther >> 5) > 1)
+    return true;
+
   // If a symbol is a weak undefined and we are compiling an executable
   // it doesn't need a range-extending thunk since it can't be called.
   if (s.isUndefWeak() && !config->shared)
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 4637a3b306daf..30bff945f1a54 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -330,8 +330,6 @@ static void checkOptions() {
   if (config->relocatable) {
     if (config->shared)
       error("-r and -shared may not be used together");
-    if (config->gcSections)
-      error("-r and --gc-sections may not be used together");
     if (config->gdbIndex)
       error("-r and --gdb-index may not be used together");
     if (config->icf != ICFLevel::None)
diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp
index c2f1830a981b8..c142c00517ccf 100644
--- a/lld/ELF/InputFiles.cpp
+++ b/lld/ELF/InputFiles.cpp
@@ -926,20 +926,6 @@ InputSectionBase *ObjFile<ELFT>::createInputSection(const Elf_Shdr &sec) {
       this->sections[sec.sh_info] = target;
     }
 
-    // This section contains relocation information.
-    // If -r is given, we do not interpret or apply relocation
-    // but just copy relocation sections to output.
-    if (config->relocatable) {
-      InputSection *relocSec = make<InputSection>(*this, sec, name);
-      // We want to add a dependency to target, similar like we do for
-      // -emit-relocs below. This is useful for the case when linker script
-      // contains the "/DISCARD/". It is perhaps uncommon to use a script with
-      // -r, but we faced it in the Linux kernel and have to handle such case
-      // and not to crash.
-      target->dependentSections.push_back(relocSec);
-      return relocSec;
-    }
-
     if (target->firstRelocation)
       fatal(toString(this) +
             ": multiple relocation sections to one section are not supported");
@@ -957,17 +943,17 @@ InputSectionBase *ObjFile<ELFT>::createInputSection(const Elf_Shdr &sec) {
     }
     assert(isUInt<31>(target->numRelocations));
 
-    // Relocation sections processed by the linker are usually removed
-    // from the output, so returning `nullptr` for the normal case.
-    // However, if -emit-relocs is given, we need to leave them in the output.
-    // (Some post link analysis tools need this information.)
-    if (config->emitRelocs) {
-      InputSection *relocSec = make<InputSection>(*this, sec, name);
-      // We will not emit relocation section if target was discarded.
-      target->dependentSections.push_back(relocSec);
-      return relocSec;
-    }
-    return nullptr;
+    // Relocation sections are usually removed from the output, so return
+    // `nullptr` for the normal case. However, if -r or --emit-relocs is
+    // specified, we need to copy them to the output. (Some post link analysis
+    // tools specify --emit-relocs to obtain the information.)
+    if (!config->relocatable && !config->emitRelocs)
+      return nullptr;
+    InputSection *relocSec = make<InputSection>(*this, sec, name);
+    // If the relocated section is discarded (due to /DISCARD/ or
+    // --gc-sections), the relocation section should be discarded as well.
+    target->dependentSections.push_back(relocSec);
+    return relocSec;
   }
   }
 
diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp
index 7a7ebd9749099..0a97b6aa2f508 100644
--- a/lld/ELF/InputSection.cpp
+++ b/lld/ELF/InputSection.cpp
@@ -28,6 +28,7 @@
 #include <algorithm>
 #include <mutex>
 #include <set>
+#include <unordered_set>
 #include <vector>
 
 using namespace llvm;
@@ -389,11 +390,16 @@ template <class ELFT> void InputSection::copyShtGroup(uint8_t *buf) {
   // The first entry is not a section number but a flag.
   *to++ = from[0];
 
-  // Adjust section numbers because section numbers in an input object
-  // files are different in the output.
+  // Adjust section numbers because section numbers in an input object files are
+  // different in the output. We also need to handle combined or discarded
+  // members.
   ArrayRef<InputSectionBase *> sections = file->getSections();
-  for (uint32_t idx : from.slice(1))
-    *to++ = sections[idx]->getOutputSection()->sectionIndex;
+  std::unordered_set<uint32_t> seen;
+  for (uint32_t idx : from.slice(1)) {
+    OutputSection *osec = sections[idx]->getOutputSection();
+    if (osec && seen.insert(osec->sectionIndex).second)
+      *to++ = osec->sectionIndex;
+  }
 }
 
 InputSectionBase *InputSection::getRelocatedSection() const {
diff --git a/lld/ELF/OutputSections.cpp b/lld/ELF/OutputSections.cpp
index 7e9e76b070ec8..3864a4143d2bb 100644
--- a/lld/ELF/OutputSections.cpp
+++ b/lld/ELF/OutputSections.cpp
@@ -21,6 +21,7 @@
 #include "llvm/Support/Parallel.h"
 #include "llvm/Support/SHA1.h"
 #include <regex>
+#include <unordered_set>
 
 using namespace llvm;
 using namespace llvm::dwarf;
@@ -376,6 +377,15 @@ static void finalizeShtGroup(OutputSection *os,
   // provides signature of the section group.
   ArrayRef<Symbol *> symbols = section->file->getSymbols();
   os->info = in.symTab->getSymbolIndex(symbols[section->info]);
+
+  // Some group members may be combined or discarded, so we need to compute the
+  // new size. The content will be rewritten in InputSection::copyShtGroup.
+  std::unordered_set<uint32_t> seen;
+  ArrayRef<InputSectionBase *> sections = section->file->getSections();
+  for (const uint32_t &idx : section->getDataAs<uint32_t>().slice(1))
+    if (OutputSection *osec = sections[read32(&idx)]->getOutputSection())
+      seen.insert(osec->sectionIndex);
+  os->size = (1 + seen.size()) * sizeof(uint32_t);
 }
 
 void OutputSection::finalize() {
diff --git a/lld/ELF/ScriptLexer.cpp b/lld/ELF/ScriptLexer.cpp
index 9ac8447eef0e2..4b16974dd1346 100644
--- a/lld/ELF/ScriptLexer.cpp
+++ b/lld/ELF/ScriptLexer.cpp
@@ -146,7 +146,7 @@ StringRef ScriptLexer::skipSpace(StringRef s) {
     if (s.startswith("/*")) {
       size_t e = s.find("*/", 2);
       if (e == StringRef::npos) {
-        error("unclosed comment in a linker script");
+        setError("unclosed comment in a linker script");
         return "";
       }
       s = s.substr(e + 2);
diff --git a/lld/ELF/ScriptLexer.h b/lld/ELF/ScriptLexer.h
index 306d428e98fe9..526268e3f65b6 100644
--- a/lld/ELF/ScriptLexer.h
+++ b/lld/ELF/ScriptLexer.h
@@ -24,7 +24,7 @@ class ScriptLexer {
 
   void setError(const Twine &msg);
   void tokenize(MemoryBufferRef mb);
-  static StringRef skipSpace(StringRef s);
+  StringRef skipSpace(StringRef s);
   bool atEOF();
   StringRef next();
   StringRef peek();
diff --git a/lld/ELF/Target.h b/lld/ELF/Target.h
index 47905ae64a47d..e53ac4d066272 100644
--- a/lld/ELF/Target.h
+++ b/lld/ELF/Target.h
@@ -213,6 +213,11 @@ unsigned getPPC64GlobalEntryToLocalEntryOffset(uint8_t stOther);
 // the .toc section.
 bool isPPC64SmallCodeModelTocReloc(RelType type);
 
+// Write a prefixed instruction, which is a 4-byte prefix followed by a 4-byte
+// instruction (regardless of endianness). Therefore, the prefix is always in
+// lower memory than the instruction.
+void writePrefixedInstruction(uint8_t *loc, uint64_t insn);
+
 void addPPC64SaveRestore();
 uint64_t getPPC64TocBase();
 uint64_t getAArch64Page(uint64_t expr);
diff --git a/lld/ELF/Thunks.cpp b/lld/ELF/Thunks.cpp
index ea74d343ebb26..091a291b26cd4 100644
--- a/lld/ELF/Thunks.cpp
+++ b/lld/ELF/Thunks.cpp
@@ -293,6 +293,18 @@ class PPC64R2SaveStub final : public Thunk {
   void addSymbols(ThunkSection &isec) override;
 };
 
+// PPC64 R12 Setup Stub
+// When a caller that does not maintain a toc-pointer performs a local call to
+// a callee which requires a toc-pointer then we need this stub to place the
+// callee's global entry point into r12 without a save of R2.
+class PPC64R12SetupStub final : public Thunk {
+public:
+  PPC64R12SetupStub(Symbol &dest) : Thunk(dest, 0) {}
+  uint32_t size() override { return 16; }
+  void writeTo(uint8_t *buf) override;
+  void addSymbols(ThunkSection &isec) override;
+};
+
 // A bl instruction uses a signed 24 bit offset, with an implicit 4 byte
 // alignment. This gives a possible 26 bits of 'reach'. If the call offset is
 // larger then that we need to emit a long-branch thunk. The target address
@@ -851,6 +863,23 @@ void PPC64R2SaveStub::addSymbols(ThunkSection &isec) {
   s->needsTocRestore = true;
 }
 
+void PPC64R12SetupStub::writeTo(uint8_t *buf) {
+  int64_t offset = destination.getVA() - getThunkTargetSym()->getVA();
+  if (!isInt<34>(offset))
+    fatal("offset must fit in 34 bits to encode in the instruction");
+  uint64_t paddi = PADDI_R12_NO_DISP | (((offset >> 16) & 0x3ffff) << 32) |
+                   (offset & 0xffff);
+
+  writePrefixedInstruction(buf + 0, paddi); // paddi r12, 0, func@pcrel, 1
+  write32(buf + 8, MTCTR_R12);              // mtctr r12
+  write32(buf + 12, BCTR);                  // bctr
+}
+
+void PPC64R12SetupStub::addSymbols(ThunkSection &isec) {
+  addSymbol(saver.save("__gep_setup_" + destination.getName()), STT_FUNC, 0,
+            isec);
+}
+
 void PPC64LongBranchThunk::writeTo(uint8_t *buf) {
   int64_t offset = in.ppc64LongBranchTarget->getEntryVA(&destination, addend) -
                    getPPC64TocBase();
@@ -974,7 +1003,8 @@ static Thunk *addThunkPPC32(const InputSection &isec, const Relocation &rel,
 }
 
 static Thunk *addThunkPPC64(RelType type, Symbol &s, int64_t a) {
-  assert((type == R_PPC64_REL14 || type == R_PPC64_REL24) &&
+  assert((type == R_PPC64_REL14 || type == R_PPC64_REL24 ||
+          type == R_PPC64_REL24_NOTOC) &&
          "unexpected relocation type for thunk");
   if (s.isInPlt())
     return make<PPC64PltCallStub>(s);
@@ -984,6 +1014,9 @@ static Thunk *addThunkPPC64(RelType type, Symbol &s, int64_t a) {
   if ((s.stOther >> 5) == 1)
     return make<PPC64R2SaveStub>(s);
 
+  if (type == R_PPC64_REL24_NOTOC && (s.stOther >> 5) > 1)
+    return make<PPC64R12SetupStub>(s);
+
   if (config->picThunk)
     return make<PPC64PILongBranchThunk>(s, a);
 
diff --git a/lld/test/CMakeLists.txt b/lld/test/CMakeLists.txt
index 4fbd2534b5a97..753eacf4d255b 100644
--- a/lld/test/CMakeLists.txt
+++ b/lld/test/CMakeLists.txt
@@ -4,17 +4,8 @@ set(LLVM_BUILD_MODE "%(build_mode)s")
 set(LLVM_TOOLS_DIR "${LLVM_TOOLS_BINARY_DIR}/%(build_config)s")
 set(LLVM_LIBS_DIR "${LLVM_BINARY_DIR}/lib${LLVM_LIBDIR_SUFFIX}/%(build_config)s")
 
-if(LLD_BUILT_STANDALONE)
-  # Set HAVE_LIBZ according to recorded LLVM_ENABLE_ZLIB value. This
-  # value is forced to 0 if zlib was not found, so it is fine to use it
-  # instead of HAVE_LIBZ (not recorded).
-  if(LLVM_ENABLE_ZLIB)
-    set(HAVE_LIBZ 1)
-  endif()
-endif()
-
 llvm_canonicalize_cmake_booleans(
-  HAVE_LIBZ
+  LLVM_ENABLE_ZLIB
   LLVM_LIBXML2_ENABLED
   )
 
@@ -34,7 +25,7 @@ configure_lit_site_cfg(
 set(LLD_TEST_DEPS lld)
 if (NOT LLD_BUILT_STANDALONE)
   list(APPEND LLD_TEST_DEPS
-    FileCheck count llc llvm-ar llvm-as llvm-bcanalyzer llvm-config llvm-cvtres
+    FileCheck count extract llc llvm-ar llvm-as llvm-bcanalyzer llvm-config llvm-cvtres
     llvm-dis llvm-dwarfdump llvm-lib llvm-lipo llvm-mc llvm-nm llvm-objcopy
     llvm-objdump llvm-pdbutil llvm-readelf llvm-readobj llvm-strip not obj2yaml
     opt yaml2obj
diff --git a/lld/test/COFF/Inputs/undefined-symbol-multi-lto.ll b/lld/test/COFF/Inputs/undefined-symbol-multi-lto.ll
new file mode 100644
index 0000000000000..5f6730272e610
--- /dev/null
+++ b/lld/test/COFF/Inputs/undefined-symbol-multi-lto.ll
@@ -0,0 +1,23 @@
+target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-windows-msvc"
+
+define dso_local i32 @"?baz@@YAHXZ"() #0 {
+  %1 = call i32 @"?foo@@YAHXZ"()
+  %2 = call i32 @"?foo@@YAHXZ"()
+  %3 = call i32 @"?bar@@YAHXZ"()
+  %4 = call i32 @"?bar@@YAHXZ"()
+  ret i32 0
+}
+
+declare dso_local i32 @"?foo@@YAHXZ"() #1
+
+declare dso_local i32 @"?bar@@YAHXZ"() #1
+
+attributes #0 = { noinline optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 2}
+!1 = !{!"clang version 11.0.0 (https://github.com/llvm/llvm-project.git bed3e1a99b41f5a9525bc0edf12ecbcf63aab0cf)"}
diff --git a/lld/test/COFF/undefined-symbol-multi.s b/lld/test/COFF/undefined-symbol-multi.s
index 2d3d8285338c5..086d12828b2d6 100644
--- a/lld/test/COFF/undefined-symbol-multi.s
+++ b/lld/test/COFF/undefined-symbol-multi.s
@@ -16,17 +16,20 @@
 # RUN: echo '  call "?foo@@YAHXZ"' >> %t.moreref.s
 # RUN: echo '  call "?foo@@YAHXZ"' >> %t.moreref.s
 # RUN: llvm-mc -triple=x86_64-windows-msvc -filetype=obj -o %t2.obj %t.moreref.s
-# RUN: not lld-link /out:/dev/null  %t.obj %t2.obj 2>&1 | FileCheck %s
+# RUN: llvm-as %S/Inputs/undefined-symbol-multi-lto.ll -o %t3.obj
+# RUN: not lld-link /out:/dev/null  %t.obj %t2.obj %t3.obj 2>&1 | FileCheck %s
 
 # CHECK: error: undefined symbol: int __cdecl foo(void)
 # CHECK-NEXT: >>> referenced by {{.*}}tmp.obj:(main)
 # CHECK-NEXT: >>> referenced by {{.*}}tmp.obj:(main)
 # CHECK-NEXT: >>> referenced by {{.*}}tmp2.obj:(bar)
-# CHECK-NEXT: >>> referenced 9 more times
+# CHECK-NEXT: >>> referenced 10 more times
 # CHECK-EMPTY:
 # CHECK-NEXT: error: undefined symbol: int __cdecl bar(void)
 # CHECK-NEXT: >>> referenced by {{.*}}.obj:(main)
 # CHECK-NEXT: >>> referenced by {{.*}}.obj:(f1)
+# CHECK-NEXT: >>> referenced by {{.*}}undefined-symbol-multi-lto.ll
+# CHECK-NEXT: >>>               {{.*}}tmp3.obj
 
         .section        .text,"xr",one_only,main
 .globl main
diff --git a/lld/test/ELF/arm-exidx-range.s b/lld/test/ELF/arm-exidx-range.s
index 69763705276c1..623c6a81c97b8 100644
--- a/lld/test/ELF/arm-exidx-range.s
+++ b/lld/test/ELF/arm-exidx-range.s
@@ -1,4 +1,4 @@
-// REQUIRES: arm
+// REQUIRES: arm, shell
 // RUN: llvm-mc --arm-add-build-attributes --triple=armv7a-linux-gnueabihf -filetype=obj %s -o %t.o
 // RUN: echo "SECTIONS { \
 // RUN:         . = 0x80000000; \
diff --git a/lld/test/ELF/driver.test b/lld/test/ELF/driver.test
index bec8301bb9a04..cd7edc6478544 100644
--- a/lld/test/ELF/driver.test
+++ b/lld/test/ELF/driver.test
@@ -34,10 +34,6 @@
 # RUN: not ld.lld -r -shared %t -o /dev/null 2>&1 | FileCheck -check-prefix=ERR2 %s
 # ERR2: -r and -shared may not be used together
 
-## Attempt to use -r and --gc-sections together
-# RUN: not ld.lld -r --gc-sections %t -o /dev/null 2>&1 | FileCheck -check-prefix=ERR3 %s
-# ERR3: -r and --gc-sections may not be used together
-
 ## Attempt to use -r and --gdb-index together
 # RUN: not ld.lld -r --gdb-index %t -o /dev/null 2>&1 | FileCheck -check-prefix=ERR4 %s
 # ERR4: -r and --gdb-index may not be used together
diff --git a/lld/test/ELF/linkerscript/diag6.test b/lld/test/ELF/linkerscript/diag6.test
index 276fd1e3cc735..0ec0400040b54 100644
--- a/lld/test/ELF/linkerscript/diag6.test
+++ b/lld/test/ELF/linkerscript/diag6.test
@@ -4,4 +4,4 @@
 
 SECTIONS /*
 
-CHECK: error: unclosed comment in a linker script
+CHECK: error: {{.*}}diag6.test:1: unclosed comment in a linker script
diff --git a/lld/test/ELF/linkerscript/noload.s b/lld/test/ELF/linkerscript/noload.s
index 2f52b465854e2..c2014722985d3 100644
--- a/lld/test/ELF/linkerscript/noload.s
+++ b/lld/test/ELF/linkerscript/noload.s
@@ -1,11 +1,7 @@
 # REQUIRES: x86
-# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.o
-# RUN: echo "SECTIONS { \
-# RUN:        .data_noload_a (NOLOAD) : { *(.data_noload_a) } \
-# RUN:        .data_noload_b (0x10000) (NOLOAD) : { *(.data_noload_b) } \
-# RUN:        .no_input_sec_noload (NOLOAD) : { . += 1; } \
-# RUN:        .text (0x20000) : { *(.text) } };" > %t.script
-# RUN: ld.lld -o %t --script %t.script %t.o
+# RUN: extract asm %s -o %t.s && extract lds %s -o %t.lds
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %t.s -o %t.o
+# RUN: ld.lld -o %t --script %t.lds %t.o
 # RUN: llvm-readelf -S -l %t | FileCheck %s
 
 # CHECK:      Name                 Type   Address          Off               Size
@@ -16,6 +12,7 @@
 # CHECK:      Type Offset   VirtAddr           PhysAddr
 # CHECK-NEXT: LOAD 0x001000 0x0000000000020000 0x0000000000020000
 
+#--- asm
 .section .text,"ax",@progbits
   nop
 
@@ -24,3 +21,11 @@
 
 .section .data_noload_b,"aw",@progbits
 .zero 4096
+
+#--- lds
+SECTIONS {
+  .data_noload_a (NOLOAD) : { *(.data_noload_a) }
+  .data_noload_b (0x10000) (NOLOAD) : { *(.data_noload_b) }
+  .no_input_sec_noload (NOLOAD) : { . += 1; }
+  .text (0x20000) : { *(.text) }
+}
diff --git a/lld/test/ELF/mips-merge-abiflags.s b/lld/test/ELF/mips-merge-abiflags.s
index e7abbe79421bd..15fe44276f9ca 100644
--- a/lld/test/ELF/mips-merge-abiflags.s
+++ b/lld/test/ELF/mips-merge-abiflags.s
@@ -5,7 +5,7 @@
 # RUN: llvm-mc -filetype=obj -triple=mips64-unknown-freebsd %s -o %t.o
 # RUN: ld.lld %t.o %p/Inputs/mips-concatenated-abiflags.o -o %t.exe
 # RUN: llvm-readobj --sections -A %t.exe | FileCheck %s
-# RUN: llvm-readobj --sections -A %p/Inputs/mips-concatenated-abiflags.o \
+# RUN: llvm-readobj --sections -A %p/Inputs/mips-concatenated-abiflags.o 2>&1 \
 # RUN:   | FileCheck --check-prefix=INPUT-OBJECT %s
 
         .globl  __start
@@ -59,4 +59,4 @@ __start:
 # INPUT-OBJECT-NEXT:    AddressAlignment: 8
 # INPUT-OBJECT-NEXT:    EntrySize: 0
 # INPUT-OBJECT-NEXT:  }
-# INPUT-OBJECT:       The .MIPS.abiflags section has a wrong size.
+# INPUT-OBJECT:       unable to read the .MIPS.abiflags section: it has a wrong size (48)
diff --git a/lld/test/ELF/ppc64-pcrel-call-to-toc.s b/lld/test/ELF/ppc64-pcrel-call-to-toc.s
new file mode 100644
index 0000000000000..d85b559a9cf3d
--- /dev/null
+++ b/lld/test/ELF/ppc64-pcrel-call-to-toc.s
@@ -0,0 +1,67 @@
+# REQUIRES: ppc
+# RUN: echo 'SECTIONS { \
+# RUN:   .text_func   0x10010000 : { *(.text_func) } \
+# RUN:   .text_callee 0x10020000 : { *(.text_callee) } \
+# RUN:   .text_caller 0x10030000 : { *(.text_caller) } \
+# RUN:   }' > %t.script
+
+# RUN: llvm-mc -filetype=obj -triple=powerpc64le %s -o %t.o
+# RUN: ld.lld -T %t.script %t.o -o %t
+# RUN: llvm-readelf -s %t | FileCheck %s --check-prefix=SYMBOL
+# RUN: llvm-objdump -d --no-show-raw-insn --mcpu=pwr10 %t | FileCheck %s
+
+# RUN: llvm-mc -filetype=obj -triple=powerpc64 %s -o %t.o
+# RUN: ld.lld -T %t.script %t.o -o %t
+# RUN: llvm-readelf -s %t | FileCheck %s --check-prefix=SYMBOL
+# RUN: llvm-objdump -d --no-show-raw-insn --mcpu=pwr10 %t | FileCheck %s
+
+## When a function without TOC accesses a function using TOC, an r12 setup stub
+## is inserted
+
+# SYMBOL:      1: 0000000010020000 0 NOTYPE LOCAL DEFAULT [<other: 0x60>] 2 callee
+# SYMBOL-NEXT: 2: 0000000010030000 0 NOTYPE LOCAL DEFAULT [<other: 0x20>] 3 caller
+# SYMBOL-NEXT: 3: 0000000010010000 0 NOTYPE LOCAL DEFAULT 1 func
+# SYMBOL:      6: 000000001003000c 16 FUNC LOCAL DEFAULT 3 __gep_setup_callee
+
+# CHECK-LABEL: <func>:
+# CHECK-NEXT:  blr
+
+# CHECK-LABEL: <callee>:
+# CHECK:       bl 0x10010000
+# CHECK-NEXT:  addis 4, 2, -1
+# CHECK-NEXT:  lwz 4, 32744(4)
+# CHECK-NEXT:  blr
+
+# CHECK-LABEL: <caller>:
+# CHECK-NEXT:  bl 0x1003000c
+# CHECK-NEXT:  blr
+
+# CHECK-LABEL: <__gep_setup_callee>:
+# CHECK-NEXT:  paddi 12, 0, -65548, 1
+# CHECK-NEXT:  mtctr 12
+# CHECK-NEXT:  bctr
+
+.section .text_func, "ax", %progbits
+func:
+  blr
+
+.section .text_callee, "ax", %progbits
+callee:
+.Lfunc_gep1:
+  addis 2, 12, .TOC.-.Lfunc_gep1@ha
+  addi 2, 2, .TOC.-.Lfunc_gep1@l
+.Lfunc_lep1:
+  .localentry callee, .Lfunc_lep1-.Lfunc_gep1
+  bl func
+  addis 4, 2, global@toc@ha
+  lwz 4, global@toc@l(4)
+  blr
+
+.section .text_caller, "ax", %progbits
+caller:
+  .localentry caller, 1
+  bl callee@notoc
+  blr
+global:
+  .long	0
+  .size	global, 4
diff --git a/lld/test/ELF/relocatable-comdat.s b/lld/test/ELF/relocatable-comdat.s
index 213d14bd2cc6f..b290bae4ee328 100644
--- a/lld/test/ELF/relocatable-comdat.s
+++ b/lld/test/ELF/relocatable-comdat.s
@@ -1,45 +1,52 @@
 # REQUIRES: x86
+## Test that SHT_GROUP sections are retained in relocatable output. The content
+## may be rewritten because group members may change their indices. Additionally,
+## group member may be combined or discarded (e.g. /DISCARD/ or --gc-sections).
+
 # RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.o
-# RUN: ld.lld -r %t.o %t.o -o %t
-# RUN: llvm-readobj --elf-section-groups --sections %t | FileCheck %s
-
-# CHECK:        Name: .text.bar
-# CHECK-NEXT:   Type: SHT_PROGBITS
-# CHECK-NEXT:   Flags [
-# CHECK-NEXT:     SHF_ALLOC
-# CHECK-NEXT:     SHF_EXECINSTR
-# CHECK-NEXT:     SHF_GROUP
-# CHECK-NEXT:   ]
-# CHECK-NEXT:   Address:
-# CHECK-NEXT:   Offset:
-# CHECK-NEXT:   Size: 8
-# CHECK:      Section {
-# CHECK-NEXT:   Index: 4
-# CHECK-NEXT:   Name: .text.foo
-# CHECK-NEXT:   Type: SHT_PROGBITS
-# CHECK-NEXT:   Flags [
-# CHECK-NEXT:     SHF_ALLOC
-# CHECK-NEXT:     SHF_EXECINSTR
-# CHECK-NEXT:     SHF_GROUP
-# CHECK-NEXT:   ]
-# CHECK-NEXT:   Address:
-# CHECK-NEXT:   Offset:
-# CHECK-NEXT:   Size: 4
-
-# CHECK:       Groups {
-# CHECK-NEXT:    Group {
-# CHECK-NEXT:      Name: .group
-# CHECK-NEXT:      Index: 2
-# CHECK-NEXT:      Link: 6
-# CHECK-NEXT:      Info: 1 
-# CHECK-NEXT:      Type: COMDAT
-# CHECK-NEXT:      Signature: abc
-# CHECK-NEXT:      Section(s) in group [
-# CHECK-NEXT:        .text.bar
-# CHECK-NEXT:        .text.foo
-# CHECK-NEXT:      ]
-# CHECK-NEXT:    }
-# CHECK-NEXT:  }
+# RUN: ld.lld -r %t.o %t.o -o %t.ro
+# RUN: llvm-readelf -g -S %t.ro | FileCheck %s
+
+# CHECK: Name    Type  Address          Off    Size   ES Flg Lk        Inf Al
+# CHECK: .group  GROUP 0000000000000000 {{.*}} 000014 04     {{[1-9]}}   1  4
+
+# CHECK: COMDAT group section [{{.*}}] `.group' [abc] contains 4 sections:
+# CHECK-NEXT: Name
+# CHECK-NEXT: .rodata.bar
+# CHECK-NEXT: .rodata.foo
+# CHECK-NEXT: .text.bar
+# CHECK-NEXT: .text.foo
+
+## Rewrite SHT_GROUP content if some members are combined.
+# RUN: echo 'SECTIONS { .rodata : {*(.rodata.*)} .text : {*(.text.*)} }' > %t1.lds
+# RUN: ld.lld -r -T %t1.lds %t.o %t.o -o %t1.ro
+# RUN: llvm-readelf -g -S %t1.ro | FileCheck %s --check-prefix=SCRIPT1
+
+# SCRIPT1: Name    Type  Address          Off    Size   ES Flg Lk        Inf Al
+# SCRIPT1: .group  GROUP 0000000000000000 {{.*}} 00000c 04     {{[1-9]}}   1  4
+
+# SCRIPT1: COMDAT group section [{{.*}}] `.group' [abc] contains 2 sections:
+# SCRIPT1-NEXT: Name
+# SCRIPT1-NEXT: .rodata
+# SCRIPT1-NEXT: .text
+
+# RUN: echo 'SECTIONS { /DISCARD/ : {*(.rodata.*)} }' > %t2.lds
+# RUN: ld.lld -r -T %t2.lds %t.o %t.o -o %t2.ro
+# RUN: llvm-readelf -g -S %t2.ro | FileCheck %s --check-prefix=SCRIPT2
+
+## Handle discarded group members.
+# SCRIPT2: [Nr] Name    Type  Address          Off    Size   ES Flg Lk        Inf Al
+# SCRIPT2: [ 2] .group  GROUP 0000000000000000 {{.*}} 00000c 04     {{[1-9]}}   1  4
+
+# SCRIPT2: COMDAT group section [{{.*}}] `.group' [abc] contains 2 sections:
+# SCRIPT2-NEXT: Name
+# SCRIPT2-NEXT: .text.bar
+# SCRIPT2-NEXT: .text.foo
+
+.section .rodata.bar,"aG",@progbits,abc,comdat
+.byte 42
+.section .rodata.foo,"aG",@progbits,abc,comdat
+.byte 42
 
 .section .text.bar,"axG",@progbits,abc,comdat
 .quad 42
diff --git a/lld/test/ELF/relocatable-gc.s b/lld/test/ELF/relocatable-gc.s
new file mode 100644
index 0000000000000..caa23314f37aa
--- /dev/null
+++ b/lld/test/ELF/relocatable-gc.s
@@ -0,0 +1,80 @@
+# REQUIRES: x86
+## Test -r --gc-sections.
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o
+
+## By default all regular sections are discarded. We currently don't track
+## usage of group signature symbols and will retain them and their associated
+## STT_SECTION symbols.
+# RUN: ld.lld -r --gc-sections --print-gc-sections %t.o -o %t
+# RUN: llvm-readelf -S -s %t | FileCheck %s
+
+# CHECK:      [ 1] .group
+# CHECK-NEXT: [ 2] .note.GNU-stack
+
+# CHECK:      Symbol table '.symtab' contains 3 entries:
+# CHECK-NEXT: Num:
+# CHECK-NEXT: 0:
+# CHECK-NEXT: 1: {{.*}} NOTYPE  LOCAL DEFAULT  1 group
+# CHECK-NEXT: 2: {{.*}} SECTION LOCAL DEFAULT  1
+
+## -u keeps .text.bar alive. Other group members are kept alive as well.
+# RUN: ld.lld -r --gc-sections -u bar %t.o -o - | llvm-readelf -Ss - | \
+# RUN:   FileCheck %s --check-prefix=KEEP_GROUP
+## -e, --init and --fini are similar.
+# RUN: ld.lld -r --gc-sections -e bar %t.o -o - | llvm-readelf -Ss - | \
+# RUN:   FileCheck %s --check-prefix=KEEP_GROUP
+# RUN: ld.lld -r --gc-sections --init=bar %t.o -o - | llvm-readelf -Ss - | \
+# RUN:   FileCheck %s --check-prefix=KEEP_GROUP
+# RUN: ld.lld -r --gc-sections --fini=bar %t.o -o - | llvm-readelf -Ss - | \
+# RUN:   FileCheck %s --check-prefix=KEEP_GROUP
+
+# KEEP_GROUP:      [ 1] .group
+# KEEP_GROUP-NEXT: [ 2] .text.bar
+# KEEP_GROUP-NEXT: [ 3] .text.foo
+# KEEP_GROUP-NEXT: [ 4] .note.GNU-stack
+
+# KEEP_GROUP:      Symbol table '.symtab' contains 7 entries:
+# KEEP_GROUP:      4: {{.*}} SECTION
+# KEEP_GROUP-NEXT: 5: {{.*}}   2 bar
+# KEEP_GROUP-NEXT: 6: {{.*}}   3 foo
+
+## If .text is retained, its referenced qux and .fred are retained as well.
+## fred_und is used (by .fred) and thus emitted.
+## Note, GNU ld does not retain qux.
+# RUN: ld.lld -r --gc-sections -e _start %t.o -o %tstart.ro
+# RUN: llvm-readelf -Ss %tstart.ro | FileCheck %s --check-prefix=KEEP_START
+
+# KEEP_START:      [ 1] .text
+# KEEP_START-NEXT: [ 2] .rela.text
+# KEEP_START-NEXT: [ 3] qux
+# KEEP_START-NEXT: [ 4] .group
+# KEEP_START-NEXT: [ 5] .fred
+# KEEP_START-NEXT: [ 6] .rela.fred
+# KEEP_START-NEXT: [ 7] .note.GNU-stack
+
+# KEEP_START:      Symbol table '.symtab' contains 10 entries:
+# KEEP_START:      5: {{.*}} SECTION
+# KEEP_START-NEXT: 6: {{.*}} UND __start_qux
+# KEEP_START-NEXT: 7: {{.*}}   1 _start
+# KEEP_START-NEXT: 8: {{.*}}   5 fred
+# KEEP_START-NEXT: 9: {{.*}} UND fred_und
+
+.section qux,"a",@progbits
+  .byte 0
+
+.text
+.globl _start, bar, foo, fred
+_start:
+  call fred
+  .quad __start_qux
+
+.section .text.bar,"axG",@progbits,group,comdat
+bar:
+  .byte 1
+.section .text.foo,"axG",@progbits,group,comdat
+foo:
+  .byte 2
+.section .fred,"ax",@progbits
+fred:
+  call fred_und
diff --git a/lld/test/lit.cfg.py b/lld/test/lit.cfg.py
index 267f8c5178584..0fa9b48c3c792 100644
--- a/lld/test/lit.cfg.py
+++ b/lld/test/lit.cfg.py
@@ -39,9 +39,9 @@
 llvm_config.use_lld()
 
 tool_patterns = [
-    'llc', 'llvm-as', 'llvm-mc', 'llvm-nm', 'llvm-objdump', 'llvm-pdbutil',
-    'llvm-dwarfdump', 'llvm-readelf', 'llvm-readobj', 'obj2yaml', 'yaml2obj',
-    'opt', 'llvm-dis']
+    'extract', 'llc', 'llvm-as', 'llvm-mc', 'llvm-nm', 'llvm-objdump',
+    'llvm-pdbutil', 'llvm-dwarfdump', 'llvm-readelf', 'llvm-readobj',
+    'obj2yaml', 'yaml2obj', 'opt', 'llvm-dis']
 
 llvm_config.add_tool_substitutions(tool_patterns)
 
@@ -87,7 +87,7 @@
 # Indirectly check if the mt.exe Microsoft utility exists by searching for
 # cvtres, which always accompanies it.  Alternatively, check if we can use
 # libxml2 to merge manifests.
-if (lit.util.which('cvtres', config.environment['PATH']) or 
+if (lit.util.which('cvtres', config.environment['PATH']) or
         config.llvm_libxml2_enabled):
     config.available_features.add('manifest_tool')
 
diff --git a/lld/test/lit.site.cfg.py.in b/lld/test/lit.site.cfg.py.in
index 4aa2fcda73bb4..3d4c51f4ab647 100644
--- a/lld/test/lit.site.cfg.py.in
+++ b/lld/test/lit.site.cfg.py.in
@@ -14,7 +14,7 @@ config.lld_libs_dir = "@LLVM_LIBRARY_OUTPUT_INTDIR@"
 config.lld_tools_dir = "@LLVM_RUNTIME_OUTPUT_INTDIR@"
 config.target_triple = "@TARGET_TRIPLE@"
 config.python_executable = "@Python3_EXECUTABLE@"
-config.have_zlib = @HAVE_LIBZ@
+config.have_zlib = @LLVM_ENABLE_ZLIB@
 config.sizeof_void_p = @CMAKE_SIZEOF_VOID_P@
 
 # Support substitution of the tools and libs dirs with user parameters. This is
diff --git a/lld/unittests/CMakeLists.txt b/lld/unittests/CMakeLists.txt
index 84d35d43f4e87..88cb85a084017 100644
--- a/lld/unittests/CMakeLists.txt
+++ b/lld/unittests/CMakeLists.txt
@@ -12,5 +12,9 @@ function(add_lld_unittest test_dirname)
   target_link_libraries(${test_dirname} ${LLVM_COMMON_LIBS})
 endfunction()
 
+if (CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
+  add_compile_options("-Wno-suggest-override")
+endif()
+
 add_subdirectory(DriverTests)
 add_subdirectory(MachOTests)
diff --git a/lldb/CMakeLists.txt b/lldb/CMakeLists.txt
index b1c0597cf3b3f..14cfba1f64ff5 100644
--- a/lldb/CMakeLists.txt
+++ b/lldb/CMakeLists.txt
@@ -1,7 +1,4 @@
-cmake_minimum_required(VERSION 3.4.3)
-if(CMAKE_SYSTEM_NAME STREQUAL Windows)
-  cmake_minimum_required(VERSION 3.13)
-endif()
+cmake_minimum_required(VERSION 3.13.4)
 
 if(POLICY CMP0075)
   cmake_policy(SET CMP0075 NEW)
diff --git a/lldb/bindings/python/python-wrapper.swig b/lldb/bindings/python/python-wrapper.swig
index f9e89373fe252..516590ed57713 100644
--- a/lldb/bindings/python/python-wrapper.swig
+++ b/lldb/bindings/python/python-wrapper.swig
@@ -271,9 +271,6 @@ LLDBSwigPythonCreateScriptedThreadPlan
     if (python_class_name == NULL || python_class_name[0] == '\0' || !session_dictionary_name)
         Py_RETURN_NONE;
 
-    // I do not want the SBThreadPlan to be deallocated when going out of scope because python
-    // has ownership of it and will manage memory for this object by itself
-    lldb::SBThreadPlan *tp_value = new lldb::SBThreadPlan(thread_plan_sp);
 
     PyErr_Cleaner py_err_cleaner(true);
 
@@ -286,7 +283,10 @@ LLDBSwigPythonCreateScriptedThreadPlan
         return nullptr;
     }
 
-    PythonObject tp_arg(PyRefType::Owned, SBTypeToSWIGWrapper(tp_value));
+    // I do not want the SBThreadPlan to be deallocated when going out of scope
+    // because python has ownership of it and will manage memory for this
+    // object by itself
+    PythonObject tp_arg(PyRefType::Owned, SBTypeToSWIGWrapper(new lldb::SBThreadPlan(thread_plan_sp)));
 
     if (!tp_arg.IsAllocated())
         Py_RETURN_NONE;
@@ -312,8 +312,7 @@ LLDBSwigPythonCreateScriptedThreadPlan
         }
         result = pfunc(tp_arg, dict);
     } else if (arg_info.get().max_positional_args >= 3) {
-        lldb::SBStructuredData *args_value = new lldb::SBStructuredData(args_impl);
-        PythonObject args_arg(PyRefType::Owned, SBTypeToSWIGWrapper(args_value));
+        PythonObject args_arg(PyRefType::Owned, SBTypeToSWIGWrapper(new lldb::SBStructuredData(args_impl)));
         result = pfunc(tp_arg, args_arg, dict);
     } else {
         error_string.assign("wrong number of arguments in __init__, should be 2 or 3 (not including self)");
diff --git a/lldb/include/lldb/API/SBThreadPlan.h b/lldb/include/lldb/API/SBThreadPlan.h
index 8f16f4f5c4d2d..6639c10e437b3 100644
--- a/lldb/include/lldb/API/SBThreadPlan.h
+++ b/lldb/include/lldb/API/SBThreadPlan.h
@@ -117,10 +117,11 @@ class LLDB_API SBThreadPlan {
   friend class lldb_private::QueueImpl;
   friend class SBQueueItem;
 
-  lldb_private::ThreadPlan *get();
+  lldb::ThreadPlanSP GetSP() const { return m_opaque_wp.lock(); }
+  lldb_private::ThreadPlan *get() const { return GetSP().get(); }
   void SetThreadPlan(const lldb::ThreadPlanSP &lldb_object_sp);
 
-  lldb::ThreadPlanSP m_opaque_sp;
+  lldb::ThreadPlanWP m_opaque_wp;
 };
 
 } // namespace lldb
diff --git a/lldb/include/lldb/DataFormatters/DataVisualization.h b/lldb/include/lldb/DataFormatters/DataVisualization.h
index b053aa074d9e6..7be07d65acdde 100644
--- a/lldb/include/lldb/DataFormatters/DataVisualization.h
+++ b/lldb/include/lldb/DataFormatters/DataVisualization.h
@@ -69,9 +69,9 @@ class DataVisualization {
 
     static void Clear();
 
-    static void
-    ForEach(std::function<bool(ConstString, const lldb::TypeSummaryImplSP &)>
-                callback);
+    static void ForEach(std::function<bool(const TypeMatcher &,
+                                           const lldb::TypeSummaryImplSP &)>
+                            callback);
 
     static uint32_t GetCount();
   };
diff --git a/lldb/include/lldb/DataFormatters/FormatManager.h b/lldb/include/lldb/DataFormatters/FormatManager.h
index 56a0303f9b027..978ad148d6c49 100644
--- a/lldb/include/lldb/DataFormatters/FormatManager.h
+++ b/lldb/include/lldb/DataFormatters/FormatManager.h
@@ -34,7 +34,7 @@ namespace lldb_private {
 // this file's objects directly
 
 class FormatManager : public IFormatChangeListener {
-  typedef FormatMap<ConstString, TypeSummaryImpl> NamedSummariesMap;
+  typedef FormattersContainer<TypeSummaryImpl> NamedSummariesMap;
   typedef TypeCategoryMap::MapType::iterator CategoryMapIterator;
 
 public:
@@ -144,13 +144,6 @@ class FormatManager : public IFormatChangeListener {
 
   static const char *GetFormatAsCString(lldb::Format format);
 
-  // if the user tries to add formatters for, say, "struct Foo" those will not
-  // match any type because of the way we strip qualifiers from typenames this
-  // method looks for the case where the user is adding a
-  // "class","struct","enum" or "union" Foo and strips the unnecessary
-  // qualifier
-  static ConstString GetValidTypeName(ConstString type);
-
   // when DataExtractor dumps a vectorOfT, it uses a predefined format for each
   // item this method returns it, or eFormatInvalid if vector_format is not a
   // vectorOf
diff --git a/lldb/include/lldb/DataFormatters/FormattersContainer.h b/lldb/include/lldb/DataFormatters/FormattersContainer.h
index d414882bae194..a30ad920ef45a 100644
--- a/lldb/include/lldb/DataFormatters/FormattersContainer.h
+++ b/lldb/include/lldb/DataFormatters/FormattersContainer.h
@@ -37,57 +37,103 @@ class IFormatChangeListener {
   virtual uint32_t GetCurrentRevision() = 0;
 };
 
-// if the user tries to add formatters for, say, "struct Foo" those will not
-// match any type because of the way we strip qualifiers from typenames this
-// method looks for the case where the user is adding a "class","struct","enum"
-// or "union" Foo and strips the unnecessary qualifier
-static inline ConstString GetValidTypeName_Impl(ConstString type) {
-  if (type.IsEmpty())
-    return type;
+/// Class for matching type names.
+class TypeMatcher {
+  RegularExpression m_type_name_regex;
+  ConstString m_type_name;
+  /// False if m_type_name_regex should be used for matching. False if this is
+  /// just matching by comparing with m_type_name string.
+  bool m_is_regex;
 
-  std::string type_cstr(type.AsCString());
-  StringLexer type_lexer(type_cstr);
+  // if the user tries to add formatters for, say, "struct Foo" those will not
+  // match any type because of the way we strip qualifiers from typenames this
+  // method looks for the case where the user is adding a
+  // "class","struct","enum" or "union" Foo and strips the unnecessary qualifier
+  static ConstString StripTypeName(ConstString type) {
+    if (type.IsEmpty())
+      return type;
 
-  type_lexer.AdvanceIf("class ");
-  type_lexer.AdvanceIf("enum ");
-  type_lexer.AdvanceIf("struct ");
-  type_lexer.AdvanceIf("union ");
+    std::string type_cstr(type.AsCString());
+    StringLexer type_lexer(type_cstr);
 
-  while (type_lexer.NextIf({' ', '\t', '\v', '\f'}).first)
-    ;
+    type_lexer.AdvanceIf("class ");
+    type_lexer.AdvanceIf("enum ");
+    type_lexer.AdvanceIf("struct ");
+    type_lexer.AdvanceIf("union ");
 
-  return ConstString(type_lexer.GetUnlexed());
-}
+    while (type_lexer.NextIf({' ', '\t', '\v', '\f'}).first)
+      ;
 
-template <typename KeyType, typename ValueType> class FormattersContainer;
+    return ConstString(type_lexer.GetUnlexed());
+  }
+
+public:
+  TypeMatcher() = delete;
+  /// Creates a matcher that accepts any type with exactly the given type name.
+  TypeMatcher(ConstString type_name)
+      : m_type_name(type_name), m_is_regex(false) {}
+  /// Creates a matcher that accepts any type matching the given regex.
+  TypeMatcher(RegularExpression regex)
+      : m_type_name_regex(regex), m_is_regex(true) {}
+
+  /// True iff this matches the given type name.
+  bool Matches(ConstString type_name) const {
+    if (m_is_regex)
+      return m_type_name_regex.Execute(type_name.GetStringRef());
+    return m_type_name == type_name ||
+           StripTypeName(m_type_name) == StripTypeName(type_name);
+  }
+
+  /// Returns the underlying match string for this TypeMatcher.
+  ConstString GetMatchString() const {
+    if (m_is_regex)
+      return ConstString(m_type_name_regex.GetText());
+    return StripTypeName(m_type_name);
+  }
+
+  /// Returns true if this TypeMatcher and the given one were most created by
+  /// the same match string.
+  /// The main purpose of this function is to find existing TypeMatcher
+  /// instances by the user input that created them. This is necessary as LLDB
+  /// allows referencing existing TypeMatchers in commands by the user input
+  /// that originally created them:
+  /// (lldb) type summary add --summary-string \"A\" -x TypeName
+  /// (lldb) type summary delete TypeName
+  bool CreatedBySameMatchString(TypeMatcher other) const {
+    return GetMatchString() == other.GetMatchString();
+  }
+};
 
-template <typename KeyType, typename ValueType> class FormatMap {
+template <typename ValueType> class FormattersContainer {
 public:
-  typedef typename ValueType::SharedPointer ValueSP;
-  typedef std::vector<std::pair<KeyType, ValueSP>> MapType;
-  typedef typename MapType::iterator MapIterator;
-  typedef std::function<bool(const KeyType &, const ValueSP &)> ForEachCallback;
+  typedef typename std::shared_ptr<ValueType> ValueSP;
+  typedef std::vector<std::pair<TypeMatcher, ValueSP>> MapType;
+  typedef std::function<bool(const TypeMatcher &, const ValueSP &)>
+      ForEachCallback;
+  typedef typename std::shared_ptr<FormattersContainer<ValueType>>
+      SharedPointer;
+
+  friend class TypeCategoryImpl;
 
-  FormatMap(IFormatChangeListener *lst)
-      : m_map(), m_map_mutex(), listener(lst) {}
+  FormattersContainer(IFormatChangeListener *lst) : listener(lst) {}
 
-  void Add(KeyType name, const ValueSP &entry) {
+  void Add(TypeMatcher matcher, const ValueSP &entry) {
     if (listener)
       entry->GetRevision() = listener->GetCurrentRevision();
     else
       entry->GetRevision() = 0;
 
     std::lock_guard<std::recursive_mutex> guard(m_map_mutex);
-    Delete(name);
-    m_map.emplace_back(std::move(name), std::move(entry));
+    Delete(matcher);
+    m_map.emplace_back(std::move(matcher), std::move(entry));
     if (listener)
       listener->Changed();
   }
 
-  bool Delete(const KeyType &name) {
+  bool Delete(TypeMatcher matcher) {
     std::lock_guard<std::recursive_mutex> guard(m_map_mutex);
-    for (MapIterator iter = m_map.begin(); iter != m_map.end(); ++iter)
-      if (iter->first == name) {
+    for (auto iter = m_map.begin(); iter != m_map.end(); ++iter)
+      if (iter->first.CreatedBySameMatchString(matcher)) {
         m_map.erase(iter);
         if (listener)
           listener->Changed();
@@ -96,217 +142,71 @@ template <typename KeyType, typename ValueType> class FormatMap {
     return false;
   }
 
-  void Clear() {
+  bool Get(ConstString type, ValueSP &entry) {
     std::lock_guard<std::recursive_mutex> guard(m_map_mutex);
-    m_map.clear();
-    if (listener)
-      listener->Changed();
+    for (auto &formatter : llvm::reverse(m_map)) {
+      if (formatter.first.Matches(type)) {
+        entry = formatter.second;
+        return true;
+      }
+    }
+    return false;
   }
 
-  bool Get(const KeyType &name, ValueSP &entry) {
+  bool GetExact(TypeMatcher matcher, ValueSP &entry) {
     std::lock_guard<std::recursive_mutex> guard(m_map_mutex);
     for (const auto &pos : m_map)
-      if (pos.first == name) {
+      if (pos.first.CreatedBySameMatchString(matcher)) {
         entry = pos.second;
         return true;
       }
     return false;
   }
 
-  void ForEach(ForEachCallback callback) {
-    if (callback) {
-      std::lock_guard<std::recursive_mutex> guard(m_map_mutex);
-      for (const auto &pos : m_map) {
-        const KeyType &type = pos.first;
-        if (!callback(type, pos.second))
-          break;
-      }
-    }
-  }
-
-  uint32_t GetCount() { return m_map.size(); }
-
-  ValueSP GetValueAtIndex(size_t index) {
+  ValueSP GetAtIndex(size_t index) {
     std::lock_guard<std::recursive_mutex> guard(m_map_mutex);
     if (index >= m_map.size())
       return ValueSP();
     return m_map[index].second;
   }
 
-  // If caller holds the mutex we could return a reference without copy ctor.
-  KeyType GetKeyAtIndex(size_t index) {
+  lldb::TypeNameSpecifierImplSP GetTypeNameSpecifierAtIndex(size_t index) {
     std::lock_guard<std::recursive_mutex> guard(m_map_mutex);
     if (index >= m_map.size())
-      return {};
-    return m_map[index].first;
-  }
-
-protected:
-  MapType m_map;
-  std::recursive_mutex m_map_mutex;
-  IFormatChangeListener *listener;
-
-  MapType &map() { return m_map; }
-
-  std::recursive_mutex &mutex() { return m_map_mutex; }
-
-  friend class FormattersContainer<KeyType, ValueType>;
-  friend class FormatManager;
-};
-
-template <typename KeyType, typename ValueType> class FormattersContainer {
-protected:
-  typedef FormatMap<KeyType, ValueType> BackEndType;
-
-public:
-  typedef typename BackEndType::MapType MapType;
-  typedef typename MapType::iterator MapIterator;
-  typedef KeyType MapKeyType;
-  typedef std::shared_ptr<ValueType> MapValueType;
-  typedef typename BackEndType::ForEachCallback ForEachCallback;
-  typedef typename std::shared_ptr<FormattersContainer<KeyType, ValueType>>
-      SharedPointer;
-
-  friend class TypeCategoryImpl;
-
-  FormattersContainer(std::string name, IFormatChangeListener *lst)
-      : m_format_map(lst), m_name(name) {}
-
-  void Add(MapKeyType type, const MapValueType &entry) {
-    Add_Impl(std::move(type), entry, static_cast<KeyType *>(nullptr));
-  }
-
-  bool Delete(ConstString type) {
-    return Delete_Impl(type, static_cast<KeyType *>(nullptr));
-  }
-
-  bool Get(ValueObject &valobj, MapValueType &entry,
-           lldb::DynamicValueType use_dynamic) {
-    CompilerType ast_type(valobj.GetCompilerType());
-    bool ret = Get(valobj, ast_type, entry, use_dynamic);
-    if (ret)
-      entry = MapValueType(entry);
-    else
-      entry = MapValueType();
-    return ret;
-  }
-
-  bool Get(ConstString type, MapValueType &entry) {
-    return Get_Impl(type, entry, static_cast<KeyType *>(nullptr));
+      return lldb::TypeNameSpecifierImplSP();
+    TypeMatcher type_matcher = m_map[index].first;
+    return std::make_shared<TypeNameSpecifierImpl>(
+        type_matcher.GetMatchString().GetStringRef(), true);
   }
 
-  bool GetExact(ConstString type, MapValueType &entry) {
-    return GetExact_Impl(type, entry, static_cast<KeyType *>(nullptr));
+  void Clear() {
+    std::lock_guard<std::recursive_mutex> guard(m_map_mutex);
+    m_map.clear();
+    if (listener)
+      listener->Changed();
   }
 
-  MapValueType GetAtIndex(size_t index) {
-    return m_format_map.GetValueAtIndex(index);
+  void ForEach(ForEachCallback callback) {
+    if (callback) {
+      std::lock_guard<std::recursive_mutex> guard(m_map_mutex);
+      for (const auto &pos : m_map) {
+        const TypeMatcher &type = pos.first;
+        if (!callback(type, pos.second))
+          break;
+      }
+    }
   }
 
-  lldb::TypeNameSpecifierImplSP GetTypeNameSpecifierAtIndex(size_t index) {
-    return GetTypeNameSpecifierAtIndex_Impl(index,
-                                            static_cast<KeyType *>(nullptr));
+  uint32_t GetCount() {
+    std::lock_guard<std::recursive_mutex> guard(m_map_mutex);
+    return m_map.size();
   }
 
-  void Clear() { m_format_map.Clear(); }
-
-  void ForEach(ForEachCallback callback) { m_format_map.ForEach(callback); }
-
-  uint32_t GetCount() { return m_format_map.GetCount(); }
-
 protected:
-  BackEndType m_format_map;
-  std::string m_name;
-
   FormattersContainer(const FormattersContainer &) = delete;
   const FormattersContainer &operator=(const FormattersContainer &) = delete;
 
-  void Add_Impl(MapKeyType type, const MapValueType &entry,
-                RegularExpression *dummy) {
-    m_format_map.Add(std::move(type), entry);
-  }
-
-  void Add_Impl(ConstString type, const MapValueType &entry,
-                ConstString *dummy) {
-    m_format_map.Add(GetValidTypeName_Impl(type), entry);
-  }
-
-  bool Delete_Impl(ConstString type, ConstString *dummy) {
-    return m_format_map.Delete(type);
-  }
-
-  bool Delete_Impl(ConstString type, RegularExpression *dummy) {
-    std::lock_guard<std::recursive_mutex> guard(m_format_map.mutex());
-    MapIterator pos, end = m_format_map.map().end();
-    for (pos = m_format_map.map().begin(); pos != end; pos++) {
-      const RegularExpression &regex = pos->first;
-      if (type.GetStringRef() == regex.GetText()) {
-        m_format_map.map().erase(pos);
-        if (m_format_map.listener)
-          m_format_map.listener->Changed();
-        return true;
-      }
-    }
-    return false;
-  }
-
-  bool Get_Impl(ConstString type, MapValueType &entry, ConstString *dummy) {
-    return m_format_map.Get(type, entry);
-  }
-
-  bool GetExact_Impl(ConstString type, MapValueType &entry,
-                     ConstString *dummy) {
-    return Get_Impl(type, entry, static_cast<KeyType *>(nullptr));
-  }
-
-  lldb::TypeNameSpecifierImplSP
-  GetTypeNameSpecifierAtIndex_Impl(size_t index, ConstString *dummy) {
-    ConstString key = m_format_map.GetKeyAtIndex(index);
-    if (key)
-      return lldb::TypeNameSpecifierImplSP(
-          new TypeNameSpecifierImpl(key.GetStringRef(), false));
-    else
-      return lldb::TypeNameSpecifierImplSP();
-  }
-
-  lldb::TypeNameSpecifierImplSP
-  GetTypeNameSpecifierAtIndex_Impl(size_t index, RegularExpression *dummy) {
-    RegularExpression regex = m_format_map.GetKeyAtIndex(index);
-    if (regex == RegularExpression())
-      return lldb::TypeNameSpecifierImplSP();
-    return lldb::TypeNameSpecifierImplSP(
-        new TypeNameSpecifierImpl(regex.GetText().str().c_str(), true));
-  }
-
-  bool Get_Impl(ConstString key, MapValueType &value,
-                RegularExpression *dummy) {
-    llvm::StringRef key_str = key.GetStringRef();
-    std::lock_guard<std::recursive_mutex> guard(m_format_map.mutex());
-    // Patterns are matched in reverse-chronological order.
-    for (const auto &pos : llvm::reverse(m_format_map.map())) {
-      const RegularExpression &regex = pos.first;
-      if (regex.Execute(key_str)) {
-        value = pos.second;
-        return true;
-      }
-    }
-    return false;
-  }
-
-  bool GetExact_Impl(ConstString key, MapValueType &value,
-                     RegularExpression *dummy) {
-    std::lock_guard<std::recursive_mutex> guard(m_format_map.mutex());
-    for (const auto &pos : m_format_map.map()) {
-      const RegularExpression &regex = pos.first;
-      if (regex.GetText() == key.GetStringRef()) {
-        value = pos.second;
-        return true;
-      }
-    }
-    return false;
-  }
-
-  bool Get(const FormattersMatchVector &candidates, MapValueType &entry) {
+  bool Get(const FormattersMatchVector &candidates, ValueSP &entry) {
     for (const FormattersMatchCandidate &candidate : candidates) {
       if (Get(candidate.GetTypeName(), entry)) {
         if (candidate.IsMatch(entry) == false) {
@@ -319,6 +219,10 @@ template <typename KeyType, typename ValueType> class FormattersContainer {
     }
     return false;
   }
+
+  MapType m_map;
+  std::recursive_mutex m_map_mutex;
+  IFormatChangeListener *listener;
 };
 
 } // namespace lldb_private
diff --git a/lldb/include/lldb/DataFormatters/TypeCategory.h b/lldb/include/lldb/DataFormatters/TypeCategory.h
index 820872a59bda6..2662ffc5aeac6 100644
--- a/lldb/include/lldb/DataFormatters/TypeCategory.h
+++ b/lldb/include/lldb/DataFormatters/TypeCategory.h
@@ -25,14 +25,13 @@ namespace lldb_private {
 
 template <typename FormatterImpl> class FormatterContainerPair {
 public:
-  typedef FormattersContainer<ConstString, FormatterImpl> ExactMatchContainer;
-  typedef FormattersContainer<RegularExpression, FormatterImpl>
-      RegexMatchContainer;
+  typedef FormattersContainer<FormatterImpl> ExactMatchContainer;
+  typedef FormattersContainer<FormatterImpl> RegexMatchContainer;
 
-  typedef typename ExactMatchContainer::MapType ExactMatchMap;
-  typedef typename RegexMatchContainer::MapType RegexMatchMap;
+  typedef TypeMatcher ExactMatchMap;
+  typedef TypeMatcher RegexMatchMap;
 
-  typedef typename ExactMatchContainer::MapValueType MapValueType;
+  typedef typename ExactMatchContainer::ValueSP MapValueType;
 
   typedef typename ExactMatchContainer::SharedPointer ExactMatchContainerSP;
   typedef typename RegexMatchContainer::SharedPointer RegexMatchContainerSP;
@@ -42,10 +41,9 @@ template <typename FormatterImpl> class FormatterContainerPair {
   typedef
       typename RegexMatchContainer::ForEachCallback RegexMatchForEachCallback;
 
-  FormatterContainerPair(const char *exact_name, const char *regex_name,
-                         IFormatChangeListener *clist)
-      : m_exact_sp(new ExactMatchContainer(std::string(exact_name), clist)),
-        m_regex_sp(new RegexMatchContainer(std::string(regex_name), clist)) {}
+  FormatterContainerPair(IFormatChangeListener *clist)
+      : m_exact_sp(new ExactMatchContainer(clist)),
+        m_regex_sp(new RegexMatchContainer(clist)) {}
 
   ~FormatterContainerPair() = default;
 
@@ -349,19 +347,13 @@ class TypeCategoryImpl {
   friend class LanguageCategory;
   friend class TypeCategoryMap;
 
-  friend class FormattersContainer<ConstString, TypeFormatImpl>;
-  friend class FormattersContainer<lldb::RegularExpressionSP, TypeFormatImpl>;
+  friend class FormattersContainer<TypeFormatImpl>;
 
-  friend class FormattersContainer<ConstString, TypeSummaryImpl>;
-  friend class FormattersContainer<lldb::RegularExpressionSP, TypeSummaryImpl>;
+  friend class FormattersContainer<TypeSummaryImpl>;
 
-  friend class FormattersContainer<ConstString, TypeFilterImpl>;
-  friend class FormattersContainer<lldb::RegularExpressionSP, TypeFilterImpl>;
-
-  friend class FormattersContainer<ConstString, ScriptedSyntheticChildren>;
-  friend class FormattersContainer<lldb::RegularExpressionSP,
-                                   ScriptedSyntheticChildren>;
+  friend class FormattersContainer<TypeFilterImpl>;
 
+  friend class FormattersContainer<ScriptedSyntheticChildren>;
 };
 
 } // namespace lldb_private
diff --git a/lldb/include/lldb/DataFormatters/TypeCategoryMap.h b/lldb/include/lldb/DataFormatters/TypeCategoryMap.h
index 832652f7d745d..6cd773786386d 100644
--- a/lldb/include/lldb/DataFormatters/TypeCategoryMap.h
+++ b/lldb/include/lldb/DataFormatters/TypeCategoryMap.h
@@ -103,7 +103,7 @@ class TypeCategoryMap {
 
   std::recursive_mutex &mutex() { return m_map_mutex; }
 
-  friend class FormattersContainer<KeyType, ValueType>;
+  friend class FormattersContainer<ValueType>;
   friend class FormatManager;
 };
 } // namespace lldb_private
diff --git a/lldb/include/lldb/Interpreter/CommandInterpreter.h b/lldb/include/lldb/Interpreter/CommandInterpreter.h
index 8a9dce7a19bc2..72638f3d807f6 100644
--- a/lldb/include/lldb/Interpreter/CommandInterpreter.h
+++ b/lldb/include/lldb/Interpreter/CommandInterpreter.h
@@ -20,6 +20,7 @@
 #include "lldb/Utility/CompletionRequest.h"
 #include "lldb/Utility/Event.h"
 #include "lldb/Utility/Log.h"
+#include "lldb/Utility/StreamString.h"
 #include "lldb/Utility/StringList.h"
 #include "lldb/lldb-forward.h"
 #include "lldb/lldb-private.h"
@@ -485,9 +486,11 @@ class CommandInterpreter : public Broadcaster,
   bool GetExpandRegexAliases() const;
 
   bool GetPromptOnQuit() const;
-
   void SetPromptOnQuit(bool enable);
 
+  bool GetSaveSessionOnQuit() const;
+  void SetSaveSessionOnQuit(bool enable);
+
   bool GetEchoCommands() const;
   void SetEchoCommands(bool enable);
 
@@ -526,6 +529,18 @@ class CommandInterpreter : public Broadcaster,
 
   bool GetSpaceReplPrompts() const;
 
+  /// Save the current debugger session transcript to a file on disk.
+  /// \param output_file
+  ///     The file path to which the session transcript will be written. Since
+  ///     the argument is optional, an arbitrary temporary file will be create
+  ///     when no argument is passed.
+  /// \param result
+  ///     This is used to pass function output and error messages.
+  /// \return \b true if the session transcript was successfully written to
+  /// disk, \b false otherwise.
+  bool SaveTranscript(CommandReturnObject &result,
+                      llvm::Optional<std::string> output_file = llvm::None);
+
 protected:
   friend class Debugger;
 
@@ -621,6 +636,8 @@ class CommandInterpreter : public Broadcaster,
   llvm::Optional<int> m_quit_exit_code;
   // If the driver is accepts custom exit codes for the 'quit' command.
   bool m_allow_exit_code = false;
+
+  StreamString m_transcript_stream;
 };
 
 } // namespace lldb_private
diff --git a/lldb/include/lldb/Interpreter/CommandObject.h b/lldb/include/lldb/Interpreter/CommandObject.h
index cc4d40b23c312..d5ad969cda66b 100644
--- a/lldb/include/lldb/Interpreter/CommandObject.h
+++ b/lldb/include/lldb/Interpreter/CommandObject.h
@@ -90,14 +90,15 @@ class CommandObject {
   {
     lldb::CommandArgumentType arg_type;
     ArgumentRepetitionType arg_repetition;
-    uint32_t arg_opt_set_association; // This arg might be associated only with
-                                      // some particular option set(s).
-    CommandArgumentData()
-        : arg_type(lldb::eArgTypeNone), arg_repetition(eArgRepeatPlain),
-          arg_opt_set_association(LLDB_OPT_SET_ALL) // By default, the arg
-                                                    // associates to all option
-                                                    // sets.
-    {}
+    /// This arg might be associated only with some particular option set(s). By
+    /// default the arg associates to all option sets.
+    uint32_t arg_opt_set_association;
+
+    CommandArgumentData(lldb::CommandArgumentType type = lldb::eArgTypeNone,
+                        ArgumentRepetitionType repetition = eArgRepeatPlain,
+                        uint32_t opt_set = LLDB_OPT_SET_ALL)
+        : arg_type(type), arg_repetition(repetition),
+          arg_opt_set_association(opt_set) {}
   };
 
   typedef std::vector<CommandArgumentData>
diff --git a/lldb/include/lldb/Interpreter/OptionValue.h b/lldb/include/lldb/Interpreter/OptionValue.h
index 5b07427094bf2..a8176e39940a0 100644
--- a/lldb/include/lldb/Interpreter/OptionValue.h
+++ b/lldb/include/lldb/Interpreter/OptionValue.h
@@ -31,6 +31,7 @@ class OptionValue {
     eTypeChar,
     eTypeDictionary,
     eTypeEnum,
+    eTypeFileLineColumn,
     eTypeFileSpec,
     eTypeFileSpecList,
     eTypeFormat,
@@ -84,7 +85,7 @@ class OptionValue {
   SetValueFromString(llvm::StringRef value,
                      VarSetOperationType op = eVarSetOperationAssign);
 
-  virtual bool Clear() = 0;
+  virtual void Clear() = 0;
 
   virtual lldb::OptionValueSP DeepCopy() const = 0;
 
@@ -135,6 +136,8 @@ class OptionValue {
       return eTypeDictionary;
     case 1u << eTypeEnum:
       return eTypeEnum;
+    case 1u << eTypeFileLineColumn:
+      return eTypeFileLineColumn;
     case 1u << eTypeFileSpec:
       return eTypeFileSpec;
     case 1u << eTypeFileSpecList:
diff --git a/lldb/include/lldb/Interpreter/OptionValueArch.h b/lldb/include/lldb/Interpreter/OptionValueArch.h
index 7b63c68fddbf0..809261ef22c36 100644
--- a/lldb/include/lldb/Interpreter/OptionValueArch.h
+++ b/lldb/include/lldb/Interpreter/OptionValueArch.h
@@ -47,10 +47,9 @@ class OptionValueArch : public OptionValue {
   SetValueFromString(const char *,
                      VarSetOperationType = eVarSetOperationAssign) = delete;
 
-  bool Clear() override {
+  void Clear() override {
     m_current_value = m_default_value;
     m_value_was_set = false;
-    return true;
   }
 
   lldb::OptionValueSP DeepCopy() const override;
diff --git a/lldb/include/lldb/Interpreter/OptionValueArray.h b/lldb/include/lldb/Interpreter/OptionValueArray.h
index 000351c2f5867..4546bbb803941 100644
--- a/lldb/include/lldb/Interpreter/OptionValueArray.h
+++ b/lldb/include/lldb/Interpreter/OptionValueArray.h
@@ -36,10 +36,9 @@ class OptionValueArray : public OptionValue {
   SetValueFromString(const char *,
                      VarSetOperationType = eVarSetOperationAssign) = delete;
 
-  bool Clear() override {
+  void Clear() override {
     m_values.clear();
     m_value_was_set = false;
-    return true;
   }
 
   lldb::OptionValueSP DeepCopy() const override;
diff --git a/lldb/include/lldb/Interpreter/OptionValueBoolean.h b/lldb/include/lldb/Interpreter/OptionValueBoolean.h
index d221f6d034c2d..1af14a4980ed9 100644
--- a/lldb/include/lldb/Interpreter/OptionValueBoolean.h
+++ b/lldb/include/lldb/Interpreter/OptionValueBoolean.h
@@ -37,10 +37,9 @@ class OptionValueBoolean : public OptionValue {
   SetValueFromString(const char *,
                      VarSetOperationType = eVarSetOperationAssign) = delete;
 
-  bool Clear() override {
+  void Clear() override {
     m_current_value = m_default_value;
     m_value_was_set = false;
-    return true;
   }
 
   void AutoComplete(CommandInterpreter &interpreter,
diff --git a/lldb/include/lldb/Interpreter/OptionValueChar.h b/lldb/include/lldb/Interpreter/OptionValueChar.h
index 8d0aa91d7076a..a8ecf507a4cfc 100644
--- a/lldb/include/lldb/Interpreter/OptionValueChar.h
+++ b/lldb/include/lldb/Interpreter/OptionValueChar.h
@@ -38,10 +38,9 @@ class OptionValueChar : public OptionValue {
   SetValueFromString(const char *,
                      VarSetOperationType = eVarSetOperationAssign) = delete;
 
-  bool Clear() override {
+  void Clear() override {
     m_current_value = m_default_value;
     m_value_was_set = false;
-    return true;
   }
 
   // Subclass specific functions
diff --git a/lldb/include/lldb/Interpreter/OptionValueDictionary.h b/lldb/include/lldb/Interpreter/OptionValueDictionary.h
index 1bc45252607c4..dab1c3ea0c1cf 100644
--- a/lldb/include/lldb/Interpreter/OptionValueDictionary.h
+++ b/lldb/include/lldb/Interpreter/OptionValueDictionary.h
@@ -35,10 +35,9 @@ class OptionValueDictionary : public OptionValue {
   SetValueFromString(llvm::StringRef value,
                      VarSetOperationType op = eVarSetOperationAssign) override;
 
-  bool Clear() override {
+  void Clear() override {
     m_values.clear();
     m_value_was_set = false;
-    return true;
   }
 
   lldb::OptionValueSP DeepCopy() const override;
diff --git a/lldb/include/lldb/Interpreter/OptionValueEnumeration.h b/lldb/include/lldb/Interpreter/OptionValueEnumeration.h
index 26ba7ad5f646a..12c6473c7f1c2 100644
--- a/lldb/include/lldb/Interpreter/OptionValueEnumeration.h
+++ b/lldb/include/lldb/Interpreter/OptionValueEnumeration.h
@@ -47,10 +47,9 @@ class OptionValueEnumeration : public OptionValue {
   SetValueFromString(const char *,
                      VarSetOperationType = eVarSetOperationAssign) = delete;
 
-  bool Clear() override {
+  void Clear() override {
     m_current_value = m_default_value;
     m_value_was_set = false;
-    return true;
   }
 
   lldb::OptionValueSP DeepCopy() const override;
diff --git a/lldb/include/lldb/Interpreter/OptionValueFileColonLine.h b/lldb/include/lldb/Interpreter/OptionValueFileColonLine.h
new file mode 100644
index 0000000000000..713deea9e1415
--- /dev/null
+++ b/lldb/include/lldb/Interpreter/OptionValueFileColonLine.h
@@ -0,0 +1,64 @@
+//===-- OptionValueFileColonLine.h ------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_INTERPRETER_OPTIONVALUEFILECOLONLINE_H
+#define LLDB_INTERPRETER_OPTIONVALUEFILECOLONLINE_H
+
+#include "lldb/Interpreter/OptionValue.h"
+
+#include "lldb/Utility/FileSpec.h"
+#include "llvm/Support/Chrono.h"
+
+namespace lldb_private {
+
+class OptionValueFileColonLine : public OptionValue {
+public:
+  OptionValueFileColonLine();
+  OptionValueFileColonLine(const llvm::StringRef input);
+
+  ~OptionValueFileColonLine() override {}
+
+  OptionValue::Type GetType() const override { return eTypeFileLineColumn; }
+
+  void DumpValue(const ExecutionContext *exe_ctx, Stream &strm,
+                 uint32_t dump_mask) override;
+
+  Status
+  SetValueFromString(llvm::StringRef value,
+                     VarSetOperationType op = eVarSetOperationAssign) override;
+  Status
+  SetValueFromString(const char *,
+                     VarSetOperationType = eVarSetOperationAssign) = delete;
+
+  void Clear() override {
+    m_file_spec.Clear();
+    m_line_number = LLDB_INVALID_LINE_NUMBER;
+    m_column_number = LLDB_INVALID_COLUMN_NUMBER;
+  }
+
+  lldb::OptionValueSP DeepCopy() const override;
+
+  void AutoComplete(CommandInterpreter &interpreter,
+                    CompletionRequest &request) override;
+
+  FileSpec &GetFileSpec() { return m_file_spec; }
+  uint32_t GetLineNumber() { return m_line_number; }
+  uint32_t GetColumnNumber() { return m_column_number; }
+
+  void SetCompletionMask(uint32_t mask) { m_completion_mask = mask; }
+
+protected:
+  FileSpec m_file_spec;
+  uint32_t m_line_number;
+  uint32_t m_column_number;
+  uint32_t m_completion_mask;
+};
+
+} // namespace lldb_private
+
+#endif // LLDB_INTERPRETER_OPTIONVALUEFILECOLONLINE_H
diff --git a/lldb/include/lldb/Interpreter/OptionValueFileSpec.h b/lldb/include/lldb/Interpreter/OptionValueFileSpec.h
index 2b18c9533f912..4fde3f6e3c8a2 100644
--- a/lldb/include/lldb/Interpreter/OptionValueFileSpec.h
+++ b/lldb/include/lldb/Interpreter/OptionValueFileSpec.h
@@ -41,12 +41,11 @@ class OptionValueFileSpec : public OptionValue {
   SetValueFromString(const char *,
                      VarSetOperationType = eVarSetOperationAssign) = delete;
 
-  bool Clear() override {
+  void Clear() override {
     m_current_value = m_default_value;
     m_value_was_set = false;
     m_data_sp.reset();
     m_data_mod_time = llvm::sys::TimePoint<>();
-    return true;
   }
 
   lldb::OptionValueSP DeepCopy() const override;
diff --git a/lldb/include/lldb/Interpreter/OptionValueFileSpecList.h b/lldb/include/lldb/Interpreter/OptionValueFileSpecList.h
index 7b762bf6b3098..38773525c8db9 100644
--- a/lldb/include/lldb/Interpreter/OptionValueFileSpecList.h
+++ b/lldb/include/lldb/Interpreter/OptionValueFileSpecList.h
@@ -39,11 +39,10 @@ class OptionValueFileSpecList : public OptionValue {
   SetValueFromString(const char *,
                      VarSetOperationType = eVarSetOperationAssign) = delete;
 
-  bool Clear() override {
+  void Clear() override {
     std::lock_guard<std::recursive_mutex> lock(m_mutex);
     m_current_value.Clear();
     m_value_was_set = false;
-    return true;
   }
 
   lldb::OptionValueSP DeepCopy() const override;
diff --git a/lldb/include/lldb/Interpreter/OptionValueFormat.h b/lldb/include/lldb/Interpreter/OptionValueFormat.h
index 6904c93a2f338..5a83ff3b8983d 100644
--- a/lldb/include/lldb/Interpreter/OptionValueFormat.h
+++ b/lldb/include/lldb/Interpreter/OptionValueFormat.h
@@ -38,10 +38,9 @@ class OptionValueFormat : public OptionValue {
   SetValueFromString(const char *,
                      VarSetOperationType = eVarSetOperationAssign) = delete;
 
-  bool Clear() override {
+  void Clear() override {
     m_current_value = m_default_value;
     m_value_was_set = false;
-    return true;
   }
 
   lldb::OptionValueSP DeepCopy() const override;
diff --git a/lldb/include/lldb/Interpreter/OptionValueFormatEntity.h b/lldb/include/lldb/Interpreter/OptionValueFormatEntity.h
index beb5d6843a98b..7c2f9fba0242e 100644
--- a/lldb/include/lldb/Interpreter/OptionValueFormatEntity.h
+++ b/lldb/include/lldb/Interpreter/OptionValueFormatEntity.h
@@ -34,7 +34,7 @@ class OptionValueFormatEntity : public OptionValue {
   SetValueFromString(const char *,
                      VarSetOperationType = eVarSetOperationAssign) = delete;
 
-  bool Clear() override;
+  void Clear() override;
 
   lldb::OptionValueSP DeepCopy() const override;
 
diff --git a/lldb/include/lldb/Interpreter/OptionValueLanguage.h b/lldb/include/lldb/Interpreter/OptionValueLanguage.h
index f4ca2fd69ab59..1293652967596 100644
--- a/lldb/include/lldb/Interpreter/OptionValueLanguage.h
+++ b/lldb/include/lldb/Interpreter/OptionValueLanguage.h
@@ -41,10 +41,9 @@ class OptionValueLanguage : public OptionValue {
   SetValueFromString(const char *,
                      VarSetOperationType = eVarSetOperationAssign) = delete;
 
-  bool Clear() override {
+  void Clear() override {
     m_current_value = m_default_value;
     m_value_was_set = false;
-    return true;
   }
 
   lldb::OptionValueSP DeepCopy() const override;
diff --git a/lldb/include/lldb/Interpreter/OptionValuePathMappings.h b/lldb/include/lldb/Interpreter/OptionValuePathMappings.h
index 18f5cbaf43366..6d1a0816f45b8 100644
--- a/lldb/include/lldb/Interpreter/OptionValuePathMappings.h
+++ b/lldb/include/lldb/Interpreter/OptionValuePathMappings.h
@@ -35,10 +35,9 @@ class OptionValuePathMappings : public OptionValue {
   SetValueFromString(const char *,
                      VarSetOperationType = eVarSetOperationAssign) = delete;
 
-  bool Clear() override {
+  void Clear() override {
     m_path_mappings.Clear(m_notify_changes);
     m_value_was_set = false;
-    return true;
   }
 
   lldb::OptionValueSP DeepCopy() const override;
diff --git a/lldb/include/lldb/Interpreter/OptionValueProperties.h b/lldb/include/lldb/Interpreter/OptionValueProperties.h
index 76f09cc771235..bd944b6a5adff 100644
--- a/lldb/include/lldb/Interpreter/OptionValueProperties.h
+++ b/lldb/include/lldb/Interpreter/OptionValueProperties.h
@@ -34,7 +34,7 @@ class OptionValueProperties
 
   Type GetType() const override { return eTypeProperties; }
 
-  bool Clear() override;
+  void Clear() override;
 
   lldb::OptionValueSP DeepCopy() const override;
 
diff --git a/lldb/include/lldb/Interpreter/OptionValueRegex.h b/lldb/include/lldb/Interpreter/OptionValueRegex.h
index b09b8414d5bf4..4751a1dc27ed0 100644
--- a/lldb/include/lldb/Interpreter/OptionValueRegex.h
+++ b/lldb/include/lldb/Interpreter/OptionValueRegex.h
@@ -36,10 +36,9 @@ class OptionValueRegex : public OptionValue {
   SetValueFromString(const char *,
                      VarSetOperationType = eVarSetOperationAssign) = delete;
 
-  bool Clear() override {
+  void Clear() override {
     m_regex = RegularExpression(m_default_regex_str);
     m_value_was_set = false;
-    return true;
   }
 
   lldb::OptionValueSP DeepCopy() const override;
diff --git a/lldb/include/lldb/Interpreter/OptionValueSInt64.h b/lldb/include/lldb/Interpreter/OptionValueSInt64.h
index fbabaaeb2ff4b..87917c1088651 100644
--- a/lldb/include/lldb/Interpreter/OptionValueSInt64.h
+++ b/lldb/include/lldb/Interpreter/OptionValueSInt64.h
@@ -50,10 +50,9 @@ class OptionValueSInt64 : public OptionValue {
   SetValueFromString(const char *,
                      VarSetOperationType = eVarSetOperationAssign) = delete;
 
-  bool Clear() override {
+  void Clear() override {
     m_current_value = m_default_value;
     m_value_was_set = false;
-    return true;
   }
 
   lldb::OptionValueSP DeepCopy() const override;
diff --git a/lldb/include/lldb/Interpreter/OptionValueString.h b/lldb/include/lldb/Interpreter/OptionValueString.h
index cd371c5670204..ed44dae67d1dc 100644
--- a/lldb/include/lldb/Interpreter/OptionValueString.h
+++ b/lldb/include/lldb/Interpreter/OptionValueString.h
@@ -85,10 +85,9 @@ class OptionValueString : public OptionValue {
   SetValueFromString(const char *,
                      VarSetOperationType = eVarSetOperationAssign) = delete;
 
-  bool Clear() override {
+  void Clear() override {
     m_current_value = m_default_value;
     m_value_was_set = false;
-    return true;
   }
 
   lldb::OptionValueSP DeepCopy() const override;
diff --git a/lldb/include/lldb/Interpreter/OptionValueUInt64.h b/lldb/include/lldb/Interpreter/OptionValueUInt64.h
index 0096e87de3677..1164fb802f682 100644
--- a/lldb/include/lldb/Interpreter/OptionValueUInt64.h
+++ b/lldb/include/lldb/Interpreter/OptionValueUInt64.h
@@ -47,10 +47,9 @@ class OptionValueUInt64 : public OptionValue {
   SetValueFromString(const char *,
                      VarSetOperationType = eVarSetOperationAssign) = delete;
 
-  bool Clear() override {
+  void Clear() override {
     m_current_value = m_default_value;
     m_value_was_set = false;
-    return true;
   }
 
   lldb::OptionValueSP DeepCopy() const override;
diff --git a/lldb/include/lldb/Interpreter/OptionValueUUID.h b/lldb/include/lldb/Interpreter/OptionValueUUID.h
index 2fb8caa3aa531..1f663e999b9d0 100644
--- a/lldb/include/lldb/Interpreter/OptionValueUUID.h
+++ b/lldb/include/lldb/Interpreter/OptionValueUUID.h
@@ -36,10 +36,9 @@ class OptionValueUUID : public OptionValue {
   SetValueFromString(const char *,
                      VarSetOperationType = eVarSetOperationAssign) = delete;
 
-  bool Clear() override {
+  void Clear() override {
     m_uuid.Clear();
     m_value_was_set = false;
-    return true;
   }
 
   lldb::OptionValueSP DeepCopy() const override;
diff --git a/lldb/include/lldb/Interpreter/OptionValues.h b/lldb/include/lldb/Interpreter/OptionValues.h
index 36e7c192d60a3..6efc9e1ad064c 100644
--- a/lldb/include/lldb/Interpreter/OptionValues.h
+++ b/lldb/include/lldb/Interpreter/OptionValues.h
@@ -17,6 +17,7 @@
 #include "lldb/Interpreter/OptionValueChar.h"
 #include "lldb/Interpreter/OptionValueDictionary.h"
 #include "lldb/Interpreter/OptionValueEnumeration.h"
+#include "lldb/Interpreter/OptionValueFileColonLine.h"
 #include "lldb/Interpreter/OptionValueFileSpec.h"
 #include "lldb/Interpreter/OptionValueFileSpecList.h"
 #include "lldb/Interpreter/OptionValueFormat.h"
diff --git a/lldb/include/lldb/Symbol/CompilerType.h b/lldb/include/lldb/Symbol/CompilerType.h
index 280966a327ece..7d5d71cbc55c5 100644
--- a/lldb/include/lldb/Symbol/CompilerType.h
+++ b/lldb/include/lldb/Symbol/CompilerType.h
@@ -177,7 +177,8 @@ class CompilerType {
 
   /// Creating related types.
   /// \{
-  CompilerType GetArrayElementType(uint64_t *stride = nullptr) const;
+  CompilerType GetArrayElementType(ExecutionContextScope *exe_scope,
+                                   uint64_t *stride = nullptr) const;
 
   CompilerType GetArrayType(uint64_t size) const;
 
@@ -383,7 +384,8 @@ class CompilerType {
   /// \}
 
   bool GetValueAsScalar(const DataExtractor &data, lldb::offset_t data_offset,
-                        size_t data_byte_size, Scalar &value) const;
+                        size_t data_byte_size, Scalar &value,
+                        ExecutionContextScope *exe_scope) const;
   void Clear() {
     m_type = nullptr;
     m_type_system = nullptr;
diff --git a/lldb/include/lldb/Symbol/Type.h b/lldb/include/lldb/Symbol/Type.h
index 8735d016bb225..efd3ede034457 100644
--- a/lldb/include/lldb/Symbol/Type.h
+++ b/lldb/include/lldb/Symbol/Type.h
@@ -114,14 +114,15 @@ class Type : public std::enable_shared_from_this<Type>, public UserID {
   // Type instances. They can store a weak pointer to the Module;
   lldb::ModuleSP GetModule();
 
-  void GetDescription(Stream *s, lldb::DescriptionLevel level, bool show_name);
+  void GetDescription(Stream *s, lldb::DescriptionLevel level, bool show_name,
+                      ExecutionContextScope *exe_scope);
 
   SymbolFile *GetSymbolFile() { return m_symbol_file; }
   const SymbolFile *GetSymbolFile() const { return m_symbol_file; }
 
   ConstString GetName();
 
-  llvm::Optional<uint64_t> GetByteSize();
+  llvm::Optional<uint64_t> GetByteSize(ExecutionContextScope *exe_scope);
 
   uint32_t GetNumChildren(bool omit_empty_base_classes);
 
diff --git a/lldb/include/lldb/Symbol/TypeSystem.h b/lldb/include/lldb/Symbol/TypeSystem.h
index e188f29354b89..c19909506349d 100644
--- a/lldb/include/lldb/Symbol/TypeSystem.h
+++ b/lldb/include/lldb/Symbol/TypeSystem.h
@@ -217,8 +217,9 @@ class TypeSystem : public PluginInterface {
 
   // Creating related types
 
-  virtual CompilerType GetArrayElementType(lldb::opaque_compiler_type_t type,
-                                           uint64_t *stride) = 0;
+  virtual CompilerType
+  GetArrayElementType(lldb::opaque_compiler_type_t type, uint64_t *stride,
+                      ExecutionContextScope *exe_scope) = 0;
 
   virtual CompilerType GetArrayType(lldb::opaque_compiler_type_t type,
                                     uint64_t size);
diff --git a/lldb/include/lldb/Target/StackFrameRecognizer.h b/lldb/include/lldb/Target/StackFrameRecognizer.h
index 38b21e7c9856b..302b56bec907b 100644
--- a/lldb/include/lldb/Target/StackFrameRecognizer.h
+++ b/lldb/include/lldb/Target/StackFrameRecognizer.h
@@ -125,7 +125,6 @@ class StackFrameRecognizerManager {
 private:
   struct RegisteredEntry {
     uint32_t recognizer_id;
-    bool deleted;
     lldb::StackFrameRecognizerSP recognizer;
     bool is_regexp;
     ConstString module;
diff --git a/lldb/include/lldb/Utility/ArchSpec.h b/lldb/include/lldb/Utility/ArchSpec.h
index 5466e573c1a52..a727d5ca4f79d 100644
--- a/lldb/include/lldb/Utility/ArchSpec.h
+++ b/lldb/include/lldb/Utility/ArchSpec.h
@@ -505,7 +505,7 @@ class ArchSpec {
 
   void SetFlags(uint32_t flags) { m_flags = flags; }
 
-  void SetFlags(std::string elf_abi);
+  void SetFlags(const std::string &elf_abi);
 
 protected:
   bool IsEqualTo(const ArchSpec &rhs, bool exact_match) const;
diff --git a/lldb/include/lldb/Utility/Broadcaster.h b/lldb/include/lldb/Utility/Broadcaster.h
index 03995454ecb02..9c025a7c7dd88 100644
--- a/lldb/include/lldb/Utility/Broadcaster.h
+++ b/lldb/include/lldb/Utility/Broadcaster.h
@@ -39,7 +39,7 @@ namespace lldb_private {
 /// Debugger maintains a list of BroadcastEventSpec's and when it is made
 class BroadcastEventSpec {
 public:
-  BroadcastEventSpec(ConstString broadcaster_class, uint32_t event_bits)
+  BroadcastEventSpec(const ConstString &broadcaster_class, uint32_t event_bits)
       : m_broadcaster_class(broadcaster_class), m_event_bits(event_bits) {}
 
   ~BroadcastEventSpec() = default;
@@ -117,7 +117,7 @@ class BroadcasterManager
 
   class BroadcasterClassMatches {
   public:
-    BroadcasterClassMatches(ConstString broadcaster_class)
+    BroadcasterClassMatches(const ConstString &broadcaster_class)
         : m_broadcaster_class(broadcaster_class) {}
 
     ~BroadcasterClassMatches() = default;
diff --git a/lldb/include/lldb/Utility/ConstString.h b/lldb/include/lldb/Utility/ConstString.h
index 1e55b2ebb9577..8a67faf5b54a4 100644
--- a/lldb/include/lldb/Utility/ConstString.h
+++ b/lldb/include/lldb/Utility/ConstString.h
@@ -42,15 +42,7 @@ class ConstString {
   /// Default constructor
   ///
   /// Initializes the string to an empty string.
-  ConstString() : m_string(nullptr) {}
-
-  /// Copy constructor
-  ///
-  /// Copies the string value in \a rhs into this object.
-  ///
-  /// \param[in] rhs
-  ///     Another string object to copy.
-  ConstString(const ConstString &rhs) : m_string(rhs.m_string) {}
+  ConstString() = default;
 
   explicit ConstString(const llvm::StringRef &s);
 
@@ -86,12 +78,6 @@ class ConstString {
   ///     from \a cstr.
   explicit ConstString(const char *cstr, size_t max_cstr_len);
 
-  /// Destructor
-  ///
-  /// Since constant string values are currently not reference counted, there
-  /// isn't much to do here.
-  ~ConstString() = default;
-
   /// C string equality binary predicate function object for ConstString
   /// objects.
   struct StringIsEqual {
@@ -124,20 +110,6 @@ class ConstString {
   ///     false otherwise.
   explicit operator bool() const { return !IsEmpty(); }
 
-  /// Assignment operator
-  ///
-  /// Assigns the string in this object with the value from \a rhs.
-  ///
-  /// \param[in] rhs
-  ///     Another string object to copy into this object.
-  ///
-  /// \return
-  ///     A const reference to this object.
-  ConstString operator=(ConstString rhs) {
-    m_string = rhs.m_string;
-    return *this;
-  }
-
   /// Equal to operator
   ///
   /// Returns true if this string is equal to the string in \a rhs. This
@@ -192,9 +164,7 @@ class ConstString {
   /// \return
   ///     \b true if this object is not equal to \a rhs.
   ///     \b false if this object is equal to \a rhs.
-  bool operator!=(ConstString rhs) const {
-    return m_string != rhs.m_string;
-  }
+  bool operator!=(ConstString rhs) const { return m_string != rhs.m_string; }
 
   /// Not equal to operator against a non-ConstString value.
   ///
@@ -447,8 +417,7 @@ class ConstString {
     return s;
   };
 
-  // Member variables
-  const char *m_string;
+  const char *m_string = nullptr;
 };
 
 /// Stream the string value \a str to the stream \a s
diff --git a/lldb/include/lldb/Utility/RegisterValue.h b/lldb/include/lldb/Utility/RegisterValue.h
index c9f295a8d95ae..4885d58063847 100644
--- a/lldb/include/lldb/Utility/RegisterValue.h
+++ b/lldb/include/lldb/Utility/RegisterValue.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/StringRef.h"
 #include <cstdint>
 #include <cstring>
+#include <utility>
 
 namespace lldb_private {
 class DataExtractor;
@@ -60,7 +61,7 @@ class RegisterValue {
   }
 
   explicit RegisterValue(llvm::APInt inst) : m_type(eTypeUInt128) {
-    m_scalar = llvm::APInt(inst);
+    m_scalar = llvm::APInt(std::move(inst));
   }
 
   explicit RegisterValue(float value) : m_type(eTypeFloat) { m_scalar = value; }
@@ -169,7 +170,7 @@ class RegisterValue {
 
   void operator=(llvm::APInt uint) {
     m_type = eTypeUInt128;
-    m_scalar = llvm::APInt(uint);
+    m_scalar = llvm::APInt(std::move(uint));
   }
 
   void operator=(float f) {
@@ -209,7 +210,7 @@ class RegisterValue {
 
   void SetUInt128(llvm::APInt uint) {
     m_type = eTypeUInt128;
-    m_scalar = uint;
+    m_scalar = std::move(uint);
   }
 
   bool SetUInt(uint64_t uint, uint32_t byte_size);
diff --git a/lldb/include/lldb/Utility/Reproducer.h b/lldb/include/lldb/Utility/Reproducer.h
index ab673e5e00192..21f43b6ad5e01 100644
--- a/lldb/include/lldb/Utility/Reproducer.h
+++ b/lldb/include/lldb/Utility/Reproducer.h
@@ -11,12 +11,14 @@
 
 #include "lldb/Utility/FileSpec.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FileCollector.h"
 #include "llvm/Support/YAMLTraits.h"
 
 #include <mutex>
 #include <string>
+#include <utility>
 #include <vector>
 
 namespace lldb_private {
@@ -99,7 +101,8 @@ class FileProvider : public Provider<FileProvider> {
     return m_collector;
   }
 
-  void recordInterestingDirectory(const llvm::Twine &dir);
+  void RecordInterestingDirectory(const llvm::Twine &dir);
+  void RecordInterestingDirectoryRecursive(const llvm::Twine &dir);
 
   void Keep() override {
     auto mapping = GetRoot().CopyByAppendingPathComponent(Info::file);
@@ -149,6 +152,7 @@ class WorkingDirectoryProvider : public Provider<WorkingDirectoryProvider> {
   }
 
   void Update(llvm::StringRef path) { m_cwd = std::string(path); }
+  llvm::StringRef GetWorkingDirectory() { return m_cwd; }
 
   struct Info {
     static const char *name;
@@ -432,7 +436,7 @@ class Reproducer {
 /// return the path to the files in the index.
 template <typename T> class MultiLoader {
 public:
-  MultiLoader(std::vector<std::string> files) : m_files(files) {}
+  MultiLoader(std::vector<std::string> files) : m_files(std::move(files)) {}
 
   static std::unique_ptr<MultiLoader> Create(Loader *loader) {
     if (!loader)
diff --git a/lldb/include/lldb/Utility/Scalar.h b/lldb/include/lldb/Utility/Scalar.h
index f215fa71c84c5..524b715230744 100644
--- a/lldb/include/lldb/Utility/Scalar.h
+++ b/lldb/include/lldb/Utility/Scalar.h
@@ -9,14 +9,15 @@
 #ifndef LLDB_UTILITY_SCALAR_H
 #define LLDB_UTILITY_SCALAR_H
 
+#include "lldb/Utility/LLDBAssert.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-private-types.h"
-#include "lldb/Utility/LLDBAssert.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/APInt.h"
 #include <cstddef>
 #include <cstdint>
+#include <utility>
 
 namespace lldb_private {
 class DataExtractor;
@@ -89,7 +90,7 @@ class Scalar {
                 llvm::APInt(BITWIDTH_INT128, NUM_OF_WORDS_INT128,
                             (reinterpret_cast<type128 *>(&v))->x)) {}
   Scalar(llvm::APInt v) : m_type(), m_float(static_cast<float>(0)) {
-    m_integer = llvm::APInt(v);
+    m_integer = llvm::APInt(std::move(v));
     m_type = GetBestTypeForBitSize(m_integer.getBitWidth(), true);
   }
   // Scalar(const RegisterValue& reg_value);
@@ -217,7 +218,7 @@ class Scalar {
   Status SetValueFromCString(const char *s, lldb::Encoding encoding,
                              size_t byte_size);
 
-  Status SetValueFromData(DataExtractor &data, lldb::Encoding encoding,
+  Status SetValueFromData(const DataExtractor &data, lldb::Encoding encoding,
                           size_t byte_size);
 
   static bool UIntValueIsValidForSize(uint64_t uval64, size_t total_byte_size) {
diff --git a/lldb/include/lldb/Utility/StringList.h b/lldb/include/lldb/Utility/StringList.h
index 591a158615933..34930abeac3c1 100644
--- a/lldb/include/lldb/Utility/StringList.h
+++ b/lldb/include/lldb/Utility/StringList.h
@@ -102,7 +102,7 @@ class StringList {
 
   StringList &operator<<(const std::string &s);
 
-  StringList &operator<<(StringList strings);
+  StringList &operator<<(const StringList &strings);
 
   // Copy assignment for a vector of strings
   StringList &operator=(const std::vector<std::string> &rhs);
diff --git a/lldb/include/lldb/Utility/StructuredData.h b/lldb/include/lldb/Utility/StructuredData.h
index 14c82e669676c..4d03af18e527b 100644
--- a/lldb/include/lldb/Utility/StructuredData.h
+++ b/lldb/include/lldb/Utility/StructuredData.h
@@ -271,9 +271,9 @@ class StructuredData {
       return false;
     }
 
-    void Push(ObjectSP item) { m_items.push_back(item); }
+    void Push(const ObjectSP &item) { m_items.push_back(item); }
 
-    void AddItem(ObjectSP item) { m_items.push_back(item); }
+    void AddItem(const ObjectSP &item) { m_items.push_back(item); }
 
     void Serialize(llvm::json::OStream &s) const override;
 
@@ -493,7 +493,7 @@ class StructuredData {
 
     void AddItem(llvm::StringRef key, ObjectSP value_sp) {
       ConstString key_cs(key);
-      m_dict[key_cs] = value_sp;
+      m_dict[key_cs] = std::move(value_sp);
     }
 
     void AddIntegerItem(llvm::StringRef key, uint64_t value) {
@@ -547,7 +547,7 @@ class StructuredData {
     void *m_object;
   };
 
-  static ObjectSP ParseJSON(std::string json_text);
+  static ObjectSP ParseJSON(const std::string &json_text);
   static ObjectSP ParseJSONFromFile(const FileSpec &file, Status &error);
 };
 
diff --git a/lldb/include/lldb/Utility/XcodeSDK.h b/lldb/include/lldb/Utility/XcodeSDK.h
index 307fe7f46798e..878b131a18145 100644
--- a/lldb/include/lldb/Utility/XcodeSDK.h
+++ b/lldb/include/lldb/Utility/XcodeSDK.h
@@ -65,11 +65,11 @@ class XcodeSDK {
   /// The merge function follows a strict order to maintain monotonicity:
   /// 1. SDK with the higher SDKType wins.
   /// 2. The newer SDK wins.
-  void Merge(XcodeSDK other);
+  void Merge(const XcodeSDK &other);
 
-  XcodeSDK &operator=(XcodeSDK other);
+  XcodeSDK &operator=(const XcodeSDK &other);
   XcodeSDK(const XcodeSDK&) = default;
-  bool operator==(XcodeSDK other);
+  bool operator==(const XcodeSDK &other);
 
   /// Return parsed SDK type and version number.
   Info Parse() const;
diff --git a/lldb/include/lldb/lldb-defines.h b/lldb/include/lldb/lldb-defines.h
index fea8079779a13..487cd0b01d5c5 100644
--- a/lldb/include/lldb/lldb-defines.h
+++ b/lldb/include/lldb/lldb-defines.h
@@ -95,6 +95,7 @@
 #define LLDB_INVALID_SIGNAL_NUMBER INT32_MAX
 #define LLDB_INVALID_OFFSET UINT64_MAX // Must match max of lldb::offset_t
 #define LLDB_INVALID_LINE_NUMBER UINT32_MAX
+#define LLDB_INVALID_COLUMN_NUMBER 0
 #define LLDB_INVALID_QUEUE_ID 0
 
 /// CPU Type definitions
@@ -119,6 +120,7 @@
 #define LLDB_OPT_SET_9 (1U << 8)
 #define LLDB_OPT_SET_10 (1U << 9)
 #define LLDB_OPT_SET_11 (1U << 10)
+#define LLDB_OPT_SET_12 (1U << 11)
 #define LLDB_OPT_SET_FROM_TO(A, B)                                             \
   (((1U << (B)) - 1) ^ (((1U << (A)) - 1) >> 1))
 
diff --git a/lldb/include/lldb/lldb-enumerations.h b/lldb/include/lldb/lldb-enumerations.h
index b3e8d604913f0..051289de439de 100644
--- a/lldb/include/lldb/lldb-enumerations.h
+++ b/lldb/include/lldb/lldb-enumerations.h
@@ -526,6 +526,7 @@ enum CommandArgumentType {
   eArgTypeExpression,
   eArgTypeExpressionPath,
   eArgTypeExprFormat,
+  eArgTypeFileLineColumn,
   eArgTypeFilename,
   eArgTypeFormat,
   eArgTypeFrameIndex,
diff --git a/lldb/include/lldb/lldb-forward.h b/lldb/include/lldb/lldb-forward.h
index 478ed1a06443e..0bf9e2c105377 100644
--- a/lldb/include/lldb/lldb-forward.h
+++ b/lldb/include/lldb/lldb-forward.h
@@ -432,6 +432,7 @@ typedef std::shared_ptr<lldb_private::Thread> ThreadSP;
 typedef std::weak_ptr<lldb_private::Thread> ThreadWP;
 typedef std::shared_ptr<lldb_private::ThreadCollection> ThreadCollectionSP;
 typedef std::shared_ptr<lldb_private::ThreadPlan> ThreadPlanSP;
+typedef std::weak_ptr<lldb_private::ThreadPlan> ThreadPlanWP;
 typedef std::shared_ptr<lldb_private::ThreadPlanTracer> ThreadPlanTracerSP;
 typedef std::shared_ptr<lldb_private::TraceOptions> TraceOptionsSP;
 typedef std::shared_ptr<lldb_private::Type> TypeSP;
diff --git a/lldb/packages/Python/lldbsuite/test/decorators.py b/lldb/packages/Python/lldbsuite/test/decorators.py
index 534bcbf59ac20..8c8f2509a8639 100644
--- a/lldb/packages/Python/lldbsuite/test/decorators.py
+++ b/lldb/packages/Python/lldbsuite/test/decorators.py
@@ -19,7 +19,6 @@
 from . import configuration
 from . import test_categories
 from . import lldbtest_config
-from lldbsuite.test_event.event_builder import EventBuilder
 from lldbsuite.support import funcutils
 from lldbsuite.test import lldbplatform
 from lldbsuite.test import lldbplatformutil
@@ -100,10 +99,6 @@ def wrapper(*args, **kwargs):
             else:
                 xfail_reason = expected_fn()
             if xfail_reason is not None:
-                if configuration.results_formatter_object is not None:
-                    # Mark this test as expected to fail.
-                    configuration.results_formatter_object.handle_event(
-                        EventBuilder.event_for_mark_test_expected_failure(self))
                 xfail_func = unittest2.expectedFailure(func)
                 xfail_func(*args, **kwargs)
             else:
@@ -445,21 +440,11 @@ def expectedFailureNetBSD(bugnumber=None):
         ['netbsd'],
         bugnumber)
 
-# Flakey tests get two chances to run. If they fail the first time round, the result formatter
-# makes sure it is run one more time.
-
-
+# TODO: This decorator does not do anything. Remove it.
 def expectedFlakey(expected_fn, bugnumber=None):
     def expectedFailure_impl(func):
         @wraps(func)
         def wrapper(*args, **kwargs):
-            self = args[0]
-            if expected_fn(self):
-                # Send event marking test as explicitly eligible for rerunning.
-                if configuration.results_formatter_object is not None:
-                    # Mark this test as rerunnable.
-                    configuration.results_formatter_object.handle_event(
-                        EventBuilder.event_for_mark_test_rerun_eligible(self))
             func(*args, **kwargs)
         return wrapper
     # Some decorators can be called both with no arguments (e.g. @expectedFailureWindows)
diff --git a/lldb/packages/Python/lldbsuite/test/dotest.py b/lldb/packages/Python/lldbsuite/test/dotest.py
index 67f227cad7155..3fb802f1c1aa5 100644
--- a/lldb/packages/Python/lldbsuite/test/dotest.py
+++ b/lldb/packages/Python/lldbsuite/test/dotest.py
@@ -44,9 +44,7 @@
 from . import dotest_args
 from . import lldbtest_config
 from . import test_categories
-from lldbsuite.test_event import formatter
 from . import test_result
-from lldbsuite.test_event.event_builder import EventBuilder
 from ..support import seven
 
 
@@ -453,27 +451,6 @@ def parseOptionsAndInitTestdirs():
 
     lldbtest_config.codesign_identity = args.codesign_identity
 
-
-def setupTestResults():
-    """Sets up test results-related objects based on arg settings."""
-
-    # Create the results formatter.
-    formatter_spec = formatter.create_results_formatter(
-            "lldbsuite.test_event.formatter.results_formatter.ResultsFormatter")
-    if formatter_spec is not None and formatter_spec.formatter is not None:
-        configuration.results_formatter_object = formatter_spec.formatter
-
-        # Send an initialize message to the formatter.
-        initialize_event = EventBuilder.bare_event("initialize")
-        initialize_event["worker_count"] = 1
-
-        formatter_spec.formatter.handle_event(initialize_event)
-
-        # Make sure we clean up the formatter on shutdown.
-        if formatter_spec.cleanup_func is not None:
-            atexit.register(formatter_spec.cleanup_func)
-
-
 def setupSysPath():
     """
     Add LLDB.framework/Resources/Python to the search paths for modules.
@@ -732,31 +709,17 @@ def visit(prefix, dir, names):
 
     # Visit all the python test files.
     for name in python_test_files:
-        try:
-            # Ensure we error out if we have multiple tests with the same
-            # base name.
-            # Future improvement: find all the places where we work with base
-            # names and convert to full paths.  We have directory structure
-            # to disambiguate these, so we shouldn't need this constraint.
-            if name in configuration.all_tests:
-                raise Exception("Found multiple tests with the name %s" % name)
-            configuration.all_tests.add(name)
-
-            # Run the relevant tests in the python file.
-            visit_file(dir, name)
-        except Exception as ex:
-            # Convert this exception to a test event error for the file.
-            test_filename = os.path.abspath(os.path.join(dir, name))
-            if configuration.results_formatter_object is not None:
-                # Grab the backtrace for the exception.
-                import traceback
-                backtrace = traceback.format_exc()
-
-                # Generate the test event.
-                configuration.results_formatter_object.handle_event(
-                    EventBuilder.event_for_job_test_add_error(
-                        test_filename, ex, backtrace))
-            raise
+        # Ensure we error out if we have multiple tests with the same
+        # base name.
+        # Future improvement: find all the places where we work with base
+        # names and convert to full paths.  We have directory structure
+        # to disambiguate these, so we shouldn't need this constraint.
+        if name in configuration.all_tests:
+            raise Exception("Found multiple tests with the name %s" % name)
+        configuration.all_tests.add(name)
+
+        # Run the relevant tests in the python file.
+        visit_file(dir, name)
 
 
 # ======================================== #
@@ -933,9 +896,6 @@ def run_suite():
     #
     parseOptionsAndInitTestdirs()
 
-    # Setup test results (test results formatter and output handling).
-    setupTestResults()
-
     setupSysPath()
 
     import lldbconfig
diff --git a/lldb/packages/Python/lldbsuite/test/lldbtest.py b/lldb/packages/Python/lldbsuite/test/lldbtest.py
index 25805726f9b39..4aad2867cf8f4 100644
--- a/lldb/packages/Python/lldbsuite/test/lldbtest.py
+++ b/lldb/packages/Python/lldbsuite/test/lldbtest.py
@@ -426,7 +426,7 @@ def system(commands, **kwargs):
         process = Popen(
             shellCommand,
             stdout=PIPE,
-            stderr=PIPE,
+            stderr=STDOUT,
             shell=True,
             **kwargs)
         pid = process.pid
@@ -440,14 +440,12 @@ def system(commands, **kwargs):
             cpe = CalledProcessError(retcode, cmd)
             # Ensure caller can access the stdout/stderr.
             cpe.lldb_extensions = {
-                "stdout_content": this_output,
-                "stderr_content": this_error,
+                "combined_output": this_output,
                 "command": shellCommand
             }
             raise cpe
         output = output + this_output.decode("utf-8")
-        error = error + this_error.decode("utf-8")
-    return (output, error)
+    return output
 
 
 def getsource_if_available(obj):
@@ -1280,7 +1278,7 @@ def getCompilerVersion(self):
         version = 'unknown'
 
         compiler = self.getCompilerBinary()
-        version_output = system([[compiler, "-v"]])[1]
+        version_output = system([[compiler, "-v"]])
         for line in version_output.split(os.linesep):
             m = re.search('version ([0-9\.]+)', line)
             if m:
@@ -1881,8 +1879,7 @@ def registerSharedLibrariesWithTarget(self, target, shlibs):
         shlib_prefix = self.platformContext.shlib_prefix
         shlib_extension = '.' + self.platformContext.shlib_extension
 
-        working_dir = self.get_process_working_directory()
-        environment = ['%s=%s' % (shlib_environment_var, working_dir)]
+        dirs = []
         # Add any shared libraries to our target if remote so they get
         # uploaded into the working directory on the remote side
         for name in shlibs:
@@ -1905,6 +1902,7 @@ def registerSharedLibrariesWithTarget(self, target, shlibs):
                 # Make sure we found the local shared library in the above code
                 self.assertTrue(os.path.exists(local_shlib_path))
 
+
             # Add the shared library to our target
             shlib_module = target.AddModule(local_shlib_path, None, None, None)
             if lldb.remote_platform:
@@ -1914,8 +1912,15 @@ def registerSharedLibrariesWithTarget(self, target, shlibs):
                     os.path.basename(local_shlib_path))
                 shlib_module.SetRemoteInstallFileSpec(
                     lldb.SBFileSpec(remote_shlib_path, False))
+                dir_to_add = self.get_process_working_directory()
+            else:
+                dir_to_add = os.path.dirname(local_shlib_path)
+
+            if dir_to_add not in dirs:
+                dirs.append(dir_to_add)
 
-        return environment
+        env_value = self.platformContext.shlib_path_separator.join(dirs)
+        return ['%s=%s' % (shlib_environment_var, env_value)]
 
     def registerSanitizerLibrariesWithTarget(self, target):
         runtimes = []
diff --git a/lldb/packages/Python/lldbsuite/test/lldbutil.py b/lldb/packages/Python/lldbsuite/test/lldbutil.py
index 25fcd5f29ee21..1ce6844d973c4 100644
--- a/lldb/packages/Python/lldbsuite/test/lldbutil.py
+++ b/lldb/packages/Python/lldbsuite/test/lldbutil.py
@@ -502,6 +502,29 @@ def run_break_set_by_source_regexp(
 
     return get_bpno_from_match(break_results)
 
+def run_break_set_by_file_colon_line(
+        test,
+        specifier,
+        path,
+        line_number,
+        column_number = 0,
+        extra_options=None,
+        num_expected_locations=-1):
+    command = 'breakpoint set -y "%s"'%(specifier)
+    if extra_options:
+        command += " " + extra_options
+
+    print("About to run: '%s'", command)
+    break_results = run_break_set_command(test, command)
+    check_breakpoint_result(
+        test,
+        break_results,
+        num_locations = num_expected_locations,
+        file_name = path,
+        line_number = line_number,
+        column_number = column_number)
+    
+    return get_bpno_from_match(break_results)
 
 def run_break_set_command(test, command):
     """Run the command passed in - it must be some break set variant - and analyze the result.
@@ -515,6 +538,7 @@ def run_break_set_command(test, command):
     If there is only one location, the dictionary MAY contain:
         file          - source file name
         line_no       - source line number
+        column        - source column number
         symbol        - symbol name
         inline_symbol - inlined symbol name
         offset        - offset from the original symbol
@@ -566,6 +590,7 @@ def check_breakpoint_result(
         break_results,
         file_name=None,
         line_number=-1,
+        column_number=0,
         symbol_name=None,
         symbol_match_exact=True,
         module_name=None,
@@ -605,6 +630,17 @@ def check_breakpoint_result(
             (line_number,
              out_line_number))
 
+    if column_number != 0:
+        out_column_number = 0
+        if 'column' in break_results:
+            out_column_number = break_results['column']
+
+        test.assertTrue(
+            column_number == out_column_number,
+            "Breakpoint column number %s doesn't match resultant column %s." %
+            (column_number,
+             out_column_number))
+
     if symbol_name:
         out_symbol_name = ""
         # Look first for the inlined symbol name, otherwise use the symbol
diff --git a/lldb/packages/Python/lldbsuite/test/make/Makefile.rules b/lldb/packages/Python/lldbsuite/test/make/Makefile.rules
index b9a6937650d05..43352b620b163 100644
--- a/lldb/packages/Python/lldbsuite/test/make/Makefile.rules
+++ b/lldb/packages/Python/lldbsuite/test/make/Makefile.rules
@@ -727,28 +727,16 @@ endif
 # and the -MM option will list all non-system dependencies.
 #----------------------------------------------------------------------
 %.d: %.c
-	@rm -f $@
-	@$(CC) -M $(CFLAGS) $< > $@.tmp && \
-	sed $(QUOTE)s,\($*\)\.o[ :]*,\1.o $@ : ,g$(QUOTE) < $@.tmp > $@
-	@rm -f $@.tmp
+	$(CC) -M $(CFLAGS) $< -MF $@ -MT $@ -MT $*.o
 
 %.d: %.cpp
-	@rm -f $@
-	@$(CXX) -M $(CXXFLAGS) $< > $@.tmp && \
-	sed $(QUOTE)s,\($*\)\.o[ :]*,\1.o $@ : ,g$(QUOTE) < $@.tmp > $@
-	@rm -f $@.tmp
+	@$(CXX) -M $(CXXFLAGS) $< -MF $@ -MT $@ -MT $*.o
 
 %.d: %.m
-	@rm -f $@
-	@$(CC) -M $(CFLAGS) $< > $@.tmp && \
-	sed $(QUOTE)s,\($*\)\.o[ :]*,\1.o $@ : ,g$(QUOTE) < $@.tmp > $@
-	@rm -f $@.tmp
+	@$(CC) -M $(CFLAGS) $< -MF $@ -MT $@ -MT $*.o
 
 %.d: %.mm
-	@rm -f $@
-	@$(CXX) -M $(CXXFLAGS) $< > $@.tmp && \
-	sed $(QUOTE)s,\($*\)\.o[ :]*,\1.o $@ : ,g$(QUOTE) < $@.tmp > $@
-	@rm -f $@.tmp
+	@$(CXX) -M $(CXXFLAGS) $< -MF $@ -MT $@ -MT $*.o
 
 #----------------------------------------------------------------------
 # Include all of the makefiles for each source file so we don't have
diff --git a/lldb/packages/Python/lldbsuite/test/test_result.py b/lldb/packages/Python/lldbsuite/test/test_result.py
index 7e13e09d9bf05..54affca86057e 100644
--- a/lldb/packages/Python/lldbsuite/test/test_result.py
+++ b/lldb/packages/Python/lldbsuite/test/test_result.py
@@ -15,7 +15,6 @@
 
 # LLDB Modules
 from . import configuration
-from lldbsuite.test_event.event_builder import EventBuilder
 from lldbsuite.test_event import build_exception
 
 
@@ -72,7 +71,6 @@ def __init__(self, *args):
         # This counts from 1 .. suite.countTestCases().
         self.counter = 0
         (width, height) = LLDBTestResult.getTerminalSize()
-        self.results_formatter = configuration.results_formatter_object
 
     def _config_string(self, test):
         compiler = getattr(test, "getCompiler", None)
@@ -181,9 +179,6 @@ def startTest(self, test):
         if self.showAll:
             self.stream.write(self.fmt % self.counter)
         super(LLDBTestResult, self).startTest(test)
-        if self.results_formatter:
-            self.results_formatter.handle_event(
-                EventBuilder.event_for_start(test))
 
     def addSuccess(self, test):
         if (self.checkExclusion(
@@ -197,9 +192,6 @@ def addSuccess(self, test):
         self.stream.write(
             "PASS: LLDB (%s) :: %s\n" %
             (self._config_string(test), str(test)))
-        if self.results_formatter:
-            self.results_formatter.handle_event(
-                EventBuilder.event_for_success(test))
 
     def _isBuildError(self, err_tuple):
         exception = err_tuple[1]
@@ -228,13 +220,6 @@ def addError(self, test, err):
         self.stream.write(
             "FAIL: LLDB (%s) :: %s\n" %
             (self._config_string(test), str(test)))
-        if self.results_formatter:
-            # Handle build errors as a separate event type
-            if self._isBuildError(err):
-                error_event = EventBuilder.event_for_build_error(test, err)
-            else:
-                error_event = EventBuilder.event_for_error(test, err)
-            self.results_formatter.handle_event(error_event)
 
     def addCleanupError(self, test, err):
         configuration.sdir_has_content = True
@@ -245,10 +230,6 @@ def addCleanupError(self, test, err):
         self.stream.write(
             "CLEANUP ERROR: LLDB (%s) :: %s\n" %
             (self._config_string(test), str(test)))
-        if self.results_formatter:
-            self.results_formatter.handle_event(
-                EventBuilder.event_for_cleanup_error(
-                    test, err))
 
     def addFailure(self, test, err):
         if (self.checkExclusion(
@@ -274,9 +255,6 @@ def addFailure(self, test, err):
                         category] = configuration.failures_per_category[category] + 1
                 else:
                     configuration.failures_per_category[category] = 1
-        if self.results_formatter:
-            self.results_formatter.handle_event(
-                EventBuilder.event_for_failure(test, err))
 
     def addExpectedFailure(self, test, err, bugnumber):
         configuration.sdir_has_content = True
@@ -287,10 +265,6 @@ def addExpectedFailure(self, test, err, bugnumber):
         self.stream.write(
             "XFAIL: LLDB (%s) :: %s\n" %
             (self._config_string(test), str(test)))
-        if self.results_formatter:
-            self.results_formatter.handle_event(
-                EventBuilder.event_for_expected_failure(
-                    test, err, bugnumber))
 
     def addSkip(self, test, reason):
         configuration.sdir_has_content = True
@@ -301,9 +275,6 @@ def addSkip(self, test, reason):
         self.stream.write(
             "UNSUPPORTED: LLDB (%s) :: %s (%s) \n" %
             (self._config_string(test), str(test), reason))
-        if self.results_formatter:
-            self.results_formatter.handle_event(
-                EventBuilder.event_for_skip(test, reason))
 
     def addUnexpectedSuccess(self, test, bugnumber):
         configuration.sdir_has_content = True
@@ -314,7 +285,3 @@ def addUnexpectedSuccess(self, test, bugnumber):
         self.stream.write(
             "XPASS: LLDB (%s) :: %s\n" %
             (self._config_string(test), str(test)))
-        if self.results_formatter:
-            self.results_formatter.handle_event(
-                EventBuilder.event_for_unexpected_success(
-                    test, bugnumber))
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py
index d702d8ee68208..71e6fdd8857f5 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py
@@ -1537,7 +1537,7 @@ def single_step_only_steps_one_instruction(
         g_c2_address = int(context.get("g_c2_address"), 16)
 
         # Set a breakpoint at the given address.
-        if self.getArchitecture() == "arm":
+        if self.getArchitecture().startswith("arm"):
             # TODO: Handle case when setting breakpoint in thumb code
             BREAKPOINT_KIND = 4
         else:
@@ -1601,8 +1601,18 @@ def single_step_only_steps_one_instruction(
         # variable value
         if re.match("s390x", arch):
             expected_step_count = 2
+        # ARM64 requires "4" instructions: 2 to compute the address (adrp, add),
+        # one to materialize the constant (mov) and the store
+        if re.match("arm64", arch):
+            expected_step_count = 4
+
         self.assertEqual(step_count, expected_step_count)
 
+        # ARM64: Once addresses and constants are materialized, only one
+        # instruction is needed.
+        if re.match("arm64", arch):
+            expected_step_count = 1
+
         # Verify we hit the next state.
         args["expected_g_c1"] = "0"
         args["expected_g_c2"] = "0"
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/socket_packet_pump.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-server/socket_packet_pump.py
index 958d6449b5162..3de76345896dc 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/socket_packet_pump.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-server/socket_packet_pump.py
@@ -6,7 +6,6 @@
 import select
 import threading
 import traceback
-import codecs
 
 from six.moves import queue
 from lldbsuite.support import seven
@@ -25,7 +24,7 @@ def _handle_output_packet_string(packet_contents):
 
 def _dump_queue(the_queue):
     while not the_queue.empty():
-        print(codecs.encode(the_queue.get(True), "string_escape"))
+        print(repr(the_queue.get(True)))
         print("\n")
 
 
diff --git a/lldb/packages/Python/lldbsuite/test_event/build_exception.py b/lldb/packages/Python/lldbsuite/test_event/build_exception.py
index 3347d9fd7cf7e..993214edd8711 100644
--- a/lldb/packages/Python/lldbsuite/test_event/build_exception.py
+++ b/lldb/packages/Python/lldbsuite/test_event/build_exception.py
@@ -4,8 +4,7 @@ def __init__(self, called_process_error):
         super(BuildError, self).__init__("Error when building test subject")
         self.command = called_process_error.lldb_extensions.get(
             "command", "<command unavailable>")
-        self.build_error = called_process_error.lldb_extensions.get(
-            "stderr_content", "<error output unavailable>")
+        self.build_error = called_process_error.lldb_extensions["combined_output"]
 
     def __str__(self):
         return self.format_build_error(self.command, self.build_error)
diff --git a/lldb/packages/Python/lldbsuite/test_event/event_builder.py b/lldb/packages/Python/lldbsuite/test_event/event_builder.py
deleted file mode 100644
index 8759011abe5ef..0000000000000
--- a/lldb/packages/Python/lldbsuite/test_event/event_builder.py
+++ /dev/null
@@ -1,481 +0,0 @@
-"""
-Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-See https://llvm.org/LICENSE.txt for license information.
-SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-Provides a class to build Python test event data structures.
-"""
-
-from __future__ import print_function
-from __future__ import absolute_import
-
-# System modules
-import inspect
-import time
-import traceback
-
-# Third-party modules
-
-# LLDB modules
-from . import build_exception
-
-
-class EventBuilder(object):
-    """Helper class to build test result event dictionaries."""
-
-    BASE_DICTIONARY = None
-
-    # Test Event Types
-    TYPE_JOB_RESULT = "job_result"
-    TYPE_TEST_RESULT = "test_result"
-    TYPE_TEST_START = "test_start"
-    TYPE_MARK_TEST_RERUN_ELIGIBLE = "test_eligible_for_rerun"
-    TYPE_MARK_TEST_EXPECTED_FAILURE = "test_expected_failure"
-    TYPE_SESSION_TERMINATE = "terminate"
-
-    RESULT_TYPES = {TYPE_JOB_RESULT, TYPE_TEST_RESULT}
-
-    # Test/Job Status Tags
-    STATUS_EXCEPTIONAL_EXIT = "exceptional_exit"
-    STATUS_SUCCESS = "success"
-    STATUS_FAILURE = "failure"
-    STATUS_EXPECTED_FAILURE = "expected_failure"
-    STATUS_EXPECTED_TIMEOUT = "expected_timeout"
-    STATUS_UNEXPECTED_SUCCESS = "unexpected_success"
-    STATUS_SKIP = "skip"
-    STATUS_ERROR = "error"
-    STATUS_TIMEOUT = "timeout"
-
-    """Test methods or jobs with a status matching any of these
-    status values will cause a testrun failure, unless
-    the test methods rerun and do not trigger an issue when rerun."""
-    TESTRUN_ERROR_STATUS_VALUES = {
-        STATUS_ERROR,
-        STATUS_EXCEPTIONAL_EXIT,
-        STATUS_FAILURE,
-        STATUS_TIMEOUT}
-
-    @staticmethod
-    def _get_test_name_info(test):
-        """Returns (test-class-name, test-method-name) from a test case instance.
-
-        @param test a unittest.TestCase instance.
-
-        @return tuple containing (test class name, test method name)
-        """
-        test_class_components = test.id().split(".")
-        test_class_name = ".".join(test_class_components[:-1])
-        test_name = test_class_components[-1]
-        return test_class_name, test_name
-
-    @staticmethod
-    def bare_event(event_type):
-        """Creates an event with default additions, event type and timestamp.
-
-        @param event_type the value set for the "event" key, used
-        to distinguish events.
-
-        @returns an event dictionary with all default additions, the "event"
-        key set to the passed in event_type, and the event_time value set to
-        time.time().
-        """
-        if EventBuilder.BASE_DICTIONARY is not None:
-            # Start with a copy of the "always include" entries.
-            event = dict(EventBuilder.BASE_DICTIONARY)
-        else:
-            event = {}
-
-        event.update({
-            "event": event_type,
-            "event_time": time.time()
-        })
-        return event
-
-    @staticmethod
-    def _assert_is_python_sourcefile(test_filename):
-        if test_filename is not None:
-            if not test_filename.endswith(".py"):
-                raise Exception(
-                    "source python filename has unexpected extension: {}".format(test_filename))
-        return test_filename
-
-    @staticmethod
-    def _event_dictionary_common(test, event_type):
-        """Returns an event dictionary setup with values for the given event type.
-
-        @param test the unittest.TestCase instance
-
-        @param event_type the name of the event type (string).
-
-        @return event dictionary with common event fields set.
-        """
-        test_class_name, test_name = EventBuilder._get_test_name_info(test)
-
-        # Determine the filename for the test case.  If there is an attribute
-        # for it, use it.  Otherwise, determine from the TestCase class path.
-        if hasattr(test, "test_filename"):
-            test_filename = EventBuilder._assert_is_python_sourcefile(
-                test.test_filename)
-        else:
-            test_filename = EventBuilder._assert_is_python_sourcefile(
-                inspect.getsourcefile(test.__class__))
-
-        event = EventBuilder.bare_event(event_type)
-        event.update({
-            "test_class": test_class_name,
-            "test_name": test_name,
-            "test_filename": test_filename
-        })
-
-        return event
-
-    @staticmethod
-    def _error_tuple_class(error_tuple):
-        """Returns the unittest error tuple's error class as a string.
-
-        @param error_tuple the error tuple provided by the test framework.
-
-        @return the error type (typically an exception) raised by the
-        test framework.
-        """
-        type_var = error_tuple[0]
-        module = inspect.getmodule(type_var)
-        if module:
-            return "{}.{}".format(module.__name__, type_var.__name__)
-        else:
-            return type_var.__name__
-
-    @staticmethod
-    def _error_tuple_message(error_tuple):
-        """Returns the unittest error tuple's error message.
-
-        @param error_tuple the error tuple provided by the test framework.
-
-        @return the error message provided by the test framework.
-        """
-        return str(error_tuple[1])
-
-    @staticmethod
-    def _error_tuple_traceback(error_tuple):
-        """Returns the unittest error tuple's error message.
-
-        @param error_tuple the error tuple provided by the test framework.
-
-        @return the error message provided by the test framework.
-        """
-        return error_tuple[2]
-
-    @staticmethod
-    def _event_dictionary_test_result(test, status):
-        """Returns an event dictionary with common test result fields set.
-
-        @param test a unittest.TestCase instance.
-
-        @param status the status/result of the test
-        (e.g. "success", "failure", etc.)
-
-        @return the event dictionary
-        """
-        event = EventBuilder._event_dictionary_common(
-            test, EventBuilder.TYPE_TEST_RESULT)
-        event["status"] = status
-        return event
-
-    @staticmethod
-    def _event_dictionary_issue(test, status, error_tuple):
-        """Returns an event dictionary with common issue-containing test result
-        fields set.
-
-        @param test a unittest.TestCase instance.
-
-        @param status the status/result of the test
-        (e.g. "success", "failure", etc.)
-
-        @param error_tuple the error tuple as reported by the test runner.
-        This is of the form (type<error>, error).
-
-        @return the event dictionary
-        """
-        event = EventBuilder._event_dictionary_test_result(test, status)
-        event["issue_class"] = EventBuilder._error_tuple_class(error_tuple)
-        event["issue_message"] = EventBuilder._error_tuple_message(error_tuple)
-        backtrace = EventBuilder._error_tuple_traceback(error_tuple)
-        if backtrace is not None:
-            event["issue_backtrace"] = traceback.format_tb(backtrace)
-        return event
-
-    @staticmethod
-    def event_for_start(test):
-        """Returns an event dictionary for the test start event.
-
-        @param test a unittest.TestCase instance.
-
-        @return the event dictionary
-        """
-        return EventBuilder._event_dictionary_common(
-            test, EventBuilder.TYPE_TEST_START)
-
-    @staticmethod
-    def event_for_success(test):
-        """Returns an event dictionary for a successful test.
-
-        @param test a unittest.TestCase instance.
-
-        @return the event dictionary
-        """
-        return EventBuilder._event_dictionary_test_result(
-            test, EventBuilder.STATUS_SUCCESS)
-
-    @staticmethod
-    def event_for_unexpected_success(test, bugnumber):
-        """Returns an event dictionary for a test that succeeded but was
-        expected to fail.
-
-        @param test a unittest.TestCase instance.
-
-        @param bugnumber the issue identifier for the bug tracking the
-        fix request for the test expected to fail (but is in fact
-        passing here).
-
-        @return the event dictionary
-
-        """
-        event = EventBuilder._event_dictionary_test_result(
-            test, EventBuilder.STATUS_UNEXPECTED_SUCCESS)
-        if bugnumber:
-            event["bugnumber"] = str(bugnumber)
-        return event
-
-    @staticmethod
-    def event_for_failure(test, error_tuple):
-        """Returns an event dictionary for a test that failed.
-
-        @param test a unittest.TestCase instance.
-
-        @param error_tuple the error tuple as reported by the test runner.
-        This is of the form (type<error>, error).
-
-        @return the event dictionary
-        """
-        return EventBuilder._event_dictionary_issue(
-            test, EventBuilder.STATUS_FAILURE, error_tuple)
-
-    @staticmethod
-    def event_for_expected_failure(test, error_tuple, bugnumber):
-        """Returns an event dictionary for a test that failed as expected.
-
-        @param test a unittest.TestCase instance.
-
-        @param error_tuple the error tuple as reported by the test runner.
-        This is of the form (type<error>, error).
-
-        @param bugnumber the issue identifier for the bug tracking the
-        fix request for the test expected to fail.
-
-        @return the event dictionary
-
-        """
-        event = EventBuilder._event_dictionary_issue(
-            test, EventBuilder.STATUS_EXPECTED_FAILURE, error_tuple)
-        if bugnumber:
-            event["bugnumber"] = str(bugnumber)
-        return event
-
-    @staticmethod
-    def event_for_skip(test, reason):
-        """Returns an event dictionary for a test that was skipped.
-
-        @param test a unittest.TestCase instance.
-
-        @param reason the reason why the test is being skipped.
-
-        @return the event dictionary
-        """
-        event = EventBuilder._event_dictionary_test_result(
-            test, EventBuilder.STATUS_SKIP)
-        event["skip_reason"] = reason
-        return event
-
-    @staticmethod
-    def event_for_error(test, error_tuple):
-        """Returns an event dictionary for a test that hit a test execution error.
-
-        @param test a unittest.TestCase instance.
-
-        @param error_tuple the error tuple as reported by the test runner.
-        This is of the form (type<error>, error).
-
-        @return the event dictionary
-        """
-        event = EventBuilder._event_dictionary_issue(
-            test, EventBuilder.STATUS_ERROR, error_tuple)
-        event["issue_phase"] = "test"
-        return event
-
-    @staticmethod
-    def event_for_build_error(test, error_tuple):
-        """Returns an event dictionary for a test that hit a test execution error
-        during the test cleanup phase.
-
-        @param test a unittest.TestCase instance.
-
-        @param error_tuple the error tuple as reported by the test runner.
-        This is of the form (type<error>, error).
-
-        @return the event dictionary
-        """
-        event = EventBuilder._event_dictionary_issue(
-            test, EventBuilder.STATUS_ERROR, error_tuple)
-        event["issue_phase"] = "build"
-
-        build_error = error_tuple[1]
-        event["build_command"] = build_error.command
-        event["build_error"] = build_error.build_error
-        return event
-
-    @staticmethod
-    def event_for_cleanup_error(test, error_tuple):
-        """Returns an event dictionary for a test that hit a test execution error
-        during the test cleanup phase.
-
-        @param test a unittest.TestCase instance.
-
-        @param error_tuple the error tuple as reported by the test runner.
-        This is of the form (type<error>, error).
-
-        @return the event dictionary
-        """
-        event = EventBuilder._event_dictionary_issue(
-            test, EventBuilder.STATUS_ERROR, error_tuple)
-        event["issue_phase"] = "cleanup"
-        return event
-
-    @staticmethod
-    def event_for_job_test_add_error(test_filename, exception, backtrace):
-        event = EventBuilder.bare_event(EventBuilder.TYPE_JOB_RESULT)
-        event["status"] = EventBuilder.STATUS_ERROR
-        if test_filename is not None:
-            event["test_filename"] = EventBuilder._assert_is_python_sourcefile(
-                test_filename)
-        if exception is not None and "__class__" in dir(exception):
-            event["issue_class"] = exception.__class__
-        event["issue_message"] = exception
-        if backtrace is not None:
-            event["issue_backtrace"] = backtrace
-        return event
-
-    @staticmethod
-    def event_for_job_exceptional_exit(
-            pid, worker_index, exception_code, exception_description,
-            test_filename, command_line):
-        """Creates an event for a job (i.e. process) exit due to signal.
-
-        @param pid the process id for the job that failed
-        @param worker_index optional id for the job queue running the process
-        @param exception_code optional code
-        (e.g. SIGTERM integer signal number)
-        @param exception_description optional string containing symbolic
-        representation of the issue (e.g. "SIGTERM")
-        @param test_filename the path to the test filename that exited
-        in some exceptional way.
-        @param command_line the Popen()-style list provided as the command line
-        for the process that timed out.
-
-        @return an event dictionary coding the job completion description.
-        """
-        event = EventBuilder.bare_event(EventBuilder.TYPE_JOB_RESULT)
-        event["status"] = EventBuilder.STATUS_EXCEPTIONAL_EXIT
-        if pid is not None:
-            event["pid"] = pid
-        if worker_index is not None:
-            event["worker_index"] = int(worker_index)
-        if exception_code is not None:
-            event["exception_code"] = exception_code
-        if exception_description is not None:
-            event["exception_description"] = exception_description
-        if test_filename is not None:
-            event["test_filename"] = EventBuilder._assert_is_python_sourcefile(
-                test_filename)
-        if command_line is not None:
-            event["command_line"] = command_line
-        return event
-
-    @staticmethod
-    def event_for_job_timeout(pid, worker_index, test_filename, command_line):
-        """Creates an event for a job (i.e. process) timeout.
-
-        @param pid the process id for the job that timed out
-        @param worker_index optional id for the job queue running the process
-        @param test_filename the path to the test filename that timed out.
-        @param command_line the Popen-style list provided as the command line
-        for the process that timed out.
-
-        @return an event dictionary coding the job completion description.
-        """
-        event = EventBuilder.bare_event(EventBuilder.TYPE_JOB_RESULT)
-        event["status"] = "timeout"
-        if pid is not None:
-            event["pid"] = pid
-        if worker_index is not None:
-            event["worker_index"] = int(worker_index)
-        if test_filename is not None:
-            event["test_filename"] = EventBuilder._assert_is_python_sourcefile(
-                test_filename)
-        if command_line is not None:
-            event["command_line"] = command_line
-        return event
-
-    @staticmethod
-    def event_for_mark_test_rerun_eligible(test):
-        """Creates an event that indicates the specified test is explicitly
-        eligible for rerun.
-
-        Note there is a mode that will enable test rerun eligibility at the
-        global level.  These markings for explicit rerun eligibility are
-        intended for the mode of running where only explicitly re-runnable
-        tests are rerun upon hitting an issue.
-
-        @param test the TestCase instance to which this pertains.
-
-        @return an event that specifies the given test as being eligible to
-        be rerun.
-        """
-        event = EventBuilder._event_dictionary_common(
-            test,
-            EventBuilder.TYPE_MARK_TEST_RERUN_ELIGIBLE)
-        return event
-
-    @staticmethod
-    def event_for_mark_test_expected_failure(test):
-        """Creates an event that indicates the specified test is expected
-        to fail.
-
-        @param test the TestCase instance to which this pertains.
-
-        @return an event that specifies the given test is expected to fail.
-        """
-        event = EventBuilder._event_dictionary_common(
-            test,
-            EventBuilder.TYPE_MARK_TEST_EXPECTED_FAILURE)
-        return event
-
-    @staticmethod
-    def add_entries_to_all_events(entries_dict):
-        """Specifies a dictionary of entries to add to all test events.
-
-        This provides a mechanism for, say, a parallel test runner to
-        indicate to each inferior dotest.py that it should add a
-        worker index to each.
-
-        Calling this method replaces all previous entries added
-        by a prior call to this.
-
-        Event build methods will overwrite any entries that collide.
-        Thus, the passed in dictionary is the base, which gets merged
-        over by event building when keys collide.
-
-        @param entries_dict a dictionary containing key and value
-        pairs that should be merged into all events created by the
-        event generator.  May be None to clear out any extra entries.
-        """
-        EventBuilder.BASE_DICTIONARY = dict(entries_dict)
diff --git a/lldb/packages/Python/lldbsuite/test_event/formatter/__init__.py b/lldb/packages/Python/lldbsuite/test_event/formatter/__init__.py
deleted file mode 100644
index d6609d353c856..0000000000000
--- a/lldb/packages/Python/lldbsuite/test_event/formatter/__init__.py
+++ /dev/null
@@ -1,66 +0,0 @@
-"""
-Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-See https://llvm.org/LICENSE.txt for license information.
-SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-"""
-
-from __future__ import print_function
-from __future__ import absolute_import
-
-# System modules
-import importlib
-import socket
-import sys
-
-# Third-party modules
-
-# LLDB modules
-
-
-# Ignore method count on DTOs.
-# pylint: disable=too-few-public-methods
-class CreatedFormatter(object):
-    """Provides transfer object for returns from create_results_formatter()."""
-
-    def __init__(self, formatter, cleanup_func):
-        self.formatter = formatter
-        self.cleanup_func = cleanup_func
-
-
-def create_results_formatter(formatter_name):
-    """Sets up a test results formatter.
-
-    @param config an instance of FormatterConfig
-    that indicates how to setup the ResultsFormatter.
-
-    @return an instance of CreatedFormatter.
-    """
-
-    # Create an instance of the class.
-    # First figure out the package/module.
-    components = formatter_name.split(".")
-    module = importlib.import_module(".".join(components[:-1]))
-
-    # Create the class name we need to load.
-    cls = getattr(module, components[-1])
-
-    # Handle formatter options for the results formatter class.
-    formatter_arg_parser = cls.arg_parser()
-    command_line_options = []
-
-    formatter_options = formatter_arg_parser.parse_args(
-        command_line_options)
-
-    # Create the TestResultsFormatter given the processed options.
-    results_formatter_object = cls(sys.stdout, formatter_options)
-
-    def shutdown_formatter():
-        """Shuts down the formatter when it is no longer needed."""
-        # Tell the formatter to write out anything it may have
-        # been saving until the very end (e.g. xUnit results
-        # can't complete its output until this point).
-        results_formatter_object.send_terminate_as_needed()
-
-    return CreatedFormatter(
-        results_formatter_object,
-        shutdown_formatter)
diff --git a/lldb/packages/Python/lldbsuite/test_event/formatter/results_formatter.py b/lldb/packages/Python/lldbsuite/test_event/formatter/results_formatter.py
deleted file mode 100644
index 140a319287407..0000000000000
--- a/lldb/packages/Python/lldbsuite/test_event/formatter/results_formatter.py
+++ /dev/null
@@ -1,764 +0,0 @@
-"""
-Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-See https://llvm.org/LICENSE.txt for license information.
-SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-Provides classes used by the test results reporting infrastructure
-within the LLDB test suite.
-"""
-
-from __future__ import print_function
-from __future__ import absolute_import
-
-# System modules
-import argparse
-import os
-import re
-import sys
-import threading
-
-# Third-party modules
-
-
-# LLDB modules
-from lldbsuite.test import configuration
-from ..event_builder import EventBuilder
-
-import lldbsuite
-
-
-FILE_LEVEL_KEY_RE = re.compile(r"^(.+\.py)[^.:]*$")
-
-
-class ResultsFormatter(object):
-    """Provides interface to formatting test results out to a file-like object.
-
-    This class allows the LLDB test framework's raw test-related
-    events to be processed and formatted in any manner desired.
-    Test events are represented by python dictionaries, formatted
-    as in the EventBuilder class above.
-
-    ResultFormatter instances are given a file-like object in which
-    to write their results.
-
-    ResultFormatter lifetime looks like the following:
-
-    # The result formatter is created.
-    # The argparse options dictionary is generated from calling
-    # the SomeResultFormatter.arg_parser() with the options data
-    # passed to dotest.py via the "--results-formatter-options"
-    # argument.  See the help on that for syntactic requirements
-    # on getting that parsed correctly.
-    formatter = SomeResultFormatter(file_like_object, argparse_options_dict)
-
-    # Single call to session start, before parsing any events.
-    formatter.begin_session()
-
-    formatter.handle_event({"event":"initialize",...})
-
-    # Zero or more calls specified for events recorded during the test session.
-    # The parallel test runner manages getting results from all the inferior
-    # dotest processes, so from a new format perspective, don't worry about
-    # that.  The formatter will be presented with a single stream of events
-    # sandwiched between a single begin_session()/end_session() pair in the
-    # parallel test runner process/thread.
-    for event in zero_or_more_test_events():
-        formatter.handle_event(event)
-
-    # Single call to terminate/wrap-up. For formatters that need all the
-    # data before they can print a correct result (e.g. xUnit/JUnit),
-    # this is where the final report can be generated.
-    formatter.handle_event({"event":"terminate",...})
-
-    It is not the formatter's responsibility to close the file_like_object.
-    (i.e. do not close it).
-
-    The lldb test framework passes these test events in real time, so they
-    arrive as they come in.
-
-    In the case of the parallel test runner, the dotest inferiors
-    add a 'pid' field to the dictionary that indicates which inferior
-    pid generated the event.
-
-    Note more events may be added in the future to support richer test
-    reporting functionality. One example: creating a true flaky test
-    result category so that unexpected successes really mean the test
-    is marked incorrectly (either should be marked flaky, or is indeed
-    passing consistently now and should have the xfail marker
-    removed). In this case, a flaky_success and flaky_fail event
-    likely will be added to capture these and support reporting things
-    like percentages of flaky test passing so we can see if we're
-    making some things worse/better with regards to failure rates.
-
-    Another example: announcing all the test methods that are planned
-    to be run, so we can better support redo operations of various kinds
-    (redo all non-run tests, redo non-run tests except the one that
-    was running [perhaps crashed], etc.)
-
-    Implementers are expected to override all the public methods
-    provided in this class. See each method's docstring to see
-    expectations about when the call should be chained.
-
-    """
-    @classmethod
-    def arg_parser(cls):
-        """@return arg parser used to parse formatter-specific options."""
-        parser = argparse.ArgumentParser(
-            description='{} options'.format(cls.__name__),
-            usage=('dotest.py --results-formatter-options='
-                   '"--option1 value1 [--option2 value2 [...]]"'))
-        parser.add_argument(
-            "--dump-results",
-            action="store_true",
-            help=('dump the raw results data after printing '
-                  'the summary output.'))
-        return parser
-
-    def __init__(self, out_file, options):
-        super(ResultsFormatter, self).__init__()
-        self.out_file = out_file
-        self.options = options
-        self.using_terminal = False
-        if not self.out_file:
-            raise Exception("ResultsFormatter created with no file object")
-        self.start_time_by_test = {}
-        self.terminate_called = False
-
-        # Track the most recent test start event by worker index.
-        # We'll use this to assign TIMEOUT and exceptional
-        # exits to the most recent test started on a given
-        # worker index.
-        self.started_tests_by_worker = {}
-
-        # Store the most recent test_method/job status.
-        self.result_events = {}
-
-        # Track the number of test method reruns.
-        self.test_method_rerun_count = 0
-
-        # Lock that we use while mutating inner state, like the
-        # total test count and the elements.  We minimize how
-        # long we hold the lock just to keep inner state safe, not
-        # entirely consistent from the outside.
-        self.lock = threading.RLock()
-
-        # Keeps track of the test base filenames for tests that
-        # are expected to timeout.  If a timeout occurs in any test
-        # basename that matches this list, that result should be
-        # converted into a non-issue.  We'll create an expected
-        # timeout test status for this.
-        self.expected_timeouts_by_basename = set()
-
-        # Tests which have reported that they are expecting to fail. These will
-        # be marked as expected failures even if they return a failing status,
-        # probably because they crashed or deadlocked.
-        self.expected_failures = set()
-
-        # Keep track of rerun-eligible tests.
-        # This is a set that contains tests saved as:
-        # {test_filename}:{test_class}:{test_name}
-        self.rerun_eligible_tests = set()
-
-        # A dictionary of test files that had a failing
-        # test, in the format of:
-        # key = test path, value = array of test methods that need rerun
-        self.tests_for_rerun = {}
-
-    @classmethod
-    def _make_key(cls, result_event):
-        """Creates a key from a test or job result event.
-
-        This key attempts to be as unique as possible.  For
-        test result events, it will be unique per test method.
-        For job events (ones not promoted to a test result event),
-        it will be unique per test case file.
-
-        @return a string-based key of the form
-        {test_filename}:{test_class}.{test_name}
-        """
-        if result_event is None:
-            return None
-        component_count = 0
-        if "test_filename" in result_event:
-            key = result_event["test_filename"]
-            component_count += 1
-        else:
-            key = "<no_filename>"
-        if "test_class" in result_event:
-            if component_count > 0:
-                key += ":"
-            key += result_event["test_class"]
-            component_count += 1
-        if "test_name" in result_event:
-            if component_count > 0:
-                key += "."
-            key += result_event["test_name"]
-            component_count += 1
-        return key
-
-    @classmethod
-    def _is_file_level_issue(cls, key, event):
-        """Returns whether a given key represents a file-level event.
-
-        @param cls this class.  Unused, but following PEP8 for
-        preferring @classmethod over @staticmethod.
-
-        @param key the key for the issue being tested.
-
-        @param event the event for the issue being tested.
-
-        @return True when the given key (as made by _make_key())
-        represents an event that is at the test file level (i.e.
-        it isn't scoped to a test class or method).
-        """
-        if key is None:
-            return False
-        else:
-            return FILE_LEVEL_KEY_RE.match(key) is not None
-
-    def _mark_test_as_expected_failure(self, test_result_event):
-        key = self._make_key(test_result_event)
-        if key is not None:
-            self.expected_failures.add(key)
-        else:
-            sys.stderr.write(
-                "\nerror: test marked as expected failure but "
-                "failed to create key.\n")
-
-    def _mark_test_for_rerun_eligibility(self, test_result_event):
-        key = self._make_key(test_result_event)
-        if key is not None:
-            self.rerun_eligible_tests.add(key)
-        else:
-            sys.stderr.write(
-                "\nerror: test marked for re-run eligibility but "
-                "failed to create key.\n")
-
-    def _maybe_add_test_to_rerun_list(self, result_event):
-        key = self._make_key(result_event)
-        if key is not None:
-            if (key in self.rerun_eligible_tests or
-                    configuration.rerun_all_issues):
-                test_filename = result_event.get("test_filename", None)
-                if test_filename is not None:
-                    test_name = result_event.get("test_name", None)
-                    if test_filename not in self.tests_for_rerun:
-                        self.tests_for_rerun[test_filename] = []
-                    if test_name is not None:
-                        self.tests_for_rerun[test_filename].append(test_name)
-        else:
-            sys.stderr.write(
-                "\nerror: couldn't add testrun-failing test to rerun "
-                "list because no eligibility key could be created.\n")
-
-    def _maybe_remap_job_result_event(self, test_event):
-        """Remaps timeout/exceptional exit job results to last test method running.
-
-        @param test_event the job_result test event.  This is an in/out
-        parameter.  It will be modified if it can be mapped to a test_result
-        of the same status, using details from the last-running test method
-        known to be most recently started on the same worker index.
-        """
-        test_start = None
-
-        job_status = test_event["status"]
-        if job_status in [
-                EventBuilder.STATUS_TIMEOUT,
-                EventBuilder.STATUS_EXCEPTIONAL_EXIT]:
-            worker_index = test_event.get("worker_index", None)
-            if worker_index is not None:
-                test_start = self.started_tests_by_worker.get(
-                    worker_index, None)
-
-        # If we have a test start to remap, do it here.
-        if test_start is not None:
-            test_event["event"] = EventBuilder.TYPE_TEST_RESULT
-
-            # Fill in all fields from test start not present in
-            # job status message.
-            for (start_key, start_value) in test_start.items():
-                if start_key not in test_event:
-                    test_event[start_key] = start_value
-
-    def _maybe_remap_expected_timeout(self, event):
-        if event is None:
-            return
-
-        status = event.get("status", None)
-        if status is None or status != EventBuilder.STATUS_TIMEOUT:
-            return
-
-        # Check if the timeout test's basename is in the expected timeout
-        # list.  If so, convert to an expected timeout.
-        basename = os.path.basename(event.get("test_filename", ""))
-        if basename in self.expected_timeouts_by_basename:
-            # Convert to an expected timeout.
-            event["status"] = EventBuilder.STATUS_EXPECTED_TIMEOUT
-
-    def _maybe_remap_expected_failure(self, event):
-        if event is None:
-            return
-
-        key = self._make_key(event)
-        if key not in self.expected_failures:
-            return
-
-        status = event.get("status", None)
-        if status in EventBuilder.TESTRUN_ERROR_STATUS_VALUES:
-            event["status"] = EventBuilder.STATUS_EXPECTED_FAILURE
-        elif status == EventBuilder.STATUS_SUCCESS:
-            event["status"] = EventBuilder.STATUS_UNEXPECTED_SUCCESS
-
-    def handle_event(self, test_event):
-        """Handles the test event for collection into the formatter output.
-
-        Derived classes may override this but should call down to this
-        implementation first.
-
-        @param test_event the test event as formatted by one of the
-        event_for_* calls.
-        """
-        with self.lock:
-            # Keep track of whether terminate was received.  We do this so
-            # that a process can call the 'terminate' event on its own, to
-            # close down a formatter at the appropriate time.  Then the
-            # atexit() cleanup can call the "terminate if it hasn't been
-            # called yet".
-            if test_event is not None:
-                event_type = test_event.get("event", "")
-                # We intentionally allow event_type to be checked anew
-                # after this check below since this check may rewrite
-                # the event type
-                if event_type == EventBuilder.TYPE_JOB_RESULT:
-                    # Possibly convert the job status (timeout,
-                    # exceptional exit) # to an appropriate test_result event.
-                    self._maybe_remap_job_result_event(test_event)
-                    event_type = test_event.get("event", "")
-
-                # Remap timeouts to expected timeouts.
-                if event_type in EventBuilder.RESULT_TYPES:
-                    self._maybe_remap_expected_timeout(test_event)
-                    self._maybe_remap_expected_failure(test_event)
-                    event_type = test_event.get("event", "")
-
-                if event_type == "terminate":
-                    self.terminate_called = True
-                elif event_type in EventBuilder.RESULT_TYPES:
-                    # Clear the most recently started test for the related
-                    # worker.
-                    worker_index = test_event.get("worker_index", None)
-                    if worker_index is not None:
-                        self.started_tests_by_worker.pop(worker_index, None)
-                    status = test_event["status"]
-                    if status in EventBuilder.TESTRUN_ERROR_STATUS_VALUES:
-                        # A test/job status value in any of those status values
-                        # causes a testrun failure. If such a test fails, check
-                        # whether it can be rerun. If it can be rerun, add it
-                        # to the rerun job.
-                        self._maybe_add_test_to_rerun_list(test_event)
-
-                    # Build the test key.
-                    test_key = self._make_key(test_event)
-                    if test_key is None:
-                        raise Exception(
-                            "failed to find test filename for "
-                            "test event {}".format(test_event))
-
-                    # Save the most recent test event for the test key. This
-                    # allows a second test phase to overwrite the most recent
-                    # result for the test key (unique per method). We do final
-                    # reporting at the end, so we'll report based on final
-                    # results. We do this so that a re-run caused by, perhaps,
-                    # the need to run a low-load, single-worker test run can
-                    # have the final run's results to always be used.
-                    if test_key in self.result_events:
-                        self.test_method_rerun_count += 1
-                    self.result_events[test_key] = test_event
-                elif event_type == EventBuilder.TYPE_TEST_START:
-                    # Track the start time for the test method.
-                    self.track_start_time(
-                        test_event["test_class"],
-                        test_event["test_name"],
-                        test_event["event_time"])
-                    # Track of the most recent test method start event
-                    # for the related worker.  This allows us to figure
-                    # out whether a process timeout or exceptional exit
-                    # can be charged (i.e. assigned) to a test method.
-                    worker_index = test_event.get("worker_index", None)
-                    if worker_index is not None:
-                        self.started_tests_by_worker[worker_index] = test_event
-
-                elif event_type == EventBuilder.TYPE_MARK_TEST_RERUN_ELIGIBLE:
-                    self._mark_test_for_rerun_eligibility(test_event)
-                elif (event_type ==
-                      EventBuilder.TYPE_MARK_TEST_EXPECTED_FAILURE):
-                    self._mark_test_as_expected_failure(test_event)
-
-    def set_expected_timeouts_by_basename(self, basenames):
-        """Specifies a list of test file basenames that are allowed to timeout
-        without being called out as a timeout issue.
-
-        These fall into a new status category called STATUS_EXPECTED_TIMEOUT.
-        """
-        if basenames is not None:
-            for basename in basenames:
-                self.expected_timeouts_by_basename.add(basename)
-
-    def track_start_time(self, test_class, test_name, start_time):
-        """tracks the start time of a test so elapsed time can be computed.
-
-        this alleviates the need for test results to be processed serially
-        by test.  it will save the start time for the test so that
-        elapsed_time_for_test() can compute the elapsed time properly.
-        """
-        if test_class is None or test_name is None:
-            return
-
-        test_key = "{}.{}".format(test_class, test_name)
-        self.start_time_by_test[test_key] = start_time
-
-    def elapsed_time_for_test(self, test_class, test_name, end_time):
-        """returns the elapsed time for a test.
-
-        this function can only be called once per test and requires that
-        the track_start_time() method be called sometime prior to calling
-        this method.
-        """
-        if test_class is None or test_name is None:
-            return -2.0
-
-        test_key = "{}.{}".format(test_class, test_name)
-        if test_key not in self.start_time_by_test:
-            return -1.0
-        else:
-            start_time = self.start_time_by_test[test_key]
-        del self.start_time_by_test[test_key]
-        return end_time - start_time
-
-    def is_using_terminal(self):
-        """returns true if this results formatter is using the terminal and
-        output should be avoided."""
-        return self.using_terminal
-
-    def send_terminate_as_needed(self):
-        """sends the terminate event if it hasn't been received yet."""
-        if not self.terminate_called:
-            terminate_event = EventBuilder.bare_event("terminate")
-            self.handle_event(terminate_event)
-
-    # Derived classes may require self access
-    # pylint: disable=no-self-use
-    # noinspection PyMethodMayBeStatic,PyMethodMayBeStatic
-    def replaces_summary(self):
-        """Returns whether the results formatter includes a summary
-        suitable to replace the old lldb test run results.
-
-        @return True if the lldb test runner can skip its summary
-        generation when using this results formatter; False otherwise.
-        """
-        return False
-
-    def counts_by_test_result_status(self, status):
-        """Returns number of test method results for the given status.
-
-        @status_result a test result status (e.g. success, fail, skip)
-        as defined by the EventBuilder.STATUS_* class members.
-
-        @return an integer returning the number of test methods matching
-        the given test result status.
-        """
-        return len([
-            [key, event] for (key, event) in self.result_events.items()
-            if event.get("status", "") == status])
-
-    @classmethod
-    def _event_sort_key(cls, event):
-        """Returns the sort key to be used for a test event.
-
-        This method papers over the differences in a test method result vs. a
-        job (i.e. inferior process) result.
-
-        @param event a test result or job result event.
-        @return a key useful for sorting events by name (test name preferably,
-        then by test filename).
-        """
-        if "test_name" in event:
-            return event["test_name"]
-        else:
-            return event.get("test_filename", None)
-
-    def _partition_results_by_status(self, categories):
-        """Partitions the captured test results by event status.
-
-        This permits processing test results by the category ids.
-
-        @param categories the list of categories on which to partition.
-        Follows the format described in _report_category_details().
-
-        @return a dictionary where each key is the test result status,
-        and each entry is a list containing all the test result events
-        that matched that test result status.  Result status IDs with
-        no matching entries will have a zero-length list.
-        """
-        partitioned_events = {}
-        for category in categories:
-            result_status_id = category[0]
-            matching_events = [
-                [key, event] for (key, event) in self.result_events.items()
-                if event.get("status", "") == result_status_id]
-            partitioned_events[result_status_id] = sorted(
-                matching_events,
-                key=lambda x: self._event_sort_key(x[1]))
-        return partitioned_events
-
-    @staticmethod
-    def _print_banner(out_file, banner_text):
-        """Prints an ASCII banner around given text.
-
-        Output goes to the out file for the results formatter.
-
-        @param out_file a file-like object where output will be written.
-        @param banner_text the text to display, with a banner
-        of '=' around the line above and line below.
-        """
-        banner_separator = "".ljust(len(banner_text), "=")
-
-        out_file.write("\n{}\n{}\n{}\n".format(
-            banner_separator,
-            banner_text,
-            banner_separator))
-
-    def _print_summary_counts(
-            self, out_file, categories, result_events_by_status, extra_rows):
-        """Prints summary counts for all categories.
-
-        @param out_file a file-like object used to print output.
-
-        @param categories the list of categories on which to partition.
-        Follows the format described in _report_category_details().
-
-        @param result_events_by_status the partitioned list of test
-        result events in a dictionary, with the key set to the test
-        result status id and the value set to the list of test method
-        results that match the status id.
-        """
-
-        # Get max length for category printed name
-        category_with_max_printed_name = max(
-            categories, key=lambda x: len(x[1]))
-        max_category_name_length = len(category_with_max_printed_name[1])
-
-        # If we are provided with extra rows, consider these row name lengths.
-        if extra_rows is not None:
-            for row in extra_rows:
-                name_length = len(row[0])
-                if name_length > max_category_name_length:
-                    max_category_name_length = name_length
-
-        self._print_banner(out_file, "Test Result Summary")
-
-        # Prepend extra rows
-        if extra_rows is not None:
-            for row in extra_rows:
-                extra_label = "{}:".format(row[0]).ljust(
-                    max_category_name_length + 1)
-                out_file.write("{} {:4}\n".format(extra_label, row[1]))
-
-        for category in categories:
-            result_status_id = category[0]
-            result_label = "{}:".format(category[1]).ljust(
-                max_category_name_length + 1)
-            count = len(result_events_by_status[result_status_id])
-            out_file.write("{} {:4}\n".format(
-                result_label,
-                count))
-
-    @classmethod
-    def _has_printable_details(cls, categories, result_events_by_status):
-        """Returns whether there are any test result details that need to be printed.
-
-        This will spin through the results and see if any result in a category
-        that is printable has any results to print.
-
-        @param categories the list of categories on which to partition.
-        Follows the format described in _report_category_details().
-
-        @param result_events_by_status the partitioned list of test
-        result events in a dictionary, with the key set to the test
-        result status id and the value set to the list of test method
-        results that match the status id.
-
-        @return True if there are any details (i.e. test results
-        for failures, errors, unexpected successes); False otherwise.
-        """
-        for category in categories:
-            result_status_id = category[0]
-            print_matching_tests = category[2]
-            if print_matching_tests:
-                if len(result_events_by_status[result_status_id]) > 0:
-                    # We found a printable details test result status
-                    # that has details to print.
-                    return True
-        # We didn't find any test result category with printable
-        # details.
-        return False
-
-    @staticmethod
-    def _report_category_details(out_file, category, result_events_by_status):
-        """Reports all test results matching the given category spec.
-
-        @param out_file a file-like object used to print output.
-
-        @param category a category spec of the format [test_event_name,
-        printed_category_name, print_matching_entries?]
-
-        @param result_events_by_status the partitioned list of test
-        result events in a dictionary, with the key set to the test
-        result status id and the value set to the list of test method
-        results that match the status id.
-        """
-        result_status_id = category[0]
-        print_matching_tests = category[2]
-        detail_label = category[3]
-
-        if print_matching_tests:
-            # Sort by test name
-            for (_, event) in result_events_by_status[result_status_id]:
-                # Convert full test path into test-root-relative.
-                test_relative_path = os.path.relpath(
-                    os.path.realpath(event["test_filename"]),
-                    lldbsuite.lldb_test_root)
-
-                # Create extra info component (used for exceptional exit info)
-                if result_status_id == EventBuilder.STATUS_EXCEPTIONAL_EXIT:
-                    extra_info = "[EXCEPTIONAL EXIT {} ({})] ".format(
-                        event["exception_code"],
-                        event["exception_description"])
-                else:
-                    extra_info = ""
-
-                # Figure out the identity we will use for this test.
-                if configuration.verbose and ("test_class" in event):
-                    test_id = "{}.{}".format(
-                        event["test_class"], event["test_name"])
-                elif "test_name" in event:
-                    test_id = event["test_name"]
-                else:
-                    test_id = "<no_running_test_method>"
-
-                # Display the info.
-                out_file.write("{}: {}{} ({})\n".format(
-                    detail_label,
-                    extra_info,
-                    test_id,
-                    test_relative_path))
-
-    def print_results(self, out_file):
-        """Writes the test result report to the output file.
-
-        @param out_file a file-like object used for printing summary
-        results.  This is different than self.out_file, which might
-        be something else for non-summary data.
-        """
-        extra_results = [
-            # Total test methods processed, excluding reruns.
-            ["Test Methods", len(self.result_events)],
-            ["Reruns", self.test_method_rerun_count]]
-
-        # Output each of the test result entries.
-        categories = [
-            # result id, printed name, print matching tests?, detail label
-            [EventBuilder.STATUS_SUCCESS,
-             "Success", False, None],
-            [EventBuilder.STATUS_EXPECTED_FAILURE,
-             "Expected Failure", False, None],
-            [EventBuilder.STATUS_FAILURE,
-             "Failure", True, "FAIL"],
-            [EventBuilder.STATUS_ERROR,
-             "Error", True, "ERROR"],
-            [EventBuilder.STATUS_EXCEPTIONAL_EXIT,
-             "Exceptional Exit", True, "ERROR"],
-            [EventBuilder.STATUS_UNEXPECTED_SUCCESS,
-             "Unexpected Success", True, "UNEXPECTED SUCCESS"],
-            [EventBuilder.STATUS_SKIP, "Skip", False, None],
-            [EventBuilder.STATUS_TIMEOUT,
-             "Timeout", True, "TIMEOUT"],
-            [EventBuilder.STATUS_EXPECTED_TIMEOUT,
-             # Intentionally using the unusual hyphenation in TIME-OUT to
-             # prevent buildbots from thinking it is an issue when scanning
-             # for TIMEOUT.
-             "Expected Timeout", True, "EXPECTED TIME-OUT"]
-        ]
-
-        # Partition all the events by test result status
-        result_events_by_status = self._partition_results_by_status(
-            categories)
-
-        # Print the details
-        have_details = self._has_printable_details(
-            categories, result_events_by_status)
-        if have_details:
-            self._print_banner(out_file, "Issue Details")
-            for category in categories:
-                self._report_category_details(
-                    out_file, category, result_events_by_status)
-
-        # Print the summary
-        self._print_summary_counts(
-            out_file, categories, result_events_by_status, extra_results)
-
-        if self.options.dump_results:
-            # Debug dump of the key/result info for all categories.
-            self._print_banner(out_file, "Results Dump")
-            for status, events_by_key in result_events_by_status.items():
-                out_file.write("\nSTATUS: {}\n".format(status))
-                for key, event in events_by_key:
-                    out_file.write("key:   {}\n".format(key))
-                    out_file.write("event: {}\n".format(event))
-
-    def clear_file_level_issues(self, tests_for_rerun, out_file):
-        """Clear file-charged issues in any of the test rerun files.
-
-        @param tests_for_rerun the list of test-dir-relative paths that have
-        functions that require rerunning.  This is the test list
-        returned by the results_formatter at the end of the previous run.
-
-        @return the number of file-level issues that were cleared.
-        """
-        if tests_for_rerun is None:
-            return 0
-
-        cleared_file_level_issues = 0
-        # Find the unique set of files that are covered by the given tests
-        # that are to be rerun.  We derive the files that are eligible for
-        # having their markers cleared, because we support running in a mode
-        # where only flaky tests are eligible for rerun.  If the file-level
-        # issue occurred in a file that was not marked as flaky, then we
-        # shouldn't be clearing the event here.
-        basename_set = set()
-        for test_file_relpath in tests_for_rerun:
-            basename_set.add(os.path.basename(test_file_relpath))
-
-        # Find all the keys for file-level events that are considered
-        # test issues.
-        file_level_issues = [(key, event)
-                             for key, event in self.result_events.items()
-                             if ResultsFormatter._is_file_level_issue(
-                                     key, event)
-                             and event.get("status", "") in
-                             EventBuilder.TESTRUN_ERROR_STATUS_VALUES]
-
-        # Now remove any file-level error for the given test base name.
-        for key, event in file_level_issues:
-            # If the given file base name is in the rerun set, then we
-            # clear that entry from the result set.
-            if os.path.basename(key) in basename_set:
-                self.result_events.pop(key, None)
-                cleared_file_level_issues += 1
-                if out_file is not None:
-                    out_file.write(
-                        "clearing file-level issue for file {} "
-                        "(issue type: {})"
-                        .format(key, event.get("status", "<unset-status>")))
-
-        return cleared_file_level_issues
diff --git a/lldb/source/API/SBLaunchInfo.cpp b/lldb/source/API/SBLaunchInfo.cpp
index ba13072e8f9bc..cda8134c9853b 100644
--- a/lldb/source/API/SBLaunchInfo.cpp
+++ b/lldb/source/API/SBLaunchInfo.cpp
@@ -190,9 +190,10 @@ void SBLaunchInfo::SetEnvironment(const SBEnvironment &env, bool append) {
   LLDB_RECORD_METHOD(void, SBLaunchInfo, SetEnvironment,
                      (const lldb::SBEnvironment &, bool), env, append);
   Environment &refEnv = env.ref();
-  if (append)
-    m_opaque_sp->GetEnvironment().insert(refEnv.begin(), refEnv.end());
-  else
+  if (append) {
+    for (auto &KV : refEnv)
+      m_opaque_sp->GetEnvironment().insert_or_assign(KV.first(), KV.second);
+  } else
     m_opaque_sp->GetEnvironment() = refEnv;
   m_opaque_sp->RegenerateEnvp();
 }
diff --git a/lldb/source/API/SBReproducer.cpp b/lldb/source/API/SBReproducer.cpp
index 0eb3429c4fefc..9815bf11263c5 100644
--- a/lldb/source/API/SBReproducer.cpp
+++ b/lldb/source/API/SBReproducer.cpp
@@ -234,7 +234,10 @@ const char *SBReproducer::GetPath() {
 
 void SBReproducer::SetWorkingDirectory(const char *path) {
   if (auto *g = lldb_private::repro::Reproducer::Instance().GetGenerator()) {
-    g->GetOrCreate<WorkingDirectoryProvider>().Update(path);
+    auto &wp = g->GetOrCreate<repro::WorkingDirectoryProvider>();
+    wp.Update(path);
+    auto &fp = g->GetOrCreate<repro::FileProvider>();
+    fp.RecordInterestingDirectory(wp.GetWorkingDirectory());
   }
 }
 
diff --git a/lldb/source/API/SBThreadPlan.cpp b/lldb/source/API/SBThreadPlan.cpp
index 1a947bbc26081..c740c8b7492fc 100644
--- a/lldb/source/API/SBThreadPlan.cpp
+++ b/lldb/source/API/SBThreadPlan.cpp
@@ -53,13 +53,13 @@ using namespace lldb_private;
 SBThreadPlan::SBThreadPlan() { LLDB_RECORD_CONSTRUCTOR_NO_ARGS(SBThreadPlan); }
 
 SBThreadPlan::SBThreadPlan(const ThreadPlanSP &lldb_object_sp)
-    : m_opaque_sp(lldb_object_sp) {
+    : m_opaque_wp(lldb_object_sp) {
   LLDB_RECORD_CONSTRUCTOR(SBThreadPlan, (const lldb::ThreadPlanSP &),
                           lldb_object_sp);
 }
 
 SBThreadPlan::SBThreadPlan(const SBThreadPlan &rhs)
-    : m_opaque_sp(rhs.m_opaque_sp) {
+    : m_opaque_wp(rhs.m_opaque_wp) {
   LLDB_RECORD_CONSTRUCTOR(SBThreadPlan, (const lldb::SBThreadPlan &), rhs);
 }
 
@@ -69,8 +69,8 @@ SBThreadPlan::SBThreadPlan(lldb::SBThread &sb_thread, const char *class_name) {
 
   Thread *thread = sb_thread.get();
   if (thread)
-    m_opaque_sp = std::make_shared<ThreadPlanPython>(*thread, class_name, 
-                                                     nullptr);
+    m_opaque_wp =
+        std::make_shared<ThreadPlanPython>(*thread, class_name, nullptr);
 }
 
 SBThreadPlan::SBThreadPlan(lldb::SBThread &sb_thread, const char *class_name,
@@ -81,7 +81,7 @@ SBThreadPlan::SBThreadPlan(lldb::SBThread &sb_thread, const char *class_name,
 
   Thread *thread = sb_thread.get();
   if (thread)
-    m_opaque_sp = std::make_shared<ThreadPlanPython>(*thread, class_name, 
+    m_opaque_wp = std::make_shared<ThreadPlanPython>(*thread, class_name,
                                                      args_data.m_impl_up.get());
 }
 
@@ -92,14 +92,12 @@ const lldb::SBThreadPlan &SBThreadPlan::operator=(const SBThreadPlan &rhs) {
                      SBThreadPlan, operator=,(const lldb::SBThreadPlan &), rhs);
 
   if (this != &rhs)
-    m_opaque_sp = rhs.m_opaque_sp;
+    m_opaque_wp = rhs.m_opaque_wp;
   return LLDB_RECORD_RESULT(*this);
 }
 // Destructor
 SBThreadPlan::~SBThreadPlan() = default;
 
-lldb_private::ThreadPlan *SBThreadPlan::get() { return m_opaque_sp.get(); }
-
 bool SBThreadPlan::IsValid() const {
   LLDB_RECORD_METHOD_CONST_NO_ARGS(bool, SBThreadPlan, IsValid);
   return this->operator bool();
@@ -107,13 +105,13 @@ bool SBThreadPlan::IsValid() const {
 SBThreadPlan::operator bool() const {
   LLDB_RECORD_METHOD_CONST_NO_ARGS(bool, SBThreadPlan, operator bool);
 
-  return m_opaque_sp.get() != nullptr;
+  return static_cast<bool>(GetSP());
 }
 
 void SBThreadPlan::Clear() {
   LLDB_RECORD_METHOD_NO_ARGS(void, SBThreadPlan, Clear);
 
-  m_opaque_sp.reset();
+  m_opaque_wp.reset();
 }
 
 lldb::StopReason SBThreadPlan::GetStopReason() {
@@ -138,9 +136,10 @@ uint64_t SBThreadPlan::GetStopReasonDataAtIndex(uint32_t idx) {
 SBThread SBThreadPlan::GetThread() const {
   LLDB_RECORD_METHOD_CONST_NO_ARGS(lldb::SBThread, SBThreadPlan, GetThread);
 
-  if (m_opaque_sp) {
+  ThreadPlanSP thread_plan_sp(GetSP());
+  if (thread_plan_sp) {
     return LLDB_RECORD_RESULT(
-        SBThread(m_opaque_sp->GetThread().shared_from_this()));
+        SBThread(thread_plan_sp->GetThread().shared_from_this()));
   } else
     return LLDB_RECORD_RESULT(SBThread());
 }
@@ -149,50 +148,52 @@ bool SBThreadPlan::GetDescription(lldb::SBStream &description) const {
   LLDB_RECORD_METHOD_CONST(bool, SBThreadPlan, GetDescription,
                            (lldb::SBStream &), description);
 
-  if (m_opaque_sp) {
-    m_opaque_sp->GetDescription(description.get(), eDescriptionLevelFull);
+  ThreadPlanSP thread_plan_sp(GetSP());
+  if (thread_plan_sp) {
+    thread_plan_sp->GetDescription(description.get(), eDescriptionLevelFull);
   } else {
     description.Printf("Empty SBThreadPlan");
   }
   return true;
 }
 
-void SBThreadPlan::SetThreadPlan(const ThreadPlanSP &lldb_object_sp) {
-  m_opaque_sp = lldb_object_sp;
+void SBThreadPlan::SetThreadPlan(const ThreadPlanSP &lldb_object_wp) {
+  m_opaque_wp = lldb_object_wp;
 }
 
 void SBThreadPlan::SetPlanComplete(bool success) {
   LLDB_RECORD_METHOD(void, SBThreadPlan, SetPlanComplete, (bool), success);
 
-  if (m_opaque_sp)
-    m_opaque_sp->SetPlanComplete(success);
+  ThreadPlanSP thread_plan_sp(GetSP());
+  if (thread_plan_sp)
+    thread_plan_sp->SetPlanComplete(success);
 }
 
 bool SBThreadPlan::IsPlanComplete() {
   LLDB_RECORD_METHOD_NO_ARGS(bool, SBThreadPlan, IsPlanComplete);
 
-  if (m_opaque_sp)
-    return m_opaque_sp->IsPlanComplete();
-  else
-    return true;
+  ThreadPlanSP thread_plan_sp(GetSP());
+  if (thread_plan_sp)
+    return thread_plan_sp->IsPlanComplete();
+  return true;
 }
 
 bool SBThreadPlan::IsPlanStale() {
   LLDB_RECORD_METHOD_NO_ARGS(bool, SBThreadPlan, IsPlanStale);
 
-  if (m_opaque_sp)
-    return m_opaque_sp->IsPlanStale();
-  else
-    return true;
+  ThreadPlanSP thread_plan_sp(GetSP());
+  if (thread_plan_sp)
+    return thread_plan_sp->IsPlanStale();
+  return true;
 }
 
 bool SBThreadPlan::IsValid() {
   LLDB_RECORD_METHOD_NO_ARGS(bool, SBThreadPlan, IsValid);
 
-  if (m_opaque_sp)
-    return m_opaque_sp->ValidatePlan(nullptr);
-  else
-    return false;
+  ThreadPlanSP thread_plan_sp(GetSP());
+  if (thread_plan_sp)
+    return thread_plan_sp->ValidatePlan(nullptr);
+  return false;
 }
 
 // This section allows an SBThreadPlan to push another of the common types of
@@ -220,7 +221,8 @@ SBThreadPlan SBThreadPlan::QueueThreadPlanForStepOverRange(
                      (lldb::SBAddress &, lldb::addr_t, lldb::SBError &),
                      sb_start_address, size, error);
 
-  if (m_opaque_sp) {
+  ThreadPlanSP thread_plan_sp(GetSP());
+  if (thread_plan_sp) {
     Address *start_address = sb_start_address.get();
     if (!start_address) {
       return LLDB_RECORD_RESULT(SBThreadPlan());
@@ -231,19 +233,18 @@ SBThreadPlan SBThreadPlan::QueueThreadPlanForStepOverRange(
     start_address->CalculateSymbolContext(&sc);
     Status plan_status;
 
-    SBThreadPlan plan =
-        SBThreadPlan(m_opaque_sp->GetThread().QueueThreadPlanForStepOverRange(
+    SBThreadPlan plan = SBThreadPlan(
+        thread_plan_sp->GetThread().QueueThreadPlanForStepOverRange(
             false, range, sc, eAllThreads, plan_status));
 
     if (plan_status.Fail())
       error.SetErrorString(plan_status.AsCString());
     else
-      plan.m_opaque_sp->SetPrivate(true);
-    
+      plan.GetSP()->SetPrivate(true);
+
     return LLDB_RECORD_RESULT(plan);
-  } else {
-    return LLDB_RECORD_RESULT(SBThreadPlan());
   }
+  return LLDB_RECORD_RESULT(SBThreadPlan());
 }
 
 SBThreadPlan
@@ -266,7 +267,8 @@ SBThreadPlan::QueueThreadPlanForStepInRange(SBAddress &sb_start_address,
                      (lldb::SBAddress &, lldb::addr_t, lldb::SBError &),
                      sb_start_address, size, error);
 
-  if (m_opaque_sp) {
+  ThreadPlanSP thread_plan_sp(GetSP());
+  if (thread_plan_sp) {
     Address *start_address = sb_start_address.get();
     if (!start_address) {
       return LLDB_RECORD_RESULT(SBThreadPlan());
@@ -278,18 +280,17 @@ SBThreadPlan::QueueThreadPlanForStepInRange(SBAddress &sb_start_address,
 
     Status plan_status;
     SBThreadPlan plan =
-        SBThreadPlan(m_opaque_sp->GetThread().QueueThreadPlanForStepInRange(
+        SBThreadPlan(thread_plan_sp->GetThread().QueueThreadPlanForStepInRange(
             false, range, sc, nullptr, eAllThreads, plan_status));
 
     if (plan_status.Fail())
       error.SetErrorString(plan_status.AsCString());
     else
-      plan.m_opaque_sp->SetPrivate(true);
+      plan.GetSP()->SetPrivate(true);
 
     return LLDB_RECORD_RESULT(plan);
-  } else {
-    return LLDB_RECORD_RESULT(SBThreadPlan());
   }
+  return LLDB_RECORD_RESULT(SBThreadPlan());
 }
 
 SBThreadPlan
@@ -312,26 +313,26 @@ SBThreadPlan::QueueThreadPlanForStepOut(uint32_t frame_idx_to_step_to,
                      (uint32_t, bool, lldb::SBError &), frame_idx_to_step_to,
                      first_insn, error);
 
-  if (m_opaque_sp) {
+  ThreadPlanSP thread_plan_sp(GetSP());
+  if (thread_plan_sp) {
     SymbolContext sc;
-    sc = m_opaque_sp->GetThread().GetStackFrameAtIndex(0)->GetSymbolContext(
+    sc = thread_plan_sp->GetThread().GetStackFrameAtIndex(0)->GetSymbolContext(
         lldb::eSymbolContextEverything);
 
     Status plan_status;
     SBThreadPlan plan =
-        SBThreadPlan(m_opaque_sp->GetThread().QueueThreadPlanForStepOut(
+        SBThreadPlan(thread_plan_sp->GetThread().QueueThreadPlanForStepOut(
             false, &sc, first_insn, false, eVoteYes, eVoteNoOpinion,
             frame_idx_to_step_to, plan_status));
 
     if (plan_status.Fail())
       error.SetErrorString(plan_status.AsCString());
     else
-      plan.m_opaque_sp->SetPrivate(true);
+      plan.GetSP()->SetPrivate(true);
 
     return LLDB_RECORD_RESULT(plan);
-  } else {
-    return LLDB_RECORD_RESULT(SBThreadPlan());
   }
+  return LLDB_RECORD_RESULT(SBThreadPlan());
 }
 
 SBThreadPlan
@@ -350,25 +351,25 @@ SBThreadPlan SBThreadPlan::QueueThreadPlanForRunToAddress(SBAddress sb_address,
                      QueueThreadPlanForRunToAddress,
                      (lldb::SBAddress, lldb::SBError &), sb_address, error);
 
-  if (m_opaque_sp) {
+  ThreadPlanSP thread_plan_sp(GetSP());
+  if (thread_plan_sp) {
     Address *address = sb_address.get();
     if (!address)
       return LLDB_RECORD_RESULT(SBThreadPlan());
 
     Status plan_status;
     SBThreadPlan plan =
-        SBThreadPlan(m_opaque_sp->GetThread().QueueThreadPlanForRunToAddress(
+        SBThreadPlan(thread_plan_sp->GetThread().QueueThreadPlanForRunToAddress(
             false, *address, false, plan_status));
 
     if (plan_status.Fail())
       error.SetErrorString(plan_status.AsCString());
     else
-      plan.m_opaque_sp->SetPrivate(true);
+      plan.GetSP()->SetPrivate(true);
 
     return LLDB_RECORD_RESULT(plan);
-  } else {
-    return LLDB_RECORD_RESULT(SBThreadPlan());
   }
+  return LLDB_RECORD_RESULT(SBThreadPlan());
 }
 
 SBThreadPlan
@@ -389,22 +390,22 @@ SBThreadPlan::QueueThreadPlanForStepScripted(const char *script_class_name,
                      QueueThreadPlanForStepScripted,
                      (const char *, lldb::SBError &), script_class_name, error);
 
-  if (m_opaque_sp) {
+  ThreadPlanSP thread_plan_sp(GetSP());
+  if (thread_plan_sp) {
     Status plan_status;
     StructuredData::ObjectSP empty_args;
     SBThreadPlan plan =
-        SBThreadPlan(m_opaque_sp->GetThread().QueueThreadPlanForStepScripted(
+        SBThreadPlan(thread_plan_sp->GetThread().QueueThreadPlanForStepScripted(
             false, script_class_name, empty_args, false, plan_status));
 
     if (plan_status.Fail())
       error.SetErrorString(plan_status.AsCString());
     else
-      plan.m_opaque_sp->SetPrivate(true);
+      plan.GetSP()->SetPrivate(true);
 
     return LLDB_RECORD_RESULT(plan);
-  } else {
-    return LLDB_RECORD_RESULT(SBThreadPlan());
   }
+  return LLDB_RECORD_RESULT(SBThreadPlan());
 }
 
 SBThreadPlan
@@ -416,17 +417,18 @@ SBThreadPlan::QueueThreadPlanForStepScripted(const char *script_class_name,
                      (const char *, lldb::SBStructuredData &, lldb::SBError &), 
                      script_class_name, args_data, error);
 
-  if (m_opaque_sp) {
+  ThreadPlanSP thread_plan_sp(GetSP());
+  if (thread_plan_sp) {
     Status plan_status;
     StructuredData::ObjectSP args_obj = args_data.m_impl_up->GetObjectSP();
     SBThreadPlan plan =
-        SBThreadPlan(m_opaque_sp->GetThread().QueueThreadPlanForStepScripted(
+        SBThreadPlan(thread_plan_sp->GetThread().QueueThreadPlanForStepScripted(
             false, script_class_name, args_obj, false, plan_status));
 
     if (plan_status.Fail())
       error.SetErrorString(plan_status.AsCString());
     else
-      plan.m_opaque_sp->SetPrivate(true);
+      plan.GetSP()->SetPrivate(true);
 
     return LLDB_RECORD_RESULT(plan);
   } else {
diff --git a/lldb/source/API/SBType.cpp b/lldb/source/API/SBType.cpp
index 852630f2d01a6..1c1d4e3a3ed9c 100644
--- a/lldb/source/API/SBType.cpp
+++ b/lldb/source/API/SBType.cpp
@@ -214,8 +214,8 @@ SBType SBType::GetArrayElementType() {
     return LLDB_RECORD_RESULT(SBType());
   CompilerType canonical_type =
       m_opaque_sp->GetCompilerType(true).GetCanonicalType();
-  return LLDB_RECORD_RESULT(
-      SBType(TypeImplSP(new TypeImpl(canonical_type.GetArrayElementType()))));
+  return LLDB_RECORD_RESULT(SBType(
+      TypeImplSP(new TypeImpl(canonical_type.GetArrayElementType(nullptr)))));
 }
 
 SBType SBType::GetArrayType(uint64_t size) {
diff --git a/lldb/source/Breakpoint/Breakpoint.cpp b/lldb/source/Breakpoint/Breakpoint.cpp
index 317dfa2310942..88c7f3352f426 100644
--- a/lldb/source/Breakpoint/Breakpoint.cpp
+++ b/lldb/source/Breakpoint/Breakpoint.cpp
@@ -144,8 +144,7 @@ lldb::BreakpointSP Breakpoint::CreateFromStructuredData(
   bool success = breakpoint_dict->GetValueForKeyAsDictionary(
       BreakpointResolver::GetSerializationKey(), resolver_dict);
   if (!success) {
-    error.SetErrorStringWithFormat(
-        "Breakpoint data missing toplevel resolver key");
+    error.SetErrorString("Breakpoint data missing toplevel resolver key");
     return result_sp;
   }
 
diff --git a/lldb/source/Breakpoint/BreakpointID.cpp b/lldb/source/Breakpoint/BreakpointID.cpp
index 293baf4ad1c77..a37abcfa52aba 100644
--- a/lldb/source/Breakpoint/BreakpointID.cpp
+++ b/lldb/source/Breakpoint/BreakpointID.cpp
@@ -96,7 +96,7 @@ bool BreakpointID::StringIsBreakpointName(llvm::StringRef str, Status &error) {
   error.Clear();
   if (str.empty())
   {
-    error.SetErrorStringWithFormat("Empty breakpoint names are not allowed");
+    error.SetErrorString("Empty breakpoint names are not allowed");
     return false;
   }
 
diff --git a/lldb/source/Breakpoint/BreakpointIDList.cpp b/lldb/source/Breakpoint/BreakpointIDList.cpp
index 705bc5ee1c8dc..e6a5dceeb93b2 100644
--- a/lldb/source/Breakpoint/BreakpointIDList.cpp
+++ b/lldb/source/Breakpoint/BreakpointIDList.cpp
@@ -220,10 +220,10 @@ void BreakpointIDList::FindAndReplaceIDRanges(Args &old_args, Target *target,
         ((start_loc_id != LLDB_INVALID_BREAK_ID) &&
          (end_loc_id == LLDB_INVALID_BREAK_ID))) {
       new_args.Clear();
-      result.AppendErrorWithFormat("Invalid breakpoint id range:  Either "
-                                   "both ends of range must specify"
-                                   " a breakpoint location, or neither can "
-                                   "specify a breakpoint location.\n");
+      result.AppendError("Invalid breakpoint id range:  Either "
+                         "both ends of range must specify"
+                         " a breakpoint location, or neither can "
+                         "specify a breakpoint location.");
       result.SetStatus(eReturnStatusFailed);
       return;
     }
diff --git a/lldb/source/Breakpoint/BreakpointOptions.cpp b/lldb/source/Breakpoint/BreakpointOptions.cpp
index 0ce7b0f852e8c..f6bb7633d0a95 100644
--- a/lldb/source/Breakpoint/BreakpointOptions.cpp
+++ b/lldb/source/Breakpoint/BreakpointOptions.cpp
@@ -319,7 +319,7 @@ std::unique_ptr<BreakpointOptions> BreakpointOptions::CreateFromStructuredData(
     else {
       ScriptInterpreter *interp = target.GetDebugger().GetScriptInterpreter();
       if (!interp) {
-        error.SetErrorStringWithFormat(
+        error.SetErrorString(
             "Can't set script commands - no script interpreter");
         return nullptr;
       }
diff --git a/lldb/source/Breakpoint/BreakpointResolver.cpp b/lldb/source/Breakpoint/BreakpointResolver.cpp
index 7c03a0745ac61..cfd073367b00a 100644
--- a/lldb/source/Breakpoint/BreakpointResolver.cpp
+++ b/lldb/source/Breakpoint/BreakpointResolver.cpp
@@ -81,8 +81,7 @@ BreakpointResolverSP BreakpointResolver::CreateFromStructuredData(
       GetSerializationSubclassKey(), subclass_name);
 
   if (!success) {
-    error.SetErrorStringWithFormat(
-        "Resolver data missing subclass resolver key");
+    error.SetErrorString("Resolver data missing subclass resolver key");
     return result_sp;
   }
 
diff --git a/lldb/source/Breakpoint/BreakpointResolverName.cpp b/lldb/source/Breakpoint/BreakpointResolverName.cpp
index 25f5bb3f6eed7..6fab20af5e597 100644
--- a/lldb/source/Breakpoint/BreakpointResolverName.cpp
+++ b/lldb/source/Breakpoint/BreakpointResolverName.cpp
@@ -108,7 +108,7 @@ BreakpointResolver *BreakpointResolverName::CreateFromStructuredData(
   success =
       options_dict.GetValueForKeyAsInteger(GetKey(OptionNames::Offset), offset);
   if (!success) {
-    error.SetErrorStringWithFormat("BRN::CFSD: Missing offset entry.");
+    error.SetErrorString("BRN::CFSD: Missing offset entry.");
     return nullptr;
   }
 
@@ -116,7 +116,7 @@ BreakpointResolver *BreakpointResolverName::CreateFromStructuredData(
   success = options_dict.GetValueForKeyAsBoolean(
       GetKey(OptionNames::SkipPrologue), skip_prologue);
   if (!success) {
-    error.SetErrorStringWithFormat("BRN::CFSD: Missing Skip prologue entry.");
+    error.SetErrorString("BRN::CFSD: Missing Skip prologue entry.");
     return nullptr;
   }
 
@@ -131,15 +131,14 @@ BreakpointResolver *BreakpointResolverName::CreateFromStructuredData(
     success = options_dict.GetValueForKeyAsArray(
         GetKey(OptionNames::SymbolNameArray), names_array);
     if (!success) {
-      error.SetErrorStringWithFormat("BRN::CFSD: Missing symbol names entry.");
+      error.SetErrorString("BRN::CFSD: Missing symbol names entry.");
       return nullptr;
     }
     StructuredData::Array *names_mask_array;
     success = options_dict.GetValueForKeyAsArray(
         GetKey(OptionNames::NameMaskArray), names_mask_array);
     if (!success) {
-      error.SetErrorStringWithFormat(
-          "BRN::CFSD: Missing symbol names mask entry.");
+      error.SetErrorString("BRN::CFSD: Missing symbol names mask entry.");
       return nullptr;
     }
 
diff --git a/lldb/source/Commands/CMakeLists.txt b/lldb/source/Commands/CMakeLists.txt
index 28cbb2cd6cf64..28bcfacdf3e87 100644
--- a/lldb/source/Commands/CMakeLists.txt
+++ b/lldb/source/Commands/CMakeLists.txt
@@ -13,6 +13,7 @@ add_lldb_library(lldbCommands
   CommandObjectFrame.cpp
   CommandObjectGUI.cpp
   CommandObjectHelp.cpp
+  CommandObjectLanguage.cpp
   CommandObjectLog.cpp
   CommandObjectMemory.cpp
   CommandObjectMultiword.cpp
@@ -22,6 +23,7 @@ add_lldb_library(lldbCommands
   CommandObjectQuit.cpp
   CommandObjectRegister.cpp
   CommandObjectReproducer.cpp
+  CommandObjectSession.cpp
   CommandObjectSettings.cpp
   CommandObjectSource.cpp
   CommandObjectStats.cpp
@@ -31,7 +33,6 @@ add_lldb_library(lldbCommands
   CommandObjectVersion.cpp
   CommandObjectWatchpoint.cpp
   CommandObjectWatchpointCommand.cpp
-  CommandObjectLanguage.cpp
 
   LINK_LIBS
     lldbBase
diff --git a/lldb/source/Commands/CommandObjectBreakpoint.cpp b/lldb/source/Commands/CommandObjectBreakpoint.cpp
index be7ef8a1b60bd..b62fe6c93cd8b 100644
--- a/lldb/source/Commands/CommandObjectBreakpoint.cpp
+++ b/lldb/source/Commands/CommandObjectBreakpoint.cpp
@@ -17,6 +17,7 @@
 #include "lldb/Interpreter/OptionArgParser.h"
 #include "lldb/Interpreter/OptionGroupPythonClassWithDict.h"
 #include "lldb/Interpreter/OptionValueBoolean.h"
+#include "lldb/Interpreter/OptionValueFileColonLine.h"
 #include "lldb/Interpreter/OptionValueString.h"
 #include "lldb/Interpreter/OptionValueUInt64.h"
 #include "lldb/Interpreter/Options.h"
@@ -443,7 +444,22 @@ class CommandObjectBreakpointSet : public CommandObjectParsed {
       case 'X':
         m_source_regex_func_names.insert(std::string(option_arg));
         break;
-
+        
+      case 'y':
+      {
+        OptionValueFileColonLine value;
+        Status fcl_err = value.SetValueFromString(option_arg);
+        if (!fcl_err.Success()) {
+          error.SetErrorStringWithFormat(
+              "Invalid value for file:line specifier: %s",
+              fcl_err.AsCString());
+        } else {
+          m_filenames.AppendIfUnique(value.GetFileSpec());
+          m_line_num = value.GetLineNumber();
+          m_column = value.GetColumnNumber();
+        }
+      } break;
+      
       default:
         llvm_unreachable("Unimplemented option");
       }
diff --git a/lldb/source/Commands/CommandObjectCommands.cpp b/lldb/source/Commands/CommandObjectCommands.cpp
index d77e69c6f6a60..255fbe53fb2ea 100644
--- a/lldb/source/Commands/CommandObjectCommands.cpp
+++ b/lldb/source/Commands/CommandObjectCommands.cpp
@@ -30,155 +30,6 @@ using namespace lldb_private;
 
 // CommandObjectCommandsSource
 
-#define LLDB_OPTIONS_history
-#include "CommandOptions.inc"
-
-class CommandObjectCommandsHistory : public CommandObjectParsed {
-public:
-  CommandObjectCommandsHistory(CommandInterpreter &interpreter)
-      : CommandObjectParsed(interpreter, "command history",
-                            "Dump the history of commands in this session.\n"
-                            "Commands in the history list can be run again "
-                            "using \"!<INDEX>\".   \"!-<OFFSET>\" will re-run "
-                            "the command that is <OFFSET> commands from the end"
-                            " of the list (counting the current command).",
-                            nullptr),
-        m_options() {}
-
-  ~CommandObjectCommandsHistory() override = default;
-
-  Options *GetOptions() override { return &m_options; }
-
-protected:
-  class CommandOptions : public Options {
-  public:
-    CommandOptions()
-        : Options(), m_start_idx(0), m_stop_idx(0), m_count(0), m_clear(false) {
-    }
-
-    ~CommandOptions() override = default;
-
-    Status SetOptionValue(uint32_t option_idx, llvm::StringRef option_arg,
-                          ExecutionContext *execution_context) override {
-      Status error;
-      const int short_option = m_getopt_table[option_idx].val;
-
-      switch (short_option) {
-      case 'c':
-        error = m_count.SetValueFromString(option_arg, eVarSetOperationAssign);
-        break;
-      case 's':
-        if (option_arg == "end") {
-          m_start_idx.SetCurrentValue(UINT64_MAX);
-          m_start_idx.SetOptionWasSet();
-        } else
-          error = m_start_idx.SetValueFromString(option_arg,
-                                                 eVarSetOperationAssign);
-        break;
-      case 'e':
-        error =
-            m_stop_idx.SetValueFromString(option_arg, eVarSetOperationAssign);
-        break;
-      case 'C':
-        m_clear.SetCurrentValue(true);
-        m_clear.SetOptionWasSet();
-        break;
-      default:
-        llvm_unreachable("Unimplemented option");
-      }
-
-      return error;
-    }
-
-    void OptionParsingStarting(ExecutionContext *execution_context) override {
-      m_start_idx.Clear();
-      m_stop_idx.Clear();
-      m_count.Clear();
-      m_clear.Clear();
-    }
-
-    llvm::ArrayRef<OptionDefinition> GetDefinitions() override {
-      return llvm::makeArrayRef(g_history_options);
-    }
-
-    // Instance variables to hold the values for command options.
-
-    OptionValueUInt64 m_start_idx;
-    OptionValueUInt64 m_stop_idx;
-    OptionValueUInt64 m_count;
-    OptionValueBoolean m_clear;
-  };
-
-  bool DoExecute(Args &command, CommandReturnObject &result) override {
-    if (m_options.m_clear.GetCurrentValue() &&
-        m_options.m_clear.OptionWasSet()) {
-      m_interpreter.GetCommandHistory().Clear();
-      result.SetStatus(lldb::eReturnStatusSuccessFinishNoResult);
-    } else {
-      if (m_options.m_start_idx.OptionWasSet() &&
-          m_options.m_stop_idx.OptionWasSet() &&
-          m_options.m_count.OptionWasSet()) {
-        result.AppendError("--count, --start-index and --end-index cannot be "
-                           "all specified in the same invocation");
-        result.SetStatus(lldb::eReturnStatusFailed);
-      } else {
-        std::pair<bool, uint64_t> start_idx(
-            m_options.m_start_idx.OptionWasSet(),
-            m_options.m_start_idx.GetCurrentValue());
-        std::pair<bool, uint64_t> stop_idx(
-            m_options.m_stop_idx.OptionWasSet(),
-            m_options.m_stop_idx.GetCurrentValue());
-        std::pair<bool, uint64_t> count(m_options.m_count.OptionWasSet(),
-                                        m_options.m_count.GetCurrentValue());
-
-        const CommandHistory &history(m_interpreter.GetCommandHistory());
-
-        if (start_idx.first && start_idx.second == UINT64_MAX) {
-          if (count.first) {
-            start_idx.second = history.GetSize() - count.second;
-            stop_idx.second = history.GetSize() - 1;
-          } else if (stop_idx.first) {
-            start_idx.second = stop_idx.second;
-            stop_idx.second = history.GetSize() - 1;
-          } else {
-            start_idx.second = 0;
-            stop_idx.second = history.GetSize() - 1;
-          }
-        } else {
-          if (!start_idx.first && !stop_idx.first && !count.first) {
-            start_idx.second = 0;
-            stop_idx.second = history.GetSize() - 1;
-          } else if (start_idx.first) {
-            if (count.first) {
-              stop_idx.second = start_idx.second + count.second - 1;
-            } else if (!stop_idx.first) {
-              stop_idx.second = history.GetSize() - 1;
-            }
-          } else if (stop_idx.first) {
-            if (count.first) {
-              if (stop_idx.second >= count.second)
-                start_idx.second = stop_idx.second - count.second + 1;
-              else
-                start_idx.second = 0;
-            }
-          } else /* if (count.first) */
-          {
-            start_idx.second = 0;
-            stop_idx.second = count.second - 1;
-          }
-        }
-        history.Dump(result.GetOutputStream(), start_idx.second,
-                     stop_idx.second);
-      }
-    }
-    return result.Succeeded();
-  }
-
-  CommandOptions m_options;
-};
-
-// CommandObjectCommandsSource
-
 #define LLDB_OPTIONS_source
 #include "CommandOptions.inc"
 
@@ -1850,8 +1701,6 @@ CommandObjectMultiwordCommands::CommandObjectMultiwordCommands(
                  CommandObjectSP(new CommandObjectCommandsDelete(interpreter)));
   LoadSubCommand(
       "regex", CommandObjectSP(new CommandObjectCommandsAddRegex(interpreter)));
-  LoadSubCommand("history", CommandObjectSP(
-                                new CommandObjectCommandsHistory(interpreter)));
   LoadSubCommand(
       "script",
       CommandObjectSP(new CommandObjectMultiwordCommandsScript(interpreter)));
diff --git a/lldb/source/Commands/CommandObjectFrame.cpp b/lldb/source/Commands/CommandObjectFrame.cpp
index b42020d76751e..c7b67b59d288c 100644
--- a/lldb/source/Commands/CommandObjectFrame.cpp
+++ b/lldb/source/Commands/CommandObjectFrame.cpp
@@ -999,9 +999,14 @@ class CommandObjectFrameRecognizerDelete : public CommandObjectParsed {
       return false;
     }
 
-    GetSelectedOrDummyTarget()
-        .GetFrameRecognizerManager()
-        .RemoveRecognizerWithID(recognizer_id);
+    if (!GetSelectedOrDummyTarget()
+             .GetFrameRecognizerManager()
+             .RemoveRecognizerWithID(recognizer_id)) {
+      result.AppendErrorWithFormat("'%s' is not a valid recognizer id.\n",
+                                   command.GetArgumentAtIndex(0));
+      result.SetStatus(eReturnStatusFailed);
+      return false;
+    }
     result.SetStatus(eReturnStatusSuccessFinishResult);
     return result.Succeeded();
   }
diff --git a/lldb/source/Commands/CommandObjectQuit.cpp b/lldb/source/Commands/CommandObjectQuit.cpp
index d0c7bbd3abf89..d4d15bea9a8ea 100644
--- a/lldb/source/Commands/CommandObjectQuit.cpp
+++ b/lldb/source/Commands/CommandObjectQuit.cpp
@@ -103,5 +103,9 @@ bool CommandObjectQuit::DoExecute(Args &command, CommandReturnObject &result) {
       CommandInterpreter::eBroadcastBitQuitCommandReceived;
   m_interpreter.BroadcastEvent(event_type);
   result.SetStatus(eReturnStatusQuit);
+
+  if (m_interpreter.GetSaveSessionOnQuit())
+    m_interpreter.SaveTranscript(result);
+
   return true;
 }
diff --git a/lldb/source/Commands/CommandObjectSession.cpp b/lldb/source/Commands/CommandObjectSession.cpp
new file mode 100644
index 0000000000000..c2cdfa29a3f64
--- /dev/null
+++ b/lldb/source/Commands/CommandObjectSession.cpp
@@ -0,0 +1,208 @@
+#include "CommandObjectSession.h"
+#include "lldb/Host/OptionParser.h"
+#include "lldb/Interpreter/CommandInterpreter.h"
+#include "lldb/Interpreter/CommandReturnObject.h"
+#include "lldb/Interpreter/OptionArgParser.h"
+#include "lldb/Interpreter/OptionValue.h"
+#include "lldb/Interpreter/OptionValueBoolean.h"
+#include "lldb/Interpreter/OptionValueString.h"
+#include "lldb/Interpreter/OptionValueUInt64.h"
+#include "lldb/Interpreter/Options.h"
+
+using namespace lldb;
+using namespace lldb_private;
+
+class CommandObjectSessionSave : public CommandObjectParsed {
+public:
+  CommandObjectSessionSave(CommandInterpreter &interpreter)
+      : CommandObjectParsed(interpreter, "session save",
+                            "Save the current session transcripts to a file.\n"
+                            "If no file if specified, transcripts will be "
+                            "saved to a temporary file.",
+                            "session save [file]") {
+    CommandArgumentEntry arg1;
+    arg1.emplace_back(eArgTypePath, eArgRepeatOptional);
+    m_arguments.push_back(arg1);
+  }
+
+  ~CommandObjectSessionSave() override = default;
+
+  void
+  HandleArgumentCompletion(CompletionRequest &request,
+                           OptionElementVector &opt_element_vector) override {
+    CommandCompletions::InvokeCommonCompletionCallbacks(
+        GetCommandInterpreter(), CommandCompletions::eDiskFileCompletion,
+        request, nullptr);
+  }
+
+protected:
+  bool DoExecute(Args &args, CommandReturnObject &result) override {
+    llvm::StringRef file_path;
+
+    if (!args.empty())
+      file_path = args[0].ref();
+
+    if (m_interpreter.SaveTranscript(result, file_path.str()))
+      result.SetStatus(eReturnStatusSuccessFinishNoResult);
+    else
+      result.SetStatus(eReturnStatusFailed);
+    return result.Succeeded();
+  }
+};
+
+#define LLDB_OPTIONS_history
+#include "CommandOptions.inc"
+
+class CommandObjectSessionHistory : public CommandObjectParsed {
+public:
+  CommandObjectSessionHistory(CommandInterpreter &interpreter)
+      : CommandObjectParsed(interpreter, "session history",
+                            "Dump the history of commands in this session.\n"
+                            "Commands in the history list can be run again "
+                            "using \"!<INDEX>\".   \"!-<OFFSET>\" will re-run "
+                            "the command that is <OFFSET> commands from the end"
+                            " of the list (counting the current command).",
+                            nullptr),
+        m_options() {}
+
+  ~CommandObjectSessionHistory() override = default;
+
+  Options *GetOptions() override { return &m_options; }
+
+protected:
+  class CommandOptions : public Options {
+  public:
+    CommandOptions()
+        : Options(), m_start_idx(0), m_stop_idx(0), m_count(0), m_clear(false) {
+    }
+
+    ~CommandOptions() override = default;
+
+    Status SetOptionValue(uint32_t option_idx, llvm::StringRef option_arg,
+                          ExecutionContext *execution_context) override {
+      Status error;
+      const int short_option = m_getopt_table[option_idx].val;
+
+      switch (short_option) {
+      case 'c':
+        error = m_count.SetValueFromString(option_arg, eVarSetOperationAssign);
+        break;
+      case 's':
+        if (option_arg == "end") {
+          m_start_idx.SetCurrentValue(UINT64_MAX);
+          m_start_idx.SetOptionWasSet();
+        } else
+          error = m_start_idx.SetValueFromString(option_arg,
+                                                 eVarSetOperationAssign);
+        break;
+      case 'e':
+        error =
+            m_stop_idx.SetValueFromString(option_arg, eVarSetOperationAssign);
+        break;
+      case 'C':
+        m_clear.SetCurrentValue(true);
+        m_clear.SetOptionWasSet();
+        break;
+      default:
+        llvm_unreachable("Unimplemented option");
+      }
+
+      return error;
+    }
+
+    void OptionParsingStarting(ExecutionContext *execution_context) override {
+      m_start_idx.Clear();
+      m_stop_idx.Clear();
+      m_count.Clear();
+      m_clear.Clear();
+    }
+
+    llvm::ArrayRef<OptionDefinition> GetDefinitions() override {
+      return llvm::makeArrayRef(g_history_options);
+    }
+
+    // Instance variables to hold the values for command options.
+
+    OptionValueUInt64 m_start_idx;
+    OptionValueUInt64 m_stop_idx;
+    OptionValueUInt64 m_count;
+    OptionValueBoolean m_clear;
+  };
+
+  bool DoExecute(Args &command, CommandReturnObject &result) override {
+    if (m_options.m_clear.GetCurrentValue() &&
+        m_options.m_clear.OptionWasSet()) {
+      m_interpreter.GetCommandHistory().Clear();
+      result.SetStatus(lldb::eReturnStatusSuccessFinishNoResult);
+    } else {
+      if (m_options.m_start_idx.OptionWasSet() &&
+          m_options.m_stop_idx.OptionWasSet() &&
+          m_options.m_count.OptionWasSet()) {
+        result.AppendError("--count, --start-index and --end-index cannot be "
+                           "all specified in the same invocation");
+        result.SetStatus(lldb::eReturnStatusFailed);
+      } else {
+        std::pair<bool, uint64_t> start_idx(
+            m_options.m_start_idx.OptionWasSet(),
+            m_options.m_start_idx.GetCurrentValue());
+        std::pair<bool, uint64_t> stop_idx(
+            m_options.m_stop_idx.OptionWasSet(),
+            m_options.m_stop_idx.GetCurrentValue());
+        std::pair<bool, uint64_t> count(m_options.m_count.OptionWasSet(),
+                                        m_options.m_count.GetCurrentValue());
+
+        const CommandHistory &history(m_interpreter.GetCommandHistory());
+
+        if (start_idx.first && start_idx.second == UINT64_MAX) {
+          if (count.first) {
+            start_idx.second = history.GetSize() - count.second;
+            stop_idx.second = history.GetSize() - 1;
+          } else if (stop_idx.first) {
+            start_idx.second = stop_idx.second;
+            stop_idx.second = history.GetSize() - 1;
+          } else {
+            start_idx.second = 0;
+            stop_idx.second = history.GetSize() - 1;
+          }
+        } else {
+          if (!start_idx.first && !stop_idx.first && !count.first) {
+            start_idx.second = 0;
+            stop_idx.second = history.GetSize() - 1;
+          } else if (start_idx.first) {
+            if (count.first) {
+              stop_idx.second = start_idx.second + count.second - 1;
+            } else if (!stop_idx.first) {
+              stop_idx.second = history.GetSize() - 1;
+            }
+          } else if (stop_idx.first) {
+            if (count.first) {
+              if (stop_idx.second >= count.second)
+                start_idx.second = stop_idx.second - count.second + 1;
+              else
+                start_idx.second = 0;
+            }
+          } else /* if (count.first) */
+          {
+            start_idx.second = 0;
+            stop_idx.second = count.second - 1;
+          }
+        }
+        history.Dump(result.GetOutputStream(), start_idx.second,
+                     stop_idx.second);
+      }
+    }
+    return result.Succeeded();
+  }
+
+  CommandOptions m_options;
+};
+
+CommandObjectSession::CommandObjectSession(CommandInterpreter &interpreter)
+    : CommandObjectMultiword(interpreter, "session",
+                             "Commands controlling LLDB session.",
+                             "session <subcommand> [<command-options>]") {
+  LoadSubCommand("save",
+                 CommandObjectSP(new CommandObjectSessionSave(interpreter)));
+  LoadSubCommand("history",
+                 CommandObjectSP(new CommandObjectSessionHistory(interpreter)));
+}
diff --git a/lldb/source/Commands/CommandObjectSession.h b/lldb/source/Commands/CommandObjectSession.h
new file mode 100644
index 0000000000000..0af0e279f47eb
--- /dev/null
+++ b/lldb/source/Commands/CommandObjectSession.h
@@ -0,0 +1,23 @@
+//===-- CommandObjectSession.h ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_SOURCE_COMMANDS_COMMANDOBJECTSESSION_H
+#define LLDB_SOURCE_COMMANDS_COMMANDOBJECTSESSION_H
+
+#include "lldb/Interpreter/CommandObjectMultiword.h"
+
+namespace lldb_private {
+
+class CommandObjectSession : public CommandObjectMultiword {
+public:
+  CommandObjectSession(CommandInterpreter &interpreter);
+};
+
+} // namespace lldb_private
+
+#endif // LLDB_SOURCE_COMMANDS_COMMANDOBJECTSESSION_H
diff --git a/lldb/source/Commands/CommandObjectSource.cpp b/lldb/source/Commands/CommandObjectSource.cpp
index 1ccfd3a5166f4..8fff22a06366c 100644
--- a/lldb/source/Commands/CommandObjectSource.cpp
+++ b/lldb/source/Commands/CommandObjectSource.cpp
@@ -16,6 +16,7 @@
 #include "lldb/Host/OptionParser.h"
 #include "lldb/Interpreter/CommandReturnObject.h"
 #include "lldb/Interpreter/OptionArgParser.h"
+#include "lldb/Interpreter/OptionValueFileColonLine.h"
 #include "lldb/Interpreter/Options.h"
 #include "lldb/Symbol/CompileUnit.h"
 #include "lldb/Symbol/Function.h"
@@ -667,6 +668,22 @@ class CommandObjectSourceList : public CommandObjectParsed {
       case 'r':
         reverse = true;
         break;
+      case 'y':
+      {
+        OptionValueFileColonLine value;
+        Status fcl_err = value.SetValueFromString(option_arg);
+        if (!fcl_err.Success()) {
+          error.SetErrorStringWithFormat(
+              "Invalid value for file:line specifier: %s",
+              fcl_err.AsCString());
+        } else {
+          file_name = value.GetFileSpec().GetPath();
+          start_line = value.GetLineNumber();
+          // I don't see anything useful to do with a column number, but I don't
+          // want to complain since someone may well have cut and pasted a
+          // listing from somewhere that included a column.
+        }
+      } break;
       default:
         llvm_unreachable("Unimplemented option");
       }
diff --git a/lldb/source/Commands/CommandObjectTarget.cpp b/lldb/source/Commands/CommandObjectTarget.cpp
index e50415f930b3b..0a14a25107a46 100644
--- a/lldb/source/Commands/CommandObjectTarget.cpp
+++ b/lldb/source/Commands/CommandObjectTarget.cpp
@@ -1624,7 +1624,8 @@ static size_t LookupFunctionInModule(CommandInterpreter &interpreter,
   return 0;
 }
 
-static size_t LookupTypeInModule(CommandInterpreter &interpreter, Stream &strm,
+static size_t LookupTypeInModule(Target *target,
+                                 CommandInterpreter &interpreter, Stream &strm,
                                  Module *module, const char *name_cstr,
                                  bool name_is_regex) {
   TypeList type_list;
@@ -1652,7 +1653,7 @@ static size_t LookupTypeInModule(CommandInterpreter &interpreter, Stream &strm,
       // Resolve the clang type so that any forward references to types
       // that haven't yet been parsed will get parsed.
       type_sp->GetFullCompilerType();
-      type_sp->GetDescription(&strm, eDescriptionLevelFull, true);
+      type_sp->GetDescription(&strm, eDescriptionLevelFull, true, target);
       // Print all typedef chains
       TypeSP typedef_type_sp(type_sp);
       TypeSP typedefed_type_sp(typedef_type_sp->GetTypedefType());
@@ -1661,7 +1662,8 @@ static size_t LookupTypeInModule(CommandInterpreter &interpreter, Stream &strm,
         strm.Printf("     typedef '%s': ",
                     typedef_type_sp->GetName().GetCString());
         typedefed_type_sp->GetFullCompilerType();
-        typedefed_type_sp->GetDescription(&strm, eDescriptionLevelFull, true);
+        typedefed_type_sp->GetDescription(&strm, eDescriptionLevelFull, true,
+                                          target);
         typedef_type_sp = typedefed_type_sp;
         typedefed_type_sp = typedef_type_sp->GetTypedefType();
       }
@@ -1671,9 +1673,9 @@ static size_t LookupTypeInModule(CommandInterpreter &interpreter, Stream &strm,
   return type_list.GetSize();
 }
 
-static size_t LookupTypeHere(CommandInterpreter &interpreter, Stream &strm,
-                             Module &module, const char *name_cstr,
-                             bool name_is_regex) {
+static size_t LookupTypeHere(Target *target, CommandInterpreter &interpreter,
+                             Stream &strm, Module &module,
+                             const char *name_cstr, bool name_is_regex) {
   TypeList type_list;
   const uint32_t max_num_matches = UINT32_MAX;
   bool name_is_fully_qualified = false;
@@ -1696,8 +1698,8 @@ static size_t LookupTypeHere(CommandInterpreter &interpreter, Stream &strm,
     // Resolve the clang type so that any forward references to types that
     // haven't yet been parsed will get parsed.
     type_sp->GetFullCompilerType();
-    type_sp->GetDescription(&strm, eDescriptionLevelFull, true);
-    // Print all typedef chains
+    type_sp->GetDescription(&strm, eDescriptionLevelFull, true, target);
+    // Print all typedef chains.
     TypeSP typedef_type_sp(type_sp);
     TypeSP typedefed_type_sp(typedef_type_sp->GetTypedefType());
     while (typedefed_type_sp) {
@@ -1705,7 +1707,8 @@ static size_t LookupTypeHere(CommandInterpreter &interpreter, Stream &strm,
       strm.Printf("     typedef '%s': ",
                   typedef_type_sp->GetName().GetCString());
       typedefed_type_sp->GetFullCompilerType();
-      typedefed_type_sp->GetDescription(&strm, eDescriptionLevelFull, true);
+      typedefed_type_sp->GetDescription(&strm, eDescriptionLevelFull, true,
+                                        target);
       typedef_type_sp = typedefed_type_sp;
       typedefed_type_sp = typedef_type_sp->GetTypedefType();
     }
@@ -3745,9 +3748,9 @@ class CommandObjectTargetModulesLookup : public CommandObjectParsed {
       return false;
     case eLookupTypeType:
       if (!m_options.m_str.empty()) {
-        if (LookupTypeHere(m_interpreter, result.GetOutputStream(),
-                           *sym_ctx.module_sp, m_options.m_str.c_str(),
-                           m_options.m_use_regex)) {
+        if (LookupTypeHere(&GetSelectedTarget(), m_interpreter,
+                           result.GetOutputStream(), *sym_ctx.module_sp,
+                           m_options.m_str.c_str(), m_options.m_use_regex)) {
           result.SetStatus(eReturnStatusSuccessFinishResult);
           return true;
         }
@@ -3817,9 +3820,9 @@ class CommandObjectTargetModulesLookup : public CommandObjectParsed {
 
     case eLookupTypeType:
       if (!m_options.m_str.empty()) {
-        if (LookupTypeInModule(m_interpreter, result.GetOutputStream(), module,
-                               m_options.m_str.c_str(),
-                               m_options.m_use_regex)) {
+        if (LookupTypeInModule(
+                &GetSelectedTarget(), m_interpreter, result.GetOutputStream(),
+                module, m_options.m_str.c_str(), m_options.m_use_regex)) {
           result.SetStatus(eReturnStatusSuccessFinishResult);
           return true;
         }
diff --git a/lldb/source/Commands/CommandObjectType.cpp b/lldb/source/Commands/CommandObjectType.cpp
index b2020f26621f8..b23f91de0ce6c 100644
--- a/lldb/source/Commands/CommandObjectType.cpp
+++ b/lldb/source/Commands/CommandObjectType.cpp
@@ -1066,13 +1066,15 @@ class CommandObjectTypeFormatterList : public CommandObjectParsed {
       TypeCategoryImpl::ForEachCallbacks<FormatterType> foreach;
       foreach
         .SetExact([&result, &formatter_regex, &any_printed](
-                      ConstString name,
+                      const TypeMatcher &type_matcher,
                       const FormatterSharedPointer &format_sp) -> bool {
           if (formatter_regex) {
             bool escape = true;
-            if (name.GetStringRef() == formatter_regex->GetText()) {
+            if (type_matcher.CreatedBySameMatchString(
+                    ConstString(formatter_regex->GetText()))) {
               escape = false;
-            } else if (formatter_regex->Execute(name.GetStringRef())) {
+            } else if (formatter_regex->Execute(
+                           type_matcher.GetMatchString().GetStringRef())) {
               escape = false;
             }
 
@@ -1081,20 +1083,23 @@ class CommandObjectTypeFormatterList : public CommandObjectParsed {
           }
 
           any_printed = true;
-          result.GetOutputStream().Printf("%s: %s\n", name.AsCString(),
-                                          format_sp->GetDescription().c_str());
+          result.GetOutputStream().Printf(
+              "%s: %s\n", type_matcher.GetMatchString().GetCString(),
+              format_sp->GetDescription().c_str());
           return true;
         });
 
       foreach
         .SetWithRegex([&result, &formatter_regex, &any_printed](
-                          const RegularExpression &regex,
+                          const TypeMatcher &type_matcher,
                           const FormatterSharedPointer &format_sp) -> bool {
           if (formatter_regex) {
             bool escape = true;
-            if (regex.GetText() == formatter_regex->GetText()) {
+            if (type_matcher.CreatedBySameMatchString(
+                    ConstString(formatter_regex->GetText()))) {
               escape = false;
-            } else if (formatter_regex->Execute(regex.GetText())) {
+            } else if (formatter_regex->Execute(
+                           type_matcher.GetMatchString().GetStringRef())) {
               escape = false;
             }
 
@@ -1103,9 +1108,9 @@ class CommandObjectTypeFormatterList : public CommandObjectParsed {
           }
 
           any_printed = true;
-          result.GetOutputStream().Printf("%s: %s\n",
-                                          regex.GetText().str().c_str(),
-                                          format_sp->GetDescription().c_str());
+          result.GetOutputStream().Printf(
+              "%s: %s\n", type_matcher.GetMatchString().GetCString(),
+              format_sp->GetDescription().c_str());
           return true;
         });
 
@@ -1681,10 +1686,10 @@ class CommandObjectTypeSummaryList
     if (DataVisualization::NamedSummaryFormats::GetCount() > 0) {
       result.GetOutputStream().Printf("Named summaries:\n");
       DataVisualization::NamedSummaryFormats::ForEach(
-          [&result](ConstString name,
+          [&result](const TypeMatcher &type_matcher,
                     const TypeSummaryImplSP &summary_sp) -> bool {
             result.GetOutputStream().Printf(
-                "%s: %s\n", name.AsCString(),
+                "%s: %s\n", type_matcher.GetMatchString().GetCString(),
                 summary_sp->GetDescription().c_str());
             return true;
           });
diff --git a/lldb/source/Commands/Options.td b/lldb/source/Commands/Options.td
index d6f1e0a3c96d3..6384179f0ef11 100644
--- a/lldb/source/Commands/Options.td
+++ b/lldb/source/Commands/Options.td
@@ -105,7 +105,7 @@ let Command = "breakpoint dummy" in {
 
 let Command = "breakpoint set" in {
   def breakpoint_set_shlib : Option<"shlib", "s">, Arg<"ShlibName">,
-    Completion<"Module">, Groups<[1,2,3,4,5,6,7,8,9,11]>, // *not* in group 10
+    Completion<"Module">, Groups<[1,2,3,4,5,6,7,8,9,11,12]>, // *not* in group 10
     Desc<"Set the breakpoint only in this shared library.  Can repeat this "
     "option multiple times to specify multiple shared libraries.">;
   def breakpoint_set_hardware : Option<"hardware", "H">,
@@ -186,21 +186,24 @@ let Command = "breakpoint set" in {
     "expression (note: currently only implemented for setting breakpoints on "
     "identifiers). If not set the target.language setting is used.">;
   def breakpoint_set_skip_prologue : Option<"skip-prologue", "K">,
-    Arg<"Boolean">, Groups<[1,3,4,5,6,7,8]>,
+    Arg<"Boolean">, Groups<[1,3,4,5,6,7,8,12]>,
     Desc<"sKip the prologue if the breakpoint is at the beginning of a "
     "function. If not set the target.skip-prologue setting is used.">;
   def breakpoint_set_breakpoint_name : Option<"breakpoint-name", "N">,
     Arg<"BreakpointName">,
     Desc<"Adds this to the list of names for this breakpoint.">;
   def breakpoint_set_address_slide : Option<"address-slide", "R">,
-    Arg<"Address">, Groups<[1,3,4,5,6,7,8]>,
+    Arg<"Address">, Groups<[1,3,4,5,6,7,8,12]>,
     Desc<"Add the specified offset to whatever address(es) the breakpoint "
     "resolves to. At present this applies the offset directly as given, and "
     "doesn't try to align it to instruction boundaries.">;
   def breakpoint_set_move_to_nearest_code : Option<"move-to-nearest-code", "m">,
-    Groups<[1, 9]>, Arg<"Boolean">,
+    Groups<[1,9,12]>, Arg<"Boolean">,
     Desc<"Move breakpoints to nearest code. If not set the "
     "target.move-to-nearest-codesetting is used.">;
+  def breakpoint_set_file_colon_line : Option<"joint-specifier", "y">, Group<12>, Arg<"FileLineColumn">,
+    Required, Completion<"SourceFile">, 
+    Desc<"A specifier in the form filename:line[:column] for setting file & line breakpoints.">;  
   /* Don't add this option till it actually does something useful...
   def breakpoint_set_exception_typename : Option<"exception-typename", "O">,
     Arg<"TypeName">, Desc<"The breakpoint will only stop if an "
@@ -729,7 +732,7 @@ let Command = "source info" in {
 let Command = "source list" in {
   def source_list_count : Option<"count", "c">, Arg<"Count">,
     Desc<"The number of source lines to display.">;
-  def source_list_shlib : Option<"shlib", "s">, Groups<[1,2]>, Arg<"ShlibName">,
+  def source_list_shlib : Option<"shlib", "s">, Groups<[1,2,5]>, Arg<"ShlibName">,
     Completion<"Module">,
     Desc<"Look up the source file in the given shared library.">;
   def source_list_show_breakpoints : Option<"show-breakpoints", "b">,
@@ -747,6 +750,10 @@ let Command = "source list" in {
     " information for the corresponding file and line.">;
   def source_list_reverse : Option<"reverse", "r">, Group<4>, Desc<"Reverse the"
     " listing to look backwards from the last displayed block of source.">;
+  def source_list_file_colon_line : Option<"joint-specifier", "y">, Group<5>,
+    Arg<"FileLineColumn">, Completion<"SourceFile">, 
+    Desc<"A specifier in the form filename:line[:column] from which to display"
+         " source.">;
 }
 
 let Command = "target dependents" in {
diff --git a/lldb/source/Core/IOHandlerCursesGUI.cpp b/lldb/source/Core/IOHandlerCursesGUI.cpp
index f8fc917721983..de4bd2a79095a 100644
--- a/lldb/source/Core/IOHandlerCursesGUI.cpp
+++ b/lldb/source/Core/IOHandlerCursesGUI.cpp
@@ -1189,9 +1189,7 @@ class Application {
 
     ListenerSP listener_sp(
         Listener::MakeListener("lldb.IOHandler.curses.Application"));
-    ConstString broadcaster_class_target(Target::GetStaticBroadcasterClass());
     ConstString broadcaster_class_process(Process::GetStaticBroadcasterClass());
-    ConstString broadcaster_class_thread(Thread::GetStaticBroadcasterClass());
     debugger.EnableForwardEvents(listener_sp);
 
     bool update = true;
diff --git a/lldb/source/Core/ModuleList.cpp b/lldb/source/Core/ModuleList.cpp
index 0345678ddafff..164d0740acfd8 100644
--- a/lldb/source/Core/ModuleList.cpp
+++ b/lldb/source/Core/ModuleList.cpp
@@ -291,14 +291,24 @@ size_t ModuleList::RemoveOrphans(bool mandatory) {
     if (!lock.try_lock())
       return 0;
   }
-  collection::iterator pos = m_modules.begin();
   size_t remove_count = 0;
-  while (pos != m_modules.end()) {
-    if (pos->unique()) {
-      pos = RemoveImpl(pos);
-      ++remove_count;
-    } else {
-      ++pos;
+  // Modules might hold shared pointers to other modules, so removing one
+  // module might make other other modules orphans. Keep removing modules until
+  // there are no further modules that can be removed.
+  bool made_progress = true;
+  while (made_progress) {
+    // Keep track if we make progress this iteration.
+    made_progress = false;
+    collection::iterator pos = m_modules.begin();
+    while (pos != m_modules.end()) {
+      if (pos->unique()) {
+        pos = RemoveImpl(pos);
+        ++remove_count;
+        // We did make progress.
+        made_progress = true;
+      } else {
+        ++pos;
+      }
     }
   }
   return remove_count;
@@ -968,7 +978,7 @@ Status ModuleList::GetSharedModule(const ModuleSpec &module_spec,
             error.SetErrorStringWithFormat(
                 "cannot locate a module for UUID '%s'", uuid_str.c_str());
           else
-            error.SetErrorStringWithFormat("cannot locate a module");
+            error.SetErrorString("cannot locate a module");
         }
       }
     }
diff --git a/lldb/source/Core/SearchFilter.cpp b/lldb/source/Core/SearchFilter.cpp
index ea51fb3791818..fc1ceff719128 100644
--- a/lldb/source/Core/SearchFilter.cpp
+++ b/lldb/source/Core/SearchFilter.cpp
@@ -89,7 +89,7 @@ SearchFilterSP SearchFilter::CreateFromStructuredData(
   bool success = filter_dict.GetValueForKeyAsString(
       GetSerializationSubclassKey(), subclass_name);
   if (!success) {
-    error.SetErrorStringWithFormat("Filter data missing subclass key");
+    error.SetErrorString("Filter data missing subclass key");
     return result_sp;
   }
 
diff --git a/lldb/source/Core/Value.cpp b/lldb/source/Core/Value.cpp
index 63467644cdef8..64e9a507bcfa6 100644
--- a/lldb/source/Core/Value.cpp
+++ b/lldb/source/Core/Value.cpp
@@ -354,7 +354,7 @@ Status Value::GetValueAsData(ExecutionContext *exe_ctx, DataExtractor &data,
         return error; // Success;
     }
 
-    error.SetErrorStringWithFormat("extracting data from value failed");
+    error.SetErrorString("extracting data from value failed");
     break;
   }
   case eValueTypeLoadAddress:
@@ -535,8 +535,7 @@ Status Value::GetValueAsData(ExecutionContext *exe_ctx, DataExtractor &data,
     if (address_type == eAddressTypeHost) {
       // The address is an address in this process, so just copy it.
       if (address == 0) {
-        error.SetErrorStringWithFormat(
-            "trying to read from host address of 0.");
+        error.SetErrorString("trying to read from host address of 0.");
         return error;
       }
       memcpy(dst, reinterpret_cast<uint8_t *>(address), byte_size);
@@ -580,7 +579,7 @@ Status Value::GetValueAsData(ExecutionContext *exe_ctx, DataExtractor &data,
                                      address_type);
     }
   } else {
-    error.SetErrorStringWithFormat("out of memory");
+    error.SetErrorString("out of memory");
   }
 
   return error;
@@ -604,8 +603,9 @@ Scalar &Value::ResolveValue(ExecutionContext *exe_ctx) {
       Status error(GetValueAsData(exe_ctx, data, nullptr));
       if (error.Success()) {
         Scalar scalar;
-        if (compiler_type.GetValueAsScalar(data, 0, data.GetByteSize(),
-                                           scalar)) {
+        if (compiler_type.GetValueAsScalar(
+                data, 0, data.GetByteSize(), scalar,
+                exe_ctx ? exe_ctx->GetBestExecutionContextScope() : nullptr)) {
           m_value = scalar;
           m_value_type = eValueTypeScalar;
         } else {
diff --git a/lldb/source/Core/ValueObjectMemory.cpp b/lldb/source/Core/ValueObjectMemory.cpp
index 91b2c6084928a..8e7d3ebc93f69 100644
--- a/lldb/source/Core/ValueObjectMemory.cpp
+++ b/lldb/source/Core/ValueObjectMemory.cpp
@@ -140,9 +140,12 @@ size_t ValueObjectMemory::CalculateNumChildren(uint32_t max) {
 }
 
 uint64_t ValueObjectMemory::GetByteSize() {
+  ExecutionContext exe_ctx(GetExecutionContextRef());
   if (m_type_sp)
-    return m_type_sp->GetByteSize().getValueOr(0);
-  return m_compiler_type.GetByteSize(nullptr).getValueOr(0);
+    return m_type_sp->GetByteSize(exe_ctx.GetBestExecutionContextScope())
+        .getValueOr(0);
+  return m_compiler_type.GetByteSize(exe_ctx.GetBestExecutionContextScope())
+      .getValueOr(0);
 }
 
 lldb::ValueType ValueObjectMemory::GetValueType() const {
diff --git a/lldb/source/DataFormatters/DataVisualization.cpp b/lldb/source/DataFormatters/DataVisualization.cpp
index 450a5cbc3ef36..ded8bbd90391a 100644
--- a/lldb/source/DataFormatters/DataVisualization.cpp
+++ b/lldb/source/DataFormatters/DataVisualization.cpp
@@ -169,13 +169,12 @@ DataVisualization::Categories::GetCategoryAtIndex(size_t index) {
 
 bool DataVisualization::NamedSummaryFormats::GetSummaryFormat(
     ConstString type, lldb::TypeSummaryImplSP &entry) {
-  return GetFormatManager().GetNamedSummaryContainer().Get(type, entry);
+  return GetFormatManager().GetNamedSummaryContainer().GetExact(type, entry);
 }
 
 void DataVisualization::NamedSummaryFormats::Add(
     ConstString type, const lldb::TypeSummaryImplSP &entry) {
-  GetFormatManager().GetNamedSummaryContainer().Add(
-      FormatManager::GetValidTypeName(type), entry);
+  GetFormatManager().GetNamedSummaryContainer().Add(type, entry);
 }
 
 bool DataVisualization::NamedSummaryFormats::Delete(ConstString type) {
@@ -187,7 +186,7 @@ void DataVisualization::NamedSummaryFormats::Clear() {
 }
 
 void DataVisualization::NamedSummaryFormats::ForEach(
-    std::function<bool(ConstString, const lldb::TypeSummaryImplSP &)>
+    std::function<bool(const TypeMatcher &, const lldb::TypeSummaryImplSP &)>
         callback) {
   GetFormatManager().GetNamedSummaryContainer().ForEach(callback);
 }
diff --git a/lldb/source/DataFormatters/FormatManager.cpp b/lldb/source/DataFormatters/FormatManager.cpp
index ad02d37360b84..6c824d1f77284 100644
--- a/lldb/source/DataFormatters/FormatManager.cpp
+++ b/lldb/source/DataFormatters/FormatManager.cpp
@@ -237,7 +237,9 @@ void FormatManager::GetPossibleMatches(
   // stripped.
   uint64_t array_size;
   if (compiler_type.IsArrayType(nullptr, &array_size, nullptr)) {
-    CompilerType element_type = compiler_type.GetArrayElementType();
+    ExecutionContext exe_ctx(valobj.GetExecutionContextRef());
+    CompilerType element_type = compiler_type.GetArrayElementType(
+        exe_ctx.GetBestExecutionContextScope());
     if (element_type.IsTypedefType()) {
       // Get the stripped element type and compute the stripped array type
       // from it.
@@ -551,10 +553,6 @@ bool FormatManager::ShouldPrintAsOneLiner(ValueObject &valobj) {
   return true;
 }
 
-ConstString FormatManager::GetValidTypeName(ConstString type) {
-  return ::GetValidTypeName_Impl(type);
-}
-
 ConstString FormatManager::GetTypeForCache(ValueObject &valobj,
                                            lldb::DynamicValueType use_dynamic) {
   ValueObjectSP valobj_sp = valobj.GetQualifiedRepresentationIfAvailable(
diff --git a/lldb/source/DataFormatters/TypeCategory.cpp b/lldb/source/DataFormatters/TypeCategory.cpp
index 8368c91a57f12..f1c6210edd1a7 100644
--- a/lldb/source/DataFormatters/TypeCategory.cpp
+++ b/lldb/source/DataFormatters/TypeCategory.cpp
@@ -15,11 +15,9 @@ using namespace lldb_private;
 
 TypeCategoryImpl::TypeCategoryImpl(IFormatChangeListener *clist,
                                    ConstString name)
-    : m_format_cont("format", "regex-format", clist),
-      m_summary_cont("summary", "regex-summary", clist),
-      m_filter_cont("filter", "regex-filter", clist),
-      m_synth_cont("synth", "regex-synth", clist), m_enabled(false),
-      m_change_listener(clist), m_mutex(), m_name(name), m_languages() {}
+    : m_format_cont(clist), m_summary_cont(clist), m_filter_cont(clist),
+      m_synth_cont(clist), m_enabled(false), m_change_listener(clist),
+      m_mutex(), m_name(name), m_languages() {}
 
 static bool IsApplicable(lldb::LanguageType category_lang,
                          lldb::LanguageType valobj_lang) {
diff --git a/lldb/source/Expression/DWARFExpression.cpp b/lldb/source/Expression/DWARFExpression.cpp
index 6050c1922564c..87e2c82439392 100644
--- a/lldb/source/Expression/DWARFExpression.cpp
+++ b/lldb/source/Expression/DWARFExpression.cpp
@@ -158,7 +158,7 @@ static bool ReadRegisterValueAsScalar(RegisterContext *reg_ctx,
                                       Value &value) {
   if (reg_ctx == nullptr) {
     if (error_ptr)
-      error_ptr->SetErrorStringWithFormat("No register context in frame.\n");
+      error_ptr->SetErrorString("No register context in frame.\n");
   } else {
     uint32_t native_reg =
         reg_ctx->ConvertRegisterKindToRegisterNumber(reg_kind, reg_num);
@@ -1036,22 +1036,21 @@ bool DWARFExpression::Evaluate(
             LLDB_INVALID_ADDRESS);
         if (!module_sp) {
           if (error_ptr)
-            error_ptr->SetErrorStringWithFormat(
+            error_ptr->SetErrorString(
                 "need module to resolve file address for DW_OP_deref");
           return false;
         }
         Address so_addr;
         if (!module_sp->ResolveFileAddress(file_addr, so_addr)) {
           if (error_ptr)
-            error_ptr->SetErrorStringWithFormat(
+            error_ptr->SetErrorString(
                 "failed to resolve file address in module");
           return false;
         }
         addr_t load_Addr = so_addr.GetLoadAddress(exe_ctx->GetTargetPtr());
         if (load_Addr == LLDB_INVALID_ADDRESS) {
           if (error_ptr)
-            error_ptr->SetErrorStringWithFormat(
-                "failed to resolve load address");
+            error_ptr->SetErrorString("failed to resolve load address");
           return false;
         }
         stack.back().GetScalar() = load_Addr;
@@ -1079,13 +1078,12 @@ bool DWARFExpression::Evaluate(
             }
           } else {
             if (error_ptr)
-              error_ptr->SetErrorStringWithFormat(
-                  "NULL process for DW_OP_deref.\n");
+              error_ptr->SetErrorString("NULL process for DW_OP_deref.\n");
             return false;
           }
         } else {
           if (error_ptr)
-            error_ptr->SetErrorStringWithFormat(
+            error_ptr->SetErrorString(
                 "NULL execution context for DW_OP_deref.\n");
           return false;
         }
@@ -1199,13 +1197,12 @@ bool DWARFExpression::Evaluate(
             }
           } else {
             if (error_ptr)
-              error_ptr->SetErrorStringWithFormat(
-                  "NULL process for DW_OP_deref.\n");
+              error_ptr->SetErrorString("NULL process for DW_OP_deref.\n");
             return false;
           }
         } else {
           if (error_ptr)
-            error_ptr->SetErrorStringWithFormat(
+            error_ptr->SetErrorString(
                 "NULL execution context for DW_OP_deref.\n");
           return false;
         }
@@ -2036,7 +2033,7 @@ bool DWARFExpression::Evaluate(
         }
       } else {
         if (error_ptr)
-          error_ptr->SetErrorStringWithFormat(
+          error_ptr->SetErrorString(
               "NULL execution context for DW_OP_fbreg.\n");
         return false;
       }
diff --git a/lldb/source/Expression/IRInterpreter.cpp b/lldb/source/Expression/IRInterpreter.cpp
index 4c7a656265981..b87a759aadc58 100644
--- a/lldb/source/Expression/IRInterpreter.cpp
+++ b/lldb/source/Expression/IRInterpreter.cpp
@@ -1356,14 +1356,14 @@ bool IRInterpreter::Interpret(llvm::Module &module, llvm::Function &function,
       // Check we can actually get a thread
       if (exe_ctx.GetThreadPtr() == nullptr) {
         error.SetErrorToGenericError();
-        error.SetErrorStringWithFormat("unable to acquire thread");
+        error.SetErrorString("unable to acquire thread");
         return false;
       }
 
       // Make sure we have a valid process
       if (!exe_ctx.GetProcessPtr()) {
         error.SetErrorToGenericError();
-        error.SetErrorStringWithFormat("unable to get the process");
+        error.SetErrorString("unable to get the process");
         return false;
       }
 
@@ -1404,7 +1404,7 @@ bool IRInterpreter::Interpret(llvm::Module &module, llvm::Function &function,
       static lldb_private::ABI::CallArgument rawArgs[16];
       if (numArgs >= 16) {
         error.SetErrorToGenericError();
-        error.SetErrorStringWithFormat("function takes too many arguments");
+        error.SetErrorString("function takes too many arguments");
         return false;
       }
 
@@ -1490,7 +1490,7 @@ bool IRInterpreter::Interpret(llvm::Module &module, llvm::Function &function,
       // Check that the thread plan completed successfully
       if (res != lldb::ExpressionResults::eExpressionCompleted) {
         error.SetErrorToGenericError();
-        error.SetErrorStringWithFormat("ThreadPlanCallFunctionUsingABI failed");
+        error.SetErrorString("ThreadPlanCallFunctionUsingABI failed");
         return false;
       }
 
@@ -1511,7 +1511,7 @@ bool IRInterpreter::Interpret(llvm::Module &module, llvm::Function &function,
         // Check if the return value is valid
         if (vobj == nullptr || !retVal) {
           error.SetErrorToGenericError();
-          error.SetErrorStringWithFormat("unable to get the return value");
+          error.SetErrorString("unable to get the return value");
           return false;
         }
 
diff --git a/lldb/source/Expression/Materializer.cpp b/lldb/source/Expression/Materializer.cpp
index f33462053f22a..6f8d9b154570a 100644
--- a/lldb/source/Expression/Materializer.cpp
+++ b/lldb/source/Expression/Materializer.cpp
@@ -514,7 +514,7 @@ class EntityVariable : public Materializer::Entity {
           return;
         }
 
-        if (data.GetByteSize() < m_variable_sp->GetType()->GetByteSize()) {
+        if (data.GetByteSize() < m_variable_sp->GetType()->GetByteSize(scope)) {
           if (data.GetByteSize() == 0 &&
               !m_variable_sp->LocationExpression().IsValid()) {
             err.SetErrorStringWithFormat("the variable '%s' has no location, "
@@ -525,7 +525,7 @@ class EntityVariable : public Materializer::Entity {
                 "size of variable %s (%" PRIu64
                 ") is larger than the ValueObject's size (%" PRIu64 ")",
                 m_variable_sp->GetName().AsCString(),
-                m_variable_sp->GetType()->GetByteSize().getValueOr(0),
+                m_variable_sp->GetType()->GetByteSize(scope).getValueOr(0),
                 data.GetByteSize());
           }
           return;
@@ -797,7 +797,7 @@ class EntityResultVariable : public Materializer::Entity {
 
       llvm::Optional<size_t> opt_bit_align = m_type.GetTypeBitAlign(exe_scope);
       if (!opt_bit_align) {
-        err.SetErrorStringWithFormat("can't get the type alignment");
+        err.SetErrorString("can't get the type alignment");
         return;
       }
 
diff --git a/lldb/source/Host/windows/ProcessLauncherWindows.cpp b/lldb/source/Host/windows/ProcessLauncherWindows.cpp
index 00470f558e94d..bbfe4d0d01755 100644
--- a/lldb/source/Host/windows/ProcessLauncherWindows.cpp
+++ b/lldb/source/Host/windows/ProcessLauncherWindows.cpp
@@ -42,7 +42,7 @@ void CreateEnvironmentBuffer(const Environment &env,
   buffer.push_back(0);
 }
 
-bool GetFlattenedWindowsCommandString(Args args, std::string &command) {
+bool GetFlattenedWindowsCommandString(Args args, std::wstring &command) {
   if (args.empty())
     return false;
 
@@ -50,7 +50,12 @@ bool GetFlattenedWindowsCommandString(Args args, std::string &command) {
   for (auto &entry : args.entries())
     args_ref.push_back(entry.ref());
 
-  command = llvm::sys::flattenWindowsCommandLine(args_ref);
+  llvm::ErrorOr<std::wstring> result =
+      llvm::sys::flattenWindowsCommandLine(args_ref);
+  if (result.getError())
+    return false;
+
+  command = *result;
   return true;
 }
 } // namespace
@@ -61,7 +66,6 @@ ProcessLauncherWindows::LaunchProcess(const ProcessLaunchInfo &launch_info,
   error.Clear();
 
   std::string executable;
-  std::string commandLine;
   std::vector<char> environment;
   STARTUPINFO startupinfo = {};
   PROCESS_INFORMATION pi = {};
@@ -99,11 +103,11 @@ ProcessLauncherWindows::LaunchProcess(const ProcessLaunchInfo &launch_info,
   env_block = environment.data();
 
   executable = launch_info.GetExecutableFile().GetPath();
-  GetFlattenedWindowsCommandString(launch_info.GetArguments(), commandLine);
+  std::wstring wcommandLine;
+  GetFlattenedWindowsCommandString(launch_info.GetArguments(), wcommandLine);
 
-  std::wstring wexecutable, wcommandLine, wworkingDirectory;
+  std::wstring wexecutable, wworkingDirectory;
   llvm::ConvertUTF8toWide(executable, wexecutable);
-  llvm::ConvertUTF8toWide(commandLine, wcommandLine);
   llvm::ConvertUTF8toWide(launch_info.GetWorkingDirectory().GetCString(),
                           wworkingDirectory);
   // If the command line is empty, it's best to pass a null pointer to tell
diff --git a/lldb/source/Initialization/SystemInitializerCommon.cpp b/lldb/source/Initialization/SystemInitializerCommon.cpp
index 0cef2a48dcb46..3d07f92259beb 100644
--- a/lldb/source/Initialization/SystemInitializerCommon.cpp
+++ b/lldb/source/Initialization/SystemInitializerCommon.cpp
@@ -94,6 +94,8 @@ llvm::Error SystemInitializerCommon::Initialize() {
     vp.SetVersion(lldb_private::GetVersion());
     repro::FileProvider &fp = g->GetOrCreate<repro::FileProvider>();
     FileSystem::Initialize(fp.GetFileCollector());
+    repro::WorkingDirectoryProvider &wp = g->GetOrCreate<repro::WorkingDirectoryProvider>();
+    fp.RecordInterestingDirectory(wp.GetWorkingDirectory());
   } else {
     FileSystem::Initialize();
   }
diff --git a/lldb/source/Interpreter/CMakeLists.txt b/lldb/source/Interpreter/CMakeLists.txt
index 0ed39869467ec..7a8c826d040cb 100644
--- a/lldb/source/Interpreter/CMakeLists.txt
+++ b/lldb/source/Interpreter/CMakeLists.txt
@@ -35,6 +35,7 @@ add_lldb_library(lldbInterpreter
   OptionValueChar.cpp
   OptionValueDictionary.cpp
   OptionValueEnumeration.cpp
+  OptionValueFileColonLine.cpp
   OptionValueFileSpec.cpp
   OptionValueFileSpecList.cpp
   OptionValueFormat.cpp
diff --git a/lldb/source/Interpreter/CommandInterpreter.cpp b/lldb/source/Interpreter/CommandInterpreter.cpp
index e55b255001798..50a69b29260ca 100644
--- a/lldb/source/Interpreter/CommandInterpreter.cpp
+++ b/lldb/source/Interpreter/CommandInterpreter.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <limits>
 #include <memory>
 #include <stdlib.h>
 #include <string>
@@ -31,6 +32,7 @@
 #include "Commands/CommandObjectQuit.h"
 #include "Commands/CommandObjectRegister.h"
 #include "Commands/CommandObjectReproducer.h"
+#include "Commands/CommandObjectSession.h"
 #include "Commands/CommandObjectSettings.h"
 #include "Commands/CommandObjectSource.h"
 #include "Commands/CommandObjectStats.h"
@@ -52,6 +54,8 @@
 #if LLDB_ENABLE_LIBEDIT
 #include "lldb/Host/Editline.h"
 #endif
+#include "lldb/Host/File.h"
+#include "lldb/Host/FileCache.h"
 #include "lldb/Host/Host.h"
 #include "lldb/Host/HostInfo.h"
 
@@ -74,6 +78,7 @@
 #include "llvm/Support/FormatAdapters.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/PrettyStackTrace.h"
+#include "llvm/Support/ScopedPrinter.h"
 
 using namespace lldb;
 using namespace lldb_private;
@@ -116,7 +121,7 @@ CommandInterpreter::CommandInterpreter(Debugger &debugger,
       m_skip_lldbinit_files(false), m_skip_app_init_files(false),
       m_command_io_handler_sp(), m_comment_char('#'),
       m_batch_command_mode(false), m_truncation_warning(eNoTruncation),
-      m_command_source_depth(0), m_result() {
+      m_command_source_depth(0), m_result(), m_transcript_stream() {
   SetEventName(eBroadcastBitThreadShouldExit, "thread-should-exit");
   SetEventName(eBroadcastBitResetPrompt, "reset-prompt");
   SetEventName(eBroadcastBitQuitCommandReceived, "quit");
@@ -142,6 +147,17 @@ void CommandInterpreter::SetPromptOnQuit(bool enable) {
   m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, enable);
 }
 
+bool CommandInterpreter::GetSaveSessionOnQuit() const {
+  const uint32_t idx = ePropertySaveSessionOnQuit;
+  return m_collection_sp->GetPropertyAtIndexAsBoolean(
+      nullptr, idx, g_interpreter_properties[idx].default_uint_value != 0);
+}
+
+void CommandInterpreter::SetSaveSessionOnQuit(bool enable) {
+  const uint32_t idx = ePropertySaveSessionOnQuit;
+  m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, enable);
+}
+
 bool CommandInterpreter::GetEchoCommands() const {
   const uint32_t idx = ePropertyEchoCommands;
   return m_collection_sp->GetPropertyAtIndexAsBoolean(
@@ -441,6 +457,11 @@ void CommandInterpreter::Initialize() {
   if (cmd_obj_sp) {
     AddAlias("re", cmd_obj_sp);
   }
+
+  cmd_obj_sp = GetCommandSPExact("session history", false);
+  if (cmd_obj_sp) {
+    AddAlias("history", cmd_obj_sp);
+  }
 }
 
 void CommandInterpreter::Clear() {
@@ -460,54 +481,40 @@ const char *CommandInterpreter::ProcessEmbeddedScriptCommands(const char *arg) {
   return arg;
 }
 
+#define REGISTER_COMMAND_OBJECT(NAME, CLASS)                                   \
+  m_command_dict[NAME] = std::make_shared<CLASS>(*this);
+
 void CommandInterpreter::LoadCommandDictionary() {
   static Timer::Category func_cat(LLVM_PRETTY_FUNCTION);
   Timer scoped_timer(func_cat, LLVM_PRETTY_FUNCTION);
 
-  lldb::ScriptLanguage script_language = m_debugger.GetScriptLanguage();
-
-  m_command_dict["apropos"] = CommandObjectSP(new CommandObjectApropos(*this));
-  m_command_dict["breakpoint"] =
-      CommandObjectSP(new CommandObjectMultiwordBreakpoint(*this));
-  m_command_dict["command"] =
-      CommandObjectSP(new CommandObjectMultiwordCommands(*this));
-  m_command_dict["disassemble"] =
-      CommandObjectSP(new CommandObjectDisassemble(*this));
-  m_command_dict["expression"] =
-      CommandObjectSP(new CommandObjectExpression(*this));
-  m_command_dict["frame"] =
-      CommandObjectSP(new CommandObjectMultiwordFrame(*this));
-  m_command_dict["gui"] = CommandObjectSP(new CommandObjectGUI(*this));
-  m_command_dict["help"] = CommandObjectSP(new CommandObjectHelp(*this));
-  m_command_dict["log"] = CommandObjectSP(new CommandObjectLog(*this));
-  m_command_dict["memory"] = CommandObjectSP(new CommandObjectMemory(*this));
-  m_command_dict["platform"] =
-      CommandObjectSP(new CommandObjectPlatform(*this));
-  m_command_dict["plugin"] = CommandObjectSP(new CommandObjectPlugin(*this));
-  m_command_dict["process"] =
-      CommandObjectSP(new CommandObjectMultiwordProcess(*this));
-  m_command_dict["quit"] = CommandObjectSP(new CommandObjectQuit(*this));
-  m_command_dict["register"] =
-      CommandObjectSP(new CommandObjectRegister(*this));
-  m_command_dict["reproducer"] =
-      CommandObjectSP(new CommandObjectReproducer(*this));
-  m_command_dict["script"] =
-      CommandObjectSP(new CommandObjectScript(*this, script_language));
-  m_command_dict["settings"] =
-      CommandObjectSP(new CommandObjectMultiwordSettings(*this));
-  m_command_dict["source"] =
-      CommandObjectSP(new CommandObjectMultiwordSource(*this));
-  m_command_dict["statistics"] = CommandObjectSP(new CommandObjectStats(*this));
-  m_command_dict["target"] =
-      CommandObjectSP(new CommandObjectMultiwordTarget(*this));
-  m_command_dict["thread"] =
-      CommandObjectSP(new CommandObjectMultiwordThread(*this));
-  m_command_dict["type"] = CommandObjectSP(new CommandObjectType(*this));
-  m_command_dict["version"] = CommandObjectSP(new CommandObjectVersion(*this));
-  m_command_dict["watchpoint"] =
-      CommandObjectSP(new CommandObjectMultiwordWatchpoint(*this));
-  m_command_dict["language"] =
-      CommandObjectSP(new CommandObjectLanguage(*this));
+  REGISTER_COMMAND_OBJECT("apropos", CommandObjectApropos);
+  REGISTER_COMMAND_OBJECT("breakpoint", CommandObjectMultiwordBreakpoint);
+  REGISTER_COMMAND_OBJECT("command", CommandObjectMultiwordCommands);
+  REGISTER_COMMAND_OBJECT("disassemble", CommandObjectDisassemble);
+  REGISTER_COMMAND_OBJECT("expression", CommandObjectExpression);
+  REGISTER_COMMAND_OBJECT("frame", CommandObjectMultiwordFrame);
+  REGISTER_COMMAND_OBJECT("gui", CommandObjectGUI);
+  REGISTER_COMMAND_OBJECT("help", CommandObjectHelp);
+  REGISTER_COMMAND_OBJECT("log", CommandObjectLog);
+  REGISTER_COMMAND_OBJECT("memory", CommandObjectMemory);
+  REGISTER_COMMAND_OBJECT("platform", CommandObjectPlatform);
+  REGISTER_COMMAND_OBJECT("plugin", CommandObjectPlugin);
+  REGISTER_COMMAND_OBJECT("process", CommandObjectMultiwordProcess);
+  REGISTER_COMMAND_OBJECT("quit", CommandObjectQuit);
+  REGISTER_COMMAND_OBJECT("register", CommandObjectRegister);
+  REGISTER_COMMAND_OBJECT("reproducer", CommandObjectReproducer);
+  REGISTER_COMMAND_OBJECT("script", CommandObjectScript);
+  REGISTER_COMMAND_OBJECT("settings", CommandObjectMultiwordSettings);
+  REGISTER_COMMAND_OBJECT("session", CommandObjectSession);
+  REGISTER_COMMAND_OBJECT("source", CommandObjectMultiwordSource);
+  REGISTER_COMMAND_OBJECT("statistics", CommandObjectStats);
+  REGISTER_COMMAND_OBJECT("target", CommandObjectMultiwordTarget);
+  REGISTER_COMMAND_OBJECT("thread", CommandObjectMultiwordThread);
+  REGISTER_COMMAND_OBJECT("type", CommandObjectType);
+  REGISTER_COMMAND_OBJECT("version", CommandObjectVersion);
+  REGISTER_COMMAND_OBJECT("watchpoint", CommandObjectMultiwordWatchpoint);
+  REGISTER_COMMAND_OBJECT("language", CommandObjectLanguage);
 
   // clang-format off
   const char *break_regexes[][2] = {
@@ -1667,6 +1674,8 @@ bool CommandInterpreter::HandleCommand(const char *command_line,
   else
     add_to_history = (lazy_add_to_history == eLazyBoolYes);
 
+  m_transcript_stream << "(lldb) " << command_line << '\n';
+
   bool empty_command = false;
   bool comment_command = false;
   if (command_string.empty())
@@ -1708,7 +1717,7 @@ bool CommandInterpreter::HandleCommand(const char *command_line,
         command_string = command_line;
         original_command_string = command_line;
         if (m_repeat_command.empty()) {
-          result.AppendErrorWithFormat("No auto repeat.\n");
+          result.AppendError("No auto repeat.");
           result.SetStatus(eReturnStatusFailed);
           return false;
         }
@@ -1799,6 +1808,9 @@ bool CommandInterpreter::HandleCommand(const char *command_line,
   LLDB_LOGF(log, "HandleCommand, command %s",
             (result.Succeeded() ? "succeeded" : "did not succeed"));
 
+  m_transcript_stream << result.GetOutputData();
+  m_transcript_stream << result.GetErrorData();
+
   return result.Succeeded();
 }
 
@@ -2141,7 +2153,7 @@ void CommandInterpreter::SourceInitFileCwd(CommandReturnObject &result) {
         llvm::sys::path::parent_path(home_init_file)) {
       result.SetStatus(eReturnStatusSuccessFinishNoResult);
     } else {
-      result.AppendErrorWithFormat(InitFileWarning);
+      result.AppendError(InitFileWarning);
       result.SetStatus(eReturnStatusFailed);
     }
   }
@@ -2877,6 +2889,51 @@ bool CommandInterpreter::IOHandlerInterrupt(IOHandler &io_handler) {
   return false;
 }
 
+bool CommandInterpreter::SaveTranscript(
+    CommandReturnObject &result, llvm::Optional<std::string> output_file) {
+  if (output_file == llvm::None || output_file->empty()) {
+    std::string now = llvm::to_string(std::chrono::system_clock::now());
+    std::replace(now.begin(), now.end(), ' ', '_');
+    const std::string file_name = "lldb_session_" + now + ".log";
+    FileSpec tmp = HostInfo::GetGlobalTempDir();
+    tmp.AppendPathComponent(file_name);
+    output_file = tmp.GetPath();
+  }
+
+  auto error_out = [&](llvm::StringRef error_message, std::string description) {
+    LLDB_LOG(GetLogIfAllCategoriesSet(LIBLLDB_LOG_COMMANDS), "{0} ({1}:{2})",
+             error_message, output_file, description);
+    result.AppendErrorWithFormatv(
+        "Failed to save session's transcripts to {0}!", *output_file);
+    return false;
+  };
+
+  File::OpenOptions flags = File::eOpenOptionWrite |
+                            File::eOpenOptionCanCreate |
+                            File::eOpenOptionTruncate;
+
+  auto opened_file = FileSystem::Instance().Open(FileSpec(*output_file), flags);
+
+  if (!opened_file)
+    return error_out("Unable to create file",
+                     llvm::toString(opened_file.takeError()));
+
+  FileUP file = std::move(opened_file.get());
+
+  size_t byte_size = m_transcript_stream.GetSize();
+
+  Status error = file->Write(m_transcript_stream.GetData(), byte_size);
+
+  if (error.Fail() || byte_size != m_transcript_stream.GetSize())
+    return error_out("Unable to write to destination file",
+                     "Bytes written do not match transcript size.");
+
+  result.AppendMessageWithFormat("Session's transcripts saved to %s\n",
+                                 output_file->c_str());
+
+  return true;
+}
+
 void CommandInterpreter::GetLLDBCommandsFromIOHandler(
     const char *prompt, IOHandlerDelegate &delegate, void *baton) {
   Debugger &debugger = GetDebugger();
diff --git a/lldb/source/Interpreter/CommandObject.cpp b/lldb/source/Interpreter/CommandObject.cpp
index 538f7a1ba6930..6f58b8ba0ea77 100644
--- a/lldb/source/Interpreter/CommandObject.cpp
+++ b/lldb/source/Interpreter/CommandObject.cpp
@@ -1064,6 +1064,7 @@ CommandObject::ArgumentTableEntry CommandObject::g_arguments_data[] = {
     { eArgTypeIndex, "index", CommandCompletions::eNoCompletion, { nullptr, false }, "An index into a list." },
     { eArgTypeLanguage, "source-language", CommandCompletions::eNoCompletion, { LanguageTypeHelpTextCallback, true }, nullptr },
     { eArgTypeLineNum, "linenum", CommandCompletions::eNoCompletion, { nullptr, false }, "Line number in a source file." },
+    { eArgTypeFileLineColumn, "linespec", CommandCompletions::eNoCompletion, { nullptr, false }, "A source specifier in the form file:line[:column]" },
     { eArgTypeLogCategory, "log-category", CommandCompletions::eNoCompletion, { nullptr, false }, "The name of a category within a log channel, e.g. all (try \"log list\" to see a list of all channels and their categories." },
     { eArgTypeLogChannel, "log-channel", CommandCompletions::eNoCompletion, { nullptr, false }, "The name of a log channel, e.g. process.gdb-remote (try \"log list\" to see a list of all channels and their categories)." },
     { eArgTypeMethod, "method", CommandCompletions::eNoCompletion, { nullptr, false }, "A C++ method name." },
diff --git a/lldb/source/Interpreter/CommandObjectScript.cpp b/lldb/source/Interpreter/CommandObjectScript.cpp
index 5e98c658ebdb3..e5ae244cade19 100644
--- a/lldb/source/Interpreter/CommandObjectScript.cpp
+++ b/lldb/source/Interpreter/CommandObjectScript.cpp
@@ -20,8 +20,7 @@ using namespace lldb_private;
 
 // CommandObjectScript
 
-CommandObjectScript::CommandObjectScript(CommandInterpreter &interpreter,
-                                         ScriptLanguage script_lang)
+CommandObjectScript::CommandObjectScript(CommandInterpreter &interpreter)
     : CommandObjectRaw(
           interpreter, "script",
           "Invoke the script interpreter with provided code and display any "
diff --git a/lldb/source/Interpreter/CommandObjectScript.h b/lldb/source/Interpreter/CommandObjectScript.h
index a5f659995de31..40abf8bd730c7 100644
--- a/lldb/source/Interpreter/CommandObjectScript.h
+++ b/lldb/source/Interpreter/CommandObjectScript.h
@@ -13,13 +13,9 @@
 
 namespace lldb_private {
 
-// CommandObjectScript
-
 class CommandObjectScript : public CommandObjectRaw {
 public:
-  CommandObjectScript(CommandInterpreter &interpreter,
-                      lldb::ScriptLanguage script_lang);
-
+  CommandObjectScript(CommandInterpreter &interpreter);
   ~CommandObjectScript() override;
 
 protected:
diff --git a/lldb/source/Interpreter/InterpreterProperties.td b/lldb/source/Interpreter/InterpreterProperties.td
index 600c1e3edb9b8..e5346d14a6e1f 100644
--- a/lldb/source/Interpreter/InterpreterProperties.td
+++ b/lldb/source/Interpreter/InterpreterProperties.td
@@ -9,6 +9,10 @@ let Definition = "interpreter" in {
     Global,
     DefaultTrue,
     Desc<"If true, LLDB will prompt you before quitting if there are any live processes being debugged. If false, LLDB will quit without asking in any case.">;
+  def SaveSessionOnQuit: Property<"save-session-on-quit", "Boolean">,
+    Global,
+    DefaultFalse,
+    Desc<"If true, LLDB will save the session's transcripts before quitting.">;
   def StopCmdSourceOnError: Property<"stop-command-source-on-error", "Boolean">,
     Global,
     DefaultTrue,
diff --git a/lldb/source/Interpreter/OptionValue.cpp b/lldb/source/Interpreter/OptionValue.cpp
index 198be85a7b47b..0bd9a591af678 100644
--- a/lldb/source/Interpreter/OptionValue.cpp
+++ b/lldb/source/Interpreter/OptionValue.cpp
@@ -39,7 +39,7 @@ Status OptionValue::SetSubValue(const ExecutionContext *exe_ctx,
                                 VarSetOperationType op, llvm::StringRef name,
                                 llvm::StringRef value) {
   Status error;
-  error.SetErrorStringWithFormat("SetSubValue is not supported");
+  error.SetErrorString("SetSubValue is not supported");
   return error;
 }
 
@@ -471,6 +471,8 @@ const char *OptionValue::GetBuiltinTypeAsCString(Type t) {
     return "dictionary";
   case eTypeEnum:
     return "enum";
+  case eTypeFileLineColumn:
+    return "file:line:column specifier";
   case eTypeFileSpec:
     return "file";
   case eTypeFileSpecList:
diff --git a/lldb/source/Interpreter/OptionValueArray.cpp b/lldb/source/Interpreter/OptionValueArray.cpp
index 9be11e32e2dbc..0b293ccfc248f 100644
--- a/lldb/source/Interpreter/OptionValueArray.cpp
+++ b/lldb/source/Interpreter/OptionValueArray.cpp
@@ -52,6 +52,7 @@ void OptionValueArray::DumpValue(const ExecutionContext *exe_ctx, Stream &strm,
       case eTypeChar:
       case eTypeEnum:
       case eTypeFileSpec:
+      case eTypeFileLineColumn:
       case eTypeFormat:
       case eTypeSInt64:
       case eTypeString:
diff --git a/lldb/source/Interpreter/OptionValueDictionary.cpp b/lldb/source/Interpreter/OptionValueDictionary.cpp
index caadccd042329..79323f502d179 100644
--- a/lldb/source/Interpreter/OptionValueDictionary.cpp
+++ b/lldb/source/Interpreter/OptionValueDictionary.cpp
@@ -62,6 +62,7 @@ void OptionValueDictionary::DumpValue(const ExecutionContext *exe_ctx,
       case eTypeBoolean:
       case eTypeChar:
       case eTypeEnum:
+      case eTypeFileLineColumn:
       case eTypeFileSpec:
       case eTypeFormat:
       case eTypeSInt64:
diff --git a/lldb/source/Interpreter/OptionValueFileColonLine.cpp b/lldb/source/Interpreter/OptionValueFileColonLine.cpp
new file mode 100644
index 0000000000000..dac557c4248aa
--- /dev/null
+++ b/lldb/source/Interpreter/OptionValueFileColonLine.cpp
@@ -0,0 +1,145 @@
+//===-- OptionValueFileColonLine.cpp---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Interpreter/OptionValueFileColonLine.h"
+
+#include "lldb/DataFormatters/FormatManager.h"
+#include "lldb/Interpreter/CommandCompletions.h"
+#include "lldb/Interpreter/CommandInterpreter.h"
+#include "lldb/Utility/Args.h"
+#include "lldb/Utility/State.h"
+
+using namespace lldb;
+using namespace lldb_private;
+
+// This is an OptionValue for parsing file:line:column specifications.
+// I set the completer to "source file" which isn't quite right, but we can
+// only usefully complete in the file name part of it so it should be good
+// enough.
+OptionValueFileColonLine::OptionValueFileColonLine()
+    : OptionValue(), m_file_spec(), m_line_number(LLDB_INVALID_LINE_NUMBER),
+      m_column_number(LLDB_INVALID_COLUMN_NUMBER),
+      m_completion_mask(CommandCompletions::eSourceFileCompletion) {}
+
+OptionValueFileColonLine::OptionValueFileColonLine(llvm::StringRef input)
+    : OptionValue(), m_file_spec(), m_line_number(LLDB_INVALID_LINE_NUMBER),
+      m_column_number(LLDB_INVALID_COLUMN_NUMBER),
+      m_completion_mask(CommandCompletions::eSourceFileCompletion) {
+  SetValueFromString(input, eVarSetOperationAssign);
+}
+
+void OptionValueFileColonLine::DumpValue(const ExecutionContext *exe_ctx,
+                                         Stream &strm, uint32_t dump_mask) {
+  if (dump_mask & eDumpOptionType)
+    strm.Printf("(%s)", GetTypeAsCString());
+  if (dump_mask & eDumpOptionValue) {
+    if (dump_mask & eDumpOptionType)
+      strm.PutCString(" = ");
+
+    if (m_file_spec)
+      strm << '"' << m_file_spec.GetPath().c_str() << '"';
+    if (m_line_number != LLDB_INVALID_LINE_NUMBER)
+      strm.Printf(":%d", m_line_number);
+    if (m_column_number != LLDB_INVALID_COLUMN_NUMBER)
+      strm.Printf(":%d", m_column_number);
+  }
+}
+
+Status OptionValueFileColonLine::SetValueFromString(llvm::StringRef value,
+                                                    VarSetOperationType op) {
+  Status error;
+  switch (op) {
+  case eVarSetOperationClear:
+    Clear();
+    NotifyValueChanged();
+    break;
+
+  case eVarSetOperationReplace:
+  case eVarSetOperationAssign:
+    if (value.size() > 0) {
+      // This is in the form filename:linenumber:column.
+      // I wish we could use filename:linenumber.column, that would make the
+      // parsing unambiguous and so much easier...
+      // But clang & gcc both print the output with two : so we're stuck with
+      // the two colons.  Practically, the only actual ambiguity this introduces
+      // is with files like "foo:10", which doesn't seem terribly likely.
+
+      // Providing the column is optional, so the input value might have one or
+      // two colons.  First pick off the last colon separated piece.
+      // It has to be there, since the line number is required:
+      llvm::StringRef last_piece;
+      llvm::StringRef left_of_last_piece;
+
+      std::tie(left_of_last_piece, last_piece) = value.rsplit(':');
+      if (last_piece.empty()) {
+        error.SetErrorStringWithFormat("Line specifier must include file and "
+                                       "line: '%s'",
+                                       value.str().c_str());
+        return error;
+      }
+
+      // Now see if there's another colon and if so pull out the middle piece:
+      // Then check whether the middle piece is an integer.  If it is, then it
+      // was the line number, and if it isn't we're going to assume that there
+      // was a colon in the filename (see note at the beginning of the function)
+      // and ignore it.
+      llvm::StringRef file_name;
+      llvm::StringRef middle_piece;
+
+      std::tie(file_name, middle_piece) = left_of_last_piece.rsplit(':');
+      if (middle_piece.empty() || !llvm::to_integer(middle_piece, 
+                                                    m_line_number)) {
+        // The middle piece was empty or not an integer, so there were only two
+        // legit pieces; our original division was right.  Reassign the file
+        // name and pull out the line number:
+        file_name = left_of_last_piece;
+        if (!llvm::to_integer(last_piece, m_line_number)) {
+          error.SetErrorStringWithFormat("Bad line number value '%s' in: '%s'",
+                                         last_piece.str().c_str(),
+                                         value.str().c_str());
+          return error;
+        }
+      } else {
+        // There were three pieces, and we've got the line number.  So now
+        // we just need to check the column number which was the last peice.
+        if (!llvm::to_integer(last_piece, m_column_number)) {
+          error.SetErrorStringWithFormat("Bad column value '%s' in: '%s'",
+                                         last_piece.str().c_str(),
+                                         value.str().c_str());
+          return error;
+        }
+      }
+
+      m_value_was_set = true;
+      m_file_spec.SetFile(file_name, FileSpec::Style::native);
+      NotifyValueChanged();
+    } else {
+      error.SetErrorString("invalid value string");
+    }
+    break;
+
+  case eVarSetOperationInsertBefore:
+  case eVarSetOperationInsertAfter:
+  case eVarSetOperationRemove:
+  case eVarSetOperationAppend:
+  case eVarSetOperationInvalid:
+    error = OptionValue::SetValueFromString(value, op);
+    break;
+  }
+  return error;
+}
+
+lldb::OptionValueSP OptionValueFileColonLine::DeepCopy() const {
+  return OptionValueSP(new OptionValueFileColonLine(*this));
+}
+
+void OptionValueFileColonLine::AutoComplete(CommandInterpreter &interpreter,
+                                            CompletionRequest &request) {
+  CommandCompletions::InvokeCommonCompletionCallbacks(
+      interpreter, m_completion_mask, request, nullptr);
+}
diff --git a/lldb/source/Interpreter/OptionValueFileSpec.cpp b/lldb/source/Interpreter/OptionValueFileSpec.cpp
index 15acb7e5e5b08..a03fd55d0385a 100644
--- a/lldb/source/Interpreter/OptionValueFileSpec.cpp
+++ b/lldb/source/Interpreter/OptionValueFileSpec.cpp
@@ -64,13 +64,6 @@ Status OptionValueFileSpec::SetValueFromString(llvm::StringRef value,
   case eVarSetOperationReplace:
   case eVarSetOperationAssign:
     if (value.size() > 0) {
-      // The setting value may have whitespace, double-quotes, or single-quotes
-      // around the file path to indicate that internal spaces are not word
-      // breaks.  Strip off any ws & quotes from the start and end of the file
-      // path - we aren't doing any word // breaking here so the quoting is
-      // unnecessary.  NB this will cause a problem if someone tries to specify
-      // a file path that legitimately begins or ends with a " or ' character,
-      // or whitespace.
       value = value.trim("\"' \t");
       m_value_was_set = true;
       m_current_value.SetFile(value.str(), FileSpec::Style::native);
diff --git a/lldb/source/Interpreter/OptionValueFormatEntity.cpp b/lldb/source/Interpreter/OptionValueFormatEntity.cpp
index 6b36bd575186c..509a21752e480 100644
--- a/lldb/source/Interpreter/OptionValueFormatEntity.cpp
+++ b/lldb/source/Interpreter/OptionValueFormatEntity.cpp
@@ -29,11 +29,10 @@ OptionValueFormatEntity::OptionValueFormatEntity(const char *default_format)
   }
 }
 
-bool OptionValueFormatEntity::Clear() {
+void OptionValueFormatEntity::Clear() {
   m_current_entry = m_default_entry;
   m_current_format = m_default_format;
   m_value_was_set = false;
-  return true;
 }
 
 static void EscapeBackticks(llvm::StringRef str, std::string &dst) {
@@ -85,7 +84,7 @@ Status OptionValueFormatEntity::SetValueFromString(llvm::StringRef value_str,
       if (first_char == '"' || first_char == '\'') {
         const size_t trimmed_len = trimmed_value_str.size();
         if (trimmed_len == 1 || value_str[trimmed_len - 1] != first_char) {
-          error.SetErrorStringWithFormat("mismatched quotes");
+          error.SetErrorString("mismatched quotes");
           return error;
         }
         value_str = trimmed_value_str.substr(1, trimmed_len - 2);
diff --git a/lldb/source/Interpreter/OptionValueProperties.cpp b/lldb/source/Interpreter/OptionValueProperties.cpp
index 24cda056977ea..5b82008ca5717 100644
--- a/lldb/source/Interpreter/OptionValueProperties.cpp
+++ b/lldb/source/Interpreter/OptionValueProperties.cpp
@@ -517,11 +517,10 @@ bool OptionValueProperties::SetPropertyAtIndexAsUInt64(
   return false;
 }
 
-bool OptionValueProperties::Clear() {
+void OptionValueProperties::Clear() {
   const size_t num_properties = m_properties.size();
   for (size_t i = 0; i < num_properties; ++i)
     m_properties[i].GetValue()->Clear();
-  return true;
 }
 
 Status OptionValueProperties::SetValueFromString(llvm::StringRef value,
diff --git a/lldb/source/Interpreter/Options.cpp b/lldb/source/Interpreter/Options.cpp
index c14b6b6ce66a9..d9af6cc03fd29 100644
--- a/lldb/source/Interpreter/Options.cpp
+++ b/lldb/source/Interpreter/Options.cpp
@@ -1308,7 +1308,7 @@ llvm::Expected<Args> Options::Parse(const Args &args,
                               &long_options_index);
 
     if (val == ':') {
-      error.SetErrorStringWithFormat("last option requires an argument");
+      error.SetErrorString("last option requires an argument");
       break;
     }
 
@@ -1317,7 +1317,7 @@ llvm::Expected<Args> Options::Parse(const Args &args,
 
     // Did we get an error?
     if (val == '?') {
-      error.SetErrorStringWithFormat("unknown or ambiguous option");
+      error.SetErrorString("unknown or ambiguous option");
       break;
     }
     // The option auto-set itself
diff --git a/lldb/source/Interpreter/Property.cpp b/lldb/source/Interpreter/Property.cpp
index 9238112498533..a02497662ff5f 100644
--- a/lldb/source/Interpreter/Property.cpp
+++ b/lldb/source/Interpreter/Property.cpp
@@ -99,6 +99,12 @@ Property::Property(const PropertyDefinition &definition)
     }
     break;
 
+  case OptionValue::eTypeFileLineColumn:
+    // "definition.default_uint_value" is not used for a
+    // OptionValue::eTypeFileSpecList
+    m_value_sp = std::make_shared<OptionValueFileColonLine>();
+    break;
+
   case OptionValue::eTypeFileSpec: {
     // "definition.default_uint_value" represents if the
     // "definition.default_cstr_value" should be resolved or not
diff --git a/lldb/source/Plugins/ABI/AArch64/ABIMacOSX_arm64.cpp b/lldb/source/Plugins/ABI/AArch64/ABIMacOSX_arm64.cpp
index 983da26a2a6df..09b319a84a082 100644
--- a/lldb/source/Plugins/ABI/AArch64/ABIMacOSX_arm64.cpp
+++ b/lldb/source/Plugins/ABI/AArch64/ABIMacOSX_arm64.cpp
@@ -492,7 +492,8 @@ static bool LoadValueFromConsecutiveGPRRegisters(
     uint32_t &NGRN,       // NGRN (see ABI documentation)
     uint32_t &NSRN,       // NSRN (see ABI documentation)
     DataExtractor &data) {
-  llvm::Optional<uint64_t> byte_size = value_type.GetByteSize(nullptr);
+  llvm::Optional<uint64_t> byte_size =
+      value_type.GetByteSize(exe_ctx.GetBestExecutionContextScope());
   if (!byte_size || *byte_size == 0)
     return false;
 
@@ -509,7 +510,8 @@ static bool LoadValueFromConsecutiveGPRRegisters(
     if (NSRN < 8 && (8 - NSRN) >= homogeneous_count) {
       if (!base_type)
         return false;
-      llvm::Optional<uint64_t> base_byte_size = base_type.GetByteSize(nullptr);
+      llvm::Optional<uint64_t> base_byte_size =
+          base_type.GetByteSize(exe_ctx.GetBestExecutionContextScope());
       if (!base_byte_size)
         return false;
       uint32_t data_offset = 0;
@@ -646,7 +648,7 @@ ValueObjectSP ABIMacOSX_arm64::GetReturnValueObjectImpl(
     return return_valobj_sp;
 
   llvm::Optional<uint64_t> byte_size =
-      return_compiler_type.GetByteSize(nullptr);
+      return_compiler_type.GetByteSize(&thread);
   if (!byte_size)
     return return_valobj_sp;
 
diff --git a/lldb/source/Plugins/ABI/AArch64/ABISysV_arm64.cpp b/lldb/source/Plugins/ABI/AArch64/ABISysV_arm64.cpp
index 831c8aa0d760b..1214195dcfc04 100644
--- a/lldb/source/Plugins/ABI/AArch64/ABISysV_arm64.cpp
+++ b/lldb/source/Plugins/ABI/AArch64/ABISysV_arm64.cpp
@@ -466,7 +466,8 @@ static bool LoadValueFromConsecutiveGPRRegisters(
     uint32_t &NGRN,       // NGRN (see ABI documentation)
     uint32_t &NSRN,       // NSRN (see ABI documentation)
     DataExtractor &data) {
-  llvm::Optional<uint64_t> byte_size = value_type.GetByteSize(nullptr);
+  llvm::Optional<uint64_t> byte_size =
+      value_type.GetByteSize(exe_ctx.GetBestExecutionContextScope());
 
   if (byte_size || *byte_size == 0)
     return false;
@@ -484,7 +485,8 @@ static bool LoadValueFromConsecutiveGPRRegisters(
     if (NSRN < 8 && (8 - NSRN) >= homogeneous_count) {
       if (!base_type)
         return false;
-      llvm::Optional<uint64_t> base_byte_size = base_type.GetByteSize(nullptr);
+      llvm::Optional<uint64_t> base_byte_size =
+          base_type.GetByteSize(exe_ctx.GetBestExecutionContextScope());
       if (!base_byte_size)
         return false;
       uint32_t data_offset = 0;
@@ -614,7 +616,7 @@ ValueObjectSP ABISysV_arm64::GetReturnValueObjectImpl(
     return return_valobj_sp;
 
   llvm::Optional<uint64_t> byte_size =
-      return_compiler_type.GetByteSize(nullptr);
+      return_compiler_type.GetByteSize(&thread);
   if (!byte_size)
     return return_valobj_sp;
 
diff --git a/lldb/source/Plugins/ABI/ARC/ABISysV_arc.cpp b/lldb/source/Plugins/ABI/ARC/ABISysV_arc.cpp
index a212eef2ab8a6..540270fb95444 100644
--- a/lldb/source/Plugins/ABI/ARC/ABISysV_arc.cpp
+++ b/lldb/source/Plugins/ABI/ARC/ABISysV_arc.cpp
@@ -458,7 +458,7 @@ ABISysV_arc::GetReturnValueObjectSimple(Thread &thread,
   const uint32_t type_flags = compiler_type.GetTypeInfo();
   // Integer return type.
   if (type_flags & eTypeIsInteger) {
-    const size_t byte_size = compiler_type.GetByteSize(nullptr).getValueOr(0);
+    const size_t byte_size = compiler_type.GetByteSize(&thread).getValueOr(0);
     auto raw_value = ReadRawValue(reg_ctx, byte_size);
 
     const bool is_signed = (type_flags & eTypeIsSigned) != 0;
@@ -482,7 +482,7 @@ ABISysV_arc::GetReturnValueObjectSimple(Thread &thread,
 
     if (compiler_type.IsFloatingPointType(float_count, is_complex) &&
         1 == float_count && !is_complex) {
-      const size_t byte_size = compiler_type.GetByteSize(nullptr).getValueOr(0);
+      const size_t byte_size = compiler_type.GetByteSize(&thread).getValueOr(0);
       auto raw_value = ReadRawValue(reg_ctx, byte_size);
 
       if (!SetSizedFloat(value.GetScalar(), raw_value, byte_size))
diff --git a/lldb/source/Plugins/ABI/ARM/ABISysV_arm.cpp b/lldb/source/Plugins/ABI/ARM/ABISysV_arm.cpp
index 1a93bac564f72..c63c569f875a9 100644
--- a/lldb/source/Plugins/ABI/ARM/ABISysV_arm.cpp
+++ b/lldb/source/Plugins/ABI/ARM/ABISysV_arm.cpp
@@ -1718,7 +1718,7 @@ ValueObjectSP ABISysV_arm::GetReturnValueObjectImpl(
 
       if (homogeneous_count > 0 && homogeneous_count <= 4) {
         llvm::Optional<uint64_t> base_byte_size =
-            base_type.GetByteSize(nullptr);
+            base_type.GetByteSize(&thread);
         if (base_type.IsVectorType(nullptr, nullptr)) {
           if (base_byte_size &&
               (*base_byte_size == 8 || *base_byte_size == 16)) {
@@ -1747,7 +1747,7 @@ ValueObjectSP ABISysV_arm::GetReturnValueObjectImpl(
 
             if (base_type.IsFloatingPointType(float_count, is_complex)) {
               llvm::Optional<uint64_t> base_byte_size =
-                  base_type.GetByteSize(nullptr);
+                  base_type.GetByteSize(&thread);
               if (float_count == 2 && is_complex) {
                 if (index != 0 && base_byte_size &&
                     vfp_byte_size != *base_byte_size)
diff --git a/lldb/source/Plugins/ABI/Mips/ABISysV_mips64.cpp b/lldb/source/Plugins/ABI/Mips/ABISysV_mips64.cpp
index bb28a50e5f4ab..751555722dacc 100644
--- a/lldb/source/Plugins/ABI/Mips/ABISysV_mips64.cpp
+++ b/lldb/source/Plugins/ABI/Mips/ABISysV_mips64.cpp
@@ -753,7 +753,7 @@ ValueObjectSP ABISysV_mips64::GetReturnValueObjectImpl(
   const ArchSpec target_arch = target->GetArchitecture();
   ByteOrder target_byte_order = target_arch.GetByteOrder();
   llvm::Optional<uint64_t> byte_size =
-      return_compiler_type.GetByteSize(nullptr);
+      return_compiler_type.GetByteSize(&thread);
   if (!byte_size)
     return return_valobj_sp;
   const uint32_t type_flags = return_compiler_type.GetTypeInfo(nullptr);
@@ -962,7 +962,7 @@ ValueObjectSP ABISysV_mips64::GetReturnValueObjectImpl(
                 return_compiler_type.GetFieldAtIndex(
                     idx, name, &field_bit_offset, nullptr, nullptr);
             llvm::Optional<uint64_t> field_byte_width =
-                field_compiler_type.GetByteSize(nullptr);
+                field_compiler_type.GetByteSize(&thread);
             if (!field_byte_width)
               return return_valobj_sp;
 
@@ -1034,7 +1034,7 @@ ValueObjectSP ABISysV_mips64::GetReturnValueObjectImpl(
         CompilerType field_compiler_type = return_compiler_type.GetFieldAtIndex(
             idx, name, &field_bit_offset, nullptr, nullptr);
         llvm::Optional<uint64_t> field_byte_width =
-            field_compiler_type.GetByteSize(nullptr);
+            field_compiler_type.GetByteSize(&thread);
 
         // if we don't know the size of the field (e.g. invalid type), just
         // bail out
diff --git a/lldb/source/Plugins/ABI/PowerPC/ABISysV_ppc.cpp b/lldb/source/Plugins/ABI/PowerPC/ABISysV_ppc.cpp
index 6f5eded7b0315..91d2e59ed632e 100644
--- a/lldb/source/Plugins/ABI/PowerPC/ABISysV_ppc.cpp
+++ b/lldb/source/Plugins/ABI/PowerPC/ABISysV_ppc.cpp
@@ -527,7 +527,7 @@ ValueObjectSP ABISysV_ppc::GetReturnValueObjectSimple(
       // Extract the register context so we can read arguments from registers
 
       llvm::Optional<uint64_t> byte_size =
-          return_compiler_type.GetByteSize(nullptr);
+          return_compiler_type.GetByteSize(&thread);
       if (!byte_size)
         return return_valobj_sp;
       uint64_t raw_value = thread.GetRegisterContext()->ReadRegisterAsUnsigned(
@@ -574,7 +574,7 @@ ValueObjectSP ABISysV_ppc::GetReturnValueObjectSimple(
         // Don't handle complex yet.
       } else {
         llvm::Optional<uint64_t> byte_size =
-            return_compiler_type.GetByteSize(nullptr);
+            return_compiler_type.GetByteSize(&thread);
         if (byte_size && *byte_size <= sizeof(long double)) {
           const RegisterInfo *f1_info = reg_ctx->GetRegisterInfoByName("f1", 0);
           RegisterValue f1_value;
@@ -608,7 +608,7 @@ ValueObjectSP ABISysV_ppc::GetReturnValueObjectSimple(
         thread.GetStackFrameAtIndex(0).get(), value, ConstString(""));
   } else if (type_flags & eTypeIsVector) {
     llvm::Optional<uint64_t> byte_size =
-        return_compiler_type.GetByteSize(nullptr);
+        return_compiler_type.GetByteSize(&thread);
     if (byte_size && *byte_size > 0) {
       const RegisterInfo *altivec_reg = reg_ctx->GetRegisterInfoByName("v2", 0);
       if (altivec_reg) {
diff --git a/lldb/source/Plugins/ABI/PowerPC/ABISysV_ppc64.cpp b/lldb/source/Plugins/ABI/PowerPC/ABISysV_ppc64.cpp
index 251ac972fd768..c7cb7736df9f6 100644
--- a/lldb/source/Plugins/ABI/PowerPC/ABISysV_ppc64.cpp
+++ b/lldb/source/Plugins/ABI/PowerPC/ABISysV_ppc64.cpp
@@ -567,7 +567,7 @@ class ReturnValueExtractor {
   ReturnValueExtractor(Thread &thread, CompilerType &type,
                        RegisterContext *reg_ctx, ProcessSP process_sp)
       : m_thread(thread), m_type(type),
-        m_byte_size(m_type.GetByteSize(nullptr).getValueOr(0)),
+        m_byte_size(m_type.GetByteSize(&thread).getValueOr(0)),
         m_data_up(new DataBufferHeap(m_byte_size, 0)), m_reg_ctx(reg_ctx),
         m_process_sp(process_sp), m_byte_order(process_sp->GetByteOrder()),
         m_addr_size(
@@ -643,7 +643,7 @@ class ReturnValueExtractor {
     DataExtractor de(&raw_data, sizeof(raw_data), m_byte_order, m_addr_size);
 
     offset_t offset = 0;
-    llvm::Optional<uint64_t> byte_size = type.GetByteSize(nullptr);
+    llvm::Optional<uint64_t> byte_size = type.GetByteSize(m_process_sp.get());
     if (!byte_size)
       return {};
     switch (*byte_size) {
@@ -777,7 +777,8 @@ class ReturnValueExtractor {
     CompilerType elem_type;
     if (m_type.IsHomogeneousAggregate(&elem_type)) {
       uint32_t type_flags = elem_type.GetTypeInfo();
-      llvm::Optional<uint64_t> elem_size = elem_type.GetByteSize(nullptr);
+      llvm::Optional<uint64_t> elem_size =
+          elem_type.GetByteSize(m_process_sp.get());
       if (!elem_size)
         return {};
       if (type_flags & eTypeIsComplex || !(type_flags & eTypeIsFloat)) {
diff --git a/lldb/source/Plugins/ABI/SystemZ/ABISysV_s390x.cpp b/lldb/source/Plugins/ABI/SystemZ/ABISysV_s390x.cpp
index eced2adc7591a..29ad4d97e7184 100644
--- a/lldb/source/Plugins/ABI/SystemZ/ABISysV_s390x.cpp
+++ b/lldb/source/Plugins/ABI/SystemZ/ABISysV_s390x.cpp
@@ -508,7 +508,7 @@ ValueObjectSP ABISysV_s390x::GetReturnValueObjectSimple(
     if (type_flags & eTypeIsInteger) {
       // Extract the register context so we can read arguments from registers.
       llvm::Optional<uint64_t> byte_size =
-          return_compiler_type.GetByteSize(nullptr);
+          return_compiler_type.GetByteSize(&thread);
       if (!byte_size)
         return return_valobj_sp;
       uint64_t raw_value = thread.GetRegisterContext()->ReadRegisterAsUnsigned(
@@ -555,7 +555,7 @@ ValueObjectSP ABISysV_s390x::GetReturnValueObjectSimple(
         // Don't handle complex yet.
       } else {
         llvm::Optional<uint64_t> byte_size =
-            return_compiler_type.GetByteSize(nullptr);
+            return_compiler_type.GetByteSize(&thread);
         if (byte_size && *byte_size <= sizeof(long double)) {
           const RegisterInfo *f0_info = reg_ctx->GetRegisterInfoByName("f0", 0);
           RegisterValue f0_value;
diff --git a/lldb/source/Plugins/ABI/X86/ABISysV_i386.cpp b/lldb/source/Plugins/ABI/X86/ABISysV_i386.cpp
index 2ac87d1512e97..2f47d3f462d22 100644
--- a/lldb/source/Plugins/ABI/X86/ABISysV_i386.cpp
+++ b/lldb/source/Plugins/ABI/X86/ABISysV_i386.cpp
@@ -387,7 +387,7 @@ ValueObjectSP ABISysV_i386::GetReturnValueObjectSimple(
   {
     value.SetValueType(Value::eValueTypeScalar);
     llvm::Optional<uint64_t> byte_size =
-        return_compiler_type.GetByteSize(nullptr);
+        return_compiler_type.GetByteSize(&thread);
     if (!byte_size)
       return return_valobj_sp;
     bool success = false;
@@ -512,7 +512,7 @@ ValueObjectSP ABISysV_i386::GetReturnValueObjectSimple(
   } else if (type_flags & eTypeIsVector) // 'Packed'
   {
     llvm::Optional<uint64_t> byte_size =
-        return_compiler_type.GetByteSize(nullptr);
+        return_compiler_type.GetByteSize(&thread);
     if (byte_size && *byte_size > 0) {
       const RegisterInfo *vec_reg = reg_ctx->GetRegisterInfoByName("xmm0", 0);
       if (vec_reg == nullptr)
diff --git a/lldb/source/Plugins/ABI/X86/ABISysV_x86_64.cpp b/lldb/source/Plugins/ABI/X86/ABISysV_x86_64.cpp
index 7729e58f85809..2aa2c02b2e87e 100644
--- a/lldb/source/Plugins/ABI/X86/ABISysV_x86_64.cpp
+++ b/lldb/source/Plugins/ABI/X86/ABISysV_x86_64.cpp
@@ -406,7 +406,7 @@ ValueObjectSP ABISysV_x86_64::GetReturnValueObjectSimple(
       // Extract the register context so we can read arguments from registers
 
       llvm::Optional<uint64_t> byte_size =
-          return_compiler_type.GetByteSize(nullptr);
+          return_compiler_type.GetByteSize(&thread);
       if (!byte_size)
         return return_valobj_sp;
       uint64_t raw_value = thread.GetRegisterContext()->ReadRegisterAsUnsigned(
@@ -453,7 +453,7 @@ ValueObjectSP ABISysV_x86_64::GetReturnValueObjectSimple(
         // Don't handle complex yet.
       } else {
         llvm::Optional<uint64_t> byte_size =
-            return_compiler_type.GetByteSize(nullptr);
+            return_compiler_type.GetByteSize(&thread);
         if (byte_size && *byte_size <= sizeof(long double)) {
           const RegisterInfo *xmm0_info =
               reg_ctx->GetRegisterInfoByName("xmm0", 0);
@@ -492,7 +492,7 @@ ValueObjectSP ABISysV_x86_64::GetReturnValueObjectSimple(
         thread.GetStackFrameAtIndex(0).get(), value, ConstString(""));
   } else if (type_flags & eTypeIsVector) {
     llvm::Optional<uint64_t> byte_size =
-        return_compiler_type.GetByteSize(nullptr);
+        return_compiler_type.GetByteSize(&thread);
     if (byte_size && *byte_size > 0) {
       const RegisterInfo *altivec_reg =
           reg_ctx->GetRegisterInfoByName("xmm0", 0);
diff --git a/lldb/source/Plugins/ABI/X86/ABIWindows_x86_64.cpp b/lldb/source/Plugins/ABI/X86/ABIWindows_x86_64.cpp
index 63b670b07277d..06e0ce40836d4 100644
--- a/lldb/source/Plugins/ABI/X86/ABIWindows_x86_64.cpp
+++ b/lldb/source/Plugins/ABI/X86/ABIWindows_x86_64.cpp
@@ -414,7 +414,7 @@ ValueObjectSP ABIWindows_x86_64::GetReturnValueObjectSimple(
     if (type_flags & eTypeIsInteger) {
       // Extract the register context so we can read arguments from registers
       llvm::Optional<uint64_t> byte_size =
-          return_compiler_type.GetByteSize(nullptr);
+          return_compiler_type.GetByteSize(&thread);
       if (!byte_size)
         return return_valobj_sp;
       uint64_t raw_value = thread.GetRegisterContext()->ReadRegisterAsUnsigned(
@@ -461,7 +461,7 @@ ValueObjectSP ABIWindows_x86_64::GetReturnValueObjectSimple(
         // Don't handle complex yet.
       } else {
         llvm::Optional<uint64_t> byte_size =
-            return_compiler_type.GetByteSize(nullptr);
+            return_compiler_type.GetByteSize(&thread);
         if (byte_size && *byte_size <= sizeof(long double)) {
           const RegisterInfo *xmm0_info =
               reg_ctx->GetRegisterInfoByName("xmm0", 0);
@@ -499,7 +499,7 @@ ValueObjectSP ABIWindows_x86_64::GetReturnValueObjectSimple(
         thread.GetStackFrameAtIndex(0).get(), value, ConstString(""));
   } else if (type_flags & eTypeIsVector) {
     llvm::Optional<uint64_t> byte_size =
-        return_compiler_type.GetByteSize(nullptr);
+        return_compiler_type.GetByteSize(&thread);
     if (byte_size && *byte_size > 0) {
       const RegisterInfo *xmm_reg =
           reg_ctx->GetRegisterInfoByName("xmm0", 0);
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ASTUtils.h b/lldb/source/Plugins/ExpressionParser/Clang/ASTUtils.h
index 3787c572d45b8..769b18d54cedd 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ASTUtils.h
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ASTUtils.h
@@ -404,13 +404,6 @@ class SemaSourceWithPriorities : public clang::ExternalSemaSource {
     return nullptr;
   }
 
-  bool DeclIsFromPCHWithObjectFile(const clang::Decl *D) override {
-    for (auto *S : Sources)
-      if (S->DeclIsFromPCHWithObjectFile(D))
-        return true;
-    return false;
-  }
-
   bool layoutRecordType(
       const clang::RecordDecl *Record, uint64_t &Size, uint64_t &Alignment,
       llvm::DenseMap<const clang::FieldDecl *, uint64_t> &FieldOffsets,
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp b/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp
index 8511e554509ac..b35bf07034bdc 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp
@@ -305,9 +305,7 @@ bool IRForTarget::CreateResultVariable(llvm::Function &llvm_function) {
   }
 
   lldb::TargetSP target_sp(m_execution_unit.GetTarget());
-  lldb_private::ExecutionContext exe_ctx(target_sp, true);
-  llvm::Optional<uint64_t> bit_size =
-      m_result_type.GetBitSize(exe_ctx.GetBestExecutionContextScope());
+  llvm::Optional<uint64_t> bit_size = m_result_type.GetBitSize(target_sp.get());
   if (!bit_size) {
     lldb_private::StreamString type_desc_stream;
     m_result_type.DumpTypeDescription(&type_desc_stream);
@@ -330,7 +328,8 @@ bool IRForTarget::CreateResultVariable(llvm::Function &llvm_function) {
   m_result_name = lldb_private::ConstString("$RESULT_NAME");
 
   LLDB_LOG(log, "Creating a new result global: \"{0}\" with size {1}",
-           m_result_name, m_result_type.GetByteSize(nullptr).getValueOr(0));
+           m_result_name,
+           m_result_type.GetByteSize(target_sp.get()).getValueOr(0));
 
   // Construct a new result global and set up its metadata
 
@@ -1242,10 +1241,12 @@ bool IRForTarget::MaybeHandleVariable(Value *llvm_value_ptr) {
       value_type = global_variable->getType();
     }
 
-    llvm::Optional<uint64_t> value_size = compiler_type.GetByteSize(nullptr);
+    auto *target = m_execution_unit.GetTarget().get();
+    llvm::Optional<uint64_t> value_size = compiler_type.GetByteSize(target);
     if (!value_size)
       return false;
-    llvm::Optional<size_t> opt_alignment = compiler_type.GetTypeBitAlign(nullptr);
+    llvm::Optional<size_t> opt_alignment =
+        compiler_type.GetTypeBitAlign(target);
     if (!opt_alignment)
       return false;
     lldb::offset_t value_alignment = (*opt_alignment + 7ull) / 8ull;
diff --git a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
index 08e43ae6b3e81..83a140ebd5e7f 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
@@ -601,8 +601,7 @@ static void LoadLibCxxFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
       ConstString("^std::__[[:alnum:]]+::atomic<.+>$"), stl_synth_flags, true);
 
   cpp_category_sp->GetRegexTypeSyntheticsContainer()->Add(
-      RegularExpression(
-          llvm::StringRef("^(std::__[[:alnum:]]+::)deque<.+>(( )?&)?$")),
+      RegularExpression("^(std::__[[:alnum:]]+::)deque<.+>(( )?&)?$"),
       SyntheticChildrenSP(new ScriptedSyntheticChildren(
           stl_synth_flags,
           "lldb.formatters.cpp.libcxx.stddeque_SynthProvider")));
@@ -823,32 +822,32 @@ static void LoadLibStdcppFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
       false);
 
   cpp_category_sp->GetRegexTypeSyntheticsContainer()->Add(
-      RegularExpression(llvm::StringRef("^std::vector<.+>(( )?&)?$")),
+      RegularExpression("^std::vector<.+>(( )?&)?$"),
       SyntheticChildrenSP(new ScriptedSyntheticChildren(
           stl_synth_flags,
           "lldb.formatters.cpp.gnu_libstdcpp.StdVectorSynthProvider")));
   cpp_category_sp->GetRegexTypeSyntheticsContainer()->Add(
-      RegularExpression(llvm::StringRef("^std::map<.+> >(( )?&)?$")),
+      RegularExpression("^std::map<.+> >(( )?&)?$"),
       SyntheticChildrenSP(new ScriptedSyntheticChildren(
           stl_synth_flags,
           "lldb.formatters.cpp.gnu_libstdcpp.StdMapSynthProvider")));
   cpp_category_sp->GetRegexTypeSyntheticsContainer()->Add(
-      RegularExpression(llvm::StringRef("^std::(__cxx11::)?list<.+>(( )?&)?$")),
+      RegularExpression("^std::(__cxx11::)?list<.+>(( )?&)?$"),
       SyntheticChildrenSP(new ScriptedSyntheticChildren(
           stl_synth_flags,
           "lldb.formatters.cpp.gnu_libstdcpp.StdListSynthProvider")));
   stl_summary_flags.SetDontShowChildren(false);
   stl_summary_flags.SetSkipPointers(true);
   cpp_category_sp->GetRegexTypeSummariesContainer()->Add(
-      RegularExpression(llvm::StringRef("^std::vector<.+>(( )?&)?$")),
+      RegularExpression("^std::vector<.+>(( )?&)?$"),
       TypeSummaryImplSP(
           new StringSummaryFormat(stl_summary_flags, "size=${svar%#}")));
   cpp_category_sp->GetRegexTypeSummariesContainer()->Add(
-      RegularExpression(llvm::StringRef("^std::map<.+> >(( )?&)?$")),
+      RegularExpression("^std::map<.+> >(( )?&)?$"),
       TypeSummaryImplSP(
           new StringSummaryFormat(stl_summary_flags, "size=${svar%#}")));
   cpp_category_sp->GetRegexTypeSummariesContainer()->Add(
-      RegularExpression(llvm::StringRef("^std::(__cxx11::)?list<.+>(( )?&)?$")),
+      RegularExpression("^std::(__cxx11::)?list<.+>(( )?&)?$"),
       TypeSummaryImplSP(
           new StringSummaryFormat(stl_summary_flags, "size=${svar%#}")));
 
diff --git a/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp b/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
index 5feec8167186d..9073672740e08 100644
--- a/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
+++ b/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
@@ -686,7 +686,7 @@ Symtab *ObjectFilePECOFF::GetSymtab() {
 
             if (symbol.naux > 0) {
               i += symbol.naux;
-              offset += symbol_size;
+              offset += symbol.naux * symbol_size;
             }
           }
         }
diff --git a/lldb/source/Plugins/OperatingSystem/Python/OperatingSystemPython.cpp b/lldb/source/Plugins/OperatingSystem/Python/OperatingSystemPython.cpp
index 417aa2e21436f..4350010f02965 100644
--- a/lldb/source/Plugins/OperatingSystem/Python/OperatingSystemPython.cpp
+++ b/lldb/source/Plugins/OperatingSystem/Python/OperatingSystemPython.cpp
@@ -133,8 +133,8 @@ DynamicRegisterInfo *OperatingSystemPython::GetDynamicRegisterInfo() {
     if (!dictionary)
       return nullptr;
 
-    m_register_info_up.reset(new DynamicRegisterInfo(
-        *dictionary, m_process->GetTarget().GetArchitecture()));
+    m_register_info_up = std::make_unique<DynamicRegisterInfo>(
+        *dictionary, m_process->GetTarget().GetArchitecture());
     assert(m_register_info_up->GetNumRegisters() > 0);
     assert(m_register_info_up->GetNumRegisterSets() > 0);
   }
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp
index d31559bc90183..baa9d7b50ad63 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp
@@ -570,12 +570,23 @@ bool PlatformDarwin::ARMGetSupportedArchitectureAtIndex(uint32_t idx,
 #define OSNAME "watchos"
 #elif defined(TARGET_OS_BRIDGE) && TARGET_OS_BRIDGE == 1
 #define OSNAME "bridgeos"
-#elif defined(TARGET_OS_OSX) && TARGET_OS_OSX == 1
-#define OSNAME "macosx"
 #else
 #define OSNAME "ios"
 #endif
 
+#if TARGET_OS_OSX
+  if (IsHost()) {
+    if (idx == 0) {
+      arch.SetTriple("arm64e-apple-macosx");
+      return true;
+    } else if (idx == 1) {
+      arch.SetTriple("arm64-apple-macosx");
+      return true;
+    }
+    return false;
+  }
+#endif
+
   const ArchSpec::Core system_core = system_arch.GetCore();
   switch (system_core) {
   default:
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformMacOSX.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformMacOSX.cpp
index 0b7f898ee0d33..93860c2579798 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformMacOSX.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformMacOSX.cpp
@@ -282,7 +282,33 @@ PlatformMacOSX::GetFileWithUUID(const lldb_private::FileSpec &platform_file,
 bool PlatformMacOSX::GetSupportedArchitectureAtIndex(uint32_t idx,
                                                      ArchSpec &arch) {
 #if defined(__arm__) || defined(__arm64__) || defined(__aarch64__)
-  return ARMGetSupportedArchitectureAtIndex(idx, arch);
+  // macOS for ARM64 support both native and translated x86_64 processes
+  if (!m_num_arm_arches || idx < m_num_arm_arches) {
+    bool res = ARMGetSupportedArchitectureAtIndex(idx, arch);
+    if (res)
+      return true;
+    if (!m_num_arm_arches)
+      m_num_arm_arches = idx;
+  }
+
+  // We can't use x86GetSupportedArchitectureAtIndex() because it uses
+  // the system architecture for some of its return values and also
+  // has a 32bits variant.
+  if (idx == m_num_arm_arches) {
+    arch.SetTriple("x86_64-apple-macosx");
+    return true;
+  } else if (idx == m_num_arm_arches + 1) {
+    arch.SetTriple("x86_64-apple-ios-macabi");
+    return true;
+  } else if (idx == m_num_arm_arches + 2) {
+    arch.SetTriple("arm64-apple-ios");
+    return true;
+  } else if (idx == m_num_arm_arches + 3) {
+    arch.SetTriple("arm64e-apple-ios");
+    return true;
+  }
+
+  return false;
 #else
   return x86GetSupportedArchitectureAtIndex(idx, arch);
 #endif
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformMacOSX.h b/lldb/source/Plugins/Platform/MacOSX/PlatformMacOSX.h
index 30b11eb376849..3e9a0b3aab906 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformMacOSX.h
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformMacOSX.h
@@ -79,6 +79,10 @@ class PlatformMacOSX : public PlatformDarwin {
 private:
   PlatformMacOSX(const PlatformMacOSX &) = delete;
   const PlatformMacOSX &operator=(const PlatformMacOSX &) = delete;
+
+#if defined(__arm__) || defined(__arm64__) || defined(__aarch64__)
+  uint32_t m_num_arm_arches = 0;
+#endif
 };
 
 #endif // LLDB_SOURCE_PLUGINS_PLATFORM_MACOSX_PLATFORMMACOSX_H
diff --git a/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm64.cpp b/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm64.cpp
index d3eafae1e6de4..035bf59cc9c9a 100644
--- a/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm64.cpp
+++ b/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm64.cpp
@@ -24,7 +24,10 @@ RegisterContextPOSIXProcessMonitor_arm64::
     RegisterContextPOSIXProcessMonitor_arm64(
         lldb_private::Thread &thread,
         std::unique_ptr<RegisterInfoPOSIX_arm64> register_info)
-    : RegisterContextPOSIX_arm64(thread, std::move(register_info)) {}
+    : RegisterContextPOSIX_arm64(thread, std::move(register_info)) {
+  ::memset(&m_fpr, 0, sizeof m_gpr_arm64);
+  ::memset(&m_fpr, 0, sizeof m_fpr);
+}
 
 ProcessMonitor &RegisterContextPOSIXProcessMonitor_arm64::GetMonitor() {
   lldb::ProcessSP base = CalculateProcess();
@@ -226,11 +229,11 @@ bool RegisterContextPOSIXProcessMonitor_arm64::UpdateAfterBreakpoint() {
 unsigned RegisterContextPOSIXProcessMonitor_arm64::GetRegisterIndexFromOffset(
     unsigned offset) {
   unsigned reg;
-  for (reg = 0; reg < k_num_registers_arm64; reg++) {
+  for (reg = 0; reg < GetRegisterCount(); reg++) {
     if (GetRegisterInfo()[reg].byte_offset == offset)
       break;
   }
-  assert(reg < k_num_registers_arm64 && "Invalid register offset.");
+  assert(reg < GetRegisterCount() && "Invalid register offset.");
   return reg;
 }
 
diff --git a/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm64.h b/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm64.h
index f100d905e28fd..579fa9d8cebb9 100644
--- a/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm64.h
+++ b/lldb/source/Plugins/Process/FreeBSD/RegisterContextPOSIXProcessMonitor_arm64.h
@@ -71,6 +71,11 @@ class RegisterContextPOSIXProcessMonitor_arm64
   uint32_t NumSupportedHardwareWatchpoints();
 
 private:
+  RegisterInfoPOSIX_arm64::GPR m_gpr_arm64; // 64-bit general purpose registers.
+
+  RegisterInfoPOSIX_arm64::FPU
+      m_fpr; // floating-point registers including extended register sets.
+
   ProcessMonitor &GetMonitor();
 };
 
diff --git a/lldb/source/Plugins/Process/Linux/LinuxPTraceDefines_arm64sve.h b/lldb/source/Plugins/Process/Utility/LinuxPTraceDefines_arm64sve.h
similarity index 88%
rename from lldb/source/Plugins/Process/Linux/LinuxPTraceDefines_arm64sve.h
rename to lldb/source/Plugins/Process/Utility/LinuxPTraceDefines_arm64sve.h
index 18ba39d55b1b1..fe3aed96d8594 100644
--- a/lldb/source/Plugins/Process/Linux/LinuxPTraceDefines_arm64sve.h
+++ b/lldb/source/Plugins/Process/Utility/LinuxPTraceDefines_arm64sve.h
@@ -6,15 +6,27 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef lldb_LinuxPTraceDefines_arm64sve_h
-#define lldb_LinuxPTraceDefines_arm64sve_h
+#ifndef LLDB_SOURCE_PLUGINS_PROCESS_UTILITY_LINUXPTRACEDEFINES_ARM64SVE_H
+#define LLDB_SOURCE_PLUGINS_PROCESS_UTILITY_LINUXPTRACEDEFINES_ARM64SVE_H
+
+// LinuxPTraceDefines_arm64sve.h defines essential macros for manipulating
+// AArch64 SVE core dump registers. Add guard for aarch64/Linux hosts where
+// newer versions of ptrace.h or sigcontext.h might already define SVE macros.
+#ifndef SVE_SIG_REGS_OFFSET
+
+#include <stdint.h>
+
+struct aarch64_context {
+  uint16_t magic;
+  uint16_t size;
+};
 
 #define SVE_MAGIC 0x53564501
 
 struct sve_context {
-  struct _aarch64_ctx head;
-  __u16 vl;
-  __u16 __reserved[3];
+  struct aarch64_context head;
+  uint16_t vl;
+  uint16_t reserved[3];
 };
 
 /*
@@ -92,8 +104,8 @@ struct sve_context {
  * Additional data might be appended in the future.
  */
 
-#define SVE_SIG_ZREG_SIZE(vq) ((__u32)(vq)*SVE_VQ_BYTES)
-#define SVE_SIG_PREG_SIZE(vq) ((__u32)(vq) * (SVE_VQ_BYTES / 8))
+#define SVE_SIG_ZREG_SIZE(vq) ((uint32_t)(vq)*SVE_VQ_BYTES)
+#define SVE_SIG_PREG_SIZE(vq) ((uint32_t)(vq) * (SVE_VQ_BYTES / 8))
 #define SVE_SIG_FFR_SIZE(vq) SVE_SIG_PREG_SIZE(vq)
 
 #define SVE_SIG_REGS_OFFSET                                                    \
@@ -123,12 +135,12 @@ struct sve_context {
 /* SVE/FP/SIMD state (NT_ARM_SVE) */
 
 struct user_sve_header {
-  __u32 size;     /* total meaningful regset content in bytes */
-  __u32 max_size; /* maxmium possible size for this thread */
-  __u16 vl;       /* current vector length */
-  __u16 max_vl;   /* maximum possible vector length */
-  __u16 flags;
-  __u16 __reserved;
+  uint32_t size;     /* total meaningful regset content in bytes */
+  uint32_t max_size; /* maxmium possible size for this thread */
+  uint16_t vl;       /* current vector length */
+  uint16_t max_vl;   /* maximum possible vector length */
+  uint16_t flags;
+  uint16_t reserved;
 };
 
 /* Definitions for user_sve_header.flags: */
@@ -206,8 +218,8 @@ struct user_sve_header {
 #define SVE_PT_SVE_ZREG_SIZE(vq) SVE_SIG_ZREG_SIZE(vq)
 #define SVE_PT_SVE_PREG_SIZE(vq) SVE_SIG_PREG_SIZE(vq)
 #define SVE_PT_SVE_FFR_SIZE(vq) SVE_SIG_FFR_SIZE(vq)
-#define SVE_PT_SVE_FPSR_SIZE sizeof(__u32)
-#define SVE_PT_SVE_FPCR_SIZE sizeof(__u32)
+#define SVE_PT_SVE_FPSR_SIZE sizeof(uint32_t)
+#define SVE_PT_SVE_FPCR_SIZE sizeof(uint32_t)
 
 #define __SVE_SIG_TO_PT(offset)                                                \
   ((offset)-SVE_SIG_REGS_OFFSET + SVE_PT_REGS_OFFSET)
@@ -250,4 +262,6 @@ struct user_sve_header {
        ? SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, flags)                        \
        : SVE_PT_FPSIMD_OFFSET + SVE_PT_FPSIMD_SIZE(vq, flags))
 
-#endif
+#endif // SVE_SIG_REGS_OFFSET
+
+#endif // LLDB_SOURCE_PLUGINS_PROCESS_UTILITY_LINUXPTRACEDEFINES_ARM64SVE_H
diff --git a/lldb/source/Plugins/Process/Utility/RegisterContextPOSIX_arm64.cpp b/lldb/source/Plugins/Process/Utility/RegisterContextPOSIX_arm64.cpp
index 8ef587f13e3ae..3f52501c35f3b 100644
--- a/lldb/source/Plugins/Process/Utility/RegisterContextPOSIX_arm64.cpp
+++ b/lldb/source/Plugins/Process/Utility/RegisterContextPOSIX_arm64.cpp
@@ -39,14 +39,18 @@ bool RegisterContextPOSIX_arm64::IsFPR(unsigned reg) {
   return false;
 }
 
+bool RegisterContextPOSIX_arm64::IsSVE(unsigned reg) const {
+  if (m_register_info_up->GetRegisterSetFromRegisterIndex(reg) ==
+      RegisterInfoPOSIX_arm64::SVERegSet)
+    return true;
+  return false;
+}
+
 RegisterContextPOSIX_arm64::RegisterContextPOSIX_arm64(
     lldb_private::Thread &thread,
     std::unique_ptr<RegisterInfoPOSIX_arm64> register_info)
     : lldb_private::RegisterContext(thread, 0),
-      m_register_info_up(std::move(register_info)) {
-
-  ::memset(&m_fpr, 0, sizeof m_fpr);
-}
+      m_register_info_up(std::move(register_info)) {}
 
 RegisterContextPOSIX_arm64::~RegisterContextPOSIX_arm64() {}
 
@@ -82,8 +86,8 @@ const lldb_private::RegisterInfo *
 RegisterContextPOSIX_arm64::GetRegisterInfoAtIndex(size_t reg) {
   if (reg < GetRegisterCount())
     return &GetRegisterInfo()[reg];
-  else
-    return nullptr;
+
+  return nullptr;
 }
 
 size_t RegisterContextPOSIX_arm64::GetRegisterSetCount() {
diff --git a/lldb/source/Plugins/Process/Utility/RegisterContextPOSIX_arm64.h b/lldb/source/Plugins/Process/Utility/RegisterContextPOSIX_arm64.h
index c2d5aee7f73c9..fdb8b9cfeee84 100644
--- a/lldb/source/Plugins/Process/Utility/RegisterContextPOSIX_arm64.h
+++ b/lldb/source/Plugins/Process/Utility/RegisterContextPOSIX_arm64.h
@@ -11,7 +11,6 @@
 
 #include "RegisterInfoInterface.h"
 #include "RegisterInfoPOSIX_arm64.h"
-#include "lldb-arm64-register-enums.h"
 #include "lldb/Target/RegisterContext.h"
 #include "lldb/Utility/Log.h"
 
@@ -46,13 +45,6 @@ class RegisterContextPOSIX_arm64 : public lldb_private::RegisterContext {
   const char *GetRegisterName(unsigned reg);
 
 protected:
-  uint64_t m_gpr_arm64[lldb_private::k_num_gpr_registers_arm64]; // 64-bit
-                                                                 // general
-                                                                 // purpose
-                                                                 // registers.
-
-  struct RegisterInfoPOSIX_arm64::FPU
-      m_fpr; // floating-point registers including extended register sets.
   std::unique_ptr<RegisterInfoPOSIX_arm64> m_register_info_up;
 
   virtual const lldb_private::RegisterInfo *GetRegisterInfo();
@@ -63,6 +55,21 @@ class RegisterContextPOSIX_arm64 : public lldb_private::RegisterContext {
 
   size_t GetFPUSize() { return sizeof(RegisterInfoPOSIX_arm64::FPU); }
 
+  bool IsSVE(unsigned reg) const;
+
+  bool IsSVEZ(unsigned reg) const { return m_register_info_up->IsSVEZReg(reg); }
+  bool IsSVEP(unsigned reg) const { return m_register_info_up->IsSVEPReg(reg); }
+  bool IsSVEVG(unsigned reg) const {
+    return m_register_info_up->IsSVERegVG(reg);
+  }
+
+  uint32_t GetRegNumSVEZ0() const {
+    return m_register_info_up->GetRegNumSVEZ0();
+  }
+
+  uint32_t GetRegNumFPCR() const { return m_register_info_up->GetRegNumFPCR(); }
+  uint32_t GetRegNumFPSR() const { return m_register_info_up->GetRegNumFPSR(); }
+
   virtual bool ReadGPR() = 0;
   virtual bool ReadFPR() = 0;
   virtual bool WriteGPR() = 0;
diff --git a/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.cpp b/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.cpp
index 4537cee42ad96..e28e46449b1d2 100644
--- a/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.cpp
+++ b/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.cpp
@@ -25,6 +25,24 @@
   (LLVM_EXTENSION offsetof(RegisterInfoPOSIX_arm64::FPU, reg) +                \
    sizeof(RegisterInfoPOSIX_arm64::GPR))
 
+// This information is based on AArch64 with SVE architecture reference manual.
+// AArch64 with SVE has 32 Z and 16 P vector registers. There is also an FFR
+// (First Fault) register and a VG (Vector Granule) pseudo register.
+
+// SVE 16-byte quad word is the basic unit of expansion in vector length.
+#define SVE_QUAD_WORD_BYTES 16
+
+// Vector length is the multiplier which decides the no of quad words,
+// (multiples of 128-bits or 16-bytes) present in a Z register. Vector length
+// is decided during execution and can change at runtime. SVE AArch64 register
+// infos have modes one for each valid value of vector length. A change in
+// vector length requires register context to update sizes of SVE Z, P and FFR.
+// Also register context needs to update byte offsets of all registers affected
+// by the change in vector length.
+#define SVE_REGS_DEFAULT_OFFSET_LINUX sizeof(RegisterInfoPOSIX_arm64::GPR)
+
+#define SVE_OFFSET_VG SVE_REGS_DEFAULT_OFFSET_LINUX
+
 #define EXC_OFFSET_NAME(reg)                                                   \
   (LLVM_EXTENSION offsetof(RegisterInfoPOSIX_arm64::EXC, reg) +                \
    sizeof(RegisterInfoPOSIX_arm64::GPR) +                                      \
@@ -51,6 +69,7 @@
 // Include RegisterInfos_arm64 to declare our g_register_infos_arm64 structure.
 #define DECLARE_REGISTER_INFOS_ARM64_STRUCT
 #include "RegisterInfos_arm64.h"
+#include "RegisterInfos_arm64_sve.h"
 #undef DECLARE_REGISTER_INFOS_ARM64_STRUCT
 
 static const lldb_private::RegisterInfo *
@@ -69,7 +88,8 @@ GetRegisterInfoPtr(const lldb_private::ArchSpec &target_arch) {
 enum {
   k_num_gpr_registers = gpr_w28 - gpr_x0 + 1,
   k_num_fpr_registers = fpu_fpcr - fpu_v0 + 1,
-  k_num_register_sets = 2
+  k_num_sve_registers = sve_ffr - sve_vg + 1,
+  k_num_register_sets = 3
 };
 
 // ARM64 general purpose registers.
@@ -133,13 +153,38 @@ static const uint32_t g_fpu_regnums_arm64[] = {
 static_assert(((sizeof g_fpu_regnums_arm64 / sizeof g_fpu_regnums_arm64[0]) -
                1) == k_num_fpr_registers,
               "g_fpu_regnums_arm64 has wrong number of register infos");
-// clang-format on
+
+// ARM64 SVE registers.
+static const uint32_t g_sve_regnums_arm64[] = {
+    sve_vg,  sve_z0,  sve_z1,
+    sve_z2,  sve_z3,  sve_z4,
+    sve_z5,  sve_z6,  sve_z7,
+    sve_z8,  sve_z9,  sve_z10,
+    sve_z11, sve_z12, sve_z13,
+    sve_z14, sve_z15, sve_z16,
+    sve_z17, sve_z18, sve_z19,
+    sve_z20, sve_z21, sve_z22,
+    sve_z23, sve_z24, sve_z25,
+    sve_z26, sve_z27, sve_z28,
+    sve_z29, sve_z30, sve_z31,
+    sve_p0,  sve_p1,  sve_p2,
+    sve_p3,  sve_p4,  sve_p5,
+    sve_p6,  sve_p7,  sve_p8,
+    sve_p9,  sve_p10, sve_p11,
+    sve_p12, sve_p13, sve_p14,
+    sve_p15, sve_ffr, LLDB_INVALID_REGNUM};
+static_assert(((sizeof g_sve_regnums_arm64 / sizeof g_sve_regnums_arm64[0]) -
+               1) == k_num_sve_registers,
+              "g_sve_regnums_arm64 has wrong number of register infos");
+
 // Register sets for ARM64.
 static const lldb_private::RegisterSet g_reg_sets_arm64[k_num_register_sets] = {
     {"General Purpose Registers", "gpr", k_num_gpr_registers,
      g_gpr_regnums_arm64},
     {"Floating Point Registers", "fpu", k_num_fpr_registers,
-     g_fpu_regnums_arm64}};
+     g_fpu_regnums_arm64},
+    {"Scalable Vector Extension Registers", "sve", k_num_sve_registers,
+     g_sve_regnums_arm64}};
 
 static uint32_t
 GetRegisterInfoCount(const lldb_private::ArchSpec &target_arch) {
@@ -159,25 +204,13 @@ RegisterInfoPOSIX_arm64::RegisterInfoPOSIX_arm64(
     : lldb_private::RegisterInfoAndSetInterface(target_arch),
       m_register_info_p(GetRegisterInfoPtr(target_arch)),
       m_register_info_count(GetRegisterInfoCount(target_arch)) {
-
-  switch (target_arch.GetMachine()) {
-  case llvm::Triple::aarch64:
-  case llvm::Triple::aarch64_32:
-    num_registers = k_num_gpr_registers + k_num_fpr_registers;
-    num_gpr_registers = k_num_gpr_registers;
-    num_fpr_registers = k_num_fpr_registers;
-    last_gpr = gpr_w28;
-    first_fpr = fpu_v0;
-    last_fpr = fpu_fpcr;
-    break;
-  default:
-    assert(false && "Unhandled target architecture.");
-    break;
-  }
 }
 
 uint32_t RegisterInfoPOSIX_arm64::GetRegisterCount() const {
-  return num_gpr_registers + num_fpr_registers;
+  if (IsSVEEnabled())
+    return k_num_gpr_registers + k_num_fpr_registers + k_num_sve_registers;
+
+  return k_num_gpr_registers + k_num_fpr_registers;
 }
 
 size_t RegisterInfoPOSIX_arm64::GetGPRSize() const {
@@ -194,22 +227,115 @@ RegisterInfoPOSIX_arm64::GetRegisterInfo() const {
 }
 
 size_t RegisterInfoPOSIX_arm64::GetRegisterSetCount() const {
-  return k_num_register_sets;
+  if (IsSVEEnabled())
+    return k_num_register_sets;
+  return k_num_register_sets - 1;
 }
 
 size_t RegisterInfoPOSIX_arm64::GetRegisterSetFromRegisterIndex(
     uint32_t reg_index) const {
-  if (reg_index <= last_gpr)
+  if (reg_index <= gpr_w28)
     return GPRegSet;
-  else if (reg_index <= last_fpr)
+  if (reg_index <= fpu_fpcr)
     return FPRegSet;
+  if (reg_index <= sve_ffr)
+    return SVERegSet;
   return LLDB_INVALID_REGNUM;
 }
 
 const lldb_private::RegisterSet *
 RegisterInfoPOSIX_arm64::GetRegisterSet(size_t set_index) const {
-  if (set_index < k_num_register_sets)
+  if (set_index < GetRegisterSetCount())
     return &g_reg_sets_arm64[set_index];
-
   return nullptr;
 }
+
+uint32_t
+RegisterInfoPOSIX_arm64::ConfigureVectorRegisterInfos(uint32_t sve_vq) {
+  // sve_vq contains SVE Quad vector length in context of AArch64 SVE.
+  // SVE register infos if enabled cannot be disabled by selecting sve_vq = 0.
+  // Also if an invalid or previously set vector length is passed to this
+  // function then it will exit immediately with previously set vector length.
+  if (!VectorSizeIsValid(sve_vq) || m_vector_reg_vq == sve_vq)
+    return m_vector_reg_vq;
+
+  // We cannot enable AArch64 only mode if SVE was enabled.
+  if (sve_vq == eVectorQuadwordAArch64 &&
+      m_vector_reg_vq > eVectorQuadwordAArch64)
+    sve_vq = eVectorQuadwordAArch64SVE;
+
+  m_vector_reg_vq = sve_vq;
+
+  if (sve_vq == eVectorQuadwordAArch64) {
+    m_register_info_count =
+        static_cast<uint32_t>(sizeof(g_register_infos_arm64_le) /
+                              sizeof(g_register_infos_arm64_le[0]));
+    m_register_info_p = g_register_infos_arm64_le;
+
+    return m_vector_reg_vq;
+  }
+
+  m_register_info_count =
+      static_cast<uint32_t>(sizeof(g_register_infos_arm64_sve_le) /
+                            sizeof(g_register_infos_arm64_sve_le[0]));
+
+  std::vector<lldb_private::RegisterInfo> &reg_info_ref =
+      m_per_vq_reg_infos[sve_vq];
+
+  if (reg_info_ref.empty()) {
+    reg_info_ref = llvm::makeArrayRef(g_register_infos_arm64_sve_le,
+                                      m_register_info_count);
+
+    uint32_t offset = SVE_REGS_DEFAULT_OFFSET_LINUX;
+
+    reg_info_ref[sve_vg].byte_offset = offset;
+    offset += reg_info_ref[sve_vg].byte_size;
+
+    // Update Z registers size and offset
+    uint32_t s_reg_base = fpu_s0;
+    uint32_t d_reg_base = fpu_d0;
+    uint32_t v_reg_base = fpu_v0;
+    uint32_t z_reg_base = sve_z0;
+
+    for (uint32_t index = 0; index < 32; index++) {
+      reg_info_ref[s_reg_base + index].byte_offset = offset;
+      reg_info_ref[d_reg_base + index].byte_offset = offset;
+      reg_info_ref[v_reg_base + index].byte_offset = offset;
+      reg_info_ref[z_reg_base + index].byte_offset = offset;
+
+      reg_info_ref[z_reg_base + index].byte_size = sve_vq * SVE_QUAD_WORD_BYTES;
+      offset += reg_info_ref[z_reg_base + index].byte_size;
+    }
+
+    // Update P registers and FFR size and offset
+    for (uint32_t it = sve_p0; it <= sve_ffr; it++) {
+      reg_info_ref[it].byte_offset = offset;
+      reg_info_ref[it].byte_size = sve_vq * SVE_QUAD_WORD_BYTES / 8;
+      offset += reg_info_ref[it].byte_size;
+    }
+
+    reg_info_ref[fpu_fpsr].byte_offset = offset;
+    reg_info_ref[fpu_fpcr].byte_offset = offset + 4;
+  }
+
+  m_register_info_p = reg_info_ref.data();
+  return m_vector_reg_vq;
+}
+
+bool RegisterInfoPOSIX_arm64::IsSVEZReg(unsigned reg) const {
+  return (sve_z0 <= reg && reg <= sve_z31);
+}
+
+bool RegisterInfoPOSIX_arm64::IsSVEPReg(unsigned reg) const {
+  return (sve_p0 <= reg && reg <= sve_p15);
+}
+
+bool RegisterInfoPOSIX_arm64::IsSVERegVG(unsigned reg) const {
+  return sve_vg == reg;
+}
+
+uint32_t RegisterInfoPOSIX_arm64::GetRegNumSVEZ0() const { return sve_z0; }
+
+uint32_t RegisterInfoPOSIX_arm64::GetRegNumFPCR() const { return fpu_fpcr; }
+
+uint32_t RegisterInfoPOSIX_arm64::GetRegNumFPSR() const { return fpu_fpsr; }
diff --git a/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h b/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h
index 2da6a531a6b60..892a759b91158 100644
--- a/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h
+++ b/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h
@@ -12,11 +12,21 @@
 #include "RegisterInfoAndSetInterface.h"
 #include "lldb/Target/RegisterContext.h"
 #include "lldb/lldb-private.h"
+#include <map>
+
+enum class SVEState { Unknown, Disabled, FPSIMD, Full };
 
 class RegisterInfoPOSIX_arm64
     : public lldb_private::RegisterInfoAndSetInterface {
 public:
-  enum { GPRegSet = 0, FPRegSet };
+  enum { GPRegSet = 0, FPRegSet, SVERegSet };
+
+  // AArch64 Register set FP/SIMD feature configuration
+  enum {
+    eVectorQuadwordAArch64,
+    eVectorQuadwordAArch64SVE,
+    eVectorQuadwordAArch64SVEMax = 256
+  };
 
   // based on RegisterContextDarwin_arm64.h
   struct GPR {
@@ -73,14 +83,31 @@ class RegisterInfoPOSIX_arm64
 
   size_t GetRegisterSetFromRegisterIndex(uint32_t reg_index) const override;
 
+  uint32_t ConfigureVectorRegisterInfos(uint32_t mode);
+
+  bool VectorSizeIsValid(uint32_t vq) {
+    if (vq >= eVectorQuadwordAArch64 && vq <= eVectorQuadwordAArch64SVEMax)
+      return true;
+    return false;
+  }
+
+  bool IsSVEEnabled() const { return m_vector_reg_vq > eVectorQuadwordAArch64; }
+
+  bool IsSVEZReg(unsigned reg) const;
+  bool IsSVEPReg(unsigned reg) const;
+  bool IsSVERegVG(unsigned reg) const;
+
+  uint32_t GetRegNumSVEZ0() const;
+  uint32_t GetRegNumFPCR() const;
+  uint32_t GetRegNumFPSR() const;
+
 private:
-  uint32_t num_registers;
-  uint32_t num_gpr_registers;
-  uint32_t num_fpr_registers;
+  typedef std::map<uint32_t, std::vector<lldb_private::RegisterInfo>>
+      per_vq_register_infos;
+
+  per_vq_register_infos m_per_vq_reg_infos;
 
-  uint32_t last_gpr;
-  uint32_t first_fpr;
-  uint32_t last_fpr;
+  uint32_t m_vector_reg_vq = eVectorQuadwordAArch64;
 
   const lldb_private::RegisterInfo *m_register_info_p;
   uint32_t m_register_info_count;
diff --git a/lldb/source/Plugins/Process/Utility/RegisterInfos_arm64_sve.h b/lldb/source/Plugins/Process/Utility/RegisterInfos_arm64_sve.h
new file mode 100644
index 0000000000000..752f738fd470d
--- /dev/null
+++ b/lldb/source/Plugins/Process/Utility/RegisterInfos_arm64_sve.h
@@ -0,0 +1,640 @@
+//===-- RegisterInfos_arm64_sve.h -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifdef DECLARE_REGISTER_INFOS_ARM64_STRUCT
+
+enum {
+  sve_fpsr = fpu_fpsr,
+  sve_fpcr = fpu_fpcr,
+
+  sve_vg = exc_far,
+
+  sve_z0,
+  sve_z1,
+  sve_z2,
+  sve_z3,
+  sve_z4,
+  sve_z5,
+  sve_z6,
+  sve_z7,
+  sve_z8,
+  sve_z9,
+  sve_z10,
+  sve_z11,
+  sve_z12,
+  sve_z13,
+  sve_z14,
+  sve_z15,
+  sve_z16,
+  sve_z17,
+  sve_z18,
+  sve_z19,
+  sve_z20,
+  sve_z21,
+  sve_z22,
+  sve_z23,
+  sve_z24,
+  sve_z25,
+  sve_z26,
+  sve_z27,
+  sve_z28,
+  sve_z29,
+  sve_z30,
+  sve_z31,
+
+  sve_p0,
+  sve_p1,
+  sve_p2,
+  sve_p3,
+  sve_p4,
+  sve_p5,
+  sve_p6,
+  sve_p7,
+  sve_p8,
+  sve_p9,
+  sve_p10,
+  sve_p11,
+  sve_p12,
+  sve_p13,
+  sve_p14,
+  sve_p15,
+
+  sve_ffr,
+};
+
+#ifndef SVE_OFFSET_VG
+#error SVE_OFFSET_VG must be defined before including this header file
+#endif
+
+static uint32_t g_sve_s0_invalidates[] = {sve_z0, fpu_v0, fpu_d0,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s1_invalidates[] = {sve_z1, fpu_v1, fpu_d1,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s2_invalidates[] = {sve_z2, fpu_v2, fpu_d2,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s3_invalidates[] = {sve_z3, fpu_v3, fpu_d3,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s4_invalidates[] = {sve_z4, fpu_v4, fpu_d4,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s5_invalidates[] = {sve_z5, fpu_v5, fpu_d5,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s6_invalidates[] = {sve_z6, fpu_v6, fpu_d6,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s7_invalidates[] = {sve_z7, fpu_v7, fpu_d7,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s8_invalidates[] = {sve_z8, fpu_v8, fpu_d8,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s9_invalidates[] = {sve_z9, fpu_v9, fpu_d9,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s10_invalidates[] = {sve_z10, fpu_v10, fpu_d10,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s11_invalidates[] = {sve_z11, fpu_v11, fpu_d11,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s12_invalidates[] = {sve_z12, fpu_v12, fpu_d12,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s13_invalidates[] = {sve_z13, fpu_v13, fpu_d13,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s14_invalidates[] = {sve_z14, fpu_v14, fpu_d14,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s15_invalidates[] = {sve_z15, fpu_v15, fpu_d15,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s16_invalidates[] = {sve_z16, fpu_v16, fpu_d16,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s17_invalidates[] = {sve_z17, fpu_v17, fpu_d17,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s18_invalidates[] = {sve_z18, fpu_v18, fpu_d18,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s19_invalidates[] = {sve_z19, fpu_v19, fpu_d19,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s20_invalidates[] = {sve_z20, fpu_v20, fpu_d20,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s21_invalidates[] = {sve_z21, fpu_v21, fpu_d21,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s22_invalidates[] = {sve_z22, fpu_v22, fpu_d22,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s23_invalidates[] = {sve_z23, fpu_v23, fpu_d23,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s24_invalidates[] = {sve_z24, fpu_v24, fpu_d24,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s25_invalidates[] = {sve_z25, fpu_v25, fpu_d25,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s26_invalidates[] = {sve_z26, fpu_v26, fpu_d26,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s27_invalidates[] = {sve_z27, fpu_v27, fpu_d27,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s28_invalidates[] = {sve_z28, fpu_v28, fpu_d28,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s29_invalidates[] = {sve_z29, fpu_v29, fpu_d29,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s30_invalidates[] = {sve_z30, fpu_v30, fpu_d30,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_s31_invalidates[] = {sve_z31, fpu_v31, fpu_d31,
+                                           LLDB_INVALID_REGNUM};
+
+static uint32_t g_sve_d0_invalidates[] = {sve_z0, fpu_v0, fpu_s0,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d1_invalidates[] = {sve_z1, fpu_v1, fpu_s1,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d2_invalidates[] = {sve_z2, fpu_v2, fpu_s2,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d3_invalidates[] = {sve_z3, fpu_v3, fpu_s3,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d4_invalidates[] = {sve_z4, fpu_v4, fpu_s4,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d5_invalidates[] = {sve_z5, fpu_v5, fpu_s5,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d6_invalidates[] = {sve_z6, fpu_v6, fpu_s6,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d7_invalidates[] = {sve_z7, fpu_v7, fpu_s7,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d8_invalidates[] = {sve_z8, fpu_v8, fpu_s8,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d9_invalidates[] = {sve_z9, fpu_v9, fpu_s9,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d10_invalidates[] = {sve_z10, fpu_v10, fpu_s10,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d11_invalidates[] = {sve_z11, fpu_v11, fpu_s11,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d12_invalidates[] = {sve_z12, fpu_v12, fpu_s12,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d13_invalidates[] = {sve_z13, fpu_v13, fpu_s13,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d14_invalidates[] = {sve_z14, fpu_v14, fpu_s14,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d15_invalidates[] = {sve_z15, fpu_v15, fpu_s15,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d16_invalidates[] = {sve_z16, fpu_v16, fpu_s16,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d17_invalidates[] = {sve_z17, fpu_v17, fpu_s17,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d18_invalidates[] = {sve_z18, fpu_v18, fpu_s18,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d19_invalidates[] = {sve_z19, fpu_v19, fpu_s19,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d20_invalidates[] = {sve_z20, fpu_v20, fpu_s20,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d21_invalidates[] = {sve_z21, fpu_v21, fpu_s21,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d22_invalidates[] = {sve_z22, fpu_v22, fpu_s22,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d23_invalidates[] = {sve_z23, fpu_v23, fpu_s23,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d24_invalidates[] = {sve_z24, fpu_v24, fpu_s24,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d25_invalidates[] = {sve_z25, fpu_v25, fpu_s25,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d26_invalidates[] = {sve_z26, fpu_v26, fpu_s26,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d27_invalidates[] = {sve_z27, fpu_v27, fpu_s27,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d28_invalidates[] = {sve_z28, fpu_v28, fpu_s28,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d29_invalidates[] = {sve_z29, fpu_v29, fpu_s29,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d30_invalidates[] = {sve_z30, fpu_v30, fpu_s30,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_d31_invalidates[] = {sve_z31, fpu_v31, fpu_s31,
+                                           LLDB_INVALID_REGNUM};
+
+static uint32_t g_sve_v0_invalidates[] = {sve_z0, fpu_d0, fpu_s0,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v1_invalidates[] = {sve_z1, fpu_d1, fpu_s1,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v2_invalidates[] = {sve_z2, fpu_d2, fpu_s2,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v3_invalidates[] = {sve_z3, fpu_d3, fpu_s3,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v4_invalidates[] = {sve_z4, fpu_d4, fpu_s4,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v5_invalidates[] = {sve_z5, fpu_d5, fpu_s5,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v6_invalidates[] = {sve_z6, fpu_d6, fpu_s6,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v7_invalidates[] = {sve_z7, fpu_d7, fpu_s7,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v8_invalidates[] = {sve_z8, fpu_d8, fpu_s8,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v9_invalidates[] = {sve_z9, fpu_d9, fpu_s9,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v10_invalidates[] = {sve_z10, fpu_d10, fpu_s10,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v11_invalidates[] = {sve_z11, fpu_d11, fpu_s11,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v12_invalidates[] = {sve_z12, fpu_d12, fpu_s12,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v13_invalidates[] = {sve_z13, fpu_d13, fpu_s13,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v14_invalidates[] = {sve_z14, fpu_d14, fpu_s14,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v15_invalidates[] = {sve_z15, fpu_d15, fpu_s15,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v16_invalidates[] = {sve_z16, fpu_d16, fpu_s16,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v17_invalidates[] = {sve_z17, fpu_d17, fpu_s17,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v18_invalidates[] = {sve_z18, fpu_d18, fpu_s18,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v19_invalidates[] = {sve_z19, fpu_d19, fpu_s19,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v20_invalidates[] = {sve_z20, fpu_d20, fpu_s20,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v21_invalidates[] = {sve_z21, fpu_d21, fpu_s21,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v22_invalidates[] = {sve_z22, fpu_d22, fpu_s22,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v23_invalidates[] = {sve_z23, fpu_d23, fpu_s23,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v24_invalidates[] = {sve_z24, fpu_d24, fpu_s24,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v25_invalidates[] = {sve_z25, fpu_d25, fpu_s25,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v26_invalidates[] = {sve_z26, fpu_d26, fpu_s26,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v27_invalidates[] = {sve_z27, fpu_d27, fpu_s27,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v28_invalidates[] = {sve_z28, fpu_d28, fpu_s28,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v29_invalidates[] = {sve_z29, fpu_d29, fpu_s29,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v30_invalidates[] = {sve_z30, fpu_d30, fpu_s30,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_v31_invalidates[] = {sve_z31, fpu_d31, fpu_s31,
+                                           LLDB_INVALID_REGNUM};
+
+static uint32_t g_sve_z0_invalidates[] = {fpu_v0, fpu_d0, fpu_s0,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z1_invalidates[] = {fpu_v1, fpu_d1, fpu_s1,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z2_invalidates[] = {fpu_v2, fpu_d2, fpu_s2,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z3_invalidates[] = {fpu_v3, fpu_d3, fpu_s3,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z4_invalidates[] = {fpu_v4, fpu_d4, fpu_s4,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z5_invalidates[] = {fpu_v5, fpu_d5, fpu_s5,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z6_invalidates[] = {fpu_v6, fpu_d6, fpu_s6,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z7_invalidates[] = {fpu_v7, fpu_d7, fpu_s7,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z8_invalidates[] = {fpu_v8, fpu_d8, fpu_s8,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z9_invalidates[] = {fpu_v9, fpu_d9, fpu_s9,
+                                          LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z10_invalidates[] = {fpu_v10, fpu_d10, fpu_s10,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z11_invalidates[] = {fpu_v11, fpu_d11, fpu_s11,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z12_invalidates[] = {fpu_v12, fpu_d12, fpu_s12,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z13_invalidates[] = {fpu_v13, fpu_d13, fpu_s13,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z14_invalidates[] = {fpu_v14, fpu_d14, fpu_s14,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z15_invalidates[] = {fpu_v15, fpu_d15, fpu_s15,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z16_invalidates[] = {fpu_v16, fpu_d16, fpu_s16,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z17_invalidates[] = {fpu_v17, fpu_d17, fpu_s17,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z18_invalidates[] = {fpu_v18, fpu_d18, fpu_s18,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z19_invalidates[] = {fpu_v19, fpu_d19, fpu_s19,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z20_invalidates[] = {fpu_v20, fpu_d20, fpu_s20,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z21_invalidates[] = {fpu_v21, fpu_d21, fpu_s21,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z22_invalidates[] = {fpu_v22, fpu_d22, fpu_s22,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z23_invalidates[] = {fpu_v23, fpu_d23, fpu_s23,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z24_invalidates[] = {fpu_v24, fpu_d24, fpu_s24,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z25_invalidates[] = {fpu_v25, fpu_d25, fpu_s25,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z26_invalidates[] = {fpu_v26, fpu_d26, fpu_s26,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z27_invalidates[] = {fpu_v27, fpu_d27, fpu_s27,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z28_invalidates[] = {fpu_v28, fpu_d28, fpu_s28,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z29_invalidates[] = {fpu_v29, fpu_d29, fpu_s29,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z30_invalidates[] = {fpu_v30, fpu_d30, fpu_s30,
+                                           LLDB_INVALID_REGNUM};
+static uint32_t g_sve_z31_invalidates[] = {fpu_v31, fpu_d31, fpu_s31,
+                                           LLDB_INVALID_REGNUM};
+
+static uint32_t g_contained_z0[] = {sve_z0, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z1[] = {sve_z1, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z2[] = {sve_z2, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z3[] = {sve_z3, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z4[] = {sve_z4, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z5[] = {sve_z5, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z6[] = {sve_z6, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z7[] = {sve_z7, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z8[] = {sve_z8, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z9[] = {sve_z9, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z10[] = {sve_z10, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z11[] = {sve_z11, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z12[] = {sve_z12, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z13[] = {sve_z13, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z14[] = {sve_z14, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z15[] = {sve_z15, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z16[] = {sve_z16, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z17[] = {sve_z17, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z18[] = {sve_z18, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z19[] = {sve_z19, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z20[] = {sve_z20, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z21[] = {sve_z21, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z22[] = {sve_z22, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z23[] = {sve_z23, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z24[] = {sve_z24, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z25[] = {sve_z25, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z26[] = {sve_z26, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z27[] = {sve_z27, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z28[] = {sve_z28, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z29[] = {sve_z29, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z30[] = {sve_z30, LLDB_INVALID_REGNUM};
+static uint32_t g_contained_z31[] = {sve_z31, LLDB_INVALID_REGNUM};
+
+#define VG_OFFSET_NAME(reg) SVE_OFFSET_VG
+
+#define SVE_REG_KIND(reg) MISC_KIND(reg, sve, LLDB_INVALID_REGNUM)
+#define MISC_VG_KIND(lldb_kind) MISC_KIND(vg, sve, LLDB_INVALID_REGNUM)
+
+// Default offset SVE Z registers and all corresponding pseudo registers
+// ( S, D and V registers) is zero and will be configured during execution.
+
+// Defines sve pseudo vector (V) register with 16-byte size
+#define DEFINE_VREG_SVE(vreg, zreg)                                            \
+  {                                                                            \
+    #vreg, nullptr, 16, 0, lldb::eEncodingVector, lldb::eFormatVectorOfUInt8,  \
+        VREG_KIND(vreg), g_contained_##zreg, g_sve_##vreg##_invalidates,       \
+        nullptr, 0                                                             \
+  }
+
+// Defines S and D pseudo registers mapping over corresponding vector register
+#define DEFINE_FPU_PSEUDO_SVE(reg, size, zreg)                                 \
+  {                                                                            \
+    #reg, nullptr, size, 0, lldb::eEncodingIEEE754, lldb::eFormatFloat,        \
+        LLDB_KIND(fpu_##reg), g_contained_##zreg, g_sve_##reg##_invalidates,   \
+        nullptr, 0                                                             \
+  }
+
+// Defines a Z vector register with 16-byte default size
+#define DEFINE_ZREG(reg)                                                       \
+  {                                                                            \
+    #reg, nullptr, 16, 0, lldb::eEncodingVector, lldb::eFormatVectorOfUInt8,   \
+        SVE_REG_KIND(reg), nullptr, g_sve_##reg##_invalidates, nullptr, 0      \
+  }
+
+// Defines a P vector register with 2-byte default size
+#define DEFINE_PREG(reg)                                                       \
+  {                                                                            \
+    #reg, nullptr, 2, 0, lldb::eEncodingVector, lldb::eFormatVectorOfUInt8,    \
+        SVE_REG_KIND(reg), nullptr, nullptr, nullptr, 0                        \
+  }
+
+static lldb_private::RegisterInfo g_register_infos_arm64_sve_le[] = {
+    // clang-format off
+    // DEFINE_GPR64(name, GENERIC KIND)
+    DEFINE_GPR64(x0, LLDB_REGNUM_GENERIC_ARG1),
+    DEFINE_GPR64(x1, LLDB_REGNUM_GENERIC_ARG2),
+    DEFINE_GPR64(x2, LLDB_REGNUM_GENERIC_ARG3),
+    DEFINE_GPR64(x3, LLDB_REGNUM_GENERIC_ARG4),
+    DEFINE_GPR64(x4, LLDB_REGNUM_GENERIC_ARG5),
+    DEFINE_GPR64(x5, LLDB_REGNUM_GENERIC_ARG6),
+    DEFINE_GPR64(x6, LLDB_REGNUM_GENERIC_ARG7),
+    DEFINE_GPR64(x7, LLDB_REGNUM_GENERIC_ARG8),
+    DEFINE_GPR64(x8, LLDB_INVALID_REGNUM),
+    DEFINE_GPR64(x9, LLDB_INVALID_REGNUM),
+    DEFINE_GPR64(x10, LLDB_INVALID_REGNUM),
+    DEFINE_GPR64(x11, LLDB_INVALID_REGNUM),
+    DEFINE_GPR64(x12, LLDB_INVALID_REGNUM),
+    DEFINE_GPR64(x13, LLDB_INVALID_REGNUM),
+    DEFINE_GPR64(x14, LLDB_INVALID_REGNUM),
+    DEFINE_GPR64(x15, LLDB_INVALID_REGNUM),
+    DEFINE_GPR64(x16, LLDB_INVALID_REGNUM),
+    DEFINE_GPR64(x17, LLDB_INVALID_REGNUM),
+    DEFINE_GPR64(x18, LLDB_INVALID_REGNUM),
+    DEFINE_GPR64(x19, LLDB_INVALID_REGNUM),
+    DEFINE_GPR64(x20, LLDB_INVALID_REGNUM),
+    DEFINE_GPR64(x21, LLDB_INVALID_REGNUM),
+    DEFINE_GPR64(x22, LLDB_INVALID_REGNUM),
+    DEFINE_GPR64(x23, LLDB_INVALID_REGNUM),
+    DEFINE_GPR64(x24, LLDB_INVALID_REGNUM),
+    DEFINE_GPR64(x25, LLDB_INVALID_REGNUM),
+    DEFINE_GPR64(x26, LLDB_INVALID_REGNUM),
+    DEFINE_GPR64(x27, LLDB_INVALID_REGNUM),
+    DEFINE_GPR64(x28, LLDB_INVALID_REGNUM),
+    // DEFINE_GPR64(name, GENERIC KIND)
+    DEFINE_GPR64_ALT(fp, x29, LLDB_REGNUM_GENERIC_FP),
+    DEFINE_GPR64_ALT(lr, x30, LLDB_REGNUM_GENERIC_RA),
+    DEFINE_GPR64_ALT(sp, x31, LLDB_REGNUM_GENERIC_SP),
+    DEFINE_GPR64(pc, LLDB_REGNUM_GENERIC_PC),
+
+    // DEFINE_MISC_REGS(name, size, TYPE, lldb kind)
+    DEFINE_MISC_REGS(cpsr, 4, GPR, gpr_cpsr),
+
+    // DEFINE_GPR32(name, parent name)
+    DEFINE_GPR32(w0, x0),
+    DEFINE_GPR32(w1, x1),
+    DEFINE_GPR32(w2, x2),
+    DEFINE_GPR32(w3, x3),
+    DEFINE_GPR32(w4, x4),
+    DEFINE_GPR32(w5, x5),
+    DEFINE_GPR32(w6, x6),
+    DEFINE_GPR32(w7, x7),
+    DEFINE_GPR32(w8, x8),
+    DEFINE_GPR32(w9, x9),
+    DEFINE_GPR32(w10, x10),
+    DEFINE_GPR32(w11, x11),
+    DEFINE_GPR32(w12, x12),
+    DEFINE_GPR32(w13, x13),
+    DEFINE_GPR32(w14, x14),
+    DEFINE_GPR32(w15, x15),
+    DEFINE_GPR32(w16, x16),
+    DEFINE_GPR32(w17, x17),
+    DEFINE_GPR32(w18, x18),
+    DEFINE_GPR32(w19, x19),
+    DEFINE_GPR32(w20, x20),
+    DEFINE_GPR32(w21, x21),
+    DEFINE_GPR32(w22, x22),
+    DEFINE_GPR32(w23, x23),
+    DEFINE_GPR32(w24, x24),
+    DEFINE_GPR32(w25, x25),
+    DEFINE_GPR32(w26, x26),
+    DEFINE_GPR32(w27, x27),
+    DEFINE_GPR32(w28, x28),
+
+    // DEFINE_VREG_SVE(v register, z register)
+    DEFINE_VREG_SVE(v0, z0),
+    DEFINE_VREG_SVE(v1, z1),
+    DEFINE_VREG_SVE(v2, z2),
+    DEFINE_VREG_SVE(v3, z3),
+    DEFINE_VREG_SVE(v4, z4),
+    DEFINE_VREG_SVE(v5, z5),
+    DEFINE_VREG_SVE(v6, z6),
+    DEFINE_VREG_SVE(v7, z7),
+    DEFINE_VREG_SVE(v8, z8),
+    DEFINE_VREG_SVE(v9, z9),
+    DEFINE_VREG_SVE(v10, z10),
+    DEFINE_VREG_SVE(v11, z11),
+    DEFINE_VREG_SVE(v12, z12),
+    DEFINE_VREG_SVE(v13, z13),
+    DEFINE_VREG_SVE(v14, z14),
+    DEFINE_VREG_SVE(v15, z15),
+    DEFINE_VREG_SVE(v16, z16),
+    DEFINE_VREG_SVE(v17, z17),
+    DEFINE_VREG_SVE(v18, z18),
+    DEFINE_VREG_SVE(v19, z19),
+    DEFINE_VREG_SVE(v20, z20),
+    DEFINE_VREG_SVE(v21, z21),
+    DEFINE_VREG_SVE(v22, z22),
+    DEFINE_VREG_SVE(v23, z23),
+    DEFINE_VREG_SVE(v24, z24),
+    DEFINE_VREG_SVE(v25, z25),
+    DEFINE_VREG_SVE(v26, z26),
+    DEFINE_VREG_SVE(v27, z27),
+    DEFINE_VREG_SVE(v28, z28),
+    DEFINE_VREG_SVE(v29, z29),
+    DEFINE_VREG_SVE(v30, z30),
+    DEFINE_VREG_SVE(v31, z31),
+
+    // DEFINE_FPU_PSEUDO(name, size, ENDIAN OFFSET, parent register)
+    DEFINE_FPU_PSEUDO_SVE(s0, 4, z0),
+    DEFINE_FPU_PSEUDO_SVE(s1, 4, z1),
+    DEFINE_FPU_PSEUDO_SVE(s2, 4, z2),
+    DEFINE_FPU_PSEUDO_SVE(s3, 4, z3),
+    DEFINE_FPU_PSEUDO_SVE(s4, 4, z4),
+    DEFINE_FPU_PSEUDO_SVE(s5, 4, z5),
+    DEFINE_FPU_PSEUDO_SVE(s6, 4, z6),
+    DEFINE_FPU_PSEUDO_SVE(s7, 4, z7),
+    DEFINE_FPU_PSEUDO_SVE(s8, 4, z8),
+    DEFINE_FPU_PSEUDO_SVE(s9, 4, z9),
+    DEFINE_FPU_PSEUDO_SVE(s10, 4, z10),
+    DEFINE_FPU_PSEUDO_SVE(s11, 4, z11),
+    DEFINE_FPU_PSEUDO_SVE(s12, 4, z12),
+    DEFINE_FPU_PSEUDO_SVE(s13, 4, z13),
+    DEFINE_FPU_PSEUDO_SVE(s14, 4, z14),
+    DEFINE_FPU_PSEUDO_SVE(s15, 4, z15),
+    DEFINE_FPU_PSEUDO_SVE(s16, 4, z16),
+    DEFINE_FPU_PSEUDO_SVE(s17, 4, z17),
+    DEFINE_FPU_PSEUDO_SVE(s18, 4, z18),
+    DEFINE_FPU_PSEUDO_SVE(s19, 4, z19),
+    DEFINE_FPU_PSEUDO_SVE(s20, 4, z20),
+    DEFINE_FPU_PSEUDO_SVE(s21, 4, z21),
+    DEFINE_FPU_PSEUDO_SVE(s22, 4, z22),
+    DEFINE_FPU_PSEUDO_SVE(s23, 4, z23),
+    DEFINE_FPU_PSEUDO_SVE(s24, 4, z24),
+    DEFINE_FPU_PSEUDO_SVE(s25, 4, z25),
+    DEFINE_FPU_PSEUDO_SVE(s26, 4, z26),
+    DEFINE_FPU_PSEUDO_SVE(s27, 4, z27),
+    DEFINE_FPU_PSEUDO_SVE(s28, 4, z28),
+    DEFINE_FPU_PSEUDO_SVE(s29, 4, z29),
+    DEFINE_FPU_PSEUDO_SVE(s30, 4, z30),
+    DEFINE_FPU_PSEUDO_SVE(s31, 4, z31),
+
+    DEFINE_FPU_PSEUDO_SVE(d0, 8, z0),
+    DEFINE_FPU_PSEUDO_SVE(d1, 8, z1),
+    DEFINE_FPU_PSEUDO_SVE(d2, 8, z2),
+    DEFINE_FPU_PSEUDO_SVE(d3, 8, z3),
+    DEFINE_FPU_PSEUDO_SVE(d4, 8, z4),
+    DEFINE_FPU_PSEUDO_SVE(d5, 8, z5),
+    DEFINE_FPU_PSEUDO_SVE(d6, 8, z6),
+    DEFINE_FPU_PSEUDO_SVE(d7, 8, z7),
+    DEFINE_FPU_PSEUDO_SVE(d8, 8, z8),
+    DEFINE_FPU_PSEUDO_SVE(d9, 8, z9),
+    DEFINE_FPU_PSEUDO_SVE(d10, 8, z10),
+    DEFINE_FPU_PSEUDO_SVE(d11, 8, z11),
+    DEFINE_FPU_PSEUDO_SVE(d12, 8, z12),
+    DEFINE_FPU_PSEUDO_SVE(d13, 8, z13),
+    DEFINE_FPU_PSEUDO_SVE(d14, 8, z14),
+    DEFINE_FPU_PSEUDO_SVE(d15, 8, z15),
+    DEFINE_FPU_PSEUDO_SVE(d16, 8, z16),
+    DEFINE_FPU_PSEUDO_SVE(d17, 8, z17),
+    DEFINE_FPU_PSEUDO_SVE(d18, 8, z18),
+    DEFINE_FPU_PSEUDO_SVE(d19, 8, z19),
+    DEFINE_FPU_PSEUDO_SVE(d20, 8, z20),
+    DEFINE_FPU_PSEUDO_SVE(d21, 8, z21),
+    DEFINE_FPU_PSEUDO_SVE(d22, 8, z22),
+    DEFINE_FPU_PSEUDO_SVE(d23, 8, z23),
+    DEFINE_FPU_PSEUDO_SVE(d24, 8, z24),
+    DEFINE_FPU_PSEUDO_SVE(d25, 8, z25),
+    DEFINE_FPU_PSEUDO_SVE(d26, 8, z26),
+    DEFINE_FPU_PSEUDO_SVE(d27, 8, z27),
+    DEFINE_FPU_PSEUDO_SVE(d28, 8, z28),
+    DEFINE_FPU_PSEUDO_SVE(d29, 8, z29),
+    DEFINE_FPU_PSEUDO_SVE(d30, 8, z30),
+    DEFINE_FPU_PSEUDO_SVE(d31, 8, z31),
+
+    // DEFINE_MISC_REGS(name, size, TYPE, lldb kind)
+    DEFINE_MISC_REGS(fpsr, 4, FPU, fpu_fpsr),
+    DEFINE_MISC_REGS(fpcr, 4, FPU, fpu_fpcr),
+
+    DEFINE_MISC_REGS(vg, 8, VG, sve_vg),
+    // DEFINE_ZREG(name)
+    DEFINE_ZREG(z0),
+    DEFINE_ZREG(z1),
+    DEFINE_ZREG(z2),
+    DEFINE_ZREG(z3),
+    DEFINE_ZREG(z4),
+    DEFINE_ZREG(z5),
+    DEFINE_ZREG(z6),
+    DEFINE_ZREG(z7),
+    DEFINE_ZREG(z8),
+    DEFINE_ZREG(z9),
+    DEFINE_ZREG(z10),
+    DEFINE_ZREG(z11),
+    DEFINE_ZREG(z12),
+    DEFINE_ZREG(z13),
+    DEFINE_ZREG(z14),
+    DEFINE_ZREG(z15),
+    DEFINE_ZREG(z16),
+    DEFINE_ZREG(z17),
+    DEFINE_ZREG(z18),
+    DEFINE_ZREG(z19),
+    DEFINE_ZREG(z20),
+    DEFINE_ZREG(z21),
+    DEFINE_ZREG(z22),
+    DEFINE_ZREG(z23),
+    DEFINE_ZREG(z24),
+    DEFINE_ZREG(z25),
+    DEFINE_ZREG(z26),
+    DEFINE_ZREG(z27),
+    DEFINE_ZREG(z28),
+    DEFINE_ZREG(z29),
+    DEFINE_ZREG(z30),
+    DEFINE_ZREG(z31),
+
+    // DEFINE_PREG(name)
+    DEFINE_PREG(p0),
+    DEFINE_PREG(p1),
+    DEFINE_PREG(p2),
+    DEFINE_PREG(p3),
+    DEFINE_PREG(p4),
+    DEFINE_PREG(p5),
+    DEFINE_PREG(p6),
+    DEFINE_PREG(p7),
+    DEFINE_PREG(p8),
+    DEFINE_PREG(p9),
+    DEFINE_PREG(p10),
+    DEFINE_PREG(p11),
+    DEFINE_PREG(p12),
+    DEFINE_PREG(p13),
+    DEFINE_PREG(p14),
+    DEFINE_PREG(p15),
+
+    // DEFINE FFR
+    DEFINE_PREG(ffr)
+    // clang-format on
+};
+
+#endif // DECLARE_REGISTER_INFOS_ARM64_SVE_STRUCT
diff --git a/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.cpp b/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.cpp
index 685567416983c..5dea227bf6651 100644
--- a/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.cpp
+++ b/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "RegisterContextPOSIXCore_arm64.h"
+#include "Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h"
 
 #include "Plugins/Process/elf-core/RegisterUtilities.h"
 #include "lldb/Target/Thread.h"
@@ -27,6 +28,12 @@ RegisterContextCorePOSIX_arm64::RegisterContextCorePOSIX_arm64(
 
   m_fpregset = getRegset(
       notes, m_register_info_up->GetTargetArchitecture().GetTriple(), FPR_Desc);
+
+  m_sveregset =
+      getRegset(notes, m_register_info_up->GetTargetArchitecture().GetTriple(),
+                AARCH64_SVE_Desc);
+
+  ConfigureRegisterContext();
 }
 
 RegisterContextCorePOSIX_arm64::~RegisterContextCorePOSIX_arm64() {}
@@ -45,9 +52,55 @@ bool RegisterContextCorePOSIX_arm64::WriteFPR() {
   return false;
 }
 
+const uint8_t *RegisterContextCorePOSIX_arm64::GetSVEBuffer(uint64_t offset) {
+  return m_sveregset.GetDataStart() + offset;
+}
+
+void RegisterContextCorePOSIX_arm64::ConfigureRegisterContext() {
+  if (m_sveregset.GetByteSize() > sizeof(user_sve_header)) {
+    uint64_t sve_header_field_offset = 8;
+    m_sve_vector_length = m_sveregset.GetU16(&sve_header_field_offset);
+    sve_header_field_offset = 12;
+    uint16_t sve_header_flags_field =
+        m_sveregset.GetU16(&sve_header_field_offset);
+    if ((sve_header_flags_field & SVE_PT_REGS_MASK) == SVE_PT_REGS_FPSIMD)
+      m_sve_state = SVEState::FPSIMD;
+    else if ((sve_header_flags_field & SVE_PT_REGS_MASK) == SVE_PT_REGS_SVE)
+      m_sve_state = SVEState::Full;
+
+    if (sve_vl_valid(m_sve_vector_length))
+      m_register_info_up->ConfigureVectorRegisterInfos(
+          sve_vq_from_vl(m_sve_vector_length));
+    else {
+      m_sve_state = SVEState::Disabled;
+      m_sve_vector_length = 0;
+    }
+  } else
+    m_sve_state = SVEState::Disabled;
+}
+
+uint32_t RegisterContextCorePOSIX_arm64::CalculateSVEOffset(
+    const RegisterInfo *reg_info) {
+  // Start of Z0 data is after GPRs plus 8 bytes of vg register
+  uint32_t sve_reg_offset = LLDB_INVALID_INDEX32;
+  if (m_sve_state == SVEState::FPSIMD) {
+    const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB];
+    sve_reg_offset = SVE_PT_FPSIMD_OFFSET + (reg - GetRegNumSVEZ0()) * 16;
+  } else if (m_sve_state == SVEState::Full) {
+    uint32_t sve_z0_offset = GetGPRSize() + 8;
+    sve_reg_offset =
+        SVE_SIG_REGS_OFFSET + reg_info->byte_offset - sve_z0_offset;
+  }
+
+  return sve_reg_offset;
+}
+
 bool RegisterContextCorePOSIX_arm64::ReadRegister(const RegisterInfo *reg_info,
                                                   RegisterValue &value) {
-  lldb::offset_t offset = reg_info->byte_offset;
+  Status error;
+  lldb::offset_t offset;
+
+  offset = reg_info->byte_offset;
   if (offset + reg_info->byte_size <= GetGPRSize()) {
     uint64_t v = m_gpr.GetMaxU64(&offset, reg_info->byte_size);
     if (offset == reg_info->byte_offset + reg_info->byte_size) {
@@ -60,15 +113,86 @@ bool RegisterContextCorePOSIX_arm64::ReadRegister(const RegisterInfo *reg_info,
   if (reg == LLDB_INVALID_REGNUM)
     return false;
 
-  offset -= GetGPRSize();
-  if (IsFPR(reg) && offset + reg_info->byte_size <= GetFPUSize()) {
-    Status error;
-    value.SetFromMemoryData(reg_info, m_fpregset.GetDataStart() + offset,
-                            reg_info->byte_size, lldb::eByteOrderLittle, error);
-    return error.Success();
-  }
+  if (IsFPR(reg)) {
+    if (m_sve_state == SVEState::Disabled) {
+      // SVE is disabled take legacy route for FPU register access
+      offset -= GetGPRSize();
+      if (offset < m_fpregset.GetByteSize()) {
+        value.SetFromMemoryData(reg_info, m_fpregset.GetDataStart() + offset,
+                                reg_info->byte_size, lldb::eByteOrderLittle,
+                                error);
+        return error.Success();
+      }
+    } else {
+      // FPSR and FPCR will be located right after Z registers in
+      // SVEState::FPSIMD while in SVEState::Full they will be located at the
+      // end of register data after an alignment correction based on currently
+      // selected vector length.
+      uint32_t sve_reg_num = LLDB_INVALID_REGNUM;
+      if (reg == GetRegNumFPSR()) {
+        sve_reg_num = reg;
+        if (m_sve_state == SVEState::Full)
+          offset = SVE_PT_SVE_FPSR_OFFSET(sve_vq_from_vl(m_sve_vector_length));
+        else if (m_sve_state == SVEState::FPSIMD)
+          offset = SVE_PT_FPSIMD_OFFSET + (32 * 16);
+      } else if (reg == GetRegNumFPCR()) {
+        sve_reg_num = reg;
+        if (m_sve_state == SVEState::Full)
+          offset = SVE_PT_SVE_FPCR_OFFSET(sve_vq_from_vl(m_sve_vector_length));
+        else if (m_sve_state == SVEState::FPSIMD)
+          offset = SVE_PT_FPSIMD_OFFSET + (32 * 16) + 4;
+      } else {
+        // Extract SVE Z register value register number for this reg_info
+        if (reg_info->value_regs &&
+            reg_info->value_regs[0] != LLDB_INVALID_REGNUM)
+          sve_reg_num = reg_info->value_regs[0];
+        offset = CalculateSVEOffset(GetRegisterInfoAtIndex(sve_reg_num));
+      }
+
+      assert(sve_reg_num != LLDB_INVALID_REGNUM);
+      assert(offset < m_sveregset.GetByteSize());
+      value.SetFromMemoryData(reg_info, GetSVEBuffer(offset),
+                              reg_info->byte_size, lldb::eByteOrderLittle,
+                              error);
+    }
+  } else if (IsSVE(reg)) {
+    if (IsSVEVG(reg)) {
+      value = GetSVERegVG();
+      return true;
+    }
 
-  return false;
+    switch (m_sve_state) {
+    case SVEState::FPSIMD: {
+      // In FPSIMD state SVE payload mirrors legacy fpsimd struct and so just
+      // copy 16 bytes of v register to the start of z register. All other
+      // SVE register will be set to zero.
+      uint64_t byte_size = 1;
+      uint8_t zeros = 0;
+      const uint8_t *src = &zeros;
+      if (IsSVEZ(reg)) {
+        byte_size = 16;
+        offset = CalculateSVEOffset(reg_info);
+        assert(offset < m_sveregset.GetByteSize());
+        src = GetSVEBuffer(offset);
+      }
+      value.SetFromMemoryData(reg_info, src, byte_size, lldb::eByteOrderLittle,
+                              error);
+    } break;
+    case SVEState::Full:
+      offset = CalculateSVEOffset(reg_info);
+      assert(offset < m_sveregset.GetByteSize());
+      value.SetFromMemoryData(reg_info, GetSVEBuffer(offset),
+                              reg_info->byte_size, lldb::eByteOrderLittle,
+                              error);
+      break;
+    case SVEState::Disabled:
+    default:
+      return false;
+    }
+  } else
+    return false;
+
+  return error.Success();
 }
 
 bool RegisterContextCorePOSIX_arm64::ReadAllRegisterValues(
diff --git a/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.h b/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.h
index 830e0ff91e4c6..a4fdc4f143285 100644
--- a/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.h
+++ b/lldb/source/Plugins/Process/elf-core/RegisterContextPOSIXCore_arm64.h
@@ -9,7 +9,9 @@
 #ifndef LLDB_SOURCE_PLUGINS_PROCESS_ELF_CORE_REGISTERCONTEXTPOSIXCORE_ARM64_H
 #define LLDB_SOURCE_PLUGINS_PROCESS_ELF_CORE_REGISTERCONTEXTPOSIXCORE_ARM64_H
 
+#include "Plugins/Process/Utility/LinuxPTraceDefines_arm64sve.h"
 #include "Plugins/Process/Utility/RegisterContextPOSIX_arm64.h"
+
 #include "Plugins/Process/elf-core/RegisterUtilities.h"
 #include "lldb/Utility/DataBufferHeap.h"
 #include "lldb/Utility/DataExtractor.h"
@@ -49,6 +51,18 @@ class RegisterContextCorePOSIX_arm64 : public RegisterContextPOSIX_arm64 {
   lldb::DataBufferSP m_gpr_buffer;
   lldb_private::DataExtractor m_gpr;
   lldb_private::DataExtractor m_fpregset;
+  lldb_private::DataExtractor m_sveregset;
+
+  SVEState m_sve_state;
+  uint16_t m_sve_vector_length = 0;
+
+  const uint8_t *GetSVEBuffer(uint64_t offset = 0);
+
+  void ConfigureRegisterContext();
+
+  uint32_t CalculateSVEOffset(const lldb_private::RegisterInfo *reg_info);
+
+  uint64_t GetSVERegVG() { return m_sve_vector_length / 8; }
 };
 
 #endif // LLDB_SOURCE_PLUGINS_PROCESS_ELF_CORE_REGISTERCONTEXTPOSIXCORE_ARM64_H
diff --git a/lldb/source/Plugins/Process/elf-core/RegisterUtilities.h b/lldb/source/Plugins/Process/elf-core/RegisterUtilities.h
index 4e08aa280817a..25abd7ed54b7f 100644
--- a/lldb/source/Plugins/Process/elf-core/RegisterUtilities.h
+++ b/lldb/source/Plugins/Process/elf-core/RegisterUtilities.h
@@ -107,6 +107,10 @@ constexpr RegsetDesc FPR_Desc[] = {
     {llvm::Triple::OpenBSD, llvm::Triple::UnknownArch, OPENBSD::NT_FPREGS},
 };
 
+constexpr RegsetDesc AARCH64_SVE_Desc[] = {
+    {llvm::Triple::Linux, llvm::Triple::aarch64, llvm::ELF::NT_ARM_SVE},
+};
+
 constexpr RegsetDesc PPC_VMX_Desc[] = {
     {llvm::Triple::FreeBSD, llvm::Triple::UnknownArch, llvm::ELF::NT_PPC_VMX},
     {llvm::Triple::Linux, llvm::Triple::UnknownArch, llvm::ELF::NT_PPC_VMX},
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
index bfacd41dc1a3a..6a60502416959 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
@@ -50,7 +50,7 @@
 #include <compression.h>
 #endif
 
-#if defined(HAVE_LIBZ)
+#if LLVM_ENABLE_ZLIB
 #include <zlib.h>
 #endif
 
@@ -582,7 +582,7 @@ bool GDBRemoteCommunication::DecompressPacket() {
   }
 #endif
 
-#if defined(HAVE_LIBZ)
+#if LLVM_ENABLE_ZLIB
   if (decompressed_bytes == 0 && decompressed_bufsize != ULONG_MAX &&
       decompressed_buffer != nullptr &&
       m_compression_type == CompressionType::ZlibDeflate) {
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
index c75d5e106cd02..d77f7a0b5a379 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
@@ -1053,7 +1053,7 @@ void GDBRemoteCommunicationClient::MaybeEnableCompression(
   }
 #endif
 
-#if defined(HAVE_LIBZ)
+#if LLVM_ENABLE_ZLIB
   if (avail_type == CompressionType::None) {
     for (auto compression : supported_compressions) {
       if (compression == "zlib-deflate") {
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
index 2d1db66e7fd91..35e7c34734e2c 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
@@ -209,11 +209,12 @@ TypeSP DWARFASTParserClang::ParseTypeFromClangModule(const SymbolContext &sc,
     GetClangASTImporter().RequireCompleteType(ClangUtil::GetQualType(type));
 
   SymbolFileDWARF *dwarf = die.GetDWARF();
-  TypeSP type_sp(new Type(
-      die.GetID(), dwarf, pcm_type_sp->GetName(), pcm_type_sp->GetByteSize(),
-      nullptr, LLDB_INVALID_UID, Type::eEncodingInvalid,
-      &pcm_type_sp->GetDeclaration(), type, Type::ResolveState::Forward,
-      TypePayloadClang(GetOwningClangModule(die))));
+  TypeSP type_sp(new Type(die.GetID(), dwarf, pcm_type_sp->GetName(),
+                          pcm_type_sp->GetByteSize(nullptr), nullptr,
+                          LLDB_INVALID_UID, Type::eEncodingInvalid,
+                          &pcm_type_sp->GetDeclaration(), type,
+                          Type::ResolveState::Forward,
+                          TypePayloadClang(GetOwningClangModule(die))));
 
   dwarf->GetTypeList().Insert(type_sp);
   dwarf->GetDIEToType()[die.GetDIE()] = type_sp.get();
@@ -810,7 +811,7 @@ TypeSP DWARFASTParserClang::ParseEnum(const SymbolContext &sc,
       bool is_signed = false;
       enumerator_clang_type.IsIntegerType(is_signed);
       ParseChildEnumerators(clang_type, is_signed,
-                            type_sp->GetByteSize().getValueOr(0), die);
+                            type_sp->GetByteSize(nullptr).getValueOr(0), die);
     }
     TypeSystemClang::CompleteTagDeclarationDefinition(clang_type);
   } else {
@@ -1261,7 +1262,7 @@ TypeSP DWARFASTParserClang::ParseArrayType(const DWARFDIE &die,
     attrs.bit_stride = array_info->bit_stride;
   }
   if (attrs.byte_stride == 0 && attrs.bit_stride == 0)
-    attrs.byte_stride = element_type->GetByteSize().getValueOr(0);
+    attrs.byte_stride = element_type->GetByteSize(nullptr).getValueOr(0);
   CompilerType array_element_type = element_type->GetForwardCompilerType();
   CompleteType(array_element_type);
 
@@ -2057,7 +2058,7 @@ bool DWARFASTParserClang::CompleteRecordType(const DWARFDIE &die,
   if (!layout_info.field_offsets.empty() || !layout_info.base_offsets.empty() ||
       !layout_info.vbase_offsets.empty()) {
     if (type)
-      layout_info.bit_size = type->GetByteSize().getValueOr(0) * 8;
+      layout_info.bit_size = type->GetByteSize(nullptr).getValueOr(0) * 8;
     if (layout_info.bit_size == 0)
       layout_info.bit_size =
           die.GetAttributeValueAsUnsigned(DW_AT_byte_size, 0) * 8;
@@ -2079,7 +2080,7 @@ bool DWARFASTParserClang::CompleteEnumType(const DWARFDIE &die,
       bool is_signed = false;
       clang_type.IsIntegerType(is_signed);
       ParseChildEnumerators(clang_type, is_signed,
-                            type->GetByteSize().getValueOr(0), die);
+                            type->GetByteSize(nullptr).getValueOr(0), die);
     }
     TypeSystemClang::CompleteTagDeclarationDefinition(clang_type);
   }
@@ -2567,7 +2568,7 @@ void DWARFASTParserClang::ParseSingleMember(
             this_field_info.bit_offset = data_bit_offset;
           } else {
             if (!byte_size)
-              byte_size = member_type->GetByteSize();
+              byte_size = member_type->GetByteSize(nullptr);
 
             ObjectFile *objfile = die.GetDWARF()->GetObjectFile();
             if (objfile->GetByteOrder() == eByteOrderLittle) {
@@ -2578,10 +2579,14 @@ void DWARFASTParserClang::ParseSingleMember(
             }
           }
 
-          if ((this_field_info.bit_offset >= parent_bit_size) ||
-              (last_field_info.IsBitfield() &&
-               !last_field_info.NextBitfieldOffsetIsValid(
-                   this_field_info.bit_offset))) {
+          // The ObjC runtime knows the byte offset but we still need to provide
+          // the bit-offset in the layout. It just means something different then
+          // what it does in C and C++. So we skip this check for ObjC types.
+          if (!TypeSystemClang::IsObjCObjectOrInterfaceType(class_clang_type) &&
+              ((this_field_info.bit_offset >= parent_bit_size) ||
+               (last_field_info.IsBitfield() &&
+                !last_field_info.NextBitfieldOffsetIsValid(
+                    this_field_info.bit_offset)))) {
             ObjectFile *objfile = die.GetDWARF()->GetObjectFile();
             objfile->GetModule()->ReportWarning(
                 "0x%8.8" PRIx64 ": %s bitfield named \"%s\" has invalid "
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
index 0b7e31ae2d1df..5ce392a57e0c8 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
@@ -1791,7 +1791,7 @@ SymbolFileDWARF::GlobalVariableMap &SymbolFileDWARF::GetGlobalAranges() {
                     lldb::addr_t byte_size = 1;
                     if (var_sp->GetType())
                       byte_size =
-                          var_sp->GetType()->GetByteSize().getValueOr(0);
+                          var_sp->GetType()->GetByteSize(nullptr).getValueOr(0);
                     m_global_aranges_up->Append(GlobalVariableMap::Entry(
                         file_addr, byte_size, var_sp.get()));
                   }
@@ -3445,9 +3445,10 @@ VariableSP SymbolFileDWARF::ParseVariableDIE(const SymbolContext &sc,
             new SymbolFileType(*this, GetUID(type_die_form.Reference())));
 
         if (const_value.Form() && type_sp && type_sp->GetType())
-          location.UpdateValue(const_value.Unsigned(),
-                               type_sp->GetType()->GetByteSize().getValueOr(0),
-                               die.GetCU()->GetAddressByteSize());
+          location.UpdateValue(
+              const_value.Unsigned(),
+              type_sp->GetType()->GetByteSize(nullptr).getValueOr(0),
+              die.GetCU()->GetAddressByteSize());
 
         var_sp = std::make_shared<Variable>(
             die.GetID(), name, mangled, type_sp, scope, symbol_context_scope,
diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp
index cce06473d92f3..9f53e92afa0b1 100644
--- a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp
+++ b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp
@@ -460,7 +460,7 @@ lldb::TypeSP SymbolFileNativePDB::CreateModifierType(PdbTypeSymId type_id,
   lldb::TypeSP modified_type = GetOrCreateType(mr.ModifiedType);
 
   return std::make_shared<Type>(toOpaqueUid(type_id), this, ConstString(name),
-                                modified_type->GetByteSize(), nullptr,
+                                modified_type->GetByteSize(nullptr), nullptr,
                                 LLDB_INVALID_UID, Type::eEncodingIsUID, decl,
                                 ct, Type::ResolveState::Full);
 }
@@ -584,7 +584,7 @@ lldb::TypeSP SymbolFileNativePDB::CreateTagType(PdbTypeSymId type_id,
 
   return std::make_shared<lldb_private::Type>(
       toOpaqueUid(type_id), this, ConstString(uname),
-      underlying_type->GetByteSize(), nullptr, LLDB_INVALID_UID,
+      underlying_type->GetByteSize(nullptr), nullptr, LLDB_INVALID_UID,
       lldb_private::Type::eEncodingIsUID, decl, ct,
       lldb_private::Type::ResolveState::Forward);
 }
@@ -1376,9 +1376,10 @@ TypeSP SymbolFileNativePDB::CreateTypedef(PdbGlobalSymId id) {
 
   Declaration decl;
   return std::make_shared<lldb_private::Type>(
-      toOpaqueUid(id), this, ConstString(udt.Name), target_type->GetByteSize(),
-      nullptr, target_type->GetID(), lldb_private::Type::eEncodingIsTypedefUID,
-      decl, target_type->GetForwardCompilerType(),
+      toOpaqueUid(id), this, ConstString(udt.Name),
+      target_type->GetByteSize(nullptr), nullptr, target_type->GetID(),
+      lldb_private::Type::eEncodingIsTypedefUID, decl,
+      target_type->GetForwardCompilerType(),
       lldb_private::Type::ResolveState::Forward);
 }
 
diff --git a/lldb/source/Plugins/SymbolVendor/MacOSX/SymbolVendorMacOSX.cpp b/lldb/source/Plugins/SymbolVendor/MacOSX/SymbolVendorMacOSX.cpp
index fc426d1c9c376..336538241b6aa 100644
--- a/lldb/source/Plugins/SymbolVendor/MacOSX/SymbolVendorMacOSX.cpp
+++ b/lldb/source/Plugins/SymbolVendor/MacOSX/SymbolVendorMacOSX.cpp
@@ -299,7 +299,7 @@ SymbolVendorMacOSX::CreateInstance(const lldb::ModuleSP &module_sp,
           if (repro::Generator *g =
                   repro::Reproducer::Instance().GetGenerator()) {
             repro::FileProvider &fp = g->GetOrCreate<repro::FileProvider>();
-            fp.recordInterestingDirectory(dsym_root);
+            fp.RecordInterestingDirectoryRecursive(dsym_root);
           }
         }
         return symbol_vendor;
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index 8825b473cd33d..a1bde107bf7ad 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -1657,9 +1657,9 @@ bool TypeSystemClang::FieldIsBitfield(FieldDecl *field,
   if (field->isBitField()) {
     Expr *bit_width_expr = field->getBitWidth();
     if (bit_width_expr) {
-      llvm::APSInt bit_width_apsint;
-      if (bit_width_expr->isIntegerConstantExpr(bit_width_apsint, ast)) {
-        bitfield_bit_size = bit_width_apsint.getLimitedValue(UINT32_MAX);
+      if (Optional<llvm::APSInt> bit_width_apsint =
+              bit_width_expr->getIntegerConstantExpr(ast)) {
+        bitfield_bit_size = bit_width_apsint->getLimitedValue(UINT32_MAX);
         return true;
       }
     }
@@ -4089,7 +4089,8 @@ unsigned TypeSystemClang::GetTypeQualifiers(lldb::opaque_compiler_type_t type) {
 
 CompilerType
 TypeSystemClang::GetArrayElementType(lldb::opaque_compiler_type_t type,
-                                     uint64_t *stride) {
+                                     uint64_t *stride,
+                                     ExecutionContextScope *exe_scope) {
   if (type) {
     clang::QualType qual_type(GetQualType(type));
 
@@ -4103,7 +4104,7 @@ TypeSystemClang::GetArrayElementType(lldb::opaque_compiler_type_t type,
 
     // TODO: the real stride will be >= this value.. find the real one!
     if (stride)
-      if (Optional<uint64_t> size = element_type.GetByteSize(nullptr))
+      if (Optional<uint64_t> size = element_type.GetByteSize(exe_scope))
         *stride = *size;
 
     return element_type;
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
index 9475e4d9f4424..78c45ae32d3bb 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
@@ -689,7 +689,8 @@ class TypeSystemClang : public TypeSystem {
                     uint32_t opaque_payload);
 
   CompilerType GetArrayElementType(lldb::opaque_compiler_type_t type,
-                                   uint64_t *stride) override;
+                                   uint64_t *stride,
+                                   ExecutionContextScope *exe_scope) override;
 
   CompilerType GetArrayType(lldb::opaque_compiler_type_t type,
                             uint64_t size) override;
diff --git a/lldb/source/Symbol/CompilerType.cpp b/lldb/source/Symbol/CompilerType.cpp
index f819c9f8ce23d..c27bba26a208b 100644
--- a/lldb/source/Symbol/CompilerType.cpp
+++ b/lldb/source/Symbol/CompilerType.cpp
@@ -317,9 +317,10 @@ unsigned CompilerType::GetTypeQualifiers() const {
 
 // Creating related types
 
-CompilerType CompilerType::GetArrayElementType(uint64_t *stride) const {
+CompilerType CompilerType::GetArrayElementType(ExecutionContextScope *exe_scope,
+                                               uint64_t *stride) const {
   if (IsValid()) {
-    return m_type_system->GetArrayElementType(m_type, stride);
+    return m_type_system->GetArrayElementType(m_type, stride, exe_scope);
   }
   return CompilerType();
 }
@@ -766,8 +767,8 @@ LLVM_DUMP_METHOD void CompilerType::dump() const {
 
 bool CompilerType::GetValueAsScalar(const lldb_private::DataExtractor &data,
                                     lldb::offset_t data_byte_offset,
-                                    size_t data_byte_size,
-                                    Scalar &value) const {
+                                    size_t data_byte_size, Scalar &value,
+                                    ExecutionContextScope *exe_scope) const {
   if (!IsValid())
     return false;
 
@@ -780,7 +781,7 @@ bool CompilerType::GetValueAsScalar(const lldb_private::DataExtractor &data,
     if (encoding == lldb::eEncodingInvalid || count != 1)
       return false;
 
-    llvm::Optional<uint64_t> byte_size = GetByteSize(nullptr);
+    llvm::Optional<uint64_t> byte_size = GetByteSize(exe_scope);
     if (!byte_size)
       return false;
     lldb::offset_t offset = data_byte_offset;
diff --git a/lldb/source/Symbol/SymbolContext.cpp b/lldb/source/Symbol/SymbolContext.cpp
index 12c2077154b91..51f56704cca66 100644
--- a/lldb/source/Symbol/SymbolContext.cpp
+++ b/lldb/source/Symbol/SymbolContext.cpp
@@ -204,7 +204,7 @@ void SymbolContext::GetDescription(Stream *s, lldb::DescriptionLevel level,
     Type *func_type = function->GetType();
     if (func_type) {
       s->Indent("   FuncType: ");
-      func_type->GetDescription(s, level, false);
+      func_type->GetDescription(s, level, false, target);
       s->EOL();
     }
   }
diff --git a/lldb/source/Symbol/Type.cpp b/lldb/source/Symbol/Type.cpp
index 307e99ac84b69..ecf0575b9a57e 100644
--- a/lldb/source/Symbol/Type.cpp
+++ b/lldb/source/Symbol/Type.cpp
@@ -170,7 +170,7 @@ Type::Type()
 }
 
 void Type::GetDescription(Stream *s, lldb::DescriptionLevel level,
-                          bool show_name) {
+                          bool show_name, ExecutionContextScope *exe_scope) {
   *s << "id = " << (const UserID &)*this;
 
   // Call the name accessor to make sure we resolve the type name
@@ -186,7 +186,7 @@ void Type::GetDescription(Stream *s, lldb::DescriptionLevel level,
   }
 
   // Call the get byte size accesor so we resolve our byte size
-  if (GetByteSize())
+  if (GetByteSize(exe_scope))
     s->Printf(", byte-size = %" PRIu64, m_byte_size);
   bool show_fullpaths = (level == lldb::eDescriptionLevelVerbose);
   m_decl.Dump(s, show_fullpaths);
@@ -323,7 +323,9 @@ void Type::DumpValue(ExecutionContext *exe_ctx, Stream *s,
 
     GetForwardCompilerType().DumpValue(
         exe_ctx, s, format == lldb::eFormatDefault ? GetFormat() : format, data,
-        data_byte_offset, GetByteSize().getValueOr(0),
+        data_byte_offset,
+        GetByteSize(exe_ctx ? exe_ctx->GetBestExecutionContextScope() : nullptr)
+            .getValueOr(0),
         0, // Bitfield bit size
         0, // Bitfield bit offset
         show_types, show_summary, verbose, 0);
@@ -336,7 +338,7 @@ Type *Type::GetEncodingType() {
   return m_encoding_type;
 }
 
-llvm::Optional<uint64_t> Type::GetByteSize() {
+llvm::Optional<uint64_t> Type::GetByteSize(ExecutionContextScope *exe_scope) {
   if (m_byte_size_has_value)
     return m_byte_size;
 
@@ -352,14 +354,14 @@ llvm::Optional<uint64_t> Type::GetByteSize() {
   case eEncodingIsTypedefUID: {
     Type *encoding_type = GetEncodingType();
     if (encoding_type)
-      if (llvm::Optional<uint64_t> size = encoding_type->GetByteSize()) {
+      if (llvm::Optional<uint64_t> size = encoding_type->GetByteSize(exe_scope)) {
         m_byte_size = *size;
         m_byte_size_has_value = true;
         return m_byte_size;
       }
 
     if (llvm::Optional<uint64_t> size =
-            GetLayoutCompilerType().GetByteSize(nullptr)) {
+            GetLayoutCompilerType().GetByteSize(exe_scope)) {
       m_byte_size = *size;
       m_byte_size_has_value = true;
         return m_byte_size;
@@ -430,7 +432,9 @@ bool Type::ReadFromMemory(ExecutionContext *exe_ctx, lldb::addr_t addr,
     return false;
   }
 
-  const uint64_t byte_size = GetByteSize().getValueOr(0);
+  const uint64_t byte_size =
+      GetByteSize(exe_ctx ? exe_ctx->GetBestExecutionContextScope() : nullptr)
+          .getValueOr(0);
   if (data.GetByteSize() < byte_size) {
     lldb::DataBufferSP data_sp(new DataBufferHeap(byte_size, '\0'));
     data.SetData(data_sp);
diff --git a/lldb/source/Target/Process.cpp b/lldb/source/Target/Process.cpp
index d777a27139119..b7694ec43f341 100644
--- a/lldb/source/Target/Process.cpp
+++ b/lldb/source/Target/Process.cpp
@@ -3154,7 +3154,7 @@ Status Process::PrivateResume() {
     if (m_thread_list.WillResume()) {
       // Last thing, do the PreResumeActions.
       if (!RunPreResumeActions()) {
-        error.SetErrorStringWithFormat(
+        error.SetErrorString(
             "Process::PrivateResume PreResumeActions failed, not resuming.");
       } else {
         m_mod_id.BumpResumeID();
diff --git a/lldb/source/Target/StackFrameRecognizer.cpp b/lldb/source/Target/StackFrameRecognizer.cpp
index 6fa09a387ad2e..73d22d5bb4e61 100644
--- a/lldb/source/Target/StackFrameRecognizer.cpp
+++ b/lldb/source/Target/StackFrameRecognizer.cpp
@@ -50,17 +50,17 @@ ScriptedStackFrameRecognizer::RecognizeFrame(lldb::StackFrameSP frame) {
 void StackFrameRecognizerManager::AddRecognizer(
     StackFrameRecognizerSP recognizer, ConstString module,
     llvm::ArrayRef<ConstString> symbols, bool first_instruction_only) {
-  m_recognizers.push_front({(uint32_t)m_recognizers.size(), false, recognizer,
-                            false, module, RegularExpressionSP(), symbols,
+  m_recognizers.push_front({(uint32_t)m_recognizers.size(), recognizer, false,
+                            module, RegularExpressionSP(), symbols,
                             RegularExpressionSP(), first_instruction_only});
 }
 
 void StackFrameRecognizerManager::AddRecognizer(
     StackFrameRecognizerSP recognizer, RegularExpressionSP module,
     RegularExpressionSP symbol, bool first_instruction_only) {
-  m_recognizers.push_front(
-      {(uint32_t)m_recognizers.size(), false, recognizer, true, ConstString(),
-       module, std::vector<ConstString>(), symbol, first_instruction_only});
+  m_recognizers.push_front({(uint32_t)m_recognizers.size(), recognizer, true,
+                            ConstString(), module, std::vector<ConstString>(),
+                            symbol, first_instruction_only});
 }
 
 void StackFrameRecognizerManager::ForEach(
@@ -90,9 +90,13 @@ bool StackFrameRecognizerManager::RemoveRecognizerWithID(
     uint32_t recognizer_id) {
   if (recognizer_id >= m_recognizers.size())
     return false;
-  if (m_recognizers[recognizer_id].deleted)
+  auto found =
+      llvm::find_if(m_recognizers, [recognizer_id](const RegisteredEntry &e) {
+        return e.recognizer_id == recognizer_id;
+      });
+  if (found == m_recognizers.end())
     return false;
-  m_recognizers[recognizer_id].deleted = true;
+  m_recognizers.erase(found);
   return true;
 }
 
@@ -116,8 +120,6 @@ StackFrameRecognizerManager::GetRecognizerForFrame(StackFrameSP frame) {
   Address current_addr = frame->GetFrameCodeAddress();
 
   for (auto entry : m_recognizers) {
-    if (entry.deleted)
-      continue;
     if (entry.module)
       if (entry.module != module_name)
         continue;
diff --git a/lldb/source/Utility/ArchSpec.cpp b/lldb/source/Utility/ArchSpec.cpp
index ca1ce4b3d378a..f220f4e30b29e 100644
--- a/lldb/source/Utility/ArchSpec.cpp
+++ b/lldb/source/Utility/ArchSpec.cpp
@@ -613,7 +613,7 @@ std::string ArchSpec::GetTargetABI() const {
   return abi;
 }
 
-void ArchSpec::SetFlags(std::string elf_abi) {
+void ArchSpec::SetFlags(const std::string &elf_abi) {
 
   uint32_t flag = GetFlags();
   if (IsMIPS()) {
diff --git a/lldb/source/Utility/ConstString.cpp b/lldb/source/Utility/ConstString.cpp
index 62f79b3df7a5a..11f48d0fcaba4 100644
--- a/lldb/source/Utility/ConstString.cpp
+++ b/lldb/source/Utility/ConstString.cpp
@@ -308,7 +308,7 @@ void ConstString::SetString(const llvm::StringRef &s) {
 }
 
 void ConstString::SetStringWithMangledCounterpart(llvm::StringRef demangled,
-                                                   ConstString mangled) {
+                                                  ConstString mangled) {
   m_string = StringPool().GetConstCStringAndSetMangledCounterPart(
       demangled, mangled.m_string);
 }
diff --git a/lldb/source/Utility/Reproducer.cpp b/lldb/source/Utility/Reproducer.cpp
index 7620ab2c389d6..4441f3d475ea3 100644
--- a/lldb/source/Utility/Reproducer.cpp
+++ b/lldb/source/Utility/Reproducer.cpp
@@ -157,7 +157,7 @@ FileSpec Reproducer::GetReproducerPath() const {
   return {};
 }
 
-static FileSpec MakeAbsolute(FileSpec file_spec) {
+static FileSpec MakeAbsolute(const FileSpec &file_spec) {
   SmallString<128> path;
   file_spec.GetPath(path, false);
   llvm::sys::fs::make_absolute(path);
@@ -299,7 +299,12 @@ void WorkingDirectoryProvider::Keep() {
   os << m_cwd << "\n";
 }
 
-void FileProvider::recordInterestingDirectory(const llvm::Twine &dir) {
+void FileProvider::RecordInterestingDirectory(const llvm::Twine &dir) {
+  if (m_collector)
+    m_collector->addFile(dir);
+}
+
+void FileProvider::RecordInterestingDirectoryRecursive(const llvm::Twine &dir) {
   if (m_collector)
     m_collector->addDirectory(dir);
 }
diff --git a/lldb/source/Utility/Scalar.cpp b/lldb/source/Utility/Scalar.cpp
index 6c48bbde532f9..27d5b3b88d338 100644
--- a/lldb/source/Utility/Scalar.cpp
+++ b/lldb/source/Utility/Scalar.cpp
@@ -160,7 +160,7 @@ bool Scalar::GetData(DataExtractor &data, size_t limit_byte_size) const {
 void Scalar::GetBytes(llvm::MutableArrayRef<uint8_t> storage) const {
   assert(storage.size() >= GetByteSize());
 
-  const auto &store = [&](const llvm::APInt val) {
+  const auto &store = [&](const llvm::APInt &val) {
     StoreIntToMemory(val, storage.data(), (val.getBitWidth() + 7) / 8);
   };
   switch (GetCategory(m_type)) {
@@ -1068,12 +1068,9 @@ Status Scalar::SetValueFromCString(const char *value_str, Encoding encoding,
   return error;
 }
 
-Status Scalar::SetValueFromData(DataExtractor &data, lldb::Encoding encoding,
-                                size_t byte_size) {
+Status Scalar::SetValueFromData(const DataExtractor &data,
+                                lldb::Encoding encoding, size_t byte_size) {
   Status error;
-
-  type128 int128;
-  type256 int256;
   switch (encoding) {
   case lldb::eEncodingInvalid:
     error.SetErrorString("invalid encoding");
@@ -1081,100 +1078,26 @@ Status Scalar::SetValueFromData(DataExtractor &data, lldb::Encoding encoding,
   case lldb::eEncodingVector:
     error.SetErrorString("vector encoding unsupported");
     break;
-  case lldb::eEncodingUint: {
-    lldb::offset_t offset = 0;
-
-    switch (byte_size) {
-    case 1:
-      operator=(data.GetU8(&offset));
-      break;
-    case 2:
-      operator=(data.GetU16(&offset));
-      break;
-    case 4:
-      operator=(data.GetU32(&offset));
-      break;
-    case 8:
-      operator=(data.GetU64(&offset));
-      break;
-    case 16:
-      if (data.GetByteOrder() == eByteOrderBig) {
-        int128.x[1] = data.GetU64(&offset);
-        int128.x[0] = data.GetU64(&offset);
-      } else {
-        int128.x[0] = data.GetU64(&offset);
-        int128.x[1] = data.GetU64(&offset);
-      }
-      operator=(llvm::APInt(BITWIDTH_INT128, NUM_OF_WORDS_INT128, int128.x));
-      break;
-    case 32:
-      if (data.GetByteOrder() == eByteOrderBig) {
-        int256.x[3] = data.GetU64(&offset);
-        int256.x[2] = data.GetU64(&offset);
-        int256.x[1] = data.GetU64(&offset);
-        int256.x[0] = data.GetU64(&offset);
-      } else {
-        int256.x[0] = data.GetU64(&offset);
-        int256.x[1] = data.GetU64(&offset);
-        int256.x[2] = data.GetU64(&offset);
-        int256.x[3] = data.GetU64(&offset);
-      }
-      operator=(llvm::APInt(BITWIDTH_INT256, NUM_OF_WORDS_INT256, int256.x));
-      break;
-    default:
-      error.SetErrorStringWithFormat(
-          "unsupported unsigned integer byte size: %" PRIu64 "",
-          static_cast<uint64_t>(byte_size));
-      break;
-    }
-  } break;
+  case lldb::eEncodingUint:
   case lldb::eEncodingSint: {
-    lldb::offset_t offset = 0;
-
-    switch (byte_size) {
-    case 1:
-      operator=(static_cast<int8_t>(data.GetU8(&offset)));
-      break;
-    case 2:
-      operator=(static_cast<int16_t>(data.GetU16(&offset)));
-      break;
-    case 4:
-      operator=(static_cast<int32_t>(data.GetU32(&offset)));
-      break;
-    case 8:
-      operator=(static_cast<int64_t>(data.GetU64(&offset)));
-      break;
-    case 16:
-      if (data.GetByteOrder() == eByteOrderBig) {
-        int128.x[1] = data.GetU64(&offset);
-        int128.x[0] = data.GetU64(&offset);
-      } else {
-        int128.x[0] = data.GetU64(&offset);
-        int128.x[1] = data.GetU64(&offset);
-      }
-      operator=(llvm::APInt(BITWIDTH_INT128, NUM_OF_WORDS_INT128, int128.x));
-      break;
-    case 32:
-      if (data.GetByteOrder() == eByteOrderBig) {
-        int256.x[3] = data.GetU64(&offset);
-        int256.x[2] = data.GetU64(&offset);
-        int256.x[1] = data.GetU64(&offset);
-        int256.x[0] = data.GetU64(&offset);
-      } else {
-        int256.x[0] = data.GetU64(&offset);
-        int256.x[1] = data.GetU64(&offset);
-        int256.x[2] = data.GetU64(&offset);
-        int256.x[3] = data.GetU64(&offset);
-      }
-      operator=(llvm::APInt(BITWIDTH_INT256, NUM_OF_WORDS_INT256, int256.x));
-      break;
-    default:
-      error.SetErrorStringWithFormat(
-          "unsupported signed integer byte size: %" PRIu64 "",
-          static_cast<uint64_t>(byte_size));
-      break;
+    if (data.GetByteSize() < byte_size)
+      return Status("insufficient data");
+    Type type = GetBestTypeForBitSize(byte_size*8, encoding == lldb::eEncodingSint);
+    if (type == e_void) {
+      return Status("unsupported integer byte size: %" PRIu64 "",
+                    static_cast<uint64_t>(byte_size));
     }
-  } break;
+    m_type = type;
+    if (data.GetByteOrder() == endian::InlHostByteOrder()) {
+      m_integer = APInt::getNullValue(8 * byte_size);
+      llvm::LoadIntFromMemory(m_integer, data.GetDataStart(), byte_size);
+    } else {
+      std::vector<uint8_t> buffer(byte_size);
+      std::copy_n(data.GetDataStart(), byte_size, buffer.rbegin());
+      llvm::LoadIntFromMemory(m_integer, buffer.data(), byte_size);
+    }
+    break;
+  }
   case lldb::eEncodingIEEE754: {
     lldb::offset_t offset = 0;
 
diff --git a/lldb/source/Utility/StringLexer.cpp b/lldb/source/Utility/StringLexer.cpp
index 947472a014ebe..26185517e2753 100644
--- a/lldb/source/Utility/StringLexer.cpp
+++ b/lldb/source/Utility/StringLexer.cpp
@@ -10,10 +10,11 @@
 
 #include <algorithm>
 #include <assert.h>
+#include <utility>
 
 using namespace lldb_private;
 
-StringLexer::StringLexer(std::string s) : m_data(s), m_position(0) {}
+StringLexer::StringLexer(std::string s) : m_data(std::move(s)), m_position(0) {}
 
 StringLexer::Character StringLexer::Peek() { return m_data[m_position]; }
 
diff --git a/lldb/source/Utility/StringList.cpp b/lldb/source/Utility/StringList.cpp
index 809c5a02acf6b..d3896ca60ac9c 100644
--- a/lldb/source/Utility/StringList.cpp
+++ b/lldb/source/Utility/StringList.cpp
@@ -212,7 +212,7 @@ StringList &StringList::operator<<(const std::string &str) {
   return *this;
 }
 
-StringList &StringList::operator<<(StringList strings) {
+StringList &StringList::operator<<(const StringList &strings) {
   AppendList(strings);
   return *this;
 }
diff --git a/lldb/source/Utility/StructuredData.cpp b/lldb/source/Utility/StructuredData.cpp
index df1b35618e4a7..359c49ae254b0 100644
--- a/lldb/source/Utility/StructuredData.cpp
+++ b/lldb/source/Utility/StructuredData.cpp
@@ -21,7 +21,8 @@ static StructuredData::ObjectSP ParseJSONValue(json::Value &value);
 static StructuredData::ObjectSP ParseJSONObject(json::Object *object);
 static StructuredData::ObjectSP ParseJSONArray(json::Array *array);
 
-StructuredData::ObjectSP StructuredData::ParseJSON(std::string json_text) {
+StructuredData::ObjectSP
+StructuredData::ParseJSON(const std::string &json_text) {
   llvm::Expected<json::Value> value = json::parse(json_text);
   if (!value) {
     llvm::consumeError(value.takeError());
diff --git a/lldb/source/Utility/XcodeSDK.cpp b/lldb/source/Utility/XcodeSDK.cpp
index 066bf457966c9..4f64042e3643d 100644
--- a/lldb/source/Utility/XcodeSDK.cpp
+++ b/lldb/source/Utility/XcodeSDK.cpp
@@ -54,12 +54,12 @@ XcodeSDK::XcodeSDK(XcodeSDK::Info info) : m_name(GetName(info.type).str()) {
   }
 }
 
-XcodeSDK &XcodeSDK::operator=(XcodeSDK other) {
+XcodeSDK &XcodeSDK::operator=(const XcodeSDK &other) {
   m_name = other.m_name;
   return *this;
 }
 
-bool XcodeSDK::operator==(XcodeSDK other) {
+bool XcodeSDK::operator==(const XcodeSDK &other) {
   return m_name == other.m_name;
 }
 
@@ -147,7 +147,7 @@ bool XcodeSDK::Info::operator==(const Info &other) const {
          std::tie(other.type, other.version, other.internal);
 }
 
-void XcodeSDK::Merge(XcodeSDK other) {
+void XcodeSDK::Merge(const XcodeSDK &other) {
   // The "bigger" SDK always wins.
   auto l = Parse();
   auto r = other.Parse();
diff --git a/lldb/test/API/CMakeLists.txt b/lldb/test/API/CMakeLists.txt
index 34f3522c8dfec..a80992f287e9e 100644
--- a/lldb/test/API/CMakeLists.txt
+++ b/lldb/test/API/CMakeLists.txt
@@ -13,10 +13,10 @@ function(add_python_test_target name test_script args comment)
   add_dependencies(${name} lldb-test-deps)
 endfunction()
 
-# The default architecture with which to compile test executables is the default LLVM target
-# architecture, which itself defaults to the host architecture.
-string(TOLOWER "${LLVM_TARGET_ARCH}" LLDB_DEFAULT_TEST_ARCH)
-if( LLDB_DEFAULT_TEST_ARCH STREQUAL "host" )
+# The default architecture with which to compile test executables is the
+# default LLVM target architecture, which itself defaults to the host
+# architecture.
+if(NOT LLDB_DEFAULT_TEST_ARCH)
   string(REGEX MATCH "^[^-]*" LLDB_DEFAULT_TEST_ARCH ${LLVM_HOST_TRIPLE})
 endif ()
 
diff --git a/lldb/test/API/commands/expression/weak_symbols/TestWeakSymbols.py b/lldb/test/API/commands/expression/weak_symbols/TestWeakSymbols.py
index b67b5b4efe0bd..2b63fba00e658 100644
--- a/lldb/test/API/commands/expression/weak_symbols/TestWeakSymbols.py
+++ b/lldb/test/API/commands/expression/weak_symbols/TestWeakSymbols.py
@@ -48,10 +48,6 @@ def do_test(self):
 
         launch_info = lldb.SBLaunchInfo(None)
         launch_info.SetWorkingDirectory(self.getBuildDir())
-        # We have to point to the hidden directory to pick up the
-        # version of the dylib without the weak symbols:
-        env_expr = self.platformContext.shlib_environment_var + "=" + hidden_dir
-        launch_info.SetEnvironmentEntries([env_expr], True)
 
         (self.target, _, thread, _) = lldbutil.run_to_source_breakpoint(
                                               self, "Set a breakpoint here",
diff --git a/lldb/test/API/commands/frame/recognizer/TestFrameRecognizer.py b/lldb/test/API/commands/frame/recognizer/TestFrameRecognizer.py
index f807937f2f17b..b025a2255f07b 100644
--- a/lldb/test/API/commands/frame/recognizer/TestFrameRecognizer.py
+++ b/lldb/test/API/commands/frame/recognizer/TestFrameRecognizer.py
@@ -44,8 +44,24 @@ def test_frame_recognizer_1(self):
 
         self.runCmd("frame recognizer delete 0")
 
+        # Test that it deleted the recognizer with id 0.
         self.expect("frame recognizer list",
                     substrs=['1: recognizer.MyOtherFrameRecognizer, module a.out, symbol bar (regexp)'])
+        self.expect("frame recognizer list", matching=False,
+                    substrs=['MyFrameRecognizer'])
+
+        # Test that an invalid index and deleting the same index again
+        # is an error and doesn't do any changes.
+        self.expect("frame recognizer delete 2", error=True,
+                    substrs=["error: '2' is not a valid recognizer id."])
+        self.expect("frame recognizer delete 0", error=True,
+                    substrs=["error: '0' is not a valid recognizer id."])
+        # Recognizers should have the same state as above.
+        self.expect("frame recognizer list",
+                    substrs=['1: recognizer.MyOtherFrameRecognizer, module a.out, symbol bar (regexp)'])
+        self.expect("frame recognizer list", matching=False,
+                    substrs=['MyFrameRecognizer'])
+
 
         self.runCmd("frame recognizer clear")
 
diff --git a/lldb/test/API/commands/command/history/TestCommandHistory.py b/lldb/test/API/commands/session/history/TestSessionHistory.py
similarity index 62%
rename from lldb/test/API/commands/command/history/TestCommandHistory.py
rename to lldb/test/API/commands/session/history/TestSessionHistory.py
index c866198f26b6f..56e7336781fa3 100644
--- a/lldb/test/API/commands/command/history/TestCommandHistory.py
+++ b/lldb/test/API/commands/session/history/TestSessionHistory.py
@@ -1,5 +1,5 @@
 """
-Test the command history mechanism
+Test the session history command
 """
 
 
@@ -10,13 +10,13 @@
 from lldbsuite.test import lldbutil
 
 
-class CommandHistoryTestCase(TestBase):
+class SessionHistoryTestCase(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
     @no_debug_info_test
     def test_history(self):
-        self.runCmd('command history --clear', inHistory=False)
+        self.runCmd('session history --clear', inHistory=False)
         self.runCmd('breakpoint list', check=False, inHistory=True)  # 0
         self.runCmd('register read', check=False, inHistory=True)  # 1
         self.runCmd('apropos hello', check=False, inHistory=True)  # 2
@@ -32,31 +32,31 @@ def test_history(self):
         self.runCmd('frame select 1', check=False, inHistory=True)  # 9
 
         self.expect(
-            "command history -s 3 -c 3",
+            "session history -s 3 -c 3",
             inHistory=True,
             substrs=[
                 '3: memory write',
                 '4: log list',
                 '5: disassemble'])
 
-        self.expect("command history -s 3 -e 3", inHistory=True,
+        self.expect("session history -s 3 -e 3", inHistory=True,
                     substrs=['3: memory write'])
 
         self.expect(
-            "command history -s 6 -e 7",
+            "session history -s 6 -e 7",
             inHistory=True,
             substrs=[
                 '6: expression 1',
                 '7: type summary list -w default'])
 
-        self.expect("command history -c 2", inHistory=True,
+        self.expect("session history -c 2", inHistory=True,
                     substrs=['0: breakpoint list', '1: register read'])
 
-        self.expect("command history -e 3 -c 1", inHistory=True,
+        self.expect("session history -e 3 -c 1", inHistory=True,
                     substrs=['3: memory write'])
 
         self.expect(
-            "command history -e 2",
+            "session history -e 2",
             inHistory=True,
             substrs=[
                 '0: breakpoint list',
@@ -64,43 +64,43 @@ def test_history(self):
                 '2: apropos hello'])
 
         self.expect(
-            "command history -s 12",
+            "session history -s 12",
             inHistory=True,
             substrs=[
-                '12: command history -s 6 -e 7',
-                '13: command history -c 2',
-                '14: command history -e 3 -c 1',
-                '15: command history -e 2',
-                '16: command history -s 12'])
+                '12: session history -s 6 -e 7',
+                '13: session history -c 2',
+                '14: session history -e 3 -c 1',
+                '15: session history -e 2',
+                '16: session history -s 12'])
 
         self.expect(
-            "command history -s end -c 3",
+            "session history -s end -c 3",
             inHistory=True,
             substrs=[
-                '15: command history -e 2',
-                '16: command history -s 12',
-                '17: command history -s end -c 3'])
+                '15: session history -e 2',
+                '16: session history -s 12',
+                '17: session history -s end -c 3'])
 
         self.expect(
-            "command history -s end -e 15",
+            "session history -s end -e 15",
             inHistory=True,
             substrs=[
-                '15: command history -e 2',
-                '16: command history -s 12',
-                '17: command history -s end -c 3',
-                'command history -s end -e 15'])
+                '15: session history -e 2',
+                '16: session history -s 12',
+                '17: session history -s end -c 3',
+                'session history -s end -e 15'])
 
-        self.expect("command history -s 5 -c 1", inHistory=True,
+        self.expect("session history -s 5 -c 1", inHistory=True,
                     substrs=['5: disassemble'])
 
-        self.expect("command history -c 1 -s 5", inHistory=True,
+        self.expect("session history -c 1 -s 5", inHistory=True,
                     substrs=['5: disassemble'])
 
-        self.expect("command history -c 1 -e 3", inHistory=True,
+        self.expect("session history -c 1 -e 3", inHistory=True,
                     substrs=['3: memory write'])
 
         self.expect(
-            "command history -c 1 -e 3 -s 5",
+            "session history -c 1 -e 3 -s 5",
             error=True,
             inHistory=True,
             substrs=['error: --count, --start-index and --end-index cannot be all specified in the same invocation'])
diff --git a/lldb/test/API/commands/session/save/TestSessionSave.py b/lldb/test/API/commands/session/save/TestSessionSave.py
new file mode 100644
index 0000000000000..ec244a42efcac
--- /dev/null
+++ b/lldb/test/API/commands/session/save/TestSessionSave.py
@@ -0,0 +1,74 @@
+"""
+Test the session save feature
+"""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class SessionSaveTestCase(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    def raw_transcript_builder(self, cmd, res):
+        raw = "(lldb) " + cmd + "\n"
+        if res.GetOutputSize():
+          raw += res.GetOutput()
+        if res.GetErrorSize():
+          raw += res.GetError()
+        return raw
+
+
+    @skipIfWindows
+    @skipIfReproducer
+    @no_debug_info_test
+    def test_session_save(self):
+        raw = ""
+        interpreter = self.dbg.GetCommandInterpreter()
+
+        settings = [
+          'settings set interpreter.echo-commands true',
+          'settings set interpreter.echo-comment-commands true',
+          'settings set interpreter.stop-command-source-on-error false'
+        ]
+
+        for setting in settings:
+          interpreter.HandleCommand(setting, lldb.SBCommandReturnObject())
+
+        inputs = [
+          '# This is a comment',  # Comment
+          'help session',         # Valid command
+          'Lorem ipsum'           # Invalid command
+        ]
+
+        for cmd in inputs:
+          res = lldb.SBCommandReturnObject()
+          interpreter.HandleCommand(cmd, res)
+          raw += self.raw_transcript_builder(cmd, res)
+
+        self.assertTrue(interpreter.HasCommands())
+        self.assertTrue(len(raw) != 0)
+
+        # Check for error
+        cmd = 'session save /root/file'
+        interpreter.HandleCommand(cmd, res)
+        self.assertFalse(res.Succeeded())
+        raw += self.raw_transcript_builder(cmd, res)
+
+        import tempfile
+        tf = tempfile.NamedTemporaryFile()
+        output_file = tf.name
+
+        res = lldb.SBCommandReturnObject()
+        interpreter.HandleCommand('session save ' + output_file, res)
+        self.assertTrue(res.Succeeded())
+        raw += self.raw_transcript_builder(cmd, res)
+
+        with open(output_file, "r") as file:
+          content = file.read()
+          # Exclude last line, since session won't record it's own output
+          lines = raw.splitlines()[:-1]
+          for line in lines:
+            self.assertIn(line, content)
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_by_file_colon_line/Makefile b/lldb/test/API/functionalities/breakpoint/breakpoint_by_file_colon_line/Makefile
new file mode 100644
index 0000000000000..ad42b20df4ed5
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_by_file_colon_line/Makefile
@@ -0,0 +1,4 @@
+C_SOURCES := main.c
+CFLAGS_EXTRAS := -std=c99 -gcolumn-info
+
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_by_file_colon_line/TestBreakpointByFileColonLine.py b/lldb/test/API/functionalities/breakpoint/breakpoint_by_file_colon_line/TestBreakpointByFileColonLine.py
new file mode 100644
index 0000000000000..011fcdce8da46
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_by_file_colon_line/TestBreakpointByFileColonLine.py
@@ -0,0 +1,42 @@
+"""
+Test setting a breakpoint by line and column.
+"""
+
+
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class BreakpointByLineAndColumnTestCase(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    def testBreakpointSpecWithLine(self):
+        self.build()
+        exe = self.getBuildArtifact("a.out")
+
+        target = self.dbg.CreateTarget(exe)
+        self.assertTrue(target, VALID_TARGET)
+        
+        # This one should work:
+        lldbutil.run_break_set_by_file_colon_line(self, "main.c:11", "main.c", 11, num_expected_locations = 1)
+        # Let's try an illegal specifier to make sure the command fails.  I'm not being exhaustive
+        # since the UnitTest has more bad patterns.  I'm just testing that if the SetFromString
+        # fails, we propagate the error.
+        self.expect("break set -y 'foo.c'", error=True)
+        
+    ## Skip gcc version less 7.1 since it doesn't support -gcolumn-info
+    @skipIf(compiler="gcc", compiler_version=['<', '7.1'])
+    def testBreakpointByLine(self):
+        self.build()
+        exe = self.getBuildArtifact("a.out")
+
+        target = self.dbg.CreateTarget(exe)
+        self.assertTrue(target, VALID_TARGET)
+
+        main_c = lldb.SBFileSpec("main.c")
+        lldbutil.run_break_set_by_file_colon_line(self, "main.c:11:50", "main.c", 11, num_expected_locations = 1)
+        
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_by_file_colon_line/main.c b/lldb/test/API/functionalities/breakpoint/breakpoint_by_file_colon_line/main.c
new file mode 100644
index 0000000000000..6e3f7e2ddbf93
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_by_file_colon_line/main.c
@@ -0,0 +1,14 @@
+int square(int x)
+{
+  return x * x;
+}
+
+int main (int argc, char const *argv[])
+{
+  int did_call = 0;
+
+  // Line 11.                                    v Column 50.
+  if(square(argc+1) != 0) { did_call = 1; return square(argc); }
+  //                                             ^
+  return square(0);
+}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-skip-summary/TestDataFormatterSkipSummary.py b/lldb/test/API/functionalities/data-formatter/data-formatter-skip-summary/TestDataFormatterSkipSummary.py
index f5cf525427c8f..aa5e20bd2f757 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-skip-summary/TestDataFormatterSkipSummary.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-skip-summary/TestDataFormatterSkipSummary.py
@@ -146,7 +146,7 @@ def cleanup():
         if self.getCompiler().endswith('gcc') and not self.getCompiler().endswith('llvm-gcc'):
             import re
             gcc_version_output = system(
-                [[lldbutil.which(self.getCompiler()), "-v"]])[1]
+                [[lldbutil.which(self.getCompiler()), "-v"]])
             self.trace("my output:", gcc_version_output)
             for line in gcc_version_output.split(os.linesep):
                 m = re.search('\(Apple Inc\. build ([0-9]+)\)', line)
diff --git a/lldb/test/API/functionalities/history/TestHistoryRecall.py b/lldb/test/API/functionalities/history/TestHistoryRecall.py
index 557b4a4c082b9..18ab2ca0fe6e7 100644
--- a/lldb/test/API/functionalities/history/TestHistoryRecall.py
+++ b/lldb/test/API/functionalities/history/TestHistoryRecall.py
@@ -25,12 +25,12 @@ def test_history_recall(self):
     def sample_test(self):
         interp = self.dbg.GetCommandInterpreter()
         result = lldb.SBCommandReturnObject()
-        interp.HandleCommand("command history", result, True)
+        interp.HandleCommand("session history", result, True)
         interp.HandleCommand("platform list", result, True)
 
         interp.HandleCommand("!0", result, False)
         self.assertTrue(result.Succeeded(), "!0 command did not work: %s"%(result.GetError()))
-        self.assertTrue("command history" in result.GetOutput(), "!0 didn't rerun command history")
+        self.assertTrue("session history" in result.GetOutput(), "!0 didn't rerun session history")
 
         interp.HandleCommand("!-1", result, False)
         self.assertTrue(result.Succeeded(), "!-1 command did not work: %s"%(result.GetError()))
diff --git a/lldb/test/API/functionalities/postmortem/elf-core/TestLinuxCore.py b/lldb/test/API/functionalities/postmortem/elf-core/TestLinuxCore.py
index e0046f7108898..162503c33d37d 100644
--- a/lldb/test/API/functionalities/postmortem/elf-core/TestLinuxCore.py
+++ b/lldb/test/API/functionalities/postmortem/elf-core/TestLinuxCore.py
@@ -35,69 +35,69 @@ class LinuxCoreTestCase(TestBase):
     _mips_regions = 5
     _ppc64le_regions = 2
 
-
     @skipIf(triple='^mips')
     @skipIfLLVMTargetMissing("AArch64")
-    @skipIfReproducer # lldb::FileSP used in typemap cannot be instrumented.
+    @skipIfReproducer  # lldb::FileSP used in typemap cannot be instrumented.
     def test_aarch64(self):
         """Test that lldb can read the process information from an aarch64 linux core file."""
-        self.do_test("linux-aarch64", self._aarch64_pid, self._aarch64_regions, "a.out")
+        self.do_test("linux-aarch64", self._aarch64_pid,
+                     self._aarch64_regions, "a.out")
 
     @skipIf(triple='^mips')
     @skipIfLLVMTargetMissing("X86")
-    @skipIfReproducer # lldb::FileSP used in typemap cannot be instrumented.
+    @skipIfReproducer  # lldb::FileSP used in typemap cannot be instrumented.
     def test_i386(self):
         """Test that lldb can read the process information from an i386 linux core file."""
         self.do_test("linux-i386", self._i386_pid, self._i386_regions, "a.out")
 
     @skipIfLLVMTargetMissing("Mips")
-    @skipIfReproducer # lldb::FileSP used in typemap cannot be instrumented.
+    @skipIfReproducer  # lldb::FileSP used in typemap cannot be instrumented.
     def test_mips_o32(self):
         """Test that lldb can read the process information from an MIPS O32 linux core file."""
         self.do_test("linux-mipsel-gnuabio32", self._mips_o32_pid,
-                self._mips_regions, "linux-mipsel-gn")
+                     self._mips_regions, "linux-mipsel-gn")
 
     @skipIfLLVMTargetMissing("Mips")
-    @skipIfReproducer # lldb::FileSP used in typemap cannot be instrumented.
+    @skipIfReproducer  # lldb::FileSP used in typemap cannot be instrumented.
     def test_mips_n32(self):
         """Test that lldb can read the process information from an MIPS N32 linux core file """
         self.do_test("linux-mips64el-gnuabin32", self._mips64_n32_pid,
-                self._mips_regions, "linux-mips64el-")
+                     self._mips_regions, "linux-mips64el-")
 
     @skipIfLLVMTargetMissing("Mips")
-    @skipIfReproducer # lldb::FileSP used in typemap cannot be instrumented.
+    @skipIfReproducer  # lldb::FileSP used in typemap cannot be instrumented.
     def test_mips_n64(self):
         """Test that lldb can read the process information from an MIPS N64 linux core file """
         self.do_test("linux-mips64el-gnuabi64", self._mips64_n64_pid,
-                self._mips_regions, "linux-mips64el-")
+                     self._mips_regions, "linux-mips64el-")
 
     @skipIf(triple='^mips')
     @skipIfLLVMTargetMissing("PowerPC")
-    @skipIfReproducer # lldb::FileSP used in typemap cannot be instrumented.
+    @skipIfReproducer  # lldb::FileSP used in typemap cannot be instrumented.
     def test_ppc64le(self):
         """Test that lldb can read the process information from an ppc64le linux core file."""
         self.do_test("linux-ppc64le", self._ppc64le_pid, self._ppc64le_regions,
-                "linux-ppc64le.ou")
+                     "linux-ppc64le.ou")
 
     @skipIf(triple='^mips')
     @skipIfLLVMTargetMissing("X86")
-    @skipIfReproducer # lldb::FileSP used in typemap cannot be instrumented.
+    @skipIfReproducer  # lldb::FileSP used in typemap cannot be instrumented.
     def test_x86_64(self):
         """Test that lldb can read the process information from an x86_64 linux core file."""
         self.do_test("linux-x86_64", self._x86_64_pid, self._x86_64_regions,
-        "a.out")
+                     "a.out")
 
     @skipIf(triple='^mips')
     @skipIfLLVMTargetMissing("SystemZ")
-    @skipIfReproducer # lldb::FileSP used in typemap cannot be instrumented.
+    @skipIfReproducer  # lldb::FileSP used in typemap cannot be instrumented.
     def test_s390x(self):
         """Test that lldb can read the process information from an s390x linux core file."""
         self.do_test("linux-s390x", self._s390x_pid, self._s390x_regions,
-        "a.out")
+                     "a.out")
 
     @skipIf(triple='^mips')
     @skipIfLLVMTargetMissing("X86")
-    @skipIfReproducer # lldb::FileSP used in typemap cannot be instrumented.
+    @skipIfReproducer  # lldb::FileSP used in typemap cannot be instrumented.
     def test_same_pid_running(self):
         """Test that we read the information from the core correctly even if we have a running
         process with the same PID around"""
@@ -122,11 +122,11 @@ def test_same_pid_running(self):
                 f.seek(pid_offset)
                 f.write(struct.pack("<I", os.getpid()))
         self.do_test(self.getBuildArtifact("linux-x86_64-pid"), os.getpid(),
-                self._x86_64_regions, "a.out")
+                     self._x86_64_regions, "a.out")
 
     @skipIf(triple='^mips')
     @skipIfLLVMTargetMissing("X86")
-    @skipIfReproducer # lldb::FileSP used in typemap cannot be instrumented.
+    @skipIfReproducer  # lldb::FileSP used in typemap cannot be instrumented.
     def test_two_cores_same_pid(self):
         """Test that we handle the situation if we have two core files with the same PID
         around"""
@@ -153,7 +153,7 @@ def test_two_cores_same_pid(self):
         # without destroying this process, run the test which opens another core file with the
         # same pid
         self.do_test("linux-x86_64", self._x86_64_pid, self._x86_64_regions,
-                "a.out")
+                     "a.out")
 
     @skipIf(triple='^mips')
     @skipIfLLVMTargetMissing("X86")
@@ -192,8 +192,8 @@ def test_FPR_SSE(self):
         values["xmm7"] = "{0x13 0xf1 0x30 0xcd 0x13 0xf1 0x30 0xcd 0x13 0xf1 0x30 0xcd 0x13 0xf1 0x30 0xcd}"
 
         for regname, value in values.items():
-            self.expect("register read {}".format(regname), substrs=["{} = {}".format(regname, value)])
-
+            self.expect("register read {}".format(regname),
+                        substrs=["{} = {}".format(regname, value)])
 
         # now check i386 core file
         target = self.dbg.CreateTarget(None)
@@ -203,23 +203,27 @@ def test_FPR_SSE(self):
         values["fioff"] = "0x080480cc"
 
         for regname, value in values.items():
-            self.expect("register read {}".format(regname), substrs=["{} = {}".format(regname, value)])
+            self.expect("register read {}".format(regname),
+                        substrs=["{} = {}".format(regname, value)])
 
     @skipIf(triple='^mips')
     @skipIfLLVMTargetMissing("X86")
-    @skipIfReproducer # lldb::FileSP used in typemap cannot be instrumented.
+    @skipIfReproducer  # lldb::FileSP used in typemap cannot be instrumented.
     def test_i386_sysroot(self):
         """Test that lldb can find the exe for an i386 linux core file using the sysroot."""
 
         # Copy linux-i386.out to tmp_sysroot/home/labath/test/a.out (since it was compiled as
         # /home/labath/test/a.out)
-        tmp_sysroot = os.path.join(self.getBuildDir(), "lldb_i386_mock_sysroot")
-        executable = os.path.join(tmp_sysroot, "home", "labath", "test", "a.out")
+        tmp_sysroot = os.path.join(
+            self.getBuildDir(), "lldb_i386_mock_sysroot")
+        executable = os.path.join(
+            tmp_sysroot, "home", "labath", "test", "a.out")
         lldbutil.mkdir_p(os.path.dirname(executable))
         shutil.copyfile("linux-i386.out", executable)
 
         # Set sysroot and load core
-        self.runCmd("platform select remote-linux --sysroot '%s'" % tmp_sysroot)
+        self.runCmd("platform select remote-linux --sysroot '%s'" %
+                    tmp_sysroot)
         target = self.dbg.CreateTarget(None)
         self.assertTrue(target, VALID_TARGET)
         process = target.LoadCore("linux-i386.core")
@@ -232,7 +236,7 @@ def test_i386_sysroot(self):
     @skipIf(triple='^mips')
     @skipIfLLVMTargetMissing("X86")
     @skipIfWindows
-    @skipIfReproducer # lldb::FileSP used in typemap cannot be instrumented.
+    @skipIfReproducer  # lldb::FileSP used in typemap cannot be instrumented.
     def test_x86_64_sysroot(self):
         """Test that sysroot has more priority then local filesystem."""
 
@@ -243,7 +247,8 @@ def test_x86_64_sysroot(self):
 
         # Copy correct executable to the location inside sysroot
         tmp_sysroot = os.path.join(self.getBuildDir(), "mock_sysroot")
-        exe_inside = os.path.join(tmp_sysroot, os.path.relpath(exe_outside, "/"))
+        exe_inside = os.path.join(
+            tmp_sysroot, os.path.relpath(exe_outside, "/"))
         lldbutil.mkdir_p(os.path.dirname(exe_inside))
         shutil.copyfile("linux-x86_64.out", exe_inside)
 
@@ -256,7 +261,8 @@ def test_x86_64_sysroot(self):
             f.write(core)
 
         # Set sysroot and load core
-        self.runCmd("platform select remote-linux --sysroot '%s'" % tmp_sysroot)
+        self.runCmd("platform select remote-linux --sysroot '%s'" %
+                    tmp_sysroot)
         target = self.dbg.CreateTarget(None)
         self.assertTrue(target, VALID_TARGET)
         process = target.LoadCore(core_file)
@@ -264,7 +270,8 @@ def test_x86_64_sysroot(self):
         # Check that we found executable from the sysroot
         mod_path = str(target.GetModuleAtIndex(0).GetFileSpec())
         self.assertEqual(mod_path, exe_inside)
-        self.check_all(process, self._x86_64_pid, self._x86_64_regions, "a.out")
+        self.check_all(process, self._x86_64_pid,
+                       self._x86_64_regions, "a.out")
 
         self.dbg.DeleteTarget(target)
 
@@ -319,7 +326,137 @@ def test_aarch64_regs(self):
         values["fpcr"] = "0x00000000"
 
         for regname, value in values.items():
-            self.expect("register read {}".format(regname), substrs=["{} = {}".format(regname, value)])
+            self.expect("register read {}".format(regname),
+                        substrs=["{} = {}".format(regname, value)])
+
+        self.expect("register read --all")
+
+    @skipIf(triple='^mips')
+    @skipIfLLVMTargetMissing("AArch64")
+    def test_aarch64_sve_regs_fpsimd(self):
+        # check 64 bit ARM core files
+        target = self.dbg.CreateTarget(None)
+        self.assertTrue(target, VALID_TARGET)
+        process = target.LoadCore("linux-aarch64-sve-fpsimd.core")
+
+        values = {}
+        values["x1"] = "0x000000000000002f"
+        values["w1"] = "0x0000002f"
+        values["fp"] = "0x0000ffffcbad8d50"
+        values["lr"] = "0x0000000000400180"
+        values["sp"] = "0x0000ffffcbad8d30"
+        values["pc"] = "0x000000000040014c"
+        values["cpsr"] = "0x00001000"
+        values["v0"] = "{0x00 0x00 0x00 0x00 0x00 0x00 0xe0 0x3f 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00}"
+        values["v1"] = "{0x00 0x00 0x00 0x00 0x00 0x00 0xf8 0x3f 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00}"
+        values["v2"] = "{0x00 0x00 0x00 0x00 0x00 0x00 0x04 0x40 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00}"
+        values["v3"] = "{0x00 0x00 0x00 0x00 0x00 0x00 0x0c 0x40 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00}"
+        values["v4"] = "{0x00 0x00 0x90 0x40 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00}"
+        values["v5"] = "{0x00 0x00 0xb0 0x40 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00}"
+        values["v6"] = "{0x00 0x00 0xd0 0x40 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00}"
+        values["v7"] = "{0x00 0x00 0xf0 0x40 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00}"
+        values["v8"] = "{0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11}"
+        values["v27"] = "{0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00}"
+        values["v28"] = "{0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00}"
+        values["v31"] = "{0x30 0x30 0x30 0x30 0x30 0x30 0x30 0x30 0x30 0x30 0x30 0x30 0x30 0x30 0x30 0x30}"
+        values["s2"] = "0"
+        values["s3"] = "0"
+        values["s4"] = "4.5"
+        values["s5"] = "5.5"
+        values["s6"] = "6.5"
+        values["s7"] = "7.5"
+        values["s8"] = "1.14437e-28"
+        values["s30"] = "0"
+        values["s31"] = "6.40969e-10"
+        values["d0"] = "0.5"
+        values["d1"] = "1.5"
+        values["d2"] = "2.5"
+        values["d3"] = "3.5"
+        values["d4"] = "5.35161536149201e-315"
+        values["d5"] = "5.36197666906508e-315"
+        values["d6"] = "5.37233797663815e-315"
+        values["d7"] = "5.38269928421123e-315"
+        values["d8"] = "1.80107573659442e-226"
+        values["d30"] = "0"
+        values["d31"] = "1.39804328609529e-76"
+        values["fpsr"] = "0x00000000"
+        values["fpcr"] = "0x00000000"
+        values["vg"] = "0x0000000000000004"
+        values["z0"] = "{0x00 0x00 0x00 0x00 0x00 0x00 0xe0 0x3f 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00}"
+        values["z1"] = "{0x00 0x00 0x00 0x00 0x00 0x00 0xf8 0x3f 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00}"
+        values["z2"] = "{0x00 0x00 0x00 0x00 0x00 0x00 0x04 0x40 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00}"
+        values["z3"] = "{0x00 0x00 0x00 0x00 0x00 0x00 0x0c 0x40 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00}"
+        values["z4"] = "{0x00 0x00 0x90 0x40 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00}"
+        values["z5"] = "{0x00 0x00 0xb0 0x40 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00}"
+        values["z6"] = "{0x00 0x00 0xd0 0x40 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00}"
+        values["z7"] = "{0x00 0x00 0xf0 0x40 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00}"
+        values["z8"] = "{0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00}"
+        values["z27"] = "{0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00}"
+        values["z28"] = "{0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00}"
+        values["z31"] = "{0x30 0x30 0x30 0x30 0x30 0x30 0x30 0x30 0x30 0x30 0x30 0x30 0x30 0x30 0x30 0x30 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00}"
+        values["p0"] = "{0x00 0x00 0x00 0x00}"
+        values["p1"] = "{0x00 0x00 0x00 0x00}"
+        values["p2"] = "{0x00 0x00 0x00 0x00}"
+        values["p4"] = "{0x00 0x00 0x00 0x00}"
+        values["p3"] = "{0x00 0x00 0x00 0x00}"
+        values["p6"] = "{0x00 0x00 0x00 0x00}"
+        values["p5"] = "{0x00 0x00 0x00 0x00}"
+        values["p7"] = "{0x00 0x00 0x00 0x00}"
+        values["p8"] = "{0x00 0x00 0x00 0x00}"
+        values["p9"] = "{0x00 0x00 0x00 0x00}"
+        values["p11"] = "{0x00 0x00 0x00 0x00}"
+        values["p10"] = "{0x00 0x00 0x00 0x00}"
+        values["p12"] = "{0x00 0x00 0x00 0x00}"
+        values["p13"] = "{0x00 0x00 0x00 0x00}"
+        values["p14"] = "{0x00 0x00 0x00 0x00}"
+        values["p15"] = "{0x00 0x00 0x00 0x00}"
+        values["ffr"] = "{0x00 0x00 0x00 0x00}"
+
+        for regname, value in values.items():
+            self.expect("register read {}".format(regname),
+                        substrs=["{} = {}".format(regname, value)])
+
+        self.expect("register read --all")
+
+    @skipIf(triple='^mips')
+    @skipIfLLVMTargetMissing("AArch64")
+    def test_aarch64_sve_regs_full(self):
+        # check 64 bit ARM core files
+        target = self.dbg.CreateTarget(None)
+        self.assertTrue(target, VALID_TARGET)
+        process = target.LoadCore("linux-aarch64-sve-full.core")
+
+        values = {}
+        values["fp"] = "0x0000fffffc1ff4f0"
+        values["lr"] = "0x0000000000400170"
+        values["sp"] = "0x0000fffffc1ff4d0"
+        values["pc"] = "0x000000000040013c"
+        values["v0"] = "{0x00 0x00 0xf0 0x40 0x00 0x00 0xf0 0x40 0x00 0x00 0xf0 0x40 0x00 0x00 0xf0 0x40}"
+        values["v1"] = "{0x00 0x00 0x38 0x41 0x00 0x00 0x38 0x41 0x00 0x00 0x38 0x41 0x00 0x00 0x38 0x41}"
+        values["v2"] = "{0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00}"
+        values["v3"] = "{0x00 0x00 0x78 0x41 0x00 0x00 0x78 0x41 0x00 0x00 0x78 0x41 0x00 0x00 0x78 0x41}"
+        values["s0"] = "7.5"
+        values["s1"] = "11.5"
+        values["s2"] = "0"
+        values["s3"] = "15.5"
+        values["d0"] = "65536.0158538818"
+        values["d1"] = "1572864.25476074"
+        values["d2"] = "0"
+        values["d3"] = "25165828.0917969"
+        values["vg"] = "0x0000000000000004"
+        values["z0"] = "{0x00 0x00 0xf0 0x40 0x00 0x00 0xf0 0x40 0x00 0x00 0xf0 0x40 0x00 0x00 0xf0 0x40 0x00 0x00 0xf0 0x40 0x00 0x00 0xf0 0x40 0x00 0x00 0xf0 0x40 0x00 0x00 0xf0 0x40}"
+        values["z1"] = "{0x00 0x00 0x38 0x41 0x00 0x00 0x38 0x41 0x00 0x00 0x38 0x41 0x00 0x00 0x38 0x41 0x00 0x00 0x38 0x41 0x00 0x00 0x38 0x41 0x00 0x00 0x38 0x41 0x00 0x00 0x38 0x41}"
+        values["z2"] = "{0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00}"
+        values["z3"] = "{0x00 0x00 0x78 0x41 0x00 0x00 0x78 0x41 0x00 0x00 0x78 0x41 0x00 0x00 0x78 0x41 0x00 0x00 0x78 0x41 0x00 0x00 0x78 0x41 0x00 0x00 0x78 0x41 0x00 0x00 0x78 0x41}"
+        values["p0"] = "{0x11 0x11 0x11 0x11}"
+        values["p1"] = "{0x11 0x11 0x11 0x11}"
+        values["p2"] = "{0x00 0x00 0x00 0x00}"
+        values["p3"] = "{0x11 0x11 0x11 0x11}"
+        values["p4"] = "{0x00 0x00 0x00 0x00}"
+
+        for regname, value in values.items():
+            self.expect("register read {}".format(regname),
+                        substrs=["{} = {}".format(regname, value)])
 
         self.expect("register read --all")
 
@@ -350,7 +487,8 @@ def test_arm_core(self):
         values["pc"] = "0x0000000f"
         values["cpsr"] = "0x00000010"
         for regname, value in values.items():
-            self.expect("register read {}".format(regname), substrs=["{} = {}".format(regname, value)])
+            self.expect("register read {}".format(regname),
+                        substrs=["{} = {}".format(regname, value)])
 
         self.expect("register read --all")
 
@@ -486,6 +624,7 @@ def do_test(self, filename, pid, region_count, thread_name):
 
         self.dbg.DeleteTarget(target)
 
+
 def replace_path(binary, replace_from, replace_to):
     src = replace_from.encode()
     dst = replace_to.encode()
diff --git a/lldb/test/API/functionalities/postmortem/elf-core/linux-aarch64-sve-fpsimd.core b/lldb/test/API/functionalities/postmortem/elf-core/linux-aarch64-sve-fpsimd.core
new file mode 100644
index 0000000000000..c58c8fa97795b
Binary files /dev/null and b/lldb/test/API/functionalities/postmortem/elf-core/linux-aarch64-sve-fpsimd.core differ
diff --git a/lldb/test/API/functionalities/postmortem/elf-core/linux-aarch64-sve-full.core b/lldb/test/API/functionalities/postmortem/elf-core/linux-aarch64-sve-full.core
new file mode 100644
index 0000000000000..6f408e4fea01a
Binary files /dev/null and b/lldb/test/API/functionalities/postmortem/elf-core/linux-aarch64-sve-full.core differ
diff --git a/lldb/test/API/functionalities/postmortem/elf-core/linux-aarch64-sve.c b/lldb/test/API/functionalities/postmortem/elf-core/linux-aarch64-sve.c
new file mode 100644
index 0000000000000..f9f48e5b8ba68
--- /dev/null
+++ b/lldb/test/API/functionalities/postmortem/elf-core/linux-aarch64-sve.c
@@ -0,0 +1,24 @@
+// compile with -march=armv8-a+sve on compatible aarch64 compiler
+// linux-aarch64-sve.core was generated by: aarch64-linux-gnu-gcc-8
+// commandline: -march=armv8-a+sve -nostdlib -static -g linux-aarch64-sve.c
+static void bar(char *boom) {
+  char F = 'b';
+  asm volatile("ptrue p0.s\n\t");
+  asm volatile("fcpy  z0.s, p0/m, #7.5\n\t");
+  asm volatile("ptrue p1.s\n\t");
+  asm volatile("fcpy  z1.s, p1/m, #11.5\n\t");
+  asm volatile("ptrue p3.s\n\t");
+  asm volatile("fcpy  z3.s, p3/m, #15.5\n\t");
+
+  *boom = 47; // Frame bar
+}
+
+static void foo(char *boom, void (*boomer)(char *)) {
+  char F = 'f';
+  boomer(boom); // Frame foo
+}
+
+void _start(void) {
+  char F = '_';
+  foo(0, bar); // Frame _start
+}
diff --git a/lldb/test/API/lang/objc/bitfield_ivars/Makefile b/lldb/test/API/lang/objc/bitfield_ivars/Makefile
new file mode 100644
index 0000000000000..a68dad547ec63
--- /dev/null
+++ b/lldb/test/API/lang/objc/bitfield_ivars/Makefile
@@ -0,0 +1,4 @@
+OBJC_SOURCES := main.m
+LD_EXTRAS = -framework Foundation
+
+include Makefile.rules
diff --git a/lldb/test/API/lang/objc/bitfield_ivars/TestBitfieldIvars.py b/lldb/test/API/lang/objc/bitfield_ivars/TestBitfieldIvars.py
index c0d006ee53a87..6118854131024 100644
--- a/lldb/test/API/lang/objc/bitfield_ivars/TestBitfieldIvars.py
+++ b/lldb/test/API/lang/objc/bitfield_ivars/TestBitfieldIvars.py
@@ -1,12 +1,41 @@
-from lldbsuite.test import lldbinline
-from lldbsuite.test import decorators
-
-lldbinline.MakeInlineTest(
-    __file__,
-    globals(),
-    [
-        # This is a Darwin-only failure related to incorrect expression-
-        # evaluation for single-bit ObjC bitfields.
-        decorators.skipUnlessDarwin,
-        decorators.expectedFailureAll(
-            bugnumber="rdar://problem/17990991")])
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+class TestBitfieldIvars(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+
+    @skipUnlessDarwin
+    def test(self):
+        self.build()
+        lldbutil.run_to_source_breakpoint(self, "// break here", lldb.SBFileSpec("main.m"))
+
+        self.expect_expr("chb->hb->field1", result_type="unsigned int", result_value="0")
+
+        ## FIXME field2 should have a value of 1
+        self.expect("expr chb->hb->field2", matching=False, substrs = ["= 1"]) # this must happen second
+
+        self.expect_expr("hb2->field1", result_type="unsigned int", result_value="10")
+        self.expect_expr("hb2->field2", result_type="unsigned int", result_value="3")
+        self.expect_expr("hb2->field3", result_type="unsigned int", result_value="4")
+
+        self.expect("frame var *hb2", substrs = [ 'x =', '100',
+                                             'field1 =', '10',
+                                             'field2 =', '3',
+                                             'field3 =', '4'])
+
+    # This test is meant to be xfailed, but running the test triggers an ASan
+    # issue, so it must be skipped for now.
+    @skipIf
+    def testExprWholeObject(self):
+        self.build()
+        lldbutil.run_to_source_breakpoint(self, "// break here", lldb.SBFileSpec("main.m"))
+
+        ## FIXME expression with individual bit-fields obtains correct values but not with the whole object
+        self.expect("expr *hb2", substrs = [ 'x =', '100',
+                                             'field1 =', '10',
+                                             'field2 =', '3',
+                                             'field3 =', '4'])
diff --git a/lldb/test/API/lang/objc/bitfield_ivars/main.m b/lldb/test/API/lang/objc/bitfield_ivars/main.m
index 1ba1cf004abf6..841464d38fe74 100644
--- a/lldb/test/API/lang/objc/bitfield_ivars/main.m
+++ b/lldb/test/API/lang/objc/bitfield_ivars/main.m
@@ -34,10 +34,31 @@ -(id)init {
 
 @end
 
+@interface HasBitfield2 : NSObject {
+@public
+  unsigned int x;
+
+  unsigned field1 : 15;
+  unsigned field2 : 4;
+  unsigned field3 : 4;
+}
+@end
+
+@implementation HasBitfield2
+- (id)init {
+  return (self = [super init]);
+}
+@end
+
 int main(int argc, const char * argv[]) {
     ContainsAHasBitfield *chb = [[ContainsAHasBitfield alloc] init];
-    printf("%d\n", chb->hb->field2); //% self.expect("expression -- chb->hb->field1", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["= 0"])
-                                     //% self.expect("expression -- chb->hb->field2", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["= 1"]) # this must happen second
-    return 0;
+    HasBitfield2 *hb2 = [[HasBitfield2 alloc] init];
+
+    hb2->x = 100;
+    hb2->field1 = 10;
+    hb2->field2 = 3;
+    hb2->field3 = 4;
+
+    return 0; // break here
 }
 
diff --git a/lldb/test/API/python_api/sbenvironment/TestSBEnvironment.py b/lldb/test/API/python_api/sbenvironment/TestSBEnvironment.py
index c1937f985e283..6389854ce58f5 100644
--- a/lldb/test/API/python_api/sbenvironment/TestSBEnvironment.py
+++ b/lldb/test/API/python_api/sbenvironment/TestSBEnvironment.py
@@ -53,6 +53,11 @@ def test_launch_info(self):
         launch_info.SetEnvironment(env, append=True)
         self.assertEqual(launch_info.GetEnvironment().GetNumValues(), env_count + 1)
 
+        env.Set("FOO", "baz", overwrite=True)
+        launch_info.SetEnvironment(env, append=True)
+        self.assertEqual(launch_info.GetEnvironment().GetNumValues(), env_count + 1)
+        self.assertEqual(launch_info.GetEnvironment().Get("FOO"), "baz")
+
         # Make sure we can replace the launchInfo's environment
         env.Clear()
         env.Set("BAR", "foo", overwrite=True)
@@ -120,6 +125,11 @@ def test_creating_and_modifying_environment(self):
         env.SetEntries(entries, append=False)
         self.assertEqualEntries(env, ["X=x", "Y=y"])
 
+        entries.Clear()
+        entries.AppendList(["X=y", "Y=x"], 2)
+        env.SetEntries(entries, append=True)
+        self.assertEqualEntries(env, ["X=y", "Y=x"])
+
         # Test clear
         env.Clear()
         self.assertEqualEntries(env, [])
diff --git a/lldb/test/API/source-manager/TestSourceManager.py b/lldb/test/API/source-manager/TestSourceManager.py
index 714b736da02e6..3ce056f2d0294 100644
--- a/lldb/test/API/source-manager/TestSourceManager.py
+++ b/lldb/test/API/source-manager/TestSourceManager.py
@@ -197,6 +197,14 @@ def test_modify_source_file_while_debugging(self):
             SOURCE_DISPLAYED_CORRECTLY,
             substrs=['Hello world'])
 
+        # Do the same thing with a file & line spec:
+        self.expect(
+            "source list -y main-copy.c:%d" %
+            self.line,
+            SOURCE_DISPLAYED_CORRECTLY,
+            substrs=['Hello world'])
+
+        
         # The '-b' option shows the line table locations from the debug information
         # that indicates valid places to set source level breakpoints.
 
diff --git a/lldb/test/Shell/ObjectFile/PECOFF/symbol.yaml b/lldb/test/Shell/ObjectFile/PECOFF/symbol.yaml
index 1d79e702333b8..9dbc93b9b9185 100644
--- a/lldb/test/Shell/ObjectFile/PECOFF/symbol.yaml
+++ b/lldb/test/Shell/ObjectFile/PECOFF/symbol.yaml
@@ -1,6 +1,9 @@
 # RUN: yaml2obj %s > %t
 # RUN: lldb-test symbols %t | FileCheck %s
 
+## The .file symbol isn't checked, but is included to test that the symbol
+## table iteration handles cases with a symbol with more than one aux symbol.
+
 # CHECK: Type File Address/Value {{.*}} Size            Flags           Name
 # CHECK: Code 0x0000000040001000        0x{{[0-9a-f]+}} 0x{{[0-9a-f]+}} entry
 # CHECK:      0x0000000040002000        0x{{[0-9a-f]+}} 0x{{[0-9a-f]+}} variable
@@ -101,6 +104,13 @@ symbols:
     SimpleType:      IMAGE_SYM_TYPE_NULL
     ComplexType:     IMAGE_SYM_DTYPE_NULL
     StorageClass:    IMAGE_SYM_CLASS_STATIC
+  - Name:            .file
+    Value:           0
+    SectionNumber:   -2
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_FILE
+    File:            longfilenameusingtwoauxsymbols
   - Name:            entry
     Value:           0
     SectionNumber:   1
diff --git a/lldb/test/Shell/Reproducer/TestWorkingDir.test b/lldb/test/Shell/Reproducer/TestWorkingDir.test
index 1d4c7694211af..54266e76c2799 100644
--- a/lldb/test/Shell/Reproducer/TestWorkingDir.test
+++ b/lldb/test/Shell/Reproducer/TestWorkingDir.test
@@ -15,3 +15,18 @@
 
 # RUN: cat %t.repro/cwd.txt | FileCheck %t.check
 # RUN: %lldb --replay %t.repro | FileCheck %t.check
+
+# Make sure the current working directory is recorded even when it's not
+# referenced.
+
+# RUN: rm -rf %t.repro
+# RUN: rm -rf %t
+# RUN: mkdir -p %t/probably_unique
+# RUN: touch %t/probably_unique/dont_include_me
+# RUN: cd %t/probably_unique
+# RUN: %lldb -x -b -o 'reproducer generate' --capture --capture-path %t.repro
+# RUN: cat %t.repro/cwd.txt | FileCheck %s
+# CHECK: probably_unique
+# RUN: cat %t.repro/files.yaml | FileCheck %s --check-prefix VFS
+# VFS: probably_unique
+# VFS-NOT: dont_include_me
diff --git a/lldb/tools/debugserver/CMakeLists.txt b/lldb/tools/debugserver/CMakeLists.txt
index fc23cf3c7e206..eba5c41491329 100644
--- a/lldb/tools/debugserver/CMakeLists.txt
+++ b/lldb/tools/debugserver/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.4.3)
+cmake_minimum_required(VERSION 3.13.4)
 
 project(Debugserver LANGUAGES C CXX ASM-ATT)
 
diff --git a/lldb/unittests/CMakeLists.txt b/lldb/unittests/CMakeLists.txt
index 6422b726ca69d..37a5f972cdec6 100644
--- a/lldb/unittests/CMakeLists.txt
+++ b/lldb/unittests/CMakeLists.txt
@@ -5,6 +5,10 @@ add_dependencies(lldb-test-deps LLDBUnitTests)
 include_directories(${LLDB_SOURCE_ROOT})
 include_directories(${LLDB_PROJECT_ROOT}/unittests)
 
+if (CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
+  add_compile_options("-Wno-suggest-override")
+endif()
+
 set(LLDB_GTEST_COMMON_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/gtest_common.h)
 if (MSVC)
   list(APPEND LLVM_COMPILE_FLAGS /FI ${LLDB_GTEST_COMMON_INCLUDE})
diff --git a/lldb/unittests/DataFormatter/CMakeLists.txt b/lldb/unittests/DataFormatter/CMakeLists.txt
index 45011c56b0b0f..9d967a72bfd1f 100644
--- a/lldb/unittests/DataFormatter/CMakeLists.txt
+++ b/lldb/unittests/DataFormatter/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_lldb_unittest(LLDBFormatterTests
   FormatManagerTests.cpp
+  FormattersContainerTest.cpp
   StringPrinterTests.cpp
 
   LINK_LIBS
diff --git a/lldb/unittests/DataFormatter/FormattersContainerTest.cpp b/lldb/unittests/DataFormatter/FormattersContainerTest.cpp
new file mode 100644
index 0000000000000..a28212391eae9
--- /dev/null
+++ b/lldb/unittests/DataFormatter/FormattersContainerTest.cpp
@@ -0,0 +1,159 @@
+//===-- FormattersContainerTests.cpp --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/DataFormatters/FormattersContainer.h"
+
+#include "gtest/gtest.h"
+
+using namespace lldb;
+using namespace lldb_private;
+
+// All the prefixes that the exact name matching will strip from the type.
+static const std::vector<std::string> exact_name_prefixes = {
+    "", // no prefix.
+    "class ", "struct ", "union ", "enum ",
+};
+
+// TypeMatcher that uses a exact type name string that needs to be matched.
+TEST(TypeMatcherTests, ExactName) {
+  for (const std::string &prefix : exact_name_prefixes) {
+    SCOPED_TRACE("Prefix: " + prefix);
+
+    TypeMatcher matcher(ConstString(prefix + "Name"));
+    EXPECT_TRUE(matcher.Matches(ConstString("class Name")));
+    EXPECT_TRUE(matcher.Matches(ConstString("struct Name")));
+    EXPECT_TRUE(matcher.Matches(ConstString("union Name")));
+    EXPECT_TRUE(matcher.Matches(ConstString("enum Name")));
+    EXPECT_TRUE(matcher.Matches(ConstString("Name")));
+
+    EXPECT_FALSE(matcher.Matches(ConstString("Name ")));
+    EXPECT_FALSE(matcher.Matches(ConstString("ame")));
+    EXPECT_FALSE(matcher.Matches(ConstString("Nam")));
+    EXPECT_FALSE(matcher.Matches(ConstString("am")));
+    EXPECT_FALSE(matcher.Matches(ConstString("a")));
+    EXPECT_FALSE(matcher.Matches(ConstString(" ")));
+    EXPECT_FALSE(matcher.Matches(ConstString("class N")));
+    EXPECT_FALSE(matcher.Matches(ConstString("class ")));
+    EXPECT_FALSE(matcher.Matches(ConstString("class")));
+  }
+}
+
+// TypeMatcher that uses a regex to match a type name.
+TEST(TypeMatcherTests, RegexName) {
+  TypeMatcher matcher(RegularExpression("^a[a-z]c$"));
+  EXPECT_TRUE(matcher.Matches(ConstString("abc")));
+  EXPECT_TRUE(matcher.Matches(ConstString("azc")));
+
+  // FIXME: This isn't consistent with the 'exact' type name matches above.
+  EXPECT_FALSE(matcher.Matches(ConstString("class abc")));
+
+  EXPECT_FALSE(matcher.Matches(ConstString("abbc")));
+  EXPECT_FALSE(matcher.Matches(ConstString(" abc")));
+  EXPECT_FALSE(matcher.Matches(ConstString("abc ")));
+  EXPECT_FALSE(matcher.Matches(ConstString(" abc ")));
+  EXPECT_FALSE(matcher.Matches(ConstString("XabcX")));
+  EXPECT_FALSE(matcher.Matches(ConstString("ac")));
+  EXPECT_FALSE(matcher.Matches(ConstString("a[a-z]c")));
+  EXPECT_FALSE(matcher.Matches(ConstString("aAc")));
+  EXPECT_FALSE(matcher.Matches(ConstString("ABC")));
+  EXPECT_FALSE(matcher.Matches(ConstString("")));
+}
+
+// TypeMatcher that only searches the type name.
+TEST(TypeMatcherTests, RegexMatchPart) {
+  TypeMatcher matcher(RegularExpression("a[a-z]c"));
+  EXPECT_TRUE(matcher.Matches(ConstString("class abc")));
+  EXPECT_TRUE(matcher.Matches(ConstString("abc")));
+  EXPECT_TRUE(matcher.Matches(ConstString(" abc ")));
+  EXPECT_TRUE(matcher.Matches(ConstString("azc")));
+  EXPECT_TRUE(matcher.Matches(ConstString("abc ")));
+  EXPECT_TRUE(matcher.Matches(ConstString(" abc ")));
+  EXPECT_TRUE(matcher.Matches(ConstString(" abc")));
+  EXPECT_TRUE(matcher.Matches(ConstString("XabcX")));
+
+  EXPECT_FALSE(matcher.Matches(ConstString("abbc")));
+  EXPECT_FALSE(matcher.Matches(ConstString("ac")));
+  EXPECT_FALSE(matcher.Matches(ConstString("a[a-z]c")));
+  EXPECT_FALSE(matcher.Matches(ConstString("aAc")));
+  EXPECT_FALSE(matcher.Matches(ConstString("ABC")));
+  EXPECT_FALSE(matcher.Matches(ConstString("")));
+}
+
+// GetMatchString for exact type name matchers.
+TEST(TypeMatcherTests, GetMatchStringExactName) {
+  EXPECT_EQ(TypeMatcher(ConstString("aa")).GetMatchString(), "aa");
+  EXPECT_EQ(TypeMatcher(ConstString("")).GetMatchString(), "");
+  EXPECT_EQ(TypeMatcher(ConstString("[a]")).GetMatchString(), "[a]");
+}
+
+// GetMatchString for regex matchers.
+TEST(TypeMatcherTests, GetMatchStringRegex) {
+  EXPECT_EQ(TypeMatcher(RegularExpression("aa")).GetMatchString(), "aa");
+  EXPECT_EQ(TypeMatcher(RegularExpression("")).GetMatchString(), "");
+  EXPECT_EQ(TypeMatcher(RegularExpression("[a]")).GetMatchString(), "[a]");
+}
+
+// GetMatchString for regex matchers.
+TEST(TypeMatcherTests, CreatedBySameMatchString) {
+  TypeMatcher empty_str(ConstString(""));
+  TypeMatcher empty_regex(RegularExpression(""));
+  EXPECT_TRUE(empty_str.CreatedBySameMatchString(empty_str));
+  EXPECT_TRUE(empty_str.CreatedBySameMatchString(empty_regex));
+
+  TypeMatcher a_str(ConstString("a"));
+  TypeMatcher a_regex(RegularExpression("a"));
+  EXPECT_TRUE(a_str.CreatedBySameMatchString(a_str));
+  EXPECT_TRUE(a_str.CreatedBySameMatchString(a_regex));
+
+  TypeMatcher digit_str(ConstString("[0-9]"));
+  TypeMatcher digit_regex(RegularExpression("[0-9]"));
+  EXPECT_TRUE(digit_str.CreatedBySameMatchString(digit_str));
+  EXPECT_TRUE(digit_str.CreatedBySameMatchString(digit_regex));
+
+  EXPECT_FALSE(empty_str.CreatedBySameMatchString(a_str));
+  EXPECT_FALSE(empty_str.CreatedBySameMatchString(a_regex));
+  EXPECT_FALSE(empty_str.CreatedBySameMatchString(digit_str));
+  EXPECT_FALSE(empty_str.CreatedBySameMatchString(digit_regex));
+
+  EXPECT_FALSE(empty_regex.CreatedBySameMatchString(a_str));
+  EXPECT_FALSE(empty_regex.CreatedBySameMatchString(a_regex));
+  EXPECT_FALSE(empty_regex.CreatedBySameMatchString(digit_str));
+  EXPECT_FALSE(empty_regex.CreatedBySameMatchString(digit_regex));
+
+  EXPECT_FALSE(a_str.CreatedBySameMatchString(empty_str));
+  EXPECT_FALSE(a_str.CreatedBySameMatchString(empty_regex));
+  EXPECT_FALSE(a_str.CreatedBySameMatchString(digit_str));
+  EXPECT_FALSE(a_str.CreatedBySameMatchString(digit_regex));
+
+  EXPECT_FALSE(a_regex.CreatedBySameMatchString(empty_str));
+  EXPECT_FALSE(a_regex.CreatedBySameMatchString(empty_regex));
+  EXPECT_FALSE(a_regex.CreatedBySameMatchString(digit_str));
+  EXPECT_FALSE(a_regex.CreatedBySameMatchString(digit_regex));
+
+  EXPECT_FALSE(digit_str.CreatedBySameMatchString(empty_str));
+  EXPECT_FALSE(digit_str.CreatedBySameMatchString(empty_regex));
+  EXPECT_FALSE(digit_str.CreatedBySameMatchString(a_str));
+  EXPECT_FALSE(digit_str.CreatedBySameMatchString(a_regex));
+
+  EXPECT_FALSE(digit_regex.CreatedBySameMatchString(empty_str));
+  EXPECT_FALSE(digit_regex.CreatedBySameMatchString(empty_regex));
+  EXPECT_FALSE(digit_regex.CreatedBySameMatchString(a_str));
+  EXPECT_FALSE(digit_regex.CreatedBySameMatchString(a_regex));
+}
+
+// Test CreatedBySameMatchString with stripped exact name prefixes.
+TEST(TypeMatcherTests, CreatedBySameMatchStringExactNamePrefixes) {
+  for (const std::string &prefix : exact_name_prefixes) {
+    SCOPED_TRACE("Prefix: " + prefix);
+    TypeMatcher with_prefix(ConstString(prefix + "Name"));
+    TypeMatcher without_prefix(RegularExpression(""));
+
+    EXPECT_TRUE(with_prefix.CreatedBySameMatchString(with_prefix));
+    EXPECT_TRUE(without_prefix.CreatedBySameMatchString(without_prefix));
+  }
+}
diff --git a/lldb/unittests/Expression/DWARFExpressionTest.cpp b/lldb/unittests/Expression/DWARFExpressionTest.cpp
index 78c400c33d4e3..ffa3775a6113b 100644
--- a/lldb/unittests/Expression/DWARFExpressionTest.cpp
+++ b/lldb/unittests/Expression/DWARFExpressionTest.cpp
@@ -94,8 +94,7 @@ TEST(DWARFExpression, DW_OP_convert) {
       "      - Attribute:       DW_AT_byte_size\n"
       "        Form:            DW_FORM_data1\n"
       "debug_info:\n"
-      "  - Length:          0\n"
-      "    Version:         4\n"
+      "  - Version:         4\n"
       "    AbbrOffset:      0\n"
       "    AddrSize:        8\n"
       "    Entries:\n"
diff --git a/lldb/unittests/Interpreter/CMakeLists.txt b/lldb/unittests/Interpreter/CMakeLists.txt
index 0de5b0b72488e..28663ec02a2a9 100644
--- a/lldb/unittests/Interpreter/CMakeLists.txt
+++ b/lldb/unittests/Interpreter/CMakeLists.txt
@@ -1,6 +1,7 @@
 add_lldb_unittest(InterpreterTests
   TestCompletion.cpp
   TestOptionArgParser.cpp
+  TestOptionValueFileColonLine.cpp
 
   LINK_LIBS
     lldbInterpreter
diff --git a/lldb/unittests/Interpreter/TestOptionValueFileColonLine.cpp b/lldb/unittests/Interpreter/TestOptionValueFileColonLine.cpp
new file mode 100644
index 0000000000000..079c26bdd8ab7
--- /dev/null
+++ b/lldb/unittests/Interpreter/TestOptionValueFileColonLine.cpp
@@ -0,0 +1,58 @@
+//===-- ArgsTest.cpp ------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Interpreter/OptionValueFileColonLine.h"
+#include "lldb/Utility/FileSpec.h"
+#include "lldb/Utility/Status.h"
+#include "gtest/gtest.h"
+
+using namespace lldb_private;
+
+void CheckSetting(const char *input, bool success, FileSpec path = {},
+                  uint32_t line_number = LLDB_INVALID_LINE_NUMBER,
+                  uint32_t column_number = LLDB_INVALID_COLUMN_NUMBER) {
+
+  OptionValueFileColonLine value;
+  Status error;
+  llvm::StringRef s_ref(input);
+  error = value.SetValueFromString(s_ref);
+  ASSERT_EQ(error.Success(), success);
+
+  // If we were meant to fail, we don't need to do more checks:
+  if (!success)
+    return;
+
+  ASSERT_EQ(value.GetLineNumber(), line_number);
+  ASSERT_EQ(value.GetColumnNumber(), column_number);
+  ASSERT_EQ(value.GetFileSpec(), path);
+}
+
+TEST(OptionValueFileColonLine, setFromString) {
+  OptionValueFileColonLine value;
+  Status error;
+
+  // Make sure a default constructed value is invalid:
+  ASSERT_EQ(value.GetLineNumber(),
+            static_cast<uint32_t>(LLDB_INVALID_LINE_NUMBER));
+  ASSERT_EQ(value.GetColumnNumber(),
+            static_cast<uint32_t>(LLDB_INVALID_COLUMN_NUMBER));
+  ASSERT_FALSE(value.GetFileSpec());
+
+  // Make sure it is an error to pass a specifier with no line number:
+  CheckSetting("foo.c", false);
+
+  // Now try with just a file & line:
+  CheckSetting("foo.c:12", true, FileSpec("foo.c"), 12);
+  CheckSetting("foo.c:12:20", true, FileSpec("foo.c"), 12, 20);
+  // Make sure a colon doesn't mess us up:
+  CheckSetting("foo:bar.c:12", true, FileSpec("foo:bar.c"), 12);
+  CheckSetting("foo:bar.c:12:20", true, FileSpec("foo:bar.c"), 12, 20);
+  // Try errors in the line number:
+  CheckSetting("foo.c:12c", false);
+  CheckSetting("foo.c:12:20c", false);
+}
diff --git a/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp b/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp
index 858aa61b19bb8..6b84a21c9d7f4 100644
--- a/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp
+++ b/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp
@@ -56,8 +56,7 @@ TEST_F(DWARFASTParserClangTests,
       "      - Attribute:       DW_AT_byte_size\n"
       "        Form:            DW_FORM_data1\n"
       "debug_info:\n"
-      "  - Length:          0\n"
-      "    Version:         4\n"
+      "  - Version:         4\n"
       "    AbbrOffset:      0\n"
       "    AddrSize:        8\n"
       "    Entries:\n"
diff --git a/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp b/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp
index 072fe6aaf0f9f..63bb6b7c40436 100644
--- a/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp
+++ b/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp
@@ -363,7 +363,7 @@ TEST_F(SymbolFilePDBTests, TestSimpleClassTypes) {
   CompilerType compiler_type = udt_type->GetForwardCompilerType();
   EXPECT_TRUE(TypeSystemClang::IsClassType(compiler_type.GetOpaqueQualType()));
   EXPECT_EQ(GetGlobalConstantInteger(session, "sizeof_Class"),
-            udt_type->GetByteSize());
+            udt_type->GetByteSize(nullptr));
 }
 
 TEST_F(SymbolFilePDBTests, TestNestedClassTypes) {
@@ -417,7 +417,7 @@ TEST_F(SymbolFilePDBTests, TestNestedClassTypes) {
   EXPECT_TRUE(TypeSystemClang::IsClassType(compiler_type.GetOpaqueQualType()));
 
   EXPECT_EQ(GetGlobalConstantInteger(session, "sizeof_NestedClass"),
-            udt_type->GetByteSize());
+            udt_type->GetByteSize(nullptr));
 }
 
 TEST_F(SymbolFilePDBTests, TestClassInNamespace) {
@@ -461,7 +461,7 @@ TEST_F(SymbolFilePDBTests, TestClassInNamespace) {
   EXPECT_TRUE(TypeSystemClang::IsClassType(compiler_type.GetOpaqueQualType()));
 
   EXPECT_EQ(GetGlobalConstantInteger(session, "sizeof_NSClass"),
-            udt_type->GetByteSize());
+            udt_type->GetByteSize(nullptr));
 }
 
 TEST_F(SymbolFilePDBTests, TestEnumTypes) {
@@ -491,7 +491,7 @@ TEST_F(SymbolFilePDBTests, TestEnumTypes) {
     std::string sizeof_var = "sizeof_";
     sizeof_var.append(Enum);
     EXPECT_EQ(GetGlobalConstantInteger(session, sizeof_var),
-              enum_type->GetByteSize());
+              enum_type->GetByteSize(nullptr));
   }
 }
 
@@ -539,7 +539,7 @@ TEST_F(SymbolFilePDBTests, TestTypedefs) {
     std::string sizeof_var = "sizeof_";
     sizeof_var.append(Typedef);
     EXPECT_EQ(GetGlobalConstantInteger(session, sizeof_var),
-              typedef_type->GetByteSize());
+              typedef_type->GetByteSize(nullptr));
   }
 }
 
diff --git a/lldb/unittests/TestingSupport/Symbol/YAMLModuleTester.cpp b/lldb/unittests/TestingSupport/Symbol/YAMLModuleTester.cpp
index 54c2c9441e579..248e87825ac99 100644
--- a/lldb/unittests/TestingSupport/Symbol/YAMLModuleTester.cpp
+++ b/lldb/unittests/TestingSupport/Symbol/YAMLModuleTester.cpp
@@ -93,7 +93,7 @@ class YAMLObjectFile : public lldb_private::ObjectFile {
 
 YAMLModuleTester::YAMLModuleTester(llvm::StringRef yaml_data,
                                    llvm::StringRef triple) {
-  auto sections_map = llvm::DWARFYAML::emitDebugSections(yaml_data, true);
+  auto sections_map = llvm::DWARFYAML::emitDebugSections(yaml_data);
   if (!sections_map)
     return;
   m_sections_map = std::move(*sections_map);
diff --git a/lldb/unittests/Utility/ScalarTest.cpp b/lldb/unittests/Utility/ScalarTest.cpp
index dd4683145b968..f6bc6a404c159 100644
--- a/lldb/unittests/Utility/ScalarTest.cpp
+++ b/lldb/unittests/Utility/ScalarTest.cpp
@@ -135,12 +135,8 @@ TEST(ScalarTest, GetBytes) {
   Scalar f_scalar;
   DataExtractor e_data(e, sizeof(e), endian::InlHostByteOrder(),
                        sizeof(void *));
-  Status e_error =
-      e_scalar.SetValueFromData(e_data, lldb::eEncodingUint, sizeof(e));
   DataExtractor f_data(f, sizeof(f), endian::InlHostByteOrder(),
                        sizeof(void *));
-  Status f_error =
-      f_scalar.SetValueFromData(f_data, lldb::eEncodingUint, sizeof(f));
   a_scalar.GetBytes(Storage);
   ASSERT_EQ(0, memcmp(&a, Storage, sizeof(a)));
   b_scalar.GetBytes(Storage);
@@ -149,14 +145,39 @@ TEST(ScalarTest, GetBytes) {
   ASSERT_EQ(0, memcmp(&c, Storage, sizeof(c)));
   d_scalar.GetBytes(Storage);
   ASSERT_EQ(0, memcmp(&d, Storage, sizeof(d)));
-  ASSERT_EQ(0, e_error.Fail());
+  ASSERT_THAT_ERROR(
+      e_scalar.SetValueFromData(e_data, lldb::eEncodingUint, sizeof(e))
+          .ToError(),
+      llvm::Succeeded());
   e_scalar.GetBytes(Storage);
   ASSERT_EQ(0, memcmp(e, Storage, sizeof(e)));
-  ASSERT_EQ(0, f_error.Fail());
+  ASSERT_THAT_ERROR(
+      f_scalar.SetValueFromData(f_data, lldb::eEncodingUint, sizeof(f))
+          .ToError(),
+      llvm::Succeeded());
   f_scalar.GetBytes(Storage);
   ASSERT_EQ(0, memcmp(f, Storage, sizeof(f)));
 }
 
+TEST(ScalarTest, SetValueFromData) {
+  uint8_t a[] = {1, 2, 3, 4};
+  Scalar s;
+  ASSERT_THAT_ERROR(
+      s.SetValueFromData(
+           DataExtractor(a, sizeof(a), lldb::eByteOrderLittle, sizeof(void *)),
+           lldb::eEncodingSint, sizeof(a))
+          .ToError(),
+      llvm::Succeeded());
+  EXPECT_EQ(0x04030201, s);
+  ASSERT_THAT_ERROR(
+      s.SetValueFromData(
+           DataExtractor(a, sizeof(a), lldb::eByteOrderBig, sizeof(void *)),
+           lldb::eEncodingSint, sizeof(a))
+          .ToError(),
+      llvm::Succeeded());
+  EXPECT_EQ(0x01020304, s);
+}
+
 TEST(ScalarTest, CastOperations) {
   long long a = 0xf1f2f3f4f5f6f7f8LL;
   Scalar a_scalar(a);
diff --git a/llvm-spirv/.travis.yml b/llvm-spirv/.travis.yml
index f593a238e1d20..45d909221c9b6 100644
--- a/llvm-spirv/.travis.yml
+++ b/llvm-spirv/.travis.yml
@@ -24,15 +24,18 @@ before_install:
     if [ $TRAVIS_OS_NAME == "linux" ]; then
       curl -L "https://apt.llvm.org/llvm-snapshot.gpg.key" | sudo apt-key add -
       curl -L "http://packages.lunarg.com/lunarg-signing-key-pub.asc" | sudo apt-key add -
+      curl -L "https://apt.kitware.com/keys/kitware-archive-latest.asc" | sudo apt-key add -
       echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic main" | sudo tee -a ${TRAVIS_ROOT}/etc/apt/sources.list
       echo "deb https://packages.lunarg.com/vulkan bionic main" | sudo tee -a ${TRAVIS_ROOT}/etc/apt/sources.list
+      echo "deb https://apt.kitware.com/ubuntu/ bionic main" | sudo tee -a ${TRAVIS_ROOT}/etc/apt/sources.list
       sudo apt-get update
       sudo apt-get -yq --no-install-suggests --no-install-recommends install \
         llvm-${LLVM_VERSION}-dev \
         clang-${LLVM_VERSION} \
         clang-format-${LLVM_VERSION} \
         clang-tidy-${LLVM_VERSION} \
-        spirv-tools
+        spirv-tools \
+        cmake
     fi
 
 compiler:
@@ -60,17 +63,16 @@ matrix:
   include:
     - os: osx
       env: BUILD_TYPE=Release BUILD_EXTERNAL=0 MAKE_TARGETS="llvm-spirv" MAKE_TEST_TARGET="check-llvm-spirv"
-      osx_image: xcode9.3
+      osx_image: xcode12
 
     - os: osx
       env: BUILD_TYPE=Debug BUILD_EXTERNAL=0 MAKE_TARGETS="llvm-spirv" MAKE_TEST_TARGET="check-llvm-spirv"
-      osx_image: xcode9.3
+      osx_image: xcode12
 
     - env: BUILD_EXTERNAL=1 CHECK_FORMAT=1
 
     - env: BUILD_EXTERNAL=1 CHECK_TIDY=1
   allow_failures:
-    - compiler: clang
     - os: osx
   fast_finish: true
 
@@ -90,6 +92,7 @@ script:
       export PKG_CONFIG_PATH=$PWD/install/lib/pkgconfig
     fi
   - mkdir build && cd build
+  - PATH=/usr/bin:$PATH
   - |
     if [ $BUILD_EXTERNAL == "1" ]; then
       if [ $CHECK_FORMAT != "1" ]; then
diff --git a/llvm-spirv/include/LLVMSPIRVExtensions.inc b/llvm-spirv/include/LLVMSPIRVExtensions.inc
index ecf31f72cc190..adaa48efadc20 100644
--- a/llvm-spirv/include/LLVMSPIRVExtensions.inc
+++ b/llvm-spirv/include/LLVMSPIRVExtensions.inc
@@ -10,6 +10,7 @@ EXT(SPV_INTEL_media_block_io)
 EXT(SPV_INTEL_device_side_avc_motion_estimation)
 EXT(SPV_INTEL_fpga_loop_controls)
 EXT(SPV_INTEL_fpga_memory_attributes)
+EXT(SPV_INTEL_fpga_memory_accesses)
 EXT(SPV_INTEL_unstructured_loop_controls)
 EXT(SPV_INTEL_fpga_reg)
 EXT(SPV_INTEL_blocking_pipes)
@@ -22,3 +23,4 @@ EXT(SPV_INTEL_optimization_hints)
 EXT(SPV_INTEL_float_controls2)
 EXT(SPV_INTEL_vector_compute)
 EXT(SPV_INTEL_usm_storage_classes)
+EXT(SPV_INTEL_fpga_buffer_location)
diff --git a/llvm-spirv/lib/SPIRV/OCLUtil.h b/llvm-spirv/lib/SPIRV/OCLUtil.h
index 3d4ae9b9ae2e5..00b8bb1ae8602 100644
--- a/llvm-spirv/lib/SPIRV/OCLUtil.h
+++ b/llvm-spirv/lib/SPIRV/OCLUtil.h
@@ -255,6 +255,13 @@ const OCLMemOrderKind OCLLegacyAtomicMemOrder = OCLMO_seq_cst;
 /// OCL 1.x atomic memory scope when translated to 2.0 atomics.
 const OCLScopeKind OCLLegacyAtomicMemScope = OCLMS_device;
 
+enum IntelFPGAMemoryAccessesVal {
+  BurstCoalesce = 0x1,
+  CacheSizeFlag = 0x2,
+  DontStaticallyCoalesce = 0x4,
+  PrefetchFlag = 0x8
+};
+
 namespace kOCLVer {
 const unsigned CL12 = 102000;
 const unsigned CL20 = 200000;
diff --git a/llvm-spirv/lib/SPIRV/SPIRVReader.cpp b/llvm-spirv/lib/SPIRV/SPIRVReader.cpp
index bd6317c9bdfb1..bc7386e26d765 100644
--- a/llvm-spirv/lib/SPIRV/SPIRVReader.cpp
+++ b/llvm-spirv/lib/SPIRV/SPIRVReader.cpp
@@ -168,6 +168,27 @@ static void addOCLKernelArgumentMetadata(
   Fn->setMetadata(MDName, MDNode::get(*Context, ValueVec));
 }
 
+static void addBufferLocationMetadata(
+    LLVMContext *Context, SPIRVFunction *BF, llvm::Function *Fn,
+    std::function<Metadata *(SPIRVFunctionParameter *)> Func) {
+  std::vector<Metadata *> ValueVec;
+  bool DecorationFound = false;
+  BF->foreachArgument([&](SPIRVFunctionParameter *Arg) {
+    if (Arg->getType()->isTypePointer() &&
+        Arg->hasDecorate(DecorationBufferLocationINTEL)) {
+      DecorationFound = true;
+      ValueVec.push_back(Func(Arg));
+    } else {
+      llvm::Metadata *DefaultNode = ConstantAsMetadata::get(
+          ConstantInt::get(Type::getInt32Ty(*Context), -1));
+      ValueVec.push_back(DefaultNode);
+    }
+  });
+  if (DecorationFound)
+    Fn->setMetadata("kernel_arg_buffer_location",
+                    MDNode::get(*Context, ValueVec));
+}
+
 Value *SPIRVToLLVM::getTranslatedValue(SPIRVValue *BV) {
   auto Loc = ValueMap.find(BV);
   if (Loc != ValueMap.end())
@@ -243,11 +264,30 @@ Value *SPIRVToLLVM::mapFunction(SPIRVFunction *BF, Function *F) {
 
 // Variable like GlobalInvolcationId[x] -> get_global_id(x).
 // Variable like WorkDim -> get_work_dim().
+// Replace the following pattern:
+// %a = addrspacecast i32 addrspace(1)* @__spirv_BuiltInSubgroupMaxSize to
+// i32 addrspace(4)*
+// %b = load i32, i32 addrspace(4)* %a, align 4
+// %c = load i32, i32 addrspace(4)* %a, align 4
+// With:
+// %b = call spir_func i32 @_Z22get_max_sub_group_sizev()
+// %c = call spir_func i32 @_Z22get_max_sub_group_sizev()
+
+// And replace the following pattern:
+// %a = addrspacecast <3 x i64> addrspace(1)* @__spirv_BuiltInWorkgroupId to
+// <3 x i64> addrspace(4)*
+// %b = load <3 x i64>, <3 x i64> addrspace(4)* %a, align 32
+// %c = extractelement <3 x i64> %b, i32 idx
+// %d = extractelement <3 x i64> %b, i32 idx
+// With:
+// %c = call spir_func i64 @_Z12get_group_idj(idx)
+// %d = call spir_func i64 @_Z12get_group_idj(idx)
 bool SPIRVToLLVM::transOCLBuiltinFromVariable(GlobalVariable *GV,
                                               SPIRVBuiltinVariableKind Kind) {
   std::string FuncName = SPIRSPIRVBuiltinVariableMap::rmap(Kind);
-  std::string MangledName;
   Type *ReturnTy = GV->getType()->getPointerElementType();
+  // Some SPIR-V builtin variables are translated to a function with an index
+  // argument.
   bool HasIndexArg =
       ReturnTy->isVectorTy() &&
       !(BuiltInSubgroupEqMask <= Kind && Kind <= BuiltInSubgroupLtMask);
@@ -256,6 +296,7 @@ bool SPIRVToLLVM::transOCLBuiltinFromVariable(GlobalVariable *GV,
   std::vector<Type *> ArgTy;
   if (HasIndexArg)
     ArgTy.push_back(Type::getInt32Ty(*Context));
+  std::string MangledName;
   mangleOpenClBuiltin(FuncName, ArgTy, MangledName);
   Function *Func = M->getFunction(MangledName);
   if (!Func) {
@@ -265,48 +306,70 @@ bool SPIRVToLLVM::transOCLBuiltinFromVariable(GlobalVariable *GV,
     Func->addFnAttr(Attribute::NoUnwind);
     Func->addFnAttr(Attribute::ReadNone);
   }
-  std::vector<Instruction *> Deletes;
-  std::vector<Instruction *> Uses;
-  for (auto UI = GV->user_begin(), UE = GV->user_end(); UI != UE; ++UI) {
-    LoadInst *LD = nullptr;
-    AddrSpaceCastInst *ASCast = dyn_cast<AddrSpaceCastInst>(*UI);
-    if (ASCast) {
-      LD = cast<LoadInst>(*ASCast->user_begin());
-    } else {
-      LD = cast<LoadInst>(*UI);
-    }
-    if (!HasIndexArg) {
-      Uses.push_back(LD);
-      Deletes.push_back(LD);
-      if (ASCast)
-        Deletes.push_back(ASCast);
-      continue;
-    }
-    for (auto LDUI = LD->user_begin(), LDUE = LD->user_end(); LDUI != LDUE;
-         ++LDUI) {
-      assert(isa<ExtractElementInst>(*LDUI) && "Unsupported use");
-      auto EEI = dyn_cast<ExtractElementInst>(*LDUI);
-      Uses.push_back(EEI);
-      Deletes.push_back(EEI);
-    }
-    Deletes.push_back(LD);
-    if (ASCast)
-      Deletes.push_back(ASCast);
-  }
-  for (auto &I : Uses) {
-    std::vector<Value *> Arg;
-    if (auto EEI = dyn_cast<ExtractElementInst>(I))
-      Arg.push_back(EEI->getIndexOperand());
+
+  // Collect instructions in these containers to remove them later.
+  std::vector<Instruction *> Extracts;
+  std::vector<Instruction *> Loads;
+  std::vector<Instruction *> Casts;
+
+  auto Replace = [&](std::vector<Value *> Arg, Instruction *I) {
     auto Call = CallInst::Create(Func, Arg, "", I);
     Call->takeName(I);
     setAttrByCalledFunc(Call);
     SPIRVDBG(dbgs() << "[transOCLBuiltinFromVariable] " << *I << " -> " << *Call
                     << '\n';)
     I->replaceAllUsesWith(Call);
+  };
+
+  // If HasIndexArg is true, we are going over users of the Load instruction,
+  // which are expected to be ExtractElement instructions. The result of the
+  // ExtractElement instructions is considered as a usage of the GV and should
+  // be replaced with Func.
+  // If HasIndexArg is false, the result of the Load instruction is the value
+  // which should be replaced with the Func.
+  auto FindAndReplace = [&](LoadInst *LD) {
+    Loads.push_back(LD);
+    if (HasIndexArg) {
+      for (auto *LoadUser : LD->users()) {
+        if (auto *Extract = dyn_cast<ExtractElementInst>(LoadUser)) {
+          Extracts.push_back(Extract);
+          Replace({Extract->getIndexOperand()}, Extract);
+        }
+      }
+    } else {
+      Replace({}, LD);
+    }
+  };
+
+  // Go over the GV users, find Load and ExtractElement instructions and
+  // replace them with the corresponding function call.
+  for (auto *UI : GV->users()) {
+    // There might or might not be an addrspacecast instruction.
+    if (auto *ASCast = dyn_cast<AddrSpaceCastInst>(UI)) {
+      Casts.push_back(ASCast);
+      for (auto *CastUser : ASCast->users()) {
+        if (auto *LD = dyn_cast<LoadInst>(CastUser)) {
+          FindAndReplace(LD);
+        }
+      }
+    } else if (auto *LD = dyn_cast<LoadInst>(UI)) {
+      FindAndReplace(LD);
+    } else {
+      llvm_unreachable("Unexpected pattern!");
+    }
   }
-  for (auto &I : Deletes) {
-    I->eraseFromParent();
-  }
+
+  auto Erase = [](std::vector<Instruction *> &ToErase) {
+    for (Instruction *I : ToErase) {
+      assert(I->hasNUses(0));
+      I->eraseFromParent();
+    }
+  };
+  // Order of erasing is important.
+  Erase(Extracts);
+  Erase(Loads);
+  Erase(Casts);
+
   return true;
 }
 
@@ -1680,6 +1743,13 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *BV, Function *F,
     StoreInst *SI = nullptr;
     auto *Src = transValue(BS->getSrc(), F, BB);
     auto *Dst = transValue(BS->getDst(), F, BB);
+    // An intrinsic call may have been generated for the source variable.
+    if (Use *SingleUse = Src->getSingleUndroppableUse()) {
+      if (auto *II = dyn_cast<IntrinsicInst>(SingleUse->getUser())) {
+        // Overwrite the future store operand with the intrinsic call result.
+        Src = II;
+      }
+    }
     bool isVolatile = BS->SPIRVMemoryAccess::isVolatile();
     uint64_t AlignValue = BS->SPIRVMemoryAccess::getAlignment();
     if (0 == AlignValue)
@@ -3046,6 +3116,26 @@ void generateIntelFPGAAnnotation(const SPIRVEntry *E,
     Out << "{force_pow2_depth:" << Result << '}';
   if (E->hasDecorate(DecorationUserSemantic))
     Out << E->getDecorationStringLiteral(DecorationUserSemantic).front();
+
+  unsigned LSUParamsBitmask = 0;
+  llvm::SmallString<32> AdditionalParamsStr;
+  llvm::raw_svector_ostream ParamsOut(AdditionalParamsStr);
+  if (E->hasDecorate(DecorationBurstCoalesceINTEL, 0))
+    LSUParamsBitmask |= IntelFPGAMemoryAccessesVal::BurstCoalesce;
+  if (E->hasDecorate(DecorationCacheSizeINTEL, 0, &Result)) {
+    LSUParamsBitmask |= IntelFPGAMemoryAccessesVal::CacheSizeFlag;
+    ParamsOut << "{cache-size:" << Result << "}";
+  }
+  if (E->hasDecorate(DecorationDontStaticallyCoalesceINTEL, 0))
+    LSUParamsBitmask |= IntelFPGAMemoryAccessesVal::DontStaticallyCoalesce;
+  if (E->hasDecorate(DecorationPrefetchINTEL, 0, &Result)) {
+    LSUParamsBitmask |= IntelFPGAMemoryAccessesVal::PrefetchFlag;
+    // TODO: Enable prefetch size backwards translation
+    // once it is supported
+  }
+  if (LSUParamsBitmask == 0)
+    return;
+  Out << "{params:" << LSUParamsBitmask << "}" << AdditionalParamsStr;
 }
 
 void generateIntelFPGAAnnotationForStructMember(
@@ -3104,13 +3194,14 @@ void generateIntelFPGAAnnotationForStructMember(
 }
 
 void SPIRVToLLVM::transIntelFPGADecorations(SPIRVValue *BV, Value *V) {
-  if (!BV->isVariable())
+  if (!BV->isVariable() && !BV->isInst())
     return;
 
-  if (auto AL = dyn_cast<AllocaInst>(V)) {
-    IRBuilder<> Builder(AL->getParent());
+  if (auto *Inst = dyn_cast<Instruction>(V)) {
+    auto *AL = dyn_cast<AllocaInst>(Inst);
+    Type *AllocatedTy = AL ? AL->getAllocatedType() : Inst->getType();
 
-    SPIRVType *ST = BV->getType()->getPointerElementType();
+    IRBuilder<> Builder(Inst->getParent());
 
     Type *Int8PtrTyPrivate = Type::getInt8PtrTy(*Context, SPIRAS_Private);
     IntegerType *Int32Ty = IntegerType::get(*Context, 32);
@@ -3118,7 +3209,8 @@ void SPIRVToLLVM::transIntelFPGADecorations(SPIRVValue *BV, Value *V) {
     Value *UndefInt8Ptr = UndefValue::get(Int8PtrTyPrivate);
     Value *UndefInt32 = UndefValue::get(Int32Ty);
 
-    if (ST->isTypeStruct()) {
+    if (AL && BV->getType()->getPointerElementType()->isTypeStruct()) {
+      auto *ST = BV->getType()->getPointerElementType();
       SPIRVTypeStruct *STS = static_cast<SPIRVTypeStruct *>(ST);
 
       for (SPIRVWord I = 0; I < STS->getMemberCount(); ++I) {
@@ -3127,8 +3219,7 @@ void SPIRVToLLVM::transIntelFPGADecorations(SPIRVValue *BV, Value *V) {
         if (!AnnotStr.empty()) {
           auto *GS = Builder.CreateGlobalStringPtr(AnnotStr);
 
-          auto GEP = Builder.CreateConstInBoundsGEP2_32(AL->getAllocatedType(),
-                                                        AL, 0, I);
+          auto GEP = Builder.CreateConstInBoundsGEP2_32(AllocatedTy, AL, 0, I);
 
           Type *IntTy = GEP->getType()->getPointerElementType()->isIntegerTy()
                             ? GEP->getType()
@@ -3151,13 +3242,18 @@ void SPIRVToLLVM::transIntelFPGADecorations(SPIRVValue *BV, Value *V) {
     if (!AnnotStr.empty()) {
       auto *GS = Builder.CreateGlobalStringPtr(AnnotStr);
 
-      auto AnnotationFn =
-          llvm::Intrinsic::getDeclaration(M, Intrinsic::var_annotation);
+      Value *BaseInst =
+          AL ? Builder.CreateBitCast(V, Int8PtrTyPrivate, V->getName()) : Inst;
 
-      llvm::Value *Args[] = {
-          Builder.CreateBitCast(V, Int8PtrTyPrivate, V->getName()),
-          Builder.CreateBitCast(GS, Int8PtrTyPrivate), UndefInt8Ptr,
-          UndefInt32};
+      auto AnnotationFn =
+          AL // Static memory attributes if true
+              ? llvm::Intrinsic::getDeclaration(M, Intrinsic::var_annotation)
+              : llvm::Intrinsic::getDeclaration(M, Intrinsic::ptr_annotation,
+                                                AllocatedTy);
+
+      llvm::Value *Args[] = {BaseInst,
+                             Builder.CreateBitCast(GS, Int8PtrTyPrivate),
+                             UndefInt8Ptr, UndefInt32};
       Builder.CreateCall(AnnotationFn, Args);
     }
   } else if (auto *GV = dyn_cast<GlobalVariable>(V)) {
@@ -3444,6 +3540,15 @@ bool SPIRVToLLVM::transOCLMetadata(SPIRVFunction *BF) {
                                                         Arg->getName());
                                  });
   }
+  // Generate metadata for kernel_arg_buffer_location
+  addBufferLocationMetadata(Context, BF, F, [=](SPIRVFunctionParameter *Arg) {
+    auto Literals = Arg->getDecorationLiterals(DecorationBufferLocationINTEL);
+    assert(Literals.size() == 1 &&
+           "BufferLocationINTEL decoration shall have 1 ID literal");
+
+    return ConstantAsMetadata::get(
+        ConstantInt::get(Type::getInt32Ty(*Context), Literals[0]));
+  });
   return true;
 }
 
diff --git a/llvm-spirv/lib/SPIRV/SPIRVWriter.cpp b/llvm-spirv/lib/SPIRV/SPIRVWriter.cpp
index 347111bb34454..25988f58ea76d 100644
--- a/llvm-spirv/lib/SPIRV/SPIRVWriter.cpp
+++ b/llvm-spirv/lib/SPIRV/SPIRVWriter.cpp
@@ -527,6 +527,13 @@ SPIRVFunction *LLVMToSPIRV::transFunctionDecl(Function *F) {
     BM->addEntryPoint(ExecutionModelKernel, BF->getId());
   else if (F->getLinkage() != GlobalValue::InternalLinkage)
     BF->setLinkageType(transLinkageType(F));
+
+  // Translate OpenCL/SYCL buffer_location metadata if it's attached to the
+  // translated function declaration
+  MDNode *BufferLocation = nullptr;
+  if (BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_fpga_buffer_location))
+    BufferLocation = ((*F).getMetadata("kernel_arg_buffer_location"));
+
   auto Attrs = F->getAttributes();
 
   for (Function::arg_iterator I = F->arg_begin(), E = F->arg_end(); I != E;
@@ -552,6 +559,15 @@ SPIRVFunction *LLVMToSPIRV::transFunctionDecl(Function *F) {
       BA->addDecorate(DecorationMaxByteOffset,
                       Attrs.getAttribute(ArgNo + 1, Attribute::Dereferenceable)
                           .getDereferenceableBytes());
+    if (BufferLocation && I->getType()->isPointerTy()) {
+      // Order of integer numbers in MD node follows the order of function
+      // parameters on which we shall attach the appropriate decoration. Add
+      // decoration only if MD value is not negative.
+      BM->addCapability(CapabilityFPGABufferLocationINTEL);
+      int LocID = getMDOperandAsInt(BufferLocation, ArgNo);
+      if (LocID >= 0)
+        BA->addDecorate(DecorationBufferLocationINTEL, LocID);
+    }
   }
   if (Attrs.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt))
     BF->addDecorate(DecorationFuncParamAttr, FunctionParameterAttributeZext);
@@ -1667,13 +1683,58 @@ SPIRVValue *LLVMToSPIRV::oclTransSpvcCastSampler(CallInst *CI,
   return BV;
 }
 
-std::vector<std::pair<Decoration, std::string>>
+using DecorationsInfoVec = std::vector<std::pair<Decoration, std::string>>;
+
+struct IntelFPGADecorations {
+  DecorationsInfoVec MemoryAttributesVec;
+  DecorationsInfoVec MemoryAccessesVec;
+};
+
+struct IntelLSUControlsInfo {
+  void setWithBitMask(unsigned ParamsBitMask) {
+    if (ParamsBitMask & IntelFPGAMemoryAccessesVal::BurstCoalesce)
+      BurstCoalesce = true;
+    if (ParamsBitMask & IntelFPGAMemoryAccessesVal::CacheSizeFlag)
+      CacheSizeInfo = 0;
+    if (ParamsBitMask & IntelFPGAMemoryAccessesVal::DontStaticallyCoalesce)
+      DontStaticallyCoalesce = true;
+    if (ParamsBitMask & IntelFPGAMemoryAccessesVal::PrefetchFlag)
+      PrefetchInfo = 0;
+  }
+
+  DecorationsInfoVec getDecorationsFromCurrentState() {
+    DecorationsInfoVec ResultVec;
+    // Simple flags
+    if (BurstCoalesce)
+      ResultVec.emplace_back(DecorationBurstCoalesceINTEL, "");
+    if (DontStaticallyCoalesce)
+      ResultVec.emplace_back(DecorationDontStaticallyCoalesceINTEL, "");
+    // Conditional values
+    if (CacheSizeInfo.hasValue()) {
+      ResultVec.emplace_back(DecorationCacheSizeINTEL,
+                             std::to_string(CacheSizeInfo.getValue()));
+    }
+    if (PrefetchInfo.hasValue()) {
+      ResultVec.emplace_back(DecorationPrefetchINTEL,
+                             std::to_string(PrefetchInfo.getValue()));
+    }
+    return ResultVec;
+  }
+
+  bool BurstCoalesce = false;
+  llvm::Optional<unsigned> CacheSizeInfo;
+  bool DontStaticallyCoalesce = false;
+  llvm::Optional<unsigned> PrefetchInfo;
+};
+
+IntelFPGADecorations
 tryParseIntelFPGAAnnotationString(StringRef AnnotatedCode) {
-  std::vector<std::pair<Decoration, std::string>> Decorates;
+  IntelFPGADecorations Decorates;
+  IntelLSUControlsInfo LSUControls;
 
   // Intel FPGA decorations are separated into
   // {word} OR {word:value,value,...} blocks
-  std::regex DecorationRegex("\\{[\\w:,]+\\}");
+  std::regex DecorationRegex("\\{[\\w:,-]+\\}");
   using RegexIterT = std::regex_iterator<StringRef::const_iterator>;
   RegexIterT DecorationsIt(AnnotatedCode.begin(), AnnotatedCode.end(),
                            DecorationRegex);
@@ -1685,35 +1746,52 @@ tryParseIntelFPGAAnnotationString(StringRef AnnotatedCode) {
     std::pair<StringRef, StringRef> Split = AnnotatedDecoration.split(':');
     StringRef Name = Split.first, ValueStr = Split.second;
 
-    StringRef Annotation;
-    Decoration Dec;
-    if (Name == "pump") {
-      Dec = llvm::StringSwitch<Decoration>(ValueStr)
-                .Case("1", DecorationSinglepumpINTEL)
-                .Case("2", DecorationDoublepumpINTEL);
-    } else if (Name == "register") {
-      Dec = DecorationRegisterINTEL;
-    } else if (Name == "simple_dual_port") {
-      Dec = DecorationSimpleDualPortINTEL;
-    } else {
-      Dec = llvm::StringSwitch<Decoration>(Name)
-                .Case("memory", DecorationMemoryINTEL)
-                .Case("numbanks", DecorationNumbanksINTEL)
-                .Case("bankwidth", DecorationBankwidthINTEL)
-                .Case("private_copies", DecorationMaxPrivateCopiesINTEL)
-                .Case("max_replicates", DecorationMaxReplicatesINTEL)
-                .Case("bank_bits", DecorationBankBitsINTEL)
-                .Case("merge", DecorationMergeINTEL)
-                .Case("force_pow2_depth", DecorationForcePow2DepthINTEL)
-                .Default(DecorationUserSemantic);
-      if (Dec == DecorationUserSemantic)
-        Annotation = AnnotatedDecoration;
-      else
-        Annotation = ValueStr;
+    if (Name == "params") {
+      unsigned ParamsBitMask = 0;
+      bool Failure = ValueStr.getAsInteger(10, ParamsBitMask);
+      assert(!Failure && "Non-integer LSU controls value");
+      (void)Failure;
+      LSUControls.setWithBitMask(ParamsBitMask);
+    } else if (Name == "cache-size") {
+      if (!LSUControls.CacheSizeInfo.hasValue())
+        continue;
+      unsigned CacheSizeValue = 0;
+      bool Failure = ValueStr.getAsInteger(10, CacheSizeValue);
+      assert(!Failure && "Non-integer cache size value");
+      (void)Failure;
+      LSUControls.CacheSizeInfo = CacheSizeValue;
+    } // TODO: Support LSU prefetch size, which currently defaults to 0
+    else /* Memory attributes */ {
+      StringRef Annotation;
+      Decoration Dec;
+      if (Name == "pump") {
+        Dec = llvm::StringSwitch<Decoration>(ValueStr)
+                  .Case("1", DecorationSinglepumpINTEL)
+                  .Case("2", DecorationDoublepumpINTEL);
+      } else if (Name == "register") {
+        Dec = DecorationRegisterINTEL;
+      } else if (Name == "simple_dual_port") {
+        Dec = DecorationSimpleDualPortINTEL;
+      } else {
+        Dec = llvm::StringSwitch<Decoration>(Name)
+                  .Case("memory", DecorationMemoryINTEL)
+                  .Case("numbanks", DecorationNumbanksINTEL)
+                  .Case("bankwidth", DecorationBankwidthINTEL)
+                  .Case("private_copies", DecorationMaxPrivateCopiesINTEL)
+                  .Case("max_replicates", DecorationMaxReplicatesINTEL)
+                  .Case("bank_bits", DecorationBankBitsINTEL)
+                  .Case("merge", DecorationMergeINTEL)
+                  .Case("force_pow2_depth", DecorationForcePow2DepthINTEL)
+                  .Default(DecorationUserSemantic);
+        if (Dec == DecorationUserSemantic)
+          Annotation = AnnotatedDecoration;
+        else
+          Annotation = ValueStr;
+      }
+      Decorates.MemoryAttributesVec.emplace_back(Dec, Annotation.str());
     }
-
-    Decorates.emplace_back(Dec, Annotation.str());
   }
+  Decorates.MemoryAccessesVec = LSUControls.getDecorationsFromCurrentState();
   return Decorates;
 }
 
@@ -1729,11 +1807,11 @@ std::vector<SPIRVWord> getBankBitsFromString(StringRef S) {
   return Bits;
 }
 
-void addIntelFPGADecorations(
-    SPIRVEntry *E,
-    std::vector<std::pair<Decoration, std::string>> &Decorations) {
-  if (!E->getModule()->isAllowedToUseExtension(
-          ExtensionID::SPV_INTEL_fpga_memory_attributes))
+void addIntelFPGADecorations(SPIRVEntry *E, DecorationsInfoVec &Decorations) {
+  SPIRVModule *M = E->getModule();
+  if (!M->isAllowedToUseExtension(
+          ExtensionID::SPV_INTEL_fpga_memory_attributes) &&
+      !M->isAllowedToUseExtension(ExtensionID::SPV_INTEL_fpga_memory_accesses))
     return;
 
   for (const auto &I : Decorations) {
@@ -1763,6 +1841,8 @@ void addIntelFPGADecorations(
     case DecorationSinglepumpINTEL:
     case DecorationDoublepumpINTEL:
     case DecorationSimpleDualPortINTEL:
+    case DecorationBurstCoalesceINTEL:
+    case DecorationDontStaticallyCoalesceINTEL:
       assert(I.second.empty());
       E->addDecorate(I.first);
       break;
@@ -1772,6 +1852,8 @@ void addIntelFPGADecorations(
     // DecorationMaxPrivateCopiesINTEL
     // DecorationMaxReplicatesINTEL
     // DecorationForcePow2DepthINTEL
+    // DecorationCacheSizeINTEL
+    // DecorationPrefetchINTEL
     default:
       SPIRVWord Result = 0;
       StringRef(I.second).getAsInteger(10, Result);
@@ -1781,13 +1863,9 @@ void addIntelFPGADecorations(
   }
 }
 
-void addIntelFPGADecorationsForStructMember(
-    SPIRVEntry *E, SPIRVWord MemberNumber,
-    std::vector<std::pair<Decoration, std::string>> &Decorations) {
-  if (!E->getModule()->isAllowedToUseExtension(
-          ExtensionID::SPV_INTEL_fpga_memory_attributes))
-    return;
-
+void addIntelFPGADecorationsForStructMember(SPIRVEntry *E,
+                                            SPIRVWord MemberNumber,
+                                            DecorationsInfoVec &Decorations) {
   for (const auto &I : Decorations) {
     // Such decoration already exists on a type, skip it
     if (E->hasMemberDecorate(I.first, /*Index=*/0, MemberNumber,
@@ -2064,12 +2142,13 @@ SPIRVValue *LLVMToSPIRV::transIntrinsicInst(IntrinsicInst *II,
     StringRef AnnotationString;
     getConstantStringInfo(C, AnnotationString);
 
-    std::vector<std::pair<Decoration, std::string>> Decorations;
+    DecorationsInfoVec Decorations;
     if (BB->getModule()->isAllowedToUseExtension(
             ExtensionID::SPV_INTEL_fpga_memory_attributes))
       // If it is allowed, let's try to parse annotation string to find
       // IntelFPGA-specific decorations
-      Decorations = tryParseIntelFPGAAnnotationString(AnnotationString);
+      Decorations = tryParseIntelFPGAAnnotationString(AnnotationString)
+                        .MemoryAttributesVec;
 
     // If we didn't find any IntelFPGA-specific decorations, let's add the whole
     // annotation string as UserSemantic Decoration
@@ -2094,26 +2173,49 @@ SPIRVValue *LLVMToSPIRV::transIntrinsicInst(IntrinsicInst *II,
     while (isa<BitCastInst>(AnnotSubj) || isa<AddrSpaceCastInst>(AnnotSubj)) {
       AnnotSubj = cast<CastInst>(AnnotSubj)->getOperand(0);
     }
-    // If the pointer is a GEP, then we have to emit a member decoration
+
+    IntelFPGADecorations Decorations;
+    if (BB->getModule()->isAllowedToUseExtension(
+            ExtensionID::SPV_INTEL_fpga_memory_attributes) ||
+        BB->getModule()->isAllowedToUseExtension(
+            ExtensionID::SPV_INTEL_fpga_memory_accesses))
+      // If it is allowed, let's try to parse annotation string to find
+      // IntelFPGA-specific decorations
+      // TODO: The check is not entirely correct, since it allows usage
+      // of extension A even if extension B is the only one enabled.
+      // With the existing stack, these are always enabled/disabled
+      // simultaneously, however we should find a way to separate the
+      // actions for each individual extension.
+      Decorations = tryParseIntelFPGAAnnotationString(AnnotationString);
+
+    // If the pointer is a GEP, then we have to emit a member decoration for the
+    // GEP-accessed struct, or a memory access decoration for the GEP itself.
     if (auto *GI = dyn_cast<GetElementPtrInst>(AnnotSubj)) {
       auto *Ty = transType(GI->getSourceElementType());
+      auto *ResPtr = transValue(GI, BB);
       SPIRVWord MemberNumber =
           dyn_cast<ConstantInt>(GI->getOperand(2))->getZExtValue();
 
-      std::vector<std::pair<Decoration, std::string>> Decorations;
-      if (BB->getModule()->isAllowedToUseExtension(
-              ExtensionID::SPV_INTEL_fpga_memory_attributes))
-        // If it is allowed, let's try to parse annotation string to find
-        // IntelFPGA-specific decorations
-        Decorations = tryParseIntelFPGAAnnotationString(AnnotationString);
-
       // If we didn't find any IntelFPGA-specific decorations, let's add the
       // whole annotation string as UserSemantic Decoration
-      if (Decorations.empty()) {
+      if (Decorations.MemoryAttributesVec.empty() &&
+          Decorations.MemoryAccessesVec.empty()) {
+        // TODO: Is there a way to detect that the annotation belongs solely
+        // to struct member memory atributes or struct member memory access
+        // controls? This would allow emitting just the necessary decoration.
         Ty->addMemberDecorate(new SPIRVMemberDecorateUserSemanticAttr(
             Ty, MemberNumber, AnnotationString.str()));
+        ResPtr->addDecorate(
+            new SPIRVDecorateUserSemanticAttr(ResPtr, AnnotationString.str()));
       } else {
-        addIntelFPGADecorationsForStructMember(Ty, MemberNumber, Decorations);
+        addIntelFPGADecorationsForStructMember(Ty, MemberNumber,
+                                               Decorations.MemoryAttributesVec);
+        // Apply the LSU parameter decoration to the pointer result of a GEP
+        // to the given struct member (InBoundsPtrAccessChain in SPIR-V).
+        // Decorating the member itself with a MemberDecoration is not feasible,
+        // because multiple accesses to the struct-held memory can require
+        // different LSU parameters.
+        addIntelFPGADecorations(ResPtr, Decorations.MemoryAccessesVec);
       }
       II->replaceAllUsesWith(II->getOperand(0));
     } else {
@@ -2122,11 +2224,23 @@ SPIRVValue *LLVMToSPIRV::transIntrinsicInst(IntrinsicInst *II,
       if (AnnotationString == kOCLBuiltinName::FPGARegIntel) {
         if (BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_fpga_reg))
           return BM->addFPGARegINTELInst(Ty, transValue(BI, BB), BB);
+        return transValue(BI, BB);
+      } else {
+        // Memory accesses to a standalone pointer variable
+        auto *DecSubj = transValue(II->getArgOperand(0), BB);
+        if (Decorations.MemoryAccessesVec.empty())
+          DecSubj->addDecorate(new SPIRVDecorateUserSemanticAttr(
+              DecSubj, AnnotationString.str()));
         else
-          return transValue(BI, BB);
+          // Apply the LSU parameter decoration to the pointer result of an
+          // instruction. Note it's the address to the accessed memory that's
+          // loaded from the original pointer variable, and not the value
+          // accessed by the latter.
+          addIntelFPGADecorations(DecSubj, Decorations.MemoryAccessesVec);
+        II->replaceAllUsesWith(II->getOperand(0));
       }
     }
-    return 0;
+    return nullptr;
   }
   // We can just ignore/drop some intrinsics, like optimizations hint.
   case Intrinsic::invariant_start:
@@ -2331,10 +2445,11 @@ void LLVMToSPIRV::transGlobalAnnotation(GlobalVariable *V) {
     StringRef AnnotationString;
     getConstantStringInfo(GV, AnnotationString);
 
-    std::vector<std::pair<Decoration, std::string>> Decorations;
+    DecorationsInfoVec Decorations;
     if (BM->isAllowedToUseExtension(
             ExtensionID::SPV_INTEL_fpga_memory_attributes))
-      Decorations = tryParseIntelFPGAAnnotationString(AnnotationString);
+      Decorations = tryParseIntelFPGAAnnotationString(AnnotationString)
+                        .MemoryAttributesVec;
 
     // If we didn't find any IntelFPGA-specific decorations, let's
     // add the whole annotation string as UserSemantic Decoration
diff --git a/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVDecorate.h b/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVDecorate.h
index 4f5883083f8e6..ba03b181cf3ca 100644
--- a/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVDecorate.h
+++ b/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVDecorate.h
@@ -155,10 +155,17 @@ class SPIRVDecorate : public SPIRVDecorateGeneric {
     case DecorationBankBitsINTEL:
     case DecorationForcePow2DepthINTEL:
       return getSet(ExtensionID::SPV_INTEL_fpga_memory_attributes);
+    case DecorationBurstCoalesceINTEL:
+    case DecorationCacheSizeINTEL:
+    case DecorationDontStaticallyCoalesceINTEL:
+    case DecorationPrefetchINTEL:
+      return getSet(ExtensionID::SPV_INTEL_fpga_memory_accesses);
     case DecorationReferencedIndirectlyINTEL:
       return getSet(ExtensionID::SPV_INTEL_function_pointers);
     case DecorationIOPipeStorageINTEL:
       return getSet(ExtensionID::SPV_INTEL_io_pipes);
+    case DecorationBufferLocationINTEL:
+      return getSet(ExtensionID::SPV_INTEL_fpga_buffer_location);
     default:
       return SPIRVExtSet();
     }
@@ -255,8 +262,15 @@ class SPIRVMemberDecorate : public SPIRVDecorateGeneric {
     case DecorationBankBitsINTEL:
     case DecorationForcePow2DepthINTEL:
       return getSet(ExtensionID::SPV_INTEL_fpga_memory_attributes);
+    case DecorationBurstCoalesceINTEL:
+    case DecorationCacheSizeINTEL:
+    case DecorationDontStaticallyCoalesceINTEL:
+    case DecorationPrefetchINTEL:
+      return getSet(ExtensionID::SPV_INTEL_fpga_memory_accesses);
     case DecorationIOPipeStorageINTEL:
       return getSet(ExtensionID::SPV_INTEL_io_pipes);
+    case DecorationBufferLocationINTEL:
+      return getSet(ExtensionID::SPV_INTEL_fpga_buffer_location);
     default:
       return SPIRVExtSet();
     }
diff --git a/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVEnum.h b/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVEnum.h
index 078a6ec426376..c35b4f01f1730 100644
--- a/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVEnum.h
+++ b/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVEnum.h
@@ -396,6 +396,14 @@ template <> inline void SPIRVMap<Decoration, SPIRVCapVec>::init() {
   ADD_VEC_INIT(DecorationSIMTCallINTEL, {CapabilityVectorComputeINTEL});
   ADD_VEC_INIT(DecorationFuncParamKindINTEL, {CapabilityVectorComputeINTEL});
   ADD_VEC_INIT(DecorationFuncParamDescINTEL, {CapabilityVectorComputeINTEL});
+  ADD_VEC_INIT(DecorationBurstCoalesceINTEL,
+               {CapabilityFPGAMemoryAccessesINTEL});
+  ADD_VEC_INIT(DecorationCacheSizeINTEL, {CapabilityFPGAMemoryAccessesINTEL});
+  ADD_VEC_INIT(DecorationDontStaticallyCoalesceINTEL,
+               {CapabilityFPGAMemoryAccessesINTEL});
+  ADD_VEC_INIT(DecorationPrefetchINTEL, {CapabilityFPGAMemoryAccessesINTEL});
+  ADD_VEC_INIT(DecorationBufferLocationINTEL,
+               {CapabilityFPGABufferLocationINTEL});
 }
 
 template <> inline void SPIRVMap<BuiltIn, SPIRVCapVec>::init() {
diff --git a/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVIsValidEnum.h b/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVIsValidEnum.h
index 2b96e6c298497..06e7223aa2473 100644
--- a/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVIsValidEnum.h
+++ b/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVIsValidEnum.h
@@ -423,6 +423,10 @@ inline bool isValid(spv::Decoration V) {
   case DecorationDoublepumpINTEL:
   case DecorationBankBitsINTEL:
   case DecorationForcePow2DepthINTEL:
+  case DecorationBurstCoalesceINTEL:
+  case DecorationCacheSizeINTEL:
+  case DecorationDontStaticallyCoalesceINTEL:
+  case DecorationPrefetchINTEL:
   case DecorationReferencedIndirectlyINTEL:
   case DecorationVectorComputeFunctionINTEL:
   case DecorationStackCallINTEL:
@@ -606,6 +610,7 @@ inline bool isValid(spv::Capability V) {
   case CapabilityVectorComputeINTEL:
   case CapabilityVectorAnyINTEL:
   case CapabilityFPGAMemoryAttributesINTEL:
+  case CapabilityFPGAMemoryAccessesINTEL:
   case CapabilityArbitraryPrecisionIntegersINTEL:
   case CapabilityFPGALoopControlsINTEL:
   case CapabilityBlockingPipesINTEL:
diff --git a/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h b/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h
index 5290bdb9bb5a0..c9915382e4fcc 100644
--- a/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h
+++ b/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h
@@ -362,6 +362,10 @@ template <> inline void SPIRVMap<Decoration, std::string>::init() {
   add(DecorationMergeINTEL, "MergeINTEL");
   add(DecorationBankBitsINTEL, "BankBitsINTEL");
   add(DecorationForcePow2DepthINTEL, "ForcePow2DepthINTEL");
+  add(DecorationBurstCoalesceINTEL, "BurstCoalesceINTEL");
+  add(DecorationCacheSizeINTEL, "CacheSizeINTEL");
+  add(DecorationDontStaticallyCoalesceINTEL, "DontStaticallyCoalesceINTEL");
+  add(DecorationPrefetchINTEL, "PrefetchINTEL");
   add(DecorationReferencedIndirectlyINTEL, "ReferencedIndirectlyINTEL");
   add(DecorationIOPipeStorageINTEL, "IOPipeStorageINTEL");
   add(DecorationVectorComputeFunctionINTEL, "VectorComputeFunctionINTEL");
@@ -372,6 +376,7 @@ template <> inline void SPIRVMap<Decoration, std::string>::init() {
   add(DecorationGlobalVariableOffsetINTEL, "GlobalVariableOffsetINTEL");
   add(DecorationFuncParamIOKind, "FuncParamIOKind");
   add(DecorationSIMTCallINTEL, "SIMTCallINTEL");
+  add(DecorationBufferLocationINTEL, "BufferLocationINTEL");
 }
 SPIRV_DEF_NAMEMAP(Decoration, SPIRVDecorationNameMap)
 
@@ -534,6 +539,7 @@ template <> inline void SPIRVMap<Capability, std::string>::init() {
   add(CapabilitySubgroupAvcMotionEstimationChromaINTEL,
       "SubgroupAvcMotionEstimationChromaINTEL");
   add(CapabilityFPGAMemoryAttributesINTEL, "FPGAMemoryAttributesINTEL");
+  add(CapabilityFPGAMemoryAccessesINTEL, "FPGAMemoryAccessesINTEL");
   add(CapabilityArbitraryPrecisionIntegersINTEL,
       "ArbitraryPrecisionIntegersINTEL");
   add(CapabilityFPGALoopControlsINTEL, "FPGALoopControlsINTEL");
@@ -555,6 +561,7 @@ template <> inline void SPIRVMap<Capability, std::string>::init() {
       "GroupNonUniformShuffleRelative");
   add(CapabilityGroupNonUniformClustered, "GroupNonUniformClustered");
   add(CapabilityUSMStorageClassesINTEL, "USMStorageClassesINTEL");
+  add(CapabilityFPGABufferLocationINTEL, "FPGABufferLocationINTEL");
 }
 SPIRV_DEF_NAMEMAP(Capability, SPIRVCapabilityNameMap)
 
diff --git a/llvm-spirv/lib/SPIRV/libSPIRV/spirv.hpp b/llvm-spirv/lib/SPIRV/libSPIRV/spirv.hpp
index 862c45e4f4ae6..53492791c7e7b 100644
--- a/llvm-spirv/lib/SPIRV/libSPIRV/spirv.hpp
+++ b/llvm-spirv/lib/SPIRV/libSPIRV/spirv.hpp
@@ -504,6 +504,11 @@ enum Decoration {
   DecorationMergeINTEL = 5834,
   DecorationBankBitsINTEL = 5835,
   DecorationForcePow2DepthINTEL = 5836,
+  DecorationBurstCoalesceINTEL = 5899,
+  DecorationCacheSizeINTEL = 5900,
+  DecorationDontStaticallyCoalesceINTEL = 5901,
+  DecorationPrefetchINTEL = 5902,
+  DecorationBufferLocationINTEL = 5921,
   DecorationIOPipeStorageINTEL = 5944,
   DecorationMax = 0x7fffffff,
 };
@@ -958,7 +963,9 @@ enum Capability {
   CapabilityFPGARegINTEL = 5948,
   CapabilityKernelAttributesINTEL = 5892,
   CapabilityFPGAKernelAttributesINTEL = 5897,
+  CapabilityFPGABufferLocationINTEL = 5920,
   CapabilityUSMStorageClassesINTEL = 5935,
+  CapabilityFPGAMemoryAccessesINTEL = 5898,
   CapabilityIOPipeINTEL = 5943,
   CapabilityMax = 0x7fffffff,
 };
diff --git a/llvm-spirv/test/DebugInfo/X86/live-debug-variables.ll b/llvm-spirv/test/DebugInfo/X86/live-debug-variables.ll
index c1de99ef9564a..900f71aa579c9 100644
--- a/llvm-spirv/test/DebugInfo/X86/live-debug-variables.ll
+++ b/llvm-spirv/test/DebugInfo/X86/live-debug-variables.ll
@@ -2,7 +2,8 @@
 ; RUN: llvm-spirv %t.bc -o %t.spv
 ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll
 
-; RUN: llc -mtriple=x86_64-linux-gnu -filetype=obj -o - %t.ll | llvm-dwarfdump -debug-loc - | FileCheck %s
+; RUN: llc -mtriple=x86_64-linux-gnu -filetype=obj -o - %t.ll | llvm-dwarfdump -name i4 - \
+; RUN:     | FileCheck %s
 
 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
 target triple = "spir64-unknown-unknown"
@@ -17,11 +18,11 @@ target triple = "spir64-unknown-unknown"
 ; Generated from:
 ;
 ; extern int foobar(int, int, int, int, int);
-; 
+;
 ; int F(int i1, int i2, int i3, int i4, int i5) {
 ;   return foobar(i1, i2, i3, i4, i5);
 ; }
-; 
+;
 ; int foo(int a, int b, int c, int d, int e) {
 ;   return F(a,b,c,d,e) +
 ;          F(a,b,c,d,e) +
@@ -29,12 +30,13 @@ target triple = "spir64-unknown-unknown"
 ;          F(a,b,c,d,e);
 ; }
 
-; CHECK:      .debug_loc contents:
-; CHECK-NEXT: 0x00000000:
-;   We currently emit an entry for the function prologue, too, which could be optimized away.
-; CHECK:              (0x0000000000000018, 0x0000000000000072): DW_OP_reg3 RBX
-;   We should only have one entry inside the function.
-; CHECK-NOT: :
+; Ignore the abstract entry.
+; CHECK: DW_TAG_formal_parameter
+; Check concrete entry has a single location.
+; CHECK:      DW_TAG_formal_parameter
+; CHECK-NEXT:   DW_AT_location (DW_OP_reg3 RBX)
+; CHECK-NEXT:   DW_AT_abstract_origin
+; CHECK-NOT:  DW_TAG_formal_parameter
 
 declare i32 @foobar(i32, i32, i32, i32, i32)
 
diff --git a/llvm-spirv/test/DebugInfo/X86/xray-split-dwarf-interaction.ll b/llvm-spirv/test/DebugInfo/X86/xray-split-dwarf-interaction.ll
deleted file mode 100644
index ff31e06377769..0000000000000
--- a/llvm-spirv/test/DebugInfo/X86/xray-split-dwarf-interaction.ll
+++ /dev/null
@@ -1,90 +0,0 @@
-; RUN: llvm-as < %s -o %t.bc
-; RUN: llvm-spirv %t.bc -o %t.spv
-; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll
-
-; RUN: llc -mtriple=x86_64-pc-linux -split-dwarf-file=input.dwo -O3 \
-; RUN:     -function-sections -data-sections \
-; RUN:     -relocation-model=pic -filetype=asm \
-; RUN:     -generate-type-units -o - %s | \
-; RUN:     FileCheck %s --check-prefix=CHECK-ASM
-; RUN: llc -mtriple=x86_64-pc-linux -split-dwarf-file=input.dwo -O3 \
-; RUN:     -function-sections -data-sections \
-; RUN:     -relocation-model=pic -filetype=obj \
-; RUN:     -generate-type-units -o - %s | \
-; RUN:     llvm-readelf -sections | \
-; RUN:     FileCheck %s --check-prefix=CHECK-ELF
-
-target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
-target triple = "spir64-unknown-unknown"
-; Created from `clang++ -fxray-instrument -gsplit-dwarf -fdebug-types-section
-; -ffunction-sections -fdata-sections -emit-llvm -S input.cc`:
-; input.cc:
-;
-; class a {
-;   int b();
-; };
-; int a::b() {
-;   for (;;)
-;     ;
-; }
-;
-; In this test we want to make sure that the xray_instr_map section for
-; `a::b()` is actually associated with the function's symbol instead of the
-; .debug_types.dwo section.
-;
-; CHECK-ASM: xray_fn_idx,"awo",@progbits,_ZN1a1bEv{{$}}
-;
-; CHECK-ELF-DAG: [[FSECT:[0-9]+]]] .text._ZN1a1bEv PROGBITS
-; CHECK-ELF-DAG: [{{.*}}] .debug_types.dwo PROGBITS
-; CHECK-ELF-DAG: [{{.*}}] xray_instr_map PROGBITS {{.*}} {{.*}} {{.*}} {{.*}} AL [[FSECT]]
-
-%class.a = type { i8 }
-
-; Function Attrs: nounwind readnone uwtable
-define i32 @_ZN1a1bEv(%class.a* nocapture readnone) local_unnamed_addr #0 align 2 !dbg !8 {
-  tail call void @llvm.dbg.value(metadata %class.a* %0, metadata !17, metadata !DIExpression()), !dbg !19
-  br label %2, !dbg !20
-
-; <label>:2:                                      ; preds = %2, %1
-  br label %2, !dbg !21, !llvm.loop !25
-}
-
-
-; Function Attrs: nounwind readnone speculatable
-declare void @llvm.dbg.value(metadata, metadata, metadata) #1
-
-attributes #0 = { nounwind readnone uwtable "xray-instruction-threshold"="200" }
-attributes #1 = { nounwind readnone speculatable }
-
-!llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!3, !4, !5, !6}
-!llvm.ident = !{!7}
-
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version trunk (trunk r312634)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, debugInfoForProfiling: true)
-!1 = !DIFile(filename: "input.cc", directory: "/usr/local/google/home/dberris/tmp")
-!2 = !{}
-!3 = !{i32 2, !"Debug Info Version", i32 3}
-!4 = !{i32 1, !"wchar_size", i32 4}
-!5 = !{i32 7, !"PIC Level", i32 2}
-!6 = !{i32 7, !"PIE Level", i32 2}
-!7 = !{!"clang version trunk (trunk r312634)"}
-!8 = distinct !DISubprogram(name: "b", linkageName: "_ZN1a1bEv", scope: !9, file: !1, line: 4, type: !12, isLocal: false, isDefinition: true, scopeLine: 4, flags: DIFlagPrototyped, isOptimized: true, unit: !0, declaration: !11, retainedNodes: !16)
-!9 = distinct !DICompositeType(tag: DW_TAG_class_type, name: "a", file: !1, line: 1, size: 8, elements: !10, identifier: "_ZTS1a")
-!10 = !{!11}
-!11 = !DISubprogram(name: "b", linkageName: "_ZN1a1bEv", scope: !9, file: !1, line: 2, type: !12, isLocal: false, isDefinition: false, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: true)
-!12 = !DISubroutineType(types: !13)
-!13 = !{!14, !15}
-!14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-!15 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !9, size: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
-!16 = !{!17}
-!17 = !DILocalVariable(name: "this", arg: 1, scope: !8, type: !18, flags: DIFlagArtificial | DIFlagObjectPointer)
-!18 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !9, size: 64)
-!19 = !DILocation(line: 0, scope: !8)
-!20 = !DILocation(line: 5, column: 3, scope: !8)
-!21 = !DILocation(line: 5, column: 3, scope: !22)
-!22 = !DILexicalBlockFile(scope: !23, file: !1, discriminator: 2)
-!23 = distinct !DILexicalBlock(scope: !24, file: !1, line: 5, column: 3)
-!24 = distinct !DILexicalBlock(scope: !8, file: !1, line: 5, column: 3)
-!25 = distinct !{!25, !26, !27}
-!26 = !DILocation(line: 5, column: 3, scope: !24)
-!27 = !DILocation(line: 6, column: 5, scope: !24)
diff --git a/llvm-spirv/test/transcoding/FPGABufferLocation.ll b/llvm-spirv/test/transcoding/FPGABufferLocation.ll
new file mode 100644
index 0000000000000..e649a89625d5e
--- /dev/null
+++ b/llvm-spirv/test/transcoding/FPGABufferLocation.ll
@@ -0,0 +1,61 @@
+; RUN: llvm-as %s -o %t.bc
+; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_INTEL_fpga_buffer_location -o %t.spv
+; RUN: llvm-spirv %t.spv -to-text -o %t.spt
+; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV
+
+; RUN: llvm-spirv -r %t.spv -o %t.rev.bc
+; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM
+
+; RUN: llvm-spirv -spirv-text -r %t.spt -o %t.rev.bc
+; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM
+
+; CHECK-SPIRV: 2 Capability FPGABufferLocationINTEL
+; CHECK-SPIRV: 9 Extension "SPV_INTEL_fpga_buffer_location"
+; CHECK-SPIRV: 3 Name [[ARGA:[0-9]+]] "a"
+; CHECK-SPIRV: 3 Name [[ARGB:[0-9]+]] "b"
+; CHECK-SPIRV: 3 Name [[ARGC:[0-9]+]] "c"
+; CHECK-SPIRV: 3 Name [[ARGD:[0-9]+]] "d"
+; CHECK-SPIRV: 3 Name [[ARGE:[0-9]+]] "e"
+; CHECK-SPIRV-NOT: 4 Decorate [[ARGC]] BufferLocationINTEL -1
+; CHECK-SPIRV-NOT: 4 Decorate [[ARGC]] BufferLocationINTEL -1
+; CHECK-SPIRV: 4 Decorate [[ARGA]] BufferLocationINTEL 1
+; CHECK-SPIRV: 4 Decorate [[ARGB]] BufferLocationINTEL 2
+; CHECK-SPIRV-NOT: 4 Decorate [[ARGD]] BufferLocationINTEL -1
+; CHECK-SPIRV-NOT: 4 Decorate [[ARGE]] BufferLocationINTEL 3
+
+; CHECK-SPIRV: 5 Function
+; CHECK-SPIRV: 3 FunctionParameter {{[0-9]+}} [[ARGA]]
+; CHECK-SPIRV: 3 FunctionParameter {{[0-9]+}} [[ARGB]]
+; CHECK-SPIRV: 3 FunctionParameter {{[0-9]+}} [[ARGC]]
+; CHECK-SPIRV: 3 FunctionParameter {{[0-9]+}} [[ARGD]]
+; CHECK-SPIRV: 3 FunctionParameter {{[0-9]+}} [[ARGE]]
+
+; CHECK-LLVM: define spir_kernel void @test{{.*}} !kernel_arg_buffer_location ![[BUFLOC_MD:[0-9]+]] {{.*}}
+; CHECK-LLVM: ![[BUFLOC_MD]] = !{i32 1, i32 2, i32 -1, i32 -1, i32 -1}
+
+; ModuleID = 'buffer_location.cl'
+source_filename = "buffer_location.cl"
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024"
+target triple = "spir64-unknown-unknown"
+
+; Function Attrs: norecurse nounwind readnone
+define spir_kernel void @test(i32 addrspace(1)* %a, float addrspace(1)* %b, i32 addrspace(1)* %c, i32 %d, i32 %e) local_unnamed_addr !kernel_arg_addr_space !3 !kernel_arg_access_qual !4 !kernel_arg_type !5 !kernel_arg_base_type !5 !kernel_arg_buffer_location !6 {
+entry:
+  ret void
+}
+
+!opencl.enable.FP_CONTRACT = !{}
+!opencl.ocl.version = !{!0}
+!opencl.spir.version = !{!0}
+!opencl.used.extensions = !{!1}
+!opencl.used.optional.core.features = !{!1}
+!opencl.compiler.options = !{!1}
+!llvm.ident = !{!2}
+
+!0 = !{i32 2, i32 0}
+!1 = !{}
+!2 = !{!""}
+!3 = !{i32 1, i32 1, i32 1}
+!4 = !{!"none", !"none", !"none"}
+!5 = !{!"int*", !"float*", !"int*"}
+!6 = !{i32 1, i32 2, i32 -1, i32 -1, i32 3}
diff --git a/llvm-spirv/test/transcoding/IntelFPGAMemoryAccesses.ll b/llvm-spirv/test/transcoding/IntelFPGAMemoryAccesses.ll
new file mode 100644
index 0000000000000..0d1f6424a1818
--- /dev/null
+++ b/llvm-spirv/test/transcoding/IntelFPGAMemoryAccesses.ll
@@ -0,0 +1,253 @@
+; LLVM IR generated by Intel SYCL Clang compiler (https://github.com/intel/llvm)
+; SYCL source code can be found below:
+
+; #define BURST_COAL 0x1
+; #define CACHE_SIZE_FLAG 0x2
+; #define DONT_STATICALLY_COAL 0x4
+; #define PREFETCH 0x8
+; struct State {
+;   float Field1;
+;   int Field2;
+; };
+;
+; void foo(float *A, int *B, State *C) {
+;   float *x;
+;   int *y;
+;   State *z;
+;   double *t;
+;   x = __builtin_intel_fpga_mem(A, BURST_COAL | CACHE_SIZE_FLAG, 0);
+;   y = __builtin_intel_fpga_mem(B, DONT_STATICALLY_COAL | PREFETCH, 0);
+;   z = __builtin_intel_fpga_mem(C, CACHE_SIZE_FLAG, 127);
+;   x = __builtin_intel_fpga_mem(&C->Field1, BURST_COAL | CACHE_SIZE_FLAG, 127);
+;   y = __builtin_intel_fpga_mem(&C->Field2, 0, 127);
+;   z = __builtin_intel_fpga_mem(C, BURST_COAL | CACHE_SIZE_FLAG | DONT_STATICALLY_COAL | PREFETCH, 127);
+;   t = __builtin_intel_fpga_mem((double *) A, BURST_COAL | CACHE_SIZE_FLAG, 0);
+; }
+;
+; template <typename name, typename Func>
+; __attribute__((sycl_kernel)) void kernel_single_task(Func kernelFunc) {
+;   kernelFunc();
+; }
+;
+; int main() {
+;   kernel_single_task<class fake_kernel>([]() {
+;     float *A;
+;     int *B;
+;     State *C;
+;     foo(A, B, C);
+;   });
+;   return 0;
+; }
+
+; RUN: llvm-as %s -o %t.bc
+; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_INTEL_fpga_memory_accesses -o %t.spv
+; RUN: llvm-spirv %t.spv -to-text -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+
+; RUN: llvm-spirv -r %t.spv -o %t.rev.bc
+; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM
+
+; CHECK-SPIRV: Capability FPGAMemoryAccessesINTEL
+; CHECK-SPIRV: Extension "SPV_INTEL_fpga_memory_accesses"
+; CHECK-SPIRV: Decorate {{[0-9]+}} BurstCoalesceINTEL
+; CHECK-SPIRV: Decorate {{[0-9]+}} CacheSizeINTEL 0
+; CHECK-SPIRV: Decorate {{[0-9]+}} CacheSizeINTEL 127
+; CHECK-SPIRV: Decorate {{[0-9]+}} DontStaticallyCoalesceINTEL
+; CHECK-SPIRV: Decorate {{[0-9]+}} PrefetchINTEL 0
+; Check that the semantically meaningless decoration was
+; translated as a mere annotation
+; CHECK-SPIRV: Decorate {{[0-9]+}} UserSemantic "{params:0}{cache-size:127}"
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
+target triple = "spir64-unknown-unknown-sycldevice"
+
+%"class._ZTSZ4mainE3$_0.anon" = type { i8 }
+%struct._ZTS5State.State = type { float, i32 }
+
+; CHECK-LLVM: [[PARAM_3_CACHE_0:@[a-z0-9_.]+]] = {{.*}}{params:3}{cache-size:0}
+@.str = private unnamed_addr constant [25 x i8] c"{params:3}{cache-size:0}\00", section "llvm.metadata"
+@.str.1 = private unnamed_addr constant [14 x i8] c"<invalid loc>\00", section "llvm.metadata"
+; "params" bitmask doesn't hold 0x2 (cache size ON), so cache-size can be dropped
+; during translation
+; CHECK-LLVM: [[PARAM_12_CACHE_0:@[a-z0-9_.]+]] = {{.*}}{params:12}
+@.str.2 = private unnamed_addr constant [26 x i8] c"{params:12}{cache-size:0}\00", section "llvm.metadata"
+; CHECK-LLVM: [[PARAM_2_CACHE_127:@[a-z0-9_.]+]] = {{.*}}{params:2}{cache-size:127}
+@.str.3 = private unnamed_addr constant [27 x i8] c"{params:2}{cache-size:127}\00", section "llvm.metadata"
+; CHECK-LLVM: [[PARAM_3_CACHE_127:@[a-z0-9_.]+]] = {{.*}}{params:3}{cache-size:127}
+@.str.4 = private unnamed_addr constant [27 x i8] c"{params:3}{cache-size:127}\00", section "llvm.metadata"
+; Since "params" bitmask is set to 0, the next string isn't required to be preserved
+; during translation. Neither is the corresponding pointer annotation intrinsic.
+@.str.5 = private unnamed_addr constant [27 x i8] c"{params:0}{cache-size:127}\00", section "llvm.metadata"
+; CHECK-LLVM: [[PARAM_15_CACHE_127:@[a-z0-9_.]+]] = {{.*}}{params:15}{cache-size:127}
+@.str.6 = private unnamed_addr constant [28 x i8] c"{params:15}{cache-size:127}\00", section "llvm.metadata"
+; TODO: Investigate why the same global annotation string shows up twice in backwards translation.
+; CHECK-LLVM: [[PARAM_3_CACHE_0_DOUBLE:@[a-z0-9_.]+]] = {{.*}}{params:3}{cache-size:0}
+
+; Function Attrs: norecurse nounwind
+define spir_kernel void @_ZTSZ4mainE11fake_kernel() #0 !kernel_arg_addr_space !4 !kernel_arg_access_qual !4 !kernel_arg_type !4 !kernel_arg_base_type !4 !kernel_arg_type_qual !4 {
+entry:
+  %0 = alloca %"class._ZTSZ4mainE3$_0.anon", align 1
+  %1 = bitcast %"class._ZTSZ4mainE3$_0.anon"* %0 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %1) #5
+  %2 = addrspacecast %"class._ZTSZ4mainE3$_0.anon"* %0 to %"class._ZTSZ4mainE3$_0.anon" addrspace(4)*
+  call spir_func void @"_ZZ4mainENK3$_0clEv"(%"class._ZTSZ4mainE3$_0.anon" addrspace(4)* %2)
+  %3 = bitcast %"class._ZTSZ4mainE3$_0.anon"* %0 to i8*
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %3) #5
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+; Function Attrs: inlinehint norecurse nounwind
+define internal spir_func void @"_ZZ4mainENK3$_0clEv"(%"class._ZTSZ4mainE3$_0.anon" addrspace(4)* %this) #2 align 2 {
+entry:
+  %this.addr = alloca %"class._ZTSZ4mainE3$_0.anon" addrspace(4)*, align 8
+  %A = alloca float addrspace(4)*, align 8
+  %B = alloca i32 addrspace(4)*, align 8
+  %C = alloca %struct._ZTS5State.State addrspace(4)*, align 8
+  store %"class._ZTSZ4mainE3$_0.anon" addrspace(4)* %this, %"class._ZTSZ4mainE3$_0.anon" addrspace(4)** %this.addr, align 8, !tbaa !5
+  %0 = bitcast float addrspace(4)** %A to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8, i8* %0) #5
+  %1 = bitcast i32 addrspace(4)** %B to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8, i8* %1) #5
+  %2 = bitcast %struct._ZTS5State.State addrspace(4)** %C to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8, i8* %2) #5
+  %3 = load float addrspace(4)*, float addrspace(4)** %A, align 8, !tbaa !5
+  %4 = load i32 addrspace(4)*, i32 addrspace(4)** %B, align 8, !tbaa !5
+  %5 = load %struct._ZTS5State.State addrspace(4)*, %struct._ZTS5State.State addrspace(4)** %C, align 8, !tbaa !5
+  call spir_func void @_Z3fooPfPiP5State(float addrspace(4)* %3, i32 addrspace(4)* %4, %struct._ZTS5State.State addrspace(4)* %5)
+  %6 = bitcast %struct._ZTS5State.State addrspace(4)** %C to i8*
+  call void @llvm.lifetime.end.p0i8(i64 8, i8* %6) #5
+  %7 = bitcast i32 addrspace(4)** %B to i8*
+  call void @llvm.lifetime.end.p0i8(i64 8, i8* %7) #5
+  %8 = bitcast float addrspace(4)** %A to i8*
+  call void @llvm.lifetime.end.p0i8(i64 8, i8* %8) #5
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
+
+; CHECK-LLVM: define spir_func void @{{.*}}foo
+; Function Attrs: norecurse nounwind
+define spir_func void @_Z3fooPfPiP5State(float addrspace(4)* %A, i32 addrspace(4)* %B, %struct._ZTS5State.State addrspace(4)* %C) #3 {
+entry:
+; CHECK-LLVM: %[[FLOAT_FUNC_PARAM:[[:alnum:].]+]] = alloca float addrspace(4)*, align 8
+; CHECK-LLVM: %[[INT_FUNC_PARAM:[[:alnum:].]+]] = alloca i32 addrspace(4)*, align 8
+; CHECK-LLVM: %[[STRUCT_FUNC_PARAM:[[:alnum:].]+]] = alloca %struct{{.*}}State addrspace(4)*, align 8
+  %A.addr = alloca float addrspace(4)*, align 8
+  %B.addr = alloca i32 addrspace(4)*, align 8
+  %C.addr = alloca %struct._ZTS5State.State addrspace(4)*, align 8
+; CHECK-LLVM: %[[FLOAT_VAR:[[:alnum:].]+]] = alloca float addrspace(4)*, align 8
+; CHECK-LLVM: %[[INT_VAR:[[:alnum:].]+]] = alloca i32 addrspace(4)*, align 8
+; CHECK-LLVM: %[[STRUCT_VAR:[[:alnum:].]+]] = alloca %struct{{.*}}State addrspace(4)*, align 8
+; CHECK-LLVM: %[[DOUBLE_VAR:[[:alnum:].]+]] = alloca double addrspace(4)*, align 8
+  %x = alloca float addrspace(4)*, align 8
+  %y = alloca i32 addrspace(4)*, align 8
+  %z = alloca %struct._ZTS5State.State addrspace(4)*, align 8
+  %t = alloca double addrspace(4)*, align 8
+  store float addrspace(4)* %A, float addrspace(4)** %A.addr, align 8, !tbaa !5
+  store i32 addrspace(4)* %B, i32 addrspace(4)** %B.addr, align 8, !tbaa !5
+  store %struct._ZTS5State.State addrspace(4)* %C, %struct._ZTS5State.State addrspace(4)** %C.addr, align 8, !tbaa !5
+  %0 = bitcast float addrspace(4)** %x to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8, i8* %0) #5
+  %1 = bitcast i32 addrspace(4)** %y to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8, i8* %1) #5
+  %2 = bitcast %struct._ZTS5State.State addrspace(4)** %z to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8, i8* %2) #5
+; CHECK-LLVM: %[[FLOAT_FUNC_PARAM_LOAD:[[:alnum:].]+]] = load float addrspace(4)*, float addrspace(4)** %[[FLOAT_FUNC_PARAM]]
+; CHECK-LLVM: %[[INTRINSIC_CALL:[[:alnum:].]+]] = call float addrspace(4)* @llvm.ptr.annotation.p4f32(float addrspace(4)* %[[FLOAT_FUNC_PARAM_LOAD]], i8* getelementptr inbounds ({{.*}} [[PARAM_3_CACHE_0]]
+; CHECK-LLVM: store float addrspace(4)* %[[INTRINSIC_CALL]], float addrspace(4)** %[[FLOAT_VAR]]
+  %3 = load float addrspace(4)*, float addrspace(4)** %A.addr, align 8, !tbaa !5
+  %4 = call float addrspace(4)* @llvm.ptr.annotation.p4f32(float addrspace(4)* %3, i8* getelementptr inbounds ([25 x i8], [25 x i8]* @.str, i32 0, i32 0), i8* getelementptr inbounds ([14 x i8], [14 x i8]* @.str.1, i32 0, i32 0), i32 0) #6
+  store float addrspace(4)* %4, float addrspace(4)** %x, align 8, !tbaa !5
+; CHECK-LLVM: %[[INT_FUNC_PARAM_LOAD:[[:alnum:].]+]] = load i32 addrspace(4)*, i32 addrspace(4)** %[[INT_FUNC_PARAM]]
+; CHECK-LLVM: %[[INTRINSIC_CALL:[[:alnum:].]+]] = call i32 addrspace(4)* @llvm.ptr.annotation.p4i32(i32 addrspace(4)* %[[INT_FUNC_PARAM_LOAD]], i8* getelementptr inbounds ({{.*}} [[PARAM_12_CACHE_0]]
+; CHECK-LLVM: store i32 addrspace(4)* %[[INTRINSIC_CALL]], i32 addrspace(4)** %[[INT_VAR]]
+  %5 = load i32 addrspace(4)*, i32 addrspace(4)** %B.addr, align 8, !tbaa !5
+  %6 = call i32 addrspace(4)* @llvm.ptr.annotation.p4i32(i32 addrspace(4)* %5, i8* getelementptr inbounds ([26 x i8], [26 x i8]* @.str.2, i32 0, i32 0), i8* getelementptr inbounds ([14 x i8], [14 x i8]* @.str.1, i32 0, i32 0), i32 0) #6
+  store i32 addrspace(4)* %6, i32 addrspace(4)** %y, align 8, !tbaa !5
+; CHECK-LLVM: %[[WHOLE_STRUCT_LOAD:[0-9]+]] = [[WHOLE_STRUCT_LOAD_INST:load\ %struct.*State\ addrspace\(4\)\*,\ %struct.*State\ addrspace\(4\)\*\*.*]][[STRUCT_FUNC_PARAM]]
+; CHECK-LLVM: %[[INTRINSIC_CALL:[[:alnum:].]+]] = call %struct{{.*}}State addrspace(4)* @llvm.ptr.annotation.p4s_struct{{.*}}States(%struct{{.*}}State addrspace(4)* %[[WHOLE_STRUCT_LOAD]], i8* getelementptr inbounds ({{.*}} [[PARAM_2_CACHE_127]]
+; CHECK-LLVM: store %struct{{.*}}State addrspace(4)* %[[INTRINSIC_CALL]], %struct{{.*}}State addrspace(4)** %[[STRUCT_VAR]]
+  %7 = load %struct._ZTS5State.State addrspace(4)*, %struct._ZTS5State.State addrspace(4)** %C.addr, align 8, !tbaa !5
+  %8 = call %struct._ZTS5State.State addrspace(4)* @llvm.ptr.annotation.p4s_struct._ZTS5State.States(%struct._ZTS5State.State addrspace(4)* %7, i8* getelementptr inbounds ([27 x i8], [27 x i8]* @.str.3, i32 0, i32 0), i8* getelementptr inbounds ([14 x i8], [14 x i8]* @.str.1, i32 0, i32 0), i32 0) #6
+  store %struct._ZTS5State.State addrspace(4)* %8, %struct._ZTS5State.State addrspace(4)** %z, align 8, !tbaa !5
+; CHECK-LLVM: %[[WHOLE_STRUCT_LOAD_FOR_FLOAT:[0-9]+]] = [[WHOLE_STRUCT_LOAD_INST]][[STRUCT_FUNC_PARAM]]
+; CHECK-LLVM: %[[FLOAT_FIELD_GEP:[[:alnum:].]+]] = getelementptr inbounds %struct{{.*}}State, %struct{{.*}}State addrspace(4)* %[[WHOLE_STRUCT_LOAD_FOR_FLOAT]], i32 0, i32 0
+; CHECK-LLVM: %[[INTRINSIC_CALL:[[:alnum:].]+]] = call float addrspace(4)* @llvm.ptr.annotation.p4f32(float addrspace(4)* %[[FLOAT_FIELD_GEP]], i8* getelementptr inbounds ({{.*}} [[PARAM_3_CACHE_127]]
+; CHECK-LLVM: store float addrspace(4)* %[[INTRINSIC_CALL]], float addrspace(4)** %[[FLOAT_VAR]]
+  %9 = load %struct._ZTS5State.State addrspace(4)*, %struct._ZTS5State.State addrspace(4)** %C.addr, align 8, !tbaa !5
+  %Field1 = getelementptr inbounds %struct._ZTS5State.State, %struct._ZTS5State.State addrspace(4)* %9, i32 0, i32 0
+  %10 = call float addrspace(4)* @llvm.ptr.annotation.p4f32(float addrspace(4)* %Field1, i8* getelementptr inbounds ([27 x i8], [27 x i8]* @.str.4, i32 0, i32 0), i8* getelementptr inbounds ([14 x i8], [14 x i8]* @.str.1, i32 0, i32 0), i32 0) #6
+  store float addrspace(4)* %10, float addrspace(4)** %x, align 8, !tbaa !5
+; CHECK-LLVM: %[[WHOLE_STRUCT_LOAD_FOR_INT:[0-9]+]] = [[WHOLE_STRUCT_LOAD_INST]][[STRUCT_FUNC_PARAM]]
+; CHECK-LLVM: %[[INT_FIELD_GEP:[[:alnum:].]+]] = getelementptr inbounds %struct{{.*}}State, %struct{{.*}}State addrspace(4)* %[[WHOLE_STRUCT_LOAD_FOR_INT]], i32 0, i32 1
+; The annotation for the succeeding intrinsic isn't required to be preserved
+; during translation
+; CHECK-LLVM: store i32 addrspace(4)* %{{.*}}, i32 addrspace(4)** %[[INT_VAR]]
+  %11 = load %struct._ZTS5State.State addrspace(4)*, %struct._ZTS5State.State addrspace(4)** %C.addr, align 8, !tbaa !5
+  %Field2 = getelementptr inbounds %struct._ZTS5State.State, %struct._ZTS5State.State addrspace(4)* %11, i32 0, i32 1
+  %12 = call i32 addrspace(4)* @llvm.ptr.annotation.p4i32(i32 addrspace(4)* %Field2, i8* getelementptr inbounds ([27 x i8], [27 x i8]* @.str.5, i32 0, i32 0), i8* getelementptr inbounds ([14 x i8], [14 x i8]* @.str.1, i32 0, i32 0), i32 0) #6
+  store i32 addrspace(4)* %12, i32 addrspace(4)** %y, align 8, !tbaa !5
+; CHECK-LLVM: %[[WHOLE_STRUCT_LOAD:[0-9]+]] = [[WHOLE_STRUCT_LOAD_INST]][[STRUCT_FUNC_PARAM]]
+; CHECK-LLVM: %[[INTRINSIC_CALL:[[:alnum:].]+]] = call %struct{{.*}}State addrspace(4)* @llvm.ptr.annotation.p4s_struct{{.*}}States(%struct{{.*}}State addrspace(4)* %[[WHOLE_STRUCT_LOAD]], i8* getelementptr inbounds ({{.*}} [[PARAM_15_CACHE_127]]
+; CHECK-LLVM: store %struct{{.*}}State addrspace(4)* %[[INTRINSIC_CALL]], %struct{{.*}}State addrspace(4)** %[[STRUCT_VAR]]
+  %13 = load %struct._ZTS5State.State addrspace(4)*, %struct._ZTS5State.State addrspace(4)** %C.addr, align 8, !tbaa !5
+  %14 = call %struct._ZTS5State.State addrspace(4)* @llvm.ptr.annotation.p4s_struct._ZTS5State.States(%struct._ZTS5State.State addrspace(4)* %13, i8* getelementptr inbounds ([28 x i8], [28 x i8]* @.str.6, i32 0, i32 0), i8* getelementptr inbounds ([14 x i8], [14 x i8]* @.str.1, i32 0, i32 0), i32 0) #6
+  store %struct._ZTS5State.State addrspace(4)* %14, %struct._ZTS5State.State addrspace(4)** %z, align 8, !tbaa !5
+  %15 = bitcast double addrspace(4)** %t to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8, i8* %15) #5
+; CHECK-LLVM: %[[FLOAT_FUNC_PARAM_LOAD:[[:alnum:].]+]] = load float addrspace(4)*, float addrspace(4)** %[[FLOAT_FUNC_PARAM]]
+; CHECK-LLVM: %[[BITCAST_FLOAT_TO_DOUBLE:[[:alnum:].]+]] = bitcast float addrspace(4)* %[[FLOAT_FUNC_PARAM_LOAD]] to double addrspace(4)*
+; CHECK-LLVM: %[[INTRINSIC_CALL:[[:alnum:].]+]] = call double addrspace(4)* @llvm.ptr.annotation.p4f64(double addrspace(4)* %[[BITCAST_FLOAT_TO_DOUBLE]], i8* getelementptr inbounds ({{.*}} [[PARAM_3_CACHE_0_DOUBLE]]
+; CHECK-LLVM: store double addrspace(4)* %[[INTRINSIC_CALL]], double addrspace(4)** %[[DOUBLE_VAR]]
+  %16 = load float addrspace(4)*, float addrspace(4)** %A.addr, align 8, !tbaa !5
+  %17 = bitcast float addrspace(4)* %16 to double addrspace(4)*
+  %18 = call double addrspace(4)* @llvm.ptr.annotation.p4f64(double addrspace(4)* %17, i8* getelementptr inbounds ([25 x i8], [25 x i8]* @.str, i32 0, i32 0), i8* getelementptr inbounds ([14 x i8], [14 x i8]* @.str.1, i32 0, i32 0), i32 0) #6
+  store double addrspace(4)* %18, double addrspace(4)** %t, align 8, !tbaa !5
+  %19 = bitcast double addrspace(4)** %t to i8*
+  call void @llvm.lifetime.end.p0i8(i64 8, i8* %19) #5
+  %20 = bitcast %struct._ZTS5State.State addrspace(4)** %z to i8*
+  call void @llvm.lifetime.end.p0i8(i64 8, i8* %20) #5
+  %21 = bitcast i32 addrspace(4)** %y to i8*
+  call void @llvm.lifetime.end.p0i8(i64 8, i8* %21) #5
+  %22 = bitcast float addrspace(4)** %x to i8*
+  call void @llvm.lifetime.end.p0i8(i64 8, i8* %22) #5
+  ret void
+}
+
+; Function Attrs: nounwind willreturn
+declare float addrspace(4)* @llvm.ptr.annotation.p4f32(float addrspace(4)*, i8*, i8*, i32) #4
+
+; Function Attrs: nounwind willreturn
+declare i32 addrspace(4)* @llvm.ptr.annotation.p4i32(i32 addrspace(4)*, i8*, i8*, i32) #4
+
+; Function Attrs: nounwind willreturn
+declare %struct._ZTS5State.State addrspace(4)* @llvm.ptr.annotation.p4s_struct._ZTS5State.States(%struct._ZTS5State.State addrspace(4)*, i8*, i8*, i32) #4
+
+; Function Attrs: nounwind willreturn
+declare double addrspace(4)* @llvm.ptr.annotation.p4f64(double addrspace(4)*, i8*, i8*, i32) #4
+
+attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "sycl-module-id"="/tmp/lsu.cpp" "uniform-work-group-size"="true" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind willreturn }
+attributes #2 = { inlinehint norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { nounwind willreturn }
+attributes #5 = { nounwind }
+attributes #6 = { readnone }
+
+!llvm.module.flags = !{!0}
+!opencl.spir.version = !{!1}
+!spirv.Source = !{!2}
+!llvm.ident = !{!3}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 1, i32 2}
+!2 = !{i32 4, i32 100000}
+!3 = !{!"clang version 11.0.0"}
+!4 = !{}
+!5 = !{!6, !6, i64 0}
+!6 = !{!"any pointer", !7, i64 0}
+!7 = !{!"omnipotent char", !8, i64 0}
+!8 = !{!"Simple C++ TBAA"}
diff --git a/llvm-spirv/test/transcoding/builtin_vars_opt.ll b/llvm-spirv/test/transcoding/builtin_vars_opt.ll
new file mode 100644
index 0000000000000..7cc4ba5bdd106
--- /dev/null
+++ b/llvm-spirv/test/transcoding/builtin_vars_opt.ll
@@ -0,0 +1,119 @@
+; RUN: llvm-as < %s -o %t.bc
+; RUN: llvm-spirv %t.bc -o %t.spv
+; RUN: spirv-val %t.spv
+; RUN: llvm-spirv %t.spv -to-text -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: llvm-spirv %t.spv -r -o %t.rev.bc
+; RUN: llvm-dis %t.rev.bc -o - | FileCheck %s --check-prefix=CHECK-LLVM
+
+; The IR was generated from the following source:
+; #include <CL/sycl.hpp>
+;
+; template <typename T, int N>
+; class sycl_subgr;
+;
+; using namespace cl::sycl;
+;
+; int main() {
+;   queue Queue;
+;   int X = 8;
+;   nd_range<1> NdRange(X, X);
+;   buffer<int> buf(X);
+;   Queue.submit([&](handler &cgh) {
+;     auto acc = buf.template get_access<access::mode::read_write>(cgh);
+;     cgh.parallel_for<sycl_subgr<int, 0>>(NdRange, [=](nd_item<1> NdItem) {
+;       intel::sub_group SG = NdItem.get_sub_group();
+;       if (X % 2) {
+;         acc[0] = SG.get_max_local_range()[0];
+;       }
+;       acc[1] = (X % 3) ? 1 : SG.get_max_local_range()[0];
+;     });
+;   });
+;   return 0;
+; }
+; Command line:
+; clang -fsycl -fsycl-device-only -Xclang -fsycl-enable-optimizations tmp.cpp -o tmp.bc
+; llvm-spirv tmp.bc -s -o builtin_vars_opt.ll
+
+; CHECK-SPIRV: Decorate [[#SG_MaxSize_BI:]] BuiltIn 37
+; CHECK-SPIRV: Decorate [[#SG_MaxSize_BI:]] Constant
+; CHECK-SPIRV: Decorate [[#SG_MaxSize_BI:]] LinkageAttributes "__spirv_BuiltInSubgroupMaxSize" Import
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
+target triple = "spir64-unknown-linux-sycldevice"
+
+%"class._ZTSN2cl4sycl5rangeILi1EEE.cl::sycl::range" = type { %"class._ZTSN2cl4sycl6detail5arrayILi1EEE.cl::sycl::detail::array" }
+%"class._ZTSN2cl4sycl6detail5arrayILi1EEE.cl::sycl::detail::array" = type { [1 x i64] }
+%"class._ZTSN2cl4sycl2idILi1EEE.cl::sycl::id" = type { %"class._ZTSN2cl4sycl6detail5arrayILi1EEE.cl::sycl::detail::array" }
+
+$_ZTS10sycl_subgrIiLi0EE = comdat any
+
+; CHECK-LLVM-NOT: @__spirv_BuiltInSubgroupMaxSize
+@__spirv_BuiltInSubgroupMaxSize = external dso_local local_unnamed_addr addrspace(1) constant i32, align 4
+
+
+; Function Attrs: norecurse
+define weak_odr dso_local spir_kernel void @_ZTS10sycl_subgrIiLi0EE(i32 %_arg_, i32 addrspace(1)* %_arg_1, %"class._ZTSN2cl4sycl5rangeILi1EEE.cl::sycl::range"* byval(%"class._ZTSN2cl4sycl5rangeILi1EEE.cl::sycl::range") align 8 %_arg_3, %"class._ZTSN2cl4sycl5rangeILi1EEE.cl::sycl::range"* byval(%"class._ZTSN2cl4sycl5rangeILi1EEE.cl::sycl::range") align 8 %_arg_4, %"class._ZTSN2cl4sycl2idILi1EEE.cl::sycl::id"* byval(%"class._ZTSN2cl4sycl2idILi1EEE.cl::sycl::id") align 8 %_arg_5) local_unnamed_addr #0 comdat !kernel_arg_addr_space !4 !kernel_arg_access_qual !5 !kernel_arg_type !6 !kernel_arg_base_type !6 !kernel_arg_type_qual !7 {
+entry:
+  %0 = getelementptr inbounds %"class._ZTSN2cl4sycl2idILi1EEE.cl::sycl::id", %"class._ZTSN2cl4sycl2idILi1EEE.cl::sycl::id"* %_arg_5, i64 0, i32 0, i32 0, i64 0
+  %1 = load i64, i64* %0, align 8
+  %add.ptr.i = getelementptr inbounds i32, i32 addrspace(1)* %_arg_1, i64 %1
+  %2 = and i32 %_arg_, 1
+  %tobool.not.i = icmp eq i32 %2, 0
+; CHECK-LLVM-NOT: addrspacecast i32 addrspace(1)* @__spirv_BuiltInSubgroupMaxSize to i32 addrspace(4)*
+  %3 = addrspacecast i32 addrspace(1)* @__spirv_BuiltInSubgroupMaxSize to i32 addrspace(4)*
+  br i1 %tobool.not.i, label %if.end.i, label %if.then.i
+
+if.then.i:                                        ; preds = %entry
+; CHECK-LLVM: if.then.i
+; CHECK-LLVM-NOT: load
+; CHECK-LLVM: call spir_func i32 @_Z22get_max_sub_group_sizev()
+  %4 = load i32, i32 addrspace(4)* %3, align 4, !noalias !8
+  %ptridx.ascast.i14.i = addrspacecast i32 addrspace(1)* %add.ptr.i to i32 addrspace(4)*
+  store i32 %4, i32 addrspace(4)* %ptridx.ascast.i14.i, align 4
+  br label %if.end.i
+
+if.end.i:                                         ; preds = %if.then.i, %entry
+  %rem3.i = srem i32 %_arg_, 3
+  %tobool4.not.i = icmp eq i32 %rem3.i, 0
+  br i1 %tobool4.not.i, label %cond.false.i, label %"_ZZZ4mainENK3$_0clERN2cl4sycl7handlerEENKUlNS1_7nd_itemILi1EEEE_clES5_.exit"
+
+cond.false.i:                                     ; preds = %if.end.i
+; CHECK-LLVM: cond.false.i:
+; CHECK-LLVM-NOT: load
+; CHECK-LLVM: call spir_func i32 @_Z22get_max_sub_group_sizev()
+  %5 = load i32, i32 addrspace(4)* %3, align 4, !noalias !11
+  br label %"_ZZZ4mainENK3$_0clERN2cl4sycl7handlerEENKUlNS1_7nd_itemILi1EEEE_clES5_.exit"
+
+"_ZZZ4mainENK3$_0clERN2cl4sycl7handlerEENKUlNS1_7nd_itemILi1EEEE_clES5_.exit": ; preds = %cond.false.i, %if.end.i
+  %cond.i = phi i32 [ %5, %cond.false.i ], [ 1, %if.end.i ]
+  %ptridx.i.i = getelementptr inbounds i32, i32 addrspace(1)* %add.ptr.i, i64 1
+  %ptridx.ascast.i.i = addrspacecast i32 addrspace(1)* %ptridx.i.i to i32 addrspace(4)*
+  store i32 %cond.i, i32 addrspace(4)* %ptridx.ascast.i.i, align 4
+  ret void
+}
+
+attributes #0 = { norecurse "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "sycl-module-id"="../sycl/test/sub_group/shuffle.cpp" "uniform-work-group-size"="true" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.module.flags = !{!0}
+!opencl.spir.version = !{!1}
+!spirv.Source = !{!2}
+!llvm.ident = !{!3}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 1, i32 2}
+!2 = !{i32 4, i32 100000}
+!3 = !{!"clang version 12.0.0 (https://github.com/intel/llvm.git e5ea144e480c7b28162a1477b3d462cfc221ff61)"}
+!4 = !{i32 0, i32 1, i32 0, i32 0, i32 0}
+!5 = !{!"none", !"none", !"none", !"none", !"none"}
+!6 = !{!"int", !"int*", !"cl::sycl::range<1>", !"cl::sycl::range<1>", !"cl::sycl::id<1>"}
+!7 = !{!"", !"", !"", !"", !""}
+!8 = !{!9, !9, i64 0}
+!9 = !{!"int", !10, i64 0}
+!10 = !{!"omnipotent char", !11, i64 0}
+!11 = !{!"Simple C++ TBAA"}
+!12 = !{!13}
+!13 = distinct !{!13, !14, !"_ZNK2cl4sycl5intel9sub_group19get_max_local_rangeEv: %agg.result"}
+!14 = distinct !{!14, !"_ZNK2cl4sycl5intel9sub_group19get_max_local_rangeEv"}
+!15 = !{!16}
+!16 = distinct !{!16, !17, !"_ZNK2cl4sycl5intel9sub_group19get_max_local_rangeEv: %agg.result"}
+!17 = distinct !{!17, !"_ZNK2cl4sycl5intel9sub_group19get_max_local_rangeEv"}
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
index 303b39221d9e6..4f92cd8432a6f 100644
--- a/llvm/CMakeLists.txt
+++ b/llvm/CMakeLists.txt
@@ -1,12 +1,6 @@
 # See docs/CMake.html for instructions about how to build LLVM with CMake.
 
-cmake_minimum_required(VERSION 3.4.3)
-
-if ("${CMAKE_VERSION}" VERSION_LESS "3.13.4")
-  message(FATAL_ERROR
-    "Your CMake version is ${CMAKE_VERSION}. The minimum version of CMake "
-    "required to build LLVM is now 3.13.4.")
-endif()
+cmake_minimum_required(VERSION 3.14)
 
 if(POLICY CMP0068)
   cmake_policy(SET CMP0068 NEW)
@@ -845,6 +839,29 @@ if (tensorflow_c_api)
   include_directories(${TENSORFLOW_C_LIB_PATH}/include)
 endif()
 
+# They are not referenced. See set_output_directory().
+set( CMAKE_RUNTIME_OUTPUT_DIRECTORY ${LLVM_BINARY_DIR}/bin )
+set( CMAKE_LIBRARY_OUTPUT_DIRECTORY ${LLVM_BINARY_DIR}/lib${LLVM_LIBDIR_SUFFIX} )
+set( CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${LLVM_BINARY_DIR}/lib${LLVM_LIBDIR_SUFFIX} )
+
+# For up-to-date instructions for installing the Tensorflow dependency, refer to
+# the bot setup script: https://github.com/google/ml-compiler-opt/blob/master/buildbot/buildbot_init.sh
+# Specifically, assuming python3 is installed:
+# python3 -m pip install --upgrade pip && python3 -m pip install --user tf_nightly==2.3.0.dev20200528
+# Then set TENSORFLOW_AOT_PATH to the package install - usually it's ~/.local/lib/python3.7/site-packages/tensorflow
+#
+set(TENSORFLOW_AOT_PATH "" CACHE PATH "Path to TensorFlow pip install dir")
+
+if (NOT TENSORFLOW_AOT_PATH STREQUAL "")
+  set(LLVM_HAVE_TF_AOT "ON" CACHE BOOL "Tensorflow AOT available")
+  set(TENSORFLOW_AOT_COMPILER
+    "${TENSORFLOW_AOT_PATH}/../../../../bin/saved_model_cli"
+    CACHE PATH "Path to the Tensorflow AOT compiler")
+  include_directories(${TENSORFLOW_AOT_PATH}/include)
+  add_subdirectory(${TENSORFLOW_AOT_PATH}/xla_aot_runtime_src
+    ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}/tf_runtime)
+endif()
+
 # Configure the three LLVM configuration header files.
 configure_file(
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Config/config.h.cmake
@@ -875,12 +892,6 @@ add_custom_target(srpm
   COMMAND rpmbuild -bs --define '_topdir ${LLVM_SRPM_DIR}' ${LLVM_SRPM_BINARY_SPECFILE})
 set_target_properties(srpm PROPERTIES FOLDER "Misc")
 
-
-# They are not referenced. See set_output_directory().
-set( CMAKE_RUNTIME_OUTPUT_DIRECTORY ${LLVM_BINARY_DIR}/bin )
-set( CMAKE_LIBRARY_OUTPUT_DIRECTORY ${LLVM_BINARY_DIR}/lib${LLVM_LIBDIR_SUFFIX} )
-set( CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${LLVM_BINARY_DIR}/lib${LLVM_LIBDIR_SUFFIX} )
-
 if(APPLE AND DARWIN_LTO_LIBRARY)
   set(CMAKE_EXE_LINKER_FLAGS
     "${CMAKE_EXE_LINKER_FLAGS} -Wl,-lto_library -Wl,${DARWIN_LTO_LIBRARY}")
@@ -975,28 +986,6 @@ if( MINGW AND NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang" )
   llvm_replace_compiler_option(CMAKE_CXX_FLAGS_RELEASE "-O3" "-O2")
 endif()
 
-# For up-to-date instructions for installing the Tensorflow dependency, refer to
-# the bot setup script: https://github.com/google/ml-compiler-opt/blob/master/buildbot/buildbot_init.sh
-# Specifically, assuming python3 is installed:
-# python3 -m pip install --upgrade pip && python3 -m pip install --user tf_nightly==2.3.0.dev20200528
-# Then set TENSORFLOW_AOT_PATH to the package install - usually it's ~/.local/lib/python3.7/site-packages/tensorflow
-#
-set(TENSORFLOW_AOT_PATH "" CACHE PATH "Path to TensorFlow pip install dir")
-
-if (NOT TENSORFLOW_AOT_PATH STREQUAL "")
-  set(LLVM_HAVE_TF_AOT "ON" CACHE BOOL "Tensorflow AOT available")
-  set(TENSORFLOW_AOT_COMPILER
-    "${TENSORFLOW_AOT_PATH}/../../../../bin/saved_model_cli"
-    CACHE PATH "Path to the Tensorflow AOT compiler")
-  # Unlike the LLVM_HAVE_TF_API case, we don't need to expose this through
-  # llvm-config.h, because it's an internal implementation detail. A user of the llvm library that wants to also
-  # use the TF AOT compiler may do so through their custom build step.
-  add_definitions("-DLLVM_HAVE_TF_AOT")
-  include_directories(${TENSORFLOW_AOT_PATH}/include)
-  add_subdirectory(${TENSORFLOW_AOT_PATH}/xla_aot_runtime_src
-    ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}/tf_runtime)
-endif()
-
 # Put this before tblgen. Else we have a circular dependence.
 add_subdirectory(lib/Demangle)
 add_subdirectory(lib/Support)
diff --git a/llvm/cmake/config-ix.cmake b/llvm/cmake/config-ix.cmake
index 90e5d327c7577..9d0397fc79422 100644
--- a/llvm/cmake/config-ix.cmake
+++ b/llvm/cmake/config-ix.cmake
@@ -56,7 +56,6 @@ check_include_file(sys/types.h HAVE_SYS_TYPES_H)
 check_include_file(termios.h HAVE_TERMIOS_H)
 check_include_file(unistd.h HAVE_UNISTD_H)
 check_include_file(valgrind/valgrind.h HAVE_VALGRIND_VALGRIND_H)
-check_include_file(zlib.h HAVE_ZLIB_H)
 check_include_file(fenv.h HAVE_FENV_H)
 check_symbol_exists(FE_ALL_EXCEPT "fenv.h" HAVE_DECL_FE_ALL_EXCEPT)
 check_symbol_exists(FE_INEXACT "fenv.h" HAVE_DECL_FE_INEXACT)
@@ -117,68 +116,60 @@ endif()
 
 # Don't look for these libraries if we're using MSan, since uninstrumented third
 # party code may call MSan interceptors like strlen, leading to false positives.
-if(NOT LLVM_USE_SANITIZER MATCHES "Memory.*")
-  set(HAVE_LIBZ 0)
-  if(LLVM_ENABLE_ZLIB)
-    foreach(library z zlib_static zlib)
+# Don't look for these libraries on Windows.
+if(NOT LLVM_USE_SANITIZER MATCHES "Memory.*" AND NOT PURE_WINDOWS)
+  # Skip libedit if using ASan as it contains memory leaks.
+  if (LLVM_ENABLE_LIBEDIT AND HAVE_HISTEDIT_H AND NOT LLVM_USE_SANITIZER MATCHES ".*Address.*")
+    check_library_exists(edit el_init "" HAVE_LIBEDIT)
+  else()
+    set(HAVE_LIBEDIT 0)
+  endif()
+  if(LLVM_ENABLE_TERMINFO)
+    set(HAVE_TERMINFO 0)
+    foreach(library terminfo tinfo curses ncurses ncursesw)
       string(TOUPPER ${library} library_suffix)
-      check_library_exists(${library} compress2 "" HAVE_LIBZ_${library_suffix})
-      if(HAVE_LIBZ_${library_suffix})
-        set(HAVE_LIBZ 1)
-        set(ZLIB_LIBRARIES "${library}")
+      check_library_exists(${library} setupterm "" HAVE_TERMINFO_${library_suffix})
+      if(HAVE_TERMINFO_${library_suffix})
+        set(HAVE_TERMINFO 1)
+        set(TERMINFO_LIBS "${library}")
         break()
       endif()
     endforeach()
+  else()
+    set(HAVE_TERMINFO 0)
   endif()
 
-  # Don't look for these libraries on Windows.
-  if (NOT PURE_WINDOWS)
-    # Skip libedit if using ASan as it contains memory leaks.
-    if (LLVM_ENABLE_LIBEDIT AND HAVE_HISTEDIT_H AND NOT LLVM_USE_SANITIZER MATCHES ".*Address.*")
-      check_library_exists(edit el_init "" HAVE_LIBEDIT)
-    else()
-      set(HAVE_LIBEDIT 0)
-    endif()
-    if(LLVM_ENABLE_TERMINFO)
-      set(HAVE_TERMINFO 0)
-      foreach(library terminfo tinfo curses ncurses ncursesw)
-        string(TOUPPER ${library} library_suffix)
-        check_library_exists(${library} setupterm "" HAVE_TERMINFO_${library_suffix})
-        if(HAVE_TERMINFO_${library_suffix})
-          set(HAVE_TERMINFO 1)
-          set(TERMINFO_LIBS "${library}")
-          break()
-        endif()
-      endforeach()
+  if(LLVM_ENABLE_ZLIB)
+    if(LLVM_ENABLE_ZLIB STREQUAL FORCE_ON)
+      find_package(ZLIB REQUIRED)
     else()
-      set(HAVE_TERMINFO 0)
+      find_package(ZLIB)
     endif()
+    set(LLVM_ENABLE_ZLIB "${ZLIB_FOUND}")
+  endif()
 
-    find_library(ICONV_LIBRARY_PATH NAMES iconv libiconv libiconv-2 c)
-    set(LLVM_LIBXML2_ENABLED 0)
-    set(LIBXML2_FOUND 0)
-    if((LLVM_ENABLE_LIBXML2) AND ((CMAKE_SYSTEM_NAME MATCHES "Linux") AND (ICONV_LIBRARY_PATH) OR APPLE))
-      find_package(LibXml2)
-      if (LIBXML2_FOUND)
-        set(LLVM_LIBXML2_ENABLED 1)
-        if ((CMAKE_OSX_SYSROOT) AND (EXISTS ${CMAKE_OSX_SYSROOT}/${LIBXML2_INCLUDE_DIR}))
-          include_directories(${CMAKE_OSX_SYSROOT}/${LIBXML2_INCLUDE_DIR})
-        else()
-          include_directories(${LIBXML2_INCLUDE_DIR})
-        endif()
+  find_library(ICONV_LIBRARY_PATH NAMES iconv libiconv libiconv-2 c)
+  set(LLVM_LIBXML2_ENABLED 0)
+  set(LIBXML2_FOUND 0)
+  if((LLVM_ENABLE_LIBXML2) AND ((CMAKE_SYSTEM_NAME MATCHES "Linux") AND (ICONV_LIBRARY_PATH) OR APPLE))
+    find_package(LibXml2)
+    if (LIBXML2_FOUND)
+      set(LLVM_LIBXML2_ENABLED 1)
+      if ((CMAKE_OSX_SYSROOT) AND (EXISTS ${CMAKE_OSX_SYSROOT}/${LIBXML2_INCLUDE_DIR}))
+        include_directories(${CMAKE_OSX_SYSROOT}/${LIBXML2_INCLUDE_DIR})
+      else()
+        include_directories(${LIBXML2_INCLUDE_DIR})
       endif()
     endif()
   endif()
+else()
+  set(LLVM_ENABLE_ZLIB FALSE)
 endif()
 
 if (LLVM_ENABLE_LIBXML2 STREQUAL "FORCE_ON" AND NOT LLVM_LIBXML2_ENABLED)
   message(FATAL_ERROR "Failed to congifure libxml2")
 endif()
 
-if (LLVM_ENABLE_ZLIB STREQUAL "FORCE_ON" AND NOT HAVE_LIBZ)
-  message(FATAL_ERROR "Failed to configure zlib")
-endif()
-
 check_library_exists(xar xar_open "" HAVE_LIBXAR)
 if(HAVE_LIBXAR)
   set(XAR_LIB xar)
@@ -517,13 +508,6 @@ else( LLVM_ENABLE_THREADS )
   message(STATUS "Threads disabled.")
 endif()
 
-if (LLVM_ENABLE_ZLIB )
-  # Check if zlib is available in the system.
-  if ( NOT HAVE_ZLIB_H OR NOT HAVE_LIBZ )
-    set(LLVM_ENABLE_ZLIB 0)
-  endif()
-endif()
-
 if (LLVM_ENABLE_DOXYGEN)
   message(STATUS "Doxygen enabled.")
   find_package(Doxygen REQUIRED)
diff --git a/llvm/cmake/modules/HandleLLVMOptions.cmake b/llvm/cmake/modules/HandleLLVMOptions.cmake
index 2e249593e12f1..62dd0ef79cf48 100644
--- a/llvm/cmake/modules/HandleLLVMOptions.cmake
+++ b/llvm/cmake/modules/HandleLLVMOptions.cmake
@@ -672,6 +672,21 @@ if (LLVM_ENABLE_WARNINGS AND (LLVM_COMPILER_IS_GCC_COMPATIBLE OR CLANG_CL))
   # Enable -Wdelete-non-virtual-dtor if available.
   add_flag_if_supported("-Wdelete-non-virtual-dtor" DELETE_NON_VIRTUAL_DTOR_FLAG)
 
+  # Enable -Wsuggest-override if it's available, and only if it doesn't
+  # suggest adding 'override' to functions that are already marked 'final'
+  # (which means it is disabled for GCC < 9.2).
+  check_cxx_compiler_flag("-Wsuggest-override" CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
+  if (CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
+    set(OLD_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
+    set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -Werror=suggest-override")
+    CHECK_CXX_SOURCE_COMPILES("class base {public: virtual void anchor();};
+                               class derived : base {public: void anchor() final;};
+                               int main() { return 0; }"
+                              CXX_WSUGGEST_OVERRIDE_ALLOWS_ONLY_FINAL)
+    set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQUIRED_FLAGS})
+    append_if(CXX_WSUGGEST_OVERRIDE_ALLOWS_ONLY_FINAL "-Wsuggest-override" CMAKE_CXX_FLAGS)
+  endif()
+
   # Check if -Wcomment is OK with an // comment ending with '\' if the next
   # line is also a // comment.
   set(OLD_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
diff --git a/llvm/docs/CMake.rst b/llvm/docs/CMake.rst
index 1f908d3e95b12..f96e34f21e803 100644
--- a/llvm/docs/CMake.rst
+++ b/llvm/docs/CMake.rst
@@ -34,7 +34,7 @@ Quick start
 We use here the command-line, non-interactive CMake interface.
 
 #. `Download <http://www.cmake.org/cmake/resources/software.html>`_ and install
-   CMake. Version 3.4.3 is the minimum required.
+   CMake. Version 3.13.4 is the minimum required.
 
 #. Open a shell. Your development tools must be reachable from this shell
    through the PATH environment variable.
@@ -717,7 +717,7 @@ and uses them to build a simple application ``simple-tool``.
 
 .. code-block:: cmake
 
-  cmake_minimum_required(VERSION 3.4.3)
+  cmake_minimum_required(VERSION 3.13.4)
   project(SimpleProject)
 
   find_package(LLVM REQUIRED CONFIG)
diff --git a/llvm/docs/CMakePrimer.rst b/llvm/docs/CMakePrimer.rst
index abfd08017fbfa..bc63ac1c02734 100644
--- a/llvm/docs/CMakePrimer.rst
+++ b/llvm/docs/CMakePrimer.rst
@@ -54,7 +54,7 @@ program. The example uses only CMake language-defined functions.
 
 .. code-block:: cmake
 
-   cmake_minimum_required(VERSION 3.2)
+   cmake_minimum_required(VERSION 3.15)
    project(HelloWorld)
    add_executable(HelloWorld HelloWorld.cpp)
 
@@ -64,13 +64,13 @@ block to define "APPLE" when targeting Apple platforms:
 
 .. code-block:: cmake
 
-   cmake_minimum_required(VERSION 3.2)
+   cmake_minimum_required(VERSION 3.15)
    project(HelloWorld)
    add_executable(HelloWorld HelloWorld.cpp)
    if(APPLE)
      target_compile_definitions(HelloWorld PUBLIC APPLE)
    endif()
-   
+
 Variables, Types, and Scope
 ===========================
 
@@ -93,7 +93,7 @@ example:
    set(var_name var1)
    set(${var_name} foo) # same as "set(var1 foo)"
    set(${${var_name}}_var bar) # same as "set(foo_var bar)"
-   
+
 Dereferencing an unset variable results in an empty expansion. It is a common
 pattern in CMake to conditionally set variables knowing that it will be used in
 code paths that the variable isn't set. There are examples of this throughout
@@ -107,7 +107,7 @@ An example of variable empty expansion is:
      set(extra_sources Apple.cpp)
    endif()
    add_executable(HelloWorld HelloWorld.cpp ${extra_sources})
-   
+
 In this example the ``extra_sources`` variable is only defined if you're
 targeting an Apple platform. For all other targets the ``extra_sources`` will be
 evaluated as empty before add_executable is given its arguments.
@@ -124,7 +124,7 @@ defining lists:
    # Creates a list with members a, b, c, and d
    set(my_list a b c d)
    set(my_list "a;b;c;d")
-   
+
    # Creates a string "a b c d"
    set(my_string "a b c d")
 
@@ -141,7 +141,7 @@ make a list of variable names that refer to other lists. For example:
    set(a 1 2 3)
    set(b 4 5 6)
    set(c 7 8 9)
-   
+
 With this layout you can iterate through the list of lists printing each value
 with the following code:
 
@@ -152,7 +152,7 @@ with the following code:
        message(${value})
      endforeach()
    endforeach()
-   
+
 You'll notice that the inner foreach loop's list is doubly dereferenced. This is
 because the first dereference turns ``list_name`` into the name of the sub-list
 (a, b, or c in the example), then the second dereference is to get the value of
@@ -385,7 +385,7 @@ result in some unexpected behavior if using unreferenced variables. For example:
        message("${var}")
      endforeach()
    endmacro()
-   
+
    set(my_list a b c d)
    set(my_list_of_numbers 1 2 3 4)
    print_list(my_list_of_numbers)
diff --git a/llvm/docs/CommandGuide/llc.rst b/llvm/docs/CommandGuide/llc.rst
index 9b07b0b29d49e..a15eb0e6aaab2 100644
--- a/llvm/docs/CommandGuide/llc.rst
+++ b/llvm/docs/CommandGuide/llc.rst
@@ -163,9 +163,9 @@ End-user Options
 Tuning/Configuration Options
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. option:: --print-machineinstrs
+.. option:: --print-after-isel
 
- Print generated machine code between compilation phases (useful for debugging).
+ Print generated machine code after instruction selection (useful for debugging).
 
 .. option:: --regalloc=<allocator>
 
diff --git a/llvm/docs/CommandGuide/lli.rst b/llvm/docs/CommandGuide/lli.rst
index b9892b64801c0..76d6e5f472b78 100644
--- a/llvm/docs/CommandGuide/lli.rst
+++ b/llvm/docs/CommandGuide/lli.rst
@@ -165,7 +165,7 @@ CODE GENERATION OPTIONS
       =simple-noitin: Simple two pass scheduling: Same as simple except using generic latency
       =list-burr: Bottom-up register reduction list scheduling
       =list-tdrr: Top-down register reduction list scheduling
-      =list-td: Top-down list scheduler -print-machineinstrs - Print generated machine code
+      =list-td: Top-down list scheduler
 
 .. option:: -regalloc=allocator
 
diff --git a/llvm/docs/CommandGuide/llvm-libtool-darwin.rst b/llvm/docs/CommandGuide/llvm-libtool-darwin.rst
index 0baacfd88e8a5..6bae95c3a6698 100644
--- a/llvm/docs/CommandGuide/llvm-libtool-darwin.rst
+++ b/llvm/docs/CommandGuide/llvm-libtool-darwin.rst
@@ -42,6 +42,10 @@ OPTIONS
 
   Specify the output file name. Must be specified exactly once.
 
+.. option:: -static
+
+ Produces a static library from the input files.
+
 EXIT STATUS
 -----------
 
diff --git a/llvm/docs/CommandGuide/llvm-readelf.rst b/llvm/docs/CommandGuide/llvm-readelf.rst
index f74f02c9a6676..fa54c1dc84dc4 100644
--- a/llvm/docs/CommandGuide/llvm-readelf.rst
+++ b/llvm/docs/CommandGuide/llvm-readelf.rst
@@ -14,7 +14,7 @@ DESCRIPTION
 The :program:`llvm-readelf` tool displays low-level format-specific information
 about one or more object files.
 
-If ``input`` is "``-``" or omitted, :program:`llvm-readelf` reads from standard
+If ``input`` is "``-``", :program:`llvm-readelf` reads from standard
 input. Otherwise, it will read from the specified ``filenames``.
 
 OPTIONS
diff --git a/llvm/docs/CommandGuide/llvm-readobj.rst b/llvm/docs/CommandGuide/llvm-readobj.rst
index 51cd266ff79e0..9b1b5ba92bc07 100644
--- a/llvm/docs/CommandGuide/llvm-readobj.rst
+++ b/llvm/docs/CommandGuide/llvm-readobj.rst
@@ -14,7 +14,7 @@ DESCRIPTION
 The :program:`llvm-readobj` tool displays low-level format-specific information
 about one or more object files.
 
-If ``input`` is "``-``" or omitted, :program:`llvm-readobj` reads from standard
+If ``input`` is "``-``", :program:`llvm-readobj` reads from standard
 input. Otherwise, it will read from the specified ``filenames``.
 
 DIFFERENCES TO LLVM-READELF
diff --git a/llvm/docs/GettingStarted.rst b/llvm/docs/GettingStarted.rst
index 478571bd4c238..c30ebecc89e03 100644
--- a/llvm/docs/GettingStarted.rst
+++ b/llvm/docs/GettingStarted.rst
@@ -167,7 +167,7 @@ uses the package and provides other details.
 =========================================================== ============ ==========================================
 Package                                                     Version      Notes
 =========================================================== ============ ==========================================
-`CMake <http://cmake.org/>`__                               >=3.4.3      Makefile/workspace generator
+`CMake <http://cmake.org/>`__                               >=3.13.4     Makefile/workspace generator
 `GCC <http://gcc.gnu.org/>`_                                >=5.1.0      C/C++ compiler\ :sup:`1`
 `python <http://www.python.org/>`_                          >=2.7        Automated test suite\ :sup:`2`
 `zlib <http://zlib.net>`_                                   >=1.2.3.4    Compression library\ :sup:`3`
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 86d8c62af2b7e..6b9c5c6899819 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -1066,6 +1066,30 @@ Currently, only the following parameter attributes are defined:
     site. If the alignment is not specified, then the code generator
     makes a target-specific assumption.
 
+.. _attr_byref:
+
+``byref(<ty>)``
+
+    The ``byref`` argument attribute allows specifying the pointee
+    memory type of an argument. This is similar to ``byval``, but does
+    not imply a copy is made anywhere, or that the argument is passed
+    on the stack. This implies the pointer is dereferenceable up to
+    the storage size of the type.
+
+    It is not generally permissible to introduce a write to an
+    ``byref`` pointer. The pointer may have any address space and may
+    be read only.
+
+    This is not a valid attribute for return values.
+
+    The alignment for an ``byref`` parameter can be explicitly
+    specified by combining it with the ``align`` attribute, similar to
+    ``byval``. If the alignment is not specified, then the code generator
+    makes a target-specific assumption.
+
+     This is intended for representing ABI constraints, and is not
+    intended to be inferred for optimization use.
+
 .. _attr_preallocated:
 
 ``preallocated(<ty>)``
@@ -4862,7 +4886,12 @@ DIExpression that describes how to get from an object's address to the actual
 raw data, if they aren't equivalent. This is only supported for array types,
 particularly to describe Fortran arrays, which have an array descriptor in
 addition to the array data. Alternatively it can also be DIVariable which
-has the address of the actual raw data.
+has the address of the actual raw data. The Fortran language supports pointer
+arrays which can be attached to actual arrays, this attachement between pointer
+and pointee is called association.  The optional ``associated`` is a
+DIExpression that describes whether the pointer array is currently associated.
+The optional ``allocated`` is a DIExpression that describes whether the
+allocatable array is currently allocated.
 
 For ``DW_TAG_enumeration_type``, the ``elements:`` should be :ref:`enumerator
 descriptors <DIEnumerator>`, each representing the definition of an enumeration
@@ -12229,14 +12258,172 @@ Example:
         call void @llvm.call.preallocated.teardown(token %cs)
         ret void
 
-Standard C Library Intrinsics
------------------------------
+Standard C/C++ Library Intrinsics
+---------------------------------
 
-LLVM provides intrinsics for a few important standard C library
+LLVM provides intrinsics for a few important standard C/C++ library
 functions. These intrinsics allow source-language front-ends to pass
 information about the alignment of the pointer arguments to the code
 generator, providing opportunity for more efficient code generation.
 
+
+'``llvm.abs.*``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+This is an overloaded intrinsic. You can use ``llvm.abs`` on any
+integer bit width or any vector of integer elements.
+
+::
+
+      declare i32 @llvm.abs.i32(i32 <src>, i1 <is_int_min_poison>)
+      declare <4 x i32> @llvm.abs.v4i32(<4 x i32> <src>, i1 <is_int_min_poison>)
+
+Overview:
+"""""""""
+
+The '``llvm.abs``' family of intrinsic functions returns the absolute value
+of an argument.
+
+Arguments:
+""""""""""
+
+The first argument is the value for which the absolute value is to be returned.
+This argument may be of any integer type or a vector with integer element type.
+The return type must match the first argument type.
+
+The second argument must be a constant and is a flag to indicate whether the
+result value of the '``llvm.abs``' intrinsic is a
+:ref:`poison value <poisonvalues>` if the argument is statically or dynamically
+an ``INT_MIN`` value.
+
+Semantics:
+""""""""""
+
+The '``llvm.abs``' intrinsic returns the magnitude (always positive) of the
+argument or each element of a vector argument.". If the argument is ``INT_MIN``,
+then the result is also ``INT_MIN`` if ``is_int_min_poison == 0`` and
+``poison`` otherwise.
+
+
+'``llvm.smax.*``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+This is an overloaded intrinsic. You can use ``@llvm.smax`` on any
+integer bit width or any vector of integer elements.
+
+::
+
+      declare i32 @llvm.smax.i32(i32 %a, i32 %b)
+      declare <4 x i32> @llvm.smax.v4i32(<4 x i32> %a, <4 x i32> %b)
+
+Overview:
+"""""""""
+
+Return the larger of ``%a`` and ``%b`` comparing the values as signed integers.
+Vector intrinsics operate on a per-element basis. The larger element of ``%a``
+and ``%b`` at a given index is returned for that index.
+
+Arguments:
+""""""""""
+
+The arguments (``%a`` and ``%b``) may be of any integer type or a vector with
+integer element type. The argument types must match each other, and the return
+type must match the argument type.
+
+
+'``llvm.smin.*``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+This is an overloaded intrinsic. You can use ``@llvm.smin`` on any
+integer bit width or any vector of integer elements.
+
+::
+
+      declare i32 @llvm.smin.i32(i32 %a, i32 %b)
+      declare <4 x i32> @llvm.smin.v4i32(<4 x i32> %a, <4 x i32> %b)
+
+Overview:
+"""""""""
+
+Return the smaller of ``%a`` and ``%b`` comparing the values as signed integers.
+Vector intrinsics operate on a per-element basis. The smaller element of ``%a``
+and ``%b`` at a given index is returned for that index.
+
+Arguments:
+""""""""""
+
+The arguments (``%a`` and ``%b``) may be of any integer type or a vector with
+integer element type. The argument types must match each other, and the return
+type must match the argument type.
+
+
+'``llvm.umax.*``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+This is an overloaded intrinsic. You can use ``@llvm.umax`` on any
+integer bit width or any vector of integer elements.
+
+::
+
+      declare i32 @llvm.umax.i32(i32 %a, i32 %b)
+      declare <4 x i32> @llvm.umax.v4i32(<4 x i32> %a, <4 x i32> %b)
+
+Overview:
+"""""""""
+
+Return the larger of ``%a`` and ``%b`` comparing the values as unsigned
+integers. Vector intrinsics operate on a per-element basis. The larger element
+of ``%a`` and ``%b`` at a given index is returned for that index.
+
+Arguments:
+""""""""""
+
+The arguments (``%a`` and ``%b``) may be of any integer type or a vector with
+integer element type. The argument types must match each other, and the return
+type must match the argument type.
+
+
+'``llvm.umin.*``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+This is an overloaded intrinsic. You can use ``@llvm.umin`` on any
+integer bit width or any vector of integer elements.
+
+::
+
+      declare i32 @llvm.umin.i32(i32 %a, i32 %b)
+      declare <4 x i32> @llvm.umin.v4i32(<4 x i32> %a, <4 x i32> %b)
+
+Overview:
+"""""""""
+
+Return the smaller of ``%a`` and ``%b`` comparing the values as unsigned
+integers. Vector intrinsics operate on a per-element basis. The smaller element
+of ``%a`` and ``%b`` at a given index is returned for that index.
+
+Arguments:
+""""""""""
+
+The arguments (``%a`` and ``%b``) may be of any integer type or a vector with
+integer element type. The argument types must match each other, and the return
+type must match the argument type.
+
+
 .. _int_memcpy:
 
 '``llvm.memcpy``' Intrinsic
diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index e234965aaa7c0..a939a4af43c1e 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -59,6 +59,9 @@ Changes to the LLVM IR
 
 * ...
 
+* Added the ``byref`` attribute to better represent argument passing
+  for the `amdgpu_kernel` calling convention.
+
 Changes to building LLVM
 ------------------------
 
@@ -83,11 +86,18 @@ Changes to the X86 Target
 
 During this release ...
 
+* The 'mpx' feature was removed from the backend. It had been removed from clang
+  frontend in 10.0. Mention of the 'mpx' feature in an IR file will print a
+  message to stderr, but IR should still compile.
+
 Changes to the AMDGPU Target
 -----------------------------
 
 During this release ...
 
+* The new ``byref`` attribute is now the preferred method for
+  representing aggregate kernel arguments.
+
 Changes to the AVR Target
 -----------------------------
 
@@ -123,7 +133,9 @@ During this release ...
 Changes to the LLVM tools
 ---------------------------------
 
-During this release ...
+* llvm-readobj and llvm-readelf behavior has changed to report an error when
+  executed with no input files instead of reading an input from stdin.
+  Reading from stdin can still be achieved by specifying `-` as an input file.
 
 Changes to LLDB
 ---------------------------------
diff --git a/llvm/docs/TestSuiteGuide.md b/llvm/docs/TestSuiteGuide.md
index 6128636ce5eed..1392ad0033679 100644
--- a/llvm/docs/TestSuiteGuide.md
+++ b/llvm/docs/TestSuiteGuide.md
@@ -71,8 +71,8 @@ Quickstart
 6. Show and compare result files (optional):
 
    ```bash
-   # Make sure pandas is installed. Prepend `sudo` if necessary.
-   % pip install pandas
+   # Make sure pandas and scipy are installed. Prepend `sudo` if necessary.
+   % pip install pandas scipy
    # Show a single result file:
    % test-suite/utils/compare.py results.json
    # Compare two result files:
diff --git a/llvm/docs/TestingGuide.rst b/llvm/docs/TestingGuide.rst
index 2e937f0006272..6fd9ab2d24ca4 100644
--- a/llvm/docs/TestingGuide.rst
+++ b/llvm/docs/TestingGuide.rst
@@ -271,8 +271,27 @@ adding your code there instead of creating a new file.
 Extra files
 -----------
 
-If your test requires extra files besides the file containing the ``RUN:``
-lines, the idiomatic place to put them is in a subdirectory ``Inputs``.
+If your test requires extra files besides the file containing the ``RUN:`` lines
+and the extra files are small, consider specifying them in the same file and
+using ``extract`` to extract them. For example,
+
+.. code-block:: llvm
+
+  ; RUN: extract b %s -o %tb.ll
+  ; RUN: extract a %s | llvm-link - %tb.ll -S | FileCheck %s
+
+  ; CHECK: ...
+
+  ;--- a
+  ...
+  ;--- b
+  ...
+
+The parts are separated by the regex ``^(.|//)--- <part>``. By default the
+extracted content has leading empty lines to preserve line numbers. Specify
+``--no-leading-lines`` to drop leading lines.
+
+If the extra files are large, the idiomatic place to put them is in a subdirectory ``Inputs``.
 You can then refer to the extra files as ``%S/Inputs/foo.bar``.
 
 For example, consider ``test/Linker/ident.ll``. The directory structure is
diff --git a/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl03.rst b/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl03.rst
index 32472e3a48202..c37c9600f51e7 100644
--- a/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl03.rst
+++ b/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl03.rst
@@ -67,7 +67,7 @@ way to model this. Again, this tutorial won't dwell on good software
 engineering practices: for our purposes, adding a virtual method is
 simplest.
 
-The second thing we want is an "LogError" method like we used for the
+The second thing we want is a "LogError" method like we used for the
 parser, which will be used to report errors found during code generation
 (for example, use of an undeclared parameter):
 
diff --git a/llvm/examples/HowToUseLLJIT/HowToUseLLJIT.cpp b/llvm/examples/HowToUseLLJIT/HowToUseLLJIT.cpp
index 4383b5a1eeb0b..c71c4d9293ab8 100644
--- a/llvm/examples/HowToUseLLJIT/HowToUseLLJIT.cpp
+++ b/llvm/examples/HowToUseLLJIT/HowToUseLLJIT.cpp
@@ -10,6 +10,7 @@
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Module.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/InitLLVM.h"
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Support/raw_ostream.h"
diff --git a/llvm/examples/OrcV2Examples/ExampleModules.h b/llvm/examples/OrcV2Examples/ExampleModules.h
index ae0089fd9dd54..72b0cc767f119 100644
--- a/llvm/examples/OrcV2Examples/ExampleModules.h
+++ b/llvm/examples/OrcV2Examples/ExampleModules.h
@@ -18,6 +18,7 @@
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IRReader/IRReader.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/SourceMgr.h"
 
diff --git a/llvm/examples/OrcV2Examples/LLJITWithTargetProcessControl/LLJITWithTargetProcessControl.cpp b/llvm/examples/OrcV2Examples/LLJITWithTargetProcessControl/LLJITWithTargetProcessControl.cpp
index a88360e3a8cd6..d2ce26bddd72e 100644
--- a/llvm/examples/OrcV2Examples/LLJITWithTargetProcessControl/LLJITWithTargetProcessControl.cpp
+++ b/llvm/examples/OrcV2Examples/LLJITWithTargetProcessControl/LLJITWithTargetProcessControl.cpp
@@ -25,6 +25,7 @@
 #include "llvm/ExecutionEngine/Orc/LLJIT.h"
 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
 #include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
+#include "llvm/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.h"
 #include "llvm/ExecutionEngine/Orc/TPCIndirectionUtils.h"
 #include "llvm/ExecutionEngine/Orc/TargetProcessControl.h"
 #include "llvm/Support/InitLLVM.h"
@@ -51,17 +52,23 @@ ExitOnError ExitOnErr;
 
 const llvm::StringRef FooMod =
     R"(
+  declare i32 @return1()
+
   define i32 @foo_body() {
   entry:
-    ret i32 1
+    %0 = call i32 @return1()
+    ret i32 %0
   }
 )";
 
 const llvm::StringRef BarMod =
     R"(
+  declare i32 @return2()
+
   define i32 @bar_body() {
   entry:
-    ret i32 2
+    %0 = call i32 @return2()
+    ret i32 %0
   }
 )";
 
@@ -91,6 +98,9 @@ const llvm::StringRef MainMod =
   declare i32 @bar()
 )";
 
+extern "C" int32_t return1() { return 1; }
+extern "C" int32_t return2() { return 2; }
+
 static void *reenter(void *Ctx, void *TrampolineAddr) {
   std::promise<void *> LandingAddressP;
   auto LandingAddressF = LandingAddressP.get_future();
@@ -105,6 +115,11 @@ static void *reenter(void *Ctx, void *TrampolineAddr) {
   return LandingAddressF.get();
 }
 
+static void reportErrorAndExit() {
+  errs() << "Unable to lazily compile function. Exiting.\n";
+  exit(1);
+}
+
 cl::list<std::string> InputArgv(cl::Positional,
                                 cl::desc("<program arguments>..."));
 
@@ -119,7 +134,8 @@ int main(int argc, char *argv[]) {
   ExitOnErr.setBanner(std::string(argv[0]) + ": ");
 
   // (1) Create LLJIT instance.
-  auto J = ExitOnErr(LLJITBuilder().create());
+  auto TPC = ExitOnErr(SelfTargetProcessControl::Create());
+  auto J = ExitOnErr(LLJITBuilder().setTargetProcessControl(*TPC).create());
 
   // (2) Install transform to print modules as they are compiled:
   J->getIRTransformLayer().setTransform(
@@ -130,13 +146,14 @@ int main(int argc, char *argv[]) {
       });
 
   // (3) Create stubs and call-through managers:
-
-  auto TPC = ExitOnErr(SelfTargetProcessControl::Create());
   auto TPCIU = ExitOnErr(TPCIndirectionUtils::Create(*TPC));
   ExitOnErr(TPCIU->writeResolverBlock(pointerToJITTargetAddress(&reenter),
                                       pointerToJITTargetAddress(TPCIU.get())));
-  TPCIU->createLazyCallThroughManager(J->getExecutionSession(), 0);
+  TPCIU->createLazyCallThroughManager(
+      J->getExecutionSession(), pointerToJITTargetAddress(&reportErrorAndExit));
   auto ISM = TPCIU->createIndirectStubsManager();
+  J->getMainJITDylib().addGenerator(
+      ExitOnErr(TPCDynamicLibrarySearchGenerator::GetForTargetProcess(*TPC)));
 
   // (4) Add modules.
   ExitOnErr(J->addIRModule(ExitOnErr(parseExampleModule(FooMod, "foo-mod"))));
diff --git a/llvm/include/llvm/ADT/PointerUnion.h b/llvm/include/llvm/ADT/PointerUnion.h
index 6fecff8d756f0..c39691061b720 100644
--- a/llvm/include/llvm/ADT/PointerUnion.h
+++ b/llvm/include/llvm/ADT/PointerUnion.h
@@ -93,13 +93,6 @@ namespace pointer_union_detail {
     static constexpr int NumLowBitsAvailable = lowBitsAvailable<PTs...>();
   };
 
-  /// Implement assignment in terms of construction.
-  template <typename Derived, typename T> struct AssignableFrom {
-    Derived &operator=(T t) {
-      return static_cast<Derived &>(*this) = Derived(t);
-    }
-  };
-
   template <typename Derived, typename ValTy, int I, typename ...Types>
   class PointerUnionMembers;
 
diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h
index eed676bb74e14..2e7a097174763 100644
--- a/llvm/include/llvm/ADT/STLExtras.h
+++ b/llvm/include/llvm/ADT/STLExtras.h
@@ -206,15 +206,6 @@ class function_ref<Ret(Params...)> {
   explicit operator bool() const { return callback; }
 };
 
-// deleter - Very very very simple method that is used to invoke operator
-// delete on something.  It is used like this:
-//
-//   for_each(V.begin(), B.end(), deleter<Interval>);
-template <class T>
-inline void deleter(T *Ptr) {
-  delete Ptr;
-}
-
 //===----------------------------------------------------------------------===//
 //     Extra additions to <iterator>
 //===----------------------------------------------------------------------===//
@@ -1471,10 +1462,11 @@ inline void sort(Container &&C, Compare Comp) {
 /// which is only enabled when the operation is O(1).
 template <typename R>
 auto size(R &&Range,
-          std::enable_if_t<std::is_same<typename std::iterator_traits<decltype(
-                                            Range.begin())>::iterator_category,
-                                        std::random_access_iterator_tag>::value,
-                           void> * = nullptr) {
+          std::enable_if_t<
+              std::is_base_of<std::random_access_iterator_tag,
+                              typename std::iterator_traits<decltype(
+                                  Range.begin())>::iterator_category>::value,
+              void> * = nullptr) {
   return std::distance(Range.begin(), Range.end());
 }
 
@@ -1911,16 +1903,16 @@ decltype(auto) apply_tuple(F &&f, Tuple &&t) {
 /// Return true if the sequence [Begin, End) has exactly N items. Runs in O(N)
 /// time. Not meant for use with random-access iterators.
 /// Can optionally take a predicate to filter lazily some items.
-template<typename IterTy,
-         typename Pred = bool (*)(const decltype(*std::declval<IterTy>()) &)>
+template <typename IterTy,
+          typename Pred = bool (*)(const decltype(*std::declval<IterTy>()) &)>
 bool hasNItems(
     IterTy &&Begin, IterTy &&End, unsigned N,
     Pred &&ShouldBeCounted =
         [](const decltype(*std::declval<IterTy>()) &) { return true; },
     std::enable_if_t<
-        !std::is_same<typename std::iterator_traits<std::remove_reference_t<
-                          decltype(Begin)>>::iterator_category,
-                      std::random_access_iterator_tag>::value,
+        !std::is_base_of<std::random_access_iterator_tag,
+                         typename std::iterator_traits<std::remove_reference_t<
+                             decltype(Begin)>>::iterator_category>::value,
         void> * = nullptr) {
   for (; N; ++Begin) {
     if (Begin == End)
@@ -1936,16 +1928,16 @@ bool hasNItems(
 /// Return true if the sequence [Begin, End) has N or more items. Runs in O(N)
 /// time. Not meant for use with random-access iterators.
 /// Can optionally take a predicate to lazily filter some items.
-template<typename IterTy,
-         typename Pred = bool (*)(const decltype(*std::declval<IterTy>()) &)>
+template <typename IterTy,
+          typename Pred = bool (*)(const decltype(*std::declval<IterTy>()) &)>
 bool hasNItemsOrMore(
     IterTy &&Begin, IterTy &&End, unsigned N,
     Pred &&ShouldBeCounted =
         [](const decltype(*std::declval<IterTy>()) &) { return true; },
     std::enable_if_t<
-        !std::is_same<typename std::iterator_traits<std::remove_reference_t<
-                          decltype(Begin)>>::iterator_category,
-                      std::random_access_iterator_tag>::value,
+        !std::is_base_of<std::random_access_iterator_tag,
+                         typename std::iterator_traits<std::remove_reference_t<
+                             decltype(Begin)>>::iterator_category>::value,
         void> * = nullptr) {
   for (; N; ++Begin) {
     if (Begin == End)
diff --git a/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h b/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h
new file mode 100644
index 0000000000000..a39c4e5413d8d
--- /dev/null
+++ b/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h
@@ -0,0 +1,86 @@
+//=- FunctionPropertiesAnalysis.h - Function Properties Analysis --*- C++ -*-=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the FunctionPropertiesInfo and FunctionPropertiesAnalysis
+// classes used to extract function properties.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FUNCTIONPROPERTIESANALYSIS_H_
+#define LLVM_FUNCTIONPROPERTIESANALYSIS_H_
+
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+class Function;
+
+class FunctionPropertiesInfo {
+public:
+  static FunctionPropertiesInfo getFunctionPropertiesInfo(const Function &F,
+                                                          const LoopInfo &LI);
+
+  void print(raw_ostream &OS) const;
+
+  /// Number of basic blocks
+  int64_t BasicBlockCount = 0;
+
+  /// Number of blocks reached from a conditional instruction, or that are
+  /// 'cases' of a SwitchInstr.
+  // FIXME: We may want to replace this with a more meaningful metric, like
+  // number of conditionally executed blocks:
+  // 'if (a) s();' would be counted here as 2 blocks, just like
+  // 'if (a) s(); else s2(); s3();' would.
+  int64_t BlocksReachedFromConditionalInstruction = 0;
+
+  /// Number of uses of this function, plus 1 if the function is callable
+  /// outside the module.
+  int64_t Uses = 0;
+
+  /// Number of direct calls made from this function to other functions
+  /// defined in this module.
+  int64_t DirectCallsToDefinedFunctions = 0;
+
+  // Load Instruction Count
+  int64_t LoadInstCount = 0;
+
+  // Store Instruction Count
+  int64_t StoreInstCount = 0;
+
+  // Maximum Loop Depth in the Function
+  int64_t MaxLoopDepth = 0;
+
+  // Number of Top Level Loops in the Function
+  int64_t TopLevelLoopCount = 0;
+};
+
+// Analysis pass
+class FunctionPropertiesAnalysis
+    : public AnalysisInfoMixin<FunctionPropertiesAnalysis> {
+
+public:
+  static AnalysisKey Key;
+
+  using Result = FunctionPropertiesInfo;
+
+  Result run(Function &F, FunctionAnalysisManager &FAM);
+};
+
+/// Printer pass for the FunctionPropertiesAnalysis results.
+class FunctionPropertiesPrinterPass
+    : public PassInfoMixin<FunctionPropertiesPrinterPass> {
+  raw_ostream &OS;
+
+public:
+  explicit FunctionPropertiesPrinterPass(raw_ostream &OS) : OS(OS) {}
+
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+};
+
+} // namespace llvm
+#endif // LLVM_FUNCTIONPROPERTIESANALYSIS_H_
\ No newline at end of file
diff --git a/llvm/include/llvm/Analysis/InlineAdvisor.h b/llvm/include/llvm/Analysis/InlineAdvisor.h
index 3480d93385a8e..c0ea40eb1e98a 100644
--- a/llvm/include/llvm/Analysis/InlineAdvisor.h
+++ b/llvm/include/llvm/Analysis/InlineAdvisor.h
@@ -14,6 +14,7 @@
 #include <vector>
 
 #include "llvm/Analysis/InlineCost.h"
+#include "llvm/Config/llvm-config.h"
 #include "llvm/IR/PassManager.h"
 
 namespace llvm {
@@ -208,6 +209,12 @@ std::unique_ptr<InlineAdvisor>
 getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM);
 #endif
 
+#ifdef LLVM_HAVE_TF_API
+std::unique_ptr<InlineAdvisor>
+getDevelopmentModeAdvisor(Module &M, ModuleAnalysisManager &MAM,
+                          std::function<bool(CallBase &)> GetDefaultAdvice);
+#endif
+
 // Default (manual policy) decision making helper APIs. Shared with the legacy
 // pass manager inliner.
 
diff --git a/llvm/include/llvm/Analysis/InlineFeaturesAnalysis.h b/llvm/include/llvm/Analysis/InlineFeaturesAnalysis.h
deleted file mode 100644
index cc3f96c424e9c..0000000000000
--- a/llvm/include/llvm/Analysis/InlineFeaturesAnalysis.h
+++ /dev/null
@@ -1,45 +0,0 @@
-//===- InlineFeaturesAnalysis.h - ML Policy Feature extraction  -*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_INLINEFEATURESANALYSIS_H_
-#define LLVM_INLINEFEATURESANALYSIS_H_
-
-#include "llvm/IR/PassManager.h"
-
-namespace llvm {
-class Function;
-
-class InlineFeaturesAnalysis
-    : public AnalysisInfoMixin<InlineFeaturesAnalysis> {
-public:
-  static AnalysisKey Key;
-  struct Result {
-    /// Number of basic blocks
-    int64_t BasicBlockCount = 0;
-
-    /// Number of blocks reached from a conditional instruction, or that are
-    /// 'cases' of a SwitchInstr.
-    // FIXME: We may want to replace this with a more meaningful metric, like
-    // number of conditionally executed blocks:
-    // 'if (a) s();' would be counted here as 2 blocks, just like
-    // 'if (a) s(); else s2(); s3();' would.
-    int64_t BlocksReachedFromConditionalInstruction = 0;
-
-    /// Number of uses of this function, plus 1 if the function is callable
-    /// outside the module.
-    int64_t Uses = 0;
-
-    /// Number of direct calls made from this function to other functions
-    /// defined in this module.
-    int64_t DirectCallsToDefinedFunctions = 0;
-  };
-  Result run(const Function &F, FunctionAnalysisManager &FAM);
-};
-
-} // namespace llvm
-#endif // LLVM_INLINEFEATURESANALYSIS_H_
\ No newline at end of file
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index a5237e9ba59e3..663b9c9935db7 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -418,7 +418,7 @@ class RuntimePointerChecking {
                       bool UseDependencies);
 
   /// Returns the checks that generateChecks created.
-  const SmallVector<RuntimePointerCheck, 4> &getChecks() const {
+  const SmallVectorImpl<RuntimePointerCheck> &getChecks() const {
     return Checks;
   }
 
diff --git a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h
index a1fea9fefc9a9..fe1891e440c26 100644
--- a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h
+++ b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h
@@ -25,7 +25,6 @@ class BasicBlock;
 class BlockFrequencyInfo;
 class CallBase;
 class Function;
-class ProfileSummary;
 
 /// Analysis providing profile information.
 ///
diff --git a/llvm/include/llvm/Analysis/RegionInfo.h b/llvm/include/llvm/Analysis/RegionInfo.h
index b0336c5597741..f93081d6f51d0 100644
--- a/llvm/include/llvm/Analysis/RegionInfo.h
+++ b/llvm/include/llvm/Analysis/RegionInfo.h
@@ -59,7 +59,6 @@
 namespace llvm {
 
 class DominanceFrontier;
-class DominatorTree;
 class Loop;
 class LoopInfo;
 class PostDominatorTree;
@@ -877,8 +876,6 @@ class RegionInfoBase {
   void verifyAnalysis() const;
 };
 
-class Region;
-
 class RegionNode : public RegionNodeBase<RegionTraits<Function>> {
 public:
   inline RegionNode(Region *Parent, BasicBlock *Entry, bool isSubRegion = false)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index b6698eefdb011..e1426e3e5192a 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -42,6 +42,7 @@ class CallBase;
 class ExtractElementInst;
 class Function;
 class GlobalValue;
+class InstCombiner;
 class IntrinsicInst;
 class LoadInst;
 class LoopAccessInfo;
@@ -56,6 +57,7 @@ class TargetLibraryInfo;
 class Type;
 class User;
 class Value;
+struct KnownBits;
 template <typename T> class Optional;
 
 /// Information about a load/store intrinsic defined by the target.
@@ -542,6 +544,29 @@ class TargetTransformInfo {
   /// target-independent defaults with information from \p L and \p SE.
   void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
                              PeelingPreferences &PP) const;
+
+  /// Targets can implement their own combinations for target-specific
+  /// intrinsics. This function will be called from the InstCombine pass every
+  /// time a target-specific intrinsic is encountered.
+  ///
+  /// \returns None to not do anything target specific or a value that will be
+  /// returned from the InstCombiner. It is possible to return null and stop
+  /// further processing of the intrinsic by returning nullptr.
+  Optional<Instruction *> instCombineIntrinsic(InstCombiner &IC,
+                                               IntrinsicInst &II) const;
+  /// Can be used to implement target-specific instruction combining.
+  /// \see instCombineIntrinsic
+  Optional<Value *>
+  simplifyDemandedUseBitsIntrinsic(InstCombiner &IC, IntrinsicInst &II,
+                                   APInt DemandedMask, KnownBits &Known,
+                                   bool &KnownBitsComputed) const;
+  /// Can be used to implement target-specific instruction combining.
+  /// \see instCombineIntrinsic
+  Optional<Value *> simplifyDemandedVectorEltsIntrinsic(
+      InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
+      APInt &UndefElts2, APInt &UndefElts3,
+      std::function<void(Instruction *, unsigned, APInt, APInt &)>
+          SimplifyAndSetOp) const;
   /// @}
 
   /// \name Scalar Target Information
@@ -1301,6 +1326,17 @@ class TargetTransformInfo::Concept {
                               AssumptionCache &AC, TargetLibraryInfo *TLI,
                               DominatorTree *DT, const LoopAccessInfo *LAI) = 0;
   virtual bool emitGetActiveLaneMask() = 0;
+  virtual Optional<Instruction *> instCombineIntrinsic(InstCombiner &IC,
+                                                       IntrinsicInst &II) = 0;
+  virtual Optional<Value *>
+  simplifyDemandedUseBitsIntrinsic(InstCombiner &IC, IntrinsicInst &II,
+                                   APInt DemandedMask, KnownBits &Known,
+                                   bool &KnownBitsComputed) = 0;
+  virtual Optional<Value *> simplifyDemandedVectorEltsIntrinsic(
+      InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
+      APInt &UndefElts2, APInt &UndefElts3,
+      std::function<void(Instruction *, unsigned, APInt, APInt &)>
+          SimplifyAndSetOp) = 0;
   virtual bool isLegalAddImmediate(int64_t Imm) = 0;
   virtual bool isLegalICmpImmediate(int64_t Imm) = 0;
   virtual bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV,
@@ -1588,6 +1624,26 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
   bool emitGetActiveLaneMask() override {
     return Impl.emitGetActiveLaneMask();
   }
+  Optional<Instruction *> instCombineIntrinsic(InstCombiner &IC,
+                                               IntrinsicInst &II) override {
+    return Impl.instCombineIntrinsic(IC, II);
+  }
+  Optional<Value *>
+  simplifyDemandedUseBitsIntrinsic(InstCombiner &IC, IntrinsicInst &II,
+                                   APInt DemandedMask, KnownBits &Known,
+                                   bool &KnownBitsComputed) override {
+    return Impl.simplifyDemandedUseBitsIntrinsic(IC, II, DemandedMask, Known,
+                                                 KnownBitsComputed);
+  }
+  Optional<Value *> simplifyDemandedVectorEltsIntrinsic(
+      InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
+      APInt &UndefElts2, APInt &UndefElts3,
+      std::function<void(Instruction *, unsigned, APInt, APInt &)>
+          SimplifyAndSetOp) override {
+    return Impl.simplifyDemandedVectorEltsIntrinsic(
+        IC, II, DemandedElts, UndefElts, UndefElts2, UndefElts3,
+        SimplifyAndSetOp);
+  }
   bool isLegalAddImmediate(int64_t Imm) override {
     return Impl.isLegalAddImmediate(Imm);
   }
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 0ce975d6d4b5d..73e5ff60a4be6 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -147,6 +147,26 @@ class TargetTransformInfoImplBase {
     return false;
   }
 
+  Optional<Instruction *> instCombineIntrinsic(InstCombiner &IC,
+                                               IntrinsicInst &II) const {
+    return None;
+  }
+
+  Optional<Value *>
+  simplifyDemandedUseBitsIntrinsic(InstCombiner &IC, IntrinsicInst &II,
+                                   APInt DemandedMask, KnownBits &Known,
+                                   bool &KnownBitsComputed) const {
+    return None;
+  }
+
+  Optional<Value *> simplifyDemandedVectorEltsIntrinsic(
+      InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
+      APInt &UndefElts2, APInt &UndefElts3,
+      std::function<void(Instruction *, unsigned, APInt, APInt &)>
+          SimplifyAndSetOp) const {
+    return None;
+  }
+
   void getUnrollingPreferences(Loop *, ScalarEvolution &,
                                TTI::UnrollingPreferences &) {}
 
@@ -652,7 +672,7 @@ class TargetTransformInfoImplBase {
 
       // In case of a vector need to pick the max between the min
       // required size for each element
-      auto *VT = cast<VectorType>(Val->getType());
+      auto *VT = cast<FixedVectorType>(Val->getType());
 
       // Assume unsigned elements
       isSigned = false;
diff --git a/llvm/include/llvm/Analysis/Utils/Local.h b/llvm/include/llvm/Analysis/Utils/Local.h
index f31b563454244..9da0c6586dac8 100644
--- a/llvm/include/llvm/Analysis/Utils/Local.h
+++ b/llvm/include/llvm/Analysis/Utils/Local.h
@@ -78,7 +78,7 @@ Value *EmitGEPOffset(IRBuilderTy *Builder, const DataLayout &DL, User *GEP,
     // Splat the index if needed.
     if (IntIdxTy->isVectorTy() && !Op->getType()->isVectorTy())
       Op = Builder->CreateVectorSplat(
-          cast<VectorType>(IntIdxTy)->getNumElements(), Op);
+          cast<FixedVectorType>(IntIdxTy)->getNumElements(), Op);
 
     // Convert to correct type.
     if (Op->getType() != IntIdxTy)
diff --git a/llvm/include/llvm/Analysis/Utils/TFUtils.h b/llvm/include/llvm/Analysis/Utils/TFUtils.h
index 2ab2c7a57d941..b93ca88ec1c93 100644
--- a/llvm/include/llvm/Analysis/Utils/TFUtils.h
+++ b/llvm/include/llvm/Analysis/Utils/TFUtils.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_ANALYSIS_UTILS_TFUTILS_H
 #define LLVM_ANALYSIS_UTILS_TFUTILS_H
 
-#include "llvm/Config/config.h"
+#include "llvm/Config/llvm-config.h"
 
 #ifdef LLVM_HAVE_TF_API
 #include "llvm/IR/LLVMContext.h"
diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index 178f61563cd7f..ce5aea2e8d34d 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -607,7 +607,8 @@ class Value;
   bool canCreatePoison(const Operator *Op);
 
   /// Return true if this function can prove that V is never undef value
-  /// or poison value.
+  /// or poison value. If V is an aggregate value or vector, check whether all
+  /// elements (except padding) are not undef or poison.
   /// Note that this is different from canCreateUndefOrPoison because the
   /// function assumes Op's operands are not poison/undef.
   ///
diff --git a/llvm/include/llvm/BinaryFormat/ELFRelocs/PowerPC64.def b/llvm/include/llvm/BinaryFormat/ELFRelocs/PowerPC64.def
index e28c9caaefaf5..ddcafa2065bd4 100644
--- a/llvm/include/llvm/BinaryFormat/ELFRelocs/PowerPC64.def
+++ b/llvm/include/llvm/BinaryFormat/ELFRelocs/PowerPC64.def
@@ -97,6 +97,7 @@
 #undef R_PPC64_DTPREL16_HIGH
 #undef R_PPC64_DTPREL16_HIGHA
 #undef R_PPC64_REL24_NOTOC
+#undef R_PPC64_PCREL_OPT
 #undef R_PPC64_PCREL34
 #undef R_PPC64_GOT_PCREL34
 #undef R_PPC64_IRELATIVE
@@ -194,6 +195,7 @@ ELF_RELOC(R_PPC64_TPREL16_HIGHA,        113)
 ELF_RELOC(R_PPC64_DTPREL16_HIGH,        114)
 ELF_RELOC(R_PPC64_DTPREL16_HIGHA,       115)
 ELF_RELOC(R_PPC64_REL24_NOTOC,          116)
+ELF_RELOC(R_PPC64_PCREL_OPT,            123)
 ELF_RELOC(R_PPC64_PCREL34,              132)
 ELF_RELOC(R_PPC64_GOT_PCREL34,          133)
 ELF_RELOC(R_PPC64_IRELATIVE,            248)
diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
index de4fe66303246..452918cde64bc 100644
--- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -644,6 +644,7 @@ enum AttributeKindCodes {
   ATTR_KIND_NO_MERGE = 66,
   ATTR_KIND_NULL_POINTER_IS_VALID = 67,
   ATTR_KIND_NOUNDEF = 68,
+  ATTR_KIND_BYREF = 69,
 };
 
 enum ComdatSelectionKindCodes {
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 0ea9a70f07f58..9ee1d13a34d67 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -477,6 +477,30 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     return BaseT::emitGetActiveLaneMask();
   }
 
+  Optional<Instruction *> instCombineIntrinsic(InstCombiner &IC,
+                                               IntrinsicInst &II) {
+    return BaseT::instCombineIntrinsic(IC, II);
+  }
+
+  Optional<Value *> simplifyDemandedUseBitsIntrinsic(InstCombiner &IC,
+                                                     IntrinsicInst &II,
+                                                     APInt DemandedMask,
+                                                     KnownBits &Known,
+                                                     bool &KnownBitsComputed) {
+    return BaseT::simplifyDemandedUseBitsIntrinsic(IC, II, DemandedMask, Known,
+                                                   KnownBitsComputed);
+  }
+
+  Optional<Value *> simplifyDemandedVectorEltsIntrinsic(
+      InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
+      APInt &UndefElts2, APInt &UndefElts3,
+      std::function<void(Instruction *, unsigned, APInt, APInt &)>
+          SimplifyAndSetOp) {
+    return BaseT::simplifyDemandedVectorEltsIntrinsic(
+        IC, II, DemandedElts, UndefElts, UndefElts2, UndefElts3,
+        SimplifyAndSetOp);
+  }
+
   int getInstructionLatency(const Instruction *I) {
     if (isa<LoadInst>(I))
       return getST()->getSchedModel().DefaultLoadLatency;
@@ -1605,7 +1629,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       }
     }
 
-    auto MinLegalCostI = std::min_element(LegalCost.begin(), LegalCost.end());
+    auto *MinLegalCostI = std::min_element(LegalCost.begin(), LegalCost.end());
     if (MinLegalCostI != LegalCost.end())
       return *MinLegalCostI;
 
diff --git a/llvm/include/llvm/CodeGen/DbgEntityHistoryCalculator.h b/llvm/include/llvm/CodeGen/DbgEntityHistoryCalculator.h
index f7fc74a27fcab..5a1085ea3a37c 100644
--- a/llvm/include/llvm/CodeGen/DbgEntityHistoryCalculator.h
+++ b/llvm/include/llvm/CodeGen/DbgEntityHistoryCalculator.h
@@ -12,6 +12,7 @@
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/LexicalScopes.h"
 #include <utility>
 
 namespace llvm {
@@ -52,6 +53,8 @@ class DbgValueHistoryMap {
   ///   register-described debug values that have their end index
   ///   set to this entry's position in the entry vector.
   class Entry {
+    friend DbgValueHistoryMap;
+
   public:
     enum EntryKind { DbgValue, Clobber };
 
@@ -89,6 +92,8 @@ class DbgValueHistoryMap {
     return Entries[Index];
   }
 
+  /// Drop location ranges which exist entirely outside each variable's scope.
+  void trimLocationRanges(const MachineFunction &MF, LexicalScopes &LScopes);
   bool empty() const { return VarEntries.empty(); }
   void clear() { VarEntries.clear(); }
   EntriesMap::const_iterator begin() const { return VarEntries.begin(); }
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CSEInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/CSEInfo.h
index 8bd9e9443552e..a705e1c278175 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CSEInfo.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CSEInfo.h
@@ -16,14 +16,12 @@
 #include "llvm/CodeGen/CSEConfigBase.h"
 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
 #include "llvm/CodeGen/GlobalISel/GISelWorkList.h"
-#include "llvm/CodeGen/GlobalISel/Utils.h"
-#include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/IR/PassManager.h"
-#include "llvm/Pass.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/CodeGen.h"
 
 namespace llvm {
+class MachineBasicBlock;
 
 /// A class that wraps MachineInstrs and derives from FoldingSetNode in order to
 /// be uniqued in a CSEMap. The tradeoff here is extra memory allocations for
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
index 4d60dffb91dbd..6eb5c853bb362 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
@@ -111,15 +111,18 @@ class CallLowering {
   /// argument should go, exactly what happens can vary slightly. This
   /// class abstracts the differences.
   struct ValueHandler {
-    ValueHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
-                 CCAssignFn *AssignFn)
-      : MIRBuilder(MIRBuilder), MRI(MRI), AssignFn(AssignFn) {}
+    ValueHandler(bool IsIncoming, MachineIRBuilder &MIRBuilder,
+                 MachineRegisterInfo &MRI, CCAssignFn *AssignFn)
+        : MIRBuilder(MIRBuilder), MRI(MRI), AssignFn(AssignFn),
+          IsIncomingArgumentHandler(IsIncoming) {}
 
     virtual ~ValueHandler() = default;
 
     /// Returns true if the handler is dealing with incoming arguments,
     /// i.e. those that move values from some physical location to vregs.
-    virtual bool isIncomingArgumentHandler() const = 0;
+    bool isIncomingArgumentHandler() const {
+      return IsIncomingArgumentHandler;
+    }
 
     /// Materialize a VReg containing the address of the specified
     /// stack-based object. This is either based on a FrameIndex or
@@ -147,6 +150,7 @@ class CallLowering {
     virtual void assignValueToAddress(const ArgInfo &Arg, Register Addr,
                                       uint64_t Size, MachinePointerInfo &MPO,
                                       CCValAssign &VA) {
+      assert(Arg.Regs.size() == 1);
       assignValueToAddress(Arg.Regs[0], Addr, Size, MPO, VA);
     }
 
@@ -177,9 +181,22 @@ class CallLowering {
     CCAssignFn *AssignFn;
 
   private:
+    bool IsIncomingArgumentHandler;
     virtual void anchor();
   };
 
+  struct IncomingValueHandler : public ValueHandler {
+    IncomingValueHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
+                         CCAssignFn *AssignFn)
+        : ValueHandler(true, MIRBuilder, MRI, AssignFn) {}
+  };
+
+  struct OutgoingValueHandler : public ValueHandler {
+    OutgoingValueHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
+                         CCAssignFn *AssignFn)
+        : ValueHandler(false, MIRBuilder, MRI, AssignFn) {}
+  };
+
 protected:
   /// Getter for generic TargetLowering class.
   const TargetLowering *getTLI() const {
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index c317b7ed4c54b..e694e7ad2c834 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -107,8 +107,8 @@ class CombinerHelper {
   bool matchCombineIndexedLoadStore(MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo);
   void applyCombineIndexedLoadStore(MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo);
 
-  bool matchSextAlreadyExtended(MachineInstr &MI);
-  bool applySextAlreadyExtended(MachineInstr &MI);
+  bool matchSextTruncSextLoad(MachineInstr &MI);
+  bool applySextTruncSextLoad(MachineInstr &MI);
 
   bool matchElideBrByInvertingCond(MachineInstr &MI);
   void applyElideBrByInvertingCond(MachineInstr &MI);
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GISelKnownBits.h b/llvm/include/llvm/CodeGen/GlobalISel/GISelKnownBits.h
index 55cf54d6e9469..f5662a27debe3 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GISelKnownBits.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GISelKnownBits.h
@@ -13,13 +13,11 @@
 #ifndef LLVM_CODEGEN_GLOBALISEL_KNOWNBITSINFO_H
 #define LLVM_CODEGEN_GLOBALISEL_KNOWNBITSINFO_H
 
-#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/DenseMap.h"
 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/Register.h"
-#include "llvm/IR/PassManager.h"
 #include "llvm/InitializePasses.h"
-#include "llvm/Pass.h"
 #include "llvm/Support/KnownBits.h"
 
 namespace llvm {
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index 058aacf38634d..0a490249f1568 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -302,6 +302,7 @@ class LegalizerHelper {
 
   LegalizeResult lowerFPTRUNC_F64_TO_F16(MachineInstr &MI);
   LegalizeResult lowerFPTRUNC(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
+  LegalizeResult lowerFPOWI(MachineInstr &MI);
 
   LegalizeResult lowerMinMax(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
   LegalizeResult lowerFCopySign(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
@@ -316,6 +317,8 @@ class LegalizerHelper {
   LegalizeResult lowerExtract(MachineInstr &MI);
   LegalizeResult lowerInsert(MachineInstr &MI);
   LegalizeResult lowerSADDO_SSUBO(MachineInstr &MI);
+  LegalizeResult lowerAddSubSatToMinMax(MachineInstr &MI);
+  LegalizeResult lowerAddSubSatToAddoSubo(MachineInstr &MI);
   LegalizeResult lowerBswap(MachineInstr &MI);
   LegalizeResult lowerBitreverse(MachineInstr &MI);
   LegalizeResult lowerReadWriteRegister(MachineInstr &MI);
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
index 61e0418757bce..b50d7cd7a5f91 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
@@ -308,6 +308,11 @@ LegalizeMutation changeElementTo(unsigned TypeIdx, unsigned FromTypeIdx);
 /// Keep the same scalar or element type as the given type.
 LegalizeMutation changeElementTo(unsigned TypeIdx, LLT Ty);
 
+/// Change the scalar size or element size to have the same scalar size as type
+/// index \p FromIndex. Unlike changeElementTo, this discards pointer types and
+/// only changes the size.
+LegalizeMutation changeElementSizeTo(unsigned TypeIdx, unsigned FromTypeIdx);
+
 /// Widen the scalar type or vector element type for the given type index to the
 /// next power of 2.
 LegalizeMutation widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned Min = 0);
@@ -696,6 +701,13 @@ class LegalizeRuleSet {
     markAllIdxsAsCovered();
     return actionIf(LegalizeAction::NarrowScalar, Predicate, Mutation);
   }
+  /// Narrow the scalar, specified in mutation, when type indexes 0 and 1 is any
+  /// type pair in the given list.
+  LegalizeRuleSet &
+  narrowScalarFor(std::initializer_list<std::pair<LLT, LLT>> Types,
+                  LegalizeMutation Mutation) {
+    return actionFor(LegalizeAction::NarrowScalar, Types, Mutation);
+  }
 
   /// Add more elements to reach the type selected by the mutation if the
   /// predicate is true.
@@ -857,7 +869,10 @@ class LegalizeRuleSet {
     return actionIf(
         LegalizeAction::NarrowScalar,
         [=](const LegalityQuery &Query) {
-          return scalarWiderThan(TypeIdx, Ty.getSizeInBits()) && Predicate(Query);
+          const LLT QueryTy = Query.Types[TypeIdx];
+          return QueryTy.isScalar() &&
+                 QueryTy.getSizeInBits() > Ty.getSizeInBits() &&
+                 Predicate(Query);
         },
         changeElementTo(typeIdx(TypeIdx), Ty));
   }
@@ -883,11 +898,25 @@ class LegalizeRuleSet {
           return Query.Types[LargeTypeIdx].getScalarSizeInBits() >
                  Query.Types[TypeIdx].getSizeInBits();
         },
+        LegalizeMutations::changeElementSizeTo(TypeIdx, LargeTypeIdx));
+  }
+
+  /// Narrow the scalar to match the size of another.
+  LegalizeRuleSet &maxScalarSameAs(unsigned TypeIdx, unsigned NarrowTypeIdx) {
+    typeIdx(TypeIdx);
+    return narrowScalarIf(
         [=](const LegalityQuery &Query) {
-          LLT T = Query.Types[LargeTypeIdx];
-          return std::make_pair(TypeIdx,
-                                T.isVector() ? T.getElementType() : T);
-        });
+          return Query.Types[NarrowTypeIdx].getScalarSizeInBits() <
+                 Query.Types[TypeIdx].getSizeInBits();
+        },
+        LegalizeMutations::changeElementSizeTo(TypeIdx, NarrowTypeIdx));
+  }
+
+  /// Change the type \p TypeIdx to have the same scalar size as type \p
+  /// SameSizeIdx.
+  LegalizeRuleSet &scalarSameSizeAs(unsigned TypeIdx, unsigned SameSizeIdx) {
+    return minScalarSameAs(TypeIdx, SameSizeIdx)
+          .maxScalarSameAs(TypeIdx, SameSizeIdx);
   }
 
   /// Conditionally widen the scalar or elt to match the size of another.
@@ -1207,6 +1236,12 @@ class LegalizerInfo {
   bool isLegal(const LegalityQuery &Query) const {
     return getAction(Query).Action == LegalizeAction::Legal;
   }
+
+  bool isLegalOrCustom(const LegalityQuery &Query) const {
+    auto Action = getAction(Query).Action;
+    return Action == LegalizeAction::Legal || Action == LegalizeAction::Custom;
+  }
+
   bool isLegal(const MachineInstr &MI, const MachineRegisterInfo &MRI) const;
   bool isLegalOrCustom(const MachineInstr &MI,
                        const MachineRegisterInfo &MRI) const;
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index d6498345f25c8..3ad2effea1b61 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -20,7 +20,7 @@
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DebugLoc.h"
-
+#include "llvm/IR/Module.h"
 
 namespace llvm {
 
@@ -1583,6 +1583,13 @@ class MachineIRBuilder {
     return buildInstr(TargetOpcode::G_FEXP2, {Dst}, {Src}, Flags);
   }
 
+  /// Build and insert \p Dst = G_FPOW \p Src0, \p Src1
+  MachineInstrBuilder buildFPow(const DstOp &Dst, const SrcOp &Src0,
+                                const SrcOp &Src1,
+                                Optional<unsigned> Flags = None) {
+    return buildInstr(TargetOpcode::G_FPOW, {Dst}, {Src0, Src1}, Flags);
+  }
+
   /// Build and insert \p Res = G_FCOPYSIGN \p Op0, \p Op1
   MachineInstrBuilder buildFCopysign(const DstOp &Dst, const SrcOp &Src0,
                                      const SrcOp &Src1) {
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
index 42d86917721a2..8eb95a8899913 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
@@ -190,16 +190,27 @@ inline bool isKnownNeverSNaN(Register Val, const MachineRegisterInfo &MRI) {
 
 Align inferAlignFromPtrInfo(MachineFunction &MF, const MachinePointerInfo &MPO);
 
-/// Return the least common multiple type of \p Ty0 and \p Ty1, by changing
-/// the number of vector elements or scalar bitwidth. The intent is a
-/// G_MERGE_VALUES can be constructed from \p Ty0 elements, and unmerged into
-/// \p Ty1.
-LLT getLCMType(LLT Ty0, LLT Ty1);
-
-/// Return a type that is greatest common divisor of \p OrigTy and \p
-/// TargetTy. This will either change the number of vector elements, or
-/// bitwidth of scalars. The intent is the result type can be used as the
-/// result of a G_UNMERGE_VALUES from \p OrigTy.
+/// Return the least common multiple type of \p OrigTy and \p TargetTy, by changing the
+/// number of vector elements or scalar bitwidth. The intent is a
+/// G_MERGE_VALUES, G_BUILD_VECTOR, or G_CONCAT_VECTORS can be constructed from
+/// \p OrigTy elements, and unmerged into \p TargetTy
+LLVM_READNONE
+LLT getLCMType(LLT OrigTy, LLT TargetTy);
+
+/// Return a type where the total size is the greatest common divisor of \p
+/// OrigTy and \p TargetTy. This will try to either change the number of vector
+/// elements, or bitwidth of scalars. The intent is the result type can be used
+/// as the result of a G_UNMERGE_VALUES from \p OrigTy, and then some
+/// combination of G_MERGE_VALUES, G_BUILD_VECTOR and G_CONCAT_VECTORS (possibly
+/// with intermediate casts) can re-form \p TargetTy.
+///
+/// If these are vectors with different element types, this will try to produce
+/// a vector with a compatible total size, but the element type of \p OrigTy. If
+/// this can't be satisfied, this will produce a scalar smaller than the
+/// original vector elements.
+///
+/// In the worst case, this returns LLT::scalar(1)
+LLVM_READNONE
 LLT getGCDType(LLT OrigTy, LLT TargetTy);
 
 } // End namespace llvm.
diff --git a/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h b/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
index 0f8d69ebd7dab..7ce11c784b08f 100644
--- a/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
+++ b/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
@@ -58,18 +58,29 @@ class MachineBlockFrequencyInfo : public MachineFunctionPass {
   /// information. Please note that initial frequency is equal to 1024. It means
   /// that we should not rely on the value itself, but only on the comparison to
   /// the other block frequencies. We do this to avoid using of floating points.
-  ///
+  /// For example, to get the frequency of a block relative to the entry block,
+  /// divide the integral value returned by this function (the
+  /// BlockFrequency::getFrequency() value) by getEntryFreq().
   BlockFrequency getBlockFreq(const MachineBasicBlock *MBB) const;
 
+  /// Compute the frequency of the block, relative to the entry block.
+  /// This API assumes getEntryFreq() is non-zero.
+  float getBlockFreqRelativeToEntryBlock(const MachineBasicBlock *MBB) const {
+    return getBlockFreq(MBB).getFrequency() * (1.0f / getEntryFreq());
+  }
+
   Optional<uint64_t> getBlockProfileCount(const MachineBasicBlock *MBB) const;
   Optional<uint64_t> getProfileCountFromFreq(uint64_t Freq) const;
 
-  bool isIrrLoopHeader(const MachineBasicBlock *MBB);
+  bool isIrrLoopHeader(const MachineBasicBlock *MBB) const;
 
   void setBlockFreq(const MachineBasicBlock *MBB, uint64_t Freq);
 
   const MachineFunction *getFunction() const;
   const MachineBranchProbabilityInfo *getMBPI() const;
+
+  /// Pop up a ghostview window with the current block frequency propagation
+  /// rendered using dot.
   void view(const Twine &Name, bool isSimple = true) const;
 
   // Print the block frequency Freq to OS using the current functions entry
@@ -81,6 +92,8 @@ class MachineBlockFrequencyInfo : public MachineFunctionPass {
   raw_ostream &printBlockFreq(raw_ostream &OS,
                               const MachineBasicBlock *MBB) const;
 
+  /// Divide a block's BlockFrequency::getFrequency() value by this value to
+  /// obtain the entry block - relative frequency of said block.
   uint64_t getEntryFreq() const;
 };
 
diff --git a/llvm/include/llvm/CodeGen/TargetCallingConv.h b/llvm/include/llvm/CodeGen/TargetCallingConv.h
index 347d7ff404047..7baf5b2f7834b 100644
--- a/llvm/include/llvm/CodeGen/TargetCallingConv.h
+++ b/llvm/include/llvm/CodeGen/TargetCallingConv.h
@@ -31,6 +31,7 @@ namespace ISD {
     unsigned IsInReg : 1;    ///< Passed in register
     unsigned IsSRet : 1;     ///< Hidden struct-ret ptr
     unsigned IsByVal : 1;    ///< Struct passed by value
+    unsigned IsByRef : 1;    ///< Passed in memory
     unsigned IsNest : 1;     ///< Nested fn static chain
     unsigned IsReturned : 1; ///< Always returned
     unsigned IsSplit : 1;
@@ -43,25 +44,31 @@ namespace ISD {
     unsigned IsHva : 1;        ///< HVA field for
     unsigned IsHvaStart : 1;   ///< HVA structure start
     unsigned IsSecArgPass : 1; ///< Second argument
-    unsigned ByValAlign : 4;   ///< Log 2 of byval alignment
+    unsigned ByValOrByRefAlign : 4; ///< Log 2 of byval/byref alignment
     unsigned OrigAlign : 5;    ///< Log 2 of original alignment
     unsigned IsInConsecutiveRegsLast : 1;
     unsigned IsInConsecutiveRegs : 1;
     unsigned IsCopyElisionCandidate : 1; ///< Argument copy elision candidate
     unsigned IsPointer : 1;
 
-    unsigned ByValSize; ///< Byval struct size
+    unsigned ByValOrByRefSize; ///< Byval or byref struct size
 
     unsigned PointerAddrSpace; ///< Address space of pointer argument
 
+    /// Set the alignment used by byref or byval parameters.
+    void setAlignImpl(Align A) {
+      ByValOrByRefAlign = encode(A);
+      assert(getNonZeroByValAlign() == A && "bitfield overflow");
+    }
+
   public:
     ArgFlagsTy()
-        : IsZExt(0), IsSExt(0), IsInReg(0), IsSRet(0), IsByVal(0), IsNest(0),
-          IsReturned(0), IsSplit(0), IsInAlloca(0), IsPreallocated(0),
+      : IsZExt(0), IsSExt(0), IsInReg(0), IsSRet(0), IsByVal(0), IsByRef(0),
+          IsNest(0), IsReturned(0), IsSplit(0), IsInAlloca(0), IsPreallocated(0),
           IsSplitEnd(0), IsSwiftSelf(0), IsSwiftError(0), IsCFGuardTarget(0),
-          IsHva(0), IsHvaStart(0), IsSecArgPass(0), ByValAlign(0), OrigAlign(0),
-          IsInConsecutiveRegsLast(0), IsInConsecutiveRegs(0),
-          IsCopyElisionCandidate(0), IsPointer(0), ByValSize(0),
+          IsHva(0), IsHvaStart(0), IsSecArgPass(0), ByValOrByRefAlign(0),
+          OrigAlign(0), IsInConsecutiveRegsLast(0), IsInConsecutiveRegs(0),
+          IsCopyElisionCandidate(0), IsPointer(0), ByValOrByRefSize(0),
           PointerAddrSpace(0) {
       static_assert(sizeof(*this) == 3 * sizeof(unsigned), "flags are too big");
     }
@@ -81,6 +88,9 @@ namespace ISD {
     bool isByVal() const { return IsByVal; }
     void setByVal() { IsByVal = 1; }
 
+    bool isByRef() const { return IsByRef; }
+    void setByRef() { IsByRef = 1; }
+
     bool isInAlloca() const { return IsInAlloca; }
     void setInAlloca() { IsInAlloca = 1; }
 
@@ -131,17 +141,22 @@ namespace ISD {
 
     LLVM_ATTRIBUTE_DEPRECATED(unsigned getByValAlign() const,
                               "Use getNonZeroByValAlign() instead") {
-      MaybeAlign A = decodeMaybeAlign(ByValAlign);
+      MaybeAlign A = decodeMaybeAlign(ByValOrByRefAlign);
       return A ? A->value() : 0;
     }
     Align getNonZeroByValAlign() const {
-      MaybeAlign A = decodeMaybeAlign(ByValAlign);
+      MaybeAlign A = decodeMaybeAlign(ByValOrByRefAlign);
       assert(A && "ByValAlign must be defined");
       return *A;
     }
     void setByValAlign(Align A) {
-      ByValAlign = encode(A);
-      assert(getNonZeroByValAlign() == A && "bitfield overflow");
+      assert(isByVal() && !isByRef());
+      setAlignImpl(A);
+    }
+
+    void setByRefAlign(Align A) {
+      assert(!isByVal() && isByRef());
+      setAlignImpl(A);
     }
 
     LLVM_ATTRIBUTE_DEPRECATED(unsigned getOrigAlign() const,
@@ -157,8 +172,23 @@ namespace ISD {
       assert(getNonZeroOrigAlign() == A && "bitfield overflow");
     }
 
-    unsigned getByValSize() const { return ByValSize; }
-    void setByValSize(unsigned S) { ByValSize = S; }
+    unsigned getByValSize() const {
+      assert(isByVal() && !isByRef());
+      return ByValOrByRefSize;
+    }
+    void setByValSize(unsigned S) {
+      assert(isByVal() && !isByRef());
+      ByValOrByRefSize = S;
+    }
+
+    unsigned getByRefSize() const {
+      assert(!isByVal() && isByRef());
+      return ByValOrByRefSize;
+    }
+    void setByRefSize(unsigned S) {
+      assert(!isByVal() && isByRef());
+      ByValOrByRefSize = S;
+    }
 
     unsigned getPointerAddrSpace() const { return PointerAddrSpace; }
     void setPointerAddrSpace(unsigned AS) { PointerAddrSpace = AS; }
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 06f2b3ca38ea7..a92761abd2f82 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -278,6 +278,7 @@ class TargetLoweringBase {
     bool IsSRet : 1;
     bool IsNest : 1;
     bool IsByVal : 1;
+    bool IsByRef : 1;
     bool IsInAlloca : 1;
     bool IsPreallocated : 1;
     bool IsReturned : 1;
@@ -290,7 +291,7 @@ class TargetLoweringBase {
 
     ArgListEntry()
         : IsSExt(false), IsZExt(false), IsInReg(false), IsSRet(false),
-          IsNest(false), IsByVal(false), IsInAlloca(false),
+          IsNest(false), IsByVal(false), IsByRef(false), IsInAlloca(false),
           IsPreallocated(false), IsReturned(false), IsSwiftSelf(false),
           IsSwiftError(false), IsCFGuardTarget(false) {}
 
diff --git a/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h b/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
index 6e2c0973e3547..a333d971bbcb9 100644
--- a/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
+++ b/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
@@ -246,7 +246,7 @@ class TargetLoweringObjectFileXCOFF : public TargetLoweringObjectFile {
                                    const Constant *C,
                                    Align &Alignment) const override;
 
-  static XCOFF::StorageClass getStorageClassForGlobal(const GlobalObject *GO);
+  static XCOFF::StorageClass getStorageClassForGlobal(const GlobalValue *GV);
 
   MCSection *
   getSectionForFunctionDescriptor(const Function *F,
@@ -263,7 +263,7 @@ class TargetLoweringObjectFileXCOFF : public TargetLoweringObjectFile {
   MCSymbol *getTargetSymbol(const GlobalValue *GV,
                             const TargetMachine &TM) const override;
 
-  MCSymbol *getFunctionEntryPointSymbol(const Function *F,
+  MCSymbol *getFunctionEntryPointSymbol(const GlobalValue *Func,
                                         const TargetMachine &TM) const override;
 };
 
diff --git a/llvm/include/llvm/CodeGen/TargetPassConfig.h b/llvm/include/llvm/CodeGen/TargetPassConfig.h
index a18c8b16bf1c0..fc52452169416 100644
--- a/llvm/include/llvm/CodeGen/TargetPassConfig.h
+++ b/llvm/include/llvm/CodeGen/TargetPassConfig.h
@@ -187,7 +187,7 @@ class TargetPassConfig : public ImmutablePass {
 
   /// Insert InsertedPassID pass after TargetPassID pass.
   void insertPass(AnalysisID TargetPassID, IdentifyingPassPtr InsertedPassID,
-                  bool VerifyAfter = true, bool PrintAfter = true);
+                  bool VerifyAfter = true);
 
   /// Allow the target to enable a specific standard pass by default.
   void enablePass(AnalysisID PassID) { substitutePass(PassID, PassID); }
@@ -319,8 +319,8 @@ class TargetPassConfig : public ImmutablePass {
 
   /// Add standard passes after a pass that has just been added. For example,
   /// the MachineVerifier if it is enabled.
-  void addMachinePostPasses(const std::string &Banner, bool AllowPrint = true,
-                            bool AllowVerify = true, bool AllowStrip = true);
+  void addMachinePostPasses(const std::string &Banner, bool AllowVerify = true,
+                            bool AllowStrip = true);
 
   /// Check whether or not GlobalISel should abort on error.
   /// When this is disabled, GlobalISel will fall back on SDISel instead of
@@ -441,21 +441,16 @@ class TargetPassConfig : public ImmutablePass {
 
   /// Add a CodeGen pass at this point in the pipeline after checking overrides.
   /// Return the pass that was added, or zero if no pass was added.
-  /// @p printAfter    if true and adding a machine function pass add an extra
-  ///                  machine printer pass afterwards
   /// @p verifyAfter   if true and adding a machine function pass add an extra
   ///                  machine verification pass afterwards.
-  AnalysisID addPass(AnalysisID PassID, bool verifyAfter = true,
-                     bool printAfter = true);
+  AnalysisID addPass(AnalysisID PassID, bool verifyAfter = true);
 
   /// Add a pass to the PassManager if that pass is supposed to be run, as
   /// determined by the StartAfter and StopAfter options. Takes ownership of the
   /// pass.
-  /// @p printAfter    if true and adding a machine function pass add an extra
-  ///                  machine printer pass afterwards
   /// @p verifyAfter   if true and adding a machine function pass add an extra
   ///                  machine verification pass afterwards.
-  void addPass(Pass *P, bool verifyAfter = true, bool printAfter = true);
+  void addPass(Pass *P, bool verifyAfter = true);
 
   /// addMachinePasses helper to create the target-selected or overriden
   /// regalloc pass.
diff --git a/llvm/include/llvm/Config/config.h.cmake b/llvm/include/llvm/Config/config.h.cmake
index 290f74bd02d26..298fab318c10d 100644
--- a/llvm/include/llvm/Config/config.h.cmake
+++ b/llvm/include/llvm/Config/config.h.cmake
@@ -109,9 +109,6 @@
 /* Define to 1 if you have the `pthread_setname_np' function. */
 #cmakedefine HAVE_PTHREAD_SETNAME_NP ${HAVE_PTHREAD_SETNAME_NP}
 
-/* Define to 1 if you have the `z' library (-lz). */
-#cmakedefine HAVE_LIBZ ${HAVE_LIBZ}
-
 /* Define to 1 if you have the <link.h> header file. */
 #cmakedefine HAVE_LINK_H ${HAVE_LINK_H}
 
@@ -220,9 +217,6 @@
 /* Define to 1 if you have the <valgrind/valgrind.h> header file. */
 #cmakedefine HAVE_VALGRIND_VALGRIND_H ${HAVE_VALGRIND_VALGRIND_H}
 
-/* Define to 1 if you have the <zlib.h> header file. */
-#cmakedefine HAVE_ZLIB_H ${HAVE_ZLIB_H}
-
 /* Have host's _alloca */
 #cmakedefine HAVE__ALLOCA ${HAVE__ALLOCA}
 
diff --git a/llvm/include/llvm/Config/llvm-config.h.cmake b/llvm/include/llvm/Config/llvm-config.h.cmake
index 82b682ddb3dc5..01892bc5b610b 100644
--- a/llvm/include/llvm/Config/llvm-config.h.cmake
+++ b/llvm/include/llvm/Config/llvm-config.h.cmake
@@ -82,4 +82,7 @@
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
+/* Define if LLVM was built with a dependency to the tensorflow compiler */
+#cmakedefine LLVM_HAVE_TF_AOT
+
 #endif
diff --git a/llvm/include/llvm/DebugInfo/CodeView/DebugSubsectionVisitor.h b/llvm/include/llvm/DebugInfo/CodeView/DebugSubsectionVisitor.h
index 720b1b49581fa..624a623e75b80 100644
--- a/llvm/include/llvm/DebugInfo/CodeView/DebugSubsectionVisitor.h
+++ b/llvm/include/llvm/DebugInfo/CodeView/DebugSubsectionVisitor.h
@@ -10,10 +10,8 @@
 #define LLVM_DEBUGINFO_CODEVIEW_MODULEDEBUGFRAGMENTVISITOR_H
 
 #include "llvm/DebugInfo/CodeView/CodeView.h"
-#include "llvm/DebugInfo/CodeView/DebugSubsectionRecord.h"
 #include "llvm/DebugInfo/CodeView/StringsAndChecksums.h"
 #include "llvm/Support/Error.h"
-#include <cstdint>
 
 namespace llvm {
 
@@ -30,7 +28,6 @@ class DebugStringTableSubsectionRef;
 class DebugSymbolRVASubsectionRef;
 class DebugSymbolsSubsectionRef;
 class DebugUnknownSubsectionRef;
-class StringsAndChecksumsRef;
 
 class DebugSubsectionVisitor {
 public:
diff --git a/llvm/include/llvm/DebugInfo/MSF/MappedBlockStream.h b/llvm/include/llvm/DebugInfo/MSF/MappedBlockStream.h
index 593d781b990e1..473c89e8106f3 100644
--- a/llvm/include/llvm/DebugInfo/MSF/MappedBlockStream.h
+++ b/llvm/include/llvm/DebugInfo/MSF/MappedBlockStream.h
@@ -24,8 +24,6 @@
 namespace llvm {
 namespace msf {
 
-struct MSFLayout;
-
 /// MappedBlockStream represents data stored in an MSF file into chunks of a
 /// particular size (called the Block Size), and whose chunks may not be
 /// necessarily contiguous.  The arrangement of these chunks MSF the file
diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/NativeFunctionSymbol.h b/llvm/include/llvm/DebugInfo/PDB/Native/NativeFunctionSymbol.h
index 4adf89f0d69aa..9d3ba8895be2f 100644
--- a/llvm/include/llvm/DebugInfo/PDB/Native/NativeFunctionSymbol.h
+++ b/llvm/include/llvm/DebugInfo/PDB/Native/NativeFunctionSymbol.h
@@ -30,7 +30,6 @@ class NativeFunctionSymbol : public NativeRawSymbol {
   uint32_t getAddressOffset() const override;
   uint32_t getAddressSection() const override;
   std::string getName() const override;
-  PDB_SymType getSymTag() const override;
   uint64_t getLength() const override;
   uint32_t getRelativeVirtualAddress() const override;
   uint64_t getVirtualAddress() const override;
diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/NativeLineNumber.h b/llvm/include/llvm/DebugInfo/PDB/Native/NativeLineNumber.h
index a7ce82c70b08c..5dedc70f11ba6 100644
--- a/llvm/include/llvm/DebugInfo/PDB/Native/NativeLineNumber.h
+++ b/llvm/include/llvm/DebugInfo/PDB/Native/NativeLineNumber.h
@@ -22,7 +22,7 @@ class NativeLineNumber : public IPDBLineNumber {
                             const codeview::LineInfo Line,
                             uint32_t ColumnNumber, uint32_t Length,
                             uint32_t Section, uint32_t Offset,
-                            uint32_t SrcFileId);
+                            uint32_t SrcFileId, uint32_t CompilandId);
 
   uint32_t getLineNumber() const override;
   uint32_t getLineNumberEnd() const override;
@@ -45,6 +45,7 @@ class NativeLineNumber : public IPDBLineNumber {
   uint32_t Offset;
   uint32_t Length;
   uint32_t SrcFileId;
+  uint32_t CompilandId;
 };
 } // namespace pdb
 } // namespace llvm
diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/NativePublicSymbol.h b/llvm/include/llvm/DebugInfo/PDB/Native/NativePublicSymbol.h
index 0a1451530f188..9f410e27f4cbe 100644
--- a/llvm/include/llvm/DebugInfo/PDB/Native/NativePublicSymbol.h
+++ b/llvm/include/llvm/DebugInfo/PDB/Native/NativePublicSymbol.h
@@ -30,7 +30,6 @@ class NativePublicSymbol : public NativeRawSymbol {
   uint32_t getAddressOffset() const override;
   uint32_t getAddressSection() const override;
   std::string getName() const override;
-  PDB_SymType getSymTag() const override;
   uint32_t getRelativeVirtualAddress() const override;
   uint64_t getVirtualAddress() const override;
 
diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/SymbolCache.h b/llvm/include/llvm/DebugInfo/PDB/Native/SymbolCache.h
index 90fd19a7a2fba..e78c53451d8ef 100644
--- a/llvm/include/llvm/DebugInfo/PDB/Native/SymbolCache.h
+++ b/llvm/include/llvm/DebugInfo/PDB/Native/SymbolCache.h
@@ -61,7 +61,7 @@ class SymbolCache {
   DenseMap<uint32_t, SymIndexId> GlobalOffsetToSymbolId;
 
   /// Map from segment and code offset to SymIndexId.
-  DenseMap<std::pair<uint32_t, uint32_t>, SymIndexId> AddressToFunctionSymId;
+  DenseMap<std::pair<uint32_t, uint32_t>, SymIndexId> AddressToSymbolId;
   DenseMap<std::pair<uint32_t, uint32_t>, SymIndexId> AddressToPublicSymId;
 
   /// Map from virtual address to module index.
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h
index e0cfd8bf24099..78e3ceef50e21 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h
@@ -62,14 +62,33 @@ class TrampolinePool {
       JITTargetAddress TrampolineAddr,
       NotifyLandingResolvedFunction OnLandingResolved) const>;
 
-  virtual ~TrampolinePool() {}
+  virtual ~TrampolinePool();
 
   /// Get an available trampoline address.
   /// Returns an error if no trampoline can be created.
-  virtual Expected<JITTargetAddress> getTrampoline() = 0;
+  Expected<JITTargetAddress> getTrampoline() {
+    std::lock_guard<std::mutex> Lock(TPMutex);
+    if (AvailableTrampolines.empty()) {
+      if (auto Err = grow())
+        return std::move(Err);
+    }
+    assert(!AvailableTrampolines.empty() && "Failed to grow trampoline pool");
+    auto TrampolineAddr = AvailableTrampolines.back();
+    AvailableTrampolines.pop_back();
+    return TrampolineAddr;
+  }
 
-private:
-  virtual void anchor();
+  /// Returns the given trampoline to the pool for re-use.
+  void releaseTrampoline(JITTargetAddress TrampolineAddr) {
+    std::lock_guard<std::mutex> Lock(TPMutex);
+    AvailableTrampolines.push_back(TrampolineAddr);
+  }
+
+protected:
+  virtual Error grow() = 0;
+
+  std::mutex TPMutex;
+  std::vector<JITTargetAddress> AvailableTrampolines;
 };
 
 /// A trampoline pool for trampolines within the current process.
@@ -90,26 +109,6 @@ template <typename ORCABI> class LocalTrampolinePool : public TrampolinePool {
     return std::move(LTP);
   }
 
-  /// Get a free trampoline. Returns an error if one can not be provided (e.g.
-  /// because the pool is empty and can not be grown).
-  Expected<JITTargetAddress> getTrampoline() override {
-    std::lock_guard<std::mutex> Lock(LTPMutex);
-    if (AvailableTrampolines.empty()) {
-      if (auto Err = grow())
-        return std::move(Err);
-    }
-    assert(!AvailableTrampolines.empty() && "Failed to grow trampoline pool");
-    auto TrampolineAddr = AvailableTrampolines.back();
-    AvailableTrampolines.pop_back();
-    return TrampolineAddr;
-  }
-
-  /// Returns the given trampoline to the pool for re-use.
-  void releaseTrampoline(JITTargetAddress TrampolineAddr) {
-    std::lock_guard<std::mutex> Lock(LTPMutex);
-    AvailableTrampolines.push_back(TrampolineAddr);
-  }
-
 private:
   static JITTargetAddress reenter(void *TrampolinePoolPtr, void *TrampolineId) {
     LocalTrampolinePool<ORCABI> *TrampolinePool =
@@ -154,8 +153,8 @@ template <typename ORCABI> class LocalTrampolinePool : public TrampolinePool {
     }
   }
 
-  Error grow() {
-    assert(this->AvailableTrampolines.empty() && "Growing prematurely?");
+  Error grow() override {
+    assert(AvailableTrampolines.empty() && "Growing prematurely?");
 
     std::error_code EC;
     auto TrampolineBlock =
@@ -175,7 +174,7 @@ template <typename ORCABI> class LocalTrampolinePool : public TrampolinePool {
         pointerToJITTargetAddress(ResolverBlock.base()), NumTrampolines);
 
     for (unsigned I = 0; I < NumTrampolines; ++I)
-      this->AvailableTrampolines.push_back(pointerToJITTargetAddress(
+      AvailableTrampolines.push_back(pointerToJITTargetAddress(
           TrampolineMem + (I * ORCABI::TrampolineSize)));
 
     if (auto EC = sys::Memory::protectMappedMemory(
@@ -189,10 +188,8 @@ template <typename ORCABI> class LocalTrampolinePool : public TrampolinePool {
 
   ResolveLandingFunction ResolveLanding;
 
-  std::mutex LTPMutex;
   sys::OwningMemoryBlock ResolverBlock;
   std::vector<sys::OwningMemoryBlock> TrampolineBlocks;
-  std::vector<JITTargetAddress> AvailableTrampolines;
 };
 
 /// Target-independent base class for compile callback management.
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h b/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h
index 96f8e169e7dcc..c256be9843446 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h
@@ -29,6 +29,7 @@ namespace orc {
 
 class LLJITBuilderState;
 class LLLazyJITBuilderState;
+class TargetProcessControl;
 
 /// A pre-fabricated ORC JIT stack that can serve as an alternative to MCJIT.
 ///
@@ -272,6 +273,7 @@ class LLJITBuilderState {
   CompileFunctionCreator CreateCompileFunction;
   PlatformSetupFunction SetUpPlatform;
   unsigned NumCompileThreads = 0;
+  TargetProcessControl *TPC = nullptr;
 
   /// Called prior to JIT class construcion to fix up defaults.
   Error prepareForConstruction();
@@ -354,6 +356,17 @@ class LLJITBuilderSetters {
     return impl();
   }
 
+  /// Set a TargetProcessControl object.
+  ///
+  /// If the platform uses ObjectLinkingLayer by default and no
+  /// ObjectLinkingLayerCreator has been set then the TargetProcessControl
+  /// object will be used to supply the memory manager for the
+  /// ObjectLinkingLayer.
+  SetterImpl &setTargetProcessControl(TargetProcessControl &TPC) {
+    impl().TPC = &TPC;
+    return impl();
+  }
+
   /// Create an instance of the JIT.
   Expected<std::unique_ptr<JITType>> create() {
     if (auto Err = impl().prepareForConstruction())
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h
index 2bfe3b001709c..fbf9bde8a9d55 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h
@@ -90,8 +90,14 @@ class ObjectLinkingLayer : public ObjectLayer {
   using ReturnObjectBufferFunction =
       std::function<void(std::unique_ptr<MemoryBuffer>)>;
 
-  /// Construct an ObjectLinkingLayer with the given NotifyLoaded,
-  /// and NotifyEmitted functors.
+  /// Construct an ObjectLinkingLayer.
+  ObjectLinkingLayer(ExecutionSession &ES,
+                     jitlink::JITLinkMemoryManager &MemMgr);
+
+  /// Construct an ObjectLinkingLayer. Takes ownership of the given
+  /// JITLinkMemoryManager. This method is a temporary hack to simplify
+  /// co-existence with RTDyldObjectLinkingLayer (which also owns its
+  /// allocators).
   ObjectLinkingLayer(ExecutionSession &ES,
                      std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr);
 
@@ -159,7 +165,8 @@ class ObjectLinkingLayer : public ObjectLayer {
   Error removeAllModules();
 
   mutable std::mutex LayerMutex;
-  std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr;
+  jitlink::JITLinkMemoryManager &MemMgr;
+  std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgrOwnership;
   bool OverrideObjectFlags = false;
   bool AutoClaimObjectSymbols = false;
   ReturnObjectBufferFunction ReturnObjectBuffer;
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h b/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h
index 86e8d5df3ad93..5f4b98a32e251 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h
@@ -453,20 +453,8 @@ class OrcRemoteTargetClient
   public:
     RemoteTrampolinePool(OrcRemoteTargetClient &Client) : Client(Client) {}
 
-    Expected<JITTargetAddress> getTrampoline() override {
-      std::lock_guard<std::mutex> Lock(RTPMutex);
-      if (AvailableTrampolines.empty()) {
-        if (auto Err = grow())
-          return std::move(Err);
-      }
-      assert(!AvailableTrampolines.empty() && "Failed to grow trampoline pool");
-      auto TrampolineAddr = AvailableTrampolines.back();
-      AvailableTrampolines.pop_back();
-      return TrampolineAddr;
-    }
-
   private:
-    Error grow() {
+    Error grow() override {
       JITTargetAddress BlockAddr = 0;
       uint32_t NumTrampolines = 0;
       if (auto TrampolineInfoOrErr = Client.emitTrampolineBlock())
@@ -476,14 +464,12 @@ class OrcRemoteTargetClient
 
       uint32_t TrampolineSize = Client.getTrampolineSize();
       for (unsigned I = 0; I < NumTrampolines; ++I)
-        this->AvailableTrampolines.push_back(BlockAddr + (I * TrampolineSize));
+        AvailableTrampolines.push_back(BlockAddr + (I * TrampolineSize));
 
       return Error::success();
     }
 
-    std::mutex RTPMutex;
     OrcRemoteTargetClient &Client;
-    std::vector<JITTargetAddress> AvailableTrampolines;
   };
 
   /// Remote compile callback manager.
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Speculation.h b/llvm/include/llvm/ExecutionEngine/Orc/Speculation.h
index a6537dd3093bd..10f78c8bc6beb 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Speculation.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Speculation.h
@@ -18,14 +18,10 @@
 #include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/ExecutionEngine/Orc/DebugUtils.h"
 #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
-#include "llvm/IR/PassManager.h"
-#include "llvm/Passes/PassBuilder.h"
 #include "llvm/Support/Debug.h"
-
 #include <mutex>
 #include <type_traits>
 #include <utility>
-#include <vector>
 
 namespace llvm {
 namespace orc {
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.h b/llvm/include/llvm/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.h
new file mode 100644
index 0000000000000..6c95e22a4257d
--- /dev/null
+++ b/llvm/include/llvm/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.h
@@ -0,0 +1,60 @@
+//===------------ TPCDynamicLibrarySearchGenerator.h ------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Support loading and searching of dynamic libraries in a target process via
+// the TargetProcessControl class.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_TPCDYNAMICLIBRARYSEARCHGENERATOR_H
+#define LLVM_EXECUTIONENGINE_ORC_TPCDYNAMICLIBRARYSEARCHGENERATOR_H
+
+#include "llvm/ExecutionEngine/Orc/TargetProcessControl.h"
+
+namespace llvm {
+namespace orc {
+
+class TPCDynamicLibrarySearchGenerator : public JITDylib::DefinitionGenerator {
+public:
+  /// Create a DynamicLibrarySearchGenerator that searches for symbols in the
+  /// library with the given handle.
+  ///
+  /// If the Allow predicate is given then only symbols matching the predicate
+  /// will be searched for. If the predicate is not given then all symbols will
+  /// be searched for.
+  TPCDynamicLibrarySearchGenerator(
+      TargetProcessControl &TPC,
+      TargetProcessControl::DynamicLibraryHandle DylibHandle)
+      : TPC(TPC), DylibHandle(DylibHandle) {}
+
+  /// Permanently loads the library at the given path and, on success, returns
+  /// a DynamicLibrarySearchGenerator that will search it for symbol definitions
+  /// in the library. On failure returns the reason the library failed to load.
+  static Expected<std::unique_ptr<TPCDynamicLibrarySearchGenerator>>
+  Load(TargetProcessControl &TPC, const char *LibraryPath);
+
+  /// Creates a TPCDynamicLibrarySearchGenerator that searches for symbols in
+  /// the target process.
+  static Expected<std::unique_ptr<TPCDynamicLibrarySearchGenerator>>
+  GetForTargetProcess(TargetProcessControl &TPC) {
+    return Load(TPC, nullptr);
+  }
+
+  Error tryToGenerate(LookupKind K, JITDylib &JD,
+                      JITDylibLookupFlags JDLookupFlags,
+                      const SymbolLookupSet &Symbols) override;
+
+private:
+  TargetProcessControl &TPC;
+  TargetProcessControl::DynamicLibraryHandle DylibHandle;
+};
+
+} // end namespace orc
+} // end namespace llvm
+
+#endif // LLVM_EXECUTIONENGINE_ORC_TPCDYNAMICLIBRARYSEARCHGENERATOR_H
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcessControl.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcessControl.h
index facafd8836530..37bfa5a45b6c0 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcessControl.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcessControl.h
@@ -13,11 +13,16 @@
 #ifndef LLVM_EXECUTIONENGINE_ORC_TARGETPROCESSCONTROL_H
 #define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESSCONTROL_H
 
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h"
+#include "llvm/ExecutionEngine/Orc/Core.h"
+#include "llvm/Support/DynamicLibrary.h"
 #include "llvm/Support/MSVCErrorWorkarounds.h"
 
 #include <future>
+#include <vector>
 
 namespace llvm {
 namespace orc {
@@ -105,6 +110,22 @@ class TargetProcessControl {
     }
   };
 
+  using DynamicLibraryHandle = JITTargetAddress;
+
+  /// Request lookup within a single library.
+  /// If Library is None then the whole target process should be searched.
+  struct LookupRequestElement {
+    LookupRequestElement(DynamicLibraryHandle Handle,
+                         const SymbolLookupSet &Symbols)
+        : Handle(Handle), Symbols(Symbols) {}
+    DynamicLibraryHandle Handle;
+    const SymbolLookupSet &Symbols;
+  };
+
+  using LookupRequest = ArrayRef<LookupRequestElement>;
+
+  using LookupResult = std::vector<std::vector<JITTargetAddress>>;
+
   virtual ~TargetProcessControl();
 
   /// Return the Triple for the target process.
@@ -119,6 +140,17 @@ class TargetProcessControl {
   /// Return a MemoryAccess object for the target process.
   MemoryAccess &getMemoryAccess() const { return *MemAccess; }
 
+  /// Load the library at the given path. Returns a handle to the loaded
+  /// library. If LibraryPath is null this function will return the global
+  /// handle for the target process.
+  virtual Expected<DynamicLibraryHandle>
+  loadLibrary(const char *LibraryPath) = 0;
+
+  /// Search for symbols in the target process.
+  /// The result of the lookup is a 2-dimentional array of target addresses
+  /// that correspond to the lookup order.
+  virtual Expected<LookupResult> lookupSymbols(LookupRequest Request) = 0;
+
 protected:
   TargetProcessControl(Triple TT, unsigned PageSize);
 
@@ -136,6 +168,10 @@ class SelfTargetProcessControl : public TargetProcessControl,
 
   static Expected<std::unique_ptr<SelfTargetProcessControl>> Create();
 
+  Expected<DynamicLibraryHandle> loadLibrary(const char *LibraryPath) override;
+
+  Expected<LookupResult> lookupSymbols(LookupRequest Request) override;
+
 private:
   void writeUInt8s(ArrayRef<UInt8Write> Ws,
                    WriteResultFn OnWriteComplete) override;
@@ -154,6 +190,9 @@ class SelfTargetProcessControl : public TargetProcessControl,
 
   std::unique_ptr<jitlink::InProcessMemoryManager> IPMM =
       std::make_unique<jitlink::InProcessMemoryManager>();
+
+  char GlobalManglingPrefix = 0;
+  std::vector<std::unique_ptr<sys::DynamicLibrary>> DynamicLibraries;
 };
 
 } // end namespace orc
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 2df1d2c2bec26..a0d694533e4d5 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -402,7 +402,6 @@ def OMP_Requires : Directive<"requires"> {
 }
 def OMP_TargetData : Directive<"target data"> {
   let allowedClauses = [
-    VersionedClause<OMPC_Map>,
     VersionedClause<OMPC_UseDevicePtr>,
     VersionedClause<OMPC_UseDeviceAddr, 50>
   ];
@@ -416,19 +415,20 @@ def OMP_TargetData : Directive<"target data"> {
 }
 def OMP_TargetEnterData : Directive<"target enter data"> {
   let allowedClauses = [
-    VersionedClause<OMPC_Depend>,
-    VersionedClause<OMPC_Map>
+    VersionedClause<OMPC_Depend>
   ];
   let allowedOnceClauses = [
     VersionedClause<OMPC_If>,
     VersionedClause<OMPC_Device>,
     VersionedClause<OMPC_NoWait>
   ];
+  let requiredClauses = [
+    VersionedClause<OMPC_Map>
+  ];
 }
 def OMP_TargetExitData : Directive<"target exit data"> {
   let allowedClauses = [
-    VersionedClause<OMPC_Depend>,
-    VersionedClause<OMPC_Map>
+    VersionedClause<OMPC_Depend>
   ];
   let allowedOnceClauses = [
     VersionedClause<OMPC_Device>,
diff --git a/llvm/include/llvm/IR/Argument.h b/llvm/include/llvm/IR/Argument.h
index 2ca18c6103495..2dd7c6cbc8b8b 100644
--- a/llvm/include/llvm/IR/Argument.h
+++ b/llvm/include/llvm/IR/Argument.h
@@ -65,6 +65,9 @@ class Argument final : public Value {
   /// Return true if this argument has the byval attribute.
   bool hasByValAttr() const;
 
+  /// Return true if this argument has the byref attribute.
+  bool hasByRefAttr() const;
+
   /// Return true if this argument has the swiftself attribute.
   bool hasSwiftSelfAttr() const;
 
@@ -80,6 +83,15 @@ class Argument final : public Value {
   /// in-memory ABI size copied to the stack for the call. Otherwise, return 0.
   uint64_t getPassPointeeByValueCopySize(const DataLayout &DL) const;
 
+  /// Return true if this argument has the byval, inalloca, preallocated, or
+  /// byref attribute. These attributes represent arguments being passed by
+  /// value (which may or may not involve a stack copy)
+  bool hasPointeeInMemoryValueAttr() const;
+
+  /// If hasPointeeInMemoryValueAttr returns true, the in-memory ABI type is
+  /// returned. Otherwise, nullptr.
+  Type *getPointeeInMemoryValueType() const;
+
   /// If this is a byval or inalloca argument, return its alignment.
   /// FIXME: Remove this function once transition to Align is over.
   /// Use getParamAlign() instead.
@@ -91,6 +103,9 @@ class Argument final : public Value {
   /// If this is a byval argument, return its type.
   Type *getParamByValType() const;
 
+  /// If this is a byref argument, return its type.
+  Type *getParamByRefType() const;
+
   /// Return true if this argument has the nest attribute.
   bool hasNestAttr() const;
 
diff --git a/llvm/include/llvm/IR/Attributes.h b/llvm/include/llvm/IR/Attributes.h
index 58365aa2b7645..03f2fc123cba4 100644
--- a/llvm/include/llvm/IR/Attributes.h
+++ b/llvm/include/llvm/IR/Attributes.h
@@ -108,6 +108,7 @@ class Attribute {
                                         unsigned ElemSizeArg,
                                         const Optional<unsigned> &NumElemsArg);
   static Attribute getWithByValType(LLVMContext &Context, Type *Ty);
+  static Attribute getWithByRefType(LLVMContext &Context, Type *Ty);
   static Attribute getWithPreallocatedType(LLVMContext &Context, Type *Ty);
 
   static Attribute::AttrKind getAttrKindFromName(StringRef AttrName);
@@ -303,6 +304,7 @@ class AttributeSet {
   uint64_t getDereferenceableBytes() const;
   uint64_t getDereferenceableOrNullBytes() const;
   Type *getByValType() const;
+  Type *getByRefType() const;
   Type *getPreallocatedType() const;
   std::pair<unsigned, Optional<unsigned>> getAllocSizeArgs() const;
   std::string getAsString(bool InAttrGrp = false) const;
@@ -626,6 +628,9 @@ class AttributeList {
   /// Return the byval type for the specified function parameter.
   Type *getParamByValType(unsigned ArgNo) const;
 
+  /// Return the byref type for the specified function parameter.
+  Type *getParamByRefType(unsigned ArgNo) const;
+
   /// Return the preallocated type for the specified function parameter.
   Type *getParamPreallocatedType(unsigned ArgNo) const;
 
@@ -729,6 +734,7 @@ class AttrBuilder {
   uint64_t DerefOrNullBytes = 0;
   uint64_t AllocSizeArgs = 0;
   Type *ByValType = nullptr;
+  Type *ByRefType = nullptr;
   Type *PreallocatedType = nullptr;
 
 public:
@@ -808,6 +814,9 @@ class AttrBuilder {
   /// Retrieve the byval type.
   Type *getByValType() const { return ByValType; }
 
+  /// Retrieve the byref type.
+  Type *getByRefType() const { return ByRefType; }
+
   /// Retrieve the preallocated type.
   Type *getPreallocatedType() const { return PreallocatedType; }
 
@@ -854,6 +863,9 @@ class AttrBuilder {
   /// This turns a byval type into the form used internally in Attribute.
   AttrBuilder &addByValAttr(Type *Ty);
 
+  /// This turns a byref type into the form used internally in Attribute.
+  AttrBuilder &addByRefAttr(Type *Ty);
+
   /// This turns a preallocated type into the form used internally in Attribute.
   AttrBuilder &addPreallocatedAttr(Type *Ty);
 
diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td
index 395f9dbfb1760..214eb84b6a470 100644
--- a/llvm/include/llvm/IR/Attributes.td
+++ b/llvm/include/llvm/IR/Attributes.td
@@ -39,6 +39,9 @@ def Builtin : EnumAttr<"builtin">;
 /// Pass structure by value.
 def ByVal : TypeAttr<"byval">;
 
+/// Mark in-memory ABI type.
+def ByRef : TypeAttr<"byref">;
+
 /// Parameter or return value may not contain uninitialized or poison bits.
 def NoUndef : EnumAttr<"noundef">;
 
diff --git a/llvm/include/llvm/IR/DebugInfoMetadata.h b/llvm/include/llvm/IR/DebugInfoMetadata.h
index 7d7cc4de79374..6e57e6d79cb78 100644
--- a/llvm/include/llvm/IR/DebugInfoMetadata.h
+++ b/llvm/include/llvm/IR/DebugInfoMetadata.h
@@ -942,13 +942,14 @@ class DICompositeType : public DIType {
           DINodeArray Elements, unsigned RuntimeLang, DIType *VTableHolder,
           DITemplateParameterArray TemplateParams, StringRef Identifier,
           DIDerivedType *Discriminator, Metadata *DataLocation,
-          StorageType Storage, bool ShouldCreate = true) {
+          Metadata *Associated, Metadata *Allocated, StorageType Storage,
+          bool ShouldCreate = true) {
     return getImpl(Context, Tag, getCanonicalMDString(Context, Name), File,
                    Line, Scope, BaseType, SizeInBits, AlignInBits, OffsetInBits,
                    Flags, Elements.get(), RuntimeLang, VTableHolder,
                    TemplateParams.get(),
                    getCanonicalMDString(Context, Identifier), Discriminator,
-                   DataLocation, Storage, ShouldCreate);
+                   DataLocation, Associated, Allocated, Storage, ShouldCreate);
   }
   static DICompositeType *
   getImpl(LLVMContext &Context, unsigned Tag, MDString *Name, Metadata *File,
@@ -957,15 +958,16 @@ class DICompositeType : public DIType {
           DIFlags Flags, Metadata *Elements, unsigned RuntimeLang,
           Metadata *VTableHolder, Metadata *TemplateParams,
           MDString *Identifier, Metadata *Discriminator, Metadata *DataLocation,
-          StorageType Storage, bool ShouldCreate = true);
+          Metadata *Associated, Metadata *Allocated, StorageType Storage,
+          bool ShouldCreate = true);
 
   TempDICompositeType cloneImpl() const {
-    return getTemporary(getContext(), getTag(), getName(), getFile(), getLine(),
-                        getScope(), getBaseType(), getSizeInBits(),
-                        getAlignInBits(), getOffsetInBits(), getFlags(),
-                        getElements(), getRuntimeLang(), getVTableHolder(),
-                        getTemplateParams(), getIdentifier(),
-                        getDiscriminator(), getRawDataLocation());
+    return getTemporary(
+        getContext(), getTag(), getName(), getFile(), getLine(), getScope(),
+        getBaseType(), getSizeInBits(), getAlignInBits(), getOffsetInBits(),
+        getFlags(), getElements(), getRuntimeLang(), getVTableHolder(),
+        getTemplateParams(), getIdentifier(), getDiscriminator(),
+        getRawDataLocation(), getRawAssociated(), getRawAllocated());
   }
 
 public:
@@ -977,10 +979,11 @@ class DICompositeType : public DIType {
        DINodeArray Elements, unsigned RuntimeLang, DIType *VTableHolder,
        DITemplateParameterArray TemplateParams = nullptr,
        StringRef Identifier = "", DIDerivedType *Discriminator = nullptr,
-       Metadata *DataLocation = nullptr),
+       Metadata *DataLocation = nullptr, Metadata *Associated = nullptr,
+       Metadata *Allocated = nullptr),
       (Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
        OffsetInBits, Flags, Elements, RuntimeLang, VTableHolder, TemplateParams,
-       Identifier, Discriminator, DataLocation))
+       Identifier, Discriminator, DataLocation, Associated, Allocated))
   DEFINE_MDNODE_GET(
       DICompositeType,
       (unsigned Tag, MDString *Name, Metadata *File, unsigned Line,
@@ -988,10 +991,11 @@ class DICompositeType : public DIType {
        uint32_t AlignInBits, uint64_t OffsetInBits, DIFlags Flags,
        Metadata *Elements, unsigned RuntimeLang, Metadata *VTableHolder,
        Metadata *TemplateParams = nullptr, MDString *Identifier = nullptr,
-       Metadata *Discriminator = nullptr, Metadata *DataLocation = nullptr),
+       Metadata *Discriminator = nullptr, Metadata *DataLocation = nullptr,
+       Metadata *Associated = nullptr, Metadata *Allocated = nullptr),
       (Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
        OffsetInBits, Flags, Elements, RuntimeLang, VTableHolder, TemplateParams,
-       Identifier, Discriminator, DataLocation))
+       Identifier, Discriminator, DataLocation, Associated, Allocated))
 
   TempDICompositeType clone() const { return cloneImpl(); }
 
@@ -1009,7 +1013,7 @@ class DICompositeType : public DIType {
              uint64_t OffsetInBits, DIFlags Flags, Metadata *Elements,
              unsigned RuntimeLang, Metadata *VTableHolder,
              Metadata *TemplateParams, Metadata *Discriminator,
-             Metadata *DataLocation);
+             Metadata *DataLocation, Metadata *Associated, Metadata *Allocated);
   static DICompositeType *getODRTypeIfExists(LLVMContext &Context,
                                              MDString &Identifier);
 
@@ -1022,14 +1026,13 @@ class DICompositeType : public DIType {
   ///
   /// If not \a LLVMContext::isODRUniquingDebugTypes(), this function returns
   /// nullptr.
-  static DICompositeType *
-  buildODRType(LLVMContext &Context, MDString &Identifier, unsigned Tag,
-               MDString *Name, Metadata *File, unsigned Line, Metadata *Scope,
-               Metadata *BaseType, uint64_t SizeInBits, uint32_t AlignInBits,
-               uint64_t OffsetInBits, DIFlags Flags, Metadata *Elements,
-               unsigned RuntimeLang, Metadata *VTableHolder,
-               Metadata *TemplateParams, Metadata *Discriminator,
-               Metadata *DataLocation);
+  static DICompositeType *buildODRType(
+      LLVMContext &Context, MDString &Identifier, unsigned Tag, MDString *Name,
+      Metadata *File, unsigned Line, Metadata *Scope, Metadata *BaseType,
+      uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits,
+      DIFlags Flags, Metadata *Elements, unsigned RuntimeLang,
+      Metadata *VTableHolder, Metadata *TemplateParams, Metadata *Discriminator,
+      Metadata *DataLocation, Metadata *Associated, Metadata *Allocated);
 
   DIType *getBaseType() const { return cast_or_null<DIType>(getRawBaseType()); }
   DINodeArray getElements() const {
@@ -1058,6 +1061,20 @@ class DICompositeType : public DIType {
   DIExpression *getDataLocationExp() const {
     return dyn_cast_or_null<DIExpression>(getRawDataLocation());
   }
+  Metadata *getRawAssociated() const { return getOperand(10); }
+  DIVariable *getAssociated() const {
+    return dyn_cast_or_null<DIVariable>(getRawAssociated());
+  }
+  DIExpression *getAssociatedExp() const {
+    return dyn_cast_or_null<DIExpression>(getRawAssociated());
+  }
+  Metadata *getRawAllocated() const { return getOperand(11); }
+  DIVariable *getAllocated() const {
+    return dyn_cast_or_null<DIVariable>(getRawAllocated());
+  }
+  DIExpression *getAllocatedExp() const {
+    return dyn_cast_or_null<DIExpression>(getRawAllocated());
+  }
 
   /// Replace operands.
   ///
diff --git a/llvm/include/llvm/IR/Function.h b/llvm/include/llvm/IR/Function.h
index bb4ec13c7610f..ad7e5ecb9f0f4 100644
--- a/llvm/include/llvm/IR/Function.h
+++ b/llvm/include/llvm/IR/Function.h
@@ -199,6 +199,11 @@ class Function : public GlobalObject, public ilist_node<Function> {
   /// returns Intrinsic::not_intrinsic!
   bool isIntrinsic() const { return HasLLVMReservedName; }
 
+  /// isTargetIntrinsic - Returns true if this function is an intrinsic and the
+  /// intrinsic is specific to a certain target. If this is not an intrinsic
+  /// or a generic intrinsic, false is returned.
+  bool isTargetIntrinsic() const;
+
   /// Returns true if the function is one of the "Constrained Floating-Point
   /// Intrinsics". Returns false if not, and returns false when
   /// getIntrinsicID() returns Intrinsic::not_intrinsic.
@@ -467,6 +472,11 @@ class Function : public GlobalObject, public ilist_node<Function> {
     return Ty ? Ty : (arg_begin() + ArgNo)->getType()->getPointerElementType();
   }
 
+  /// Extract the byref type for a parameter.
+  Type *getParamByRefType(unsigned ArgNo) const {
+    return AttributeSets.getParamByRefType(ArgNo);
+  }
+
   /// Extract the number of dereferenceable bytes for a call or
   /// parameter (0=unknown).
   /// @param i AttributeList index, referring to a return value or argument.
diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h
index 07af00ec92402..ce0744bbf0876 100644
--- a/llvm/include/llvm/IR/InstrTypes.h
+++ b/llvm/include/llvm/IR/InstrTypes.h
@@ -1393,14 +1393,18 @@ class CallBase : public Instruction {
   ///
   void setAttributes(AttributeList A) { Attrs = A; }
 
-  /// Determine whether this call has the given attribute.
+  /// Determine whether this call has the given attribute. If it does not
+  /// then determine if the called function has the attribute, but only if
+  /// the attribute is allowed for the call.
   bool hasFnAttr(Attribute::AttrKind Kind) const {
     assert(Kind != Attribute::NoBuiltin &&
            "Use CallBase::isNoBuiltin() to check for Attribute::NoBuiltin");
     return hasFnAttrImpl(Kind);
   }
 
-  /// Determine whether this call has the given attribute.
+  /// Determine whether this call has the given attribute. If it does not
+  /// then determine if the called function has the attribute, but only if
+  /// the attribute is allowed for the call.
   bool hasFnAttr(StringRef Kind) const { return hasFnAttrImpl(Kind); }
 
   /// adds the attribute to the list of attributes.
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 4918ea876df65..0a071464804eb 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -1039,6 +1039,25 @@ def int_udiv_fix_sat : Intrinsic<[llvm_anyint_ty],
                                  [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty],
                                  [IntrNoMem, ImmArg<ArgIndex<2>>]>;
 
+//===------------------ Integer Min/Max/Abs Intrinsics --------------------===//
+//
+def int_abs : Intrinsic<
+    [llvm_anyint_ty], [LLVMMatchType<0>, llvm_i1_ty],
+    [IntrNoMem, IntrSpeculatable, IntrWillReturn, ImmArg<ArgIndex<1>>]>;
+
+def int_smax : Intrinsic<
+    [llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
+    [IntrNoMem, IntrSpeculatable, IntrWillReturn]>;
+def int_smin : Intrinsic<
+    [llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
+    [IntrNoMem, IntrSpeculatable, IntrWillReturn]>;
+def int_umax : Intrinsic<
+    [llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
+    [IntrNoMem, IntrSpeculatable, IntrWillReturn]>;
+def int_umin : Intrinsic<
+    [llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
+    [IntrNoMem, IntrSpeculatable, IntrWillReturn]>;
+
 //===------------------------- Memory Use Markers -------------------------===//
 //
 def int_lifetime_start  : Intrinsic<[],
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 01380afae0068..79432afb0dcc1 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -1667,7 +1667,8 @@ class AMDGPUGlobalAtomicRtn<LLVMType vt> : Intrinsic <
   [vt],
   [llvm_anyptr_ty,    // vaddr
    vt],               // vdata(VGPR)
-  [IntrArgMemOnly, NoCapture<ArgIndex<0>>], "", [SDNPMemOperand]>;
+  [IntrArgMemOnly, IntrWillReturn, NoCapture<ArgIndex<0>>], "",
+  [SDNPMemOperand]>;
 
 def int_amdgcn_global_atomic_csub : AMDGPUGlobalAtomicRtn<llvm_i32_ty>;
 
diff --git a/llvm/include/llvm/IR/IntrinsicsPowerPC.td b/llvm/include/llvm/IR/IntrinsicsPowerPC.td
index 2abb6b4e55fe7..23bcf3ce1959c 100644
--- a/llvm/include/llvm/IR/IntrinsicsPowerPC.td
+++ b/llvm/include/llvm/IR/IntrinsicsPowerPC.td
@@ -500,27 +500,27 @@ let TargetPrefix = "ppc" in {  // All intrinsics start with "llvm.ppc.".
                         [IntrNoMem]>;
   def int_ppc_altivec_vinsbvlx : GCCBuiltin<"__builtin_altivec_vinsbvlx">,
               Intrinsic<[llvm_v16i8_ty],
-                        [llvm_v16i8_ty, llvm_i64_ty, llvm_v16i8_ty],
+                        [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty],
                         [IntrNoMem]>;
   def int_ppc_altivec_vinsbvrx : GCCBuiltin<"__builtin_altivec_vinsbvrx">,
               Intrinsic<[llvm_v16i8_ty],
-                        [llvm_v16i8_ty, llvm_i64_ty, llvm_v16i8_ty],
+                        [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty],
                         [IntrNoMem]>;
   def int_ppc_altivec_vinshvlx : GCCBuiltin<"__builtin_altivec_vinshvlx">,
               Intrinsic<[llvm_v8i16_ty],
-                        [llvm_v8i16_ty, llvm_i64_ty, llvm_v8i16_ty],
+                        [llvm_v8i16_ty, llvm_i32_ty, llvm_v8i16_ty],
                         [IntrNoMem]>;
  def int_ppc_altivec_vinshvrx : GCCBuiltin<"__builtin_altivec_vinshvrx">,
               Intrinsic<[llvm_v8i16_ty],
-                        [llvm_v8i16_ty, llvm_i64_ty, llvm_v8i16_ty],
+                        [llvm_v8i16_ty, llvm_i32_ty, llvm_v8i16_ty],
                         [IntrNoMem]>;
   def int_ppc_altivec_vinswvlx : GCCBuiltin<"__builtin_altivec_vinswvlx">,
               Intrinsic<[llvm_v4i32_ty],
-                        [llvm_v4i32_ty, llvm_i64_ty, llvm_v4i32_ty],
+                        [llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty],
                         [IntrNoMem]>;
   def int_ppc_altivec_vinswvrx : GCCBuiltin<"__builtin_altivec_vinswvrx">,
               Intrinsic<[llvm_v4i32_ty],
-                        [llvm_v4i32_ty, llvm_i64_ty, llvm_v4i32_ty],
+                        [llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty],
                         [IntrNoMem]>;
   // P10 Vector Insert with immediate.
   def int_ppc_altivec_vinsw :
@@ -1069,7 +1069,7 @@ def int_ppc_vsx_xxinsertw :
                             [IntrNoMem]>;
 def int_ppc_vsx_xvtlsbb :
       PowerPC_VSX_Intrinsic<"xvtlsbb", [llvm_i32_ty],
-                            [llvm_v16i8_ty, llvm_i1_ty], [IntrNoMem]>;
+                            [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>;
 def int_ppc_vsx_xxeval :
       PowerPC_VSX_Intrinsic<"xxeval", [llvm_v2i64_ty],
                            [llvm_v2i64_ty, llvm_v2i64_ty,
diff --git a/llvm/include/llvm/IR/PassManager.h b/llvm/include/llvm/IR/PassManager.h
index f503871e23609..c2f535037a56d 100644
--- a/llvm/include/llvm/IR/PassManager.h
+++ b/llvm/include/llvm/IR/PassManager.h
@@ -1061,7 +1061,16 @@ extern template class InnerAnalysisManagerProxy<FunctionAnalysisManager,
 ///
 /// This proxy only exposes the const interface of the outer analysis manager,
 /// to indicate that you cannot cause an outer analysis to run from within an
-/// inner pass.  Instead, you must rely on the \c getCachedResult API.
+/// inner pass.  Instead, you must rely on the \c getCachedResult API.  This is
+/// due to keeping potential future concurrency in mind. To give an example,
+/// running a module analysis before any function passes may give a different
+/// result than running it in a function pass. Both may be valid, but it would
+/// produce non-deterministic results. GlobalsAA is a good analysis example,
+/// because the cached information has the mod/ref info for all memory for each
+/// function at the time the analysis was computed. The information is still
+/// valid after a function transformation, but it may be *different* if
+/// recomputed after that transform. GlobalsAA is never invalidated.
+
 ///
 /// This proxy doesn't manage invalidation in any way -- that is handled by the
 /// recursive return path of each layer of the pass manager.  A consequence of
diff --git a/llvm/include/llvm/IR/PassTimingInfo.h b/llvm/include/llvm/IR/PassTimingInfo.h
index b70850fd64d76..a42d6aa3bede6 100644
--- a/llvm/include/llvm/IR/PassTimingInfo.h
+++ b/llvm/include/llvm/IR/PassTimingInfo.h
@@ -17,11 +17,13 @@
 
 #include "llvm/ADT/Any.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Timer.h"
-#include "llvm/Support/TypeName.h"
 #include <memory>
+#include <utility>
+
 namespace llvm {
 
 class Pass;
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index f4acb00a0a3fb..b43902ff8a551 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -267,6 +267,7 @@ void initializeLowerInvokeLegacyPassPass(PassRegistry&);
 void initializeLowerSwitchPass(PassRegistry&);
 void initializeLowerTypeTestsPass(PassRegistry&);
 void initializeLowerMatrixIntrinsicsLegacyPassPass(PassRegistry &);
+void initializeLowerMatrixIntrinsicsMinimalLegacyPassPass(PassRegistry &);
 void initializeMIRCanonicalizerPass(PassRegistry &);
 void initializeMIRNamerPass(PassRegistry &);
 void initializeMIRPrintingPassPass(PassRegistry&);
@@ -418,6 +419,7 @@ void initializeStructurizeCFGPass(PassRegistry&);
 void initializeSYCLLowerWGScopeLegacyPassPass(PassRegistry &);
 void initializeSYCLLowerESIMDLegacyPassPass(PassRegistry &);
 void initializeESIMDLowerLoadStorePass(PassRegistry &);
+void initializeESIMDLowerVecArgLegacyPassPass(PassRegistry &);
 void initializeTailCallElimPass(PassRegistry&);
 void initializeTailDuplicatePass(PassRegistry&);
 void initializeTargetLibraryInfoWrapperPassPass(PassRegistry&);
diff --git a/llvm/include/llvm/LinkAllPasses.h b/llvm/include/llvm/LinkAllPasses.h
index 8f4bcd9e6e75f..c50459347643d 100644
--- a/llvm/include/llvm/LinkAllPasses.h
+++ b/llvm/include/llvm/LinkAllPasses.h
@@ -204,6 +204,7 @@ namespace {
       (void)llvm::createSYCLLowerWGScopePass();
       (void)llvm::createSYCLLowerESIMDPass();
       (void)llvm::createESIMDLowerLoadStorePass();
+      (void)llvm::createESIMDLowerVecArgPass();
       std::string buf;
       llvm::raw_string_ostream os(buf);
       (void) llvm::createPrintModulePass(os);
diff --git a/llvm/include/llvm/MC/MCAsmInfo.h b/llvm/include/llvm/MC/MCAsmInfo.h
index 46c5a111c891e..0f9d503045d88 100644
--- a/llvm/include/llvm/MC/MCAsmInfo.h
+++ b/llvm/include/llvm/MC/MCAsmInfo.h
@@ -209,6 +209,9 @@ class MCAsmInfo {
   const char *Data32bitsDirective;
   const char *Data64bitsDirective;
 
+  /// True if data directives support signed values
+  bool SupportsSignedData = true;
+
   /// If non-null, a directive that is used to emit a word which should be
   /// relocated as a 64-bit GP-relative offset, e.g. .gpdword on Mips.  Defaults
   /// to nullptr.
@@ -436,6 +439,7 @@ class MCAsmInfo {
   const char *getData16bitsDirective() const { return Data16bitsDirective; }
   const char *getData32bitsDirective() const { return Data32bitsDirective; }
   const char *getData64bitsDirective() const { return Data64bitsDirective; }
+  bool supportsSignedData() const { return SupportsSignedData; }
   const char *getGPRel64Directive() const { return GPRel64Directive; }
   const char *getGPRel32Directive() const { return GPRel32Directive; }
   const char *getDTPRel64Directive() const { return DTPRel64Directive; }
diff --git a/llvm/include/llvm/MC/MCExpr.h b/llvm/include/llvm/MC/MCExpr.h
index 803c0d443beec..a424957bb0b22 100644
--- a/llvm/include/llvm/MC/MCExpr.h
+++ b/llvm/include/llvm/MC/MCExpr.h
@@ -302,6 +302,7 @@ class MCSymbolRefExpr : public MCExpr {
     VK_PPC_TLSLD,           // symbol@tlsld
     VK_PPC_LOCAL,           // symbol@local
     VK_PPC_NOTOC,           // symbol@notoc
+    VK_PPC_PCREL_OPT,       // .reloc expr, R_PPC64_PCREL_OPT, expr
 
     VK_COFF_IMGREL32, // symbol@imgrel (image-relative)
 
diff --git a/llvm/include/llvm/MC/MCFixup.h b/llvm/include/llvm/MC/MCFixup.h
index affc846cbdd44..b3a23911d636e 100644
--- a/llvm/include/llvm/MC/MCFixup.h
+++ b/llvm/include/llvm/MC/MCFixup.h
@@ -9,7 +9,6 @@
 #ifndef LLVM_MC_MCFIXUP_H
 #define LLVM_MC_MCFIXUP_H
 
-#include "llvm/MC/MCExpr.h"
 #include "llvm/Support/DataTypes.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/SMLoc.h"
diff --git a/llvm/include/llvm/MC/MCSymbol.h b/llvm/include/llvm/MC/MCSymbol.h
index 84263bf94035e..a89de5dfa22f1 100644
--- a/llvm/include/llvm/MC/MCSymbol.h
+++ b/llvm/include/llvm/MC/MCSymbol.h
@@ -16,6 +16,7 @@
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFragment.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
@@ -27,7 +28,6 @@ namespace llvm {
 
 class MCAsmInfo;
 class MCContext;
-class MCExpr;
 class MCSection;
 class raw_ostream;
 
diff --git a/llvm/include/llvm/Object/ELF.h b/llvm/include/llvm/Object/ELF.h
index b44dd3f486614..3dab0471a2391 100644
--- a/llvm/include/llvm/Object/ELF.h
+++ b/llvm/include/llvm/Object/ELF.h
@@ -48,6 +48,12 @@ static inline Error createError(const Twine &Err) {
   return make_error<StringError>(Err, object_error::parse_failed);
 }
 
+enum PPCInstrMasks : uint64_t {
+  PADDI_R12_NO_DISP = 0x0610000039800000,
+  MTCTR_R12 = 0x7D8903A6,
+  BCTR = 0x4E800420,
+};
+
 template <class ELFT> class ELFFile;
 
 template <class ELFT>
@@ -196,7 +202,7 @@ class ELFFile {
     return getSectionContentsAsArray<Elf_Relr>(Sec);
   }
 
-  Expected<std::vector<Elf_Rela>> decode_relrs(Elf_Relr_Range relrs) const;
+  Expected<std::vector<Elf_Rel>> decode_relrs(Elf_Relr_Range relrs) const;
 
   Expected<std::vector<Elf_Rela>> android_relas(const Elf_Shdr *Sec) const;
 
diff --git a/llvm/include/llvm/ObjectYAML/DWARFEmitter.h b/llvm/include/llvm/ObjectYAML/DWARFEmitter.h
index 1552ec7c178e1..5837c69ed59f9 100644
--- a/llvm/include/llvm/ObjectYAML/DWARFEmitter.h
+++ b/llvm/include/llvm/ObjectYAML/DWARFEmitter.h
@@ -39,9 +39,10 @@ Error emitDebugInfo(raw_ostream &OS, const Data &DI);
 Error emitDebugLine(raw_ostream &OS, const Data &DI);
 Error emitDebugAddr(raw_ostream &OS, const Data &DI);
 Error emitDebugStrOffsets(raw_ostream &OS, const Data &DI);
+Error emitDebugRnglists(raw_ostream &OS, const Data &DI);
 
 Expected<StringMap<std::unique_ptr<MemoryBuffer>>>
-emitDebugSections(StringRef YAMLString, bool ApplyFixups = false,
+emitDebugSections(StringRef YAMLString,
                   bool IsLittleEndian = sys::IsLittleEndianHost);
 } // end namespace DWARFYAML
 } // end namespace llvm
diff --git a/llvm/include/llvm/ObjectYAML/DWARFYAML.h b/llvm/include/llvm/ObjectYAML/DWARFYAML.h
index 244a5ff19298e..7b18ceebdb1cf 100644
--- a/llvm/include/llvm/ObjectYAML/DWARFYAML.h
+++ b/llvm/include/llvm/ObjectYAML/DWARFYAML.h
@@ -119,12 +119,10 @@ struct DWARFContext {
 };
 
 struct Unit {
-  dwarf::DwarfFormat Format;
-  uint64_t Length;
-  uint16_t Version;
+  dwarf::FormParams FormParams;
+  Optional<yaml::Hex64> Length;
   llvm::dwarf::UnitType Type; // Added in DWARF 5
   yaml::Hex64 AbbrOffset;
-  uint8_t AddrSize;
   std::vector<Entry> Entries;
 };
 
@@ -185,6 +183,26 @@ struct StringOffsetsTable {
   std::vector<yaml::Hex64> Offsets;
 };
 
+struct RnglistEntry {
+  dwarf::RnglistEntries Operator;
+  std::vector<yaml::Hex64> Values;
+};
+
+template <typename EntryType> struct ListEntries {
+  std::vector<EntryType> Entries;
+};
+
+template <typename EntryType> struct ListTable {
+  dwarf::DwarfFormat Format;
+  Optional<yaml::Hex64> Length;
+  yaml::Hex16 Version;
+  Optional<yaml::Hex8> AddrSize;
+  yaml::Hex8 SegSelectorSize;
+  Optional<uint32_t> OffsetEntryCount;
+  Optional<std::vector<yaml::Hex64>> Offsets;
+  std::vector<ListEntries<EntryType>> Lists;
+};
+
 struct Data {
   bool IsLittleEndian;
   bool Is64BitAddrSize;
@@ -203,6 +221,7 @@ struct Data {
   std::vector<Unit> CompileUnits;
 
   std::vector<LineTable> DebugLines;
+  Optional<std::vector<ListTable<RnglistEntry>>> DebugRnglists;
 
   bool isEmpty() const;
 
@@ -228,6 +247,11 @@ LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::DWARFYAML::LineTableOpcode)
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::DWARFYAML::SegAddrPair)
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::DWARFYAML::AddrTableEntry)
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::DWARFYAML::StringOffsetsTable)
+LLVM_YAML_IS_SEQUENCE_VECTOR(
+    llvm::DWARFYAML::ListTable<DWARFYAML::RnglistEntry>)
+LLVM_YAML_IS_SEQUENCE_VECTOR(
+    llvm::DWARFYAML::ListEntries<DWARFYAML::RnglistEntry>)
+LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::DWARFYAML::RnglistEntry)
 
 namespace llvm {
 namespace yaml {
@@ -296,6 +320,20 @@ template <> struct MappingTraits<DWARFYAML::SegAddrPair> {
   static void mapping(IO &IO, DWARFYAML::SegAddrPair &SegAddrPair);
 };
 
+template <typename EntryType>
+struct MappingTraits<DWARFYAML::ListTable<EntryType>> {
+  static void mapping(IO &IO, DWARFYAML::ListTable<EntryType> &ListTable);
+};
+
+template <typename EntryType>
+struct MappingTraits<DWARFYAML::ListEntries<EntryType>> {
+  static void mapping(IO &IO, DWARFYAML::ListEntries<EntryType> &ListEntries);
+};
+
+template <> struct MappingTraits<DWARFYAML::RnglistEntry> {
+  static void mapping(IO &IO, DWARFYAML::RnglistEntry &RnglistEntry);
+};
+
 template <> struct MappingTraits<DWARFYAML::AddrTableEntry> {
   static void mapping(IO &IO, DWARFYAML::AddrTableEntry &AddrTable);
 };
@@ -383,6 +421,15 @@ template <> struct ScalarEnumerationTraits<dwarf::Constants> {
   }
 };
 
+#define HANDLE_DW_RLE(unused, name)                                            \
+  io.enumCase(value, "DW_RLE_" #name, dwarf::DW_RLE_##name);
+
+template <> struct ScalarEnumerationTraits<dwarf::RnglistEntries> {
+  static void enumeration(IO &io, dwarf::RnglistEntries &value) {
+#include "llvm/BinaryFormat/Dwarf.def"
+  }
+};
+
 } // end namespace yaml
 } // end namespace llvm
 
diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index 3d3002eecce98..bd8b886a2bb3c 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -16,6 +16,7 @@
 #define LLVM_PASSES_STANDARDINSTRUMENTATIONS_H
 
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/IR/PassInstrumentation.h"
 #include "llvm/IR/PassTimingInfo.h"
 
@@ -53,14 +54,27 @@ class PrintIRInstrumentation {
   bool StoreModuleDesc = false;
 };
 
+class OptNoneInstrumentation {
+public:
+  OptNoneInstrumentation(bool DebugLogging) : DebugLogging(DebugLogging) {}
+  void registerCallbacks(PassInstrumentationCallbacks &PIC);
+
+private:
+  bool skip(StringRef PassID, Any IR);
+
+  bool DebugLogging;
+};
+
 /// This class provides an interface to register all the standard pass
 /// instrumentations and manages their state (if any).
 class StandardInstrumentations {
   PrintIRInstrumentation PrintIR;
   TimePassesHandler TimePasses;
+  OptNoneInstrumentation OptNone;
 
 public:
-  StandardInstrumentations() = default;
+  StandardInstrumentations(bool DebugLogging)
+      : PrintIR(), TimePasses(), OptNone(DebugLogging) {}
 
   void registerCallbacks(PassInstrumentationCallbacks &PIC);
 
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index 3d9fca9422c41..a3359ca901339 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -75,18 +75,10 @@ inline StringRef getInstrProfValueProfFuncName() {
 }
 
 /// Return the name profile runtime entry point to do value range profiling.
-// FIXME: This is to be removed after switching to the new memop value
-// profiling.
 inline StringRef getInstrProfValueRangeProfFuncName() {
   return INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR;
 }
 
-/// Return the name profile runtime entry point to do memop size value
-/// profiling.
-inline StringRef getInstrProfValueProfMemOpFuncName() {
-  return INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR;
-}
-
 /// Return the name prefix of variables containing instrumented function names.
 inline StringRef getInstrProfNameVarPrefix() { return "__profn_"; }
 
@@ -1146,7 +1138,8 @@ void getMemOPSizeRangeFromOption(StringRef Str, int64_t &RangeStart,
 
 // Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime
 // aware this is an ir_level profile so it can set the version flag.
-void createIRLevelProfileFlagVar(Module &M, bool IsCS);
+void createIRLevelProfileFlagVar(Module &M, bool IsCS,
+                                 bool InstrEntryBBEnabled);
 
 // Create the variable for the profile file name.
 void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput);
diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc
index 6d0ffb12294b0..e56623afad644 100644
--- a/llvm/include/llvm/ProfileData/InstrProfData.inc
+++ b/llvm/include/llvm/ProfileData/InstrProfData.inc
@@ -157,8 +157,6 @@ VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA
 #ifndef VALUE_RANGE_PROF
 VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
 #else /* VALUE_RANGE_PROF */
-/* FIXME: This is to be removed after switching to the new memop value
- * profiling. */
 VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) \
                       INSTR_PROF_COMMA
 VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeStart, Type::getInt64Ty(Ctx)) \
@@ -673,6 +671,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
 #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
 #define VARIANT_MASK_IR_PROF (0x1ULL << 56)
 #define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
+#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
 #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
 #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
 
@@ -755,14 +754,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
 #define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
 #define INSTR_PROF_VALUE_PROF_FUNC_STR \
         INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
-/* FIXME: This is to be removed after switching to the new memop value
- * profiling. */
 #define INSTR_PROF_VALUE_RANGE_PROF_FUNC __llvm_profile_instrument_range
 #define INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR \
         INSTR_PROF_QUOTE(INSTR_PROF_VALUE_RANGE_PROF_FUNC)
-#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
-#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR                                   \
-  INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
 
 /* InstrProfile per-function control data alignment.  */
 #define INSTR_PROF_DATA_ALIGNMENT 8
@@ -790,121 +784,3 @@ typedef struct InstrProfValueData {
 #endif
 
 #undef COVMAP_V2_OR_V3
-
-#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
-
-#ifdef __cplusplus
-#define INSTR_PROF_INLINE inline
-#else
-#define INSTR_PROF_INLINE
-#endif
-
-/* The value range buckets (22 buckets) for the memop size value profiling looks
- * like:
- *
- *   [0, 0]
- *   [1, 1]
- *   [2, 2]
- *   [3, 3]
- *   [4, 4]
- *   [5, 5]
- *   [6, 6]
- *   [7, 7]
- *   [8, 8]
- *   [9, 15]
- *   [16, 16]
- *   [17, 31]
- *   [32, 32]
- *   [33, 63]
- *   [64, 64]
- *   [65, 127]
- *   [128, 128]
- *   [129, 255]
- *   [256, 256]
- *   [257, 511]
- *   [512, 512]
- *   [513, UINT64_MAX]
- *
- * Each range has a 'representative value' which is the lower end value of the
- * range and used to store in the runtime profile data records and the VP
- * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127].
- */
-
-/*
- * Clz and Popcount. This code was copied from
- * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and
- * llvm/include/llvm/Support/MathExtras.h.
- */
-#if defined(_MSC_VER) && !defined(__clang__)
-
-#include <intrin.h>
-INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
-int InstProfClzll(unsigned long long X) {
-  unsigned long LeadZeroIdx = 0;
-#if !defined(_M_ARM64) && !defined(_M_X64)
-  // Scan the high 32 bits.
-  if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32)))
-    return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset
-                                                      // from the MSB.
-  // Scan the low 32 bits.
-  if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X)))
-    return (int)(63 - LeadZeroIdx);
-#else
-  if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx;
-#endif
-  return 64;
-}
-INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
-int InstProfPopcountll(unsigned long long X) {
-  // This code originates from https://reviews.llvm.org/rG30626254510f.
-  unsigned long long v = X;
-  v = v - ((v >> 1) & 0x5555555555555555ULL);
-  v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
-  v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
-  return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56);
-}
-
-#else
-
-INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
-int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); }
-INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
-int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); }
-
-#endif  /* defined(_MSC_VER) && !defined(__clang__) */
-
-/* Map an (observed) memop size value to the representative value of its range.
- * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */
-INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t
-InstrProfGetRangeRepValue(uint64_t Value) {
-  if (Value <= 8)
-    // The first ranges are individually tracked. Use the value as is.
-    return Value;
-  else if (Value >= 513)
-    // The last range is mapped to its lowest value.
-    return 513;
-  else if (InstProfPopcountll(Value) == 1)
-    // If it's a power of two, use it as is.
-    return Value;
-  else
-    // Otherwise, take to the previous power of two + 1.
-    return (1 << (64 - InstProfClzll(Value) - 1)) + 1;
-}
-
-/* Return true if the range that an (observed) memop size value belongs to has
- * only a single value in the range.  For example, 0 -> true, 8 -> true, 10 ->
- * false, 64 -> true, 100 -> false, 513 -> false. */
-INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned
-InstrProfIsSingleValRange(uint64_t Value) {
-  if (Value <= 8)
-    // The first ranges are individually tracked.
-    return 1;
-  else if (InstProfPopcountll(Value) == 1)
-    // If it's a power of two, there's only one value.
-    return 1;
-  else
-    // Otherwise, there's more than one value in the range.
-    return 0;
-}
-
-#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */
diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h
index f5f552672bf03..e4e81219855ec 100644
--- a/llvm/include/llvm/ProfileData/InstrProfReader.h
+++ b/llvm/include/llvm/ProfileData/InstrProfReader.h
@@ -79,6 +79,8 @@ class InstrProfReader {
 
   virtual bool hasCSIRLevelProfile() const = 0;
 
+  virtual bool instrEntryBBEnabled() const = 0;
+
   /// Return the PGO symtab. There are three different readers:
   /// Raw, Text, and Indexed profile readers. The first two types
   /// of readers are used only by llvm-profdata tool, while the indexed
@@ -148,6 +150,7 @@ class TextInstrProfReader : public InstrProfReader {
   line_iterator Line;
   bool IsIRLevelProfile = false;
   bool HasCSIRLevelProfile = false;
+  bool InstrEntryBBEnabled = false;
 
   Error readValueProfileData(InstrProfRecord &Record);
 
@@ -164,6 +167,8 @@ class TextInstrProfReader : public InstrProfReader {
 
   bool hasCSIRLevelProfile() const override { return HasCSIRLevelProfile; }
 
+  bool instrEntryBBEnabled() const override { return InstrEntryBBEnabled; }
+
   /// Read the header.
   Error readHeader() override;
 
@@ -224,6 +229,10 @@ class RawInstrProfReader : public InstrProfReader {
     return (Version & VARIANT_MASK_CSIR_PROF) != 0;
   }
 
+  bool instrEntryBBEnabled() const override {
+    return (Version & VARIANT_MASK_INSTR_ENTRY) != 0;
+  }
+
   InstrProfSymtab &getSymtab() override {
     assert(Symtab.get());
     return *Symtab.get();
@@ -360,6 +369,7 @@ struct InstrProfReaderIndexBase {
   virtual uint64_t getVersion() const = 0;
   virtual bool isIRLevelProfile() const = 0;
   virtual bool hasCSIRLevelProfile() const = 0;
+  virtual bool instrEntryBBEnabled() const = 0;
   virtual Error populateSymtab(InstrProfSymtab &) = 0;
 };
 
@@ -408,6 +418,10 @@ class InstrProfReaderIndex : public InstrProfReaderIndexBase {
     return (FormatVersion & VARIANT_MASK_CSIR_PROF) != 0;
   }
 
+  bool instrEntryBBEnabled() const override {
+    return (FormatVersion & VARIANT_MASK_INSTR_ENTRY) != 0;
+  }
+
   Error populateSymtab(InstrProfSymtab &Symtab) override {
     return Symtab.create(HashTable->keys());
   }
@@ -462,6 +476,10 @@ class IndexedInstrProfReader : public InstrProfReader {
     return Index->hasCSIRLevelProfile();
   }
 
+  bool instrEntryBBEnabled() const override {
+    return Index->instrEntryBBEnabled();
+  }
+
   /// Return true if the given buffer is in an indexed instrprof format.
   static bool hasFormat(const MemoryBuffer &DataBuffer);
 
diff --git a/llvm/include/llvm/ProfileData/InstrProfWriter.h b/llvm/include/llvm/ProfileData/InstrProfWriter.h
index 5882fa2781e27..2d69bba26a29c 100644
--- a/llvm/include/llvm/ProfileData/InstrProfWriter.h
+++ b/llvm/include/llvm/ProfileData/InstrProfWriter.h
@@ -40,11 +40,12 @@ class InstrProfWriter {
   bool Sparse;
   StringMap<ProfilingData> FunctionData;
   ProfKind ProfileKind = PF_Unknown;
+  bool InstrEntryBBEnabled;
   // Use raw pointer here for the incomplete type object.
   InstrProfRecordWriterTrait *InfoObj;
 
 public:
-  InstrProfWriter(bool Sparse = false);
+  InstrProfWriter(bool Sparse = false, bool InstrEntryBBEnabled = false);
   ~InstrProfWriter();
 
   /// Add function counts for the given function. If there are already counts
@@ -97,6 +98,7 @@ class InstrProfWriter {
     return Error::success();
   }
 
+  void setInstrEntryBBEnabled(bool Enabled) { InstrEntryBBEnabled = Enabled; }
   // Internal interface for testing purpose only.
   void setValueProfDataEndianness(support::endianness Endianness);
   void setOutputSparse(bool Sparse);
diff --git a/llvm/include/llvm/SYCLLowerIR/LowerESIMD.h b/llvm/include/llvm/SYCLLowerIR/LowerESIMD.h
index ac76459edafce..632459c2efd09 100644
--- a/llvm/include/llvm/SYCLLowerIR/LowerESIMD.h
+++ b/llvm/include/llvm/SYCLLowerIR/LowerESIMD.h
@@ -42,6 +42,9 @@ class ESIMDLowerLoadStorePass : public PassInfoMixin<ESIMDLowerLoadStorePass> {
 FunctionPass *createESIMDLowerLoadStorePass();
 void initializeESIMDLowerLoadStorePass(PassRegistry &);
 
+ModulePass *createESIMDLowerVecArgPass();
+void initializeESIMDLowerVecArgLegacyPassPass(PassRegistry &);
+
 } // namespace llvm
 
 #endif // LLVM_SYCLLOWERIR_LOWERESIMD_H
diff --git a/llvm/include/llvm/Support/CrashRecoveryContext.h b/llvm/include/llvm/Support/CrashRecoveryContext.h
index 61a1bd405a4d4..1529c197a301f 100644
--- a/llvm/include/llvm/Support/CrashRecoveryContext.h
+++ b/llvm/include/llvm/Support/CrashRecoveryContext.h
@@ -181,7 +181,7 @@ class CrashRecoveryContextDestructorCleanup : public
       : CrashRecoveryContextCleanupBase<
             CrashRecoveryContextDestructorCleanup<T>, T>(context, resource) {}
 
-  virtual void recoverResources() {
+  void recoverResources() override {
     this->resource->~T();
   }
 };
diff --git a/llvm/include/llvm/Support/FormatVariadic.h b/llvm/include/llvm/Support/FormatVariadic.h
index dfafc3ccb44e4..094b054f773fb 100644
--- a/llvm/include/llvm/Support/FormatVariadic.h
+++ b/llvm/include/llvm/Support/FormatVariadic.h
@@ -205,10 +205,10 @@ template <typename Tuple> class formatv_object : public formatv_object_base {
 //
 // The characters '{' and '}' are reserved and cannot appear anywhere within a
 // replacement sequence.  Outside of a replacement sequence, in order to print
-// a literal '{' or '}' it must be doubled -- "{{" to print a literal '{' and
-// "}}" to print a literal '}'.
+// a literal '{' it must be doubled as "{{".
 //
 // ===Parameter Indexing===
+//
 // `index` specifies the index of the parameter in the parameter pack to format
 // into the output.  Note that it is possible to refer to the same parameter
 // index multiple times in a given format string.  This makes it possible to
diff --git a/llvm/include/llvm/Support/Program.h b/llvm/include/llvm/Support/Program.h
index dbda064cda058..d729d38836503 100644
--- a/llvm/include/llvm/Support/Program.h
+++ b/llvm/include/llvm/Support/Program.h
@@ -218,7 +218,7 @@ namespace sys {
   /// to build a single flat command line appropriate for calling CreateProcess
   /// on
   /// Windows.
-  std::string flattenWindowsCommandLine(ArrayRef<StringRef> Args);
+  ErrorOr<std::wstring> flattenWindowsCommandLine(ArrayRef<StringRef> Args);
 #endif
   }
 }
diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def
index c069f5d22ba83..f61b3f9acb3aa 100644
--- a/llvm/include/llvm/Support/TargetOpcodes.def
+++ b/llvm/include/llvm/Support/TargetOpcodes.def
@@ -493,6 +493,9 @@ HANDLE_TARGET_OPCODE(G_FREM)
 /// Generic FP exponentiation.
 HANDLE_TARGET_OPCODE(G_FPOW)
 
+/// Generic FP exponentiation, with an integer exponent.
+HANDLE_TARGET_OPCODE(G_FPOWI)
+
 /// Generic base-e exponential of a value.
 HANDLE_TARGET_OPCODE(G_FEXP)
 
diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index 3d8262b2404f8..03795c0050a2c 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -732,6 +732,13 @@ def G_FPOW : GenericInstruction {
   let hasSideEffects = 0;
 }
 
+// Floating point exponentiation, with an integer power.
+def G_FPOWI : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src0, type1:$src1);
+  let hasSideEffects = 0;
+}
+
 // Floating point base-e exponential of a value.
 def G_FEXP : GenericInstruction {
   let OutOperandList = (outs type0:$dst);
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index 1dd3e374b5245..40ed6be089ac8 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -125,11 +125,11 @@ def extending_loads : GICombineRule<
   (apply [{ Helper.applyCombineExtendingLoads(*${root}, ${matchinfo}); }])>;
 def combines_for_extload: GICombineGroup<[extending_loads]>;
 
-def sext_already_extended : GICombineRule<
+def sext_trunc_sextload : GICombineRule<
   (defs root:$d),
   (match (wip_match_opcode G_SEXT_INREG):$d,
-         [{ return Helper.matchSextAlreadyExtended(*${d}); }]),
-  (apply [{ Helper.applySextAlreadyExtended(*${d}); }])>;
+         [{ return Helper.matchSextTruncSextLoad(*${d}); }]),
+  (apply [{ Helper.applySextTruncSextLoad(*${d}); }])>;
 
 def combine_indexed_load_store : GICombineRule<
   (defs root:$root, indexed_load_store_matchdata:$matchinfo),
diff --git a/llvm/include/llvm/Target/TargetLoweringObjectFile.h b/llvm/include/llvm/Target/TargetLoweringObjectFile.h
index cc6c93b6ee2b9..0cb888e750781 100644
--- a/llvm/include/llvm/Target/TargetLoweringObjectFile.h
+++ b/llvm/include/llvm/Target/TargetLoweringObjectFile.h
@@ -247,7 +247,8 @@ class TargetLoweringObjectFile : public MCObjectFileInfo {
 
   /// If supported, return the function entry point symbol.
   /// Otherwise, returns nulltpr.
-  virtual MCSymbol *getFunctionEntryPointSymbol(const Function *F,
+  /// Func must be a function or an alias which has a function as base object.
+  virtual MCSymbol *getFunctionEntryPointSymbol(const GlobalValue *Func,
                                                 const TargetMachine &TM) const {
     return nullptr;
   }
diff --git a/llvm/include/llvm/Target/TargetMachine.h b/llvm/include/llvm/Target/TargetMachine.h
index 6d539f1145ee1..f59bc5e5bae59 100644
--- a/llvm/include/llvm/Target/TargetMachine.h
+++ b/llvm/include/llvm/Target/TargetMachine.h
@@ -241,8 +241,6 @@ class TargetMachine {
     Options.SupportsDebugEntryValues = Enable;
   }
 
-  bool shouldPrintMachineCode() const { return Options.PrintMachineCode; }
-
   bool getUniqueSectionNames() const { return Options.UniqueSectionNames; }
 
   /// Return true if unique basic block section names must be generated.
diff --git a/llvm/include/llvm/Target/TargetOptions.h b/llvm/include/llvm/Target/TargetOptions.h
index d73686b2bdd89..c3f0963f59970 100644
--- a/llvm/include/llvm/Target/TargetOptions.h
+++ b/llvm/include/llvm/Target/TargetOptions.h
@@ -113,9 +113,8 @@ namespace llvm {
   class TargetOptions {
   public:
     TargetOptions()
-        : PrintMachineCode(false), UnsafeFPMath(false), NoInfsFPMath(false),
-          NoNaNsFPMath(false), NoTrappingFPMath(true),
-          NoSignedZerosFPMath(false),
+        : UnsafeFPMath(false), NoInfsFPMath(false), NoNaNsFPMath(false),
+          NoTrappingFPMath(true), NoSignedZerosFPMath(false),
           HonorSignDependentRoundingFPMathOption(false), NoZerosInBSS(false),
           GuaranteedTailCallOpt(false), StackSymbolOrdering(true),
           EnableFastISel(false), EnableGlobalISel(false), UseInitArray(false),
@@ -131,11 +130,6 @@ namespace llvm {
           XRayOmitFunctionIndex(false),
           FPDenormalMode(DenormalMode::IEEE, DenormalMode::IEEE) {}
 
-    /// PrintMachineCode - This flag is enabled when the -print-machineinstrs
-    /// option is specified on the command line, and should enable debugging
-    /// output from the code generator.
-    unsigned PrintMachineCode : 1;
-
     /// DisableFramePointerElim - This returns true if frame pointer elimination
     /// optimization should be disabled for the given machine function.
     bool DisableFramePointerElim(const MachineFunction &MF) const;
diff --git a/llvm/include/llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h b/llvm/include/llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h
index 887c8807904e4..e5e24e0b6311a 100644
--- a/llvm/include/llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h
+++ b/llvm/include/llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h
@@ -17,7 +17,6 @@
 #ifndef LLVM_TRANSFORMS_AGGRESSIVE_INSTCOMBINE_INSTCOMBINE_H
 #define LLVM_TRANSFORMS_AGGRESSIVE_INSTCOMBINE_INSTCOMBINE_H
 
-#include "llvm/IR/Function.h"
 #include "llvm/IR/PassManager.h"
 
 namespace llvm {
diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h
index bed180e6717a2..1eead8ee788de 100644
--- a/llvm/include/llvm/Transforms/IPO/Attributor.h
+++ b/llvm/include/llvm/Transforms/IPO/Attributor.h
@@ -715,6 +715,21 @@ struct InformationCache {
   /// Return the map conaining all the knowledge we have from `llvm.assume`s.
   const RetainedKnowledgeMap &getKnowledgeMap() const { return KnowledgeMap; }
 
+  /// Return if \p To is potentially reachable form \p From or not
+  /// If the same query was answered, return cached result
+  bool getPotentiallyReachable(const Instruction &From, const Instruction &To) {
+    auto KeyPair = std::make_pair(&From, &To);
+    auto Iter = PotentiallyReachableMap.find(KeyPair);
+    if (Iter != PotentiallyReachableMap.end())
+      return Iter->second;
+    const Function &F = *From.getFunction();
+    bool Result = isPotentiallyReachable(
+        &From, &To, nullptr, AG.getAnalysis<DominatorTreeAnalysis>(F),
+        AG.getAnalysis<LoopAnalysis>(F));
+    PotentiallyReachableMap.insert(std::make_pair(KeyPair, Result));
+    return Result;
+  }
+
 private:
   struct FunctionInfo {
     ~FunctionInfo();
@@ -774,6 +789,10 @@ struct InformationCache {
   /// Set of inlineable functions
   SmallPtrSet<const Function *, 8> InlineableFunctions;
 
+  /// A map for caching results of queries for isPotentiallyReachable
+  DenseMap<std::pair<const Instruction *, const Instruction *>, bool>
+      PotentiallyReachableMap;
+
   /// Give the Attributor access to the members so
   /// Attributor::identifyDefaultAbstractAttributes(...) can initialize them.
   friend struct Attributor;
@@ -2374,16 +2393,17 @@ struct AAReachability : public StateWrapper<BooleanState, AbstractAttribute> {
   /// Returns true if 'From' instruction is assumed to reach, 'To' instruction.
   /// Users should provide two positions they are interested in, and the class
   /// determines (and caches) reachability.
-  bool isAssumedReachable(const Instruction *From,
-                          const Instruction *To) const {
-    return isPotentiallyReachable(From, To);
+  bool isAssumedReachable(Attributor &A, const Instruction &From,
+                          const Instruction &To) const {
+    return A.getInfoCache().getPotentiallyReachable(From, To);
   }
 
   /// Returns true if 'From' instruction is known to reach, 'To' instruction.
   /// Users should provide two positions they are interested in, and the class
   /// determines (and caches) reachability.
-  bool isKnownReachable(const Instruction *From, const Instruction *To) const {
-    return isPotentiallyReachable(From, To);
+  bool isKnownReachable(Attributor &A, const Instruction &From,
+                        const Instruction &To) const {
+    return A.getInfoCache().getPotentiallyReachable(From, To);
   }
 
   /// Create an abstract attribute view for the position \p IRP.
diff --git a/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h b/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h
new file mode 100644
index 0000000000000..c51b0297e3422
--- /dev/null
+++ b/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h
@@ -0,0 +1,518 @@
+//===- InstCombiner.h - InstCombine implementation --------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+///
+/// This file provides the interface for the instcombine pass implementation.
+/// The interface is used for generic transformations in this folder and
+/// target specific combinations in the targets.
+/// The visitor implementation is in \c InstCombinerImpl in
+/// \c InstCombineInternal.h.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_INSTCOMBINE_INSTCOMBINER_H
+#define LLVM_TRANSFORMS_INSTCOMBINE_INSTCOMBINER_H
+
+#include "llvm/Analysis/InstructionSimplify.h"
+#include "llvm/Analysis/TargetFolder.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/PatternMatch.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/KnownBits.h"
+#include "llvm/Transforms/InstCombine/InstCombineWorklist.h"
+#include <cassert>
+
+#define DEBUG_TYPE "instcombine"
+
+namespace llvm {
+
+class AAResults;
+class AssumptionCache;
+class ProfileSummaryInfo;
+class TargetLibraryInfo;
+class TargetTransformInfo;
+struct KnownBits;
+
+/// The core instruction combiner logic.
+///
+/// This class provides both the logic to recursively visit instructions and
+/// combine them.
+class LLVM_LIBRARY_VISIBILITY InstCombiner {
+  /// Only used to call target specific inst combining.
+  TargetTransformInfo &TTI;
+
+public:
+  /// Maximum size of array considered when transforming.
+  uint64_t MaxArraySizeForCombine = 0;
+
+  /// An IRBuilder that automatically inserts new instructions into the
+  /// worklist.
+  using BuilderTy = IRBuilder<TargetFolder, IRBuilderCallbackInserter>;
+  BuilderTy &Builder;
+
+protected:
+  /// A worklist of the instructions that need to be simplified.
+  InstCombineWorklist &Worklist;
+
+  // Mode in which we are running the combiner.
+  const bool MinimizeSize;
+
+  AAResults *AA;
+
+  // Required analyses.
+  AssumptionCache &AC;
+  TargetLibraryInfo &TLI;
+  DominatorTree &DT;
+  const DataLayout &DL;
+  const SimplifyQuery SQ;
+  OptimizationRemarkEmitter &ORE;
+  BlockFrequencyInfo *BFI;
+  ProfileSummaryInfo *PSI;
+
+  // Optional analyses. When non-null, these can both be used to do better
+  // combining and will be updated to reflect any changes.
+  LoopInfo *LI;
+
+  bool MadeIRChange = false;
+
+public:
+  InstCombiner(InstCombineWorklist &Worklist, BuilderTy &Builder,
+               bool MinimizeSize, AAResults *AA, AssumptionCache &AC,
+               TargetLibraryInfo &TLI, TargetTransformInfo &TTI,
+               DominatorTree &DT, OptimizationRemarkEmitter &ORE,
+               BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
+               const DataLayout &DL, LoopInfo *LI)
+      : TTI(TTI), Builder(Builder), Worklist(Worklist),
+        MinimizeSize(MinimizeSize), AA(AA), AC(AC), TLI(TLI), DT(DT), DL(DL),
+        SQ(DL, &TLI, &DT, &AC), ORE(ORE), BFI(BFI), PSI(PSI), LI(LI) {}
+
+  virtual ~InstCombiner() {}
+
+  /// Return the source operand of a potentially bitcasted value while
+  /// optionally checking if it has one use. If there is no bitcast or the one
+  /// use check is not met, return the input value itself.
+  static Value *peekThroughBitcast(Value *V, bool OneUseOnly = false) {
+    if (auto *BitCast = dyn_cast<BitCastInst>(V))
+      if (!OneUseOnly || BitCast->hasOneUse())
+        return BitCast->getOperand(0);
+
+    // V is not a bitcast or V has more than one use and OneUseOnly is true.
+    return V;
+  }
+
+  /// Assign a complexity or rank value to LLVM Values. This is used to reduce
+  /// the amount of pattern matching needed for compares and commutative
+  /// instructions. For example, if we have:
+  ///   icmp ugt X, Constant
+  /// or
+  ///   xor (add X, Constant), cast Z
+  ///
+  /// We do not have to consider the commuted variants of these patterns because
+  /// canonicalization based on complexity guarantees the above ordering.
+  ///
+  /// This routine maps IR values to various complexity ranks:
+  ///   0 -> undef
+  ///   1 -> Constants
+  ///   2 -> Other non-instructions
+  ///   3 -> Arguments
+  ///   4 -> Cast and (f)neg/not instructions
+  ///   5 -> Other instructions
+  static unsigned getComplexity(Value *V) {
+    if (isa<Instruction>(V)) {
+      if (isa<CastInst>(V) || match(V, m_Neg(PatternMatch::m_Value())) ||
+          match(V, m_Not(PatternMatch::m_Value())) ||
+          match(V, m_FNeg(PatternMatch::m_Value())))
+        return 4;
+      return 5;
+    }
+    if (isa<Argument>(V))
+      return 3;
+    return isa<Constant>(V) ? (isa<UndefValue>(V) ? 0 : 1) : 2;
+  }
+
+  /// Predicate canonicalization reduces the number of patterns that need to be
+  /// matched by other transforms. For example, we may swap the operands of a
+  /// conditional branch or select to create a compare with a canonical
+  /// (inverted) predicate which is then more likely to be matched with other
+  /// values.
+  static bool isCanonicalPredicate(CmpInst::Predicate Pred) {
+    switch (Pred) {
+    case CmpInst::ICMP_NE:
+    case CmpInst::ICMP_ULE:
+    case CmpInst::ICMP_SLE:
+    case CmpInst::ICMP_UGE:
+    case CmpInst::ICMP_SGE:
+    // TODO: There are 16 FCMP predicates. Should others be (not) canonical?
+    case CmpInst::FCMP_ONE:
+    case CmpInst::FCMP_OLE:
+    case CmpInst::FCMP_OGE:
+      return false;
+    default:
+      return true;
+    }
+  }
+
+  /// Given an exploded icmp instruction, return true if the comparison only
+  /// checks the sign bit. If it only checks the sign bit, set TrueIfSigned if
+  /// the result of the comparison is true when the input value is signed.
+  static bool isSignBitCheck(ICmpInst::Predicate Pred, const APInt &RHS,
+                             bool &TrueIfSigned) {
+    switch (Pred) {
+    case ICmpInst::ICMP_SLT: // True if LHS s< 0
+      TrueIfSigned = true;
+      return RHS.isNullValue();
+    case ICmpInst::ICMP_SLE: // True if LHS s<= -1
+      TrueIfSigned = true;
+      return RHS.isAllOnesValue();
+    case ICmpInst::ICMP_SGT: // True if LHS s> -1
+      TrueIfSigned = false;
+      return RHS.isAllOnesValue();
+    case ICmpInst::ICMP_SGE: // True if LHS s>= 0
+      TrueIfSigned = false;
+      return RHS.isNullValue();
+    case ICmpInst::ICMP_UGT:
+      // True if LHS u> RHS and RHS == sign-bit-mask - 1
+      TrueIfSigned = true;
+      return RHS.isMaxSignedValue();
+    case ICmpInst::ICMP_UGE:
+      // True if LHS u>= RHS and RHS == sign-bit-mask (2^7, 2^15, 2^31, etc)
+      TrueIfSigned = true;
+      return RHS.isMinSignedValue();
+    case ICmpInst::ICMP_ULT:
+      // True if LHS u< RHS and RHS == sign-bit-mask (2^7, 2^15, 2^31, etc)
+      TrueIfSigned = false;
+      return RHS.isMinSignedValue();
+    case ICmpInst::ICMP_ULE:
+      // True if LHS u<= RHS and RHS == sign-bit-mask - 1
+      TrueIfSigned = false;
+      return RHS.isMaxSignedValue();
+    default:
+      return false;
+    }
+  }
+
+  /// Add one to a Constant
+  static Constant *AddOne(Constant *C) {
+    return ConstantExpr::getAdd(C, ConstantInt::get(C->getType(), 1));
+  }
+
+  /// Subtract one from a Constant
+  static Constant *SubOne(Constant *C) {
+    return ConstantExpr::getSub(C, ConstantInt::get(C->getType(), 1));
+  }
+
+  llvm::Optional<std::pair<
+      CmpInst::Predicate,
+      Constant *>> static getFlippedStrictnessPredicateAndConstant(CmpInst::
+                                                                       Predicate
+                                                                           Pred,
+                                                                   Constant *C);
+
+  /// Return true if the specified value is free to invert (apply ~ to).
+  /// This happens in cases where the ~ can be eliminated.  If WillInvertAllUses
+  /// is true, work under the assumption that the caller intends to remove all
+  /// uses of V and only keep uses of ~V.
+  ///
+  /// See also: canFreelyInvertAllUsersOf()
+  static bool isFreeToInvert(Value *V, bool WillInvertAllUses) {
+    // ~(~(X)) -> X.
+    if (match(V, m_Not(PatternMatch::m_Value())))
+      return true;
+
+    // Constants can be considered to be not'ed values.
+    if (match(V, PatternMatch::m_AnyIntegralConstant()))
+      return true;
+
+    // Compares can be inverted if all of their uses are being modified to use
+    // the ~V.
+    if (isa<CmpInst>(V))
+      return WillInvertAllUses;
+
+    // If `V` is of the form `A + Constant` then `-1 - V` can be folded into
+    // `(-1 - Constant) - A` if we are willing to invert all of the uses.
+    if (BinaryOperator *BO = dyn_cast<BinaryOperator>(V))
+      if (BO->getOpcode() == Instruction::Add ||
+          BO->getOpcode() == Instruction::Sub)
+        if (isa<Constant>(BO->getOperand(0)) ||
+            isa<Constant>(BO->getOperand(1)))
+          return WillInvertAllUses;
+
+    // Selects with invertible operands are freely invertible
+    if (match(V,
+              m_Select(PatternMatch::m_Value(), m_Not(PatternMatch::m_Value()),
+                       m_Not(PatternMatch::m_Value()))))
+      return WillInvertAllUses;
+
+    return false;
+  }
+
+  /// Given i1 V, can every user of V be freely adapted if V is changed to !V ?
+  /// InstCombine's canonicalizeICmpPredicate() must be kept in sync with this
+  /// fn.
+  ///
+  /// See also: isFreeToInvert()
+  static bool canFreelyInvertAllUsersOf(Value *V, Value *IgnoredUser) {
+    // Look at every user of V.
+    for (Use &U : V->uses()) {
+      if (U.getUser() == IgnoredUser)
+        continue; // Don't consider this user.
+
+      auto *I = cast<Instruction>(U.getUser());
+      switch (I->getOpcode()) {
+      case Instruction::Select:
+        if (U.getOperandNo() != 0) // Only if the value is used as select cond.
+          return false;
+        break;
+      case Instruction::Br:
+        assert(U.getOperandNo() == 0 && "Must be branching on that value.");
+        break; // Free to invert by swapping true/false values/destinations.
+      case Instruction::Xor: // Can invert 'xor' if it's a 'not', by ignoring
+                             // it.
+        if (!match(I, m_Not(PatternMatch::m_Value())))
+          return false; // Not a 'not'.
+        break;
+      default:
+        return false; // Don't know, likely not freely invertible.
+      }
+      // So far all users were free to invert...
+    }
+    return true; // Can freely invert all users!
+  }
+
+  /// Some binary operators require special handling to avoid poison and
+  /// undefined behavior. If a constant vector has undef elements, replace those
+  /// undefs with identity constants if possible because those are always safe
+  /// to execute. If no identity constant exists, replace undef with some other
+  /// safe constant.
+  static Constant *
+  getSafeVectorConstantForBinop(BinaryOperator::BinaryOps Opcode, Constant *In,
+                                bool IsRHSConstant) {
+    auto *InVTy = dyn_cast<VectorType>(In->getType());
+    assert(InVTy && "Not expecting scalars here");
+
+    Type *EltTy = InVTy->getElementType();
+    auto *SafeC = ConstantExpr::getBinOpIdentity(Opcode, EltTy, IsRHSConstant);
+    if (!SafeC) {
+      // TODO: Should this be available as a constant utility function? It is
+      // similar to getBinOpAbsorber().
+      if (IsRHSConstant) {
+        switch (Opcode) {
+        case Instruction::SRem: // X % 1 = 0
+        case Instruction::URem: // X %u 1 = 0
+          SafeC = ConstantInt::get(EltTy, 1);
+          break;
+        case Instruction::FRem: // X % 1.0 (doesn't simplify, but it is safe)
+          SafeC = ConstantFP::get(EltTy, 1.0);
+          break;
+        default:
+          llvm_unreachable(
+              "Only rem opcodes have no identity constant for RHS");
+        }
+      } else {
+        switch (Opcode) {
+        case Instruction::Shl:  // 0 << X = 0
+        case Instruction::LShr: // 0 >>u X = 0
+        case Instruction::AShr: // 0 >> X = 0
+        case Instruction::SDiv: // 0 / X = 0
+        case Instruction::UDiv: // 0 /u X = 0
+        case Instruction::SRem: // 0 % X = 0
+        case Instruction::URem: // 0 %u X = 0
+        case Instruction::Sub:  // 0 - X (doesn't simplify, but it is safe)
+        case Instruction::FSub: // 0.0 - X (doesn't simplify, but it is safe)
+        case Instruction::FDiv: // 0.0 / X (doesn't simplify, but it is safe)
+        case Instruction::FRem: // 0.0 % X = 0
+          SafeC = Constant::getNullValue(EltTy);
+          break;
+        default:
+          llvm_unreachable("Expected to find identity constant for opcode");
+        }
+      }
+    }
+    assert(SafeC && "Must have safe constant for binop");
+    unsigned NumElts = InVTy->getNumElements();
+    SmallVector<Constant *, 16> Out(NumElts);
+    for (unsigned i = 0; i != NumElts; ++i) {
+      Constant *C = In->getAggregateElement(i);
+      Out[i] = isa<UndefValue>(C) ? SafeC : C;
+    }
+    return ConstantVector::get(Out);
+  }
+
+  /// Create and insert the idiom we use to indicate a block is unreachable
+  /// without having to rewrite the CFG from within InstCombine.
+  static void CreateNonTerminatorUnreachable(Instruction *InsertAt) {
+    auto &Ctx = InsertAt->getContext();
+    new StoreInst(ConstantInt::getTrue(Ctx),
+                  UndefValue::get(Type::getInt1PtrTy(Ctx)), InsertAt);
+  }
+
+  void addToWorklist(Instruction *I) { Worklist.push(I); }
+
+  AssumptionCache &getAssumptionCache() const { return AC; }
+  TargetLibraryInfo &getTargetLibraryInfo() const { return TLI; }
+  DominatorTree &getDominatorTree() const { return DT; }
+  const DataLayout &getDataLayout() const { return DL; }
+  const SimplifyQuery &getSimplifyQuery() const { return SQ; }
+  OptimizationRemarkEmitter &getOptimizationRemarkEmitter() const {
+    return ORE;
+  }
+  BlockFrequencyInfo *getBlockFrequencyInfo() const { return BFI; }
+  ProfileSummaryInfo *getProfileSummaryInfo() const { return PSI; }
+  LoopInfo *getLoopInfo() const { return LI; }
+
+  // Call target specific combiners
+  Optional<Instruction *> targetInstCombineIntrinsic(IntrinsicInst &II);
+  Optional<Value *>
+  targetSimplifyDemandedUseBitsIntrinsic(IntrinsicInst &II, APInt DemandedMask,
+                                         KnownBits &Known,
+                                         bool &KnownBitsComputed);
+  Optional<Value *> targetSimplifyDemandedVectorEltsIntrinsic(
+      IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
+      APInt &UndefElts2, APInt &UndefElts3,
+      std::function<void(Instruction *, unsigned, APInt, APInt &)>
+          SimplifyAndSetOp);
+
+  /// Inserts an instruction \p New before instruction \p Old
+  ///
+  /// Also adds the new instruction to the worklist and returns \p New so that
+  /// it is suitable for use as the return from the visitation patterns.
+  Instruction *InsertNewInstBefore(Instruction *New, Instruction &Old) {
+    assert(New && !New->getParent() &&
+           "New instruction already inserted into a basic block!");
+    BasicBlock *BB = Old.getParent();
+    BB->getInstList().insert(Old.getIterator(), New); // Insert inst
+    Worklist.push(New);
+    return New;
+  }
+
+  /// Same as InsertNewInstBefore, but also sets the debug loc.
+  Instruction *InsertNewInstWith(Instruction *New, Instruction &Old) {
+    New->setDebugLoc(Old.getDebugLoc());
+    return InsertNewInstBefore(New, Old);
+  }
+
+  /// A combiner-aware RAUW-like routine.
+  ///
+  /// This method is to be used when an instruction is found to be dead,
+  /// replaceable with another preexisting expression. Here we add all uses of
+  /// I to the worklist, replace all uses of I with the new value, then return
+  /// I, so that the inst combiner will know that I was modified.
+  Instruction *replaceInstUsesWith(Instruction &I, Value *V) {
+    // If there are no uses to replace, then we return nullptr to indicate that
+    // no changes were made to the program.
+    if (I.use_empty())
+      return nullptr;
+
+    Worklist.pushUsersToWorkList(I); // Add all modified instrs to worklist.
+
+    // If we are replacing the instruction with itself, this must be in a
+    // segment of unreachable code, so just clobber the instruction.
+    if (&I == V)
+      V = UndefValue::get(I.getType());
+
+    LLVM_DEBUG(dbgs() << "IC: Replacing " << I << "\n"
+                      << "    with " << *V << '\n');
+
+    I.replaceAllUsesWith(V);
+    return &I;
+  }
+
+  /// Replace operand of instruction and add old operand to the worklist.
+  Instruction *replaceOperand(Instruction &I, unsigned OpNum, Value *V) {
+    Worklist.addValue(I.getOperand(OpNum));
+    I.setOperand(OpNum, V);
+    return &I;
+  }
+
+  /// Replace use and add the previously used value to the worklist.
+  void replaceUse(Use &U, Value *NewValue) {
+    Worklist.addValue(U);
+    U = NewValue;
+  }
+
+  /// Combiner aware instruction erasure.
+  ///
+  /// When dealing with an instruction that has side effects or produces a void
+  /// value, we can't rely on DCE to delete the instruction. Instead, visit
+  /// methods should return the value returned by this function.
+  virtual Instruction *eraseInstFromFunction(Instruction &I) = 0;
+
+  void computeKnownBits(const Value *V, KnownBits &Known, unsigned Depth,
+                        const Instruction *CxtI) const {
+    llvm::computeKnownBits(V, Known, DL, Depth, &AC, CxtI, &DT);
+  }
+
+  KnownBits computeKnownBits(const Value *V, unsigned Depth,
+                             const Instruction *CxtI) const {
+    return llvm::computeKnownBits(V, DL, Depth, &AC, CxtI, &DT);
+  }
+
+  bool isKnownToBeAPowerOfTwo(const Value *V, bool OrZero = false,
+                              unsigned Depth = 0,
+                              const Instruction *CxtI = nullptr) {
+    return llvm::isKnownToBeAPowerOfTwo(V, DL, OrZero, Depth, &AC, CxtI, &DT);
+  }
+
+  bool MaskedValueIsZero(const Value *V, const APInt &Mask, unsigned Depth = 0,
+                         const Instruction *CxtI = nullptr) const {
+    return llvm::MaskedValueIsZero(V, Mask, DL, Depth, &AC, CxtI, &DT);
+  }
+
+  unsigned ComputeNumSignBits(const Value *Op, unsigned Depth = 0,
+                              const Instruction *CxtI = nullptr) const {
+    return llvm::ComputeNumSignBits(Op, DL, Depth, &AC, CxtI, &DT);
+  }
+
+  OverflowResult computeOverflowForUnsignedMul(const Value *LHS,
+                                               const Value *RHS,
+                                               const Instruction *CxtI) const {
+    return llvm::computeOverflowForUnsignedMul(LHS, RHS, DL, &AC, CxtI, &DT);
+  }
+
+  OverflowResult computeOverflowForSignedMul(const Value *LHS, const Value *RHS,
+                                             const Instruction *CxtI) const {
+    return llvm::computeOverflowForSignedMul(LHS, RHS, DL, &AC, CxtI, &DT);
+  }
+
+  OverflowResult computeOverflowForUnsignedAdd(const Value *LHS,
+                                               const Value *RHS,
+                                               const Instruction *CxtI) const {
+    return llvm::computeOverflowForUnsignedAdd(LHS, RHS, DL, &AC, CxtI, &DT);
+  }
+
+  OverflowResult computeOverflowForSignedAdd(const Value *LHS, const Value *RHS,
+                                             const Instruction *CxtI) const {
+    return llvm::computeOverflowForSignedAdd(LHS, RHS, DL, &AC, CxtI, &DT);
+  }
+
+  OverflowResult computeOverflowForUnsignedSub(const Value *LHS,
+                                               const Value *RHS,
+                                               const Instruction *CxtI) const {
+    return llvm::computeOverflowForUnsignedSub(LHS, RHS, DL, &AC, CxtI, &DT);
+  }
+
+  OverflowResult computeOverflowForSignedSub(const Value *LHS, const Value *RHS,
+                                             const Instruction *CxtI) const {
+    return llvm::computeOverflowForSignedSub(LHS, RHS, DL, &AC, CxtI, &DT);
+  }
+
+  virtual bool SimplifyDemandedBits(Instruction *I, unsigned OpNo,
+                                    const APInt &DemandedMask, KnownBits &Known,
+                                    unsigned Depth = 0) = 0;
+  virtual Value *
+  SimplifyDemandedVectorElts(Value *V, APInt DemandedElts, APInt &UndefElts,
+                             unsigned Depth = 0,
+                             bool AllowMultipleUsers = false) = 0;
+};
+
+} // namespace llvm
+
+#undef DEBUG_TYPE
+
+#endif
diff --git a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
index a7052f7b6a2b1..263d3b629589c 100644
--- a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
+++ b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
@@ -68,8 +68,6 @@ class InstrProfiling : public PassInfoMixin<InstrProfiling> {
   // vector of counter load/store pairs to be register promoted.
   std::vector<LoadStorePair> PromotionCandidates;
 
-  // FIXME: These are to be removed after switching to the new memop value
-  // profiling.
   // The start value of precise value profile range for memory intrinsic sizes.
   int64_t MemOPSizeRangeStart;
   // The end value of precise value profile range for memory intrinsic sizes.
diff --git a/llvm/include/llvm/Transforms/Scalar.h b/llvm/include/llvm/Transforms/Scalar.h
index 7f5583570a44f..07d968efbcbbd 100644
--- a/llvm/include/llvm/Transforms/Scalar.h
+++ b/llvm/include/llvm/Transforms/Scalar.h
@@ -368,6 +368,13 @@ Pass *createLowerGuardIntrinsicPass();
 //
 Pass *createLowerMatrixIntrinsicsPass();
 
+//===----------------------------------------------------------------------===//
+//
+// LowerMatrixIntrinsicsMinimal - Lower matrix intrinsics to vector operations
+//                               (lightweight, does not require extra analysis)
+//
+Pass *createLowerMatrixIntrinsicsMinimalPass();
+
 //===----------------------------------------------------------------------===//
 //
 // LowerWidenableCondition - Lower widenable condition to i1 true.
diff --git a/llvm/include/llvm/Transforms/Utils/Debugify.h b/llvm/include/llvm/Transforms/Utils/Debugify.h
index 6f11d0a7d0624..1b9d43b775e75 100644
--- a/llvm/include/llvm/Transforms/Utils/Debugify.h
+++ b/llvm/include/llvm/Transforms/Utils/Debugify.h
@@ -13,8 +13,11 @@
 #ifndef LLVM_TRANSFORM_UTILS_DEBUGIFY_H
 #define LLVM_TRANSFORM_UTILS_DEBUGIFY_H
 
-#include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Bitcode/BitcodeWriterPass.h"
+#include "llvm/IR/IRPrintingPasses.h"
+#include "llvm/IR/LegacyPassManager.h"
 #include "llvm/IR/PassManager.h"
 
 namespace llvm {
@@ -89,4 +92,55 @@ struct NewPMCheckDebugifyPass
   llvm::PreservedAnalyses run(llvm::Module &M, llvm::ModuleAnalysisManager &AM);
 };
 
+namespace llvm {
+/// DebugifyCustomPassManager wraps each pass with the debugify passes if
+/// needed.
+/// NOTE: We support legacy custom pass manager only.
+/// TODO: Add New PM support for custom pass manager.
+class DebugifyCustomPassManager : public legacy::PassManager {
+  DebugifyStatsMap DIStatsMap;
+  bool EnableDebugifyEach = false;
+
+public:
+  using super = legacy::PassManager;
+
+  void add(Pass *P) override {
+    // Wrap each pass with (-check)-debugify passes if requested, making
+    // exceptions for passes which shouldn't see -debugify instrumentation.
+    bool WrapWithDebugify = EnableDebugifyEach && !P->getAsImmutablePass() &&
+                            !isIRPrintingPass(P) && !isBitcodeWriterPass(P);
+    if (!WrapWithDebugify) {
+      super::add(P);
+      return;
+    }
+
+    // Apply -debugify/-check-debugify before/after each pass and collect
+    // debug info loss statistics.
+    PassKind Kind = P->getPassKind();
+    StringRef Name = P->getPassName();
+
+    // TODO: Implement Debugify for LoopPass.
+    switch (Kind) {
+    case PT_Function:
+      super::add(createDebugifyFunctionPass());
+      super::add(P);
+      super::add(createCheckDebugifyFunctionPass(true, Name, &DIStatsMap));
+      break;
+    case PT_Module:
+      super::add(createDebugifyModulePass());
+      super::add(P);
+      super::add(createCheckDebugifyModulePass(true, Name, &DIStatsMap));
+      break;
+    default:
+      super::add(P);
+      break;
+    }
+  }
+
+  void enableDebugifyEach() { EnableDebugifyEach = true; }
+
+  const DebugifyStatsMap &getDebugifyStatsMap() const { return DIStatsMap; }
+};
+} // namespace llvm
+
 #endif // LLVM_TRANSFORM_UTILS_DEBUGIFY_H
diff --git a/llvm/include/llvm/Transforms/Utils/MatrixUtils.h b/llvm/include/llvm/Transforms/Utils/MatrixUtils.h
new file mode 100644
index 0000000000000..39a0d4bf40ccb
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Utils/MatrixUtils.h
@@ -0,0 +1,94 @@
+//===- MatrixUtils.h - Utilities to lower matrix intrinsics -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Utilities for generating tiled loops for matrix operations.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_UTILS_MATRIXUTILS_H
+#define LLVM_TRANSFORMS_UTILS_MATRIXUTILS_H
+
+#include "llvm/ADT/StringRef.h"
+
+namespace llvm {
+class DomTreeUpdater;
+class BasicBlock;
+class Value;
+class Loop;
+class LoopInfo;
+class IRBuilderBase;
+
+/// A helper struct to create IR loop nests for tiling in IR of the following
+/// form:
+///   for CurrentColumn = 0..NumColumns
+///     for CurrentRow = 0..NumRows
+///       for CurrentInner = 0..NumInner
+struct TileInfo {
+  /// Number of rows of the matrix.
+  unsigned NumRows;
+
+  /// Number of columns of the matrix.
+  unsigned NumColumns;
+
+  /// Number of columns of the first matrix of a multiply /
+  /// number of rows of the second matrix of a multiply.
+  unsigned NumInner;
+
+  /// Number of rows/columns in a tile.
+  unsigned TileSize = -1;
+
+  /// Start row of the current tile to compute.
+  Value *CurrentRow;
+
+  /// Start column of the current tile to compute.
+  Value *CurrentCol;
+
+  /// Current tile offset during the tile computation.
+  Value *CurrentK;
+
+  /// Header of the outermost loop iterating from 0..NumColumns.
+  BasicBlock *ColumnLoopHeader = nullptr;
+
+  /// Header of the second loop iterating from 0..NumRows.
+  BasicBlock *RowLoopHeader = nullptr;
+  /// Latch of the second loop iterating from 0..NumRows.
+  BasicBlock *RowLoopLatch = nullptr;
+  /// Header of the innermost loop iterating from 0..NumInner.
+  BasicBlock *InnerLoopHeader = nullptr;
+  /// Latch of the innermost loop iterating from 0..NumInner.
+  BasicBlock *InnerLoopLatch = nullptr;
+
+  TileInfo(unsigned NumRows, unsigned NumColumns, unsigned NumInner,
+           unsigned TileSize)
+      : NumRows(NumRows), NumColumns(NumColumns), NumInner(NumInner),
+        TileSize(TileSize) {}
+
+  /// Creates an IR loop nests for tiling of the form below. Returns the block
+  /// for the inner loop body and sets {Column,Row,Inner}LoopHeader/Latch
+  /// fields.
+  ///
+  /// for CurrentColumn = 0..NumColumns
+  ///   for CurrentRow = 0..NumRows
+  ///     for CurrentInner = 0..NumInner
+  BasicBlock *CreateTiledLoops(BasicBlock *Start, BasicBlock *End,
+                               IRBuilderBase &B, DomTreeUpdater &DTU,
+                               LoopInfo &LI);
+
+private:
+  /// Creates a new loop with header, body and latch blocks that iterates from
+  /// [0, Bound). Updates \p Preheader to branch to the new header and uses \p
+  /// Exit as exit block.  Adds the new loop blocks to \L and applies dominator
+  /// tree updates to \p DTU.
+  static BasicBlock *CreateLoop(BasicBlock *Preheader, BasicBlock *Exit,
+                                Value *Bound, Value *Step, StringRef Name,
+                                IRBuilderBase &B, DomTreeUpdater &DTU, Loop *L,
+                                LoopInfo &LI);
+};
+} // namespace llvm
+
+#endif
diff --git a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
index 0c88f9f79e766..6e53d15d25fc7 100644
--- a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
+++ b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
@@ -26,410 +26,400 @@
 #include "llvm/Support/CommandLine.h"
 
 namespace llvm {
-  extern cl::opt<unsigned> SCEVCheapExpansionBudget;
+extern cl::opt<unsigned> SCEVCheapExpansionBudget;
+
+/// Return true if the given expression is safe to expand in the sense that
+/// all materialized values are safe to speculate anywhere their operands are
+/// defined.
+bool isSafeToExpand(const SCEV *S, ScalarEvolution &SE);
+
+/// Return true if the given expression is safe to expand in the sense that
+/// all materialized values are defined and safe to speculate at the specified
+/// location and their operands are defined at this location.
+bool isSafeToExpandAt(const SCEV *S, const Instruction *InsertionPoint,
+                      ScalarEvolution &SE);
+
+/// This class uses information about analyze scalars to rewrite expressions
+/// in canonical form.
+///
+/// Clients should create an instance of this class when rewriting is needed,
+/// and destroy it when finished to allow the release of the associated
+/// memory.
+class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
+  ScalarEvolution &SE;
+  const DataLayout &DL;
+
+  // New instructions receive a name to identify them with the current pass.
+  const char *IVName;
+
+  // InsertedExpressions caches Values for reuse, so must track RAUW.
+  DenseMap<std::pair<const SCEV *, Instruction *>, TrackingVH<Value>>
+      InsertedExpressions;
+
+  // InsertedValues only flags inserted instructions so needs no RAUW.
+  DenseSet<AssertingVH<Value>> InsertedValues;
+  DenseSet<AssertingVH<Value>> InsertedPostIncValues;
+
+  /// A memoization of the "relevant" loop for a given SCEV.
+  DenseMap<const SCEV *, const Loop *> RelevantLoops;
+
+  /// Addrecs referring to any of the given loops are expanded in post-inc
+  /// mode. For example, expanding {1,+,1}<L> in post-inc mode returns the add
+  /// instruction that adds one to the phi for {0,+,1}<L>, as opposed to a new
+  /// phi starting at 1. This is only supported in non-canonical mode.
+  PostIncLoopSet PostIncLoops;
+
+  /// When this is non-null, addrecs expanded in the loop it indicates should
+  /// be inserted with increments at IVIncInsertPos.
+  const Loop *IVIncInsertLoop;
+
+  /// When expanding addrecs in the IVIncInsertLoop loop, insert the IV
+  /// increment at this position.
+  Instruction *IVIncInsertPos;
+
+  /// Phis that complete an IV chain. Reuse
+  DenseSet<AssertingVH<PHINode>> ChainedPhis;
+
+  /// When true, SCEVExpander tries to expand expressions in "canonical" form.
+  /// When false, expressions are expanded in a more literal form.
+  ///
+  /// In "canonical" form addrecs are expanded as arithmetic based on a
+  /// canonical induction variable. Note that CanonicalMode doesn't guarantee
+  /// that all expressions are expanded in "canonical" form. For some
+  /// expressions literal mode can be preferred.
+  bool CanonicalMode;
+
+  /// When invoked from LSR, the expander is in "strength reduction" mode. The
+  /// only difference is that phi's are only reused if they are already in
+  /// "expanded" form.
+  bool LSRMode;
+
+  typedef IRBuilder<TargetFolder, IRBuilderCallbackInserter> BuilderType;
+  BuilderType Builder;
+
+  // RAII object that stores the current insertion point and restores it when
+  // the object is destroyed. This includes the debug location.  Duplicated
+  // from InsertPointGuard to add SetInsertPoint() which is used to updated
+  // InsertPointGuards stack when insert points are moved during SCEV
+  // expansion.
+  class SCEVInsertPointGuard {
+    IRBuilderBase &Builder;
+    AssertingVH<BasicBlock> Block;
+    BasicBlock::iterator Point;
+    DebugLoc DbgLoc;
+    SCEVExpander *SE;
+
+    SCEVInsertPointGuard(const SCEVInsertPointGuard &) = delete;
+    SCEVInsertPointGuard &operator=(const SCEVInsertPointGuard &) = delete;
+
+  public:
+    SCEVInsertPointGuard(IRBuilderBase &B, SCEVExpander *SE)
+        : Builder(B), Block(B.GetInsertBlock()), Point(B.GetInsertPoint()),
+          DbgLoc(B.getCurrentDebugLocation()), SE(SE) {
+      SE->InsertPointGuards.push_back(this);
+    }
 
-  /// Return true if the given expression is safe to expand in the sense that
-  /// all materialized values are safe to speculate anywhere their operands are
-  /// defined.
-  bool isSafeToExpand(const SCEV *S, ScalarEvolution &SE);
+    ~SCEVInsertPointGuard() {
+      // These guards should always created/destroyed in FIFO order since they
+      // are used to guard lexically scoped blocks of code in
+      // ScalarEvolutionExpander.
+      assert(SE->InsertPointGuards.back() == this);
+      SE->InsertPointGuards.pop_back();
+      Builder.restoreIP(IRBuilderBase::InsertPoint(Block, Point));
+      Builder.SetCurrentDebugLocation(DbgLoc);
+    }
 
-  /// Return true if the given expression is safe to expand in the sense that
-  /// all materialized values are defined and safe to speculate at the specified
-  /// location and their operands are defined at this location.
-  bool isSafeToExpandAt(const SCEV *S, const Instruction *InsertionPoint,
-                        ScalarEvolution &SE);
+    BasicBlock::iterator GetInsertPoint() const { return Point; }
+    void SetInsertPoint(BasicBlock::iterator I) { Point = I; }
+  };
 
-  /// This class uses information about analyze scalars to rewrite expressions
-  /// in canonical form.
-  ///
-  /// Clients should create an instance of this class when rewriting is needed,
-  /// and destroy it when finished to allow the release of the associated
-  /// memory.
-  class SCEVExpander : public SCEVVisitor<SCEVExpander, Value*> {
-    ScalarEvolution &SE;
-    const DataLayout &DL;
-
-    // New instructions receive a name to identify them with the current pass.
-    const char* IVName;
-
-    // InsertedExpressions caches Values for reuse, so must track RAUW.
-    DenseMap<std::pair<const SCEV *, Instruction *>, TrackingVH<Value>>
-        InsertedExpressions;
-
-    // InsertedValues only flags inserted instructions so needs no RAUW.
-    DenseSet<AssertingVH<Value>> InsertedValues;
-    DenseSet<AssertingVH<Value>> InsertedPostIncValues;
-
-    /// A memoization of the "relevant" loop for a given SCEV.
-    DenseMap<const SCEV *, const Loop *> RelevantLoops;
-
-    /// Addrecs referring to any of the given loops are expanded in post-inc
-    /// mode. For example, expanding {1,+,1}<L> in post-inc mode returns the add
-    /// instruction that adds one to the phi for {0,+,1}<L>, as opposed to a new
-    /// phi starting at 1. This is only supported in non-canonical mode.
-    PostIncLoopSet PostIncLoops;
-
-    /// When this is non-null, addrecs expanded in the loop it indicates should
-    /// be inserted with increments at IVIncInsertPos.
-    const Loop *IVIncInsertLoop;
-
-    /// When expanding addrecs in the IVIncInsertLoop loop, insert the IV
-    /// increment at this position.
-    Instruction *IVIncInsertPos;
-
-    /// Phis that complete an IV chain. Reuse
-    DenseSet<AssertingVH<PHINode>> ChainedPhis;
-
-    /// When true, SCEVExpander tries to expand expressions in "canonical" form.
-    /// When false, expressions are expanded in a more literal form.
-    ///
-    /// In "canonical" form addrecs are expanded as arithmetic based on a
-    /// canonical induction variable. Note that CanonicalMode doesn't guarantee
-    /// that all expressions are expanded in "canonical" form. For some
-    /// expressions literal mode can be preferred.
-    bool CanonicalMode;
-
-    /// When invoked from LSR, the expander is in "strength reduction" mode. The
-    /// only difference is that phi's are only reused if they are already in
-    /// "expanded" form.
-    bool LSRMode;
-
-    typedef IRBuilder<TargetFolder> BuilderType;
-    BuilderType Builder;
-
-    // RAII object that stores the current insertion point and restores it when
-    // the object is destroyed. This includes the debug location.  Duplicated
-    // from InsertPointGuard to add SetInsertPoint() which is used to updated
-    // InsertPointGuards stack when insert points are moved during SCEV
-    // expansion.
-    class SCEVInsertPointGuard {
-      IRBuilderBase &Builder;
-      AssertingVH<BasicBlock> Block;
-      BasicBlock::iterator Point;
-      DebugLoc DbgLoc;
-      SCEVExpander *SE;
-
-      SCEVInsertPointGuard(const SCEVInsertPointGuard &) = delete;
-      SCEVInsertPointGuard &operator=(const SCEVInsertPointGuard &) = delete;
-
-    public:
-      SCEVInsertPointGuard(IRBuilderBase &B, SCEVExpander *SE)
-          : Builder(B), Block(B.GetInsertBlock()), Point(B.GetInsertPoint()),
-            DbgLoc(B.getCurrentDebugLocation()), SE(SE) {
-        SE->InsertPointGuards.push_back(this);
-      }
-
-      ~SCEVInsertPointGuard() {
-        // These guards should always created/destroyed in FIFO order since they
-        // are used to guard lexically scoped blocks of code in
-        // ScalarEvolutionExpander.
-        assert(SE->InsertPointGuards.back() == this);
-        SE->InsertPointGuards.pop_back();
-        Builder.restoreIP(IRBuilderBase::InsertPoint(Block, Point));
-        Builder.SetCurrentDebugLocation(DbgLoc);
-      }
-
-      BasicBlock::iterator GetInsertPoint() const { return Point; }
-      void SetInsertPoint(BasicBlock::iterator I) { Point = I; }
-    };
-
-    /// Stack of pointers to saved insert points, used to keep insert points
-    /// consistent when instructions are moved.
-    SmallVector<SCEVInsertPointGuard *, 8> InsertPointGuards;
+  /// Stack of pointers to saved insert points, used to keep insert points
+  /// consistent when instructions are moved.
+  SmallVector<SCEVInsertPointGuard *, 8> InsertPointGuards;
 
 #ifndef NDEBUG
-    const char *DebugType;
+  const char *DebugType;
 #endif
 
-    friend struct SCEVVisitor<SCEVExpander, Value*>;
-
-  public:
-    /// Construct a SCEVExpander in "canonical" mode.
-    explicit SCEVExpander(ScalarEvolution &se, const DataLayout &DL,
-                          const char *name)
-        : SE(se), DL(DL), IVName(name), IVIncInsertLoop(nullptr),
-          IVIncInsertPos(nullptr), CanonicalMode(true), LSRMode(false),
-          Builder(se.getContext(), TargetFolder(DL)) {
+  friend struct SCEVVisitor<SCEVExpander, Value *>;
+
+public:
+  /// Construct a SCEVExpander in "canonical" mode.
+  explicit SCEVExpander(ScalarEvolution &se, const DataLayout &DL,
+                        const char *name)
+      : SE(se), DL(DL), IVName(name), IVIncInsertLoop(nullptr),
+        IVIncInsertPos(nullptr), CanonicalMode(true), LSRMode(false),
+        Builder(se.getContext(), TargetFolder(DL),
+                IRBuilderCallbackInserter(
+                    [this](Instruction *I) { rememberInstruction(I); })) {
 #ifndef NDEBUG
-      DebugType = "";
+    DebugType = "";
 #endif
-    }
+  }
 
-    ~SCEVExpander() {
-      // Make sure the insert point guard stack is consistent.
-      assert(InsertPointGuards.empty());
-    }
+  ~SCEVExpander() {
+    // Make sure the insert point guard stack is consistent.
+    assert(InsertPointGuards.empty());
+  }
 
 #ifndef NDEBUG
-    void setDebugType(const char* s) { DebugType = s; }
+  void setDebugType(const char *s) { DebugType = s; }
 #endif
 
-    /// Erase the contents of the InsertedExpressions map so that users trying
-    /// to expand the same expression into multiple BasicBlocks or different
-    /// places within the same BasicBlock can do so.
-    void clear() {
-      InsertedExpressions.clear();
-      InsertedValues.clear();
-      InsertedPostIncValues.clear();
-      ChainedPhis.clear();
-    }
-
-    /// Return true for expressions that can't be evaluated at runtime
-    /// within given \b Budget.
-    ///
-    /// At is a parameter which specifies point in code where user is going to
-    /// expand this expression. Sometimes this knowledge can lead to
-    /// a less pessimistic cost estimation.
-    bool isHighCostExpansion(const SCEV *Expr, Loop *L, unsigned Budget,
-                             const TargetTransformInfo *TTI,
-                             const Instruction *At) {
-      assert(TTI && "This function requires TTI to be provided.");
-      assert(At && "This function requires At instruction to be provided.");
-      if (!TTI)      // In assert-less builds, avoid crashing
-        return true; // by always claiming to be high-cost.
-      SmallVector<const SCEV *, 8> Worklist;
-      SmallPtrSet<const SCEV *, 8> Processed;
-      int BudgetRemaining = Budget * TargetTransformInfo::TCC_Basic;
-      Worklist.emplace_back(Expr);
-      while (!Worklist.empty()) {
-        const SCEV *S = Worklist.pop_back_val();
-        if (isHighCostExpansionHelper(S, L, *At, BudgetRemaining, *TTI,
-                                      Processed, Worklist))
-          return true;
-      }
-      assert(BudgetRemaining >= 0 && "Should have returned from inner loop.");
-      return false;
-    }
-
-    /// This method returns the canonical induction variable of the specified
-    /// type for the specified loop (inserting one if there is none).  A
-    /// canonical induction variable starts at zero and steps by one on each
-    /// iteration.
-    PHINode *getOrInsertCanonicalInductionVariable(const Loop *L, Type *Ty);
-
-    /// Return the induction variable increment's IV operand.
-    Instruction *getIVIncOperand(Instruction *IncV, Instruction *InsertPos,
-                                 bool allowScale);
-
-    /// Utility for hoisting an IV increment.
-    bool hoistIVInc(Instruction *IncV, Instruction *InsertPos);
-
-    /// replace congruent phis with their most canonical representative. Return
-    /// the number of phis eliminated.
-    unsigned replaceCongruentIVs(Loop *L, const DominatorTree *DT,
-                                 SmallVectorImpl<WeakTrackingVH> &DeadInsts,
-                                 const TargetTransformInfo *TTI = nullptr);
-
-    /// Insert code to directly compute the specified SCEV expression into the
-    /// program.  The inserted code is inserted into the specified block.
-    Value *expandCodeFor(const SCEV *SH, Type *Ty, Instruction *I);
-
-    /// Insert code to directly compute the specified SCEV expression into the
-    /// program.  The inserted code is inserted into the SCEVExpander's current
-    /// insertion point. If a type is specified, the result will be expanded to
-    /// have that type, with a cast if necessary.
-    Value *expandCodeFor(const SCEV *SH, Type *Ty = nullptr);
-
-
-    /// Generates a code sequence that evaluates this predicate.  The inserted
-    /// instructions will be at position \p Loc.  The result will be of type i1
-    /// and will have a value of 0 when the predicate is false and 1 otherwise.
-    Value *expandCodeForPredicate(const SCEVPredicate *Pred, Instruction *Loc);
-
-    /// A specialized variant of expandCodeForPredicate, handling the case when
-    /// we are expanding code for a SCEVEqualPredicate.
-    Value *expandEqualPredicate(const SCEVEqualPredicate *Pred,
-                                Instruction *Loc);
-
-    /// Generates code that evaluates if the \p AR expression will overflow.
-    Value *generateOverflowCheck(const SCEVAddRecExpr *AR, Instruction *Loc,
-                                 bool Signed);
-
-    /// A specialized variant of expandCodeForPredicate, handling the case when
-    /// we are expanding code for a SCEVWrapPredicate.
-    Value *expandWrapPredicate(const SCEVWrapPredicate *P, Instruction *Loc);
-
-    /// A specialized variant of expandCodeForPredicate, handling the case when
-    /// we are expanding code for a SCEVUnionPredicate.
-    Value *expandUnionPredicate(const SCEVUnionPredicate *Pred,
-                                Instruction *Loc);
-
-    /// Set the current IV increment loop and position.
-    void setIVIncInsertPos(const Loop *L, Instruction *Pos) {
-      assert(!CanonicalMode &&
-             "IV increment positions are not supported in CanonicalMode");
-      IVIncInsertLoop = L;
-      IVIncInsertPos = Pos;
-    }
-
-    /// Enable post-inc expansion for addrecs referring to the given
-    /// loops. Post-inc expansion is only supported in non-canonical mode.
-    void setPostInc(const PostIncLoopSet &L) {
-      assert(!CanonicalMode &&
-             "Post-inc expansion is not supported in CanonicalMode");
-      PostIncLoops = L;
+  /// Erase the contents of the InsertedExpressions map so that users trying
+  /// to expand the same expression into multiple BasicBlocks or different
+  /// places within the same BasicBlock can do so.
+  void clear() {
+    InsertedExpressions.clear();
+    InsertedValues.clear();
+    InsertedPostIncValues.clear();
+    ChainedPhis.clear();
+  }
+
+  /// Return true for expressions that can't be evaluated at runtime
+  /// within given \b Budget.
+  ///
+  /// At is a parameter which specifies point in code where user is going to
+  /// expand this expression. Sometimes this knowledge can lead to
+  /// a less pessimistic cost estimation.
+  bool isHighCostExpansion(const SCEV *Expr, Loop *L, unsigned Budget,
+                           const TargetTransformInfo *TTI,
+                           const Instruction *At) {
+    assert(TTI && "This function requires TTI to be provided.");
+    assert(At && "This function requires At instruction to be provided.");
+    if (!TTI)      // In assert-less builds, avoid crashing
+      return true; // by always claiming to be high-cost.
+    SmallVector<const SCEV *, 8> Worklist;
+    SmallPtrSet<const SCEV *, 8> Processed;
+    int BudgetRemaining = Budget * TargetTransformInfo::TCC_Basic;
+    Worklist.emplace_back(Expr);
+    while (!Worklist.empty()) {
+      const SCEV *S = Worklist.pop_back_val();
+      if (isHighCostExpansionHelper(S, L, *At, BudgetRemaining, *TTI, Processed,
+                                    Worklist))
+        return true;
     }
+    assert(BudgetRemaining >= 0 && "Should have returned from inner loop.");
+    return false;
+  }
+
+  /// This method returns the canonical induction variable of the specified
+  /// type for the specified loop (inserting one if there is none).  A
+  /// canonical induction variable starts at zero and steps by one on each
+  /// iteration.
+  PHINode *getOrInsertCanonicalInductionVariable(const Loop *L, Type *Ty);
+
+  /// Return the induction variable increment's IV operand.
+  Instruction *getIVIncOperand(Instruction *IncV, Instruction *InsertPos,
+                               bool allowScale);
+
+  /// Utility for hoisting an IV increment.
+  bool hoistIVInc(Instruction *IncV, Instruction *InsertPos);
+
+  /// replace congruent phis with their most canonical representative. Return
+  /// the number of phis eliminated.
+  unsigned replaceCongruentIVs(Loop *L, const DominatorTree *DT,
+                               SmallVectorImpl<WeakTrackingVH> &DeadInsts,
+                               const TargetTransformInfo *TTI = nullptr);
+
+  /// Insert code to directly compute the specified SCEV expression into the
+  /// program.  The inserted code is inserted into the specified block.
+  Value *expandCodeFor(const SCEV *SH, Type *Ty, Instruction *I);
+
+  /// Insert code to directly compute the specified SCEV expression into the
+  /// program.  The inserted code is inserted into the SCEVExpander's current
+  /// insertion point. If a type is specified, the result will be expanded to
+  /// have that type, with a cast if necessary.
+  Value *expandCodeFor(const SCEV *SH, Type *Ty = nullptr);
+
+  /// Generates a code sequence that evaluates this predicate.  The inserted
+  /// instructions will be at position \p Loc.  The result will be of type i1
+  /// and will have a value of 0 when the predicate is false and 1 otherwise.
+  Value *expandCodeForPredicate(const SCEVPredicate *Pred, Instruction *Loc);
+
+  /// A specialized variant of expandCodeForPredicate, handling the case when
+  /// we are expanding code for a SCEVEqualPredicate.
+  Value *expandEqualPredicate(const SCEVEqualPredicate *Pred, Instruction *Loc);
+
+  /// Generates code that evaluates if the \p AR expression will overflow.
+  Value *generateOverflowCheck(const SCEVAddRecExpr *AR, Instruction *Loc,
+                               bool Signed);
+
+  /// A specialized variant of expandCodeForPredicate, handling the case when
+  /// we are expanding code for a SCEVWrapPredicate.
+  Value *expandWrapPredicate(const SCEVWrapPredicate *P, Instruction *Loc);
+
+  /// A specialized variant of expandCodeForPredicate, handling the case when
+  /// we are expanding code for a SCEVUnionPredicate.
+  Value *expandUnionPredicate(const SCEVUnionPredicate *Pred, Instruction *Loc);
+
+  /// Set the current IV increment loop and position.
+  void setIVIncInsertPos(const Loop *L, Instruction *Pos) {
+    assert(!CanonicalMode &&
+           "IV increment positions are not supported in CanonicalMode");
+    IVIncInsertLoop = L;
+    IVIncInsertPos = Pos;
+  }
+
+  /// Enable post-inc expansion for addrecs referring to the given
+  /// loops. Post-inc expansion is only supported in non-canonical mode.
+  void setPostInc(const PostIncLoopSet &L) {
+    assert(!CanonicalMode &&
+           "Post-inc expansion is not supported in CanonicalMode");
+    PostIncLoops = L;
+  }
+
+  /// Disable all post-inc expansion.
+  void clearPostInc() {
+    PostIncLoops.clear();
+
+    // When we change the post-inc loop set, cached expansions may no
+    // longer be valid.
+    InsertedPostIncValues.clear();
+  }
+
+  /// Disable the behavior of expanding expressions in canonical form rather
+  /// than in a more literal form. Non-canonical mode is useful for late
+  /// optimization passes.
+  void disableCanonicalMode() { CanonicalMode = false; }
+
+  void enableLSRMode() { LSRMode = true; }
+
+  /// Set the current insertion point. This is useful if multiple calls to
+  /// expandCodeFor() are going to be made with the same insert point and the
+  /// insert point may be moved during one of the expansions (e.g. if the
+  /// insert point is not a block terminator).
+  void setInsertPoint(Instruction *IP) {
+    assert(IP);
+    Builder.SetInsertPoint(IP);
+  }
+
+  /// Clear the current insertion point. This is useful if the instruction
+  /// that had been serving as the insertion point may have been deleted.
+  void clearInsertPoint() { Builder.ClearInsertionPoint(); }
+
+  /// Set location information used by debugging information.
+  void SetCurrentDebugLocation(DebugLoc L) {
+    Builder.SetCurrentDebugLocation(std::move(L));
+  }
+
+  /// Get location information used by debugging information.
+  const DebugLoc &getCurrentDebugLocation() const {
+    return Builder.getCurrentDebugLocation();
+  }
+
+  /// Return true if the specified instruction was inserted by the code
+  /// rewriter.  If so, the client should not modify the instruction.
+  bool isInsertedInstruction(Instruction *I) const {
+    return InsertedValues.count(I) || InsertedPostIncValues.count(I);
+  }
+
+  void setChainedPhi(PHINode *PN) { ChainedPhis.insert(PN); }
+
+  /// Try to find existing LLVM IR value for S available at the point At.
+  Value *getExactExistingExpansion(const SCEV *S, const Instruction *At,
+                                   Loop *L);
+
+  /// Try to find the ValueOffsetPair for S. The function is mainly used to
+  /// check whether S can be expanded cheaply.  If this returns a non-None
+  /// value, we know we can codegen the `ValueOffsetPair` into a suitable
+  /// expansion identical with S so that S can be expanded cheaply.
+  ///
+  /// L is a hint which tells in which loop to look for the suitable value.
+  /// On success return value which is equivalent to the expanded S at point
+  /// At. Return nullptr if value was not found.
+  ///
+  /// Note that this function does not perform an exhaustive search. I.e if it
+  /// didn't find any value it does not mean that there is no such value.
+  ///
+  Optional<ScalarEvolution::ValueOffsetPair>
+  getRelatedExistingExpansion(const SCEV *S, const Instruction *At, Loop *L);
 
-    /// Disable all post-inc expansion.
-    void clearPostInc() {
-      PostIncLoops.clear();
+private:
+  LLVMContext &getContext() const { return SE.getContext(); }
 
-      // When we change the post-inc loop set, cached expansions may no
-      // longer be valid.
-      InsertedPostIncValues.clear();
-    }
+  /// Recursive helper function for isHighCostExpansion.
+  bool isHighCostExpansionHelper(const SCEV *S, Loop *L, const Instruction &At,
+                                 int &BudgetRemaining,
+                                 const TargetTransformInfo &TTI,
+                                 SmallPtrSetImpl<const SCEV *> &Processed,
+                                 SmallVectorImpl<const SCEV *> &Worklist);
 
-    /// Disable the behavior of expanding expressions in canonical form rather
-    /// than in a more literal form. Non-canonical mode is useful for late
-    /// optimization passes.
-    void disableCanonicalMode() { CanonicalMode = false; }
+  /// Insert the specified binary operator, doing a small amount of work to
+  /// avoid inserting an obviously redundant operation, and hoisting to an
+  /// outer loop when the opportunity is there and it is safe.
+  Value *InsertBinop(Instruction::BinaryOps Opcode, Value *LHS, Value *RHS,
+                     SCEV::NoWrapFlags Flags, bool IsSafeToHoist);
 
-    void enableLSRMode() { LSRMode = true; }
+  /// Arrange for there to be a cast of V to Ty at IP, reusing an existing
+  /// cast if a suitable one exists, moving an existing cast if a suitable one
+  /// exists but isn't in the right place, or creating a new one.
+  Value *ReuseOrCreateCast(Value *V, Type *Ty, Instruction::CastOps Op,
+                           BasicBlock::iterator IP);
 
-    /// Set the current insertion point. This is useful if multiple calls to
-    /// expandCodeFor() are going to be made with the same insert point and the
-    /// insert point may be moved during one of the expansions (e.g. if the
-    /// insert point is not a block terminator).
-    void setInsertPoint(Instruction *IP) {
-      assert(IP);
-      Builder.SetInsertPoint(IP);
-    }
+  /// Insert a cast of V to the specified type, which must be possible with a
+  /// noop cast, doing what we can to share the casts.
+  Value *InsertNoopCastOfTo(Value *V, Type *Ty);
 
-    /// Clear the current insertion point. This is useful if the instruction
-    /// that had been serving as the insertion point may have been deleted.
-    void clearInsertPoint() { Builder.ClearInsertionPoint(); }
+  /// Expand a SCEVAddExpr with a pointer type into a GEP instead of using
+  /// ptrtoint+arithmetic+inttoptr.
+  Value *expandAddToGEP(const SCEV *const *op_begin, const SCEV *const *op_end,
+                        PointerType *PTy, Type *Ty, Value *V);
+  Value *expandAddToGEP(const SCEV *Op, PointerType *PTy, Type *Ty, Value *V);
 
-    /// Set location information used by debugging information.
-    void SetCurrentDebugLocation(DebugLoc L) {
-      Builder.SetCurrentDebugLocation(std::move(L));
-    }
+  /// Find a previous Value in ExprValueMap for expand.
+  ScalarEvolution::ValueOffsetPair
+  FindValueInExprValueMap(const SCEV *S, const Instruction *InsertPt);
 
-    /// Get location information used by debugging information.
-    const DebugLoc &getCurrentDebugLocation() const {
-      return Builder.getCurrentDebugLocation();
-    }
+  Value *expand(const SCEV *S);
 
-    /// Return true if the specified instruction was inserted by the code
-    /// rewriter.  If so, the client should not modify the instruction.
-    bool isInsertedInstruction(Instruction *I) const {
-      return InsertedValues.count(I) || InsertedPostIncValues.count(I);
-    }
+  /// Determine the most "relevant" loop for the given SCEV.
+  const Loop *getRelevantLoop(const SCEV *);
 
-    void setChainedPhi(PHINode *PN) { ChainedPhis.insert(PN); }
-
-    /// Try to find existing LLVM IR value for S available at the point At.
-    Value *getExactExistingExpansion(const SCEV *S, const Instruction *At,
-                                     Loop *L);
-
-    /// Try to find the ValueOffsetPair for S. The function is mainly used to
-    /// check whether S can be expanded cheaply.  If this returns a non-None
-    /// value, we know we can codegen the `ValueOffsetPair` into a suitable
-    /// expansion identical with S so that S can be expanded cheaply.
-    ///
-    /// L is a hint which tells in which loop to look for the suitable value.
-    /// On success return value which is equivalent to the expanded S at point
-    /// At. Return nullptr if value was not found.
-    ///
-    /// Note that this function does not perform an exhaustive search. I.e if it
-    /// didn't find any value it does not mean that there is no such value.
-    ///
-    Optional<ScalarEvolution::ValueOffsetPair>
-    getRelatedExistingExpansion(const SCEV *S, const Instruction *At, Loop *L);
-
-  private:
-    LLVMContext &getContext() const { return SE.getContext(); }
-
-    /// Recursive helper function for isHighCostExpansion.
-    bool isHighCostExpansionHelper(const SCEV *S, Loop *L,
-                                   const Instruction &At, int &BudgetRemaining,
-                                   const TargetTransformInfo &TTI,
-                                   SmallPtrSetImpl<const SCEV *> &Processed,
-                                   SmallVectorImpl<const SCEV *> &Worklist);
-
-    /// Insert the specified binary operator, doing a small amount of work to
-    /// avoid inserting an obviously redundant operation, and hoisting to an
-    /// outer loop when the opportunity is there and it is safe.
-    Value *InsertBinop(Instruction::BinaryOps Opcode, Value *LHS, Value *RHS,
-                       SCEV::NoWrapFlags Flags, bool IsSafeToHoist);
-
-    /// Arrange for there to be a cast of V to Ty at IP, reusing an existing
-    /// cast if a suitable one exists, moving an existing cast if a suitable one
-    /// exists but isn't in the right place, or creating a new one.
-    Value *ReuseOrCreateCast(Value *V, Type *Ty,
-                             Instruction::CastOps Op,
-                             BasicBlock::iterator IP);
-
-    /// Insert a cast of V to the specified type, which must be possible with a
-    /// noop cast, doing what we can to share the casts.
-    Value *InsertNoopCastOfTo(Value *V, Type *Ty);
-
-    /// Expand a SCEVAddExpr with a pointer type into a GEP instead of using
-    /// ptrtoint+arithmetic+inttoptr.
-    Value *expandAddToGEP(const SCEV *const *op_begin,
-                          const SCEV *const *op_end,
-                          PointerType *PTy, Type *Ty, Value *V);
-    Value *expandAddToGEP(const SCEV *Op, PointerType *PTy, Type *Ty, Value *V);
-
-    /// Find a previous Value in ExprValueMap for expand.
-    ScalarEvolution::ValueOffsetPair
-    FindValueInExprValueMap(const SCEV *S, const Instruction *InsertPt);
-
-    Value *expand(const SCEV *S);
-
-    /// Determine the most "relevant" loop for the given SCEV.
-    const Loop *getRelevantLoop(const SCEV *);
-
-    Value *visitConstant(const SCEVConstant *S) {
-      return S->getValue();
-    }
+  Value *visitConstant(const SCEVConstant *S) { return S->getValue(); }
 
-    Value *visitTruncateExpr(const SCEVTruncateExpr *S);
+  Value *visitTruncateExpr(const SCEVTruncateExpr *S);
 
-    Value *visitZeroExtendExpr(const SCEVZeroExtendExpr *S);
+  Value *visitZeroExtendExpr(const SCEVZeroExtendExpr *S);
 
-    Value *visitSignExtendExpr(const SCEVSignExtendExpr *S);
+  Value *visitSignExtendExpr(const SCEVSignExtendExpr *S);
 
-    Value *visitAddExpr(const SCEVAddExpr *S);
+  Value *visitAddExpr(const SCEVAddExpr *S);
 
-    Value *visitMulExpr(const SCEVMulExpr *S);
+  Value *visitMulExpr(const SCEVMulExpr *S);
 
-    Value *visitUDivExpr(const SCEVUDivExpr *S);
+  Value *visitUDivExpr(const SCEVUDivExpr *S);
 
-    Value *visitAddRecExpr(const SCEVAddRecExpr *S);
+  Value *visitAddRecExpr(const SCEVAddRecExpr *S);
 
-    Value *visitSMaxExpr(const SCEVSMaxExpr *S);
+  Value *visitSMaxExpr(const SCEVSMaxExpr *S);
 
-    Value *visitUMaxExpr(const SCEVUMaxExpr *S);
+  Value *visitUMaxExpr(const SCEVUMaxExpr *S);
 
-    Value *visitSMinExpr(const SCEVSMinExpr *S);
+  Value *visitSMinExpr(const SCEVSMinExpr *S);
 
-    Value *visitUMinExpr(const SCEVUMinExpr *S);
+  Value *visitUMinExpr(const SCEVUMinExpr *S);
 
-    Value *visitUnknown(const SCEVUnknown *S) {
-      return S->getValue();
-    }
+  Value *visitUnknown(const SCEVUnknown *S) { return S->getValue(); }
 
-    void rememberInstruction(Value *I);
+  void rememberInstruction(Value *I);
 
-    bool isNormalAddRecExprPHI(PHINode *PN, Instruction *IncV, const Loop *L);
+  bool isNormalAddRecExprPHI(PHINode *PN, Instruction *IncV, const Loop *L);
 
-    bool isExpandedAddRecExprPHI(PHINode *PN, Instruction *IncV, const Loop *L);
+  bool isExpandedAddRecExprPHI(PHINode *PN, Instruction *IncV, const Loop *L);
 
-    Value *expandAddRecExprLiterally(const SCEVAddRecExpr *);
-    PHINode *getAddRecExprPHILiterally(const SCEVAddRecExpr *Normalized,
-                                       const Loop *L,
-                                       Type *ExpandTy,
-                                       Type *IntTy,
-                                       Type *&TruncTy,
-                                       bool &InvertStep);
-    Value *expandIVInc(PHINode *PN, Value *StepV, const Loop *L,
-                       Type *ExpandTy, Type *IntTy, bool useSubtract);
+  Value *expandAddRecExprLiterally(const SCEVAddRecExpr *);
+  PHINode *getAddRecExprPHILiterally(const SCEVAddRecExpr *Normalized,
+                                     const Loop *L, Type *ExpandTy, Type *IntTy,
+                                     Type *&TruncTy, bool &InvertStep);
+  Value *expandIVInc(PHINode *PN, Value *StepV, const Loop *L, Type *ExpandTy,
+                     Type *IntTy, bool useSubtract);
 
-    void hoistBeforePos(DominatorTree *DT, Instruction *InstToHoist,
-                        Instruction *Pos, PHINode *LoopPhi);
+  void hoistBeforePos(DominatorTree *DT, Instruction *InstToHoist,
+                      Instruction *Pos, PHINode *LoopPhi);
 
-    void fixupInsertPoints(Instruction *I);
-  };
-}
+  void fixupInsertPoints(Instruction *I);
+};
+} // namespace llvm
 
 #endif
diff --git a/llvm/include/llvm/Transforms/Utils/SimplifyCFGOptions.h b/llvm/include/llvm/Transforms/Utils/SimplifyCFGOptions.h
index ca9a7e7223dbe..46f6ca0462f8b 100644
--- a/llvm/include/llvm/Transforms/Utils/SimplifyCFGOptions.h
+++ b/llvm/include/llvm/Transforms/Utils/SimplifyCFGOptions.h
@@ -25,6 +25,7 @@ struct SimplifyCFGOptions {
   bool ForwardSwitchCondToPhi = false;
   bool ConvertSwitchToLookupTable = false;
   bool NeedCanonicalLoop = true;
+  bool HoistCommonInsts = true;
   bool SinkCommonInsts = false;
   bool SimplifyCondBranch = true;
   bool FoldTwoEntryPHINode = true;
@@ -48,6 +49,10 @@ struct SimplifyCFGOptions {
     NeedCanonicalLoop = B;
     return *this;
   }
+  SimplifyCFGOptions &hoistCommonInsts(bool B) {
+    HoistCommonInsts = B;
+    return *this;
+  }
   SimplifyCFGOptions &sinkCommonInsts(bool B) {
     SinkCommonInsts = B;
     return *this;
diff --git a/llvm/include/llvm/Transforms/Utils/SizeOpts.h b/llvm/include/llvm/Transforms/Utils/SizeOpts.h
index 08d963475f23f..3c1173b747d33 100644
--- a/llvm/include/llvm/Transforms/Utils/SizeOpts.h
+++ b/llvm/include/llvm/Transforms/Utils/SizeOpts.h
@@ -19,7 +19,6 @@
 
 extern llvm::cl::opt<bool> EnablePGSO;
 extern llvm::cl::opt<bool> PGSOLargeWorkingSetSizeOnly;
-extern llvm::cl::opt<bool> PGSOIRPassOrTestOnly;
 extern llvm::cl::opt<bool> PGSOColdCodeOnly;
 extern llvm::cl::opt<bool> PGSOColdCodeOnlyForInstrPGO;
 extern llvm::cl::opt<bool> PGSOColdCodeOnlyForSamplePGO;
@@ -60,11 +59,6 @@ bool shouldFuncOptimizeForSizeImpl(const FuncT *F, ProfileSummaryInfo *PSI,
     return true;
   if (!EnablePGSO)
     return false;
-  // Temporarily enable size optimizations only for the IR pass or test query
-  // sites for gradual commit/rollout. This is to be removed later.
-  if (PGSOIRPassOrTestOnly && !(QueryType == PGSOQueryType::IRPass ||
-                                QueryType == PGSOQueryType::Test))
-    return false;
   if (isPGSOColdCodeOnly(PSI))
     return AdapterT::isFunctionColdInCallGraph(F, PSI, *BFI);
   if (PSI->hasSampleProfile())
@@ -85,11 +79,6 @@ bool shouldOptimizeForSizeImpl(BlockTOrBlockFreq BBOrBlockFreq, ProfileSummaryIn
     return true;
   if (!EnablePGSO)
     return false;
-  // Temporarily enable size optimizations only for the IR pass or test query
-  // sites for gradual commit/rollout. This is to be removed later.
-  if (PGSOIRPassOrTestOnly && !(QueryType == PGSOQueryType::IRPass ||
-                                QueryType == PGSOQueryType::Test))
-    return false;
   if (isPGSOColdCodeOnly(PSI))
     return AdapterT::isColdBlock(BBOrBlockFreq, PSI, BFI);
   if (PSI->hasSampleProfile())
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index c6c3450f77608..7235aa5861120 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -202,9 +202,10 @@ class LoopVectorizationLegality {
       Function *F, std::function<const LoopAccessInfo &(Loop &)> *GetLAA,
       LoopInfo *LI, OptimizationRemarkEmitter *ORE,
       LoopVectorizationRequirements *R, LoopVectorizeHints *H, DemandedBits *DB,
-      AssumptionCache *AC)
+      AssumptionCache *AC, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI)
       : TheLoop(L), LI(LI), PSE(PSE), TTI(TTI), TLI(TLI), DT(DT),
-        GetLAA(GetLAA), ORE(ORE), Requirements(R), Hints(H), DB(DB), AC(AC) {}
+        GetLAA(GetLAA), ORE(ORE), Requirements(R), Hints(H), DB(DB), AC(AC),
+        BFI(BFI), PSI(PSI) {}
 
   /// ReductionList contains the reduction descriptors for all
   /// of the reductions that were found in the loop.
@@ -478,6 +479,10 @@ class LoopVectorizationLegality {
   /// Assume instructions in predicated blocks must be dropped if the CFG gets
   /// flattened.
   SmallPtrSet<Instruction *, 8> ConditionalAssumes;
+
+  /// BFI and PSI are used to check for profile guided size optimizations.
+  BlockFrequencyInfo *BFI;
+  ProfileSummaryInfo *PSI;
 };
 
 } // namespace llvm
diff --git a/llvm/lib/Analysis/CMakeLists.txt b/llvm/lib/Analysis/CMakeLists.txt
index 703623396d96a..8bb37697a6aff 100644
--- a/llvm/lib/Analysis/CMakeLists.txt
+++ b/llvm/lib/Analysis/CMakeLists.txt
@@ -1,35 +1,17 @@
-set(CommonMLSources MLInlineAdvisor.cpp)
-set(ReleaseModeMLSources ReleaseModeModelRunner.cpp)
-set(DevelopmentModeMLSources TFUtils.cpp)
-
 if (DEFINED LLVM_HAVE_TF_AOT OR DEFINED LLVM_HAVE_TF_API)
-  set(MLPolicySources ${CommonMLSources})
   if (DEFINED LLVM_HAVE_TF_AOT)
     include(TensorFlowCompile)
     tfcompile(models/inliner serve action InlinerSizeModel llvm::InlinerSizeModel)
-    list(APPEND ReleaseModeMLSources
+    list(APPEND GeneratedMLSources
       $<TARGET_OBJECTS:tf_xla_runtime_objects>
       ${GENERATED_OBJS}
     )
-    LIST(APPEND MLPolicySources ${ReleaseModeMLSources})
-  else()
-    LIST(APPEND LLVM_OPTIONAL_SOURCES ${ReleaseModeMLSources})
   endif()
 
   if (DEFINED LLVM_HAVE_TF_API)
-    LIST(APPEND MLPolicySources ${DevelopmentModeMLSources})
     LIST(APPEND MLLinkDeps ${tensorflow_c_api})
-  else()
-    LIST(APPEND LLVM_OPTIONAL_SOURCES ${DevelopmentModeMLSources})
   endif()
-else()
-  LIST(APPEND LLVM_OPTIONAL_SOURCES 
-    ${CommonMLSources}
-    ${DevelopmentModeMLSources}
-    ${ReleaseModeMLSources}
-    )
 endif()
-  
 
 add_llvm_component_library(LLVMAnalysis
   AliasAnalysis.cpp
@@ -61,11 +43,13 @@ add_llvm_component_library(LLVMAnalysis
   DemandedBits.cpp
   DependenceAnalysis.cpp
   DependenceGraphBuilder.cpp
+  DevelopmentModeInlineAdvisor.cpp
   DivergenceAnalysis.cpp
   DomPrinter.cpp
   DomTreeUpdater.cpp
   DominanceFrontier.cpp
   EHPersonalities.cpp
+  FunctionPropertiesAnalysis.cpp
   GlobalsModRef.cpp
   GuardUtils.cpp
   HeatUtils.cpp
@@ -74,7 +58,6 @@ add_llvm_component_library(LLVMAnalysis
   IndirectCallPromotionAnalysis.cpp
   InlineCost.cpp
   InlineAdvisor.cpp
-  InlineFeaturesAnalysis.cpp
   InlineSizeEstimatorAnalysis.cpp
   InstCount.cpp
   InstructionPrecedenceTracking.cpp
@@ -95,6 +78,7 @@ add_llvm_component_library(LLVMAnalysis
   LoopUnrollAnalyzer.cpp
   LoopInfo.cpp
   LoopPass.cpp
+  MLInlineAdvisor.cpp
   MemDepPrinter.cpp
   MemDerefPrinter.cpp
   MemoryBuiltins.cpp
@@ -117,6 +101,7 @@ add_llvm_component_library(LLVMAnalysis
   RegionInfo.cpp
   RegionPass.cpp
   RegionPrinter.cpp
+  ReleaseModeModelRunner.cpp
   ScalarEvolution.cpp
   ScalarEvolutionAliasAnalysis.cpp
   ScalarEvolutionDivision.cpp
@@ -125,6 +110,7 @@ add_llvm_component_library(LLVMAnalysis
   StackSafetyAnalysis.cpp
   SyncDependenceAnalysis.cpp
   SyntheticCountsUtils.cpp
+  TFUtils.cpp
   TargetLibraryInfo.cpp
   TargetTransformInfo.cpp
   Trace.cpp
@@ -136,7 +122,7 @@ add_llvm_component_library(LLVMAnalysis
   ValueTracking.cpp
   VectorUtils.cpp
   VFABIDemangling.cpp
-  ${MLPolicySources}
+  ${GeneratedMLSources}
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Analysis
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 6feffcbb98e1f..794edeb714fd2 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -41,6 +41,7 @@
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/IR/IntrinsicsARM.h"
 #include "llvm/IR/IntrinsicsX86.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/IR/Type.h"
@@ -111,7 +112,7 @@ Constant *FoldBitCast(Constant *C, Type *DestTy, const DataLayout &DL) {
   if (auto *VTy = dyn_cast<VectorType>(C->getType())) {
     // Handle a vector->scalar integer/fp cast.
     if (isa<IntegerType>(DestTy) || DestTy->isFloatingPointTy()) {
-      unsigned NumSrcElts = VTy->getNumElements();
+      unsigned NumSrcElts = cast<FixedVectorType>(VTy)->getNumElements();
       Type *SrcEltTy = VTy->getElementType();
 
       // If the vector is a vector of floating point, convert it to vector of int
@@ -154,8 +155,8 @@ Constant *FoldBitCast(Constant *C, Type *DestTy, const DataLayout &DL) {
     return ConstantExpr::getBitCast(C, DestTy);
 
   // If the element types match, IR can fold it.
-  unsigned NumDstElt = DestVTy->getNumElements();
-  unsigned NumSrcElt = cast<VectorType>(C->getType())->getNumElements();
+  unsigned NumDstElt = cast<FixedVectorType>(DestVTy)->getNumElements();
+  unsigned NumSrcElt = cast<FixedVectorType>(C->getType())->getNumElements();
   if (NumDstElt == NumSrcElt)
     return ConstantExpr::getBitCast(C, DestTy);
 
@@ -489,8 +490,8 @@ bool ReadDataFromGlobal(Constant *C, uint64_t ByteOffset, unsigned char *CurPtr,
       NumElts = AT->getNumElements();
       EltTy = AT->getElementType();
     } else {
-      NumElts = cast<VectorType>(C->getType())->getNumElements();
-      EltTy = cast<VectorType>(C->getType())->getElementType();
+      NumElts = cast<FixedVectorType>(C->getType())->getNumElements();
+      EltTy = cast<FixedVectorType>(C->getType())->getElementType();
     }
     uint64_t EltSize = DL.getTypeAllocSize(EltTy);
     uint64_t Index = ByteOffset / EltSize;
@@ -1456,6 +1457,11 @@ bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
   case Intrinsic::experimental_vector_reduce_smax:
   case Intrinsic::experimental_vector_reduce_umin:
   case Intrinsic::experimental_vector_reduce_umax:
+  // Target intrinsics
+  case Intrinsic::arm_mve_vctp8:
+  case Intrinsic::arm_mve_vctp16:
+  case Intrinsic::arm_mve_vctp32:
+  case Intrinsic::arm_mve_vctp64:
     return true;
 
   // Floating point operations cannot be folded in strictfp functions in
@@ -2719,7 +2725,8 @@ static Constant *ConstantFoldVectorCall(StringRef Name,
   SmallVector<Constant *, 4> Lane(Operands.size());
   Type *Ty = FVTy->getElementType();
 
-  if (IntrinsicID == Intrinsic::masked_load) {
+  switch (IntrinsicID) {
+  case Intrinsic::masked_load: {
     auto *SrcPtr = Operands[0];
     auto *Mask = Operands[2];
     auto *Passthru = Operands[3];
@@ -2757,6 +2764,32 @@ static Constant *ConstantFoldVectorCall(StringRef Name,
       return nullptr;
     return ConstantVector::get(NewElements);
   }
+  case Intrinsic::arm_mve_vctp8:
+  case Intrinsic::arm_mve_vctp16:
+  case Intrinsic::arm_mve_vctp32:
+  case Intrinsic::arm_mve_vctp64: {
+    if (auto *Op = dyn_cast<ConstantInt>(Operands[0])) {
+      unsigned Lanes = FVTy->getNumElements();
+      uint64_t Limit = Op->getZExtValue();
+      // vctp64 are currently modelled as returning a v4i1, not a v2i1. Make
+      // sure we get the limit right in that case and set all relevant lanes.
+      if (IntrinsicID == Intrinsic::arm_mve_vctp64)
+        Limit *= 2;
+
+      SmallVector<Constant *, 16> NCs;
+      for (unsigned i = 0; i < Lanes; i++) {
+        if (i < Limit)
+          NCs.push_back(ConstantInt::getTrue(Ty));
+        else
+          NCs.push_back(ConstantInt::getFalse(Ty));
+      }
+      return ConstantVector::get(NCs);
+    }
+    break;
+  }
+  default:
+    break;
+  }
 
   for (unsigned I = 0, E = FVTy->getNumElements(); I != E; ++I) {
     // Gather a column of constants.
diff --git a/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp b/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp
new file mode 100644
index 0000000000000..ba9343543dbbc
--- /dev/null
+++ b/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp
@@ -0,0 +1,489 @@
+//===- DevelopmentModeInlineAdvisor.cpp - runtime-loadable model runner  --===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a model runner using Tensorflow C APIs, allowing the
+// loading of a model from a command line option.
+//
+//===----------------------------------------------------------------------===//
+#include "llvm/Config/config.h"
+#if defined(LLVM_HAVE_TF_API)
+
+#include "llvm/Analysis/CallGraph.h"
+#include "llvm/Analysis/InlineSizeEstimatorAnalysis.h"
+#include "llvm/Analysis/MLInlineAdvisor.h"
+#include "llvm/Analysis/Utils/TFUtils.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/ManagedStatic.h"
+
+#include <vector>
+
+using namespace llvm;
+
+static cl::opt<std::string> TrainingLog(
+    "training-log", cl::Hidden,
+    cl::desc("Path where the development - mode inlining log is saved."));
+
+static cl::opt<std::string> TFModelUnderTrainingPath(
+    "ml-inliner-model-under-training", cl::Hidden,
+    cl::desc("Path to SavedModel from the previous training iteration."));
+
+static cl::opt<std::string> TFFeedPrefix("ml-inliner-trained-model-feed-prefix",
+                                         cl::Hidden, cl::init("action_"),
+                                         cl::desc("Prefix for feature names."));
+
+static cl::opt<std::string> TFDecisionName(
+    "ml-inliner-trained-model-decision-name", cl::Hidden,
+    cl::init("StatefulPartitionedCall"),
+    cl::desc("Name of the graph operation representing the decision."));
+
+namespace {
+/// An InlineEvent, used by TrainingLogger.
+struct InlineEvent {
+  /// What the default policy's decision would have been.
+  bool DefaultDecision = false;
+
+  /// What we advised. When training off the default policy, this is the same as
+  /// DefaultDecision.
+  bool AdvisedDecision = false;
+
+  /// What actually happened. This would be 'false' in the case of an inline
+  /// error, even if AdvisedDecision were true, otherwise it agrees with
+  /// AdvisedDecision.
+  bool Effect = false;
+
+  /// What the change in size was: size_after - size_before
+  int64_t Reward = 0;
+};
+
+/// Collect data we may use for training a model, and write it as a textual
+/// Tensorflow SequenceExample
+/// (https://www.tensorflow.org/api_docs/python/tf/train/SequenceExample)
+/// protobuf (https://developers.google.com/protocol-buffers).
+/// Because this is a protobuf, we cannot just stream the events as they come.
+/// Internally, TrainingLogger stores data in column-major format, because that
+/// lines up with how TF SequenceExample represents it.
+class TrainingLogger final {
+public:
+  TrainingLogger() {
+    for (size_t I = 0; I < NumberOfFeatures; ++I) {
+      Features.push_back(InlineFeatures());
+    }
+  }
+
+  /// Log one inlining event.
+  void logInlineEvent(const InlineEvent &Event,
+                      const MLModelRunner &ModelRunner) {
+    for (size_t I = 0; I < NumberOfFeatures; ++I) {
+      Features[I].push_back(ModelRunner.getFeature(I));
+    }
+    Decisions.push_back(Event.AdvisedDecision);
+    Effects.push_back(Event.Effect);
+    Rewards.push_back(Event.Reward);
+    DefaultDecisions.push_back(Event.DefaultDecision);
+  }
+
+  void printTensor(raw_fd_ostream &OutFile) {
+    if (DefaultDecisions.empty())
+      return;
+    OutFile << "feature_lists: {\n";
+
+    for (size_t I = 0; I < Features.size(); I++) {
+      writeTensor(OutFile, FeatureNameMap.at(I), Features[I]);
+    }
+    writeTensor(OutFile, DefaultDecisionName, DefaultDecisions);
+    writeTensor(OutFile, DecisionName, Decisions);
+    writeTensor(OutFile, RewardName, Rewards);
+
+    OutFile << "}\n";
+  }
+
+private:
+  template <typename T>
+  void writeTensor(raw_fd_ostream &OutFile, StringRef TensorName,
+                   const std::vector<T> &Tensor) {
+    OutFile << "  feature_list: {\n";
+    OutFile << "    key: "
+            << "\"" << TensorName << "\" ";
+    OutFile << "value: {\n";
+    for (const auto &Feature : Tensor) {
+      OutFile << "      feature: { int64_list: { value: [" << Feature
+              << "] } }\n";
+    }
+    OutFile << "    }\n";
+    OutFile << "  }\n";
+  }
+
+  std::vector<InlineFeatures> Features;
+  std::vector<bool> DefaultDecisions;
+  std::vector<bool> Decisions;
+  std::vector<bool> Effects;
+  std::vector<int64_t> Rewards;
+  std::vector<bool> Mandatory;
+};
+
+/// An extension of the MLInlineAdvisor for the 'development' mode, targeting
+/// the offline training scenario. Note that training happens outside of the
+/// compiler, this facility is concerned with producing training data ("logs").
+/// This InlineAdvisor can operate in the following modes:
+///
+/// 1) collect logs for the default policy. This is useful for bootstrapping
+/// training, which will be considerably faster by starting from a reasonable
+/// policy.
+///
+/// 2) collect logs for the ML policy, using a model from a previous
+/// training. Potentially, that model uses internally some small random
+/// perturbation of its weights, to induce exploration (setting this up is the
+/// responsibility of the training algorithm). The logs would then be used to
+/// retrain and improve on this model.
+///
+/// 3) use the provided model, with no logging. This is useful for end to end
+/// validation - the model, in this case, is a release candidate and shouldn't
+/// have random perturbations. It is a convenience feature: rather than needing
+/// to take the release candidate model and compile it in 'release' mode,
+/// validate it, then potentially discard it, it's easier to just pass the model
+/// to the compiler, albeit compilation would be slower, as a one-off. Once the
+/// model behaves satisfactorily, it can be compiled AOT, for efficiency, in
+/// release mode. The expectation is that a well-trained model provides a good
+/// policy over a sufficiently diverse codebase, over many changes (i.e.
+/// training happens seldom).
+class DevelopmentModeMLInlineAdvisor : public MLInlineAdvisor {
+public:
+  DevelopmentModeMLInlineAdvisor(
+      Module &M, ModuleAnalysisManager &MAM,
+      std::unique_ptr<MLModelRunner> ModelRunner,
+      std::function<bool(CallBase &)> GetDefaultAdvice, bool IsDoingInference);
+
+  size_t getTotalSizeEstimate();
+
+  virtual ~DevelopmentModeMLInlineAdvisor();
+  void updateNativeSizeEstimate(int64_t Change) { CurrentNativeSize += Change; }
+  void resetNativeSize(Function *F) {
+    FAM.invalidate<InlineSizeEstimatorAnalysis>(*F);
+  }
+
+  std::unique_ptr<MLInlineAdvice>
+  getMandatoryAdvice(CallBase &CB, OptimizationRemarkEmitter &ORE) override;
+  std::unique_ptr<MLInlineAdvice>
+  getAdviceFromModel(CallBase &CB, OptimizationRemarkEmitter &ORE) override;
+
+  size_t getNativeSizeEstimate(const Function &F) const;
+
+private:
+  bool isLogging() const { return !TrainingLog.empty(); }
+
+  std::function<bool(CallBase &)> GetDefaultAdvice;
+  TrainingLogger Logger;
+  const bool IsDoingInference;
+
+  const int32_t InitialNativeSize;
+  int32_t CurrentNativeSize = 0;
+};
+
+/// A variant of MLInlineAdvice that tracks all non-trivial inlining
+/// decisions, for training/logging.
+class LoggingMLInlineAdvice : public MLInlineAdvice {
+public:
+  LoggingMLInlineAdvice(DevelopmentModeMLInlineAdvisor *Advisor, CallBase &CB,
+                        OptimizationRemarkEmitter &ORE, bool Recommendation,
+                        TrainingLogger &Logger, size_t CallerSizeEstimateBefore,
+                        size_t CalleeSizeEstimateBefore, bool DefaultDecision)
+      : MLInlineAdvice(Advisor, CB, ORE, Recommendation), Logger(Logger),
+        CallerSizeEstimateBefore(CallerSizeEstimateBefore),
+        CalleeSizeEstimateBefore(CalleeSizeEstimateBefore),
+        DefaultDecision(DefaultDecision) {}
+
+  virtual ~LoggingMLInlineAdvice() = default;
+
+private:
+  DevelopmentModeMLInlineAdvisor *getAdvisor() const {
+    return static_cast<DevelopmentModeMLInlineAdvisor *>(Advisor);
+  }
+  void recordInliningImpl() override {
+    MLInlineAdvice::recordInliningImpl();
+    getAdvisor()->resetNativeSize(Caller);
+    int Reward = std::numeric_limits<int>::max();
+    if (!getAdvisor()->isForcedToStop()) {
+      int NativeSizeAfter = getAdvisor()->getNativeSizeEstimate(*Caller) +
+                            CalleeSizeEstimateBefore;
+      Reward = NativeSizeAfter -
+               (CallerSizeEstimateBefore + CalleeSizeEstimateBefore);
+      getAdvisor()->updateNativeSizeEstimate(Reward);
+    }
+    log(Reward, /*Success=*/true);
+  }
+
+  void recordInliningWithCalleeDeletedImpl() override {
+    MLInlineAdvice::recordInliningWithCalleeDeletedImpl();
+    getAdvisor()->resetNativeSize(Caller);
+    if (!getAdvisor()->isForcedToStop()) {
+      int NativeSizeAfter = getAdvisor()->getNativeSizeEstimate(*Caller);
+      int Reward = NativeSizeAfter -
+                   (CallerSizeEstimateBefore + CalleeSizeEstimateBefore);
+      getAdvisor()->updateNativeSizeEstimate(Reward);
+      log(Reward, /*Success=*/true);
+    }
+  }
+
+  void recordUnsuccessfulInliningImpl(const InlineResult &Result) override {
+    MLInlineAdvice::recordUnsuccessfulInliningImpl(Result);
+    log(NoReward, /*Success=*/false);
+  }
+
+  void recordUnattemptedInliningImpl() override {
+    MLInlineAdvice::recordUnattemptedInliningImpl();
+    log(NoReward, /*Success=*/false);
+  }
+
+  void log(int64_t Reward, bool Success) {
+    InlineEvent Event;
+    Event.AdvisedDecision = isInliningRecommended();
+    Event.DefaultDecision = DefaultDecision;
+    Event.Effect = Success;
+    Event.Reward = Reward;
+    Logger.logInlineEvent(Event, getAdvisor()->getModelRunner());
+  }
+
+  static const int64_t NoReward = 0;
+  TrainingLogger &Logger;
+  const size_t CallerSizeEstimateBefore;
+  const size_t CalleeSizeEstimateBefore;
+  const bool DefaultDecision;
+};
+
+/// A pseudo model runner. We use it to store feature values when collecting
+/// logs for the default policy, but never ask it to 'run'.
+class NoInferenceModelRunner : public MLModelRunner {
+public:
+  NoInferenceModelRunner(LLVMContext &Ctx)
+      : MLModelRunner(Ctx), Features(NumberOfFeatures) {}
+  void setFeature(FeatureIndex Index, int64_t Value) override {
+    Features[static_cast<int>(Index)] = Value;
+  }
+
+  int64_t getFeature(int Index) const override { return Features[Index]; }
+  bool run() override {
+    llvm_unreachable("We shouldn't call run on this model runner.");
+  }
+
+private:
+  InlineFeatures Features;
+};
+
+/// ModelUnderTrainingRunner - training mode implementation. It uses TF C APIs
+/// to dynamically load and evaluate a TF SavedModel
+/// (https://www.tensorflow.org/guide/saved_model). Runtime performance is
+/// sacrificed for ease of use while training.
+class ModelUnderTrainingRunner final : public MLModelRunner {
+public:
+  ModelUnderTrainingRunner(LLVMContext &Ctx, const std::string &ModelPath);
+
+  bool run() override;
+
+  // Disallows copy and assign.
+  ModelUnderTrainingRunner(const ModelUnderTrainingRunner &) = delete;
+  ModelUnderTrainingRunner &
+  operator=(const ModelUnderTrainingRunner &) = delete;
+
+  void setFeature(FeatureIndex Index, int64_t Value) override;
+  int64_t getFeature(int Index) const override;
+  bool isValid() const { return !!Evaluator; }
+
+private:
+  std::unique_ptr<TFModelEvaluator> Evaluator;
+
+  // The training framework needs some additional features, that just need to
+  // be set to 0.
+  struct TensorSpec {
+    std::string Name;
+    std::function<void(TFModelEvaluator *, size_t Index,
+                       const std::vector<int64_t> &Dim)>
+        Initializer;
+  };
+
+  const std::vector<TensorSpec> TrainingOnlyFeatures{
+      {"inlining_default",
+       [](TFModelEvaluator *Evaluator, size_t Index,
+          const std::vector<int64_t> &Dim) {
+         Evaluator->initInput<int64_t>(Index, Dim);
+       }},
+      {"discount",
+       [](TFModelEvaluator *Evaluator, size_t Index,
+          const std::vector<int64_t> &Dim) {
+         Evaluator->initInput<float>(Index, Dim);
+       }},
+      {"reward",
+       [](TFModelEvaluator *Evaluator, size_t Index,
+          const std::vector<int64_t> &Dim) {
+         Evaluator->initInput<float>(Index, Dim);
+       }},
+      {"step_type", [](TFModelEvaluator *Evaluator, size_t Index,
+                       const std::vector<int64_t> &Dim) {
+         Evaluator->initInput<int32_t>(Index, Dim);
+       }}};
+};
+} // namespace
+
+DevelopmentModeMLInlineAdvisor::DevelopmentModeMLInlineAdvisor(
+    Module &M, ModuleAnalysisManager &MAM,
+    std::unique_ptr<MLModelRunner> ModelRunner,
+    std::function<bool(CallBase &)> GetDefaultAdvice, bool IsDoingInference)
+    : MLInlineAdvisor(M, MAM, std::move(ModelRunner)),
+      GetDefaultAdvice(GetDefaultAdvice), IsDoingInference(IsDoingInference),
+      InitialNativeSize(isLogging() ? getTotalSizeEstimate() : 0),
+      CurrentNativeSize(InitialNativeSize) {
+  // We cannot have the case of neither inference nor logging.
+  assert(IsDoingInference || isLogging());
+}
+
+DevelopmentModeMLInlineAdvisor::~DevelopmentModeMLInlineAdvisor() {
+  if (TrainingLog.empty())
+    return;
+  std::error_code ErrorCode;
+  raw_fd_ostream OutFile(TrainingLog, ErrorCode);
+  Logger.printTensor(OutFile);
+}
+
+size_t
+DevelopmentModeMLInlineAdvisor::getNativeSizeEstimate(const Function &F) const {
+  auto &R =
+      FAM.getResult<InlineSizeEstimatorAnalysis>(const_cast<Function &>(F));
+  if (!R) {
+    F.getParent()->getContext().emitError(
+        "Native size estimator is not present.");
+    return 0;
+  }
+  return *R;
+}
+
+std::unique_ptr<MLInlineAdvice>
+DevelopmentModeMLInlineAdvisor::getMandatoryAdvice(
+    CallBase &CB, OptimizationRemarkEmitter &ORE) {
+  if (!isLogging())
+    return MLInlineAdvisor::getMandatoryAdvice(CB, ORE);
+  return std::make_unique<LoggingMLInlineAdvice>(
+      /*Advisor=*/this,
+      /*CB=*/CB, /*ORE=*/ORE, /*Recommendation=*/true, /*Logger=*/Logger,
+      /*CallerSizeEstimateBefore=*/getNativeSizeEstimate(*CB.getCaller()),
+      /*CalleeSizeEstimateBefore=*/
+      getNativeSizeEstimate(*CB.getCalledFunction()),
+      /*DefaultDecision=*/true);
+}
+
+std::unique_ptr<MLInlineAdvice>
+DevelopmentModeMLInlineAdvisor::getAdviceFromModel(
+    CallBase &CB, OptimizationRemarkEmitter &ORE) {
+  if (IsDoingInference && !isLogging())
+    return MLInlineAdvisor::getAdviceFromModel(CB, ORE);
+
+  bool DefaultAdvice = GetDefaultAdvice(CB);
+  auto Recommendation = IsDoingInference ? ModelRunner->run() : DefaultAdvice;
+  return std::make_unique<LoggingMLInlineAdvice>(
+      /*Advisor=*/this,
+      /*CB=*/CB, /*ORE=*/ORE, /*Recommendation=*/Recommendation,
+      /*Logger=*/Logger,
+      /*CallerSizeEstimateBefore=*/getNativeSizeEstimate(*CB.getCaller()),
+      /*CalleeSizeEstimateBefore=*/
+      getNativeSizeEstimate(*CB.getCalledFunction()),
+      /*DefaultDecision=*/DefaultAdvice);
+}
+
+size_t DevelopmentModeMLInlineAdvisor::getTotalSizeEstimate() {
+  size_t Ret = 0;
+  for (auto &F : M) {
+    if (F.isDeclaration())
+      continue;
+    if (isFunctionDeleted(&F))
+      continue;
+    Ret += getNativeSizeEstimate(F);
+  }
+  return Ret;
+}
+
+ModelUnderTrainingRunner::ModelUnderTrainingRunner(LLVMContext &Ctx,
+                                                   const std::string &ModelPath)
+    : MLModelRunner(Ctx) {
+  std::vector<std::string> InputNames;
+  std::vector<std::string> OutputNames;
+  for (size_t I = 0; I < NumberOfFeatures; ++I)
+    InputNames.push_back(TFFeedPrefix + FeatureNameMap[I]);
+  for (size_t I = 0; I < TrainingOnlyFeatures.size(); ++I)
+    InputNames.push_back(TFFeedPrefix + TrainingOnlyFeatures[I].Name);
+  OutputNames.push_back(TFDecisionName);
+
+  Evaluator =
+      std::make_unique<TFModelEvaluator>(ModelPath, InputNames, OutputNames);
+  if (!Evaluator || !Evaluator->isValid()) {
+    Ctx.emitError("Failed to create inliner saved model evaluator");
+    Evaluator.reset();
+    return;
+  }
+
+  static const std::vector<int64_t> Dim{1};
+
+  size_t InputIndex = 0;
+  for (; InputIndex < NumberOfFeatures; ++InputIndex) {
+    Evaluator->initInput<int64_t>(InputIndex, Dim);
+  }
+
+  for (; InputIndex < InputNames.size(); ++InputIndex) {
+    TrainingOnlyFeatures[InputIndex - NumberOfFeatures].Initializer(
+        Evaluator.get(), InputIndex, Dim);
+  }
+}
+
+bool ModelUnderTrainingRunner::run() {
+  auto ER = Evaluator->evaluate();
+  if (!ER.hasValue()) {
+    Ctx.emitError("Error evaluating model.");
+    return false;
+  }
+  int64_t Decision = *ER->getTensorValue<int64_t>(0);
+  return static_cast<bool>(Decision);
+}
+
+int64_t ModelUnderTrainingRunner::getFeature(int Index) const {
+  return *Evaluator->getInput<int64_t>(Index);
+}
+
+void ModelUnderTrainingRunner::setFeature(FeatureIndex Index, int64_t Value) {
+  size_t NumericIndex = static_cast<size_t>(Index);
+  *(Evaluator->getInput<int64_t>(NumericIndex)) = Value;
+}
+
+std::unique_ptr<InlineAdvisor> llvm::getDevelopmentModeAdvisor(
+    Module &M, ModuleAnalysisManager &MAM,
+    std::function<bool(CallBase &)> GetDefaultAdvice) {
+  auto &Ctx = M.getContext();
+  if (TrainingLog.empty() !=
+      !InlineSizeEstimatorAnalysis::isEvaluatorRequested()) {
+    Ctx.emitError("For development mode, if training logs are requested, then "
+                  "a size estimator must be available; either that, or neither "
+                  "are specified.");
+    return nullptr;
+  }
+
+  std::unique_ptr<MLModelRunner> Runner;
+
+  bool IsDoingInference = false;
+  if (TFModelUnderTrainingPath.empty())
+    Runner.reset(new NoInferenceModelRunner(Ctx));
+  else {
+    Runner = std::make_unique<ModelUnderTrainingRunner>(
+        Ctx, TFModelUnderTrainingPath);
+    if (!Runner) {
+      Ctx.emitError("Could not load the policy model from the provided path");
+      return nullptr;
+    }
+    IsDoingInference = true;
+  }
+  return std::make_unique<DevelopmentModeMLInlineAdvisor>(
+      M, MAM, std::move(Runner), GetDefaultAdvice, IsDoingInference);
+}
+#endif // defined(LLVM_HAVE_TF_API)
diff --git a/llvm/lib/Analysis/FunctionPropertiesAnalysis.cpp b/llvm/lib/Analysis/FunctionPropertiesAnalysis.cpp
new file mode 100644
index 0000000000000..b6ff27dcaf55a
--- /dev/null
+++ b/llvm/lib/Analysis/FunctionPropertiesAnalysis.cpp
@@ -0,0 +1,88 @@
+//===- FunctionPropertiesAnalysis.cpp - Function Properties Analysis ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the FunctionPropertiesInfo and FunctionPropertiesAnalysis
+// classes used to extract function properties.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/FunctionPropertiesAnalysis.h"
+#include "llvm/IR/Instructions.h"
+
+using namespace llvm;
+
+FunctionPropertiesInfo
+FunctionPropertiesInfo::getFunctionPropertiesInfo(const Function &F,
+                                                  const LoopInfo &LI) {
+
+  FunctionPropertiesInfo FPI;
+
+  FPI.Uses = ((!F.hasLocalLinkage()) ? 1 : 0) + F.getNumUses();
+
+  for (const auto &BB : F) {
+    ++FPI.BasicBlockCount;
+
+    if (const auto *BI = dyn_cast<BranchInst>(BB.getTerminator())) {
+      if (BI->isConditional())
+        FPI.BlocksReachedFromConditionalInstruction += BI->getNumSuccessors();
+    } else if (const auto *SI = dyn_cast<SwitchInst>(BB.getTerminator())) {
+      FPI.BlocksReachedFromConditionalInstruction +=
+          (SI->getNumCases() + (nullptr != SI->getDefaultDest()));
+    }
+
+    for (const auto &I : BB) {
+      if (auto *CS = dyn_cast<CallBase>(&I)) {
+        const auto *Callee = CS->getCalledFunction();
+        if (Callee && !Callee->isIntrinsic() && !Callee->isDeclaration())
+          ++FPI.DirectCallsToDefinedFunctions;
+      }
+      if (I.getOpcode() == Instruction::Load) {
+        ++FPI.LoadInstCount;
+      } else if (I.getOpcode() == Instruction::Store) {
+        ++FPI.StoreInstCount;
+      }
+    }
+    // Loop Depth of the Basic Block
+    int64_t LoopDepth;
+    LoopDepth = LI.getLoopDepth(&BB);
+    if (FPI.MaxLoopDepth < LoopDepth)
+      FPI.MaxLoopDepth = LoopDepth;
+  }
+  FPI.TopLevelLoopCount += llvm::size(LI);
+  return FPI;
+}
+
+void FunctionPropertiesInfo::print(raw_ostream &OS) const {
+  OS << "BasicBlockCount: " << BasicBlockCount << "\n"
+     << "BlocksReachedFromConditionalInstruction: "
+     << BlocksReachedFromConditionalInstruction << "\n"
+     << "Uses: " << Uses << "\n"
+     << "DirectCallsToDefinedFunctions: " << DirectCallsToDefinedFunctions
+     << "\n"
+     << "LoadInstCount: " << LoadInstCount << "\n"
+     << "StoreInstCount: " << StoreInstCount << "\n"
+     << "MaxLoopDepth: " << MaxLoopDepth << "\n"
+     << "TopLevelLoopCount: " << TopLevelLoopCount << "\n\n";
+}
+
+AnalysisKey FunctionPropertiesAnalysis::Key;
+
+FunctionPropertiesInfo
+FunctionPropertiesAnalysis::run(Function &F, FunctionAnalysisManager &FAM) {
+  return FunctionPropertiesInfo::getFunctionPropertiesInfo(
+      F, FAM.getResult<LoopAnalysis>(F));
+}
+
+PreservedAnalyses
+FunctionPropertiesPrinterPass::run(Function &F, FunctionAnalysisManager &AM) {
+  OS << "Printing analysis results of CFA for function "
+     << "'" << F.getName() << "':"
+     << "\n";
+  AM.getResult<FunctionPropertiesAnalysis>(F).print(OS);
+  return PreservedAnalyses::all();
+}
\ No newline at end of file
diff --git a/llvm/lib/Analysis/InlineAdvisor.cpp b/llvm/lib/Analysis/InlineAdvisor.cpp
index e18f681278d3a..ee92450d7e095 100644
--- a/llvm/lib/Analysis/InlineAdvisor.cpp
+++ b/llvm/lib/Analysis/InlineAdvisor.cpp
@@ -159,7 +159,13 @@ bool InlineAdvisorAnalysis::Result::tryCreate(InlineParams Params,
     Advisor.reset(new DefaultInlineAdvisor(FAM, Params));
     break;
   case InliningAdvisorMode::Development:
-    // To be added subsequently under conditional compilation.
+#ifdef LLVM_HAVE_TF_API
+    Advisor =
+        llvm::getDevelopmentModeAdvisor(M, MAM, [&FAM, Params](CallBase &CB) {
+          auto OIC = getDefaultInlineAdvice(CB, FAM, Params);
+          return OIC.hasValue();
+        });
+#endif
     break;
   case InliningAdvisorMode::Release:
 #ifdef LLVM_HAVE_TF_AOT
diff --git a/llvm/lib/Analysis/InlineFeaturesAnalysis.cpp b/llvm/lib/Analysis/InlineFeaturesAnalysis.cpp
deleted file mode 100644
index 90f521bbaab48..0000000000000
--- a/llvm/lib/Analysis/InlineFeaturesAnalysis.cpp
+++ /dev/null
@@ -1,41 +0,0 @@
-//===- InlineFeaturesAnalysis.cpp - Feature extraction for ML Policies ----===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements an analysis extracting function features, which may be
-// used by ML-driven policies, for example.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Analysis/InlineFeaturesAnalysis.h"
-#include "llvm/IR/Instructions.h"
-
-using namespace llvm;
-
-AnalysisKey InlineFeaturesAnalysis::Key;
-
-InlineFeaturesAnalysis::Result
-InlineFeaturesAnalysis::run(const Function &F, FunctionAnalysisManager &FAM) {
-  Result Ret;
-  Ret.Uses = ((!F.hasLocalLinkage()) ? 1 : 0) + F.getNumUses();
-  for (const auto &BB : F) {
-    ++Ret.BasicBlockCount;
-    if (const auto *BI = dyn_cast<BranchInst>(BB.getTerminator())) {
-      if (BI->isConditional())
-        Ret.BlocksReachedFromConditionalInstruction += BI->getNumSuccessors();
-    } else if (const auto *SI = dyn_cast<SwitchInst>(BB.getTerminator()))
-      Ret.BlocksReachedFromConditionalInstruction +=
-          (SI->getNumCases() + (nullptr != SI->getDefaultDest()));
-    for (const auto &I : BB)
-      if (auto *CS = dyn_cast<CallBase>(&I)) {
-        const auto *Callee = CS->getCalledFunction();
-        if (Callee && !Callee->isIntrinsic() && !Callee->isDeclaration())
-          ++Ret.DirectCallsToDefinedFunctions;
-      }
-  }
-  return Ret;
-}
\ No newline at end of file
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 8fbcee84a1567..49346860f251e 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -228,64 +228,54 @@ static bool valueDominatesPHI(Value *V, PHINode *P, const DominatorTree *DT) {
   return false;
 }
 
-/// Simplify "A op (B op' C)" by distributing op over op', turning it into
-/// "(A op B) op' (A op C)".  Here "op" is given by Opcode and "op'" is
-/// given by OpcodeToExpand, while "A" corresponds to LHS and "B op' C" to RHS.
-/// Also performs the transform "(A op' B) op C" -> "(A op C) op' (B op C)".
-/// Returns the simplified value, or null if no simplification was performed.
-static Value *ExpandBinOp(Instruction::BinaryOps Opcode, Value *LHS, Value *RHS,
-                          Instruction::BinaryOps OpcodeToExpand,
+/// Try to simplify a binary operator of form "V op OtherOp" where V is
+/// "(B0 opex B1)" by distributing 'op' across 'opex' as
+/// "(B0 op OtherOp) opex (B1 op OtherOp)".
+static Value *expandBinOp(Instruction::BinaryOps Opcode, Value *V,
+                          Value *OtherOp, Instruction::BinaryOps OpcodeToExpand,
                           const SimplifyQuery &Q, unsigned MaxRecurse) {
-  // Recursion is always used, so bail out at once if we already hit the limit.
-  if (!MaxRecurse--)
+  auto *B = dyn_cast<BinaryOperator>(V);
+  if (!B || B->getOpcode() != OpcodeToExpand)
+    return nullptr;
+  Value *B0 = B->getOperand(0), *B1 = B->getOperand(1);
+  Value *L = SimplifyBinOp(Opcode, B0, OtherOp, Q, MaxRecurse);
+  if (!L)
+    return nullptr;
+  Value *R = SimplifyBinOp(Opcode, B1, OtherOp, Q, MaxRecurse);
+  if (!R)
     return nullptr;
 
-  // Check whether the expression has the form "(A op' B) op C".
-  if (BinaryOperator *Op0 = dyn_cast<BinaryOperator>(LHS))
-    if (Op0->getOpcode() == OpcodeToExpand) {
-      // It does!  Try turning it into "(A op C) op' (B op C)".
-      Value *A = Op0->getOperand(0), *B = Op0->getOperand(1), *C = RHS;
-      // Do "A op C" and "B op C" both simplify?
-      if (Value *L = SimplifyBinOp(Opcode, A, C, Q, MaxRecurse))
-        if (Value *R = SimplifyBinOp(Opcode, B, C, Q, MaxRecurse)) {
-          // They do! Return "L op' R" if it simplifies or is already available.
-          // If "L op' R" equals "A op' B" then "L op' R" is just the LHS.
-          if ((L == A && R == B) || (Instruction::isCommutative(OpcodeToExpand)
-                                     && L == B && R == A)) {
-            ++NumExpand;
-            return LHS;
-          }
-          // Otherwise return "L op' R" if it simplifies.
-          if (Value *V = SimplifyBinOp(OpcodeToExpand, L, R, Q, MaxRecurse)) {
-            ++NumExpand;
-            return V;
-          }
-        }
-    }
+  // Does the expanded pair of binops simplify to the existing binop?
+  if ((L == B0 && R == B1) ||
+      (Instruction::isCommutative(OpcodeToExpand) && L == B1 && R == B0)) {
+    ++NumExpand;
+    return B;
+  }
 
-  // Check whether the expression has the form "A op (B op' C)".
-  if (BinaryOperator *Op1 = dyn_cast<BinaryOperator>(RHS))
-    if (Op1->getOpcode() == OpcodeToExpand) {
-      // It does!  Try turning it into "(A op B) op' (A op C)".
-      Value *A = LHS, *B = Op1->getOperand(0), *C = Op1->getOperand(1);
-      // Do "A op B" and "A op C" both simplify?
-      if (Value *L = SimplifyBinOp(Opcode, A, B, Q, MaxRecurse))
-        if (Value *R = SimplifyBinOp(Opcode, A, C, Q, MaxRecurse)) {
-          // They do! Return "L op' R" if it simplifies or is already available.
-          // If "L op' R" equals "B op' C" then "L op' R" is just the RHS.
-          if ((L == B && R == C) || (Instruction::isCommutative(OpcodeToExpand)
-                                     && L == C && R == B)) {
-            ++NumExpand;
-            return RHS;
-          }
-          // Otherwise return "L op' R" if it simplifies.
-          if (Value *V = SimplifyBinOp(OpcodeToExpand, L, R, Q, MaxRecurse)) {
-            ++NumExpand;
-            return V;
-          }
-        }
-    }
+  // Otherwise, return "L op' R" if it simplifies.
+  Value *S = SimplifyBinOp(OpcodeToExpand, L, R, Q, MaxRecurse);
+  if (!S)
+    return nullptr;
 
+  ++NumExpand;
+  return S;
+}
+
+/// Try to simplify binops of form "A op (B op' C)" or the commuted variant by
+/// distributing op over op'.
+static Value *expandCommutativeBinOp(Instruction::BinaryOps Opcode,
+                                     Value *L, Value *R,
+                                     Instruction::BinaryOps OpcodeToExpand,
+                                     const SimplifyQuery &Q,
+                                     unsigned MaxRecurse) {
+  // Recursion is always used, so bail out at once if we already hit the limit.
+  if (!MaxRecurse--)
+    return nullptr;
+
+  if (Value *V = expandBinOp(Opcode, L, R, OpcodeToExpand, Q, MaxRecurse))
+    return V;
+  if (Value *V = expandBinOp(Opcode, R, L, OpcodeToExpand, Q, MaxRecurse))
+    return V;
   return nullptr;
 }
 
@@ -901,8 +891,8 @@ static Value *SimplifyMulInst(Value *Op0, Value *Op1, const SimplifyQuery &Q,
     return V;
 
   // Mul distributes over Add. Try some generic simplifications based on this.
-  if (Value *V = ExpandBinOp(Instruction::Mul, Op0, Op1, Instruction::Add,
-                             Q, MaxRecurse))
+  if (Value *V = expandCommutativeBinOp(Instruction::Mul, Op0, Op1,
+                                        Instruction::Add, Q, MaxRecurse))
     return V;
 
   // If the operation is with the result of a select instruction, check whether
@@ -1222,7 +1212,8 @@ static bool isUndefShift(Value *Amount) {
 
   // If all lanes of a vector shift are undefined the whole shift is.
   if (isa<ConstantVector>(C) || isa<ConstantDataVector>(C)) {
-    for (unsigned I = 0, E = cast<VectorType>(C->getType())->getNumElements();
+    for (unsigned I = 0,
+                  E = cast<FixedVectorType>(C->getType())->getNumElements();
          I != E; ++I)
       if (!isUndefShift(C->getAggregateElement(I)))
         return false;
@@ -2095,13 +2086,13 @@ static Value *SimplifyAndInst(Value *Op0, Value *Op1, const SimplifyQuery &Q,
     return V;
 
   // And distributes over Or.  Try some generic simplifications based on this.
-  if (Value *V = ExpandBinOp(Instruction::And, Op0, Op1, Instruction::Or,
-                             Q, MaxRecurse))
+  if (Value *V = expandCommutativeBinOp(Instruction::And, Op0, Op1,
+                                        Instruction::Or, Q, MaxRecurse))
     return V;
 
   // And distributes over Xor.  Try some generic simplifications based on this.
-  if (Value *V = ExpandBinOp(Instruction::And, Op0, Op1, Instruction::Xor,
-                             Q, MaxRecurse))
+  if (Value *V = expandCommutativeBinOp(Instruction::And, Op0, Op1,
+                                        Instruction::Xor, Q, MaxRecurse))
     return V;
 
   // If the operation is with the result of a select instruction, check whether
@@ -2253,8 +2244,8 @@ static Value *SimplifyOrInst(Value *Op0, Value *Op1, const SimplifyQuery &Q,
     return V;
 
   // Or distributes over And.  Try some generic simplifications based on this.
-  if (Value *V = ExpandBinOp(Instruction::Or, Op0, Op1, Instruction::And, Q,
-                             MaxRecurse))
+  if (Value *V = expandCommutativeBinOp(Instruction::Or, Op0, Op1,
+                                        Instruction::And, Q, MaxRecurse))
     return V;
 
   // If the operation is with the result of a select instruction, check whether
@@ -4134,7 +4125,8 @@ static Value *SimplifySelectInst(Value *Cond, Value *TrueVal, Value *FalseVal,
   Constant *TrueC, *FalseC;
   if (TrueVal->getType()->isVectorTy() && match(TrueVal, m_Constant(TrueC)) &&
       match(FalseVal, m_Constant(FalseC))) {
-    unsigned NumElts = cast<VectorType>(TrueC->getType())->getNumElements();
+    unsigned NumElts =
+        cast<FixedVectorType>(TrueC->getType())->getNumElements();
     SmallVector<Constant *, 16> NewC;
     for (unsigned i = 0; i != NumElts; ++i) {
       // Bail out on incomplete vector constants.
@@ -4420,7 +4412,7 @@ static Value *SimplifyExtractElementInst(Value *Vec, Value *Idx, const SimplifyQ
   if (auto *IdxC = dyn_cast<ConstantInt>(Idx)) {
     // For fixed-length vector, fold into undef if index is out of bounds.
     if (isa<FixedVectorType>(VecVTy) &&
-        IdxC->getValue().uge(VecVTy->getNumElements()))
+        IdxC->getValue().uge(cast<FixedVectorType>(VecVTy)->getNumElements()))
       return UndefValue::get(VecVTy->getElementType());
     if (Value *Elt = findScalarElement(Vec, IdxC->getZExtValue()))
       return Elt;
@@ -4526,7 +4518,7 @@ static Value *foldIdentityShuffles(int DestElt, Value *Op0, Value *Op1,
     return nullptr;
 
   // The mask value chooses which source operand we need to look at next.
-  int InVecNumElts = cast<VectorType>(Op0->getType())->getNumElements();
+  int InVecNumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
   int RootElt = MaskVal;
   Value *SourceOp = Op0;
   if (MaskVal >= InVecNumElts) {
diff --git a/llvm/lib/Analysis/Lint.cpp b/llvm/lib/Analysis/Lint.cpp
index 564c00dbad983..38017530c3227 100644
--- a/llvm/lib/Analysis/Lint.cpp
+++ b/llvm/lib/Analysis/Lint.cpp
@@ -567,7 +567,8 @@ static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT,
 
   // For a vector, KnownZero will only be true if all values are zero, so check
   // this per component
-  for (unsigned I = 0, N = VecTy->getNumElements(); I != N; ++I) {
+  for (unsigned I = 0, N = cast<FixedVectorType>(VecTy)->getNumElements();
+       I != N; ++I) {
     Constant *Elem = C->getAggregateElement(I);
     if (isa<UndefValue>(Elem))
       return true;
@@ -625,14 +626,17 @@ void Lint::visitIndirectBrInst(IndirectBrInst &I) {
 void Lint::visitExtractElementInst(ExtractElementInst &I) {
   if (ConstantInt *CI = dyn_cast<ConstantInt>(findValue(I.getIndexOperand(),
                                                         /*OffsetOk=*/false)))
-    Assert(CI->getValue().ult(I.getVectorOperandType()->getNumElements()),
-           "Undefined result: extractelement index out of range", &I);
+    Assert(
+        CI->getValue().ult(
+            cast<FixedVectorType>(I.getVectorOperandType())->getNumElements()),
+        "Undefined result: extractelement index out of range", &I);
 }
 
 void Lint::visitInsertElementInst(InsertElementInst &I) {
   if (ConstantInt *CI = dyn_cast<ConstantInt>(findValue(I.getOperand(2),
                                                         /*OffsetOk=*/false)))
-    Assert(CI->getValue().ult(I.getType()->getNumElements()),
+    Assert(CI->getValue().ult(
+               cast<FixedVectorType>(I.getType())->getNumElements()),
            "Undefined result: insertelement index out of range", &I);
 }
 
diff --git a/llvm/lib/Analysis/MLInlineAdvisor.cpp b/llvm/lib/Analysis/MLInlineAdvisor.cpp
index 45873f260f23d..e697af97eb078 100644
--- a/llvm/lib/Analysis/MLInlineAdvisor.cpp
+++ b/llvm/lib/Analysis/MLInlineAdvisor.cpp
@@ -11,14 +11,17 @@
 // 'release' mode) or a runtime-loaded model (the 'development' case).
 //
 //===----------------------------------------------------------------------===//
+#include "llvm/Config/config.h"
+#if defined(LLVM_HAVE_TF_AOT) || defined(LLVM_HAVE_TF_API)
+
 #include <limits>
 #include <unordered_map>
 #include <unordered_set>
 
 #include "llvm/ADT/SCCIterator.h"
 #include "llvm/Analysis/CallGraph.h"
+#include "llvm/Analysis/FunctionPropertiesAnalysis.h"
 #include "llvm/Analysis/InlineCost.h"
-#include "llvm/Analysis/InlineFeaturesAnalysis.h"
 #include "llvm/Analysis/MLInlineAdvisor.h"
 #include "llvm/Analysis/MLModelRunner.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
@@ -115,7 +118,8 @@ void MLInlineAdvisor::onPassEntry() {
 }
 
 int64_t MLInlineAdvisor::getLocalCalls(Function &F) {
-  return FAM.getResult<InlineFeaturesAnalysis>(F).DirectCallsToDefinedFunctions;
+  return FAM.getResult<FunctionPropertiesAnalysis>(F)
+      .DirectCallsToDefinedFunctions;
 }
 
 // Update the internal state of the advisor, and force invalidate feature
@@ -130,7 +134,7 @@ void MLInlineAdvisor::onSuccessfulInlining(const MLInlineAdvice &Advice,
   Function *Callee = Advice.getCallee();
 
   // The caller features aren't valid anymore.
-  FAM.invalidate<InlineFeaturesAnalysis>(*Caller);
+  FAM.invalidate<FunctionPropertiesAnalysis>(*Caller);
   int64_t IRSizeAfter =
       getIRSize(*Caller) + (CalleeWasDeleted ? 0 : Advice.CalleeIRSize);
   CurrentIRSize += IRSizeAfter - (Advice.CallerIRSize + Advice.CalleeIRSize);
@@ -143,14 +147,15 @@ void MLInlineAdvisor::onSuccessfulInlining(const MLInlineAdvice &Advice,
   // For edges, we 'forget' the edges that the caller and callee used to have
   // before inlining, and add back what they currently have together.
   int64_t NewCallerAndCalleeEdges =
-      FAM.getResult<InlineFeaturesAnalysis>(*Caller)
+      FAM.getResult<FunctionPropertiesAnalysis>(*Caller)
           .DirectCallsToDefinedFunctions;
 
   if (CalleeWasDeleted)
     --NodeCount;
   else
-    NewCallerAndCalleeEdges += FAM.getResult<InlineFeaturesAnalysis>(*Callee)
-                                   .DirectCallsToDefinedFunctions;
+    NewCallerAndCalleeEdges +=
+        FAM.getResult<FunctionPropertiesAnalysis>(*Callee)
+            .DirectCallsToDefinedFunctions;
   EdgeCount += (NewCallerAndCalleeEdges - Advice.CallerAndCalleeEdges);
   assert(CurrentIRSize >= 0 && EdgeCount >= 0 && NodeCount >= 0);
 }
@@ -221,8 +226,8 @@ std::unique_ptr<InlineAdvice> MLInlineAdvisor::getAdvice(CallBase &CB) {
     NrCtantParams += (isa<Constant>(*I));
   }
 
-  auto &CallerBefore = FAM.getResult<InlineFeaturesAnalysis>(Caller);
-  auto &CalleeBefore = FAM.getResult<InlineFeaturesAnalysis>(Callee);
+  auto &CallerBefore = FAM.getResult<FunctionPropertiesAnalysis>(Caller);
+  auto &CalleeBefore = FAM.getResult<FunctionPropertiesAnalysis>(Callee);
 
   ModelRunner->setFeature(FeatureIndex::CalleeBasicBlockCount,
                           CalleeBefore.BasicBlockCount);
@@ -298,4 +303,5 @@ void MLInlineAdvice::recordUnattemptedInliningImpl() {
     reportContextForRemark(R);
     return R;
   });
-}
\ No newline at end of file
+}
+#endif // defined(LLVM_HAVE_TF_AOT) || defined(LLVM_HAVE_TF_API)
diff --git a/llvm/lib/Analysis/MemoryBuiltins.cpp b/llvm/lib/Analysis/MemoryBuiltins.cpp
index 204f855d28b33..402a2bcadf5fc 100644
--- a/llvm/lib/Analysis/MemoryBuiltins.cpp
+++ b/llvm/lib/Analysis/MemoryBuiltins.cpp
@@ -676,13 +676,14 @@ SizeOffsetType ObjectSizeOffsetVisitor::visitAllocaInst(AllocaInst &I) {
 }
 
 SizeOffsetType ObjectSizeOffsetVisitor::visitArgument(Argument &A) {
+  Type *MemoryTy = A.getPointeeInMemoryValueType();
   // No interprocedural analysis is done at the moment.
-  if (!A.hasPassPointeeByValueCopyAttr()) {
+  if (!MemoryTy) {
     ++ObjectVisitorArgument;
     return unknown();
   }
-  PointerType *PT = cast<PointerType>(A.getType());
-  APInt Size(IntTyBits, DL.getTypeAllocSize(PT->getElementType()));
+
+  APInt Size(IntTyBits, DL.getTypeAllocSize(MemoryTy));
   return std::make_pair(align(Size, A.getParamAlignment()), Zero);
 }
 
diff --git a/llvm/lib/Analysis/RegionInfo.cpp b/llvm/lib/Analysis/RegionInfo.cpp
index 88629517d484b..3ba0bb9eaf2cf 100644
--- a/llvm/lib/Analysis/RegionInfo.cpp
+++ b/llvm/lib/Analysis/RegionInfo.cpp
@@ -17,11 +17,8 @@
 #include "llvm/Analysis/RegionInfoImpl.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/Function.h"
-#include "llvm/IR/PassManager.h"
-#include "llvm/Pass.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Compiler.h"
-#include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
 
diff --git a/llvm/lib/Analysis/ReleaseModeModelRunner.cpp b/llvm/lib/Analysis/ReleaseModeModelRunner.cpp
index 4c0ffbc17ff7a..af300fb71b4b7 100644
--- a/llvm/lib/Analysis/ReleaseModeModelRunner.cpp
+++ b/llvm/lib/Analysis/ReleaseModeModelRunner.cpp
@@ -10,6 +10,8 @@
 // Only inference is supported.
 //
 //===----------------------------------------------------------------------===//
+#include "llvm/Config/config.h"
+#if defined(LLVM_HAVE_TF_AOT)
 
 #include "llvm/Analysis/InlineModelFeatureMaps.h"
 #include "llvm/Analysis/MLInlineAdvisor.h"
@@ -85,3 +87,4 @@ llvm::getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM) {
   auto AOTRunner = std::make_unique<ReleaseModeModelRunner>(M.getContext());
   return std::make_unique<MLInlineAdvisor>(M, MAM, std::move(AOTRunner));
 }
+#endif // defined(LLVM_HAVE_TF_AOT)
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 755a4e9685211..a961be8cc35e1 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -3317,10 +3317,7 @@ ScalarEvolution::getGEPExpr(GEPOperator *GEP,
   }
 
   // Add the total offset from all the GEP indices to the base.
-  auto *GEPExpr = getAddExpr(BaseExpr, TotalOffset, Wrap);
-  assert(BaseExpr->getType() == GEPExpr->getType() &&
-         "GEP should not change type mid-flight.");
-  return GEPExpr;
+  return getAddExpr(BaseExpr, TotalOffset, Wrap);
 }
 
 std::tuple<SCEV *, FoldingSetNodeID, void *>
diff --git a/llvm/lib/Analysis/TFUtils.cpp b/llvm/lib/Analysis/TFUtils.cpp
index 19e6d626e2386..a6f5b2956399e 100644
--- a/llvm/lib/Analysis/TFUtils.cpp
+++ b/llvm/lib/Analysis/TFUtils.cpp
@@ -10,6 +10,8 @@
 // This file implements utilities for interfacing with tensorflow C APIs.
 //
 //===----------------------------------------------------------------------===//
+#include "llvm/Config/config.h"
+#if defined(LLVM_HAVE_TF_API)
 
 #include "llvm/Analysis/Utils/TFUtils.h"
 #include "llvm/ADT/Twine.h"
@@ -287,3 +289,4 @@ template <> int TFModelEvaluator::getModelTypeIndex<uint64_t>() {
 
 TFModelEvaluator::EvaluationResult::~EvaluationResult() {}
 TFModelEvaluator::~TFModelEvaluator() {}
+#endif // defined(LLVM_HAVE_TF_API)
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 2f051e53790b1..c9e702ce56b81 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -322,6 +322,29 @@ bool TargetTransformInfo::emitGetActiveLaneMask() const {
   return TTIImpl->emitGetActiveLaneMask();
 }
 
+Optional<Instruction *>
+TargetTransformInfo::instCombineIntrinsic(InstCombiner &IC,
+                                          IntrinsicInst &II) const {
+  return TTIImpl->instCombineIntrinsic(IC, II);
+}
+
+Optional<Value *> TargetTransformInfo::simplifyDemandedUseBitsIntrinsic(
+    InstCombiner &IC, IntrinsicInst &II, APInt DemandedMask, KnownBits &Known,
+    bool &KnownBitsComputed) const {
+  return TTIImpl->simplifyDemandedUseBitsIntrinsic(IC, II, DemandedMask, Known,
+                                                   KnownBitsComputed);
+}
+
+Optional<Value *> TargetTransformInfo::simplifyDemandedVectorEltsIntrinsic(
+    InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
+    APInt &UndefElts2, APInt &UndefElts3,
+    std::function<void(Instruction *, unsigned, APInt, APInt &)>
+        SimplifyAndSetOp) const {
+  return TTIImpl->simplifyDemandedVectorEltsIntrinsic(
+      IC, II, DemandedElts, UndefElts, UndefElts2, UndefElts3,
+      SimplifyAndSetOp);
+}
+
 void TargetTransformInfo::getUnrollingPreferences(
     Loop *L, ScalarEvolution &SE, UnrollingPreferences &UP) const {
   return TTIImpl->getUnrollingPreferences(L, SE, UP);
@@ -966,7 +989,8 @@ static bool matchPairwiseShuffleMask(ShuffleVectorInst *SI, bool IsLeft,
   else if (!SI)
     return false;
 
-  SmallVector<int, 32> Mask(SI->getType()->getNumElements(), -1);
+  SmallVector<int, 32> Mask(
+      cast<FixedVectorType>(SI->getType())->getNumElements(), -1);
 
   // Build a mask of 0, 2, ... (left) or 1, 3, ... (right) depending on whether
   // we look at the left or right side.
@@ -1105,7 +1129,7 @@ TTI::ReductionKind TTI::matchPairwiseReduction(
   if (!RD)
     return TTI::RK_None;
 
-  auto *VecTy = cast<VectorType>(RdxStart->getType());
+  auto *VecTy = cast<FixedVectorType>(RdxStart->getType());
   unsigned NumVecElems = VecTy->getNumElements();
   if (!isPowerOf2_32(NumVecElems))
     return TTI::RK_None;
@@ -1170,7 +1194,7 @@ TTI::ReductionKind TTI::matchVectorSplittingReduction(
   if (!RD)
     return TTI::RK_None;
 
-  auto *VecTy = cast<VectorType>(ReduxRoot->getOperand(0)->getType());
+  auto *VecTy = cast<FixedVectorType>(ReduxRoot->getOperand(0)->getType());
   unsigned NumVecElems = VecTy->getNumElements();
   if (!isPowerOf2_32(NumVecElems))
     return TTI::RK_None;
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 380022c10acec..0ab2a1350af31 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -172,8 +172,8 @@ static bool getShuffleDemandedElts(const ShuffleVectorInst *Shuf,
     return false;
 
   int NumElts =
-      cast<VectorType>(Shuf->getOperand(0)->getType())->getNumElements();
-  int NumMaskElts = Shuf->getType()->getNumElements();
+      cast<FixedVectorType>(Shuf->getOperand(0)->getType())->getNumElements();
+  int NumMaskElts = cast<FixedVectorType>(Shuf->getType())->getNumElements();
   DemandedLHS = DemandedRHS = APInt::getNullValue(NumElts);
   if (DemandedElts.isNullValue())
     return true;
@@ -3489,9 +3489,10 @@ bool llvm::isKnownNeverInfinity(const Value *V, const TargetLibraryInfo *TLI,
   }
 
   // try to handle fixed width vector constants
-  if (isa<FixedVectorType>(V->getType()) && isa<Constant>(V)) {
+  auto *VFVTy = dyn_cast<FixedVectorType>(V->getType());
+  if (VFVTy && isa<Constant>(V)) {
     // For vectors, verify that each element is not infinity.
-    unsigned NumElts = cast<VectorType>(V->getType())->getNumElements();
+    unsigned NumElts = VFVTy->getNumElements();
     for (unsigned i = 0; i != NumElts; ++i) {
       Constant *Elt = cast<Constant>(V)->getAggregateElement(i);
       if (!Elt)
@@ -3593,9 +3594,10 @@ bool llvm::isKnownNeverNaN(const Value *V, const TargetLibraryInfo *TLI,
   }
 
   // Try to handle fixed width vector constants
-  if (isa<FixedVectorType>(V->getType()) && isa<Constant>(V)) {
+  auto *VFVTy = dyn_cast<FixedVectorType>(V->getType());
+  if (VFVTy && isa<Constant>(V)) {
     // For vectors, verify that each element is not NaN.
-    unsigned NumElts = cast<VectorType>(V->getType())->getNumElements();
+    unsigned NumElts = VFVTy->getNumElements();
     for (unsigned i = 0; i != NumElts; ++i) {
       Constant *Elt = cast<Constant>(V)->getAggregateElement(i);
       if (!Elt)
@@ -4779,23 +4781,13 @@ bool llvm::isGuaranteedNotToBeUndefOrPoison(const Value *V,
   if (Depth >= MaxDepth)
     return false;
 
-  // If the value is a freeze instruction, then it can never
-  // be undef or poison.
-  if (isa<FreezeInst>(V))
-    return true;
-  // TODO: Some instructions are guaranteed to return neither undef
-  // nor poison if their arguments are not poison/undef.
-
-  if (auto *A = dyn_cast<Argument>(V)) {
-    // NoUndef does not guarantee that paddings are not undef.
+  if (const auto *A = dyn_cast<Argument>(V)) {
     if (A->hasAttribute(Attribute::NoUndef))
       return true;
   }
 
   if (auto *C = dyn_cast<Constant>(V)) {
-    // TODO: We can analyze ConstExpr by opcode to determine if there is any
-    //       possibility of poison.
-    if (isa<UndefValue>(C) || isa<ConstantExpr>(C))
+    if (isa<UndefValue>(C))
       return false;
 
     if (isa<ConstantInt>(C) || isa<GlobalVariable>(C) || isa<ConstantFP>(V) ||
@@ -4804,9 +4796,6 @@ bool llvm::isGuaranteedNotToBeUndefOrPoison(const Value *V,
 
     if (C->getType()->isVectorTy())
       return !C->containsUndefElement() && !C->containsConstantExpression();
-
-    // TODO: Recursively analyze aggregates or other constants.
-    return false;
   }
 
   // Strip cast operations from a pointer value.
@@ -4826,31 +4815,22 @@ bool llvm::isGuaranteedNotToBeUndefOrPoison(const Value *V,
     return isGuaranteedNotToBeUndefOrPoison(V, CtxI, DT, Depth + 1);
   };
 
-  if (auto *I = dyn_cast<Instruction>(V)) {
-    switch (I->getOpcode()) {
-    case Instruction::GetElementPtr: {
-      auto *GEPI = dyn_cast<GetElementPtrInst>(I);
-      if (!GEPI->isInBounds() && llvm::all_of(GEPI->operands(), OpCheck))
-        return true;
-      break;
-    }
-    case Instruction::FCmp: {
-      auto *FI = dyn_cast<FCmpInst>(I);
-      if (FI->getFastMathFlags().none() &&
-          llvm::all_of(FI->operands(), OpCheck))
-        return true;
-      break;
-    }
-    case Instruction::BitCast:
-    case Instruction::PHI:
-    case Instruction::ICmp:
-      if (llvm::all_of(I->operands(), OpCheck))
+  if (auto *Opr = dyn_cast<Operator>(V)) {
+    // If the value is a freeze instruction, then it can never
+    // be undef or poison.
+    if (isa<FreezeInst>(V))
+      return true;
+
+    if (const auto *CB = dyn_cast<CallBase>(V)) {
+      if (CB->hasRetAttr(Attribute::NoUndef))
         return true;
-      break;
-    default:
-      break;
     }
 
+    if (!canCreateUndefOrPoison(Opr) && all_of(Opr->operands(), OpCheck))
+      return true;
+  }
+
+  if (auto *I = dyn_cast<Instruction>(V)) {
     if (programUndefinedIfPoison(I) && I->getType()->isIntegerTy(1))
       // Note: once we have an agreement that poison is a value-wise concept,
       // we can remove the isIntegerTy(1) constraint.
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index 23531b65ea32d..fbd5c80350627 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -817,8 +817,8 @@ static Value *concatenateTwoVectors(IRBuilderBase &Builder, Value *V1,
          VecTy1->getScalarType() == VecTy2->getScalarType() &&
          "Expect two vectors with the same element type");
 
-  unsigned NumElts1 = VecTy1->getNumElements();
-  unsigned NumElts2 = VecTy2->getNumElements();
+  unsigned NumElts1 = cast<FixedVectorType>(VecTy1)->getNumElements();
+  unsigned NumElts2 = cast<FixedVectorType>(VecTy2)->getNumElements();
   assert(NumElts1 >= NumElts2 && "Unexpect the first vector has less elements");
 
   if (NumElts1 > NumElts2) {
@@ -866,8 +866,9 @@ bool llvm::maskIsAllZeroOrUndef(Value *Mask) {
     return false;
   if (ConstMask->isNullValue() || isa<UndefValue>(ConstMask))
     return true;
-  for (unsigned I = 0,
-                E = cast<VectorType>(ConstMask->getType())->getNumElements();
+  for (unsigned
+           I = 0,
+           E = cast<FixedVectorType>(ConstMask->getType())->getNumElements();
        I != E; ++I) {
     if (auto *MaskElt = ConstMask->getAggregateElement(I))
       if (MaskElt->isNullValue() || isa<UndefValue>(MaskElt))
@@ -884,8 +885,9 @@ bool llvm::maskIsAllOneOrUndef(Value *Mask) {
     return false;
   if (ConstMask->isAllOnesValue() || isa<UndefValue>(ConstMask))
     return true;
-  for (unsigned I = 0,
-                E = cast<VectorType>(ConstMask->getType())->getNumElements();
+  for (unsigned
+           I = 0,
+           E = cast<FixedVectorType>(ConstMask->getType())->getNumElements();
        I != E; ++I) {
     if (auto *MaskElt = ConstMask->getAggregateElement(I))
       if (MaskElt->isAllOnesValue() || isa<UndefValue>(MaskElt))
@@ -899,7 +901,8 @@ bool llvm::maskIsAllOneOrUndef(Value *Mask) {
 /// vectors.  Is there something we can common this with?
 APInt llvm::possiblyDemandedEltsInMask(Value *Mask) {
 
-  const unsigned VWidth = cast<VectorType>(Mask->getType())->getNumElements();
+  const unsigned VWidth =
+      cast<FixedVectorType>(Mask->getType())->getNumElements();
   APInt DemandedElts = APInt::getAllOnesValue(VWidth);
   if (auto *CV = dyn_cast<ConstantVector>(Mask))
     for (unsigned i = 0; i < VWidth; i++)
diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp
index 777ce3abdddd5..591c6c3be2f10 100644
--- a/llvm/lib/AsmParser/LLLexer.cpp
+++ b/llvm/lib/AsmParser/LLLexer.cpp
@@ -697,6 +697,7 @@ lltok::Kind LLLexer::LexIdentifier() {
   KEYWORD(writeonly);
   KEYWORD(zeroext);
   KEYWORD(immarg);
+  KEYWORD(byref);
 
   KEYWORD(type);
   KEYWORD(opaque);
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index c9f21ee83826a..cb19b1cdb2137 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -788,13 +788,20 @@ bool LLParser::ParseStandaloneMetadata() {
 // Skips a single module summary entry.
 bool LLParser::SkipModuleSummaryEntry() {
   // Each module summary entry consists of a tag for the entry
-  // type, followed by a colon, then the fields surrounded by nested sets of
-  // parentheses. The "tag:" looks like a Label. Once parsing support is
-  // in place we will look for the tokens corresponding to the expected tags.
+  // type, followed by a colon, then the fields which may be surrounded by
+  // nested sets of parentheses. The "tag:" looks like a Label. Once parsing
+  // support is in place we will look for the tokens corresponding to the
+  // expected tags.
   if (Lex.getKind() != lltok::kw_gv && Lex.getKind() != lltok::kw_module &&
-      Lex.getKind() != lltok::kw_typeid)
+      Lex.getKind() != lltok::kw_typeid && Lex.getKind() != lltok::kw_flags &&
+      Lex.getKind() != lltok::kw_blockcount)
     return TokError(
-        "Expected 'gv', 'module', or 'typeid' at the start of summary entry");
+        "Expected 'gv', 'module', 'typeid', 'flags' or 'blockcount' at the "
+        "start of summary entry");
+  if (Lex.getKind() == lltok::kw_flags)
+    return ParseSummaryIndexFlags();
+  if (Lex.getKind() == lltok::kw_blockcount)
+    return ParseBlockCount();
   Lex.Lex();
   if (ParseToken(lltok::colon, "expected ':' at start of summary entry") ||
       ParseToken(lltok::lparen, "expected '(' at start of summary entry"))
@@ -1382,6 +1389,7 @@ bool LLParser::ParseFnAttributeValuePairs(AttrBuilder &B,
     case lltok::kw_swifterror:
     case lltok::kw_swiftself:
     case lltok::kw_immarg:
+    case lltok::kw_byref:
       HaveError |=
         Error(Lex.getLoc(),
               "invalid use of parameter-only attribute on a function");
@@ -1675,6 +1683,13 @@ bool LLParser::ParseOptionalParamAttrs(AttrBuilder &B) {
       B.addDereferenceableOrNullAttr(Bytes);
       continue;
     }
+    case lltok::kw_byref: {
+      Type *Ty;
+      if (ParseByRef(Ty))
+        return true;
+      B.addByRefAttr(Ty);
+      continue;
+    }
     case lltok::kw_inalloca:        B.addAttribute(Attribute::InAlloca); break;
     case lltok::kw_inreg:           B.addAttribute(Attribute::InReg); break;
     case lltok::kw_nest:            B.addAttribute(Attribute::Nest); break;
@@ -1795,6 +1810,7 @@ bool LLParser::ParseOptionalReturnAttrs(AttrBuilder &B) {
     case lltok::kw_swifterror:
     case lltok::kw_swiftself:
     case lltok::kw_immarg:
+    case lltok::kw_byref:
       HaveError |= Error(Lex.getLoc(), "invalid use of parameter-only attribute");
       break;
 
@@ -2568,11 +2584,11 @@ bool LLParser::ParseByValWithOptionalType(Type *&Result) {
   return false;
 }
 
-/// ParsePreallocated
-///   ::= preallocated(<ty>)
-bool LLParser::ParsePreallocated(Type *&Result) {
+/// ParseRequiredTypeAttr
+///   ::= attrname(<ty>)
+bool LLParser::ParseRequiredTypeAttr(Type *&Result, lltok::Kind AttrName) {
   Result = nullptr;
-  if (!EatIfPresent(lltok::kw_preallocated))
+  if (!EatIfPresent(AttrName))
     return true;
   if (!EatIfPresent(lltok::lparen))
     return Error(Lex.getLoc(), "expected '('");
@@ -2583,6 +2599,18 @@ bool LLParser::ParsePreallocated(Type *&Result) {
   return false;
 }
 
+/// ParsePreallocated
+///   ::= preallocated(<ty>)
+bool LLParser::ParsePreallocated(Type *&Result) {
+  return ParseRequiredTypeAttr(Result, lltok::kw_preallocated);
+}
+
+/// ParseByRef
+///   ::= byref(<type>)
+bool LLParser::ParseByRef(Type *&Result) {
+  return ParseRequiredTypeAttr(Result, lltok::kw_byref);
+}
+
 /// ParseOptionalOperandBundles
 ///    ::= /*empty*/
 ///    ::= '[' OperandBundle [, OperandBundle ]* ']'
@@ -4661,7 +4689,9 @@ bool LLParser::ParseDICompositeType(MDNode *&Result, bool IsDistinct) {
   OPTIONAL(templateParams, MDField, );                                         \
   OPTIONAL(identifier, MDStringField, );                                       \
   OPTIONAL(discriminator, MDField, );                                          \
-  OPTIONAL(dataLocation, MDField, );
+  OPTIONAL(dataLocation, MDField, );                                           \
+  OPTIONAL(associated, MDField, );                                             \
+  OPTIONAL(allocated, MDField, );
   PARSE_MD_FIELDS();
 #undef VISIT_MD_FIELDS
 
@@ -4671,7 +4701,8 @@ bool LLParser::ParseDICompositeType(MDNode *&Result, bool IsDistinct) {
             Context, *identifier.Val, tag.Val, name.Val, file.Val, line.Val,
             scope.Val, baseType.Val, size.Val, align.Val, offset.Val, flags.Val,
             elements.Val, runtimeLang.Val, vtableHolder.Val, templateParams.Val,
-            discriminator.Val, dataLocation.Val)) {
+            discriminator.Val, dataLocation.Val, associated.Val,
+            allocated.Val)) {
       Result = CT;
       return false;
     }
@@ -4683,7 +4714,7 @@ bool LLParser::ParseDICompositeType(MDNode *&Result, bool IsDistinct) {
       (Context, tag.Val, name.Val, file.Val, line.Val, scope.Val, baseType.Val,
        size.Val, align.Val, offset.Val, flags.Val, elements.Val,
        runtimeLang.Val, vtableHolder.Val, templateParams.Val, identifier.Val,
-       discriminator.Val, dataLocation.Val));
+       discriminator.Val, dataLocation.Val, associated.Val, allocated.Val));
   return false;
 }
 
@@ -8145,7 +8176,8 @@ bool LLParser::ParseSummaryIndexFlags() {
   uint64_t Flags;
   if (ParseUInt64(Flags))
     return true;
-  Index->setFlags(Flags);
+  if (Index)
+    Index->setFlags(Flags);
   return false;
 }
 
@@ -8160,7 +8192,8 @@ bool LLParser::ParseBlockCount() {
   uint64_t BlockCount;
   if (ParseUInt64(BlockCount))
     return true;
-  Index->setBlockCount(BlockCount);
+  if (Index)
+    Index->setBlockCount(BlockCount);
   return false;
 }
 
diff --git a/llvm/lib/AsmParser/LLParser.h b/llvm/lib/AsmParser/LLParser.h
index ebd8655dc35e9..333b8a35cfce1 100644
--- a/llvm/lib/AsmParser/LLParser.h
+++ b/llvm/lib/AsmParser/LLParser.h
@@ -333,7 +333,10 @@ namespace llvm {
                                     std::vector<unsigned> &FwdRefAttrGrps,
                                     bool inAttrGrp, LocTy &BuiltinLoc);
     bool ParseByValWithOptionalType(Type *&Result);
+
+    bool ParseRequiredTypeAttr(Type *&Result, lltok::Kind AttrName);
     bool ParsePreallocated(Type *&Result);
+    bool ParseByRef(Type *&Result);
 
     // Module Summary Index Parsing.
     bool SkipModuleSummaryEntry();
diff --git a/llvm/lib/AsmParser/LLToken.h b/llvm/lib/AsmParser/LLToken.h
index 0fb3bae77dd3f..8bdeb57999c60 100644
--- a/llvm/lib/AsmParser/LLToken.h
+++ b/llvm/lib/AsmParser/LLToken.h
@@ -240,6 +240,7 @@ enum Kind {
   kw_writeonly,
   kw_zeroext,
   kw_immarg,
+  kw_byref,
 
   kw_type,
   kw_opaque,
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 659e26c2bd250..9632b5700e8af 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -1532,6 +1532,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) {
     return Attribute::Preallocated;
   case bitc::ATTR_KIND_NOUNDEF:
     return Attribute::NoUndef;
+  case bitc::ATTR_KIND_BYREF:
+    return Attribute::ByRef;
   }
 }
 
@@ -1649,6 +1651,8 @@ Error BitcodeReader::parseAttributeGroupBlock() {
             return Err;
           if (Kind == Attribute::ByVal) {
             B.addByValAttr(HasType ? getTypeByID(Record[++i]) : nullptr);
+          } else if (Kind == Attribute::ByRef) {
+            B.addByRefAttr(getTypeByID(Record[++i]));
           } else if (Kind == Attribute::Preallocated) {
             B.addPreallocatedAttr(getTypeByID(Record[++i]));
           }
@@ -2965,12 +2969,15 @@ Error BitcodeReader::materializeMetadata() {
   }
 
   // Upgrade "Linker Options" module flag to "llvm.linker.options" module-level
-  // metadata.
-  if (Metadata *Val = TheModule->getModuleFlag("Linker Options")) {
-    NamedMDNode *LinkerOpts =
-        TheModule->getOrInsertNamedMetadata("llvm.linker.options");
-    for (const MDOperand &MDOptions : cast<MDNode>(Val)->operands())
-      LinkerOpts->addOperand(cast<MDNode>(MDOptions));
+  // metadata. Only upgrade if the new option doesn't exist to avoid upgrade
+  // multiple times.
+  if (!TheModule->getNamedMetadata("llvm.linker.options")) {
+    if (Metadata *Val = TheModule->getModuleFlag("Linker Options")) {
+      NamedMDNode *LinkerOpts =
+          TheModule->getOrInsertNamedMetadata("llvm.linker.options");
+      for (const MDOperand &MDOptions : cast<MDNode>(Val)->operands())
+        LinkerOpts->addOperand(cast<MDNode>(MDOptions));
+    }
   }
 
   DeferredMetadataInfo.clear();
@@ -5390,6 +5397,36 @@ Error BitcodeReader::materialize(GlobalValue *GV) {
     }
   }
 
+  // "Upgrade" older incorrect branch weights by dropping them.
+  for (auto &I : instructions(F)) {
+    if (auto *MD = I.getMetadata(LLVMContext::MD_prof)) {
+      if (MD->getOperand(0) != nullptr && isa<MDString>(MD->getOperand(0))) {
+        MDString *MDS = cast<MDString>(MD->getOperand(0));
+        StringRef ProfName = MDS->getString();
+        // Check consistency of !prof branch_weights metadata.
+        if (!ProfName.equals("branch_weights"))
+          continue;
+        unsigned ExpectedNumOperands = 0;
+        if (BranchInst *BI = dyn_cast<BranchInst>(&I))
+          ExpectedNumOperands = BI->getNumSuccessors();
+        else if (SwitchInst *SI = dyn_cast<SwitchInst>(&I))
+          ExpectedNumOperands = SI->getNumSuccessors();
+        else if (isa<CallInst>(&I))
+          ExpectedNumOperands = 1;
+        else if (IndirectBrInst *IBI = dyn_cast<IndirectBrInst>(&I))
+          ExpectedNumOperands = IBI->getNumDestinations();
+        else if (isa<SelectInst>(&I))
+          ExpectedNumOperands = 2;
+        else
+          continue; // ignore and continue.
+
+        // If branch weight doesn't match, just strip branch weight.
+        if (MD->getNumOperands() != 1 + ExpectedNumOperands)
+          I.setMetadata(LLVMContext::MD_prof, nullptr);
+      }
+    }
+  }
+
   // Look for functions that rely on old function attribute behavior.
   UpgradeFunctionAttributes(*F);
 
diff --git a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
index a8bf579bd1801..6fccb312164b1 100644
--- a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
+++ b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
@@ -1350,7 +1350,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
     break;
   }
   case bitc::METADATA_COMPOSITE_TYPE: {
-    if (Record.size() < 16 || Record.size() > 18)
+    if (Record.size() < 16 || Record.size() > 20)
       return error("Invalid record");
 
     // If we have a UUID and this is not a forward declaration, lookup the
@@ -1375,6 +1375,8 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
     Metadata *TemplateParams = nullptr;
     Metadata *Discriminator = nullptr;
     Metadata *DataLocation = nullptr;
+    Metadata *Associated = nullptr;
+    Metadata *Allocated = nullptr;
     auto *Identifier = getMDString(Record[15]);
     // If this module is being parsed so that it can be ThinLTO imported
     // into another module, composite types only need to be imported
@@ -1399,13 +1401,18 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
         Discriminator = getMDOrNull(Record[16]);
       if (Record.size() > 17)
         DataLocation = getMDOrNull(Record[17]);
+      if (Record.size() > 19) {
+        Associated = getMDOrNull(Record[18]);
+        Allocated = getMDOrNull(Record[19]);
+      }
     }
     DICompositeType *CT = nullptr;
     if (Identifier)
       CT = DICompositeType::buildODRType(
           Context, *Identifier, Tag, Name, File, Line, Scope, BaseType,
           SizeInBits, AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang,
-          VTableHolder, TemplateParams, Discriminator, DataLocation);
+          VTableHolder, TemplateParams, Discriminator, DataLocation, Associated,
+          Allocated);
 
     // Create a node if we didn't get a lazy ODR type.
     if (!CT)
@@ -1413,7 +1420,8 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
                            (Context, Tag, Name, File, Line, Scope, BaseType,
                             SizeInBits, AlignInBits, OffsetInBits, Flags,
                             Elements, RuntimeLang, VTableHolder, TemplateParams,
-                            Identifier, Discriminator, DataLocation));
+                            Identifier, Discriminator, DataLocation, Associated,
+                            Allocated));
     if (!IsNotUsedInTypeRef && Identifier)
       MetadataList.addTypeRef(*Identifier, *cast<DICompositeType>(CT));
 
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 9c15a5f9f193a..7d410b24573e7 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -733,6 +733,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
     return bitc::ATTR_KIND_PREALLOCATED;
   case Attribute::NoUndef:
     return bitc::ATTR_KIND_NOUNDEF;
+  case Attribute::ByRef:
+    return bitc::ATTR_KIND_BYREF;
   case Attribute::EndAttrKinds:
     llvm_unreachable("Can not encode end-attribute kinds marker.");
   case Attribute::None:
@@ -1630,6 +1632,8 @@ void ModuleBitcodeWriter::writeDICompositeType(
   Record.push_back(VE.getMetadataOrNullID(N->getRawIdentifier()));
   Record.push_back(VE.getMetadataOrNullID(N->getDiscriminator()));
   Record.push_back(VE.getMetadataOrNullID(N->getRawDataLocation()));
+  Record.push_back(VE.getMetadataOrNullID(N->getRawAssociated()));
+  Record.push_back(VE.getMetadataOrNullID(N->getRawAllocated()));
 
   Stream.EmitRecord(bitc::METADATA_COMPOSITE_TYPE, Record, Abbrev);
   Record.clear();
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index f8f7b74baf916..b50cae89b3494 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -1390,16 +1390,7 @@ void AsmPrinter::emitGlobalGOTEquivs() {
 void AsmPrinter::emitGlobalIndirectSymbol(Module &M,
                                           const GlobalIndirectSymbol& GIS) {
   MCSymbol *Name = getSymbol(&GIS);
-
-  if (GIS.hasExternalLinkage() || !MAI->getWeakRefDirective())
-    OutStreamer->emitSymbolAttribute(Name, MCSA_Global);
-  else if (GIS.hasWeakLinkage() || GIS.hasLinkOnceLinkage())
-    OutStreamer->emitSymbolAttribute(Name, MCSA_WeakReference);
-  else
-    assert(GIS.hasLocalLinkage() && "Invalid alias or ifunc linkage");
-
   bool IsFunction = GIS.getValueType()->isFunctionTy();
-
   // Treat bitcasts of functions as functions also. This is important at least
   // on WebAssembly where object and function addresses can't alias each other.
   if (!IsFunction)
@@ -1408,6 +1399,30 @@ void AsmPrinter::emitGlobalIndirectSymbol(Module &M,
         IsFunction =
           CE->getOperand(0)->getType()->getPointerElementType()->isFunctionTy();
 
+  // AIX's assembly directive `.set` is not usable for aliasing purpose,
+  // so AIX has to use the extra-label-at-definition strategy. At this
+  // point, all the extra label is emitted, we just have to emit linkage for
+  // those labels.
+  if (TM.getTargetTriple().isOSBinFormatXCOFF()) {
+    assert(!isa<GlobalIFunc>(GIS) && "IFunc is not supported on AIX.");
+    assert(MAI->hasVisibilityOnlyWithLinkage() &&
+           "Visibility should be handled with emitLinkage() on AIX.");
+    emitLinkage(&GIS, Name);
+    // If it's a function, also emit linkage for aliases of function entry
+    // point.
+    if (IsFunction)
+      emitLinkage(&GIS,
+                  getObjFileLowering().getFunctionEntryPointSymbol(&GIS, TM));
+    return;
+  }
+
+  if (GIS.hasExternalLinkage() || !MAI->getWeakRefDirective())
+    OutStreamer->emitSymbolAttribute(Name, MCSA_Global);
+  else if (GIS.hasWeakLinkage() || GIS.hasLinkOnceLinkage())
+    OutStreamer->emitSymbolAttribute(Name, MCSA_WeakReference);
+  else
+    assert(GIS.hasLocalLinkage() && "Invalid alias or ifunc linkage");
+
   // Set the symbol type to function if the alias has a function type.
   // This affects codegen when the aliasee is not a function.
   if (IsFunction)
diff --git a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp
index 3f053c7a38c77..5d52e7fb80920 100644
--- a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp
@@ -13,15 +13,10 @@
 #include "CodeViewDebug.h"
 #include "DwarfExpression.h"
 #include "llvm/ADT/APSInt.h"
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/DenseSet.h"
-#include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/TinyPtrVector.h"
 #include "llvm/ADT/Triple.h"
@@ -40,7 +35,6 @@
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/DebugInfo/CodeView/CVTypeVisitor.h"
-#include "llvm/DebugInfo/CodeView/CodeView.h"
 #include "llvm/DebugInfo/CodeView/CodeViewRecordIO.h"
 #include "llvm/DebugInfo/CodeView/ContinuationRecordBuilder.h"
 #include "llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h"
@@ -48,14 +42,12 @@
 #include "llvm/DebugInfo/CodeView/Line.h"
 #include "llvm/DebugInfo/CodeView/SymbolRecord.h"
 #include "llvm/DebugInfo/CodeView/TypeDumpVisitor.h"
-#include "llvm/DebugInfo/CodeView/TypeIndex.h"
 #include "llvm/DebugInfo/CodeView/TypeRecord.h"
 #include "llvm/DebugInfo/CodeView/TypeTableCollection.h"
 #include "llvm/DebugInfo/CodeView/TypeVisitorCallbackPipeline.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DebugInfoMetadata.h"
-#include "llvm/IR/DebugLoc.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/IR/GlobalVariable.h"
@@ -71,7 +63,6 @@
 #include "llvm/Support/BinaryStreamWriter.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
-#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -85,12 +76,8 @@
 #include <cassert>
 #include <cctype>
 #include <cstddef>
-#include <cstdint>
 #include <iterator>
 #include <limits>
-#include <string>
-#include <utility>
-#include <vector>
 
 using namespace llvm;
 using namespace llvm::codeview;
diff --git a/llvm/lib/CodeGen/AsmPrinter/DbgEntityHistoryCalculator.cpp b/llvm/lib/CodeGen/AsmPrinter/DbgEntityHistoryCalculator.cpp
index 584b7614915d8..ebf871600564c 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DbgEntityHistoryCalculator.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DbgEntityHistoryCalculator.cpp
@@ -8,9 +8,11 @@
 
 #include "llvm/CodeGen/DbgEntityHistoryCalculator.h"
 #include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/Optional.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/LexicalScopes.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstr.h"
@@ -90,6 +92,192 @@ void DbgValueHistoryMap::Entry::endEntry(EntryIndex Index) {
   EndIndex = Index;
 }
 
+using OrderMap = DenseMap<const MachineInstr *, unsigned>;
+/// Number instructions so that we can compare instruction positions within MF.
+/// Meta instructions are given the same nubmer as the preceding instruction.
+/// Because the block ordering will not change it is possible (and safe) to
+/// compare instruction positions between blocks.
+static void numberInstructions(const MachineFunction &MF, OrderMap &Ordering) {
+  // We give meta instructions the same number as the peceding instruction
+  // because this function is written for the task of comparing positions of
+  // variable location ranges against scope ranges. To reflect what we'll see
+  // in the binary, when we look at location ranges we must consider all
+  // DBG_VALUEs between two real instructions at the same position. And a
+  // scope range which ends on a meta instruction should be considered to end
+  // at the last seen real instruction. E.g.
+  //
+  //  1 instruction p      Both the variable location for x and for y start
+  //  1 DBG_VALUE for "x"  after instruction p so we give them all the same
+  //  1 DBG_VALUE for "y"  number. If a scope range ends at DBG_VALUE for "y",
+  //  2 instruction q      we should treat it as ending after instruction p
+  //                       because it will be the last real instruction in the
+  //                       range. DBG_VALUEs at or after this position for
+  //                       variables declared in the scope will have no effect.
+  unsigned position = 0;
+  for (const MachineBasicBlock &MBB : MF)
+    for (const MachineInstr &MI : MBB)
+      Ordering[&MI] = MI.isMetaInstruction() ? position : ++position;
+}
+
+/// Check if instruction A comes before B. Meta instructions have the same
+/// position as the preceding non-meta instruction. See numberInstructions for
+/// more info.
+static bool isBefore(const MachineInstr *A, const MachineInstr *B,
+                     const OrderMap &Ordering) {
+  return Ordering.lookup(A) < Ordering.lookup(B);
+}
+
+/// Check if the instruction range [StartMI, EndMI] intersects any instruction
+/// range in Ranges. EndMI can be nullptr to indicate that the range is
+/// unbounded. Assumes Ranges is ordered and disjoint. Returns true and points
+/// to the first intersecting scope range if one exists.
+static Optional<ArrayRef<InsnRange>::iterator>
+intersects(const MachineInstr *StartMI, const MachineInstr *EndMI,
+           const ArrayRef<InsnRange> &Ranges, const OrderMap &Ordering) {
+  for (auto RangesI = Ranges.begin(), RangesE = Ranges.end();
+       RangesI != RangesE; ++RangesI) {
+    if (EndMI && isBefore(EndMI, RangesI->first, Ordering))
+      return None;
+    if (EndMI && !isBefore(RangesI->second, EndMI, Ordering))
+      return RangesI;
+    if (isBefore(StartMI, RangesI->second, Ordering))
+      return RangesI;
+  }
+  return None;
+}
+
+void DbgValueHistoryMap::trimLocationRanges(const MachineFunction &MF,
+                                            LexicalScopes &LScopes) {
+  OrderMap Ordering;
+  numberInstructions(MF, Ordering);
+
+  // The indices of the entries we're going to remove for each variable.
+  SmallVector<EntryIndex, 4> ToRemove;
+  // Entry reference count for each variable. Clobbers left with no references
+  // will be removed.
+  SmallVector<int, 4> ReferenceCount;
+  // Entries reference other entries by index. Offsets is used to remap these
+  // references if any entries are removed.
+  SmallVector<size_t, 4> Offsets;
+
+  for (auto &Record : VarEntries) {
+    auto &HistoryMapEntries = Record.second;
+    if (HistoryMapEntries.empty())
+      continue;
+
+    InlinedEntity Entity = Record.first;
+    const DILocalVariable *LocalVar = cast<DILocalVariable>(Entity.first);
+
+    LexicalScope *Scope = nullptr;
+    if (const DILocation *InlinedAt = Entity.second) {
+      Scope = LScopes.findInlinedScope(LocalVar->getScope(), InlinedAt);
+    } else {
+      Scope = LScopes.findLexicalScope(LocalVar->getScope());
+      // Ignore variables for non-inlined function level scopes. The scope
+      // ranges (from scope->getRanges()) will not include any instructions
+      // before the first one with a debug-location, which could cause us to
+      // incorrectly drop a location. We could introduce special casing for
+      // these variables, but it doesn't seem worth it because no out-of-scope
+      // locations have been observed for variables declared in function level
+      // scopes.
+      if (Scope &&
+          (Scope->getScopeNode() == Scope->getScopeNode()->getSubprogram()) &&
+          (Scope->getScopeNode() == LocalVar->getScope()))
+        continue;
+    }
+
+    // If there is no scope for the variable then something has probably gone
+    // wrong.
+    if (!Scope)
+      continue;
+
+    ToRemove.clear();
+    // Zero the reference counts.
+    ReferenceCount.assign(HistoryMapEntries.size(), 0);
+    // Index of the DBG_VALUE which marks the start of the current location
+    // range.
+    EntryIndex StartIndex = 0;
+    ArrayRef<InsnRange> ScopeRanges(Scope->getRanges());
+    for (auto EI = HistoryMapEntries.begin(), EE = HistoryMapEntries.end();
+         EI != EE; ++EI, ++StartIndex) {
+      // Only DBG_VALUEs can open location ranges so skip anything else.
+      if (!EI->isDbgValue())
+        continue;
+
+      // Index of the entry which closes this range.
+      EntryIndex EndIndex = EI->getEndIndex();
+      // If this range is closed bump the reference count of the closing entry.
+      if (EndIndex != NoEntry)
+        ReferenceCount[EndIndex] += 1;
+      // Skip this location range if the opening entry is still referenced. It
+      // may close a location range which intersects a scope range.
+      // TODO: We could be 'smarter' and trim these kinds of ranges such that
+      // they do not leak out of the scope ranges if they partially overlap.
+      if (ReferenceCount[StartIndex] > 0)
+        continue;
+
+      const MachineInstr *StartMI = EI->getInstr();
+      const MachineInstr *EndMI = EndIndex != NoEntry
+                                      ? HistoryMapEntries[EndIndex].getInstr()
+                                      : nullptr;
+      // Check if the location range [StartMI, EndMI] intersects with any scope
+      // range for the variable.
+      if (auto R = intersects(StartMI, EndMI, ScopeRanges, Ordering)) {
+        // Adjust ScopeRanges to exclude ranges which subsequent location ranges
+        // cannot possibly intersect.
+        ScopeRanges = ArrayRef<InsnRange>(R.getValue(), ScopeRanges.end());
+      } else {
+        // If the location range does not intersect any scope range then the
+        // DBG_VALUE which opened this location range is usless, mark it for
+        // removal.
+        ToRemove.push_back(StartIndex);
+        // Because we'll be removing this entry we need to update the reference
+        // count of the closing entry, if one exists.
+        if (EndIndex != NoEntry)
+          ReferenceCount[EndIndex] -= 1;
+      }
+    }
+
+    // If there is nothing to remove then jump to next variable.
+    if (ToRemove.empty())
+      continue;
+
+    // Mark clobbers that will no longer close any location ranges for removal.
+    for (size_t i = 0; i < HistoryMapEntries.size(); ++i)
+      if (ReferenceCount[i] <= 0 && HistoryMapEntries[i].isClobber())
+        ToRemove.push_back(i);
+
+    std::sort(ToRemove.begin(), ToRemove.end());
+
+    // Build an offset map so we can update the EndIndex of the remaining
+    // entries.
+    // Zero the offsets.
+    Offsets.assign(HistoryMapEntries.size(), 0);
+    size_t CurOffset = 0;
+    auto ToRemoveItr = ToRemove.begin();
+    for (size_t EntryIdx = *ToRemoveItr; EntryIdx < HistoryMapEntries.size();
+         ++EntryIdx) {
+      // Check if this is an entry which will be removed.
+      if (ToRemoveItr != ToRemove.end() && *ToRemoveItr == EntryIdx) {
+        ++ToRemoveItr;
+        ++CurOffset;
+      }
+      Offsets[EntryIdx] = CurOffset;
+    }
+
+    // Update the EndIndex of the entries to account for those which will be
+    // removed.
+    for (auto &Entry : HistoryMapEntries)
+      if (Entry.isClosed())
+        Entry.EndIndex -= Offsets[Entry.EndIndex];
+
+    // Now actually remove the entries. Iterate backwards so that our remaining
+    // ToRemove indices are valid after each erase.
+    for (auto Itr = ToRemove.rbegin(), End = ToRemove.rend(); Itr != End; ++Itr)
+      HistoryMapEntries.erase(HistoryMapEntries.begin() + *Itr);
+  }
+}
+
 void DbgLabelInstrMap::addInstr(InlinedEntity Label, const MachineInstr &MI) {
   assert(MI.isDebugLabel() && "not a DBG_LABEL");
   LabelInstr[Label] = &MI;
diff --git a/llvm/lib/CodeGen/AsmPrinter/DebugHandlerBase.cpp b/llvm/lib/CodeGen/AsmPrinter/DebugHandlerBase.cpp
index 880791a06d931..a46de83e555cc 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DebugHandlerBase.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DebugHandlerBase.cpp
@@ -21,11 +21,16 @@
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/IR/DebugInfo.h"
 #include "llvm/MC/MCStreamer.h"
+#include "llvm/Support/CommandLine.h"
 
 using namespace llvm;
 
 #define DEBUG_TYPE "dwarfdebug"
 
+/// If true, we drop variable location ranges which exist entirely outside the
+/// variable's lexical scope instruction ranges.
+static cl::opt<bool> TrimVarLocs("trim-var-locs", cl::Hidden, cl::init(true));
+
 Optional<DbgVariableLocation>
 DbgVariableLocation::extractFromMachineInstruction(
     const MachineInstr &Instruction) {
@@ -191,6 +196,8 @@ void DebugHandlerBase::beginFunction(const MachineFunction *MF) {
   assert(DbgLabels.empty() && "DbgLabels map wasn't cleaned!");
   calculateDbgEntityHistory(MF, Asm->MF->getSubtarget().getRegisterInfo(),
                             DbgValues, DbgLabels);
+  if (TrimVarLocs)
+    DbgValues.trimLocationRanges(*MF, LScopes);
   LLVM_DEBUG(DbgValues.dump());
 
   // Request labels for the full history.
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
index d90e49c1c2511..ece6665e99f6c 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
@@ -12,18 +12,12 @@
 
 #include "DwarfCompileUnit.h"
 #include "AddressPool.h"
-#include "DwarfDebug.h"
 #include "DwarfExpression.h"
-#include "DwarfUnit.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/CodeGen/AsmPrinter.h"
 #include "llvm/CodeGen/DIE.h"
-#include "llvm/CodeGen/LexicalScopes.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineOperand.h"
@@ -32,22 +26,16 @@
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DebugInfo.h"
-#include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCSymbolWasm.h"
 #include "llvm/MC/MachineLocation.h"
-#include "llvm/Support/Casting.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
-#include <algorithm>
-#include <cassert>
-#include <cstdint>
 #include <iterator>
-#include <memory>
 #include <string>
 #include <utility>
 
@@ -117,7 +105,7 @@ unsigned DwarfCompileUnit::getOrCreateSourceID(const DIFile *File) {
     return Asm->OutStreamer->emitDwarfFileDirective(0, "", "", None, None,
                                                     CUID);
   return Asm->OutStreamer->emitDwarfFileDirective(
-      0, File->getDirectory(), File->getFilename(), getMD5AsBytes(File),
+      0, File->getDirectory(), File->getFilename(), DD->getMD5AsBytes(File),
       File->getSource(), CUID);
 }
 
@@ -803,6 +791,10 @@ static SmallVector<const DIVariable *, 2> dependencies(DbgVariable *Var) {
     return Result;
   if (auto *DLVar = Array->getDataLocation())
     Result.push_back(DLVar);
+  if (auto *AsVar = Array->getAssociated())
+    Result.push_back(AsVar);
+  if (auto *AlVar = Array->getAllocated())
+    Result.push_back(AlVar);
   for (auto *El : Array->getElements()) {
     if (auto *Subrange = dyn_cast<DISubrange>(El)) {
       if (auto Count = Subrange->getCount())
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
index 4ccd8c96dd0df..85aee2307e067 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
@@ -22,7 +22,6 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/CodeGen/DbgEntityHistoryCalculator.h"
-#include "llvm/CodeGen/DIE.h"
 #include "llvm/CodeGen/LexicalScopes.h"
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/Support/Casting.h"
@@ -34,6 +33,9 @@
 namespace llvm {
 
 class AsmPrinter;
+class DIE;
+class DIELoc;
+class DIEValueList;
 class DwarfFile;
 class GlobalVariable;
 class MCExpr;
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index 45ed5256deb99..5a4c4dfb90a2a 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -95,10 +95,6 @@ static cl::opt<bool> UseDwarfRangesBaseAddressSpecifier(
     "use-dwarf-ranges-base-address-specifier", cl::Hidden,
     cl::desc("Use base address specifiers in debug_ranges"), cl::init(false));
 
-static cl::opt<bool> EmitDwarfDebugEntryValues(
-    "emit-debug-entry-values", cl::Hidden,
-    cl::desc("Emit the debug entry values"), cl::init(false));
-
 static cl::opt<bool> GenerateARangeSection("generate-arange-section",
                                            cl::Hidden,
                                            cl::desc("Generate dwarf aranges"),
@@ -430,9 +426,7 @@ DwarfDebug::DwarfDebug(AsmPrinter *A, Module *M)
 
   // Emit call-site-param debug info for GDB and LLDB, if the target supports
   // the debug entry values feature. It can also be enabled explicitly.
-  EmitDebugEntryValues = (Asm->TM.Options.ShouldEmitDebugEntryValues() &&
-                          (tuneForGDB() || tuneForLLDB())) ||
-                         EmitDwarfDebugEntryValues;
+  EmitDebugEntryValues = Asm->TM.Options.ShouldEmitDebugEntryValues();
 
   Asm->OutStreamer->getContext().setDwarfVersion(DwarfVersion);
 }
@@ -1071,9 +1065,8 @@ DwarfDebug::getOrCreateDwarfCompileUnit(const DICompileUnit *DIUnit) {
   // compilation directory.
   if (!Asm->OutStreamer->hasRawTextSupport() || SingleCU)
     Asm->OutStreamer->emitDwarfFile0Directive(
-        CompilationDir, DIUnit->getFilename(),
-        NewCU.getMD5AsBytes(DIUnit->getFile()), DIUnit->getSource(),
-        NewCU.getUniqueID());
+        CompilationDir, DIUnit->getFilename(), getMD5AsBytes(DIUnit->getFile()),
+        DIUnit->getSource(), NewCU.getUniqueID());
 
   if (useSplitDwarf()) {
     NewCU.setSkeleton(constructSkeletonCU(NewCU));
@@ -2495,6 +2488,7 @@ void DwarfDebug::emitDebugLocValue(const AsmPrinter &AP, const DIBasicType *BT,
       DwarfExpr.addSignedConstant(Value.getInt());
     else
       DwarfExpr.addUnsignedConstant(Value.getInt());
+    DwarfExpr.addExpression(std::move(ExprCursor));
   } else if (Value.isLocation()) {
     MachineLocation Location = Value.getLoc();
     DwarfExpr.setLocation(Location, DIExpr);
@@ -2512,11 +2506,27 @@ void DwarfDebug::emitDebugLocValue(const AsmPrinter &AP, const DIBasicType *BT,
     // TODO TargetIndexLocation is a target-independent. Currently only the WebAssembly-specific
     // encoding is supported.
     DwarfExpr.addWasmLocation(Loc.Index, static_cast<uint64_t>(Loc.Offset));
+      DwarfExpr.addExpression(std::move(ExprCursor));
+      return;
   } else if (Value.isConstantFP()) {
-    APInt RawBytes = Value.getConstantFP()->getValueAPF().bitcastToAPInt();
-    DwarfExpr.addUnsignedConstant(RawBytes);
+    if (AP.getDwarfVersion() >= 4 && AP.getDwarfDebug()->tuneForGDB()) {
+      DwarfExpr.addConstantFP(Value.getConstantFP()->getValueAPF());
+      return;
+    } else if (Value.getConstantFP()
+                   ->getValueAPF()
+                   .bitcastToAPInt()
+                   .getBitWidth() <= 64 /*bits*/) {
+      DwarfExpr.addUnsignedConstant(
+          Value.getConstantFP()->getValueAPF().bitcastToAPInt());
+      DwarfExpr.addExpression(std::move(ExprCursor));
+      return;
+    }
+    LLVM_DEBUG(
+        dbgs()
+        << "Skipped DwarfExpression creation for ConstantFP of size: "
+        << Value.getConstantFP()->getValueAPF().bitcastToAPInt().getBitWidth()
+        << " bits\n");
   }
-  DwarfExpr.addExpression(std::move(ExprCursor));
 }
 
 void DebugLocEntry::finalize(const AsmPrinter &AP,
@@ -3001,7 +3011,10 @@ static void emitMacroHeader(AsmPrinter *Asm, const DwarfDebug &DD,
   Asm->OutStreamer->AddComment("Flags: 32 bit, debug_line_offset present");
   Asm->emitInt8(Flags);
   Asm->OutStreamer->AddComment("debug_line_offset");
-  Asm->OutStreamer->emitSymbolValue(CU.getLineTableStartSym(), /*Size=*/4);
+  if (DD.useSplitDwarf())
+    Asm->OutStreamer->emitIntValue(0, /*Size=*/4);
+  else
+    Asm->OutStreamer->emitSymbolValue(CU.getLineTableStartSym(), /*Size=*/4);
 }
 
 void DwarfDebug::handleMacroNodes(DIMacroNodeArray Nodes, DwarfCompileUnit &U) {
@@ -3057,16 +3070,22 @@ void DwarfDebug::emitMacro(DIMacro &M) {
 }
 
 void DwarfDebug::emitMacroFileImpl(
-    DIMacroFile &F, DwarfCompileUnit &U, unsigned StartFile, unsigned EndFile,
+    DIMacroFile &MF, DwarfCompileUnit &U, unsigned StartFile, unsigned EndFile,
     StringRef (*MacroFormToString)(unsigned Form)) {
 
   Asm->OutStreamer->AddComment(MacroFormToString(StartFile));
   Asm->emitULEB128(StartFile);
   Asm->OutStreamer->AddComment("Line Number");
-  Asm->emitULEB128(F.getLine());
+  Asm->emitULEB128(MF.getLine());
   Asm->OutStreamer->AddComment("File Number");
-  Asm->emitULEB128(U.getOrCreateSourceID(F.getFile()));
-  handleMacroNodes(F.getElements(), U);
+  DIFile &F = *MF.getFile();
+  if (useSplitDwarf())
+    Asm->emitULEB128(getDwoLineTable(U)->getFile(
+        F.getDirectory(), F.getFilename(), getMD5AsBytes(&F),
+        Asm->OutContext.getDwarfVersion(), F.getSource()));
+  else
+    Asm->emitULEB128(U.getOrCreateSourceID(&F));
+  handleMacroNodes(MF.getElements(), U);
   Asm->OutStreamer->AddComment(MacroFormToString(EndFile));
   Asm->emitULEB128(EndFile);
 }
@@ -3200,7 +3219,7 @@ MCDwarfDwoLineTable *DwarfDebug::getDwoLineTable(const DwarfCompileUnit &CU) {
   const DICompileUnit *DIUnit = CU.getCUNode();
   SplitTypeUnitFileTable.maybeSetRootFile(
       DIUnit->getDirectory(), DIUnit->getFilename(),
-      CU.getMD5AsBytes(DIUnit->getFile()), DIUnit->getSource());
+      getMD5AsBytes(DIUnit->getFile()), DIUnit->getSource());
   return &SplitTypeUnitFileTable;
 }
 
@@ -3383,3 +3402,20 @@ void DwarfDebug::insertSectionLabel(const MCSymbol *S) {
     if (useSplitDwarf() || getDwarfVersion() >= 5)
       AddrPool.getIndex(S);
 }
+
+Optional<MD5::MD5Result> DwarfDebug::getMD5AsBytes(const DIFile *File) const {
+  assert(File);
+  if (getDwarfVersion() < 5)
+    return None;
+  Optional<DIFile::ChecksumInfo<StringRef>> Checksum = File->getChecksum();
+  if (!Checksum || Checksum->Kind != DIFile::CSK_MD5)
+    return None;
+
+  // Convert the string checksum to an MD5Result for the streamer.
+  // The verifier validates the checksum so we assume it's okay.
+  // An MD5 checksum is 16 bytes.
+  std::string ChecksumString = fromHex(Checksum->Value);
+  MD5::MD5Result CKMem;
+  std::copy(ChecksumString.begin(), ChecksumString.end(), CKMem.Bytes.data());
+  return CKMem;
+}
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
index ad2f2f3edd8e6..2587ee61b0599 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
@@ -784,6 +784,10 @@ class DwarfDebug : public DebugHandlerBase {
   static void emitDebugLocValue(const AsmPrinter &AP, const DIBasicType *BT,
                                 const DbgValueLoc &Value,
                                 DwarfExpression &DwarfExpr);
+
+  /// If the \p File has an MD5 checksum, return it as an MD5Result
+  /// allocated in the MCContext.
+  Optional<MD5::MD5Result> getMD5AsBytes(const DIFile *File) const;
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
index d4762121d1050..7e6f4a0425d87 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
@@ -25,6 +25,8 @@
 
 using namespace llvm;
 
+#define DEBUG_TYPE "dwarfdebug"
+
 void DwarfExpression::emitConstu(uint64_t Value) {
   if (Value < 32)
     emitOp(dwarf::DW_OP_lit0 + Value);
@@ -219,6 +221,36 @@ void DwarfExpression::addUnsignedConstant(const APInt &Value) {
   }
 }
 
+void DwarfExpression::addConstantFP(const APFloat &Value) {
+  assert(isImplicitLocation() || isUnknownLocation());
+  APInt RawBytes = Value.bitcastToAPInt();
+  int NumBytes = RawBytes.getBitWidth() / 8;
+  const char *Data = (const char *)RawBytes.getRawData();
+  emitOp(dwarf::DW_OP_implicit_value);
+  if (NumBytes == 4 /*float*/ || NumBytes == 8 /*double*/) {
+    emitUnsigned(NumBytes /*Size of the block in bytes*/);
+    for (int i = 0; i < NumBytes; ++i)
+      emitData1(Data[i]);
+    return;
+  }
+  if (NumBytes == 10 /*long double*/) {
+    // long double IEEE representation uses 80 bits(10 bytes).
+    // 6 bytes are padded to make it 128 bits(16 bytes) due to
+    // addressing restrictions.
+    emitUnsigned(16 /*Size of the block in bytes*/);
+    // Emit the block of bytes.
+    for (int i = 0; i < NumBytes; ++i)
+      emitData1(Data[i]);
+    // Emit the rest as padding bytes.
+    for (int i = 0; i < 16 - NumBytes; ++i)
+      emitData1(0);
+    return;
+  }
+  LLVM_DEBUG(
+      dbgs() << "Skipped DW_OP_implicit_value creation for ConstantFP of size: "
+             << RawBytes.getBitWidth() << " bits\n");
+}
+
 bool DwarfExpression::addMachineRegExpression(const TargetRegisterInfo &TRI,
                                               DIExpressionCursor &ExprCursor,
                                               unsigned MachineReg,
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h
index 757b175114535..097208f1cfade 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h
@@ -299,6 +299,9 @@ class DwarfExpression {
   /// Emit an unsigned constant.
   void addUnsignedConstant(const APInt &Value);
 
+  /// Emit floating point constant.
+  void addConstantFP(const APFloat &Value);
+
   /// Lock this down to become a memory location description.
   void setMemoryLocationKind() {
     assert(isUnknownLocation());
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
index e958f38e486b0..1ba2afe48bf4b 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
@@ -307,23 +307,6 @@ void DwarfUnit::addSectionOffset(DIE &Die, dwarf::Attribute Attribute,
     addUInt(Die, Attribute, dwarf::DW_FORM_data4, Integer);
 }
 
-Optional<MD5::MD5Result> DwarfUnit::getMD5AsBytes(const DIFile *File) const {
-  assert(File);
-  if (DD->getDwarfVersion() < 5)
-    return None;
-  Optional<DIFile::ChecksumInfo<StringRef>> Checksum = File->getChecksum();
-  if (!Checksum || Checksum->Kind != DIFile::CSK_MD5)
-    return None;
-
-  // Convert the string checksum to an MD5Result for the streamer.
-  // The verifier validates the checksum so we assume it's okay.
-  // An MD5 checksum is 16 bytes.
-  std::string ChecksumString = fromHex(Checksum->Value);
-  MD5::MD5Result CKMem;
-  std::copy(ChecksumString.begin(), ChecksumString.end(), CKMem.Bytes.data());
-  return CKMem;
-}
-
 unsigned DwarfTypeUnit::getOrCreateSourceID(const DIFile *File) {
   if (!SplitLineTable)
     return getCU().getOrCreateSourceID(File);
@@ -332,10 +315,9 @@ unsigned DwarfTypeUnit::getOrCreateSourceID(const DIFile *File) {
     // This is a split type unit that needs a line table.
     addSectionOffset(getUnitDie(), dwarf::DW_AT_stmt_list, 0);
   }
-  return SplitLineTable->getFile(File->getDirectory(), File->getFilename(),
-                                 getMD5AsBytes(File),
-                                 Asm->OutContext.getDwarfVersion(),
-                                 File->getSource());
+  return SplitLineTable->getFile(
+      File->getDirectory(), File->getFilename(), DD->getMD5AsBytes(File),
+      Asm->OutContext.getDwarfVersion(), File->getSource());
 }
 
 void DwarfUnit::addOpAddress(DIELoc &Die, const MCSymbol *Sym) {
@@ -1445,6 +1427,28 @@ void DwarfUnit::constructArrayTypeDIE(DIE &Buffer, const DICompositeType *CTy) {
     addBlock(Buffer, dwarf::DW_AT_data_location, DwarfExpr.finalize());
   }
 
+  if (DIVariable *Var = CTy->getAssociated()) {
+    if (auto *VarDIE = getDIE(Var))
+      addDIEEntry(Buffer, dwarf::DW_AT_associated, *VarDIE);
+  } else if (DIExpression *Expr = CTy->getAssociatedExp()) {
+    DIELoc *Loc = new (DIEValueAllocator) DIELoc;
+    DIEDwarfExpression DwarfExpr(*Asm, getCU(), *Loc);
+    DwarfExpr.setMemoryLocationKind();
+    DwarfExpr.addExpression(Expr);
+    addBlock(Buffer, dwarf::DW_AT_associated, DwarfExpr.finalize());
+  }
+
+  if (DIVariable *Var = CTy->getAllocated()) {
+    if (auto *VarDIE = getDIE(Var))
+      addDIEEntry(Buffer, dwarf::DW_AT_allocated, *VarDIE);
+  } else if (DIExpression *Expr = CTy->getAllocatedExp()) {
+    DIELoc *Loc = new (DIEValueAllocator) DIELoc;
+    DIEDwarfExpression DwarfExpr(*Asm, getCU(), *Loc);
+    DwarfExpr.setMemoryLocationKind();
+    DwarfExpr.addExpression(Expr);
+    addBlock(Buffer, dwarf::DW_AT_allocated, DwarfExpr.finalize());
+  }
+
   // Emit the element type.
   addType(Buffer, CTy->getBaseType());
 
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h
index 34f3a34ed3362..8af0cdc538cda 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h
@@ -284,10 +284,6 @@ class DwarfUnit : public DIEUnit {
                                       const MCSymbol *Label,
                                       const MCSymbol *Sec);
 
-  /// If the \p File has an MD5 checksum, return it as an MD5Result
-  /// allocated in the MCContext.
-  Optional<MD5::MD5Result> getMD5AsBytes(const DIFile *File) const;
-
   /// Get context owner's DIE.
   DIE *createTypeDIE(const DICompositeType *Ty);
 
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 465ba08dbfcdb..42cffafbb1ce9 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -439,7 +439,11 @@ char CodeGenPrepare::ID = 0;
 
 INITIALIZE_PASS_BEGIN(CodeGenPrepare, DEBUG_TYPE,
                       "Optimize for code generation", false, false)
+INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
+INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
 INITIALIZE_PASS_END(CodeGenPrepare, DEBUG_TYPE,
                     "Optimize for code generation", false, false)
 
diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
index a7146515c4c96..867a00693b9aa 100644
--- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
@@ -313,80 +313,87 @@ bool CallLowering::handleAssignments(CCState &CCInfo,
     EVT VAVT = VA.getValVT();
     const LLT OrigTy = getLLTForType(*Args[i].Ty, DL);
 
-    if (VA.isRegLoc()) {
-      if (Handler.isIncomingArgumentHandler() && VAVT != OrigVT) {
-        if (VAVT.getSizeInBits() < OrigVT.getSizeInBits()) {
-          // Expected to be multiple regs for a single incoming arg.
-          unsigned NumArgRegs = Args[i].Regs.size();
-          if (NumArgRegs < 2)
-            return false;
-
-          assert((j + (NumArgRegs - 1)) < ArgLocs.size() &&
-                 "Too many regs for number of args");
-          for (unsigned Part = 0; Part < NumArgRegs; ++Part) {
-            // There should be Regs.size() ArgLocs per argument.
-            VA = ArgLocs[j + Part];
-            Handler.assignValueToReg(Args[i].Regs[Part], VA.getLocReg(), VA);
-          }
-          j += NumArgRegs - 1;
-          // Merge the split registers into the expected larger result vreg
-          // of the original call.
-          MIRBuilder.buildMerge(Args[i].OrigRegs[0], Args[i].Regs);
-          continue;
-        }
-        const LLT VATy(VAVT.getSimpleVT());
-        Register NewReg =
-            MIRBuilder.getMRI()->createGenericVirtualRegister(VATy);
-        Handler.assignValueToReg(NewReg, VA.getLocReg(), VA);
-        // If it's a vector type, we either need to truncate the elements
-        // or do an unmerge to get the lower block of elements.
-        if (VATy.isVector() &&
-            VATy.getNumElements() > OrigVT.getVectorNumElements()) {
-          // Just handle the case where the VA type is 2 * original type.
-          if (VATy.getNumElements() != OrigVT.getVectorNumElements() * 2) {
-            LLVM_DEBUG(dbgs()
-                       << "Incoming promoted vector arg has too many elts");
-            return false;
-          }
-          auto Unmerge = MIRBuilder.buildUnmerge({OrigTy, OrigTy}, {NewReg});
-          MIRBuilder.buildCopy(ArgReg, Unmerge.getReg(0));
-        } else {
-          MIRBuilder.buildTrunc(ArgReg, {NewReg}).getReg(0);
+    // Expected to be multiple regs for a single incoming arg.
+    // There should be Regs.size() ArgLocs per argument.
+    unsigned NumArgRegs = Args[i].Regs.size();
+
+    assert((j + (NumArgRegs - 1)) < ArgLocs.size() &&
+           "Too many regs for number of args");
+    for (unsigned Part = 0; Part < NumArgRegs; ++Part) {
+      // There should be Regs.size() ArgLocs per argument.
+      VA = ArgLocs[j + Part];
+      if (VA.isMemLoc()) {
+        // Don't currently support loading/storing a type that needs to be split
+        // to the stack. Should be easy, just not implemented yet.
+        if (NumArgRegs > 1) {
+          LLVM_DEBUG(
+            dbgs()
+            << "Load/store a split arg to/from the stack not implemented yet\n");
+          return false;
         }
-      } else if (!Handler.isIncomingArgumentHandler()) {
-        assert((j + (Args[i].Regs.size() - 1)) < ArgLocs.size() &&
-               "Too many regs for number of args");
-        // This is an outgoing argument that might have been split.
-        for (unsigned Part = 0; Part < Args[i].Regs.size(); ++Part) {
-          // There should be Regs.size() ArgLocs per argument.
-          VA = ArgLocs[j + Part];
-          Handler.assignValueToReg(Args[i].Regs[Part], VA.getLocReg(), VA);
+
+        // FIXME: Use correct address space for pointer size
+        EVT LocVT = VA.getValVT();
+        unsigned MemSize = LocVT == MVT::iPTR ? DL.getPointerSize()
+                                              : LocVT.getStoreSize();
+        unsigned Offset = VA.getLocMemOffset();
+        MachinePointerInfo MPO;
+        Register StackAddr = Handler.getStackAddress(MemSize, Offset, MPO);
+        Handler.assignValueToAddress(Args[i], StackAddr,
+                                     MemSize, MPO, VA);
+        continue;
+      }
+
+      assert(VA.isRegLoc() && "custom loc should have been handled already");
+
+      if (OrigVT.getSizeInBits() >= VAVT.getSizeInBits() ||
+          !Handler.isIncomingArgumentHandler()) {
+        // This is an argument that might have been split. There should be
+        // Regs.size() ArgLocs per argument.
+
+        // Insert the argument copies. If VAVT < OrigVT, we'll insert the merge
+        // to the original register after handling all of the parts.
+        Handler.assignValueToReg(Args[i].Regs[Part], VA.getLocReg(), VA);
+        continue;
+      }
+
+      // This ArgLoc covers multiple pieces, so we need to split it.
+      const LLT VATy(VAVT.getSimpleVT());
+      Register NewReg =
+        MIRBuilder.getMRI()->createGenericVirtualRegister(VATy);
+      Handler.assignValueToReg(NewReg, VA.getLocReg(), VA);
+      // If it's a vector type, we either need to truncate the elements
+      // or do an unmerge to get the lower block of elements.
+      if (VATy.isVector() &&
+          VATy.getNumElements() > OrigVT.getVectorNumElements()) {
+        // Just handle the case where the VA type is 2 * original type.
+        if (VATy.getNumElements() != OrigVT.getVectorNumElements() * 2) {
+          LLVM_DEBUG(dbgs()
+                     << "Incoming promoted vector arg has too many elts");
+          return false;
         }
-        j += Args[i].Regs.size() - 1;
+        auto Unmerge = MIRBuilder.buildUnmerge({OrigTy, OrigTy}, {NewReg});
+        MIRBuilder.buildCopy(ArgReg, Unmerge.getReg(0));
       } else {
-        Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA);
+        MIRBuilder.buildTrunc(ArgReg, {NewReg}).getReg(0);
       }
-    } else if (VA.isMemLoc()) {
-      // Don't currently support loading/storing a type that needs to be split
-      // to the stack. Should be easy, just not implemented yet.
-      if (Args[i].Regs.size() > 1) {
-        LLVM_DEBUG(
-            dbgs()
-            << "Load/store a split arg to/from the stack not implemented yet");
-        return false;
+    }
+
+    // Now that all pieces have been handled, re-pack any arguments into any
+    // wider, original registers.
+    if (Handler.isIncomingArgumentHandler()) {
+      if (VAVT.getSizeInBits() < OrigVT.getSizeInBits()) {
+        assert(NumArgRegs >= 2);
+
+        // Merge the split registers into the expected larger result vreg
+        // of the original call.
+        MIRBuilder.buildMerge(Args[i].OrigRegs[0], Args[i].Regs);
       }
-      MVT VT = MVT::getVT(Args[i].Ty);
-      unsigned Size = VT == MVT::iPTR ? DL.getPointerSize()
-                                      : alignTo(VT.getSizeInBits(), 8) / 8;
-      unsigned Offset = VA.getLocMemOffset();
-      MachinePointerInfo MPO;
-      Register StackAddr = Handler.getStackAddress(Size, Offset, MPO);
-      Handler.assignValueToAddress(Args[i], StackAddr, Size, MPO, VA);
-    } else {
-      // FIXME: Support byvals and other weirdness
-      return false;
     }
+
+    j += NumArgRegs - 1;
   }
+
   return true;
 }
 
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 194961ae3b216..422b71b28b6fe 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -576,20 +576,36 @@ bool CombinerHelper::dominates(const MachineInstr &DefMI,
   return isPredecessor(DefMI, UseMI);
 }
 
-bool CombinerHelper::matchSextAlreadyExtended(MachineInstr &MI) {
+bool CombinerHelper::matchSextTruncSextLoad(MachineInstr &MI) {
   assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
   Register SrcReg = MI.getOperand(1).getReg();
-  unsigned SrcSignBits = KB->computeNumSignBits(SrcReg);
-  unsigned NumSextBits =
-      MRI.getType(MI.getOperand(0).getReg()).getScalarSizeInBits() -
-      MI.getOperand(2).getImm();
-  return SrcSignBits >= NumSextBits;
+  Register LoadUser = SrcReg;
+
+  if (MRI.getType(SrcReg).isVector())
+    return false;
+
+  Register TruncSrc;
+  if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc))))
+    LoadUser = TruncSrc;
+
+  uint64_t SizeInBits = MI.getOperand(2).getImm();
+  // If the source is a G_SEXTLOAD from the same bit width, then we don't
+  // need any extend at all, just a truncate.
+  if (auto *LoadMI = getOpcodeDef(TargetOpcode::G_SEXTLOAD, LoadUser, MRI)) {
+    const auto &MMO = **LoadMI->memoperands_begin();
+    // If truncating more than the original extended value, abort.
+    if (TruncSrc && MRI.getType(TruncSrc).getSizeInBits() < MMO.getSizeInBits())
+      return false;
+    if (MMO.getSizeInBits() == SizeInBits)
+      return true;
+  }
+  return false;
 }
 
-bool CombinerHelper::applySextAlreadyExtended(MachineInstr &MI) {
+bool CombinerHelper::applySextTruncSextLoad(MachineInstr &MI) {
   assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
-  MachineIRBuilder MIB(MI);
-  MIB.buildCopy(MI.getOperand(0).getReg(), MI.getOperand(1).getReg());
+  Builder.setInstrAndDebugLoc(MI);
+  Builder.buildCopy(MI.getOperand(0).getReg(), MI.getOperand(1).getReg());
   MI.eraseFromParent();
   return true;
 }
diff --git a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
index 0e9c6e4fab9f9..e5d77b0eb8578 100644
--- a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
@@ -11,7 +11,6 @@
 //
 //===------------------
 #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
-#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/CodeGen/GlobalISel/Utils.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
@@ -442,16 +441,6 @@ unsigned GISelKnownBits::computeNumSignBits(Register R,
     unsigned Tmp = DstTy.getScalarSizeInBits() - SrcTy.getScalarSizeInBits();
     return computeNumSignBits(Src, DemandedElts, Depth + 1) + Tmp;
   }
-  case TargetOpcode::G_SEXTLOAD: {
-    Register Dst = MI.getOperand(0).getReg();
-    LLT Ty = MRI.getType(Dst);
-    // TODO: add vector support
-    if (Ty.isVector())
-      break;
-    if (MI.hasOneMemOperand())
-      return Ty.getSizeInBits() - (*MI.memoperands_begin())->getSizeInBits();
-    break;
-  }
   case TargetOpcode::G_TRUNC: {
     Register Src = MI.getOperand(1).getReg();
     LLT SrcTy = MRI.getType(Src);
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 8f6643b2f1935..aa898d5a61896 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1264,6 +1264,8 @@ unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
       return TargetOpcode::G_FNEARBYINT;
     case Intrinsic::pow:
       return TargetOpcode::G_FPOW;
+    case Intrinsic::powi:
+      return TargetOpcode::G_FPOWI;
     case Intrinsic::rint:
       return TargetOpcode::G_FRINT;
     case Intrinsic::round:
@@ -1484,6 +1486,14 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
     return translateBinaryOp(TargetOpcode::G_USUBSAT, CI, MIRBuilder);
   case Intrinsic::ssub_sat:
     return translateBinaryOp(TargetOpcode::G_SSUBSAT, CI, MIRBuilder);
+  case Intrinsic::umin:
+    return translateBinaryOp(TargetOpcode::G_UMIN, CI, MIRBuilder);
+  case Intrinsic::umax:
+    return translateBinaryOp(TargetOpcode::G_UMAX, CI, MIRBuilder);
+  case Intrinsic::smin:
+    return translateBinaryOp(TargetOpcode::G_SMIN, CI, MIRBuilder);
+  case Intrinsic::smax:
+    return translateBinaryOp(TargetOpcode::G_SMAX, CI, MIRBuilder);
   case Intrinsic::fmuladd: {
     const TargetMachine &TM = MF->getTarget();
     const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
diff --git a/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp b/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
index 2ce1d414e7550..502f3cb85726c 100644
--- a/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
@@ -558,6 +558,11 @@ bool InlineAsmLowering::lowerInlineAsm(
       }
 
       unsigned Flag = InlineAsm::getFlagWord(InlineAsm::Kind_RegUse, NumRegs);
+      if (OpInfo.Regs.front().isVirtual()) {
+        // Put the register class of the virtual registers in the flag word.
+        const TargetRegisterClass *RC = MRI->getRegClass(OpInfo.Regs.front());
+        Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
+      }
       Inst.addImm(Flag);
       if (!buildAnyextOrCopy(OpInfo.Regs[0], SourceRegs[0], MIRBuilder))
         return false;
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizeMutations.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizeMutations.cpp
index fcbecf90a8455..f3ba3f0801988 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizeMutations.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizeMutations.cpp
@@ -43,6 +43,16 @@ LegalizeMutation LegalizeMutations::changeElementTo(unsigned TypeIdx,
   };
 }
 
+LegalizeMutation LegalizeMutations::changeElementSizeTo(unsigned TypeIdx,
+                                                        unsigned FromTypeIdx) {
+  return [=](const LegalityQuery &Query) {
+    const LLT OldTy = Query.Types[TypeIdx];
+    const LLT NewTy = Query.Types[FromTypeIdx];
+    const LLT NewEltTy = LLT::scalar(NewTy.getScalarSizeInBits());
+    return std::make_pair(TypeIdx, OldTy.changeElementType(NewEltTy));
+  };
+}
+
 LegalizeMutation LegalizeMutations::widenScalarOrEltToNextPow2(unsigned TypeIdx,
                                                                unsigned Min) {
   return [=](const LegalityQuery &Query) {
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 5b6937e471cdc..5dcb5b3271d8a 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -252,7 +252,7 @@ LLT LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts, LLT DstTy,
                                     LLT NarrowTy, Register SrcReg) {
   LLT SrcTy = MRI.getType(SrcReg);
 
-  LLT GCDTy = getGCDType(DstTy, getGCDType(SrcTy, NarrowTy));
+  LLT GCDTy = getGCDType(getGCDType(SrcTy, NarrowTy), DstTy);
   if (SrcTy == GCDTy) {
     // If the source already evenly divides the result type, we don't need to do
     // anything.
@@ -459,7 +459,8 @@ static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
 
 /// True if an instruction is in tail position in its caller. Intended for
 /// legalizing libcalls as tail calls when possible.
-static bool isLibCallInTailPosition(MachineInstr &MI) {
+static bool isLibCallInTailPosition(const TargetInstrInfo &TII,
+                                    MachineInstr &MI) {
   MachineBasicBlock &MBB = *MI.getParent();
   const Function &F = MBB.getParent()->getFunction();
 
@@ -479,7 +480,6 @@ static bool isLibCallInTailPosition(MachineInstr &MI) {
     return false;
 
   // Only tail call if the following instruction is a standard return.
-  auto &TII = *MI.getMF()->getSubtarget().getInstrInfo();
   auto Next = next_nodbg(MI.getIterator(), MBB.instr_end());
   if (Next == MBB.instr_end() || TII.isTailCall(*Next) || !Next->isReturn())
     return false;
@@ -575,7 +575,7 @@ llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
   Info.Callee = MachineOperand::CreateES(Name);
   Info.OrigRet = CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx));
   Info.IsTailCall = MI.getOperand(MI.getNumOperands() - 1).getImm() == 1 &&
-                    isLibCallInTailPosition(MI);
+                    isLibCallInTailPosition(MIRBuilder.getTII(), MI);
 
   std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
   if (!CLI.lowerCall(MIRBuilder, Info))
@@ -1212,6 +1212,29 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
     Observer.changedInstr(MI);
     return Legalized;
   }
+  case TargetOpcode::G_FPTOUI: {
+    if (TypeIdx != 0)
+      return UnableToLegalize;
+    Observer.changingInstr(MI);
+    narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
+    Observer.changedInstr(MI);
+    return Legalized;
+  }
+  case TargetOpcode::G_FPTOSI: {
+    if (TypeIdx != 0)
+      return UnableToLegalize;
+    Observer.changingInstr(MI);
+    narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_SEXT);
+    Observer.changedInstr(MI);
+    return Legalized;
+  }
+  case TargetOpcode::G_FPEXT:
+    if (TypeIdx != 0)
+      return UnableToLegalize;
+    Observer.changingInstr(MI);
+    narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_FPEXT);
+    Observer.changedInstr(MI);
+    return Legalized;
   }
 }
 
@@ -1629,7 +1652,7 @@ LegalizerHelper::widenScalarExtract(MachineInstr &MI, unsigned TypeIdx,
 LegalizerHelper::LegalizeResult
 LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx,
                                    LLT WideTy) {
-  if (TypeIdx != 0)
+  if (TypeIdx != 0 || WideTy.isVector())
     return UnableToLegalize;
   Observer.changingInstr(MI);
   widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
@@ -2147,6 +2170,15 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
     widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
     Observer.changedInstr(MI);
     return Legalized;
+  case TargetOpcode::G_FPOWI: {
+    if (TypeIdx != 0)
+      return UnableToLegalize;
+    Observer.changingInstr(MI);
+    widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT);
+    widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
+    Observer.changedInstr(MI);
+    return Legalized;
+  }
   case TargetOpcode::G_INTTOPTR:
     if (TypeIdx != 1)
       return UnableToLegalize;
@@ -2173,8 +2205,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
     // Avoid changing the result vector type if the source element type was
     // requested.
     if (TypeIdx == 1) {
-      auto &TII = *MI.getMF()->getSubtarget().getInstrInfo();
-      MI.setDesc(TII.get(TargetOpcode::G_BUILD_VECTOR_TRUNC));
+      MI.setDesc(MIRBuilder.getTII().get(TargetOpcode::G_BUILD_VECTOR_TRUNC));
     } else {
       widenScalarDst(MI, WideTy, 0);
     }
@@ -2651,6 +2682,8 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
     return lowerFPTOSI(MI);
   case G_FPTRUNC:
     return lowerFPTRUNC(MI, TypeIdx, Ty);
+  case G_FPOWI:
+    return lowerFPOWI(MI);
   case G_SMIN:
   case G_SMAX:
   case G_UMIN:
@@ -2695,6 +2728,27 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
   case G_READ_REGISTER:
   case G_WRITE_REGISTER:
     return lowerReadWriteRegister(MI);
+  case G_UADDSAT:
+  case G_USUBSAT: {
+    // Try to make a reasonable guess about which lowering strategy to use. The
+    // target can override this with custom lowering and calling the
+    // implementation functions.
+    LLT Ty = MRI.getType(MI.getOperand(0).getReg());
+    if (LI.isLegalOrCustom({G_UMIN, Ty}))
+      return lowerAddSubSatToMinMax(MI);
+    return lowerAddSubSatToAddoSubo(MI);
+  }
+  case G_SADDSAT:
+  case G_SSUBSAT: {
+    LLT Ty = MRI.getType(MI.getOperand(0).getReg());
+
+    // FIXME: It would probably make more sense to see if G_SADDO is preferred,
+    // since it's a shorter expansion. However, we would need to figure out the
+    // preferred boolean type for the carry out for the query.
+    if (LI.isLegalOrCustom({G_SMIN, Ty}) && LI.isLegalOrCustom({G_SMAX, Ty}))
+      return lowerAddSubSatToMinMax(MI);
+    return lowerAddSubSatToAddoSubo(MI);
+  }
   }
 }
 
@@ -4273,7 +4327,7 @@ LegalizerHelper::narrowScalarCTPOP(MachineInstr &MI, unsigned TypeIdx,
 LegalizerHelper::LegalizeResult
 LegalizerHelper::lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
   unsigned Opc = MI.getOpcode();
-  auto &TII = *MI.getMF()->getSubtarget().getInstrInfo();
+  const auto &TII = MIRBuilder.getTII();
   auto isSupported = [this](const LegalityQuery &Q) {
     auto QAction = LI.getAction(Q).Action;
     return QAction == Legal || QAction == Libcall || QAction == Custom;
@@ -4799,6 +4853,20 @@ LegalizerHelper::lowerFPTRUNC(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
   return UnableToLegalize;
 }
 
+// TODO: If RHS is a constant SelectionDAGBuilder expands this into a
+// multiplication tree.
+LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPOWI(MachineInstr &MI) {
+  Register Dst = MI.getOperand(0).getReg();
+  Register Src0 = MI.getOperand(1).getReg();
+  Register Src1 = MI.getOperand(2).getReg();
+  LLT Ty = MRI.getType(Dst);
+
+  auto CvtSrc1 = MIRBuilder.buildSITOFP(Ty, Src1);
+  MIRBuilder.buildFPow(Dst, Src0, CvtSrc1, MI.getFlags());
+  MI.eraseFromParent();
+  return Legalized;
+}
+
 static CmpInst::Predicate minMaxToCompare(unsigned Opc) {
   switch (Opc) {
   case TargetOpcode::G_SMIN:
@@ -5268,6 +5336,151 @@ LegalizerHelper::lowerSADDO_SSUBO(MachineInstr &MI) {
   return Legalized;
 }
 
+LegalizerHelper::LegalizeResult
+LegalizerHelper::lowerAddSubSatToMinMax(MachineInstr &MI) {
+  Register Res = MI.getOperand(0).getReg();
+  Register LHS = MI.getOperand(1).getReg();
+  Register RHS = MI.getOperand(2).getReg();
+  LLT Ty = MRI.getType(Res);
+  bool IsSigned;
+  bool IsAdd;
+  unsigned BaseOp;
+  switch (MI.getOpcode()) {
+  default:
+    llvm_unreachable("unexpected addsat/subsat opcode");
+  case TargetOpcode::G_UADDSAT:
+    IsSigned = false;
+    IsAdd = true;
+    BaseOp = TargetOpcode::G_ADD;
+    break;
+  case TargetOpcode::G_SADDSAT:
+    IsSigned = true;
+    IsAdd = true;
+    BaseOp = TargetOpcode::G_ADD;
+    break;
+  case TargetOpcode::G_USUBSAT:
+    IsSigned = false;
+    IsAdd = false;
+    BaseOp = TargetOpcode::G_SUB;
+    break;
+  case TargetOpcode::G_SSUBSAT:
+    IsSigned = true;
+    IsAdd = false;
+    BaseOp = TargetOpcode::G_SUB;
+    break;
+  }
+
+  if (IsSigned) {
+    // sadd.sat(a, b) ->
+    //   hi = 0x7fffffff - smax(a, 0)
+    //   lo = 0x80000000 - smin(a, 0)
+    //   a + smin(smax(lo, b), hi)
+    // ssub.sat(a, b) ->
+    //   lo = smax(a, -1) - 0x7fffffff
+    //   hi = smin(a, -1) - 0x80000000
+    //   a - smin(smax(lo, b), hi)
+    // TODO: AMDGPU can use a "median of 3" instruction here:
+    //   a +/- med3(lo, b, hi)
+    uint64_t NumBits = Ty.getScalarSizeInBits();
+    auto MaxVal =
+        MIRBuilder.buildConstant(Ty, APInt::getSignedMaxValue(NumBits));
+    auto MinVal =
+        MIRBuilder.buildConstant(Ty, APInt::getSignedMinValue(NumBits));
+    MachineInstrBuilder Hi, Lo;
+    if (IsAdd) {
+      auto Zero = MIRBuilder.buildConstant(Ty, 0);
+      Hi = MIRBuilder.buildSub(Ty, MaxVal, MIRBuilder.buildSMax(Ty, LHS, Zero));
+      Lo = MIRBuilder.buildSub(Ty, MinVal, MIRBuilder.buildSMin(Ty, LHS, Zero));
+    } else {
+      auto NegOne = MIRBuilder.buildConstant(Ty, -1);
+      Lo = MIRBuilder.buildSub(Ty, MIRBuilder.buildSMax(Ty, LHS, NegOne),
+                               MaxVal);
+      Hi = MIRBuilder.buildSub(Ty, MIRBuilder.buildSMin(Ty, LHS, NegOne),
+                               MinVal);
+    }
+    auto RHSClamped =
+        MIRBuilder.buildSMin(Ty, MIRBuilder.buildSMax(Ty, Lo, RHS), Hi);
+    MIRBuilder.buildInstr(BaseOp, {Res}, {LHS, RHSClamped});
+  } else {
+    // uadd.sat(a, b) -> a + umin(~a, b)
+    // usub.sat(a, b) -> a - umin(a, b)
+    Register Not = IsAdd ? MIRBuilder.buildNot(Ty, LHS).getReg(0) : LHS;
+    auto Min = MIRBuilder.buildUMin(Ty, Not, RHS);
+    MIRBuilder.buildInstr(BaseOp, {Res}, {LHS, Min});
+  }
+
+  MI.eraseFromParent();
+  return Legalized;
+}
+
+LegalizerHelper::LegalizeResult
+LegalizerHelper::lowerAddSubSatToAddoSubo(MachineInstr &MI) {
+  Register Res = MI.getOperand(0).getReg();
+  Register LHS = MI.getOperand(1).getReg();
+  Register RHS = MI.getOperand(2).getReg();
+  LLT Ty = MRI.getType(Res);
+  LLT BoolTy = Ty.changeElementSize(1);
+  bool IsSigned;
+  bool IsAdd;
+  unsigned OverflowOp;
+  switch (MI.getOpcode()) {
+  default:
+    llvm_unreachable("unexpected addsat/subsat opcode");
+  case TargetOpcode::G_UADDSAT:
+    IsSigned = false;
+    IsAdd = true;
+    OverflowOp = TargetOpcode::G_UADDO;
+    break;
+  case TargetOpcode::G_SADDSAT:
+    IsSigned = true;
+    IsAdd = true;
+    OverflowOp = TargetOpcode::G_SADDO;
+    break;
+  case TargetOpcode::G_USUBSAT:
+    IsSigned = false;
+    IsAdd = false;
+    OverflowOp = TargetOpcode::G_USUBO;
+    break;
+  case TargetOpcode::G_SSUBSAT:
+    IsSigned = true;
+    IsAdd = false;
+    OverflowOp = TargetOpcode::G_SSUBO;
+    break;
+  }
+
+  auto OverflowRes =
+      MIRBuilder.buildInstr(OverflowOp, {Ty, BoolTy}, {LHS, RHS});
+  Register Tmp = OverflowRes.getReg(0);
+  Register Ov = OverflowRes.getReg(1);
+  MachineInstrBuilder Clamp;
+  if (IsSigned) {
+    // sadd.sat(a, b) ->
+    //   {tmp, ov} = saddo(a, b)
+    //   ov ? (tmp >>s 31) + 0x80000000 : r
+    // ssub.sat(a, b) ->
+    //   {tmp, ov} = ssubo(a, b)
+    //   ov ? (tmp >>s 31) + 0x80000000 : r
+    uint64_t NumBits = Ty.getScalarSizeInBits();
+    auto ShiftAmount = MIRBuilder.buildConstant(Ty, NumBits - 1);
+    auto Sign = MIRBuilder.buildAShr(Ty, Tmp, ShiftAmount);
+    auto MinVal =
+        MIRBuilder.buildConstant(Ty, APInt::getSignedMinValue(NumBits));
+    Clamp = MIRBuilder.buildAdd(Ty, Sign, MinVal);
+  } else {
+    // uadd.sat(a, b) ->
+    //   {tmp, ov} = uaddo(a, b)
+    //   ov ? 0xffffffff : tmp
+    // usub.sat(a, b) ->
+    //   {tmp, ov} = usubo(a, b)
+    //   ov ? 0 : tmp
+    Clamp = MIRBuilder.buildConstant(Ty, IsAdd ? -1 : 0);
+  }
+  MIRBuilder.buildSelect(Res, Ov, Clamp, Tmp);
+
+  MI.eraseFromParent();
+  return Legalized;
+}
+
 LegalizerHelper::LegalizeResult
 LegalizerHelper::lowerBswap(MachineInstr &MI) {
   Register Dst = MI.getOperand(0).getReg();
diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index 10f696d6a3b30..6a6137d466766 100644
--- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -106,8 +106,8 @@ MachineInstrBuilder MachineIRBuilder::buildConstDbgValue(const Constant &C,
   } else if (auto *CFP = dyn_cast<ConstantFP>(&C)) {
     MIB.addFPImm(CFP);
   } else {
-    // Insert %noreg if we didn't find a usable constant and had to drop it.
-    MIB.addReg(0U);
+    // Insert $noreg if we didn't find a usable constant and had to drop it.
+    MIB.addReg(Register());
   }
 
   MIB.addImm(0).addMetadata(Variable).addMetadata(Expr);
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 8a7fb4fbbf2de..584a691efb238 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -510,54 +510,97 @@ void llvm::getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU) {
   AU.addPreserved<StackProtector>();
 }
 
-LLT llvm::getLCMType(LLT Ty0, LLT Ty1) {
-  if (!Ty0.isVector() && !Ty1.isVector()) {
-    unsigned Mul = Ty0.getSizeInBits() * Ty1.getSizeInBits();
-    int GCDSize = greatestCommonDivisor(Ty0.getSizeInBits(),
-                                        Ty1.getSizeInBits());
-    return LLT::scalar(Mul / GCDSize);
-  }
+static unsigned getLCMSize(unsigned OrigSize, unsigned TargetSize) {
+  unsigned Mul = OrigSize * TargetSize;
+  unsigned GCDSize = greatestCommonDivisor(OrigSize, TargetSize);
+  return Mul / GCDSize;
+}
 
-  if (Ty0.isVector() && !Ty1.isVector()) {
-    assert(Ty0.getElementType() == Ty1 && "not yet handled");
-    return Ty0;
-  }
+LLT llvm::getLCMType(LLT OrigTy, LLT TargetTy) {
+  const unsigned OrigSize = OrigTy.getSizeInBits();
+  const unsigned TargetSize = TargetTy.getSizeInBits();
 
-  if (Ty1.isVector() && !Ty0.isVector()) {
-    assert(Ty1.getElementType() == Ty0 && "not yet handled");
-    return Ty1;
-  }
+  if (OrigSize == TargetSize)
+    return OrigTy;
+
+  if (OrigTy.isVector()) {
+    const LLT OrigElt = OrigTy.getElementType();
 
-  if (Ty0.isVector() && Ty1.isVector()) {
-    assert(Ty0.getElementType() == Ty1.getElementType() && "not yet handled");
+    if (TargetTy.isVector()) {
+      const LLT TargetElt = TargetTy.getElementType();
 
-    int GCDElts = greatestCommonDivisor(Ty0.getNumElements(),
-                                        Ty1.getNumElements());
+      if (OrigElt.getSizeInBits() == TargetElt.getSizeInBits()) {
+        int GCDElts = greatestCommonDivisor(OrigTy.getNumElements(),
+                                            TargetTy.getNumElements());
+        // Prefer the original element type.
+        int Mul = OrigTy.getNumElements() * TargetTy.getNumElements();
+        return LLT::vector(Mul / GCDElts, OrigTy.getElementType());
+      }
+    } else {
+      if (OrigElt.getSizeInBits() == TargetSize)
+        return OrigTy;
+    }
+
+    unsigned LCMSize = getLCMSize(OrigSize, TargetSize);
+    return LLT::vector(LCMSize / OrigElt.getSizeInBits(), OrigElt);
+  }
 
-    int Mul = Ty0.getNumElements() * Ty1.getNumElements();
-    return LLT::vector(Mul / GCDElts, Ty0.getElementType());
+  if (TargetTy.isVector()) {
+    unsigned LCMSize = getLCMSize(OrigSize, TargetSize);
+    return LLT::vector(LCMSize / OrigSize, OrigTy);
   }
 
-  llvm_unreachable("not yet handled");
+  unsigned LCMSize = getLCMSize(OrigSize, TargetSize);
+
+  // Preserve pointer types.
+  if (LCMSize == OrigSize)
+    return OrigTy;
+  if (LCMSize == TargetSize)
+    return TargetTy;
+
+  return LLT::scalar(LCMSize);
 }
 
 LLT llvm::getGCDType(LLT OrigTy, LLT TargetTy) {
-  if (OrigTy.isVector() && TargetTy.isVector()) {
-    assert(OrigTy.getElementType() == TargetTy.getElementType());
-    int GCD = greatestCommonDivisor(OrigTy.getNumElements(),
-                                    TargetTy.getNumElements());
-    return LLT::scalarOrVector(GCD, OrigTy.getElementType());
-  }
+  const unsigned OrigSize = OrigTy.getSizeInBits();
+  const unsigned TargetSize = TargetTy.getSizeInBits();
+
+  if (OrigSize == TargetSize)
+    return OrigTy;
+
+  if (OrigTy.isVector()) {
+    LLT OrigElt = OrigTy.getElementType();
+    if (TargetTy.isVector()) {
+      LLT TargetElt = TargetTy.getElementType();
+      if (OrigElt.getSizeInBits() == TargetElt.getSizeInBits()) {
+        int GCD = greatestCommonDivisor(OrigTy.getNumElements(),
+                                        TargetTy.getNumElements());
+        return LLT::scalarOrVector(GCD, OrigElt);
+      }
+    } else {
+      // If the source is a vector of pointers, return a pointer element.
+      if (OrigElt.getSizeInBits() == TargetSize)
+        return OrigElt;
+    }
 
-  if (OrigTy.isVector() && !TargetTy.isVector()) {
-    assert(OrigTy.getElementType() == TargetTy);
-    return TargetTy;
+    unsigned GCD = greatestCommonDivisor(OrigSize, TargetSize);
+    if (GCD == OrigElt.getSizeInBits())
+      return OrigElt;
+
+    // If we can't produce the original element type, we have to use a smaller
+    // scalar.
+    if (GCD < OrigElt.getSizeInBits())
+      return LLT::scalar(GCD);
+    return LLT::vector(GCD / OrigElt.getSizeInBits(), OrigElt);
   }
 
-  assert(!OrigTy.isVector() && !TargetTy.isVector() &&
-         "GCD type of vector and scalar not implemented");
+  if (TargetTy.isVector()) {
+    // Try to preserve the original element type.
+    LLT TargetElt = TargetTy.getElementType();
+    if (TargetElt.getSizeInBits() == OrigSize)
+      return OrigTy;
+  }
 
-  int GCD = greatestCommonDivisor(OrigTy.getSizeInBits(),
-                                  TargetTy.getSizeInBits());
+  unsigned GCD = greatestCommonDivisor(OrigSize, TargetSize);
   return LLT::scalar(GCD);
 }
diff --git a/llvm/lib/CodeGen/LLVMTargetMachine.cpp b/llvm/lib/CodeGen/LLVMTargetMachine.cpp
index b485f2cf72610..e94b7ed4de039 100644
--- a/llvm/lib/CodeGen/LLVMTargetMachine.cpp
+++ b/llvm/lib/CodeGen/LLVMTargetMachine.cpp
@@ -196,18 +196,9 @@ bool LLVMTargetMachine::addPassesToEmitFile(
   if (!PassConfig)
     return true;
 
-  if (!TargetPassConfig::willCompleteCodeGenPipeline()) {
-    if (this->getTargetTriple().isOSAIX()) {
-      // On AIX, we might manifest MCSymbols during SDAG lowering. For MIR
-      // testing to be meaningful, we need to ensure that the symbols created
-      // are MCSymbolXCOFF variants, which requires that
-      // the TargetLoweringObjectFile instance has been initialized.
-      MCContext &Ctx = MMIWP->getMMI().getContext();
-      const_cast<TargetLoweringObjectFile &>(*this->getObjFileLowering())
-          .Initialize(Ctx, *this);
-    }
+  if (!TargetPassConfig::willCompleteCodeGenPipeline())
     PM.add(createPrintMIRPass(Out));
-  } else if (addAsmPrinter(PM, Out, DwoOut, FileType,
+  else if (addAsmPrinter(PM, Out, DwoOut, FileType,
                            MMIWP->getMMI().getContext()))
     return true;
 
diff --git a/llvm/lib/CodeGen/LiveIntervals.cpp b/llvm/lib/CodeGen/LiveIntervals.cpp
index e8ee0599e1a2c..6539de107279b 100644
--- a/llvm/lib/CodeGen/LiveIntervals.cpp
+++ b/llvm/lib/CodeGen/LiveIntervals.cpp
@@ -868,9 +868,7 @@ float LiveIntervals::getSpillWeight(bool isDef, bool isUse,
 float LiveIntervals::getSpillWeight(bool isDef, bool isUse,
                                     const MachineBlockFrequencyInfo *MBFI,
                                     const MachineBasicBlock *MBB) {
-  BlockFrequency Freq = MBFI->getBlockFreq(MBB);
-  const float Scale = 1.0f / MBFI->getEntryFreq();
-  return (isDef + isUse) * (Freq.getFrequency() * Scale);
+  return (isDef + isUse) * MBFI->getBlockFreqRelativeToEntryBlock(MBB);
 }
 
 LiveRange::Segment
diff --git a/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp b/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp
index 1168b01a835f1..c836997060149 100644
--- a/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp
+++ b/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp
@@ -241,8 +241,8 @@ MachineBlockFrequencyInfo::getProfileCountFromFreq(uint64_t Freq) const {
   return MBFI ? MBFI->getProfileCountFromFreq(F, Freq) : None;
 }
 
-bool
-MachineBlockFrequencyInfo::isIrrLoopHeader(const MachineBasicBlock *MBB) {
+bool MachineBlockFrequencyInfo::isIrrLoopHeader(
+    const MachineBasicBlock *MBB) const {
   assert(MBFI && "Expected analysis to be available");
   return MBFI->isIrrLoopHeader(MBB);
 }
diff --git a/llvm/lib/CodeGen/MachineBlockPlacement.cpp b/llvm/lib/CodeGen/MachineBlockPlacement.cpp
index 783d22fafee9d..1729670f51103 100644
--- a/llvm/lib/CodeGen/MachineBlockPlacement.cpp
+++ b/llvm/lib/CodeGen/MachineBlockPlacement.cpp
@@ -177,6 +177,14 @@ static cl::opt<unsigned> TailDupPlacementPenalty(
     cl::init(2),
     cl::Hidden);
 
+// Heuristic for tail duplication if profile count is used in cost model.
+static cl::opt<unsigned> TailDupProfilePercentThreshold(
+    "tail-dup-profile-percent-threshold",
+    cl::desc("If profile count information is used in tail duplication cost "
+             "model, the gained fall through number from tail duplication "
+             "should be at least this percent of hot count."),
+    cl::init(50), cl::Hidden);
+
 // Heuristic for triangle chains.
 static cl::opt<unsigned> TriangleChainCount(
     "triangle-chain-count",
@@ -377,6 +385,10 @@ class MachineBlockPlacement : public MachineFunctionPass {
   /// Partial tail duplication threshold.
   BlockFrequency DupThreshold;
 
+  /// True:  use block profile count to compute tail duplication cost.
+  /// False: use block frequency to compute tail duplication cost.
+  bool UseProfileCount;
+
   /// Allocator and owner of BlockChain structures.
   ///
   /// We build BlockChains lazily while processing the loop structure of
@@ -402,6 +414,19 @@ class MachineBlockPlacement : public MachineFunctionPass {
   SmallPtrSet<MachineBasicBlock *, 4> BlocksWithUnanalyzableExits;
 #endif
 
+  /// Get block profile count or frequency according to UseProfileCount.
+  /// The return value is used to model tail duplication cost.
+  BlockFrequency getBlockCountOrFrequency(const MachineBasicBlock *BB) {
+    if (UseProfileCount) {
+      auto Count = MBFI->getMBFI().getBlockProfileCount(BB);
+      if (Count)
+        return *Count;
+      else
+        return 0;
+    } else
+      return MBFI->getBlockFreq(BB);
+  }
+
   /// Scale the DupThreshold according to basic block size.
   BlockFrequency scaleThreshold(MachineBasicBlock *BB);
   void initDupThreshold();
@@ -3120,7 +3145,7 @@ bool MachineBlockPlacement::isBestSuccessor(MachineBasicBlock *BB,
 
   // Compute the number of reduced taken branches if Pred falls through to BB
   // instead of another successor. Then compare it with threshold.
-  BlockFrequency PredFreq = MBFI->getBlockFreq(Pred);
+  BlockFrequency PredFreq = getBlockCountOrFrequency(Pred);
   BlockFrequency Gain = PredFreq * (BBProb - BestProb);
   return Gain > scaleThreshold(BB);
 }
@@ -3194,7 +3219,7 @@ void MachineBlockPlacement::findDuplicateCandidates(
   // it. But it can beneficially fall through to BB, and duplicate BB into other
   // predecessors.
   for (MachineBasicBlock *Pred : Preds) {
-    BlockFrequency PredFreq = MBFI->getBlockFreq(Pred);
+    BlockFrequency PredFreq = getBlockCountOrFrequency(Pred);
 
     if (!TailDup.canTailDuplicate(BB, Pred)) {
       // BB can't be duplicated into Pred, but it is possible to be layout
@@ -3243,6 +3268,15 @@ void MachineBlockPlacement::initDupThreshold() {
   if (!F->getFunction().hasProfileData())
     return;
 
+  // We prefer to use prifile count.
+  uint64_t HotThreshold = PSI->getOrCompHotCountThreshold();
+  if (HotThreshold != UINT64_MAX) {
+    UseProfileCount = true;
+    DupThreshold = HotThreshold * TailDupProfilePercentThreshold / 100;
+    return;
+  }
+
+  // Profile count is not available, we can use block frequency instead.
   BlockFrequency MaxFreq = 0;
   for (MachineBasicBlock &MBB : *F) {
     BlockFrequency Freq = MBFI->getBlockFreq(&MBB);
@@ -3250,10 +3284,9 @@ void MachineBlockPlacement::initDupThreshold() {
       MaxFreq = Freq;
   }
 
-  // FIXME: we may use profile count instead of frequency,
-  // and need more fine tuning.
   BranchProbability ThresholdProb(TailDupPlacementPenalty, 100);
   DupThreshold = MaxFreq * ThresholdProb;
+  UseProfileCount = false;
 }
 
 bool MachineBlockPlacement::runOnMachineFunction(MachineFunction &MF) {
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index 2b4fd654e46c0..cece914d2eb17 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -1142,7 +1142,7 @@ void MachineMemOperand::print(raw_ostream &OS, ModuleSlotTracker &MST,
       const MIRFormatter *Formatter = TII->getMIRFormatter();
       // FIXME: This is not necessarily the correct MIR serialization format for
       // a custom pseudo source value, but at least it allows
-      // -print-machineinstrs to work on a target with custom pseudo source
+      // MIR printing to work on a target with custom pseudo source
       // values.
       OS << "custom \"";
       Formatter->printCustomPseudoSourceValue(OS, MST, *PVal);
diff --git a/llvm/lib/CodeGen/RegAllocPBQP.cpp b/llvm/lib/CodeGen/RegAllocPBQP.cpp
index 7590dbf1b9774..34701b71f2816 100644
--- a/llvm/lib/CodeGen/RegAllocPBQP.cpp
+++ b/llvm/lib/CodeGen/RegAllocPBQP.cpp
@@ -451,8 +451,7 @@ class Coalescing : public PBQPRAConstraint {
         unsigned DstReg = CP.getDstReg();
         unsigned SrcReg = CP.getSrcReg();
 
-        const float Scale = 1.0f / MBFI.getEntryFreq();
-        PBQP::PBQPNum CBenefit = MBFI.getBlockFreq(&MBB).getFrequency() * Scale;
+        PBQP::PBQPNum CBenefit = MBFI.getBlockFreqRelativeToEntryBlock(&MBB);
 
         if (CP.isPhys()) {
           if (!MF.getRegInfo().isAllocatable(DstReg))
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index f14b3dba4f318..a026d39600264 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -11372,9 +11372,10 @@ SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
       // Stop if more than one members are non-undef.
       if (NumDefs > 1)
         break;
+
       VTs.push_back(EVT::getVectorVT(*DAG.getContext(),
                                      VT.getVectorElementType(),
-                                     X.getValueType().getVectorNumElements()));
+                                     X.getValueType().getVectorElementCount()));
     }
 
     if (NumDefs == 0)
@@ -18795,6 +18796,11 @@ static SDValue combineConcatVectorOfScalars(SDNode *N, SelectionDAG &DAG) {
 static SDValue combineConcatVectorOfExtracts(SDNode *N, SelectionDAG &DAG) {
   EVT VT = N->getValueType(0);
   EVT OpVT = N->getOperand(0).getValueType();
+
+  // We currently can't generate an appropriate shuffle for a scalable vector.
+  if (VT.isScalableVector())
+    return SDValue();
+
   int NumElts = VT.getVectorNumElements();
   int NumOpElts = OpVT.getVectorNumElements();
 
@@ -19055,11 +19061,14 @@ SDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) {
     return V;
 
   // Type legalization of vectors and DAG canonicalization of SHUFFLE_VECTOR
-  // nodes often generate nop CONCAT_VECTOR nodes.
-  // Scan the CONCAT_VECTOR operands and look for a CONCAT operations that
-  // place the incoming vectors at the exact same location.
+  // nodes often generate nop CONCAT_VECTOR nodes. Scan the CONCAT_VECTOR
+  // operands and look for a CONCAT operations that place the incoming vectors
+  // at the exact same location.
+  //
+  // For scalable vectors, EXTRACT_SUBVECTOR indexes are implicitly scaled.
   SDValue SingleSource = SDValue();
-  unsigned PartNumElem = N->getOperand(0).getValueType().getVectorNumElements();
+  unsigned PartNumElem =
+      N->getOperand(0).getValueType().getVectorMinNumElements();
 
   for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
     SDValue Op = N->getOperand(i);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index cbbcaf1601ed1..ce185be162bcd 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -3903,6 +3903,23 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
           == TargetLowering::Legal)
         return true;
       break;
+    case ISD::STRICT_FSUB: {
+      if (TLI.getStrictFPOperationAction(Node->getOpcode(),
+                                         Node->getValueType(0))
+          == TargetLowering::Legal)
+        return true;
+
+      EVT VT = Node->getValueType(0);
+      const SDNodeFlags Flags = Node->getFlags();
+      SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags);
+      SDValue Fadd = DAG.getNode(ISD::STRICT_FADD, dl, Node->getVTList(),
+                                 {Node->getOperand(0), Node->getOperand(1), Neg},
+                         Flags);
+
+      Results.push_back(Fadd);
+      Results.push_back(Fadd.getValue(1));
+      break;
+    }
     case ISD::STRICT_LRINT:
     case ISD::STRICT_LLRINT:
     case ISD::STRICT_LROUND:
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 414ba25ffd5ff..1394f084c6dc6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -2151,7 +2151,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
   EVT InVT = Lo.getValueType();
 
   EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
-                               InVT.getVectorNumElements());
+                               InVT.getVectorElementCount());
 
   if (N->isStrictFPOpcode()) {
     Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other }, 
@@ -2559,13 +2559,9 @@ SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
   SDValue InVec = N->getOperand(OpNo);
   EVT InVT = InVec->getValueType(0);
   EVT OutVT = N->getValueType(0);
-  unsigned NumElements = OutVT.getVectorNumElements();
+  ElementCount NumElements = OutVT.getVectorElementCount();
   bool IsFloat = OutVT.isFloatingPoint();
 
-  // Widening should have already made sure this is a power-two vector
-  // if we're trying to split it at all. assert() that's true, just in case.
-  assert(!(NumElements & 1) && "Splitting vector, but not in half!");
-
   unsigned InElementSize = InVT.getScalarSizeInBits();
   unsigned OutElementSize = OutVT.getScalarSizeInBits();
 
@@ -2595,6 +2591,9 @@ SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
   GetSplitVector(InVec, InLoVec, InHiVec);
 
   // Truncate them to 1/2 the element size.
+  //
+  // This assumes the number of elements is a power of two; any vector that
+  // isn't should be widened, not split.
   EVT HalfElementVT = IsFloat ?
     EVT::getFloatingPointVT(InElementSize/2) :
     EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
@@ -4913,7 +4912,8 @@ SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
 
   int LdWidth = LdVT.getSizeInBits();
   int WidthDiff = WidenWidth - LdWidth;
-  // Allow wider loads.
+  // Allow wider loads if they are sufficiently aligned to avoid memory faults
+  // and if the original load is simple.
   unsigned LdAlign = (!LD->isSimple()) ? 0 : LD->getAlignment();
 
   // Find the vector type that can load from.
@@ -4965,19 +4965,6 @@ SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
                       LD->getPointerInfo().getWithOffset(Offset),
                       LD->getOriginalAlign(), MMOFlags, AAInfo);
       LdChain.push_back(L.getValue(1));
-      if (L->getValueType(0).isVector() && NewVTWidth >= LdWidth) {
-        // Later code assumes the vector loads produced will be mergeable, so we
-        // must pad the final entry up to the previous width. Scalars are
-        // combined separately.
-        SmallVector<SDValue, 16> Loads;
-        Loads.push_back(L);
-        unsigned size = L->getValueSizeInBits(0);
-        while (size < LdOp->getValueSizeInBits(0)) {
-          Loads.push_back(DAG.getUNDEF(L->getValueType(0)));
-          size += L->getValueSizeInBits(0);
-        }
-        L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0), Loads);
-      }
     } else {
       L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
                       LD->getPointerInfo().getWithOffset(Offset),
@@ -5018,8 +5005,17 @@ SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
     EVT NewLdTy = LdOps[i].getValueType();
     if (NewLdTy != LdTy) {
       // Create a larger vector.
+      unsigned NumOps = NewLdTy.getSizeInBits() / LdTy.getSizeInBits();
+      assert(NewLdTy.getSizeInBits() % LdTy.getSizeInBits() == 0);
+      SmallVector<SDValue, 16> WidenOps(NumOps);
+      unsigned j = 0;
+      for (; j != End-Idx; ++j)
+        WidenOps[j] = ConcatOps[Idx+j];
+      for (; j != NumOps; ++j)
+        WidenOps[j] = DAG.getUNDEF(LdTy);
+
       ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy,
-                                     makeArrayRef(&ConcatOps[Idx], End - Idx));
+                                     WidenOps);
       Idx = End - 1;
       LdTy = NewLdTy;
     }
diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
index 72e68a5045c69..7a5e8ac6075e6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
@@ -1838,13 +1838,16 @@ class RegReductionPQBase : public SchedulingPriorityQueue {
 
 template<class SF>
 static SUnit *popFromQueueImpl(std::vector<SUnit *> &Q, SF &Picker) {
-  std::vector<SUnit *>::iterator Best = Q.begin();
-  for (auto I = std::next(Q.begin()), E = Q.end(); I != E; ++I)
-    if (Picker(*Best, *I))
-      Best = I;
-  SUnit *V = *Best;
-  if (Best != std::prev(Q.end()))
-    std::swap(*Best, Q.back());
+  unsigned BestIdx = 0;
+  // Only compute the cost for the first 1000 items in the queue, to avoid
+  // excessive compile-times for very large queues.
+  for (unsigned I = 1, E = std::min(Q.size(), (decltype(Q.size()))1000); I != E;
+       I++)
+    if (Picker(Q[BestIdx], Q[I]))
+      BestIdx = I;
+  SUnit *V = Q[BestIdx];
+  if (BestIdx + 1 != Q.size())
+    std::swap(Q[BestIdx], Q.back());
   Q.pop_back();
   return V;
 }
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index c0f84055cfceb..9d2f64b94df16 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -14,19 +14,15 @@
 #include "SDNodeDbgValue.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/APInt.h"
-#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/BitVector.h"
-#include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallSet.h"
-#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/ADT/Twine.h"
-#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/BlockFrequencyInfo.h"
 #include "llvm/Analysis/BranchProbabilityInfo.h"
 #include "llvm/Analysis/ConstantFolding.h"
@@ -40,7 +36,6 @@
 #include "llvm/CodeGen/Analysis.h"
 #include "llvm/CodeGen/FunctionLoweringInfo.h"
 #include "llvm/CodeGen/GCMetadata.h"
-#include "llvm/CodeGen/ISDOpcodes.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
@@ -53,17 +48,14 @@
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SelectionDAG.h"
-#include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
 #include "llvm/CodeGen/StackMaps.h"
 #include "llvm/CodeGen/SwiftErrorValueTracking.h"
 #include "llvm/CodeGen/TargetFrameLowering.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
-#include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
-#include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/CodeGen/WinEHFuncInfo.h"
 #include "llvm/IR/Argument.h"
 #include "llvm/IR/Attributes.h"
@@ -75,13 +67,11 @@
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DebugInfoMetadata.h"
-#include "llvm/IR/DebugLoc.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/GetElementPtrTypeIterator.h"
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/InstrTypes.h"
-#include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
@@ -92,38 +82,28 @@
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/IR/PatternMatch.h"
-#include "llvm/IR/Statepoint.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/User.h"
 #include "llvm/IR/Value.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Support/AtomicOrdering.h"
-#include "llvm/Support/BranchProbability.h"
 #include "llvm/Support/Casting.h"
-#include "llvm/Support/CodeGen.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetIntrinsicInfo.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/Transforms/Utils/Local.h"
-#include <algorithm>
-#include <cassert>
 #include <cstddef>
-#include <cstdint>
 #include <cstring>
 #include <iterator>
 #include <limits>
 #include <numeric>
 #include <tuple>
-#include <utility>
-#include <vector>
 
 using namespace llvm;
 using namespace PatternMatch;
@@ -6383,6 +6363,36 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
                               Op1, Op2, Op3, DAG, TLI));
     return;
   }
+  case Intrinsic::smax: {
+    SDValue Op1 = getValue(I.getArgOperand(0));
+    SDValue Op2 = getValue(I.getArgOperand(1));
+    setValue(&I, DAG.getNode(ISD::SMAX, sdl, Op1.getValueType(), Op1, Op2));
+    return;
+  }
+  case Intrinsic::smin: {
+    SDValue Op1 = getValue(I.getArgOperand(0));
+    SDValue Op2 = getValue(I.getArgOperand(1));
+    setValue(&I, DAG.getNode(ISD::SMIN, sdl, Op1.getValueType(), Op1, Op2));
+    return;
+  }
+  case Intrinsic::umax: {
+    SDValue Op1 = getValue(I.getArgOperand(0));
+    SDValue Op2 = getValue(I.getArgOperand(1));
+    setValue(&I, DAG.getNode(ISD::UMAX, sdl, Op1.getValueType(), Op1, Op2));
+    return;
+  }
+  case Intrinsic::umin: {
+    SDValue Op1 = getValue(I.getArgOperand(0));
+    SDValue Op2 = getValue(I.getArgOperand(1));
+    setValue(&I, DAG.getNode(ISD::UMIN, sdl, Op1.getValueType(), Op1, Op2));
+    return;
+  }
+  case Intrinsic::abs: {
+    // TODO: Preserve "int min is poison" arg in SDAG?
+    SDValue Op1 = getValue(I.getArgOperand(0));
+    setValue(&I, DAG.getNode(ISD::ABS, sdl, Op1.getValueType(), Op1));
+    return;
+  }
   case Intrinsic::stacksave: {
     SDValue Op = getRoot();
     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
@@ -9119,6 +9129,7 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
     Entry.IsSRet = true;
     Entry.IsNest = false;
     Entry.IsByVal = false;
+    Entry.IsByRef = false;
     Entry.IsReturned = false;
     Entry.IsSwiftSelf = false;
     Entry.IsSwiftError = false;
@@ -9239,6 +9250,8 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
         Flags.setCFGuardTarget();
       if (Args[i].IsByVal)
         Flags.setByVal();
+      if (Args[i].IsByRef)
+        Flags.setByRef();
       if (Args[i].IsPreallocated) {
         Flags.setPreallocated();
         // Set the byval flag for CCAssignFn callbacks that don't know about
@@ -9752,6 +9765,8 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
         Flags.setSwiftError();
       if (Arg.hasAttribute(Attribute::ByVal))
         Flags.setByVal();
+      if (Arg.hasAttribute(Attribute::ByRef))
+        Flags.setByRef();
       if (Arg.hasAttribute(Attribute::InAlloca)) {
         Flags.setInAlloca();
         // Set the byval flag for CCAssignFn callbacks that don't know about
@@ -9770,27 +9785,39 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
         // preallocated handling in the various CC lowering callbacks.
         Flags.setByVal();
       }
+
+      Type *ArgMemTy = nullptr;
       if (F.getCallingConv() == CallingConv::X86_INTR) {
         // IA Interrupt passes frame (1st parameter) by value in the stack.
-        if (ArgNo == 0)
+        if (ArgNo == 0) {
           Flags.setByVal();
+          // FIXME: Dependence on pointee element type. See bug 46672.
+          ArgMemTy = Arg.getType()->getPointerElementType();
+        }
       }
-      if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated()) {
-        Type *ElementTy = Arg.getParamByValType();
-
-        // For ByVal, size and alignment should be passed from FE.  BE will
-        // guess if this info is not there but there are cases it cannot get
-        // right.
-        unsigned FrameSize = DL.getTypeAllocSize(Arg.getParamByValType());
-        Flags.setByValSize(FrameSize);
-
-        unsigned FrameAlign;
-        if (Arg.getParamAlignment())
-          FrameAlign = Arg.getParamAlignment();
-        else
-          FrameAlign = TLI->getByValTypeAlignment(ElementTy, DL);
-        Flags.setByValAlign(Align(FrameAlign));
+      if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
+          Flags.isByRef()) {
+        if (!ArgMemTy)
+          ArgMemTy = Arg.getPointeeInMemoryValueType();
+
+        uint64_t MemSize = DL.getTypeAllocSize(ArgMemTy);
+
+        // For in-memory arguments, size and alignment should be passed from FE.
+        // BE will guess if this info is not there but there are cases it cannot
+        // get right.
+        MaybeAlign MemAlign = Arg.getParamAlign();
+        if (!MemAlign)
+          MemAlign = Align(TLI->getByValTypeAlignment(ArgMemTy, DL));
+
+        if (Flags.isByRef()) {
+          Flags.setByRefSize(MemSize);
+          Flags.setByRefAlign(*MemAlign);
+        } else {
+          Flags.setByValSize(MemSize);
+          Flags.setByValAlign(*MemAlign);
+        }
       }
+
       if (Arg.hasAttribute(Attribute::Nest))
         Flags.setNest();
       if (NeedsRegBlock)
diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
index 27bebe503ce66..bf3b89e006405 100644
--- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -2125,9 +2125,11 @@ const MCExpr *TargetLoweringObjectFileXCOFF::lowerRelativeReference(
   report_fatal_error("XCOFF not yet implemented.");
 }
 
-XCOFF::StorageClass TargetLoweringObjectFileXCOFF::getStorageClassForGlobal(
-    const GlobalObject *GO) {
-  switch (GO->getLinkage()) {
+XCOFF::StorageClass
+TargetLoweringObjectFileXCOFF::getStorageClassForGlobal(const GlobalValue *GV) {
+  assert(!isa<GlobalIFunc>(GV) && "GlobalIFunc is not supported on AIX.");
+
+  switch (GV->getLinkage()) {
   case GlobalValue::InternalLinkage:
   case GlobalValue::PrivateLinkage:
     return XCOFF::C_HIDEXT;
@@ -2149,10 +2151,16 @@ XCOFF::StorageClass TargetLoweringObjectFileXCOFF::getStorageClassForGlobal(
 }
 
 MCSymbol *TargetLoweringObjectFileXCOFF::getFunctionEntryPointSymbol(
-    const Function *F, const TargetMachine &TM) const {
+    const GlobalValue *Func, const TargetMachine &TM) const {
+  assert(
+      (isa<Function>(Func) ||
+       (isa<GlobalAlias>(Func) &&
+        isa_and_nonnull<Function>(cast<GlobalAlias>(Func)->getBaseObject()))) &&
+      "Func must be a function or an alias which has a function as base "
+      "object.");
   SmallString<128> NameStr;
   NameStr.push_back('.');
-  getNameWithPrefix(NameStr, F, TM);
+  getNameWithPrefix(NameStr, Func, TM);
   return getContext().getOrCreateSymbol(NameStr);
 }
 
diff --git a/llvm/lib/CodeGen/TargetOptionsImpl.cpp b/llvm/lib/CodeGen/TargetOptionsImpl.cpp
index 4866d4c171c0e..0731cf9b28f42 100644
--- a/llvm/lib/CodeGen/TargetOptionsImpl.cpp
+++ b/llvm/lib/CodeGen/TargetOptionsImpl.cpp
@@ -47,7 +47,11 @@ bool TargetOptions::HonorSignDependentRoundingFPMath() const {
 }
 
 /// NOTE: There are targets that still do not support the debug entry values
-/// production.
+/// production and that is being controlled with the SupportsDebugEntryValues.
+/// In addition, SCE debugger does not have the feature implemented, so prefer
+/// not to emit the debug entry values in that case.
+/// The EnableDebugEntryValues can be used for the testing purposes.
 bool TargetOptions::ShouldEmitDebugEntryValues() const {
-  return SupportsDebugEntryValues || EnableDebugEntryValues;
+  return (SupportsDebugEntryValues && DebuggerTuning != DebuggerKind::SCE) ||
+         EnableDebugEntryValues;
 }
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
index ffff56f545b2d..64511940249ca 100644
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -141,9 +141,11 @@ static cl::opt<cl::boolOrDefault> EnableGlobalISelOption(
     "global-isel", cl::Hidden,
     cl::desc("Enable the \"global\" instruction selector"));
 
-static cl::opt<std::string> PrintMachineInstrs(
-    "print-machineinstrs", cl::ValueOptional, cl::desc("Print machine instrs"),
-    cl::value_desc("pass-name"), cl::init("option-unspecified"), cl::Hidden);
+// FIXME: remove this after switching to NPM or GlobalISel, whichever gets there
+//        first...
+static cl::opt<bool>
+    PrintAfterISel("print-after-isel", cl::init(false), cl::Hidden,
+                   cl::desc("Print machine instrs after ISel"));
 
 static cl::opt<GlobalISelAbortMode> EnableGlobalISelAbort(
     "global-isel-abort", cl::Hidden,
@@ -294,12 +296,11 @@ struct InsertedPass {
   AnalysisID TargetPassID;
   IdentifyingPassPtr InsertedPassID;
   bool VerifyAfter;
-  bool PrintAfter;
 
   InsertedPass(AnalysisID TargetPassID, IdentifyingPassPtr InsertedPassID,
-               bool VerifyAfter, bool PrintAfter)
+               bool VerifyAfter)
       : TargetPassID(TargetPassID), InsertedPassID(InsertedPassID),
-        VerifyAfter(VerifyAfter), PrintAfter(PrintAfter) {}
+        VerifyAfter(VerifyAfter) {}
 
   Pass *getInsertedPass() const {
     assert(InsertedPassID.isValid() && "Illegal Pass ID!");
@@ -411,9 +412,6 @@ TargetPassConfig::TargetPassConfig(LLVMTargetMachine &TM, PassManagerBase &pm)
   initializeBasicAAWrapperPassPass(*PassRegistry::getPassRegistry());
   initializeAAResultsWrapperPassPass(*PassRegistry::getPassRegistry());
 
-  if (StringRef(PrintMachineInstrs.getValue()).equals(""))
-    TM.Options.PrintMachineCode = true;
-
   if (EnableIPRA.getNumOccurrences())
     TM.Options.EnableIPRA = EnableIPRA;
   else {
@@ -437,14 +435,13 @@ CodeGenOpt::Level TargetPassConfig::getOptLevel() const {
 /// Insert InsertedPassID pass after TargetPassID.
 void TargetPassConfig::insertPass(AnalysisID TargetPassID,
                                   IdentifyingPassPtr InsertedPassID,
-                                  bool VerifyAfter, bool PrintAfter) {
+                                  bool VerifyAfter) {
   assert(((!InsertedPassID.isInstance() &&
            TargetPassID != InsertedPassID.getID()) ||
           (InsertedPassID.isInstance() &&
            TargetPassID != InsertedPassID.getInstance()->getPassID())) &&
          "Insert a pass after itself!");
-  Impl->InsertedPasses.emplace_back(TargetPassID, InsertedPassID, VerifyAfter,
-                                    PrintAfter);
+  Impl->InsertedPasses.emplace_back(TargetPassID, InsertedPassID, VerifyAfter);
 }
 
 /// createPassConfig - Create a pass configuration object to be used by
@@ -522,7 +519,7 @@ bool TargetPassConfig::isPassSubstitutedOrOverridden(AnalysisID ID) const {
 /// a later pass or that it should stop after an earlier pass, then do not add
 /// the pass.  Finally, compare the current pass against the StartAfter
 /// and StopAfter options and change the Started/Stopped flags accordingly.
-void TargetPassConfig::addPass(Pass *P, bool verifyAfter, bool printAfter) {
+void TargetPassConfig::addPass(Pass *P, bool verifyAfter) {
   assert(!Initialized && "PassConfig is immutable");
 
   // Cache the Pass ID here in case the pass manager finds this pass is
@@ -540,17 +537,16 @@ void TargetPassConfig::addPass(Pass *P, bool verifyAfter, bool printAfter) {
       addMachinePrePasses();
     std::string Banner;
     // Construct banner message before PM->add() as that may delete the pass.
-    if (AddingMachinePasses && (printAfter || verifyAfter))
+    if (AddingMachinePasses && verifyAfter)
       Banner = std::string("After ") + std::string(P->getPassName());
     PM->add(P);
     if (AddingMachinePasses)
-      addMachinePostPasses(Banner, /*AllowPrint*/ printAfter,
-                           /*AllowVerify*/ verifyAfter);
+      addMachinePostPasses(Banner, /*AllowVerify*/ verifyAfter);
 
     // Add the passes after the pass P if there is any.
     for (auto IP : Impl->InsertedPasses) {
       if (IP.TargetPassID == PassID)
-        addPass(IP.getInsertedPass(), IP.VerifyAfter, IP.PrintAfter);
+        addPass(IP.getInsertedPass(), IP.VerifyAfter);
     }
   } else {
     delete P;
@@ -570,8 +566,7 @@ void TargetPassConfig::addPass(Pass *P, bool verifyAfter, bool printAfter) {
 ///
 /// addPass cannot return a pointer to the pass instance because is internal the
 /// PassManager and the instance we create here may already be freed.
-AnalysisID TargetPassConfig::addPass(AnalysisID PassID, bool verifyAfter,
-                                     bool printAfter) {
+AnalysisID TargetPassConfig::addPass(AnalysisID PassID, bool verifyAfter) {
   IdentifyingPassPtr TargetID = getPassSubstitution(PassID);
   IdentifyingPassPtr FinalPtr = overridePass(PassID, TargetID);
   if (!FinalPtr.isValid())
@@ -586,7 +581,7 @@ AnalysisID TargetPassConfig::addPass(AnalysisID PassID, bool verifyAfter,
       llvm_unreachable("Pass ID not registered");
   }
   AnalysisID FinalID = P->getPassID();
-  addPass(P, verifyAfter, printAfter); // Ends the lifetime of P.
+  addPass(P, verifyAfter); // Ends the lifetime of P.
 
   return FinalID;
 }
@@ -597,7 +592,7 @@ void TargetPassConfig::printAndVerify(const std::string &Banner) {
 }
 
 void TargetPassConfig::addPrintPass(const std::string &Banner) {
-  if (TM->shouldPrintMachineCode())
+  if (PrintAfterISel)
     PM->add(createMachineFunctionPrinterPass(dbgs(), Banner));
 }
 
@@ -625,12 +620,9 @@ void TargetPassConfig::addMachinePrePasses(bool AllowDebugify) {
 }
 
 void TargetPassConfig::addMachinePostPasses(const std::string &Banner,
-                                            bool AllowPrint, bool AllowVerify,
-                                            bool AllowStrip) {
+                                            bool AllowVerify, bool AllowStrip) {
   if (DebugifyAndStripAll == cl::BOU_TRUE && DebugifyIsSafe)
     addStripDebugPass();
-  if (AllowPrint)
-    addPrintPass(Banner);
   if (AllowVerify)
     addVerifyPass(Banner);
 }
@@ -916,20 +908,6 @@ static cl::opt<RegisterRegAlloc::FunctionPassCtor, false,
 void TargetPassConfig::addMachinePasses() {
   AddingMachinePasses = true;
 
-  // Insert a machine instr printer pass after the specified pass.
-  StringRef PrintMachineInstrsPassName = PrintMachineInstrs.getValue();
-  if (!PrintMachineInstrsPassName.equals("") &&
-      !PrintMachineInstrsPassName.equals("option-unspecified")) {
-    if (const PassInfo *TPI = getPassInfo(PrintMachineInstrsPassName)) {
-      const PassRegistry *PR = PassRegistry::getPassRegistry();
-      const PassInfo *IPI = PR->getPassInfo(StringRef("machineinstr-printer"));
-      assert(IPI && "failed to get \"machineinstr-printer\" PassInfo!");
-      const char *TID = (const char *)(TPI->getTypeInfo());
-      const char *IID = (const char *)(IPI->getTypeInfo());
-      insertPass(TID, IID);
-    }
-  }
-
   // Add passes that optimize machine instructions in SSA form.
   if (getOptLevel() != CodeGenOpt::None) {
     addMachineSSAOptimization();
@@ -1000,7 +978,7 @@ void TargetPassConfig::addMachinePasses() {
   // GC
   if (addGCPasses()) {
     if (PrintGCInfo)
-      addPass(createGCInfoPrinter(dbgs()), false, false);
+      addPass(createGCInfoPrinter(dbgs()), false);
   }
 
   // Basic block placement.
@@ -1205,6 +1183,11 @@ void TargetPassConfig::addOptimizedRegAlloc() {
   // LiveVariables can be removed completely, and LiveIntervals can be directly
   // computed. (We still either need to regenerate kill flags after regalloc, or
   // preferably fix the scavenger to not depend on them).
+  // FIXME: UnreachableMachineBlockElim is a dependant pass of LiveVariables.
+  // When LiveVariables is removed this has to be removed/moved either.
+  // Explicit addition of UnreachableMachineBlockElim allows stopping before or
+  // after it with -stop-before/-stop-after.
+  addPass(&UnreachableMachineBlockElimID, false);
   addPass(&LiveVariablesID, false);
 
   // Edge splitting is smarter with machine loop info.
diff --git a/llvm/lib/CodeGen/ValueTypes.cpp b/llvm/lib/CodeGen/ValueTypes.cpp
index 66bcdd9b2c4a4..57e26e8cb7a1e 100644
--- a/llvm/lib/CodeGen/ValueTypes.cpp
+++ b/llvm/lib/CodeGen/ValueTypes.cpp
@@ -307,9 +307,9 @@ Type *EVT::getTypeForEVT(LLVMContext &Context) const {
   case MVT::v32f16:
     return FixedVectorType::get(Type::getHalfTy(Context), 32);
   case MVT::v64f16:
-    return FixedVectorType::get(Type::getBFloatTy(Context), 64);
+    return FixedVectorType::get(Type::getHalfTy(Context), 64);
   case MVT::v128f16:
-    return FixedVectorType::get(Type::getBFloatTy(Context), 128);
+    return FixedVectorType::get(Type::getHalfTy(Context), 128);
   case MVT::v2bf16:
     return FixedVectorType::get(Type::getBFloatTy(Context), 2);
   case MVT::v3bf16:
diff --git a/llvm/lib/DebugInfo/PDB/Native/NativeFunctionSymbol.cpp b/llvm/lib/DebugInfo/PDB/Native/NativeFunctionSymbol.cpp
index 2537daa7493c6..cd77fab51e04a 100644
--- a/llvm/lib/DebugInfo/PDB/Native/NativeFunctionSymbol.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/NativeFunctionSymbol.cpp
@@ -19,7 +19,7 @@ using namespace llvm::pdb;
 NativeFunctionSymbol::NativeFunctionSymbol(NativeSession &Session,
                                            SymIndexId Id,
                                            const codeview::ProcSym &Sym)
-    : NativeRawSymbol(Session, PDB_SymType::Data, Id), Sym(Sym) {}
+    : NativeRawSymbol(Session, PDB_SymType::Function, Id), Sym(Sym) {}
 
 NativeFunctionSymbol::~NativeFunctionSymbol() {}
 
@@ -42,10 +42,6 @@ std::string NativeFunctionSymbol::getName() const {
   return std::string(Sym.Name);
 }
 
-PDB_SymType NativeFunctionSymbol::getSymTag() const {
-  return PDB_SymType::Function;
-}
-
 uint64_t NativeFunctionSymbol::getLength() const { return Sym.CodeSize; }
 
 uint32_t NativeFunctionSymbol::getRelativeVirtualAddress() const {
diff --git a/llvm/lib/DebugInfo/PDB/Native/NativeLineNumber.cpp b/llvm/lib/DebugInfo/PDB/Native/NativeLineNumber.cpp
index 2535e09baf625..155ed0cdb8284 100644
--- a/llvm/lib/DebugInfo/PDB/Native/NativeLineNumber.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/NativeLineNumber.cpp
@@ -15,9 +15,10 @@ NativeLineNumber::NativeLineNumber(const NativeSession &Session,
                                    const codeview::LineInfo Line,
                                    uint32_t ColumnNumber, uint32_t Section,
                                    uint32_t Offset, uint32_t Length,
-                                   uint32_t SrcFileId)
+                                   uint32_t SrcFileId, uint32_t CompilandId)
     : Session(Session), Line(Line), ColumnNumber(ColumnNumber),
-      Section(Section), Offset(Offset), Length(Length), SrcFileId(SrcFileId) {}
+      Section(Section), Offset(Offset), Length(Length), SrcFileId(SrcFileId),
+      CompilandId(CompilandId) {}
 
 uint32_t NativeLineNumber::getLineNumber() const { return Line.getStartLine(); }
 
@@ -45,6 +46,6 @@ uint32_t NativeLineNumber::getLength() const { return Length; }
 
 uint32_t NativeLineNumber::getSourceFileId() const { return SrcFileId; }
 
-uint32_t NativeLineNumber::getCompilandId() const { return 0; }
+uint32_t NativeLineNumber::getCompilandId() const { return CompilandId; }
 
 bool NativeLineNumber::isStatement() const { return Line.isStatement(); }
diff --git a/llvm/lib/DebugInfo/PDB/Native/NativePublicSymbol.cpp b/llvm/lib/DebugInfo/PDB/Native/NativePublicSymbol.cpp
index 7086af7e67a2a..1265e688b867f 100644
--- a/llvm/lib/DebugInfo/PDB/Native/NativePublicSymbol.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/NativePublicSymbol.cpp
@@ -18,7 +18,7 @@ using namespace llvm::pdb;
 
 NativePublicSymbol::NativePublicSymbol(NativeSession &Session, SymIndexId Id,
                                        const codeview::PublicSym32 &Sym)
-    : NativeRawSymbol(Session, PDB_SymType::Data, Id), Sym(Sym) {}
+    : NativeRawSymbol(Session, PDB_SymType::PublicSymbol, Id), Sym(Sym) {}
 
 NativePublicSymbol::~NativePublicSymbol() {}
 
@@ -39,10 +39,6 @@ std::string NativePublicSymbol::getName() const {
   return std::string(Sym.Name);
 }
 
-PDB_SymType NativePublicSymbol::getSymTag() const {
-  return PDB_SymType::PublicSymbol;
-}
-
 uint32_t NativePublicSymbol::getRelativeVirtualAddress() const {
   return Session.getRVAFromSectOffset(Sym.Segment, Sym.Offset);
 }
diff --git a/llvm/lib/DebugInfo/PDB/Native/NativeSession.cpp b/llvm/lib/DebugInfo/PDB/Native/NativeSession.cpp
index ac8449df44ffb..acd2902590928 100644
--- a/llvm/lib/DebugInfo/PDB/Native/NativeSession.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/NativeSession.cpp
@@ -272,14 +272,14 @@ NativeSession::findLineNumbersByAddress(uint64_t Address,
 
 std::unique_ptr<IPDBEnumLineNumbers>
 NativeSession::findLineNumbersByRVA(uint32_t RVA, uint32_t Length) const {
-  return findLineNumbersByAddress(getLoadAddress() + RVA, Length);
+  return Cache.findLineNumbersByVA(getLoadAddress() + RVA, Length);
 }
 
 std::unique_ptr<IPDBEnumLineNumbers>
 NativeSession::findLineNumbersBySectOffset(uint32_t Section, uint32_t Offset,
                                            uint32_t Length) const {
   uint64_t VA = getVAFromSectOffset(Section, Offset);
-  return findLineNumbersByAddress(VA, Length);
+  return Cache.findLineNumbersByVA(VA, Length);
 }
 
 std::unique_ptr<IPDBEnumSourceFiles>
diff --git a/llvm/lib/DebugInfo/PDB/Native/SymbolCache.cpp b/llvm/lib/DebugInfo/PDB/Native/SymbolCache.cpp
index 9f15907b519e8..db402d1c2b33d 100644
--- a/llvm/lib/DebugInfo/PDB/Native/SymbolCache.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/SymbolCache.cpp
@@ -319,8 +319,16 @@ SymbolCache::findSymbolBySectOffset(uint32_t Sect, uint32_t Offset,
     return findFunctionSymbolBySectOffset(Sect, Offset);
   case PDB_SymType::PublicSymbol:
     return findPublicSymbolBySectOffset(Sect, Offset);
+  case PDB_SymType::Compiland: {
+    Optional<uint16_t> Modi =
+        getModuleIndexForAddr(Session.getVAFromSectOffset(Sect, Offset));
+    if (!Modi)
+      return nullptr;
+    return getOrCreateCompiland(*Modi);
+  }
   case PDB_SymType::None: {
-    // FIXME: Implement for PDB_SymType::Data.
+    // FIXME: Implement for PDB_SymType::Data. The symbolizer calls this but
+    // only uses it to find the symbol length.
     if (auto Sym = findFunctionSymbolBySectOffset(Sect, Offset))
       return Sym;
     return nullptr;
@@ -332,8 +340,8 @@ SymbolCache::findSymbolBySectOffset(uint32_t Sect, uint32_t Offset,
 
 std::unique_ptr<PDBSymbol>
 SymbolCache::findFunctionSymbolBySectOffset(uint32_t Sect, uint32_t Offset) {
-  auto Iter = AddressToFunctionSymId.find({Sect, Offset});
-  if (Iter != AddressToFunctionSymId.end())
+  auto Iter = AddressToSymbolId.find({Sect, Offset});
+  if (Iter != AddressToSymbolId.end())
     return getSymbolById(Iter->second);
 
   if (!Dbi)
@@ -357,8 +365,14 @@ SymbolCache::findFunctionSymbolBySectOffset(uint32_t Sect, uint32_t Offset) {
     auto PS = cantFail(SymbolDeserializer::deserializeAs<ProcSym>(*I));
     if (Sect == PS.Segment && Offset >= PS.CodeOffset &&
         Offset < PS.CodeOffset + PS.CodeSize) {
+      // Check if the symbol is already cached.
+      auto Found = AddressToSymbolId.find({PS.Segment, PS.CodeOffset});
+      if (Found != AddressToSymbolId.end())
+        return getSymbolById(Found->second);
+
+      // Otherwise, create a new symbol.
       SymIndexId Id = createSymbol<NativeFunctionSymbol>(PS);
-      AddressToFunctionSymId.insert({{Sect, Offset}, Id});
+      AddressToSymbolId.insert({{PS.Segment, PS.CodeOffset}, Id});
       return getSymbolById(Id);
     }
 
@@ -418,9 +432,16 @@ SymbolCache::findPublicSymbolBySectOffset(uint32_t Sect, uint32_t Offset) {
     consumeError(Sym.takeError());
     return nullptr;
   }
+
+  // Check if the symbol is already cached.
   auto PS = cantFail(SymbolDeserializer::deserializeAs<PublicSym32>(Sym.get()));
+  auto Found = AddressToPublicSymId.find({PS.Segment, PS.Offset});
+  if (Found != AddressToPublicSymId.end())
+    return getSymbolById(Found->second);
+
+  // Otherwise, create a new symbol.
   SymIndexId Id = createSymbol<NativePublicSymbol>(PS);
-  AddressToPublicSymId.insert({{Sect, Offset}, Id});
+  AddressToPublicSymId.insert({{PS.Segment, PS.Offset}, Id});
   return getSymbolById(Id);
 }
 
@@ -587,7 +608,7 @@ SymbolCache::findLineNumbersByVA(uint64_t VA, uint32_t Length) const {
         ExpectedChecksums->getArray().at(LineIter->FileNameIndex);
     uint32_t SrcFileId = getOrCreateSourceFile(*ChecksumIter);
     NativeLineNumber LineNum(Session, LineIter->Line, LineIter->ColumnNumber,
-                             LineSect, LineOff, LineLength, SrcFileId);
+                             LineSect, LineOff, LineLength, SrcFileId, Modi);
     LineNumbers.push_back(LineNum);
     ++LineIter;
   }
diff --git a/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp b/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp
index 1d767a2b0d889..383e2c0d3054c 100644
--- a/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp
+++ b/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp
@@ -16,6 +16,7 @@
 
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/BinaryFormat/COFF.h"
+#include "llvm/Config/config.h"
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/DebugInfo/PDB/PDB.h"
 #include "llvm/DebugInfo/PDB/PDBContext.h"
@@ -555,9 +556,12 @@ LLVMSymbolizer::getOrCreateModuleInfo(const std::string &ModuleName) {
     if (!EC && DebugInfo != nullptr && !PDBFileName.empty()) {
       using namespace pdb;
       std::unique_ptr<IPDBSession> Session;
-      PDB_ReaderType ReaderType = Opts.UseNativePDBReader
-                                      ? PDB_ReaderType::Native
-                                      : PDB_ReaderType::DIA;
+
+      PDB_ReaderType ReaderType = PDB_ReaderType::Native;
+#if LLVM_ENABLE_DIA_SDK
+      if (!Opts.UseNativePDBReader)
+        ReaderType = PDB_ReaderType::DIA;
+#endif
       if (auto Err = loadDataForEXE(ReaderType, Objects.first->getFileName(),
                                     Session)) {
         Modules.emplace(ModuleName, std::unique_ptr<SymbolizableModule>());
diff --git a/llvm/lib/Demangle/Demangle.cpp b/llvm/lib/Demangle/Demangle.cpp
index 71dafa0b2e437..000f75b6a9770 100644
--- a/llvm/lib/Demangle/Demangle.cpp
+++ b/llvm/lib/Demangle/Demangle.cpp
@@ -31,6 +31,6 @@ std::string llvm::demangle(const std::string &MangledName) {
     return MangledName;
 
   std::string Ret = Demangled;
-  free(Demangled);
+  std::free(Demangled);
   return Ret;
 }
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
index 28adf9b3fb718..226955fbceb45 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
@@ -182,6 +182,8 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder {
     using namespace support;
     auto &Obj = getObject();
 
+    LLVM_DEBUG(dbgs() << "Processing relocations:\n");
+
     for (auto &S : Obj.sections()) {
 
       JITTargetAddress SectionAddress = S.getAddress();
@@ -200,8 +202,8 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder {
             getSectionByIndex(Obj.getSectionIndex(S.getRawDataRefImpl()));
         if (!NSec.GraphSection) {
           LLVM_DEBUG({
-            dbgs() << "Skipping relocations for MachO section " << NSec.SegName
-                   << "/" << NSec.SectName
+            dbgs() << "  Skipping relocations for MachO section "
+                   << NSec.SegName << "/" << NSec.SectName
                    << " which has no associated graph section\n";
           });
           continue;
@@ -222,9 +224,10 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder {
         JITTargetAddress FixupAddress = SectionAddress + (uint32_t)RI.r_address;
 
         LLVM_DEBUG({
-          dbgs() << "Processing " << getMachOARM64RelocationKindName(*Kind)
-                 << " relocation at " << format("0x%016" PRIx64, FixupAddress)
-                 << "\n";
+          auto &NSec =
+              getSectionByIndex(Obj.getSectionIndex(S.getRawDataRefImpl()));
+          dbgs() << "  " << NSec.SectName << " + "
+                 << formatv("{0:x8}", RI.r_address) << ":\n";
         });
 
         // Find the block that the fixup points to.
@@ -378,6 +381,7 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder {
         }
 
         LLVM_DEBUG({
+          dbgs() << "    ";
           Edge GE(*Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol,
                   Addend);
           printEdge(dbgs(), *BlockToFix, GE,
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
index 54d725eac144a..134b01f0f6560 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
@@ -184,6 +184,8 @@ class MachOLinkGraphBuilder_x86_64 : public MachOLinkGraphBuilder {
     using namespace support;
     auto &Obj = getObject();
 
+    LLVM_DEBUG(dbgs() << "Processing relocations:\n");
+
     for (auto &S : Obj.sections()) {
 
       JITTargetAddress SectionAddress = S.getAddress();
@@ -202,8 +204,8 @@ class MachOLinkGraphBuilder_x86_64 : public MachOLinkGraphBuilder {
             getSectionByIndex(Obj.getSectionIndex(S.getRawDataRefImpl()));
         if (!NSec.GraphSection) {
           LLVM_DEBUG({
-            dbgs() << "Skipping relocations for MachO section " << NSec.SegName
-                   << "/" << NSec.SectName
+            dbgs() << "  Skipping relocations for MachO section "
+                   << NSec.SegName << "/" << NSec.SectName
                    << " which has no associated graph section\n";
           });
           continue;
@@ -225,8 +227,10 @@ class MachOLinkGraphBuilder_x86_64 : public MachOLinkGraphBuilder {
         JITTargetAddress FixupAddress = SectionAddress + (uint32_t)RI.r_address;
 
         LLVM_DEBUG({
-          dbgs() << "Processing relocation at "
-                 << format("0x%016" PRIx64, FixupAddress) << "\n";
+          auto &NSec =
+              getSectionByIndex(Obj.getSectionIndex(S.getRawDataRefImpl()));
+          dbgs() << "  " << NSec.SectName << " + "
+                 << formatv("{0:x8}", RI.r_address) << ":\n";
         });
 
         // Find the block that the fixup points to.
@@ -341,6 +345,7 @@ class MachOLinkGraphBuilder_x86_64 : public MachOLinkGraphBuilder {
         }
 
         LLVM_DEBUG({
+          dbgs() << "    ";
           Edge GE(*Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol,
                   Addend);
           printEdge(dbgs(), *BlockToFix, GE,
diff --git a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
index f9d7924cd5e8c..1876c3512d11a 100644
--- a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
+++ b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
@@ -26,6 +26,7 @@ add_llvm_component_library(LLVMOrcJIT
   SpeculateAnalyses.cpp
   TargetProcessControl.cpp
   ThreadSafeModule.cpp
+  TPCDynamicLibrarySearchGenerator.cpp
   TPCIndirectionUtils.cpp
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/ExecutionEngine/Orc
diff --git a/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp
index 031b1afefc9da..4f7f6089e68db 100644
--- a/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp
@@ -54,8 +54,8 @@ class CompileCallbackMaterializationUnit : public orc::MaterializationUnit {
 namespace llvm {
 namespace orc {
 
+TrampolinePool::~TrampolinePool() {}
 void IndirectStubsManager::anchor() {}
-void TrampolinePool::anchor() {}
 
 Expected<JITTargetAddress>
 JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) {
diff --git a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
index 713a48fbf3eb8..ffe156dcdd62c 100644
--- a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
@@ -13,6 +13,7 @@
 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
 #include "llvm/ExecutionEngine/Orc/OrcError.h"
 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
+#include "llvm/ExecutionEngine/Orc/TargetProcessControl.h"
 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/IRBuilder.h"
@@ -973,10 +974,15 @@ Error LLJITBuilderState::prepareForConstruction() {
       JTMB->setRelocationModel(Reloc::PIC_);
       JTMB->setCodeModel(CodeModel::Small);
       CreateObjectLinkingLayer =
-          [](ExecutionSession &ES,
-             const Triple &) -> std::unique_ptr<ObjectLayer> {
-        auto ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>(
-            ES, std::make_unique<jitlink::InProcessMemoryManager>());
+          [TPC = this->TPC](ExecutionSession &ES,
+                            const Triple &) -> std::unique_ptr<ObjectLayer> {
+        std::unique_ptr<ObjectLinkingLayer> ObjLinkingLayer;
+        if (TPC)
+          ObjLinkingLayer =
+              std::make_unique<ObjectLinkingLayer>(ES, TPC->getMemMgr());
+        else
+          ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>(
+              ES, std::make_unique<jitlink::InProcessMemoryManager>());
         ObjLinkingLayer->addPlugin(std::make_unique<EHFrameRegistrationPlugin>(
             jitlink::InProcessEHFrameRegistrar::getInstance()));
         return std::move(ObjLinkingLayer);
diff --git a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
index 02066b458dfce..8ad2e338ca82a 100644
--- a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
@@ -36,7 +36,7 @@ class ObjectLinkingLayerJITLinkContext final : public JITLinkContext {
       Layer.ReturnObjectBuffer(std::move(ObjBuffer));
   }
 
-  JITLinkMemoryManager &getMemoryManager() override { return *Layer.MemMgr; }
+  JITLinkMemoryManager &getMemoryManager() override { return Layer.MemMgr; }
 
   MemoryBufferRef getObjectBuffer() const override {
     return ObjBuffer->getMemBufferRef();
@@ -447,9 +447,13 @@ class ObjectLinkingLayerJITLinkContext final : public JITLinkContext {
 
 ObjectLinkingLayer::Plugin::~Plugin() {}
 
+ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES,
+                                       JITLinkMemoryManager &MemMgr)
+    : ObjectLayer(ES), MemMgr(MemMgr) {}
+
 ObjectLinkingLayer::ObjectLinkingLayer(
     ExecutionSession &ES, std::unique_ptr<JITLinkMemoryManager> MemMgr)
-    : ObjectLayer(ES), MemMgr(std::move(MemMgr)) {}
+    : ObjectLayer(ES), MemMgr(*MemMgr), MemMgrOwnership(std::move(MemMgr)) {}
 
 ObjectLinkingLayer::~ObjectLinkingLayer() {
   if (auto Err = removeAllModules())
diff --git a/llvm/lib/ExecutionEngine/Orc/Speculation.cpp b/llvm/lib/ExecutionEngine/Orc/Speculation.cpp
index 0530b1a97b678..3dd536d8253e3 100644
--- a/llvm/lib/ExecutionEngine/Orc/Speculation.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Speculation.cpp
@@ -16,9 +16,6 @@
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Verifier.h"
-#include "llvm/Support/Debug.h"
-
-#include <vector>
 
 namespace llvm {
 
diff --git a/llvm/lib/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.cpp b/llvm/lib/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.cpp
new file mode 100644
index 0000000000000..ea8bde971d1d4
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.cpp
@@ -0,0 +1,51 @@
+//===---------------- TPCDynamicLibrarySearchGenerator.cpp ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.h"
+
+namespace llvm {
+namespace orc {
+
+Expected<std::unique_ptr<TPCDynamicLibrarySearchGenerator>>
+TPCDynamicLibrarySearchGenerator::Load(TargetProcessControl &TPC,
+                                       const char *LibraryPath) {
+  auto Handle = TPC.loadLibrary(LibraryPath);
+  if (!Handle)
+    return Handle.takeError();
+
+  return std::make_unique<TPCDynamicLibrarySearchGenerator>(TPC, *Handle);
+}
+
+Error TPCDynamicLibrarySearchGenerator::tryToGenerate(
+    LookupKind K, JITDylib &JD, JITDylibLookupFlags JDLookupFlags,
+    const SymbolLookupSet &Symbols) {
+
+  if (Symbols.empty())
+    return Error::success();
+
+  SymbolMap NewSymbols;
+
+  TargetProcessControl::LookupRequestElement Request(DylibHandle, Symbols);
+  auto Result = TPC.lookupSymbols(Request);
+  if (!Result)
+    return Result.takeError();
+
+  assert(Result->size() == 1 && "Results for more than one library returned");
+  assert(Result->front().size() == Symbols.size() &&
+         "Result has incorrect number of elements");
+
+  auto ResultI = Result->front().begin();
+  for (auto &KV : Symbols)
+    NewSymbols[KV.first] =
+        JITEvaluatedSymbol(*ResultI++, JITSymbolFlags::Exported);
+
+  return JD.define(absoluteSymbols(std::move(NewSymbols)));
+}
+
+} // end namespace orc
+} // end namespace llvm
diff --git a/llvm/lib/ExecutionEngine/Orc/TPCIndirectionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/TPCIndirectionUtils.cpp
index 150040cd11e5d..b4c21c32310ce 100644
--- a/llvm/lib/ExecutionEngine/Orc/TPCIndirectionUtils.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/TPCIndirectionUtils.cpp
@@ -37,20 +37,16 @@ class TPCTrampolinePool : public TrampolinePool {
 public:
   TPCTrampolinePool(TPCIndirectionUtils &TPCIU);
   Error deallocatePool();
-  Expected<JITTargetAddress> getTrampoline() override;
-  void releaseTrampoline(JITTargetAddress TrampolineAddr);
 
 protected:
-  Error grow();
+  Error grow() override;
 
   using Allocation = jitlink::JITLinkMemoryManager::Allocation;
 
-  std::mutex TPMutex;
   TPCIndirectionUtils &TPCIU;
   unsigned TrampolineSize = 0;
   unsigned TrampolinesPerPage = 0;
   std::vector<std::unique_ptr<Allocation>> TrampolineBlocks;
-  std::vector<JITTargetAddress> AvailableTrampolines;
 };
 
 class TPCIndirectStubsManager : public IndirectStubsManager,
@@ -96,26 +92,8 @@ Error TPCTrampolinePool::deallocatePool() {
   return Err;
 }
 
-Expected<JITTargetAddress> TPCTrampolinePool::getTrampoline() {
-  std::lock_guard<std::mutex> Lock(TPMutex);
-  if (AvailableTrampolines.empty()) {
-    if (auto Err = grow())
-      return std::move(Err);
-  }
-
-  assert(!AvailableTrampolines.empty() && "Failed to grow trampoline pool");
-  auto TrampolineAddr = AvailableTrampolines.back();
-  AvailableTrampolines.pop_back();
-  return TrampolineAddr;
-}
-
-void TPCTrampolinePool::releaseTrampoline(JITTargetAddress TrampolineAddr) {
-  std::lock_guard<std::mutex> Lock(TPMutex);
-  AvailableTrampolines.push_back(TrampolineAddr);
-}
-
 Error TPCTrampolinePool::grow() {
-  assert(this->AvailableTrampolines.empty() &&
+  assert(AvailableTrampolines.empty() &&
          "Grow called with trampolines still available");
 
   auto ResolverAddress = TPCIU.getResolverBlockAddress();
@@ -144,7 +122,7 @@ Error TPCTrampolinePool::grow() {
 
   auto TargetAddr = (*Alloc)->getTargetMemory(TrampolinePagePermissions);
   for (unsigned I = 0; I < NumTrampolines; ++I)
-    this->AvailableTrampolines.push_back(TargetAddr + (I * TrampolineSize));
+    AvailableTrampolines.push_back(TargetAddr + (I * TrampolineSize));
 
   if (auto Err = (*Alloc)->finalize())
     return Err;
diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcessControl.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcessControl.cpp
index 833b597fe712a..ab07d3ad90b11 100644
--- a/llvm/lib/ExecutionEngine/Orc/TargetProcessControl.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/TargetProcessControl.cpp
@@ -7,6 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ExecutionEngine/Orc/TargetProcessControl.h"
+
+#include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/Support/Host.h"
 #include "llvm/Support/Process.h"
 
@@ -26,6 +28,8 @@ SelfTargetProcessControl::SelfTargetProcessControl(Triple TT, unsigned PageSize)
     : TargetProcessControl(std::move(TT), PageSize) {
   this->MemMgr = IPMM.get();
   this->MemAccess = this;
+  if (this->TT.isOSBinFormatMachO())
+    GlobalManglingPrefix = '_';
 }
 
 Expected<std::unique_ptr<SelfTargetProcessControl>>
@@ -39,6 +43,48 @@ SelfTargetProcessControl::Create() {
   return std::make_unique<SelfTargetProcessControl>(std::move(TT), *PageSize);
 }
 
+Expected<TargetProcessControl::DynamicLibraryHandle>
+SelfTargetProcessControl::loadLibrary(const char *LibraryPath) {
+  std::string ErrMsg;
+  auto Dylib = std::make_unique<sys::DynamicLibrary>(
+      sys::DynamicLibrary::getPermanentLibrary(LibraryPath, &ErrMsg));
+  if (!Dylib->isValid())
+    return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode());
+  DynamicLibraries.push_back(std::move(Dylib));
+  return pointerToJITTargetAddress(DynamicLibraries.back().get());
+}
+
+Expected<TargetProcessControl::LookupResult>
+SelfTargetProcessControl::lookupSymbols(LookupRequest Request) {
+  LookupResult R;
+
+  for (auto &Elem : Request) {
+    auto *Dylib = jitTargetAddressToPointer<sys::DynamicLibrary *>(Elem.Handle);
+    assert(llvm::find_if(DynamicLibraries,
+                         [=](const std::unique_ptr<sys::DynamicLibrary> &DL) {
+                           return DL.get() == Dylib;
+                         }) != DynamicLibraries.end() &&
+           "Invalid handle");
+
+    R.push_back(std::vector<JITTargetAddress>());
+    for (auto &KV : Elem.Symbols) {
+      auto &Sym = KV.first;
+      std::string Tmp((*Sym).data() + !!GlobalManglingPrefix,
+                      (*Sym).size() - !!GlobalManglingPrefix);
+      if (void *Addr = Dylib->getAddressOfSymbol(Tmp.c_str()))
+        R.back().push_back(pointerToJITTargetAddress(Addr));
+      else if (KV.second == SymbolLookupFlags::RequiredSymbol) {
+        // FIXME: Collect all failing symbols before erroring out.
+        SymbolNameVector MissingSymbols;
+        MissingSymbols.push_back(Sym);
+        return make_error<SymbolsNotFound>(std::move(MissingSymbols));
+      }
+    }
+  }
+
+  return R;
+}
+
 void SelfTargetProcessControl::writeUInt8s(ArrayRef<UInt8Write> Ws,
                                            WriteResultFn OnWriteComplete) {
   for (auto &W : Ws)
diff --git a/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp b/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp
index 925049b2a1b4f..febcabfaa5719 100644
--- a/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp
+++ b/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp
@@ -112,6 +112,15 @@ uint8_t *SectionMemoryManager::allocateSection(
   // Save this address as the basis for our next request
   MemGroup.Near = MB;
 
+  // Copy the address to all the other groups, if they have not
+  // been initialized.
+  if (CodeMem.Near.base() == 0)
+    CodeMem.Near = MB;
+  if (RODataMem.Near.base() == 0)
+    RODataMem.Near = MB;
+  if (RWDataMem.Near.base() == 0)
+    RWDataMem.Near = MB;
+
   // Remember that we allocated this memory
   MemGroup.AllocatedMem.push_back(MB);
   Addr = (uintptr_t)MB.base();
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index fd08310316b3a..66098026b986f 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -1962,6 +1962,8 @@ static void writeDICompositeType(raw_ostream &Out, const DICompositeType *N,
   Printer.printString("identifier", N->getIdentifier());
   Printer.printMetadata("discriminator", N->getRawDiscriminator());
   Printer.printMetadata("dataLocation", N->getRawDataLocation());
+  Printer.printMetadata("associated", N->getRawAssociated());
+  Printer.printMetadata("allocated", N->getRawAllocated());
   Out << ")";
 }
 
@@ -4269,11 +4271,14 @@ void AssemblyWriter::writeAttribute(const Attribute &Attr, bool InAttrGroup) {
   }
 
   assert((Attr.hasAttribute(Attribute::ByVal) ||
+          Attr.hasAttribute(Attribute::ByRef) ||
           Attr.hasAttribute(Attribute::Preallocated)) &&
          "unexpected type attr");
 
   if (Attr.hasAttribute(Attribute::ByVal)) {
     Out << "byval";
+  } else if (Attr.hasAttribute(Attribute::ByRef)) {
+    Out << "byref";
   } else {
     Out << "preallocated";
   }
diff --git a/llvm/lib/IR/AttributeImpl.h b/llvm/lib/IR/AttributeImpl.h
index 5c334348cde37..b013474d92789 100644
--- a/llvm/lib/IR/AttributeImpl.h
+++ b/llvm/lib/IR/AttributeImpl.h
@@ -251,6 +251,7 @@ class AttributeSetNode final
   std::pair<unsigned, Optional<unsigned>> getAllocSizeArgs() const;
   std::string getAsString(bool InAttrGrp) const;
   Type *getByValType() const;
+  Type *getByRefType() const;
   Type *getPreallocatedType() const;
 
   using iterator = const Attribute *;
diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp
index 8bf4e82357c69..43074e8193b65 100644
--- a/llvm/lib/IR/Attributes.cpp
+++ b/llvm/lib/IR/Attributes.cpp
@@ -172,6 +172,10 @@ Attribute Attribute::getWithByValType(LLVMContext &Context, Type *Ty) {
   return get(Context, ByVal, Ty);
 }
 
+Attribute Attribute::getWithByRefType(LLVMContext &Context, Type *Ty) {
+  return get(Context, ByRef, Ty);
+}
+
 Attribute Attribute::getWithPreallocatedType(LLVMContext &Context, Type *Ty) {
   return get(Context, Preallocated, Ty);
 }
@@ -459,9 +463,9 @@ std::string Attribute::getAsString(bool InAttrGrp) const {
     return Result;
   }
 
-  if (hasAttribute(Attribute::Preallocated)) {
-    std::string Result;
-    Result += "preallocated";
+  const bool IsByRef = hasAttribute(Attribute::ByRef);
+  if (IsByRef || hasAttribute(Attribute::Preallocated)) {
+    std::string Result = IsByRef ? "byref" : "preallocated";
     raw_string_ostream OS(Result);
     Result += '(';
     getValueAsType()->print(OS, false, true);
@@ -742,6 +746,10 @@ uint64_t AttributeSet::getDereferenceableOrNullBytes() const {
   return SetNode ? SetNode->getDereferenceableOrNullBytes() : 0;
 }
 
+Type *AttributeSet::getByRefType() const {
+  return SetNode ? SetNode->getByRefType() : nullptr;
+}
+
 Type *AttributeSet::getByValType() const {
   return SetNode ? SetNode->getByValType() : nullptr;
 }
@@ -842,6 +850,9 @@ AttributeSetNode *AttributeSetNode::get(LLVMContext &C, const AttrBuilder &B) {
     case Attribute::ByVal:
       Attr = Attribute::getWithByValType(C, B.getByValType());
       break;
+    case Attribute::ByRef:
+      Attr = Attribute::getWithByRefType(C, B.getByRefType());
+      break;
     case Attribute::Preallocated:
       Attr = Attribute::getWithPreallocatedType(C, B.getPreallocatedType());
       break;
@@ -928,6 +939,12 @@ Type *AttributeSetNode::getByValType() const {
   return nullptr;
 }
 
+Type *AttributeSetNode::getByRefType() const {
+  if (auto A = findEnumAttribute(Attribute::ByRef))
+    return A->getValueAsType();
+  return nullptr;
+}
+
 Type *AttributeSetNode::getPreallocatedType() const {
   if (auto A = findEnumAttribute(Attribute::Preallocated))
     return A->getValueAsType();
@@ -1449,6 +1466,10 @@ Type *AttributeList::getParamByValType(unsigned Index) const {
   return getAttributes(Index+FirstArgIndex).getByValType();
 }
 
+Type *AttributeList::getParamByRefType(unsigned Index) const {
+  return getAttributes(Index + FirstArgIndex).getByRefType();
+}
+
 Type *AttributeList::getParamPreallocatedType(unsigned Index) const {
   return getAttributes(Index + FirstArgIndex).getPreallocatedType();
 }
@@ -1534,6 +1555,7 @@ void AttrBuilder::clear() {
   DerefBytes = DerefOrNullBytes = 0;
   AllocSizeArgs = 0;
   ByValType = nullptr;
+  ByRefType = nullptr;
   PreallocatedType = nullptr;
 }
 
@@ -1560,6 +1582,8 @@ AttrBuilder &AttrBuilder::addAttribute(Attribute Attr) {
     StackAlignment = Attr.getStackAlignment();
   else if (Kind == Attribute::ByVal)
     ByValType = Attr.getValueAsType();
+  else if (Kind == Attribute::ByRef)
+    ByRefType = Attr.getValueAsType();
   else if (Kind == Attribute::Preallocated)
     PreallocatedType = Attr.getValueAsType();
   else if (Kind == Attribute::Dereferenceable)
@@ -1586,6 +1610,8 @@ AttrBuilder &AttrBuilder::removeAttribute(Attribute::AttrKind Val) {
     StackAlignment.reset();
   else if (Val == Attribute::ByVal)
     ByValType = nullptr;
+  else if (Val == Attribute::ByRef)
+    ByRefType = nullptr;
   else if (Val == Attribute::Preallocated)
     PreallocatedType = nullptr;
   else if (Val == Attribute::Dereferenceable)
@@ -1676,6 +1702,12 @@ AttrBuilder &AttrBuilder::addByValAttr(Type *Ty) {
   return *this;
 }
 
+AttrBuilder &AttrBuilder::addByRefAttr(Type *Ty) {
+  Attrs[Attribute::ByRef] = true;
+  ByRefType = Ty;
+  return *this;
+}
+
 AttrBuilder &AttrBuilder::addPreallocatedAttr(Type *Ty) {
   Attrs[Attribute::Preallocated] = true;
   PreallocatedType = Ty;
@@ -1702,6 +1734,9 @@ AttrBuilder &AttrBuilder::merge(const AttrBuilder &B) {
   if (!ByValType)
     ByValType = B.ByValType;
 
+  if (!ByRefType)
+    ByRefType = B.ByRefType;
+
   if (!PreallocatedType)
     PreallocatedType = B.PreallocatedType;
 
@@ -1733,6 +1768,9 @@ AttrBuilder &AttrBuilder::remove(const AttrBuilder &B) {
   if (B.ByValType)
     ByValType = nullptr;
 
+  if (B.ByRefType)
+    ByRefType = nullptr;
+
   if (B.PreallocatedType)
     PreallocatedType = nullptr;
 
@@ -1796,7 +1834,7 @@ bool AttrBuilder::operator==(const AttrBuilder &B) {
 
   return Alignment == B.Alignment && StackAlignment == B.StackAlignment &&
          DerefBytes == B.DerefBytes && ByValType == B.ByValType &&
-         PreallocatedType == B.PreallocatedType;
+         ByRefType == B.ByRefType && PreallocatedType == B.PreallocatedType;
 }
 
 //===----------------------------------------------------------------------===//
@@ -1825,7 +1863,8 @@ AttrBuilder AttributeFuncs::typeIncompatible(Type *Ty) {
         .addAttribute(Attribute::StructRet)
         .addAttribute(Attribute::InAlloca)
         .addPreallocatedAttr(Ty)
-        .addByValAttr(Ty);
+        .addByValAttr(Ty)
+        .addByRefAttr(Ty);
 
   return Incompatible;
 }
diff --git a/llvm/lib/IR/DebugInfoMetadata.cpp b/llvm/lib/IR/DebugInfoMetadata.cpp
index 110d94116f102..bc2a4c0153ccb 100644
--- a/llvm/lib/IR/DebugInfoMetadata.cpp
+++ b/llvm/lib/IR/DebugInfoMetadata.cpp
@@ -493,7 +493,8 @@ DICompositeType *DICompositeType::getImpl(
     uint32_t AlignInBits, uint64_t OffsetInBits, DIFlags Flags,
     Metadata *Elements, unsigned RuntimeLang, Metadata *VTableHolder,
     Metadata *TemplateParams, MDString *Identifier, Metadata *Discriminator,
-    Metadata *DataLocation, StorageType Storage, bool ShouldCreate) {
+    Metadata *DataLocation, Metadata *Associated, Metadata *Allocated,
+    StorageType Storage, bool ShouldCreate) {
   assert(isCanonical(Name) && "Expected canonical MDString");
 
   // Keep this in sync with buildODRType.
@@ -501,10 +502,10 @@ DICompositeType *DICompositeType::getImpl(
                         (Tag, Name, File, Line, Scope, BaseType, SizeInBits,
                          AlignInBits, OffsetInBits, Flags, Elements,
                          RuntimeLang, VTableHolder, TemplateParams, Identifier,
-                         Discriminator, DataLocation));
+                         Discriminator, DataLocation, Associated, Allocated));
   Metadata *Ops[] = {File,          Scope,        Name,           BaseType,
                      Elements,      VTableHolder, TemplateParams, Identifier,
-                     Discriminator, DataLocation};
+                     Discriminator, DataLocation, Associated,     Allocated};
   DEFINE_GETIMPL_STORE(DICompositeType, (Tag, Line, RuntimeLang, SizeInBits,
                                          AlignInBits, OffsetInBits, Flags),
                        Ops);
@@ -516,7 +517,7 @@ DICompositeType *DICompositeType::buildODRType(
     uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits,
     DIFlags Flags, Metadata *Elements, unsigned RuntimeLang,
     Metadata *VTableHolder, Metadata *TemplateParams, Metadata *Discriminator,
-    Metadata *DataLocation) {
+    Metadata *DataLocation, Metadata *Associated, Metadata *Allocated) {
   assert(!Identifier.getString().empty() && "Expected valid identifier");
   if (!Context.isODRUniquingDebugTypes())
     return nullptr;
@@ -526,7 +527,7 @@ DICompositeType *DICompositeType::buildODRType(
                Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits,
                AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang,
                VTableHolder, TemplateParams, &Identifier, Discriminator,
-               DataLocation);
+               DataLocation, Associated, Allocated);
 
   // Only mutate CT if it's a forward declaration and the new operands aren't.
   assert(CT->getRawIdentifier() == &Identifier && "Wrong ODR identifier?");
@@ -538,7 +539,7 @@ DICompositeType *DICompositeType::buildODRType(
              Flags);
   Metadata *Ops[] = {File,          Scope,        Name,           BaseType,
                      Elements,      VTableHolder, TemplateParams, &Identifier,
-                     Discriminator, DataLocation};
+                     Discriminator, DataLocation, Associated,     Allocated};
   assert((std::end(Ops) - std::begin(Ops)) == (int)CT->getNumOperands() &&
          "Mismatched number of operands");
   for (unsigned I = 0, E = CT->getNumOperands(); I != E; ++I)
@@ -553,7 +554,7 @@ DICompositeType *DICompositeType::getODRType(
     uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits,
     DIFlags Flags, Metadata *Elements, unsigned RuntimeLang,
     Metadata *VTableHolder, Metadata *TemplateParams, Metadata *Discriminator,
-    Metadata *DataLocation) {
+    Metadata *DataLocation, Metadata *Associated, Metadata *Allocated) {
   assert(!Identifier.getString().empty() && "Expected valid identifier");
   if (!Context.isODRUniquingDebugTypes())
     return nullptr;
@@ -562,7 +563,8 @@ DICompositeType *DICompositeType::getODRType(
     CT = DICompositeType::getDistinct(
         Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits,
         AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang, VTableHolder,
-        TemplateParams, &Identifier, Discriminator, DataLocation);
+        TemplateParams, &Identifier, Discriminator, DataLocation, Associated,
+        Allocated);
   return CT;
 }
 
diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp
index 8db2389ef5428..c925adc0ea8e4 100644
--- a/llvm/lib/IR/Function.cpp
+++ b/llvm/lib/IR/Function.cpp
@@ -102,6 +102,12 @@ bool Argument::hasByValAttr() const {
   return hasAttribute(Attribute::ByVal);
 }
 
+bool Argument::hasByRefAttr() const {
+  if (!getType()->isPointerTy())
+    return false;
+  return hasAttribute(Attribute::ByRef);
+}
+
 bool Argument::hasSwiftSelfAttr() const {
   return getParent()->hasParamAttribute(getArgNo(), Attribute::SwiftSelf);
 }
@@ -129,27 +135,52 @@ bool Argument::hasPassPointeeByValueCopyAttr() const {
          Attrs.hasParamAttribute(getArgNo(), Attribute::Preallocated);
 }
 
-uint64_t Argument::getPassPointeeByValueCopySize(const DataLayout &DL) const {
-  AttributeSet ParamAttrs
-    = getParent()->getAttributes().getParamAttributes(getArgNo());
+bool Argument::hasPointeeInMemoryValueAttr() const {
+  if (!getType()->isPointerTy())
+    return false;
+  AttributeList Attrs = getParent()->getAttributes();
+  return Attrs.hasParamAttribute(getArgNo(), Attribute::ByVal) ||
+         Attrs.hasParamAttribute(getArgNo(), Attribute::InAlloca) ||
+         Attrs.hasParamAttribute(getArgNo(), Attribute::Preallocated) ||
+         Attrs.hasParamAttribute(getArgNo(), Attribute::ByRef);
+}
 
+/// For a byval, inalloca, or preallocated parameter, get the in-memory
+/// parameter type.
+static Type *getMemoryParamAllocType(AttributeSet ParamAttrs, Type *ArgTy) {
   // FIXME: All the type carrying attributes are mutually exclusive, so there
   // should be a single query to get the stored type that handles any of them.
   if (Type *ByValTy = ParamAttrs.getByValType())
-    return DL.getTypeAllocSize(ByValTy);
+    return ByValTy;
+  if (Type *ByRefTy = ParamAttrs.getByRefType())
+    return ByRefTy;
   if (Type *PreAllocTy = ParamAttrs.getPreallocatedType())
-    return DL.getTypeAllocSize(PreAllocTy);
+    return PreAllocTy;
 
   // FIXME: inalloca always depends on pointee element type. It's also possible
   // for byval to miss it.
   if (ParamAttrs.hasAttribute(Attribute::InAlloca) ||
       ParamAttrs.hasAttribute(Attribute::ByVal) ||
       ParamAttrs.hasAttribute(Attribute::Preallocated))
-    return DL.getTypeAllocSize(cast<PointerType>(getType())->getElementType());
+    return cast<PointerType>(ArgTy)->getElementType();
 
+  return nullptr;
+}
+
+uint64_t Argument::getPassPointeeByValueCopySize(const DataLayout &DL) const {
+  AttributeSet ParamAttrs =
+      getParent()->getAttributes().getParamAttributes(getArgNo());
+  if (Type *MemTy = getMemoryParamAllocType(ParamAttrs, getType()))
+    return DL.getTypeAllocSize(MemTy);
   return 0;
 }
 
+Type *Argument::getPointeeInMemoryValueType() const {
+  AttributeSet ParamAttrs =
+      getParent()->getAttributes().getParamAttributes(getArgNo());
+  return getMemoryParamAllocType(ParamAttrs, getType());
+}
+
 unsigned Argument::getParamAlignment() const {
   assert(getType()->isPointerTy() && "Only pointers have alignments");
   return getParent()->getParamAlignment(getArgNo());
@@ -165,6 +196,11 @@ Type *Argument::getParamByValType() const {
   return getParent()->getParamByValType(getArgNo());
 }
 
+Type *Argument::getParamByRefType() const {
+  assert(getType()->isPointerTy() && "Only pointers have byval types");
+  return getParent()->getParamByRefType(getArgNo());
+}
+
 uint64_t Argument::getDereferenceableBytes() const {
   assert(getType()->isPointerTy() &&
          "Only pointers have dereferenceable bytes");
@@ -582,6 +618,10 @@ static const char * const IntrinsicNameTable[] = {
 #include "llvm/IR/IntrinsicImpl.inc"
 #undef GET_INTRINSIC_TARGET_DATA
 
+bool Function::isTargetIntrinsic() const {
+  return IntID > TargetInfos[0].Count;
+}
+
 /// Find the segment of \c IntrinsicNameTable for intrinsics with the same
 /// target as \c Name, or the generic table if \c Name is not target specific.
 ///
diff --git a/llvm/lib/IR/LLVMContextImpl.h b/llvm/lib/IR/LLVMContextImpl.h
index 1c7d8746d242f..b97ac37c5fcfd 100644
--- a/llvm/lib/IR/LLVMContextImpl.h
+++ b/llvm/lib/IR/LLVMContextImpl.h
@@ -525,6 +525,8 @@ template <> struct MDNodeKeyImpl<DICompositeType> {
   MDString *Identifier;
   Metadata *Discriminator;
   Metadata *DataLocation;
+  Metadata *Associated;
+  Metadata *Allocated;
 
   MDNodeKeyImpl(unsigned Tag, MDString *Name, Metadata *File, unsigned Line,
                 Metadata *Scope, Metadata *BaseType, uint64_t SizeInBits,
@@ -532,13 +534,15 @@ template <> struct MDNodeKeyImpl<DICompositeType> {
                 Metadata *Elements, unsigned RuntimeLang,
                 Metadata *VTableHolder, Metadata *TemplateParams,
                 MDString *Identifier, Metadata *Discriminator,
-                Metadata *DataLocation)
+                Metadata *DataLocation, Metadata *Associated,
+                Metadata *Allocated)
       : Tag(Tag), Name(Name), File(File), Line(Line), Scope(Scope),
         BaseType(BaseType), SizeInBits(SizeInBits), OffsetInBits(OffsetInBits),
         AlignInBits(AlignInBits), Flags(Flags), Elements(Elements),
         RuntimeLang(RuntimeLang), VTableHolder(VTableHolder),
         TemplateParams(TemplateParams), Identifier(Identifier),
-        Discriminator(Discriminator), DataLocation(DataLocation) {}
+        Discriminator(Discriminator), DataLocation(DataLocation),
+        Associated(Associated), Allocated(Allocated) {}
   MDNodeKeyImpl(const DICompositeType *N)
       : Tag(N->getTag()), Name(N->getRawName()), File(N->getRawFile()),
         Line(N->getLine()), Scope(N->getRawScope()),
@@ -549,7 +553,8 @@ template <> struct MDNodeKeyImpl<DICompositeType> {
         TemplateParams(N->getRawTemplateParams()),
         Identifier(N->getRawIdentifier()),
         Discriminator(N->getRawDiscriminator()),
-        DataLocation(N->getRawDataLocation()) {}
+        DataLocation(N->getRawDataLocation()),
+        Associated(N->getRawAssociated()), Allocated(N->getRawAllocated()) {}
 
   bool isKeyOf(const DICompositeType *RHS) const {
     return Tag == RHS->getTag() && Name == RHS->getRawName() &&
@@ -564,7 +569,9 @@ template <> struct MDNodeKeyImpl<DICompositeType> {
            TemplateParams == RHS->getRawTemplateParams() &&
            Identifier == RHS->getRawIdentifier() &&
            Discriminator == RHS->getRawDiscriminator() &&
-           DataLocation == RHS->getRawDataLocation();
+           DataLocation == RHS->getRawDataLocation() &&
+           Associated == RHS->getRawAssociated() &&
+           Allocated == RHS->getRawAllocated();
   }
 
   unsigned getHashValue() const {
diff --git a/llvm/lib/IR/PassTimingInfo.cpp b/llvm/lib/IR/PassTimingInfo.cpp
index 25275e5733acb..829c3e8ea9ad4 100644
--- a/llvm/lib/IR/PassTimingInfo.cpp
+++ b/llvm/lib/IR/PassTimingInfo.cpp
@@ -16,9 +16,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/IR/PassTimingInfo.h"
-#include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/Statistic.h"
-#include "llvm/ADT/StringRef.h"
 #include "llvm/IR/PassInstrumentation.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/CommandLine.h"
@@ -26,9 +24,8 @@
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/Mutex.h"
-#include "llvm/Support/Timer.h"
+#include "llvm/Support/TypeName.h"
 #include "llvm/Support/raw_ostream.h"
-#include <memory>
 #include <string>
 
 using namespace llvm;
diff --git a/llvm/lib/IR/Use.cpp b/llvm/lib/IR/Use.cpp
index dc0716b85372a..99049c0232aa3 100644
--- a/llvm/lib/IR/Use.cpp
+++ b/llvm/lib/IR/Use.cpp
@@ -17,24 +17,17 @@ void Use::swap(Use &RHS) {
   if (Val == RHS.Val)
     return;
 
-  if (Val)
-    removeFromList();
-
-  Value *OldVal = Val;
-  if (RHS.Val) {
-    RHS.removeFromList();
-    Val = RHS.Val;
-    Val->addUse(*this);
-  } else {
-    Val = nullptr;
-  }
-
-  if (OldVal) {
-    RHS.Val = OldVal;
-    RHS.Val->addUse(RHS);
-  } else {
-    RHS.Val = nullptr;
-  }
+  std::swap(Val, RHS.Val);
+  std::swap(Next, RHS.Next);
+  std::swap(Prev, RHS.Prev);
+
+  *Prev = this;
+  if (Next)
+    Next->Prev = &Next;
+
+  *RHS.Prev = &RHS;
+  if (RHS.Next)
+    RHS.Next->Prev = &RHS.Next;
 }
 
 unsigned Use::getOperandNo() const {
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index c518ae87ea9b0..67a295e7a71ca 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -1035,6 +1035,16 @@ void Verifier::visitDICompositeType(const DICompositeType &N) {
     AssertDI(N.getTag() == dwarf::DW_TAG_array_type,
              "dataLocation can only appear in array type");
   }
+
+  if (N.getRawAssociated()) {
+    AssertDI(N.getTag() == dwarf::DW_TAG_array_type,
+             "associated can only appear in array type");
+  }
+
+  if (N.getRawAllocated()) {
+    AssertDI(N.getTag() == dwarf::DW_TAG_array_type,
+             "allocated can only appear in array type");
+  }
 }
 
 void Verifier::visitDISubroutineType(const DISubroutineType &N) {
@@ -1652,9 +1662,10 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty,
   AttrCount += Attrs.hasAttribute(Attribute::StructRet) ||
                Attrs.hasAttribute(Attribute::InReg);
   AttrCount += Attrs.hasAttribute(Attribute::Nest);
+  AttrCount += Attrs.hasAttribute(Attribute::ByRef);
   Assert(AttrCount <= 1,
          "Attributes 'byval', 'inalloca', 'preallocated', 'inreg', 'nest', "
-         "and 'sret' are incompatible!",
+         "'byref', and 'sret' are incompatible!",
          V);
 
   Assert(!(Attrs.hasAttribute(Attribute::InAlloca) &&
@@ -1720,9 +1731,10 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty,
     SmallPtrSet<Type*, 4> Visited;
     if (!PTy->getElementType()->isSized(&Visited)) {
       Assert(!Attrs.hasAttribute(Attribute::ByVal) &&
-                 !Attrs.hasAttribute(Attribute::InAlloca) &&
-                 !Attrs.hasAttribute(Attribute::Preallocated),
-             "Attributes 'byval', 'inalloca', and 'preallocated' do not "
+             !Attrs.hasAttribute(Attribute::ByRef) &&
+             !Attrs.hasAttribute(Attribute::InAlloca) &&
+             !Attrs.hasAttribute(Attribute::Preallocated),
+             "Attributes 'byval', 'byref', 'inalloca', and 'preallocated' do not "
              "support unsized types!",
              V);
     }
@@ -1731,10 +1743,18 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty,
              "Attribute 'swifterror' only applies to parameters "
              "with pointer to pointer type!",
              V);
+
+    if (Attrs.hasAttribute(Attribute::ByRef)) {
+      Assert(Attrs.getByRefType() == PTy->getElementType(),
+             "Attribute 'byref' type does not match parameter!", V);
+    }
   } else {
     Assert(!Attrs.hasAttribute(Attribute::ByVal),
            "Attribute 'byval' only applies to parameters with pointer type!",
            V);
+    Assert(!Attrs.hasAttribute(Attribute::ByRef),
+           "Attribute 'byref' only applies to parameters with pointer type!",
+           V);
     Assert(!Attrs.hasAttribute(Attribute::SwiftError),
            "Attribute 'swifterror' only applies to parameters "
            "with pointer type!",
@@ -1765,10 +1785,11 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs,
           !RetAttrs.hasAttribute(Attribute::Returned) &&
           !RetAttrs.hasAttribute(Attribute::InAlloca) &&
           !RetAttrs.hasAttribute(Attribute::Preallocated) &&
+          !RetAttrs.hasAttribute(Attribute::ByRef) &&
           !RetAttrs.hasAttribute(Attribute::SwiftSelf) &&
           !RetAttrs.hasAttribute(Attribute::SwiftError)),
-         "Attributes 'byval', 'inalloca', 'preallocated', 'nest', 'sret', "
-         "'nocapture', 'nofree', "
+         "Attributes 'byval', 'inalloca', 'preallocated', 'byref', "
+         "'nest', 'sret', 'nocapture', 'nofree', "
          "'returned', 'swiftself', and 'swifterror' do not apply to return "
          "values!",
          V);
@@ -2292,6 +2313,28 @@ void Verifier::visitFunction(const Function &F) {
   case CallingConv::AMDGPU_CS:
     Assert(!F.hasStructRetAttr(),
            "Calling convention does not allow sret", &F);
+    if (F.getCallingConv() != CallingConv::SPIR_KERNEL) {
+      const unsigned StackAS = DL.getAllocaAddrSpace();
+      unsigned i = 0;
+      for (const Argument &Arg : F.args()) {
+        Assert(!Attrs.hasParamAttribute(i, Attribute::ByVal),
+               "Calling convention disallows byval", &F);
+        Assert(!Attrs.hasParamAttribute(i, Attribute::Preallocated),
+               "Calling convention disallows preallocated", &F);
+        Assert(!Attrs.hasParamAttribute(i, Attribute::InAlloca),
+               "Calling convention disallows inalloca", &F);
+
+        if (Attrs.hasParamAttribute(i, Attribute::ByRef)) {
+          // FIXME: Should also disallow LDS and GDS, but we don't have the enum
+          // value here.
+          Assert(Arg.getType()->getPointerAddressSpace() != StackAS,
+                 "Calling convention disallows stack byref", &F);
+        }
+
+        ++i;
+      }
+    }
+
     LLVM_FALLTHROUGH;
   case CallingConv::Fast:
   case CallingConv::Cold:
@@ -3174,17 +3217,19 @@ static bool isTypeCongruent(Type *L, Type *R) {
 
 static AttrBuilder getParameterABIAttributes(int I, AttributeList Attrs) {
   static const Attribute::AttrKind ABIAttrs[] = {
-      Attribute::StructRet,   Attribute::ByVal,     Attribute::InAlloca,
-      Attribute::InReg,       Attribute::SwiftSelf, Attribute::SwiftError,
-      Attribute::Preallocated};
+      Attribute::StructRet,    Attribute::ByVal,     Attribute::InAlloca,
+      Attribute::InReg,        Attribute::SwiftSelf, Attribute::SwiftError,
+      Attribute::Preallocated, Attribute::ByRef};
   AttrBuilder Copy;
   for (auto AK : ABIAttrs) {
     if (Attrs.hasParamAttribute(I, AK))
       Copy.addAttribute(AK);
   }
-  // `align` is ABI-affecting only in combination with `byval`.
+
+  // `align` is ABI-affecting only in combination with `byval` or `byref`.
   if (Attrs.hasParamAttribute(I, Attribute::Alignment) &&
-      Attrs.hasParamAttribute(I, Attribute::ByVal))
+      (Attrs.hasParamAttribute(I, Attribute::ByVal) ||
+       Attrs.hasParamAttribute(I, Attribute::ByRef)))
     Copy.addAlignmentAttr(Attrs.getParamAlignment(I));
   return Copy;
 }
diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp
index 0c395f9bbf280..ca29548a4d7ca 100644
--- a/llvm/lib/LTO/LTOBackend.cpp
+++ b/llvm/lib/LTO/LTOBackend.cpp
@@ -197,7 +197,7 @@ static void runNewPMPasses(const Config &Conf, Module &Mod, TargetMachine *TM,
   }
 
   PassInstrumentationCallbacks PIC;
-  StandardInstrumentations SI;
+  StandardInstrumentations SI(Conf.DebugPassManager);
   SI.registerCallbacks(PIC);
   PassBuilder PB(TM, Conf.PTO, PGOOpt, &PIC);
   AAManager AA;
diff --git a/llvm/lib/MC/MCExpr.cpp b/llvm/lib/MC/MCExpr.cpp
index ecf63b10f73fc..11fd32e48a4ae 100644
--- a/llvm/lib/MC/MCExpr.cpp
+++ b/llvm/lib/MC/MCExpr.cpp
@@ -47,6 +47,8 @@ void MCExpr::print(raw_ostream &OS, const MCAsmInfo *MAI, bool InParens) const {
     auto Value = cast<MCConstantExpr>(*this).getValue();
     auto PrintInHex = cast<MCConstantExpr>(*this).useHexFormat();
     auto SizeInBytes = cast<MCConstantExpr>(*this).getSizeInBytes();
+    if (Value < 0 && MAI && !MAI->supportsSignedData())
+      PrintInHex = true;
     if (PrintInHex)
       switch (SizeInBytes) {
       default:
@@ -322,6 +324,7 @@ StringRef MCSymbolRefExpr::getVariantKindName(VariantKind Kind) {
   case VK_PPC_TLSLD: return "tlsld";
   case VK_PPC_LOCAL: return "local";
   case VK_PPC_NOTOC: return "notoc";
+  case VK_PPC_PCREL_OPT: return "<<invalid>>";
   case VK_COFF_IMGREL32: return "IMGREL";
   case VK_Hexagon_LO16: return "LO16";
   case VK_Hexagon_HI16: return "HI16";
diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp
index e39c4a03bc1ef..78ee215b59aae 100644
--- a/llvm/lib/MC/MCObjectStreamer.cpp
+++ b/llvm/lib/MC/MCObjectStreamer.cpp
@@ -665,6 +665,68 @@ void MCObjectStreamer::emitGPRel64Value(const MCExpr *Value) {
   DF->getContents().resize(DF->getContents().size() + 8, 0);
 }
 
+static Optional<std::pair<bool, std::string>>
+getOffsetAndDataFragment(const MCSymbol &Symbol, uint32_t &RelocOffset,
+                         MCDataFragment *&DF) {
+  if (Symbol.isVariable()) {
+    const MCExpr *SymbolExpr = Symbol.getVariableValue();
+    MCValue OffsetVal;
+    if(!SymbolExpr->evaluateAsRelocatable(OffsetVal, nullptr, nullptr))
+      return std::make_pair(false,
+                            std::string("symbol in .reloc offset is not "
+                                        "relocatable"));
+    if (OffsetVal.isAbsolute()) {
+      RelocOffset = OffsetVal.getConstant();
+      MCFragment *Fragment = Symbol.getFragment();
+      // FIXME Support symbols with no DF. For example:
+      // .reloc .data, ENUM_VALUE, <some expr>
+      if (!Fragment || Fragment->getKind() != MCFragment::FT_Data)
+        return std::make_pair(false,
+                              std::string("symbol in offset has no data "
+                                          "fragment"));
+      DF = cast<MCDataFragment>(Fragment);
+      return None;
+    }
+
+    if (OffsetVal.getSymB())
+      return std::make_pair(false,
+                            std::string(".reloc symbol offset is not "
+                                        "representable"));
+
+    const MCSymbolRefExpr &SRE = cast<MCSymbolRefExpr>(*OffsetVal.getSymA());
+    if (!SRE.getSymbol().isDefined())
+      return std::make_pair(false,
+                            std::string("symbol used in the .reloc offset is "
+                                        "not defined"));
+
+    if (SRE.getSymbol().isVariable())
+      return std::make_pair(false,
+                            std::string("symbol used in the .reloc offset is "
+                                        "variable"));
+
+    MCFragment *Fragment = SRE.getSymbol().getFragment();
+    // FIXME Support symbols with no DF. For example:
+    // .reloc .data, ENUM_VALUE, <some expr>
+    if (!Fragment || Fragment->getKind() != MCFragment::FT_Data)
+      return std::make_pair(false,
+                            std::string("symbol in offset has no data "
+                                        "fragment"));
+    RelocOffset = SRE.getSymbol().getOffset() + OffsetVal.getConstant();
+    DF = cast<MCDataFragment>(Fragment);
+  } else {
+    RelocOffset = Symbol.getOffset();
+    MCFragment *Fragment = Symbol.getFragment();
+    // FIXME Support symbols with no DF. For example:
+    // .reloc .data, ENUM_VALUE, <some expr>
+    if (!Fragment || Fragment->getKind() != MCFragment::FT_Data)
+      return std::make_pair(false,
+                            std::string("symbol in offset has no data "
+                                        "fragment"));
+    DF = cast<MCDataFragment>(Fragment);
+  }
+  return None;
+}
+
 Optional<std::pair<bool, std::string>>
 MCObjectStreamer::emitRelocDirective(const MCExpr &Offset, StringRef Name,
                                      const MCExpr *Expr, SMLoc Loc,
@@ -698,10 +760,17 @@ MCObjectStreamer::emitRelocDirective(const MCExpr &Offset, StringRef Name,
                           std::string(".reloc offset is not representable"));
 
   const MCSymbolRefExpr &SRE = cast<MCSymbolRefExpr>(*OffsetVal.getSymA());
-  if (SRE.getSymbol().isDefined()) {
-    // FIXME SRE.getSymbol() may not be relative to DF.
+  const MCSymbol &Symbol = SRE.getSymbol();
+  if (Symbol.isDefined()) {
+    uint32_t SymbolOffset = 0;
+    Optional<std::pair<bool, std::string>> Error;
+    Error = getOffsetAndDataFragment(Symbol, SymbolOffset, DF);
+
+    if (Error != None)
+      return Error;
+
     DF->getFixups().push_back(
-        MCFixup::create(SRE.getSymbol().getOffset() + OffsetVal.getConstant(),
+        MCFixup::create(SymbolOffset + OffsetVal.getConstant(),
                         Expr, Kind, Loc));
     return None;
   }
diff --git a/llvm/lib/Object/ELF.cpp b/llvm/lib/Object/ELF.cpp
index 2515695095a1c..7a675944bb440 100644
--- a/llvm/lib/Object/ELF.cpp
+++ b/llvm/lib/Object/ELF.cpp
@@ -278,7 +278,7 @@ StringRef llvm::object::getELFSectionTypeName(uint32_t Machine, unsigned Type) {
 }
 
 template <class ELFT>
-Expected<std::vector<typename ELFT::Rela>>
+Expected<std::vector<typename ELFT::Rel>>
 ELFFile<ELFT>::decode_relrs(Elf_Relr_Range relrs) const {
   // This function decodes the contents of an SHT_RELR packed relocation
   // section.
@@ -310,11 +310,10 @@ ELFFile<ELFT>::decode_relrs(Elf_Relr_Range relrs) const {
   //    even means address, odd means bitmap.
   // 2. Just a simple list of addresses is a valid encoding.
 
-  Elf_Rela Rela;
-  Rela.r_info = 0;
-  Rela.r_addend = 0;
-  Rela.setType(getRelativeRelocationType(), false);
-  std::vector<Elf_Rela> Relocs;
+  Elf_Rel Rel;
+  Rel.r_info = 0;
+  Rel.setType(getRelativeRelocationType(), false);
+  std::vector<Elf_Rel> Relocs;
 
   // Word type: uint32_t for Elf32, and uint64_t for Elf64.
   typedef typename ELFT::uint Word;
@@ -331,8 +330,8 @@ ELFFile<ELFT>::decode_relrs(Elf_Relr_Range relrs) const {
     Word Entry = R;
     if ((Entry&1) == 0) {
       // Even entry: encodes the offset for next relocation.
-      Rela.r_offset = Entry;
-      Relocs.push_back(Rela);
+      Rel.r_offset = Entry;
+      Relocs.push_back(Rel);
       // Set base offset for subsequent bitmap entries.
       Base = Entry + WordSize;
       continue;
@@ -343,8 +342,8 @@ ELFFile<ELFT>::decode_relrs(Elf_Relr_Range relrs) const {
     while (Entry != 0) {
       Entry >>= 1;
       if ((Entry&1) != 0) {
-        Rela.r_offset = Offset;
-        Relocs.push_back(Rela);
+        Rel.r_offset = Offset;
+        Relocs.push_back(Rel);
       }
       Offset += WordSize;
     }
diff --git a/llvm/lib/ObjectYAML/CMakeLists.txt b/llvm/lib/ObjectYAML/CMakeLists.txt
index bc014c1caca38..92ea1ec9866cf 100644
--- a/llvm/lib/ObjectYAML/CMakeLists.txt
+++ b/llvm/lib/ObjectYAML/CMakeLists.txt
@@ -6,7 +6,6 @@ add_llvm_component_library(LLVMObjectYAML
   COFFEmitter.cpp
   COFFYAML.cpp
   DWARFEmitter.cpp
-  DWARFVisitor.cpp
   DWARFYAML.cpp
   ELFEmitter.cpp
   ELFYAML.cpp
diff --git a/llvm/lib/ObjectYAML/DWARFEmitter.cpp b/llvm/lib/ObjectYAML/DWARFEmitter.cpp
index dc815cd69c129..f61191022fb9b 100644
--- a/llvm/lib/ObjectYAML/DWARFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/DWARFEmitter.cpp
@@ -12,7 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ObjectYAML/DWARFEmitter.h"
-#include "DWARFVisitor.h"
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/Dwarf.h"
@@ -85,6 +85,12 @@ static void writeInitialLength(const dwarf::DwarfFormat Format,
       writeVariableSizedInteger(Length, IsDWARF64 ? 8 : 4, OS, IsLittleEndian));
 }
 
+static void writeDWARFOffset(uint64_t Offset, dwarf::DwarfFormat Format,
+                             raw_ostream &OS, bool IsLittleEndian) {
+  cantFail(writeVariableSizedInteger(Offset, Format == dwarf::DWARF64 ? 8 : 4,
+                                     OS, IsLittleEndian));
+}
+
 Error DWARFYAML::emitDebugStr(raw_ostream &OS, const DWARFYAML::Data &DI) {
   for (auto Str : DI.DebugStrings) {
     OS.write(Str.data(), Str.size());
@@ -123,10 +129,7 @@ Error DWARFYAML::emitDebugAranges(raw_ostream &OS, const DWARFYAML::Data &DI) {
     auto HeaderStart = OS.tell();
     writeInitialLength(Range.Format, Range.Length, OS, DI.IsLittleEndian);
     writeInteger((uint16_t)Range.Version, OS, DI.IsLittleEndian);
-    if (Range.Format == dwarf::DWARF64)
-      writeInteger((uint64_t)Range.CuOffset, OS, DI.IsLittleEndian);
-    else
-      writeInteger((uint32_t)Range.CuOffset, OS, DI.IsLittleEndian);
+    writeDWARFOffset(Range.CuOffset, Range.Format, OS, DI.IsLittleEndian);
     writeInteger((uint8_t)Range.AddrSize, OS, DI.IsLittleEndian);
     writeInteger((uint8_t)Range.SegSize, OS, DI.IsLittleEndian);
 
@@ -204,79 +207,180 @@ Error DWARFYAML::emitPubSection(raw_ostream &OS,
   return Error::success();
 }
 
-namespace {
-/// An extension of the DWARFYAML::ConstVisitor which writes compile
-/// units and DIEs to a stream.
-class DumpVisitor : public DWARFYAML::ConstVisitor {
-  raw_ostream &OS;
-
-protected:
-  void onStartCompileUnit(const DWARFYAML::Unit &CU) override {
-    writeInitialLength(CU.Format, CU.Length, OS, DebugInfo.IsLittleEndian);
-    writeInteger((uint16_t)CU.Version, OS, DebugInfo.IsLittleEndian);
-    if (CU.Version >= 5) {
-      writeInteger((uint8_t)CU.Type, OS, DebugInfo.IsLittleEndian);
-      writeInteger((uint8_t)CU.AddrSize, OS, DebugInfo.IsLittleEndian);
-      cantFail(writeVariableSizedInteger(CU.AbbrOffset,
-                                         CU.Format == dwarf::DWARF64 ? 8 : 4,
-                                         OS, DebugInfo.IsLittleEndian));
-    } else {
-      cantFail(writeVariableSizedInteger(CU.AbbrOffset,
-                                         CU.Format == dwarf::DWARF64 ? 8 : 4,
-                                         OS, DebugInfo.IsLittleEndian));
-      writeInteger((uint8_t)CU.AddrSize, OS, DebugInfo.IsLittleEndian);
-    }
-  }
-
-  void onStartDIE(const DWARFYAML::Unit &CU,
-                  const DWARFYAML::Entry &DIE) override {
-    encodeULEB128(DIE.AbbrCode, OS);
-  }
-
-  void onValue(const uint8_t U) override {
-    writeInteger(U, OS, DebugInfo.IsLittleEndian);
-  }
-
-  void onValue(const uint16_t U) override {
-    writeInteger(U, OS, DebugInfo.IsLittleEndian);
-  }
-
-  void onValue(const uint32_t U) override {
-    writeInteger(U, OS, DebugInfo.IsLittleEndian);
+static Expected<uint64_t> writeDIE(ArrayRef<DWARFYAML::Abbrev> AbbrevDecls,
+                                   const DWARFYAML::Unit &Unit,
+                                   const DWARFYAML::Entry &Entry,
+                                   raw_ostream &OS, bool IsLittleEndian) {
+  uint64_t EntryBegin = OS.tell();
+  encodeULEB128(Entry.AbbrCode, OS);
+  uint32_t AbbrCode = Entry.AbbrCode;
+  if (AbbrCode == 0 || Entry.Values.empty())
+    return OS.tell() - EntryBegin;
+
+  if (AbbrCode > AbbrevDecls.size())
+    return createStringError(
+        errc::invalid_argument,
+        "abbrev code must be less than or equal to the number of "
+        "entries in abbreviation table");
+  const DWARFYAML::Abbrev &Abbrev = AbbrevDecls[AbbrCode - 1];
+  auto FormVal = Entry.Values.begin();
+  auto AbbrForm = Abbrev.Attributes.begin();
+  for (; FormVal != Entry.Values.end() && AbbrForm != Abbrev.Attributes.end();
+       ++FormVal, ++AbbrForm) {
+    dwarf::Form Form = AbbrForm->Form;
+    bool Indirect;
+    do {
+      Indirect = false;
+      switch (Form) {
+      case dwarf::DW_FORM_addr:
+        // TODO: Test this error.
+        if (Error Err = writeVariableSizedInteger(
+                FormVal->Value, Unit.FormParams.AddrSize, OS, IsLittleEndian))
+          return std::move(Err);
+        break;
+      case dwarf::DW_FORM_ref_addr:
+        // TODO: Test this error.
+        if (Error Err = writeVariableSizedInteger(
+                FormVal->Value, Unit.FormParams.getRefAddrByteSize(), OS,
+                IsLittleEndian))
+          return std::move(Err);
+        break;
+      case dwarf::DW_FORM_exprloc:
+      case dwarf::DW_FORM_block:
+        encodeULEB128(FormVal->BlockData.size(), OS);
+        OS.write((const char *)FormVal->BlockData.data(),
+                 FormVal->BlockData.size());
+        break;
+      case dwarf::DW_FORM_block1: {
+        writeInteger((uint8_t)FormVal->BlockData.size(), OS, IsLittleEndian);
+        OS.write((const char *)FormVal->BlockData.data(),
+                 FormVal->BlockData.size());
+        break;
+      }
+      case dwarf::DW_FORM_block2: {
+        writeInteger((uint16_t)FormVal->BlockData.size(), OS, IsLittleEndian);
+        OS.write((const char *)FormVal->BlockData.data(),
+                 FormVal->BlockData.size());
+        break;
+      }
+      case dwarf::DW_FORM_block4: {
+        writeInteger((uint32_t)FormVal->BlockData.size(), OS, IsLittleEndian);
+        OS.write((const char *)FormVal->BlockData.data(),
+                 FormVal->BlockData.size());
+        break;
+      }
+      case dwarf::DW_FORM_strx:
+      case dwarf::DW_FORM_addrx:
+      case dwarf::DW_FORM_rnglistx:
+      case dwarf::DW_FORM_loclistx:
+      case dwarf::DW_FORM_udata:
+      case dwarf::DW_FORM_ref_udata:
+      case dwarf::DW_FORM_GNU_addr_index:
+      case dwarf::DW_FORM_GNU_str_index:
+        encodeULEB128(FormVal->Value, OS);
+        break;
+      case dwarf::DW_FORM_data1:
+      case dwarf::DW_FORM_ref1:
+      case dwarf::DW_FORM_flag:
+      case dwarf::DW_FORM_strx1:
+      case dwarf::DW_FORM_addrx1:
+        writeInteger((uint8_t)FormVal->Value, OS, IsLittleEndian);
+        break;
+      case dwarf::DW_FORM_data2:
+      case dwarf::DW_FORM_ref2:
+      case dwarf::DW_FORM_strx2:
+      case dwarf::DW_FORM_addrx2:
+        writeInteger((uint16_t)FormVal->Value, OS, IsLittleEndian);
+        break;
+      case dwarf::DW_FORM_data4:
+      case dwarf::DW_FORM_ref4:
+      case dwarf::DW_FORM_ref_sup4:
+      case dwarf::DW_FORM_strx4:
+      case dwarf::DW_FORM_addrx4:
+        writeInteger((uint32_t)FormVal->Value, OS, IsLittleEndian);
+        break;
+      case dwarf::DW_FORM_data8:
+      case dwarf::DW_FORM_ref8:
+      case dwarf::DW_FORM_ref_sup8:
+      case dwarf::DW_FORM_ref_sig8:
+        writeInteger((uint64_t)FormVal->Value, OS, IsLittleEndian);
+        break;
+      case dwarf::DW_FORM_sdata:
+        encodeSLEB128(FormVal->Value, OS);
+        break;
+      case dwarf::DW_FORM_string:
+        OS.write(FormVal->CStr.data(), FormVal->CStr.size());
+        OS.write('\0');
+        break;
+      case dwarf::DW_FORM_indirect:
+        encodeULEB128(FormVal->Value, OS);
+        Indirect = true;
+        Form = static_cast<dwarf::Form>((uint64_t)FormVal->Value);
+        ++FormVal;
+        break;
+      case dwarf::DW_FORM_strp:
+      case dwarf::DW_FORM_sec_offset:
+      case dwarf::DW_FORM_GNU_ref_alt:
+      case dwarf::DW_FORM_GNU_strp_alt:
+      case dwarf::DW_FORM_line_strp:
+      case dwarf::DW_FORM_strp_sup:
+        cantFail(writeVariableSizedInteger(
+            FormVal->Value, Unit.FormParams.getDwarfOffsetByteSize(), OS,
+            IsLittleEndian));
+        break;
+      default:
+        break;
+      }
+    } while (Indirect);
   }
 
-  void onValue(const uint64_t U, const bool LEB = false) override {
-    if (LEB)
-      encodeULEB128(U, OS);
-    else
-      writeInteger(U, OS, DebugInfo.IsLittleEndian);
-  }
+  return OS.tell() - EntryBegin;
+}
 
-  void onValue(const int64_t S, const bool LEB = false) override {
-    if (LEB)
-      encodeSLEB128(S, OS);
-    else
-      writeInteger(S, OS, DebugInfo.IsLittleEndian);
-  }
+Error DWARFYAML::emitDebugInfo(raw_ostream &OS, const DWARFYAML::Data &DI) {
+  for (const DWARFYAML::Unit &Unit : DI.CompileUnits) {
+    uint64_t Length = 3; // sizeof(version) + sizeof(address_size)
+    Length += Unit.FormParams.Version >= 5 ? 1 : 0; // sizeof(unit_type)
+    Length +=
+        Unit.FormParams.getDwarfOffsetByteSize(); // sizeof(debug_abbrev_offset)
+
+    // Since the length of the current compilation unit is undetermined yet, we
+    // firstly write the content of the compilation unit to a buffer to
+    // calculate it and then serialize the buffer content to the actual output
+    // stream.
+    std::string EntryBuffer;
+    raw_string_ostream EntryBufferOS(EntryBuffer);
+
+    for (const DWARFYAML::Entry &Entry : Unit.Entries) {
+      if (Expected<uint64_t> EntryLength = writeDIE(
+              DI.AbbrevDecls, Unit, Entry, EntryBufferOS, DI.IsLittleEndian))
+        Length += *EntryLength;
+      else
+        return EntryLength.takeError();
+    }
 
-  void onValue(const StringRef String) override {
-    OS.write(String.data(), String.size());
-    OS.write('\0');
-  }
+    // If the length is specified in the YAML description, we use it instead of
+    // the actual length.
+    if (Unit.Length)
+      Length = *Unit.Length;
+
+    writeInitialLength(Unit.FormParams.Format, Length, OS, DI.IsLittleEndian);
+    writeInteger((uint16_t)Unit.FormParams.Version, OS, DI.IsLittleEndian);
+    if (Unit.FormParams.Version >= 5) {
+      writeInteger((uint8_t)Unit.Type, OS, DI.IsLittleEndian);
+      writeInteger((uint8_t)Unit.FormParams.AddrSize, OS, DI.IsLittleEndian);
+      writeDWARFOffset(Unit.AbbrOffset, Unit.FormParams.Format, OS,
+                       DI.IsLittleEndian);
+    } else {
+      writeDWARFOffset(Unit.AbbrOffset, Unit.FormParams.Format, OS,
+                       DI.IsLittleEndian);
+      writeInteger((uint8_t)Unit.FormParams.AddrSize, OS, DI.IsLittleEndian);
+    }
 
-  void onValue(const MemoryBufferRef MBR) override {
-    OS.write(MBR.getBufferStart(), MBR.getBufferSize());
+    OS.write(EntryBuffer.data(), EntryBuffer.size());
   }
 
-public:
-  DumpVisitor(const DWARFYAML::Data &DI, raw_ostream &Out)
-      : DWARFYAML::ConstVisitor(DI), OS(Out) {}
-};
-} // namespace
-
-Error DWARFYAML::emitDebugInfo(raw_ostream &OS, const DWARFYAML::Data &DI) {
-  DumpVisitor Visitor(DI, OS);
-  return Visitor.traverseDebugInfo();
+  return Error::success();
 }
 
 static void emitFileEntry(raw_ostream &OS, const DWARFYAML::File &File) {
@@ -326,7 +430,8 @@ Error DWARFYAML::emitDebugLine(raw_ostream &OS, const DWARFYAML::Data &DI) {
         case dwarf::DW_LNE_set_discriminator:
           // TODO: Test this error.
           if (Error Err = writeVariableSizedInteger(
-                  Op.Data, DI.CompileUnits[0].AddrSize, OS, DI.IsLittleEndian))
+                  Op.Data, DI.CompileUnits[0].FormParams.AddrSize, OS,
+                  DI.IsLittleEndian))
             return Err;
           break;
         case dwarf::DW_LNE_define_file:
@@ -431,16 +536,186 @@ Error DWARFYAML::emitDebugStrOffsets(raw_ostream &OS, const Data &DI) {
     writeInteger((uint16_t)Table.Version, OS, DI.IsLittleEndian);
     writeInteger((uint16_t)Table.Padding, OS, DI.IsLittleEndian);
 
-    for (uint64_t Offset : Table.Offsets) {
-      cantFail(writeVariableSizedInteger(Offset,
-                                         Table.Format == dwarf::DWARF64 ? 8 : 4,
-                                         OS, DI.IsLittleEndian));
+    for (uint64_t Offset : Table.Offsets)
+      writeDWARFOffset(Offset, Table.Format, OS, DI.IsLittleEndian);
+  }
+
+  return Error::success();
+}
+
+static Error checkListEntryOperands(StringRef EncodingString,
+                                    ArrayRef<yaml::Hex64> Values,
+                                    uint64_t ExpectedOperands) {
+  if (Values.size() != ExpectedOperands)
+    return createStringError(
+        errc::invalid_argument,
+        "invalid number (%zu) of operands for the operator: %s, %" PRIu64
+        " expected",
+        Values.size(), EncodingString.str().c_str(), ExpectedOperands);
+
+  return Error::success();
+}
+
+static Error writeListEntryAddress(StringRef EncodingName, raw_ostream &OS,
+                                   uint64_t Addr, uint8_t AddrSize,
+                                   bool IsLittleEndian) {
+  if (Error Err = writeVariableSizedInteger(Addr, AddrSize, OS, IsLittleEndian))
+    return createStringError(errc::invalid_argument,
+                             "unable to write address for the operator %s: %s",
+                             EncodingName.str().c_str(),
+                             toString(std::move(Err)).c_str());
+
+  return Error::success();
+}
+
+static Expected<uint64_t> writeListEntry(raw_ostream &OS,
+                                         const DWARFYAML::RnglistEntry &Entry,
+                                         uint8_t AddrSize,
+                                         bool IsLittleEndian) {
+  uint64_t BeginOffset = OS.tell();
+  writeInteger((uint8_t)Entry.Operator, OS, IsLittleEndian);
+
+  StringRef EncodingName = dwarf::RangeListEncodingString(Entry.Operator);
+
+  auto CheckOperands = [&](uint64_t ExpectedOperands) -> Error {
+    return checkListEntryOperands(EncodingName, Entry.Values, ExpectedOperands);
+  };
+
+  auto WriteAddress = [&](uint64_t Addr) -> Error {
+    return writeListEntryAddress(EncodingName, OS, Addr, AddrSize,
+                                 IsLittleEndian);
+  };
+
+  switch (Entry.Operator) {
+  case dwarf::DW_RLE_end_of_list:
+    if (Error Err = CheckOperands(0))
+      return std::move(Err);
+    break;
+  case dwarf::DW_RLE_base_addressx:
+    if (Error Err = CheckOperands(1))
+      return std::move(Err);
+    encodeULEB128(Entry.Values[0], OS);
+    break;
+  case dwarf::DW_RLE_startx_endx:
+  case dwarf::DW_RLE_startx_length:
+  case dwarf::DW_RLE_offset_pair:
+    if (Error Err = CheckOperands(2))
+      return std::move(Err);
+    encodeULEB128(Entry.Values[0], OS);
+    encodeULEB128(Entry.Values[1], OS);
+    break;
+  case dwarf::DW_RLE_base_address:
+    if (Error Err = CheckOperands(1))
+      return std::move(Err);
+    if (Error Err = WriteAddress(Entry.Values[0]))
+      return std::move(Err);
+    break;
+  case dwarf::DW_RLE_start_end:
+    if (Error Err = CheckOperands(2))
+      return std::move(Err);
+    if (Error Err = WriteAddress(Entry.Values[0]))
+      return std::move(Err);
+    cantFail(WriteAddress(Entry.Values[1]));
+    break;
+  case dwarf::DW_RLE_start_length:
+    if (Error Err = CheckOperands(2))
+      return std::move(Err);
+    if (Error Err = WriteAddress(Entry.Values[0]))
+      return std::move(Err);
+    encodeULEB128(Entry.Values[1], OS);
+    break;
+  }
+
+  return OS.tell() - BeginOffset;
+}
+
+template <typename EntryType>
+Error writeDWARFLists(raw_ostream &OS,
+                      ArrayRef<DWARFYAML::ListTable<EntryType>> Tables,
+                      bool IsLittleEndian, bool Is64BitAddrSize) {
+  for (const DWARFYAML::ListTable<EntryType> &Table : Tables) {
+    // sizeof(version) + sizeof(address_size) + sizeof(segment_selector_size) +
+    // sizeof(offset_entry_count) = 8
+    uint64_t Length = 8;
+
+    uint8_t AddrSize;
+    if (Table.AddrSize)
+      AddrSize = *Table.AddrSize;
+    else
+      AddrSize = Is64BitAddrSize ? 8 : 4;
+
+    // Since the length of the current range/location lists entry is
+    // undetermined yet, we firstly write the content of the range/location
+    // lists to a buffer to calculate the length and then serialize the buffer
+    // content to the actual output stream.
+    std::string ListBuffer;
+    raw_string_ostream ListBufferOS(ListBuffer);
+
+    // Offsets holds offsets for each range/location list. The i-th element is
+    // the offset from the beginning of the first range/location list to the
+    // location of the i-th range list.
+    std::vector<uint64_t> Offsets;
+
+    for (const DWARFYAML::ListEntries<EntryType> &List : Table.Lists) {
+      Offsets.push_back(ListBufferOS.tell());
+      for (const EntryType &Entry : List.Entries) {
+        Expected<uint64_t> EntrySize =
+            writeListEntry(ListBufferOS, Entry, AddrSize, IsLittleEndian);
+        if (!EntrySize)
+          return EntrySize.takeError();
+        Length += *EntrySize;
+      }
     }
+
+    // If the offset_entry_count field isn't specified, yaml2obj will infer it
+    // from the 'Offsets' field in the YAML description. If the 'Offsets' field
+    // isn't specified either, yaml2obj will infer it from the auto-generated
+    // offsets.
+    uint32_t OffsetEntryCount;
+    if (Table.OffsetEntryCount)
+      OffsetEntryCount = *Table.OffsetEntryCount;
+    else
+      OffsetEntryCount = Table.Offsets ? Table.Offsets->size() : Offsets.size();
+    uint64_t OffsetsSize =
+        OffsetEntryCount * (Table.Format == dwarf::DWARF64 ? 8 : 4);
+    Length += OffsetsSize;
+
+    // If the length is specified in the YAML description, we use it instead of
+    // the actual length.
+    if (Table.Length)
+      Length = *Table.Length;
+
+    writeInitialLength(Table.Format, Length, OS, IsLittleEndian);
+    writeInteger((uint16_t)Table.Version, OS, IsLittleEndian);
+    writeInteger((uint8_t)AddrSize, OS, IsLittleEndian);
+    writeInteger((uint8_t)Table.SegSelectorSize, OS, IsLittleEndian);
+    writeInteger((uint32_t)OffsetEntryCount, OS, IsLittleEndian);
+
+    auto EmitOffsets = [&](ArrayRef<uint64_t> Offsets, uint64_t OffsetsSize) {
+      for (uint64_t Offset : Offsets)
+        writeDWARFOffset(OffsetsSize + Offset, Table.Format, OS,
+                         IsLittleEndian);
+    };
+
+    if (Table.Offsets)
+      EmitOffsets(ArrayRef<uint64_t>((const uint64_t *)Table.Offsets->data(),
+                                     Table.Offsets->size()),
+                  0);
+    else
+      EmitOffsets(Offsets, OffsetsSize);
+
+    OS.write(ListBuffer.data(), ListBuffer.size());
   }
 
   return Error::success();
 }
 
+Error DWARFYAML::emitDebugRnglists(raw_ostream &OS, const Data &DI) {
+  assert(DI.DebugRnglists && "unexpected emitDebugRnglists() call");
+  return writeDWARFLists<DWARFYAML::RnglistEntry>(
+      OS, *DI.DebugRnglists, DI.IsLittleEndian, DI.Is64BitAddrSize);
+}
+
 using EmitFuncType = Error (*)(raw_ostream &, const DWARFYAML::Data &);
 
 static Error
@@ -458,51 +733,8 @@ emitDebugSectionImpl(const DWARFYAML::Data &DI, EmitFuncType EmitFunc,
   return Error::success();
 }
 
-namespace {
-class DIEFixupVisitor : public DWARFYAML::Visitor {
-  uint64_t Length;
-
-public:
-  DIEFixupVisitor(DWARFYAML::Data &DI) : DWARFYAML::Visitor(DI){};
-
-protected:
-  void onStartCompileUnit(DWARFYAML::Unit &CU) override {
-    // Size of the unit header, excluding the length field itself.
-    Length = CU.Version >= 5 ? 8 : 7;
-  }
-
-  void onEndCompileUnit(DWARFYAML::Unit &CU) override { CU.Length = Length; }
-
-  void onStartDIE(DWARFYAML::Unit &CU, DWARFYAML::Entry &DIE) override {
-    Length += getULEB128Size(DIE.AbbrCode);
-  }
-
-  void onValue(const uint8_t U) override { Length += 1; }
-  void onValue(const uint16_t U) override { Length += 2; }
-  void onValue(const uint32_t U) override { Length += 4; }
-  void onValue(const uint64_t U, const bool LEB = false) override {
-    if (LEB)
-      Length += getULEB128Size(U);
-    else
-      Length += 8;
-  }
-  void onValue(const int64_t S, const bool LEB = false) override {
-    if (LEB)
-      Length += getSLEB128Size(S);
-    else
-      Length += 8;
-  }
-  void onValue(const StringRef String) override { Length += String.size() + 1; }
-
-  void onValue(const MemoryBufferRef MBR) override {
-    Length += MBR.getBufferSize();
-  }
-};
-} // namespace
-
 Expected<StringMap<std::unique_ptr<MemoryBuffer>>>
-DWARFYAML::emitDebugSections(StringRef YAMLString, bool ApplyFixups,
-                             bool IsLittleEndian) {
+DWARFYAML::emitDebugSections(StringRef YAMLString, bool IsLittleEndian) {
   auto CollectDiagnostic = [](const SMDiagnostic &Diag, void *DiagContext) {
     *static_cast<SMDiagnostic *>(DiagContext) = Diag;
   };
@@ -517,12 +749,6 @@ DWARFYAML::emitDebugSections(StringRef YAMLString, bool ApplyFixups,
   if (YIn.error())
     return createStringError(YIn.error(), GeneratedDiag.getMessage());
 
-  if (ApplyFixups) {
-    DIEFixupVisitor DIFixer(DI);
-    if (Error Err = DIFixer.traverseDebugInfo())
-      return std::move(Err);
-  }
-
   StringMap<std::unique_ptr<MemoryBuffer>> DebugSections;
   Error Err = emitDebugSectionImpl(DI, &DWARFYAML::emitDebugInfo, "debug_info",
                                    DebugSections);
diff --git a/llvm/lib/ObjectYAML/DWARFVisitor.cpp b/llvm/lib/ObjectYAML/DWARFVisitor.cpp
deleted file mode 100644
index bea71fb3235d2..0000000000000
--- a/llvm/lib/ObjectYAML/DWARFVisitor.cpp
+++ /dev/null
@@ -1,188 +0,0 @@
-//===--- DWARFVisitor.cpp ---------------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-//===----------------------------------------------------------------------===//
-
-#include "DWARFVisitor.h"
-#include "llvm/BinaryFormat/Dwarf.h"
-#include "llvm/ObjectYAML/DWARFYAML.h"
-#include "llvm/Support/Errc.h"
-#include "llvm/Support/Error.h"
-
-using namespace llvm;
-
-template <typename T>
-void DWARFYAML::VisitorImpl<T>::onVariableSizeValue(uint64_t U, unsigned Size) {
-  switch (Size) {
-  case 8:
-    onValue((uint64_t)U);
-    break;
-  case 4:
-    onValue((uint32_t)U);
-    break;
-  case 2:
-    onValue((uint16_t)U);
-    break;
-  case 1:
-    onValue((uint8_t)U);
-    break;
-  default:
-    llvm_unreachable("Invalid integer write size.");
-  }
-}
-
-static unsigned getOffsetSize(const DWARFYAML::Unit &Unit) {
-  return Unit.Format == dwarf::DWARF64 ? 8 : 4;
-}
-
-static unsigned getRefSize(const DWARFYAML::Unit &Unit) {
-  if (Unit.Version == 2)
-    return Unit.AddrSize;
-  return getOffsetSize(Unit);
-}
-
-template <typename T> Error DWARFYAML::VisitorImpl<T>::traverseDebugInfo() {
-  for (auto &Unit : DebugInfo.CompileUnits) {
-    onStartCompileUnit(Unit);
-
-    for (auto &Entry : Unit.Entries) {
-      onStartDIE(Unit, Entry);
-      uint32_t AbbrCode = Entry.AbbrCode;
-      if (AbbrCode == 0 || Entry.Values.empty())
-        continue;
-
-      if (AbbrCode > DebugInfo.AbbrevDecls.size())
-        return createStringError(
-            errc::invalid_argument,
-            "abbrev code must be less than or equal to the number of "
-            "entries in abbreviation table");
-      const DWARFYAML::Abbrev &Abbrev = DebugInfo.AbbrevDecls[AbbrCode - 1];
-      auto FormVal = Entry.Values.begin();
-      auto AbbrForm = Abbrev.Attributes.begin();
-      for (;
-           FormVal != Entry.Values.end() && AbbrForm != Abbrev.Attributes.end();
-           ++FormVal, ++AbbrForm) {
-        onForm(*AbbrForm, *FormVal);
-        dwarf::Form Form = AbbrForm->Form;
-        bool Indirect;
-        do {
-          Indirect = false;
-          switch (Form) {
-          case dwarf::DW_FORM_addr:
-            onVariableSizeValue(FormVal->Value, Unit.AddrSize);
-            break;
-          case dwarf::DW_FORM_ref_addr:
-            onVariableSizeValue(FormVal->Value, getRefSize(Unit));
-            break;
-          case dwarf::DW_FORM_exprloc:
-          case dwarf::DW_FORM_block:
-            onValue((uint64_t)FormVal->BlockData.size(), true);
-            onValue(
-                MemoryBufferRef(StringRef((const char *)&FormVal->BlockData[0],
-                                          FormVal->BlockData.size()),
-                                ""));
-            break;
-          case dwarf::DW_FORM_block1: {
-            auto writeSize = FormVal->BlockData.size();
-            onValue((uint8_t)writeSize);
-            onValue(
-                MemoryBufferRef(StringRef((const char *)&FormVal->BlockData[0],
-                                          FormVal->BlockData.size()),
-                                ""));
-            break;
-          }
-          case dwarf::DW_FORM_block2: {
-            auto writeSize = FormVal->BlockData.size();
-            onValue((uint16_t)writeSize);
-            onValue(
-                MemoryBufferRef(StringRef((const char *)&FormVal->BlockData[0],
-                                          FormVal->BlockData.size()),
-                                ""));
-            break;
-          }
-          case dwarf::DW_FORM_block4: {
-            auto writeSize = FormVal->BlockData.size();
-            onValue((uint32_t)writeSize);
-            onValue(
-                MemoryBufferRef(StringRef((const char *)&FormVal->BlockData[0],
-                                          FormVal->BlockData.size()),
-                                ""));
-            break;
-          }
-          case dwarf::DW_FORM_strx:
-          case dwarf::DW_FORM_addrx:
-          case dwarf::DW_FORM_rnglistx:
-          case dwarf::DW_FORM_loclistx:
-          case dwarf::DW_FORM_udata:
-          case dwarf::DW_FORM_ref_udata:
-          case dwarf::DW_FORM_GNU_addr_index:
-          case dwarf::DW_FORM_GNU_str_index:
-            onValue((uint64_t)FormVal->Value, /*LEB=*/true);
-            break;
-          case dwarf::DW_FORM_data1:
-          case dwarf::DW_FORM_ref1:
-          case dwarf::DW_FORM_flag:
-          case dwarf::DW_FORM_strx1:
-          case dwarf::DW_FORM_addrx1:
-            onValue((uint8_t)FormVal->Value);
-            break;
-          case dwarf::DW_FORM_data2:
-          case dwarf::DW_FORM_ref2:
-          case dwarf::DW_FORM_strx2:
-          case dwarf::DW_FORM_addrx2:
-            onValue((uint16_t)FormVal->Value);
-            break;
-          case dwarf::DW_FORM_data4:
-          case dwarf::DW_FORM_ref4:
-          case dwarf::DW_FORM_ref_sup4:
-          case dwarf::DW_FORM_strx4:
-          case dwarf::DW_FORM_addrx4:
-            onValue((uint32_t)FormVal->Value);
-            break;
-          case dwarf::DW_FORM_data8:
-          case dwarf::DW_FORM_ref8:
-          case dwarf::DW_FORM_ref_sup8:
-          case dwarf::DW_FORM_ref_sig8:
-            onValue((uint64_t)FormVal->Value);
-            break;
-          case dwarf::DW_FORM_sdata:
-            onValue((int64_t)FormVal->Value, true);
-            break;
-          case dwarf::DW_FORM_string:
-            onValue(FormVal->CStr);
-            break;
-          case dwarf::DW_FORM_indirect:
-            onValue((uint64_t)FormVal->Value, true);
-            Indirect = true;
-            Form = static_cast<dwarf::Form>((uint64_t)FormVal->Value);
-            ++FormVal;
-            break;
-          case dwarf::DW_FORM_strp:
-          case dwarf::DW_FORM_sec_offset:
-          case dwarf::DW_FORM_GNU_ref_alt:
-          case dwarf::DW_FORM_GNU_strp_alt:
-          case dwarf::DW_FORM_line_strp:
-          case dwarf::DW_FORM_strp_sup:
-            onVariableSizeValue(FormVal->Value, getOffsetSize(Unit));
-            break;
-          default:
-            break;
-          }
-        } while (Indirect);
-      }
-      onEndDIE(Unit, Entry);
-    }
-    onEndCompileUnit(Unit);
-  }
-
-  return Error::success();
-}
-
-// Explicitly instantiate the two template expansions.
-template class DWARFYAML::VisitorImpl<DWARFYAML::Data>;
-template class DWARFYAML::VisitorImpl<const DWARFYAML::Data>;
diff --git a/llvm/lib/ObjectYAML/DWARFVisitor.h b/llvm/lib/ObjectYAML/DWARFVisitor.h
deleted file mode 100644
index 3b2c4303c7f7f..0000000000000
--- a/llvm/lib/ObjectYAML/DWARFVisitor.h
+++ /dev/null
@@ -1,97 +0,0 @@
-//===--- DWARFVisitor.h -----------------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_OBJECTYAML_DWARFVISITOR_H
-#define LLVM_OBJECTYAML_DWARFVISITOR_H
-
-#include "llvm/ADT/StringRef.h"
-#include "llvm/BinaryFormat/Dwarf.h"
-#include "llvm/Support/MemoryBuffer.h"
-
-namespace llvm {
-class Error;
-
-namespace DWARFYAML {
-
-struct Data;
-struct Unit;
-struct Entry;
-struct FormValue;
-struct AttributeAbbrev;
-
-/// A class to visits DWARFYAML Compile Units and DIEs in preorder.
-///
-/// Extensions of this class can either maintain const or non-const references
-/// to the DWARFYAML::Data object.
-template <typename T> class VisitorImpl {
-protected:
-  T &DebugInfo;
-
-  /// Visitor Functions
-  /// @{
-  virtual void onStartCompileUnit(Unit &CU) {}
-  virtual void onEndCompileUnit(Unit &CU) {}
-  virtual void onStartDIE(Unit &CU, Entry &DIE) {}
-  virtual void onEndDIE(Unit &CU, Entry &DIE) {}
-  virtual void onForm(AttributeAbbrev &AttAbbrev, FormValue &Value) {}
-  /// @}
-
-  /// Const Visitor Functions
-  /// @{
-  virtual void onStartCompileUnit(const Unit &CU) {}
-  virtual void onEndCompileUnit(const Unit &CU) {}
-  virtual void onStartDIE(const Unit &CU, const Entry &DIE) {}
-  virtual void onEndDIE(const Unit &CU, const Entry &DIE) {}
-  virtual void onForm(const AttributeAbbrev &AttAbbrev,
-                      const FormValue &Value) {}
-  /// @}
-
-  /// Value visitors
-  /// @{
-  virtual void onValue(const uint8_t U) {}
-  virtual void onValue(const uint16_t U) {}
-  virtual void onValue(const uint32_t U) {}
-  virtual void onValue(const uint64_t U, const bool LEB = false) {}
-  virtual void onValue(const int64_t S, const bool LEB = false) {}
-  virtual void onValue(const StringRef String) {}
-  virtual void onValue(const MemoryBufferRef MBR) {}
-  /// @}
-
-public:
-  VisitorImpl(T &DI) : DebugInfo(DI) {}
-
-  virtual ~VisitorImpl() {}
-
-  Error traverseDebugInfo();
-
-private:
-  void onVariableSizeValue(uint64_t U, unsigned Size);
-};
-
-// Making the visior instantiations extern and explicit in the cpp file. This
-// prevents them from being instantiated in every compile unit that uses the
-// visitors.
-extern template class VisitorImpl<DWARFYAML::Data>;
-extern template class VisitorImpl<const DWARFYAML::Data>;
-
-class Visitor : public VisitorImpl<Data> {
-public:
-  Visitor(Data &DI) : VisitorImpl<Data>(DI) {}
-};
-
-class ConstVisitor : public VisitorImpl<const Data> {
-public:
-  ConstVisitor(const Data &DI) : VisitorImpl<const Data>(DI) {}
-};
-
-} // namespace DWARFYAML
-} // namespace llvm
-
-#endif
diff --git a/llvm/lib/ObjectYAML/DWARFYAML.cpp b/llvm/lib/ObjectYAML/DWARFYAML.cpp
index 4ed3b48b67852..186ec8ac4bdc3 100644
--- a/llvm/lib/ObjectYAML/DWARFYAML.cpp
+++ b/llvm/lib/ObjectYAML/DWARFYAML.cpp
@@ -48,6 +48,8 @@ SetVector<StringRef> DWARFYAML::Data::getUsedSectionNames() const {
     SecNames.insert("debug_gnu_pubtypes");
   if (DebugStrOffsets)
     SecNames.insert("debug_str_offsets");
+  if (DebugRnglists)
+    SecNames.insert("debug_rnglists");
   return SecNames;
 }
 
@@ -72,6 +74,7 @@ void MappingTraits<DWARFYAML::Data>::mapping(IO &IO, DWARFYAML::Data &DWARF) {
   IO.mapOptional("debug_line", DWARF.DebugLines);
   IO.mapOptional("debug_addr", DWARF.DebugAddr);
   IO.mapOptional("debug_str_offsets", DWARF.DebugStrOffsets);
+  IO.mapOptional("debug_rnglists", DWARF.DebugRnglists);
   IO.setContext(OldContext);
 }
 
@@ -139,13 +142,13 @@ void MappingTraits<DWARFYAML::PubSection>::mapping(
 }
 
 void MappingTraits<DWARFYAML::Unit>::mapping(IO &IO, DWARFYAML::Unit &Unit) {
-  IO.mapOptional("Format", Unit.Format, dwarf::DWARF32);
-  IO.mapRequired("Length", Unit.Length);
-  IO.mapRequired("Version", Unit.Version);
-  if (Unit.Version >= 5)
+  IO.mapOptional("Format", Unit.FormParams.Format, dwarf::DWARF32);
+  IO.mapOptional("Length", Unit.Length);
+  IO.mapRequired("Version", Unit.FormParams.Version);
+  if (Unit.FormParams.Version >= 5)
     IO.mapRequired("UnitType", Unit.Type);
   IO.mapRequired("AbbrOffset", Unit.AbbrOffset);
-  IO.mapRequired("AddrSize", Unit.AddrSize);
+  IO.mapRequired("AddrSize", Unit.FormParams.AddrSize);
   IO.mapOptional("Entries", Unit.Entries);
 }
 
@@ -233,6 +236,31 @@ void MappingTraits<DWARFYAML::StringOffsetsTable>::mapping(
   IO.mapOptional("Offsets", StrOffsetsTable.Offsets);
 }
 
+void MappingTraits<DWARFYAML::RnglistEntry>::mapping(
+    IO &IO, DWARFYAML::RnglistEntry &RnglistEntry) {
+  IO.mapRequired("Operator", RnglistEntry.Operator);
+  IO.mapOptional("Values", RnglistEntry.Values);
+}
+
+template <typename EntryType>
+void MappingTraits<DWARFYAML::ListEntries<EntryType>>::mapping(
+    IO &IO, DWARFYAML::ListEntries<EntryType> &ListEntries) {
+  IO.mapOptional("Entries", ListEntries.Entries);
+}
+
+template <typename EntryType>
+void MappingTraits<DWARFYAML::ListTable<EntryType>>::mapping(
+    IO &IO, DWARFYAML::ListTable<EntryType> &ListTable) {
+  IO.mapOptional("Format", ListTable.Format, dwarf::DWARF32);
+  IO.mapOptional("Length", ListTable.Length);
+  IO.mapOptional("Version", ListTable.Version, 5);
+  IO.mapOptional("AddressSize", ListTable.AddrSize);
+  IO.mapOptional("SegmentSelectorSize", ListTable.SegSelectorSize, 0);
+  IO.mapOptional("OffsetEntryCount", ListTable.OffsetEntryCount);
+  IO.mapOptional("Offsets", ListTable.Offsets);
+  IO.mapOptional("Lists", ListTable.Lists);
+}
+
 void MappingTraits<DWARFYAML::InitialLength>::mapping(
     IO &IO, DWARFYAML::InitialLength &InitialLength) {
   IO.mapRequired("TotalLength", InitialLength.TotalLength);
diff --git a/llvm/lib/ObjectYAML/ELFEmitter.cpp b/llvm/lib/ObjectYAML/ELFEmitter.cpp
index 26fe1236752fa..fc80839064ee0 100644
--- a/llvm/lib/ObjectYAML/ELFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/ELFEmitter.cpp
@@ -976,6 +976,8 @@ Expected<uint64_t> emitDWARF(typename ELFT::Shdr &SHeader, StringRef Name,
                                     DWARF.IsLittleEndian, /*IsGNUStyle=*/true);
   else if (Name == ".debug_str_offsets")
     Err = DWARFYAML::emitDebugStrOffsets(*OS, DWARF);
+  else if (Name == ".debug_rnglists")
+    Err = DWARFYAML::emitDebugRnglists(*OS, DWARF);
   else
     llvm_unreachable("unexpected emitDWARF() call");
 
diff --git a/llvm/lib/ObjectYAML/LLVMBuild.txt b/llvm/lib/ObjectYAML/LLVMBuild.txt
index de43aafe51c90..94a830b3c4475 100644
--- a/llvm/lib/ObjectYAML/LLVMBuild.txt
+++ b/llvm/lib/ObjectYAML/LLVMBuild.txt
@@ -10,4 +10,4 @@
 type = Library
 name = ObjectYAML
 parent = Libraries
-required_libraries = Object Support DebugInfoCodeView MC
+required_libraries = BinaryFormat Object Support DebugInfoCodeView MC
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 1766e579c33d6..771675d35fdf2 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -31,10 +31,10 @@
 #include "llvm/Analysis/DemandedBits.h"
 #include "llvm/Analysis/DependenceAnalysis.h"
 #include "llvm/Analysis/DominanceFrontier.h"
+#include "llvm/Analysis/FunctionPropertiesAnalysis.h"
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/IVUsers.h"
 #include "llvm/Analysis/InlineAdvisor.h"
-#include "llvm/Analysis/InlineFeaturesAnalysis.h"
 #include "llvm/Analysis/InlineSizeEstimatorAnalysis.h"
 #include "llvm/Analysis/LazyCallGraph.h"
 #include "llvm/Analysis/LazyValueInfo.h"
@@ -275,6 +275,8 @@ extern cl::opt<bool> FlattenedProfileUsed;
 extern cl::opt<AttributorRunOption> AttributorRun;
 extern cl::opt<bool> EnableKnowledgeRetention;
 
+extern cl::opt<bool> EnableMatrix;
+
 const PassBuilder::OptimizationLevel PassBuilder::OptimizationLevel::O0 = {
     /*SpeedLevel*/ 0,
     /*SizeLevel*/ 0};
@@ -1093,6 +1095,11 @@ ModulePassManager PassBuilder::buildModuleOptimizationPipeline(
   OptimizePM.addPass(Float2IntPass());
   OptimizePM.addPass(LowerConstantIntrinsicsPass());
 
+  if (EnableMatrix) {
+    OptimizePM.addPass(LowerMatrixIntrinsicsPass());
+    OptimizePM.addPass(EarlyCSEPass());
+  }
+
   // FIXME: We need to run some loop optimizations to re-rotate loops after
   // simplify-cfg and others undo their rotation.
 
@@ -1763,6 +1770,8 @@ Expected<SimplifyCFGOptions> parseSimplifyCFGOptions(StringRef Params) {
       Result.convertSwitchToLookupTable(Enable);
     } else if (ParamName == "keep-loops") {
       Result.needCanonicalLoops(Enable);
+    } else if (ParamName == "hoist-common-insts") {
+      Result.hoistCommonInsts(Enable);
     } else if (ParamName == "sink-common-insts") {
       Result.sinkCommonInsts(Enable);
     } else if (Enable && ParamName.consume_front("bonus-inst-threshold=")) {
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index ad20d02436dae..edaca9ebf6090 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -129,10 +129,10 @@ FUNCTION_ANALYSIS("domtree", DominatorTreeAnalysis())
 FUNCTION_ANALYSIS("postdomtree", PostDominatorTreeAnalysis())
 FUNCTION_ANALYSIS("demanded-bits", DemandedBitsAnalysis())
 FUNCTION_ANALYSIS("domfrontier", DominanceFrontierAnalysis())
+FUNCTION_ANALYSIS("func-properties", FunctionPropertiesAnalysis())
 FUNCTION_ANALYSIS("loops", LoopAnalysis())
 FUNCTION_ANALYSIS("lazy-value-info", LazyValueAnalysis())
 FUNCTION_ANALYSIS("da", DependenceAnalysis())
-FUNCTION_ANALYSIS("inliner-features", InlineFeaturesAnalysis())
 FUNCTION_ANALYSIS("inliner-size-estimator", InlineSizeEstimatorAnalysis())
 FUNCTION_ANALYSIS("memdep", MemoryDependenceAnalysis())
 FUNCTION_ANALYSIS("memoryssa", MemorySSAAnalysis())
@@ -234,6 +234,7 @@ FUNCTION_PASS("print<domtree>", DominatorTreePrinterPass(dbgs()))
 FUNCTION_PASS("print<postdomtree>", PostDominatorTreePrinterPass(dbgs()))
 FUNCTION_PASS("print<demanded-bits>", DemandedBitsPrinterPass(dbgs()))
 FUNCTION_PASS("print<domfrontier>", DominanceFrontierPrinterPass(dbgs()))
+FUNCTION_PASS("print<func-properties>", FunctionPropertiesPrinterPass(dbgs()))
 FUNCTION_PASS("print<inline-cost>", InlineCostAnnotationPrinterPass(dbgs()))
 FUNCTION_PASS("print<inliner-size-estimator>", 
   InlineSizeEstimatorAnalysisPrinterPass(dbgs()))
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 1e1a6b98a65a3..22a7103715d56 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Passes/StandardInstrumentations.h"
+#include "llvm/ADT/Any.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/Analysis/CallGraphSCCPass.h"
 #include "llvm/Analysis/LazyCallGraph.h"
@@ -21,12 +22,19 @@
 #include "llvm/IR/IRPrintingPasses.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/PassInstrumentation.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
 
+// TODO: remove once all required passes are marked as such.
+static cl::opt<bool>
+    EnableOptnone("enable-npm-optnone", cl::init(false),
+                  cl::desc("Enable skipping optional passes optnone functions "
+                           "under new pass manager"));
+
 namespace {
 
 /// Extracting Module out of \p IR unit. Also fills a textual description
@@ -243,8 +251,32 @@ void PrintIRInstrumentation::registerCallbacks(
   }
 }
 
+void OptNoneInstrumentation::registerCallbacks(
+    PassInstrumentationCallbacks &PIC) {
+  PIC.registerBeforePassCallback(
+      [this](StringRef P, Any IR) { return this->skip(P, IR); });
+}
+
+bool OptNoneInstrumentation::skip(StringRef PassID, Any IR) {
+  if (!EnableOptnone)
+    return true;
+  const Function *F = nullptr;
+  if (any_isa<const Function *>(IR)) {
+    F = any_cast<const Function *>(IR);
+  } else if (any_isa<const Loop *>(IR)) {
+    F = any_cast<const Loop *>(IR)->getHeader()->getParent();
+  }
+  if (F && F->hasOptNone()) {
+    if (DebugLogging)
+      dbgs() << "Skipping pass: " << PassID << " (optnone)\n";
+    return false;
+  }
+  return true;
+}
+
 void StandardInstrumentations::registerCallbacks(
     PassInstrumentationCallbacks &PIC) {
   PrintIR.registerCallbacks(PIC);
   TimePasses.registerCallbacks(PIC);
+  OptNone.registerCallbacks(PIC);
 }
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index 8879674c324d5..9b429bf37d742 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -1111,8 +1111,6 @@ bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken) {
   return true;
 }
 
-// FIXME: This is to be removed after switching to the new memop value
-// profiling.
 // Parse the value profile options.
 void getMemOPSizeRangeFromOption(StringRef MemOPSizeRange, int64_t &RangeStart,
                                  int64_t &RangeLast) {
@@ -1136,12 +1134,15 @@ void getMemOPSizeRangeFromOption(StringRef MemOPSizeRange, int64_t &RangeStart,
 
 // Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime
 // aware this is an ir_level profile so it can set the version flag.
-void createIRLevelProfileFlagVar(Module &M, bool IsCS) {
+void createIRLevelProfileFlagVar(Module &M, bool IsCS,
+                                 bool InstrEntryBBEnabled) {
   const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
   Type *IntTy64 = Type::getInt64Ty(M.getContext());
   uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF);
   if (IsCS)
     ProfileVersion |= VARIANT_MASK_CSIR_PROF;
+  if (InstrEntryBBEnabled)
+    ProfileVersion |= VARIANT_MASK_INSTR_ENTRY;
   auto IRLevelVersionVariable = new GlobalVariable(
       M, IntTy64, true, GlobalValue::WeakAnyLinkage,
       Constant::getIntegerValue(IntTy64, APInt(64, ProfileVersion)), VarName);
diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index 16a69cb5457b8..9581e5b486a6f 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -154,23 +154,29 @@ bool TextInstrProfReader::hasFormat(const MemoryBuffer &Buffer) {
 Error TextInstrProfReader::readHeader() {
   Symtab.reset(new InstrProfSymtab());
   bool IsIRInstr = false;
-  if (!Line->startswith(":")) {
-    IsIRLevelProfile = false;
-    return success();
+  bool IsEntryFirst = false;
+  bool IsCS = false;
+
+  while (Line->startswith(":")) {
+    StringRef Str = Line->substr(1);
+    if (Str.equals_lower("ir"))
+      IsIRInstr = true;
+    else if (Str.equals_lower("fe"))
+      IsIRInstr = false;
+    else if (Str.equals_lower("csir")) {
+      IsIRInstr = true;
+      IsCS = true;
+    } else if (Str.equals_lower("entry_first"))
+      IsEntryFirst = true;
+    else if (Str.equals_lower("not_entry_first"))
+      IsEntryFirst = false;
+    else
+      return error(instrprof_error::bad_header);
+    ++Line;
   }
-  StringRef Str = (Line)->substr(1);
-  if (Str.equals_lower("ir"))
-    IsIRInstr = true;
-  else if (Str.equals_lower("fe"))
-    IsIRInstr = false;
-  else if (Str.equals_lower("csir")) {
-    IsIRInstr = true;
-    HasCSIRLevelProfile = true;
-  } else
-    return error(instrprof_error::bad_header);
-
-  ++Line;
   IsIRLevelProfile = IsIRInstr;
+  InstrEntryBBEnabled = IsEntryFirst;
+  HasCSIRLevelProfile = IsCS;
   return success();
 }
 
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index ccb270e0b719b..88445f186e835 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -165,8 +165,9 @@ class InstrProfRecordWriterTrait {
 
 } // end namespace llvm
 
-InstrProfWriter::InstrProfWriter(bool Sparse)
-    : Sparse(Sparse), InfoObj(new InstrProfRecordWriterTrait()) {}
+InstrProfWriter::InstrProfWriter(bool Sparse, bool InstrEntryBBEnabled)
+    : Sparse(Sparse), InstrEntryBBEnabled(InstrEntryBBEnabled),
+      InfoObj(new InstrProfRecordWriterTrait()) {}
 
 InstrProfWriter::~InstrProfWriter() { delete InfoObj; }
 
@@ -308,6 +309,9 @@ void InstrProfWriter::writeImpl(ProfOStream &OS) {
     Header.Version |= VARIANT_MASK_IR_PROF;
     Header.Version |= VARIANT_MASK_CSIR_PROF;
   }
+  if (InstrEntryBBEnabled)
+    Header.Version |= VARIANT_MASK_INSTR_ENTRY;
+
   Header.Unused = 0;
   Header.HashType = static_cast<uint64_t>(IndexedInstrProf::HashType);
   Header.HashOffset = 0;
@@ -441,6 +445,8 @@ Error InstrProfWriter::writeText(raw_fd_ostream &OS) {
     OS << "# IR level Instrumentation Flag\n:ir\n";
   else if (ProfileKind == PF_IRLevelWithCS)
     OS << "# CSIR level Instrumentation Flag\n:csir\n";
+  if (InstrEntryBBEnabled)
+    OS << "# Always instrument the function entry block\n:entry_first\n";
   InstrProfSymtab Symtab;
 
   using FuncPair = detail::DenseMapPair<uint64_t, InstrProfRecord>;
diff --git a/llvm/lib/SYCLLowerIR/CMakeLists.txt b/llvm/lib/SYCLLowerIR/CMakeLists.txt
index 892514384706c..809d97fcab587 100644
--- a/llvm/lib/SYCLLowerIR/CMakeLists.txt
+++ b/llvm/lib/SYCLLowerIR/CMakeLists.txt
@@ -7,7 +7,7 @@ if (NOT TARGET LLVMGenXIntrinsics)
     include(FetchContent)
     FetchContent_Declare(vc-intrinsics
       GIT_REPOSITORY https://github.com/intel/vc-intrinsics.git
-      GIT_TAG        cce6e48c28eb850d7dadd30841c0d95f009bbca1
+      GIT_TAG        5e35898100ebe3747ea50c91b05e679757b25703
     )
     FetchContent_MakeAvailable(vc-intrinsics)
     FetchContent_GetProperties(vc-intrinsics)
@@ -28,10 +28,14 @@ if (NOT TARGET LLVMGenXIntrinsics)
   )
 endif()
 
+set_property(GLOBAL PROPERTY LLVMGenXIntrinsics_SOURCE_PROP ${LLVMGenXIntrinsics_SOURCE_DIR})
+set_property(GLOBAL PROPERTY LLVMGenXIntrinsics_BINARY_PROP ${LLVMGenXIntrinsics_BINARY_DIR})
+
 add_llvm_component_library(LLVMSYCLLowerIR
   LowerWGScope.cpp
   LowerESIMD.cpp
   LowerESIMDVLoadVStore.cpp
+  LowerESIMDVecArg.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/SYCLLowerIR
diff --git a/llvm/lib/SYCLLowerIR/LowerESIMD.cpp b/llvm/lib/SYCLLowerIR/LowerESIMD.cpp
index 232706c101d23..38cec50ac814e 100644
--- a/llvm/lib/SYCLLowerIR/LowerESIMD.cpp
+++ b/llvm/lib/SYCLLowerIR/LowerESIMD.cpp
@@ -820,7 +820,7 @@ translateSpirvIntrinsic(CallInst *CI, StringRef SpirvIntrName,
   auto translateSpirvIntr = [&SpirvIntrName, &ESIMDToErases,
                              CI](StringRef SpvIName, auto TranslateFunc) {
     if (SpirvIntrName.consume_front(SpvIName)) {
-      Value *TranslatedV = TranslateFunc(*CI, SpirvIntrName);
+      Value *TranslatedV = TranslateFunc(*CI, SpirvIntrName.substr(1, 1));
       CI->replaceAllUsesWith(TranslatedV);
       ESIMDToErases.push_back(CI);
     }
diff --git a/llvm/lib/SYCLLowerIR/LowerESIMDVecArg.cpp b/llvm/lib/SYCLLowerIR/LowerESIMDVecArg.cpp
new file mode 100644
index 0000000000000..3488d9a9532d9
--- /dev/null
+++ b/llvm/lib/SYCLLowerIR/LowerESIMDVecArg.cpp
@@ -0,0 +1,320 @@
+//===-- ESIMDVecArgPass.cpp - lower Close To Metal (CM) constructs --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Change in function parameter type from simd* to native llvm vector type for
+// cmc compiler to generate correct code for subroutine parameter passing and
+// globals:
+//
+// Old IR:
+// ======
+//
+// Parameter %0 is of type simd*
+// define dso_local spir_func void @_Z3fooPiN2cm3gen4simdIiLi16EEE(i32
+// addrspace(4)* %C,
+//      "class._ZTSN2cm3gen4simdIiLi16EEE.cm::gen::simd" * %0)
+//      local_unnamed_addr #2 {
+//
+// New IR:
+// ======
+//
+// Translate simd* parameter (#1) to vector <16 x 32>* type and insert bitcast.
+// All users of old parameter will use result of the bitcast.
+//
+// define dso_local spir_func void @_Z3fooPiN2cm3gen4simdIiLi16EEE(i32
+// addrspace(4)* %C,
+//      <16 x i32>* %0) local_unnamed_addr #2 {
+// entry:
+// % 1 = bitcast<16 x i32> * % 0 to %
+// "class._ZTSN2cm3gen4simdIiLi16EEE.cm::gen::simd" *
+//
+//
+// Change in global variables:
+//
+// Old IR:
+// ======
+// @vc = global %"class._ZTSN2cm3gen4simdIiLi16EEE.cm::gen::simd"
+//          zeroinitializer, align 64 #0
+//
+// % call.cm.i.i = tail call<16 x i32> @llvm.genx.vload.v16i32.p4v16i32(
+//    <16 x i32> addrspace(4) * getelementptr(
+//    % "class._ZTSN2cm3gen4simdIiLi16EEE.cm::gen::simd",
+//    % "class._ZTSN2cm3gen4simdIiLi16EEE.cm::gen::simd" addrspace(4) *
+//    addrspacecast(% "class._ZTSN2cm3gen4simdIiLi16EEE.cm::gen::simd" * @vc to
+//    % "class._ZTSN2cm3gen4simdIiLi16EEE.cm::gen::simd" addrspace(4) *), i64 0,
+//    i32 0))
+//
+// New IR:
+// ======
+//
+// @0 = dso_local global <16 x i32> zeroinitializer, align 64 #0 <-- New Global
+// Variable
+//
+// % call.cm.i.i = tail call<16 x i32> @llvm.genx.vload.v16i32.p4v16i32(
+//        <16 x i32> addrspace(4) * getelementptr(
+//        % "class._ZTSN2cm3gen4simdIiLi16EEE.cm::gen::simd",
+//        % "class._ZTSN2cm3gen4simdIiLi16EEE.cm::gen::simd" addrspace(4) *
+//        addrspacecast(% "class._ZTSN2cm3gen4simdIiLi16EEE.cm::gen::simd" *
+//        bitcast(<16 x i32> * @0 to
+//        %"class._ZTSN2cm3gen4simdIiLi16EEE.cm::gen::simd" *) to %
+//        "class._ZTSN2cm3gen4simdIiLi16EEE.cm::gen::simd" addrspace(4) *),
+//        i64 0, i32 0))
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/Cloning.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "ESIMDLowerVecArg"
+
+namespace llvm {
+
+// Forward declarations
+void initializeESIMDLowerVecArgLegacyPassPass(PassRegistry &);
+ModulePass *createESIMDLowerVecArgPass();
+
+// Pass converts simd* function parameters and globals to
+// llvm's first-class vector* type.
+class ESIMDLowerVecArgPass {
+public:
+  bool run(Module &M);
+
+private:
+  DenseMap<GlobalVariable *, GlobalVariable *> OldNewGlobal;
+
+  Function *rewriteFunc(Function &F);
+  Type *getSimdArgPtrTyOrNull(Value *arg);
+  void fixGlobals(Module &M);
+  void replaceConstExprWithGlobals(Module &M);
+  ConstantExpr *createNewConstantExpr(GlobalVariable *newGlobalVar,
+                                      Type *oldGlobalType, Value *old);
+  void removeOldGlobals();
+};
+
+} // namespace llvm
+
+namespace {
+class ESIMDLowerVecArgLegacyPass : public ModulePass {
+public:
+  static char ID;
+  ESIMDLowerVecArgLegacyPass() : ModulePass(ID) {
+    initializeESIMDLowerVecArgLegacyPassPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool runOnModule(Module &M) override {
+    auto Modified = Impl.run(M);
+    return Modified;
+  }
+
+  bool doInitialization(Module &M) override { return false; }
+
+private:
+  ESIMDLowerVecArgPass Impl;
+};
+} // namespace
+
+char ESIMDLowerVecArgLegacyPass::ID = 0;
+INITIALIZE_PASS(ESIMDLowerVecArgLegacyPass, "ESIMDLowerVecArg",
+                "Translate simd ptr to native vector type", false, false)
+
+// Public interface to VecArgPass
+ModulePass *llvm::createESIMDLowerVecArgPass() {
+  return new ESIMDLowerVecArgLegacyPass();
+}
+
+// Return ptr to first-class vector type if Value is a simd*, else return
+// nullptr.
+Type *ESIMDLowerVecArgPass::getSimdArgPtrTyOrNull(Value *arg) {
+  auto ArgType = dyn_cast<PointerType>(arg->getType());
+  if (!ArgType || !ArgType->getElementType()->isStructTy())
+    return nullptr;
+  auto ContainedType = ArgType->getElementType();
+  if ((ContainedType->getStructNumElements() != 1) ||
+      !ContainedType->getStructElementType(0)->isVectorTy())
+    return nullptr;
+  return PointerType::get(ContainedType->getStructElementType(0),
+                          ArgType->getPointerAddressSpace());
+}
+
+// F may have multiple arguments of type simd*. This
+// function updates all parameters along with call
+// call sites of F.
+Function *ESIMDLowerVecArgPass::rewriteFunc(Function &F) {
+  FunctionType *FTy = F.getFunctionType();
+  Type *RetTy = FTy->getReturnType();
+  SmallVector<Type *, 8> ArgTys;
+
+  for (unsigned int i = 0; i != F.arg_size(); i++) {
+    auto Arg = F.getArg(i);
+    Type *NewTy = getSimdArgPtrTyOrNull(Arg);
+    if (NewTy) {
+      // Copy over byval type for simd* type
+      ArgTys.push_back(NewTy);
+    } else {
+      // Transfer all non-simd ptr arguments
+      ArgTys.push_back(Arg->getType());
+    }
+  }
+
+  FunctionType *NFTy = FunctionType::get(RetTy, ArgTys, false);
+
+  // Create new function body and insert into the module
+  Function *NF = Function::Create(NFTy, F.getLinkage(), F.getName());
+  F.getParent()->getFunctionList().insert(F.getIterator(), NF);
+
+  SmallVector<ReturnInst *, 8> Returns;
+  SmallVector<BitCastInst *, 8> BitCasts;
+  ValueToValueMapTy VMap;
+  for (unsigned int I = 0; I != F.arg_size(); I++) {
+    auto Arg = F.getArg(I);
+    Type *newTy = getSimdArgPtrTyOrNull(Arg);
+    if (newTy) {
+      // bitcast vector* -> simd*
+      auto BitCast = new BitCastInst(NF->getArg(I), Arg->getType());
+      BitCasts.push_back(BitCast);
+      VMap.insert(std::make_pair(Arg, BitCast));
+      continue;
+    }
+    VMap.insert(std::make_pair(Arg, NF->getArg(I)));
+  }
+
+  llvm::CloneFunctionInto(NF, &F, VMap, F.getSubprogram() != nullptr, Returns);
+
+  for (auto &B : BitCasts) {
+    NF->begin()->getInstList().push_front(B);
+  }
+
+  NF->takeName(&F);
+
+  // Fix call sites
+  SmallVector<std::pair<Instruction *, Instruction *>, 10> OldNewInst;
+  for (auto &use : F.uses()) {
+    // Use must be a call site
+    SmallVector<Value *, 10> Params;
+    auto Call = cast<CallInst>(use.getUser());
+    // Variadic functions not supported
+    assert(!Call->getFunction()->isVarArg() &&
+           "Variadic functions not supported");
+    for (unsigned int I = 0; I < Call->getNumArgOperands(); I++) {
+      auto SrcOpnd = Call->getOperand(I);
+      auto NewTy = getSimdArgPtrTyOrNull(SrcOpnd);
+      if (NewTy) {
+        auto BitCast = new BitCastInst(SrcOpnd, NewTy, "", Call);
+        Params.push_back(BitCast);
+      } else {
+        if (SrcOpnd != &F)
+          Params.push_back(SrcOpnd);
+        else
+          Params.push_back(NF);
+      }
+    }
+    // create new call instruction
+    auto NewCallInst = CallInst::Create(NFTy, NF, Params, "");
+    NewCallInst->setCallingConv(F.getCallingConv());
+    OldNewInst.push_back(std::make_pair(Call, NewCallInst));
+  }
+
+  for (auto InstPair : OldNewInst) {
+    auto OldInst = InstPair.first;
+    auto NewInst = InstPair.second;
+    ReplaceInstWithInst(OldInst, NewInst);
+  }
+
+  F.eraseFromParent();
+
+  return NF;
+}
+
+// Replace ConstantExpr if it contains old global variable.
+ConstantExpr *
+ESIMDLowerVecArgPass::createNewConstantExpr(GlobalVariable *NewGlobalVar,
+                                            Type *OldGlobalType, Value *Old) {
+  ConstantExpr *NewConstantExpr = nullptr;
+
+  if (isa<GlobalVariable>(Old)) {
+    NewConstantExpr = cast<ConstantExpr>(
+        ConstantExpr::getBitCast(NewGlobalVar, OldGlobalType));
+    return NewConstantExpr;
+  }
+
+  auto InnerMost = createNewConstantExpr(
+      NewGlobalVar, OldGlobalType, cast<ConstantExpr>(Old)->getOperand(0));
+
+  NewConstantExpr = cast<ConstantExpr>(
+      cast<ConstantExpr>(Old)->getWithOperandReplaced(0, InnerMost));
+
+  return NewConstantExpr;
+}
+
+// Globals are part of ConstantExpr. This loop iterates over
+// all such instances and replaces them with a new ConstantExpr
+// consisting of new global vector* variable.
+void ESIMDLowerVecArgPass::replaceConstExprWithGlobals(Module &M) {
+  for (auto &GlobalVars : OldNewGlobal) {
+    auto &G = *GlobalVars.first;
+    for (auto UseOfG : G.users()) {
+      auto NewGlobal = GlobalVars.second;
+      auto NewConstExpr = createNewConstantExpr(NewGlobal, G.getType(), UseOfG);
+      UseOfG->replaceAllUsesWith(NewConstExpr);
+    }
+  }
+}
+
+// This function creates new global variables of type vector* type
+// when old one is of simd* type.
+void ESIMDLowerVecArgPass::fixGlobals(Module &M) {
+  for (auto &G : M.getGlobalList()) {
+    auto NewTy = getSimdArgPtrTyOrNull(&G);
+    if (NewTy && !G.user_empty()) {
+      // Peel off ptr type that getSimdArgPtrTyOrNull applies
+      NewTy = NewTy->getPointerElementType();
+      auto ZeroInit = ConstantAggregateZero::get(NewTy);
+      auto NewGlobalVar =
+          new GlobalVariable(NewTy, G.isConstant(), G.getLinkage(), ZeroInit,
+                             "", G.getThreadLocalMode(), G.getAddressSpace());
+      NewGlobalVar->setExternallyInitialized(G.isExternallyInitialized());
+      NewGlobalVar->copyAttributesFrom(&G);
+      NewGlobalVar->takeName(&G);
+      NewGlobalVar->copyMetadata(&G, 0);
+      M.getGlobalList().push_back(NewGlobalVar);
+      OldNewGlobal.insert(std::make_pair(&G, NewGlobalVar));
+    }
+  }
+
+  replaceConstExprWithGlobals(M);
+
+  removeOldGlobals();
+}
+
+// Remove old global variables from the program.
+void ESIMDLowerVecArgPass::removeOldGlobals() {
+  for (auto &G : OldNewGlobal) {
+    G.first->removeDeadConstantUsers();
+    G.first->eraseFromParent();
+  }
+}
+
+bool ESIMDLowerVecArgPass::run(Module &M) {
+  fixGlobals(M);
+
+  SmallVector<Function *, 10> functions;
+  for (auto &F : M) {
+    functions.push_back(&F);
+  }
+
+  for (auto F : functions) {
+    for (unsigned int I = 0; I != F->arg_size(); I++) {
+      auto Arg = F->getArg(I);
+      if (getSimdArgPtrTyOrNull(Arg)) {
+        rewriteFunc(*F);
+        break;
+      }
+    }
+  }
+
+  return true;
+}
diff --git a/llvm/lib/SYCLLowerIR/LowerWGScope.cpp b/llvm/lib/SYCLLowerIR/LowerWGScope.cpp
index 326de30c30fb3..14087420587e3 100644
--- a/llvm/lib/SYCLLowerIR/LowerWGScope.cpp
+++ b/llvm/lib/SYCLLowerIR/LowerWGScope.cpp
@@ -265,6 +265,7 @@ static void guardBlockWithIsLeaderCheck(BasicBlock *IfBB, BasicBlock *TrueBB,
   auto *Ty = LinearLocalID->getType();
   Value *Zero = Constant::getNullValue(Ty);
   IRBuilder<> Builder(IfBB->getContext());
+  spirv::genWGBarrier(*(IfBB->getTerminator()), TT);
   Builder.SetInsertPoint(IfBB->getTerminator());
   Value *Cmp = Builder.CreateICmpEQ(LinearLocalID, Zero, "cmpz");
   Builder.SetCurrentDebugLocation(DbgLoc);
@@ -685,7 +686,7 @@ static void fixupPrivateMemoryPFWILambdaCaptures(CallInst *PFWICall) {
 
 // Go through "byval" parameters which are passed as AS(0) pointers
 // and: (1) create local shadows for them (2) and initialize them from the
-// leader's copy and (3) replace usages with pointer to the shadow
+// leader's copy and (3) materialize the value in the local variable before use
 //
 // Do the same for 'this' pointer which points to PFWG lamda object which is
 // allocated in the caller. Caller is a kernel function which is generated by
@@ -707,7 +708,7 @@ static void sharePFWGPrivateObjects(Function &F, const Triple &TT) {
   BasicBlock *LeaderBB = EntryBB->splitBasicBlock(SplitPoint, "leader");
   BasicBlock *MergeBB = LeaderBB->splitBasicBlock(&LeaderBB->front(), "merge");
 
-  // 1) rewire the above basic blocks so that LeaderBB is executed only for the
+  // Rewire the above basic blocks so that LeaderBB is executed only for the
   // leader workitem
   guardBlockWithIsLeaderCheck(EntryBB, LeaderBB, MergeBB,
                               EntryBB->back().getDebugLoc(), TT);
@@ -719,28 +720,13 @@ static void sharePFWGPrivateObjects(Function &F, const Triple &TT) {
     IRBuilder<> Builder(Ctx);
     Builder.SetInsertPoint(&LeaderBB->front());
 
-    // 2) create the shared copy - "shadow" - for current arg
+    // Create the shared copy - "shadow" - for current arg
     GlobalVariable *Shadow = nullptr;
-    Value *RepVal = nullptr;
     if (Arg.hasByValAttr()) {
       assert(Arg.getType()->getPointerAddressSpace() ==
              asUInt(spirv::AddrSpace::Private));
       T = Arg.getParamByValType();
       Shadow = spirv::createWGLocalVariable(*F.getParent(), T, "ArgShadow");
-      RepVal = Shadow;
-      if (TT.isNVPTX()) {
-        // For NVPTX target address space inference for kernel arguments and
-        // allocas is happening in the backend (NVPTXLowerArgs and
-        // NVPTXLowerAlloca passes). After the frontend these pointers are in
-        // LLVM default address space 0 which is the generic address space for
-        // NVPTX target.
-        assert(Arg.getType()->getPointerAddressSpace() == 0);
-
-        // Cast a pointer in the shared address space to the generic address
-        // space.
-        RepVal = ConstantExpr::getPointerBitCastOrAddrSpaceCast(Shadow,
-                                                                Arg.getType());
-      }
     }
     // Process 'this' pointer which points to PFWG lambda object
     else if (Arg.getArgNo() == 0) {
@@ -748,21 +734,20 @@ static void sharePFWGPrivateObjects(Function &F, const Triple &TT) {
       assert(PtrT && "Expected this pointer as the first argument");
       T = PtrT->getElementType();
       Shadow = spirv::createWGLocalVariable(*F.getParent(), T, "ArgShadow");
-      RepVal =
-          Builder.CreatePointerBitCastOrAddrSpaceCast(Shadow, Arg.getType());
     }
 
-    if (!Shadow || !RepVal)
+    if (!Shadow)
       continue;
 
-    // 3) replace argument with shadow in all uses
-    for (auto *U : Arg.users())
-      U->replaceUsesOfWith(&Arg, RepVal);
-
     copyBetweenPrivateAndShadow(&Arg, Shadow, Builder,
                                 true /*private->shadow*/);
+    // Materialize the value in the local variable before use
+    Builder.SetInsertPoint(&MergeBB->front());
+    copyBetweenPrivateAndShadow(&Arg, Shadow, Builder,
+                                false /*shadow->private*/);
   }
-  // 5) make sure workers use up-to-date shared values written by the leader
+  // Insert barrier to make sure workers use up-to-date shared values written by
+  // the leader
   spirv::genWGBarrier(MergeBB->front(), TT);
 }
 
@@ -897,7 +882,7 @@ GlobalVariable *spirv::createWGLocalVariable(Module &M, Type *T,
       );
   G->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
   const DataLayout &DL = M.getDataLayout();
-  G->setAlignment(MaybeAlign(DL.getPreferredAlignment(G)));
+  G->setAlignment(MaybeAlign(DL.getPreferredAlign(G)));
   LocalMemUsed += DL.getTypeStoreSize(G->getValueType());
   LLVM_DEBUG(llvm::dbgs() << "Local AS Var created: " << G->getName() << "\n");
   LLVM_DEBUG(llvm::dbgs() << "  Local mem used: " << LocalMemUsed << "B\n");
@@ -961,8 +946,8 @@ Value *spirv::genPseudoLocalID(Instruction &Before, const Triple &TT) {
                              // asUInt(spirv::AddrSpace::Input) // AddressSpace
                              asUInt(spirv::AddrSpace::Global) // AddressSpace
       );
-      unsigned Align = M.getDataLayout().getPreferredAlignment(G);
-      G->setAlignment(MaybeAlign(Align));
+      Align Alignment = M.getDataLayout().getPreferredAlign(G);
+      G->setAlignment(MaybeAlign(Alignment));
     }
     Value *Res = new LoadInst(G->getType()->getElementType(), G, "", &Before);
     return Res;
diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt
index 9ea83c2524df2..8c04fa3fc0b54 100644
--- a/llvm/lib/Support/CMakeLists.txt
+++ b/llvm/lib/Support/CMakeLists.txt
@@ -1,7 +1,7 @@
-set(system_libs)
-if ( LLVM_ENABLE_ZLIB AND HAVE_LIBZ )
-  set(system_libs ${system_libs} ${ZLIB_LIBRARIES})
+if(LLVM_ENABLE_ZLIB)
+  set(imported_libs ZLIB::ZLIB)
 endif()
+
 if( MSVC OR MINGW )
   # libuuid required for FOLDERID_Profile usage in lib/Support/Windows/Path.inc.
   # advapi32 required for CryptAcquireContextW in lib/Support/Windows/Path.inc.
@@ -196,10 +196,32 @@ add_llvm_component_library(LLVMSupport
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/ADT
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Support
   ${Backtrace_INCLUDE_DIRS}
-  LINK_LIBS ${system_libs} ${delayload_flags} ${Z3_LINK_FILES}
+  LINK_LIBS ${system_libs} ${imported_libs} ${delayload_flags} ${Z3_LINK_FILES}
   )
 
-set_property(TARGET LLVMSupport PROPERTY LLVM_SYSTEM_LIBS "${system_libs}")
+set(llvm_system_libs ${system_libs})
+
+if(LLVM_ENABLE_ZLIB)
+  string(TOUPPER ${CMAKE_BUILD_TYPE} build_type)
+  get_property(zlib_library TARGET ZLIB::ZLIB PROPERTY LOCATION_${build_type})
+  if(NOT zlib_library)
+    get_property(zlib_library TARGET ZLIB::ZLIB PROPERTY LOCATION)
+  endif()
+  get_filename_component(zlib_library ${zlib_library} NAME)
+  if(CMAKE_STATIC_LIBRARY_PREFIX AND CMAKE_STATIC_LIBRARY_SUFFIX AND
+      zlib_library MATCHES "^${CMAKE_STATIC_LIBRARY_PREFIX}.*${CMAKE_STATIC_LIBRARY_SUFFIX}$")
+    STRING(REGEX REPLACE "^${CMAKE_STATIC_LIBRARY_PREFIX}" "" zlib_library ${zlib_library})
+    STRING(REGEX REPLACE "${CMAKE_STATIC_LIBRARY_SUFFIX}$" "" zlib_library ${zlib_library})
+  endif()
+  if(CMAKE_SHARED_LIBRARY_PREFIX AND CMAKE_SHARED_LIBRARY_SUFFIX AND
+      zlib_library MATCHES "^${CMAKE_SHARED_LIBRARY_PREFIX}.*${CMAKE_SHARED_LIBRARY_SUFFIX}$")
+    STRING(REGEX REPLACE "^${CMAKE_SHARED_LIBRARY_PREFIX}" "" zlib_library ${zlib_library})
+    STRING(REGEX REPLACE "${CMAKE_SHARED_LIBRARY_SUFFIX}$" "" zlib_library ${zlib_library})
+  endif()
+  set(llvm_system_libs ${llvm_system_libs} "${zlib_library}")
+endif()
+
+set_property(TARGET LLVMSupport PROPERTY LLVM_SYSTEM_LIBS "${llvm_system_libs}")
 
 if(LLVM_WITH_Z3)
   target_include_directories(LLVMSupport SYSTEM
diff --git a/llvm/lib/Support/CRC.cpp b/llvm/lib/Support/CRC.cpp
index 7ff09debe3b7c..2bc668beed322 100644
--- a/llvm/lib/Support/CRC.cpp
+++ b/llvm/lib/Support/CRC.cpp
@@ -25,7 +25,7 @@
 
 using namespace llvm;
 
-#if LLVM_ENABLE_ZLIB == 0 || !HAVE_ZLIB_H
+#if !LLVM_ENABLE_ZLIB
 
 static const uint32_t CRCTable[256] = {
     0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f,
diff --git a/llvm/lib/Support/Compression.cpp b/llvm/lib/Support/Compression.cpp
index 27d92f0e0aec2..b8c77cf69b95f 100644
--- a/llvm/lib/Support/Compression.cpp
+++ b/llvm/lib/Support/Compression.cpp
@@ -17,13 +17,13 @@
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
-#if LLVM_ENABLE_ZLIB == 1 && HAVE_ZLIB_H
+#if LLVM_ENABLE_ZLIB
 #include <zlib.h>
 #endif
 
 using namespace llvm;
 
-#if LLVM_ENABLE_ZLIB == 1 && HAVE_LIBZ
+#if LLVM_ENABLE_ZLIB
 static Error createError(StringRef Err) {
   return make_error<StringError>(Err, inconvertibleErrorCode());
 }
diff --git a/llvm/lib/Support/FormatVariadic.cpp b/llvm/lib/Support/FormatVariadic.cpp
index 632e879e540dd..f6d48bcd50e83 100644
--- a/llvm/lib/Support/FormatVariadic.cpp
+++ b/llvm/lib/Support/FormatVariadic.cpp
@@ -91,27 +91,26 @@ formatv_object_base::parseReplacementItem(StringRef Spec) {
 
 std::pair<ReplacementItem, StringRef>
 formatv_object_base::splitLiteralAndReplacement(StringRef Fmt) {
-  std::size_t From = 0;
-  while (From < Fmt.size() && From != StringRef::npos) {
-    std::size_t BO = Fmt.find_first_of('{', From);
+  while (!Fmt.empty()) {
     // Everything up until the first brace is a literal.
-    if (BO != 0)
+    if (Fmt.front() != '{') {
+      std::size_t BO = Fmt.find_first_of('{');
       return std::make_pair(ReplacementItem{Fmt.substr(0, BO)}, Fmt.substr(BO));
+    }
 
-    StringRef Braces =
-        Fmt.drop_front(BO).take_while([](char C) { return C == '{'; });
+    StringRef Braces = Fmt.take_while([](char C) { return C == '{'; });
     // If there is more than one brace, then some of them are escaped.  Treat
     // these as replacements.
     if (Braces.size() > 1) {
       size_t NumEscapedBraces = Braces.size() / 2;
-      StringRef Middle = Fmt.substr(BO, NumEscapedBraces);
-      StringRef Right = Fmt.drop_front(BO + NumEscapedBraces * 2);
+      StringRef Middle = Fmt.take_front(NumEscapedBraces);
+      StringRef Right = Fmt.drop_front(NumEscapedBraces * 2);
       return std::make_pair(ReplacementItem{Middle}, Right);
     }
     // An unterminated open brace is undefined.  We treat the rest of the string
     // as a literal replacement, but we assert to indicate that this is
     // undefined and that we consider it an error.
-    std::size_t BC = Fmt.find_first_of('}', BO);
+    std::size_t BC = Fmt.find_first_of('}');
     if (BC == StringRef::npos) {
       assert(
           false &&
@@ -122,12 +121,12 @@ formatv_object_base::splitLiteralAndReplacement(StringRef Fmt) {
     // Even if there is a closing brace, if there is another open brace before
     // this closing brace, treat this portion as literal, and try again with the
     // next one.
-    std::size_t BO2 = Fmt.find_first_of('{', BO + 1);
+    std::size_t BO2 = Fmt.find_first_of('{', 1);
     if (BO2 < BC)
       return std::make_pair(ReplacementItem{Fmt.substr(0, BO2)},
                             Fmt.substr(BO2));
 
-    StringRef Spec = Fmt.slice(BO + 1, BC);
+    StringRef Spec = Fmt.slice(1, BC);
     StringRef Right = Fmt.substr(BC + 1);
 
     auto RI = parseReplacementItem(Spec);
@@ -136,7 +135,7 @@ formatv_object_base::splitLiteralAndReplacement(StringRef Fmt) {
 
     // If there was an error parsing the replacement item, treat it as an
     // invalid replacement spec, and just continue.
-    From = BC + 1;
+    Fmt = Fmt.drop_front(BC + 1);
   }
   return std::make_pair(ReplacementItem{Fmt}, StringRef());
 }
diff --git a/llvm/lib/Support/Windows/Program.inc b/llvm/lib/Support/Windows/Program.inc
index 9fe05d24ec2eb..36840016db392 100644
--- a/llvm/lib/Support/Windows/Program.inc
+++ b/llvm/lib/Support/Windows/Program.inc
@@ -189,7 +189,13 @@ static bool Execute(ProcessInfo &PI, StringRef Program,
   // Windows wants a command line, not an array of args, to pass to the new
   // process.  We have to concatenate them all, while quoting the args that
   // have embedded spaces (or are empty).
-  std::string Command = flattenWindowsCommandLine(Args);
+  auto Result = flattenWindowsCommandLine(Args);
+  if (std::error_code ec = Result.getError()) {
+    SetLastError(ec.value());
+    MakeErrMsg(ErrMsg, std::string("Unable to convert command-line to UTF-16"));
+    return false;
+  }
+  std::wstring Command = *Result;
 
   // The pointer to the environment block for the new process.
   std::vector<wchar_t> EnvBlock;
@@ -271,18 +277,11 @@ static bool Execute(ProcessInfo &PI, StringRef Program,
     return false;
   }
 
-  SmallVector<wchar_t, MAX_PATH> CommandUtf16;
-  if (std::error_code ec = windows::UTF8ToUTF16(Command, CommandUtf16)) {
-    SetLastError(ec.value());
-    MakeErrMsg(ErrMsg,
-               std::string("Unable to convert command-line to UTF-16"));
-    return false;
-  }
-
-  BOOL rc = CreateProcessW(ProgramUtf16.data(), CommandUtf16.data(), 0, 0,
-                           TRUE, CREATE_UNICODE_ENVIRONMENT,
-                           EnvBlock.empty() ? 0 : EnvBlock.data(), 0, &si,
-                           &pi);
+  std::vector<wchar_t> CommandUtf16(Command.size() + 1, 0);
+  std::copy(Command.begin(), Command.end(), CommandUtf16.begin());
+  BOOL rc = CreateProcessW(ProgramUtf16.data(), CommandUtf16.data(), 0, 0, TRUE,
+                           CREATE_UNICODE_ENVIRONMENT,
+                           EnvBlock.empty() ? 0 : EnvBlock.data(), 0, &si, &pi);
   DWORD err = GetLastError();
 
   // Regardless of whether the process got created or not, we are done with
@@ -376,7 +375,7 @@ static std::string quoteSingleArg(StringRef Arg) {
 }
 
 namespace llvm {
-std::string sys::flattenWindowsCommandLine(ArrayRef<StringRef> Args) {
+ErrorOr<std::wstring> sys::flattenWindowsCommandLine(ArrayRef<StringRef> Args) {
   std::string Command;
   for (StringRef Arg : Args) {
     if (argNeedsQuotes(Arg))
@@ -387,7 +386,11 @@ std::string sys::flattenWindowsCommandLine(ArrayRef<StringRef> Args) {
     Command.push_back(' ');
   }
 
-  return Command;
+  SmallVector<wchar_t, MAX_PATH> CommandUtf16;
+  if (std::error_code ec = windows::UTF8ToUTF16(Command, CommandUtf16))
+    return ec;
+
+  return std::wstring(CommandUtf16.begin(), CommandUtf16.end());
 }
 
 ProcessInfo sys::Wait(const ProcessInfo &PI, unsigned SecondsToWait,
@@ -532,12 +535,16 @@ llvm::sys::writeFileWithEncoding(StringRef FileName, StringRef Contents,
 
 bool llvm::sys::commandLineFitsWithinSystemLimits(StringRef Program,
                                                   ArrayRef<StringRef> Args) {
-  // The documented max length of the command line passed to CreateProcess.
-  static const size_t MaxCommandStringLength = 32768;
+  // The documentation on CreateProcessW states that the size of the argument
+  // lpCommandLine must not be greater than 32767 characters, including the
+  // Unicode terminating null character. We use smaller value to reduce risk
+  // of getting invalid command line due to unaccounted factors.
+  static const size_t MaxCommandStringLength = 32000;
   SmallVector<StringRef, 8> FullArgs;
   FullArgs.push_back(Program);
   FullArgs.append(Args.begin(), Args.end());
-  std::string Result = flattenWindowsCommandLine(FullArgs);
-  return (Result.size() + 1) <= MaxCommandStringLength;
+  auto Result = flattenWindowsCommandLine(FullArgs);
+  assert(!Result.getError());
+  return (Result->size() + 1) <= MaxCommandStringLength;
 }
 }
diff --git a/llvm/lib/Target/AArch64/AArch64Combine.td b/llvm/lib/Target/AArch64/AArch64Combine.td
index aa41cae289e8b..b8da73de0f8c8 100644
--- a/llvm/lib/Target/AArch64/AArch64Combine.td
+++ b/llvm/lib/Target/AArch64/AArch64Combine.td
@@ -79,6 +79,6 @@ def shuffle_vector_pseudos : GICombineGroup<[dup, rev, ext, zip, uzp, trn]>;
 def AArch64PostLegalizerCombinerHelper
     : GICombinerHelper<"AArch64GenPostLegalizerCombinerHelper",
                        [erase_undef_store, combines_for_extload,
-                        sext_already_extended, shuffle_vector_pseudos]> {
+                        sext_trunc_sextload, shuffle_vector_pseudos]> {
   let DisableRuleOption = "aarch64postlegalizercombiner-disable-rule";
 }
diff --git a/llvm/lib/Target/AArch64/AArch64FastISel.cpp b/llvm/lib/Target/AArch64/AArch64FastISel.cpp
index 0f63f4ca62e5e..f0b35c2aef262 100644
--- a/llvm/lib/Target/AArch64/AArch64FastISel.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FastISel.cpp
@@ -3652,14 +3652,10 @@ bool AArch64FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::BRK))
         .addImm(1);
     return true;
-  case Intrinsic::debugtrap: {
-    if (Subtarget->isTargetWindows()) {
-      BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::BRK))
-          .addImm(0xF000);
-      return true;
-    }
-    break;
-  }
+  case Intrinsic::debugtrap:
+    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::BRK))
+        .addImm(0xF000);
+    return true;
 
   case Intrinsic::sqrt: {
     Type *RetTy = II->getCalledFunction()->getReturnType();
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index dae347cd8c2b9..af77171e92849 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -677,8 +677,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
 
   // Trap.
   setOperationAction(ISD::TRAP, MVT::Other, Legal);
-  if (Subtarget->isTargetWindows())
-    setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
+  setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
 
   // We combine OR nodes for bitfield operations.
   setTargetDAGCombine(ISD::OR);
@@ -704,6 +703,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
   setTargetDAGCombine(ISD::ZERO_EXTEND);
   setTargetDAGCombine(ISD::SIGN_EXTEND);
   setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
+  setTargetDAGCombine(ISD::TRUNCATE);
   setTargetDAGCombine(ISD::CONCAT_VECTORS);
   setTargetDAGCombine(ISD::STORE);
   if (Subtarget->supportsAddressTopByteIgnored())
@@ -850,8 +850,6 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::UADDSAT, VT, Legal);
       setOperationAction(ISD::SSUBSAT, VT, Legal);
       setOperationAction(ISD::USUBSAT, VT, Legal);
-
-      setOperationAction(ISD::TRUNCATE, VT, Custom);
     }
     for (MVT VT : { MVT::v4f16, MVT::v2f32,
                     MVT::v8f16, MVT::v4f32, MVT::v2f64 }) {
@@ -932,8 +930,11 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
         setOperationAction(ISD::SHL, VT, Custom);
         setOperationAction(ISD::SRL, VT, Custom);
         setOperationAction(ISD::SRA, VT, Custom);
-        if (VT.getScalarType() == MVT::i1)
+        if (VT.getScalarType() == MVT::i1) {
           setOperationAction(ISD::SETCC, VT, Custom);
+          setOperationAction(ISD::TRUNCATE, VT, Custom);
+          setOperationAction(ISD::CONCAT_VECTORS, VT, Legal);
+        }
       }
     }
 
@@ -1074,6 +1075,10 @@ void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
   // Lower fixed length vector operations to scalable equivalents.
   setOperationAction(ISD::ADD, VT, Custom);
   setOperationAction(ISD::FADD, VT, Custom);
+  setOperationAction(ISD::FDIV, VT, Custom);
+  setOperationAction(ISD::FMA, VT, Custom);
+  setOperationAction(ISD::FMUL, VT, Custom);
+  setOperationAction(ISD::FSUB, VT, Custom);
   setOperationAction(ISD::LOAD, VT, Custom);
   setOperationAction(ISD::STORE, VT, Custom);
   setOperationAction(ISD::TRUNCATE, VT, Custom);
@@ -1466,6 +1471,8 @@ const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
     MAKE_CASE(AArch64ISD::UADDV)
     MAKE_CASE(AArch64ISD::SRHADD)
     MAKE_CASE(AArch64ISD::URHADD)
+    MAKE_CASE(AArch64ISD::SHADD)
+    MAKE_CASE(AArch64ISD::UHADD)
     MAKE_CASE(AArch64ISD::SMINV)
     MAKE_CASE(AArch64ISD::UMINV)
     MAKE_CASE(AArch64ISD::SMAXV)
@@ -3306,9 +3313,16 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
   }
 
   case Intrinsic::aarch64_neon_srhadd:
-  case Intrinsic::aarch64_neon_urhadd: {
-    bool IsSignedAdd = IntNo == Intrinsic::aarch64_neon_srhadd;
-    unsigned Opcode = IsSignedAdd ? AArch64ISD::SRHADD : AArch64ISD::URHADD;
+  case Intrinsic::aarch64_neon_urhadd:
+  case Intrinsic::aarch64_neon_shadd:
+  case Intrinsic::aarch64_neon_uhadd: {
+    bool IsSignedAdd = (IntNo == Intrinsic::aarch64_neon_srhadd ||
+                        IntNo == Intrinsic::aarch64_neon_shadd);
+    bool IsRoundingAdd = (IntNo == Intrinsic::aarch64_neon_srhadd ||
+                          IntNo == Intrinsic::aarch64_neon_urhadd);
+    unsigned Opcode =
+        IsSignedAdd ? (IsRoundingAdd ? AArch64ISD::SRHADD : AArch64ISD::SHADD)
+                    : (IsRoundingAdd ? AArch64ISD::URHADD : AArch64ISD::UHADD);
     return DAG.getNode(Opcode, dl, Op.getValueType(), Op.getOperand(1),
                        Op.getOperand(2));
   }
@@ -3475,24 +3489,23 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
   case ISD::UMULO:
     return LowerXALUO(Op, DAG);
   case ISD::FADD:
-    if (Op.getValueType().isScalableVector() ||
-        useSVEForFixedLengthVectorVT(Op.getValueType()))
-      return LowerToPredicatedOp(Op, DAG, AArch64ISD::FADD_PRED);
-    return LowerF128Call(Op, DAG, RTLIB::ADD_F128);
+    if (Op.getValueType() == MVT::f128)
+      return LowerF128Call(Op, DAG, RTLIB::ADD_F128);
+    return LowerToPredicatedOp(Op, DAG, AArch64ISD::FADD_PRED);
   case ISD::FSUB:
-    if (Op.getValueType().isScalableVector())
-      return LowerToPredicatedOp(Op, DAG, AArch64ISD::FSUB_PRED);
-    return LowerF128Call(Op, DAG, RTLIB::SUB_F128);
+    if (Op.getValueType() == MVT::f128)
+      return LowerF128Call(Op, DAG, RTLIB::SUB_F128);
+    return LowerToPredicatedOp(Op, DAG, AArch64ISD::FSUB_PRED);
   case ISD::FMUL:
-    if (Op.getValueType().isScalableVector())
-      return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMUL_PRED);
-    return LowerF128Call(Op, DAG, RTLIB::MUL_F128);
+    if (Op.getValueType() == MVT::f128)
+      return LowerF128Call(Op, DAG, RTLIB::MUL_F128);
+    return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMUL_PRED);
   case ISD::FMA:
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMA_PRED);
   case ISD::FDIV:
-    if (Op.getValueType().isScalableVector())
-      return LowerToPredicatedOp(Op, DAG, AArch64ISD::FDIV_PRED);
-    return LowerF128Call(Op, DAG, RTLIB::DIV_F128);
+    if (Op.getValueType() == MVT::f128)
+      return LowerF128Call(Op, DAG, RTLIB::DIV_F128);
+    return LowerToPredicatedOp(Op, DAG, AArch64ISD::FDIV_PRED);
   case ISD::FP_ROUND:
   case ISD::STRICT_FP_ROUND:
     return LowerFP_ROUND(Op, DAG);
@@ -8860,79 +8873,27 @@ static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt) {
   return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits / 2 : ElementBits));
 }
 
-// Attempt to form urhadd(OpA, OpB) from
-// truncate(vlshr(sub(zext(OpB), xor(zext(OpA), Ones(ElemSizeInBits))), 1)).
-// The original form of this expression is
-// truncate(srl(add(zext(OpB), add(zext(OpA), 1)), 1)) and before this function
-// is called the srl will have been lowered to AArch64ISD::VLSHR and the
-// ((OpA + OpB + 1) >> 1) expression will have been changed to (OpB - (~OpA)).
-// This pass can also recognize a variant of this pattern that uses sign
-// extension instead of zero extension and form a srhadd(OpA, OpB) from it.
 SDValue AArch64TargetLowering::LowerTRUNCATE(SDValue Op,
                                              SelectionDAG &DAG) const {
   EVT VT = Op.getValueType();
 
+  if (VT.getScalarType() == MVT::i1) {
+    // Lower i1 truncate to `(x & 1) != 0`.
+    SDLoc dl(Op);
+    EVT OpVT = Op.getOperand(0).getValueType();
+    SDValue Zero = DAG.getConstant(0, dl, OpVT);
+    SDValue One = DAG.getConstant(1, dl, OpVT);
+    SDValue And = DAG.getNode(ISD::AND, dl, OpVT, Op.getOperand(0), One);
+    return DAG.getSetCC(dl, VT, And, Zero, ISD::SETNE);
+  }
+
   if (!VT.isVector() || VT.isScalableVector())
-    return Op;
+    return SDValue();
 
   if (useSVEForFixedLengthVectorVT(Op.getOperand(0).getValueType()))
     return LowerFixedLengthVectorTruncateToSVE(Op, DAG);
 
-  // Since we are looking for a right shift by a constant value of 1 and we are
-  // operating on types at least 16 bits in length (sign/zero extended OpA and
-  // OpB, which are at least 8 bits), it follows that the truncate will always
-  // discard the shifted-in bit and therefore the right shift will be logical
-  // regardless of the signedness of OpA and OpB.
-  SDValue Shift = Op.getOperand(0);
-  if (Shift.getOpcode() != AArch64ISD::VLSHR)
-    return Op;
-
-  // Is the right shift using an immediate value of 1?
-  uint64_t ShiftAmount = Shift.getConstantOperandVal(1);
-  if (ShiftAmount != 1)
-    return Op;
-
-  SDValue Sub = Shift->getOperand(0);
-  if (Sub.getOpcode() != ISD::SUB)
-    return Op;
-
-  SDValue Xor = Sub.getOperand(1);
-  if (Xor.getOpcode() != ISD::XOR)
-    return Op;
-
-  SDValue ExtendOpA = Xor.getOperand(0);
-  SDValue ExtendOpB = Sub.getOperand(0);
-  unsigned ExtendOpAOpc = ExtendOpA.getOpcode();
-  unsigned ExtendOpBOpc = ExtendOpB.getOpcode();
-  if (!(ExtendOpAOpc == ExtendOpBOpc &&
-        (ExtendOpAOpc == ISD::ZERO_EXTEND || ExtendOpAOpc == ISD::SIGN_EXTEND)))
-    return Op;
-
-  // Is the result of the right shift being truncated to the same value type as
-  // the original operands, OpA and OpB?
-  SDValue OpA = ExtendOpA.getOperand(0);
-  SDValue OpB = ExtendOpB.getOperand(0);
-  EVT OpAVT = OpA.getValueType();
-  assert(ExtendOpA.getValueType() == ExtendOpB.getValueType());
-  if (!(VT == OpAVT && OpAVT == OpB.getValueType()))
-    return Op;
-
-  // Is the XOR using a constant amount of all ones in the right hand side?
-  uint64_t C;
-  if (!isAllConstantBuildVector(Xor.getOperand(1), C))
-    return Op;
-
-  unsigned ElemSizeInBits = VT.getScalarSizeInBits();
-  APInt CAsAPInt(ElemSizeInBits, C);
-  if (CAsAPInt != APInt::getAllOnesValue(ElemSizeInBits))
-    return Op;
-
-  SDLoc DL(Op);
-  bool IsSignExtend = ExtendOpAOpc == ISD::SIGN_EXTEND;
-  unsigned RHADDOpc = IsSignExtend ? AArch64ISD::SRHADD : AArch64ISD::URHADD;
-  SDValue ResultURHADD = DAG.getNode(RHADDOpc, DL, VT, OpA, OpB);
-
-  return ResultURHADD;
+  return SDValue();
 }
 
 SDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(SDValue Op,
@@ -11138,6 +11099,89 @@ static SDValue performSRLCombine(SDNode *N,
   return SDValue();
 }
 
+// Attempt to form urhadd(OpA, OpB) from
+// truncate(vlshr(sub(zext(OpB), xor(zext(OpA), Ones(ElemSizeInBits))), 1))
+// or uhadd(OpA, OpB) from truncate(vlshr(add(zext(OpA), zext(OpB)), 1)).
+// The original form of the first expression is
+// truncate(srl(add(zext(OpB), add(zext(OpA), 1)), 1)) and the
+// (OpA + OpB + 1) subexpression will have been changed to (OpB - (~OpA)).
+// Before this function is called the srl will have been lowered to
+// AArch64ISD::VLSHR.
+// This pass can also recognize signed variants of the patterns that use sign
+// extension instead of zero extension and form a srhadd(OpA, OpB) or a
+// shadd(OpA, OpB) from them.
+static SDValue
+performVectorTruncateCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
+                             SelectionDAG &DAG) {
+  EVT VT = N->getValueType(0);
+
+  // Since we are looking for a right shift by a constant value of 1 and we are
+  // operating on types at least 16 bits in length (sign/zero extended OpA and
+  // OpB, which are at least 8 bits), it follows that the truncate will always
+  // discard the shifted-in bit and therefore the right shift will be logical
+  // regardless of the signedness of OpA and OpB.
+  SDValue Shift = N->getOperand(0);
+  if (Shift.getOpcode() != AArch64ISD::VLSHR)
+    return SDValue();
+
+  // Is the right shift using an immediate value of 1?
+  uint64_t ShiftAmount = Shift.getConstantOperandVal(1);
+  if (ShiftAmount != 1)
+    return SDValue();
+
+  SDValue ExtendOpA, ExtendOpB;
+  SDValue ShiftOp0 = Shift.getOperand(0);
+  unsigned ShiftOp0Opc = ShiftOp0.getOpcode();
+  if (ShiftOp0Opc == ISD::SUB) {
+
+    SDValue Xor = ShiftOp0.getOperand(1);
+    if (Xor.getOpcode() != ISD::XOR)
+      return SDValue();
+
+    // Is the XOR using a constant amount of all ones in the right hand side?
+    uint64_t C;
+    if (!isAllConstantBuildVector(Xor.getOperand(1), C))
+      return SDValue();
+
+    unsigned ElemSizeInBits = VT.getScalarSizeInBits();
+    APInt CAsAPInt(ElemSizeInBits, C);
+    if (CAsAPInt != APInt::getAllOnesValue(ElemSizeInBits))
+      return SDValue();
+
+    ExtendOpA = Xor.getOperand(0);
+    ExtendOpB = ShiftOp0.getOperand(0);
+  } else if (ShiftOp0Opc == ISD::ADD) {
+    ExtendOpA = ShiftOp0.getOperand(0);
+    ExtendOpB = ShiftOp0.getOperand(1);
+  } else
+    return SDValue();
+
+  unsigned ExtendOpAOpc = ExtendOpA.getOpcode();
+  unsigned ExtendOpBOpc = ExtendOpB.getOpcode();
+  if (!(ExtendOpAOpc == ExtendOpBOpc &&
+        (ExtendOpAOpc == ISD::ZERO_EXTEND || ExtendOpAOpc == ISD::SIGN_EXTEND)))
+    return SDValue();
+
+  // Is the result of the right shift being truncated to the same value type as
+  // the original operands, OpA and OpB?
+  SDValue OpA = ExtendOpA.getOperand(0);
+  SDValue OpB = ExtendOpB.getOperand(0);
+  EVT OpAVT = OpA.getValueType();
+  assert(ExtendOpA.getValueType() == ExtendOpB.getValueType());
+  if (!(VT == OpAVT && OpAVT == OpB.getValueType()))
+    return SDValue();
+
+  SDLoc DL(N);
+  bool IsSignExtend = ExtendOpAOpc == ISD::SIGN_EXTEND;
+  bool IsRHADD = ShiftOp0Opc == ISD::SUB;
+  unsigned HADDOpc = IsSignExtend
+                         ? (IsRHADD ? AArch64ISD::SRHADD : AArch64ISD::SHADD)
+                         : (IsRHADD ? AArch64ISD::URHADD : AArch64ISD::UHADD);
+  SDValue ResultHADD = DAG.getNode(HADDOpc, DL, VT, OpA, OpB);
+
+  return ResultHADD;
+}
+
 static SDValue performConcatVectorsCombine(SDNode *N,
                                            TargetLowering::DAGCombinerInfo &DCI,
                                            SelectionDAG &DAG) {
@@ -11183,9 +11227,9 @@ static SDValue performConcatVectorsCombine(SDNode *N,
   if (DCI.isBeforeLegalizeOps())
     return SDValue();
 
-  // Optimise concat_vectors of two [us]rhadds that use extracted subvectors
-  // from the same original vectors. Combine these into a single [us]rhadd that
-  // operates on the two original vectors. Example:
+  // Optimise concat_vectors of two [us]rhadds or [us]hadds that use extracted
+  // subvectors from the same original vectors. Combine these into a single
+  // [us]rhadd or [us]hadd that operates on the two original vectors. Example:
   //  (v16i8 (concat_vectors (v8i8 (urhadd (extract_subvector (v16i8 OpA, <0>),
   //                                        extract_subvector (v16i8 OpB,
   //                                        <0>))),
@@ -11195,7 +11239,8 @@ static SDValue performConcatVectorsCombine(SDNode *N,
   // ->
   //  (v16i8(urhadd(v16i8 OpA, v16i8 OpB)))
   if (N->getNumOperands() == 2 && N0Opc == N1Opc &&
-      (N0Opc == AArch64ISD::URHADD || N0Opc == AArch64ISD::SRHADD)) {
+      (N0Opc == AArch64ISD::URHADD || N0Opc == AArch64ISD::SRHADD ||
+       N0Opc == AArch64ISD::UHADD || N0Opc == AArch64ISD::SHADD)) {
     SDValue N00 = N0->getOperand(0);
     SDValue N01 = N0->getOperand(1);
     SDValue N10 = N1->getOperand(0);
@@ -13890,6 +13935,8 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
     return performExtendCombine(N, DCI, DAG);
   case ISD::SIGN_EXTEND_INREG:
     return performSignExtendInRegCombine(N, DCI, DAG);
+  case ISD::TRUNCATE:
+    return performVectorTruncateCombine(N, DCI, DAG);
   case ISD::CONCAT_VECTORS:
     return performConcatVectorsCombine(N, DCI, DAG);
   case ISD::SELECT:
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 982dbc86d1694..344ed96d79609 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -191,6 +191,10 @@ enum NodeType : unsigned {
   SADDV,
   UADDV,
 
+  // Vector halving addition
+  SHADD,
+  UHADD,
+
   // Vector rounding halving addition
   SRHADD,
   URHADD,
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 5139ae5ccaf19..08f80c9aa361b 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -2744,6 +2744,35 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
     return;
   }
 
+  // Copy a Z register pair by copying the individual sub-registers.
+  if (AArch64::ZPR2RegClass.contains(DestReg) &&
+      AArch64::ZPR2RegClass.contains(SrcReg)) {
+    static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1};
+    copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORR_ZZZ,
+                     Indices);
+    return;
+  }
+
+  // Copy a Z register triple by copying the individual sub-registers.
+  if (AArch64::ZPR3RegClass.contains(DestReg) &&
+      AArch64::ZPR3RegClass.contains(SrcReg)) {
+    static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
+                                       AArch64::zsub2};
+    copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORR_ZZZ,
+                     Indices);
+    return;
+  }
+
+  // Copy a Z register quad by copying the individual sub-registers.
+  if (AArch64::ZPR4RegClass.contains(DestReg) &&
+      AArch64::ZPR4RegClass.contains(SrcReg)) {
+    static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
+                                       AArch64::zsub2, AArch64::zsub3};
+    copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORR_ZZZ,
+                     Indices);
+    return;
+  }
+
   if (AArch64::GPR64spRegClass.contains(DestReg) &&
       (AArch64::GPR64spRegClass.contains(SrcReg) || SrcReg == AArch64::XZR)) {
     if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index f4a5f639e4973..d4e984754d0e9 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -556,6 +556,8 @@ def AArch64umaxv    : SDNode<"AArch64ISD::UMAXV", SDT_AArch64UnaryVec>;
 
 def AArch64srhadd   : SDNode<"AArch64ISD::SRHADD", SDT_AArch64binvec>;
 def AArch64urhadd   : SDNode<"AArch64ISD::URHADD", SDT_AArch64binvec>;
+def AArch64shadd   : SDNode<"AArch64ISD::SHADD", SDT_AArch64binvec>;
+def AArch64uhadd   : SDNode<"AArch64ISD::UHADD", SDT_AArch64binvec>;
 
 def SDT_AArch64SETTAG : SDTypeProfile<0, 2, [SDTCisPtrTy<0>, SDTCisPtrTy<1>]>;
 def AArch64stg : SDNode<"AArch64ISD::STG", SDT_AArch64SETTAG, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
@@ -4064,7 +4066,7 @@ defm PMUL     : SIMDThreeSameVectorB<1, 0b10011, "pmul", int_aarch64_neon_pmul>;
 defm SABA     : SIMDThreeSameVectorBHSTied<0, 0b01111, "saba",
       TriOpFrag<(add node:$LHS, (int_aarch64_neon_sabd node:$MHS, node:$RHS))> >;
 defm SABD     : SIMDThreeSameVectorBHS<0,0b01110,"sabd", int_aarch64_neon_sabd>;
-defm SHADD    : SIMDThreeSameVectorBHS<0,0b00000,"shadd", int_aarch64_neon_shadd>;
+defm SHADD    : SIMDThreeSameVectorBHS<0,0b00000,"shadd", AArch64shadd>;
 defm SHSUB    : SIMDThreeSameVectorBHS<0,0b00100,"shsub", int_aarch64_neon_shsub>;
 defm SMAXP    : SIMDThreeSameVectorBHS<0,0b10100,"smaxp", int_aarch64_neon_smaxp>;
 defm SMAX     : SIMDThreeSameVectorBHS<0,0b01100,"smax", smax>;
@@ -4083,7 +4085,7 @@ defm SUB      : SIMDThreeSameVector<1,0b10000,"sub", sub>;
 defm UABA     : SIMDThreeSameVectorBHSTied<1, 0b01111, "uaba",
       TriOpFrag<(add node:$LHS, (int_aarch64_neon_uabd node:$MHS, node:$RHS))> >;
 defm UABD     : SIMDThreeSameVectorBHS<1,0b01110,"uabd", int_aarch64_neon_uabd>;
-defm UHADD    : SIMDThreeSameVectorBHS<1,0b00000,"uhadd", int_aarch64_neon_uhadd>;
+defm UHADD    : SIMDThreeSameVectorBHS<1,0b00000,"uhadd", AArch64uhadd>;
 defm UHSUB    : SIMDThreeSameVectorBHS<1,0b00100,"uhsub", int_aarch64_neon_uhsub>;
 defm UMAXP    : SIMDThreeSameVectorBHS<1,0b10100,"umaxp", int_aarch64_neon_umaxp>;
 defm UMAX     : SIMDThreeSameVectorBHS<1,0b01100,"umax", umax>;
@@ -6663,7 +6665,7 @@ def : Pat<(i32 (trunc GPR64sp:$src)),
 
 // __builtin_trap() uses the BRK instruction on AArch64.
 def : Pat<(trap), (BRK 1)>;
-def : Pat<(debugtrap), (BRK 0xF000)>, Requires<[IsWindows]>;
+def : Pat<(debugtrap), (BRK 0xF000)>;
 
 // Multiply high patterns which multiply the lower subvector using smull/umull
 // and the upper subvector with smull2/umull2. Then shuffle the high the high
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
index 886158ca44901..83a488afc7972 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
@@ -40,6 +40,14 @@ AArch64RegisterInfo::AArch64RegisterInfo(const Triple &TT)
   AArch64_MC::initLLVMToCVRegMapping(this);
 }
 
+static bool hasSVEArgsOrReturn(const MachineFunction *MF) {
+  const Function &F = MF->getFunction();
+  return isa<ScalableVectorType>(F.getReturnType()) ||
+         any_of(F.args(), [](const Argument &Arg) {
+           return isa<ScalableVectorType>(Arg.getType());
+         });
+}
+
 const MCPhysReg *
 AArch64RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
   assert(MF && "Invalid MachineFunction pointer.");
@@ -75,6 +83,8 @@ AArch64RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
     // This is for OSes other than Windows; Windows is a separate case further
     // above.
     return CSR_AArch64_AAPCS_X18_SaveList;
+  if (hasSVEArgsOrReturn(MF))
+    return CSR_AArch64_SVE_AAPCS_SaveList;
   return CSR_AArch64_AAPCS_SaveList;
 }
 
@@ -343,6 +353,15 @@ bool AArch64RegisterInfo::hasBasePointer(const MachineFunction &MF) const {
   if (MFI.hasVarSizedObjects() || MF.hasEHFunclets()) {
     if (needsStackRealignment(MF))
       return true;
+
+    if (MF.getSubtarget<AArch64Subtarget>().hasSVE()) {
+      const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
+      // Frames that have variable sized objects and scalable SVE objects,
+      // should always use a basepointer.
+      if (!AFI->hasCalculatedStackSizeSVE() || AFI->getStackSizeSVE())
+        return true;
+    }
+
     // Conservatively estimate whether the negative offset from the frame
     // pointer will be sufficient to reach. If a function has a smallish
     // frame, it's less likely to have lots of spills and callee saved
@@ -379,8 +398,15 @@ AArch64RegisterInfo::useFPForScavengingIndex(const MachineFunction &MF) const {
   // (closer to SP).
   //
   // The beginning works most reliably if we have a frame pointer.
+  // In the presence of any non-constant space between FP and locals,
+  // (e.g. in case of stack realignment or a scalable SVE area), it is
+  // better to use SP or BP.
   const AArch64FrameLowering &TFI = *getFrameLowering(MF);
-  return TFI.hasFP(MF);
+  const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
+  assert((!MF.getSubtarget<AArch64Subtarget>().hasSVE() ||
+          AFI->hasCalculatedStackSizeSVE()) &&
+         "Expected SVE area to be calculated by this point");
+  return TFI.hasFP(MF) && !needsStackRealignment(MF) && !AFI->getStackSizeSVE();
 }
 
 bool AArch64RegisterInfo::requiresFrameIndexScavenging(
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 1d7b774f2ee43..dc501a9536b9a 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -1135,6 +1135,14 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
   def : Pat<(nxv8i1 (extract_subvector (nxv16i1 PPR:$Ps), (i64 8))),
             (ZIP2_PPP_B PPR:$Ps, (PFALSE))>;
 
+  // Concatenate two predicates.
+  def : Pat<(nxv4i1 (concat_vectors nxv2i1:$p1, nxv2i1:$p2)),
+            (UZP1_PPP_S $p1, $p2)>;
+  def : Pat<(nxv8i1 (concat_vectors nxv4i1:$p1, nxv4i1:$p2)),
+            (UZP1_PPP_H $p1, $p2)>;
+  def : Pat<(nxv16i1 (concat_vectors nxv8i1:$p1, nxv8i1:$p2)),
+            (UZP1_PPP_B $p1, $p2)>;
+
   defm CMPHS_PPzZZ : sve_int_cmp_0<0b000, "cmphs", SETUGE, SETULE>;
   defm CMPHI_PPzZZ : sve_int_cmp_0<0b001, "cmphi", SETUGT, SETULT>;
   defm CMPGE_PPzZZ : sve_int_cmp_0<0b100, "cmpge", SETGE, SETLE>;
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
index 11a8d5def4296..5bfed7e472b06 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
@@ -52,10 +52,10 @@ AArch64CallLowering::AArch64CallLowering(const AArch64TargetLowering &TLI)
   : CallLowering(&TLI) {}
 
 namespace {
-struct IncomingArgHandler : public CallLowering::ValueHandler {
+struct IncomingArgHandler : public CallLowering::IncomingValueHandler {
   IncomingArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
                      CCAssignFn *AssignFn)
-      : ValueHandler(MIRBuilder, MRI, AssignFn), StackUsed(0) {}
+      : IncomingValueHandler(MIRBuilder, MRI, AssignFn), StackUsed(0) {}
 
   Register getStackAddress(uint64_t Size, int64_t Offset,
                            MachinePointerInfo &MPO) override {
@@ -96,9 +96,7 @@ struct IncomingArgHandler : public CallLowering::ValueHandler {
   /// How the physical register gets marked varies between formal
   /// parameters (it's a basic-block live-in), and a call instruction
   /// (it's an implicit-def of the BL).
-  virtual void markPhysRegUsed(unsigned PhysReg) = 0;
-
-  bool isIncomingArgumentHandler() const override { return true; }
+  virtual void markPhysRegUsed(MCRegister PhysReg) = 0;
 
   uint64_t StackUsed;
 };
@@ -108,7 +106,7 @@ struct FormalArgHandler : public IncomingArgHandler {
                    CCAssignFn *AssignFn)
     : IncomingArgHandler(MIRBuilder, MRI, AssignFn) {}
 
-  void markPhysRegUsed(unsigned PhysReg) override {
+  void markPhysRegUsed(MCRegister PhysReg) override {
     MIRBuilder.getMRI()->addLiveIn(PhysReg);
     MIRBuilder.getMBB().addLiveIn(PhysReg);
   }
@@ -119,24 +117,22 @@ struct CallReturnHandler : public IncomingArgHandler {
                     MachineInstrBuilder MIB, CCAssignFn *AssignFn)
     : IncomingArgHandler(MIRBuilder, MRI, AssignFn), MIB(MIB) {}
 
-  void markPhysRegUsed(unsigned PhysReg) override {
+  void markPhysRegUsed(MCRegister PhysReg) override {
     MIB.addDef(PhysReg, RegState::Implicit);
   }
 
   MachineInstrBuilder MIB;
 };
 
-struct OutgoingArgHandler : public CallLowering::ValueHandler {
+struct OutgoingArgHandler : public CallLowering::OutgoingValueHandler {
   OutgoingArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
                      MachineInstrBuilder MIB, CCAssignFn *AssignFn,
                      CCAssignFn *AssignFnVarArg, bool IsTailCall = false,
                      int FPDiff = 0)
-      : ValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB),
+      : OutgoingValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB),
         AssignFnVarArg(AssignFnVarArg), IsTailCall(IsTailCall), FPDiff(FPDiff),
         StackSize(0), SPReg(0) {}
 
-  bool isIncomingArgumentHandler() const override { return false; }
-
   Register getStackAddress(uint64_t Size, int64_t Offset,
                            MachinePointerInfo &MPO) override {
     MachineFunction &MF = MIRBuilder.getMF();
@@ -186,6 +182,8 @@ struct OutgoingArgHandler : public CallLowering::ValueHandler {
     if (!Arg.IsFixed)
       MaxSize = 0;
 
+    assert(Arg.Regs.size() == 1);
+
     Register ValVReg = VA.getLocInfo() != CCValAssign::LocInfo::FPExt
                            ? extendRegister(Arg.Regs[0], VA, MaxSize)
                            : Arg.Regs[0];
@@ -416,7 +414,7 @@ static void handleMustTailForwardedRegisters(MachineIRBuilder &MIRBuilder,
   // Conservatively forward X8, since it might be used for an aggregate
   // return.
   if (!CCInfo.isAllocated(AArch64::X8)) {
-    unsigned X8VReg = MF.addLiveIn(AArch64::X8, &AArch64::GPR64RegClass);
+    Register X8VReg = MF.addLiveIn(AArch64::X8, &AArch64::GPR64RegClass);
     Forwards.push_back(ForwardedRegister(X8VReg, AArch64::X8, MVT::i64));
   }
 
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 408f0cb77e738..eb6a4aa3d8266 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -2945,17 +2945,20 @@ bool AArch64InstructionSelector::selectTLSGlobalValue(
   const GlobalValue &GV = *I.getOperand(1).getGlobal();
   MachineIRBuilder MIB(I);
 
-  MIB.buildInstr(AArch64::LOADgot, {AArch64::X0}, {})
-      .addGlobalAddress(&GV, 0, AArch64II::MO_TLS);
+  auto LoadGOT =
+      MIB.buildInstr(AArch64::LOADgot, {&AArch64::GPR64commonRegClass}, {})
+          .addGlobalAddress(&GV, 0, AArch64II::MO_TLS);
 
   auto Load = MIB.buildInstr(AArch64::LDRXui, {&AArch64::GPR64commonRegClass},
-                             {Register(AArch64::X0)})
+                             {LoadGOT.getReg(0)})
                   .addImm(0);
 
+  MIB.buildCopy(Register(AArch64::X0), LoadGOT.getReg(0));
   // TLS calls preserve all registers except those that absolutely must be
   // trashed: X0 (it takes an argument), LR (it's a call) and NZCV (let's not be
   // silly).
   MIB.buildInstr(getBLRCallOpcode(MF), {}, {Load})
+      .addUse(AArch64::X0, RegState::Implicit)
       .addDef(AArch64::X0, RegState::Implicit)
       .addRegMask(TRI.getTLSCallPreservedMask());
 
@@ -4633,8 +4636,6 @@ bool AArch64InstructionSelector::selectIntrinsicWithSideEffects(
     MIRBuilder.buildInstr(AArch64::BRK, {}, {}).addImm(1);
     break;
   case Intrinsic::debugtrap:
-    if (!STI.isTargetWindows())
-      return false;
     MIRBuilder.buildInstr(AArch64::BRK, {}, {}).addImm(0xF000);
     break;
   }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.h
index 576e6cfe929e2..22d29a2c99355 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.h
@@ -27,7 +27,7 @@ struct ArgDescriptor {
   friend class AMDGPUArgumentUsageInfo;
 
   union {
-    Register Reg;
+    MCRegister Reg;
     unsigned StackOffset;
   };
 
@@ -69,7 +69,7 @@ struct ArgDescriptor {
     return !IsStack;
   }
 
-  Register getRegister() const {
+  MCRegister getRegister() const {
     assert(!IsStack);
     return Reg;
   }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
index 05a4e3462a263..a72d100b13075 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
@@ -15,30 +15,49 @@
 #include "AMDGPUCallLowering.h"
 #include "AMDGPU.h"
 #include "AMDGPUISelLowering.h"
+#include "AMDGPULegalizerInfo.h"
 #include "AMDGPUSubtarget.h"
 #include "AMDGPUTargetMachine.h"
+#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
 #include "SIISelLowering.h"
 #include "SIMachineFunctionInfo.h"
 #include "SIRegisterInfo.h"
-#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
 #include "llvm/CodeGen/Analysis.h"
 #include "llvm/CodeGen/CallingConvLower.h"
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/Support/LowLevelTypeImpl.h"
 
+#define DEBUG_TYPE "amdgpu-call-lowering"
+
 using namespace llvm;
 
 namespace {
 
-struct OutgoingValueHandler : public CallLowering::ValueHandler {
-  OutgoingValueHandler(MachineIRBuilder &B, MachineRegisterInfo &MRI,
-                       MachineInstrBuilder MIB, CCAssignFn *AssignFn)
-      : ValueHandler(B, MRI, AssignFn), MIB(MIB) {}
+struct AMDGPUValueHandler : public CallLowering::ValueHandler {
+  AMDGPUValueHandler(bool IsIncoming, MachineIRBuilder &B,
+                     MachineRegisterInfo &MRI, CCAssignFn *AssignFn)
+      : ValueHandler(IsIncoming, B, MRI, AssignFn) {}
 
-  MachineInstrBuilder MIB;
+  /// Wrapper around extendRegister to ensure we extend to a full 32-bit
+  /// register.
+  Register extendRegisterMin32(Register ValVReg, CCValAssign &VA) {
+    if (VA.getLocVT().getSizeInBits() < 32) {
+      // 16-bit types are reported as legal for 32-bit registers. We need to
+      // extend and do a 32-bit copy to avoid the verifier complaining about it.
+      return MIRBuilder.buildAnyExt(LLT::scalar(32), ValVReg).getReg(0);
+    }
+
+    return extendRegister(ValVReg, VA);
+  }
+};
+
+struct AMDGPUOutgoingValueHandler : public AMDGPUValueHandler {
+  AMDGPUOutgoingValueHandler(MachineIRBuilder &B, MachineRegisterInfo &MRI,
+                             MachineInstrBuilder MIB, CCAssignFn *AssignFn)
+      : AMDGPUValueHandler(false, B, MRI, AssignFn), MIB(MIB) {}
 
-  bool isIncomingArgumentHandler() const override { return false; }
+  MachineInstrBuilder MIB;
 
   Register getStackAddress(uint64_t Size, int64_t Offset,
                            MachinePointerInfo &MPO) override {
@@ -52,13 +71,7 @@ struct OutgoingValueHandler : public CallLowering::ValueHandler {
 
   void assignValueToReg(Register ValVReg, Register PhysReg,
                         CCValAssign &VA) override {
-    Register ExtReg;
-    if (VA.getLocVT().getSizeInBits() < 32) {
-      // 16-bit types are reported as legal for 32-bit registers. We need to
-      // extend and do a 32-bit copy to avoid the verifier complaining about it.
-      ExtReg = MIRBuilder.buildAnyExt(LLT::scalar(32), ValVReg).getReg(0);
-    } else
-      ExtReg = extendRegister(ValVReg, VA);
+    Register ExtReg = extendRegisterMin32(ValVReg, VA);
 
     // If this is a scalar return, insert a readfirstlane just in case the value
     // ends up in a VGPR.
@@ -85,12 +98,12 @@ struct OutgoingValueHandler : public CallLowering::ValueHandler {
   }
 };
 
-struct IncomingArgHandler : public CallLowering::ValueHandler {
+struct AMDGPUIncomingArgHandler : public AMDGPUValueHandler {
   uint64_t StackUsed = 0;
 
-  IncomingArgHandler(MachineIRBuilder &B, MachineRegisterInfo &MRI,
-                     CCAssignFn *AssignFn)
-    : ValueHandler(B, MRI, AssignFn) {}
+  AMDGPUIncomingArgHandler(MachineIRBuilder &B, MachineRegisterInfo &MRI,
+                           CCAssignFn *AssignFn)
+      : AMDGPUValueHandler(true, B, MRI, AssignFn) {}
 
   Register getStackAddress(uint64_t Size, int64_t Offset,
                            MachinePointerInfo &MPO) override {
@@ -129,13 +142,17 @@ struct IncomingArgHandler : public CallLowering::ValueHandler {
     }
   }
 
-  void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size,
+  void assignValueToAddress(Register ValVReg, Register Addr, uint64_t MemSize,
                             MachinePointerInfo &MPO, CCValAssign &VA) override {
     MachineFunction &MF = MIRBuilder.getMF();
 
+    // The reported memory location may be wider than the value.
+    const LLT RegTy = MRI.getType(ValVReg);
+    MemSize = std::min(static_cast<uint64_t>(RegTy.getSizeInBytes()), MemSize);
+
     // FIXME: Get alignment
     auto MMO = MF.getMachineMemOperand(
-        MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, Size,
+        MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, MemSize,
         inferAlignFromPtrInfo(MF, MPO));
     MIRBuilder.buildLoad(ValVReg, Addr, *MMO);
   }
@@ -144,21 +161,107 @@ struct IncomingArgHandler : public CallLowering::ValueHandler {
   /// parameters (it's a basic-block live-in), and a call instruction
   /// (it's an implicit-def of the BL).
   virtual void markPhysRegUsed(unsigned PhysReg) = 0;
-
-  // FIXME: What is the point of this being a callback?
-  bool isIncomingArgumentHandler() const override { return true; }
 };
 
-struct FormalArgHandler : public IncomingArgHandler {
+struct FormalArgHandler : public AMDGPUIncomingArgHandler {
   FormalArgHandler(MachineIRBuilder &B, MachineRegisterInfo &MRI,
                    CCAssignFn *AssignFn)
-    : IncomingArgHandler(B, MRI, AssignFn) {}
+      : AMDGPUIncomingArgHandler(B, MRI, AssignFn) {}
 
   void markPhysRegUsed(unsigned PhysReg) override {
     MIRBuilder.getMBB().addLiveIn(PhysReg);
   }
 };
 
+struct CallReturnHandler : public AMDGPUIncomingArgHandler {
+  CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
+                    MachineInstrBuilder MIB, CCAssignFn *AssignFn)
+      : AMDGPUIncomingArgHandler(MIRBuilder, MRI, AssignFn), MIB(MIB) {}
+
+  void markPhysRegUsed(unsigned PhysReg) override {
+    MIB.addDef(PhysReg, RegState::Implicit);
+  }
+
+  MachineInstrBuilder MIB;
+};
+
+struct AMDGPUOutgoingArgHandler : public AMDGPUValueHandler {
+  MachineInstrBuilder MIB;
+  CCAssignFn *AssignFnVarArg;
+
+  /// For tail calls, the byte offset of the call's argument area from the
+  /// callee's. Unused elsewhere.
+  int FPDiff;
+
+  // Cache the SP register vreg if we need it more than once in this call site.
+  Register SPReg;
+
+  bool IsTailCall;
+
+  AMDGPUOutgoingArgHandler(MachineIRBuilder &MIRBuilder,
+                           MachineRegisterInfo &MRI, MachineInstrBuilder MIB,
+                           CCAssignFn *AssignFn, CCAssignFn *AssignFnVarArg,
+                           bool IsTailCall = false, int FPDiff = 0)
+      : AMDGPUValueHandler(false, MIRBuilder, MRI, AssignFn), MIB(MIB),
+        AssignFnVarArg(AssignFnVarArg), FPDiff(FPDiff), IsTailCall(IsTailCall) {
+  }
+
+  Register getStackAddress(uint64_t Size, int64_t Offset,
+                           MachinePointerInfo &MPO) override {
+    MachineFunction &MF = MIRBuilder.getMF();
+    const LLT PtrTy = LLT::pointer(AMDGPUAS::PRIVATE_ADDRESS, 32);
+    const LLT S32 = LLT::scalar(32);
+
+    if (IsTailCall) {
+      llvm_unreachable("implement me");
+    }
+
+    const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+
+    if (!SPReg)
+      SPReg = MIRBuilder.buildCopy(PtrTy, MFI->getStackPtrOffsetReg()).getReg(0);
+
+    auto OffsetReg = MIRBuilder.buildConstant(S32, Offset);
+
+    auto AddrReg = MIRBuilder.buildPtrAdd(PtrTy, SPReg, OffsetReg);
+    MPO = MachinePointerInfo::getStack(MF, Offset);
+    return AddrReg.getReg(0);
+  }
+
+  void assignValueToReg(Register ValVReg, Register PhysReg,
+                        CCValAssign &VA) override {
+    MIB.addUse(PhysReg, RegState::Implicit);
+    Register ExtReg = extendRegisterMin32(ValVReg, VA);
+    MIRBuilder.buildCopy(PhysReg, ExtReg);
+  }
+
+  void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size,
+                            MachinePointerInfo &MPO, CCValAssign &VA) override {
+    MachineFunction &MF = MIRBuilder.getMF();
+    uint64_t LocMemOffset = VA.getLocMemOffset();
+    const auto &ST = MF.getSubtarget<GCNSubtarget>();
+
+    auto MMO = MF.getMachineMemOperand(
+      MPO, MachineMemOperand::MOStore, Size,
+      commonAlignment(ST.getStackAlignment(), LocMemOffset));
+    MIRBuilder.buildStore(ValVReg, Addr, *MMO);
+  }
+
+  void assignValueToAddress(const CallLowering::ArgInfo &Arg, Register Addr,
+                            uint64_t Size, MachinePointerInfo &MPO,
+                            CCValAssign &VA) override {
+    Register ValVReg = VA.getLocInfo() != CCValAssign::LocInfo::FPExt
+                           ? extendRegister(Arg.Regs[0], VA)
+                           : Arg.Regs[0];
+
+    // If we extended we might need to adjust the MMO's Size.
+    const LLT RegTy = MRI.getType(ValVReg);
+    if (RegTy.getSizeInBytes() > Size)
+      Size = RegTy.getSizeInBytes();
+
+    assignValueToAddress(ValVReg, Addr, Size, MPO, VA);
+  }
+};
 }
 
 AMDGPUCallLowering::AMDGPUCallLowering(const AMDGPUTargetLowering &TLI)
@@ -181,9 +284,10 @@ static ISD::NodeType extOpcodeToISDExtOpcode(unsigned MIOpc) {
 
 void AMDGPUCallLowering::splitToValueTypes(
   MachineIRBuilder &B,
-  const ArgInfo &OrigArg, unsigned OrigArgIdx,
+  const ArgInfo &OrigArg,
   SmallVectorImpl<ArgInfo> &SplitArgs,
   const DataLayout &DL, CallingConv::ID CallConv,
+  bool IsOutgoing,
   SplitArgTy PerformArgSplit) const {
   const SITargetLowering &TLI = *getTLI<SITargetLowering>();
   LLVMContext &Ctx = OrigArg.Ty->getContext();
@@ -202,7 +306,7 @@ void AMDGPUCallLowering::splitToValueTypes(
     Type *Ty = VT.getTypeForEVT(Ctx);
     LLT LLTy = getLLTForType(*Ty, DL);
 
-    if (OrigArgIdx == AttributeList::ReturnIndex && VT.isScalarInteger()) {
+    if (IsOutgoing && VT.isScalarInteger()) {
       unsigned ExtendOp = TargetOpcode::G_ANYEXT;
       if (OrigArg.Flags[0].isSExt()) {
         assert(OrigArg.Regs.size() == 1 && "expect only simple return values");
@@ -214,7 +318,7 @@ void AMDGPUCallLowering::splitToValueTypes(
 
       EVT ExtVT = TLI.getTypeForExtReturn(Ctx, VT,
                                           extOpcodeToISDExtOpcode(ExtendOp));
-      if (ExtVT != VT) {
+      if (ExtVT.getSizeInBits() != VT.getSizeInBits()) {
         VT = ExtVT;
         Ty = ExtVT.getTypeForEVT(Ctx);
         LLTy = getLLTForType(*Ty, DL);
@@ -323,7 +427,7 @@ bool AMDGPUCallLowering::lowerReturnVal(MachineIRBuilder &B,
   SmallVector<ArgInfo, 4> SplitRetInfos;
 
   splitToValueTypes(
-    B, OrigRetInfo, AttributeList::ReturnIndex, SplitRetInfos, DL, CC,
+    B, OrigRetInfo, SplitRetInfos, DL, CC, true,
     [&](ArrayRef<Register> Regs, Register SrcReg, LLT LLTy, LLT PartLLT,
         int VTSplitIdx) {
       unpackRegsToOrigType(B, Regs, SrcReg,
@@ -332,7 +436,7 @@ bool AMDGPUCallLowering::lowerReturnVal(MachineIRBuilder &B,
     });
 
   CCAssignFn *AssignFn = TLI.CCAssignFnForReturn(CC, F.isVarArg());
-  OutgoingValueHandler RetHandler(B, *MRI, Ret, AssignFn);
+  AMDGPUOutgoingValueHandler RetHandler(B, *MRI, Ret, AssignFn);
   return handleAssignments(B, SplitRetInfos, RetHandler);
 }
 
@@ -385,24 +489,19 @@ bool AMDGPUCallLowering::lowerReturn(MachineIRBuilder &B,
   return true;
 }
 
-Register AMDGPUCallLowering::lowerParameterPtr(MachineIRBuilder &B,
-                                               Type *ParamTy,
-                                               uint64_t Offset) const {
-
+void AMDGPUCallLowering::lowerParameterPtr(Register DstReg, MachineIRBuilder &B,
+                                           Type *ParamTy,
+                                           uint64_t Offset) const {
   MachineFunction &MF = B.getMF();
   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
   MachineRegisterInfo &MRI = MF.getRegInfo();
-  const Function &F = MF.getFunction();
-  const DataLayout &DL = F.getParent()->getDataLayout();
-  PointerType *PtrTy = PointerType::get(ParamTy, AMDGPUAS::CONSTANT_ADDRESS);
-  LLT PtrType = getLLTForType(*PtrTy, DL);
   Register KernArgSegmentPtr =
     MFI->getPreloadedReg(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
   Register KernArgSegmentVReg = MRI.getLiveInVirtReg(KernArgSegmentPtr);
 
   auto OffsetReg = B.buildConstant(LLT::scalar(64), Offset);
 
-  return B.buildPtrAdd(PtrType, KernArgSegmentVReg, OffsetReg).getReg(0);
+  B.buildPtrAdd(DstReg, KernArgSegmentVReg, OffsetReg);
 }
 
 void AMDGPUCallLowering::lowerParameter(MachineIRBuilder &B, Type *ParamTy,
@@ -413,7 +512,10 @@ void AMDGPUCallLowering::lowerParameter(MachineIRBuilder &B, Type *ParamTy,
   const DataLayout &DL = F.getParent()->getDataLayout();
   MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
   unsigned TypeSize = DL.getTypeStoreSize(ParamTy);
-  Register PtrReg = lowerParameterPtr(B, ParamTy, Offset);
+
+  LLT PtrTy = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
+  Register PtrReg = B.getMRI()->createGenericVirtualRegister(PtrTy);
+  lowerParameterPtr(PtrReg, B, ParamTy, Offset);
 
   MachineMemOperand *MMO = MF.getMachineMemOperand(
       PtrInfo,
@@ -500,12 +602,15 @@ bool AMDGPUCallLowering::lowerFormalArgumentsKernel(
 
   // TODO: Align down to dword alignment and extract bits for extending loads.
   for (auto &Arg : F.args()) {
-    Type *ArgTy = Arg.getType();
+    const bool IsByRef = Arg.hasByRefAttr();
+    Type *ArgTy = IsByRef ? Arg.getParamByRefType() : Arg.getType();
     unsigned AllocSize = DL.getTypeAllocSize(ArgTy);
     if (AllocSize == 0)
       continue;
 
-    Align ABIAlign = DL.getABITypeAlign(ArgTy);
+    MaybeAlign ABIAlign = IsByRef ? Arg.getParamAlign() : None;
+    if (!ABIAlign)
+      ABIAlign = DL.getABITypeAlign(ArgTy);
 
     uint64_t ArgOffset = alignTo(ExplicitArgOffset, ABIAlign) + BaseOffset;
     ExplicitArgOffset = alignTo(ExplicitArgOffset, ABIAlign) + AllocSize;
@@ -515,16 +620,34 @@ bool AMDGPUCallLowering::lowerFormalArgumentsKernel(
       continue;
     }
 
-    ArrayRef<Register> OrigArgRegs = VRegs[i];
-    Register ArgReg =
-      OrigArgRegs.size() == 1
-      ? OrigArgRegs[0]
-      : MRI.createGenericVirtualRegister(getLLTForType(*ArgTy, DL));
-
     Align Alignment = commonAlignment(KernArgBaseAlign, ArgOffset);
-    lowerParameter(B, ArgTy, ArgOffset, Alignment, ArgReg);
-    if (OrigArgRegs.size() > 1)
-      unpackRegs(OrigArgRegs, ArgReg, ArgTy, B);
+
+    if (IsByRef) {
+      unsigned ByRefAS = cast<PointerType>(Arg.getType())->getAddressSpace();
+
+      assert(VRegs[i].size() == 1 &&
+             "expected only one register for byval pointers");
+      if (ByRefAS == AMDGPUAS::CONSTANT_ADDRESS) {
+        lowerParameterPtr(VRegs[i][0], B, ArgTy, ArgOffset);
+      } else {
+        const LLT ConstPtrTy = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
+        Register PtrReg = MRI.createGenericVirtualRegister(ConstPtrTy);
+        lowerParameterPtr(PtrReg, B, ArgTy, ArgOffset);
+
+        B.buildAddrSpaceCast(VRegs[i][0], PtrReg);
+      }
+    } else {
+      ArrayRef<Register> OrigArgRegs = VRegs[i];
+      Register ArgReg =
+        OrigArgRegs.size() == 1
+        ? OrigArgRegs[0]
+        : MRI.createGenericVirtualRegister(getLLTForType(*ArgTy, DL));
+
+      lowerParameter(B, ArgTy, ArgOffset, Alignment, ArgReg);
+      if (OrigArgRegs.size() > 1)
+        unpackRegs(OrigArgRegs, ArgReg, ArgTy, B);
+    }
+
     ++i;
   }
 
@@ -730,7 +853,7 @@ bool AMDGPUCallLowering::lowerFormalArguments(
     setArgFlags(OrigArg, OrigArgIdx, DL, F);
 
     splitToValueTypes(
-      B, OrigArg, OrigArgIdx, SplitArgs, DL, CC,
+      B, OrigArg, SplitArgs, DL, CC, false,
       // FIXME: We should probably be passing multiple registers to
       // handleAssignments to do this
       [&](ArrayRef<Register> Regs, Register DstReg,
@@ -818,3 +941,345 @@ bool AMDGPUCallLowering::lowerFormalArguments(
 
   return true;
 }
+
+bool AMDGPUCallLowering::passSpecialInputs(MachineIRBuilder &MIRBuilder,
+                                           CCState &CCInfo,
+                                           SmallVectorImpl<std::pair<MCRegister, Register>> &ArgRegs,
+                                           CallLoweringInfo &Info) const {
+  MachineFunction &MF = MIRBuilder.getMF();
+
+  const AMDGPUFunctionArgInfo *CalleeArgInfo
+    = &AMDGPUArgumentUsageInfo::FixedABIFunctionInfo;
+
+  const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+  const AMDGPUFunctionArgInfo &CallerArgInfo = MFI->getArgInfo();
+
+
+  // TODO: Unify with private memory register handling. This is complicated by
+  // the fact that at least in kernels, the input argument is not necessarily
+  // in the same location as the input.
+  AMDGPUFunctionArgInfo::PreloadedValue InputRegs[] = {
+    AMDGPUFunctionArgInfo::DISPATCH_PTR,
+    AMDGPUFunctionArgInfo::QUEUE_PTR,
+    AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR,
+    AMDGPUFunctionArgInfo::DISPATCH_ID,
+    AMDGPUFunctionArgInfo::WORKGROUP_ID_X,
+    AMDGPUFunctionArgInfo::WORKGROUP_ID_Y,
+    AMDGPUFunctionArgInfo::WORKGROUP_ID_Z
+  };
+
+  MachineRegisterInfo &MRI = MF.getRegInfo();
+
+  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+  const AMDGPULegalizerInfo *LI
+    = static_cast<const AMDGPULegalizerInfo*>(ST.getLegalizerInfo());
+
+  for (auto InputID : InputRegs) {
+    const ArgDescriptor *OutgoingArg;
+    const TargetRegisterClass *ArgRC;
+    LLT ArgTy;
+
+    std::tie(OutgoingArg, ArgRC, ArgTy) =
+        CalleeArgInfo->getPreloadedValue(InputID);
+    if (!OutgoingArg)
+      continue;
+
+    const ArgDescriptor *IncomingArg;
+    const TargetRegisterClass *IncomingArgRC;
+    std::tie(IncomingArg, IncomingArgRC, ArgTy) =
+        CallerArgInfo.getPreloadedValue(InputID);
+    assert(IncomingArgRC == ArgRC);
+
+    Register InputReg = MRI.createGenericVirtualRegister(ArgTy);
+
+    if (IncomingArg) {
+      LI->loadInputValue(InputReg, MIRBuilder, IncomingArg);
+    } else {
+      assert(InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR);
+      LI->getImplicitArgPtr(InputReg, MRI, MIRBuilder);
+    }
+
+    if (OutgoingArg->isRegister()) {
+      ArgRegs.emplace_back(OutgoingArg->getRegister(), InputReg);
+      if (!CCInfo.AllocateReg(OutgoingArg->getRegister()))
+        report_fatal_error("failed to allocate implicit input argument");
+    } else {
+      LLVM_DEBUG(dbgs() << "Unhandled stack passed implicit input argument\n");
+      return false;
+    }
+  }
+
+  // Pack workitem IDs into a single register or pass it as is if already
+  // packed.
+  const ArgDescriptor *OutgoingArg;
+  const TargetRegisterClass *ArgRC;
+  LLT ArgTy;
+
+  std::tie(OutgoingArg, ArgRC, ArgTy) =
+      CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X);
+  if (!OutgoingArg)
+    std::tie(OutgoingArg, ArgRC, ArgTy) =
+        CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
+  if (!OutgoingArg)
+    std::tie(OutgoingArg, ArgRC, ArgTy) =
+        CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
+  if (!OutgoingArg)
+    return false;
+
+  const ArgDescriptor *IncomingArgX = std::get<0>(
+      CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X));
+  const ArgDescriptor *IncomingArgY = std::get<0>(
+      CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y));
+  const ArgDescriptor *IncomingArgZ = std::get<0>(
+      CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z));
+
+  const LLT S32 = LLT::scalar(32);
+
+  // If incoming ids are not packed we need to pack them.
+  // FIXME: Should consider known workgroup size to eliminate known 0 cases.
+  Register InputReg;
+  if (IncomingArgX && !IncomingArgX->isMasked() && CalleeArgInfo->WorkItemIDX) {
+    InputReg = MRI.createGenericVirtualRegister(S32);
+    LI->loadInputValue(InputReg, MIRBuilder, IncomingArgX);
+  }
+
+  if (IncomingArgY && !IncomingArgY->isMasked() && CalleeArgInfo->WorkItemIDY) {
+    Register Y = MRI.createGenericVirtualRegister(S32);
+    LI->loadInputValue(Y, MIRBuilder, IncomingArgY);
+
+    Y = MIRBuilder.buildShl(S32, Y, MIRBuilder.buildConstant(S32, 10)).getReg(0);
+    InputReg = InputReg ? MIRBuilder.buildOr(S32, InputReg, Y).getReg(0) : Y;
+  }
+
+  if (IncomingArgZ && !IncomingArgZ->isMasked() && CalleeArgInfo->WorkItemIDZ) {
+    Register Z = MRI.createGenericVirtualRegister(S32);
+    LI->loadInputValue(Z, MIRBuilder, IncomingArgZ);
+
+    Z = MIRBuilder.buildShl(S32, Z, MIRBuilder.buildConstant(S32, 20)).getReg(0);
+    InputReg = InputReg ? MIRBuilder.buildOr(S32, InputReg, Z).getReg(0) : Z;
+  }
+
+  if (!InputReg) {
+    InputReg = MRI.createGenericVirtualRegister(S32);
+
+    // Workitem ids are already packed, any of present incoming arguments will
+    // carry all required fields.
+    ArgDescriptor IncomingArg = ArgDescriptor::createArg(
+      IncomingArgX ? *IncomingArgX :
+        IncomingArgY ? *IncomingArgY : *IncomingArgZ, ~0u);
+    LI->loadInputValue(InputReg, MIRBuilder, &IncomingArg);
+  }
+
+  if (OutgoingArg->isRegister()) {
+    ArgRegs.emplace_back(OutgoingArg->getRegister(), InputReg);
+    if (!CCInfo.AllocateReg(OutgoingArg->getRegister()))
+      report_fatal_error("failed to allocate implicit input argument");
+  } else {
+    LLVM_DEBUG(dbgs() << "Unhandled stack passed implicit input argument\n");
+    return false;
+  }
+
+  return true;
+}
+
+/// Returns a pair containing the fixed CCAssignFn and the vararg CCAssignFn for
+/// CC.
+static std::pair<CCAssignFn *, CCAssignFn *>
+getAssignFnsForCC(CallingConv::ID CC, const SITargetLowering &TLI) {
+  return {TLI.CCAssignFnForCall(CC, false), TLI.CCAssignFnForCall(CC, true)};
+}
+
+static unsigned getCallOpcode(const MachineFunction &CallerF, bool IsIndirect,
+                              bool IsTailCall) {
+  return AMDGPU::SI_CALL;
+}
+
+// Add operands to call instruction to track the callee.
+static bool addCallTargetOperands(MachineInstrBuilder &CallInst,
+                                  MachineIRBuilder &MIRBuilder,
+                                  AMDGPUCallLowering::CallLoweringInfo &Info) {
+  if (Info.Callee.isReg()) {
+    CallInst.addReg(Info.Callee.getReg());
+    CallInst.addImm(0);
+  } else if (Info.Callee.isGlobal() && Info.Callee.getOffset() == 0) {
+    // The call lowering lightly assumed we can directly encode a call target in
+    // the instruction, which is not the case. Materialize the address here.
+    const GlobalValue *GV = Info.Callee.getGlobal();
+    auto Ptr = MIRBuilder.buildGlobalValue(
+      LLT::pointer(GV->getAddressSpace(), 64), GV);
+    CallInst.addReg(Ptr.getReg(0));
+    CallInst.add(Info.Callee);
+  } else
+    return false;
+
+  return true;
+}
+
+bool AMDGPUCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
+                                   CallLoweringInfo &Info) const {
+  if (!AMDGPUTargetMachine::EnableFixedFunctionABI) {
+    LLVM_DEBUG(dbgs() << "Variable function ABI not implemented\n");
+    return false;
+  }
+
+  if (Info.IsVarArg) {
+    LLVM_DEBUG(dbgs() << "Variadic functions not implemented\n");
+    return false;
+  }
+
+  MachineFunction &MF = MIRBuilder.getMF();
+  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+  const SIRegisterInfo *TRI = ST.getRegisterInfo();
+
+  const Function &F = MF.getFunction();
+  MachineRegisterInfo &MRI = MF.getRegInfo();
+  const SITargetLowering &TLI = *getTLI<SITargetLowering>();
+  const DataLayout &DL = F.getParent()->getDataLayout();
+
+  if (AMDGPU::isShader(F.getCallingConv())) {
+    LLVM_DEBUG(dbgs() << "Unhandled call from graphics shader\n");
+    return false;
+  }
+
+  SmallVector<ArgInfo, 8> OutArgs;
+  SmallVector<ArgInfo, 4> SplitRetInfos;
+
+  for (auto &OrigArg : Info.OrigArgs) {
+    splitToValueTypes(
+      MIRBuilder, OrigArg, OutArgs, DL, Info.CallConv, true,
+      // FIXME: We should probably be passing multiple registers to
+      // handleAssignments to do this
+      [&](ArrayRef<Register> Regs, Register SrcReg, LLT LLTy, LLT PartLLT,
+          int VTSplitIdx) {
+        unpackRegsToOrigType(MIRBuilder, Regs, SrcReg, OrigArg, LLTy, PartLLT);
+      });
+  }
+
+  // If we can lower as a tail call, do that instead.
+  bool CanTailCallOpt = false;
+
+  // We must emit a tail call if we have musttail.
+  if (Info.IsMustTailCall && !CanTailCallOpt) {
+    LLVM_DEBUG(dbgs() << "Failed to lower musttail call as tail call\n");
+    return false;
+  }
+
+  // Find out which ABI gets to decide where things go.
+  CCAssignFn *AssignFnFixed;
+  CCAssignFn *AssignFnVarArg;
+  std::tie(AssignFnFixed, AssignFnVarArg) =
+      getAssignFnsForCC(Info.CallConv, TLI);
+
+  MIRBuilder.buildInstr(AMDGPU::ADJCALLSTACKUP)
+    .addImm(0)
+    .addImm(0);
+
+  // Create a temporarily-floating call instruction so we can add the implicit
+  // uses of arg registers.
+  unsigned Opc = getCallOpcode(MF, Info.Callee.isReg(), false);
+
+  auto MIB = MIRBuilder.buildInstrNoInsert(Opc);
+  MIB.addDef(TRI->getReturnAddressReg(MF));
+
+  if (!addCallTargetOperands(MIB, MIRBuilder, Info))
+    return false;
+
+  // Tell the call which registers are clobbered.
+  const uint32_t *Mask = TRI->getCallPreservedMask(MF, Info.CallConv);
+  MIB.addRegMask(Mask);
+
+  SmallVector<CCValAssign, 16> ArgLocs;
+  CCState CCInfo(Info.CallConv, Info.IsVarArg, MF, ArgLocs, F.getContext());
+
+  // We could pass MIB and directly add the implicit uses to the call
+  // now. However, as an aesthetic choice, place implicit argument operands
+  // after the ordinary user argument registers.
+  SmallVector<std::pair<MCRegister, Register>, 12> ImplicitArgRegs;
+
+  if (AMDGPUTargetMachine::EnableFixedFunctionABI) {
+    // With a fixed ABI, allocate fixed registers before user arguments.
+    if (!passSpecialInputs(MIRBuilder, CCInfo, ImplicitArgRegs, Info))
+      return false;
+  }
+
+  // Do the actual argument marshalling.
+  SmallVector<Register, 8> PhysRegs;
+  AMDGPUOutgoingArgHandler Handler(MIRBuilder, MRI, MIB, AssignFnFixed,
+                                   AssignFnVarArg, false);
+  if (!handleAssignments(CCInfo, ArgLocs, MIRBuilder, OutArgs, Handler))
+    return false;
+
+  const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+
+  // Insert copies for the SRD. In the HSA case, this should be an identity
+  // copy.
+  auto ScratchRSrcReg = MIRBuilder.buildCopy(LLT::vector(4, 32),
+                                             MFI->getScratchRSrcReg());
+  MIRBuilder.buildCopy(AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3, ScratchRSrcReg);
+  MIB.addReg(AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3, RegState::Implicit);
+
+  for (std::pair<MCRegister, Register> ArgReg : ImplicitArgRegs) {
+    MIRBuilder.buildCopy((Register)ArgReg.first, ArgReg.second);
+    MIB.addReg(ArgReg.first, RegState::Implicit);
+  }
+
+  // Get a count of how many bytes are to be pushed on the stack.
+  unsigned NumBytes = CCInfo.getNextStackOffset();
+
+  // If Callee is a reg, since it is used by a target specific
+  // instruction, it must have a register class matching the
+  // constraint of that instruction.
+
+  // FIXME: We should define regbankselectable call instructions to handle
+  // divergent call targets.
+  if (MIB->getOperand(1).isReg()) {
+    MIB->getOperand(1).setReg(constrainOperandRegClass(
+        MF, *TRI, MRI, *ST.getInstrInfo(),
+        *ST.getRegBankInfo(), *MIB, MIB->getDesc(), MIB->getOperand(1),
+        1));
+  }
+
+  auto OrigInsertPt = MIRBuilder.getInsertPt();
+
+  // Now we can add the actual call instruction to the correct position.
+  MIRBuilder.insertInstr(MIB);
+
+  // Insert this now to give us an anchor point for managing the insert point.
+  MachineInstrBuilder CallSeqEnd =
+    MIRBuilder.buildInstr(AMDGPU::ADJCALLSTACKDOWN);
+
+  SmallVector<ArgInfo, 8> InArgs;
+  if (!Info.OrigRet.Ty->isVoidTy()) {
+    splitToValueTypes(
+      MIRBuilder, Info.OrigRet, InArgs, DL, Info.CallConv, false,
+      [&](ArrayRef<Register> Regs, Register DstReg,
+          LLT LLTy, LLT PartLLT, int VTSplitIdx) {
+        assert(DstReg == Info.OrigRet.Regs[VTSplitIdx]);
+        packSplitRegsToOrigType(MIRBuilder,  Info.OrigRet.Regs[VTSplitIdx],
+                                Regs, LLTy, PartLLT);
+      });
+  }
+
+  // Make sure the raw argument copies are inserted before the marshalling to
+  // the original types.
+  MIRBuilder.setInsertPt(MIRBuilder.getMBB(), CallSeqEnd);
+
+  // Finally we can copy the returned value back into its virtual-register. In
+  // symmetry with the arguments, the physical register must be an
+  // implicit-define of the call instruction.
+  if (!Info.OrigRet.Ty->isVoidTy()) {
+    CCAssignFn *RetAssignFn = TLI.CCAssignFnForReturn(Info.CallConv,
+                                                      Info.IsVarArg);
+    CallReturnHandler Handler(MIRBuilder, MRI, MIB, RetAssignFn);
+    if (!handleAssignments(MIRBuilder, InArgs, Handler))
+      return false;
+  }
+
+  uint64_t CalleePopBytes = NumBytes;
+  CallSeqEnd.addImm(0)
+            .addImm(CalleePopBytes);
+
+  // Restore the insert point to after the call sequence.
+  MIRBuilder.setInsertPt(MIRBuilder.getMBB(), OrigInsertPt);
+  return true;
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.h
index 446619d1502ee..d11b9801b9716 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.h
@@ -23,8 +23,8 @@ class AMDGPUTargetLowering;
 class MachineInstrBuilder;
 
 class AMDGPUCallLowering: public CallLowering {
-  Register lowerParameterPtr(MachineIRBuilder &B, Type *ParamTy,
-                             uint64_t Offset) const;
+  void lowerParameterPtr(Register DstReg, MachineIRBuilder &B, Type *ParamTy,
+                         uint64_t Offset) const;
 
   void lowerParameter(MachineIRBuilder &B, Type *ParamTy, uint64_t Offset,
                       Align Alignment, Register DstReg) const;
@@ -34,10 +34,10 @@ class AMDGPUCallLowering: public CallLowering {
 
   void splitToValueTypes(MachineIRBuilder &B,
                          const ArgInfo &OrigArgInfo,
-                         unsigned OrigArgIdx,
                          SmallVectorImpl<ArgInfo> &SplitArgs,
                          const DataLayout &DL,
                          CallingConv::ID CallConv,
+                         bool IsOutgoing,
                          SplitArgTy SplitArg) const;
 
   bool lowerReturnVal(MachineIRBuilder &B, const Value *Val,
@@ -54,6 +54,15 @@ class AMDGPUCallLowering: public CallLowering {
 
   bool lowerFormalArguments(MachineIRBuilder &B, const Function &F,
                             ArrayRef<ArrayRef<Register>> VRegs) const override;
+
+  bool passSpecialInputs(MachineIRBuilder &MIRBuilder,
+                         CCState &CCInfo,
+                         SmallVectorImpl<std::pair<MCRegister, Register>> &ArgRegs,
+                         CallLoweringInfo &Info) const;
+
+  bool lowerCall(MachineIRBuilder &MIRBuilder,
+                 CallLoweringInfo &Info) const override;
+
   static CCAssignFn *CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg);
   static CCAssignFn *CCAssignFnForReturn(CallingConv::ID CC, bool IsVarArg);
 };
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
index c6f6a3b84e367..5ecb048797963 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
@@ -24,6 +24,23 @@
 #include "llvm/IR/Module.h"
 #include "llvm/Support/raw_ostream.h"
 
+using namespace llvm;
+
+static std::pair<Type *, Align> getArgumentTypeAlign(const Argument &Arg,
+                                                     const DataLayout &DL) {
+  Type *Ty = Arg.getType();
+  MaybeAlign ArgAlign;
+  if (Arg.hasByRefAttr()) {
+    Ty = Arg.getParamByRefType();
+    ArgAlign = Arg.getParamAlign();
+  }
+
+  if (!ArgAlign)
+    ArgAlign = DL.getABITypeAlign(Ty);
+
+  return std::make_pair(Ty, *ArgAlign);
+}
+
 namespace llvm {
 
 static cl::opt<bool> DumpHSAMetadata(
@@ -311,23 +328,28 @@ void MetadataStreamerV2::emitKernelArg(const Argument &Arg) {
   if (Node && ArgNo < Node->getNumOperands())
     TypeQual = cast<MDString>(Node->getOperand(ArgNo))->getString();
 
-  Type *Ty = Arg.getType();
   const DataLayout &DL = Func->getParent()->getDataLayout();
 
   MaybeAlign PointeeAlign;
-  if (auto PtrTy = dyn_cast<PointerType>(Ty)) {
+  if (auto PtrTy = dyn_cast<PointerType>(Arg.getType())) {
     if (PtrTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
+      // FIXME: Should report this for all address spaces
       PointeeAlign = DL.getValueOrABITypeAlignment(Arg.getParamAlign(),
                                                    PtrTy->getElementType());
     }
   }
 
-  emitKernelArg(DL, Ty, getValueKind(Arg.getType(), TypeQual, BaseTypeName),
-                PointeeAlign, Name, TypeName, BaseTypeName, AccQual, TypeQual);
+  Type *ArgTy;
+  Align ArgAlign;
+  std::tie(ArgTy, ArgAlign) = getArgumentTypeAlign(Arg, DL);
+
+  emitKernelArg(DL, ArgTy, ArgAlign,
+                getValueKind(ArgTy, TypeQual, BaseTypeName), PointeeAlign, Name,
+                TypeName, BaseTypeName, AccQual, TypeQual);
 }
 
 void MetadataStreamerV2::emitKernelArg(const DataLayout &DL, Type *Ty,
-                                       ValueKind ValueKind,
+                                       Align Alignment, ValueKind ValueKind,
                                        MaybeAlign PointeeAlign, StringRef Name,
                                        StringRef TypeName,
                                        StringRef BaseTypeName,
@@ -338,7 +360,7 @@ void MetadataStreamerV2::emitKernelArg(const DataLayout &DL, Type *Ty,
   Arg.mName = std::string(Name);
   Arg.mTypeName = std::string(TypeName);
   Arg.mSize = DL.getTypeAllocSize(Ty);
-  Arg.mAlign = DL.getABITypeAlign(Ty).value();
+  Arg.mAlign = Alignment.value();
   Arg.mValueKind = ValueKind;
   Arg.mPointeeAlign = PointeeAlign ? PointeeAlign->value() : 0;
 
@@ -374,11 +396,11 @@ void MetadataStreamerV2::emitHiddenKernelArgs(const Function &Func) {
   auto Int64Ty = Type::getInt64Ty(Func.getContext());
 
   if (HiddenArgNumBytes >= 8)
-    emitKernelArg(DL, Int64Ty, ValueKind::HiddenGlobalOffsetX);
+    emitKernelArg(DL, Int64Ty, Align(8), ValueKind::HiddenGlobalOffsetX);
   if (HiddenArgNumBytes >= 16)
-    emitKernelArg(DL, Int64Ty, ValueKind::HiddenGlobalOffsetY);
+    emitKernelArg(DL, Int64Ty, Align(8), ValueKind::HiddenGlobalOffsetY);
   if (HiddenArgNumBytes >= 24)
-    emitKernelArg(DL, Int64Ty, ValueKind::HiddenGlobalOffsetZ);
+    emitKernelArg(DL, Int64Ty, Align(8), ValueKind::HiddenGlobalOffsetZ);
 
   auto Int8PtrTy = Type::getInt8PtrTy(Func.getContext(),
                                       AMDGPUAS::GLOBAL_ADDRESS);
@@ -387,31 +409,31 @@ void MetadataStreamerV2::emitHiddenKernelArgs(const Function &Func) {
   // "none" argument.
   if (HiddenArgNumBytes >= 32) {
     if (Func.getParent()->getNamedMetadata("llvm.printf.fmts"))
-      emitKernelArg(DL, Int8PtrTy, ValueKind::HiddenPrintfBuffer);
+      emitKernelArg(DL, Int8PtrTy, Align(8), ValueKind::HiddenPrintfBuffer);
     else if (Func.getParent()->getFunction("__ockl_hostcall_internal")) {
       // The printf runtime binding pass should have ensured that hostcall and
       // printf are not used in the same module.
       assert(!Func.getParent()->getNamedMetadata("llvm.printf.fmts"));
-      emitKernelArg(DL, Int8PtrTy, ValueKind::HiddenHostcallBuffer);
+      emitKernelArg(DL, Int8PtrTy, Align(8), ValueKind::HiddenHostcallBuffer);
     } else
-      emitKernelArg(DL, Int8PtrTy, ValueKind::HiddenNone);
+      emitKernelArg(DL, Int8PtrTy, Align(8), ValueKind::HiddenNone);
   }
 
   // Emit "default queue" and "completion action" arguments if enqueue kernel is
   // used, otherwise emit dummy "none" arguments.
   if (HiddenArgNumBytes >= 48) {
     if (Func.hasFnAttribute("calls-enqueue-kernel")) {
-      emitKernelArg(DL, Int8PtrTy, ValueKind::HiddenDefaultQueue);
-      emitKernelArg(DL, Int8PtrTy, ValueKind::HiddenCompletionAction);
+      emitKernelArg(DL, Int8PtrTy, Align(8), ValueKind::HiddenDefaultQueue);
+      emitKernelArg(DL, Int8PtrTy, Align(8), ValueKind::HiddenCompletionAction);
     } else {
-      emitKernelArg(DL, Int8PtrTy, ValueKind::HiddenNone);
-      emitKernelArg(DL, Int8PtrTy, ValueKind::HiddenNone);
+      emitKernelArg(DL, Int8PtrTy, Align(8), ValueKind::HiddenNone);
+      emitKernelArg(DL, Int8PtrTy, Align(8), ValueKind::HiddenNone);
     }
   }
 
   // Emit the pointer argument for multi-grid object.
   if (HiddenArgNumBytes >= 56)
-    emitKernelArg(DL, Int8PtrTy, ValueKind::HiddenMultiGridSyncArg);
+    emitKernelArg(DL, Int8PtrTy, Align(8), ValueKind::HiddenMultiGridSyncArg);
 }
 
 bool MetadataStreamerV2::emitTo(AMDGPUTargetStreamer &TargetStreamer) {
@@ -699,10 +721,12 @@ void MetadataStreamerV3::emitKernelArg(const Argument &Arg, unsigned &Offset,
   if (Node && ArgNo < Node->getNumOperands())
     TypeQual = cast<MDString>(Node->getOperand(ArgNo))->getString();
 
-  Type *Ty = Arg.getType();
   const DataLayout &DL = Func->getParent()->getDataLayout();
 
   MaybeAlign PointeeAlign;
+  Type *Ty = Arg.hasByRefAttr() ? Arg.getParamByRefType() : Arg.getType();
+
+  // FIXME: Need to distinguish in memory alignment from pointer alignment.
   if (auto PtrTy = dyn_cast<PointerType>(Ty)) {
     if (PtrTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
       PointeeAlign = DL.getValueOrABITypeAlignment(Arg.getParamAlign(),
@@ -710,19 +734,21 @@ void MetadataStreamerV3::emitKernelArg(const Argument &Arg, unsigned &Offset,
     }
   }
 
-  emitKernelArg(Func->getParent()->getDataLayout(), Arg.getType(),
-                getValueKind(Arg.getType(), TypeQual, BaseTypeName), Offset,
-                Args, PointeeAlign, Name, TypeName, BaseTypeName, AccQual,
-                TypeQual);
+  // There's no distinction between byval aggregates and raw aggregates.
+  Type *ArgTy;
+  Align ArgAlign;
+  std::tie(ArgTy, ArgAlign) = getArgumentTypeAlign(Arg, DL);
+
+  emitKernelArg(DL, ArgTy, ArgAlign,
+                getValueKind(ArgTy, TypeQual, BaseTypeName), Offset, Args,
+                PointeeAlign, Name, TypeName, BaseTypeName, AccQual, TypeQual);
 }
 
-void MetadataStreamerV3::emitKernelArg(const DataLayout &DL, Type *Ty,
-                                       StringRef ValueKind, unsigned &Offset,
-                                       msgpack::ArrayDocNode Args,
-                                       MaybeAlign PointeeAlign, StringRef Name,
-                                       StringRef TypeName,
-                                       StringRef BaseTypeName,
-                                       StringRef AccQual, StringRef TypeQual) {
+void MetadataStreamerV3::emitKernelArg(
+    const DataLayout &DL, Type *Ty, Align Alignment, StringRef ValueKind,
+    unsigned &Offset, msgpack::ArrayDocNode Args, MaybeAlign PointeeAlign,
+    StringRef Name, StringRef TypeName, StringRef BaseTypeName,
+    StringRef AccQual, StringRef TypeQual) {
   auto Arg = Args.getDocument()->getMapNode();
 
   if (!Name.empty())
@@ -730,7 +756,6 @@ void MetadataStreamerV3::emitKernelArg(const DataLayout &DL, Type *Ty,
   if (!TypeName.empty())
     Arg[".type_name"] = Arg.getDocument()->getNode(TypeName, /*Copy=*/true);
   auto Size = DL.getTypeAllocSize(Ty);
-  Align Alignment = DL.getABITypeAlign(Ty);
   Arg[".size"] = Arg.getDocument()->getNode(Size);
   Offset = alignTo(Offset, Alignment);
   Arg[".offset"] = Arg.getDocument()->getNode(Offset);
@@ -777,11 +802,14 @@ void MetadataStreamerV3::emitHiddenKernelArgs(const Function &Func,
   auto Int64Ty = Type::getInt64Ty(Func.getContext());
 
   if (HiddenArgNumBytes >= 8)
-    emitKernelArg(DL, Int64Ty, "hidden_global_offset_x", Offset, Args);
+    emitKernelArg(DL, Int64Ty, Align(8), "hidden_global_offset_x", Offset,
+                  Args);
   if (HiddenArgNumBytes >= 16)
-    emitKernelArg(DL, Int64Ty, "hidden_global_offset_y", Offset, Args);
+    emitKernelArg(DL, Int64Ty, Align(8), "hidden_global_offset_y", Offset,
+                  Args);
   if (HiddenArgNumBytes >= 24)
-    emitKernelArg(DL, Int64Ty, "hidden_global_offset_z", Offset, Args);
+    emitKernelArg(DL, Int64Ty, Align(8), "hidden_global_offset_z", Offset,
+                  Args);
 
   auto Int8PtrTy =
       Type::getInt8PtrTy(Func.getContext(), AMDGPUAS::GLOBAL_ADDRESS);
@@ -790,31 +818,36 @@ void MetadataStreamerV3::emitHiddenKernelArgs(const Function &Func,
   // "none" argument.
   if (HiddenArgNumBytes >= 32) {
     if (Func.getParent()->getNamedMetadata("llvm.printf.fmts"))
-      emitKernelArg(DL, Int8PtrTy, "hidden_printf_buffer", Offset, Args);
+      emitKernelArg(DL, Int8PtrTy, Align(8), "hidden_printf_buffer", Offset,
+                    Args);
     else if (Func.getParent()->getFunction("__ockl_hostcall_internal")) {
       // The printf runtime binding pass should have ensured that hostcall and
       // printf are not used in the same module.
       assert(!Func.getParent()->getNamedMetadata("llvm.printf.fmts"));
-      emitKernelArg(DL, Int8PtrTy, "hidden_hostcall_buffer", Offset, Args);
+      emitKernelArg(DL, Int8PtrTy, Align(8), "hidden_hostcall_buffer", Offset,
+                    Args);
     } else
-      emitKernelArg(DL, Int8PtrTy, "hidden_none", Offset, Args);
+      emitKernelArg(DL, Int8PtrTy, Align(8), "hidden_none", Offset, Args);
   }
 
   // Emit "default queue" and "completion action" arguments if enqueue kernel is
   // used, otherwise emit dummy "none" arguments.
   if (HiddenArgNumBytes >= 48) {
     if (Func.hasFnAttribute("calls-enqueue-kernel")) {
-      emitKernelArg(DL, Int8PtrTy, "hidden_default_queue", Offset, Args);
-      emitKernelArg(DL, Int8PtrTy, "hidden_completion_action", Offset, Args);
+      emitKernelArg(DL, Int8PtrTy, Align(8), "hidden_default_queue", Offset,
+                    Args);
+      emitKernelArg(DL, Int8PtrTy, Align(8), "hidden_completion_action", Offset,
+                    Args);
     } else {
-      emitKernelArg(DL, Int8PtrTy, "hidden_none", Offset, Args);
-      emitKernelArg(DL, Int8PtrTy, "hidden_none", Offset, Args);
+      emitKernelArg(DL, Int8PtrTy, Align(8), "hidden_none", Offset, Args);
+      emitKernelArg(DL, Int8PtrTy, Align(8), "hidden_none", Offset, Args);
     }
   }
 
   // Emit the pointer argument for multi-grid object.
   if (HiddenArgNumBytes >= 56)
-    emitKernelArg(DL, Int8PtrTy, "hidden_multigrid_sync_arg", Offset, Args);
+    emitKernelArg(DL, Int8PtrTy, Align(8), "hidden_multigrid_sync_arg", Offset,
+                  Args);
 }
 
 msgpack::MapDocNode
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h
index 9534fffd228d3..8390ea90e2719 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h
@@ -87,11 +87,12 @@ class MetadataStreamerV3 final : public MetadataStreamer {
   void emitKernelArg(const Argument &Arg, unsigned &Offset,
                      msgpack::ArrayDocNode Args);
 
-  void emitKernelArg(const DataLayout &DL, Type *Ty, StringRef ValueKind,
-                     unsigned &Offset, msgpack::ArrayDocNode Args,
-                     MaybeAlign PointeeAlign = None, StringRef Name = "",
-                     StringRef TypeName = "", StringRef BaseTypeName = "",
-                     StringRef AccQual = "", StringRef TypeQual = "");
+  void emitKernelArg(const DataLayout &DL, Type *Ty, Align Alignment,
+                     StringRef ValueKind, unsigned &Offset,
+                     msgpack::ArrayDocNode Args, MaybeAlign PointeeAlign = None,
+                     StringRef Name = "", StringRef TypeName = "",
+                     StringRef BaseTypeName = "", StringRef AccQual = "",
+                     StringRef TypeQual = "");
 
   void emitHiddenKernelArgs(const Function &Func, unsigned &Offset,
                             msgpack::ArrayDocNode Args);
@@ -156,10 +157,11 @@ class MetadataStreamerV2 final : public MetadataStreamer {
 
   void emitKernelArg(const Argument &Arg);
 
-  void emitKernelArg(const DataLayout &DL, Type *Ty, ValueKind ValueKind,
-                     MaybeAlign PointeeAlign = None, StringRef Name = "",
-                     StringRef TypeName = "", StringRef BaseTypeName = "",
-                     StringRef AccQual = "", StringRef TypeQual = "");
+  void emitKernelArg(const DataLayout &DL, Type *Ty, Align Alignment,
+                     ValueKind ValueKind, MaybeAlign PointeeAlign = None,
+                     StringRef Name = "", StringRef TypeName = "",
+                     StringRef BaseTypeName = "", StringRef AccQual = "",
+                     StringRef TypeQual = "");
 
   void emitHiddenKernelArgs(const Function &Func);
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 940ec6f31c698..f7cdc7461eacd 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -1017,10 +1017,14 @@ void AMDGPUTargetLowering::analyzeFormalArgumentsCompute(
   unsigned InIndex = 0;
 
   for (const Argument &Arg : Fn.args()) {
+    const bool IsByRef = Arg.hasByRefAttr();
     Type *BaseArgTy = Arg.getType();
-    Align Alignment = DL.getABITypeAlign(BaseArgTy);
-    MaxAlign = std::max(Alignment, MaxAlign);
-    unsigned AllocSize = DL.getTypeAllocSize(BaseArgTy);
+    Type *MemArgTy = IsByRef ? Arg.getParamByRefType() : BaseArgTy;
+    MaybeAlign Alignment = IsByRef ? Arg.getParamAlign() : None;
+    if (!Alignment)
+      Alignment = DL.getABITypeAlign(MemArgTy);
+    MaxAlign = max(Alignment, MaxAlign);
+    uint64_t AllocSize = DL.getTypeAllocSize(MemArgTy);
 
     uint64_t ArgOffset = alignTo(ExplicitArgOffset, Alignment) + ExplicitOffset;
     ExplicitArgOffset = alignTo(ExplicitArgOffset, Alignment) + AllocSize;
@@ -1224,7 +1228,7 @@ SDValue AMDGPUTargetLowering::LowerOperation(SDValue Op,
   switch (Op.getOpcode()) {
   default:
     Op->print(errs(), &DAG);
-    llvm_unreachable("Custom lowering code for this"
+    llvm_unreachable("Custom lowering code for this "
                      "instruction is not implemented yet!");
     break;
   case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG);
@@ -2698,14 +2702,12 @@ SDValue AMDGPUTargetLowering::LowerFP_TO_SINT(SDValue Op,
   // TODO: Factor out code common with LowerFP_TO_UINT.
 
   EVT SrcVT = Src.getValueType();
-  if (Subtarget->has16BitInsts() && SrcVT == MVT::f16) {
+  if (SrcVT == MVT::f16 ||
+      (SrcVT == MVT::f32 && Src.getOpcode() == ISD::FP16_TO_FP)) {
     SDLoc DL(Op);
 
-    SDValue FPExtend = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src);
-    SDValue FpToInt32 =
-        DAG.getNode(Op.getOpcode(), DL, MVT::i64, FPExtend);
-
-    return FpToInt32;
+    SDValue FpToInt32 = DAG.getNode(Op.getOpcode(), DL, MVT::i32, Src);
+    return DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, FpToInt32);
   }
 
   if (Op.getValueType() == MVT::i64 && Src.getValueType() == MVT::f64)
@@ -2721,14 +2723,12 @@ SDValue AMDGPUTargetLowering::LowerFP_TO_UINT(SDValue Op,
   // TODO: Factor out code common with LowerFP_TO_SINT.
 
   EVT SrcVT = Src.getValueType();
-  if (Subtarget->has16BitInsts() && SrcVT == MVT::f16) {
+  if (SrcVT == MVT::f16 ||
+      (SrcVT == MVT::f32 && Src.getOpcode() == ISD::FP16_TO_FP)) {
     SDLoc DL(Op);
 
-    SDValue FPExtend = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src);
-    SDValue FpToInt32 =
-        DAG.getNode(Op.getOpcode(), DL, MVT::i64, FPExtend);
-
-    return FpToInt32;
+    SDValue FpToUInt32 = DAG.getNode(Op.getOpcode(), DL, MVT::i32, Src);
+    return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, FpToUInt32);
   }
 
   if (Op.getValueType() == MVT::i64 && Src.getValueType() == MVT::f64)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
new file mode 100644
index 0000000000000..58f251fec5e98
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -0,0 +1,895 @@
+//===- AMDGPInstCombineIntrinsic.cpp - AMDGPU specific InstCombine pass ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// \file
+// This file implements a TargetTransformInfo analysis pass specific to the
+// AMDGPU target machine. It uses the target's detailed information to provide
+// more precise answers to certain TTI queries, while letting the target
+// independent and default TTI implementations handle the rest.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUTargetTransformInfo.h"
+#include "llvm/Transforms/InstCombine/InstCombiner.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "AMDGPUtti"
+
+namespace {
+
+struct AMDGPUImageDMaskIntrinsic {
+  unsigned Intr;
+};
+
+#define GET_AMDGPUImageDMaskIntrinsicTable_IMPL
+#include "InstCombineTables.inc"
+
+} // end anonymous namespace
+
+// Constant fold llvm.amdgcn.fmed3 intrinsics for standard inputs.
+//
+// A single NaN input is folded to minnum, so we rely on that folding for
+// handling NaNs.
+static APFloat fmed3AMDGCN(const APFloat &Src0, const APFloat &Src1,
+                           const APFloat &Src2) {
+  APFloat Max3 = maxnum(maxnum(Src0, Src1), Src2);
+
+  APFloat::cmpResult Cmp0 = Max3.compare(Src0);
+  assert(Cmp0 != APFloat::cmpUnordered && "nans handled separately");
+  if (Cmp0 == APFloat::cmpEqual)
+    return maxnum(Src1, Src2);
+
+  APFloat::cmpResult Cmp1 = Max3.compare(Src1);
+  assert(Cmp1 != APFloat::cmpUnordered && "nans handled separately");
+  if (Cmp1 == APFloat::cmpEqual)
+    return maxnum(Src0, Src2);
+
+  return maxnum(Src0, Src1);
+}
+
+Optional<Instruction *>
+GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
+  Intrinsic::ID IID = II.getIntrinsicID();
+  switch (IID) {
+  default:
+    break;
+  case Intrinsic::amdgcn_rcp: {
+    Value *Src = II.getArgOperand(0);
+
+    // TODO: Move to ConstantFolding/InstSimplify?
+    if (isa<UndefValue>(Src)) {
+      Type *Ty = II.getType();
+      auto *QNaN = ConstantFP::get(Ty, APFloat::getQNaN(Ty->getFltSemantics()));
+      return IC.replaceInstUsesWith(II, QNaN);
+    }
+
+    if (II.isStrictFP())
+      break;
+
+    if (const ConstantFP *C = dyn_cast<ConstantFP>(Src)) {
+      const APFloat &ArgVal = C->getValueAPF();
+      APFloat Val(ArgVal.getSemantics(), 1);
+      Val.divide(ArgVal, APFloat::rmNearestTiesToEven);
+
+      // This is more precise than the instruction may give.
+      //
+      // TODO: The instruction always flushes denormal results (except for f16),
+      // should this also?
+      return IC.replaceInstUsesWith(II, ConstantFP::get(II.getContext(), Val));
+    }
+
+    break;
+  }
+  case Intrinsic::amdgcn_rsq: {
+    Value *Src = II.getArgOperand(0);
+
+    // TODO: Move to ConstantFolding/InstSimplify?
+    if (isa<UndefValue>(Src)) {
+      Type *Ty = II.getType();
+      auto *QNaN = ConstantFP::get(Ty, APFloat::getQNaN(Ty->getFltSemantics()));
+      return IC.replaceInstUsesWith(II, QNaN);
+    }
+
+    break;
+  }
+  case Intrinsic::amdgcn_frexp_mant:
+  case Intrinsic::amdgcn_frexp_exp: {
+    Value *Src = II.getArgOperand(0);
+    if (const ConstantFP *C = dyn_cast<ConstantFP>(Src)) {
+      int Exp;
+      APFloat Significand =
+          frexp(C->getValueAPF(), Exp, APFloat::rmNearestTiesToEven);
+
+      if (IID == Intrinsic::amdgcn_frexp_mant) {
+        return IC.replaceInstUsesWith(
+            II, ConstantFP::get(II.getContext(), Significand));
+      }
+
+      // Match instruction special case behavior.
+      if (Exp == APFloat::IEK_NaN || Exp == APFloat::IEK_Inf)
+        Exp = 0;
+
+      return IC.replaceInstUsesWith(II, ConstantInt::get(II.getType(), Exp));
+    }
+
+    if (isa<UndefValue>(Src)) {
+      return IC.replaceInstUsesWith(II, UndefValue::get(II.getType()));
+    }
+
+    break;
+  }
+  case Intrinsic::amdgcn_class: {
+    enum {
+      S_NAN = 1 << 0,       // Signaling NaN
+      Q_NAN = 1 << 1,       // Quiet NaN
+      N_INFINITY = 1 << 2,  // Negative infinity
+      N_NORMAL = 1 << 3,    // Negative normal
+      N_SUBNORMAL = 1 << 4, // Negative subnormal
+      N_ZERO = 1 << 5,      // Negative zero
+      P_ZERO = 1 << 6,      // Positive zero
+      P_SUBNORMAL = 1 << 7, // Positive subnormal
+      P_NORMAL = 1 << 8,    // Positive normal
+      P_INFINITY = 1 << 9   // Positive infinity
+    };
+
+    const uint32_t FullMask = S_NAN | Q_NAN | N_INFINITY | N_NORMAL |
+                              N_SUBNORMAL | N_ZERO | P_ZERO | P_SUBNORMAL |
+                              P_NORMAL | P_INFINITY;
+
+    Value *Src0 = II.getArgOperand(0);
+    Value *Src1 = II.getArgOperand(1);
+    const ConstantInt *CMask = dyn_cast<ConstantInt>(Src1);
+    if (!CMask) {
+      if (isa<UndefValue>(Src0)) {
+        return IC.replaceInstUsesWith(II, UndefValue::get(II.getType()));
+      }
+
+      if (isa<UndefValue>(Src1)) {
+        return IC.replaceInstUsesWith(II,
+                                      ConstantInt::get(II.getType(), false));
+      }
+      break;
+    }
+
+    uint32_t Mask = CMask->getZExtValue();
+
+    // If all tests are made, it doesn't matter what the value is.
+    if ((Mask & FullMask) == FullMask) {
+      return IC.replaceInstUsesWith(II, ConstantInt::get(II.getType(), true));
+    }
+
+    if ((Mask & FullMask) == 0) {
+      return IC.replaceInstUsesWith(II, ConstantInt::get(II.getType(), false));
+    }
+
+    if (Mask == (S_NAN | Q_NAN)) {
+      // Equivalent of isnan. Replace with standard fcmp.
+      Value *FCmp = IC.Builder.CreateFCmpUNO(Src0, Src0);
+      FCmp->takeName(&II);
+      return IC.replaceInstUsesWith(II, FCmp);
+    }
+
+    if (Mask == (N_ZERO | P_ZERO)) {
+      // Equivalent of == 0.
+      Value *FCmp =
+          IC.Builder.CreateFCmpOEQ(Src0, ConstantFP::get(Src0->getType(), 0.0));
+
+      FCmp->takeName(&II);
+      return IC.replaceInstUsesWith(II, FCmp);
+    }
+
+    // fp_class (nnan x), qnan|snan|other -> fp_class (nnan x), other
+    if (((Mask & S_NAN) || (Mask & Q_NAN)) &&
+        isKnownNeverNaN(Src0, &IC.getTargetLibraryInfo())) {
+      return IC.replaceOperand(
+          II, 1, ConstantInt::get(Src1->getType(), Mask & ~(S_NAN | Q_NAN)));
+    }
+
+    const ConstantFP *CVal = dyn_cast<ConstantFP>(Src0);
+    if (!CVal) {
+      if (isa<UndefValue>(Src0)) {
+        return IC.replaceInstUsesWith(II, UndefValue::get(II.getType()));
+      }
+
+      // Clamp mask to used bits
+      if ((Mask & FullMask) != Mask) {
+        CallInst *NewCall = IC.Builder.CreateCall(
+            II.getCalledFunction(),
+            {Src0, ConstantInt::get(Src1->getType(), Mask & FullMask)});
+
+        NewCall->takeName(&II);
+        return IC.replaceInstUsesWith(II, NewCall);
+      }
+
+      break;
+    }
+
+    const APFloat &Val = CVal->getValueAPF();
+
+    bool Result =
+        ((Mask & S_NAN) && Val.isNaN() && Val.isSignaling()) ||
+        ((Mask & Q_NAN) && Val.isNaN() && !Val.isSignaling()) ||
+        ((Mask & N_INFINITY) && Val.isInfinity() && Val.isNegative()) ||
+        ((Mask & N_NORMAL) && Val.isNormal() && Val.isNegative()) ||
+        ((Mask & N_SUBNORMAL) && Val.isDenormal() && Val.isNegative()) ||
+        ((Mask & N_ZERO) && Val.isZero() && Val.isNegative()) ||
+        ((Mask & P_ZERO) && Val.isZero() && !Val.isNegative()) ||
+        ((Mask & P_SUBNORMAL) && Val.isDenormal() && !Val.isNegative()) ||
+        ((Mask & P_NORMAL) && Val.isNormal() && !Val.isNegative()) ||
+        ((Mask & P_INFINITY) && Val.isInfinity() && !Val.isNegative());
+
+    return IC.replaceInstUsesWith(II, ConstantInt::get(II.getType(), Result));
+  }
+  case Intrinsic::amdgcn_cvt_pkrtz: {
+    Value *Src0 = II.getArgOperand(0);
+    Value *Src1 = II.getArgOperand(1);
+    if (const ConstantFP *C0 = dyn_cast<ConstantFP>(Src0)) {
+      if (const ConstantFP *C1 = dyn_cast<ConstantFP>(Src1)) {
+        const fltSemantics &HalfSem =
+            II.getType()->getScalarType()->getFltSemantics();
+        bool LosesInfo;
+        APFloat Val0 = C0->getValueAPF();
+        APFloat Val1 = C1->getValueAPF();
+        Val0.convert(HalfSem, APFloat::rmTowardZero, &LosesInfo);
+        Val1.convert(HalfSem, APFloat::rmTowardZero, &LosesInfo);
+
+        Constant *Folded =
+            ConstantVector::get({ConstantFP::get(II.getContext(), Val0),
+                                 ConstantFP::get(II.getContext(), Val1)});
+        return IC.replaceInstUsesWith(II, Folded);
+      }
+    }
+
+    if (isa<UndefValue>(Src0) && isa<UndefValue>(Src1)) {
+      return IC.replaceInstUsesWith(II, UndefValue::get(II.getType()));
+    }
+
+    break;
+  }
+  case Intrinsic::amdgcn_cvt_pknorm_i16:
+  case Intrinsic::amdgcn_cvt_pknorm_u16:
+  case Intrinsic::amdgcn_cvt_pk_i16:
+  case Intrinsic::amdgcn_cvt_pk_u16: {
+    Value *Src0 = II.getArgOperand(0);
+    Value *Src1 = II.getArgOperand(1);
+
+    if (isa<UndefValue>(Src0) && isa<UndefValue>(Src1)) {
+      return IC.replaceInstUsesWith(II, UndefValue::get(II.getType()));
+    }
+
+    break;
+  }
+  case Intrinsic::amdgcn_ubfe:
+  case Intrinsic::amdgcn_sbfe: {
+    // Decompose simple cases into standard shifts.
+    Value *Src = II.getArgOperand(0);
+    if (isa<UndefValue>(Src)) {
+      return IC.replaceInstUsesWith(II, Src);
+    }
+
+    unsigned Width;
+    Type *Ty = II.getType();
+    unsigned IntSize = Ty->getIntegerBitWidth();
+
+    ConstantInt *CWidth = dyn_cast<ConstantInt>(II.getArgOperand(2));
+    if (CWidth) {
+      Width = CWidth->getZExtValue();
+      if ((Width & (IntSize - 1)) == 0) {
+        return IC.replaceInstUsesWith(II, ConstantInt::getNullValue(Ty));
+      }
+
+      // Hardware ignores high bits, so remove those.
+      if (Width >= IntSize) {
+        return IC.replaceOperand(
+            II, 2, ConstantInt::get(CWidth->getType(), Width & (IntSize - 1)));
+      }
+    }
+
+    unsigned Offset;
+    ConstantInt *COffset = dyn_cast<ConstantInt>(II.getArgOperand(1));
+    if (COffset) {
+      Offset = COffset->getZExtValue();
+      if (Offset >= IntSize) {
+        return IC.replaceOperand(
+            II, 1,
+            ConstantInt::get(COffset->getType(), Offset & (IntSize - 1)));
+      }
+    }
+
+    bool Signed = IID == Intrinsic::amdgcn_sbfe;
+
+    if (!CWidth || !COffset)
+      break;
+
+    // The case of Width == 0 is handled above, which makes this tranformation
+    // safe.  If Width == 0, then the ashr and lshr instructions become poison
+    // value since the shift amount would be equal to the bit size.
+    assert(Width != 0);
+
+    // TODO: This allows folding to undef when the hardware has specific
+    // behavior?
+    if (Offset + Width < IntSize) {
+      Value *Shl = IC.Builder.CreateShl(Src, IntSize - Offset - Width);
+      Value *RightShift = Signed ? IC.Builder.CreateAShr(Shl, IntSize - Width)
+                                 : IC.Builder.CreateLShr(Shl, IntSize - Width);
+      RightShift->takeName(&II);
+      return IC.replaceInstUsesWith(II, RightShift);
+    }
+
+    Value *RightShift = Signed ? IC.Builder.CreateAShr(Src, Offset)
+                               : IC.Builder.CreateLShr(Src, Offset);
+
+    RightShift->takeName(&II);
+    return IC.replaceInstUsesWith(II, RightShift);
+  }
+  case Intrinsic::amdgcn_exp:
+  case Intrinsic::amdgcn_exp_compr: {
+    ConstantInt *En = cast<ConstantInt>(II.getArgOperand(1));
+    unsigned EnBits = En->getZExtValue();
+    if (EnBits == 0xf)
+      break; // All inputs enabled.
+
+    bool IsCompr = IID == Intrinsic::amdgcn_exp_compr;
+    bool Changed = false;
+    for (int I = 0; I < (IsCompr ? 2 : 4); ++I) {
+      if ((!IsCompr && (EnBits & (1 << I)) == 0) ||
+          (IsCompr && ((EnBits & (0x3 << (2 * I))) == 0))) {
+        Value *Src = II.getArgOperand(I + 2);
+        if (!isa<UndefValue>(Src)) {
+          IC.replaceOperand(II, I + 2, UndefValue::get(Src->getType()));
+          Changed = true;
+        }
+      }
+    }
+
+    if (Changed) {
+      return &II;
+    }
+
+    break;
+  }
+  case Intrinsic::amdgcn_fmed3: {
+    // Note this does not preserve proper sNaN behavior if IEEE-mode is enabled
+    // for the shader.
+
+    Value *Src0 = II.getArgOperand(0);
+    Value *Src1 = II.getArgOperand(1);
+    Value *Src2 = II.getArgOperand(2);
+
+    // Checking for NaN before canonicalization provides better fidelity when
+    // mapping other operations onto fmed3 since the order of operands is
+    // unchanged.
+    CallInst *NewCall = nullptr;
+    if (match(Src0, PatternMatch::m_NaN()) || isa<UndefValue>(Src0)) {
+      NewCall = IC.Builder.CreateMinNum(Src1, Src2);
+    } else if (match(Src1, PatternMatch::m_NaN()) || isa<UndefValue>(Src1)) {
+      NewCall = IC.Builder.CreateMinNum(Src0, Src2);
+    } else if (match(Src2, PatternMatch::m_NaN()) || isa<UndefValue>(Src2)) {
+      NewCall = IC.Builder.CreateMaxNum(Src0, Src1);
+    }
+
+    if (NewCall) {
+      NewCall->copyFastMathFlags(&II);
+      NewCall->takeName(&II);
+      return IC.replaceInstUsesWith(II, NewCall);
+    }
+
+    bool Swap = false;
+    // Canonicalize constants to RHS operands.
+    //
+    // fmed3(c0, x, c1) -> fmed3(x, c0, c1)
+    if (isa<Constant>(Src0) && !isa<Constant>(Src1)) {
+      std::swap(Src0, Src1);
+      Swap = true;
+    }
+
+    if (isa<Constant>(Src1) && !isa<Constant>(Src2)) {
+      std::swap(Src1, Src2);
+      Swap = true;
+    }
+
+    if (isa<Constant>(Src0) && !isa<Constant>(Src1)) {
+      std::swap(Src0, Src1);
+      Swap = true;
+    }
+
+    if (Swap) {
+      II.setArgOperand(0, Src0);
+      II.setArgOperand(1, Src1);
+      II.setArgOperand(2, Src2);
+      return &II;
+    }
+
+    if (const ConstantFP *C0 = dyn_cast<ConstantFP>(Src0)) {
+      if (const ConstantFP *C1 = dyn_cast<ConstantFP>(Src1)) {
+        if (const ConstantFP *C2 = dyn_cast<ConstantFP>(Src2)) {
+          APFloat Result = fmed3AMDGCN(C0->getValueAPF(), C1->getValueAPF(),
+                                       C2->getValueAPF());
+          return IC.replaceInstUsesWith(
+              II, ConstantFP::get(IC.Builder.getContext(), Result));
+        }
+      }
+    }
+
+    break;
+  }
+  case Intrinsic::amdgcn_icmp:
+  case Intrinsic::amdgcn_fcmp: {
+    const ConstantInt *CC = cast<ConstantInt>(II.getArgOperand(2));
+    // Guard against invalid arguments.
+    int64_t CCVal = CC->getZExtValue();
+    bool IsInteger = IID == Intrinsic::amdgcn_icmp;
+    if ((IsInteger && (CCVal < CmpInst::FIRST_ICMP_PREDICATE ||
+                       CCVal > CmpInst::LAST_ICMP_PREDICATE)) ||
+        (!IsInteger && (CCVal < CmpInst::FIRST_FCMP_PREDICATE ||
+                        CCVal > CmpInst::LAST_FCMP_PREDICATE)))
+      break;
+
+    Value *Src0 = II.getArgOperand(0);
+    Value *Src1 = II.getArgOperand(1);
+
+    if (auto *CSrc0 = dyn_cast<Constant>(Src0)) {
+      if (auto *CSrc1 = dyn_cast<Constant>(Src1)) {
+        Constant *CCmp = ConstantExpr::getCompare(CCVal, CSrc0, CSrc1);
+        if (CCmp->isNullValue()) {
+          return IC.replaceInstUsesWith(
+              II, ConstantExpr::getSExt(CCmp, II.getType()));
+        }
+
+        // The result of V_ICMP/V_FCMP assembly instructions (which this
+        // intrinsic exposes) is one bit per thread, masked with the EXEC
+        // register (which contains the bitmask of live threads). So a
+        // comparison that always returns true is the same as a read of the
+        // EXEC register.
+        Function *NewF = Intrinsic::getDeclaration(
+            II.getModule(), Intrinsic::read_register, II.getType());
+        Metadata *MDArgs[] = {MDString::get(II.getContext(), "exec")};
+        MDNode *MD = MDNode::get(II.getContext(), MDArgs);
+        Value *Args[] = {MetadataAsValue::get(II.getContext(), MD)};
+        CallInst *NewCall = IC.Builder.CreateCall(NewF, Args);
+        NewCall->addAttribute(AttributeList::FunctionIndex,
+                              Attribute::Convergent);
+        NewCall->takeName(&II);
+        return IC.replaceInstUsesWith(II, NewCall);
+      }
+
+      // Canonicalize constants to RHS.
+      CmpInst::Predicate SwapPred =
+          CmpInst::getSwappedPredicate(static_cast<CmpInst::Predicate>(CCVal));
+      II.setArgOperand(0, Src1);
+      II.setArgOperand(1, Src0);
+      II.setArgOperand(
+          2, ConstantInt::get(CC->getType(), static_cast<int>(SwapPred)));
+      return &II;
+    }
+
+    if (CCVal != CmpInst::ICMP_EQ && CCVal != CmpInst::ICMP_NE)
+      break;
+
+    // Canonicalize compare eq with true value to compare != 0
+    // llvm.amdgcn.icmp(zext (i1 x), 1, eq)
+    //   -> llvm.amdgcn.icmp(zext (i1 x), 0, ne)
+    // llvm.amdgcn.icmp(sext (i1 x), -1, eq)
+    //   -> llvm.amdgcn.icmp(sext (i1 x), 0, ne)
+    Value *ExtSrc;
+    if (CCVal == CmpInst::ICMP_EQ &&
+        ((match(Src1, PatternMatch::m_One()) &&
+          match(Src0, m_ZExt(PatternMatch::m_Value(ExtSrc)))) ||
+         (match(Src1, PatternMatch::m_AllOnes()) &&
+          match(Src0, m_SExt(PatternMatch::m_Value(ExtSrc))))) &&
+        ExtSrc->getType()->isIntegerTy(1)) {
+      IC.replaceOperand(II, 1, ConstantInt::getNullValue(Src1->getType()));
+      IC.replaceOperand(II, 2,
+                        ConstantInt::get(CC->getType(), CmpInst::ICMP_NE));
+      return &II;
+    }
+
+    CmpInst::Predicate SrcPred;
+    Value *SrcLHS;
+    Value *SrcRHS;
+
+    // Fold compare eq/ne with 0 from a compare result as the predicate to the
+    // intrinsic. The typical use is a wave vote function in the library, which
+    // will be fed from a user code condition compared with 0. Fold in the
+    // redundant compare.
+
+    // llvm.amdgcn.icmp([sz]ext ([if]cmp pred a, b), 0, ne)
+    //   -> llvm.amdgcn.[if]cmp(a, b, pred)
+    //
+    // llvm.amdgcn.icmp([sz]ext ([if]cmp pred a, b), 0, eq)
+    //   -> llvm.amdgcn.[if]cmp(a, b, inv pred)
+    if (match(Src1, PatternMatch::m_Zero()) &&
+        match(Src0, PatternMatch::m_ZExtOrSExt(
+                        m_Cmp(SrcPred, PatternMatch::m_Value(SrcLHS),
+                              PatternMatch::m_Value(SrcRHS))))) {
+      if (CCVal == CmpInst::ICMP_EQ)
+        SrcPred = CmpInst::getInversePredicate(SrcPred);
+
+      Intrinsic::ID NewIID = CmpInst::isFPPredicate(SrcPred)
+                                 ? Intrinsic::amdgcn_fcmp
+                                 : Intrinsic::amdgcn_icmp;
+
+      Type *Ty = SrcLHS->getType();
+      if (auto *CmpType = dyn_cast<IntegerType>(Ty)) {
+        // Promote to next legal integer type.
+        unsigned Width = CmpType->getBitWidth();
+        unsigned NewWidth = Width;
+
+        // Don't do anything for i1 comparisons.
+        if (Width == 1)
+          break;
+
+        if (Width <= 16)
+          NewWidth = 16;
+        else if (Width <= 32)
+          NewWidth = 32;
+        else if (Width <= 64)
+          NewWidth = 64;
+        else if (Width > 64)
+          break; // Can't handle this.
+
+        if (Width != NewWidth) {
+          IntegerType *CmpTy = IC.Builder.getIntNTy(NewWidth);
+          if (CmpInst::isSigned(SrcPred)) {
+            SrcLHS = IC.Builder.CreateSExt(SrcLHS, CmpTy);
+            SrcRHS = IC.Builder.CreateSExt(SrcRHS, CmpTy);
+          } else {
+            SrcLHS = IC.Builder.CreateZExt(SrcLHS, CmpTy);
+            SrcRHS = IC.Builder.CreateZExt(SrcRHS, CmpTy);
+          }
+        }
+      } else if (!Ty->isFloatTy() && !Ty->isDoubleTy() && !Ty->isHalfTy())
+        break;
+
+      Function *NewF = Intrinsic::getDeclaration(
+          II.getModule(), NewIID, {II.getType(), SrcLHS->getType()});
+      Value *Args[] = {SrcLHS, SrcRHS,
+                       ConstantInt::get(CC->getType(), SrcPred)};
+      CallInst *NewCall = IC.Builder.CreateCall(NewF, Args);
+      NewCall->takeName(&II);
+      return IC.replaceInstUsesWith(II, NewCall);
+    }
+
+    break;
+  }
+  case Intrinsic::amdgcn_ballot: {
+    if (auto *Src = dyn_cast<ConstantInt>(II.getArgOperand(0))) {
+      if (Src->isZero()) {
+        // amdgcn.ballot(i1 0) is zero.
+        return IC.replaceInstUsesWith(II, Constant::getNullValue(II.getType()));
+      }
+
+      if (Src->isOne()) {
+        // amdgcn.ballot(i1 1) is exec.
+        const char *RegName = "exec";
+        if (II.getType()->isIntegerTy(32))
+          RegName = "exec_lo";
+        else if (!II.getType()->isIntegerTy(64))
+          break;
+
+        Function *NewF = Intrinsic::getDeclaration(
+            II.getModule(), Intrinsic::read_register, II.getType());
+        Metadata *MDArgs[] = {MDString::get(II.getContext(), RegName)};
+        MDNode *MD = MDNode::get(II.getContext(), MDArgs);
+        Value *Args[] = {MetadataAsValue::get(II.getContext(), MD)};
+        CallInst *NewCall = IC.Builder.CreateCall(NewF, Args);
+        NewCall->addAttribute(AttributeList::FunctionIndex,
+                              Attribute::Convergent);
+        NewCall->takeName(&II);
+        return IC.replaceInstUsesWith(II, NewCall);
+      }
+    }
+    break;
+  }
+  case Intrinsic::amdgcn_wqm_vote: {
+    // wqm_vote is identity when the argument is constant.
+    if (!isa<Constant>(II.getArgOperand(0)))
+      break;
+
+    return IC.replaceInstUsesWith(II, II.getArgOperand(0));
+  }
+  case Intrinsic::amdgcn_kill: {
+    const ConstantInt *C = dyn_cast<ConstantInt>(II.getArgOperand(0));
+    if (!C || !C->getZExtValue())
+      break;
+
+    // amdgcn.kill(i1 1) is a no-op
+    return IC.eraseInstFromFunction(II);
+  }
+  case Intrinsic::amdgcn_update_dpp: {
+    Value *Old = II.getArgOperand(0);
+
+    auto *BC = cast<ConstantInt>(II.getArgOperand(5));
+    auto *RM = cast<ConstantInt>(II.getArgOperand(3));
+    auto *BM = cast<ConstantInt>(II.getArgOperand(4));
+    if (BC->isZeroValue() || RM->getZExtValue() != 0xF ||
+        BM->getZExtValue() != 0xF || isa<UndefValue>(Old))
+      break;
+
+    // If bound_ctrl = 1, row mask = bank mask = 0xf we can omit old value.
+    return IC.replaceOperand(II, 0, UndefValue::get(Old->getType()));
+  }
+  case Intrinsic::amdgcn_permlane16:
+  case Intrinsic::amdgcn_permlanex16: {
+    // Discard vdst_in if it's not going to be read.
+    Value *VDstIn = II.getArgOperand(0);
+    if (isa<UndefValue>(VDstIn))
+      break;
+
+    ConstantInt *FetchInvalid = cast<ConstantInt>(II.getArgOperand(4));
+    ConstantInt *BoundCtrl = cast<ConstantInt>(II.getArgOperand(5));
+    if (!FetchInvalid->getZExtValue() && !BoundCtrl->getZExtValue())
+      break;
+
+    return IC.replaceOperand(II, 0, UndefValue::get(VDstIn->getType()));
+  }
+  case Intrinsic::amdgcn_readfirstlane:
+  case Intrinsic::amdgcn_readlane: {
+    // A constant value is trivially uniform.
+    if (Constant *C = dyn_cast<Constant>(II.getArgOperand(0))) {
+      return IC.replaceInstUsesWith(II, C);
+    }
+
+    // The rest of these may not be safe if the exec may not be the same between
+    // the def and use.
+    Value *Src = II.getArgOperand(0);
+    Instruction *SrcInst = dyn_cast<Instruction>(Src);
+    if (SrcInst && SrcInst->getParent() != II.getParent())
+      break;
+
+    // readfirstlane (readfirstlane x) -> readfirstlane x
+    // readlane (readfirstlane x), y -> readfirstlane x
+    if (match(Src,
+              PatternMatch::m_Intrinsic<Intrinsic::amdgcn_readfirstlane>())) {
+      return IC.replaceInstUsesWith(II, Src);
+    }
+
+    if (IID == Intrinsic::amdgcn_readfirstlane) {
+      // readfirstlane (readlane x, y) -> readlane x, y
+      if (match(Src, PatternMatch::m_Intrinsic<Intrinsic::amdgcn_readlane>())) {
+        return IC.replaceInstUsesWith(II, Src);
+      }
+    } else {
+      // readlane (readlane x, y), y -> readlane x, y
+      if (match(Src, PatternMatch::m_Intrinsic<Intrinsic::amdgcn_readlane>(
+                         PatternMatch::m_Value(),
+                         PatternMatch::m_Specific(II.getArgOperand(1))))) {
+        return IC.replaceInstUsesWith(II, Src);
+      }
+    }
+
+    break;
+  }
+  case Intrinsic::amdgcn_ldexp: {
+    // FIXME: This doesn't introduce new instructions and belongs in
+    // InstructionSimplify.
+    Type *Ty = II.getType();
+    Value *Op0 = II.getArgOperand(0);
+    Value *Op1 = II.getArgOperand(1);
+
+    // Folding undef to qnan is safe regardless of the FP mode.
+    if (isa<UndefValue>(Op0)) {
+      auto *QNaN = ConstantFP::get(Ty, APFloat::getQNaN(Ty->getFltSemantics()));
+      return IC.replaceInstUsesWith(II, QNaN);
+    }
+
+    const APFloat *C = nullptr;
+    match(Op0, PatternMatch::m_APFloat(C));
+
+    // FIXME: Should flush denorms depending on FP mode, but that's ignored
+    // everywhere else.
+    //
+    // These cases should be safe, even with strictfp.
+    // ldexp(0.0, x) -> 0.0
+    // ldexp(-0.0, x) -> -0.0
+    // ldexp(inf, x) -> inf
+    // ldexp(-inf, x) -> -inf
+    if (C && (C->isZero() || C->isInfinity())) {
+      return IC.replaceInstUsesWith(II, Op0);
+    }
+
+    // With strictfp, be more careful about possibly needing to flush denormals
+    // or not, and snan behavior depends on ieee_mode.
+    if (II.isStrictFP())
+      break;
+
+    if (C && C->isNaN()) {
+      // FIXME: We just need to make the nan quiet here, but that's unavailable
+      // on APFloat, only IEEEfloat
+      auto *Quieted =
+          ConstantFP::get(Ty, scalbn(*C, 0, APFloat::rmNearestTiesToEven));
+      return IC.replaceInstUsesWith(II, Quieted);
+    }
+
+    // ldexp(x, 0) -> x
+    // ldexp(x, undef) -> x
+    if (isa<UndefValue>(Op1) || match(Op1, PatternMatch::m_ZeroInt())) {
+      return IC.replaceInstUsesWith(II, Op0);
+    }
+
+    break;
+  }
+  }
+  return None;
+}
+
+/// Implement SimplifyDemandedVectorElts for amdgcn buffer and image intrinsics.
+///
+/// Note: This only supports non-TFE/LWE image intrinsic calls; those have
+///       struct returns.
+Value *simplifyAMDGCNMemoryIntrinsicDemanded(InstCombiner &IC,
+                                             IntrinsicInst &II,
+                                             APInt DemandedElts,
+                                             int DMaskIdx = -1) {
+
+  auto *IIVTy = cast<VectorType>(II.getType());
+  unsigned VWidth = IIVTy->getNumElements();
+  if (VWidth == 1)
+    return nullptr;
+
+  IRBuilderBase::InsertPointGuard Guard(IC.Builder);
+  IC.Builder.SetInsertPoint(&II);
+
+  // Assume the arguments are unchanged and later override them, if needed.
+  SmallVector<Value *, 16> Args(II.arg_begin(), II.arg_end());
+
+  if (DMaskIdx < 0) {
+    // Buffer case.
+
+    const unsigned ActiveBits = DemandedElts.getActiveBits();
+    const unsigned UnusedComponentsAtFront = DemandedElts.countTrailingZeros();
+
+    // Start assuming the prefix of elements is demanded, but possibly clear
+    // some other bits if there are trailing zeros (unused components at front)
+    // and update offset.
+    DemandedElts = (1 << ActiveBits) - 1;
+
+    if (UnusedComponentsAtFront > 0) {
+      static const unsigned InvalidOffsetIdx = 0xf;
+
+      unsigned OffsetIdx;
+      switch (II.getIntrinsicID()) {
+      case Intrinsic::amdgcn_raw_buffer_load:
+        OffsetIdx = 1;
+        break;
+      case Intrinsic::amdgcn_s_buffer_load:
+        // If resulting type is vec3, there is no point in trimming the
+        // load with updated offset, as the vec3 would most likely be widened to
+        // vec4 anyway during lowering.
+        if (ActiveBits == 4 && UnusedComponentsAtFront == 1)
+          OffsetIdx = InvalidOffsetIdx;
+        else
+          OffsetIdx = 1;
+        break;
+      case Intrinsic::amdgcn_struct_buffer_load:
+        OffsetIdx = 2;
+        break;
+      default:
+        // TODO: handle tbuffer* intrinsics.
+        OffsetIdx = InvalidOffsetIdx;
+        break;
+      }
+
+      if (OffsetIdx != InvalidOffsetIdx) {
+        // Clear demanded bits and update the offset.
+        DemandedElts &= ~((1 << UnusedComponentsAtFront) - 1);
+        auto *Offset = II.getArgOperand(OffsetIdx);
+        unsigned SingleComponentSizeInBits =
+            IC.getDataLayout().getTypeSizeInBits(II.getType()->getScalarType());
+        unsigned OffsetAdd =
+            UnusedComponentsAtFront * SingleComponentSizeInBits / 8;
+        auto *OffsetAddVal = ConstantInt::get(Offset->getType(), OffsetAdd);
+        Args[OffsetIdx] = IC.Builder.CreateAdd(Offset, OffsetAddVal);
+      }
+    }
+  } else {
+    // Image case.
+
+    ConstantInt *DMask = cast<ConstantInt>(II.getArgOperand(DMaskIdx));
+    unsigned DMaskVal = DMask->getZExtValue() & 0xf;
+
+    // Mask off values that are undefined because the dmask doesn't cover them
+    DemandedElts &= (1 << countPopulation(DMaskVal)) - 1;
+
+    unsigned NewDMaskVal = 0;
+    unsigned OrigLoadIdx = 0;
+    for (unsigned SrcIdx = 0; SrcIdx < 4; ++SrcIdx) {
+      const unsigned Bit = 1 << SrcIdx;
+      if (!!(DMaskVal & Bit)) {
+        if (!!DemandedElts[OrigLoadIdx])
+          NewDMaskVal |= Bit;
+        OrigLoadIdx++;
+      }
+    }
+
+    if (DMaskVal != NewDMaskVal)
+      Args[DMaskIdx] = ConstantInt::get(DMask->getType(), NewDMaskVal);
+  }
+
+  unsigned NewNumElts = DemandedElts.countPopulation();
+  if (!NewNumElts)
+    return UndefValue::get(II.getType());
+
+  // FIXME: Allow v3i16/v3f16 in buffer and image intrinsics when the types are
+  // fully supported.
+  if (II.getType()->getScalarSizeInBits() == 16 && NewNumElts == 3)
+    return nullptr;
+
+  if (NewNumElts >= VWidth && DemandedElts.isMask()) {
+    if (DMaskIdx >= 0)
+      II.setArgOperand(DMaskIdx, Args[DMaskIdx]);
+    return nullptr;
+  }
+
+  // Validate function argument and return types, extracting overloaded types
+  // along the way.
+  SmallVector<Type *, 6> OverloadTys;
+  if (!Intrinsic::getIntrinsicSignature(II.getCalledFunction(), OverloadTys))
+    return nullptr;
+
+  Module *M = II.getParent()->getParent()->getParent();
+  Type *EltTy = IIVTy->getElementType();
+  Type *NewTy =
+      (NewNumElts == 1) ? EltTy : FixedVectorType::get(EltTy, NewNumElts);
+
+  OverloadTys[0] = NewTy;
+  Function *NewIntrin =
+      Intrinsic::getDeclaration(M, II.getIntrinsicID(), OverloadTys);
+
+  CallInst *NewCall = IC.Builder.CreateCall(NewIntrin, Args);
+  NewCall->takeName(&II);
+  NewCall->copyMetadata(II);
+
+  if (NewNumElts == 1) {
+    return IC.Builder.CreateInsertElement(UndefValue::get(II.getType()),
+                                          NewCall,
+                                          DemandedElts.countTrailingZeros());
+  }
+
+  SmallVector<int, 8> EltMask;
+  unsigned NewLoadIdx = 0;
+  for (unsigned OrigLoadIdx = 0; OrigLoadIdx < VWidth; ++OrigLoadIdx) {
+    if (!!DemandedElts[OrigLoadIdx])
+      EltMask.push_back(NewLoadIdx++);
+    else
+      EltMask.push_back(NewNumElts);
+  }
+
+  Value *Shuffle =
+      IC.Builder.CreateShuffleVector(NewCall, UndefValue::get(NewTy), EltMask);
+
+  return Shuffle;
+}
+
+Optional<Value *> GCNTTIImpl::simplifyDemandedVectorEltsIntrinsic(
+    InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
+    APInt &UndefElts2, APInt &UndefElts3,
+    std::function<void(Instruction *, unsigned, APInt, APInt &)>
+        SimplifyAndSetOp) const {
+  switch (II.getIntrinsicID()) {
+  case Intrinsic::amdgcn_buffer_load:
+  case Intrinsic::amdgcn_buffer_load_format:
+  case Intrinsic::amdgcn_raw_buffer_load:
+  case Intrinsic::amdgcn_raw_buffer_load_format:
+  case Intrinsic::amdgcn_raw_tbuffer_load:
+  case Intrinsic::amdgcn_s_buffer_load:
+  case Intrinsic::amdgcn_struct_buffer_load:
+  case Intrinsic::amdgcn_struct_buffer_load_format:
+  case Intrinsic::amdgcn_struct_tbuffer_load:
+  case Intrinsic::amdgcn_tbuffer_load:
+    return simplifyAMDGCNMemoryIntrinsicDemanded(IC, II, DemandedElts);
+  default: {
+    if (getAMDGPUImageDMaskIntrinsic(II.getIntrinsicID())) {
+      return simplifyAMDGCNMemoryIntrinsicDemanded(IC, II, DemandedElts, 0);
+    }
+    break;
+  }
+  }
+  return None;
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 74e6f0c438b2d..a637442002932 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -446,10 +446,8 @@ bool AMDGPUInstructionSelector::selectG_UADDO_USUBO_UADDE_USUBE(
                           I.getOpcode() == AMDGPU::G_USUBE;
 
   if (isVCC(Dst1Reg, *MRI)) {
-      // The name of the opcodes are misleading. v_add_i32/v_sub_i32 have unsigned
-      // carry out despite the _i32 name. These were renamed in VI to _U32.
-      // FIXME: We should probably rename the opcodes here.
-    unsigned NoCarryOpc = IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
+    unsigned NoCarryOpc =
+        IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
     unsigned CarryOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
     I.setDesc(TII.get(HasCarryIn ? CarryOpc : NoCarryOpc));
     I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true));
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 92ff345cd78c4..80633bbc6179d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -422,26 +422,71 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
     .legalIf(isPointer(0));
 
   if (ST.hasVOP3PInsts()) {
+    assert(ST.hasIntClamp() && "all targets with VOP3P should support clamp");
     getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL})
       .legalFor({S32, S16, V2S16})
       .clampScalar(0, S16, S32)
       .clampMaxNumElements(0, S16, 2)
       .scalarize(0)
       .widenScalarToNextPow2(0, 32);
+
+    getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT, G_SADDSAT, G_SSUBSAT})
+      .lowerFor({S32, S16, V2S16}) // FIXME: legal and merge with add/sub/mul
+      .minScalar(0, S16)
+      .clampMaxNumElements(0, S16, 2)
+      .scalarize(0)
+      .widenScalarToNextPow2(0, 32)
+      .lower();
   } else if (ST.has16BitInsts()) {
     getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL})
       .legalFor({S32, S16})
       .clampScalar(0, S16, S32)
       .scalarize(0)
-      .widenScalarToNextPow2(0, 32);
+      .widenScalarToNextPow2(0, 32); // FIXME: min should be 16
+
+    assert(ST.hasIntClamp() && "all targets with 16-bit should support clamp");
+
+    // Technically the saturating operations require clamp bit support, but this
+    // was introduced at the same time as 16-bit operations.
+    getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT})
+      .lowerFor({S32, S16}) // FIXME: legal with clamp modifier
+      .minScalar(0, S16)
+      .scalarize(0)
+      .widenScalarToNextPow2(0, 16)
+      .lower();
+
+    // We're just lowering this, but it helps get a better result to try to
+    // coerce to the desired type first.
+    getActionDefinitionsBuilder({G_SADDSAT, G_SSUBSAT})
+      .minScalar(0, S16)
+      .scalarize(0)
+      .lower();
   } else {
     getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL})
       .legalFor({S32})
       .clampScalar(0, S32, S32)
       .scalarize(0);
+
+    if (ST.hasIntClamp()) {
+      getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT})
+        .lowerFor({S32}) // FIXME: legal with clamp modifier.
+        .scalarize(0)
+        .minScalarOrElt(0, S32)
+        .lower();
+    } else {
+      // Clamp bit support was added in VI, along with 16-bit operations.
+      getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT})
+        .minScalar(0, S32)
+        .scalarize(0)
+        .lower();
+    }
+
+    getActionDefinitionsBuilder({G_SADDSAT, G_SSUBSAT})
+      .minScalar(0, S32)
+      .scalarize(0)
+      .lower();
   }
 
-  // FIXME: Not really legal. Placeholder for custom lowering.
   getActionDefinitionsBuilder({G_SDIV, G_UDIV, G_SREM, G_UREM})
     .customFor({S32, S64})
     .clampScalar(0, S32, S64)
@@ -599,7 +644,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
 
   getActionDefinitionsBuilder(G_FPEXT)
     .legalFor({{S64, S32}, {S32, S16}})
-    .lowerFor({{S64, S16}}) // FIXME: Implement
+    .narrowScalarFor({{S64, S16}}, changeTo(0, S32))
     .scalarize(0);
 
   getActionDefinitionsBuilder(G_FSUB)
@@ -654,7 +699,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
 
   auto &FPToI = getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI})
     .legalFor({{S32, S32}, {S32, S64}, {S32, S16}})
-    .customFor({{S64, S64}});
+    .customFor({{S64, S64}})
+    .narrowScalarFor({{S64, S16}}, changeTo(0, S32));
   if (ST.has16BitInsts())
     FPToI.legalFor({{S16, S16}});
   else
@@ -692,10 +738,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
     .scalarize(0);
 
   getActionDefinitionsBuilder(G_PTRMASK)
-    .legalIf(typeInSet(1, {S64, S32}))
-    .minScalar(1, S32)
-    .maxScalarIf(sizeIs(0, 32), 1, S32)
-    .maxScalarIf(sizeIs(0, 64), 1, S64)
+    .legalIf(all(sameSize(0, 1), typeInSet(1, {S64, S32})))
+    .scalarSameSizeAs(1, 0)
     .scalarize(0);
 
   auto &CmpBuilder =
@@ -747,6 +791,10 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
   ExpOps.clampScalar(0, MinScalarFPTy, S32)
         .scalarize(0);
 
+  getActionDefinitionsBuilder(G_FPOWI)
+    .clampScalar(0, MinScalarFPTy, S32)
+    .lower();
+
   // The 64-bit versions produce 32-bit results, but only on the SALU.
   getActionDefinitionsBuilder(G_CTPOP)
     .legalFor({{S32, S32}, {S32, S64}})
@@ -1428,12 +1476,6 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
     SextInReg.lowerFor({{S32}, {S64}});
   }
 
-  // FIXME: Placeholder rule. Really depends on whether the clamp modifier is
-  // available, and is selectively legal for s16, s32, v2s16.
-  getActionDefinitionsBuilder({G_SADDSAT, G_SSUBSAT, G_UADDSAT, G_USUBSAT})
-    .scalarize(0)
-    .clampScalar(0, S16, S32);
-
   SextInReg
     .scalarize(0)
     .clampScalar(0, S32, S64)
@@ -1722,6 +1764,7 @@ bool AMDGPULegalizerInfo::legalizeFrint(
 
   auto Cond = B.buildFCmp(CmpInst::FCMP_OGT, LLT::scalar(1), Fabs, C2);
   B.buildSelect(MI.getOperand(0).getReg(), Cond, Src, Tmp2);
+  MI.eraseFromParent();
   return true;
 }
 
@@ -1753,7 +1796,7 @@ bool AMDGPULegalizerInfo::legalizeFceil(
   return true;
 }
 
-static MachineInstrBuilder extractF64Exponent(unsigned Hi,
+static MachineInstrBuilder extractF64Exponent(Register Hi,
                                               MachineIRBuilder &B) {
   const unsigned FractBits = 52;
   const unsigned ExpBits = 11;
@@ -1763,6 +1806,7 @@ static MachineInstrBuilder extractF64Exponent(unsigned Hi,
   auto Const1 = B.buildConstant(S32, ExpBits);
 
   auto ExpPart = B.buildIntrinsic(Intrinsic::amdgcn_ubfe, {S32}, false)
+    .addUse(Hi)
     .addUse(Const0.getReg(0))
     .addUse(Const1.getReg(0));
 
@@ -1810,6 +1854,7 @@ bool AMDGPULegalizerInfo::legalizeIntrinsicTrunc(
 
   auto Tmp1 = B.buildSelect(S64, ExpLt0, SignBit64, Tmp0);
   B.buildSelect(MI.getOperand(0).getReg(), ExpGt51, Src, Tmp1);
+  MI.eraseFromParent();
   return true;
 }
 
@@ -3110,19 +3155,13 @@ bool AMDGPULegalizerInfo::legalizeFDIVFastIntrin(MachineInstr &MI,
   return true;
 }
 
-bool AMDGPULegalizerInfo::legalizeImplicitArgPtr(MachineInstr &MI,
-                                                 MachineRegisterInfo &MRI,
-                                                 MachineIRBuilder &B) const {
+bool AMDGPULegalizerInfo::getImplicitArgPtr(Register DstReg,
+                                            MachineRegisterInfo &MRI,
+                                            MachineIRBuilder &B) const {
   const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
-  if (!MFI->isEntryFunction()) {
-    return legalizePreloadedArgIntrin(MI, MRI, B,
-                                      AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR);
-  }
-
   uint64_t Offset =
     ST.getTargetLowering()->getImplicitParameterOffset(
       B.getMF(), AMDGPUTargetLowering::FIRST_IMPLICIT);
-  Register DstReg = MI.getOperand(0).getReg();
   LLT DstTy = MRI.getType(DstReg);
   LLT IdxTy = LLT::scalar(DstTy.getSizeInBits());
 
@@ -3138,7 +3177,24 @@ bool AMDGPULegalizerInfo::legalizeImplicitArgPtr(MachineInstr &MI,
   if (!loadInputValue(KernargPtrReg, B, Arg))
     return false;
 
+  // FIXME: This should be nuw
   B.buildPtrAdd(DstReg, KernargPtrReg, B.buildConstant(IdxTy, Offset).getReg(0));
+  return true;
+}
+
+bool AMDGPULegalizerInfo::legalizeImplicitArgPtr(MachineInstr &MI,
+                                                 MachineRegisterInfo &MRI,
+                                                 MachineIRBuilder &B) const {
+  const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
+  if (!MFI->isEntryFunction()) {
+    return legalizePreloadedArgIntrin(MI, MRI, B,
+                                      AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR);
+  }
+
+  Register DstReg = MI.getOperand(0).getReg();
+  if (!getImplicitArgPtr(DstReg, MRI, B))
+    return false;
+
   MI.eraseFromParent();
   return true;
 }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
index ce32bbf76b34f..d932cab4659ab 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
@@ -133,6 +133,9 @@ class AMDGPULegalizerInfo : public LegalizerInfo {
   bool legalizeFDIVFastIntrin(MachineInstr &MI, MachineRegisterInfo &MRI,
                               MachineIRBuilder &B) const;
 
+  bool getImplicitArgPtr(Register DstReg, MachineRegisterInfo &MRI,
+                         MachineIRBuilder &B) const;
+
   bool legalizeImplicitArgPtr(MachineInstr &MI, MachineRegisterInfo &MRI,
                               MachineIRBuilder &B) const;
   bool legalizeIsAddrSpace(MachineInstr &MI, MachineRegisterInfo &MRI,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
index 62ab5bb55a16a..650c872182eca 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
@@ -108,10 +108,14 @@ bool AMDGPULowerKernelArguments::runOnFunction(Function &F) {
   uint64_t ExplicitArgOffset = 0;
 
   for (Argument &Arg : F.args()) {
-    Type *ArgTy = Arg.getType();
-    Align ABITypeAlign = DL.getABITypeAlign(ArgTy);
-    unsigned Size = DL.getTypeSizeInBits(ArgTy);
-    unsigned AllocSize = DL.getTypeAllocSize(ArgTy);
+    const bool IsByRef = Arg.hasByRefAttr();
+    Type *ArgTy = IsByRef ? Arg.getParamByRefType() : Arg.getType();
+    MaybeAlign ABITypeAlign = IsByRef ? Arg.getParamAlign() : None;
+    if (!ABITypeAlign)
+      ABITypeAlign = DL.getABITypeAlign(ArgTy);
+
+    uint64_t Size = DL.getTypeSizeInBits(ArgTy);
+    uint64_t AllocSize = DL.getTypeAllocSize(ArgTy);
 
     uint64_t EltOffset = alignTo(ExplicitArgOffset, ABITypeAlign) + BaseOffset;
     ExplicitArgOffset = alignTo(ExplicitArgOffset, ABITypeAlign) + AllocSize;
@@ -119,6 +123,19 @@ bool AMDGPULowerKernelArguments::runOnFunction(Function &F) {
     if (Arg.use_empty())
       continue;
 
+    // If this is byval, the loads are already explicit in the function. We just
+    // need to rewrite the pointer values.
+    if (IsByRef) {
+      Value *ArgOffsetPtr = Builder.CreateConstInBoundsGEP1_64(
+          Builder.getInt8Ty(), KernArgSegment, EltOffset,
+          Arg.getName() + ".byval.kernarg.offset");
+
+      Value *CastOffsetPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
+          ArgOffsetPtr, Arg.getType());
+      Arg.replaceAllUsesWith(CastOffsetPtr);
+      continue;
+    }
+
     if (PointerType *PT = dyn_cast<PointerType>(ArgTy)) {
       // FIXME: Hack. We rely on AssertZext to be able to fold DS addressing
       // modes on SI to know the high bits are 0 so pointer adds don't wrap. We
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index 727f71b350490..de05c2119d985 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -759,8 +759,14 @@ bool AMDGPUPromoteAlloca::hasSufficientLocalMem(const Function &F) {
 
     for (const User *U : GV.users()) {
       const Instruction *Use = dyn_cast<Instruction>(U);
-      if (!Use)
-        continue;
+      if (!Use) {
+        // FIXME: This is probably a constant expression use. We should
+        // recursively search the users of it for the parent function instead of
+        // bailing.
+        LLVM_DEBUG(dbgs() << "Giving up on LDS size estimate "
+                             "due to constant expression\n");
+        return false;
+      }
 
       if (Use->getParent()->getParent() == &F) {
         Align Alignment =
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index be4f3e1be0386..93006bb68ed37 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -3214,7 +3214,7 @@ AMDGPURegisterBankInfo::getImageMapping(const MachineRegisterInfo &MRI,
 
     if (MustBeSGPR) {
       // If this must be an SGPR, so we must report whatever it is as legal.
-      unsigned NewBank = getRegBankID(OpReg, MRI, *TRI, AMDGPU::SGPRRegBankID);
+      unsigned NewBank = getRegBankID(OpReg, MRI, AMDGPU::SGPRRegBankID);
       OpdsMapping[I] = AMDGPU::getValueMapping(NewBank, Size);
     } else {
       // Some operands must be VGPR, and these are easy to copy to.
@@ -3292,9 +3292,8 @@ AMDGPURegisterBankInfo::getInstrMappingForLoad(const MachineInstr &MI) const {
 unsigned
 AMDGPURegisterBankInfo::getRegBankID(Register Reg,
                                      const MachineRegisterInfo &MRI,
-                                     const TargetRegisterInfo &TRI,
                                      unsigned Default) const {
-  const RegisterBank *Bank = getRegBank(Reg, MRI, TRI);
+  const RegisterBank *Bank = getRegBank(Reg, MRI, *TRI);
   return Bank ? Bank->getID() : Default;
 }
 
@@ -3326,7 +3325,7 @@ AMDGPURegisterBankInfo::getSGPROpMapping(Register Reg,
                                          const TargetRegisterInfo &TRI) const {
   // Lie and claim anything is legal, even though this needs to be an SGPR
   // applyMapping will have to deal with it as a waterfall loop.
-  unsigned Bank = getRegBankID(Reg, MRI, TRI, AMDGPU::SGPRRegBankID);
+  unsigned Bank = getRegBankID(Reg, MRI, AMDGPU::SGPRRegBankID);
   unsigned Size = getSizeInBits(Reg, MRI, TRI);
   return AMDGPU::getValueMapping(Bank, Size);
 }
@@ -3394,7 +3393,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     unsigned BankID = AMDGPU::SGPRRegBankID;
 
     for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
-      auto OpBank = getRegBankID(MI.getOperand(I).getReg(), MRI, *TRI);
+      auto OpBank = getRegBankID(MI.getOperand(I).getReg(), MRI);
       // It doesn't make sense to use vcc or scc banks here, so just ignore
       // them.
       if (OpBank != AMDGPU::SGPRRegBankID) {
@@ -3477,15 +3476,15 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
           BankLHS = AMDGPU::VCCRegBankID;
           BankRHS = AMDGPU::VCCRegBankID;
         } else {
-          BankLHS = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI,
+          BankLHS = getRegBankID(MI.getOperand(1).getReg(), MRI,
                                  AMDGPU::SGPRRegBankID);
-          BankRHS = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI,
+          BankRHS = getRegBankID(MI.getOperand(2).getReg(), MRI,
                                  AMDGPU::SGPRRegBankID);
         }
       } else {
-        BankLHS = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI,
+        BankLHS = getRegBankID(MI.getOperand(1).getReg(), MRI,
                                AMDGPU::VCCRegBankID);
-        BankRHS = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI,
+        BankRHS = getRegBankID(MI.getOperand(2).getReg(), MRI,
                                AMDGPU::VCCRegBankID);
 
         // Both inputs should be true booleans to produce a boolean result.
@@ -3513,10 +3512,10 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
         OpdsMapping[1] = OpdsMapping[2] = OpdsMapping[0];
       } else {
         OpdsMapping[0] = getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size);
-        unsigned Bank1 = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI/*, DefaultBankID*/);
+        unsigned Bank1 = getRegBankID(MI.getOperand(1).getReg(), MRI /*, DefaultBankID*/);
         OpdsMapping[1] = AMDGPU::getValueMapping(Bank1, Size);
 
-        unsigned Bank2 = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI/*, DefaultBankID*/);
+        unsigned Bank2 = getRegBankID(MI.getOperand(2).getReg(), MRI /*, DefaultBankID*/);
         OpdsMapping[2] = AMDGPU::getValueMapping(Bank2, Size);
       }
 
@@ -3548,6 +3547,10 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
       return getDefaultMappingSOP(MI);
     LLVM_FALLTHROUGH;
 
+  case AMDGPU::G_SADDSAT: // FIXME: Could lower sat ops for SALU
+  case AMDGPU::G_SSUBSAT:
+  case AMDGPU::G_UADDSAT:
+  case AMDGPU::G_USUBSAT:
   case AMDGPU::G_FADD:
   case AMDGPU::G_FSUB:
   case AMDGPU::G_FPTOSI:
@@ -3612,7 +3615,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   case AMDGPU::G_DYN_STACKALLOC: {
     // Result is always uniform, and a wave reduction is needed for the source.
     OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32);
-    unsigned SrcBankID = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI);
+    unsigned SrcBankID = getRegBankID(MI.getOperand(1).getReg(), MRI);
     OpdsMapping[1] = AMDGPU::getValueMapping(SrcBankID, 32);
     break;
   }
@@ -3629,7 +3632,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     break;
   }
   case AMDGPU::G_EXTRACT: {
-    unsigned BankID = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI);
+    unsigned BankID = getRegBankID(MI.getOperand(1).getReg(), MRI);
     unsigned DstSize = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
     unsigned SrcSize = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI);
     OpdsMapping[0] = AMDGPU::getValueMapping(BankID, DstSize);
@@ -3643,8 +3646,8 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     if (DstTy == LLT::vector(2, 16)) {
       unsigned DstSize = DstTy.getSizeInBits();
       unsigned SrcSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
-      unsigned Src0BankID = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI);
-      unsigned Src1BankID = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI);
+      unsigned Src0BankID = getRegBankID(MI.getOperand(1).getReg(), MRI);
+      unsigned Src1BankID = getRegBankID(MI.getOperand(2).getReg(), MRI);
       unsigned DstBankID = regBankUnion(Src0BankID, Src1BankID);
 
       OpdsMapping[0] = AMDGPU::getValueMapping(DstBankID, DstSize);
@@ -3675,7 +3678,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   case AMDGPU::G_FABS:
   case AMDGPU::G_FNEG: {
     unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
-    unsigned BankID = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI);
+    unsigned BankID = getRegBankID(MI.getOperand(1).getReg(), MRI);
     OpdsMapping[0] = OpdsMapping[1] = AMDGPU::getValueMapping(BankID, Size);
     break;
   }
@@ -3683,7 +3686,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   case AMDGPU::G_CTTZ_ZERO_UNDEF:
   case AMDGPU::G_CTPOP: {
     unsigned Size = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
-    unsigned BankID = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI);
+    unsigned BankID = getRegBankID(MI.getOperand(1).getReg(), MRI);
     OpdsMapping[0] = AMDGPU::getValueMapping(BankID, 32);
 
     // This should really be getValueMappingSGPR64Only, but allowing the generic
@@ -3695,7 +3698,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   case AMDGPU::G_TRUNC: {
     Register Dst = MI.getOperand(0).getReg();
     Register Src = MI.getOperand(1).getReg();
-    unsigned Bank = getRegBankID(Src, MRI, *TRI);
+    unsigned Bank = getRegBankID(Src, MRI);
     unsigned DstSize = getSizeInBits(Dst, MRI, *TRI);
     unsigned SrcSize = getSizeInBits(Src, MRI, *TRI);
     OpdsMapping[0] = AMDGPU::getValueMapping(Bank, DstSize);
@@ -3732,7 +3735,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   }
   case AMDGPU::G_FCMP: {
     unsigned Size = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
-    unsigned Op2Bank = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI);
+    unsigned Op2Bank = getRegBankID(MI.getOperand(2).getReg(), MRI);
     OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1);
     OpdsMapping[1] = nullptr; // Predicate Operand.
     OpdsMapping[2] = AMDGPU::getValueMapping(Op2Bank, Size);
@@ -3757,10 +3760,10 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
 
     // See if the result register has already been constrained to vcc, which may
     // happen due to control flow intrinsic lowering.
-    unsigned DstBank = getRegBankID(MI.getOperand(0).getReg(), MRI, *TRI,
+    unsigned DstBank = getRegBankID(MI.getOperand(0).getReg(), MRI,
                                     AMDGPU::SGPRRegBankID);
-    unsigned Op2Bank = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI);
-    unsigned Op3Bank = getRegBankID(MI.getOperand(3).getReg(), MRI, *TRI);
+    unsigned Op2Bank = getRegBankID(MI.getOperand(2).getReg(), MRI);
+    unsigned Op3Bank = getRegBankID(MI.getOperand(3).getReg(), MRI);
 
     bool CanUseSCC = DstBank == AMDGPU::SGPRRegBankID &&
                      Op2Bank == AMDGPU::SGPRRegBankID &&
@@ -3783,11 +3786,11 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   }
   case AMDGPU::G_EXTRACT_VECTOR_ELT: {
     // VGPR index can be used for waterfall when indexing a SGPR vector.
-    unsigned SrcBankID = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI);
+    unsigned SrcBankID = getRegBankID(MI.getOperand(1).getReg(), MRI);
     unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
     unsigned SrcSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
     unsigned IdxSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
-    unsigned IdxBank = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI);
+    unsigned IdxBank = getRegBankID(MI.getOperand(2).getReg(), MRI);
     unsigned OutputBankID = regBankUnion(SrcBankID, IdxBank);
 
     OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(OutputBankID, DstSize);
@@ -3804,9 +3807,8 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     unsigned VecSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
     unsigned InsertSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
     unsigned IdxSize = MRI.getType(MI.getOperand(3).getReg()).getSizeInBits();
-    unsigned InsertEltBankID = getRegBankID(MI.getOperand(2).getReg(),
-                                            MRI, *TRI);
-    unsigned IdxBankID = getRegBankID(MI.getOperand(3).getReg(), MRI, *TRI);
+    unsigned InsertEltBankID = getRegBankID(MI.getOperand(2).getReg(), MRI);
+    unsigned IdxBankID = getRegBankID(MI.getOperand(3).getReg(), MRI);
 
     OpdsMapping[0] = AMDGPU::getValueMapping(OutputBankID, VecSize);
     OpdsMapping[1] = AMDGPU::getValueMapping(OutputBankID, VecSize);
@@ -4017,6 +4019,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     case Intrinsic::amdgcn_wwm:
     case Intrinsic::amdgcn_wqm:
     case Intrinsic::amdgcn_softwqm:
+    case Intrinsic::amdgcn_set_inactive:
       return getDefaultMappingAllVGPR(MI);
     case Intrinsic::amdgcn_kernarg_segment_ptr:
     case Intrinsic::amdgcn_s_getpc:
@@ -4071,7 +4074,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
       // This must be an SGPR, but accept a VGPR.
       Register IdxReg = MI.getOperand(3).getReg();
       unsigned IdxSize = MRI.getType(IdxReg).getSizeInBits();
-      unsigned IdxBank = getRegBankID(IdxReg, MRI, *TRI, AMDGPU::SGPRRegBankID);
+      unsigned IdxBank = getRegBankID(IdxReg, MRI, AMDGPU::SGPRRegBankID);
       OpdsMapping[3] = AMDGPU::getValueMapping(IdxBank, IdxSize);
       LLVM_FALLTHROUGH;
     }
@@ -4086,10 +4089,10 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
       unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
       Register SrcReg = MI.getOperand(2).getReg();
       unsigned SrcSize = MRI.getType(SrcReg).getSizeInBits();
-      unsigned SrcBank = getRegBankID(SrcReg, MRI, *TRI, AMDGPU::SGPRRegBankID);
+      unsigned SrcBank = getRegBankID(SrcReg, MRI, AMDGPU::SGPRRegBankID);
       Register IdxReg = MI.getOperand(3).getReg();
       unsigned IdxSize = MRI.getType(IdxReg).getSizeInBits();
-      unsigned IdxBank = getRegBankID(IdxReg, MRI, *TRI, AMDGPU::SGPRRegBankID);
+      unsigned IdxBank = getRegBankID(IdxReg, MRI, AMDGPU::SGPRRegBankID);
       OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize);
 
       // These 2 must be SGPRs, but accept VGPRs. Readfirstlane will be inserted
@@ -4155,7 +4158,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     case Intrinsic::amdgcn_interp_p2_f16: {
       const int M0Idx = MI.getNumOperands() - 1;
       Register M0Reg = MI.getOperand(M0Idx).getReg();
-      unsigned M0Bank = getRegBankID(M0Reg, MRI, *TRI, AMDGPU::SGPRRegBankID);
+      unsigned M0Bank = getRegBankID(M0Reg, MRI, AMDGPU::SGPRRegBankID);
       unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
 
       OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize);
@@ -4207,7 +4210,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     case Intrinsic::amdgcn_ds_ordered_swap: {
       unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
       OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize);
-      unsigned M0Bank = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI,
+      unsigned M0Bank = getRegBankID(MI.getOperand(2).getReg(), MRI,
                                  AMDGPU::SGPRRegBankID);
       OpdsMapping[2] = AMDGPU::getValueMapping(M0Bank, 32);
       OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
@@ -4234,14 +4237,14 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     case Intrinsic::amdgcn_s_sendmsg:
     case Intrinsic::amdgcn_s_sendmsghalt: {
       // This must be an SGPR, but accept a VGPR.
-      unsigned Bank = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI,
+      unsigned Bank = getRegBankID(MI.getOperand(2).getReg(), MRI,
                                    AMDGPU::SGPRRegBankID);
       OpdsMapping[2] = AMDGPU::getValueMapping(Bank, 32);
       break;
     }
     case Intrinsic::amdgcn_s_setreg: {
       // This must be an SGPR, but accept a VGPR.
-      unsigned Bank = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI,
+      unsigned Bank = getRegBankID(MI.getOperand(2).getReg(), MRI,
                                    AMDGPU::SGPRRegBankID);
       OpdsMapping[2] = AMDGPU::getValueMapping(Bank, 32);
       break;
@@ -4310,7 +4313,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
       OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
 
       // This must be an SGPR, but accept a VGPR.
-      unsigned Bank = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI,
+      unsigned Bank = getRegBankID(MI.getOperand(2).getReg(), MRI,
                                    AMDGPU::SGPRRegBankID);
       OpdsMapping[2] = AMDGPU::getValueMapping(Bank, 32);
       break;
@@ -4319,7 +4322,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     case Intrinsic::amdgcn_ds_gws_sema_p:
     case Intrinsic::amdgcn_ds_gws_sema_release_all: {
       // This must be an SGPR, but accept a VGPR.
-      unsigned Bank = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI,
+      unsigned Bank = getRegBankID(MI.getOperand(1).getReg(), MRI,
                                    AMDGPU::SGPRRegBankID);
       OpdsMapping[1] = AMDGPU::getValueMapping(Bank, 32);
       break;
@@ -4331,16 +4334,16 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   }
   case AMDGPU::G_SELECT: {
     unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
-    unsigned Op2Bank = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI,
+    unsigned Op2Bank = getRegBankID(MI.getOperand(2).getReg(), MRI,
                                     AMDGPU::SGPRRegBankID);
-    unsigned Op3Bank = getRegBankID(MI.getOperand(3).getReg(), MRI, *TRI,
+    unsigned Op3Bank = getRegBankID(MI.getOperand(3).getReg(), MRI,
                                     AMDGPU::SGPRRegBankID);
     bool SGPRSrcs = Op2Bank == AMDGPU::SGPRRegBankID &&
                     Op3Bank == AMDGPU::SGPRRegBankID;
 
     unsigned CondBankDefault = SGPRSrcs ?
       AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID;
-    unsigned CondBank = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI,
+    unsigned CondBank = getRegBankID(MI.getOperand(1).getReg(), MRI,
                                      CondBankDefault);
     if (CondBank == AMDGPU::SGPRRegBankID)
       CondBank = SGPRSrcs ? AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID;
@@ -4400,7 +4403,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     break;
   }
   case AMDGPU::G_BRCOND: {
-    unsigned Bank = getRegBankID(MI.getOperand(0).getReg(), MRI, *TRI,
+    unsigned Bank = getRegBankID(MI.getOperand(0).getReg(), MRI,
                                  AMDGPU::SGPRRegBankID);
     assert(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() == 1);
     if (Bank != AMDGPU::SGPRRegBankID)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h
index 8f38ec4eeb3a4..23a466e9bd85b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h
@@ -105,7 +105,6 @@ class AMDGPURegisterBankInfo : public AMDGPUGenRegisterBankInfo {
   getInstrMappingForLoad(const MachineInstr &MI) const;
 
   unsigned getRegBankID(Register Reg, const MachineRegisterInfo &MRI,
-                        const TargetRegisterInfo &TRI,
                         unsigned Default = AMDGPU::VGPRRegBankID) const;
 
   // Return a value mapping for an operand that is required to be an SGPR.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
index 213788ae0f67b..ac30137943bc2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
@@ -508,12 +508,15 @@ uint64_t AMDGPUSubtarget::getExplicitKernArgSize(const Function &F,
   MaxAlign = Align(1);
 
   for (const Argument &Arg : F.args()) {
-    Type *ArgTy = Arg.getType();
+    const bool IsByRef = Arg.hasByRefAttr();
+    Type *ArgTy = IsByRef ? Arg.getParamByRefType() : Arg.getType();
+    MaybeAlign Alignment = IsByRef ? Arg.getParamAlign() : None;
+    if (!Alignment)
+      Alignment = DL.getABITypeAlign(ArgTy);
 
-    const Align Alignment = DL.getABITypeAlign(ArgTy);
     uint64_t AllocSize = DL.getTypeAllocSize(ArgTy);
     ExplicitArgBytes = alignTo(ExplicitArgBytes, Alignment) + AllocSize;
-    MaxAlign = std::max(MaxAlign, Alignment);
+    MaxAlign = max(MaxAlign, Alignment);
   }
 
   return ExplicitArgBytes;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 9ca851c4d7463..7998b0cb9f6ac 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -723,10 +723,9 @@ static bool isArgPassedInSGPR(const Argument *A) {
   case CallingConv::AMDGPU_GS:
   case CallingConv::AMDGPU_PS:
   case CallingConv::AMDGPU_CS:
-    // For non-compute shaders, SGPR inputs are marked with either inreg or byval.
+    // For non-compute shaders, SGPR inputs are marked with either inreg.
     // Everything else is in VGPRs.
-    return F->getAttributes().hasParamAttribute(A->getArgNo(), Attribute::InReg) ||
-           F->getAttributes().hasParamAttribute(A->getArgNo(), Attribute::ByVal);
+    return F->getAttributes().hasParamAttribute(A->getArgNo(), Attribute::InReg);
   default:
     // TODO: Should calls support inreg for SGPR inputs?
     return false;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index 3364a9bcaccbb..b060221330cb2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -33,6 +33,7 @@
 namespace llvm {
 
 class AMDGPUTargetLowering;
+class InstCombiner;
 class Loop;
 class ScalarEvolution;
 class Type;
@@ -223,6 +224,14 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
   Value *rewriteIntrinsicWithAddressSpace(IntrinsicInst *II, Value *OldV,
                                           Value *NewV) const;
 
+  Optional<Instruction *> instCombineIntrinsic(InstCombiner &IC,
+                                               IntrinsicInst &II) const;
+  Optional<Value *> simplifyDemandedVectorEltsIntrinsic(
+      InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
+      APInt &UndefElts2, APInt &UndefElts3,
+      std::function<void(Instruction *, unsigned, APInt, APInt &)>
+          SimplifyAndSetOp) const;
+
   unsigned getVectorSplitCost() { return 0; }
 
   unsigned getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, int Index,
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index 1f6b37cf25fe2..32d1b05e49f08 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -34,6 +34,10 @@ tablegen(LLVM R600GenSubtargetInfo.inc -gen-subtarget)
 
 add_public_tablegen_target(AMDGPUCommonTableGen)
 
+set(LLVM_TARGET_DEFINITIONS InstCombineTables.td)
+tablegen(LLVM InstCombineTables.inc -gen-searchable-tables)
+add_public_tablegen_target(InstCombineTableGen)
+
 add_llvm_target(AMDGPUCodeGen
   AMDGPUAliasAnalysis.cpp
   AMDGPUAlwaysInlinePass.cpp
@@ -48,6 +52,7 @@ add_llvm_target(AMDGPUCodeGen
   AMDGPUFixFunctionBitcasts.cpp
   AMDGPUFrameLowering.cpp
   AMDGPUHSAMetadataStreamer.cpp
+  AMDGPUInstCombineIntrinsic.cpp
   AMDGPUInstrInfo.cpp
   AMDGPUInstructionSelector.cpp
   AMDGPUISelDAGToDAG.cpp
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineTables.td b/llvm/lib/Target/AMDGPU/InstCombineTables.td
similarity index 100%
rename from llvm/lib/Transforms/InstCombine/InstCombineTables.td
rename to llvm/lib/Target/AMDGPU/InstCombineTables.td
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
index 00bf404dea279..e44c0194e811b 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
@@ -391,10 +391,12 @@ void AMDGPUInstPrinter::printImmediateInt16(uint32_t Imm,
                                             const MCSubtargetInfo &STI,
                                             raw_ostream &O) {
   int16_t SImm = static_cast<int16_t>(Imm);
-  if (isInlinableIntLiteral(SImm))
+  if (isInlinableIntLiteral(SImm)) {
     O << SImm;
-  else
-    O << formatHex(static_cast<uint64_t>(Imm));
+  } else {
+    uint64_t Imm16 = static_cast<uint16_t>(Imm);
+    O << formatHex(Imm16);
+  }
 }
 
 void AMDGPUInstPrinter::printImmediate16(uint32_t Imm,
@@ -422,11 +424,13 @@ void AMDGPUInstPrinter::printImmediate16(uint32_t Imm,
     O<< "4.0";
   else if (Imm == 0xC400)
     O<< "-4.0";
-  else if (Imm == 0x3118) {
-    assert(STI.getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm]);
+  else if (Imm == 0x3118 &&
+           STI.getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm]) {
     O << "0.15915494";
-  } else
-    O << formatHex(static_cast<uint64_t>(Imm));
+  } else {
+    uint64_t Imm16 = static_cast<uint16_t>(Imm);
+    O << formatHex(Imm16);
+  }
 }
 
 void AMDGPUInstPrinter::printImmediateV216(uint32_t Imm,
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
index 6dfd23ea72e67..78a66a78ec341 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
@@ -109,8 +109,6 @@ class AMDGPUInstPrinter : public MCInstPrinter {
                            raw_ostream &O);
   void printImmediate16(uint32_t Imm, const MCSubtargetInfo &STI,
                         raw_ostream &O);
-  void printImmediateIntV216(uint32_t Imm, const MCSubtargetInfo &STI,
-                             raw_ostream &O);
   void printImmediateV216(uint32_t Imm, const MCSubtargetInfo &STI,
                           raw_ostream &O);
   void printImmediate32(uint32_t Imm, const MCSubtargetInfo &STI,
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 0986e1efb9840..ea1d20f7387df 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -35,7 +35,7 @@ struct FoldCandidate {
     int FrameIndexToFold;
   };
   int ShrinkOpcode;
-  unsigned char UseOpNo;
+  unsigned UseOpNo;
   MachineOperand::MachineOperandType Kind;
   bool Commuted;
 
@@ -662,6 +662,11 @@ void SIFoldOperands::foldOperand(
           Use = MRI->use_begin(DestReg), E = MRI->use_end();
           Use != E; Use = NextUse) {
           NextUse = std::next(Use);
+
+          // There's no point trying to fold into an implicit operand.
+          if (Use->isImplicit())
+            continue;
+
           FoldCandidate FC = FoldCandidate(Use->getParent(),
            Use.getOperandNo(), &UseMI->getOperand(1));
           CopyUses.push_back(FC);
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index c2666f3302151..fa2a96d33952b 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -2254,9 +2254,21 @@ SDValue SITargetLowering::LowerFormalArguments(
       const uint64_t Offset = VA.getLocMemOffset();
       Align Alignment = commonAlignment(KernelArgBaseAlign, Offset);
 
-      SDValue Arg =
-          lowerKernargMemParameter(DAG, VT, MemVT, DL, Chain, Offset, Alignment,
-                                   Ins[i].Flags.isSExt(), &Ins[i]);
+      if (Arg.Flags.isByRef()) {
+        SDValue Ptr = lowerKernArgParameterPtr(DAG, DL, Chain, Offset);
+
+        if (!isNoopAddrSpaceCast(AMDGPUAS::CONSTANT_ADDRESS,
+                                 Arg.Flags.getPointerAddrSpace())) {
+          Ptr = DAG.getAddrSpaceCast(DL, VT, Ptr, AMDGPUAS::CONSTANT_ADDRESS,
+                                     Arg.Flags.getPointerAddrSpace());
+        }
+
+        InVals.push_back(Ptr);
+        continue;
+      }
+
+      SDValue Arg = lowerKernargMemParameter(
+        DAG, VT, MemVT, DL, Chain, Offset, Alignment, Ins[i].Flags.isSExt(), &Ins[i]);
       Chains.push_back(Arg.getValue(1));
 
       auto *ParamTy =
@@ -4479,7 +4491,7 @@ static SDValue adjustLoadValueTypeImpl(SDValue Result, EVT LoadVT,
     EVT IntLoadVT = LoadVT.changeTypeToInteger();
 
     // Workaround legalizer not scalarizing truncate after vector op
-    // legalization byt not creating intermediate vector trunc.
+    // legalization but not creating intermediate vector trunc.
     SmallVector<SDValue, 4> Elts;
     DAG.ExtractVectorElements(Result, Elts);
     for (SDValue &Elt : Elts)
diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
index cf1629fda0aff..57a89d9acc815 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
@@ -684,9 +684,9 @@ class SIMachineFunctionInfo final : public AMDGPUMachineFunction {
     return ArgInfo.getPreloadedValue(Value);
   }
 
-  Register getPreloadedReg(AMDGPUFunctionArgInfo::PreloadedValue Value) const {
+  MCRegister getPreloadedReg(AMDGPUFunctionArgInfo::PreloadedValue Value) const {
     auto Arg = std::get<0>(ArgInfo.getPreloadedValue(Value));
-    return Arg ? Arg->getRegister() : Register();
+    return Arg ? Arg->getRegister() : MCRegister();
   }
 
   unsigned getGITPtrHigh() const {
diff --git a/llvm/lib/Target/AMDGPU/VOP2Instructions.td b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
index 55b64c4e614c4..d9dcfdca23b35 100644
--- a/llvm/lib/Target/AMDGPU/VOP2Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
@@ -472,7 +472,7 @@ def V_MADMK_F32 : VOP2_Pseudo <"v_madmk_f32", VOP_MADMK_F32, []>;
 
 let isCommutable = 1 in {
 defm V_ADD_F32 : VOP2Inst <"v_add_f32", VOP_F32_F32_F32, any_fadd>;
-defm V_SUB_F32 : VOP2Inst <"v_sub_f32", VOP_F32_F32_F32, fsub>;
+defm V_SUB_F32 : VOP2Inst <"v_sub_f32", VOP_F32_F32_F32, any_fsub>;
 defm V_SUBREV_F32 : VOP2Inst <"v_subrev_f32", VOP_F32_F32_F32, null_frag, "v_sub_f32">;
 defm V_MUL_LEGACY_F32 : VOP2Inst <"v_mul_legacy_f32", VOP_F32_F32_F32, AMDGPUfmul_legacy>;
 defm V_MUL_F32 : VOP2Inst <"v_mul_f32", VOP_F32_F32_F32, any_fmul>;
@@ -636,7 +636,7 @@ defm V_ASHRREV_I16 : VOP2Inst <"v_ashrrev_i16", VOP_I16_I16_I16, ashr_rev>;
 let isCommutable = 1 in {
 let FPDPRounding = 1 in {
 defm V_ADD_F16 : VOP2Inst <"v_add_f16", VOP_F16_F16_F16, any_fadd>;
-defm V_SUB_F16 : VOP2Inst <"v_sub_f16", VOP_F16_F16_F16, fsub>;
+defm V_SUB_F16 : VOP2Inst <"v_sub_f16", VOP_F16_F16_F16, any_fsub>;
 defm V_SUBREV_F16 : VOP2Inst <"v_subrev_f16", VOP_F16_F16_F16, null_frag, "v_sub_f16">;
 defm V_MUL_F16 : VOP2Inst <"v_mul_f16", VOP_F16_F16_F16, any_fmul>;
 
diff --git a/llvm/lib/Target/ARC/ARCISelLowering.cpp b/llvm/lib/Target/ARC/ARCISelLowering.cpp
index 4a6510f10eeb4..ca33f52974711 100644
--- a/llvm/lib/Target/ARC/ARCISelLowering.cpp
+++ b/llvm/lib/Target/ARC/ARCISelLowering.cpp
@@ -22,7 +22,6 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineJumpTableInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/SelectionDAGISel.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/Intrinsics.h"
diff --git a/llvm/lib/Target/ARM/ARMCallLowering.cpp b/llvm/lib/Target/ARM/ARMCallLowering.cpp
index 4f1410eecff4f..0b2427e414200 100644
--- a/llvm/lib/Target/ARM/ARMCallLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMCallLowering.cpp
@@ -85,12 +85,11 @@ namespace {
 
 /// Helper class for values going out through an ABI boundary (used for handling
 /// function return values and call parameters).
-struct OutgoingValueHandler : public CallLowering::ValueHandler {
-  OutgoingValueHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
-                       MachineInstrBuilder &MIB, CCAssignFn *AssignFn)
-      : ValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB) {}
-
-  bool isIncomingArgumentHandler() const override { return false; }
+struct ARMOutgoingValueHandler : public CallLowering::OutgoingValueHandler {
+  ARMOutgoingValueHandler(MachineIRBuilder &MIRBuilder,
+                          MachineRegisterInfo &MRI, MachineInstrBuilder &MIB,
+                          CCAssignFn *AssignFn)
+      : OutgoingValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB) {}
 
   Register getStackAddress(uint64_t Size, int64_t Offset,
                            MachinePointerInfo &MPO) override {
@@ -258,7 +257,8 @@ bool ARMCallLowering::lowerReturnVal(MachineIRBuilder &MIRBuilder,
   CCAssignFn *AssignFn =
       TLI.CCAssignFnForReturn(F.getCallingConv(), F.isVarArg());
 
-  OutgoingValueHandler RetHandler(MIRBuilder, MF.getRegInfo(), Ret, AssignFn);
+  ARMOutgoingValueHandler RetHandler(MIRBuilder, MF.getRegInfo(), Ret,
+                                     AssignFn);
   return handleAssignments(MIRBuilder, SplitRetInfos, RetHandler);
 }
 
@@ -282,12 +282,10 @@ namespace {
 
 /// Helper class for values coming in through an ABI boundary (used for handling
 /// formal arguments and call return values).
-struct IncomingValueHandler : public CallLowering::ValueHandler {
-  IncomingValueHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
-                       CCAssignFn AssignFn)
-      : ValueHandler(MIRBuilder, MRI, AssignFn) {}
-
-  bool isIncomingArgumentHandler() const override { return true; }
+struct ARMIncomingValueHandler : public CallLowering::IncomingValueHandler {
+  ARMIncomingValueHandler(MachineIRBuilder &MIRBuilder,
+                          MachineRegisterInfo &MRI, CCAssignFn AssignFn)
+      : IncomingValueHandler(MIRBuilder, MRI, AssignFn) {}
 
   Register getStackAddress(uint64_t Size, int64_t Offset,
                            MachinePointerInfo &MPO) override {
@@ -399,10 +397,10 @@ struct IncomingValueHandler : public CallLowering::ValueHandler {
   virtual void markPhysRegUsed(unsigned PhysReg) = 0;
 };
 
-struct FormalArgHandler : public IncomingValueHandler {
+struct FormalArgHandler : public ARMIncomingValueHandler {
   FormalArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
                    CCAssignFn AssignFn)
-      : IncomingValueHandler(MIRBuilder, MRI, AssignFn) {}
+      : ARMIncomingValueHandler(MIRBuilder, MRI, AssignFn) {}
 
   void markPhysRegUsed(unsigned PhysReg) override {
     MIRBuilder.getMRI()->addLiveIn(PhysReg);
@@ -469,10 +467,10 @@ bool ARMCallLowering::lowerFormalArguments(
 
 namespace {
 
-struct CallReturnHandler : public IncomingValueHandler {
+struct CallReturnHandler : public ARMIncomingValueHandler {
   CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
                     MachineInstrBuilder MIB, CCAssignFn *AssignFn)
-      : IncomingValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB) {}
+      : ARMIncomingValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB) {}
 
   void markPhysRegUsed(unsigned PhysReg) override {
     MIB.addDef(PhysReg, RegState::Implicit);
@@ -554,7 +552,7 @@ bool ARMCallLowering::lowerCall(MachineIRBuilder &MIRBuilder, CallLoweringInfo &
   }
 
   auto ArgAssignFn = TLI.CCAssignFnForCall(Info.CallConv, IsVarArg);
-  OutgoingValueHandler ArgHandler(MIRBuilder, MRI, MIB, ArgAssignFn);
+  ARMOutgoingValueHandler ArgHandler(MIRBuilder, MRI, MIB, ArgAssignFn);
   if (!handleAssignments(MIRBuilder, ArgInfos, ArgHandler))
     return false;
 
diff --git a/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp b/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp
index 195d0a89291b0..204e57fefb9a5 100644
--- a/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp
+++ b/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp
@@ -491,7 +491,11 @@ ARMConstantIslands::doInitialConstPlacement(std::vector<MachineInstr*> &CPEMIs)
 
   // The function needs to be as aligned as the basic blocks. The linker may
   // move functions around based on their alignment.
-  MF->ensureAlignment(BB->getAlignment());
+  // Special case: halfword literals still need word alignment on the function.
+  Align FuncAlign = MaxAlign;
+  if (MaxAlign == 2)
+    FuncAlign = Align(4);
+  MF->ensureAlignment(FuncAlign);
 
   // Order the entries in BB by descending alignment.  That ensures correct
   // alignment of all entries as long as BB is sufficiently aligned.  Keep
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 7c2798b0a4ba2..dd53b10797f84 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -1718,6 +1718,8 @@ const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case ARMISD::VMULLu:        return "ARMISD::VMULLu";
   case ARMISD::VADDVs:        return "ARMISD::VADDVs";
   case ARMISD::VADDVu:        return "ARMISD::VADDVu";
+  case ARMISD::VADDVps:       return "ARMISD::VADDVps";
+  case ARMISD::VADDVpu:       return "ARMISD::VADDVpu";
   case ARMISD::VADDLVs:       return "ARMISD::VADDLVs";
   case ARMISD::VADDLVu:       return "ARMISD::VADDLVu";
   case ARMISD::VADDLVAs:      return "ARMISD::VADDLVAs";
@@ -1728,10 +1730,16 @@ const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case ARMISD::VADDLVApu:     return "ARMISD::VADDLVApu";
   case ARMISD::VMLAVs:        return "ARMISD::VMLAVs";
   case ARMISD::VMLAVu:        return "ARMISD::VMLAVu";
+  case ARMISD::VMLAVps:       return "ARMISD::VMLAVps";
+  case ARMISD::VMLAVpu:       return "ARMISD::VMLAVpu";
   case ARMISD::VMLALVs:       return "ARMISD::VMLALVs";
   case ARMISD::VMLALVu:       return "ARMISD::VMLALVu";
+  case ARMISD::VMLALVps:      return "ARMISD::VMLALVps";
+  case ARMISD::VMLALVpu:      return "ARMISD::VMLALVpu";
   case ARMISD::VMLALVAs:      return "ARMISD::VMLALVAs";
   case ARMISD::VMLALVAu:      return "ARMISD::VMLALVAu";
+  case ARMISD::VMLALVAps:     return "ARMISD::VMLALVAps";
+  case ARMISD::VMLALVApu:     return "ARMISD::VMLALVApu";
   case ARMISD::UMAAL:         return "ARMISD::UMAAL";
   case ARMISD::UMLAL:         return "ARMISD::UMLAL";
   case ARMISD::SMLAL:         return "ARMISD::SMLAL";
@@ -12259,6 +12267,14 @@ static SDValue PerformADDVecReduce(SDNode *N,
     return M;
   if (SDValue M = MakeVecReduce(ARMISD::VMLALVu, ARMISD::VMLALVAu, N1, N0))
     return M;
+  if (SDValue M = MakeVecReduce(ARMISD::VMLALVps, ARMISD::VMLALVAps, N0, N1))
+    return M;
+  if (SDValue M = MakeVecReduce(ARMISD::VMLALVpu, ARMISD::VMLALVApu, N0, N1))
+    return M;
+  if (SDValue M = MakeVecReduce(ARMISD::VMLALVps, ARMISD::VMLALVAps, N1, N0))
+    return M;
+  if (SDValue M = MakeVecReduce(ARMISD::VMLALVpu, ARMISD::VMLALVApu, N1, N0))
+    return M;
   return SDValue();
 }
 
@@ -14729,6 +14745,20 @@ static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG,
       return A;
     return SDValue();
   };
+  auto IsPredVADDV = [&](MVT RetTy, unsigned ExtendCode,
+                         ArrayRef<MVT> ExtTypes, SDValue &Mask) {
+    if (ResVT != RetTy || N0->getOpcode() != ISD::VSELECT ||
+        !ISD::isBuildVectorAllZeros(N0->getOperand(2).getNode()))
+      return SDValue();
+    Mask = N0->getOperand(0);
+    SDValue Ext = N0->getOperand(1);
+    if (Ext->getOpcode() != ExtendCode)
+      return SDValue();
+    SDValue A = Ext->getOperand(0);
+    if (llvm::any_of(ExtTypes, [&A](MVT Ty) { return A.getValueType() == Ty; }))
+      return A;
+    return SDValue();
+  };
   auto IsVMLAV = [&](MVT RetTy, unsigned ExtendCode, ArrayRef<MVT> ExtTypes,
                      SDValue &A, SDValue &B) {
     if (ResVT != RetTy || N0->getOpcode() != ISD::MUL)
@@ -14744,6 +14774,26 @@ static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG,
       return true;
     return false;
   };
+  auto IsPredVMLAV = [&](MVT RetTy, unsigned ExtendCode, ArrayRef<MVT> ExtTypes,
+                     SDValue &A, SDValue &B, SDValue &Mask) {
+    if (ResVT != RetTy || N0->getOpcode() != ISD::VSELECT ||
+        !ISD::isBuildVectorAllZeros(N0->getOperand(2).getNode()))
+      return false;
+    Mask = N0->getOperand(0);
+    SDValue Mul = N0->getOperand(1);
+    if (Mul->getOpcode() != ISD::MUL)
+      return false;
+    SDValue ExtA = Mul->getOperand(0);
+    SDValue ExtB = Mul->getOperand(1);
+    if (ExtA->getOpcode() != ExtendCode && ExtB->getOpcode() != ExtendCode)
+      return false;
+    A = ExtA->getOperand(0);
+    B = ExtB->getOperand(0);
+    if (A.getValueType() == B.getValueType() &&
+        llvm::any_of(ExtTypes, [&A](MVT Ty) { return A.getValueType() == Ty; }))
+      return true;
+    return false;
+  };
   auto Create64bitNode = [&](unsigned Opcode, ArrayRef<SDValue> Ops) {
     SDValue Node = DAG.getNode(Opcode, dl, {MVT::i32, MVT::i32}, Ops);
     return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Node,
@@ -14759,6 +14809,16 @@ static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG,
   if (SDValue A = IsVADDV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v4i32}))
     return Create64bitNode(ARMISD::VADDLVu, {A});
 
+  SDValue Mask;
+  if (SDValue A = IsPredVADDV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, Mask))
+    return DAG.getNode(ARMISD::VADDVps, dl, ResVT, A, Mask);
+  if (SDValue A = IsPredVADDV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, Mask))
+    return DAG.getNode(ARMISD::VADDVpu, dl, ResVT, A, Mask);
+  if (SDValue A = IsPredVADDV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v4i32}, Mask))
+    return Create64bitNode(ARMISD::VADDLVps, {A, Mask});
+  if (SDValue A = IsPredVADDV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v4i32}, Mask))
+    return Create64bitNode(ARMISD::VADDLVpu, {A, Mask});
+
   SDValue A, B;
   if (IsVMLAV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B))
     return DAG.getNode(ARMISD::VMLAVs, dl, ResVT, A, B);
@@ -14768,6 +14828,15 @@ static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG,
     return Create64bitNode(ARMISD::VMLALVs, {A, B});
   if (IsVMLAV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v4i32}, A, B))
     return Create64bitNode(ARMISD::VMLALVu, {A, B});
+
+  if (IsPredVMLAV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B, Mask))
+    return DAG.getNode(ARMISD::VMLAVps, dl, ResVT, A, B, Mask);
+  if (IsPredVMLAV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B, Mask))
+    return DAG.getNode(ARMISD::VMLAVpu, dl, ResVT, A, B, Mask);
+  if (IsPredVMLAV(MVT::i64, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v4i32}, A, B, Mask))
+    return Create64bitNode(ARMISD::VMLALVps, {A, B, Mask});
+  if (IsPredVMLAV(MVT::i64, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v4i32}, A, B, Mask))
+    return Create64bitNode(ARMISD::VMLALVpu, {A, B, Mask});
   return SDValue();
 }
 
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h
index f4d77d4ff70f5..1428600ca5240 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.h
+++ b/llvm/lib/Target/ARM/ARMISelLowering.h
@@ -219,20 +219,28 @@ class VectorType;
       // MVE reductions
       VADDVs,       // sign- or zero-extend the elements of a vector to i32,
       VADDVu,       //   add them all together, and return an i32 of their sum
+      VADDVps,      // Same as VADDV[su] but with a v4i1 predicate mask
+      VADDVpu,
       VADDLVs,      // sign- or zero-extend elements to i64 and sum, returning
       VADDLVu,      //   the low and high 32-bit halves of the sum
-      VADDLVAs,     // same as VADDLV[su] but also add an input accumulator
+      VADDLVAs,     // Same as VADDLV[su] but also add an input accumulator
       VADDLVAu,     //   provided as low and high halves
-      VADDLVps,     // same as VADDLVs but with a v4i1 predicate mask
-      VADDLVpu,     // same as VADDLVu but with a v4i1 predicate mask
-      VADDLVAps,    // same as VADDLVps but with a v4i1 predicate mask
-      VADDLVApu,    // same as VADDLVpu but with a v4i1 predicate mask
-      VMLAVs,
-      VMLAVu,
-      VMLALVs,
-      VMLALVu,
-      VMLALVAs,
-      VMLALVAu,
+      VADDLVps,     // Same as VADDLV[su] but with a v4i1 predicate mask
+      VADDLVpu,
+      VADDLVAps,    // Same as VADDLVp[su] but with a v4i1 predicate mask
+      VADDLVApu,
+      VMLAVs,       // sign- or zero-extend the elements of two vectors to i32, multiply them
+      VMLAVu,       //   and add the results together, returning an i32 of their sum
+      VMLAVps,      // Same as VMLAV[su] with a v4i1 predicate mask
+      VMLAVpu,
+      VMLALVs,      // Same as VMLAV but with i64, returning the low and
+      VMLALVu,      //   high 32-bit halves of the sum
+      VMLALVps,     // Same as VMLALV[su] with a v4i1 predicate mask
+      VMLALVpu,
+      VMLALVAs,     // Same as VMLALV but also add an input accumulator
+      VMLALVAu,     //   provided as low and high halves
+      VMLALVAps,    // Same as VMLALVA[su] with a v4i1 predicate mask
+      VMLALVApu,
 
       SMULWB,       // Signed multiply word by half word, bottom
       SMULWT,       // Signed multiply word by half word, top
diff --git a/llvm/lib/Target/ARM/ARMInstrInfo.td b/llvm/lib/Target/ARM/ARMInstrInfo.td
index da0a836c8f95d..db2ab7a97aa66 100644
--- a/llvm/lib/Target/ARM/ARMInstrInfo.td
+++ b/llvm/lib/Target/ARM/ARMInstrInfo.td
@@ -498,6 +498,18 @@ def SubReg_i32_lane : SDNodeXForm<imm, [{
 }]>;
 
 
+def ARMimmAllZerosV: PatLeaf<(bitconvert (v4i32 (ARMvmovImm (i32 0))))>;
+def ARMimmAllZerosD: PatLeaf<(bitconvert (v2i32 (ARMvmovImm (i32 0))))>;
+def ARMimmAllOnesV: PatLeaf<(bitconvert (v16i8 (ARMvmovImm (i32 0xEFF))))>;
+def ARMimmAllOnesD: PatLeaf<(bitconvert (v8i8 (ARMvmovImm (i32 0xEFF))))>;
+
+def ARMimmOneV: PatLeaf<(ARMvmovImm (i32 timm)), [{
+  ConstantSDNode *ConstVal = cast<ConstantSDNode>(N->getOperand(0));
+  unsigned EltBits = 0;
+  uint64_t EltVal = ARM_AM::decodeVMOVModImm(ConstVal->getZExtValue(), EltBits);
+  return (EltBits == N->getValueType(0).getScalarSizeInBits() && EltVal == 0x01);
+}]>;
+
 
 //===----------------------------------------------------------------------===//
 // Operand Definitions.
diff --git a/llvm/lib/Target/ARM/ARMInstrMVE.td b/llvm/lib/Target/ARM/ARMInstrMVE.td
index d6e725084d7f9..dc701412b5950 100644
--- a/llvm/lib/Target/ARM/ARMInstrMVE.td
+++ b/llvm/lib/Target/ARM/ARMInstrMVE.td
@@ -318,6 +318,78 @@ def MVE_v2f64 : MVEVectorVTInfo<v2f64, ?,     v4i1,  ?,    0b11, "f", ?>;
 def MVE_v16p8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b11, "p", 0b0>;
 def MVE_v8p16 : MVEVectorVTInfo<v8i16, v4i32, v8i1,  v4i1, 0b11, "p", 0b1>;
 
+multiclass MVE_TwoOpPattern<MVEVectorVTInfo VTI, PatFrag Op, Intrinsic PredInt,
+                            dag PredOperands, Instruction Inst,
+                            SDPatternOperator IdentityVec = null_frag> {
+  // Unpredicated
+  def : Pat<(VTI.Vec (Op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),
+            (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
+
+  // Predicated with select
+  if !ne(VTI.Size, 0b11) then {
+    def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$mask),
+                                (VTI.Vec (Op (VTI.Vec MQPR:$Qm),
+                                             (VTI.Vec MQPR:$Qn))),
+                                (VTI.Vec MQPR:$inactive))),
+              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
+                              ARMVCCThen, (VTI.Pred VCCR:$mask),
+                              (VTI.Vec MQPR:$inactive)))>;
+
+    // Optionally with the select folded through the op
+    def : Pat<(VTI.Vec (Op (VTI.Vec MQPR:$Qm),
+                           (VTI.Vec (vselect (VTI.Pred VCCR:$mask),
+                                             (VTI.Vec MQPR:$Qn),
+                                             (VTI.Vec IdentityVec))))),
+              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
+                              ARMVCCThen, (VTI.Pred VCCR:$mask),
+                              (VTI.Vec MQPR:$Qm)))>;
+  }
+
+  // Predicated with intrinsic
+  def : Pat<(VTI.Vec !con((PredInt (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)),
+                          PredOperands,
+                          (? (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive)))),
+            (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
+                            ARMVCCThen, (VTI.Pred VCCR:$mask),
+                            (VTI.Vec MQPR:$inactive)))>;
+}
+
+multiclass MVE_TwoOpPatternDup<MVEVectorVTInfo VTI, PatFrag Op, Intrinsic PredInt,
+                               dag PredOperands, Instruction Inst,
+                               SDPatternOperator IdentityVec = null_frag> {
+  // Unpredicated
+  def : Pat<(VTI.Vec (Op (VTI.Vec MQPR:$Qm), (VTI.Vec (ARMvdup rGPR:$Rn)))),
+            (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), rGPR:$Rn))>;
+
+  // Predicated with select
+  if !ne(VTI.Size, 0b11) then {
+    def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$mask),
+                                (VTI.Vec (Op (VTI.Vec MQPR:$Qm),
+                                             (VTI.Vec (ARMvdup rGPR:$Rn)))),
+                                (VTI.Vec MQPR:$inactive))),
+              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), rGPR:$Rn,
+                              ARMVCCThen, (VTI.Pred VCCR:$mask),
+                              (VTI.Vec MQPR:$inactive)))>;
+
+    // Optionally with the select folded through the op
+    def : Pat<(VTI.Vec (Op (VTI.Vec MQPR:$Qm),
+                           (VTI.Vec (vselect (VTI.Pred VCCR:$mask),
+                                             (ARMvdup rGPR:$Rn),
+                                             (VTI.Vec IdentityVec))))),
+              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), rGPR:$Rn,
+                              ARMVCCThen, (VTI.Pred VCCR:$mask),
+                              (VTI.Vec MQPR:$Qm)))>;
+  }
+
+  // Predicated with intrinsic
+  def : Pat<(VTI.Vec !con((PredInt (VTI.Vec MQPR:$Qm), (VTI.Vec (ARMvdup rGPR:$Rn))),
+                          PredOperands,
+                          (? (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive)))),
+            (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), rGPR:$Rn,
+                            ARMVCCThen, (VTI.Pred VCCR:$mask),
+                            (VTI.Vec MQPR:$inactive)))>;
+}
+
 // --------- Start of base classes for the instructions themselves
 
 class MVE_MI<dag oops, dag iops, InstrItinClass itin, string asm,
@@ -612,8 +684,13 @@ class MVE_VADDV<string iname, string suffix, dag iops, string cstr,
   let validForTailPredication = 1;
 }
 
+def SDTVecReduceP : SDTypeProfile<1, 2, [    // VADDLVp
+  SDTCisInt<0>, SDTCisVec<1>, SDTCisVec<2>
+]>;
 def ARMVADDVs       : SDNode<"ARMISD::VADDVs", SDTVecReduce>;
 def ARMVADDVu       : SDNode<"ARMISD::VADDVu", SDTVecReduce>;
+def ARMVADDVps      : SDNode<"ARMISD::VADDVps", SDTVecReduceP>;
+def ARMVADDVpu      : SDNode<"ARMISD::VADDVpu", SDTVecReduceP>;
 
 multiclass MVE_VADDV_A<MVEVectorVTInfo VTI> {
   def acc    : MVE_VADDV<"vaddva", VTI.Suffix,
@@ -630,20 +707,39 @@ multiclass MVE_VADDV_A<MVEVectorVTInfo VTI> {
     if VTI.Unsigned then {
       def : Pat<(i32 (vecreduce_add (VTI.Vec MQPR:$vec))),
                 (i32 (InstN $vec))>;
+      def : Pat<(i32 (vecreduce_add (VTI.Vec (vselect (VTI.Pred VCCR:$pred),
+                                                      (VTI.Vec MQPR:$vec),
+                                                      (VTI.Vec ARMimmAllZerosV))))),
+                (i32 (InstN $vec, ARMVCCThen, $pred))>;
       def : Pat<(i32 (ARMVADDVu (VTI.Vec MQPR:$vec))),
                 (i32 (InstN $vec))>;
+      def : Pat<(i32 (ARMVADDVpu (VTI.Vec MQPR:$vec), (VTI.Pred VCCR:$pred))),
+                (i32 (InstN $vec, ARMVCCThen, $pred))>;
       def : Pat<(i32 (add (i32 (vecreduce_add (VTI.Vec MQPR:$vec))),
                           (i32 tGPREven:$acc))),
                 (i32 (InstA $acc, $vec))>;
+      def : Pat<(i32 (add (i32 (vecreduce_add (VTI.Vec (vselect (VTI.Pred VCCR:$pred),
+                                                                (VTI.Vec MQPR:$vec),
+                                                                (VTI.Vec ARMimmAllZerosV))))),
+                          (i32 tGPREven:$acc))),
+                (i32 (InstA $acc, $vec, ARMVCCThen, $pred))>;
       def : Pat<(i32 (add (i32 (ARMVADDVu (VTI.Vec MQPR:$vec))),
                           (i32 tGPREven:$acc))),
                 (i32 (InstA $acc, $vec))>;
+      def : Pat<(i32 (add (i32 (ARMVADDVpu (VTI.Vec MQPR:$vec), (VTI.Pred VCCR:$pred))),
+                          (i32 tGPREven:$acc))),
+                (i32 (InstA $acc, $vec, ARMVCCThen, $pred))>;
     } else {
       def : Pat<(i32 (ARMVADDVs (VTI.Vec MQPR:$vec))),
                 (i32 (InstN $vec))>;
       def : Pat<(i32 (add (i32 (ARMVADDVs (VTI.Vec MQPR:$vec))),
                           (i32 tGPREven:$acc))),
                 (i32 (InstA $acc, $vec))>;
+      def : Pat<(i32 (ARMVADDVps (VTI.Vec MQPR:$vec), (VTI.Pred VCCR:$pred))),
+                (i32 (InstN $vec, ARMVCCThen, $pred))>;
+      def : Pat<(i32 (add (i32 (ARMVADDVps (VTI.Vec MQPR:$vec), (VTI.Pred VCCR:$pred))),
+                          (i32 tGPREven:$acc))),
+                (i32 (InstA $acc, $vec, ARMVCCThen, $pred))>;
     }
 
     def : Pat<(i32 (int_arm_mve_addv_predicated (VTI.Vec MQPR:$vec),
@@ -1009,12 +1105,28 @@ def SDTVecReduce2LA : SDTypeProfile<2, 4, [    // VMLALVA
   SDTCisInt<0>, SDTCisInt<1>, SDTCisInt<2>, SDTCisInt<3>,
   SDTCisVec<4>, SDTCisVec<5>
 ]>;
+def SDTVecReduce2P : SDTypeProfile<1, 3, [    // VMLAV
+  SDTCisInt<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisVec<3>
+]>;
+def SDTVecReduce2LP : SDTypeProfile<2, 3, [    // VMLALV
+  SDTCisInt<0>, SDTCisInt<1>, SDTCisVec<2>, SDTCisVec<3>, SDTCisVec<4>
+]>;
+def SDTVecReduce2LAP : SDTypeProfile<2, 5, [    // VMLALVA
+  SDTCisInt<0>, SDTCisInt<1>, SDTCisInt<2>, SDTCisInt<3>,
+  SDTCisVec<4>, SDTCisVec<5>, SDTCisVec<6>
+]>;
 def ARMVMLAVs       : SDNode<"ARMISD::VMLAVs", SDTVecReduce2>;
 def ARMVMLAVu       : SDNode<"ARMISD::VMLAVu", SDTVecReduce2>;
 def ARMVMLALVs      : SDNode<"ARMISD::VMLALVs", SDTVecReduce2L>;
 def ARMVMLALVu      : SDNode<"ARMISD::VMLALVu", SDTVecReduce2L>;
-def ARMVMLALVAs      : SDNode<"ARMISD::VMLALVAs", SDTVecReduce2LA>;
-def ARMVMLALVAu      : SDNode<"ARMISD::VMLALVAu", SDTVecReduce2LA>;
+def ARMVMLALVAs     : SDNode<"ARMISD::VMLALVAs", SDTVecReduce2LA>;
+def ARMVMLALVAu     : SDNode<"ARMISD::VMLALVAu", SDTVecReduce2LA>;
+def ARMVMLAVps      : SDNode<"ARMISD::VMLAVps", SDTVecReduce2P>;
+def ARMVMLAVpu      : SDNode<"ARMISD::VMLAVpu", SDTVecReduce2P>;
+def ARMVMLALVps     : SDNode<"ARMISD::VMLALVps", SDTVecReduce2LP>;
+def ARMVMLALVpu     : SDNode<"ARMISD::VMLALVpu", SDTVecReduce2LP>;
+def ARMVMLALVAps    : SDNode<"ARMISD::VMLALVAps", SDTVecReduce2LAP>;
+def ARMVMLALVApu    : SDNode<"ARMISD::VMLALVApu", SDTVecReduce2LAP>;
 
 let Predicates = [HasMVEInt] in {
   def : Pat<(i32 (vecreduce_add (mul (v4i32 MQPR:$src1), (v4i32 MQPR:$src2)))),
@@ -1033,22 +1145,68 @@ let Predicates = [HasMVEInt] in {
             (i32 (MVE_VMLADAVu8 (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
 
   def : Pat<(i32 (add (i32 (vecreduce_add (mul (v4i32 MQPR:$src1), (v4i32 MQPR:$src2)))),
-                                          (i32 tGPREven:$src3))),
+                      (i32 tGPREven:$src3))),
             (i32 (MVE_VMLADAVau32 $src3, $src1, $src2))>;
   def : Pat<(i32 (add (i32 (vecreduce_add (mul (v8i16 MQPR:$src1), (v8i16 MQPR:$src2)))),
-                                          (i32 tGPREven:$src3))),
+                      (i32 tGPREven:$src3))),
             (i32 (MVE_VMLADAVau16 $src3, $src1, $src2))>;
   def : Pat<(i32 (add (ARMVMLAVs (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)), tGPREven:$Rd)),
             (i32 (MVE_VMLADAVas16 tGPREven:$Rd, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
   def : Pat<(i32 (add (ARMVMLAVu (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)), tGPREven:$Rd)),
             (i32 (MVE_VMLADAVau16 tGPREven:$Rd, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
   def : Pat<(i32 (add (i32 (vecreduce_add (mul (v16i8 MQPR:$src1), (v16i8 MQPR:$src2)))),
-                                          (i32 tGPREven:$src3))),
+                      (i32 tGPREven:$src3))),
             (i32 (MVE_VMLADAVau8 $src3, $src1, $src2))>;
   def : Pat<(i32 (add (ARMVMLAVs (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)), tGPREven:$Rd)),
             (i32 (MVE_VMLADAVas8 tGPREven:$Rd, (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
   def : Pat<(i32 (add (ARMVMLAVu (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)), tGPREven:$Rd)),
             (i32 (MVE_VMLADAVau8 tGPREven:$Rd, (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
+
+  // Predicated
+  def : Pat<(i32 (vecreduce_add (vselect (v4i1 VCCR:$pred),
+                                         (mul (v4i32 MQPR:$src1), (v4i32 MQPR:$src2)),
+                                         (v4i32 ARMimmAllZerosV)))),
+            (i32 (MVE_VMLADAVu32 $src1, $src2, ARMVCCThen, $pred))>;
+  def : Pat<(i32 (vecreduce_add (vselect (v8i1 VCCR:$pred),
+                                         (mul (v8i16 MQPR:$src1), (v8i16 MQPR:$src2)),
+                                         (v8i16 ARMimmAllZerosV)))),
+            (i32 (MVE_VMLADAVu16 $src1, $src2, ARMVCCThen, $pred))>;
+  def : Pat<(i32 (ARMVMLAVps (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred))),
+            (i32 (MVE_VMLADAVs16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred))>;
+  def : Pat<(i32 (ARMVMLAVpu (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred))),
+            (i32 (MVE_VMLADAVu16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred))>;
+  def : Pat<(i32 (vecreduce_add (vselect (v16i1 VCCR:$pred),
+                                         (mul (v16i8 MQPR:$src1), (v16i8 MQPR:$src2)),
+                                         (v16i8 ARMimmAllZerosV)))),
+            (i32 (MVE_VMLADAVu8 $src1, $src2, ARMVCCThen, $pred))>;
+  def : Pat<(i32 (ARMVMLAVps (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), (v16i1 VCCR:$pred))),
+            (i32 (MVE_VMLADAVs8 (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), ARMVCCThen, $pred))>;
+  def : Pat<(i32 (ARMVMLAVpu (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), (v16i1 VCCR:$pred))),
+            (i32 (MVE_VMLADAVu8 (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), ARMVCCThen, $pred))>;
+
+  def : Pat<(i32 (add (i32 (vecreduce_add (vselect (v4i1 VCCR:$pred),
+                                                   (mul (v4i32 MQPR:$src1), (v4i32 MQPR:$src2)),
+                                                   (v4i32 ARMimmAllZerosV)))),
+                      (i32 tGPREven:$src3))),
+            (i32 (MVE_VMLADAVau32 $src3, $src1, $src2, ARMVCCThen, $pred))>;
+  def : Pat<(i32 (add (i32 (vecreduce_add (vselect (v8i1 VCCR:$pred),
+                                                   (mul (v8i16 MQPR:$src1), (v8i16 MQPR:$src2)),
+                                                   (v8i16 ARMimmAllZerosV)))),
+                      (i32 tGPREven:$src3))),
+            (i32 (MVE_VMLADAVau16 $src3, $src1, $src2, ARMVCCThen, $pred))>;
+  def : Pat<(i32 (add (ARMVMLAVps (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred)), tGPREven:$Rd)),
+            (i32 (MVE_VMLADAVas16 tGPREven:$Rd, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred))>;
+  def : Pat<(i32 (add (ARMVMLAVpu (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred)), tGPREven:$Rd)),
+            (i32 (MVE_VMLADAVau16 tGPREven:$Rd, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred))>;
+  def : Pat<(i32 (add (i32 (vecreduce_add (vselect (v16i1 VCCR:$pred),
+                                                   (mul (v16i8 MQPR:$src1), (v16i8 MQPR:$src2)),
+                                                   (v16i8 ARMimmAllZerosV)))),
+                      (i32 tGPREven:$src3))),
+            (i32 (MVE_VMLADAVau8 $src3, $src1, $src2, ARMVCCThen, $pred))>;
+  def : Pat<(i32 (add (ARMVMLAVps (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), (v16i1 VCCR:$pred)), tGPREven:$Rd)),
+            (i32 (MVE_VMLADAVas8 tGPREven:$Rd, (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), ARMVCCThen, $pred))>;
+  def : Pat<(i32 (add (ARMVMLAVpu (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), (v16i1 VCCR:$pred)), tGPREven:$Rd)),
+            (i32 (MVE_VMLADAVau8 tGPREven:$Rd, (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), ARMVCCThen, $pred))>;
 }
 
 // vmlav aliases vmladav
@@ -1168,6 +1326,25 @@ let Predicates = [HasMVEInt] in {
             (MVE_VMLALDAVas16 tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))>;
   def : Pat<(ARMVMLALVAu tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)),
             (MVE_VMLALDAVau16 tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))>;
+
+  // Predicated
+  def : Pat<(ARMVMLALVps (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), (v4i1 VCCR:$pred)),
+            (MVE_VMLALDAVs32 (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), ARMVCCThen, $pred)>;
+  def : Pat<(ARMVMLALVpu (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), (v4i1 VCCR:$pred)),
+            (MVE_VMLALDAVu32 (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), ARMVCCThen, $pred)>;
+  def : Pat<(ARMVMLALVps (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred)),
+            (MVE_VMLALDAVs16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred)>;
+  def : Pat<(ARMVMLALVpu (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred)),
+            (MVE_VMLALDAVu16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred)>;
+
+  def : Pat<(ARMVMLALVAps tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), (v4i1 VCCR:$pred)),
+            (MVE_VMLALDAVas32 tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), ARMVCCThen, $pred)>;
+  def : Pat<(ARMVMLALVApu tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), (v4i1 VCCR:$pred)),
+            (MVE_VMLALDAVau32 tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), ARMVCCThen, $pred)>;
+  def : Pat<(ARMVMLALVAps tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred)),
+            (MVE_VMLALDAVas16 tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred)>;
+  def : Pat<(ARMVMLALVApu tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred)),
+            (MVE_VMLALDAVau16 tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred)>;
 }
 
 // vmlalv aliases vmlaldav
@@ -1215,7 +1392,7 @@ class MVE_comp<InstrItinClass itin, string iname, string suffix,
 }
 
 class MVE_VMINMAXNM<string iname, string suffix, bit sz, bit bit_21,
-                list<dag> pattern=[]>
+                    list<dag> pattern=[]>
   : MVE_comp<NoItinerary, iname, suffix, "", pattern> {
 
   let Inst{28} = 0b1;
@@ -1231,46 +1408,19 @@ class MVE_VMINMAXNM<string iname, string suffix, bit sz, bit bit_21,
   let Predicates = [HasMVEFloat];
 }
 
-def MVE_VMAXNMf32 : MVE_VMINMAXNM<"vmaxnm", "f32", 0b0, 0b0>;
-def MVE_VMAXNMf16 : MVE_VMINMAXNM<"vmaxnm", "f16", 0b1, 0b0>;
-
-let Predicates = [HasMVEFloat] in {
-  def : Pat<(v4f32 (fmaxnum (v4f32 MQPR:$val1), (v4f32 MQPR:$val2))),
-            (v4f32 (MVE_VMAXNMf32 (v4f32 MQPR:$val1), (v4f32 MQPR:$val2)))>;
-  def : Pat<(v8f16 (fmaxnum (v8f16 MQPR:$val1), (v8f16 MQPR:$val2))),
-            (v8f16 (MVE_VMAXNMf16 (v8f16 MQPR:$val1), (v8f16 MQPR:$val2)))>;
-  def : Pat<(v4f32 (int_arm_mve_max_predicated (v4f32 MQPR:$val1), (v4f32 MQPR:$val2), (i32 0),
-                          (v4i1 VCCR:$mask), (v4f32 MQPR:$inactive))),
-            (v4f32 (MVE_VMAXNMf32 (v4f32 MQPR:$val1), (v4f32 MQPR:$val2),
-                          ARMVCCThen, (v4i1 VCCR:$mask),
-                          (v4f32 MQPR:$inactive)))>;
-  def : Pat<(v8f16 (int_arm_mve_max_predicated (v8f16 MQPR:$val1), (v8f16 MQPR:$val2), (i32 0),
-                          (v8i1 VCCR:$mask), (v8f16 MQPR:$inactive))),
-            (v8f16 (MVE_VMAXNMf16 (v8f16 MQPR:$val1), (v8f16 MQPR:$val2),
-                          ARMVCCThen, (v8i1 VCCR:$mask),
-                          (v8f16 MQPR:$inactive)))>;
-}
-
-def MVE_VMINNMf32 : MVE_VMINMAXNM<"vminnm", "f32", 0b0, 0b1>;
-def MVE_VMINNMf16 : MVE_VMINMAXNM<"vminnm", "f16", 0b1, 0b1>;
+multiclass MVE_VMINMAXNM_m<string iname, bit bit_4, MVEVectorVTInfo VTI, SDNode Op, Intrinsic PredInt> {
+  def "" : MVE_VMINMAXNM<iname, VTI.Suffix, VTI.Size{0}, bit_4>;
 
-let Predicates = [HasMVEFloat] in {
-  def : Pat<(v4f32 (fminnum (v4f32 MQPR:$val1), (v4f32 MQPR:$val2))),
-            (v4f32 (MVE_VMINNMf32 (v4f32 MQPR:$val1), (v4f32 MQPR:$val2)))>;
-  def : Pat<(v8f16 (fminnum (v8f16 MQPR:$val1), (v8f16 MQPR:$val2))),
-            (v8f16 (MVE_VMINNMf16 (v8f16 MQPR:$val1), (v8f16 MQPR:$val2)))>;
-  def : Pat<(v4f32 (int_arm_mve_min_predicated (v4f32 MQPR:$val1), (v4f32 MQPR:$val2),
-                          (i32 0), (v4i1 VCCR:$mask), (v4f32 MQPR:$inactive))),
-            (v4f32 (MVE_VMINNMf32 (v4f32 MQPR:$val1), (v4f32 MQPR:$val2),
-                          ARMVCCThen, (v4i1 VCCR:$mask),
-                          (v4f32 MQPR:$inactive)))>;
-  def : Pat<(v8f16 (int_arm_mve_min_predicated (v8f16 MQPR:$val1), (v8f16 MQPR:$val2),
-                          (i32 0), (v8i1 VCCR:$mask), (v8f16 MQPR:$inactive))),
-            (v8f16 (MVE_VMINNMf16 (v8f16 MQPR:$val1), (v8f16 MQPR:$val2),
-                          ARMVCCThen, (v8i1 VCCR:$mask),
-                          (v8f16 MQPR:$inactive)))>;
+  let Predicates = [HasMVEFloat] in {
+    defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 0)), !cast<Instruction>(NAME)>;
+  }
 }
 
+defm MVE_VMAXNMf32 : MVE_VMINMAXNM_m<"vmaxnm", 0b0, MVE_v4f32, fmaxnum, int_arm_mve_max_predicated>;
+defm MVE_VMAXNMf16 : MVE_VMINMAXNM_m<"vmaxnm", 0b0, MVE_v8f16, fmaxnum, int_arm_mve_max_predicated>;
+defm MVE_VMINNMf32 : MVE_VMINMAXNM_m<"vminnm", 0b1, MVE_v4f32, fminnum, int_arm_mve_min_predicated>;
+defm MVE_VMINNMf16 : MVE_VMINMAXNM_m<"vminnm", 0b1, MVE_v8f16, fminnum, int_arm_mve_min_predicated>;
+
 
 class MVE_VMINMAX<string iname, string suffix, bit U, bits<2> size,
               bit bit_4, list<dag> pattern=[]>
@@ -1288,22 +1438,11 @@ class MVE_VMINMAX<string iname, string suffix, bit U, bits<2> size,
 }
 
 multiclass MVE_VMINMAX_m<string iname, bit bit_4, MVEVectorVTInfo VTI,
-                      SDNode unpred_op, Intrinsic pred_int> {
+                      SDNode Op, Intrinsic PredInt> {
   def "" : MVE_VMINMAX<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, bit_4>;
-  defvar Inst = !cast<Instruction>(NAME);
 
   let Predicates = [HasMVEInt] in {
-    // Unpredicated min/max
-    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),
-              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
-
-    // Predicated min/max
-    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
-                            (i32 VTI.Unsigned), (VTI.Pred VCCR:$mask),
-                            (VTI.Vec MQPR:$inactive))),
-              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
-                             ARMVCCThen, (VTI.Pred VCCR:$mask),
-                             (VTI.Vec MQPR:$inactive)))>;
+    defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)), !cast<Instruction>(NAME)>;
   }
 }
 
@@ -1476,61 +1615,41 @@ foreach s=["s8", "s16", "s32", "u8", "u16", "u32", "i8", "i16", "i32", "f16", "f
         (MVE_VAND MQPR:$QdSrc, MQPR:$QnSrc, MQPR:$QmSrc, vpred_r:$vp)>;
 }
 
-multiclass MVE_bit_op<MVEVectorVTInfo VTI, SDNode unpred_op, Intrinsic pred_int, MVE_bit_ops instruction> {
-  let Predicates = [HasMVEInt] in {
-    // Unpredicated operation
-    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),
-              (VTI.Vec (instruction (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
-    // Predicated operation
-    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
-                            (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive))),
-              (VTI.Vec (instruction
-                            (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
-                            ARMVCCThen, (VTI.Pred VCCR:$mask),
-                            (VTI.Vec MQPR:$inactive)))>;
-  }
-}
-
-defm : MVE_bit_op<MVE_v16i8, and, int_arm_mve_and_predicated, MVE_VAND>;
-defm : MVE_bit_op<MVE_v8i16, and, int_arm_mve_and_predicated, MVE_VAND>;
-defm : MVE_bit_op<MVE_v4i32, and, int_arm_mve_and_predicated, MVE_VAND>;
-defm : MVE_bit_op<MVE_v2i64, and, int_arm_mve_and_predicated, MVE_VAND>;
-
-defm : MVE_bit_op<MVE_v16i8, or, int_arm_mve_orr_predicated, MVE_VORR>;
-defm : MVE_bit_op<MVE_v8i16, or, int_arm_mve_orr_predicated, MVE_VORR>;
-defm : MVE_bit_op<MVE_v4i32, or, int_arm_mve_orr_predicated, MVE_VORR>;
-defm : MVE_bit_op<MVE_v2i64, or, int_arm_mve_orr_predicated, MVE_VORR>;
-
-defm : MVE_bit_op<MVE_v16i8, xor, int_arm_mve_eor_predicated, MVE_VEOR>;
-defm : MVE_bit_op<MVE_v8i16, xor, int_arm_mve_eor_predicated, MVE_VEOR>;
-defm : MVE_bit_op<MVE_v4i32, xor, int_arm_mve_eor_predicated, MVE_VEOR>;
-defm : MVE_bit_op<MVE_v2i64, xor, int_arm_mve_eor_predicated, MVE_VEOR>;
-
-multiclass MVE_bit_op_with_inv<MVEVectorVTInfo VTI, SDNode unpred_op, Intrinsic pred_int, MVE_bit_ops instruction> {
-  let Predicates = [HasMVEInt] in {
-    // Unpredicated operation
-    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (vnotq (VTI.Vec MQPR:$Qn)))),
-              (VTI.Vec (instruction (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
-    // Predicated operation
-    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
-                            (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive))),
-              (VTI.Vec (instruction
-                            (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
-                            ARMVCCThen, (VTI.Pred VCCR:$mask),
-                            (VTI.Vec MQPR:$inactive)))>;
-  }
+let Predicates = [HasMVEInt] in {
+  defm : MVE_TwoOpPattern<MVE_v16i8, and, int_arm_mve_and_predicated, (? ), MVE_VAND, ARMimmAllOnesV>;
+  defm : MVE_TwoOpPattern<MVE_v8i16, and, int_arm_mve_and_predicated, (? ), MVE_VAND, ARMimmAllOnesV>;
+  defm : MVE_TwoOpPattern<MVE_v4i32, and, int_arm_mve_and_predicated, (? ), MVE_VAND, ARMimmAllOnesV>;
+  defm : MVE_TwoOpPattern<MVE_v2i64, and, int_arm_mve_and_predicated, (? ), MVE_VAND, ARMimmAllOnesV>;
+
+  defm : MVE_TwoOpPattern<MVE_v16i8, or, int_arm_mve_orr_predicated, (? ), MVE_VORR, ARMimmAllZerosV>;
+  defm : MVE_TwoOpPattern<MVE_v8i16, or, int_arm_mve_orr_predicated, (? ), MVE_VORR, ARMimmAllZerosV>;
+  defm : MVE_TwoOpPattern<MVE_v4i32, or, int_arm_mve_orr_predicated, (? ), MVE_VORR, ARMimmAllZerosV>;
+  defm : MVE_TwoOpPattern<MVE_v2i64, or, int_arm_mve_orr_predicated, (? ), MVE_VORR, ARMimmAllZerosV>;
+
+  defm : MVE_TwoOpPattern<MVE_v16i8, xor, int_arm_mve_eor_predicated, (? ), MVE_VEOR, ARMimmAllZerosV>;
+  defm : MVE_TwoOpPattern<MVE_v8i16, xor, int_arm_mve_eor_predicated, (? ), MVE_VEOR, ARMimmAllZerosV>;
+  defm : MVE_TwoOpPattern<MVE_v4i32, xor, int_arm_mve_eor_predicated, (? ), MVE_VEOR, ARMimmAllZerosV>;
+  defm : MVE_TwoOpPattern<MVE_v2i64, xor, int_arm_mve_eor_predicated, (? ), MVE_VEOR, ARMimmAllZerosV>;
+
+  defm : MVE_TwoOpPattern<MVE_v16i8, BinOpFrag<(and node:$LHS, (vnotq node:$RHS))>,
+                          int_arm_mve_bic_predicated, (? ), MVE_VBIC>;
+  defm : MVE_TwoOpPattern<MVE_v8i16, BinOpFrag<(and node:$LHS, (vnotq node:$RHS))>,
+                          int_arm_mve_bic_predicated, (? ), MVE_VBIC>;
+  defm : MVE_TwoOpPattern<MVE_v4i32, BinOpFrag<(and node:$LHS, (vnotq node:$RHS))>,
+                          int_arm_mve_bic_predicated, (? ), MVE_VBIC>;
+  defm : MVE_TwoOpPattern<MVE_v2i64, BinOpFrag<(and node:$LHS, (vnotq node:$RHS))>,
+                          int_arm_mve_bic_predicated, (? ), MVE_VBIC>;
+
+  defm : MVE_TwoOpPattern<MVE_v16i8, BinOpFrag<(or node:$LHS, (vnotq node:$RHS))>,
+                          int_arm_mve_orn_predicated, (? ), MVE_VORN>;
+  defm : MVE_TwoOpPattern<MVE_v8i16, BinOpFrag<(or node:$LHS, (vnotq node:$RHS))>,
+                          int_arm_mve_orn_predicated, (? ), MVE_VORN>;
+  defm : MVE_TwoOpPattern<MVE_v4i32, BinOpFrag<(or node:$LHS, (vnotq node:$RHS))>,
+                          int_arm_mve_orn_predicated, (? ), MVE_VORN>;
+  defm : MVE_TwoOpPattern<MVE_v2i64, BinOpFrag<(or node:$LHS, (vnotq node:$RHS))>,
+                          int_arm_mve_orn_predicated, (? ), MVE_VORN>;
 }
 
-defm : MVE_bit_op_with_inv<MVE_v16i8, and, int_arm_mve_bic_predicated, MVE_VBIC>;
-defm : MVE_bit_op_with_inv<MVE_v8i16, and, int_arm_mve_bic_predicated, MVE_VBIC>;
-defm : MVE_bit_op_with_inv<MVE_v4i32, and, int_arm_mve_bic_predicated, MVE_VBIC>;
-defm : MVE_bit_op_with_inv<MVE_v2i64, and, int_arm_mve_bic_predicated, MVE_VBIC>;
-
-defm : MVE_bit_op_with_inv<MVE_v16i8, or, int_arm_mve_orn_predicated, MVE_VORN>;
-defm : MVE_bit_op_with_inv<MVE_v8i16, or, int_arm_mve_orn_predicated, MVE_VORN>;
-defm : MVE_bit_op_with_inv<MVE_v4i32, or, int_arm_mve_orn_predicated, MVE_VORN>;
-defm : MVE_bit_op_with_inv<MVE_v2i64, or, int_arm_mve_orn_predicated, MVE_VORN>;
-
 class MVE_bit_cmode<string iname, string suffix, bit halfword, dag inOps>
   : MVE_p<(outs MQPR:$Qd), inOps, NoItinerary,
           iname, suffix, "$Qd, $imm", vpred_n, "$Qd = $Qd_src"> {
@@ -1775,31 +1894,18 @@ class MVE_VMULt1<string iname, string suffix, bits<2> size,
   let validForTailPredication = 1;
 }
 
-multiclass MVE_VMUL_m<string iname, MVEVectorVTInfo VTI,
-                      SDNode unpred_op, Intrinsic pred_int> {
-  def "" : MVE_VMULt1<iname, VTI.Suffix, VTI.Size>;
-  defvar Inst = !cast<Instruction>(NAME);
+multiclass MVE_VMUL_m<MVEVectorVTInfo VTI> {
+  def "" : MVE_VMULt1<"vmul", VTI.Suffix, VTI.Size>;
 
   let Predicates = [HasMVEInt] in {
-    // Unpredicated multiply
-    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),
-              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
-
-    // Predicated multiply
-    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
-                            (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive))),
-              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
-                             ARMVCCThen, (VTI.Pred VCCR:$mask),
-                             (VTI.Vec MQPR:$inactive)))>;
+    defm : MVE_TwoOpPattern<VTI, mul, int_arm_mve_mul_predicated, (? ),
+                            !cast<Instruction>(NAME), ARMimmOneV>;
   }
 }
 
-multiclass MVE_VMUL<MVEVectorVTInfo VTI>
-  : MVE_VMUL_m<"vmul", VTI, mul, int_arm_mve_mul_predicated>;
-
-defm MVE_VMULi8  : MVE_VMUL<MVE_v16i8>;
-defm MVE_VMULi16 : MVE_VMUL<MVE_v8i16>;
-defm MVE_VMULi32 : MVE_VMUL<MVE_v4i32>;
+defm MVE_VMULi8  : MVE_VMUL_m<MVE_v16i8>;
+defm MVE_VMULi16 : MVE_VMUL_m<MVE_v8i16>;
+defm MVE_VMULi32 : MVE_VMUL_m<MVE_v4i32>;
 
 class MVE_VQxDMULH_Base<string iname, string suffix, bits<2> size, bit rounding,
                   list<dag> pattern=[]>
@@ -1828,8 +1934,8 @@ multiclass MVE_VQxDMULH_m<string iname, MVEVectorVTInfo VTI,
     def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
                             (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive))),
               (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
-                             ARMVCCThen, (VTI.Pred VCCR:$mask),
-                             (VTI.Vec MQPR:$inactive)))>;
+                            ARMVCCThen, (VTI.Pred VCCR:$mask),
+                            (VTI.Vec MQPR:$inactive)))>;
   }
 }
 
@@ -1862,21 +1968,12 @@ class MVE_VADDSUB<string iname, string suffix, bits<2> size, bit subtract,
 }
 
 multiclass MVE_VADDSUB_m<string iname, MVEVectorVTInfo VTI, bit subtract,
-                         SDNode unpred_op, Intrinsic pred_int> {
+                         SDNode Op, Intrinsic PredInt> {
   def "" : MVE_VADDSUB<iname, VTI.Suffix, VTI.Size, subtract>;
   defvar Inst = !cast<Instruction>(NAME);
 
   let Predicates = [HasMVEInt] in {
-    // Unpredicated add/subtract
-    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),
-              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
-
-    // Predicated add/subtract
-    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
-                            (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive))),
-              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
-                             ARMVCCThen, (VTI.Pred VCCR:$mask),
-                             (VTI.Vec MQPR:$inactive)))>;
+    defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? ), !cast<Instruction>(NAME), ARMimmAllZerosV>;
   }
 }
 
@@ -1914,22 +2011,13 @@ class MVE_VQSUB_<string suffix, bit U, bits<2> size>
   : MVE_VQADDSUB<"vqsub", suffix, U, 0b1, size>;
 
 multiclass MVE_VQADD_m<MVEVectorVTInfo VTI,
-                      SDNode unpred_op, Intrinsic pred_int> {
+                      SDNode Op, Intrinsic PredInt> {
   def "" : MVE_VQADD_<VTI.Suffix, VTI.Unsigned, VTI.Size>;
   defvar Inst = !cast<Instruction>(NAME);
 
   let Predicates = [HasMVEInt] in {
-    // Unpredicated saturating add
-    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),
-              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
-
-    // Predicated saturating add
-    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
-                            (i32 VTI.Unsigned), (VTI.Pred VCCR:$mask),
-                            (VTI.Vec MQPR:$inactive))),
-              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
-                             ARMVCCThen, (VTI.Pred VCCR:$mask),
-                             (VTI.Vec MQPR:$inactive)))>;
+    defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)),
+                            !cast<Instruction>(NAME)>;
   }
 }
 
@@ -1944,22 +2032,13 @@ defm MVE_VQADDu16 : MVE_VQADD<MVE_v8u16, uaddsat>;
 defm MVE_VQADDu32 : MVE_VQADD<MVE_v4u32, uaddsat>;
 
 multiclass MVE_VQSUB_m<MVEVectorVTInfo VTI,
-                      SDNode unpred_op, Intrinsic pred_int> {
+                      SDNode Op, Intrinsic PredInt> {
   def "" : MVE_VQSUB_<VTI.Suffix, VTI.Unsigned, VTI.Size>;
   defvar Inst = !cast<Instruction>(NAME);
 
   let Predicates = [HasMVEInt] in {
-    // Unpredicated saturating subtract
-    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),
-              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
-
-    // Predicated saturating subtract
-    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
-                            (i32 VTI.Unsigned), (VTI.Pred VCCR:$mask),
-                            (VTI.Vec MQPR:$inactive))),
-              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
-                             ARMVCCThen, (VTI.Pred VCCR:$mask),
-                             (VTI.Vec MQPR:$inactive)))>;
+    defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)),
+                            !cast<Instruction>(NAME)>;
   }
 }
 
@@ -3469,18 +3548,12 @@ class MVE_VMUL_fp<string iname, string suffix, bit size, list<dag> pattern=[]>
 }
 
 multiclass MVE_VMULT_fp_m<string iname, bit bit_21, MVEVectorVTInfo VTI,
-                            SDNode unpred_op, Intrinsic pred_int> {
+                            SDNode Op, Intrinsic PredInt> {
   def "" : MVE_VMUL_fp<iname, VTI.Suffix, VTI.Size{0}>;
   defvar Inst = !cast<Instruction>(NAME);
 
   let Predicates = [HasMVEFloat] in {
-    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),
-              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
-    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
-                            (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive))),
-              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
-                             ARMVCCThen, (VTI.Pred VCCR:$mask),
-                             (VTI.Vec MQPR:$inactive)))>;
+    defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? ), !cast<Instruction>(NAME)>;
   }
 }
 
@@ -3591,20 +3664,14 @@ defm MVE_VFMSf32 : MVE_VFMA_fp_multi<"vfms", 1, MVE_v4f32>;
 defm MVE_VFMSf16 : MVE_VFMA_fp_multi<"vfms", 1, MVE_v8f16>;
 
 multiclass MVE_VADDSUB_fp_m<string iname, bit bit_21, MVEVectorVTInfo VTI,
-                            SDNode unpred_op, Intrinsic pred_int> {
+                            SDNode Op, Intrinsic PredInt> {
   def "" : MVE_VADDSUBFMA_fp<iname, VTI.Suffix, VTI.Size{0}, 0, 1, bit_21> {
     let validForTailPredication = 1;
   }
   defvar Inst = !cast<Instruction>(NAME);
 
   let Predicates = [HasMVEFloat] in {
-    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),
-              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
-    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
-                            (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive))),
-              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
-                             ARMVCCThen, (VTI.Pred VCCR:$mask),
-                             (VTI.Vec MQPR:$inactive)))>;
+    defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? ), !cast<Instruction>(NAME)>;
   }
 }
 
@@ -5023,32 +5090,6 @@ multiclass MVE_vec_scalar_int_pat_m<Instruction inst, MVEVectorVTInfo VTI,
   }
 }
 
-// Patterns for vector-scalar instructions with FP operands
-multiclass MVE_vec_scalar_fp_pat_m<SDNode unpred_op, Intrinsic pred_int,
-                                   Instruction instr_f16,
-                                   Instruction instr_f32> {
-  let Predicates = [HasMVEFloat] in {
-    // Unpredicated F16
-    def : Pat<(v8f16 (unpred_op (v8f16 MQPR:$Qm), (v8f16 (ARMvdup rGPR:$val)))),
-              (v8f16 (instr_f16 (v8f16 MQPR:$Qm), (i32 rGPR:$val)))>;
-    // Unpredicated F32
-    def : Pat<(v4f32 (unpred_op (v4f32 MQPR:$Qm), (v4f32 (ARMvdup rGPR:$val)))),
-              (v4f32 (instr_f32 (v4f32 MQPR:$Qm), (i32 rGPR:$val)))>;
-    // Predicated F16
-    def : Pat<(v8f16 (pred_int (v8f16 MQPR:$Qm), (v8f16 (ARMvdup rGPR:$val)),
-                               (v8i1 VCCR:$mask), (v8f16 MQPR:$inactive))),
-              (v8f16 (instr_f16 (v8f16 MQPR:$Qm), (i32 rGPR:$val),
-                                ARMVCCThen, (v8i1 VCCR:$mask),
-                                (v8f16 MQPR:$inactive)))>;
-    // Predicated F32
-    def : Pat<(v4f32 (pred_int (v4f32 MQPR:$Qm), (v4f32 (ARMvdup rGPR:$val)),
-                               (v4i1 VCCR:$mask), (v4f32 MQPR:$inactive))),
-              (v4f32 (instr_f32 (v4f32 MQPR:$Qm), (i32 rGPR:$val),
-                                ARMVCCThen, (v4i1 VCCR:$mask),
-                                (v4f32 MQPR:$inactive)))>;
-  }
-}
-
 class MVE_VADDSUB_qr<string iname, string suffix, bits<2> size,
                      bit bit_5, bit bit_12, bit bit_16, bit bit_28>
   : MVE_qDest_rSrc<iname, suffix, ""> {
@@ -5064,10 +5105,11 @@ class MVE_VADDSUB_qr<string iname, string suffix, bits<2> size,
 
 // Vector-scalar add/sub
 multiclass MVE_VADDSUB_qr_m<string iname, MVEVectorVTInfo VTI, bit subtract,
-                            SDNode unpred_op, Intrinsic pred_int> {
+                            SDNode Op, Intrinsic PredInt> {
   def "" : MVE_VADDSUB_qr<iname, VTI.Suffix, VTI.Size, 0b0, subtract, 0b1, 0b0>;
-  defm : MVE_vec_scalar_int_pat_m<!cast<Instruction>(NAME), VTI,
-                                  unpred_op, pred_int>;
+  let Predicates = [HasMVEInt] in {
+    defm : MVE_TwoOpPatternDup<VTI, Op, PredInt, (? ), !cast<Instruction>(NAME), ARMimmAllZerosV>;
+  }
 }
 
 multiclass MVE_VADD_qr_m<MVEVectorVTInfo VTI>
@@ -5086,36 +5128,35 @@ defm MVE_VSUB_qr_i32 : MVE_VSUB_qr_m<MVE_v4i32>;
 
 // Vector-scalar saturating add/sub
 multiclass MVE_VQADDSUB_qr_m<string iname, MVEVectorVTInfo VTI, bit subtract,
-                             SDNode unpred_op_s, SDNode unpred_op_u,
-                             Intrinsic pred_int> {
+                             SDNode Op, Intrinsic PredInt> {
   def "" : MVE_VADDSUB_qr<iname, VTI.Suffix, VTI.Size, 0b1, subtract,
                           0b0, VTI.Unsigned>;
-  defvar unpred_op = !if(VTI.Unsigned, unpred_op_u, unpred_op_s);
-  defm : MVE_vec_scalar_int_pat_m<!cast<Instruction>(NAME), VTI,
-                                  unpred_op, pred_int, 0, 1>;
+
+  let Predicates = [HasMVEInt] in {
+    defm : MVE_TwoOpPatternDup<VTI, Op, PredInt, (? (i32 VTI.Unsigned)),
+                               !cast<Instruction>(NAME)>;
+  }
 }
 
-multiclass MVE_VQADD_qr_m<MVEVectorVTInfo VTI>
-  : MVE_VQADDSUB_qr_m<"vqadd", VTI, 0b0, saddsat, uaddsat,
-                     int_arm_mve_qadd_predicated>;
+multiclass MVE_VQADD_qr_m<MVEVectorVTInfo VTI, SDNode Op>
+  : MVE_VQADDSUB_qr_m<"vqadd", VTI, 0b0, Op, int_arm_mve_qadd_predicated>;
 
-multiclass MVE_VQSUB_qr_m<MVEVectorVTInfo VTI>
-  : MVE_VQADDSUB_qr_m<"vqsub", VTI, 0b1, ssubsat, usubsat,
-                     int_arm_mve_qsub_predicated>;
+multiclass MVE_VQSUB_qr_m<MVEVectorVTInfo VTI, SDNode Op>
+  : MVE_VQADDSUB_qr_m<"vqsub", VTI, 0b1, Op, int_arm_mve_qsub_predicated>;
 
-defm MVE_VQADD_qr_s8  : MVE_VQADD_qr_m<MVE_v16s8>;
-defm MVE_VQADD_qr_s16 : MVE_VQADD_qr_m<MVE_v8s16>;
-defm MVE_VQADD_qr_s32 : MVE_VQADD_qr_m<MVE_v4s32>;
-defm MVE_VQADD_qr_u8  : MVE_VQADD_qr_m<MVE_v16u8>;
-defm MVE_VQADD_qr_u16 : MVE_VQADD_qr_m<MVE_v8u16>;
-defm MVE_VQADD_qr_u32 : MVE_VQADD_qr_m<MVE_v4u32>;
+defm MVE_VQADD_qr_s8  : MVE_VQADD_qr_m<MVE_v16s8, saddsat>;
+defm MVE_VQADD_qr_s16 : MVE_VQADD_qr_m<MVE_v8s16, saddsat>;
+defm MVE_VQADD_qr_s32 : MVE_VQADD_qr_m<MVE_v4s32, saddsat>;
+defm MVE_VQADD_qr_u8  : MVE_VQADD_qr_m<MVE_v16u8, uaddsat>;
+defm MVE_VQADD_qr_u16 : MVE_VQADD_qr_m<MVE_v8u16, uaddsat>;
+defm MVE_VQADD_qr_u32 : MVE_VQADD_qr_m<MVE_v4u32, uaddsat>;
 
-defm MVE_VQSUB_qr_s8  : MVE_VQSUB_qr_m<MVE_v16s8>;
-defm MVE_VQSUB_qr_s16 : MVE_VQSUB_qr_m<MVE_v8s16>;
-defm MVE_VQSUB_qr_s32 : MVE_VQSUB_qr_m<MVE_v4s32>;
-defm MVE_VQSUB_qr_u8  : MVE_VQSUB_qr_m<MVE_v16u8>;
-defm MVE_VQSUB_qr_u16 : MVE_VQSUB_qr_m<MVE_v8u16>;
-defm MVE_VQSUB_qr_u32 : MVE_VQSUB_qr_m<MVE_v4u32>;
+defm MVE_VQSUB_qr_s8  : MVE_VQSUB_qr_m<MVE_v16s8, ssubsat>;
+defm MVE_VQSUB_qr_s16 : MVE_VQSUB_qr_m<MVE_v8s16, ssubsat>;
+defm MVE_VQSUB_qr_s32 : MVE_VQSUB_qr_m<MVE_v4s32, ssubsat>;
+defm MVE_VQSUB_qr_u8  : MVE_VQSUB_qr_m<MVE_v16u8, usubsat>;
+defm MVE_VQSUB_qr_u16 : MVE_VQSUB_qr_m<MVE_v8u16, usubsat>;
+defm MVE_VQSUB_qr_u32 : MVE_VQSUB_qr_m<MVE_v4u32, usubsat>;
 
 class MVE_VQDMULL_qr<string iname, string suffix, bit size,
                      bit T, string cstr="", list<dag> pattern=[]>
@@ -5206,19 +5247,25 @@ defm MVE_VHSUB_qr_u8  : MVE_VHSUB_qr_m<MVE_v16u8>;
 defm MVE_VHSUB_qr_u16 : MVE_VHSUB_qr_m<MVE_v8u16>;
 defm MVE_VHSUB_qr_u32 : MVE_VHSUB_qr_m<MVE_v4u32>;
 
+multiclass MVE_VADDSUB_qr_f<string iname, MVEVectorVTInfo VTI, bit subtract,
+                            SDNode Op, Intrinsic PredInt> {
+  def "" : MVE_VxADDSUB_qr<iname, VTI.Suffix, VTI.Size{0}, 0b11, subtract>;
+  defm : MVE_TwoOpPatternDup<VTI, Op, PredInt, (? ),
+                              !cast<Instruction>(NAME)>;
+}
+
 let Predicates = [HasMVEFloat] in {
-  def MVE_VADD_qr_f32 : MVE_VxADDSUB_qr<"vadd",  "f32", 0b0, 0b11, 0b0>;
-  def MVE_VADD_qr_f16 : MVE_VxADDSUB_qr<"vadd",  "f16", 0b1, 0b11, 0b0>;
+  defm MVE_VADD_qr_f32 : MVE_VADDSUB_qr_f<"vadd", MVE_v4f32, 0b0, fadd,
+                                          int_arm_mve_add_predicated>;
+  defm MVE_VADD_qr_f16 : MVE_VADDSUB_qr_f<"vadd", MVE_v8f16, 0b0, fadd,
+                                          int_arm_mve_add_predicated>;
 
-  def MVE_VSUB_qr_f32 : MVE_VxADDSUB_qr<"vsub",  "f32", 0b0, 0b11, 0b1>;
-  def MVE_VSUB_qr_f16 : MVE_VxADDSUB_qr<"vsub",  "f16", 0b1, 0b11, 0b1>;
+  defm MVE_VSUB_qr_f32 : MVE_VADDSUB_qr_f<"vsub", MVE_v4f32, 0b1, fsub,
+                                          int_arm_mve_sub_predicated>;
+  defm MVE_VSUB_qr_f16 : MVE_VADDSUB_qr_f<"vsub", MVE_v8f16, 0b1, fsub,
+                                          int_arm_mve_sub_predicated>;
 }
 
-defm : MVE_vec_scalar_fp_pat_m<fadd, int_arm_mve_add_predicated,
-                               MVE_VADD_qr_f16, MVE_VADD_qr_f32>;
-defm : MVE_vec_scalar_fp_pat_m<fsub, int_arm_mve_sub_predicated,
-                               MVE_VSUB_qr_f16, MVE_VSUB_qr_f32>;
-
 class MVE_VxSHL_qr<string iname, string suffix, bit U, bits<2> size,
                    bit bit_7, bit bit_17, list<dag> pattern=[]>
   : MVE_qDest_single_rSrc<iname, suffix, pattern> {
@@ -5346,8 +5393,10 @@ class MVE_VMUL_qr_int<string iname, string suffix, bits<2> size>
 
 multiclass MVE_VMUL_qr_int_m<MVEVectorVTInfo VTI> {
   def "" : MVE_VMUL_qr_int<"vmul", VTI.Suffix, VTI.Size>;
-  defm : MVE_vec_scalar_int_pat_m<!cast<Instruction>(NAME), VTI,
-                                  mul, int_arm_mve_mul_predicated>;
+  let Predicates = [HasMVEInt] in {
+    defm : MVE_TwoOpPatternDup<VTI, mul, int_arm_mve_mul_predicated, (? ),
+                               !cast<Instruction>(NAME), ARMimmOneV>;
+  }
 }
 
 defm MVE_VMUL_qr_i8  : MVE_VMUL_qr_int_m<MVE_v16i8>;
@@ -5389,13 +5438,17 @@ defm MVE_VQRDMULH_qr_s8   : MVE_VQRDMULH_qr_m<MVE_v16s8>;
 defm MVE_VQRDMULH_qr_s16  : MVE_VQRDMULH_qr_m<MVE_v8s16>;
 defm MVE_VQRDMULH_qr_s32  : MVE_VQRDMULH_qr_m<MVE_v4s32>;
 
-let Predicates = [HasMVEFloat], validForTailPredication = 1 in {
-  def MVE_VMUL_qr_f16   : MVE_VxxMUL_qr<"vmul", "f16", 0b1, 0b11>;
-  def MVE_VMUL_qr_f32   : MVE_VxxMUL_qr<"vmul", "f32", 0b0, 0b11>;
+multiclass MVE_VxxMUL_qr_f_m<MVEVectorVTInfo VTI> {
+  let validForTailPredication = 1 in
+  def "" : MVE_VxxMUL_qr<"vmul", VTI.Suffix, VTI.Size{0}, 0b11>;
+  defm : MVE_TwoOpPatternDup<VTI, fmul, int_arm_mve_mul_predicated, (? ),
+                             !cast<Instruction>(NAME)>;
 }
 
-defm : MVE_vec_scalar_fp_pat_m<fmul, int_arm_mve_mul_predicated,
-                               MVE_VMUL_qr_f16, MVE_VMUL_qr_f32>;
+let Predicates = [HasMVEFloat] in {
+  defm MVE_VMUL_qr_f16   : MVE_VxxMUL_qr_f_m<MVE_v8f16>;
+  defm MVE_VMUL_qr_f32   : MVE_VxxMUL_qr_f_m<MVE_v4f32>;
+}
 
 class MVE_VFMAMLA_qr<string iname, string suffix,
                      bit bit_28, bits<2> bits_21_20, bit S,
@@ -6937,7 +6990,7 @@ class MVE_vector_load_typed<ValueType Ty, Instruction RegImmInst,
 
 class MVE_vector_maskedload_typed<ValueType Ty, Instruction RegImmInst,
                                   PatFrag LoadKind, int shift>
-  : Pat<(Ty (LoadKind t2addrmode_imm7<shift>:$addr, VCCR:$pred, (Ty NEONimmAllZerosV))),
+  : Pat<(Ty (LoadKind t2addrmode_imm7<shift>:$addr, VCCR:$pred, (Ty (ARMvmovImm (i32 0))))),
         (Ty (RegImmInst t2addrmode_imm7<shift>:$addr, ARMVCCThen, VCCR:$pred))>;
 
 multiclass MVE_vector_load<Instruction RegImmInst, PatFrag LoadKind,
@@ -7104,11 +7157,11 @@ multiclass MVEExtLoadStore<Instruction LoadSInst, Instruction LoadUInst, string
             (VT (LoadUInst taddrmode_imm7<Shift>:$addr))>;
 
   // Masked ext loads
-  def : Pat<(VT (!cast<PatFrag>("aligned_extmaskedload"#Amble) taddrmode_imm7<Shift>:$addr, VCCR:$pred, (VT NEONimmAllZerosV))),
+  def : Pat<(VT (!cast<PatFrag>("aligned_extmaskedload"#Amble) taddrmode_imm7<Shift>:$addr, VCCR:$pred, (VT (ARMvmovImm (i32 0))))),
             (VT (LoadUInst taddrmode_imm7<Shift>:$addr, ARMVCCThen, VCCR:$pred))>;
-  def : Pat<(VT (!cast<PatFrag>("aligned_sextmaskedload"#Amble) taddrmode_imm7<Shift>:$addr, VCCR:$pred, (VT NEONimmAllZerosV))),
+  def : Pat<(VT (!cast<PatFrag>("aligned_sextmaskedload"#Amble) taddrmode_imm7<Shift>:$addr, VCCR:$pred, (VT (ARMvmovImm (i32 0))))),
             (VT (LoadSInst taddrmode_imm7<Shift>:$addr, ARMVCCThen, VCCR:$pred))>;
-  def : Pat<(VT (!cast<PatFrag>("aligned_zextmaskedload"#Amble) taddrmode_imm7<Shift>:$addr, VCCR:$pred, (VT NEONimmAllZerosV))),
+  def : Pat<(VT (!cast<PatFrag>("aligned_zextmaskedload"#Amble) taddrmode_imm7<Shift>:$addr, VCCR:$pred, (VT (ARMvmovImm (i32 0))))),
             (VT (LoadUInst taddrmode_imm7<Shift>:$addr, ARMVCCThen, VCCR:$pred))>;
 }
 
diff --git a/llvm/lib/Target/ARM/ARMInstrNEON.td b/llvm/lib/Target/ARM/ARMInstrNEON.td
index c097a4ad4facf..b05a38803553d 100644
--- a/llvm/lib/Target/ARM/ARMInstrNEON.td
+++ b/llvm/lib/Target/ARM/ARMInstrNEON.td
@@ -534,20 +534,6 @@ def NEONvtbl1     : SDNode<"ARMISD::VTBL1", SDTARMVTBL1>;
 def NEONvtbl2     : SDNode<"ARMISD::VTBL2", SDTARMVTBL2>;
 
 
-def NEONimmAllZerosV: PatLeaf<(ARMvmovImm (i32 timm)), [{
-  ConstantSDNode *ConstVal = cast<ConstantSDNode>(N->getOperand(0));
-  unsigned EltBits = 0;
-  uint64_t EltVal = ARM_AM::decodeVMOVModImm(ConstVal->getZExtValue(), EltBits);
-  return (EltBits == 32 && EltVal == 0);
-}]>;
-
-def NEONimmAllOnesV: PatLeaf<(ARMvmovImm (i32 timm)), [{
-  ConstantSDNode *ConstVal = cast<ConstantSDNode>(N->getOperand(0));
-  unsigned EltBits = 0;
-  uint64_t EltVal = ARM_AM::decodeVMOVModImm(ConstVal->getZExtValue(), EltBits);
-  return (EltBits == 8 && EltVal == 0xff);
-}]>;
-
 //===----------------------------------------------------------------------===//
 // NEON load / store instructions
 //===----------------------------------------------------------------------===//
@@ -5273,9 +5259,9 @@ def: NEONInstAlias<"vacle${p}.f16 $Vd, $Vm",
 // Vector Bitwise Operations.
 
 def vnotd : PatFrag<(ops node:$in),
-                    (xor node:$in, (bitconvert (v8i8 NEONimmAllOnesV)))>;
+                    (xor node:$in, ARMimmAllOnesD)>;
 def vnotq : PatFrag<(ops node:$in),
-                    (xor node:$in, (bitconvert (v16i8 NEONimmAllOnesV)))>;
+                    (xor node:$in, ARMimmAllOnesV)>;
 
 
 //   VAND     : Vector Bitwise AND
@@ -6054,9 +6040,9 @@ defm VQABS    : N2VInt_QHS<0b11, 0b11, 0b00, 0b01110, 0,
 // Vector Negate.
 
 def vnegd  : PatFrag<(ops node:$in),
-                     (sub (bitconvert (v2i32 NEONimmAllZerosV)), node:$in)>;
+                     (sub ARMimmAllZerosD, node:$in)>;
 def vnegq  : PatFrag<(ops node:$in),
-                     (sub (bitconvert (v4i32 NEONimmAllZerosV)), node:$in)>;
+                     (sub ARMimmAllZerosV, node:$in)>;
 
 class VNEGD<bits<2> size, string OpcodeStr, string Dt, ValueType Ty>
   : N2V<0b11, 0b11, size, 0b01, 0b00111, 0, 0, (outs DPR:$Vd), (ins DPR:$Vm),
@@ -6270,11 +6256,11 @@ defm : NEONImmReplicateInstAlias<i32, VMOVv2i32, VMOVv4i32,
 
 let AddedComplexity = 50, isAsCheapAsAMove = 1, isReMaterializable = 1 in {
   def VMOVD0 : ARMPseudoExpand<(outs DPR:$Vd), (ins), 4, IIC_VMOVImm,
-                               [(set DPR:$Vd, (v2i32 NEONimmAllZerosV))],
+                               [(set DPR:$Vd, (v2i32 ARMimmAllZerosD))],
                                (VMOVv2i32 DPR:$Vd, 0, (ops 14, zero_reg))>,
                Requires<[HasZCZ]>;
   def VMOVQ0 : ARMPseudoExpand<(outs QPR:$Vd), (ins), 4, IIC_VMOVImm,
-                               [(set QPR:$Vd, (v4i32 NEONimmAllZerosV))],
+                               [(set QPR:$Vd, (v4i32 ARMimmAllZerosV))],
                                (VMOVv4i32 QPR:$Vd, 0, (ops 14, zero_reg))>,
                Requires<[HasZCZ]>;
 }
diff --git a/llvm/lib/Target/ARM/ARMInstrVFP.td b/llvm/lib/Target/ARM/ARMInstrVFP.td
index 8a652c1d90f6e..aea137ac0ddb4 100644
--- a/llvm/lib/Target/ARM/ARMInstrVFP.td
+++ b/llvm/lib/Target/ARM/ARMInstrVFP.td
@@ -1551,6 +1551,8 @@ class AVConv1InsS_Encode<bits<5> opcod1, bits<2> opcod2, bits<4> opcod3,
   let Inst{5}     = Sm{0};
   let Inst{15-12} = Sd{4-1};
   let Inst{22}    = Sd{0};
+
+  let hasSideEffects = 0;
 }
 
 class AVConv1IsH_Encode<bits<5> opcod1, bits<2> opcod2, bits<4> opcod3,
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index bea4e157a1316..25d0a4c21fa6f 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -28,6 +28,8 @@
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/MachineValueType.h"
 #include "llvm/Target/TargetMachine.h"
+#include "llvm/Transforms/InstCombine/InstCombiner.h"
+#include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Transforms/Utils/LoopUtils.h"
 #include <algorithm>
 #include <cassert>
@@ -50,6 +52,28 @@ extern cl::opt<TailPredication::Mode> EnableTailPredication;
 
 extern cl::opt<bool> EnableMaskedGatherScatters;
 
+/// Convert a vector load intrinsic into a simple llvm load instruction.
+/// This is beneficial when the underlying object being addressed comes
+/// from a constant, since we get constant-folding for free.
+static Value *simplifyNeonVld1(const IntrinsicInst &II, unsigned MemAlign,
+                               InstCombiner::BuilderTy &Builder) {
+  auto *IntrAlign = dyn_cast<ConstantInt>(II.getArgOperand(1));
+
+  if (!IntrAlign)
+    return nullptr;
+
+  unsigned Alignment = IntrAlign->getLimitedValue() < MemAlign
+                           ? MemAlign
+                           : IntrAlign->getLimitedValue();
+
+  if (!isPowerOf2_32(Alignment))
+    return nullptr;
+
+  auto *BCastInst = Builder.CreateBitCast(II.getArgOperand(0),
+                                          PointerType::get(II.getType(), 0));
+  return Builder.CreateAlignedLoad(II.getType(), BCastInst, Align(Alignment));
+}
+
 bool ARMTTIImpl::areInlineCompatible(const Function *Caller,
                                      const Function *Callee) const {
   const TargetMachine &TM = getTLI()->getTargetMachine();
@@ -82,6 +106,114 @@ bool ARMTTIImpl::shouldFavorPostInc() const {
   return false;
 }
 
+Optional<Instruction *>
+ARMTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
+  Intrinsic::ID IID = II.getIntrinsicID();
+  switch (IID) {
+  default:
+    break;
+  case Intrinsic::arm_neon_vld1: {
+    Align MemAlign =
+        getKnownAlignment(II.getArgOperand(0), IC.getDataLayout(), &II,
+                          &IC.getAssumptionCache(), &IC.getDominatorTree());
+    if (Value *V = simplifyNeonVld1(II, MemAlign.value(), IC.Builder)) {
+      return IC.replaceInstUsesWith(II, V);
+    }
+    break;
+  }
+
+  case Intrinsic::arm_neon_vld2:
+  case Intrinsic::arm_neon_vld3:
+  case Intrinsic::arm_neon_vld4:
+  case Intrinsic::arm_neon_vld2lane:
+  case Intrinsic::arm_neon_vld3lane:
+  case Intrinsic::arm_neon_vld4lane:
+  case Intrinsic::arm_neon_vst1:
+  case Intrinsic::arm_neon_vst2:
+  case Intrinsic::arm_neon_vst3:
+  case Intrinsic::arm_neon_vst4:
+  case Intrinsic::arm_neon_vst2lane:
+  case Intrinsic::arm_neon_vst3lane:
+  case Intrinsic::arm_neon_vst4lane: {
+    Align MemAlign =
+        getKnownAlignment(II.getArgOperand(0), IC.getDataLayout(), &II,
+                          &IC.getAssumptionCache(), &IC.getDominatorTree());
+    unsigned AlignArg = II.getNumArgOperands() - 1;
+    Value *AlignArgOp = II.getArgOperand(AlignArg);
+    MaybeAlign Align = cast<ConstantInt>(AlignArgOp)->getMaybeAlignValue();
+    if (Align && *Align < MemAlign) {
+      return IC.replaceOperand(
+          II, AlignArg,
+          ConstantInt::get(Type::getInt32Ty(II.getContext()), MemAlign.value(),
+                           false));
+    }
+    break;
+  }
+
+  case Intrinsic::arm_mve_pred_i2v: {
+    Value *Arg = II.getArgOperand(0);
+    Value *ArgArg;
+    if (match(Arg, PatternMatch::m_Intrinsic<Intrinsic::arm_mve_pred_v2i>(
+                       PatternMatch::m_Value(ArgArg))) &&
+        II.getType() == ArgArg->getType()) {
+      return IC.replaceInstUsesWith(II, ArgArg);
+    }
+    Constant *XorMask;
+    if (match(Arg, m_Xor(PatternMatch::m_Intrinsic<Intrinsic::arm_mve_pred_v2i>(
+                             PatternMatch::m_Value(ArgArg)),
+                         PatternMatch::m_Constant(XorMask))) &&
+        II.getType() == ArgArg->getType()) {
+      if (auto *CI = dyn_cast<ConstantInt>(XorMask)) {
+        if (CI->getValue().trunc(16).isAllOnesValue()) {
+          auto TrueVector = IC.Builder.CreateVectorSplat(
+              cast<VectorType>(II.getType())->getNumElements(),
+              IC.Builder.getTrue());
+          return BinaryOperator::Create(Instruction::Xor, ArgArg, TrueVector);
+        }
+      }
+    }
+    KnownBits ScalarKnown(32);
+    if (IC.SimplifyDemandedBits(&II, 0, APInt::getLowBitsSet(32, 16),
+                                ScalarKnown, 0)) {
+      return &II;
+    }
+    break;
+  }
+  case Intrinsic::arm_mve_pred_v2i: {
+    Value *Arg = II.getArgOperand(0);
+    Value *ArgArg;
+    if (match(Arg, PatternMatch::m_Intrinsic<Intrinsic::arm_mve_pred_i2v>(
+                       PatternMatch::m_Value(ArgArg)))) {
+      return IC.replaceInstUsesWith(II, ArgArg);
+    }
+    if (!II.getMetadata(LLVMContext::MD_range)) {
+      Type *IntTy32 = Type::getInt32Ty(II.getContext());
+      Metadata *M[] = {
+          ConstantAsMetadata::get(ConstantInt::get(IntTy32, 0)),
+          ConstantAsMetadata::get(ConstantInt::get(IntTy32, 0xFFFF))};
+      II.setMetadata(LLVMContext::MD_range, MDNode::get(II.getContext(), M));
+      return &II;
+    }
+    break;
+  }
+  case Intrinsic::arm_mve_vadc:
+  case Intrinsic::arm_mve_vadc_predicated: {
+    unsigned CarryOp =
+        (II.getIntrinsicID() == Intrinsic::arm_mve_vadc_predicated) ? 3 : 2;
+    assert(II.getArgOperand(CarryOp)->getType()->getScalarSizeInBits() == 32 &&
+           "Bad type for intrinsic!");
+
+    KnownBits CarryKnown(32);
+    if (IC.SimplifyDemandedBits(&II, CarryOp, APInt::getOneBitSet(32, 29),
+                                CarryKnown)) {
+      return &II;
+    }
+    break;
+  }
+  }
+  return None;
+}
+
 int ARMTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
                               TTI::TargetCostKind CostKind) {
   assert(Ty->isIntegerTy());
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index 7bf6de4bffe07..093dfbbf5f026 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -113,6 +113,9 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
     return !ST->isTargetDarwin() && !ST->hasMVEFloatOps();
   }
 
+  Optional<Instruction *> instCombineIntrinsic(InstCombiner &IC,
+                                               IntrinsicInst &II) const;
+
   /// \name Scalar TTI Implementations
   /// @{
 
diff --git a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
index 05f870b90ecdb..99862b41b69d5 100644
--- a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
+++ b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
@@ -10598,6 +10598,12 @@ unsigned ARMAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
         (isThumb() && !hasV8Ops()))
       return Match_InvalidOperand;
     break;
+  case ARM::t2TBB:
+  case ARM::t2TBH:
+    // Rn = sp is only allowed with ARMv8-A
+    if (!hasV8Ops() && (Inst.getOperand(0).getReg() == ARM::SP))
+      return Match_RequiresV8;
+    break;
   default:
     break;
   }
diff --git a/llvm/lib/Target/ARM/Disassembler/ARMDisassembler.cpp b/llvm/lib/Target/ARM/Disassembler/ARMDisassembler.cpp
index 54ff0d9966cbd..ccef1ba3c8307 100644
--- a/llvm/lib/Target/ARM/Disassembler/ARMDisassembler.cpp
+++ b/llvm/lib/Target/ARM/Disassembler/ARMDisassembler.cpp
@@ -4529,12 +4529,14 @@ static DecodeStatus DecodeCoprocessor(MCInst &Inst, unsigned Val,
 static DecodeStatus
 DecodeThumbTableBranch(MCInst &Inst, unsigned Insn,
                        uint64_t Address, const void *Decoder) {
+  const FeatureBitset &FeatureBits =
+    ((const MCDisassembler*)Decoder)->getSubtargetInfo().getFeatureBits();
   DecodeStatus S = MCDisassembler::Success;
 
   unsigned Rn = fieldFromInstruction(Insn, 16, 4);
   unsigned Rm = fieldFromInstruction(Insn, 0, 4);
 
-  if (Rn == ARM::SP) S = MCDisassembler::SoftFail;
+  if (Rn == 13 && !FeatureBits[ARM::HasV8Ops]) S = MCDisassembler::SoftFail;
   if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
     return MCDisassembler::Fail;
   if (!Check(S, DecoderGPRRegisterClass(Inst, Rm, Address, Decoder)))
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp
index ac72abe0d9f68..49840672bf9a0 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp
@@ -13,12 +13,12 @@
 #include "MCTargetDesc/AVRAsmBackend.h"
 #include "MCTargetDesc/AVRFixupKinds.h"
 #include "MCTargetDesc/AVRMCTargetDesc.h"
-
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCDirectives.h"
 #include "llvm/MC/MCELFObjectWriter.h"
+#include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSubtargetInfo.h"
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h
index 9e150f120dd4d..43b26fd0c5653 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h
@@ -22,6 +22,7 @@
 namespace llvm {
 
 class MCAssembler;
+class MCContext;
 struct MCFixupKindInfo;
 
 /// Utilities for manipulating generated AVR machine code.
diff --git a/llvm/lib/Target/BPF/BPFISelLowering.cpp b/llvm/lib/Target/BPF/BPFISelLowering.cpp
index a02556a399098..3322b8d93b3a6 100644
--- a/llvm/lib/Target/BPF/BPFISelLowering.cpp
+++ b/llvm/lib/Target/BPF/BPFISelLowering.cpp
@@ -20,7 +20,6 @@
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/SelectionDAGISel.h"
 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/DiagnosticInfo.h"
diff --git a/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp b/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp
index f9abe76c976b1..d26da1e5048b8 100644
--- a/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp
+++ b/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp
@@ -13,6 +13,7 @@
 #include "MCTargetDesc/BPFMCTargetDesc.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/MC/MCCodeEmitter.h"
+#include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixup.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstrInfo.h"
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCompound.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCompound.cpp
index 2b0bbdafa3819..82b2074c5cd86 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCompound.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCompound.cpp
@@ -14,6 +14,7 @@
 #include "MCTargetDesc/HexagonMCInstrInfo.h"
 #include "MCTargetDesc/HexagonMCShuffler.h"
 #include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
diff --git a/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp b/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
index 821339f503551..fcb1db1e8c0a8 100644
--- a/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
+++ b/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
@@ -20,7 +20,6 @@
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/SelectionDAGISel.h"
 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/CallingConv.h"
diff --git a/llvm/lib/Target/Mips/MipsCallLowering.cpp b/llvm/lib/Target/Mips/MipsCallLowering.cpp
index cffd99affac10..5b5dc56cd3395 100644
--- a/llvm/lib/Target/Mips/MipsCallLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsCallLowering.cpp
@@ -87,9 +87,10 @@ bool MipsCallLowering::MipsHandler::handle(
 }
 
 namespace {
-class IncomingValueHandler : public MipsCallLowering::MipsHandler {
+class MipsIncomingValueHandler : public MipsCallLowering::MipsHandler {
 public:
-  IncomingValueHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI)
+  MipsIncomingValueHandler(MachineIRBuilder &MIRBuilder,
+                           MachineRegisterInfo &MRI)
       : MipsHandler(MIRBuilder, MRI) {}
 
 private:
@@ -117,11 +118,11 @@ class IncomingValueHandler : public MipsCallLowering::MipsHandler {
   }
 };
 
-class CallReturnHandler : public IncomingValueHandler {
+class CallReturnHandler : public MipsIncomingValueHandler {
 public:
   CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
                     MachineInstrBuilder &MIB)
-      : IncomingValueHandler(MIRBuilder, MRI), MIB(MIB) {}
+      : MipsIncomingValueHandler(MIRBuilder, MRI), MIB(MIB) {}
 
 private:
   void markPhysRegUsed(unsigned PhysReg) override {
@@ -133,9 +134,9 @@ class CallReturnHandler : public IncomingValueHandler {
 
 } // end anonymous namespace
 
-void IncomingValueHandler::assignValueToReg(Register ValVReg,
-                                            const CCValAssign &VA,
-                                            const EVT &VT) {
+void MipsIncomingValueHandler::assignValueToReg(Register ValVReg,
+                                                const CCValAssign &VA,
+                                                const EVT &VT) {
   Register PhysReg = VA.getLocReg();
   if (VT == MVT::f64 && PhysReg >= Mips::A0 && PhysReg <= Mips::A3) {
     const MipsSubtarget &STI =
@@ -167,8 +168,8 @@ void IncomingValueHandler::assignValueToReg(Register ValVReg,
   }
 }
 
-Register IncomingValueHandler::getStackAddress(const CCValAssign &VA,
-                                               MachineMemOperand *&MMO) {
+Register MipsIncomingValueHandler::getStackAddress(const CCValAssign &VA,
+                                                   MachineMemOperand *&MMO) {
   MachineFunction &MF = MIRBuilder.getMF();
   unsigned Size = alignTo(VA.getValVT().getSizeInBits(), 8) / 8;
   unsigned Offset = VA.getLocMemOffset();
@@ -186,8 +187,8 @@ Register IncomingValueHandler::getStackAddress(const CCValAssign &VA,
   return MIRBuilder.buildFrameIndex(LLT::pointer(0, 32), FI).getReg(0);
 }
 
-void IncomingValueHandler::assignValueToAddress(Register ValVReg,
-                                                const CCValAssign &VA) {
+void MipsIncomingValueHandler::assignValueToAddress(Register ValVReg,
+                                                    const CCValAssign &VA) {
   if (VA.getLocInfo() == CCValAssign::SExt ||
       VA.getLocInfo() == CCValAssign::ZExt ||
       VA.getLocInfo() == CCValAssign::AExt) {
@@ -197,10 +198,10 @@ void IncomingValueHandler::assignValueToAddress(Register ValVReg,
     buildLoad(ValVReg, VA);
 }
 
-bool IncomingValueHandler::handleSplit(SmallVectorImpl<Register> &VRegs,
-                                       ArrayRef<CCValAssign> ArgLocs,
-                                       unsigned ArgLocsStartIndex,
-                                       Register ArgsReg, const EVT &VT) {
+bool MipsIncomingValueHandler::handleSplit(SmallVectorImpl<Register> &VRegs,
+                                           ArrayRef<CCValAssign> ArgLocs,
+                                           unsigned ArgLocsStartIndex,
+                                           Register ArgsReg, const EVT &VT) {
   if (!assignVRegs(VRegs, ArgLocs, ArgLocsStartIndex, VT))
     return false;
   setLeastSignificantFirst(VRegs);
@@ -209,10 +210,10 @@ bool IncomingValueHandler::handleSplit(SmallVectorImpl<Register> &VRegs,
 }
 
 namespace {
-class OutgoingValueHandler : public MipsCallLowering::MipsHandler {
+class MipsOutgoingValueHandler : public MipsCallLowering::MipsHandler {
 public:
-  OutgoingValueHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
-                       MachineInstrBuilder &MIB)
+  MipsOutgoingValueHandler(MachineIRBuilder &MIRBuilder,
+                           MachineRegisterInfo &MRI, MachineInstrBuilder &MIB)
       : MipsHandler(MIRBuilder, MRI), MIB(MIB) {}
 
 private:
@@ -234,9 +235,9 @@ class OutgoingValueHandler : public MipsCallLowering::MipsHandler {
 };
 } // end anonymous namespace
 
-void OutgoingValueHandler::assignValueToReg(Register ValVReg,
-                                            const CCValAssign &VA,
-                                            const EVT &VT) {
+void MipsOutgoingValueHandler::assignValueToReg(Register ValVReg,
+                                                const CCValAssign &VA,
+                                                const EVT &VT) {
   Register PhysReg = VA.getLocReg();
   if (VT == MVT::f64 && PhysReg >= Mips::A0 && PhysReg <= Mips::A3) {
     const MipsSubtarget &STI =
@@ -254,8 +255,8 @@ void OutgoingValueHandler::assignValueToReg(Register ValVReg,
   }
 }
 
-Register OutgoingValueHandler::getStackAddress(const CCValAssign &VA,
-                                               MachineMemOperand *&MMO) {
+Register MipsOutgoingValueHandler::getStackAddress(const CCValAssign &VA,
+                                                   MachineMemOperand *&MMO) {
   MachineFunction &MF = MIRBuilder.getMF();
   const TargetFrameLowering *TFL = MF.getSubtarget().getFrameLowering();
 
@@ -278,16 +279,16 @@ Register OutgoingValueHandler::getStackAddress(const CCValAssign &VA,
   return AddrReg.getReg(0);
 }
 
-void OutgoingValueHandler::assignValueToAddress(Register ValVReg,
-                                                const CCValAssign &VA) {
+void MipsOutgoingValueHandler::assignValueToAddress(Register ValVReg,
+                                                    const CCValAssign &VA) {
   MachineMemOperand *MMO;
   Register Addr = getStackAddress(VA, MMO);
   Register ExtReg = extendRegister(ValVReg, VA);
   MIRBuilder.buildStore(ExtReg, Addr, *MMO);
 }
 
-Register OutgoingValueHandler::extendRegister(Register ValReg,
-                                              const CCValAssign &VA) {
+Register MipsOutgoingValueHandler::extendRegister(Register ValReg,
+                                                  const CCValAssign &VA) {
   LLT LocTy{VA.getLocVT()};
   switch (VA.getLocInfo()) {
   case CCValAssign::SExt: {
@@ -308,10 +309,10 @@ Register OutgoingValueHandler::extendRegister(Register ValReg,
   llvm_unreachable("unable to extend register");
 }
 
-bool OutgoingValueHandler::handleSplit(SmallVectorImpl<Register> &VRegs,
-                                       ArrayRef<CCValAssign> ArgLocs,
-                                       unsigned ArgLocsStartIndex,
-                                       Register ArgsReg, const EVT &VT) {
+bool MipsOutgoingValueHandler::handleSplit(SmallVectorImpl<Register> &VRegs,
+                                           ArrayRef<CCValAssign> ArgLocs,
+                                           unsigned ArgLocsStartIndex,
+                                           Register ArgsReg, const EVT &VT) {
   MIRBuilder.buildUnmerge(VRegs, ArgsReg);
   setLeastSignificantFirst(VRegs);
   if (!assignVRegs(VRegs, ArgLocs, ArgLocsStartIndex, VT))
@@ -403,7 +404,7 @@ bool MipsCallLowering::lowerReturn(MachineIRBuilder &MIRBuilder,
     CCInfo.AnalyzeReturn(Outs, TLI.CCAssignFnForReturn());
     setLocInfo(ArgLocs, Outs);
 
-    OutgoingValueHandler RetHandler(MIRBuilder, MF.getRegInfo(), Ret);
+    MipsOutgoingValueHandler RetHandler(MIRBuilder, MF.getRegInfo(), Ret);
     if (!RetHandler.handle(ArgLocs, RetInfos)) {
       return false;
     }
@@ -455,7 +456,7 @@ bool MipsCallLowering::lowerFormalArguments(
   CCInfo.AnalyzeFormalArguments(Ins, TLI.CCAssignFnForCall());
   setLocInfo(ArgLocs, Ins);
 
-  IncomingValueHandler Handler(MIRBuilder, MF.getRegInfo());
+  MipsIncomingValueHandler Handler(MIRBuilder, MF.getRegInfo());
   if (!Handler.handle(ArgLocs, ArgInfos))
     return false;
 
@@ -579,7 +580,7 @@ bool MipsCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
   CCInfo.AnalyzeCallOperands(Outs, TLI.CCAssignFnForCall(), FuncOrigArgs, Call);
   setLocInfo(ArgLocs, Outs);
 
-  OutgoingValueHandler RetHandler(MIRBuilder, MF.getRegInfo(), MIB);
+  MipsOutgoingValueHandler RetHandler(MIRBuilder, MF.getRegInfo(), MIB);
   if (!RetHandler.handle(ArgLocs, ArgInfos)) {
     return false;
   }
diff --git a/llvm/lib/Target/Mips/MipsTargetMachine.cpp b/llvm/lib/Target/Mips/MipsTargetMachine.cpp
index 80cb6ce7ac0c0..0d6b97fa91e31 100644
--- a/llvm/lib/Target/Mips/MipsTargetMachine.cpp
+++ b/llvm/lib/Target/Mips/MipsTargetMachine.cpp
@@ -295,8 +295,7 @@ MipsTargetMachine::getTargetTransformInfo(const Function &F) {
 }
 
 // Implemented by targets that want to run passes immediately before
-// machine code is emitted. return true if -print-machineinstrs should
-// print out the code after the passes.
+// machine code is emitted.
 void MipsPassConfig::addPreEmitPass() {
   // Expand pseudo instructions that are sensitive to register allocation.
   addPass(createMipsExpandPseudoPass());
diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXMCAsmInfo.cpp b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXMCAsmInfo.cpp
index aef0eed6ab9a4..f275011018a32 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXMCAsmInfo.cpp
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXMCAsmInfo.cpp
@@ -47,6 +47,7 @@ NVPTXMCAsmInfo::NVPTXMCAsmInfo(const Triple &TheTriple,
   AscizDirective = nullptr; // not supported
   SupportsQuotedNames = false;
   SupportsExtendedDwarfLocDirective = false;
+  SupportsSignedData = false;
 
   // @TODO: Can we just disable this?
   WeakDirective = "\t// .weak\t";
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
index 3873c73fb2e03..d4b2ae3840683 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
@@ -111,6 +111,263 @@ bool NVPTXTTIImpl::isSourceOfDivergence(const Value *V) {
   return false;
 }
 
+// Convert NVVM intrinsics to target-generic LLVM code where possible.
+static Instruction *simplifyNvvmIntrinsic(IntrinsicInst *II, InstCombiner &IC) {
+  // Each NVVM intrinsic we can simplify can be replaced with one of:
+  //
+  //  * an LLVM intrinsic,
+  //  * an LLVM cast operation,
+  //  * an LLVM binary operation, or
+  //  * ad-hoc LLVM IR for the particular operation.
+
+  // Some transformations are only valid when the module's
+  // flush-denormals-to-zero (ftz) setting is true/false, whereas other
+  // transformations are valid regardless of the module's ftz setting.
+  enum FtzRequirementTy {
+    FTZ_Any,       // Any ftz setting is ok.
+    FTZ_MustBeOn,  // Transformation is valid only if ftz is on.
+    FTZ_MustBeOff, // Transformation is valid only if ftz is off.
+  };
+  // Classes of NVVM intrinsics that can't be replaced one-to-one with a
+  // target-generic intrinsic, cast op, or binary op but that we can nonetheless
+  // simplify.
+  enum SpecialCase {
+    SPC_Reciprocal,
+  };
+
+  // SimplifyAction is a poor-man's variant (plus an additional flag) that
+  // represents how to replace an NVVM intrinsic with target-generic LLVM IR.
+  struct SimplifyAction {
+    // Invariant: At most one of these Optionals has a value.
+    Optional<Intrinsic::ID> IID;
+    Optional<Instruction::CastOps> CastOp;
+    Optional<Instruction::BinaryOps> BinaryOp;
+    Optional<SpecialCase> Special;
+
+    FtzRequirementTy FtzRequirement = FTZ_Any;
+
+    SimplifyAction() = default;
+
+    SimplifyAction(Intrinsic::ID IID, FtzRequirementTy FtzReq)
+        : IID(IID), FtzRequirement(FtzReq) {}
+
+    // Cast operations don't have anything to do with FTZ, so we skip that
+    // argument.
+    SimplifyAction(Instruction::CastOps CastOp) : CastOp(CastOp) {}
+
+    SimplifyAction(Instruction::BinaryOps BinaryOp, FtzRequirementTy FtzReq)
+        : BinaryOp(BinaryOp), FtzRequirement(FtzReq) {}
+
+    SimplifyAction(SpecialCase Special, FtzRequirementTy FtzReq)
+        : Special(Special), FtzRequirement(FtzReq) {}
+  };
+
+  // Try to generate a SimplifyAction describing how to replace our
+  // IntrinsicInstr with target-generic LLVM IR.
+  const SimplifyAction Action = [II]() -> SimplifyAction {
+    switch (II->getIntrinsicID()) {
+    // NVVM intrinsics that map directly to LLVM intrinsics.
+    case Intrinsic::nvvm_ceil_d:
+      return {Intrinsic::ceil, FTZ_Any};
+    case Intrinsic::nvvm_ceil_f:
+      return {Intrinsic::ceil, FTZ_MustBeOff};
+    case Intrinsic::nvvm_ceil_ftz_f:
+      return {Intrinsic::ceil, FTZ_MustBeOn};
+    case Intrinsic::nvvm_fabs_d:
+      return {Intrinsic::fabs, FTZ_Any};
+    case Intrinsic::nvvm_fabs_f:
+      return {Intrinsic::fabs, FTZ_MustBeOff};
+    case Intrinsic::nvvm_fabs_ftz_f:
+      return {Intrinsic::fabs, FTZ_MustBeOn};
+    case Intrinsic::nvvm_floor_d:
+      return {Intrinsic::floor, FTZ_Any};
+    case Intrinsic::nvvm_floor_f:
+      return {Intrinsic::floor, FTZ_MustBeOff};
+    case Intrinsic::nvvm_floor_ftz_f:
+      return {Intrinsic::floor, FTZ_MustBeOn};
+    case Intrinsic::nvvm_fma_rn_d:
+      return {Intrinsic::fma, FTZ_Any};
+    case Intrinsic::nvvm_fma_rn_f:
+      return {Intrinsic::fma, FTZ_MustBeOff};
+    case Intrinsic::nvvm_fma_rn_ftz_f:
+      return {Intrinsic::fma, FTZ_MustBeOn};
+    case Intrinsic::nvvm_fmax_d:
+      return {Intrinsic::maxnum, FTZ_Any};
+    case Intrinsic::nvvm_fmax_f:
+      return {Intrinsic::maxnum, FTZ_MustBeOff};
+    case Intrinsic::nvvm_fmax_ftz_f:
+      return {Intrinsic::maxnum, FTZ_MustBeOn};
+    case Intrinsic::nvvm_fmin_d:
+      return {Intrinsic::minnum, FTZ_Any};
+    case Intrinsic::nvvm_fmin_f:
+      return {Intrinsic::minnum, FTZ_MustBeOff};
+    case Intrinsic::nvvm_fmin_ftz_f:
+      return {Intrinsic::minnum, FTZ_MustBeOn};
+    case Intrinsic::nvvm_round_d:
+      return {Intrinsic::round, FTZ_Any};
+    case Intrinsic::nvvm_round_f:
+      return {Intrinsic::round, FTZ_MustBeOff};
+    case Intrinsic::nvvm_round_ftz_f:
+      return {Intrinsic::round, FTZ_MustBeOn};
+    case Intrinsic::nvvm_sqrt_rn_d:
+      return {Intrinsic::sqrt, FTZ_Any};
+    case Intrinsic::nvvm_sqrt_f:
+      // nvvm_sqrt_f is a special case.  For  most intrinsics, foo_ftz_f is the
+      // ftz version, and foo_f is the non-ftz version.  But nvvm_sqrt_f adopts
+      // the ftz-ness of the surrounding code.  sqrt_rn_f and sqrt_rn_ftz_f are
+      // the versions with explicit ftz-ness.
+      return {Intrinsic::sqrt, FTZ_Any};
+    case Intrinsic::nvvm_sqrt_rn_f:
+      return {Intrinsic::sqrt, FTZ_MustBeOff};
+    case Intrinsic::nvvm_sqrt_rn_ftz_f:
+      return {Intrinsic::sqrt, FTZ_MustBeOn};
+    case Intrinsic::nvvm_trunc_d:
+      return {Intrinsic::trunc, FTZ_Any};
+    case Intrinsic::nvvm_trunc_f:
+      return {Intrinsic::trunc, FTZ_MustBeOff};
+    case Intrinsic::nvvm_trunc_ftz_f:
+      return {Intrinsic::trunc, FTZ_MustBeOn};
+
+    // NVVM intrinsics that map to LLVM cast operations.
+    //
+    // Note that llvm's target-generic conversion operators correspond to the rz
+    // (round to zero) versions of the nvvm conversion intrinsics, even though
+    // most everything else here uses the rn (round to nearest even) nvvm ops.
+    case Intrinsic::nvvm_d2i_rz:
+    case Intrinsic::nvvm_f2i_rz:
+    case Intrinsic::nvvm_d2ll_rz:
+    case Intrinsic::nvvm_f2ll_rz:
+      return {Instruction::FPToSI};
+    case Intrinsic::nvvm_d2ui_rz:
+    case Intrinsic::nvvm_f2ui_rz:
+    case Intrinsic::nvvm_d2ull_rz:
+    case Intrinsic::nvvm_f2ull_rz:
+      return {Instruction::FPToUI};
+    case Intrinsic::nvvm_i2d_rz:
+    case Intrinsic::nvvm_i2f_rz:
+    case Intrinsic::nvvm_ll2d_rz:
+    case Intrinsic::nvvm_ll2f_rz:
+      return {Instruction::SIToFP};
+    case Intrinsic::nvvm_ui2d_rz:
+    case Intrinsic::nvvm_ui2f_rz:
+    case Intrinsic::nvvm_ull2d_rz:
+    case Intrinsic::nvvm_ull2f_rz:
+      return {Instruction::UIToFP};
+
+    // NVVM intrinsics that map to LLVM binary ops.
+    case Intrinsic::nvvm_add_rn_d:
+      return {Instruction::FAdd, FTZ_Any};
+    case Intrinsic::nvvm_add_rn_f:
+      return {Instruction::FAdd, FTZ_MustBeOff};
+    case Intrinsic::nvvm_add_rn_ftz_f:
+      return {Instruction::FAdd, FTZ_MustBeOn};
+    case Intrinsic::nvvm_mul_rn_d:
+      return {Instruction::FMul, FTZ_Any};
+    case Intrinsic::nvvm_mul_rn_f:
+      return {Instruction::FMul, FTZ_MustBeOff};
+    case Intrinsic::nvvm_mul_rn_ftz_f:
+      return {Instruction::FMul, FTZ_MustBeOn};
+    case Intrinsic::nvvm_div_rn_d:
+      return {Instruction::FDiv, FTZ_Any};
+    case Intrinsic::nvvm_div_rn_f:
+      return {Instruction::FDiv, FTZ_MustBeOff};
+    case Intrinsic::nvvm_div_rn_ftz_f:
+      return {Instruction::FDiv, FTZ_MustBeOn};
+
+    // The remainder of cases are NVVM intrinsics that map to LLVM idioms, but
+    // need special handling.
+    //
+    // We seem to be missing intrinsics for rcp.approx.{ftz.}f32, which is just
+    // as well.
+    case Intrinsic::nvvm_rcp_rn_d:
+      return {SPC_Reciprocal, FTZ_Any};
+    case Intrinsic::nvvm_rcp_rn_f:
+      return {SPC_Reciprocal, FTZ_MustBeOff};
+    case Intrinsic::nvvm_rcp_rn_ftz_f:
+      return {SPC_Reciprocal, FTZ_MustBeOn};
+
+      // We do not currently simplify intrinsics that give an approximate
+      // answer. These include:
+      //
+      //   - nvvm_cos_approx_{f,ftz_f}
+      //   - nvvm_ex2_approx_{d,f,ftz_f}
+      //   - nvvm_lg2_approx_{d,f,ftz_f}
+      //   - nvvm_sin_approx_{f,ftz_f}
+      //   - nvvm_sqrt_approx_{f,ftz_f}
+      //   - nvvm_rsqrt_approx_{d,f,ftz_f}
+      //   - nvvm_div_approx_{ftz_d,ftz_f,f}
+      //   - nvvm_rcp_approx_ftz_d
+      //
+      // Ideally we'd encode them as e.g. "fast call @llvm.cos", where "fast"
+      // means that fastmath is enabled in the intrinsic.  Unfortunately only
+      // binary operators (currently) have a fastmath bit in SelectionDAG, so
+      // this information gets lost and we can't select on it.
+      //
+      // TODO: div and rcp are lowered to a binary op, so these we could in
+      // theory lower them to "fast fdiv".
+
+    default:
+      return {};
+    }
+  }();
+
+  // If Action.FtzRequirementTy is not satisfied by the module's ftz state, we
+  // can bail out now.  (Notice that in the case that IID is not an NVVM
+  // intrinsic, we don't have to look up any module metadata, as
+  // FtzRequirementTy will be FTZ_Any.)
+  if (Action.FtzRequirement != FTZ_Any) {
+    StringRef Attr = II->getFunction()
+                         ->getFnAttribute("denormal-fp-math-f32")
+                         .getValueAsString();
+    DenormalMode Mode = parseDenormalFPAttribute(Attr);
+    bool FtzEnabled = Mode.Output != DenormalMode::IEEE;
+
+    if (FtzEnabled != (Action.FtzRequirement == FTZ_MustBeOn))
+      return nullptr;
+  }
+
+  // Simplify to target-generic intrinsic.
+  if (Action.IID) {
+    SmallVector<Value *, 4> Args(II->arg_operands());
+    // All the target-generic intrinsics currently of interest to us have one
+    // type argument, equal to that of the nvvm intrinsic's argument.
+    Type *Tys[] = {II->getArgOperand(0)->getType()};
+    return CallInst::Create(
+        Intrinsic::getDeclaration(II->getModule(), *Action.IID, Tys), Args);
+  }
+
+  // Simplify to target-generic binary op.
+  if (Action.BinaryOp)
+    return BinaryOperator::Create(*Action.BinaryOp, II->getArgOperand(0),
+                                  II->getArgOperand(1), II->getName());
+
+  // Simplify to target-generic cast op.
+  if (Action.CastOp)
+    return CastInst::Create(*Action.CastOp, II->getArgOperand(0), II->getType(),
+                            II->getName());
+
+  // All that's left are the special cases.
+  if (!Action.Special)
+    return nullptr;
+
+  switch (*Action.Special) {
+  case SPC_Reciprocal:
+    // Simplify reciprocal.
+    return BinaryOperator::Create(
+        Instruction::FDiv, ConstantFP::get(II->getArgOperand(0)->getType(), 1),
+        II->getArgOperand(0), II->getName());
+  }
+  llvm_unreachable("All SpecialCase enumerators should be handled in switch.");
+}
+
+Optional<Instruction *>
+NVPTXTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
+  if (Instruction *I = simplifyNvvmIntrinsic(&II, IC)) {
+    return I;
+  }
+  return None;
+}
+
 int NVPTXTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueKind Opd1Info,
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
index cb832031f1add..6f071040dd9df 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
@@ -48,6 +48,9 @@ class NVPTXTTIImpl : public BasicTTIImplBase<NVPTXTTIImpl> {
     return AddressSpace::ADDRESS_SPACE_GENERIC;
   }
 
+  Optional<Instruction *> instCombineIntrinsic(InstCombiner &IC,
+                                               IntrinsicInst &II) const;
+
   // Loads and stores can be vectorized if the alignment is at least as big as
   // the load/store we want to vectorize.
   bool isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes, Align Alignment,
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.cpp
index 4373778cc96cc..386d59266096a 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.cpp
@@ -20,6 +20,7 @@
 
 
 #include "PPCELFStreamer.h"
+#include "PPCFixupKinds.h"
 #include "PPCInstrInfo.h"
 #include "PPCMCCodeEmitter.h"
 #include "llvm/BinaryFormat/ELF.h"
@@ -89,12 +90,33 @@ void PPCELFStreamer::emitInstruction(const MCInst &Inst,
   PPCMCCodeEmitter *Emitter =
       static_cast<PPCMCCodeEmitter*>(getAssembler().getEmitterPtr());
 
+  // If the instruction is a part of the GOT to PC-Rel link time optimization
+  // instruction pair, return a value, otherwise return None. A true returned
+  // value means the instruction is the PLDpc and a false value means it is
+  // the user instruction.
+  Optional<bool> IsPartOfGOTToPCRelPair = isPartOfGOTToPCRelPair(Inst, STI);
+
+  // User of the GOT-indirect address.
+  // For example, the load that will get the relocation as follows:
+  // .reloc .Lpcrel1-8,R_PPC64_PCREL_OPT,.-(.Lpcrel1-8)
+  //  lwa 3, 4(3)
+  if (IsPartOfGOTToPCRelPair.hasValue() && !IsPartOfGOTToPCRelPair.getValue())
+    emitGOTToPCRelReloc(Inst);
+
   // Special handling is only for prefixed instructions.
   if (!Emitter->isPrefixedInstruction(Inst)) {
     MCELFStreamer::emitInstruction(Inst, STI);
     return;
   }
   emitPrefixedInstruction(Inst, STI);
+
+  // Producer of the GOT-indirect address.
+  // For example, the prefixed load from the got that will get the label as
+  // follows:
+  //  pld 3, vec@got@pcrel(0), 1
+  // .Lpcrel1:
+  if (IsPartOfGOTToPCRelPair.hasValue() && IsPartOfGOTToPCRelPair.getValue())
+    emitGOTToPCRelLabel(Inst);
 }
 
 void PPCELFStreamer::emitLabel(MCSymbol *Symbol, SMLoc Loc) {
@@ -103,6 +125,102 @@ void PPCELFStreamer::emitLabel(MCSymbol *Symbol, SMLoc Loc) {
   MCELFStreamer::emitLabel(Symbol);
 }
 
+// This linker time GOT PC Relative optimization relocation will look like this:
+//   pld <reg> symbol@got@pcrel
+// <Label###>:
+//   .reloc Label###-8,R_PPC64_PCREL_OPT,.-(Label###-8)
+//   load <loadedreg>, 0(<reg>)
+// The reason we place the label after the PLDpc instruction is that there
+// may be an alignment nop before it since prefixed instructions must not
+// cross a 64-byte boundary (please see
+// PPCELFStreamer::emitPrefixedInstruction()). When referring to the
+// label, we subtract the width of a prefixed instruction (8 bytes) to ensure
+// we refer to the PLDpc.
+void PPCELFStreamer::emitGOTToPCRelReloc(const MCInst &Inst) {
+  // Get the last operand which contains the symbol.
+  const MCOperand &Operand = Inst.getOperand(Inst.getNumOperands() - 1);
+  assert(Operand.isExpr() && "Expecting an MCExpr.");
+  // Cast the last operand to MCSymbolRefExpr to get the symbol.
+  const MCExpr *Expr = Operand.getExpr();
+  const MCSymbolRefExpr *SymExpr = static_cast<const MCSymbolRefExpr *>(Expr);
+  assert(SymExpr->getKind() == MCSymbolRefExpr::VK_PPC_PCREL_OPT &&
+         "Expecting a symbol of type VK_PPC_PCREL_OPT");
+  MCSymbol *LabelSym =
+      getContext().getOrCreateSymbol(SymExpr->getSymbol().getName());
+  const MCExpr *LabelExpr = MCSymbolRefExpr::create(LabelSym, getContext());
+  const MCExpr *Eight = MCConstantExpr::create(8, getContext());
+  // SubExpr is just Label###-8
+  const MCExpr *SubExpr =
+      MCBinaryExpr::createSub(LabelExpr, Eight, getContext());
+  MCSymbol *CurrentLocation = getContext().createTempSymbol();
+  const MCExpr *CurrentLocationExpr =
+      MCSymbolRefExpr::create(CurrentLocation, getContext());
+  // SubExpr2 is .-(Label###-8)
+  const MCExpr *SubExpr2 =
+      MCBinaryExpr::createSub(CurrentLocationExpr, SubExpr, getContext());
+
+  MCDataFragment *DF = static_cast<MCDataFragment *>(LabelSym->getFragment());
+  assert(DF && "Expecting a valid data fragment.");
+  MCFixupKind FixupKind = static_cast<MCFixupKind>(FirstLiteralRelocationKind +
+                                                   ELF::R_PPC64_PCREL_OPT);
+  DF->getFixups().push_back(
+      MCFixup::create(LabelSym->getOffset() - 8, SubExpr2,
+                      FixupKind, Inst.getLoc()));
+  emitLabel(CurrentLocation, Inst.getLoc());
+}
+
+// Emit the label that immediately follows the PLDpc for a link time GOT PC Rel
+// optimization.
+void PPCELFStreamer::emitGOTToPCRelLabel(const MCInst &Inst) {
+  // Get the last operand which contains the symbol.
+  const MCOperand &Operand = Inst.getOperand(Inst.getNumOperands() - 1);
+  assert(Operand.isExpr() && "Expecting an MCExpr.");
+  // Cast the last operand to MCSymbolRefExpr to get the symbol.
+  const MCExpr *Expr = Operand.getExpr();
+  const MCSymbolRefExpr *SymExpr = static_cast<const MCSymbolRefExpr *>(Expr);
+  assert(SymExpr->getKind() == MCSymbolRefExpr::VK_PPC_PCREL_OPT &&
+         "Expecting a symbol of type VK_PPC_PCREL_OPT");
+  MCSymbol *LabelSym =
+      getContext().getOrCreateSymbol(SymExpr->getSymbol().getName());
+  emitLabel(LabelSym, Inst.getLoc());
+}
+
+// This funciton checks if the parameter Inst is part of the setup for a link
+// time GOT PC Relative optimization. For example in this situation:
+// <MCInst PLDpc <MCOperand Reg:282> <MCOperand Expr:(glob_double@got@pcrel)>
+//   <MCOperand Imm:0> <MCOperand Expr:(.Lpcrel@<<invalid>>)>>
+// <MCInst SOME_LOAD <MCOperand Reg:22> <MCOperand Imm:0> <MCOperand Reg:282>
+//   <MCOperand Expr:(.Lpcrel@<<invalid>>)>>
+// The above is a pair of such instructions and this function will not return
+// None for either one of them. In both cases we are looking for the last
+// operand <MCOperand Expr:(.Lpcrel@<<invalid>>)> which needs to be an MCExpr
+// and has the flag MCSymbolRefExpr::VK_PPC_PCREL_OPT. After that we just look
+// at the opcode and in the case of PLDpc we will return true. For the load
+// (or store) this function will return false indicating it has found the second
+// instruciton in the pair.
+Optional<bool> llvm::isPartOfGOTToPCRelPair(const MCInst &Inst,
+                                            const MCSubtargetInfo &STI) {
+  // Need at least two operands.
+  if (Inst.getNumOperands() < 2)
+    return None;
+
+  unsigned LastOp = Inst.getNumOperands() - 1;
+  // The last operand needs to be an MCExpr and it needs to have a variant kind
+  // of VK_PPC_PCREL_OPT. If it does not satisfy these conditions it is not a
+  // link time GOT PC Rel opt instruction and we can ignore it and return None.
+  const MCOperand &Operand = Inst.getOperand(LastOp);
+  if (!Operand.isExpr())
+    return None;
+
+  // Check for the variant kind VK_PPC_PCREL_OPT in this expression.
+  const MCExpr *Expr = Operand.getExpr();
+  const MCSymbolRefExpr *SymExpr = static_cast<const MCSymbolRefExpr *>(Expr);
+  if (!SymExpr || SymExpr->getKind() != MCSymbolRefExpr::VK_PPC_PCREL_OPT)
+    return None;
+
+  return (Inst.getOpcode() == PPC::PLDpc);
+}
+
 MCELFStreamer *llvm::createPPCELFStreamer(
     MCContext &Context, std::unique_ptr<MCAsmBackend> MAB,
     std::unique_ptr<MCObjectWriter> OW,
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.h b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.h
index 51863232d0719..f44200104f320 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.h
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.h
@@ -43,8 +43,15 @@ class PPCELFStreamer : public MCELFStreamer {
   void emitLabel(MCSymbol *Symbol, SMLoc Loc = SMLoc()) override;
 private:
   void emitPrefixedInstruction(const MCInst &Inst, const MCSubtargetInfo &STI);
+  void emitGOTToPCRelReloc(const MCInst &Inst);
+  void emitGOTToPCRelLabel(const MCInst &Inst);
 };
 
+// Check if the instruction Inst is part of a pair of instructions that make up
+// a link time GOT PC Rel optimization.
+Optional<bool> isPartOfGOTToPCRelPair(const MCInst &Inst,
+                                      const MCSubtargetInfo &STI);
+
 MCELFStreamer *createPPCELFStreamer(MCContext &Context,
                                     std::unique_ptr<MCAsmBackend> MAB,
                                     std::unique_ptr<MCObjectWriter> OW,
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
index 16da62a74b8cd..222bf2fa82836 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
@@ -92,6 +92,36 @@ void PPCInstPrinter::printInst(const MCInst *MI, uint64_t Address,
     return;
   }
 
+  // Check if the last operand is an expression with the variant kind
+  // VK_PPC_PCREL_OPT. If this is the case then this is a linker optimization
+  // relocation and the .reloc directive needs to be added.
+  unsigned LastOp = MI->getNumOperands() - 1;
+  if (MI->getNumOperands() > 1) {
+    const MCOperand &Operand = MI->getOperand(LastOp);
+    if (Operand.isExpr()) {
+      const MCExpr *Expr = Operand.getExpr();
+      const MCSymbolRefExpr *SymExpr =
+          static_cast<const MCSymbolRefExpr *>(Expr);
+
+      if (SymExpr && SymExpr->getKind() == MCSymbolRefExpr::VK_PPC_PCREL_OPT) {
+        const MCSymbol &Symbol = SymExpr->getSymbol();
+        if (MI->getOpcode() == PPC::PLDpc) {
+          printInstruction(MI, Address, O);
+          O << "\n";
+          Symbol.print(O, &MAI);
+          O << ":";
+          return;
+        } else {
+          O << "\t.reloc ";
+          Symbol.print(O, &MAI);
+          O << "-8,R_PPC64_PCREL_OPT,.-(";
+          Symbol.print(O, &MAI);
+          O << "-8)\n";
+        }
+      }
+    }
+  }
+
   // Check for slwi/srwi mnemonics.
   if (MI->getOpcode() == PPC::RLWINM) {
     unsigned char SH = MI->getOperand(2).getImm();
diff --git a/llvm/lib/Target/PowerPC/PPC.h b/llvm/lib/Target/PowerPC/PPC.h
index 7e0aa2c6061dc..3106290442afa 100644
--- a/llvm/lib/Target/PowerPC/PPC.h
+++ b/llvm/lib/Target/PowerPC/PPC.h
@@ -107,6 +107,10 @@ namespace llvm {
     /// produce the relocation @got@pcrel. Fixup is VK_PPC_GOT_PCREL.
     MO_GOT_FLAG = 8,
 
+    // MO_PCREL_OPT_FLAG - If this bit is set the operand is part of a
+    // PC Relative linker optimization.
+    MO_PCREL_OPT_FLAG = 16,
+
     /// The next are not flags but distinct values.
     MO_ACCESS_MASK = 0xf00,
 
diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
index bf5fe741bac8a..bc869c39e3934 100644
--- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
@@ -27,6 +27,7 @@
 #include "PPCTargetStreamer.h"
 #include "TargetInfo/PowerPCTargetInfo.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/ADT/Twine.h"
@@ -47,6 +48,7 @@
 #include "llvm/IR/Module.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCDirectives.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstBuilder.h"
@@ -147,7 +149,15 @@ class PPCLinuxAsmPrinter : public PPCAsmPrinter {
 
 class PPCAIXAsmPrinter : public PPCAsmPrinter {
 private:
+  /// Symbols lowered from ExternalSymbolSDNodes, we will need to emit extern
+  /// linkage for them in AIX.
+  SmallPtrSet<MCSymbol *, 8> ExtSymSDNodeSymbols;
+
   static void ValidateGV(const GlobalVariable *GV);
+  // Record a list of GlobalAlias associated with a GlobalObject.
+  // This is used for AIX's extra-label-at-definition aliasing strategy.
+  DenseMap<const GlobalObject *, SmallVector<const GlobalAlias *, 1>>
+      GOAliasMap;
 
 public:
   PPCAIXAsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
@@ -167,9 +177,15 @@ class PPCAIXAsmPrinter : public PPCAsmPrinter {
 
   void emitFunctionDescriptor() override;
 
+  void emitFunctionEntryLabel() override;
+
   void emitEndOfAsmFile(Module &) override;
 
   void emitLinkage(const GlobalValue *GV, MCSymbol *GVSym) const override;
+
+  void emitInstruction(const MachineInstr *MI) override;
+
+  bool doFinalization(Module &M) override;
 };
 
 } // end anonymous namespace
@@ -1719,6 +1735,10 @@ void PPCAIXAsmPrinter::emitGlobalVariable(const GlobalVariable *GV) {
   emitLinkage(GV, EmittedInitSym);
   emitAlignment(getGVAlignment(GV, DL), GV);
   OutStreamer->emitLabel(EmittedInitSym);
+  // Emit aliasing label for global variable.
+  llvm::for_each(GOAliasMap[GV], [this](const GlobalAlias *Alias) {
+    OutStreamer->emitLabel(getSymbol(Alias));
+  });
   emitGlobalConstant(GV->getParent()->getDataLayout(), GV->getInitializer());
 }
 
@@ -1730,6 +1750,13 @@ void PPCAIXAsmPrinter::emitFunctionDescriptor() {
   // Emit function descriptor.
   OutStreamer->SwitchSection(
       cast<MCSymbolXCOFF>(CurrentFnDescSym)->getRepresentedCsect());
+
+  // Emit aliasing label for function descriptor csect.
+  llvm::for_each(GOAliasMap[&MF->getFunction()],
+                 [this](const GlobalAlias *Alias) {
+                   OutStreamer->emitLabel(getSymbol(Alias));
+                 });
+
   // Emit function entry point address.
   OutStreamer->emitValue(MCSymbolRefExpr::create(CurrentFnSym, OutContext),
                          PointerSize);
@@ -1745,6 +1772,16 @@ void PPCAIXAsmPrinter::emitFunctionDescriptor() {
   OutStreamer->SwitchSection(Current.first, Current.second);
 }
 
+void PPCAIXAsmPrinter::emitFunctionEntryLabel() {
+  PPCAsmPrinter::emitFunctionEntryLabel();
+  // Emit aliasing label for function entry point label.
+  llvm::for_each(
+      GOAliasMap[&MF->getFunction()], [this](const GlobalAlias *Alias) {
+        OutStreamer->emitLabel(
+            getObjFileLowering().getFunctionEntryPointSymbol(Alias, TM));
+      });
+}
+
 void PPCAIXAsmPrinter::emitEndOfAsmFile(Module &M) {
   // If there are no functions in this module, we will never need to reference
   // the TOC base.
@@ -1780,10 +1817,6 @@ void PPCAIXAsmPrinter::emitEndOfAsmFile(Module &M) {
 }
 
 bool PPCAIXAsmPrinter::doInitialization(Module &M) {
-  if (M.alias_size() > 0u)
-    report_fatal_error(
-        "module has aliases, which LLVM does not yet support for AIX");
-
   const bool Result = PPCAsmPrinter::doInitialization(M);
 
   auto setCsectAlignment = [this](const GlobalObject *GO) {
@@ -1809,9 +1842,67 @@ bool PPCAIXAsmPrinter::doInitialization(Module &M) {
   for (const auto &F : M)
     setCsectAlignment(&F);
 
+  // Construct an aliasing list for each GlobalObject.
+  for (const auto &Alias : M.aliases()) {
+    const GlobalObject *Base = Alias.getBaseObject();
+    if (!Base)
+      report_fatal_error(
+          "alias without a base object is not yet supported on AIX");
+    GOAliasMap[Base].push_back(&Alias);
+  }
+
   return Result;
 }
 
+void PPCAIXAsmPrinter::emitInstruction(const MachineInstr *MI) {
+  switch (MI->getOpcode()) {
+  default:
+    break;
+  case PPC::BL8:
+  case PPC::BL:
+  case PPC::BL8_NOP:
+  case PPC::BL_NOP: {
+    const MachineOperand &MO = MI->getOperand(0);
+    if (MO.isSymbol()) {
+      MCSymbolXCOFF *S =
+          cast<MCSymbolXCOFF>(OutContext.getOrCreateSymbol(MO.getSymbolName()));
+      if (!S->hasRepresentedCsectSet()) {
+        // On AIX, an undefined symbol needs to be associated with a
+        // MCSectionXCOFF to get the correct storage mapping class.
+        // In this case, XCOFF::XMC_PR.
+        MCSectionXCOFF *Sec = OutContext.getXCOFFSection(
+            S->getName(), XCOFF::XMC_PR, XCOFF::XTY_ER, XCOFF::C_EXT,
+            SectionKind::getMetadata());
+        S->setRepresentedCsect(Sec);
+      }
+      ExtSymSDNodeSymbols.insert(S);
+    }
+  } break;
+  case PPC::BL_TLS:
+  case PPC::BL8_TLS:
+  case PPC::BL8_TLS_:
+  case PPC::BL8_NOP_TLS:
+    report_fatal_error("TLS call not yet implemented");
+  case PPC::TAILB:
+  case PPC::TAILB8:
+  case PPC::TAILBA:
+  case PPC::TAILBA8:
+  case PPC::TAILBCTR:
+  case PPC::TAILBCTR8:
+    if (MI->getOperand(0).isSymbol())
+      report_fatal_error("Tail call for extern symbol not yet supported.");
+    break;
+  }
+  return PPCAsmPrinter::emitInstruction(MI);
+}
+
+bool PPCAIXAsmPrinter::doFinalization(Module &M) {
+  bool Ret = PPCAsmPrinter::doFinalization(M);
+  for (MCSymbol *Sym : ExtSymSDNodeSymbols)
+    OutStreamer->emitSymbolAttribute(Sym, MCSA_Extern);
+  return Ret;
+}
+
 /// createPPCAsmPrinterPass - Returns a pass that prints the PPC assembly code
 /// for a MachineFunction to the given output stream, in a format that the
 /// Darwin assembler can deal with.
diff --git a/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp b/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
index bd9174c1973dc..2ee394e9259d4 100644
--- a/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
@@ -1466,11 +1466,10 @@ void PPCFrameLowering::inlineStackProbe(MachineFunction &MF,
           .addImm(0)
           .addImm(32 - Log2(MaxAlign))
           .addImm(31);
-    BuildMI(PrologMBB, {MI}, DL, TII.get(isPPC64 ? PPC::STDUX : PPC::STWUX),
+    BuildMI(PrologMBB, {MI}, DL, TII.get(isPPC64 ? PPC::SUBFC8 : PPC::SUBFC),
             SPReg)
-        .addReg(FPReg)
-        .addReg(SPReg)
-        .addReg(ScratchReg);
+        .addReg(ScratchReg)
+        .addReg(SPReg);
   }
   // Probe residual part.
   if (NegResidualSize) {
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index ddfbd04e1ebc5..edc23b2673f38 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -809,6 +809,18 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
     else
       setOperationAction(ISD::MUL, MVT::v4i32, Custom);
 
+    if (Subtarget.isISA3_1()) {
+      setOperationAction(ISD::MUL, MVT::v2i64, Legal);
+      setOperationAction(ISD::UDIV, MVT::v2i64, Legal);
+      setOperationAction(ISD::SDIV, MVT::v2i64, Legal);
+      setOperationAction(ISD::UDIV, MVT::v4i32, Legal);
+      setOperationAction(ISD::SDIV, MVT::v4i32, Legal);
+      setOperationAction(ISD::UREM, MVT::v2i64, Legal);
+      setOperationAction(ISD::SREM, MVT::v2i64, Legal);
+      setOperationAction(ISD::UREM, MVT::v4i32, Legal);
+      setOperationAction(ISD::SREM, MVT::v4i32, Legal);
+    }
+
     setOperationAction(ISD::MUL, MVT::v8i16, Legal);
     setOperationAction(ISD::MUL, MVT::v16i8, Custom);
 
@@ -5335,13 +5347,18 @@ static SDValue transformCallee(const SDValue &Callee, SelectionDAG &DAG,
   // On AIX, direct function calls reference the symbol for the function's
   // entry point, which is named by prepending a "." before the function's
   // C-linkage name.
+  const auto getFunctionEntryPointSymbol = [&](StringRef SymName) {
+    auto &Context = DAG.getMachineFunction().getMMI().getContext();
+    return cast<MCSymbolXCOFF>(
+        Context.getOrCreateSymbol(Twine(".") + Twine(SymName)));
+  };
+
   const auto getAIXFuncEntryPointSymbolSDNode =
       [&](StringRef FuncName, bool IsDeclaration,
           const XCOFF::StorageClass &SC) {
-        auto &Context = DAG.getMachineFunction().getMMI().getContext();
+        MCSymbolXCOFF *S = getFunctionEntryPointSymbol(FuncName);
 
-        MCSymbolXCOFF *S = cast<MCSymbolXCOFF>(
-            Context.getOrCreateSymbol(Twine(".") + Twine(FuncName)));
+        auto &Context = DAG.getMachineFunction().getMMI().getContext();
 
         if (IsDeclaration && !S->hasRepresentedCsectSet()) {
           // On AIX, an undefined symbol needs to be associated with a
@@ -5367,31 +5384,29 @@ static SDValue transformCallee(const SDValue &Callee, SelectionDAG &DAG,
                                         UsePlt ? PPCII::MO_PLT : 0);
 
     assert(!isa<GlobalIFunc>(GV) && "IFunc is not supported on AIX.");
-    const GlobalObject *GO = cast<GlobalObject>(GV);
     const XCOFF::StorageClass SC =
-        TargetLoweringObjectFileXCOFF::getStorageClassForGlobal(GO);
-    return getAIXFuncEntryPointSymbolSDNode(GO->getName(), GO->isDeclaration(),
+        TargetLoweringObjectFileXCOFF::getStorageClassForGlobal(GV);
+    return getAIXFuncEntryPointSymbolSDNode(GV->getName(), GV->isDeclaration(),
                                             SC);
   }
 
   if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
     const char *SymName = S->getSymbol();
-    if (!Subtarget.isAIXABI())
-      return DAG.getTargetExternalSymbol(SymName, Callee.getValueType(),
-                                         UsePlt ? PPCII::MO_PLT : 0);
-
-    // If there exists a user-declared function whose name is the same as the
-    // ExternalSymbol's, then we pick up the user-declared version.
-    const Module *Mod = DAG.getMachineFunction().getFunction().getParent();
-    if (const Function *F =
-            dyn_cast_or_null<Function>(Mod->getNamedValue(SymName))) {
-      const XCOFF::StorageClass SC =
-          TargetLoweringObjectFileXCOFF::getStorageClassForGlobal(F);
-      return getAIXFuncEntryPointSymbolSDNode(F->getName(), F->isDeclaration(),
-                                              SC);
+    if (Subtarget.isAIXABI()) {
+      // If there exists a user-declared function whose name is the same as the
+      // ExternalSymbol's, then we pick up the user-declared version.
+      const Module *Mod = DAG.getMachineFunction().getFunction().getParent();
+      if (const Function *F =
+              dyn_cast_or_null<Function>(Mod->getNamedValue(SymName))) {
+        const XCOFF::StorageClass SC =
+            TargetLoweringObjectFileXCOFF::getStorageClassForGlobal(F);
+        return getAIXFuncEntryPointSymbolSDNode(F->getName(),
+                                                F->isDeclaration(), SC);
+      }
+      SymName = getFunctionEntryPointSymbol(SymName)->getName().data();
     }
-
-    return getAIXFuncEntryPointSymbolSDNode(SymName, true, XCOFF::C_EXT);
+    return DAG.getTargetExternalSymbol(SymName, Callee.getValueType(),
+                                       UsePlt ? PPCII::MO_PLT : 0);
   }
 
   // No transformation needed.
@@ -11950,18 +11965,34 @@ PPCTargetLowering::emitProbedAlloca(MachineInstr &MI,
   Register SPReg = isPPC64 ? PPC::X1 : PPC::R1;
   Register FinalStackPtr = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC);
   Register FramePointer = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC);
-
-  // Get the canonical FinalStackPtr like what
-  // PPCRegisterInfo::lowerDynamicAlloc does.
-  BuildMI(*MBB, {MI}, DL,
-          TII->get(isPPC64 ? PPC::PREPARE_PROBED_ALLOCA_64
-                           : PPC::PREPARE_PROBED_ALLOCA_32),
-          FramePointer)
-      .addDef(FinalStackPtr)
+  Register ActualNegSizeReg = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC);
+
+  // Since value of NegSizeReg might be realigned in prologepilog, insert a
+  // PREPARE_PROBED_ALLOCA pseudo instruction to get actual FramePointer and
+  // NegSize.
+  unsigned ProbeOpc;
+  if (!MRI.hasOneNonDBGUse(NegSizeReg))
+    ProbeOpc =
+        isPPC64 ? PPC::PREPARE_PROBED_ALLOCA_64 : PPC::PREPARE_PROBED_ALLOCA_32;
+  else
+    // By introducing PREPARE_PROBED_ALLOCA_NEGSIZE_OPT, ActualNegSizeReg
+    // and NegSizeReg will be allocated in the same phyreg to avoid
+    // redundant copy when NegSizeReg has only one use which is current MI and
+    // will be replaced by PREPARE_PROBED_ALLOCA then.
+    ProbeOpc = isPPC64 ? PPC::PREPARE_PROBED_ALLOCA_NEGSIZE_SAME_REG_64
+                       : PPC::PREPARE_PROBED_ALLOCA_NEGSIZE_SAME_REG_32;
+  BuildMI(*MBB, {MI}, DL, TII->get(ProbeOpc), FramePointer)
+      .addDef(ActualNegSizeReg)
       .addReg(NegSizeReg)
       .add(MI.getOperand(2))
       .add(MI.getOperand(3));
 
+  // Calculate final stack pointer, which equals to SP + ActualNegSize.
+  BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::ADD8 : PPC::ADD4),
+          FinalStackPtr)
+      .addReg(SPReg)
+      .addReg(ActualNegSizeReg);
+
   // Materialize a scratch register for update.
   int64_t NegProbeSize = -(int64_t)ProbeSize;
   assert(isInt<32>(NegProbeSize) && "Unhandled probe size!");
@@ -11982,7 +12013,7 @@ PPCTargetLowering::emitProbedAlloca(MachineInstr &MI,
     // Probing leading residual part.
     Register Div = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC);
     BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::DIVD : PPC::DIVW), Div)
-        .addReg(NegSizeReg)
+        .addReg(ActualNegSizeReg)
         .addReg(ScratchReg);
     Register Mul = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC);
     BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::MULLD : PPC::MULLW), Mul)
@@ -11991,7 +12022,7 @@ PPCTargetLowering::emitProbedAlloca(MachineInstr &MI,
     Register NegMod = MRI.createVirtualRegister(isPPC64 ? G8RC : GPRC);
     BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::SUBF8 : PPC::SUBF), NegMod)
         .addReg(Mul)
-        .addReg(NegSizeReg);
+        .addReg(ActualNegSizeReg);
     BuildMI(*MBB, {MI}, DL, TII->get(isPPC64 ? PPC::STDUX : PPC::STWUX), SPReg)
         .addReg(FramePointer)
         .addReg(SPReg)
@@ -16377,31 +16408,56 @@ bool PPCTargetLowering::isFMAFasterThanFMulAndFAdd(const Function &F,
   }
 }
 
-// Currently this is a copy from AArch64TargetLowering::isProfitableToHoist.
-// FIXME: add more patterns which are profitable to hoist.
+// FIXME: add more patterns which are not profitable to hoist.
 bool PPCTargetLowering::isProfitableToHoist(Instruction *I) const {
-  if (I->getOpcode() != Instruction::FMul)
-    return true;
-
   if (!I->hasOneUse())
     return true;
 
   Instruction *User = I->user_back();
   assert(User && "A single use instruction with no uses.");
 
-  if (User->getOpcode() != Instruction::FSub &&
-      User->getOpcode() != Instruction::FAdd)
-    return true;
+  switch (I->getOpcode()) {
+  case Instruction::FMul: {
+    // Don't break FMA, PowerPC prefers FMA.
+    if (User->getOpcode() != Instruction::FSub &&
+        User->getOpcode() != Instruction::FAdd)
+      return true;
 
-  const TargetOptions &Options = getTargetMachine().Options;
-  const Function *F = I->getFunction();
-  const DataLayout &DL = F->getParent()->getDataLayout();
-  Type *Ty = User->getOperand(0)->getType();
-
-  return !(
-      isFMAFasterThanFMulAndFAdd(*F, Ty) &&
-      isOperationLegalOrCustom(ISD::FMA, getValueType(DL, Ty)) &&
-      (Options.AllowFPOpFusion == FPOpFusion::Fast || Options.UnsafeFPMath));
+    const TargetOptions &Options = getTargetMachine().Options;
+    const Function *F = I->getFunction();
+    const DataLayout &DL = F->getParent()->getDataLayout();
+    Type *Ty = User->getOperand(0)->getType();
+
+    return !(
+        isFMAFasterThanFMulAndFAdd(*F, Ty) &&
+        isOperationLegalOrCustom(ISD::FMA, getValueType(DL, Ty)) &&
+        (Options.AllowFPOpFusion == FPOpFusion::Fast || Options.UnsafeFPMath));
+  }
+  case Instruction::Load: {
+    // Don't break "store (load float*)" pattern, this pattern will be combined
+    // to "store (load int32)" in later InstCombine pass. See function
+    // combineLoadToOperationType. On PowerPC, loading a float point takes more
+    // cycles than loading a 32 bit integer.
+    LoadInst *LI = cast<LoadInst>(I);
+    // For the loads that combineLoadToOperationType does nothing, like
+    // ordered load, it should be profitable to hoist them.
+    // For swifterror load, it can only be used for pointer to pointer type, so
+    // later type check should get rid of this case.
+    if (!LI->isUnordered())
+      return true;
+
+    if (User->getOpcode() != Instruction::Store)
+      return true;
+
+    if (I->getType()->getTypeID() != Type::FloatTyID)
+      return true;
+
+    return false;
+  }
+  default:
+    return true;
+  }
+  return true;
 }
 
 const MCPhysReg *
diff --git a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
index 1c457d4170d54..6956c40a70be5 100644
--- a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
+++ b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
@@ -431,9 +431,14 @@ def PROBED_ALLOCA_64 : PPCCustomInserterPseudo<(outs g8rc:$result),
                          (ins g8rc:$negsize, memri:$fpsi), "#PROBED_ALLOCA_64",
                            [(set i64:$result,
                              (PPCprobedalloca i64:$negsize, iaddr:$fpsi))]>;
-def PREPARE_PROBED_ALLOCA_64 : PPCEmitTimePseudo<(outs g8rc:$fp,
-    g8rc:$sp),
+def PREPARE_PROBED_ALLOCA_64 : PPCEmitTimePseudo<(outs
+    g8rc:$fp, g8rc:$actual_negsize),
     (ins g8rc:$negsize, memri:$fpsi), "#PREPARE_PROBED_ALLOCA_64", []>;
+def PREPARE_PROBED_ALLOCA_NEGSIZE_SAME_REG_64 : PPCEmitTimePseudo<(outs
+    g8rc:$fp, g8rc:$actual_negsize),
+    (ins g8rc:$negsize, memri:$fpsi),
+    "#PREPARE_PROBED_ALLOCA_NEGSIZE_SAME_REG_64", []>,
+    RegConstraint<"$actual_negsize = $negsize">;
 def PROBED_STACKALLOC_64 : PPCEmitTimePseudo<(outs g8rc:$scratch, g8rc:$temp),
     (ins i64imm:$stacksize),
     "#PROBED_STACKALLOC_64", []>;
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
index 11c97210ead9b..99e25bb130ce4 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
@@ -1675,6 +1675,10 @@ bool PPCInstrInfo::PredicateInstruction(MachineInstr &MI,
       bool isPPC64 = Subtarget.isPPC64();
       MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZLR8 : PPC::BDNZLR)
                                       : (isPPC64 ? PPC::BDZLR8 : PPC::BDZLR)));
+      // Need add Def and Use for CTR implicit operand.
+      MachineInstrBuilder(*MI.getParent()->getParent(), MI)
+          .addReg(Pred[1].getReg(), RegState::Implicit)
+          .addReg(Pred[1].getReg(), RegState::ImplicitDefine);
     } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
       MI.setDesc(get(PPC::BCLR));
       MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
@@ -1694,6 +1698,10 @@ bool PPCInstrInfo::PredicateInstruction(MachineInstr &MI,
       bool isPPC64 = Subtarget.isPPC64();
       MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ)
                                       : (isPPC64 ? PPC::BDZ8 : PPC::BDZ)));
+      // Need add Def and Use for CTR implicit operand.
+      MachineInstrBuilder(*MI.getParent()->getParent(), MI)
+          .addReg(Pred[1].getReg(), RegState::Implicit)
+          .addReg(Pred[1].getReg(), RegState::ImplicitDefine);
     } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
       MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
       MI.RemoveOperand(0);
@@ -1734,19 +1742,24 @@ bool PPCInstrInfo::PredicateInstruction(MachineInstr &MI,
       MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8 : PPC::BCCTR8)
                              : (setLR ? PPC::BCCTRL : PPC::BCCTR)));
       MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
-      return true;
     } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
       MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8n : PPC::BCCTR8n)
                              : (setLR ? PPC::BCCTRLn : PPC::BCCTRn)));
       MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
-      return true;
+    } else {
+      MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCCTRL8 : PPC::BCCCTR8)
+                             : (setLR ? PPC::BCCCTRL : PPC::BCCCTR)));
+      MachineInstrBuilder(*MI.getParent()->getParent(), MI)
+          .addImm(Pred[0].getImm())
+          .add(Pred[1]);
     }
 
-    MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCCTRL8 : PPC::BCCCTR8)
-                           : (setLR ? PPC::BCCCTRL : PPC::BCCCTR)));
-    MachineInstrBuilder(*MI.getParent()->getParent(), MI)
-        .addImm(Pred[0].getImm())
-        .add(Pred[1]);
+    // Need add Def and Use for LR implicit operand.
+    if (setLR)
+      MachineInstrBuilder(*MI.getParent()->getParent(), MI)
+          .addReg(isPPC64 ? PPC::LR8 : PPC::LR, RegState::Implicit)
+          .addReg(isPPC64 ? PPC::LR8 : PPC::LR, RegState::ImplicitDefine);
+
     return true;
   }
 
@@ -2270,7 +2283,8 @@ PPCInstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const {
       {MO_PLT, "ppc-plt"},
       {MO_PIC_FLAG, "ppc-pic"},
       {MO_PCREL_FLAG, "ppc-pcrel"},
-      {MO_GOT_FLAG, "ppc-got"}};
+      {MO_GOT_FLAG, "ppc-got"},
+      {MO_PCREL_OPT_FLAG, "ppc-opt-pcrel"}};
   return makeArrayRef(TargetFlags);
 }
 
@@ -2655,6 +2669,15 @@ const unsigned *PPCInstrInfo::getLoadOpcodesForSpillArray() const {
 
 void PPCInstrInfo::fixupIsDeadOrKill(MachineInstr &StartMI, MachineInstr &EndMI,
                                      unsigned RegNo) const {
+  // Conservatively clear kill flag for the register if the instructions are in
+  // different basic blocks and in SSA form, because the kill flag may no longer
+  // be right. There is no need to bother with dead flags since defs with no
+  // uses will be handled by DCE.
+  MachineRegisterInfo &MRI = StartMI.getParent()->getParent()->getRegInfo();
+  if (MRI.isSSA() && (StartMI.getParent() != EndMI.getParent())) {
+    MRI.clearKillFlags(RegNo);
+    return;
+  }
 
   // Instructions between [StartMI, EndMI] should be in same basic block.
   assert((StartMI.getParent() == EndMI.getParent()) &&
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.h b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
index d98597f483406..43973c627fcf1 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
@@ -565,14 +565,18 @@ class PPCInstrInfo : public PPCGenInstrInfo {
                              int64_t OffsetImm) const;
 
   /// Fixup killed/dead flag for register \p RegNo between instructions [\p
-  /// StartMI, \p EndMI]. Some PostRA transformations may violate register
-  /// killed/dead flags semantics, this function can be called to fix up. Before
-  /// calling this function,
+  /// StartMI, \p EndMI]. Some pre-RA or post-RA transformations may violate
+  /// register killed/dead flags semantics, this function can be called to fix
+  /// up. Before calling this function,
   /// 1. Ensure that \p RegNo liveness is killed after instruction \p EndMI.
   /// 2. Ensure that there is no new definition between (\p StartMI, \p EndMI)
   ///    and possible definition for \p RegNo is \p StartMI or \p EndMI.
-  /// 3. Ensure that all instructions between [\p StartMI, \p EndMI] are in same
-  ///    basic block.
+  /// 3. We can do accurate fixup for the case when all instructions between
+  ///    [\p StartMI, \p EndMI] are in same basic block.
+  /// 4. For the case when \p StartMI and \p EndMI are not in same basic block,
+  ///    we conservatively clear kill flag for all uses of \p RegNo for pre-RA
+  ///    and for post-RA, we give an assertion as without reaching definition
+  ///    analysis post-RA, \p StartMI and \p EndMI are hard to keep right.
   void fixupIsDeadOrKill(MachineInstr &StartMI, MachineInstr &EndMI,
                          unsigned RegNo) const;
   void replaceInstrWithLI(MachineInstr &MI, const LoadImmediateInfo &LII) const;
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
index 673ab63039cf7..c565758973bf5 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
@@ -1406,9 +1406,14 @@ def PROBED_ALLOCA_32 : PPCCustomInserterPseudo<(outs gprc:$result),
                          (ins gprc:$negsize, memri:$fpsi), "#PROBED_ALLOCA_32",
                            [(set i32:$result,
                              (PPCprobedalloca i32:$negsize, iaddr:$fpsi))]>;
-def PREPARE_PROBED_ALLOCA_32 : PPCEmitTimePseudo<(outs gprc:$fp,
-    gprc:$sp),
+def PREPARE_PROBED_ALLOCA_32 : PPCEmitTimePseudo<(outs
+    gprc:$fp, gprc:$actual_negsize),
     (ins gprc:$negsize, memri:$fpsi), "#PREPARE_PROBED_ALLOCA_32", []>;
+def PREPARE_PROBED_ALLOCA_NEGSIZE_SAME_REG_32 : PPCEmitTimePseudo<(outs
+    gprc:$fp, gprc:$actual_negsize),
+    (ins gprc:$negsize, memri:$fpsi),
+    "#PREPARE_PROBED_ALLOCA_NEGSIZE_SAME_REG_32", []>,
+    RegConstraint<"$actual_negsize = $negsize">;
 def PROBED_STACKALLOC_32 : PPCEmitTimePseudo<(outs gprc:$scratch, gprc:$temp),
     (ins i64imm:$stacksize),
     "#PROBED_STACKALLOC_32", []>;
@@ -3246,9 +3251,13 @@ def : Pat<(PPCcall (i32 texternalsym:$dst)),
 // Calls for AIX only
 def : Pat<(PPCcall (i32 mcsym:$dst)),
           (BL mcsym:$dst)>;
+
 def : Pat<(PPCcall_nop (i32 mcsym:$dst)),
           (BL_NOP mcsym:$dst)>;
 
+def : Pat<(PPCcall_nop (i32 texternalsym:$dst)),
+          (BL_NOP texternalsym:$dst)>;
+
 def : Pat<(PPCtc_return (i32 tglobaladdr:$dst),  imm:$imm),
           (TCRETURNdi tglobaladdr:$dst, imm:$imm)>;
 
diff --git a/llvm/lib/Target/PowerPC/PPCInstrPrefix.td b/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
index 742693083432c..b468a8f318ee3 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
@@ -245,7 +245,7 @@ class VXForm_RD5_N3_VB5<bits<11> xo, dag OOL, dag IOL, string asmstr,
 // VX-Form: [PO VRT RA VRB XO].
 // Destructive (insert) forms are suffixed with _ins.
 class VXForm_VTB5_RA5_ins<bits<11> xo, string opc, list<dag> pattern>
-  : VXForm_1<xo, (outs vrrc:$vD), (ins vrrc:$vDi, g8rc:$rA, vrrc:$vB),
+  : VXForm_1<xo, (outs vrrc:$vD), (ins vrrc:$vDi, gprc:$rA, vrrc:$vB),
              !strconcat(opc, " $vD, $rA, $vB"), IIC_VecGeneral, pattern>,
              RegConstraint<"$vDi = $vD">, NoEncode<"$vDi">;
 
@@ -836,32 +836,32 @@ let Predicates = [IsISA3_1] in {
   def VINSBVLX :
     VXForm_VTB5_RA5_ins<15, "vinsbvlx",
                         [(set v16i8:$vD,
-                              (int_ppc_altivec_vinsbvlx v16i8:$vDi, i64:$rA,
+                              (int_ppc_altivec_vinsbvlx v16i8:$vDi, i32:$rA,
                                                         v16i8:$vB))]>;
   def VINSBVRX :
     VXForm_VTB5_RA5_ins<271, "vinsbvrx",
                         [(set v16i8:$vD,
-                              (int_ppc_altivec_vinsbvrx v16i8:$vDi, i64:$rA,
+                              (int_ppc_altivec_vinsbvrx v16i8:$vDi, i32:$rA,
                                                         v16i8:$vB))]>;
   def VINSHVLX :
     VXForm_VTB5_RA5_ins<79, "vinshvlx",
                         [(set v8i16:$vD,
-                              (int_ppc_altivec_vinshvlx v8i16:$vDi, i64:$rA,
+                              (int_ppc_altivec_vinshvlx v8i16:$vDi, i32:$rA,
                                                         v8i16:$vB))]>;
   def VINSHVRX :
     VXForm_VTB5_RA5_ins<335, "vinshvrx",
                         [(set v8i16:$vD,
-                              (int_ppc_altivec_vinshvrx v8i16:$vDi, i64:$rA,
+                              (int_ppc_altivec_vinshvrx v8i16:$vDi, i32:$rA,
                                                         v8i16:$vB))]>;
   def VINSWVLX :
     VXForm_VTB5_RA5_ins<143, "vinswvlx",
                         [(set v4i32:$vD,
-                              (int_ppc_altivec_vinswvlx v4i32:$vDi, i64:$rA,
+                              (int_ppc_altivec_vinswvlx v4i32:$vDi, i32:$rA,
                                                         v4i32:$vB))]>;
   def VINSWVRX :
     VXForm_VTB5_RA5_ins<399, "vinswvrx",
                         [(set v4i32:$vD,
-                              (int_ppc_altivec_vinswvrx v4i32:$vDi, i64:$rA,
+                              (int_ppc_altivec_vinswvrx v4i32:$vDi, i32:$rA,
                                                         v4i32:$vB))]>;
   def VINSBLX :
     VXForm_VRT5_RAB5_ins<527, "vinsblx",
@@ -972,7 +972,49 @@ let Predicates = [IsISA3_1] in {
                          "vclrrb $vD, $vA, $rB", IIC_VecGeneral,
                          [(set v16i8:$vD,
                                (int_ppc_altivec_vclrrb v16i8:$vA, i32:$rB))]>;
-
+  def VMULLD : VXForm_1<457, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
+                        "vmulld $vD, $vA, $vB", IIC_VecGeneral,
+                        [(set v2i64:$vD, (mul v2i64:$vA, v2i64:$vB))]>;
+  def VMULHSW : VXForm_1<905, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
+                         "vmulhsw $vD, $vA, $vB", IIC_VecGeneral, []>;
+  def VMULHUW : VXForm_1<649, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
+                         "vmulhuw $vD, $vA, $vB", IIC_VecGeneral, []>;
+  def VMULHSD : VXForm_1<969, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
+                         "vmulhsd $vD, $vA, $vB", IIC_VecGeneral, []>;
+  def VMULHUD : VXForm_1<713, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
+                         "vmulhud $vD, $vA, $vB", IIC_VecGeneral, []>;
+  def VMODSW : VXForm_1<1931, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
+                        "vmodsw $vD, $vA, $vB", IIC_VecGeneral,
+                        [(set v4i32:$vD, (srem v4i32:$vA, v4i32:$vB))]>;
+  def VMODUW : VXForm_1<1675, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
+                        "vmoduw $vD, $vA, $vB", IIC_VecGeneral,
+                        [(set v4i32:$vD, (urem v4i32:$vA, v4i32:$vB))]>;
+  def VMODSD : VXForm_1<1995, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
+                        "vmodsd $vD, $vA, $vB", IIC_VecGeneral,
+                        [(set v2i64:$vD, (srem v2i64:$vA, v2i64:$vB))]>;
+  def VMODUD : VXForm_1<1739, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
+                        "vmodud $vD, $vA, $vB", IIC_VecGeneral,
+                        [(set v2i64:$vD, (urem v2i64:$vA, v2i64:$vB))]>;
+  def VDIVSW : VXForm_1<395, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
+                        "vdivsw $vD, $vA, $vB", IIC_VecGeneral,
+                        [(set v4i32:$vD, (sdiv v4i32:$vA, v4i32:$vB))]>;
+  def VDIVUW : VXForm_1<139, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
+                        "vdivuw $vD, $vA, $vB", IIC_VecGeneral,
+                        [(set v4i32:$vD, (udiv v4i32:$vA, v4i32:$vB))]>;
+  def VDIVSD : VXForm_1<459, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
+                        "vdivsd $vD, $vA, $vB", IIC_VecGeneral,
+                        [(set v2i64:$vD, (sdiv v2i64:$vA, v2i64:$vB))]>;
+  def VDIVUD : VXForm_1<203, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
+                        "vdivud $vD, $vA, $vB", IIC_VecGeneral,
+                        [(set v2i64:$vD, (udiv v2i64:$vA, v2i64:$vB))]>;
+  def VDIVESW : VXForm_1<907, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
+                         "vdivesw $vD, $vA, $vB", IIC_VecGeneral, []>;
+  def VDIVEUW : VXForm_1<651, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
+                         "vdiveuw $vD, $vA, $vB", IIC_VecGeneral, []>;
+  def VDIVESD : VXForm_1<971, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
+                         "vdivesd $vD, $vA, $vB", IIC_VecGeneral, []>;
+  def VDIVEUD : VXForm_1<715, (outs vrrc:$vD), (ins vrrc:$vA, vrrc:$vB),
+                         "vdiveud $vD, $vA, $vB", IIC_VecGeneral, []>;
   def XVTLSBB : XX2_BF3_XO5_XB6_XO9<60, 2, 475, (outs crrc:$BF), (ins vsrc:$XB),
                                     "xvtlsbb $BF, $XB", IIC_VecGeneral, []>;
 
@@ -1052,7 +1094,7 @@ let Predicates = [IsISA3_1] in {
             (v4i32 (COPY_TO_REGCLASS (XXGENPCVWM $VRB, imm:$IMM), VRRC))>;
   def : Pat<(v2i64 (int_ppc_vsx_xxgenpcvdm v2i64:$VRB, imm:$IMM)),
             (v2i64 (COPY_TO_REGCLASS (XXGENPCVDM $VRB, imm:$IMM), VRRC))>;
-  def : Pat<(i32 (int_ppc_vsx_xvtlsbb v16i8:$XB, -1)),
+  def : Pat<(i32 (int_ppc_vsx_xvtlsbb v16i8:$XB, 1)),
             (EXTRACT_SUBREG (XVTLSBB (COPY_TO_REGCLASS $XB, VSRC)), sub_lt)>;
   def : Pat<(i32 (int_ppc_vsx_xvtlsbb v16i8:$XB, 0)),
             (EXTRACT_SUBREG (XVTLSBB (COPY_TO_REGCLASS $XB, VSRC)), sub_eq)>;
diff --git a/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp b/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp
index 236f98f32e18e..1f51a55b1683e 100644
--- a/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp
+++ b/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp
@@ -100,6 +100,8 @@ static MCOperand GetSymbolRef(const MachineOperand &MO, const MCSymbol *Symbol,
         MIOpcode == PPC::BL8_NOTOC) {
       RefKind = MCSymbolRefExpr::VK_PPC_NOTOC;
     }
+    if (MO.getTargetFlags() == PPCII::MO_PCREL_OPT_FLAG)
+      RefKind = MCSymbolRefExpr::VK_PPC_PCREL_OPT;
   }
 
   const MCExpr *Expr = MCSymbolRefExpr::create(Symbol, RefKind, Ctx);
diff --git a/llvm/lib/Target/PowerPC/PPCPreEmitPeephole.cpp b/llvm/lib/Target/PowerPC/PPCPreEmitPeephole.cpp
index 4ea714ff15f7a..65539166e5efb 100644
--- a/llvm/lib/Target/PowerPC/PPCPreEmitPeephole.cpp
+++ b/llvm/lib/Target/PowerPC/PPCPreEmitPeephole.cpp
@@ -21,8 +21,8 @@
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/MC/MCContext.h"
 #include "llvm/Support/CommandLine.h"
-#include "llvm/ADT/Statistic.h"
 #include "llvm/Support/Debug.h"
 
 using namespace llvm;
@@ -43,6 +43,46 @@ RunPreEmitPeephole("ppc-late-peephole", cl::Hidden, cl::init(true),
                    cl::desc("Run pre-emit peephole optimizations."));
 
 namespace {
+
+static bool hasPCRelativeForm(MachineInstr &Use) {
+  switch (Use.getOpcode()) {
+  default:
+    return false;
+  case PPC::LBZ:
+  case PPC::LBZ8:
+  case PPC::LHA:
+  case PPC::LHA8:
+  case PPC::LHZ:
+  case PPC::LHZ8:
+  case PPC::LWZ:
+  case PPC::LWZ8:
+  case PPC::STB:
+  case PPC::STB8:
+  case PPC::STH:
+  case PPC::STH8:
+  case PPC::STW:
+  case PPC::STW8:
+  case PPC::LD:
+  case PPC::STD:
+  case PPC::LWA:
+  case PPC::LXSD:
+  case PPC::LXSSP:
+  case PPC::LXV:
+  case PPC::STXSD:
+  case PPC::STXSSP:
+  case PPC::STXV:
+  case PPC::LFD:
+  case PPC::LFS:
+  case PPC::STFD:
+  case PPC::STFS:
+  case PPC::DFLOADf32:
+  case PPC::DFLOADf64:
+  case PPC::DFSTOREf32:
+  case PPC::DFSTOREf64:
+    return true;
+  }
+}
+
   class PPCPreEmitPeephole : public MachineFunctionPass {
   public:
     static char ID;
@@ -172,6 +212,135 @@ namespace {
       return !InstrsToErase.empty();
     }
 
+    // Check if this instruction is a PLDpc that is part of a GOT indirect
+    // access.
+    bool isGOTPLDpc(MachineInstr &Instr) {
+      if (Instr.getOpcode() != PPC::PLDpc)
+        return false;
+
+      // The result must be a register.
+      const MachineOperand &LoadedAddressReg = Instr.getOperand(0);
+      if (!LoadedAddressReg.isReg())
+        return false;
+
+      // Make sure that this is a global symbol.
+      const MachineOperand &SymbolOp = Instr.getOperand(1);
+      if (!SymbolOp.isGlobal())
+        return false;
+
+      // Finally return true only if the GOT flag is present.
+      return (SymbolOp.getTargetFlags() & PPCII::MO_GOT_FLAG);
+    }
+
+    bool addLinkerOpt(MachineBasicBlock &MBB, const TargetRegisterInfo *TRI) {
+      MachineFunction *MF = MBB.getParent();
+      // Add this linker opt only if we are using PC Relative memops.
+      if (!MF->getSubtarget<PPCSubtarget>().isUsingPCRelativeCalls())
+        return false;
+
+      // Struct to keep track of one def/use pair for a GOT indirect access.
+      struct GOTDefUsePair {
+        MachineBasicBlock::iterator DefInst;
+        MachineBasicBlock::iterator UseInst;
+        Register DefReg;
+        Register UseReg;
+        bool StillValid;
+      };
+      // Vector of def/ues pairs in this basic block.
+      SmallVector<GOTDefUsePair, 4> CandPairs;
+      SmallVector<GOTDefUsePair, 4> ValidPairs;
+      bool MadeChange = false;
+
+      // Run through all of the instructions in the basic block and try to
+      // collect potential pairs of GOT indirect access instructions.
+      for (auto BBI = MBB.instr_begin(); BBI != MBB.instr_end(); ++BBI) {
+        // Look for the initial GOT indirect load.
+        if (isGOTPLDpc(*BBI)) {
+          GOTDefUsePair CurrentPair{BBI, MachineBasicBlock::iterator(),
+                                    BBI->getOperand(0).getReg(),
+                                    PPC::NoRegister, true};
+          CandPairs.push_back(CurrentPair);
+          continue;
+        }
+
+        // We haven't encountered any new PLD instructions, nothing to check.
+        if (CandPairs.empty())
+          continue;
+
+        // Run through the candidate pairs and see if any of the registers
+        // defined in the PLD instructions are used by this instruction.
+        // Note: the size of CandPairs can change in the loop.
+        for (unsigned Idx = 0; Idx < CandPairs.size(); Idx++) {
+          GOTDefUsePair &Pair = CandPairs[Idx];
+          // The instruction does not use or modify this PLD's def reg,
+          // ignore it.
+          if (!BBI->readsRegister(Pair.DefReg, TRI) &&
+              !BBI->modifiesRegister(Pair.DefReg, TRI))
+            continue;
+
+          // The use needs to be used in the address compuation and not
+          // as the register being stored for a store.
+          const MachineOperand *UseOp =
+              hasPCRelativeForm(*BBI) ? &BBI->getOperand(2) : nullptr;
+
+          // Check for a valid use.
+          if (UseOp && UseOp->isReg() && UseOp->getReg() == Pair.DefReg &&
+              UseOp->isUse() && UseOp->isKill()) {
+            Pair.UseInst = BBI;
+            Pair.UseReg = BBI->getOperand(0).getReg();
+            ValidPairs.push_back(Pair);
+          }
+          CandPairs.erase(CandPairs.begin() + Idx);
+        }
+      }
+
+      // Go through all of the pairs and check for any more valid uses.
+      for (auto Pair = ValidPairs.begin(); Pair != ValidPairs.end(); Pair++) {
+        // We shouldn't be here if we don't have a valid pair.
+        assert(Pair->UseInst.isValid() && Pair->StillValid &&
+               "Kept an invalid def/use pair for GOT PCRel opt");
+        // We have found a potential pair. Search through the instructions
+        // between the def and the use to see if it is valid to mark this as a
+        // linker opt.
+        MachineBasicBlock::iterator BBI = Pair->DefInst;
+        ++BBI;
+        for (; BBI != Pair->UseInst; ++BBI) {
+          if (BBI->readsRegister(Pair->UseReg, TRI) ||
+              BBI->modifiesRegister(Pair->UseReg, TRI)) {
+            Pair->StillValid = false;
+            break;
+          }
+        }
+
+        if (!Pair->StillValid)
+          continue;
+
+        // The load/store instruction that uses the address from the PLD will
+        // either use a register (for a store) or define a register (for the
+        // load). That register will be added as an implicit def to the PLD
+        // and as an implicit use on the second memory op. This is a precaution
+        // to prevent future passes from using that register between the two
+        // instructions.
+        MachineOperand ImplDef =
+            MachineOperand::CreateReg(Pair->UseReg, true, true);
+        MachineOperand ImplUse =
+            MachineOperand::CreateReg(Pair->UseReg, false, true);
+        Pair->DefInst->addOperand(ImplDef);
+        Pair->UseInst->addOperand(ImplUse);
+
+        // Create the symbol.
+        MCContext &Context = MF->getContext();
+        MCSymbol *Symbol =
+            Context.createTempSymbol(Twine("pcrel"), false, false);
+        MachineOperand PCRelLabel =
+            MachineOperand::CreateMCSymbol(Symbol, PPCII::MO_PCREL_OPT_FLAG);
+        Pair->DefInst->addOperand(*MF, PCRelLabel);
+        Pair->UseInst->addOperand(*MF, PCRelLabel);
+        MadeChange |= true;
+      }
+      return MadeChange;
+    }
+
     bool runOnMachineFunction(MachineFunction &MF) override {
       if (skipFunction(MF.getFunction()) || !RunPreEmitPeephole) {
         // Remove UNENCODED_NOP even when this pass is disabled.
@@ -192,6 +361,7 @@ namespace {
       SmallVector<MachineInstr *, 4> InstrsToErase;
       for (MachineBasicBlock &MBB : MF) {
         Changed |= removeRedundantLIs(MBB, TRI);
+        Changed |= addLinkerOpt(MBB, TRI);
         for (MachineInstr &MI : MBB) {
           unsigned Opc = MI.getOpcode();
           if (Opc == PPC::UNENCODED_NOP) {
diff --git a/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp b/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
index 35f5e1fbebcdf..ed8948a639728 100644
--- a/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
@@ -624,21 +624,30 @@ void PPCRegisterInfo::lowerPrepareProbedAlloca(
   bool LP64 = TM.isPPC64();
   DebugLoc dl = MI.getDebugLoc();
   Register FramePointer = MI.getOperand(0).getReg();
-  Register FinalStackPtr = MI.getOperand(1).getReg();
+  const Register ActualNegSizeReg = MI.getOperand(1).getReg();
   bool KillNegSizeReg = MI.getOperand(2).isKill();
   Register NegSizeReg = MI.getOperand(2).getReg();
-  prepareDynamicAlloca(II, NegSizeReg, KillNegSizeReg, FramePointer);
-  if (LP64) {
-    BuildMI(MBB, II, dl, TII.get(PPC::ADD8), FinalStackPtr)
-        .addReg(PPC::X1)
-        .addReg(NegSizeReg, getKillRegState(KillNegSizeReg));
-
-  } else {
-    BuildMI(MBB, II, dl, TII.get(PPC::ADD4), FinalStackPtr)
-        .addReg(PPC::R1)
-        .addReg(NegSizeReg, getKillRegState(KillNegSizeReg));
+  const MCInstrDesc &CopyInst = TII.get(LP64 ? PPC::OR8 : PPC::OR);
+  // RegAllocator might allocate FramePointer and NegSizeReg in the same phyreg.
+  if (FramePointer == NegSizeReg) {
+    assert(KillNegSizeReg && "FramePointer is a def and NegSizeReg is an use, "
+                             "NegSizeReg should be killed");
+    // FramePointer is clobbered earlier than the use of NegSizeReg in
+    // prepareDynamicAlloca, save NegSizeReg in ActualNegSizeReg to avoid
+    // misuse.
+    BuildMI(MBB, II, dl, CopyInst, ActualNegSizeReg)
+        .addReg(NegSizeReg)
+        .addReg(NegSizeReg);
+    NegSizeReg = ActualNegSizeReg;
+    KillNegSizeReg = false;
   }
-
+  prepareDynamicAlloca(II, NegSizeReg, KillNegSizeReg, FramePointer);
+  // NegSizeReg might be updated in prepareDynamicAlloca if MaxAlign >
+  // TargetAlign.
+  if (NegSizeReg != ActualNegSizeReg)
+    BuildMI(MBB, II, dl, CopyInst, ActualNegSizeReg)
+        .addReg(NegSizeReg)
+        .addReg(NegSizeReg);
   MBB.erase(II);
 }
 
@@ -1084,7 +1093,9 @@ PPCRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
 
   if (FPSI && FrameIndex == FPSI &&
       (OpC == PPC::PREPARE_PROBED_ALLOCA_64 ||
-       OpC == PPC::PREPARE_PROBED_ALLOCA_32)) {
+       OpC == PPC::PREPARE_PROBED_ALLOCA_32 ||
+       OpC == PPC::PREPARE_PROBED_ALLOCA_NEGSIZE_SAME_REG_64 ||
+       OpC == PPC::PREPARE_PROBED_ALLOCA_NEGSIZE_SAME_REG_32)) {
     lowerPrepareProbedAlloca(II);
     return;
   }
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index 53556ffc267df..bbb4239d36da5 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -13,8 +13,11 @@
 #include "llvm/CodeGen/CostTable.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetSchedule.h"
+#include "llvm/IR/IntrinsicsPowerPC.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Transforms/InstCombine/InstCombiner.h"
+#include "llvm/Transforms/Utils/Local.h"
 using namespace llvm;
 
 #define DEBUG_TYPE "ppctti"
@@ -59,6 +62,158 @@ PPCTTIImpl::getPopcntSupport(unsigned TyWidth) {
   return TTI::PSK_Software;
 }
 
+Optional<Instruction *>
+PPCTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
+  Intrinsic::ID IID = II.getIntrinsicID();
+  switch (IID) {
+  default:
+    break;
+  case Intrinsic::ppc_altivec_lvx:
+  case Intrinsic::ppc_altivec_lvxl:
+    // Turn PPC lvx -> load if the pointer is known aligned.
+    if (getOrEnforceKnownAlignment(
+            II.getArgOperand(0), Align(16), IC.getDataLayout(), &II,
+            &IC.getAssumptionCache(), &IC.getDominatorTree()) >= 16) {
+      Value *Ptr = IC.Builder.CreateBitCast(
+          II.getArgOperand(0), PointerType::getUnqual(II.getType()));
+      return new LoadInst(II.getType(), Ptr, "", false, Align(16));
+    }
+    break;
+  case Intrinsic::ppc_vsx_lxvw4x:
+  case Intrinsic::ppc_vsx_lxvd2x: {
+    // Turn PPC VSX loads into normal loads.
+    Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(0),
+                                          PointerType::getUnqual(II.getType()));
+    return new LoadInst(II.getType(), Ptr, Twine(""), false, Align(1));
+  }
+  case Intrinsic::ppc_altivec_stvx:
+  case Intrinsic::ppc_altivec_stvxl:
+    // Turn stvx -> store if the pointer is known aligned.
+    if (getOrEnforceKnownAlignment(
+            II.getArgOperand(1), Align(16), IC.getDataLayout(), &II,
+            &IC.getAssumptionCache(), &IC.getDominatorTree()) >= 16) {
+      Type *OpPtrTy = PointerType::getUnqual(II.getArgOperand(0)->getType());
+      Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(1), OpPtrTy);
+      return new StoreInst(II.getArgOperand(0), Ptr, false, Align(16));
+    }
+    break;
+  case Intrinsic::ppc_vsx_stxvw4x:
+  case Intrinsic::ppc_vsx_stxvd2x: {
+    // Turn PPC VSX stores into normal stores.
+    Type *OpPtrTy = PointerType::getUnqual(II.getArgOperand(0)->getType());
+    Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(1), OpPtrTy);
+    return new StoreInst(II.getArgOperand(0), Ptr, false, Align(1));
+  }
+  case Intrinsic::ppc_qpx_qvlfs:
+    // Turn PPC QPX qvlfs -> load if the pointer is known aligned.
+    if (getOrEnforceKnownAlignment(
+            II.getArgOperand(0), Align(16), IC.getDataLayout(), &II,
+            &IC.getAssumptionCache(), &IC.getDominatorTree()) >= 16) {
+      Type *VTy =
+          VectorType::get(IC.Builder.getFloatTy(),
+                          cast<VectorType>(II.getType())->getElementCount());
+      Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(0),
+                                            PointerType::getUnqual(VTy));
+      Value *Load = IC.Builder.CreateLoad(VTy, Ptr);
+      return new FPExtInst(Load, II.getType());
+    }
+    break;
+  case Intrinsic::ppc_qpx_qvlfd:
+    // Turn PPC QPX qvlfd -> load if the pointer is known aligned.
+    if (getOrEnforceKnownAlignment(
+            II.getArgOperand(0), Align(32), IC.getDataLayout(), &II,
+            &IC.getAssumptionCache(), &IC.getDominatorTree()) >= 32) {
+      Value *Ptr = IC.Builder.CreateBitCast(
+          II.getArgOperand(0), PointerType::getUnqual(II.getType()));
+      return new LoadInst(II.getType(), Ptr, "", false, Align(32));
+    }
+    break;
+  case Intrinsic::ppc_qpx_qvstfs:
+    // Turn PPC QPX qvstfs -> store if the pointer is known aligned.
+    if (getOrEnforceKnownAlignment(
+            II.getArgOperand(1), Align(16), IC.getDataLayout(), &II,
+            &IC.getAssumptionCache(), &IC.getDominatorTree()) >= 16) {
+      Type *VTy = VectorType::get(
+          IC.Builder.getFloatTy(),
+          cast<VectorType>(II.getArgOperand(0)->getType())->getElementCount());
+      Value *TOp = IC.Builder.CreateFPTrunc(II.getArgOperand(0), VTy);
+      Type *OpPtrTy = PointerType::getUnqual(VTy);
+      Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(1), OpPtrTy);
+      return new StoreInst(TOp, Ptr, false, Align(16));
+    }
+    break;
+  case Intrinsic::ppc_qpx_qvstfd:
+    // Turn PPC QPX qvstfd -> store if the pointer is known aligned.
+    if (getOrEnforceKnownAlignment(
+            II.getArgOperand(1), Align(32), IC.getDataLayout(), &II,
+            &IC.getAssumptionCache(), &IC.getDominatorTree()) >= 32) {
+      Type *OpPtrTy = PointerType::getUnqual(II.getArgOperand(0)->getType());
+      Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(1), OpPtrTy);
+      return new StoreInst(II.getArgOperand(0), Ptr, false, Align(32));
+    }
+    break;
+
+  case Intrinsic::ppc_altivec_vperm:
+    // Turn vperm(V1,V2,mask) -> shuffle(V1,V2,mask) if mask is a constant.
+    // Note that ppc_altivec_vperm has a big-endian bias, so when creating
+    // a vectorshuffle for little endian, we must undo the transformation
+    // performed on vec_perm in altivec.h.  That is, we must complement
+    // the permutation mask with respect to 31 and reverse the order of
+    // V1 and V2.
+    if (Constant *Mask = dyn_cast<Constant>(II.getArgOperand(2))) {
+      assert(cast<VectorType>(Mask->getType())->getNumElements() == 16 &&
+             "Bad type for intrinsic!");
+
+      // Check that all of the elements are integer constants or undefs.
+      bool AllEltsOk = true;
+      for (unsigned i = 0; i != 16; ++i) {
+        Constant *Elt = Mask->getAggregateElement(i);
+        if (!Elt || !(isa<ConstantInt>(Elt) || isa<UndefValue>(Elt))) {
+          AllEltsOk = false;
+          break;
+        }
+      }
+
+      if (AllEltsOk) {
+        // Cast the input vectors to byte vectors.
+        Value *Op0 =
+            IC.Builder.CreateBitCast(II.getArgOperand(0), Mask->getType());
+        Value *Op1 =
+            IC.Builder.CreateBitCast(II.getArgOperand(1), Mask->getType());
+        Value *Result = UndefValue::get(Op0->getType());
+
+        // Only extract each element once.
+        Value *ExtractedElts[32];
+        memset(ExtractedElts, 0, sizeof(ExtractedElts));
+
+        for (unsigned i = 0; i != 16; ++i) {
+          if (isa<UndefValue>(Mask->getAggregateElement(i)))
+            continue;
+          unsigned Idx =
+              cast<ConstantInt>(Mask->getAggregateElement(i))->getZExtValue();
+          Idx &= 31; // Match the hardware behavior.
+          if (DL.isLittleEndian())
+            Idx = 31 - Idx;
+
+          if (!ExtractedElts[Idx]) {
+            Value *Op0ToUse = (DL.isLittleEndian()) ? Op1 : Op0;
+            Value *Op1ToUse = (DL.isLittleEndian()) ? Op0 : Op1;
+            ExtractedElts[Idx] = IC.Builder.CreateExtractElement(
+                Idx < 16 ? Op0ToUse : Op1ToUse, IC.Builder.getInt32(Idx & 15));
+          }
+
+          // Insert this value into the result vector.
+          Result = IC.Builder.CreateInsertElement(Result, ExtractedElts[Idx],
+                                                  IC.Builder.getInt32(i));
+        }
+        return CastInst::Create(Instruction::BitCast, Result, II.getType());
+      }
+    }
+    break;
+  }
+  return None;
+}
+
 int PPCTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
                               TTI::TargetCostKind CostKind) {
   if (DisablePPCConstHoist)
@@ -849,7 +1004,7 @@ int PPCTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index) {
       // The cost of the load constant for a vector extract is disregarded
       // (invariant, easily schedulable).
       return vectorCostAdjustment(1, Opcode, Val, nullptr);
-      
+
     } else if (ST->hasDirectMove())
       // Assume permute has standard cost.
       // Assume move-to/move-from VSR have 2x standard cost.
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
index d998521084e1c..a3453b00a864a 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -41,6 +41,9 @@ class PPCTTIImpl : public BasicTTIImplBase<PPCTTIImpl> {
       : BaseT(TM, F.getParent()->getDataLayout()), ST(TM->getSubtargetImpl(F)),
         TLI(ST->getTargetLowering()) {}
 
+  Optional<Instruction *> instCombineIntrinsic(InstCombiner &IC,
+                                               IntrinsicInst &II) const;
+
   /// \name Scalar TTI Implementations
   /// @{
 
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 03d9eefd59d0b..cad5f8e21185f 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -25,7 +25,6 @@
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/SelectionDAGISel.h"
 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/DiagnosticInfo.h"
diff --git a/llvm/lib/Target/SystemZ/SystemZSubtarget.cpp b/llvm/lib/Target/SystemZ/SystemZSubtarget.cpp
index 68e0b7ae66a4d..dbd16ab52e608 100644
--- a/llvm/lib/Target/SystemZ/SystemZSubtarget.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZSubtarget.cpp
@@ -39,6 +39,14 @@ SystemZSubtarget::initializeSubtargetDependencies(StringRef CPU, StringRef FS) {
   if (HasSoftFloat)
     HasVector = false;
 
+  // -mno-vx implicitly disables all vector-related features.
+  if (!HasVector) {
+    HasVectorEnhancements1 = false;
+    HasVectorEnhancements2 = false;
+    HasVectorPackedDecimal = false;
+    HasVectorPackedDecimalEnhancement = false;
+  }
+
   return *this;
 }
 
diff --git a/llvm/lib/Target/X86/CMakeLists.txt b/llvm/lib/Target/X86/CMakeLists.txt
index 729934b294bae..2b0cbb676c129 100644
--- a/llvm/lib/Target/X86/CMakeLists.txt
+++ b/llvm/lib/Target/X86/CMakeLists.txt
@@ -47,6 +47,7 @@ set(sources
   X86IndirectThunks.cpp
   X86InterleavedAccess.cpp
   X86InsertPrefetch.cpp
+  X86InstCombineIntrinsic.cpp
   X86InstrFMA3Info.cpp
   X86InstrFoldTables.cpp
   X86InstrInfo.cpp
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
index 81110ba666e95..8a478354cb168 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
@@ -290,11 +290,10 @@ MCSubtargetInfo *X86_MC::createX86MCSubtargetInfo(const Triple &TT,
   if (!FS.empty())
     ArchFS = (Twine(ArchFS) + "," + FS).str();
 
-  std::string CPUName = std::string(CPU);
-  if (CPUName.empty())
-    CPUName = "generic";
+  if (CPU.empty())
+    CPU = "generic";
 
-  return createX86MCSubtargetInfoImpl(TT, CPUName, ArchFS);
+  return createX86MCSubtargetInfoImpl(TT, CPU, ArchFS);
 }
 
 static MCInstrInfo *createX86MCInstrInfo() {
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index 8ca6dac036754..69d83ab1528cb 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -234,8 +234,8 @@ def FeaturePRFCHW  : SubtargetFeature<"prfchw", "HasPRFCHW", "true",
                                       "Support PRFCHW instructions">;
 def FeatureRDSEED  : SubtargetFeature<"rdseed", "HasRDSEED", "true",
                                       "Support RDSEED instruction">;
-def FeatureLAHFSAHF : SubtargetFeature<"sahf", "HasLAHFSAHF", "true",
-                                       "Support LAHF and SAHF instructions">;
+def FeatureLAHFSAHF : SubtargetFeature<"sahf", "HasLAHFSAHF64", "true",
+                           "Support LAHF and SAHF instructions in 64-bit mode">;
 def FeatureMWAITX  : SubtargetFeature<"mwaitx", "HasMWAITX", "true",
                                       "Enable MONITORX/MWAITX timer functionality">;
 def FeatureCLZERO  : SubtargetFeature<"clzero", "HasCLZERO", "true",
@@ -244,11 +244,6 @@ def FeatureCLDEMOTE  : SubtargetFeature<"cldemote", "HasCLDEMOTE", "true",
                                       "Enable Cache Demote">;
 def FeaturePTWRITE  : SubtargetFeature<"ptwrite", "HasPTWRITE", "true",
                                       "Support ptwrite instruction">;
-// FIXME: This feature is deprecated in 10.0 and should not be used for
-// anything, but removing it would break IR files that may contain it in a
-// target-feature attribute.
-def FeatureDeprecatedMPX : SubtargetFeature<"mpx", "DeprecatedHasMPX", "false",
-                                      "Deprecated. Support MPX instructions">;
 def FeatureAMXTILE     : SubtargetFeature<"amx-tile", "HasAMXTILE", "true",
                                       "Support AMX-TILE instructions">;
 def FeatureAMXINT8     : SubtargetFeature<"amx-int8", "HasAMXINT8", "true",
@@ -1027,8 +1022,12 @@ class Proc<string Name, list<SubtargetFeature> Features>
 
 // NOTE: CMPXCHG8B is here for legacy compatibility so that it is only disabled
 // if i386/i486 is specifically requested.
+// NOTE: 64Bit is here as "generic" is the default llc CPU. The X86Subtarget
+// constructor checks that any CPU used in 64-bit mode has Feature64Bit enabled.
+// It has no effect on code generation.
 def : Proc<"generic",         [FeatureX87, FeatureSlowUAMem16,
-                               FeatureCMPXCHG8B, FeatureInsertVZEROUPPER]>;
+                               FeatureCMPXCHG8B, Feature64Bit,
+                               FeatureInsertVZEROUPPER]>;
 def : Proc<"i386",            [FeatureX87, FeatureSlowUAMem16,
                                FeatureInsertVZEROUPPER]>;
 def : Proc<"i486",            [FeatureX87, FeatureSlowUAMem16,
diff --git a/llvm/lib/Target/X86/X86CallLowering.cpp b/llvm/lib/Target/X86/X86CallLowering.cpp
index 319dc94706048..0286482ac9af8 100644
--- a/llvm/lib/Target/X86/X86CallLowering.cpp
+++ b/llvm/lib/Target/X86/X86CallLowering.cpp
@@ -95,15 +95,14 @@ bool X86CallLowering::splitToValueTypes(const ArgInfo &OrigArg,
 
 namespace {
 
-struct OutgoingValueHandler : public CallLowering::ValueHandler {
-  OutgoingValueHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
-                       MachineInstrBuilder &MIB, CCAssignFn *AssignFn)
-      : ValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB),
+struct X86OutgoingValueHandler : public CallLowering::IncomingValueHandler {
+  X86OutgoingValueHandler(MachineIRBuilder &MIRBuilder,
+                          MachineRegisterInfo &MRI, MachineInstrBuilder &MIB,
+                          CCAssignFn *AssignFn)
+      : IncomingValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB),
         DL(MIRBuilder.getMF().getDataLayout()),
         STI(MIRBuilder.getMF().getSubtarget<X86Subtarget>()) {}
 
-  bool isIncomingArgumentHandler() const override { return false; }
-
   Register getStackAddress(uint64_t Size, int64_t Offset,
                            MachinePointerInfo &MPO) override {
     LLT p0 = LLT::pointer(0, DL.getPointerSizeInBits(0));
@@ -215,7 +214,7 @@ bool X86CallLowering::lowerReturn(
         return false;
     }
 
-    OutgoingValueHandler Handler(MIRBuilder, MRI, MIB, RetCC_X86);
+    X86OutgoingValueHandler Handler(MIRBuilder, MRI, MIB, RetCC_X86);
     if (!handleAssignments(MIRBuilder, SplitArgs, Handler))
       return false;
   }
@@ -226,14 +225,12 @@ bool X86CallLowering::lowerReturn(
 
 namespace {
 
-struct IncomingValueHandler : public CallLowering::ValueHandler {
-  IncomingValueHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
-                       CCAssignFn *AssignFn)
-      : ValueHandler(MIRBuilder, MRI, AssignFn),
+struct X86IncomingValueHandler : public CallLowering::IncomingValueHandler {
+  X86IncomingValueHandler(MachineIRBuilder &MIRBuilder,
+                          MachineRegisterInfo &MRI, CCAssignFn *AssignFn)
+      : IncomingValueHandler(MIRBuilder, MRI, AssignFn),
         DL(MIRBuilder.getMF().getDataLayout()) {}
 
-  bool isIncomingArgumentHandler() const override { return true; }
-
   Register getStackAddress(uint64_t Size, int64_t Offset,
                            MachinePointerInfo &MPO) override {
     auto &MFI = MIRBuilder.getMF().getFrameInfo();
@@ -298,10 +295,10 @@ struct IncomingValueHandler : public CallLowering::ValueHandler {
   const DataLayout &DL;
 };
 
-struct FormalArgHandler : public IncomingValueHandler {
+struct FormalArgHandler : public X86IncomingValueHandler {
   FormalArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
                    CCAssignFn *AssignFn)
-      : IncomingValueHandler(MIRBuilder, MRI, AssignFn) {}
+      : X86IncomingValueHandler(MIRBuilder, MRI, AssignFn) {}
 
   void markPhysRegUsed(unsigned PhysReg) override {
     MIRBuilder.getMRI()->addLiveIn(PhysReg);
@@ -309,10 +306,10 @@ struct FormalArgHandler : public IncomingValueHandler {
   }
 };
 
-struct CallReturnHandler : public IncomingValueHandler {
+struct CallReturnHandler : public X86IncomingValueHandler {
   CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
                     CCAssignFn *AssignFn, MachineInstrBuilder &MIB)
-      : IncomingValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB) {}
+      : X86IncomingValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB) {}
 
   void markPhysRegUsed(unsigned PhysReg) override {
     MIB.addDef(PhysReg, RegState::Implicit);
@@ -421,7 +418,7 @@ bool X86CallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
       return false;
   }
   // Do the actual argument marshalling.
-  OutgoingValueHandler Handler(MIRBuilder, MRI, MIB, CC_X86);
+  X86OutgoingValueHandler Handler(MIRBuilder, MRI, MIB, CC_X86);
   if (!handleAssignments(MIRBuilder, SplitArgs, Handler))
     return false;
 
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index bb32a17bcc122..2e97bef6c106a 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -6916,25 +6916,16 @@ static bool getTargetShuffleMask(SDNode *N, MVT VT, bool AllowSentinelZero,
     DecodeZeroMoveLowMask(NumElems, Mask);
     IsUnary = true;
     break;
-  case X86ISD::VBROADCAST: {
-    SDValue N0 = N->getOperand(0);
-    // See if we're broadcasting from index 0 of an EXTRACT_SUBVECTOR. If so,
-    // add the pre-extracted value to the Ops vector.
-    if (N0.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
-        N0.getOperand(0).getValueType() == VT &&
-        N0.getConstantOperandVal(1) == 0)
-      Ops.push_back(N0.getOperand(0));
-
-    // We only decode broadcasts of same-sized vectors, unless the broadcast
-    // came from an extract from the original width. If we found one, we
-    // pushed it the Ops vector above.
-    if (N0.getValueType() == VT || !Ops.empty()) {
+  case X86ISD::VBROADCAST:
+    // We only decode broadcasts of same-sized vectors, peeking through to
+    // extracted subvectors is likely to cause hasOneUse issues with
+    // SimplifyDemandedBits etc.
+    if (N->getOperand(0).getValueType() == VT) {
       DecodeVectorBroadcast(NumElems, Mask);
       IsUnary = true;
       break;
     }
     return false;
-  }
   case X86ISD::VPERMILPV: {
     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
     IsUnary = true;
@@ -37051,7 +37042,13 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
     if (SimplifyDemandedVectorElts(Src, DemandedElts, SrcUndef, KnownZero, TLO,
                                    Depth + 1))
       return true;
-    // TODO convert SrcUndef to KnownUndef.
+
+    // Aggressively peek through ops to get at the demanded elts.
+    if (!DemandedElts.isAllOnesValue())
+      if (SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
+              Src, DemandedElts, TLO.DAG, Depth + 1))
+        return TLO.CombineTo(
+            Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc, Op.getOperand(1)));
     break;
   }
   case X86ISD::KSHIFTL: {
diff --git a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
new file mode 100644
index 0000000000000..91341f93a9943
--- /dev/null
+++ b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
@@ -0,0 +1,2007 @@
+//===-- X86InstCombineIntrinsic.cpp - X86 specific InstCombine pass -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+/// This file implements a TargetTransformInfo analysis pass specific to the
+/// X86 target machine. It uses the target's detailed information to provide
+/// more precise answers to certain TTI queries, while letting the target
+/// independent and default TTI implementations handle the rest.
+///
+//===----------------------------------------------------------------------===//
+
+#include "X86TargetTransformInfo.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IntrinsicsX86.h"
+#include "llvm/Transforms/InstCombine/InstCombiner.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "x86tti"
+
+/// Return a constant boolean vector that has true elements in all positions
+/// where the input constant data vector has an element with the sign bit set.
+static Constant *getNegativeIsTrueBoolVec(ConstantDataVector *V) {
+  SmallVector<Constant *, 32> BoolVec;
+  IntegerType *BoolTy = Type::getInt1Ty(V->getContext());
+  for (unsigned I = 0, E = V->getNumElements(); I != E; ++I) {
+    Constant *Elt = V->getElementAsConstant(I);
+    assert((isa<ConstantInt>(Elt) || isa<ConstantFP>(Elt)) &&
+           "Unexpected constant data vector element type");
+    bool Sign = V->getElementType()->isIntegerTy()
+                    ? cast<ConstantInt>(Elt)->isNegative()
+                    : cast<ConstantFP>(Elt)->isNegative();
+    BoolVec.push_back(ConstantInt::get(BoolTy, Sign));
+  }
+  return ConstantVector::get(BoolVec);
+}
+
+// TODO: If the x86 backend knew how to convert a bool vector mask back to an
+// XMM register mask efficiently, we could transform all x86 masked intrinsics
+// to LLVM masked intrinsics and remove the x86 masked intrinsic defs.
+static Instruction *simplifyX86MaskedLoad(IntrinsicInst &II, InstCombiner &IC) {
+  Value *Ptr = II.getOperand(0);
+  Value *Mask = II.getOperand(1);
+  Constant *ZeroVec = Constant::getNullValue(II.getType());
+
+  // Special case a zero mask since that's not a ConstantDataVector.
+  // This masked load instruction creates a zero vector.
+  if (isa<ConstantAggregateZero>(Mask))
+    return IC.replaceInstUsesWith(II, ZeroVec);
+
+  auto *ConstMask = dyn_cast<ConstantDataVector>(Mask);
+  if (!ConstMask)
+    return nullptr;
+
+  // The mask is constant. Convert this x86 intrinsic to the LLVM instrinsic
+  // to allow target-independent optimizations.
+
+  // First, cast the x86 intrinsic scalar pointer to a vector pointer to match
+  // the LLVM intrinsic definition for the pointer argument.
+  unsigned AddrSpace = cast<PointerType>(Ptr->getType())->getAddressSpace();
+  PointerType *VecPtrTy = PointerType::get(II.getType(), AddrSpace);
+  Value *PtrCast = IC.Builder.CreateBitCast(Ptr, VecPtrTy, "castvec");
+
+  // Second, convert the x86 XMM integer vector mask to a vector of bools based
+  // on each element's most significant bit (the sign bit).
+  Constant *BoolMask = getNegativeIsTrueBoolVec(ConstMask);
+
+  // The pass-through vector for an x86 masked load is a zero vector.
+  CallInst *NewMaskedLoad =
+      IC.Builder.CreateMaskedLoad(PtrCast, Align(1), BoolMask, ZeroVec);
+  return IC.replaceInstUsesWith(II, NewMaskedLoad);
+}
+
+// TODO: If the x86 backend knew how to convert a bool vector mask back to an
+// XMM register mask efficiently, we could transform all x86 masked intrinsics
+// to LLVM masked intrinsics and remove the x86 masked intrinsic defs.
+static bool simplifyX86MaskedStore(IntrinsicInst &II, InstCombiner &IC) {
+  Value *Ptr = II.getOperand(0);
+  Value *Mask = II.getOperand(1);
+  Value *Vec = II.getOperand(2);
+
+  // Special case a zero mask since that's not a ConstantDataVector:
+  // this masked store instruction does nothing.
+  if (isa<ConstantAggregateZero>(Mask)) {
+    IC.eraseInstFromFunction(II);
+    return true;
+  }
+
+  // The SSE2 version is too weird (eg, unaligned but non-temporal) to do
+  // anything else at this level.
+  if (II.getIntrinsicID() == Intrinsic::x86_sse2_maskmov_dqu)
+    return false;
+
+  auto *ConstMask = dyn_cast<ConstantDataVector>(Mask);
+  if (!ConstMask)
+    return false;
+
+  // The mask is constant. Convert this x86 intrinsic to the LLVM instrinsic
+  // to allow target-independent optimizations.
+
+  // First, cast the x86 intrinsic scalar pointer to a vector pointer to match
+  // the LLVM intrinsic definition for the pointer argument.
+  unsigned AddrSpace = cast<PointerType>(Ptr->getType())->getAddressSpace();
+  PointerType *VecPtrTy = PointerType::get(Vec->getType(), AddrSpace);
+  Value *PtrCast = IC.Builder.CreateBitCast(Ptr, VecPtrTy, "castvec");
+
+  // Second, convert the x86 XMM integer vector mask to a vector of bools based
+  // on each element's most significant bit (the sign bit).
+  Constant *BoolMask = getNegativeIsTrueBoolVec(ConstMask);
+
+  IC.Builder.CreateMaskedStore(Vec, PtrCast, Align(1), BoolMask);
+
+  // 'Replace uses' doesn't work for stores. Erase the original masked store.
+  IC.eraseInstFromFunction(II);
+  return true;
+}
+
+static Value *simplifyX86immShift(const IntrinsicInst &II,
+                                  InstCombiner::BuilderTy &Builder) {
+  bool LogicalShift = false;
+  bool ShiftLeft = false;
+  bool IsImm = false;
+
+  switch (II.getIntrinsicID()) {
+  default:
+    llvm_unreachable("Unexpected intrinsic!");
+  case Intrinsic::x86_sse2_psrai_d:
+  case Intrinsic::x86_sse2_psrai_w:
+  case Intrinsic::x86_avx2_psrai_d:
+  case Intrinsic::x86_avx2_psrai_w:
+  case Intrinsic::x86_avx512_psrai_q_128:
+  case Intrinsic::x86_avx512_psrai_q_256:
+  case Intrinsic::x86_avx512_psrai_d_512:
+  case Intrinsic::x86_avx512_psrai_q_512:
+  case Intrinsic::x86_avx512_psrai_w_512:
+    IsImm = true;
+    LLVM_FALLTHROUGH;
+  case Intrinsic::x86_sse2_psra_d:
+  case Intrinsic::x86_sse2_psra_w:
+  case Intrinsic::x86_avx2_psra_d:
+  case Intrinsic::x86_avx2_psra_w:
+  case Intrinsic::x86_avx512_psra_q_128:
+  case Intrinsic::x86_avx512_psra_q_256:
+  case Intrinsic::x86_avx512_psra_d_512:
+  case Intrinsic::x86_avx512_psra_q_512:
+  case Intrinsic::x86_avx512_psra_w_512:
+    LogicalShift = false;
+    ShiftLeft = false;
+    break;
+  case Intrinsic::x86_sse2_psrli_d:
+  case Intrinsic::x86_sse2_psrli_q:
+  case Intrinsic::x86_sse2_psrli_w:
+  case Intrinsic::x86_avx2_psrli_d:
+  case Intrinsic::x86_avx2_psrli_q:
+  case Intrinsic::x86_avx2_psrli_w:
+  case Intrinsic::x86_avx512_psrli_d_512:
+  case Intrinsic::x86_avx512_psrli_q_512:
+  case Intrinsic::x86_avx512_psrli_w_512:
+    IsImm = true;
+    LLVM_FALLTHROUGH;
+  case Intrinsic::x86_sse2_psrl_d:
+  case Intrinsic::x86_sse2_psrl_q:
+  case Intrinsic::x86_sse2_psrl_w:
+  case Intrinsic::x86_avx2_psrl_d:
+  case Intrinsic::x86_avx2_psrl_q:
+  case Intrinsic::x86_avx2_psrl_w:
+  case Intrinsic::x86_avx512_psrl_d_512:
+  case Intrinsic::x86_avx512_psrl_q_512:
+  case Intrinsic::x86_avx512_psrl_w_512:
+    LogicalShift = true;
+    ShiftLeft = false;
+    break;
+  case Intrinsic::x86_sse2_pslli_d:
+  case Intrinsic::x86_sse2_pslli_q:
+  case Intrinsic::x86_sse2_pslli_w:
+  case Intrinsic::x86_avx2_pslli_d:
+  case Intrinsic::x86_avx2_pslli_q:
+  case Intrinsic::x86_avx2_pslli_w:
+  case Intrinsic::x86_avx512_pslli_d_512:
+  case Intrinsic::x86_avx512_pslli_q_512:
+  case Intrinsic::x86_avx512_pslli_w_512:
+    IsImm = true;
+    LLVM_FALLTHROUGH;
+  case Intrinsic::x86_sse2_psll_d:
+  case Intrinsic::x86_sse2_psll_q:
+  case Intrinsic::x86_sse2_psll_w:
+  case Intrinsic::x86_avx2_psll_d:
+  case Intrinsic::x86_avx2_psll_q:
+  case Intrinsic::x86_avx2_psll_w:
+  case Intrinsic::x86_avx512_psll_d_512:
+  case Intrinsic::x86_avx512_psll_q_512:
+  case Intrinsic::x86_avx512_psll_w_512:
+    LogicalShift = true;
+    ShiftLeft = true;
+    break;
+  }
+  assert((LogicalShift || !ShiftLeft) && "Only logical shifts can shift left");
+
+  auto Vec = II.getArgOperand(0);
+  auto Amt = II.getArgOperand(1);
+  auto VT = cast<VectorType>(Vec->getType());
+  auto SVT = VT->getElementType();
+  auto AmtVT = Amt->getType();
+  unsigned VWidth = VT->getNumElements();
+  unsigned BitWidth = SVT->getPrimitiveSizeInBits();
+
+  // If the shift amount is guaranteed to be in-range we can replace it with a
+  // generic shift. If its guaranteed to be out of range, logical shifts combine
+  // to zero and arithmetic shifts are clamped to (BitWidth - 1).
+  if (IsImm) {
+    assert(AmtVT->isIntegerTy(32) && "Unexpected shift-by-immediate type");
+    KnownBits KnownAmtBits =
+        llvm::computeKnownBits(Amt, II.getModule()->getDataLayout());
+    if (KnownAmtBits.getMaxValue().ult(BitWidth)) {
+      Amt = Builder.CreateZExtOrTrunc(Amt, SVT);
+      Amt = Builder.CreateVectorSplat(VWidth, Amt);
+      return (LogicalShift ? (ShiftLeft ? Builder.CreateShl(Vec, Amt)
+                                        : Builder.CreateLShr(Vec, Amt))
+                           : Builder.CreateAShr(Vec, Amt));
+    }
+    if (KnownAmtBits.getMinValue().uge(BitWidth)) {
+      if (LogicalShift)
+        return ConstantAggregateZero::get(VT);
+      Amt = ConstantInt::get(SVT, BitWidth - 1);
+      return Builder.CreateAShr(Vec, Builder.CreateVectorSplat(VWidth, Amt));
+    }
+  } else {
+    // Ensure the first element has an in-range value and the rest of the
+    // elements in the bottom 64 bits are zero.
+    assert(AmtVT->isVectorTy() && AmtVT->getPrimitiveSizeInBits() == 128 &&
+           cast<VectorType>(AmtVT)->getElementType() == SVT &&
+           "Unexpected shift-by-scalar type");
+    unsigned NumAmtElts = cast<VectorType>(AmtVT)->getNumElements();
+    APInt DemandedLower = APInt::getOneBitSet(NumAmtElts, 0);
+    APInt DemandedUpper = APInt::getBitsSet(NumAmtElts, 1, NumAmtElts / 2);
+    KnownBits KnownLowerBits = llvm::computeKnownBits(
+        Amt, DemandedLower, II.getModule()->getDataLayout());
+    KnownBits KnownUpperBits = llvm::computeKnownBits(
+        Amt, DemandedUpper, II.getModule()->getDataLayout());
+    if (KnownLowerBits.getMaxValue().ult(BitWidth) &&
+        (DemandedUpper.isNullValue() || KnownUpperBits.isZero())) {
+      SmallVector<int, 16> ZeroSplat(VWidth, 0);
+      Amt = Builder.CreateShuffleVector(Amt, Amt, ZeroSplat);
+      return (LogicalShift ? (ShiftLeft ? Builder.CreateShl(Vec, Amt)
+                                        : Builder.CreateLShr(Vec, Amt))
+                           : Builder.CreateAShr(Vec, Amt));
+    }
+  }
+
+  // Simplify if count is constant vector.
+  auto CDV = dyn_cast<ConstantDataVector>(Amt);
+  if (!CDV)
+    return nullptr;
+
+  // SSE2/AVX2 uses all the first 64-bits of the 128-bit vector
+  // operand to compute the shift amount.
+  assert(AmtVT->isVectorTy() && AmtVT->getPrimitiveSizeInBits() == 128 &&
+         cast<VectorType>(AmtVT)->getElementType() == SVT &&
+         "Unexpected shift-by-scalar type");
+
+  // Concatenate the sub-elements to create the 64-bit value.
+  APInt Count(64, 0);
+  for (unsigned i = 0, NumSubElts = 64 / BitWidth; i != NumSubElts; ++i) {
+    unsigned SubEltIdx = (NumSubElts - 1) - i;
+    auto SubElt = cast<ConstantInt>(CDV->getElementAsConstant(SubEltIdx));
+    Count <<= BitWidth;
+    Count |= SubElt->getValue().zextOrTrunc(64);
+  }
+
+  // If shift-by-zero then just return the original value.
+  if (Count.isNullValue())
+    return Vec;
+
+  // Handle cases when Shift >= BitWidth.
+  if (Count.uge(BitWidth)) {
+    // If LogicalShift - just return zero.
+    if (LogicalShift)
+      return ConstantAggregateZero::get(VT);
+
+    // If ArithmeticShift - clamp Shift to (BitWidth - 1).
+    Count = APInt(64, BitWidth - 1);
+  }
+
+  // Get a constant vector of the same type as the first operand.
+  auto ShiftAmt = ConstantInt::get(SVT, Count.zextOrTrunc(BitWidth));
+  auto ShiftVec = Builder.CreateVectorSplat(VWidth, ShiftAmt);
+
+  if (ShiftLeft)
+    return Builder.CreateShl(Vec, ShiftVec);
+
+  if (LogicalShift)
+    return Builder.CreateLShr(Vec, ShiftVec);
+
+  return Builder.CreateAShr(Vec, ShiftVec);
+}
+
+// Attempt to simplify AVX2 per-element shift intrinsics to a generic IR shift.
+// Unlike the generic IR shifts, the intrinsics have defined behaviour for out
+// of range shift amounts (logical - set to zero, arithmetic - splat sign bit).
+static Value *simplifyX86varShift(const IntrinsicInst &II,
+                                  InstCombiner::BuilderTy &Builder) {
+  bool LogicalShift = false;
+  bool ShiftLeft = false;
+
+  switch (II.getIntrinsicID()) {
+  default:
+    llvm_unreachable("Unexpected intrinsic!");
+  case Intrinsic::x86_avx2_psrav_d:
+  case Intrinsic::x86_avx2_psrav_d_256:
+  case Intrinsic::x86_avx512_psrav_q_128:
+  case Intrinsic::x86_avx512_psrav_q_256:
+  case Intrinsic::x86_avx512_psrav_d_512:
+  case Intrinsic::x86_avx512_psrav_q_512:
+  case Intrinsic::x86_avx512_psrav_w_128:
+  case Intrinsic::x86_avx512_psrav_w_256:
+  case Intrinsic::x86_avx512_psrav_w_512:
+    LogicalShift = false;
+    ShiftLeft = false;
+    break;
+  case Intrinsic::x86_avx2_psrlv_d:
+  case Intrinsic::x86_avx2_psrlv_d_256:
+  case Intrinsic::x86_avx2_psrlv_q:
+  case Intrinsic::x86_avx2_psrlv_q_256:
+  case Intrinsic::x86_avx512_psrlv_d_512:
+  case Intrinsic::x86_avx512_psrlv_q_512:
+  case Intrinsic::x86_avx512_psrlv_w_128:
+  case Intrinsic::x86_avx512_psrlv_w_256:
+  case Intrinsic::x86_avx512_psrlv_w_512:
+    LogicalShift = true;
+    ShiftLeft = false;
+    break;
+  case Intrinsic::x86_avx2_psllv_d:
+  case Intrinsic::x86_avx2_psllv_d_256:
+  case Intrinsic::x86_avx2_psllv_q:
+  case Intrinsic::x86_avx2_psllv_q_256:
+  case Intrinsic::x86_avx512_psllv_d_512:
+  case Intrinsic::x86_avx512_psllv_q_512:
+  case Intrinsic::x86_avx512_psllv_w_128:
+  case Intrinsic::x86_avx512_psllv_w_256:
+  case Intrinsic::x86_avx512_psllv_w_512:
+    LogicalShift = true;
+    ShiftLeft = true;
+    break;
+  }
+  assert((LogicalShift || !ShiftLeft) && "Only logical shifts can shift left");
+
+  auto Vec = II.getArgOperand(0);
+  auto Amt = II.getArgOperand(1);
+  auto VT = cast<VectorType>(II.getType());
+  auto SVT = VT->getElementType();
+  int NumElts = VT->getNumElements();
+  int BitWidth = SVT->getIntegerBitWidth();
+
+  // If the shift amount is guaranteed to be in-range we can replace it with a
+  // generic shift.
+  APInt UpperBits =
+      APInt::getHighBitsSet(BitWidth, BitWidth - Log2_32(BitWidth));
+  if (llvm::MaskedValueIsZero(Amt, UpperBits,
+                              II.getModule()->getDataLayout())) {
+    return (LogicalShift ? (ShiftLeft ? Builder.CreateShl(Vec, Amt)
+                                      : Builder.CreateLShr(Vec, Amt))
+                         : Builder.CreateAShr(Vec, Amt));
+  }
+
+  // Simplify if all shift amounts are constant/undef.
+  auto *CShift = dyn_cast<Constant>(Amt);
+  if (!CShift)
+    return nullptr;
+
+  // Collect each element's shift amount.
+  // We also collect special cases: UNDEF = -1, OUT-OF-RANGE = BitWidth.
+  bool AnyOutOfRange = false;
+  SmallVector<int, 8> ShiftAmts;
+  for (int I = 0; I < NumElts; ++I) {
+    auto *CElt = CShift->getAggregateElement(I);
+    if (CElt && isa<UndefValue>(CElt)) {
+      ShiftAmts.push_back(-1);
+      continue;
+    }
+
+    auto *COp = dyn_cast_or_null<ConstantInt>(CElt);
+    if (!COp)
+      return nullptr;
+
+    // Handle out of range shifts.
+    // If LogicalShift - set to BitWidth (special case).
+    // If ArithmeticShift - set to (BitWidth - 1) (sign splat).
+    APInt ShiftVal = COp->getValue();
+    if (ShiftVal.uge(BitWidth)) {
+      AnyOutOfRange = LogicalShift;
+      ShiftAmts.push_back(LogicalShift ? BitWidth : BitWidth - 1);
+      continue;
+    }
+
+    ShiftAmts.push_back((int)ShiftVal.getZExtValue());
+  }
+
+  // If all elements out of range or UNDEF, return vector of zeros/undefs.
+  // ArithmeticShift should only hit this if they are all UNDEF.
+  auto OutOfRange = [&](int Idx) { return (Idx < 0) || (BitWidth <= Idx); };
+  if (llvm::all_of(ShiftAmts, OutOfRange)) {
+    SmallVector<Constant *, 8> ConstantVec;
+    for (int Idx : ShiftAmts) {
+      if (Idx < 0) {
+        ConstantVec.push_back(UndefValue::get(SVT));
+      } else {
+        assert(LogicalShift && "Logical shift expected");
+        ConstantVec.push_back(ConstantInt::getNullValue(SVT));
+      }
+    }
+    return ConstantVector::get(ConstantVec);
+  }
+
+  // We can't handle only some out of range values with generic logical shifts.
+  if (AnyOutOfRange)
+    return nullptr;
+
+  // Build the shift amount constant vector.
+  SmallVector<Constant *, 8> ShiftVecAmts;
+  for (int Idx : ShiftAmts) {
+    if (Idx < 0)
+      ShiftVecAmts.push_back(UndefValue::get(SVT));
+    else
+      ShiftVecAmts.push_back(ConstantInt::get(SVT, Idx));
+  }
+  auto ShiftVec = ConstantVector::get(ShiftVecAmts);
+
+  if (ShiftLeft)
+    return Builder.CreateShl(Vec, ShiftVec);
+
+  if (LogicalShift)
+    return Builder.CreateLShr(Vec, ShiftVec);
+
+  return Builder.CreateAShr(Vec, ShiftVec);
+}
+
+static Value *simplifyX86pack(IntrinsicInst &II,
+                              InstCombiner::BuilderTy &Builder, bool IsSigned) {
+  Value *Arg0 = II.getArgOperand(0);
+  Value *Arg1 = II.getArgOperand(1);
+  Type *ResTy = II.getType();
+
+  // Fast all undef handling.
+  if (isa<UndefValue>(Arg0) && isa<UndefValue>(Arg1))
+    return UndefValue::get(ResTy);
+
+  auto *ArgTy = cast<VectorType>(Arg0->getType());
+  unsigned NumLanes = ResTy->getPrimitiveSizeInBits() / 128;
+  unsigned NumSrcElts = ArgTy->getNumElements();
+  assert(cast<VectorType>(ResTy)->getNumElements() == (2 * NumSrcElts) &&
+         "Unexpected packing types");
+
+  unsigned NumSrcEltsPerLane = NumSrcElts / NumLanes;
+  unsigned DstScalarSizeInBits = ResTy->getScalarSizeInBits();
+  unsigned SrcScalarSizeInBits = ArgTy->getScalarSizeInBits();
+  assert(SrcScalarSizeInBits == (2 * DstScalarSizeInBits) &&
+         "Unexpected packing types");
+
+  // Constant folding.
+  if (!isa<Constant>(Arg0) || !isa<Constant>(Arg1))
+    return nullptr;
+
+  // Clamp Values - signed/unsigned both use signed clamp values, but they
+  // differ on the min/max values.
+  APInt MinValue, MaxValue;
+  if (IsSigned) {
+    // PACKSS: Truncate signed value with signed saturation.
+    // Source values less than dst minint are saturated to minint.
+    // Source values greater than dst maxint are saturated to maxint.
+    MinValue =
+        APInt::getSignedMinValue(DstScalarSizeInBits).sext(SrcScalarSizeInBits);
+    MaxValue =
+        APInt::getSignedMaxValue(DstScalarSizeInBits).sext(SrcScalarSizeInBits);
+  } else {
+    // PACKUS: Truncate signed value with unsigned saturation.
+    // Source values less than zero are saturated to zero.
+    // Source values greater than dst maxuint are saturated to maxuint.
+    MinValue = APInt::getNullValue(SrcScalarSizeInBits);
+    MaxValue = APInt::getLowBitsSet(SrcScalarSizeInBits, DstScalarSizeInBits);
+  }
+
+  auto *MinC = Constant::getIntegerValue(ArgTy, MinValue);
+  auto *MaxC = Constant::getIntegerValue(ArgTy, MaxValue);
+  Arg0 = Builder.CreateSelect(Builder.CreateICmpSLT(Arg0, MinC), MinC, Arg0);
+  Arg1 = Builder.CreateSelect(Builder.CreateICmpSLT(Arg1, MinC), MinC, Arg1);
+  Arg0 = Builder.CreateSelect(Builder.CreateICmpSGT(Arg0, MaxC), MaxC, Arg0);
+  Arg1 = Builder.CreateSelect(Builder.CreateICmpSGT(Arg1, MaxC), MaxC, Arg1);
+
+  // Shuffle clamped args together at the lane level.
+  SmallVector<int, 32> PackMask;
+  for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
+    for (unsigned Elt = 0; Elt != NumSrcEltsPerLane; ++Elt)
+      PackMask.push_back(Elt + (Lane * NumSrcEltsPerLane));
+    for (unsigned Elt = 0; Elt != NumSrcEltsPerLane; ++Elt)
+      PackMask.push_back(Elt + (Lane * NumSrcEltsPerLane) + NumSrcElts);
+  }
+  auto *Shuffle = Builder.CreateShuffleVector(Arg0, Arg1, PackMask);
+
+  // Truncate to dst size.
+  return Builder.CreateTrunc(Shuffle, ResTy);
+}
+
+static Value *simplifyX86movmsk(const IntrinsicInst &II,
+                                InstCombiner::BuilderTy &Builder) {
+  Value *Arg = II.getArgOperand(0);
+  Type *ResTy = II.getType();
+
+  // movmsk(undef) -> zero as we must ensure the upper bits are zero.
+  if (isa<UndefValue>(Arg))
+    return Constant::getNullValue(ResTy);
+
+  auto *ArgTy = dyn_cast<VectorType>(Arg->getType());
+  // We can't easily peek through x86_mmx types.
+  if (!ArgTy)
+    return nullptr;
+
+  // Expand MOVMSK to compare/bitcast/zext:
+  // e.g. PMOVMSKB(v16i8 x):
+  // %cmp = icmp slt <16 x i8> %x, zeroinitializer
+  // %int = bitcast <16 x i1> %cmp to i16
+  // %res = zext i16 %int to i32
+  unsigned NumElts = ArgTy->getNumElements();
+  Type *IntegerVecTy = VectorType::getInteger(ArgTy);
+  Type *IntegerTy = Builder.getIntNTy(NumElts);
+
+  Value *Res = Builder.CreateBitCast(Arg, IntegerVecTy);
+  Res = Builder.CreateICmpSLT(Res, Constant::getNullValue(IntegerVecTy));
+  Res = Builder.CreateBitCast(Res, IntegerTy);
+  Res = Builder.CreateZExtOrTrunc(Res, ResTy);
+  return Res;
+}
+
+static Value *simplifyX86addcarry(const IntrinsicInst &II,
+                                  InstCombiner::BuilderTy &Builder) {
+  Value *CarryIn = II.getArgOperand(0);
+  Value *Op1 = II.getArgOperand(1);
+  Value *Op2 = II.getArgOperand(2);
+  Type *RetTy = II.getType();
+  Type *OpTy = Op1->getType();
+  assert(RetTy->getStructElementType(0)->isIntegerTy(8) &&
+         RetTy->getStructElementType(1) == OpTy && OpTy == Op2->getType() &&
+         "Unexpected types for x86 addcarry");
+
+  // If carry-in is zero, this is just an unsigned add with overflow.
+  if (match(CarryIn, PatternMatch::m_ZeroInt())) {
+    Value *UAdd = Builder.CreateIntrinsic(Intrinsic::uadd_with_overflow, OpTy,
+                                          {Op1, Op2});
+    // The types have to be adjusted to match the x86 call types.
+    Value *UAddResult = Builder.CreateExtractValue(UAdd, 0);
+    Value *UAddOV = Builder.CreateZExt(Builder.CreateExtractValue(UAdd, 1),
+                                       Builder.getInt8Ty());
+    Value *Res = UndefValue::get(RetTy);
+    Res = Builder.CreateInsertValue(Res, UAddOV, 0);
+    return Builder.CreateInsertValue(Res, UAddResult, 1);
+  }
+
+  return nullptr;
+}
+
+static Value *simplifyX86insertps(const IntrinsicInst &II,
+                                  InstCombiner::BuilderTy &Builder) {
+  auto *CInt = dyn_cast<ConstantInt>(II.getArgOperand(2));
+  if (!CInt)
+    return nullptr;
+
+  VectorType *VecTy = cast<VectorType>(II.getType());
+  assert(VecTy->getNumElements() == 4 && "insertps with wrong vector type");
+
+  // The immediate permute control byte looks like this:
+  //    [3:0] - zero mask for each 32-bit lane
+  //    [5:4] - select one 32-bit destination lane
+  //    [7:6] - select one 32-bit source lane
+
+  uint8_t Imm = CInt->getZExtValue();
+  uint8_t ZMask = Imm & 0xf;
+  uint8_t DestLane = (Imm >> 4) & 0x3;
+  uint8_t SourceLane = (Imm >> 6) & 0x3;
+
+  ConstantAggregateZero *ZeroVector = ConstantAggregateZero::get(VecTy);
+
+  // If all zero mask bits are set, this was just a weird way to
+  // generate a zero vector.
+  if (ZMask == 0xf)
+    return ZeroVector;
+
+  // Initialize by passing all of the first source bits through.
+  int ShuffleMask[4] = {0, 1, 2, 3};
+
+  // We may replace the second operand with the zero vector.
+  Value *V1 = II.getArgOperand(1);
+
+  if (ZMask) {
+    // If the zero mask is being used with a single input or the zero mask
+    // overrides the destination lane, this is a shuffle with the zero vector.
+    if ((II.getArgOperand(0) == II.getArgOperand(1)) ||
+        (ZMask & (1 << DestLane))) {
+      V1 = ZeroVector;
+      // We may still move 32-bits of the first source vector from one lane
+      // to another.
+      ShuffleMask[DestLane] = SourceLane;
+      // The zero mask may override the previous insert operation.
+      for (unsigned i = 0; i < 4; ++i)
+        if ((ZMask >> i) & 0x1)
+          ShuffleMask[i] = i + 4;
+    } else {
+      // TODO: Model this case as 2 shuffles or a 'logical and' plus shuffle?
+      return nullptr;
+    }
+  } else {
+    // Replace the selected destination lane with the selected source lane.
+    ShuffleMask[DestLane] = SourceLane + 4;
+  }
+
+  return Builder.CreateShuffleVector(II.getArgOperand(0), V1, ShuffleMask);
+}
+
+/// Attempt to simplify SSE4A EXTRQ/EXTRQI instructions using constant folding
+/// or conversion to a shuffle vector.
+static Value *simplifyX86extrq(IntrinsicInst &II, Value *Op0,
+                               ConstantInt *CILength, ConstantInt *CIIndex,
+                               InstCombiner::BuilderTy &Builder) {
+  auto LowConstantHighUndef = [&](uint64_t Val) {
+    Type *IntTy64 = Type::getInt64Ty(II.getContext());
+    Constant *Args[] = {ConstantInt::get(IntTy64, Val),
+                        UndefValue::get(IntTy64)};
+    return ConstantVector::get(Args);
+  };
+
+  // See if we're dealing with constant values.
+  Constant *C0 = dyn_cast<Constant>(Op0);
+  ConstantInt *CI0 =
+      C0 ? dyn_cast_or_null<ConstantInt>(C0->getAggregateElement((unsigned)0))
+         : nullptr;
+
+  // Attempt to constant fold.
+  if (CILength && CIIndex) {
+    // From AMD documentation: "The bit index and field length are each six
+    // bits in length other bits of the field are ignored."
+    APInt APIndex = CIIndex->getValue().zextOrTrunc(6);
+    APInt APLength = CILength->getValue().zextOrTrunc(6);
+
+    unsigned Index = APIndex.getZExtValue();
+
+    // From AMD documentation: "a value of zero in the field length is
+    // defined as length of 64".
+    unsigned Length = APLength == 0 ? 64 : APLength.getZExtValue();
+
+    // From AMD documentation: "If the sum of the bit index + length field
+    // is greater than 64, the results are undefined".
+    unsigned End = Index + Length;
+
+    // Note that both field index and field length are 8-bit quantities.
+    // Since variables 'Index' and 'Length' are unsigned values
+    // obtained from zero-extending field index and field length
+    // respectively, their sum should never wrap around.
+    if (End > 64)
+      return UndefValue::get(II.getType());
+
+    // If we are inserting whole bytes, we can convert this to a shuffle.
+    // Lowering can recognize EXTRQI shuffle masks.
+    if ((Length % 8) == 0 && (Index % 8) == 0) {
+      // Convert bit indices to byte indices.
+      Length /= 8;
+      Index /= 8;
+
+      Type *IntTy8 = Type::getInt8Ty(II.getContext());
+      auto *ShufTy = FixedVectorType::get(IntTy8, 16);
+
+      SmallVector<int, 16> ShuffleMask;
+      for (int i = 0; i != (int)Length; ++i)
+        ShuffleMask.push_back(i + Index);
+      for (int i = Length; i != 8; ++i)
+        ShuffleMask.push_back(i + 16);
+      for (int i = 8; i != 16; ++i)
+        ShuffleMask.push_back(-1);
+
+      Value *SV = Builder.CreateShuffleVector(
+          Builder.CreateBitCast(Op0, ShufTy),
+          ConstantAggregateZero::get(ShufTy), ShuffleMask);
+      return Builder.CreateBitCast(SV, II.getType());
+    }
+
+    // Constant Fold - shift Index'th bit to lowest position and mask off
+    // Length bits.
+    if (CI0) {
+      APInt Elt = CI0->getValue();
+      Elt.lshrInPlace(Index);
+      Elt = Elt.zextOrTrunc(Length);
+      return LowConstantHighUndef(Elt.getZExtValue());
+    }
+
+    // If we were an EXTRQ call, we'll save registers if we convert to EXTRQI.
+    if (II.getIntrinsicID() == Intrinsic::x86_sse4a_extrq) {
+      Value *Args[] = {Op0, CILength, CIIndex};
+      Module *M = II.getModule();
+      Function *F = Intrinsic::getDeclaration(M, Intrinsic::x86_sse4a_extrqi);
+      return Builder.CreateCall(F, Args);
+    }
+  }
+
+  // Constant Fold - extraction from zero is always {zero, undef}.
+  if (CI0 && CI0->isZero())
+    return LowConstantHighUndef(0);
+
+  return nullptr;
+}
+
+/// Attempt to simplify SSE4A INSERTQ/INSERTQI instructions using constant
+/// folding or conversion to a shuffle vector.
+static Value *simplifyX86insertq(IntrinsicInst &II, Value *Op0, Value *Op1,
+                                 APInt APLength, APInt APIndex,
+                                 InstCombiner::BuilderTy &Builder) {
+  // From AMD documentation: "The bit index and field length are each six bits
+  // in length other bits of the field are ignored."
+  APIndex = APIndex.zextOrTrunc(6);
+  APLength = APLength.zextOrTrunc(6);
+
+  // Attempt to constant fold.
+  unsigned Index = APIndex.getZExtValue();
+
+  // From AMD documentation: "a value of zero in the field length is
+  // defined as length of 64".
+  unsigned Length = APLength == 0 ? 64 : APLength.getZExtValue();
+
+  // From AMD documentation: "If the sum of the bit index + length field
+  // is greater than 64, the results are undefined".
+  unsigned End = Index + Length;
+
+  // Note that both field index and field length are 8-bit quantities.
+  // Since variables 'Index' and 'Length' are unsigned values
+  // obtained from zero-extending field index and field length
+  // respectively, their sum should never wrap around.
+  if (End > 64)
+    return UndefValue::get(II.getType());
+
+  // If we are inserting whole bytes, we can convert this to a shuffle.
+  // Lowering can recognize INSERTQI shuffle masks.
+  if ((Length % 8) == 0 && (Index % 8) == 0) {
+    // Convert bit indices to byte indices.
+    Length /= 8;
+    Index /= 8;
+
+    Type *IntTy8 = Type::getInt8Ty(II.getContext());
+    auto *ShufTy = FixedVectorType::get(IntTy8, 16);
+
+    SmallVector<int, 16> ShuffleMask;
+    for (int i = 0; i != (int)Index; ++i)
+      ShuffleMask.push_back(i);
+    for (int i = 0; i != (int)Length; ++i)
+      ShuffleMask.push_back(i + 16);
+    for (int i = Index + Length; i != 8; ++i)
+      ShuffleMask.push_back(i);
+    for (int i = 8; i != 16; ++i)
+      ShuffleMask.push_back(-1);
+
+    Value *SV = Builder.CreateShuffleVector(Builder.CreateBitCast(Op0, ShufTy),
+                                            Builder.CreateBitCast(Op1, ShufTy),
+                                            ShuffleMask);
+    return Builder.CreateBitCast(SV, II.getType());
+  }
+
+  // See if we're dealing with constant values.
+  Constant *C0 = dyn_cast<Constant>(Op0);
+  Constant *C1 = dyn_cast<Constant>(Op1);
+  ConstantInt *CI00 =
+      C0 ? dyn_cast_or_null<ConstantInt>(C0->getAggregateElement((unsigned)0))
+         : nullptr;
+  ConstantInt *CI10 =
+      C1 ? dyn_cast_or_null<ConstantInt>(C1->getAggregateElement((unsigned)0))
+         : nullptr;
+
+  // Constant Fold - insert bottom Length bits starting at the Index'th bit.
+  if (CI00 && CI10) {
+    APInt V00 = CI00->getValue();
+    APInt V10 = CI10->getValue();
+    APInt Mask = APInt::getLowBitsSet(64, Length).shl(Index);
+    V00 = V00 & ~Mask;
+    V10 = V10.zextOrTrunc(Length).zextOrTrunc(64).shl(Index);
+    APInt Val = V00 | V10;
+    Type *IntTy64 = Type::getInt64Ty(II.getContext());
+    Constant *Args[] = {ConstantInt::get(IntTy64, Val.getZExtValue()),
+                        UndefValue::get(IntTy64)};
+    return ConstantVector::get(Args);
+  }
+
+  // If we were an INSERTQ call, we'll save demanded elements if we convert to
+  // INSERTQI.
+  if (II.getIntrinsicID() == Intrinsic::x86_sse4a_insertq) {
+    Type *IntTy8 = Type::getInt8Ty(II.getContext());
+    Constant *CILength = ConstantInt::get(IntTy8, Length, false);
+    Constant *CIIndex = ConstantInt::get(IntTy8, Index, false);
+
+    Value *Args[] = {Op0, Op1, CILength, CIIndex};
+    Module *M = II.getModule();
+    Function *F = Intrinsic::getDeclaration(M, Intrinsic::x86_sse4a_insertqi);
+    return Builder.CreateCall(F, Args);
+  }
+
+  return nullptr;
+}
+
+/// Attempt to convert pshufb* to shufflevector if the mask is constant.
+static Value *simplifyX86pshufb(const IntrinsicInst &II,
+                                InstCombiner::BuilderTy &Builder) {
+  Constant *V = dyn_cast<Constant>(II.getArgOperand(1));
+  if (!V)
+    return nullptr;
+
+  auto *VecTy = cast<VectorType>(II.getType());
+  unsigned NumElts = VecTy->getNumElements();
+  assert((NumElts == 16 || NumElts == 32 || NumElts == 64) &&
+         "Unexpected number of elements in shuffle mask!");
+
+  // Construct a shuffle mask from constant integers or UNDEFs.
+  int Indexes[64];
+
+  // Each byte in the shuffle control mask forms an index to permute the
+  // corresponding byte in the destination operand.
+  for (unsigned I = 0; I < NumElts; ++I) {
+    Constant *COp = V->getAggregateElement(I);
+    if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
+      return nullptr;
+
+    if (isa<UndefValue>(COp)) {
+      Indexes[I] = -1;
+      continue;
+    }
+
+    int8_t Index = cast<ConstantInt>(COp)->getValue().getZExtValue();
+
+    // If the most significant bit (bit[7]) of each byte of the shuffle
+    // control mask is set, then zero is written in the result byte.
+    // The zero vector is in the right-hand side of the resulting
+    // shufflevector.
+
+    // The value of each index for the high 128-bit lane is the least
+    // significant 4 bits of the respective shuffle control byte.
+    Index = ((Index < 0) ? NumElts : Index & 0x0F) + (I & 0xF0);
+    Indexes[I] = Index;
+  }
+
+  auto V1 = II.getArgOperand(0);
+  auto V2 = Constant::getNullValue(VecTy);
+  return Builder.CreateShuffleVector(V1, V2, makeArrayRef(Indexes, NumElts));
+}
+
+/// Attempt to convert vpermilvar* to shufflevector if the mask is constant.
+static Value *simplifyX86vpermilvar(const IntrinsicInst &II,
+                                    InstCombiner::BuilderTy &Builder) {
+  Constant *V = dyn_cast<Constant>(II.getArgOperand(1));
+  if (!V)
+    return nullptr;
+
+  auto *VecTy = cast<VectorType>(II.getType());
+  unsigned NumElts = VecTy->getNumElements();
+  bool IsPD = VecTy->getScalarType()->isDoubleTy();
+  unsigned NumLaneElts = IsPD ? 2 : 4;
+  assert(NumElts == 16 || NumElts == 8 || NumElts == 4 || NumElts == 2);
+
+  // Construct a shuffle mask from constant integers or UNDEFs.
+  int Indexes[16];
+
+  // The intrinsics only read one or two bits, clear the rest.
+  for (unsigned I = 0; I < NumElts; ++I) {
+    Constant *COp = V->getAggregateElement(I);
+    if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
+      return nullptr;
+
+    if (isa<UndefValue>(COp)) {
+      Indexes[I] = -1;
+      continue;
+    }
+
+    APInt Index = cast<ConstantInt>(COp)->getValue();
+    Index = Index.zextOrTrunc(32).getLoBits(2);
+
+    // The PD variants uses bit 1 to select per-lane element index, so
+    // shift down to convert to generic shuffle mask index.
+    if (IsPD)
+      Index.lshrInPlace(1);
+
+    // The _256 variants are a bit trickier since the mask bits always index
+    // into the corresponding 128 half. In order to convert to a generic
+    // shuffle, we have to make that explicit.
+    Index += APInt(32, (I / NumLaneElts) * NumLaneElts);
+
+    Indexes[I] = Index.getZExtValue();
+  }
+
+  auto V1 = II.getArgOperand(0);
+  auto V2 = UndefValue::get(V1->getType());
+  return Builder.CreateShuffleVector(V1, V2, makeArrayRef(Indexes, NumElts));
+}
+
+/// Attempt to convert vpermd/vpermps to shufflevector if the mask is constant.
+static Value *simplifyX86vpermv(const IntrinsicInst &II,
+                                InstCombiner::BuilderTy &Builder) {
+  auto *V = dyn_cast<Constant>(II.getArgOperand(1));
+  if (!V)
+    return nullptr;
+
+  auto *VecTy = cast<VectorType>(II.getType());
+  unsigned Size = VecTy->getNumElements();
+  assert((Size == 4 || Size == 8 || Size == 16 || Size == 32 || Size == 64) &&
+         "Unexpected shuffle mask size");
+
+  // Construct a shuffle mask from constant integers or UNDEFs.
+  int Indexes[64];
+
+  for (unsigned I = 0; I < Size; ++I) {
+    Constant *COp = V->getAggregateElement(I);
+    if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
+      return nullptr;
+
+    if (isa<UndefValue>(COp)) {
+      Indexes[I] = -1;
+      continue;
+    }
+
+    uint32_t Index = cast<ConstantInt>(COp)->getZExtValue();
+    Index &= Size - 1;
+    Indexes[I] = Index;
+  }
+
+  auto V1 = II.getArgOperand(0);
+  auto V2 = UndefValue::get(VecTy);
+  return Builder.CreateShuffleVector(V1, V2, makeArrayRef(Indexes, Size));
+}
+
+Optional<Instruction *>
+X86TTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
+  auto SimplifyDemandedVectorEltsLow = [&IC](Value *Op, unsigned Width,
+                                             unsigned DemandedWidth) {
+    APInt UndefElts(Width, 0);
+    APInt DemandedElts = APInt::getLowBitsSet(Width, DemandedWidth);
+    return IC.SimplifyDemandedVectorElts(Op, DemandedElts, UndefElts);
+  };
+
+  Intrinsic::ID IID = II.getIntrinsicID();
+  switch (IID) {
+  case Intrinsic::x86_bmi_bextr_32:
+  case Intrinsic::x86_bmi_bextr_64:
+  case Intrinsic::x86_tbm_bextri_u32:
+  case Intrinsic::x86_tbm_bextri_u64:
+    // If the RHS is a constant we can try some simplifications.
+    if (auto *C = dyn_cast<ConstantInt>(II.getArgOperand(1))) {
+      uint64_t Shift = C->getZExtValue();
+      uint64_t Length = (Shift >> 8) & 0xff;
+      Shift &= 0xff;
+      unsigned BitWidth = II.getType()->getIntegerBitWidth();
+      // If the length is 0 or the shift is out of range, replace with zero.
+      if (Length == 0 || Shift >= BitWidth) {
+        return IC.replaceInstUsesWith(II, ConstantInt::get(II.getType(), 0));
+      }
+      // If the LHS is also a constant, we can completely constant fold this.
+      if (auto *InC = dyn_cast<ConstantInt>(II.getArgOperand(0))) {
+        uint64_t Result = InC->getZExtValue() >> Shift;
+        if (Length > BitWidth)
+          Length = BitWidth;
+        Result &= maskTrailingOnes<uint64_t>(Length);
+        return IC.replaceInstUsesWith(II,
+                                      ConstantInt::get(II.getType(), Result));
+      }
+      // TODO should we turn this into 'and' if shift is 0? Or 'shl' if we
+      // are only masking bits that a shift already cleared?
+    }
+    break;
+
+  case Intrinsic::x86_bmi_bzhi_32:
+  case Intrinsic::x86_bmi_bzhi_64:
+    // If the RHS is a constant we can try some simplifications.
+    if (auto *C = dyn_cast<ConstantInt>(II.getArgOperand(1))) {
+      uint64_t Index = C->getZExtValue() & 0xff;
+      unsigned BitWidth = II.getType()->getIntegerBitWidth();
+      if (Index >= BitWidth) {
+        return IC.replaceInstUsesWith(II, II.getArgOperand(0));
+      }
+      if (Index == 0) {
+        return IC.replaceInstUsesWith(II, ConstantInt::get(II.getType(), 0));
+      }
+      // If the LHS is also a constant, we can completely constant fold this.
+      if (auto *InC = dyn_cast<ConstantInt>(II.getArgOperand(0))) {
+        uint64_t Result = InC->getZExtValue();
+        Result &= maskTrailingOnes<uint64_t>(Index);
+        return IC.replaceInstUsesWith(II,
+                                      ConstantInt::get(II.getType(), Result));
+      }
+      // TODO should we convert this to an AND if the RHS is constant?
+    }
+    break;
+  case Intrinsic::x86_bmi_pext_32:
+  case Intrinsic::x86_bmi_pext_64:
+    if (auto *MaskC = dyn_cast<ConstantInt>(II.getArgOperand(1))) {
+      if (MaskC->isNullValue()) {
+        return IC.replaceInstUsesWith(II, ConstantInt::get(II.getType(), 0));
+      }
+      if (MaskC->isAllOnesValue()) {
+        return IC.replaceInstUsesWith(II, II.getArgOperand(0));
+      }
+
+      if (auto *SrcC = dyn_cast<ConstantInt>(II.getArgOperand(0))) {
+        uint64_t Src = SrcC->getZExtValue();
+        uint64_t Mask = MaskC->getZExtValue();
+        uint64_t Result = 0;
+        uint64_t BitToSet = 1;
+
+        while (Mask) {
+          // Isolate lowest set bit.
+          uint64_t BitToTest = Mask & -Mask;
+          if (BitToTest & Src)
+            Result |= BitToSet;
+
+          BitToSet <<= 1;
+          // Clear lowest set bit.
+          Mask &= Mask - 1;
+        }
+
+        return IC.replaceInstUsesWith(II,
+                                      ConstantInt::get(II.getType(), Result));
+      }
+    }
+    break;
+  case Intrinsic::x86_bmi_pdep_32:
+  case Intrinsic::x86_bmi_pdep_64:
+    if (auto *MaskC = dyn_cast<ConstantInt>(II.getArgOperand(1))) {
+      if (MaskC->isNullValue()) {
+        return IC.replaceInstUsesWith(II, ConstantInt::get(II.getType(), 0));
+      }
+      if (MaskC->isAllOnesValue()) {
+        return IC.replaceInstUsesWith(II, II.getArgOperand(0));
+      }
+
+      if (auto *SrcC = dyn_cast<ConstantInt>(II.getArgOperand(0))) {
+        uint64_t Src = SrcC->getZExtValue();
+        uint64_t Mask = MaskC->getZExtValue();
+        uint64_t Result = 0;
+        uint64_t BitToTest = 1;
+
+        while (Mask) {
+          // Isolate lowest set bit.
+          uint64_t BitToSet = Mask & -Mask;
+          if (BitToTest & Src)
+            Result |= BitToSet;
+
+          BitToTest <<= 1;
+          // Clear lowest set bit;
+          Mask &= Mask - 1;
+        }
+
+        return IC.replaceInstUsesWith(II,
+                                      ConstantInt::get(II.getType(), Result));
+      }
+    }
+    break;
+
+  case Intrinsic::x86_sse_cvtss2si:
+  case Intrinsic::x86_sse_cvtss2si64:
+  case Intrinsic::x86_sse_cvttss2si:
+  case Intrinsic::x86_sse_cvttss2si64:
+  case Intrinsic::x86_sse2_cvtsd2si:
+  case Intrinsic::x86_sse2_cvtsd2si64:
+  case Intrinsic::x86_sse2_cvttsd2si:
+  case Intrinsic::x86_sse2_cvttsd2si64:
+  case Intrinsic::x86_avx512_vcvtss2si32:
+  case Intrinsic::x86_avx512_vcvtss2si64:
+  case Intrinsic::x86_avx512_vcvtss2usi32:
+  case Intrinsic::x86_avx512_vcvtss2usi64:
+  case Intrinsic::x86_avx512_vcvtsd2si32:
+  case Intrinsic::x86_avx512_vcvtsd2si64:
+  case Intrinsic::x86_avx512_vcvtsd2usi32:
+  case Intrinsic::x86_avx512_vcvtsd2usi64:
+  case Intrinsic::x86_avx512_cvttss2si:
+  case Intrinsic::x86_avx512_cvttss2si64:
+  case Intrinsic::x86_avx512_cvttss2usi:
+  case Intrinsic::x86_avx512_cvttss2usi64:
+  case Intrinsic::x86_avx512_cvttsd2si:
+  case Intrinsic::x86_avx512_cvttsd2si64:
+  case Intrinsic::x86_avx512_cvttsd2usi:
+  case Intrinsic::x86_avx512_cvttsd2usi64: {
+    // These intrinsics only demand the 0th element of their input vectors. If
+    // we can simplify the input based on that, do so now.
+    Value *Arg = II.getArgOperand(0);
+    unsigned VWidth = cast<VectorType>(Arg->getType())->getNumElements();
+    if (Value *V = SimplifyDemandedVectorEltsLow(Arg, VWidth, 1)) {
+      return IC.replaceOperand(II, 0, V);
+    }
+    break;
+  }
+
+  case Intrinsic::x86_mmx_pmovmskb:
+  case Intrinsic::x86_sse_movmsk_ps:
+  case Intrinsic::x86_sse2_movmsk_pd:
+  case Intrinsic::x86_sse2_pmovmskb_128:
+  case Intrinsic::x86_avx_movmsk_pd_256:
+  case Intrinsic::x86_avx_movmsk_ps_256:
+  case Intrinsic::x86_avx2_pmovmskb:
+    if (Value *V = simplifyX86movmsk(II, IC.Builder)) {
+      return IC.replaceInstUsesWith(II, V);
+    }
+    break;
+
+  case Intrinsic::x86_sse_comieq_ss:
+  case Intrinsic::x86_sse_comige_ss:
+  case Intrinsic::x86_sse_comigt_ss:
+  case Intrinsic::x86_sse_comile_ss:
+  case Intrinsic::x86_sse_comilt_ss:
+  case Intrinsic::x86_sse_comineq_ss:
+  case Intrinsic::x86_sse_ucomieq_ss:
+  case Intrinsic::x86_sse_ucomige_ss:
+  case Intrinsic::x86_sse_ucomigt_ss:
+  case Intrinsic::x86_sse_ucomile_ss:
+  case Intrinsic::x86_sse_ucomilt_ss:
+  case Intrinsic::x86_sse_ucomineq_ss:
+  case Intrinsic::x86_sse2_comieq_sd:
+  case Intrinsic::x86_sse2_comige_sd:
+  case Intrinsic::x86_sse2_comigt_sd:
+  case Intrinsic::x86_sse2_comile_sd:
+  case Intrinsic::x86_sse2_comilt_sd:
+  case Intrinsic::x86_sse2_comineq_sd:
+  case Intrinsic::x86_sse2_ucomieq_sd:
+  case Intrinsic::x86_sse2_ucomige_sd:
+  case Intrinsic::x86_sse2_ucomigt_sd:
+  case Intrinsic::x86_sse2_ucomile_sd:
+  case Intrinsic::x86_sse2_ucomilt_sd:
+  case Intrinsic::x86_sse2_ucomineq_sd:
+  case Intrinsic::x86_avx512_vcomi_ss:
+  case Intrinsic::x86_avx512_vcomi_sd:
+  case Intrinsic::x86_avx512_mask_cmp_ss:
+  case Intrinsic::x86_avx512_mask_cmp_sd: {
+    // These intrinsics only demand the 0th element of their input vectors. If
+    // we can simplify the input based on that, do so now.
+    bool MadeChange = false;
+    Value *Arg0 = II.getArgOperand(0);
+    Value *Arg1 = II.getArgOperand(1);
+    unsigned VWidth = cast<VectorType>(Arg0->getType())->getNumElements();
+    if (Value *V = SimplifyDemandedVectorEltsLow(Arg0, VWidth, 1)) {
+      IC.replaceOperand(II, 0, V);
+      MadeChange = true;
+    }
+    if (Value *V = SimplifyDemandedVectorEltsLow(Arg1, VWidth, 1)) {
+      IC.replaceOperand(II, 1, V);
+      MadeChange = true;
+    }
+    if (MadeChange) {
+      return &II;
+    }
+    break;
+  }
+  case Intrinsic::x86_avx512_cmp_pd_128:
+  case Intrinsic::x86_avx512_cmp_pd_256:
+  case Intrinsic::x86_avx512_cmp_pd_512:
+  case Intrinsic::x86_avx512_cmp_ps_128:
+  case Intrinsic::x86_avx512_cmp_ps_256:
+  case Intrinsic::x86_avx512_cmp_ps_512: {
+    // Folding cmp(sub(a,b),0) -> cmp(a,b) and cmp(0,sub(a,b)) -> cmp(b,a)
+    Value *Arg0 = II.getArgOperand(0);
+    Value *Arg1 = II.getArgOperand(1);
+    bool Arg0IsZero = match(Arg0, PatternMatch::m_PosZeroFP());
+    if (Arg0IsZero)
+      std::swap(Arg0, Arg1);
+    Value *A, *B;
+    // This fold requires only the NINF(not +/- inf) since inf minus
+    // inf is nan.
+    // NSZ(No Signed Zeros) is not needed because zeros of any sign are
+    // equal for both compares.
+    // NNAN is not needed because nans compare the same for both compares.
+    // The compare intrinsic uses the above assumptions and therefore
+    // doesn't require additional flags.
+    if ((match(Arg0,
+               PatternMatch::m_OneUse(PatternMatch::m_FSub(
+                   PatternMatch::m_Value(A), PatternMatch::m_Value(B)))) &&
+         match(Arg1, PatternMatch::m_PosZeroFP()) && isa<Instruction>(Arg0) &&
+         cast<Instruction>(Arg0)->getFastMathFlags().noInfs())) {
+      if (Arg0IsZero)
+        std::swap(A, B);
+      IC.replaceOperand(II, 0, A);
+      IC.replaceOperand(II, 1, B);
+      return &II;
+    }
+    break;
+  }
+
+  case Intrinsic::x86_avx512_add_ps_512:
+  case Intrinsic::x86_avx512_div_ps_512:
+  case Intrinsic::x86_avx512_mul_ps_512:
+  case Intrinsic::x86_avx512_sub_ps_512:
+  case Intrinsic::x86_avx512_add_pd_512:
+  case Intrinsic::x86_avx512_div_pd_512:
+  case Intrinsic::x86_avx512_mul_pd_512:
+  case Intrinsic::x86_avx512_sub_pd_512:
+    // If the rounding mode is CUR_DIRECTION(4) we can turn these into regular
+    // IR operations.
+    if (auto *R = dyn_cast<ConstantInt>(II.getArgOperand(2))) {
+      if (R->getValue() == 4) {
+        Value *Arg0 = II.getArgOperand(0);
+        Value *Arg1 = II.getArgOperand(1);
+
+        Value *V;
+        switch (IID) {
+        default:
+          llvm_unreachable("Case stmts out of sync!");
+        case Intrinsic::x86_avx512_add_ps_512:
+        case Intrinsic::x86_avx512_add_pd_512:
+          V = IC.Builder.CreateFAdd(Arg0, Arg1);
+          break;
+        case Intrinsic::x86_avx512_sub_ps_512:
+        case Intrinsic::x86_avx512_sub_pd_512:
+          V = IC.Builder.CreateFSub(Arg0, Arg1);
+          break;
+        case Intrinsic::x86_avx512_mul_ps_512:
+        case Intrinsic::x86_avx512_mul_pd_512:
+          V = IC.Builder.CreateFMul(Arg0, Arg1);
+          break;
+        case Intrinsic::x86_avx512_div_ps_512:
+        case Intrinsic::x86_avx512_div_pd_512:
+          V = IC.Builder.CreateFDiv(Arg0, Arg1);
+          break;
+        }
+
+        return IC.replaceInstUsesWith(II, V);
+      }
+    }
+    break;
+
+  case Intrinsic::x86_avx512_mask_add_ss_round:
+  case Intrinsic::x86_avx512_mask_div_ss_round:
+  case Intrinsic::x86_avx512_mask_mul_ss_round:
+  case Intrinsic::x86_avx512_mask_sub_ss_round:
+  case Intrinsic::x86_avx512_mask_add_sd_round:
+  case Intrinsic::x86_avx512_mask_div_sd_round:
+  case Intrinsic::x86_avx512_mask_mul_sd_round:
+  case Intrinsic::x86_avx512_mask_sub_sd_round:
+    // If the rounding mode is CUR_DIRECTION(4) we can turn these into regular
+    // IR operations.
+    if (auto *R = dyn_cast<ConstantInt>(II.getArgOperand(4))) {
+      if (R->getValue() == 4) {
+        // Extract the element as scalars.
+        Value *Arg0 = II.getArgOperand(0);
+        Value *Arg1 = II.getArgOperand(1);
+        Value *LHS = IC.Builder.CreateExtractElement(Arg0, (uint64_t)0);
+        Value *RHS = IC.Builder.CreateExtractElement(Arg1, (uint64_t)0);
+
+        Value *V;
+        switch (IID) {
+        default:
+          llvm_unreachable("Case stmts out of sync!");
+        case Intrinsic::x86_avx512_mask_add_ss_round:
+        case Intrinsic::x86_avx512_mask_add_sd_round:
+          V = IC.Builder.CreateFAdd(LHS, RHS);
+          break;
+        case Intrinsic::x86_avx512_mask_sub_ss_round:
+        case Intrinsic::x86_avx512_mask_sub_sd_round:
+          V = IC.Builder.CreateFSub(LHS, RHS);
+          break;
+        case Intrinsic::x86_avx512_mask_mul_ss_round:
+        case Intrinsic::x86_avx512_mask_mul_sd_round:
+          V = IC.Builder.CreateFMul(LHS, RHS);
+          break;
+        case Intrinsic::x86_avx512_mask_div_ss_round:
+        case Intrinsic::x86_avx512_mask_div_sd_round:
+          V = IC.Builder.CreateFDiv(LHS, RHS);
+          break;
+        }
+
+        // Handle the masking aspect of the intrinsic.
+        Value *Mask = II.getArgOperand(3);
+        auto *C = dyn_cast<ConstantInt>(Mask);
+        // We don't need a select if we know the mask bit is a 1.
+        if (!C || !C->getValue()[0]) {
+          // Cast the mask to an i1 vector and then extract the lowest element.
+          auto *MaskTy = FixedVectorType::get(
+              IC.Builder.getInt1Ty(),
+              cast<IntegerType>(Mask->getType())->getBitWidth());
+          Mask = IC.Builder.CreateBitCast(Mask, MaskTy);
+          Mask = IC.Builder.CreateExtractElement(Mask, (uint64_t)0);
+          // Extract the lowest element from the passthru operand.
+          Value *Passthru =
+              IC.Builder.CreateExtractElement(II.getArgOperand(2), (uint64_t)0);
+          V = IC.Builder.CreateSelect(Mask, V, Passthru);
+        }
+
+        // Insert the result back into the original argument 0.
+        V = IC.Builder.CreateInsertElement(Arg0, V, (uint64_t)0);
+
+        return IC.replaceInstUsesWith(II, V);
+      }
+    }
+    break;
+
+  // Constant fold ashr( <A x Bi>, Ci ).
+  // Constant fold lshr( <A x Bi>, Ci ).
+  // Constant fold shl( <A x Bi>, Ci ).
+  case Intrinsic::x86_sse2_psrai_d:
+  case Intrinsic::x86_sse2_psrai_w:
+  case Intrinsic::x86_avx2_psrai_d:
+  case Intrinsic::x86_avx2_psrai_w:
+  case Intrinsic::x86_avx512_psrai_q_128:
+  case Intrinsic::x86_avx512_psrai_q_256:
+  case Intrinsic::x86_avx512_psrai_d_512:
+  case Intrinsic::x86_avx512_psrai_q_512:
+  case Intrinsic::x86_avx512_psrai_w_512:
+  case Intrinsic::x86_sse2_psrli_d:
+  case Intrinsic::x86_sse2_psrli_q:
+  case Intrinsic::x86_sse2_psrli_w:
+  case Intrinsic::x86_avx2_psrli_d:
+  case Intrinsic::x86_avx2_psrli_q:
+  case Intrinsic::x86_avx2_psrli_w:
+  case Intrinsic::x86_avx512_psrli_d_512:
+  case Intrinsic::x86_avx512_psrli_q_512:
+  case Intrinsic::x86_avx512_psrli_w_512:
+  case Intrinsic::x86_sse2_pslli_d:
+  case Intrinsic::x86_sse2_pslli_q:
+  case Intrinsic::x86_sse2_pslli_w:
+  case Intrinsic::x86_avx2_pslli_d:
+  case Intrinsic::x86_avx2_pslli_q:
+  case Intrinsic::x86_avx2_pslli_w:
+  case Intrinsic::x86_avx512_pslli_d_512:
+  case Intrinsic::x86_avx512_pslli_q_512:
+  case Intrinsic::x86_avx512_pslli_w_512:
+    if (Value *V = simplifyX86immShift(II, IC.Builder)) {
+      return IC.replaceInstUsesWith(II, V);
+    }
+    break;
+
+  case Intrinsic::x86_sse2_psra_d:
+  case Intrinsic::x86_sse2_psra_w:
+  case Intrinsic::x86_avx2_psra_d:
+  case Intrinsic::x86_avx2_psra_w:
+  case Intrinsic::x86_avx512_psra_q_128:
+  case Intrinsic::x86_avx512_psra_q_256:
+  case Intrinsic::x86_avx512_psra_d_512:
+  case Intrinsic::x86_avx512_psra_q_512:
+  case Intrinsic::x86_avx512_psra_w_512:
+  case Intrinsic::x86_sse2_psrl_d:
+  case Intrinsic::x86_sse2_psrl_q:
+  case Intrinsic::x86_sse2_psrl_w:
+  case Intrinsic::x86_avx2_psrl_d:
+  case Intrinsic::x86_avx2_psrl_q:
+  case Intrinsic::x86_avx2_psrl_w:
+  case Intrinsic::x86_avx512_psrl_d_512:
+  case Intrinsic::x86_avx512_psrl_q_512:
+  case Intrinsic::x86_avx512_psrl_w_512:
+  case Intrinsic::x86_sse2_psll_d:
+  case Intrinsic::x86_sse2_psll_q:
+  case Intrinsic::x86_sse2_psll_w:
+  case Intrinsic::x86_avx2_psll_d:
+  case Intrinsic::x86_avx2_psll_q:
+  case Intrinsic::x86_avx2_psll_w:
+  case Intrinsic::x86_avx512_psll_d_512:
+  case Intrinsic::x86_avx512_psll_q_512:
+  case Intrinsic::x86_avx512_psll_w_512: {
+    if (Value *V = simplifyX86immShift(II, IC.Builder)) {
+      return IC.replaceInstUsesWith(II, V);
+    }
+
+    // SSE2/AVX2 uses only the first 64-bits of the 128-bit vector
+    // operand to compute the shift amount.
+    Value *Arg1 = II.getArgOperand(1);
+    assert(Arg1->getType()->getPrimitiveSizeInBits() == 128 &&
+           "Unexpected packed shift size");
+    unsigned VWidth = cast<VectorType>(Arg1->getType())->getNumElements();
+
+    if (Value *V = SimplifyDemandedVectorEltsLow(Arg1, VWidth, VWidth / 2)) {
+      return IC.replaceOperand(II, 1, V);
+    }
+    break;
+  }
+
+  case Intrinsic::x86_avx2_psllv_d:
+  case Intrinsic::x86_avx2_psllv_d_256:
+  case Intrinsic::x86_avx2_psllv_q:
+  case Intrinsic::x86_avx2_psllv_q_256:
+  case Intrinsic::x86_avx512_psllv_d_512:
+  case Intrinsic::x86_avx512_psllv_q_512:
+  case Intrinsic::x86_avx512_psllv_w_128:
+  case Intrinsic::x86_avx512_psllv_w_256:
+  case Intrinsic::x86_avx512_psllv_w_512:
+  case Intrinsic::x86_avx2_psrav_d:
+  case Intrinsic::x86_avx2_psrav_d_256:
+  case Intrinsic::x86_avx512_psrav_q_128:
+  case Intrinsic::x86_avx512_psrav_q_256:
+  case Intrinsic::x86_avx512_psrav_d_512:
+  case Intrinsic::x86_avx512_psrav_q_512:
+  case Intrinsic::x86_avx512_psrav_w_128:
+  case Intrinsic::x86_avx512_psrav_w_256:
+  case Intrinsic::x86_avx512_psrav_w_512:
+  case Intrinsic::x86_avx2_psrlv_d:
+  case Intrinsic::x86_avx2_psrlv_d_256:
+  case Intrinsic::x86_avx2_psrlv_q:
+  case Intrinsic::x86_avx2_psrlv_q_256:
+  case Intrinsic::x86_avx512_psrlv_d_512:
+  case Intrinsic::x86_avx512_psrlv_q_512:
+  case Intrinsic::x86_avx512_psrlv_w_128:
+  case Intrinsic::x86_avx512_psrlv_w_256:
+  case Intrinsic::x86_avx512_psrlv_w_512:
+    if (Value *V = simplifyX86varShift(II, IC.Builder)) {
+      return IC.replaceInstUsesWith(II, V);
+    }
+    break;
+
+  case Intrinsic::x86_sse2_packssdw_128:
+  case Intrinsic::x86_sse2_packsswb_128:
+  case Intrinsic::x86_avx2_packssdw:
+  case Intrinsic::x86_avx2_packsswb:
+  case Intrinsic::x86_avx512_packssdw_512:
+  case Intrinsic::x86_avx512_packsswb_512:
+    if (Value *V = simplifyX86pack(II, IC.Builder, true)) {
+      return IC.replaceInstUsesWith(II, V);
+    }
+    break;
+
+  case Intrinsic::x86_sse2_packuswb_128:
+  case Intrinsic::x86_sse41_packusdw:
+  case Intrinsic::x86_avx2_packusdw:
+  case Intrinsic::x86_avx2_packuswb:
+  case Intrinsic::x86_avx512_packusdw_512:
+  case Intrinsic::x86_avx512_packuswb_512:
+    if (Value *V = simplifyX86pack(II, IC.Builder, false)) {
+      return IC.replaceInstUsesWith(II, V);
+    }
+    break;
+
+  case Intrinsic::x86_pclmulqdq:
+  case Intrinsic::x86_pclmulqdq_256:
+  case Intrinsic::x86_pclmulqdq_512: {
+    if (auto *C = dyn_cast<ConstantInt>(II.getArgOperand(2))) {
+      unsigned Imm = C->getZExtValue();
+
+      bool MadeChange = false;
+      Value *Arg0 = II.getArgOperand(0);
+      Value *Arg1 = II.getArgOperand(1);
+      unsigned VWidth = cast<VectorType>(Arg0->getType())->getNumElements();
+
+      APInt UndefElts1(VWidth, 0);
+      APInt DemandedElts1 =
+          APInt::getSplat(VWidth, APInt(2, (Imm & 0x01) ? 2 : 1));
+      if (Value *V =
+              IC.SimplifyDemandedVectorElts(Arg0, DemandedElts1, UndefElts1)) {
+        IC.replaceOperand(II, 0, V);
+        MadeChange = true;
+      }
+
+      APInt UndefElts2(VWidth, 0);
+      APInt DemandedElts2 =
+          APInt::getSplat(VWidth, APInt(2, (Imm & 0x10) ? 2 : 1));
+      if (Value *V =
+              IC.SimplifyDemandedVectorElts(Arg1, DemandedElts2, UndefElts2)) {
+        IC.replaceOperand(II, 1, V);
+        MadeChange = true;
+      }
+
+      // If either input elements are undef, the result is zero.
+      if (DemandedElts1.isSubsetOf(UndefElts1) ||
+          DemandedElts2.isSubsetOf(UndefElts2)) {
+        return IC.replaceInstUsesWith(II,
+                                      ConstantAggregateZero::get(II.getType()));
+      }
+
+      if (MadeChange) {
+        return &II;
+      }
+    }
+    break;
+  }
+
+  case Intrinsic::x86_sse41_insertps:
+    if (Value *V = simplifyX86insertps(II, IC.Builder)) {
+      return IC.replaceInstUsesWith(II, V);
+    }
+    break;
+
+  case Intrinsic::x86_sse4a_extrq: {
+    Value *Op0 = II.getArgOperand(0);
+    Value *Op1 = II.getArgOperand(1);
+    unsigned VWidth0 = cast<VectorType>(Op0->getType())->getNumElements();
+    unsigned VWidth1 = cast<VectorType>(Op1->getType())->getNumElements();
+    assert(Op0->getType()->getPrimitiveSizeInBits() == 128 &&
+           Op1->getType()->getPrimitiveSizeInBits() == 128 && VWidth0 == 2 &&
+           VWidth1 == 16 && "Unexpected operand sizes");
+
+    // See if we're dealing with constant values.
+    Constant *C1 = dyn_cast<Constant>(Op1);
+    ConstantInt *CILength =
+        C1 ? dyn_cast_or_null<ConstantInt>(C1->getAggregateElement((unsigned)0))
+           : nullptr;
+    ConstantInt *CIIndex =
+        C1 ? dyn_cast_or_null<ConstantInt>(C1->getAggregateElement((unsigned)1))
+           : nullptr;
+
+    // Attempt to simplify to a constant, shuffle vector or EXTRQI call.
+    if (Value *V = simplifyX86extrq(II, Op0, CILength, CIIndex, IC.Builder)) {
+      return IC.replaceInstUsesWith(II, V);
+    }
+
+    // EXTRQ only uses the lowest 64-bits of the first 128-bit vector
+    // operands and the lowest 16-bits of the second.
+    bool MadeChange = false;
+    if (Value *V = SimplifyDemandedVectorEltsLow(Op0, VWidth0, 1)) {
+      IC.replaceOperand(II, 0, V);
+      MadeChange = true;
+    }
+    if (Value *V = SimplifyDemandedVectorEltsLow(Op1, VWidth1, 2)) {
+      IC.replaceOperand(II, 1, V);
+      MadeChange = true;
+    }
+    if (MadeChange) {
+      return &II;
+    }
+    break;
+  }
+
+  case Intrinsic::x86_sse4a_extrqi: {
+    // EXTRQI: Extract Length bits starting from Index. Zero pad the remaining
+    // bits of the lower 64-bits. The upper 64-bits are undefined.
+    Value *Op0 = II.getArgOperand(0);
+    unsigned VWidth = cast<VectorType>(Op0->getType())->getNumElements();
+    assert(Op0->getType()->getPrimitiveSizeInBits() == 128 && VWidth == 2 &&
+           "Unexpected operand size");
+
+    // See if we're dealing with constant values.
+    ConstantInt *CILength = dyn_cast<ConstantInt>(II.getArgOperand(1));
+    ConstantInt *CIIndex = dyn_cast<ConstantInt>(II.getArgOperand(2));
+
+    // Attempt to simplify to a constant or shuffle vector.
+    if (Value *V = simplifyX86extrq(II, Op0, CILength, CIIndex, IC.Builder)) {
+      return IC.replaceInstUsesWith(II, V);
+    }
+
+    // EXTRQI only uses the lowest 64-bits of the first 128-bit vector
+    // operand.
+    if (Value *V = SimplifyDemandedVectorEltsLow(Op0, VWidth, 1)) {
+      return IC.replaceOperand(II, 0, V);
+    }
+    break;
+  }
+
+  case Intrinsic::x86_sse4a_insertq: {
+    Value *Op0 = II.getArgOperand(0);
+    Value *Op1 = II.getArgOperand(1);
+    unsigned VWidth = cast<VectorType>(Op0->getType())->getNumElements();
+    assert(Op0->getType()->getPrimitiveSizeInBits() == 128 &&
+           Op1->getType()->getPrimitiveSizeInBits() == 128 && VWidth == 2 &&
+           cast<VectorType>(Op1->getType())->getNumElements() == 2 &&
+           "Unexpected operand size");
+
+    // See if we're dealing with constant values.
+    Constant *C1 = dyn_cast<Constant>(Op1);
+    ConstantInt *CI11 =
+        C1 ? dyn_cast_or_null<ConstantInt>(C1->getAggregateElement((unsigned)1))
+           : nullptr;
+
+    // Attempt to simplify to a constant, shuffle vector or INSERTQI call.
+    if (CI11) {
+      const APInt &V11 = CI11->getValue();
+      APInt Len = V11.zextOrTrunc(6);
+      APInt Idx = V11.lshr(8).zextOrTrunc(6);
+      if (Value *V = simplifyX86insertq(II, Op0, Op1, Len, Idx, IC.Builder)) {
+        return IC.replaceInstUsesWith(II, V);
+      }
+    }
+
+    // INSERTQ only uses the lowest 64-bits of the first 128-bit vector
+    // operand.
+    if (Value *V = SimplifyDemandedVectorEltsLow(Op0, VWidth, 1)) {
+      return IC.replaceOperand(II, 0, V);
+    }
+    break;
+  }
+
+  case Intrinsic::x86_sse4a_insertqi: {
+    // INSERTQI: Extract lowest Length bits from lower half of second source and
+    // insert over first source starting at Index bit. The upper 64-bits are
+    // undefined.
+    Value *Op0 = II.getArgOperand(0);
+    Value *Op1 = II.getArgOperand(1);
+    unsigned VWidth0 = cast<VectorType>(Op0->getType())->getNumElements();
+    unsigned VWidth1 = cast<VectorType>(Op1->getType())->getNumElements();
+    assert(Op0->getType()->getPrimitiveSizeInBits() == 128 &&
+           Op1->getType()->getPrimitiveSizeInBits() == 128 && VWidth0 == 2 &&
+           VWidth1 == 2 && "Unexpected operand sizes");
+
+    // See if we're dealing with constant values.
+    ConstantInt *CILength = dyn_cast<ConstantInt>(II.getArgOperand(2));
+    ConstantInt *CIIndex = dyn_cast<ConstantInt>(II.getArgOperand(3));
+
+    // Attempt to simplify to a constant or shuffle vector.
+    if (CILength && CIIndex) {
+      APInt Len = CILength->getValue().zextOrTrunc(6);
+      APInt Idx = CIIndex->getValue().zextOrTrunc(6);
+      if (Value *V = simplifyX86insertq(II, Op0, Op1, Len, Idx, IC.Builder)) {
+        return IC.replaceInstUsesWith(II, V);
+      }
+    }
+
+    // INSERTQI only uses the lowest 64-bits of the first two 128-bit vector
+    // operands.
+    bool MadeChange = false;
+    if (Value *V = SimplifyDemandedVectorEltsLow(Op0, VWidth0, 1)) {
+      IC.replaceOperand(II, 0, V);
+      MadeChange = true;
+    }
+    if (Value *V = SimplifyDemandedVectorEltsLow(Op1, VWidth1, 1)) {
+      IC.replaceOperand(II, 1, V);
+      MadeChange = true;
+    }
+    if (MadeChange) {
+      return &II;
+    }
+    break;
+  }
+
+  case Intrinsic::x86_sse41_pblendvb:
+  case Intrinsic::x86_sse41_blendvps:
+  case Intrinsic::x86_sse41_blendvpd:
+  case Intrinsic::x86_avx_blendv_ps_256:
+  case Intrinsic::x86_avx_blendv_pd_256:
+  case Intrinsic::x86_avx2_pblendvb: {
+    // fold (blend A, A, Mask) -> A
+    Value *Op0 = II.getArgOperand(0);
+    Value *Op1 = II.getArgOperand(1);
+    Value *Mask = II.getArgOperand(2);
+    if (Op0 == Op1) {
+      return IC.replaceInstUsesWith(II, Op0);
+    }
+
+    // Zero Mask - select 1st argument.
+    if (isa<ConstantAggregateZero>(Mask)) {
+      return IC.replaceInstUsesWith(II, Op0);
+    }
+
+    // Constant Mask - select 1st/2nd argument lane based on top bit of mask.
+    if (auto *ConstantMask = dyn_cast<ConstantDataVector>(Mask)) {
+      Constant *NewSelector = getNegativeIsTrueBoolVec(ConstantMask);
+      return SelectInst::Create(NewSelector, Op1, Op0, "blendv");
+    }
+
+    // Convert to a vector select if we can bypass casts and find a boolean
+    // vector condition value.
+    Value *BoolVec;
+    Mask = InstCombiner::peekThroughBitcast(Mask);
+    if (match(Mask, PatternMatch::m_SExt(PatternMatch::m_Value(BoolVec))) &&
+        BoolVec->getType()->isVectorTy() &&
+        BoolVec->getType()->getScalarSizeInBits() == 1) {
+      assert(Mask->getType()->getPrimitiveSizeInBits() ==
+                 II.getType()->getPrimitiveSizeInBits() &&
+             "Not expecting mask and operands with different sizes");
+
+      unsigned NumMaskElts =
+          cast<VectorType>(Mask->getType())->getNumElements();
+      unsigned NumOperandElts =
+          cast<VectorType>(II.getType())->getNumElements();
+      if (NumMaskElts == NumOperandElts) {
+        return SelectInst::Create(BoolVec, Op1, Op0);
+      }
+
+      // If the mask has less elements than the operands, each mask bit maps to
+      // multiple elements of the operands. Bitcast back and forth.
+      if (NumMaskElts < NumOperandElts) {
+        Value *CastOp0 = IC.Builder.CreateBitCast(Op0, Mask->getType());
+        Value *CastOp1 = IC.Builder.CreateBitCast(Op1, Mask->getType());
+        Value *Sel = IC.Builder.CreateSelect(BoolVec, CastOp1, CastOp0);
+        return new BitCastInst(Sel, II.getType());
+      }
+    }
+
+    break;
+  }
+
+  case Intrinsic::x86_ssse3_pshuf_b_128:
+  case Intrinsic::x86_avx2_pshuf_b:
+  case Intrinsic::x86_avx512_pshuf_b_512:
+    if (Value *V = simplifyX86pshufb(II, IC.Builder)) {
+      return IC.replaceInstUsesWith(II, V);
+    }
+    break;
+
+  case Intrinsic::x86_avx_vpermilvar_ps:
+  case Intrinsic::x86_avx_vpermilvar_ps_256:
+  case Intrinsic::x86_avx512_vpermilvar_ps_512:
+  case Intrinsic::x86_avx_vpermilvar_pd:
+  case Intrinsic::x86_avx_vpermilvar_pd_256:
+  case Intrinsic::x86_avx512_vpermilvar_pd_512:
+    if (Value *V = simplifyX86vpermilvar(II, IC.Builder)) {
+      return IC.replaceInstUsesWith(II, V);
+    }
+    break;
+
+  case Intrinsic::x86_avx2_permd:
+  case Intrinsic::x86_avx2_permps:
+  case Intrinsic::x86_avx512_permvar_df_256:
+  case Intrinsic::x86_avx512_permvar_df_512:
+  case Intrinsic::x86_avx512_permvar_di_256:
+  case Intrinsic::x86_avx512_permvar_di_512:
+  case Intrinsic::x86_avx512_permvar_hi_128:
+  case Intrinsic::x86_avx512_permvar_hi_256:
+  case Intrinsic::x86_avx512_permvar_hi_512:
+  case Intrinsic::x86_avx512_permvar_qi_128:
+  case Intrinsic::x86_avx512_permvar_qi_256:
+  case Intrinsic::x86_avx512_permvar_qi_512:
+  case Intrinsic::x86_avx512_permvar_sf_512:
+  case Intrinsic::x86_avx512_permvar_si_512:
+    if (Value *V = simplifyX86vpermv(II, IC.Builder)) {
+      return IC.replaceInstUsesWith(II, V);
+    }
+    break;
+
+  case Intrinsic::x86_avx_maskload_ps:
+  case Intrinsic::x86_avx_maskload_pd:
+  case Intrinsic::x86_avx_maskload_ps_256:
+  case Intrinsic::x86_avx_maskload_pd_256:
+  case Intrinsic::x86_avx2_maskload_d:
+  case Intrinsic::x86_avx2_maskload_q:
+  case Intrinsic::x86_avx2_maskload_d_256:
+  case Intrinsic::x86_avx2_maskload_q_256:
+    if (Instruction *I = simplifyX86MaskedLoad(II, IC)) {
+      return I;
+    }
+    break;
+
+  case Intrinsic::x86_sse2_maskmov_dqu:
+  case Intrinsic::x86_avx_maskstore_ps:
+  case Intrinsic::x86_avx_maskstore_pd:
+  case Intrinsic::x86_avx_maskstore_ps_256:
+  case Intrinsic::x86_avx_maskstore_pd_256:
+  case Intrinsic::x86_avx2_maskstore_d:
+  case Intrinsic::x86_avx2_maskstore_q:
+  case Intrinsic::x86_avx2_maskstore_d_256:
+  case Intrinsic::x86_avx2_maskstore_q_256:
+    if (simplifyX86MaskedStore(II, IC)) {
+      return nullptr;
+    }
+    break;
+
+  case Intrinsic::x86_addcarry_32:
+  case Intrinsic::x86_addcarry_64:
+    if (Value *V = simplifyX86addcarry(II, IC.Builder)) {
+      return IC.replaceInstUsesWith(II, V);
+    }
+    break;
+
+  default:
+    break;
+  }
+  return None;
+}
+
+Optional<Value *> X86TTIImpl::simplifyDemandedUseBitsIntrinsic(
+    InstCombiner &IC, IntrinsicInst &II, APInt DemandedMask, KnownBits &Known,
+    bool &KnownBitsComputed) const {
+  switch (II.getIntrinsicID()) {
+  default:
+    break;
+  case Intrinsic::x86_mmx_pmovmskb:
+  case Intrinsic::x86_sse_movmsk_ps:
+  case Intrinsic::x86_sse2_movmsk_pd:
+  case Intrinsic::x86_sse2_pmovmskb_128:
+  case Intrinsic::x86_avx_movmsk_ps_256:
+  case Intrinsic::x86_avx_movmsk_pd_256:
+  case Intrinsic::x86_avx2_pmovmskb: {
+    // MOVMSK copies the vector elements' sign bits to the low bits
+    // and zeros the high bits.
+    unsigned ArgWidth;
+    if (II.getIntrinsicID() == Intrinsic::x86_mmx_pmovmskb) {
+      ArgWidth = 8; // Arg is x86_mmx, but treated as <8 x i8>.
+    } else {
+      auto Arg = II.getArgOperand(0);
+      auto ArgType = cast<VectorType>(Arg->getType());
+      ArgWidth = ArgType->getNumElements();
+    }
+
+    // If we don't need any of low bits then return zero,
+    // we know that DemandedMask is non-zero already.
+    APInt DemandedElts = DemandedMask.zextOrTrunc(ArgWidth);
+    Type *VTy = II.getType();
+    if (DemandedElts.isNullValue()) {
+      return ConstantInt::getNullValue(VTy);
+    }
+
+    // We know that the upper bits are set to zero.
+    Known.Zero.setBitsFrom(ArgWidth);
+    KnownBitsComputed = true;
+    break;
+  }
+  case Intrinsic::x86_sse42_crc32_64_64:
+    Known.Zero.setBitsFrom(32);
+    KnownBitsComputed = true;
+    break;
+  }
+  return None;
+}
+
+Optional<Value *> X86TTIImpl::simplifyDemandedVectorEltsIntrinsic(
+    InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
+    APInt &UndefElts2, APInt &UndefElts3,
+    std::function<void(Instruction *, unsigned, APInt, APInt &)>
+        simplifyAndSetOp) const {
+  unsigned VWidth = cast<FixedVectorType>(II.getType())->getNumElements();
+  switch (II.getIntrinsicID()) {
+  default:
+    break;
+  case Intrinsic::x86_xop_vfrcz_ss:
+  case Intrinsic::x86_xop_vfrcz_sd:
+    // The instructions for these intrinsics are speced to zero upper bits not
+    // pass them through like other scalar intrinsics. So we shouldn't just
+    // use Arg0 if DemandedElts[0] is clear like we do for other intrinsics.
+    // Instead we should return a zero vector.
+    if (!DemandedElts[0]) {
+      IC.addToWorklist(&II);
+      return ConstantAggregateZero::get(II.getType());
+    }
+
+    // Only the lower element is used.
+    DemandedElts = 1;
+    simplifyAndSetOp(&II, 0, DemandedElts, UndefElts);
+
+    // Only the lower element is undefined. The high elements are zero.
+    UndefElts = UndefElts[0];
+    break;
+
+  // Unary scalar-as-vector operations that work column-wise.
+  case Intrinsic::x86_sse_rcp_ss:
+  case Intrinsic::x86_sse_rsqrt_ss:
+    simplifyAndSetOp(&II, 0, DemandedElts, UndefElts);
+
+    // If lowest element of a scalar op isn't used then use Arg0.
+    if (!DemandedElts[0]) {
+      IC.addToWorklist(&II);
+      return II.getArgOperand(0);
+    }
+    // TODO: If only low elt lower SQRT to FSQRT (with rounding/exceptions
+    // checks).
+    break;
+
+  // Binary scalar-as-vector operations that work column-wise. The high
+  // elements come from operand 0. The low element is a function of both
+  // operands.
+  case Intrinsic::x86_sse_min_ss:
+  case Intrinsic::x86_sse_max_ss:
+  case Intrinsic::x86_sse_cmp_ss:
+  case Intrinsic::x86_sse2_min_sd:
+  case Intrinsic::x86_sse2_max_sd:
+  case Intrinsic::x86_sse2_cmp_sd: {
+    simplifyAndSetOp(&II, 0, DemandedElts, UndefElts);
+
+    // If lowest element of a scalar op isn't used then use Arg0.
+    if (!DemandedElts[0]) {
+      IC.addToWorklist(&II);
+      return II.getArgOperand(0);
+    }
+
+    // Only lower element is used for operand 1.
+    DemandedElts = 1;
+    simplifyAndSetOp(&II, 1, DemandedElts, UndefElts2);
+
+    // Lower element is undefined if both lower elements are undefined.
+    // Consider things like undef&0.  The result is known zero, not undef.
+    if (!UndefElts2[0])
+      UndefElts.clearBit(0);
+
+    break;
+  }
+
+  // Binary scalar-as-vector operations that work column-wise. The high
+  // elements come from operand 0 and the low element comes from operand 1.
+  case Intrinsic::x86_sse41_round_ss:
+  case Intrinsic::x86_sse41_round_sd: {
+    // Don't use the low element of operand 0.
+    APInt DemandedElts2 = DemandedElts;
+    DemandedElts2.clearBit(0);
+    simplifyAndSetOp(&II, 0, DemandedElts2, UndefElts);
+
+    // If lowest element of a scalar op isn't used then use Arg0.
+    if (!DemandedElts[0]) {
+      IC.addToWorklist(&II);
+      return II.getArgOperand(0);
+    }
+
+    // Only lower element is used for operand 1.
+    DemandedElts = 1;
+    simplifyAndSetOp(&II, 1, DemandedElts, UndefElts2);
+
+    // Take the high undef elements from operand 0 and take the lower element
+    // from operand 1.
+    UndefElts.clearBit(0);
+    UndefElts |= UndefElts2[0];
+    break;
+  }
+
+  // Three input scalar-as-vector operations that work column-wise. The high
+  // elements come from operand 0 and the low element is a function of all
+  // three inputs.
+  case Intrinsic::x86_avx512_mask_add_ss_round:
+  case Intrinsic::x86_avx512_mask_div_ss_round:
+  case Intrinsic::x86_avx512_mask_mul_ss_round:
+  case Intrinsic::x86_avx512_mask_sub_ss_round:
+  case Intrinsic::x86_avx512_mask_max_ss_round:
+  case Intrinsic::x86_avx512_mask_min_ss_round:
+  case Intrinsic::x86_avx512_mask_add_sd_round:
+  case Intrinsic::x86_avx512_mask_div_sd_round:
+  case Intrinsic::x86_avx512_mask_mul_sd_round:
+  case Intrinsic::x86_avx512_mask_sub_sd_round:
+  case Intrinsic::x86_avx512_mask_max_sd_round:
+  case Intrinsic::x86_avx512_mask_min_sd_round:
+    simplifyAndSetOp(&II, 0, DemandedElts, UndefElts);
+
+    // If lowest element of a scalar op isn't used then use Arg0.
+    if (!DemandedElts[0]) {
+      IC.addToWorklist(&II);
+      return II.getArgOperand(0);
+    }
+
+    // Only lower element is used for operand 1 and 2.
+    DemandedElts = 1;
+    simplifyAndSetOp(&II, 1, DemandedElts, UndefElts2);
+    simplifyAndSetOp(&II, 2, DemandedElts, UndefElts3);
+
+    // Lower element is undefined if all three lower elements are undefined.
+    // Consider things like undef&0.  The result is known zero, not undef.
+    if (!UndefElts2[0] || !UndefElts3[0])
+      UndefElts.clearBit(0);
+
+    break;
+
+  case Intrinsic::x86_sse2_packssdw_128:
+  case Intrinsic::x86_sse2_packsswb_128:
+  case Intrinsic::x86_sse2_packuswb_128:
+  case Intrinsic::x86_sse41_packusdw:
+  case Intrinsic::x86_avx2_packssdw:
+  case Intrinsic::x86_avx2_packsswb:
+  case Intrinsic::x86_avx2_packusdw:
+  case Intrinsic::x86_avx2_packuswb:
+  case Intrinsic::x86_avx512_packssdw_512:
+  case Intrinsic::x86_avx512_packsswb_512:
+  case Intrinsic::x86_avx512_packusdw_512:
+  case Intrinsic::x86_avx512_packuswb_512: {
+    auto *Ty0 = II.getArgOperand(0)->getType();
+    unsigned InnerVWidth = cast<VectorType>(Ty0)->getNumElements();
+    assert(VWidth == (InnerVWidth * 2) && "Unexpected input size");
+
+    unsigned NumLanes = Ty0->getPrimitiveSizeInBits() / 128;
+    unsigned VWidthPerLane = VWidth / NumLanes;
+    unsigned InnerVWidthPerLane = InnerVWidth / NumLanes;
+
+    // Per lane, pack the elements of the first input and then the second.
+    // e.g.
+    // v8i16 PACK(v4i32 X, v4i32 Y) - (X[0..3],Y[0..3])
+    // v32i8 PACK(v16i16 X, v16i16 Y) - (X[0..7],Y[0..7]),(X[8..15],Y[8..15])
+    for (int OpNum = 0; OpNum != 2; ++OpNum) {
+      APInt OpDemandedElts(InnerVWidth, 0);
+      for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
+        unsigned LaneIdx = Lane * VWidthPerLane;
+        for (unsigned Elt = 0; Elt != InnerVWidthPerLane; ++Elt) {
+          unsigned Idx = LaneIdx + Elt + InnerVWidthPerLane * OpNum;
+          if (DemandedElts[Idx])
+            OpDemandedElts.setBit((Lane * InnerVWidthPerLane) + Elt);
+        }
+      }
+
+      // Demand elements from the operand.
+      APInt OpUndefElts(InnerVWidth, 0);
+      simplifyAndSetOp(&II, OpNum, OpDemandedElts, OpUndefElts);
+
+      // Pack the operand's UNDEF elements, one lane at a time.
+      OpUndefElts = OpUndefElts.zext(VWidth);
+      for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
+        APInt LaneElts = OpUndefElts.lshr(InnerVWidthPerLane * Lane);
+        LaneElts = LaneElts.getLoBits(InnerVWidthPerLane);
+        LaneElts <<= InnerVWidthPerLane * (2 * Lane + OpNum);
+        UndefElts |= LaneElts;
+      }
+    }
+    break;
+  }
+
+  // PSHUFB
+  case Intrinsic::x86_ssse3_pshuf_b_128:
+  case Intrinsic::x86_avx2_pshuf_b:
+  case Intrinsic::x86_avx512_pshuf_b_512:
+  // PERMILVAR
+  case Intrinsic::x86_avx_vpermilvar_ps:
+  case Intrinsic::x86_avx_vpermilvar_ps_256:
+  case Intrinsic::x86_avx512_vpermilvar_ps_512:
+  case Intrinsic::x86_avx_vpermilvar_pd:
+  case Intrinsic::x86_avx_vpermilvar_pd_256:
+  case Intrinsic::x86_avx512_vpermilvar_pd_512:
+  // PERMV
+  case Intrinsic::x86_avx2_permd:
+  case Intrinsic::x86_avx2_permps: {
+    simplifyAndSetOp(&II, 1, DemandedElts, UndefElts);
+    break;
+  }
+
+  // SSE4A instructions leave the upper 64-bits of the 128-bit result
+  // in an undefined state.
+  case Intrinsic::x86_sse4a_extrq:
+  case Intrinsic::x86_sse4a_extrqi:
+  case Intrinsic::x86_sse4a_insertq:
+  case Intrinsic::x86_sse4a_insertqi:
+    UndefElts.setHighBits(VWidth / 2);
+    break;
+  }
+  return None;
+}
diff --git a/llvm/lib/Target/X86/X86Subtarget.cpp b/llvm/lib/Target/X86/X86Subtarget.cpp
index 975cbabb30fd5..ff2a94bcac8e2 100644
--- a/llvm/lib/Target/X86/X86Subtarget.cpp
+++ b/llvm/lib/Target/X86/X86Subtarget.cpp
@@ -228,9 +228,8 @@ bool X86Subtarget::isLegalToCallImmediateAddr() const {
 }
 
 void X86Subtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
-  std::string CPUName = std::string(CPU);
-  if (CPUName.empty())
-    CPUName = "generic";
+  if (CPU.empty())
+    CPU = "generic";
 
   std::string FullFS = std::string(FS);
   if (In64BitMode) {
@@ -240,26 +239,10 @@ void X86Subtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
       FullFS = "+sse2," + FullFS;
     else
       FullFS = "+sse2";
-
-    // If no CPU was specified, enable 64bit feature to satisy later check.
-    if (CPUName == "generic") {
-      if (!FullFS.empty())
-        FullFS = "+64bit," + FullFS;
-      else
-        FullFS = "+64bit";
-    }
-  }
-
-  // LAHF/SAHF are always supported in non-64-bit mode.
-  if (!In64BitMode) {
-    if (!FullFS.empty())
-      FullFS = "+sahf," + FullFS;
-    else
-      FullFS = "+sahf";
   }
 
   // Parse features string and set the CPU.
-  ParseSubtargetFeatures(CPUName, FullFS);
+  ParseSubtargetFeatures(CPU, FullFS);
 
   // All CPUs that implement SSE4.2 or SSE4A support unaligned accesses of
   // 16-bytes and under that are reasonably fast. These features were
diff --git a/llvm/lib/Target/X86/X86Subtarget.h b/llvm/lib/Target/X86/X86Subtarget.h
index de45d357e3c2d..6f6823dea8ffd 100644
--- a/llvm/lib/Target/X86/X86Subtarget.h
+++ b/llvm/lib/Target/X86/X86Subtarget.h
@@ -50,7 +50,6 @@ enum class Style {
 } // end namespace PICStyles
 
 class X86Subtarget final : public X86GenSubtargetInfo {
-public:
   // NOTE: Do not add anything new to this list. Coarse, CPU name based flags
   // are not a good idea. We should be migrating away from these.
   enum X86ProcFamilyEnum {
@@ -59,7 +58,6 @@ class X86Subtarget final : public X86GenSubtargetInfo {
     IntelSLM
   };
 
-protected:
   enum X86SSEEnum {
     NoSSE, SSE1, SSE2, SSE3, SSSE3, SSE41, SSE42, AVX, AVX2, AVX512F
   };
@@ -191,8 +189,8 @@ class X86Subtarget final : public X86GenSubtargetInfo {
   /// Processor has RDSEED instructions.
   bool HasRDSEED = false;
 
-  /// Processor has LAHF/SAHF instructions.
-  bool HasLAHFSAHF = false;
+  /// Processor has LAHF/SAHF instructions in 64-bit mode.
+  bool HasLAHFSAHF64 = false;
 
   /// Processor has MONITORX/MWAITX instructions.
   bool HasMWAITX = false;
@@ -366,9 +364,6 @@ class X86Subtarget final : public X86GenSubtargetInfo {
   /// Processor has AVX-512 vp2intersect instructions
   bool HasVP2INTERSECT = false;
 
-  /// Deprecated flag for MPX instructions.
-  bool DeprecatedHasMPX = false;
-
   /// Processor supports CET SHSTK - Control-Flow Enforcement Technology
   /// using Shadow Stack
   bool HasSHSTK = false;
@@ -671,7 +666,7 @@ class X86Subtarget final : public X86GenSubtargetInfo {
     return hasSSE1() || (hasPRFCHW() && !has3DNow()) || hasPREFETCHWT1();
   }
   bool hasRDSEED() const { return HasRDSEED; }
-  bool hasLAHFSAHF() const { return HasLAHFSAHF; }
+  bool hasLAHFSAHF() const { return HasLAHFSAHF64 || !is64Bit(); }
   bool hasMWAITX() const { return HasMWAITX; }
   bool hasCLZERO() const { return HasCLZERO; }
   bool hasCLDEMOTE() const { return HasCLDEMOTE; }
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index d462e1f96ca26..ca875fa68523c 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -22,6 +22,8 @@
 
 namespace llvm {
 
+class InstCombiner;
+
 class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
   typedef BasicTTIImplBase<X86TTIImpl> BaseT;
   typedef TargetTransformInfo TTI;
@@ -151,6 +153,18 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
   int getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE,
                                 const SCEV *Ptr);
 
+  Optional<Instruction *> instCombineIntrinsic(InstCombiner &IC,
+                                               IntrinsicInst &II) const;
+  Optional<Value *>
+  simplifyDemandedUseBitsIntrinsic(InstCombiner &IC, IntrinsicInst &II,
+                                   APInt DemandedMask, KnownBits &Known,
+                                   bool &KnownBitsComputed) const;
+  Optional<Value *> simplifyDemandedVectorEltsIntrinsic(
+      InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
+      APInt &UndefElts2, APInt &UndefElts3,
+      std::function<void(Instruction *, unsigned, APInt, APInt &)>
+          SimplifyAndSetOp) const;
+
   unsigned getAtomicMemIntrinsicMaxElementSize() const;
 
   int getTypeBasedIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
diff --git a/llvm/lib/Target/XCore/XCoreISelLowering.cpp b/llvm/lib/Target/XCore/XCoreISelLowering.cpp
index c32653137a10b..573aee02533db 100644
--- a/llvm/lib/Target/XCore/XCoreISelLowering.cpp
+++ b/llvm/lib/Target/XCore/XCoreISelLowering.cpp
@@ -22,7 +22,6 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineJumpTableInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/SelectionDAGISel.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/Constants.h"
diff --git a/llvm/lib/Testing/Support/CMakeLists.txt b/llvm/lib/Testing/Support/CMakeLists.txt
index fe460aeefc91f..595221a105cd3 100644
--- a/llvm/lib/Testing/Support/CMakeLists.txt
+++ b/llvm/lib/Testing/Support/CMakeLists.txt
@@ -1,6 +1,10 @@
 add_definitions(-DGTEST_LANG_CXX11=1)
 add_definitions(-DGTEST_HAS_TR1_TUPLE=0)
 
+if (CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
+  add_compile_options("-Wno-suggest-override")
+endif()
+
 add_llvm_library(LLVMTestingSupport
   Annotations.cpp
   Error.cpp
diff --git a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
index d315c7f13ac2d..6acf4f4c15462 100644
--- a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
+++ b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
@@ -23,6 +23,7 @@
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Dominators.h"
+#include "llvm/IR/Function.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/LegacyPassManager.h"
 #include "llvm/IR/PatternMatch.h"
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index 7e9fd61eeb41e..dc916089d394e 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -2468,7 +2468,7 @@ struct AANoAliasCallSiteArgument final : AANoAliasImpl {
         const auto &ReachabilityAA =
             A.getAAFor<AAReachability>(*this, IRPosition::function(*ScopeFn));
 
-        if (!ReachabilityAA.isAssumedReachable(UserI, getCtxI()))
+        if (!ReachabilityAA.isAssumedReachable(A, *UserI, *getCtxI()))
           return true;
 
         if (auto *CB = dyn_cast<CallBase>(UserI)) {
diff --git a/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp b/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp
index a109d69d0f5e9..c045c277706b3 100644
--- a/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp
+++ b/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp
@@ -149,9 +149,9 @@ cl::opt<bool> EnableOrderFileInstrumentation(
     "enable-order-file-instrumentation", cl::init(false), cl::Hidden,
     cl::desc("Enable order file instrumentation (default = off)"));
 
-static cl::opt<bool>
-    EnableMatrix("enable-matrix", cl::init(false), cl::Hidden,
-                 cl::desc("Enable lowering of the matrix intrinsics"));
+cl::opt<bool> EnableMatrix(
+    "enable-matrix", cl::init(false), cl::Hidden,
+    cl::desc("Enable lowering of the matrix intrinsics"));
 
 cl::opt<AttributorRunOption> AttributorRun(
     "attributor-enable", cl::Hidden, cl::init(AttributorRunOption::NONE),
@@ -299,7 +299,7 @@ void PassManagerBuilder::populateFunctionPassManager(
   // FIXME: A lightweight version of the pass should run in the backend
   //        pipeline on demand.
   if (EnableMatrix && OptLevel == 0)
-    FPM.add(createLowerMatrixIntrinsicsPass());
+    FPM.add(createLowerMatrixIntrinsicsMinimalPass());
 
   if (OptLevel == 0) return;
 
diff --git a/llvm/lib/Transforms/InstCombine/CMakeLists.txt b/llvm/lib/Transforms/InstCombine/CMakeLists.txt
index 1a34f22f064ff..426e1c29760dc 100644
--- a/llvm/lib/Transforms/InstCombine/CMakeLists.txt
+++ b/llvm/lib/Transforms/InstCombine/CMakeLists.txt
@@ -1,7 +1,3 @@
-set(LLVM_TARGET_DEFINITIONS InstCombineTables.td)
-tablegen(LLVM InstCombineTables.inc -gen-searchable-tables)
-add_public_tablegen_target(InstCombineTableGen)
-
 add_llvm_component_library(LLVMInstCombine
   InstructionCombining.cpp
   InstCombineAddSub.cpp
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index a7f5e0a7774d2..a45fdc7fea3a9 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -29,6 +29,7 @@
 #include "llvm/Support/AlignOf.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/KnownBits.h"
+#include "llvm/Transforms/InstCombine/InstCombiner.h"
 #include <cassert>
 #include <utility>
 
@@ -860,7 +861,7 @@ static Instruction *foldNoWrapAdd(BinaryOperator &Add,
   return nullptr;
 }
 
-Instruction *InstCombiner::foldAddWithConstant(BinaryOperator &Add) {
+Instruction *InstCombinerImpl::foldAddWithConstant(BinaryOperator &Add) {
   Value *Op0 = Add.getOperand(0), *Op1 = Add.getOperand(1);
   Constant *Op1C;
   if (!match(Op1, m_Constant(Op1C)))
@@ -886,15 +887,15 @@ Instruction *InstCombiner::foldAddWithConstant(BinaryOperator &Add) {
   // zext(bool) + C -> bool ? C + 1 : C
   if (match(Op0, m_ZExt(m_Value(X))) &&
       X->getType()->getScalarSizeInBits() == 1)
-    return SelectInst::Create(X, AddOne(Op1C), Op1);
+    return SelectInst::Create(X, InstCombiner::AddOne(Op1C), Op1);
   // sext(bool) + C -> bool ? C - 1 : C
   if (match(Op0, m_SExt(m_Value(X))) &&
       X->getType()->getScalarSizeInBits() == 1)
-    return SelectInst::Create(X, SubOne(Op1C), Op1);
+    return SelectInst::Create(X, InstCombiner::SubOne(Op1C), Op1);
 
   // ~X + C --> (C-1) - X
   if (match(Op0, m_Not(m_Value(X))))
-    return BinaryOperator::CreateSub(SubOne(Op1C), X);
+    return BinaryOperator::CreateSub(InstCombiner::SubOne(Op1C), X);
 
   const APInt *C;
   if (!match(Op1, m_APInt(C)))
@@ -1021,7 +1022,7 @@ static bool MulWillOverflow(APInt &C0, APInt &C1, bool IsSigned) {
 
 // Simplifies X % C0 + (( X / C0 ) % C1) * C0 to X % (C0 * C1), where (C0 * C1)
 // does not overflow.
-Value *InstCombiner::SimplifyAddWithRemainder(BinaryOperator &I) {
+Value *InstCombinerImpl::SimplifyAddWithRemainder(BinaryOperator &I) {
   Value *LHS = I.getOperand(0), *RHS = I.getOperand(1);
   Value *X, *MulOpV;
   APInt C0, MulOpC;
@@ -1097,9 +1098,9 @@ static Instruction *foldToUnsignedSaturatedAdd(BinaryOperator &I) {
   return nullptr;
 }
 
-Instruction *
-InstCombiner::canonicalizeCondSignextOfHighBitExtractToSignextHighBitExtract(
-    BinaryOperator &I) {
+Instruction *InstCombinerImpl::
+    canonicalizeCondSignextOfHighBitExtractToSignextHighBitExtract(
+        BinaryOperator &I) {
   assert((I.getOpcode() == Instruction::Add ||
           I.getOpcode() == Instruction::Or ||
           I.getOpcode() == Instruction::Sub) &&
@@ -1198,7 +1199,7 @@ InstCombiner::canonicalizeCondSignextOfHighBitExtractToSignextHighBitExtract(
   return TruncInst::CreateTruncOrBitCast(NewAShr, I.getType());
 }
 
-Instruction *InstCombiner::visitAdd(BinaryOperator &I) {
+Instruction *InstCombinerImpl::visitAdd(BinaryOperator &I) {
   if (Value *V = SimplifyAddInst(I.getOperand(0), I.getOperand(1),
                                  I.hasNoSignedWrap(), I.hasNoUnsignedWrap(),
                                  SQ.getWithInstruction(&I)))
@@ -1486,7 +1487,7 @@ static Instruction *factorizeFAddFSub(BinaryOperator &I,
                 : BinaryOperator::CreateFDivFMF(XY, Z, &I);
 }
 
-Instruction *InstCombiner::visitFAdd(BinaryOperator &I) {
+Instruction *InstCombinerImpl::visitFAdd(BinaryOperator &I) {
   if (Value *V = SimplifyFAddInst(I.getOperand(0), I.getOperand(1),
                                   I.getFastMathFlags(),
                                   SQ.getWithInstruction(&I)))
@@ -1600,8 +1601,8 @@ Instruction *InstCombiner::visitFAdd(BinaryOperator &I) {
 /// Optimize pointer differences into the same array into a size.  Consider:
 ///  &A[10] - &A[0]: we should compile this to "10".  LHS/RHS are the pointer
 /// operands to the ptrtoint instructions for the LHS/RHS of the subtract.
-Value *InstCombiner::OptimizePointerDifference(Value *LHS, Value *RHS,
-                                               Type *Ty, bool IsNUW) {
+Value *InstCombinerImpl::OptimizePointerDifference(Value *LHS, Value *RHS,
+                                                   Type *Ty, bool IsNUW) {
   // If LHS is a gep based on RHS or RHS is a gep based on LHS, we can optimize
   // this.
   bool Swapped = false;
@@ -1692,7 +1693,7 @@ Value *InstCombiner::OptimizePointerDifference(Value *LHS, Value *RHS,
   return Builder.CreateIntCast(Result, Ty, true);
 }
 
-Instruction *InstCombiner::visitSub(BinaryOperator &I) {
+Instruction *InstCombinerImpl::visitSub(BinaryOperator &I) {
   if (Value *V = SimplifySubInst(I.getOperand(0), I.getOperand(1),
                                  I.hasNoSignedWrap(), I.hasNoUnsignedWrap(),
                                  SQ.getWithInstruction(&I)))
@@ -1806,14 +1807,14 @@ Instruction *InstCombiner::visitSub(BinaryOperator &I) {
     Value *X;
     if (match(Op1, m_ZExt(m_Value(X))) && X->getType()->isIntOrIntVectorTy(1))
       // C - (zext bool) --> bool ? C - 1 : C
-      return SelectInst::Create(X, SubOne(C), C);
+      return SelectInst::Create(X, InstCombiner::SubOne(C), C);
     if (match(Op1, m_SExt(m_Value(X))) && X->getType()->isIntOrIntVectorTy(1))
       // C - (sext bool) --> bool ? C + 1 : C
-      return SelectInst::Create(X, AddOne(C), C);
+      return SelectInst::Create(X, InstCombiner::AddOne(C), C);
 
     // C - ~X == X + (1+C)
     if (match(Op1, m_Not(m_Value(X))))
-      return BinaryOperator::CreateAdd(X, AddOne(C));
+      return BinaryOperator::CreateAdd(X, InstCombiner::AddOne(C));
 
     // Try to fold constant sub into select arguments.
     if (SelectInst *SI = dyn_cast<SelectInst>(Op1))
@@ -2094,11 +2095,11 @@ static Instruction *hoistFNegAboveFMulFDiv(Instruction &I,
   return nullptr;
 }
 
-Instruction *InstCombiner::visitFNeg(UnaryOperator &I) {
+Instruction *InstCombinerImpl::visitFNeg(UnaryOperator &I) {
   Value *Op = I.getOperand(0);
 
   if (Value *V = SimplifyFNegInst(Op, I.getFastMathFlags(),
-                                  SQ.getWithInstruction(&I)))
+                                  getSimplifyQuery().getWithInstruction(&I)))
     return replaceInstUsesWith(I, V);
 
   if (Instruction *X = foldFNegIntoConstant(I))
@@ -2117,10 +2118,10 @@ Instruction *InstCombiner::visitFNeg(UnaryOperator &I) {
   return nullptr;
 }
 
-Instruction *InstCombiner::visitFSub(BinaryOperator &I) {
+Instruction *InstCombinerImpl::visitFSub(BinaryOperator &I) {
   if (Value *V = SimplifyFSubInst(I.getOperand(0), I.getOperand(1),
                                   I.getFastMathFlags(),
-                                  SQ.getWithInstruction(&I)))
+                                  getSimplifyQuery().getWithInstruction(&I)))
     return replaceInstUsesWith(I, V);
 
   if (Instruction *X = foldVectorBinop(I))
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index 1304d46fdef4f..030d2f203ed6c 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -13,10 +13,12 @@
 #include "InstCombineInternal.h"
 #include "llvm/Analysis/CmpInstAnalysis.h"
 #include "llvm/Analysis/InstructionSimplify.h"
-#include "llvm/Transforms/Utils/Local.h"
 #include "llvm/IR/ConstantRange.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/PatternMatch.h"
+#include "llvm/Transforms/InstCombine/InstCombiner.h"
+#include "llvm/Transforms/Utils/Local.h"
+
 using namespace llvm;
 using namespace PatternMatch;
 
@@ -114,10 +116,9 @@ static Value *SimplifyBSwap(BinaryOperator &I,
 
 /// This handles expressions of the form ((val OP C1) & C2).  Where
 /// the Op parameter is 'OP', OpRHS is 'C1', and AndRHS is 'C2'.
-Instruction *InstCombiner::OptAndOp(BinaryOperator *Op,
-                                    ConstantInt *OpRHS,
-                                    ConstantInt *AndRHS,
-                                    BinaryOperator &TheAnd) {
+Instruction *InstCombinerImpl::OptAndOp(BinaryOperator *Op, ConstantInt *OpRHS,
+                                        ConstantInt *AndRHS,
+                                        BinaryOperator &TheAnd) {
   Value *X = Op->getOperand(0);
 
   switch (Op->getOpcode()) {
@@ -161,8 +162,9 @@ Instruction *InstCombiner::OptAndOp(BinaryOperator *Op,
 /// Emit a computation of: (V >= Lo && V < Hi) if Inside is true, otherwise
 /// (V < Lo || V >= Hi). This method expects that Lo < Hi. IsSigned indicates
 /// whether to treat V, Lo, and Hi as signed or not.
-Value *InstCombiner::insertRangeTest(Value *V, const APInt &Lo, const APInt &Hi,
-                                     bool isSigned, bool Inside) {
+Value *InstCombinerImpl::insertRangeTest(Value *V, const APInt &Lo,
+                                         const APInt &Hi, bool isSigned,
+                                         bool Inside) {
   assert((isSigned ? Lo.slt(Hi) : Lo.ult(Hi)) &&
          "Lo is not < Hi in range emission code!");
 
@@ -437,11 +439,10 @@ getMaskedTypeForICmpPair(Value *&A, Value *&B, Value *&C,
 /// (icmp(A & X) ==/!= Y), where the left-hand side is of type Mask_NotAllZeros
 /// and the right hand side is of type BMask_Mixed. For example,
 /// (icmp (A & 12) != 0) & (icmp (A & 15) == 8) -> (icmp (A & 15) == 8).
-static Value * foldLogOpOfMaskedICmps_NotAllZeros_BMask_Mixed(
-    ICmpInst *LHS, ICmpInst *RHS, bool IsAnd,
-    Value *A, Value *B, Value *C, Value *D, Value *E,
-    ICmpInst::Predicate PredL, ICmpInst::Predicate PredR,
-    llvm::InstCombiner::BuilderTy &Builder) {
+static Value *foldLogOpOfMaskedICmps_NotAllZeros_BMask_Mixed(
+    ICmpInst *LHS, ICmpInst *RHS, bool IsAnd, Value *A, Value *B, Value *C,
+    Value *D, Value *E, ICmpInst::Predicate PredL, ICmpInst::Predicate PredR,
+    InstCombiner::BuilderTy &Builder) {
   // We are given the canonical form:
   //   (icmp ne (A & B), 0) & (icmp eq (A & D), E).
   // where D & E == E.
@@ -568,11 +569,9 @@ static Value * foldLogOpOfMaskedICmps_NotAllZeros_BMask_Mixed(
 /// (icmp(A & X) ==/!= Y), where the left-hand side and the right hand side
 /// aren't of the common mask pattern type.
 static Value *foldLogOpOfMaskedICmpsAsymmetric(
-    ICmpInst *LHS, ICmpInst *RHS, bool IsAnd,
-    Value *A, Value *B, Value *C, Value *D, Value *E,
-    ICmpInst::Predicate PredL, ICmpInst::Predicate PredR,
-    unsigned LHSMask, unsigned RHSMask,
-    llvm::InstCombiner::BuilderTy &Builder) {
+    ICmpInst *LHS, ICmpInst *RHS, bool IsAnd, Value *A, Value *B, Value *C,
+    Value *D, Value *E, ICmpInst::Predicate PredL, ICmpInst::Predicate PredR,
+    unsigned LHSMask, unsigned RHSMask, InstCombiner::BuilderTy &Builder) {
   assert(ICmpInst::isEquality(PredL) && ICmpInst::isEquality(PredR) &&
          "Expected equality predicates for masked type of icmps.");
   // Handle Mask_NotAllZeros-BMask_Mixed cases.
@@ -603,7 +602,7 @@ static Value *foldLogOpOfMaskedICmpsAsymmetric(
 /// Try to fold (icmp(A & B) ==/!= C) &/| (icmp(A & D) ==/!= E)
 /// into a single (icmp(A & X) ==/!= Y).
 static Value *foldLogOpOfMaskedICmps(ICmpInst *LHS, ICmpInst *RHS, bool IsAnd,
-                                     llvm::InstCombiner::BuilderTy &Builder) {
+                                     InstCombiner::BuilderTy &Builder) {
   Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr, *E = nullptr;
   ICmpInst::Predicate PredL = LHS->getPredicate(), PredR = RHS->getPredicate();
   Optional<std::pair<unsigned, unsigned>> MaskPair =
@@ -748,8 +747,8 @@ static Value *foldLogOpOfMaskedICmps(ICmpInst *LHS, ICmpInst *RHS, bool IsAnd,
 /// Example: (icmp sge x, 0) & (icmp slt x, n) --> icmp ult x, n
 /// If \p Inverted is true then the check is for the inverted range, e.g.
 /// (icmp slt x, 0) | (icmp sgt x, n) --> icmp ugt x, n
-Value *InstCombiner::simplifyRangeCheck(ICmpInst *Cmp0, ICmpInst *Cmp1,
-                                        bool Inverted) {
+Value *InstCombinerImpl::simplifyRangeCheck(ICmpInst *Cmp0, ICmpInst *Cmp1,
+                                            bool Inverted) {
   // Check the lower range comparison, e.g. x >= 0
   // InstCombine already ensured that if there is a constant it's on the RHS.
   ConstantInt *RangeStart = dyn_cast<ConstantInt>(Cmp0->getOperand(1));
@@ -856,8 +855,9 @@ foldAndOrOfEqualityCmpsWithConstants(ICmpInst *LHS, ICmpInst *RHS,
 
 // Fold (iszero(A & K1) | iszero(A & K2)) -> (A & (K1 | K2)) != (K1 | K2)
 // Fold (!iszero(A & K1) & !iszero(A & K2)) -> (A & (K1 | K2)) == (K1 | K2)
-Value *InstCombiner::foldAndOrOfICmpsOfAndWithPow2(ICmpInst *LHS, ICmpInst *RHS,
-                                                   BinaryOperator &Logic) {
+Value *InstCombinerImpl::foldAndOrOfICmpsOfAndWithPow2(ICmpInst *LHS,
+                                                       ICmpInst *RHS,
+                                                       BinaryOperator &Logic) {
   bool JoinedByAnd = Logic.getOpcode() == Instruction::And;
   assert((JoinedByAnd || Logic.getOpcode() == Instruction::Or) &&
          "Wrong opcode");
@@ -1184,8 +1184,8 @@ static Value *foldAndOrOfICmpsWithConstEq(ICmpInst *Cmp0, ICmpInst *Cmp1,
 }
 
 /// Fold (icmp)&(icmp) if possible.
-Value *InstCombiner::foldAndOfICmps(ICmpInst *LHS, ICmpInst *RHS,
-                                    BinaryOperator &And) {
+Value *InstCombinerImpl::foldAndOfICmps(ICmpInst *LHS, ICmpInst *RHS,
+                                        BinaryOperator &And) {
   const SimplifyQuery Q = SQ.getWithInstruction(&And);
 
   // Fold (!iszero(A & K1) & !iszero(A & K2)) ->  (A & (K1 | K2)) == (K1 | K2)
@@ -1404,7 +1404,8 @@ Value *InstCombiner::foldAndOfICmps(ICmpInst *LHS, ICmpInst *RHS,
   return nullptr;
 }
 
-Value *InstCombiner::foldLogicOfFCmps(FCmpInst *LHS, FCmpInst *RHS, bool IsAnd) {
+Value *InstCombinerImpl::foldLogicOfFCmps(FCmpInst *LHS, FCmpInst *RHS,
+                                          bool IsAnd) {
   Value *LHS0 = LHS->getOperand(0), *LHS1 = LHS->getOperand(1);
   Value *RHS0 = RHS->getOperand(0), *RHS1 = RHS->getOperand(1);
   FCmpInst::Predicate PredL = LHS->getPredicate(), PredR = RHS->getPredicate();
@@ -1514,8 +1515,8 @@ static Instruction *matchDeMorgansLaws(BinaryOperator &I,
   Value *A, *B;
   if (match(I.getOperand(0), m_OneUse(m_Not(m_Value(A)))) &&
       match(I.getOperand(1), m_OneUse(m_Not(m_Value(B)))) &&
-      !isFreeToInvert(A, A->hasOneUse()) &&
-      !isFreeToInvert(B, B->hasOneUse())) {
+      !InstCombiner::isFreeToInvert(A, A->hasOneUse()) &&
+      !InstCombiner::isFreeToInvert(B, B->hasOneUse())) {
     Value *AndOr = Builder.CreateBinOp(Opcode, A, B, I.getName() + ".demorgan");
     return BinaryOperator::CreateNot(AndOr);
   }
@@ -1523,7 +1524,7 @@ static Instruction *matchDeMorgansLaws(BinaryOperator &I,
   return nullptr;
 }
 
-bool InstCombiner::shouldOptimizeCast(CastInst *CI) {
+bool InstCombinerImpl::shouldOptimizeCast(CastInst *CI) {
   Value *CastSrc = CI->getOperand(0);
 
   // Noop casts and casts of constants should be eliminated trivially.
@@ -1579,7 +1580,7 @@ static Instruction *foldLogicCastConstant(BinaryOperator &Logic, CastInst *Cast,
 }
 
 /// Fold {and,or,xor} (cast X), Y.
-Instruction *InstCombiner::foldCastedBitwiseLogic(BinaryOperator &I) {
+Instruction *InstCombinerImpl::foldCastedBitwiseLogic(BinaryOperator &I) {
   auto LogicOpc = I.getOpcode();
   assert(I.isBitwiseLogicOp() && "Unexpected opcode for bitwise logic folding");
 
@@ -1725,7 +1726,7 @@ static bool canNarrowShiftAmt(Constant *C, unsigned BitWidth) {
 
 /// Try to use narrower ops (sink zext ops) for an 'and' with binop operand and
 /// a common zext operand: and (binop (zext X), C), (zext X).
-Instruction *InstCombiner::narrowMaskedBinOp(BinaryOperator &And) {
+Instruction *InstCombinerImpl::narrowMaskedBinOp(BinaryOperator &And) {
   // This transform could also apply to {or, and, xor}, but there are better
   // folds for those cases, so we don't expect those patterns here. AShr is not
   // handled because it should always be transformed to LShr in this sequence.
@@ -1767,7 +1768,7 @@ Instruction *InstCombiner::narrowMaskedBinOp(BinaryOperator &And) {
 // FIXME: We use commutative matchers (m_c_*) for some, but not all, matches
 // here. We should standardize that construct where it is needed or choose some
 // other way to ensure that commutated variants of patterns are not missed.
-Instruction *InstCombiner::visitAnd(BinaryOperator &I) {
+Instruction *InstCombinerImpl::visitAnd(BinaryOperator &I) {
   if (Value *V = SimplifyAndInst(I.getOperand(0), I.getOperand(1),
                                  SQ.getWithInstruction(&I)))
     return replaceInstUsesWith(I, V);
@@ -2033,7 +2034,7 @@ Instruction *InstCombiner::visitAnd(BinaryOperator &I) {
   return nullptr;
 }
 
-Instruction *InstCombiner::matchBSwap(BinaryOperator &Or) {
+Instruction *InstCombinerImpl::matchBSwap(BinaryOperator &Or) {
   assert(Or.getOpcode() == Instruction::Or && "bswap requires an 'or'");
   Value *Op0 = Or.getOperand(0), *Op1 = Or.getOperand(1);
 
@@ -2216,7 +2217,7 @@ static bool areInverseVectorBitmasks(Constant *C1, Constant *C2) {
 /// We have an expression of the form (A & C) | (B & D). If A is a scalar or
 /// vector composed of all-zeros or all-ones values and is the bitwise 'not' of
 /// B, it can be used as the condition operand of a select instruction.
-Value *InstCombiner::getSelectCondition(Value *A, Value *B) {
+Value *InstCombinerImpl::getSelectCondition(Value *A, Value *B) {
   // Step 1: We may have peeked through bitcasts in the caller.
   // Exit immediately if we don't have (vector) integer types.
   Type *Ty = A->getType();
@@ -2273,8 +2274,8 @@ Value *InstCombiner::getSelectCondition(Value *A, Value *B) {
 
 /// We have an expression of the form (A & C) | (B & D). Try to simplify this
 /// to "A' ? C : D", where A' is a boolean or vector of booleans.
-Value *InstCombiner::matchSelectFromAndOr(Value *A, Value *C, Value *B,
-                                          Value *D) {
+Value *InstCombinerImpl::matchSelectFromAndOr(Value *A, Value *C, Value *B,
+                                              Value *D) {
   // The potential condition of the select may be bitcasted. In that case, look
   // through its bitcast and the corresponding bitcast of the 'not' condition.
   Type *OrigType = A->getType();
@@ -2294,8 +2295,8 @@ Value *InstCombiner::matchSelectFromAndOr(Value *A, Value *C, Value *B,
 }
 
 /// Fold (icmp)|(icmp) if possible.
-Value *InstCombiner::foldOrOfICmps(ICmpInst *LHS, ICmpInst *RHS,
-                                   BinaryOperator &Or) {
+Value *InstCombinerImpl::foldOrOfICmps(ICmpInst *LHS, ICmpInst *RHS,
+                                       BinaryOperator &Or) {
   const SimplifyQuery Q = SQ.getWithInstruction(&Or);
 
   // Fold (iszero(A & K1) | iszero(A & K2)) ->  (A & (K1 | K2)) != (K1 | K2)
@@ -2560,7 +2561,7 @@ Value *InstCombiner::foldOrOfICmps(ICmpInst *LHS, ICmpInst *RHS,
 // FIXME: We use commutative matchers (m_c_*) for some, but not all, matches
 // here. We should standardize that construct where it is needed or choose some
 // other way to ensure that commutated variants of patterns are not missed.
-Instruction *InstCombiner::visitOr(BinaryOperator &I) {
+Instruction *InstCombinerImpl::visitOr(BinaryOperator &I) {
   if (Value *V = SimplifyOrInst(I.getOperand(0), I.getOperand(1),
                                 SQ.getWithInstruction(&I)))
     return replaceInstUsesWith(I, V);
@@ -2929,8 +2930,8 @@ static Instruction *foldXorToXor(BinaryOperator &I,
   return nullptr;
 }
 
-Value *InstCombiner::foldXorOfICmps(ICmpInst *LHS, ICmpInst *RHS,
-                                    BinaryOperator &I) {
+Value *InstCombinerImpl::foldXorOfICmps(ICmpInst *LHS, ICmpInst *RHS,
+                                        BinaryOperator &I) {
   assert(I.getOpcode() == Instruction::Xor && I.getOperand(0) == LHS &&
          I.getOperand(1) == RHS && "Should be 'xor' with these operands");
 
@@ -3088,9 +3089,9 @@ static Instruction *sinkNotIntoXor(BinaryOperator &I,
     return nullptr;
 
   // We only want to do the transform if it is free to do.
-  if (isFreeToInvert(X, X->hasOneUse())) {
+  if (InstCombiner::isFreeToInvert(X, X->hasOneUse())) {
     // Ok, good.
-  } else if (isFreeToInvert(Y, Y->hasOneUse())) {
+  } else if (InstCombiner::isFreeToInvert(Y, Y->hasOneUse())) {
     std::swap(X, Y);
   } else
     return nullptr;
@@ -3102,7 +3103,7 @@ static Instruction *sinkNotIntoXor(BinaryOperator &I,
 // FIXME: We use commutative matchers (m_c_*) for some, but not all, matches
 // here. We should standardize that construct where it is needed or choose some
 // other way to ensure that commutated variants of patterns are not missed.
-Instruction *InstCombiner::visitXor(BinaryOperator &I) {
+Instruction *InstCombinerImpl::visitXor(BinaryOperator &I) {
   if (Value *V = SimplifyXorInst(I.getOperand(0), I.getOperand(1),
                                  SQ.getWithInstruction(&I)))
     return replaceInstUsesWith(I, V);
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp
index ba1cf982229d7..10b608d38bbc9 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp
@@ -9,8 +9,10 @@
 // This file implements the visit functions for atomic rmw instructions.
 //
 //===----------------------------------------------------------------------===//
+
 #include "InstCombineInternal.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/Transforms/InstCombine/InstCombiner.h"
 
 using namespace llvm;
 
@@ -30,7 +32,7 @@ bool isIdempotentRMW(AtomicRMWInst& RMWI) {
     default:
       return false;
     };
-  
+
   auto C = dyn_cast<ConstantInt>(RMWI.getValOperand());
   if(!C)
     return false;
@@ -93,11 +95,11 @@ bool isSaturating(AtomicRMWInst& RMWI) {
 }
 }
 
-Instruction *InstCombiner::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
+Instruction *InstCombinerImpl::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
 
   // Volatile RMWs perform a load and a store, we cannot replace this by just a
   // load or just a store. We chose not to canonicalize out of general paranoia
-  // about user expectations around volatile. 
+  // about user expectations around volatile.
   if (RMWI.isVolatile())
     return nullptr;
 
@@ -115,7 +117,7 @@ Instruction *InstCombiner::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
          "AtomicRMWs don't make sense with Unordered or NotAtomic");
 
   // Any atomicrmw xchg with no uses can be converted to a atomic store if the
-  // ordering is compatible. 
+  // ordering is compatible.
   if (RMWI.getOperation() == AtomicRMWInst::Xchg &&
       RMWI.use_empty()) {
     if (Ordering != AtomicOrdering::Release &&
@@ -127,14 +129,14 @@ Instruction *InstCombiner::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
     SI->setAlignment(DL.getABITypeAlign(RMWI.getType()));
     return eraseInstFromFunction(RMWI);
   }
-  
+
   if (!isIdempotentRMW(RMWI))
     return nullptr;
 
   // We chose to canonicalize all idempotent operations to an single
   // operation code and constant.  This makes it easier for the rest of the
   // optimizer to match easily.  The choices of or w/0 and fadd w/-0.0 are
-  // arbitrary. 
+  // arbitrary.
   if (RMWI.getType()->isIntegerTy() &&
       RMWI.getOperation() != AtomicRMWInst::Or) {
     RMWI.setOperation(AtomicRMWInst::Or);
@@ -149,7 +151,7 @@ Instruction *InstCombiner::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
   if (Ordering != AtomicOrdering::Acquire &&
       Ordering != AtomicOrdering::Monotonic)
     return nullptr;
-  
+
   LoadInst *Load = new LoadInst(RMWI.getType(), RMWI.getPointerOperand(), "",
                                 false, DL.getABITypeAlign(RMWI.getType()),
                                 Ordering, RMWI.getSyncScopeID());
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 836af6234ad5c..ed93c33c1a597 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -28,6 +28,7 @@
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/Loads.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/Attributes.h"
@@ -47,9 +48,6 @@
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/IntrinsicsARM.h"
 #include "llvm/IR/IntrinsicsHexagon.h"
-#include "llvm/IR/IntrinsicsNVPTX.h"
-#include "llvm/IR/IntrinsicsPowerPC.h"
-#include "llvm/IR/IntrinsicsX86.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/PatternMatch.h"
@@ -68,6 +66,7 @@
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/InstCombine/InstCombineWorklist.h"
+#include "llvm/Transforms/InstCombine/InstCombiner.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Transforms/Utils/SimplifyLibCalls.h"
 #include <algorithm>
@@ -100,24 +99,7 @@ static Type *getPromotedType(Type *Ty) {
   return Ty;
 }
 
-/// Return a constant boolean vector that has true elements in all positions
-/// where the input constant data vector has an element with the sign bit set.
-static Constant *getNegativeIsTrueBoolVec(ConstantDataVector *V) {
-  SmallVector<Constant *, 32> BoolVec;
-  IntegerType *BoolTy = Type::getInt1Ty(V->getContext());
-  for (unsigned I = 0, E = V->getNumElements(); I != E; ++I) {
-    Constant *Elt = V->getElementAsConstant(I);
-    assert((isa<ConstantInt>(Elt) || isa<ConstantFP>(Elt)) &&
-           "Unexpected constant data vector element type");
-    bool Sign = V->getElementType()->isIntegerTy()
-                    ? cast<ConstantInt>(Elt)->isNegative()
-                    : cast<ConstantFP>(Elt)->isNegative();
-    BoolVec.push_back(ConstantInt::get(BoolTy, Sign));
-  }
-  return ConstantVector::get(BoolVec);
-}
-
-Instruction *InstCombiner::SimplifyAnyMemTransfer(AnyMemTransferInst *MI) {
+Instruction *InstCombinerImpl::SimplifyAnyMemTransfer(AnyMemTransferInst *MI) {
   Align DstAlign = getKnownAlignment(MI->getRawDest(), DL, MI, &AC, &DT);
   MaybeAlign CopyDstAlign = MI->getDestAlign();
   if (!CopyDstAlign || *CopyDstAlign < DstAlign) {
@@ -232,7 +214,7 @@ Instruction *InstCombiner::SimplifyAnyMemTransfer(AnyMemTransferInst *MI) {
   return MI;
 }
 
-Instruction *InstCombiner::SimplifyAnyMemSet(AnyMemSetInst *MI) {
+Instruction *InstCombinerImpl::SimplifyAnyMemSet(AnyMemSetInst *MI) {
   const Align KnownAlignment =
       getKnownAlignment(MI->getDest(), DL, MI, &AC, &DT);
   MaybeAlign MemSetAlign = MI->getDestAlign();
@@ -292,820 +274,9 @@ Instruction *InstCombiner::SimplifyAnyMemSet(AnyMemSetInst *MI) {
   return nullptr;
 }
 
-static Value *simplifyX86immShift(const IntrinsicInst &II,
-                                  InstCombiner::BuilderTy &Builder) {
-  bool LogicalShift = false;
-  bool ShiftLeft = false;
-  bool IsImm = false;
-
-  switch (II.getIntrinsicID()) {
-  default: llvm_unreachable("Unexpected intrinsic!");
-  case Intrinsic::x86_sse2_psrai_d:
-  case Intrinsic::x86_sse2_psrai_w:
-  case Intrinsic::x86_avx2_psrai_d:
-  case Intrinsic::x86_avx2_psrai_w:
-  case Intrinsic::x86_avx512_psrai_q_128:
-  case Intrinsic::x86_avx512_psrai_q_256:
-  case Intrinsic::x86_avx512_psrai_d_512:
-  case Intrinsic::x86_avx512_psrai_q_512:
-  case Intrinsic::x86_avx512_psrai_w_512:
-    IsImm = true;
-    LLVM_FALLTHROUGH;
-  case Intrinsic::x86_sse2_psra_d:
-  case Intrinsic::x86_sse2_psra_w:
-  case Intrinsic::x86_avx2_psra_d:
-  case Intrinsic::x86_avx2_psra_w:
-  case Intrinsic::x86_avx512_psra_q_128:
-  case Intrinsic::x86_avx512_psra_q_256:
-  case Intrinsic::x86_avx512_psra_d_512:
-  case Intrinsic::x86_avx512_psra_q_512:
-  case Intrinsic::x86_avx512_psra_w_512:
-    LogicalShift = false;
-    ShiftLeft = false;
-    break;
-  case Intrinsic::x86_sse2_psrli_d:
-  case Intrinsic::x86_sse2_psrli_q:
-  case Intrinsic::x86_sse2_psrli_w:
-  case Intrinsic::x86_avx2_psrli_d:
-  case Intrinsic::x86_avx2_psrli_q:
-  case Intrinsic::x86_avx2_psrli_w:
-  case Intrinsic::x86_avx512_psrli_d_512:
-  case Intrinsic::x86_avx512_psrli_q_512:
-  case Intrinsic::x86_avx512_psrli_w_512:
-    IsImm = true;
-    LLVM_FALLTHROUGH;
-  case Intrinsic::x86_sse2_psrl_d:
-  case Intrinsic::x86_sse2_psrl_q:
-  case Intrinsic::x86_sse2_psrl_w:
-  case Intrinsic::x86_avx2_psrl_d:
-  case Intrinsic::x86_avx2_psrl_q:
-  case Intrinsic::x86_avx2_psrl_w:
-  case Intrinsic::x86_avx512_psrl_d_512:
-  case Intrinsic::x86_avx512_psrl_q_512:
-  case Intrinsic::x86_avx512_psrl_w_512:
-    LogicalShift = true;
-    ShiftLeft = false;
-    break;
-  case Intrinsic::x86_sse2_pslli_d:
-  case Intrinsic::x86_sse2_pslli_q:
-  case Intrinsic::x86_sse2_pslli_w:
-  case Intrinsic::x86_avx2_pslli_d:
-  case Intrinsic::x86_avx2_pslli_q:
-  case Intrinsic::x86_avx2_pslli_w:
-  case Intrinsic::x86_avx512_pslli_d_512:
-  case Intrinsic::x86_avx512_pslli_q_512:
-  case Intrinsic::x86_avx512_pslli_w_512:
-    IsImm = true;
-    LLVM_FALLTHROUGH;
-  case Intrinsic::x86_sse2_psll_d:
-  case Intrinsic::x86_sse2_psll_q:
-  case Intrinsic::x86_sse2_psll_w:
-  case Intrinsic::x86_avx2_psll_d:
-  case Intrinsic::x86_avx2_psll_q:
-  case Intrinsic::x86_avx2_psll_w:
-  case Intrinsic::x86_avx512_psll_d_512:
-  case Intrinsic::x86_avx512_psll_q_512:
-  case Intrinsic::x86_avx512_psll_w_512:
-    LogicalShift = true;
-    ShiftLeft = true;
-    break;
-  }
-  assert((LogicalShift || !ShiftLeft) && "Only logical shifts can shift left");
-
-  auto Vec = II.getArgOperand(0);
-  auto Amt = II.getArgOperand(1);
-  auto VT = cast<VectorType>(Vec->getType());
-  auto SVT = VT->getElementType();
-  auto AmtVT = Amt->getType();
-  unsigned VWidth = VT->getNumElements();
-  unsigned BitWidth = SVT->getPrimitiveSizeInBits();
-
-  // If the shift amount is guaranteed to be in-range we can replace it with a
-  // generic shift. If its guaranteed to be out of range, logical shifts combine to
-  // zero and arithmetic shifts are clamped to (BitWidth - 1).
-  if (IsImm) {
-    assert(AmtVT ->isIntegerTy(32) &&
-           "Unexpected shift-by-immediate type");
-    KnownBits KnownAmtBits =
-        llvm::computeKnownBits(Amt, II.getModule()->getDataLayout());
-    if (KnownAmtBits.getMaxValue().ult(BitWidth)) {
-      Amt = Builder.CreateZExtOrTrunc(Amt, SVT);
-      Amt = Builder.CreateVectorSplat(VWidth, Amt);
-      return (LogicalShift ? (ShiftLeft ? Builder.CreateShl(Vec, Amt)
-                                        : Builder.CreateLShr(Vec, Amt))
-                           : Builder.CreateAShr(Vec, Amt));
-    }
-    if (KnownAmtBits.getMinValue().uge(BitWidth)) {
-      if (LogicalShift)
-        return ConstantAggregateZero::get(VT);
-      Amt = ConstantInt::get(SVT, BitWidth - 1);
-      return Builder.CreateAShr(Vec, Builder.CreateVectorSplat(VWidth, Amt));
-    }
-  } else {
-    // Ensure the first element has an in-range value and the rest of the
-    // elements in the bottom 64 bits are zero.
-    assert(AmtVT->isVectorTy() && AmtVT->getPrimitiveSizeInBits() == 128 &&
-           cast<VectorType>(AmtVT)->getElementType() == SVT &&
-           "Unexpected shift-by-scalar type");
-    unsigned NumAmtElts = cast<VectorType>(AmtVT)->getNumElements();
-    APInt DemandedLower = APInt::getOneBitSet(NumAmtElts, 0);
-    APInt DemandedUpper = APInt::getBitsSet(NumAmtElts, 1, NumAmtElts / 2);
-    KnownBits KnownLowerBits = llvm::computeKnownBits(
-        Amt, DemandedLower, II.getModule()->getDataLayout());
-    KnownBits KnownUpperBits = llvm::computeKnownBits(
-        Amt, DemandedUpper, II.getModule()->getDataLayout());
-    if (KnownLowerBits.getMaxValue().ult(BitWidth) &&
-        (DemandedUpper.isNullValue() || KnownUpperBits.isZero())) {
-      SmallVector<int, 16> ZeroSplat(VWidth, 0);
-      Amt = Builder.CreateShuffleVector(Amt, Amt, ZeroSplat);
-      return (LogicalShift ? (ShiftLeft ? Builder.CreateShl(Vec, Amt)
-                                        : Builder.CreateLShr(Vec, Amt))
-                           : Builder.CreateAShr(Vec, Amt));
-    }
-  }
-
-  // Simplify if count is constant vector.
-  auto CDV = dyn_cast<ConstantDataVector>(Amt);
-  if (!CDV)
-    return nullptr;
-
-  // SSE2/AVX2 uses all the first 64-bits of the 128-bit vector
-  // operand to compute the shift amount.
-  assert(AmtVT->isVectorTy() && AmtVT->getPrimitiveSizeInBits() == 128 &&
-         cast<VectorType>(AmtVT)->getElementType() == SVT &&
-         "Unexpected shift-by-scalar type");
-
-  // Concatenate the sub-elements to create the 64-bit value.
-  APInt Count(64, 0);
-  for (unsigned i = 0, NumSubElts = 64 / BitWidth; i != NumSubElts; ++i) {
-    unsigned SubEltIdx = (NumSubElts - 1) - i;
-    auto SubElt = cast<ConstantInt>(CDV->getElementAsConstant(SubEltIdx));
-    Count <<= BitWidth;
-    Count |= SubElt->getValue().zextOrTrunc(64);
-  }
-
-  // If shift-by-zero then just return the original value.
-  if (Count.isNullValue())
-    return Vec;
-
-  // Handle cases when Shift >= BitWidth.
-  if (Count.uge(BitWidth)) {
-    // If LogicalShift - just return zero.
-    if (LogicalShift)
-      return ConstantAggregateZero::get(VT);
-
-    // If ArithmeticShift - clamp Shift to (BitWidth - 1).
-    Count = APInt(64, BitWidth - 1);
-  }
-
-  // Get a constant vector of the same type as the first operand.
-  auto ShiftAmt = ConstantInt::get(SVT, Count.zextOrTrunc(BitWidth));
-  auto ShiftVec = Builder.CreateVectorSplat(VWidth, ShiftAmt);
-
-  if (ShiftLeft)
-    return Builder.CreateShl(Vec, ShiftVec);
-
-  if (LogicalShift)
-    return Builder.CreateLShr(Vec, ShiftVec);
-
-  return Builder.CreateAShr(Vec, ShiftVec);
-}
-
-// Attempt to simplify AVX2 per-element shift intrinsics to a generic IR shift.
-// Unlike the generic IR shifts, the intrinsics have defined behaviour for out
-// of range shift amounts (logical - set to zero, arithmetic - splat sign bit).
-static Value *simplifyX86varShift(const IntrinsicInst &II,
-                                  InstCombiner::BuilderTy &Builder) {
-  bool LogicalShift = false;
-  bool ShiftLeft = false;
-
-  switch (II.getIntrinsicID()) {
-  default: llvm_unreachable("Unexpected intrinsic!");
-  case Intrinsic::x86_avx2_psrav_d:
-  case Intrinsic::x86_avx2_psrav_d_256:
-  case Intrinsic::x86_avx512_psrav_q_128:
-  case Intrinsic::x86_avx512_psrav_q_256:
-  case Intrinsic::x86_avx512_psrav_d_512:
-  case Intrinsic::x86_avx512_psrav_q_512:
-  case Intrinsic::x86_avx512_psrav_w_128:
-  case Intrinsic::x86_avx512_psrav_w_256:
-  case Intrinsic::x86_avx512_psrav_w_512:
-    LogicalShift = false;
-    ShiftLeft = false;
-    break;
-  case Intrinsic::x86_avx2_psrlv_d:
-  case Intrinsic::x86_avx2_psrlv_d_256:
-  case Intrinsic::x86_avx2_psrlv_q:
-  case Intrinsic::x86_avx2_psrlv_q_256:
-  case Intrinsic::x86_avx512_psrlv_d_512:
-  case Intrinsic::x86_avx512_psrlv_q_512:
-  case Intrinsic::x86_avx512_psrlv_w_128:
-  case Intrinsic::x86_avx512_psrlv_w_256:
-  case Intrinsic::x86_avx512_psrlv_w_512:
-    LogicalShift = true;
-    ShiftLeft = false;
-    break;
-  case Intrinsic::x86_avx2_psllv_d:
-  case Intrinsic::x86_avx2_psllv_d_256:
-  case Intrinsic::x86_avx2_psllv_q:
-  case Intrinsic::x86_avx2_psllv_q_256:
-  case Intrinsic::x86_avx512_psllv_d_512:
-  case Intrinsic::x86_avx512_psllv_q_512:
-  case Intrinsic::x86_avx512_psllv_w_128:
-  case Intrinsic::x86_avx512_psllv_w_256:
-  case Intrinsic::x86_avx512_psllv_w_512:
-    LogicalShift = true;
-    ShiftLeft = true;
-    break;
-  }
-  assert((LogicalShift || !ShiftLeft) && "Only logical shifts can shift left");
-
-  auto Vec = II.getArgOperand(0);
-  auto Amt = II.getArgOperand(1);
-  auto VT = cast<VectorType>(II.getType());
-  auto SVT = VT->getElementType();
-  int NumElts = VT->getNumElements();
-  int BitWidth = SVT->getIntegerBitWidth();
-
-  // If the shift amount is guaranteed to be in-range we can replace it with a
-  // generic shift.
-  APInt UpperBits =
-      APInt::getHighBitsSet(BitWidth, BitWidth - Log2_32(BitWidth));
-  if (llvm::MaskedValueIsZero(Amt, UpperBits,
-                              II.getModule()->getDataLayout())) {
-    return (LogicalShift ? (ShiftLeft ? Builder.CreateShl(Vec, Amt)
-                                      : Builder.CreateLShr(Vec, Amt))
-                         : Builder.CreateAShr(Vec, Amt));
-  }
-
-  // Simplify if all shift amounts are constant/undef.
-  auto *CShift = dyn_cast<Constant>(Amt);
-  if (!CShift)
-    return nullptr;
-
-  // Collect each element's shift amount.
-  // We also collect special cases: UNDEF = -1, OUT-OF-RANGE = BitWidth.
-  bool AnyOutOfRange = false;
-  SmallVector<int, 8> ShiftAmts;
-  for (int I = 0; I < NumElts; ++I) {
-    auto *CElt = CShift->getAggregateElement(I);
-    if (CElt && isa<UndefValue>(CElt)) {
-      ShiftAmts.push_back(-1);
-      continue;
-    }
-
-    auto *COp = dyn_cast_or_null<ConstantInt>(CElt);
-    if (!COp)
-      return nullptr;
-
-    // Handle out of range shifts.
-    // If LogicalShift - set to BitWidth (special case).
-    // If ArithmeticShift - set to (BitWidth - 1) (sign splat).
-    APInt ShiftVal = COp->getValue();
-    if (ShiftVal.uge(BitWidth)) {
-      AnyOutOfRange = LogicalShift;
-      ShiftAmts.push_back(LogicalShift ? BitWidth : BitWidth - 1);
-      continue;
-    }
-
-    ShiftAmts.push_back((int)ShiftVal.getZExtValue());
-  }
-
-  // If all elements out of range or UNDEF, return vector of zeros/undefs.
-  // ArithmeticShift should only hit this if they are all UNDEF.
-  auto OutOfRange = [&](int Idx) { return (Idx < 0) || (BitWidth <= Idx); };
-  if (llvm::all_of(ShiftAmts, OutOfRange)) {
-    SmallVector<Constant *, 8> ConstantVec;
-    for (int Idx : ShiftAmts) {
-      if (Idx < 0) {
-        ConstantVec.push_back(UndefValue::get(SVT));
-      } else {
-        assert(LogicalShift && "Logical shift expected");
-        ConstantVec.push_back(ConstantInt::getNullValue(SVT));
-      }
-    }
-    return ConstantVector::get(ConstantVec);
-  }
-
-  // We can't handle only some out of range values with generic logical shifts.
-  if (AnyOutOfRange)
-    return nullptr;
-
-  // Build the shift amount constant vector.
-  SmallVector<Constant *, 8> ShiftVecAmts;
-  for (int Idx : ShiftAmts) {
-    if (Idx < 0)
-      ShiftVecAmts.push_back(UndefValue::get(SVT));
-    else
-      ShiftVecAmts.push_back(ConstantInt::get(SVT, Idx));
-  }
-  auto ShiftVec = ConstantVector::get(ShiftVecAmts);
-
-  if (ShiftLeft)
-    return Builder.CreateShl(Vec, ShiftVec);
-
-  if (LogicalShift)
-    return Builder.CreateLShr(Vec, ShiftVec);
-
-  return Builder.CreateAShr(Vec, ShiftVec);
-}
-
-static Value *simplifyX86pack(IntrinsicInst &II,
-                              InstCombiner::BuilderTy &Builder, bool IsSigned) {
-  Value *Arg0 = II.getArgOperand(0);
-  Value *Arg1 = II.getArgOperand(1);
-  Type *ResTy = II.getType();
-
-  // Fast all undef handling.
-  if (isa<UndefValue>(Arg0) && isa<UndefValue>(Arg1))
-    return UndefValue::get(ResTy);
-
-  auto *ArgTy = cast<VectorType>(Arg0->getType());
-  unsigned NumLanes = ResTy->getPrimitiveSizeInBits() / 128;
-  unsigned NumSrcElts = ArgTy->getNumElements();
-  assert(cast<VectorType>(ResTy)->getNumElements() == (2 * NumSrcElts) &&
-         "Unexpected packing types");
-
-  unsigned NumSrcEltsPerLane = NumSrcElts / NumLanes;
-  unsigned DstScalarSizeInBits = ResTy->getScalarSizeInBits();
-  unsigned SrcScalarSizeInBits = ArgTy->getScalarSizeInBits();
-  assert(SrcScalarSizeInBits == (2 * DstScalarSizeInBits) &&
-         "Unexpected packing types");
-
-  // Constant folding.
-  if (!isa<Constant>(Arg0) || !isa<Constant>(Arg1))
-    return nullptr;
-
-  // Clamp Values - signed/unsigned both use signed clamp values, but they
-  // differ on the min/max values.
-  APInt MinValue, MaxValue;
-  if (IsSigned) {
-    // PACKSS: Truncate signed value with signed saturation.
-    // Source values less than dst minint are saturated to minint.
-    // Source values greater than dst maxint are saturated to maxint.
-    MinValue =
-        APInt::getSignedMinValue(DstScalarSizeInBits).sext(SrcScalarSizeInBits);
-    MaxValue =
-        APInt::getSignedMaxValue(DstScalarSizeInBits).sext(SrcScalarSizeInBits);
-  } else {
-    // PACKUS: Truncate signed value with unsigned saturation.
-    // Source values less than zero are saturated to zero.
-    // Source values greater than dst maxuint are saturated to maxuint.
-    MinValue = APInt::getNullValue(SrcScalarSizeInBits);
-    MaxValue = APInt::getLowBitsSet(SrcScalarSizeInBits, DstScalarSizeInBits);
-  }
-
-  auto *MinC = Constant::getIntegerValue(ArgTy, MinValue);
-  auto *MaxC = Constant::getIntegerValue(ArgTy, MaxValue);
-  Arg0 = Builder.CreateSelect(Builder.CreateICmpSLT(Arg0, MinC), MinC, Arg0);
-  Arg1 = Builder.CreateSelect(Builder.CreateICmpSLT(Arg1, MinC), MinC, Arg1);
-  Arg0 = Builder.CreateSelect(Builder.CreateICmpSGT(Arg0, MaxC), MaxC, Arg0);
-  Arg1 = Builder.CreateSelect(Builder.CreateICmpSGT(Arg1, MaxC), MaxC, Arg1);
-
-  // Shuffle clamped args together at the lane level.
-  SmallVector<int, 32> PackMask;
-  for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
-    for (unsigned Elt = 0; Elt != NumSrcEltsPerLane; ++Elt)
-      PackMask.push_back(Elt + (Lane * NumSrcEltsPerLane));
-    for (unsigned Elt = 0; Elt != NumSrcEltsPerLane; ++Elt)
-      PackMask.push_back(Elt + (Lane * NumSrcEltsPerLane) + NumSrcElts);
-  }
-  auto *Shuffle = Builder.CreateShuffleVector(Arg0, Arg1, PackMask);
-
-  // Truncate to dst size.
-  return Builder.CreateTrunc(Shuffle, ResTy);
-}
-
-static Value *simplifyX86movmsk(const IntrinsicInst &II,
-                                InstCombiner::BuilderTy &Builder) {
-  Value *Arg = II.getArgOperand(0);
-  Type *ResTy = II.getType();
-
-  // movmsk(undef) -> zero as we must ensure the upper bits are zero.
-  if (isa<UndefValue>(Arg))
-    return Constant::getNullValue(ResTy);
-
-  auto *ArgTy = dyn_cast<VectorType>(Arg->getType());
-  // We can't easily peek through x86_mmx types.
-  if (!ArgTy)
-    return nullptr;
-
-  // Expand MOVMSK to compare/bitcast/zext:
-  // e.g. PMOVMSKB(v16i8 x):
-  // %cmp = icmp slt <16 x i8> %x, zeroinitializer
-  // %int = bitcast <16 x i1> %cmp to i16
-  // %res = zext i16 %int to i32
-  unsigned NumElts = ArgTy->getNumElements();
-  Type *IntegerVecTy = VectorType::getInteger(ArgTy);
-  Type *IntegerTy = Builder.getIntNTy(NumElts);
-
-  Value *Res = Builder.CreateBitCast(Arg, IntegerVecTy);
-  Res = Builder.CreateICmpSLT(Res, Constant::getNullValue(IntegerVecTy));
-  Res = Builder.CreateBitCast(Res, IntegerTy);
-  Res = Builder.CreateZExtOrTrunc(Res, ResTy);
-  return Res;
-}
-
-static Value *simplifyX86addcarry(const IntrinsicInst &II,
-                                  InstCombiner::BuilderTy &Builder) {
-  Value *CarryIn = II.getArgOperand(0);
-  Value *Op1 = II.getArgOperand(1);
-  Value *Op2 = II.getArgOperand(2);
-  Type *RetTy = II.getType();
-  Type *OpTy = Op1->getType();
-  assert(RetTy->getStructElementType(0)->isIntegerTy(8) &&
-         RetTy->getStructElementType(1) == OpTy && OpTy == Op2->getType() &&
-         "Unexpected types for x86 addcarry");
-
-  // If carry-in is zero, this is just an unsigned add with overflow.
-  if (match(CarryIn, m_ZeroInt())) {
-    Value *UAdd = Builder.CreateIntrinsic(Intrinsic::uadd_with_overflow, OpTy,
-                                          { Op1, Op2 });
-    // The types have to be adjusted to match the x86 call types.
-    Value *UAddResult = Builder.CreateExtractValue(UAdd, 0);
-    Value *UAddOV = Builder.CreateZExt(Builder.CreateExtractValue(UAdd, 1),
-                                       Builder.getInt8Ty());
-    Value *Res = UndefValue::get(RetTy);
-    Res = Builder.CreateInsertValue(Res, UAddOV, 0);
-    return Builder.CreateInsertValue(Res, UAddResult, 1);
-  }
-
-  return nullptr;
-}
-
-static Value *simplifyX86insertps(const IntrinsicInst &II,
-                                  InstCombiner::BuilderTy &Builder) {
-  auto *CInt = dyn_cast<ConstantInt>(II.getArgOperand(2));
-  if (!CInt)
-    return nullptr;
-
-  VectorType *VecTy = cast<VectorType>(II.getType());
-  assert(VecTy->getNumElements() == 4 && "insertps with wrong vector type");
-
-  // The immediate permute control byte looks like this:
-  //    [3:0] - zero mask for each 32-bit lane
-  //    [5:4] - select one 32-bit destination lane
-  //    [7:6] - select one 32-bit source lane
-
-  uint8_t Imm = CInt->getZExtValue();
-  uint8_t ZMask = Imm & 0xf;
-  uint8_t DestLane = (Imm >> 4) & 0x3;
-  uint8_t SourceLane = (Imm >> 6) & 0x3;
-
-  ConstantAggregateZero *ZeroVector = ConstantAggregateZero::get(VecTy);
-
-  // If all zero mask bits are set, this was just a weird way to
-  // generate a zero vector.
-  if (ZMask == 0xf)
-    return ZeroVector;
-
-  // Initialize by passing all of the first source bits through.
-  int ShuffleMask[4] = {0, 1, 2, 3};
-
-  // We may replace the second operand with the zero vector.
-  Value *V1 = II.getArgOperand(1);
-
-  if (ZMask) {
-    // If the zero mask is being used with a single input or the zero mask
-    // overrides the destination lane, this is a shuffle with the zero vector.
-    if ((II.getArgOperand(0) == II.getArgOperand(1)) ||
-        (ZMask & (1 << DestLane))) {
-      V1 = ZeroVector;
-      // We may still move 32-bits of the first source vector from one lane
-      // to another.
-      ShuffleMask[DestLane] = SourceLane;
-      // The zero mask may override the previous insert operation.
-      for (unsigned i = 0; i < 4; ++i)
-        if ((ZMask >> i) & 0x1)
-          ShuffleMask[i] = i + 4;
-    } else {
-      // TODO: Model this case as 2 shuffles or a 'logical and' plus shuffle?
-      return nullptr;
-    }
-  } else {
-    // Replace the selected destination lane with the selected source lane.
-    ShuffleMask[DestLane] = SourceLane + 4;
-  }
-
-  return Builder.CreateShuffleVector(II.getArgOperand(0), V1, ShuffleMask);
-}
-
-/// Attempt to simplify SSE4A EXTRQ/EXTRQI instructions using constant folding
-/// or conversion to a shuffle vector.
-static Value *simplifyX86extrq(IntrinsicInst &II, Value *Op0,
-                               ConstantInt *CILength, ConstantInt *CIIndex,
-                               InstCombiner::BuilderTy &Builder) {
-  auto LowConstantHighUndef = [&](uint64_t Val) {
-    Type *IntTy64 = Type::getInt64Ty(II.getContext());
-    Constant *Args[] = {ConstantInt::get(IntTy64, Val),
-                        UndefValue::get(IntTy64)};
-    return ConstantVector::get(Args);
-  };
-
-  // See if we're dealing with constant values.
-  Constant *C0 = dyn_cast<Constant>(Op0);
-  ConstantInt *CI0 =
-      C0 ? dyn_cast_or_null<ConstantInt>(C0->getAggregateElement((unsigned)0))
-         : nullptr;
-
-  // Attempt to constant fold.
-  if (CILength && CIIndex) {
-    // From AMD documentation: "The bit index and field length are each six
-    // bits in length other bits of the field are ignored."
-    APInt APIndex = CIIndex->getValue().zextOrTrunc(6);
-    APInt APLength = CILength->getValue().zextOrTrunc(6);
-
-    unsigned Index = APIndex.getZExtValue();
-
-    // From AMD documentation: "a value of zero in the field length is
-    // defined as length of 64".
-    unsigned Length = APLength == 0 ? 64 : APLength.getZExtValue();
-
-    // From AMD documentation: "If the sum of the bit index + length field
-    // is greater than 64, the results are undefined".
-    unsigned End = Index + Length;
-
-    // Note that both field index and field length are 8-bit quantities.
-    // Since variables 'Index' and 'Length' are unsigned values
-    // obtained from zero-extending field index and field length
-    // respectively, their sum should never wrap around.
-    if (End > 64)
-      return UndefValue::get(II.getType());
-
-    // If we are inserting whole bytes, we can convert this to a shuffle.
-    // Lowering can recognize EXTRQI shuffle masks.
-    if ((Length % 8) == 0 && (Index % 8) == 0) {
-      // Convert bit indices to byte indices.
-      Length /= 8;
-      Index /= 8;
-
-      Type *IntTy8 = Type::getInt8Ty(II.getContext());
-      auto *ShufTy = FixedVectorType::get(IntTy8, 16);
-
-      SmallVector<int, 16> ShuffleMask;
-      for (int i = 0; i != (int)Length; ++i)
-        ShuffleMask.push_back(i + Index);
-      for (int i = Length; i != 8; ++i)
-        ShuffleMask.push_back(i + 16);
-      for (int i = 8; i != 16; ++i)
-        ShuffleMask.push_back(-1);
-
-      Value *SV = Builder.CreateShuffleVector(
-          Builder.CreateBitCast(Op0, ShufTy),
-          ConstantAggregateZero::get(ShufTy), ShuffleMask);
-      return Builder.CreateBitCast(SV, II.getType());
-    }
-
-    // Constant Fold - shift Index'th bit to lowest position and mask off
-    // Length bits.
-    if (CI0) {
-      APInt Elt = CI0->getValue();
-      Elt.lshrInPlace(Index);
-      Elt = Elt.zextOrTrunc(Length);
-      return LowConstantHighUndef(Elt.getZExtValue());
-    }
-
-    // If we were an EXTRQ call, we'll save registers if we convert to EXTRQI.
-    if (II.getIntrinsicID() == Intrinsic::x86_sse4a_extrq) {
-      Value *Args[] = {Op0, CILength, CIIndex};
-      Module *M = II.getModule();
-      Function *F = Intrinsic::getDeclaration(M, Intrinsic::x86_sse4a_extrqi);
-      return Builder.CreateCall(F, Args);
-    }
-  }
-
-  // Constant Fold - extraction from zero is always {zero, undef}.
-  if (CI0 && CI0->isZero())
-    return LowConstantHighUndef(0);
-
-  return nullptr;
-}
-
-/// Attempt to simplify SSE4A INSERTQ/INSERTQI instructions using constant
-/// folding or conversion to a shuffle vector.
-static Value *simplifyX86insertq(IntrinsicInst &II, Value *Op0, Value *Op1,
-                                 APInt APLength, APInt APIndex,
-                                 InstCombiner::BuilderTy &Builder) {
-  // From AMD documentation: "The bit index and field length are each six bits
-  // in length other bits of the field are ignored."
-  APIndex = APIndex.zextOrTrunc(6);
-  APLength = APLength.zextOrTrunc(6);
-
-  // Attempt to constant fold.
-  unsigned Index = APIndex.getZExtValue();
-
-  // From AMD documentation: "a value of zero in the field length is
-  // defined as length of 64".
-  unsigned Length = APLength == 0 ? 64 : APLength.getZExtValue();
-
-  // From AMD documentation: "If the sum of the bit index + length field
-  // is greater than 64, the results are undefined".
-  unsigned End = Index + Length;
-
-  // Note that both field index and field length are 8-bit quantities.
-  // Since variables 'Index' and 'Length' are unsigned values
-  // obtained from zero-extending field index and field length
-  // respectively, their sum should never wrap around.
-  if (End > 64)
-    return UndefValue::get(II.getType());
-
-  // If we are inserting whole bytes, we can convert this to a shuffle.
-  // Lowering can recognize INSERTQI shuffle masks.
-  if ((Length % 8) == 0 && (Index % 8) == 0) {
-    // Convert bit indices to byte indices.
-    Length /= 8;
-    Index /= 8;
-
-    Type *IntTy8 = Type::getInt8Ty(II.getContext());
-    auto *ShufTy = FixedVectorType::get(IntTy8, 16);
-
-    SmallVector<int, 16> ShuffleMask;
-    for (int i = 0; i != (int)Index; ++i)
-      ShuffleMask.push_back(i);
-    for (int i = 0; i != (int)Length; ++i)
-      ShuffleMask.push_back(i + 16);
-    for (int i = Index + Length; i != 8; ++i)
-      ShuffleMask.push_back(i);
-    for (int i = 8; i != 16; ++i)
-      ShuffleMask.push_back(-1);
-
-    Value *SV = Builder.CreateShuffleVector(Builder.CreateBitCast(Op0, ShufTy),
-                                            Builder.CreateBitCast(Op1, ShufTy),
-                                            ShuffleMask);
-    return Builder.CreateBitCast(SV, II.getType());
-  }
-
-  // See if we're dealing with constant values.
-  Constant *C0 = dyn_cast<Constant>(Op0);
-  Constant *C1 = dyn_cast<Constant>(Op1);
-  ConstantInt *CI00 =
-      C0 ? dyn_cast_or_null<ConstantInt>(C0->getAggregateElement((unsigned)0))
-         : nullptr;
-  ConstantInt *CI10 =
-      C1 ? dyn_cast_or_null<ConstantInt>(C1->getAggregateElement((unsigned)0))
-         : nullptr;
-
-  // Constant Fold - insert bottom Length bits starting at the Index'th bit.
-  if (CI00 && CI10) {
-    APInt V00 = CI00->getValue();
-    APInt V10 = CI10->getValue();
-    APInt Mask = APInt::getLowBitsSet(64, Length).shl(Index);
-    V00 = V00 & ~Mask;
-    V10 = V10.zextOrTrunc(Length).zextOrTrunc(64).shl(Index);
-    APInt Val = V00 | V10;
-    Type *IntTy64 = Type::getInt64Ty(II.getContext());
-    Constant *Args[] = {ConstantInt::get(IntTy64, Val.getZExtValue()),
-                        UndefValue::get(IntTy64)};
-    return ConstantVector::get(Args);
-  }
-
-  // If we were an INSERTQ call, we'll save demanded elements if we convert to
-  // INSERTQI.
-  if (II.getIntrinsicID() == Intrinsic::x86_sse4a_insertq) {
-    Type *IntTy8 = Type::getInt8Ty(II.getContext());
-    Constant *CILength = ConstantInt::get(IntTy8, Length, false);
-    Constant *CIIndex = ConstantInt::get(IntTy8, Index, false);
-
-    Value *Args[] = {Op0, Op1, CILength, CIIndex};
-    Module *M = II.getModule();
-    Function *F = Intrinsic::getDeclaration(M, Intrinsic::x86_sse4a_insertqi);
-    return Builder.CreateCall(F, Args);
-  }
-
-  return nullptr;
-}
-
-/// Attempt to convert pshufb* to shufflevector if the mask is constant.
-static Value *simplifyX86pshufb(const IntrinsicInst &II,
-                                InstCombiner::BuilderTy &Builder) {
-  Constant *V = dyn_cast<Constant>(II.getArgOperand(1));
-  if (!V)
-    return nullptr;
-
-  auto *VecTy = cast<VectorType>(II.getType());
-  unsigned NumElts = VecTy->getNumElements();
-  assert((NumElts == 16 || NumElts == 32 || NumElts == 64) &&
-         "Unexpected number of elements in shuffle mask!");
-
-  // Construct a shuffle mask from constant integers or UNDEFs.
-  int Indexes[64];
-
-  // Each byte in the shuffle control mask forms an index to permute the
-  // corresponding byte in the destination operand.
-  for (unsigned I = 0; I < NumElts; ++I) {
-    Constant *COp = V->getAggregateElement(I);
-    if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
-      return nullptr;
-
-    if (isa<UndefValue>(COp)) {
-      Indexes[I] = -1;
-      continue;
-    }
-
-    int8_t Index = cast<ConstantInt>(COp)->getValue().getZExtValue();
-
-    // If the most significant bit (bit[7]) of each byte of the shuffle
-    // control mask is set, then zero is written in the result byte.
-    // The zero vector is in the right-hand side of the resulting
-    // shufflevector.
-
-    // The value of each index for the high 128-bit lane is the least
-    // significant 4 bits of the respective shuffle control byte.
-    Index = ((Index < 0) ? NumElts : Index & 0x0F) + (I & 0xF0);
-    Indexes[I] = Index;
-  }
-
-  auto V1 = II.getArgOperand(0);
-  auto V2 = Constant::getNullValue(VecTy);
-  return Builder.CreateShuffleVector(V1, V2, makeArrayRef(Indexes, NumElts));
-}
-
-/// Attempt to convert vpermilvar* to shufflevector if the mask is constant.
-static Value *simplifyX86vpermilvar(const IntrinsicInst &II,
-                                    InstCombiner::BuilderTy &Builder) {
-  Constant *V = dyn_cast<Constant>(II.getArgOperand(1));
-  if (!V)
-    return nullptr;
-
-  auto *VecTy = cast<VectorType>(II.getType());
-  unsigned NumElts = VecTy->getNumElements();
-  bool IsPD = VecTy->getScalarType()->isDoubleTy();
-  unsigned NumLaneElts = IsPD ? 2 : 4;
-  assert(NumElts == 16 || NumElts == 8 || NumElts == 4 || NumElts == 2);
-
-  // Construct a shuffle mask from constant integers or UNDEFs.
-  int Indexes[16];
-
-  // The intrinsics only read one or two bits, clear the rest.
-  for (unsigned I = 0; I < NumElts; ++I) {
-    Constant *COp = V->getAggregateElement(I);
-    if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
-      return nullptr;
-
-    if (isa<UndefValue>(COp)) {
-      Indexes[I] = -1;
-      continue;
-    }
-
-    APInt Index = cast<ConstantInt>(COp)->getValue();
-    Index = Index.zextOrTrunc(32).getLoBits(2);
-
-    // The PD variants uses bit 1 to select per-lane element index, so
-    // shift down to convert to generic shuffle mask index.
-    if (IsPD)
-      Index.lshrInPlace(1);
-
-    // The _256 variants are a bit trickier since the mask bits always index
-    // into the corresponding 128 half. In order to convert to a generic
-    // shuffle, we have to make that explicit.
-    Index += APInt(32, (I / NumLaneElts) * NumLaneElts);
-
-    Indexes[I] = Index.getZExtValue();
-  }
-
-  auto V1 = II.getArgOperand(0);
-  auto V2 = UndefValue::get(V1->getType());
-  return Builder.CreateShuffleVector(V1, V2, makeArrayRef(Indexes, NumElts));
-}
-
-/// Attempt to convert vpermd/vpermps to shufflevector if the mask is constant.
-static Value *simplifyX86vpermv(const IntrinsicInst &II,
-                                InstCombiner::BuilderTy &Builder) {
-  auto *V = dyn_cast<Constant>(II.getArgOperand(1));
-  if (!V)
-    return nullptr;
-
-  auto *VecTy = cast<VectorType>(II.getType());
-  unsigned Size = VecTy->getNumElements();
-  assert((Size == 4 || Size == 8 || Size == 16 || Size == 32 || Size == 64) &&
-         "Unexpected shuffle mask size");
-
-  // Construct a shuffle mask from constant integers or UNDEFs.
-  int Indexes[64];
-
-  for (unsigned I = 0; I < Size; ++I) {
-    Constant *COp = V->getAggregateElement(I);
-    if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
-      return nullptr;
-
-    if (isa<UndefValue>(COp)) {
-      Indexes[I] = -1;
-      continue;
-    }
-
-    uint32_t Index = cast<ConstantInt>(COp)->getZExtValue();
-    Index &= Size - 1;
-    Indexes[I] = Index;
-  }
-
-  auto V1 = II.getArgOperand(0);
-  auto V2 = UndefValue::get(VecTy);
-  return Builder.CreateShuffleVector(V1, V2, makeArrayRef(Indexes, Size));
-}
-
 // TODO, Obvious Missing Transforms:
 // * Narrow width by halfs excluding zero/undef lanes
-Value *InstCombiner::simplifyMaskedLoad(IntrinsicInst &II) {
+Value *InstCombinerImpl::simplifyMaskedLoad(IntrinsicInst &II) {
   Value *LoadPtr = II.getArgOperand(0);
   const Align Alignment =
       cast<ConstantInt>(II.getArgOperand(1))->getAlignValue();
@@ -1132,7 +303,7 @@ Value *InstCombiner::simplifyMaskedLoad(IntrinsicInst &II) {
 // TODO, Obvious Missing Transforms:
 // * Single constant active lane -> store
 // * Narrow width by halfs excluding zero/undef lanes
-Instruction *InstCombiner::simplifyMaskedStore(IntrinsicInst &II) {
+Instruction *InstCombinerImpl::simplifyMaskedStore(IntrinsicInst &II) {
   auto *ConstMask = dyn_cast<Constant>(II.getArgOperand(3));
   if (!ConstMask)
     return nullptr;
@@ -1165,7 +336,7 @@ Instruction *InstCombiner::simplifyMaskedStore(IntrinsicInst &II) {
 // * Narrow width by halfs excluding zero/undef lanes
 // * Vector splat address w/known mask -> scalar load
 // * Vector incrementing address -> vector masked load
-Instruction *InstCombiner::simplifyMaskedGather(IntrinsicInst &II) {
+Instruction *InstCombinerImpl::simplifyMaskedGather(IntrinsicInst &II) {
   return nullptr;
 }
 
@@ -1175,7 +346,7 @@ Instruction *InstCombiner::simplifyMaskedGather(IntrinsicInst &II) {
 // * Narrow store width by halfs excluding zero/undef lanes
 // * Vector splat address w/known mask -> scalar store
 // * Vector incrementing address -> vector masked store
-Instruction *InstCombiner::simplifyMaskedScatter(IntrinsicInst &II) {
+Instruction *InstCombinerImpl::simplifyMaskedScatter(IntrinsicInst &II) {
   auto *ConstMask = dyn_cast<Constant>(II.getArgOperand(3));
   if (!ConstMask)
     return nullptr;
@@ -1206,7 +377,7 @@ Instruction *InstCombiner::simplifyMaskedScatter(IntrinsicInst &II) {
 /// This is legal because it preserves the most recent information about
 /// the presence or absence of invariant.group.
 static Instruction *simplifyInvariantGroupIntrinsic(IntrinsicInst &II,
-                                                    InstCombiner &IC) {
+                                                    InstCombinerImpl &IC) {
   auto *Arg = II.getArgOperand(0);
   auto *StrippedArg = Arg->stripPointerCasts();
   auto *StrippedInvariantGroupsArg = Arg->stripPointerCastsAndInvariantGroups();
@@ -1231,7 +402,7 @@ static Instruction *simplifyInvariantGroupIntrinsic(IntrinsicInst &II,
   return cast<Instruction>(Result);
 }
 
-static Instruction *foldCttzCtlz(IntrinsicInst &II, InstCombiner &IC) {
+static Instruction *foldCttzCtlz(IntrinsicInst &II, InstCombinerImpl &IC) {
   assert((II.getIntrinsicID() == Intrinsic::cttz ||
           II.getIntrinsicID() == Intrinsic::ctlz) &&
          "Expected cttz or ctlz intrinsic");
@@ -1301,7 +472,7 @@ static Instruction *foldCttzCtlz(IntrinsicInst &II, InstCombiner &IC) {
   return nullptr;
 }
 
-static Instruction *foldCtpop(IntrinsicInst &II, InstCombiner &IC) {
+static Instruction *foldCtpop(IntrinsicInst &II, InstCombinerImpl &IC) {
   assert(II.getIntrinsicID() == Intrinsic::ctpop &&
          "Expected ctpop intrinsic");
   Type *Ty = II.getType();
@@ -1356,107 +527,6 @@ static Instruction *foldCtpop(IntrinsicInst &II, InstCombiner &IC) {
   return nullptr;
 }
 
-// TODO: If the x86 backend knew how to convert a bool vector mask back to an
-// XMM register mask efficiently, we could transform all x86 masked intrinsics
-// to LLVM masked intrinsics and remove the x86 masked intrinsic defs.
-static Instruction *simplifyX86MaskedLoad(IntrinsicInst &II, InstCombiner &IC) {
-  Value *Ptr = II.getOperand(0);
-  Value *Mask = II.getOperand(1);
-  Constant *ZeroVec = Constant::getNullValue(II.getType());
-
-  // Special case a zero mask since that's not a ConstantDataVector.
-  // This masked load instruction creates a zero vector.
-  if (isa<ConstantAggregateZero>(Mask))
-    return IC.replaceInstUsesWith(II, ZeroVec);
-
-  auto *ConstMask = dyn_cast<ConstantDataVector>(Mask);
-  if (!ConstMask)
-    return nullptr;
-
-  // The mask is constant. Convert this x86 intrinsic to the LLVM instrinsic
-  // to allow target-independent optimizations.
-
-  // First, cast the x86 intrinsic scalar pointer to a vector pointer to match
-  // the LLVM intrinsic definition for the pointer argument.
-  unsigned AddrSpace = cast<PointerType>(Ptr->getType())->getAddressSpace();
-  PointerType *VecPtrTy = PointerType::get(II.getType(), AddrSpace);
-  Value *PtrCast = IC.Builder.CreateBitCast(Ptr, VecPtrTy, "castvec");
-
-  // Second, convert the x86 XMM integer vector mask to a vector of bools based
-  // on each element's most significant bit (the sign bit).
-  Constant *BoolMask = getNegativeIsTrueBoolVec(ConstMask);
-
-  // The pass-through vector for an x86 masked load is a zero vector.
-  CallInst *NewMaskedLoad =
-      IC.Builder.CreateMaskedLoad(PtrCast, Align(1), BoolMask, ZeroVec);
-  return IC.replaceInstUsesWith(II, NewMaskedLoad);
-}
-
-// TODO: If the x86 backend knew how to convert a bool vector mask back to an
-// XMM register mask efficiently, we could transform all x86 masked intrinsics
-// to LLVM masked intrinsics and remove the x86 masked intrinsic defs.
-static bool simplifyX86MaskedStore(IntrinsicInst &II, InstCombiner &IC) {
-  Value *Ptr = II.getOperand(0);
-  Value *Mask = II.getOperand(1);
-  Value *Vec = II.getOperand(2);
-
-  // Special case a zero mask since that's not a ConstantDataVector:
-  // this masked store instruction does nothing.
-  if (isa<ConstantAggregateZero>(Mask)) {
-    IC.eraseInstFromFunction(II);
-    return true;
-  }
-
-  // The SSE2 version is too weird (eg, unaligned but non-temporal) to do
-  // anything else at this level.
-  if (II.getIntrinsicID() == Intrinsic::x86_sse2_maskmov_dqu)
-    return false;
-
-  auto *ConstMask = dyn_cast<ConstantDataVector>(Mask);
-  if (!ConstMask)
-    return false;
-
-  // The mask is constant. Convert this x86 intrinsic to the LLVM instrinsic
-  // to allow target-independent optimizations.
-
-  // First, cast the x86 intrinsic scalar pointer to a vector pointer to match
-  // the LLVM intrinsic definition for the pointer argument.
-  unsigned AddrSpace = cast<PointerType>(Ptr->getType())->getAddressSpace();
-  PointerType *VecPtrTy = PointerType::get(Vec->getType(), AddrSpace);
-  Value *PtrCast = IC.Builder.CreateBitCast(Ptr, VecPtrTy, "castvec");
-
-  // Second, convert the x86 XMM integer vector mask to a vector of bools based
-  // on each element's most significant bit (the sign bit).
-  Constant *BoolMask = getNegativeIsTrueBoolVec(ConstMask);
-
-  IC.Builder.CreateMaskedStore(Vec, PtrCast, Align(1), BoolMask);
-
-  // 'Replace uses' doesn't work for stores. Erase the original masked store.
-  IC.eraseInstFromFunction(II);
-  return true;
-}
-
-// Constant fold llvm.amdgcn.fmed3 intrinsics for standard inputs.
-//
-// A single NaN input is folded to minnum, so we rely on that folding for
-// handling NaNs.
-static APFloat fmed3AMDGCN(const APFloat &Src0, const APFloat &Src1,
-                           const APFloat &Src2) {
-  APFloat Max3 = maxnum(maxnum(Src0, Src1), Src2);
-
-  APFloat::cmpResult Cmp0 = Max3.compare(Src0);
-  assert(Cmp0 != APFloat::cmpUnordered && "nans handled separately");
-  if (Cmp0 == APFloat::cmpEqual)
-    return maxnum(Src1, Src2);
-
-  APFloat::cmpResult Cmp1 = Max3.compare(Src1);
-  assert(Cmp1 != APFloat::cmpUnordered && "nans handled separately");
-  if (Cmp1 == APFloat::cmpEqual)
-    return maxnum(Src0, Src2);
-
-  return maxnum(Src0, Src1);
-}
-
 /// Convert a table lookup to shufflevector if the mask is constant.
 /// This could benefit tbl1 if the mask is { 7,6,5,4,3,2,1,0 }, in
 /// which case we could lower the shufflevector with rev64 instructions
@@ -1495,28 +565,6 @@ static Value *simplifyNeonTbl1(const IntrinsicInst &II,
   return Builder.CreateShuffleVector(V1, V2, makeArrayRef(Indexes));
 }
 
-/// Convert a vector load intrinsic into a simple llvm load instruction.
-/// This is beneficial when the underlying object being addressed comes
-/// from a constant, since we get constant-folding for free.
-static Value *simplifyNeonVld1(const IntrinsicInst &II,
-                               unsigned MemAlign,
-                               InstCombiner::BuilderTy &Builder) {
-  auto *IntrAlign = dyn_cast<ConstantInt>(II.getArgOperand(1));
-
-  if (!IntrAlign)
-    return nullptr;
-
-  unsigned Alignment = IntrAlign->getLimitedValue() < MemAlign ?
-                       MemAlign : IntrAlign->getLimitedValue();
-
-  if (!isPowerOf2_32(Alignment))
-    return nullptr;
-
-  auto *BCastInst = Builder.CreateBitCast(II.getArgOperand(0),
-                                          PointerType::get(II.getType(), 0));
-  return Builder.CreateAlignedLoad(II.getType(), BCastInst, Align(Alignment));
-}
-
 // Returns true iff the 2 intrinsics have the same operands, limiting the
 // comparison to the first NumOperands.
 static bool haveSameOperands(const IntrinsicInst &I, const IntrinsicInst &E,
@@ -1538,9 +586,9 @@ static bool haveSameOperands(const IntrinsicInst &I, const IntrinsicInst &E,
 //   call @llvm.foo.start(i1 0) ; This one won't be skipped: it will be removed
 //   call @llvm.foo.end(i1 0)
 //   call @llvm.foo.end(i1 0) ; &I
-static bool removeTriviallyEmptyRange(
-    IntrinsicInst &EndI, InstCombiner &IC,
-    std::function<bool(const IntrinsicInst &)> IsStart) {
+static bool
+removeTriviallyEmptyRange(IntrinsicInst &EndI, InstCombinerImpl &IC,
+                          std::function<bool(const IntrinsicInst &)> IsStart) {
   // We start from the end intrinsic and scan backwards, so that InstCombine
   // has already processed (and potentially removed) all the instructions
   // before the end intrinsic.
@@ -1566,256 +614,7 @@ static bool removeTriviallyEmptyRange(
   return false;
 }
 
-// Convert NVVM intrinsics to target-generic LLVM code where possible.
-static Instruction *SimplifyNVVMIntrinsic(IntrinsicInst *II, InstCombiner &IC) {
-  // Each NVVM intrinsic we can simplify can be replaced with one of:
-  //
-  //  * an LLVM intrinsic,
-  //  * an LLVM cast operation,
-  //  * an LLVM binary operation, or
-  //  * ad-hoc LLVM IR for the particular operation.
-
-  // Some transformations are only valid when the module's
-  // flush-denormals-to-zero (ftz) setting is true/false, whereas other
-  // transformations are valid regardless of the module's ftz setting.
-  enum FtzRequirementTy {
-    FTZ_Any,       // Any ftz setting is ok.
-    FTZ_MustBeOn,  // Transformation is valid only if ftz is on.
-    FTZ_MustBeOff, // Transformation is valid only if ftz is off.
-  };
-  // Classes of NVVM intrinsics that can't be replaced one-to-one with a
-  // target-generic intrinsic, cast op, or binary op but that we can nonetheless
-  // simplify.
-  enum SpecialCase {
-    SPC_Reciprocal,
-  };
-
-  // SimplifyAction is a poor-man's variant (plus an additional flag) that
-  // represents how to replace an NVVM intrinsic with target-generic LLVM IR.
-  struct SimplifyAction {
-    // Invariant: At most one of these Optionals has a value.
-    Optional<Intrinsic::ID> IID;
-    Optional<Instruction::CastOps> CastOp;
-    Optional<Instruction::BinaryOps> BinaryOp;
-    Optional<SpecialCase> Special;
-
-    FtzRequirementTy FtzRequirement = FTZ_Any;
-
-    SimplifyAction() = default;
-
-    SimplifyAction(Intrinsic::ID IID, FtzRequirementTy FtzReq)
-        : IID(IID), FtzRequirement(FtzReq) {}
-
-    // Cast operations don't have anything to do with FTZ, so we skip that
-    // argument.
-    SimplifyAction(Instruction::CastOps CastOp) : CastOp(CastOp) {}
-
-    SimplifyAction(Instruction::BinaryOps BinaryOp, FtzRequirementTy FtzReq)
-        : BinaryOp(BinaryOp), FtzRequirement(FtzReq) {}
-
-    SimplifyAction(SpecialCase Special, FtzRequirementTy FtzReq)
-        : Special(Special), FtzRequirement(FtzReq) {}
-  };
-
-  // Try to generate a SimplifyAction describing how to replace our
-  // IntrinsicInstr with target-generic LLVM IR.
-  const SimplifyAction Action = [II]() -> SimplifyAction {
-    switch (II->getIntrinsicID()) {
-    // NVVM intrinsics that map directly to LLVM intrinsics.
-    case Intrinsic::nvvm_ceil_d:
-      return {Intrinsic::ceil, FTZ_Any};
-    case Intrinsic::nvvm_ceil_f:
-      return {Intrinsic::ceil, FTZ_MustBeOff};
-    case Intrinsic::nvvm_ceil_ftz_f:
-      return {Intrinsic::ceil, FTZ_MustBeOn};
-    case Intrinsic::nvvm_fabs_d:
-      return {Intrinsic::fabs, FTZ_Any};
-    case Intrinsic::nvvm_fabs_f:
-      return {Intrinsic::fabs, FTZ_MustBeOff};
-    case Intrinsic::nvvm_fabs_ftz_f:
-      return {Intrinsic::fabs, FTZ_MustBeOn};
-    case Intrinsic::nvvm_floor_d:
-      return {Intrinsic::floor, FTZ_Any};
-    case Intrinsic::nvvm_floor_f:
-      return {Intrinsic::floor, FTZ_MustBeOff};
-    case Intrinsic::nvvm_floor_ftz_f:
-      return {Intrinsic::floor, FTZ_MustBeOn};
-    case Intrinsic::nvvm_fma_rn_d:
-      return {Intrinsic::fma, FTZ_Any};
-    case Intrinsic::nvvm_fma_rn_f:
-      return {Intrinsic::fma, FTZ_MustBeOff};
-    case Intrinsic::nvvm_fma_rn_ftz_f:
-      return {Intrinsic::fma, FTZ_MustBeOn};
-    case Intrinsic::nvvm_fmax_d:
-      return {Intrinsic::maxnum, FTZ_Any};
-    case Intrinsic::nvvm_fmax_f:
-      return {Intrinsic::maxnum, FTZ_MustBeOff};
-    case Intrinsic::nvvm_fmax_ftz_f:
-      return {Intrinsic::maxnum, FTZ_MustBeOn};
-    case Intrinsic::nvvm_fmin_d:
-      return {Intrinsic::minnum, FTZ_Any};
-    case Intrinsic::nvvm_fmin_f:
-      return {Intrinsic::minnum, FTZ_MustBeOff};
-    case Intrinsic::nvvm_fmin_ftz_f:
-      return {Intrinsic::minnum, FTZ_MustBeOn};
-    case Intrinsic::nvvm_round_d:
-      return {Intrinsic::round, FTZ_Any};
-    case Intrinsic::nvvm_round_f:
-      return {Intrinsic::round, FTZ_MustBeOff};
-    case Intrinsic::nvvm_round_ftz_f:
-      return {Intrinsic::round, FTZ_MustBeOn};
-    case Intrinsic::nvvm_sqrt_rn_d:
-      return {Intrinsic::sqrt, FTZ_Any};
-    case Intrinsic::nvvm_sqrt_f:
-      // nvvm_sqrt_f is a special case.  For  most intrinsics, foo_ftz_f is the
-      // ftz version, and foo_f is the non-ftz version.  But nvvm_sqrt_f adopts
-      // the ftz-ness of the surrounding code.  sqrt_rn_f and sqrt_rn_ftz_f are
-      // the versions with explicit ftz-ness.
-      return {Intrinsic::sqrt, FTZ_Any};
-    case Intrinsic::nvvm_sqrt_rn_f:
-      return {Intrinsic::sqrt, FTZ_MustBeOff};
-    case Intrinsic::nvvm_sqrt_rn_ftz_f:
-      return {Intrinsic::sqrt, FTZ_MustBeOn};
-    case Intrinsic::nvvm_trunc_d:
-      return {Intrinsic::trunc, FTZ_Any};
-    case Intrinsic::nvvm_trunc_f:
-      return {Intrinsic::trunc, FTZ_MustBeOff};
-    case Intrinsic::nvvm_trunc_ftz_f:
-      return {Intrinsic::trunc, FTZ_MustBeOn};
-
-    // NVVM intrinsics that map to LLVM cast operations.
-    //
-    // Note that llvm's target-generic conversion operators correspond to the rz
-    // (round to zero) versions of the nvvm conversion intrinsics, even though
-    // most everything else here uses the rn (round to nearest even) nvvm ops.
-    case Intrinsic::nvvm_d2i_rz:
-    case Intrinsic::nvvm_f2i_rz:
-    case Intrinsic::nvvm_d2ll_rz:
-    case Intrinsic::nvvm_f2ll_rz:
-      return {Instruction::FPToSI};
-    case Intrinsic::nvvm_d2ui_rz:
-    case Intrinsic::nvvm_f2ui_rz:
-    case Intrinsic::nvvm_d2ull_rz:
-    case Intrinsic::nvvm_f2ull_rz:
-      return {Instruction::FPToUI};
-    case Intrinsic::nvvm_i2d_rz:
-    case Intrinsic::nvvm_i2f_rz:
-    case Intrinsic::nvvm_ll2d_rz:
-    case Intrinsic::nvvm_ll2f_rz:
-      return {Instruction::SIToFP};
-    case Intrinsic::nvvm_ui2d_rz:
-    case Intrinsic::nvvm_ui2f_rz:
-    case Intrinsic::nvvm_ull2d_rz:
-    case Intrinsic::nvvm_ull2f_rz:
-      return {Instruction::UIToFP};
-
-    // NVVM intrinsics that map to LLVM binary ops.
-    case Intrinsic::nvvm_add_rn_d:
-      return {Instruction::FAdd, FTZ_Any};
-    case Intrinsic::nvvm_add_rn_f:
-      return {Instruction::FAdd, FTZ_MustBeOff};
-    case Intrinsic::nvvm_add_rn_ftz_f:
-      return {Instruction::FAdd, FTZ_MustBeOn};
-    case Intrinsic::nvvm_mul_rn_d:
-      return {Instruction::FMul, FTZ_Any};
-    case Intrinsic::nvvm_mul_rn_f:
-      return {Instruction::FMul, FTZ_MustBeOff};
-    case Intrinsic::nvvm_mul_rn_ftz_f:
-      return {Instruction::FMul, FTZ_MustBeOn};
-    case Intrinsic::nvvm_div_rn_d:
-      return {Instruction::FDiv, FTZ_Any};
-    case Intrinsic::nvvm_div_rn_f:
-      return {Instruction::FDiv, FTZ_MustBeOff};
-    case Intrinsic::nvvm_div_rn_ftz_f:
-      return {Instruction::FDiv, FTZ_MustBeOn};
-
-    // The remainder of cases are NVVM intrinsics that map to LLVM idioms, but
-    // need special handling.
-    //
-    // We seem to be missing intrinsics for rcp.approx.{ftz.}f32, which is just
-    // as well.
-    case Intrinsic::nvvm_rcp_rn_d:
-      return {SPC_Reciprocal, FTZ_Any};
-    case Intrinsic::nvvm_rcp_rn_f:
-      return {SPC_Reciprocal, FTZ_MustBeOff};
-    case Intrinsic::nvvm_rcp_rn_ftz_f:
-      return {SPC_Reciprocal, FTZ_MustBeOn};
-
-    // We do not currently simplify intrinsics that give an approximate answer.
-    // These include:
-    //
-    //   - nvvm_cos_approx_{f,ftz_f}
-    //   - nvvm_ex2_approx_{d,f,ftz_f}
-    //   - nvvm_lg2_approx_{d,f,ftz_f}
-    //   - nvvm_sin_approx_{f,ftz_f}
-    //   - nvvm_sqrt_approx_{f,ftz_f}
-    //   - nvvm_rsqrt_approx_{d,f,ftz_f}
-    //   - nvvm_div_approx_{ftz_d,ftz_f,f}
-    //   - nvvm_rcp_approx_ftz_d
-    //
-    // Ideally we'd encode them as e.g. "fast call @llvm.cos", where "fast"
-    // means that fastmath is enabled in the intrinsic.  Unfortunately only
-    // binary operators (currently) have a fastmath bit in SelectionDAG, so this
-    // information gets lost and we can't select on it.
-    //
-    // TODO: div and rcp are lowered to a binary op, so these we could in theory
-    // lower them to "fast fdiv".
-
-    default:
-      return {};
-    }
-  }();
-
-  // If Action.FtzRequirementTy is not satisfied by the module's ftz state, we
-  // can bail out now.  (Notice that in the case that IID is not an NVVM
-  // intrinsic, we don't have to look up any module metadata, as
-  // FtzRequirementTy will be FTZ_Any.)
-  if (Action.FtzRequirement != FTZ_Any) {
-    StringRef Attr = II->getFunction()
-                         ->getFnAttribute("denormal-fp-math-f32")
-                         .getValueAsString();
-    DenormalMode Mode = parseDenormalFPAttribute(Attr);
-    bool FtzEnabled = Mode.Output != DenormalMode::IEEE;
-
-    if (FtzEnabled != (Action.FtzRequirement == FTZ_MustBeOn))
-      return nullptr;
-  }
-
-  // Simplify to target-generic intrinsic.
-  if (Action.IID) {
-    SmallVector<Value *, 4> Args(II->arg_operands());
-    // All the target-generic intrinsics currently of interest to us have one
-    // type argument, equal to that of the nvvm intrinsic's argument.
-    Type *Tys[] = {II->getArgOperand(0)->getType()};
-    return CallInst::Create(
-        Intrinsic::getDeclaration(II->getModule(), *Action.IID, Tys), Args);
-  }
-
-  // Simplify to target-generic binary op.
-  if (Action.BinaryOp)
-    return BinaryOperator::Create(*Action.BinaryOp, II->getArgOperand(0),
-                                  II->getArgOperand(1), II->getName());
-
-  // Simplify to target-generic cast op.
-  if (Action.CastOp)
-    return CastInst::Create(*Action.CastOp, II->getArgOperand(0), II->getType(),
-                            II->getName());
-
-  // All that's left are the special cases.
-  if (!Action.Special)
-    return nullptr;
-
-  switch (*Action.Special) {
-  case SPC_Reciprocal:
-    // Simplify reciprocal.
-    return BinaryOperator::Create(
-        Instruction::FDiv, ConstantFP::get(II->getArgOperand(0)->getType(), 1),
-        II->getArgOperand(0), II->getName());
-  }
-  llvm_unreachable("All SpecialCase enumerators should be handled in switch.");
-}
-
-Instruction *InstCombiner::visitVAEndInst(VAEndInst &I) {
+Instruction *InstCombinerImpl::visitVAEndInst(VAEndInst &I) {
   removeTriviallyEmptyRange(I, *this, [](const IntrinsicInst &I) {
     return I.getIntrinsicID() == Intrinsic::vastart ||
            I.getIntrinsicID() == Intrinsic::vacopy;
@@ -1834,20 +633,31 @@ static Instruction *canonicalizeConstantArg0ToArg1(CallInst &Call) {
   return nullptr;
 }
 
-Instruction *InstCombiner::foldIntrinsicWithOverflowCommon(IntrinsicInst *II) {
+/// Creates a result tuple for an overflow intrinsic \p II with a given
+/// \p Result and a constant \p Overflow value.
+static Instruction *createOverflowTuple(IntrinsicInst *II, Value *Result,
+                                        Constant *Overflow) {
+  Constant *V[] = {UndefValue::get(Result->getType()), Overflow};
+  StructType *ST = cast<StructType>(II->getType());
+  Constant *Struct = ConstantStruct::get(ST, V);
+  return InsertValueInst::Create(Struct, Result, 0);
+}
+
+Instruction *
+InstCombinerImpl::foldIntrinsicWithOverflowCommon(IntrinsicInst *II) {
   WithOverflowInst *WO = cast<WithOverflowInst>(II);
   Value *OperationResult = nullptr;
   Constant *OverflowResult = nullptr;
   if (OptimizeOverflowCheck(WO->getBinaryOp(), WO->isSigned(), WO->getLHS(),
                             WO->getRHS(), *WO, OperationResult, OverflowResult))
-    return CreateOverflowTuple(WO, OperationResult, OverflowResult);
+    return createOverflowTuple(WO, OperationResult, OverflowResult);
   return nullptr;
 }
 
 /// CallInst simplification. This mostly only handles folding of intrinsic
 /// instructions. For normal calls, it allows visitCallBase to do the heavy
 /// lifting.
-Instruction *InstCombiner::visitCallInst(CallInst &CI) {
+Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
   // Don't try to simplify calls without uses. It will not do anything useful,
   // but will result in the following folds being skipped.
   if (!CI.use_empty())
@@ -1953,19 +763,8 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
     }
   }
 
-  if (Instruction *I = SimplifyNVVMIntrinsic(II, *this))
-    return I;
-
-  auto SimplifyDemandedVectorEltsLow = [this](Value *Op, unsigned Width,
-                                              unsigned DemandedWidth) {
-    APInt UndefElts(Width, 0);
-    APInt DemandedElts = APInt::getLowBitsSet(Width, DemandedWidth);
-    return SimplifyDemandedVectorElts(Op, DemandedElts, UndefElts);
-  };
-
   Intrinsic::ID IID = II->getIntrinsicID();
   switch (IID) {
-  default: break;
   case Intrinsic::objectsize:
     if (Value *V = lowerObjectSizeCall(II, DL, &TLI, /*MustSucceed=*/false))
       return replaceInstUsesWith(CI, V);
@@ -2465,932 +1264,6 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
     }
     break;
   }
-  case Intrinsic::ppc_altivec_lvx:
-  case Intrinsic::ppc_altivec_lvxl:
-    // Turn PPC lvx -> load if the pointer is known aligned.
-    if (getOrEnforceKnownAlignment(II->getArgOperand(0), Align(16), DL, II, &AC,
-                                   &DT) >= 16) {
-      Value *Ptr = Builder.CreateBitCast(II->getArgOperand(0),
-                                         PointerType::getUnqual(II->getType()));
-      return new LoadInst(II->getType(), Ptr, "", false, Align(16));
-    }
-    break;
-  case Intrinsic::ppc_vsx_lxvw4x:
-  case Intrinsic::ppc_vsx_lxvd2x: {
-    // Turn PPC VSX loads into normal loads.
-    Value *Ptr = Builder.CreateBitCast(II->getArgOperand(0),
-                                       PointerType::getUnqual(II->getType()));
-    return new LoadInst(II->getType(), Ptr, Twine(""), false, Align(1));
-  }
-  case Intrinsic::ppc_altivec_stvx:
-  case Intrinsic::ppc_altivec_stvxl:
-    // Turn stvx -> store if the pointer is known aligned.
-    if (getOrEnforceKnownAlignment(II->getArgOperand(1), Align(16), DL, II, &AC,
-                                   &DT) >= 16) {
-      Type *OpPtrTy =
-        PointerType::getUnqual(II->getArgOperand(0)->getType());
-      Value *Ptr = Builder.CreateBitCast(II->getArgOperand(1), OpPtrTy);
-      return new StoreInst(II->getArgOperand(0), Ptr, false, Align(16));
-    }
-    break;
-  case Intrinsic::ppc_vsx_stxvw4x:
-  case Intrinsic::ppc_vsx_stxvd2x: {
-    // Turn PPC VSX stores into normal stores.
-    Type *OpPtrTy = PointerType::getUnqual(II->getArgOperand(0)->getType());
-    Value *Ptr = Builder.CreateBitCast(II->getArgOperand(1), OpPtrTy);
-    return new StoreInst(II->getArgOperand(0), Ptr, false, Align(1));
-  }
-  case Intrinsic::ppc_qpx_qvlfs:
-    // Turn PPC QPX qvlfs -> load if the pointer is known aligned.
-    if (getOrEnforceKnownAlignment(II->getArgOperand(0), Align(16), DL, II, &AC,
-                                   &DT) >= 16) {
-      Type *VTy =
-          VectorType::get(Builder.getFloatTy(),
-                          cast<VectorType>(II->getType())->getElementCount());
-      Value *Ptr = Builder.CreateBitCast(II->getArgOperand(0),
-                                         PointerType::getUnqual(VTy));
-      Value *Load = Builder.CreateLoad(VTy, Ptr);
-      return new FPExtInst(Load, II->getType());
-    }
-    break;
-  case Intrinsic::ppc_qpx_qvlfd:
-    // Turn PPC QPX qvlfd -> load if the pointer is known aligned.
-    if (getOrEnforceKnownAlignment(II->getArgOperand(0), Align(32), DL, II, &AC,
-                                   &DT) >= 32) {
-      Value *Ptr = Builder.CreateBitCast(II->getArgOperand(0),
-                                         PointerType::getUnqual(II->getType()));
-      return new LoadInst(II->getType(), Ptr, "", false, Align(32));
-    }
-    break;
-  case Intrinsic::ppc_qpx_qvstfs:
-    // Turn PPC QPX qvstfs -> store if the pointer is known aligned.
-    if (getOrEnforceKnownAlignment(II->getArgOperand(1), Align(16), DL, II, &AC,
-                                   &DT) >= 16) {
-      Type *VTy = VectorType::get(
-          Builder.getFloatTy(),
-          cast<VectorType>(II->getArgOperand(0)->getType())->getElementCount());
-      Value *TOp = Builder.CreateFPTrunc(II->getArgOperand(0), VTy);
-      Type *OpPtrTy = PointerType::getUnqual(VTy);
-      Value *Ptr = Builder.CreateBitCast(II->getArgOperand(1), OpPtrTy);
-      return new StoreInst(TOp, Ptr, false, Align(16));
-    }
-    break;
-  case Intrinsic::ppc_qpx_qvstfd:
-    // Turn PPC QPX qvstfd -> store if the pointer is known aligned.
-    if (getOrEnforceKnownAlignment(II->getArgOperand(1), Align(32), DL, II, &AC,
-                                   &DT) >= 32) {
-      Type *OpPtrTy =
-        PointerType::getUnqual(II->getArgOperand(0)->getType());
-      Value *Ptr = Builder.CreateBitCast(II->getArgOperand(1), OpPtrTy);
-      return new StoreInst(II->getArgOperand(0), Ptr, false, Align(32));
-    }
-    break;
-
-  case Intrinsic::x86_bmi_bextr_32:
-  case Intrinsic::x86_bmi_bextr_64:
-  case Intrinsic::x86_tbm_bextri_u32:
-  case Intrinsic::x86_tbm_bextri_u64:
-    // If the RHS is a constant we can try some simplifications.
-    if (auto *C = dyn_cast<ConstantInt>(II->getArgOperand(1))) {
-      uint64_t Shift = C->getZExtValue();
-      uint64_t Length = (Shift >> 8) & 0xff;
-      Shift &= 0xff;
-      unsigned BitWidth = II->getType()->getIntegerBitWidth();
-      // If the length is 0 or the shift is out of range, replace with zero.
-      if (Length == 0 || Shift >= BitWidth)
-        return replaceInstUsesWith(CI, ConstantInt::get(II->getType(), 0));
-      // If the LHS is also a constant, we can completely constant fold this.
-      if (auto *InC = dyn_cast<ConstantInt>(II->getArgOperand(0))) {
-        uint64_t Result = InC->getZExtValue() >> Shift;
-        if (Length > BitWidth)
-          Length = BitWidth;
-        Result &= maskTrailingOnes<uint64_t>(Length);
-        return replaceInstUsesWith(CI, ConstantInt::get(II->getType(), Result));
-      }
-      // TODO should we turn this into 'and' if shift is 0? Or 'shl' if we
-      // are only masking bits that a shift already cleared?
-    }
-    break;
-
-  case Intrinsic::x86_bmi_bzhi_32:
-  case Intrinsic::x86_bmi_bzhi_64:
-    // If the RHS is a constant we can try some simplifications.
-    if (auto *C = dyn_cast<ConstantInt>(II->getArgOperand(1))) {
-      uint64_t Index = C->getZExtValue() & 0xff;
-      unsigned BitWidth = II->getType()->getIntegerBitWidth();
-      if (Index >= BitWidth)
-        return replaceInstUsesWith(CI, II->getArgOperand(0));
-      if (Index == 0)
-        return replaceInstUsesWith(CI, ConstantInt::get(II->getType(), 0));
-      // If the LHS is also a constant, we can completely constant fold this.
-      if (auto *InC = dyn_cast<ConstantInt>(II->getArgOperand(0))) {
-        uint64_t Result = InC->getZExtValue();
-        Result &= maskTrailingOnes<uint64_t>(Index);
-        return replaceInstUsesWith(CI, ConstantInt::get(II->getType(), Result));
-      }
-      // TODO should we convert this to an AND if the RHS is constant?
-    }
-    break;
-  case Intrinsic::x86_bmi_pext_32:
-  case Intrinsic::x86_bmi_pext_64:
-    if (auto *MaskC = dyn_cast<ConstantInt>(II->getArgOperand(1))) {
-      if (MaskC->isNullValue())
-        return replaceInstUsesWith(CI, ConstantInt::get(II->getType(), 0));
-      if (MaskC->isAllOnesValue())
-        return replaceInstUsesWith(CI, II->getArgOperand(0));
-
-      if (auto *SrcC = dyn_cast<ConstantInt>(II->getArgOperand(0))) {
-        uint64_t Src = SrcC->getZExtValue();
-        uint64_t Mask = MaskC->getZExtValue();
-        uint64_t Result = 0;
-        uint64_t BitToSet = 1;
-
-        while (Mask) {
-          // Isolate lowest set bit.
-          uint64_t BitToTest = Mask & -Mask;
-          if (BitToTest & Src)
-            Result |= BitToSet;
-
-          BitToSet <<= 1;
-          // Clear lowest set bit.
-          Mask &= Mask - 1;
-        }
-
-        return replaceInstUsesWith(CI, ConstantInt::get(II->getType(), Result));
-      }
-    }
-    break;
-  case Intrinsic::x86_bmi_pdep_32:
-  case Intrinsic::x86_bmi_pdep_64:
-    if (auto *MaskC = dyn_cast<ConstantInt>(II->getArgOperand(1))) {
-      if (MaskC->isNullValue())
-        return replaceInstUsesWith(CI, ConstantInt::get(II->getType(), 0));
-      if (MaskC->isAllOnesValue())
-        return replaceInstUsesWith(CI, II->getArgOperand(0));
-
-      if (auto *SrcC = dyn_cast<ConstantInt>(II->getArgOperand(0))) {
-        uint64_t Src = SrcC->getZExtValue();
-        uint64_t Mask = MaskC->getZExtValue();
-        uint64_t Result = 0;
-        uint64_t BitToTest = 1;
-
-        while (Mask) {
-          // Isolate lowest set bit.
-          uint64_t BitToSet = Mask & -Mask;
-          if (BitToTest & Src)
-            Result |= BitToSet;
-
-          BitToTest <<= 1;
-          // Clear lowest set bit;
-          Mask &= Mask - 1;
-        }
-
-        return replaceInstUsesWith(CI, ConstantInt::get(II->getType(), Result));
-      }
-    }
-    break;
-
-  case Intrinsic::x86_sse_cvtss2si:
-  case Intrinsic::x86_sse_cvtss2si64:
-  case Intrinsic::x86_sse_cvttss2si:
-  case Intrinsic::x86_sse_cvttss2si64:
-  case Intrinsic::x86_sse2_cvtsd2si:
-  case Intrinsic::x86_sse2_cvtsd2si64:
-  case Intrinsic::x86_sse2_cvttsd2si:
-  case Intrinsic::x86_sse2_cvttsd2si64:
-  case Intrinsic::x86_avx512_vcvtss2si32:
-  case Intrinsic::x86_avx512_vcvtss2si64:
-  case Intrinsic::x86_avx512_vcvtss2usi32:
-  case Intrinsic::x86_avx512_vcvtss2usi64:
-  case Intrinsic::x86_avx512_vcvtsd2si32:
-  case Intrinsic::x86_avx512_vcvtsd2si64:
-  case Intrinsic::x86_avx512_vcvtsd2usi32:
-  case Intrinsic::x86_avx512_vcvtsd2usi64:
-  case Intrinsic::x86_avx512_cvttss2si:
-  case Intrinsic::x86_avx512_cvttss2si64:
-  case Intrinsic::x86_avx512_cvttss2usi:
-  case Intrinsic::x86_avx512_cvttss2usi64:
-  case Intrinsic::x86_avx512_cvttsd2si:
-  case Intrinsic::x86_avx512_cvttsd2si64:
-  case Intrinsic::x86_avx512_cvttsd2usi:
-  case Intrinsic::x86_avx512_cvttsd2usi64: {
-    // These intrinsics only demand the 0th element of their input vectors. If
-    // we can simplify the input based on that, do so now.
-    Value *Arg = II->getArgOperand(0);
-    unsigned VWidth = cast<VectorType>(Arg->getType())->getNumElements();
-    if (Value *V = SimplifyDemandedVectorEltsLow(Arg, VWidth, 1))
-      return replaceOperand(*II, 0, V);
-    break;
-  }
-
-  case Intrinsic::x86_mmx_pmovmskb:
-  case Intrinsic::x86_sse_movmsk_ps:
-  case Intrinsic::x86_sse2_movmsk_pd:
-  case Intrinsic::x86_sse2_pmovmskb_128:
-  case Intrinsic::x86_avx_movmsk_pd_256:
-  case Intrinsic::x86_avx_movmsk_ps_256:
-  case Intrinsic::x86_avx2_pmovmskb:
-    if (Value *V = simplifyX86movmsk(*II, Builder))
-      return replaceInstUsesWith(*II, V);
-    break;
-
-  case Intrinsic::x86_sse_comieq_ss:
-  case Intrinsic::x86_sse_comige_ss:
-  case Intrinsic::x86_sse_comigt_ss:
-  case Intrinsic::x86_sse_comile_ss:
-  case Intrinsic::x86_sse_comilt_ss:
-  case Intrinsic::x86_sse_comineq_ss:
-  case Intrinsic::x86_sse_ucomieq_ss:
-  case Intrinsic::x86_sse_ucomige_ss:
-  case Intrinsic::x86_sse_ucomigt_ss:
-  case Intrinsic::x86_sse_ucomile_ss:
-  case Intrinsic::x86_sse_ucomilt_ss:
-  case Intrinsic::x86_sse_ucomineq_ss:
-  case Intrinsic::x86_sse2_comieq_sd:
-  case Intrinsic::x86_sse2_comige_sd:
-  case Intrinsic::x86_sse2_comigt_sd:
-  case Intrinsic::x86_sse2_comile_sd:
-  case Intrinsic::x86_sse2_comilt_sd:
-  case Intrinsic::x86_sse2_comineq_sd:
-  case Intrinsic::x86_sse2_ucomieq_sd:
-  case Intrinsic::x86_sse2_ucomige_sd:
-  case Intrinsic::x86_sse2_ucomigt_sd:
-  case Intrinsic::x86_sse2_ucomile_sd:
-  case Intrinsic::x86_sse2_ucomilt_sd:
-  case Intrinsic::x86_sse2_ucomineq_sd:
-  case Intrinsic::x86_avx512_vcomi_ss:
-  case Intrinsic::x86_avx512_vcomi_sd:
-  case Intrinsic::x86_avx512_mask_cmp_ss:
-  case Intrinsic::x86_avx512_mask_cmp_sd: {
-    // These intrinsics only demand the 0th element of their input vectors. If
-    // we can simplify the input based on that, do so now.
-    bool MadeChange = false;
-    Value *Arg0 = II->getArgOperand(0);
-    Value *Arg1 = II->getArgOperand(1);
-    unsigned VWidth = cast<VectorType>(Arg0->getType())->getNumElements();
-    if (Value *V = SimplifyDemandedVectorEltsLow(Arg0, VWidth, 1)) {
-      replaceOperand(*II, 0, V);
-      MadeChange = true;
-    }
-    if (Value *V = SimplifyDemandedVectorEltsLow(Arg1, VWidth, 1)) {
-      replaceOperand(*II, 1, V);
-      MadeChange = true;
-    }
-    if (MadeChange)
-      return II;
-    break;
-  }
-  case Intrinsic::x86_avx512_cmp_pd_128:
-  case Intrinsic::x86_avx512_cmp_pd_256:
-  case Intrinsic::x86_avx512_cmp_pd_512:
-  case Intrinsic::x86_avx512_cmp_ps_128:
-  case Intrinsic::x86_avx512_cmp_ps_256:
-  case Intrinsic::x86_avx512_cmp_ps_512: {
-    // Folding cmp(sub(a,b),0) -> cmp(a,b) and cmp(0,sub(a,b)) -> cmp(b,a)
-    Value *Arg0 = II->getArgOperand(0);
-    Value *Arg1 = II->getArgOperand(1);
-    bool Arg0IsZero = match(Arg0, m_PosZeroFP());
-    if (Arg0IsZero)
-      std::swap(Arg0, Arg1);
-    Value *A, *B;
-    // This fold requires only the NINF(not +/- inf) since inf minus
-    // inf is nan.
-    // NSZ(No Signed Zeros) is not needed because zeros of any sign are
-    // equal for both compares.
-    // NNAN is not needed because nans compare the same for both compares.
-    // The compare intrinsic uses the above assumptions and therefore
-    // doesn't require additional flags.
-    if ((match(Arg0, m_OneUse(m_FSub(m_Value(A), m_Value(B)))) &&
-         match(Arg1, m_PosZeroFP()) && isa<Instruction>(Arg0) &&
-         cast<Instruction>(Arg0)->getFastMathFlags().noInfs())) {
-      if (Arg0IsZero)
-        std::swap(A, B);
-      replaceOperand(*II, 0, A);
-      replaceOperand(*II, 1, B);
-      return II;
-    }
-    break;
-  }
-
-  case Intrinsic::x86_avx512_add_ps_512:
-  case Intrinsic::x86_avx512_div_ps_512:
-  case Intrinsic::x86_avx512_mul_ps_512:
-  case Intrinsic::x86_avx512_sub_ps_512:
-  case Intrinsic::x86_avx512_add_pd_512:
-  case Intrinsic::x86_avx512_div_pd_512:
-  case Intrinsic::x86_avx512_mul_pd_512:
-  case Intrinsic::x86_avx512_sub_pd_512:
-    // If the rounding mode is CUR_DIRECTION(4) we can turn these into regular
-    // IR operations.
-    if (auto *R = dyn_cast<ConstantInt>(II->getArgOperand(2))) {
-      if (R->getValue() == 4) {
-        Value *Arg0 = II->getArgOperand(0);
-        Value *Arg1 = II->getArgOperand(1);
-
-        Value *V;
-        switch (IID) {
-        default: llvm_unreachable("Case stmts out of sync!");
-        case Intrinsic::x86_avx512_add_ps_512:
-        case Intrinsic::x86_avx512_add_pd_512:
-          V = Builder.CreateFAdd(Arg0, Arg1);
-          break;
-        case Intrinsic::x86_avx512_sub_ps_512:
-        case Intrinsic::x86_avx512_sub_pd_512:
-          V = Builder.CreateFSub(Arg0, Arg1);
-          break;
-        case Intrinsic::x86_avx512_mul_ps_512:
-        case Intrinsic::x86_avx512_mul_pd_512:
-          V = Builder.CreateFMul(Arg0, Arg1);
-          break;
-        case Intrinsic::x86_avx512_div_ps_512:
-        case Intrinsic::x86_avx512_div_pd_512:
-          V = Builder.CreateFDiv(Arg0, Arg1);
-          break;
-        }
-
-        return replaceInstUsesWith(*II, V);
-      }
-    }
-    break;
-
-  case Intrinsic::x86_avx512_mask_add_ss_round:
-  case Intrinsic::x86_avx512_mask_div_ss_round:
-  case Intrinsic::x86_avx512_mask_mul_ss_round:
-  case Intrinsic::x86_avx512_mask_sub_ss_round:
-  case Intrinsic::x86_avx512_mask_add_sd_round:
-  case Intrinsic::x86_avx512_mask_div_sd_round:
-  case Intrinsic::x86_avx512_mask_mul_sd_round:
-  case Intrinsic::x86_avx512_mask_sub_sd_round:
-    // If the rounding mode is CUR_DIRECTION(4) we can turn these into regular
-    // IR operations.
-    if (auto *R = dyn_cast<ConstantInt>(II->getArgOperand(4))) {
-      if (R->getValue() == 4) {
-        // Extract the element as scalars.
-        Value *Arg0 = II->getArgOperand(0);
-        Value *Arg1 = II->getArgOperand(1);
-        Value *LHS = Builder.CreateExtractElement(Arg0, (uint64_t)0);
-        Value *RHS = Builder.CreateExtractElement(Arg1, (uint64_t)0);
-
-        Value *V;
-        switch (IID) {
-        default: llvm_unreachable("Case stmts out of sync!");
-        case Intrinsic::x86_avx512_mask_add_ss_round:
-        case Intrinsic::x86_avx512_mask_add_sd_round:
-          V = Builder.CreateFAdd(LHS, RHS);
-          break;
-        case Intrinsic::x86_avx512_mask_sub_ss_round:
-        case Intrinsic::x86_avx512_mask_sub_sd_round:
-          V = Builder.CreateFSub(LHS, RHS);
-          break;
-        case Intrinsic::x86_avx512_mask_mul_ss_round:
-        case Intrinsic::x86_avx512_mask_mul_sd_round:
-          V = Builder.CreateFMul(LHS, RHS);
-          break;
-        case Intrinsic::x86_avx512_mask_div_ss_round:
-        case Intrinsic::x86_avx512_mask_div_sd_round:
-          V = Builder.CreateFDiv(LHS, RHS);
-          break;
-        }
-
-        // Handle the masking aspect of the intrinsic.
-        Value *Mask = II->getArgOperand(3);
-        auto *C = dyn_cast<ConstantInt>(Mask);
-        // We don't need a select if we know the mask bit is a 1.
-        if (!C || !C->getValue()[0]) {
-          // Cast the mask to an i1 vector and then extract the lowest element.
-          auto *MaskTy = FixedVectorType::get(
-              Builder.getInt1Ty(),
-              cast<IntegerType>(Mask->getType())->getBitWidth());
-          Mask = Builder.CreateBitCast(Mask, MaskTy);
-          Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
-          // Extract the lowest element from the passthru operand.
-          Value *Passthru = Builder.CreateExtractElement(II->getArgOperand(2),
-                                                          (uint64_t)0);
-          V = Builder.CreateSelect(Mask, V, Passthru);
-        }
-
-        // Insert the result back into the original argument 0.
-        V = Builder.CreateInsertElement(Arg0, V, (uint64_t)0);
-
-        return replaceInstUsesWith(*II, V);
-      }
-    }
-    break;
-
-  // Constant fold ashr( <A x Bi>, Ci ).
-  // Constant fold lshr( <A x Bi>, Ci ).
-  // Constant fold shl( <A x Bi>, Ci ).
-  case Intrinsic::x86_sse2_psrai_d:
-  case Intrinsic::x86_sse2_psrai_w:
-  case Intrinsic::x86_avx2_psrai_d:
-  case Intrinsic::x86_avx2_psrai_w:
-  case Intrinsic::x86_avx512_psrai_q_128:
-  case Intrinsic::x86_avx512_psrai_q_256:
-  case Intrinsic::x86_avx512_psrai_d_512:
-  case Intrinsic::x86_avx512_psrai_q_512:
-  case Intrinsic::x86_avx512_psrai_w_512:
-  case Intrinsic::x86_sse2_psrli_d:
-  case Intrinsic::x86_sse2_psrli_q:
-  case Intrinsic::x86_sse2_psrli_w:
-  case Intrinsic::x86_avx2_psrli_d:
-  case Intrinsic::x86_avx2_psrli_q:
-  case Intrinsic::x86_avx2_psrli_w:
-  case Intrinsic::x86_avx512_psrli_d_512:
-  case Intrinsic::x86_avx512_psrli_q_512:
-  case Intrinsic::x86_avx512_psrli_w_512:
-  case Intrinsic::x86_sse2_pslli_d:
-  case Intrinsic::x86_sse2_pslli_q:
-  case Intrinsic::x86_sse2_pslli_w:
-  case Intrinsic::x86_avx2_pslli_d:
-  case Intrinsic::x86_avx2_pslli_q:
-  case Intrinsic::x86_avx2_pslli_w:
-  case Intrinsic::x86_avx512_pslli_d_512:
-  case Intrinsic::x86_avx512_pslli_q_512:
-  case Intrinsic::x86_avx512_pslli_w_512:
-    if (Value *V = simplifyX86immShift(*II, Builder))
-      return replaceInstUsesWith(*II, V);
-    break;
-
-  case Intrinsic::x86_sse2_psra_d:
-  case Intrinsic::x86_sse2_psra_w:
-  case Intrinsic::x86_avx2_psra_d:
-  case Intrinsic::x86_avx2_psra_w:
-  case Intrinsic::x86_avx512_psra_q_128:
-  case Intrinsic::x86_avx512_psra_q_256:
-  case Intrinsic::x86_avx512_psra_d_512:
-  case Intrinsic::x86_avx512_psra_q_512:
-  case Intrinsic::x86_avx512_psra_w_512:
-  case Intrinsic::x86_sse2_psrl_d:
-  case Intrinsic::x86_sse2_psrl_q:
-  case Intrinsic::x86_sse2_psrl_w:
-  case Intrinsic::x86_avx2_psrl_d:
-  case Intrinsic::x86_avx2_psrl_q:
-  case Intrinsic::x86_avx2_psrl_w:
-  case Intrinsic::x86_avx512_psrl_d_512:
-  case Intrinsic::x86_avx512_psrl_q_512:
-  case Intrinsic::x86_avx512_psrl_w_512:
-  case Intrinsic::x86_sse2_psll_d:
-  case Intrinsic::x86_sse2_psll_q:
-  case Intrinsic::x86_sse2_psll_w:
-  case Intrinsic::x86_avx2_psll_d:
-  case Intrinsic::x86_avx2_psll_q:
-  case Intrinsic::x86_avx2_psll_w:
-  case Intrinsic::x86_avx512_psll_d_512:
-  case Intrinsic::x86_avx512_psll_q_512:
-  case Intrinsic::x86_avx512_psll_w_512: {
-    if (Value *V = simplifyX86immShift(*II, Builder))
-      return replaceInstUsesWith(*II, V);
-
-    // SSE2/AVX2 uses only the first 64-bits of the 128-bit vector
-    // operand to compute the shift amount.
-    Value *Arg1 = II->getArgOperand(1);
-    assert(Arg1->getType()->getPrimitiveSizeInBits() == 128 &&
-           "Unexpected packed shift size");
-    unsigned VWidth = cast<VectorType>(Arg1->getType())->getNumElements();
-
-    if (Value *V = SimplifyDemandedVectorEltsLow(Arg1, VWidth, VWidth / 2))
-      return replaceOperand(*II, 1, V);
-    break;
-  }
-
-  case Intrinsic::x86_avx2_psllv_d:
-  case Intrinsic::x86_avx2_psllv_d_256:
-  case Intrinsic::x86_avx2_psllv_q:
-  case Intrinsic::x86_avx2_psllv_q_256:
-  case Intrinsic::x86_avx512_psllv_d_512:
-  case Intrinsic::x86_avx512_psllv_q_512:
-  case Intrinsic::x86_avx512_psllv_w_128:
-  case Intrinsic::x86_avx512_psllv_w_256:
-  case Intrinsic::x86_avx512_psllv_w_512:
-  case Intrinsic::x86_avx2_psrav_d:
-  case Intrinsic::x86_avx2_psrav_d_256:
-  case Intrinsic::x86_avx512_psrav_q_128:
-  case Intrinsic::x86_avx512_psrav_q_256:
-  case Intrinsic::x86_avx512_psrav_d_512:
-  case Intrinsic::x86_avx512_psrav_q_512:
-  case Intrinsic::x86_avx512_psrav_w_128:
-  case Intrinsic::x86_avx512_psrav_w_256:
-  case Intrinsic::x86_avx512_psrav_w_512:
-  case Intrinsic::x86_avx2_psrlv_d:
-  case Intrinsic::x86_avx2_psrlv_d_256:
-  case Intrinsic::x86_avx2_psrlv_q:
-  case Intrinsic::x86_avx2_psrlv_q_256:
-  case Intrinsic::x86_avx512_psrlv_d_512:
-  case Intrinsic::x86_avx512_psrlv_q_512:
-  case Intrinsic::x86_avx512_psrlv_w_128:
-  case Intrinsic::x86_avx512_psrlv_w_256:
-  case Intrinsic::x86_avx512_psrlv_w_512:
-    if (Value *V = simplifyX86varShift(*II, Builder))
-      return replaceInstUsesWith(*II, V);
-    break;
-
-  case Intrinsic::x86_sse2_packssdw_128:
-  case Intrinsic::x86_sse2_packsswb_128:
-  case Intrinsic::x86_avx2_packssdw:
-  case Intrinsic::x86_avx2_packsswb:
-  case Intrinsic::x86_avx512_packssdw_512:
-  case Intrinsic::x86_avx512_packsswb_512:
-    if (Value *V = simplifyX86pack(*II, Builder, true))
-      return replaceInstUsesWith(*II, V);
-    break;
-
-  case Intrinsic::x86_sse2_packuswb_128:
-  case Intrinsic::x86_sse41_packusdw:
-  case Intrinsic::x86_avx2_packusdw:
-  case Intrinsic::x86_avx2_packuswb:
-  case Intrinsic::x86_avx512_packusdw_512:
-  case Intrinsic::x86_avx512_packuswb_512:
-    if (Value *V = simplifyX86pack(*II, Builder, false))
-      return replaceInstUsesWith(*II, V);
-    break;
-
-  case Intrinsic::x86_pclmulqdq:
-  case Intrinsic::x86_pclmulqdq_256:
-  case Intrinsic::x86_pclmulqdq_512: {
-    if (auto *C = dyn_cast<ConstantInt>(II->getArgOperand(2))) {
-      unsigned Imm = C->getZExtValue();
-
-      bool MadeChange = false;
-      Value *Arg0 = II->getArgOperand(0);
-      Value *Arg1 = II->getArgOperand(1);
-      unsigned VWidth = cast<VectorType>(Arg0->getType())->getNumElements();
-
-      APInt UndefElts1(VWidth, 0);
-      APInt DemandedElts1 = APInt::getSplat(VWidth,
-                                            APInt(2, (Imm & 0x01) ? 2 : 1));
-      if (Value *V = SimplifyDemandedVectorElts(Arg0, DemandedElts1,
-                                                UndefElts1)) {
-        replaceOperand(*II, 0, V);
-        MadeChange = true;
-      }
-
-      APInt UndefElts2(VWidth, 0);
-      APInt DemandedElts2 = APInt::getSplat(VWidth,
-                                            APInt(2, (Imm & 0x10) ? 2 : 1));
-      if (Value *V = SimplifyDemandedVectorElts(Arg1, DemandedElts2,
-                                                UndefElts2)) {
-        replaceOperand(*II, 1, V);
-        MadeChange = true;
-      }
-
-      // If either input elements are undef, the result is zero.
-      if (DemandedElts1.isSubsetOf(UndefElts1) ||
-          DemandedElts2.isSubsetOf(UndefElts2))
-        return replaceInstUsesWith(*II,
-                                   ConstantAggregateZero::get(II->getType()));
-
-      if (MadeChange)
-        return II;
-    }
-    break;
-  }
-
-  case Intrinsic::x86_sse41_insertps:
-    if (Value *V = simplifyX86insertps(*II, Builder))
-      return replaceInstUsesWith(*II, V);
-    break;
-
-  case Intrinsic::x86_sse4a_extrq: {
-    Value *Op0 = II->getArgOperand(0);
-    Value *Op1 = II->getArgOperand(1);
-    unsigned VWidth0 = cast<VectorType>(Op0->getType())->getNumElements();
-    unsigned VWidth1 = cast<VectorType>(Op1->getType())->getNumElements();
-    assert(Op0->getType()->getPrimitiveSizeInBits() == 128 &&
-           Op1->getType()->getPrimitiveSizeInBits() == 128 && VWidth0 == 2 &&
-           VWidth1 == 16 && "Unexpected operand sizes");
-
-    // See if we're dealing with constant values.
-    Constant *C1 = dyn_cast<Constant>(Op1);
-    ConstantInt *CILength =
-        C1 ? dyn_cast_or_null<ConstantInt>(C1->getAggregateElement((unsigned)0))
-           : nullptr;
-    ConstantInt *CIIndex =
-        C1 ? dyn_cast_or_null<ConstantInt>(C1->getAggregateElement((unsigned)1))
-           : nullptr;
-
-    // Attempt to simplify to a constant, shuffle vector or EXTRQI call.
-    if (Value *V = simplifyX86extrq(*II, Op0, CILength, CIIndex, Builder))
-      return replaceInstUsesWith(*II, V);
-
-    // EXTRQ only uses the lowest 64-bits of the first 128-bit vector
-    // operands and the lowest 16-bits of the second.
-    bool MadeChange = false;
-    if (Value *V = SimplifyDemandedVectorEltsLow(Op0, VWidth0, 1)) {
-      replaceOperand(*II, 0, V);
-      MadeChange = true;
-    }
-    if (Value *V = SimplifyDemandedVectorEltsLow(Op1, VWidth1, 2)) {
-      replaceOperand(*II, 1, V);
-      MadeChange = true;
-    }
-    if (MadeChange)
-      return II;
-    break;
-  }
-
-  case Intrinsic::x86_sse4a_extrqi: {
-    // EXTRQI: Extract Length bits starting from Index. Zero pad the remaining
-    // bits of the lower 64-bits. The upper 64-bits are undefined.
-    Value *Op0 = II->getArgOperand(0);
-    unsigned VWidth = cast<VectorType>(Op0->getType())->getNumElements();
-    assert(Op0->getType()->getPrimitiveSizeInBits() == 128 && VWidth == 2 &&
-           "Unexpected operand size");
-
-    // See if we're dealing with constant values.
-    ConstantInt *CILength = dyn_cast<ConstantInt>(II->getArgOperand(1));
-    ConstantInt *CIIndex = dyn_cast<ConstantInt>(II->getArgOperand(2));
-
-    // Attempt to simplify to a constant or shuffle vector.
-    if (Value *V = simplifyX86extrq(*II, Op0, CILength, CIIndex, Builder))
-      return replaceInstUsesWith(*II, V);
-
-    // EXTRQI only uses the lowest 64-bits of the first 128-bit vector
-    // operand.
-    if (Value *V = SimplifyDemandedVectorEltsLow(Op0, VWidth, 1))
-      return replaceOperand(*II, 0, V);
-    break;
-  }
-
-  case Intrinsic::x86_sse4a_insertq: {
-    Value *Op0 = II->getArgOperand(0);
-    Value *Op1 = II->getArgOperand(1);
-    unsigned VWidth = cast<VectorType>(Op0->getType())->getNumElements();
-    assert(Op0->getType()->getPrimitiveSizeInBits() == 128 &&
-           Op1->getType()->getPrimitiveSizeInBits() == 128 && VWidth == 2 &&
-           cast<VectorType>(Op1->getType())->getNumElements() == 2 &&
-           "Unexpected operand size");
-
-    // See if we're dealing with constant values.
-    Constant *C1 = dyn_cast<Constant>(Op1);
-    ConstantInt *CI11 =
-        C1 ? dyn_cast_or_null<ConstantInt>(C1->getAggregateElement((unsigned)1))
-           : nullptr;
-
-    // Attempt to simplify to a constant, shuffle vector or INSERTQI call.
-    if (CI11) {
-      const APInt &V11 = CI11->getValue();
-      APInt Len = V11.zextOrTrunc(6);
-      APInt Idx = V11.lshr(8).zextOrTrunc(6);
-      if (Value *V = simplifyX86insertq(*II, Op0, Op1, Len, Idx, Builder))
-        return replaceInstUsesWith(*II, V);
-    }
-
-    // INSERTQ only uses the lowest 64-bits of the first 128-bit vector
-    // operand.
-    if (Value *V = SimplifyDemandedVectorEltsLow(Op0, VWidth, 1))
-      return replaceOperand(*II, 0, V);
-    break;
-  }
-
-  case Intrinsic::x86_sse4a_insertqi: {
-    // INSERTQI: Extract lowest Length bits from lower half of second source and
-    // insert over first source starting at Index bit. The upper 64-bits are
-    // undefined.
-    Value *Op0 = II->getArgOperand(0);
-    Value *Op1 = II->getArgOperand(1);
-    unsigned VWidth0 = cast<VectorType>(Op0->getType())->getNumElements();
-    unsigned VWidth1 = cast<VectorType>(Op1->getType())->getNumElements();
-    assert(Op0->getType()->getPrimitiveSizeInBits() == 128 &&
-           Op1->getType()->getPrimitiveSizeInBits() == 128 && VWidth0 == 2 &&
-           VWidth1 == 2 && "Unexpected operand sizes");
-
-    // See if we're dealing with constant values.
-    ConstantInt *CILength = dyn_cast<ConstantInt>(II->getArgOperand(2));
-    ConstantInt *CIIndex = dyn_cast<ConstantInt>(II->getArgOperand(3));
-
-    // Attempt to simplify to a constant or shuffle vector.
-    if (CILength && CIIndex) {
-      APInt Len = CILength->getValue().zextOrTrunc(6);
-      APInt Idx = CIIndex->getValue().zextOrTrunc(6);
-      if (Value *V = simplifyX86insertq(*II, Op0, Op1, Len, Idx, Builder))
-        return replaceInstUsesWith(*II, V);
-    }
-
-    // INSERTQI only uses the lowest 64-bits of the first two 128-bit vector
-    // operands.
-    bool MadeChange = false;
-    if (Value *V = SimplifyDemandedVectorEltsLow(Op0, VWidth0, 1)) {
-      replaceOperand(*II, 0, V);
-      MadeChange = true;
-    }
-    if (Value *V = SimplifyDemandedVectorEltsLow(Op1, VWidth1, 1)) {
-      replaceOperand(*II, 1, V);
-      MadeChange = true;
-    }
-    if (MadeChange)
-      return II;
-    break;
-  }
-
-  case Intrinsic::x86_sse41_pblendvb:
-  case Intrinsic::x86_sse41_blendvps:
-  case Intrinsic::x86_sse41_blendvpd:
-  case Intrinsic::x86_avx_blendv_ps_256:
-  case Intrinsic::x86_avx_blendv_pd_256:
-  case Intrinsic::x86_avx2_pblendvb: {
-    // fold (blend A, A, Mask) -> A
-    Value *Op0 = II->getArgOperand(0);
-    Value *Op1 = II->getArgOperand(1);
-    Value *Mask = II->getArgOperand(2);
-    if (Op0 == Op1)
-      return replaceInstUsesWith(CI, Op0);
-
-    // Zero Mask - select 1st argument.
-    if (isa<ConstantAggregateZero>(Mask))
-      return replaceInstUsesWith(CI, Op0);
-
-    // Constant Mask - select 1st/2nd argument lane based on top bit of mask.
-    if (auto *ConstantMask = dyn_cast<ConstantDataVector>(Mask)) {
-      Constant *NewSelector = getNegativeIsTrueBoolVec(ConstantMask);
-      return SelectInst::Create(NewSelector, Op1, Op0, "blendv");
-    }
-
-    // Convert to a vector select if we can bypass casts and find a boolean
-    // vector condition value.
-    Value *BoolVec;
-    Mask = peekThroughBitcast(Mask);
-    if (match(Mask, m_SExt(m_Value(BoolVec))) &&
-        BoolVec->getType()->isVectorTy() &&
-        BoolVec->getType()->getScalarSizeInBits() == 1) {
-      assert(Mask->getType()->getPrimitiveSizeInBits() ==
-             II->getType()->getPrimitiveSizeInBits() &&
-             "Not expecting mask and operands with different sizes");
-
-      unsigned NumMaskElts =
-          cast<VectorType>(Mask->getType())->getNumElements();
-      unsigned NumOperandElts =
-          cast<VectorType>(II->getType())->getNumElements();
-      if (NumMaskElts == NumOperandElts)
-        return SelectInst::Create(BoolVec, Op1, Op0);
-
-      // If the mask has less elements than the operands, each mask bit maps to
-      // multiple elements of the operands. Bitcast back and forth.
-      if (NumMaskElts < NumOperandElts) {
-        Value *CastOp0 = Builder.CreateBitCast(Op0, Mask->getType());
-        Value *CastOp1 = Builder.CreateBitCast(Op1, Mask->getType());
-        Value *Sel = Builder.CreateSelect(BoolVec, CastOp1, CastOp0);
-        return new BitCastInst(Sel, II->getType());
-      }
-    }
-
-    break;
-  }
-
-  case Intrinsic::x86_ssse3_pshuf_b_128:
-  case Intrinsic::x86_avx2_pshuf_b:
-  case Intrinsic::x86_avx512_pshuf_b_512:
-    if (Value *V = simplifyX86pshufb(*II, Builder))
-      return replaceInstUsesWith(*II, V);
-    break;
-
-  case Intrinsic::x86_avx_vpermilvar_ps:
-  case Intrinsic::x86_avx_vpermilvar_ps_256:
-  case Intrinsic::x86_avx512_vpermilvar_ps_512:
-  case Intrinsic::x86_avx_vpermilvar_pd:
-  case Intrinsic::x86_avx_vpermilvar_pd_256:
-  case Intrinsic::x86_avx512_vpermilvar_pd_512:
-    if (Value *V = simplifyX86vpermilvar(*II, Builder))
-      return replaceInstUsesWith(*II, V);
-    break;
-
-  case Intrinsic::x86_avx2_permd:
-  case Intrinsic::x86_avx2_permps:
-  case Intrinsic::x86_avx512_permvar_df_256:
-  case Intrinsic::x86_avx512_permvar_df_512:
-  case Intrinsic::x86_avx512_permvar_di_256:
-  case Intrinsic::x86_avx512_permvar_di_512:
-  case Intrinsic::x86_avx512_permvar_hi_128:
-  case Intrinsic::x86_avx512_permvar_hi_256:
-  case Intrinsic::x86_avx512_permvar_hi_512:
-  case Intrinsic::x86_avx512_permvar_qi_128:
-  case Intrinsic::x86_avx512_permvar_qi_256:
-  case Intrinsic::x86_avx512_permvar_qi_512:
-  case Intrinsic::x86_avx512_permvar_sf_512:
-  case Intrinsic::x86_avx512_permvar_si_512:
-    if (Value *V = simplifyX86vpermv(*II, Builder))
-      return replaceInstUsesWith(*II, V);
-    break;
-
-  case Intrinsic::x86_avx_maskload_ps:
-  case Intrinsic::x86_avx_maskload_pd:
-  case Intrinsic::x86_avx_maskload_ps_256:
-  case Intrinsic::x86_avx_maskload_pd_256:
-  case Intrinsic::x86_avx2_maskload_d:
-  case Intrinsic::x86_avx2_maskload_q:
-  case Intrinsic::x86_avx2_maskload_d_256:
-  case Intrinsic::x86_avx2_maskload_q_256:
-    if (Instruction *I = simplifyX86MaskedLoad(*II, *this))
-      return I;
-    break;
-
-  case Intrinsic::x86_sse2_maskmov_dqu:
-  case Intrinsic::x86_avx_maskstore_ps:
-  case Intrinsic::x86_avx_maskstore_pd:
-  case Intrinsic::x86_avx_maskstore_ps_256:
-  case Intrinsic::x86_avx_maskstore_pd_256:
-  case Intrinsic::x86_avx2_maskstore_d:
-  case Intrinsic::x86_avx2_maskstore_q:
-  case Intrinsic::x86_avx2_maskstore_d_256:
-  case Intrinsic::x86_avx2_maskstore_q_256:
-    if (simplifyX86MaskedStore(*II, *this))
-      return nullptr;
-    break;
-
-  case Intrinsic::x86_addcarry_32:
-  case Intrinsic::x86_addcarry_64:
-    if (Value *V = simplifyX86addcarry(*II, Builder))
-      return replaceInstUsesWith(*II, V);
-    break;
-
-  case Intrinsic::ppc_altivec_vperm:
-    // Turn vperm(V1,V2,mask) -> shuffle(V1,V2,mask) if mask is a constant.
-    // Note that ppc_altivec_vperm has a big-endian bias, so when creating
-    // a vectorshuffle for little endian, we must undo the transformation
-    // performed on vec_perm in altivec.h.  That is, we must complement
-    // the permutation mask with respect to 31 and reverse the order of
-    // V1 and V2.
-    if (Constant *Mask = dyn_cast<Constant>(II->getArgOperand(2))) {
-      assert(cast<VectorType>(Mask->getType())->getNumElements() == 16 &&
-             "Bad type for intrinsic!");
-
-      // Check that all of the elements are integer constants or undefs.
-      bool AllEltsOk = true;
-      for (unsigned i = 0; i != 16; ++i) {
-        Constant *Elt = Mask->getAggregateElement(i);
-        if (!Elt || !(isa<ConstantInt>(Elt) || isa<UndefValue>(Elt))) {
-          AllEltsOk = false;
-          break;
-        }
-      }
-
-      if (AllEltsOk) {
-        // Cast the input vectors to byte vectors.
-        Value *Op0 = Builder.CreateBitCast(II->getArgOperand(0),
-                                           Mask->getType());
-        Value *Op1 = Builder.CreateBitCast(II->getArgOperand(1),
-                                           Mask->getType());
-        Value *Result = UndefValue::get(Op0->getType());
-
-        // Only extract each element once.
-        Value *ExtractedElts[32];
-        memset(ExtractedElts, 0, sizeof(ExtractedElts));
-
-        for (unsigned i = 0; i != 16; ++i) {
-          if (isa<UndefValue>(Mask->getAggregateElement(i)))
-            continue;
-          unsigned Idx =
-            cast<ConstantInt>(Mask->getAggregateElement(i))->getZExtValue();
-          Idx &= 31;  // Match the hardware behavior.
-          if (DL.isLittleEndian())
-            Idx = 31 - Idx;
-
-          if (!ExtractedElts[Idx]) {
-            Value *Op0ToUse = (DL.isLittleEndian()) ? Op1 : Op0;
-            Value *Op1ToUse = (DL.isLittleEndian()) ? Op0 : Op1;
-            ExtractedElts[Idx] =
-              Builder.CreateExtractElement(Idx < 16 ? Op0ToUse : Op1ToUse,
-                                           Builder.getInt32(Idx&15));
-          }
-
-          // Insert this value into the result vector.
-          Result = Builder.CreateInsertElement(Result, ExtractedElts[Idx],
-                                               Builder.getInt32(i));
-        }
-        return CastInst::Create(Instruction::BitCast, Result, CI.getType());
-      }
-    }
-    break;
-
-  case Intrinsic::arm_neon_vld1: {
-    Align MemAlign = getKnownAlignment(II->getArgOperand(0), DL, II, &AC, &DT);
-    if (Value *V = simplifyNeonVld1(*II, MemAlign.value(), Builder))
-      return replaceInstUsesWith(*II, V);
-    break;
-  }
-
-  case Intrinsic::arm_neon_vld2:
-  case Intrinsic::arm_neon_vld3:
-  case Intrinsic::arm_neon_vld4:
-  case Intrinsic::arm_neon_vld2lane:
-  case Intrinsic::arm_neon_vld3lane:
-  case Intrinsic::arm_neon_vld4lane:
-  case Intrinsic::arm_neon_vst1:
-  case Intrinsic::arm_neon_vst2:
-  case Intrinsic::arm_neon_vst3:
-  case Intrinsic::arm_neon_vst4:
-  case Intrinsic::arm_neon_vst2lane:
-  case Intrinsic::arm_neon_vst3lane:
-  case Intrinsic::arm_neon_vst4lane: {
-    Align MemAlign = getKnownAlignment(II->getArgOperand(0), DL, II, &AC, &DT);
-    unsigned AlignArg = II->getNumArgOperands() - 1;
-    Value *AlignArgOp = II->getArgOperand(AlignArg);
-    MaybeAlign Align = cast<ConstantInt>(AlignArgOp)->getMaybeAlignValue();
-    if (Align && *Align < MemAlign)
-      return replaceOperand(*II, AlignArg,
-                            ConstantInt::get(Type::getInt32Ty(II->getContext()),
-                                             MemAlign.value(), false));
-    break;
-  }
 
   case Intrinsic::arm_neon_vtbl1:
   case Intrinsic::aarch64_neon_tbl1:
@@ -3453,690 +1326,6 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
     }
     break;
   }
-  case Intrinsic::arm_mve_pred_i2v: {
-    Value *Arg = II->getArgOperand(0);
-    Value *ArgArg;
-    if (match(Arg, m_Intrinsic<Intrinsic::arm_mve_pred_v2i>(m_Value(ArgArg))) &&
-        II->getType() == ArgArg->getType())
-      return replaceInstUsesWith(*II, ArgArg);
-    Constant *XorMask;
-    if (match(Arg,
-              m_Xor(m_Intrinsic<Intrinsic::arm_mve_pred_v2i>(m_Value(ArgArg)),
-                    m_Constant(XorMask))) &&
-        II->getType() == ArgArg->getType()) {
-      if (auto *CI = dyn_cast<ConstantInt>(XorMask)) {
-        if (CI->getValue().trunc(16).isAllOnesValue()) {
-          auto TrueVector = Builder.CreateVectorSplat(
-              cast<VectorType>(II->getType())->getNumElements(),
-              Builder.getTrue());
-          return BinaryOperator::Create(Instruction::Xor, ArgArg, TrueVector);
-        }
-      }
-    }
-    KnownBits ScalarKnown(32);
-    if (SimplifyDemandedBits(II, 0, APInt::getLowBitsSet(32, 16),
-                             ScalarKnown, 0))
-      return II;
-    break;
-  }
-  case Intrinsic::arm_mve_pred_v2i: {
-    Value *Arg = II->getArgOperand(0);
-    Value *ArgArg;
-    if (match(Arg, m_Intrinsic<Intrinsic::arm_mve_pred_i2v>(m_Value(ArgArg))))
-      return replaceInstUsesWith(*II, ArgArg);
-    if (!II->getMetadata(LLVMContext::MD_range)) {
-      Type *IntTy32 = Type::getInt32Ty(II->getContext());
-      Metadata *M[] = {
-        ConstantAsMetadata::get(ConstantInt::get(IntTy32, 0)),
-        ConstantAsMetadata::get(ConstantInt::get(IntTy32, 0xFFFF))
-      };
-      II->setMetadata(LLVMContext::MD_range, MDNode::get(II->getContext(), M));
-      return II;
-    }
-    break;
-  }
-  case Intrinsic::arm_mve_vadc:
-  case Intrinsic::arm_mve_vadc_predicated: {
-    unsigned CarryOp =
-        (II->getIntrinsicID() == Intrinsic::arm_mve_vadc_predicated) ? 3 : 2;
-    assert(II->getArgOperand(CarryOp)->getType()->getScalarSizeInBits() == 32 &&
-           "Bad type for intrinsic!");
-
-    KnownBits CarryKnown(32);
-    if (SimplifyDemandedBits(II, CarryOp, APInt::getOneBitSet(32, 29),
-                             CarryKnown))
-      return II;
-    break;
-  }
-  case Intrinsic::amdgcn_rcp: {
-    Value *Src = II->getArgOperand(0);
-
-    // TODO: Move to ConstantFolding/InstSimplify?
-    if (isa<UndefValue>(Src)) {
-      Type *Ty = II->getType();
-      auto *QNaN = ConstantFP::get(Ty, APFloat::getQNaN(Ty->getFltSemantics()));
-      return replaceInstUsesWith(CI, QNaN);
-    }
-
-    if (II->isStrictFP())
-      break;
-
-    if (const ConstantFP *C = dyn_cast<ConstantFP>(Src)) {
-      const APFloat &ArgVal = C->getValueAPF();
-      APFloat Val(ArgVal.getSemantics(), 1);
-      Val.divide(ArgVal, APFloat::rmNearestTiesToEven);
-
-      // This is more precise than the instruction may give.
-      //
-      // TODO: The instruction always flushes denormal results (except for f16),
-      // should this also?
-      return replaceInstUsesWith(CI, ConstantFP::get(II->getContext(), Val));
-    }
-
-    break;
-  }
-  case Intrinsic::amdgcn_rsq: {
-    Value *Src = II->getArgOperand(0);
-
-    // TODO: Move to ConstantFolding/InstSimplify?
-    if (isa<UndefValue>(Src)) {
-      Type *Ty = II->getType();
-      auto *QNaN = ConstantFP::get(Ty, APFloat::getQNaN(Ty->getFltSemantics()));
-      return replaceInstUsesWith(CI, QNaN);
-    }
-
-    break;
-  }
-  case Intrinsic::amdgcn_frexp_mant:
-  case Intrinsic::amdgcn_frexp_exp: {
-    Value *Src = II->getArgOperand(0);
-    if (const ConstantFP *C = dyn_cast<ConstantFP>(Src)) {
-      int Exp;
-      APFloat Significand = frexp(C->getValueAPF(), Exp,
-                                  APFloat::rmNearestTiesToEven);
-
-      if (IID == Intrinsic::amdgcn_frexp_mant) {
-        return replaceInstUsesWith(CI, ConstantFP::get(II->getContext(),
-                                                       Significand));
-      }
-
-      // Match instruction special case behavior.
-      if (Exp == APFloat::IEK_NaN || Exp == APFloat::IEK_Inf)
-        Exp = 0;
-
-      return replaceInstUsesWith(CI, ConstantInt::get(II->getType(), Exp));
-    }
-
-    if (isa<UndefValue>(Src))
-      return replaceInstUsesWith(CI, UndefValue::get(II->getType()));
-
-    break;
-  }
-  case Intrinsic::amdgcn_class: {
-    enum  {
-      S_NAN = 1 << 0,        // Signaling NaN
-      Q_NAN = 1 << 1,        // Quiet NaN
-      N_INFINITY = 1 << 2,   // Negative infinity
-      N_NORMAL = 1 << 3,     // Negative normal
-      N_SUBNORMAL = 1 << 4,  // Negative subnormal
-      N_ZERO = 1 << 5,       // Negative zero
-      P_ZERO = 1 << 6,       // Positive zero
-      P_SUBNORMAL = 1 << 7,  // Positive subnormal
-      P_NORMAL = 1 << 8,     // Positive normal
-      P_INFINITY = 1 << 9    // Positive infinity
-    };
-
-    const uint32_t FullMask = S_NAN | Q_NAN | N_INFINITY | N_NORMAL |
-      N_SUBNORMAL | N_ZERO | P_ZERO | P_SUBNORMAL | P_NORMAL | P_INFINITY;
-
-    Value *Src0 = II->getArgOperand(0);
-    Value *Src1 = II->getArgOperand(1);
-    const ConstantInt *CMask = dyn_cast<ConstantInt>(Src1);
-    if (!CMask) {
-      if (isa<UndefValue>(Src0))
-        return replaceInstUsesWith(*II, UndefValue::get(II->getType()));
-
-      if (isa<UndefValue>(Src1))
-        return replaceInstUsesWith(*II, ConstantInt::get(II->getType(), false));
-      break;
-    }
-
-    uint32_t Mask = CMask->getZExtValue();
-
-    // If all tests are made, it doesn't matter what the value is.
-    if ((Mask & FullMask) == FullMask)
-      return replaceInstUsesWith(*II, ConstantInt::get(II->getType(), true));
-
-    if ((Mask & FullMask) == 0)
-      return replaceInstUsesWith(*II, ConstantInt::get(II->getType(), false));
-
-    if (Mask == (S_NAN | Q_NAN)) {
-      // Equivalent of isnan. Replace with standard fcmp.
-      Value *FCmp = Builder.CreateFCmpUNO(Src0, Src0);
-      FCmp->takeName(II);
-      return replaceInstUsesWith(*II, FCmp);
-    }
-
-    if (Mask == (N_ZERO | P_ZERO)) {
-      // Equivalent of == 0.
-      Value *FCmp = Builder.CreateFCmpOEQ(
-        Src0, ConstantFP::get(Src0->getType(), 0.0));
-
-      FCmp->takeName(II);
-      return replaceInstUsesWith(*II, FCmp);
-    }
-
-    // fp_class (nnan x), qnan|snan|other -> fp_class (nnan x), other
-    if (((Mask & S_NAN) || (Mask & Q_NAN)) && isKnownNeverNaN(Src0, &TLI))
-      return replaceOperand(*II, 1, ConstantInt::get(Src1->getType(),
-                                                     Mask & ~(S_NAN | Q_NAN)));
-
-    const ConstantFP *CVal = dyn_cast<ConstantFP>(Src0);
-    if (!CVal) {
-      if (isa<UndefValue>(Src0))
-        return replaceInstUsesWith(*II, UndefValue::get(II->getType()));
-
-      // Clamp mask to used bits
-      if ((Mask & FullMask) != Mask) {
-        CallInst *NewCall = Builder.CreateCall(II->getCalledFunction(),
-          { Src0, ConstantInt::get(Src1->getType(), Mask & FullMask) }
-        );
-
-        NewCall->takeName(II);
-        return replaceInstUsesWith(*II, NewCall);
-      }
-
-      break;
-    }
-
-    const APFloat &Val = CVal->getValueAPF();
-
-    bool Result =
-      ((Mask & S_NAN) && Val.isNaN() && Val.isSignaling()) ||
-      ((Mask & Q_NAN) && Val.isNaN() && !Val.isSignaling()) ||
-      ((Mask & N_INFINITY) && Val.isInfinity() && Val.isNegative()) ||
-      ((Mask & N_NORMAL) && Val.isNormal() && Val.isNegative()) ||
-      ((Mask & N_SUBNORMAL) && Val.isDenormal() && Val.isNegative()) ||
-      ((Mask & N_ZERO) && Val.isZero() && Val.isNegative()) ||
-      ((Mask & P_ZERO) && Val.isZero() && !Val.isNegative()) ||
-      ((Mask & P_SUBNORMAL) && Val.isDenormal() && !Val.isNegative()) ||
-      ((Mask & P_NORMAL) && Val.isNormal() && !Val.isNegative()) ||
-      ((Mask & P_INFINITY) && Val.isInfinity() && !Val.isNegative());
-
-    return replaceInstUsesWith(*II, ConstantInt::get(II->getType(), Result));
-  }
-  case Intrinsic::amdgcn_cvt_pkrtz: {
-    Value *Src0 = II->getArgOperand(0);
-    Value *Src1 = II->getArgOperand(1);
-    if (const ConstantFP *C0 = dyn_cast<ConstantFP>(Src0)) {
-      if (const ConstantFP *C1 = dyn_cast<ConstantFP>(Src1)) {
-        const fltSemantics &HalfSem
-          = II->getType()->getScalarType()->getFltSemantics();
-        bool LosesInfo;
-        APFloat Val0 = C0->getValueAPF();
-        APFloat Val1 = C1->getValueAPF();
-        Val0.convert(HalfSem, APFloat::rmTowardZero, &LosesInfo);
-        Val1.convert(HalfSem, APFloat::rmTowardZero, &LosesInfo);
-
-        Constant *Folded = ConstantVector::get({
-            ConstantFP::get(II->getContext(), Val0),
-            ConstantFP::get(II->getContext(), Val1) });
-        return replaceInstUsesWith(*II, Folded);
-      }
-    }
-
-    if (isa<UndefValue>(Src0) && isa<UndefValue>(Src1))
-      return replaceInstUsesWith(*II, UndefValue::get(II->getType()));
-
-    break;
-  }
-  case Intrinsic::amdgcn_cvt_pknorm_i16:
-  case Intrinsic::amdgcn_cvt_pknorm_u16:
-  case Intrinsic::amdgcn_cvt_pk_i16:
-  case Intrinsic::amdgcn_cvt_pk_u16: {
-    Value *Src0 = II->getArgOperand(0);
-    Value *Src1 = II->getArgOperand(1);
-
-    if (isa<UndefValue>(Src0) && isa<UndefValue>(Src1))
-      return replaceInstUsesWith(*II, UndefValue::get(II->getType()));
-
-    break;
-  }
-  case Intrinsic::amdgcn_ubfe:
-  case Intrinsic::amdgcn_sbfe: {
-    // Decompose simple cases into standard shifts.
-    Value *Src = II->getArgOperand(0);
-    if (isa<UndefValue>(Src))
-      return replaceInstUsesWith(*II, Src);
-
-    unsigned Width;
-    Type *Ty = II->getType();
-    unsigned IntSize = Ty->getIntegerBitWidth();
-
-    ConstantInt *CWidth = dyn_cast<ConstantInt>(II->getArgOperand(2));
-    if (CWidth) {
-      Width = CWidth->getZExtValue();
-      if ((Width & (IntSize - 1)) == 0)
-        return replaceInstUsesWith(*II, ConstantInt::getNullValue(Ty));
-
-      // Hardware ignores high bits, so remove those.
-      if (Width >= IntSize)
-        return replaceOperand(*II, 2, ConstantInt::get(CWidth->getType(),
-                                                       Width & (IntSize - 1)));
-    }
-
-    unsigned Offset;
-    ConstantInt *COffset = dyn_cast<ConstantInt>(II->getArgOperand(1));
-    if (COffset) {
-      Offset = COffset->getZExtValue();
-      if (Offset >= IntSize)
-        return replaceOperand(*II, 1, ConstantInt::get(COffset->getType(),
-                                                       Offset & (IntSize - 1)));
-    }
-
-    bool Signed = IID == Intrinsic::amdgcn_sbfe;
-
-    if (!CWidth || !COffset)
-      break;
-
-    // The case of Width == 0 is handled above, which makes this tranformation
-    // safe.  If Width == 0, then the ashr and lshr instructions become poison
-    // value since the shift amount would be equal to the bit size.
-    assert(Width != 0);
-
-    // TODO: This allows folding to undef when the hardware has specific
-    // behavior?
-    if (Offset + Width < IntSize) {
-      Value *Shl = Builder.CreateShl(Src, IntSize - Offset - Width);
-      Value *RightShift = Signed ? Builder.CreateAShr(Shl, IntSize - Width)
-                                 : Builder.CreateLShr(Shl, IntSize - Width);
-      RightShift->takeName(II);
-      return replaceInstUsesWith(*II, RightShift);
-    }
-
-    Value *RightShift = Signed ? Builder.CreateAShr(Src, Offset)
-                               : Builder.CreateLShr(Src, Offset);
-
-    RightShift->takeName(II);
-    return replaceInstUsesWith(*II, RightShift);
-  }
-  case Intrinsic::amdgcn_exp:
-  case Intrinsic::amdgcn_exp_compr: {
-    ConstantInt *En = cast<ConstantInt>(II->getArgOperand(1));
-    unsigned EnBits = En->getZExtValue();
-    if (EnBits == 0xf)
-      break; // All inputs enabled.
-
-    bool IsCompr = IID == Intrinsic::amdgcn_exp_compr;
-    bool Changed = false;
-    for (int I = 0; I < (IsCompr ? 2 : 4); ++I) {
-      if ((!IsCompr && (EnBits & (1 << I)) == 0) ||
-          (IsCompr && ((EnBits & (0x3 << (2 * I))) == 0))) {
-        Value *Src = II->getArgOperand(I + 2);
-        if (!isa<UndefValue>(Src)) {
-          replaceOperand(*II, I + 2, UndefValue::get(Src->getType()));
-          Changed = true;
-        }
-      }
-    }
-
-    if (Changed)
-      return II;
-
-    break;
-  }
-  case Intrinsic::amdgcn_fmed3: {
-    // Note this does not preserve proper sNaN behavior if IEEE-mode is enabled
-    // for the shader.
-
-    Value *Src0 = II->getArgOperand(0);
-    Value *Src1 = II->getArgOperand(1);
-    Value *Src2 = II->getArgOperand(2);
-
-    // Checking for NaN before canonicalization provides better fidelity when
-    // mapping other operations onto fmed3 since the order of operands is
-    // unchanged.
-    CallInst *NewCall = nullptr;
-    if (match(Src0, m_NaN()) || isa<UndefValue>(Src0)) {
-      NewCall = Builder.CreateMinNum(Src1, Src2);
-    } else if (match(Src1, m_NaN()) || isa<UndefValue>(Src1)) {
-      NewCall = Builder.CreateMinNum(Src0, Src2);
-    } else if (match(Src2, m_NaN()) || isa<UndefValue>(Src2)) {
-      NewCall = Builder.CreateMaxNum(Src0, Src1);
-    }
-
-    if (NewCall) {
-      NewCall->copyFastMathFlags(II);
-      NewCall->takeName(II);
-      return replaceInstUsesWith(*II, NewCall);
-    }
-
-    bool Swap = false;
-    // Canonicalize constants to RHS operands.
-    //
-    // fmed3(c0, x, c1) -> fmed3(x, c0, c1)
-    if (isa<Constant>(Src0) && !isa<Constant>(Src1)) {
-      std::swap(Src0, Src1);
-      Swap = true;
-    }
-
-    if (isa<Constant>(Src1) && !isa<Constant>(Src2)) {
-      std::swap(Src1, Src2);
-      Swap = true;
-    }
-
-    if (isa<Constant>(Src0) && !isa<Constant>(Src1)) {
-      std::swap(Src0, Src1);
-      Swap = true;
-    }
-
-    if (Swap) {
-      II->setArgOperand(0, Src0);
-      II->setArgOperand(1, Src1);
-      II->setArgOperand(2, Src2);
-      return II;
-    }
-
-    if (const ConstantFP *C0 = dyn_cast<ConstantFP>(Src0)) {
-      if (const ConstantFP *C1 = dyn_cast<ConstantFP>(Src1)) {
-        if (const ConstantFP *C2 = dyn_cast<ConstantFP>(Src2)) {
-          APFloat Result = fmed3AMDGCN(C0->getValueAPF(), C1->getValueAPF(),
-                                       C2->getValueAPF());
-          return replaceInstUsesWith(*II,
-            ConstantFP::get(Builder.getContext(), Result));
-        }
-      }
-    }
-
-    break;
-  }
-  case Intrinsic::amdgcn_icmp:
-  case Intrinsic::amdgcn_fcmp: {
-    const ConstantInt *CC = cast<ConstantInt>(II->getArgOperand(2));
-    // Guard against invalid arguments.
-    int64_t CCVal = CC->getZExtValue();
-    bool IsInteger = IID == Intrinsic::amdgcn_icmp;
-    if ((IsInteger && (CCVal < CmpInst::FIRST_ICMP_PREDICATE ||
-                       CCVal > CmpInst::LAST_ICMP_PREDICATE)) ||
-        (!IsInteger && (CCVal < CmpInst::FIRST_FCMP_PREDICATE ||
-                        CCVal > CmpInst::LAST_FCMP_PREDICATE)))
-      break;
-
-    Value *Src0 = II->getArgOperand(0);
-    Value *Src1 = II->getArgOperand(1);
-
-    if (auto *CSrc0 = dyn_cast<Constant>(Src0)) {
-      if (auto *CSrc1 = dyn_cast<Constant>(Src1)) {
-        Constant *CCmp = ConstantExpr::getCompare(CCVal, CSrc0, CSrc1);
-        if (CCmp->isNullValue()) {
-          return replaceInstUsesWith(
-              *II, ConstantExpr::getSExt(CCmp, II->getType()));
-        }
-
-        // The result of V_ICMP/V_FCMP assembly instructions (which this
-        // intrinsic exposes) is one bit per thread, masked with the EXEC
-        // register (which contains the bitmask of live threads). So a
-        // comparison that always returns true is the same as a read of the
-        // EXEC register.
-        Function *NewF = Intrinsic::getDeclaration(
-            II->getModule(), Intrinsic::read_register, II->getType());
-        Metadata *MDArgs[] = {MDString::get(II->getContext(), "exec")};
-        MDNode *MD = MDNode::get(II->getContext(), MDArgs);
-        Value *Args[] = {MetadataAsValue::get(II->getContext(), MD)};
-        CallInst *NewCall = Builder.CreateCall(NewF, Args);
-        NewCall->addAttribute(AttributeList::FunctionIndex,
-                              Attribute::Convergent);
-        NewCall->takeName(II);
-        return replaceInstUsesWith(*II, NewCall);
-      }
-
-      // Canonicalize constants to RHS.
-      CmpInst::Predicate SwapPred
-        = CmpInst::getSwappedPredicate(static_cast<CmpInst::Predicate>(CCVal));
-      II->setArgOperand(0, Src1);
-      II->setArgOperand(1, Src0);
-      II->setArgOperand(2, ConstantInt::get(CC->getType(),
-                                            static_cast<int>(SwapPred)));
-      return II;
-    }
-
-    if (CCVal != CmpInst::ICMP_EQ && CCVal != CmpInst::ICMP_NE)
-      break;
-
-    // Canonicalize compare eq with true value to compare != 0
-    // llvm.amdgcn.icmp(zext (i1 x), 1, eq)
-    //   -> llvm.amdgcn.icmp(zext (i1 x), 0, ne)
-    // llvm.amdgcn.icmp(sext (i1 x), -1, eq)
-    //   -> llvm.amdgcn.icmp(sext (i1 x), 0, ne)
-    Value *ExtSrc;
-    if (CCVal == CmpInst::ICMP_EQ &&
-        ((match(Src1, m_One()) && match(Src0, m_ZExt(m_Value(ExtSrc)))) ||
-         (match(Src1, m_AllOnes()) && match(Src0, m_SExt(m_Value(ExtSrc))))) &&
-        ExtSrc->getType()->isIntegerTy(1)) {
-      replaceOperand(*II, 1, ConstantInt::getNullValue(Src1->getType()));
-      replaceOperand(*II, 2, ConstantInt::get(CC->getType(), CmpInst::ICMP_NE));
-      return II;
-    }
-
-    CmpInst::Predicate SrcPred;
-    Value *SrcLHS;
-    Value *SrcRHS;
-
-    // Fold compare eq/ne with 0 from a compare result as the predicate to the
-    // intrinsic. The typical use is a wave vote function in the library, which
-    // will be fed from a user code condition compared with 0. Fold in the
-    // redundant compare.
-
-    // llvm.amdgcn.icmp([sz]ext ([if]cmp pred a, b), 0, ne)
-    //   -> llvm.amdgcn.[if]cmp(a, b, pred)
-    //
-    // llvm.amdgcn.icmp([sz]ext ([if]cmp pred a, b), 0, eq)
-    //   -> llvm.amdgcn.[if]cmp(a, b, inv pred)
-    if (match(Src1, m_Zero()) &&
-        match(Src0,
-              m_ZExtOrSExt(m_Cmp(SrcPred, m_Value(SrcLHS), m_Value(SrcRHS))))) {
-      if (CCVal == CmpInst::ICMP_EQ)
-        SrcPred = CmpInst::getInversePredicate(SrcPred);
-
-      Intrinsic::ID NewIID = CmpInst::isFPPredicate(SrcPred) ?
-        Intrinsic::amdgcn_fcmp : Intrinsic::amdgcn_icmp;
-
-      Type *Ty = SrcLHS->getType();
-      if (auto *CmpType = dyn_cast<IntegerType>(Ty)) {
-        // Promote to next legal integer type.
-        unsigned Width = CmpType->getBitWidth();
-        unsigned NewWidth = Width;
-
-        // Don't do anything for i1 comparisons.
-        if (Width == 1)
-          break;
-
-        if (Width <= 16)
-          NewWidth = 16;
-        else if (Width <= 32)
-          NewWidth = 32;
-        else if (Width <= 64)
-          NewWidth = 64;
-        else if (Width > 64)
-          break; // Can't handle this.
-
-        if (Width != NewWidth) {
-          IntegerType *CmpTy = Builder.getIntNTy(NewWidth);
-          if (CmpInst::isSigned(SrcPred)) {
-            SrcLHS = Builder.CreateSExt(SrcLHS, CmpTy);
-            SrcRHS = Builder.CreateSExt(SrcRHS, CmpTy);
-          } else {
-            SrcLHS = Builder.CreateZExt(SrcLHS, CmpTy);
-            SrcRHS = Builder.CreateZExt(SrcRHS, CmpTy);
-          }
-        }
-      } else if (!Ty->isFloatTy() && !Ty->isDoubleTy() && !Ty->isHalfTy())
-        break;
-
-      Function *NewF =
-          Intrinsic::getDeclaration(II->getModule(), NewIID,
-                                    { II->getType(),
-                                      SrcLHS->getType() });
-      Value *Args[] = { SrcLHS, SrcRHS,
-                        ConstantInt::get(CC->getType(), SrcPred) };
-      CallInst *NewCall = Builder.CreateCall(NewF, Args);
-      NewCall->takeName(II);
-      return replaceInstUsesWith(*II, NewCall);
-    }
-
-    break;
-  }
-  case Intrinsic::amdgcn_ballot: {
-    if (auto *Src = dyn_cast<ConstantInt>(II->getArgOperand(0))) {
-      if (Src->isZero()) {
-        // amdgcn.ballot(i1 0) is zero.
-        return replaceInstUsesWith(*II, Constant::getNullValue(II->getType()));
-      }
-
-      if (Src->isOne()) {
-        // amdgcn.ballot(i1 1) is exec.
-        const char *RegName = "exec";
-        if (II->getType()->isIntegerTy(32))
-          RegName = "exec_lo";
-        else if (!II->getType()->isIntegerTy(64))
-          break;
-
-        Function *NewF = Intrinsic::getDeclaration(
-            II->getModule(), Intrinsic::read_register, II->getType());
-        Metadata *MDArgs[] = {MDString::get(II->getContext(), RegName)};
-        MDNode *MD = MDNode::get(II->getContext(), MDArgs);
-        Value *Args[] = {MetadataAsValue::get(II->getContext(), MD)};
-        CallInst *NewCall = Builder.CreateCall(NewF, Args);
-        NewCall->addAttribute(AttributeList::FunctionIndex,
-                              Attribute::Convergent);
-        NewCall->takeName(II);
-        return replaceInstUsesWith(*II, NewCall);
-      }
-    }
-    break;
-  }
-  case Intrinsic::amdgcn_wqm_vote: {
-    // wqm_vote is identity when the argument is constant.
-    if (!isa<Constant>(II->getArgOperand(0)))
-      break;
-
-    return replaceInstUsesWith(*II, II->getArgOperand(0));
-  }
-  case Intrinsic::amdgcn_kill: {
-    const ConstantInt *C = dyn_cast<ConstantInt>(II->getArgOperand(0));
-    if (!C || !C->getZExtValue())
-      break;
-
-    // amdgcn.kill(i1 1) is a no-op
-    return eraseInstFromFunction(CI);
-  }
-  case Intrinsic::amdgcn_update_dpp: {
-    Value *Old = II->getArgOperand(0);
-
-    auto BC = cast<ConstantInt>(II->getArgOperand(5));
-    auto RM = cast<ConstantInt>(II->getArgOperand(3));
-    auto BM = cast<ConstantInt>(II->getArgOperand(4));
-    if (BC->isZeroValue() ||
-        RM->getZExtValue() != 0xF ||
-        BM->getZExtValue() != 0xF ||
-        isa<UndefValue>(Old))
-      break;
-
-    // If bound_ctrl = 1, row mask = bank mask = 0xf we can omit old value.
-    return replaceOperand(*II, 0, UndefValue::get(Old->getType()));
-  }
-  case Intrinsic::amdgcn_permlane16:
-  case Intrinsic::amdgcn_permlanex16: {
-    // Discard vdst_in if it's not going to be read.
-    Value *VDstIn = II->getArgOperand(0);
-   if (isa<UndefValue>(VDstIn))
-     break;
-
-    ConstantInt *FetchInvalid = cast<ConstantInt>(II->getArgOperand(4));
-    ConstantInt *BoundCtrl = cast<ConstantInt>(II->getArgOperand(5));
-    if (!FetchInvalid->getZExtValue() && !BoundCtrl->getZExtValue())
-      break;
-
-    return replaceOperand(*II, 0, UndefValue::get(VDstIn->getType()));
-  }
-  case Intrinsic::amdgcn_readfirstlane:
-  case Intrinsic::amdgcn_readlane: {
-    // A constant value is trivially uniform.
-    if (Constant *C = dyn_cast<Constant>(II->getArgOperand(0)))
-      return replaceInstUsesWith(*II, C);
-
-    // The rest of these may not be safe if the exec may not be the same between
-    // the def and use.
-    Value *Src = II->getArgOperand(0);
-    Instruction *SrcInst = dyn_cast<Instruction>(Src);
-    if (SrcInst && SrcInst->getParent() != II->getParent())
-      break;
-
-    // readfirstlane (readfirstlane x) -> readfirstlane x
-    // readlane (readfirstlane x), y -> readfirstlane x
-    if (match(Src, m_Intrinsic<Intrinsic::amdgcn_readfirstlane>()))
-      return replaceInstUsesWith(*II, Src);
-
-    if (IID == Intrinsic::amdgcn_readfirstlane) {
-      // readfirstlane (readlane x, y) -> readlane x, y
-      if (match(Src, m_Intrinsic<Intrinsic::amdgcn_readlane>()))
-        return replaceInstUsesWith(*II, Src);
-    } else {
-      // readlane (readlane x, y), y -> readlane x, y
-      if (match(Src, m_Intrinsic<Intrinsic::amdgcn_readlane>(
-                  m_Value(), m_Specific(II->getArgOperand(1)))))
-        return replaceInstUsesWith(*II, Src);
-    }
-
-    break;
-  }
-  case Intrinsic::amdgcn_ldexp: {
-    // FIXME: This doesn't introduce new instructions and belongs in
-    // InstructionSimplify.
-    Type *Ty = II->getType();
-    Value *Op0 = II->getArgOperand(0);
-    Value *Op1 = II->getArgOperand(1);
-
-    // Folding undef to qnan is safe regardless of the FP mode.
-    if (isa<UndefValue>(Op0)) {
-      auto *QNaN = ConstantFP::get(Ty, APFloat::getQNaN(Ty->getFltSemantics()));
-      return replaceInstUsesWith(*II, QNaN);
-    }
-
-    const APFloat *C = nullptr;
-    match(Op0, m_APFloat(C));
-
-    // FIXME: Should flush denorms depending on FP mode, but that's ignored
-    // everywhere else.
-    //
-    // These cases should be safe, even with strictfp.
-    // ldexp(0.0, x) -> 0.0
-    // ldexp(-0.0, x) -> -0.0
-    // ldexp(inf, x) -> inf
-    // ldexp(-inf, x) -> -inf
-    if (C && (C->isZero() || C->isInfinity()))
-      return replaceInstUsesWith(*II, Op0);
-
-    // With strictfp, be more careful about possibly needing to flush denormals
-    // or not, and snan behavior depends on ieee_mode.
-    if (II->isStrictFP())
-      break;
-
-    if (C && C->isNaN()) {
-      // FIXME: We just need to make the nan quiet here, but that's unavailable
-      // on APFloat, only IEEEfloat
-      auto *Quieted = ConstantFP::get(
-        Ty, scalbn(*C, 0, APFloat::rmNearestTiesToEven));
-      return replaceInstUsesWith(*II, Quieted);
-    }
-
-    // ldexp(x, 0) -> x
-    // ldexp(x, undef) -> x
-    if (isa<UndefValue>(Op1) || match(Op1, m_ZeroInt()))
-      return replaceInstUsesWith(*II, Op0);
-
-    break;
-  }
   case Intrinsic::hexagon_V6_vandvrt:
   case Intrinsic::hexagon_V6_vandvrt_128B: {
     // Simplify Q -> V -> Q conversion.
@@ -4360,12 +1549,19 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
     }
     break;
   }
+  default: {
+    // Handle target specific intrinsics
+    Optional<Instruction *> V = targetInstCombineIntrinsic(*II);
+    if (V.hasValue())
+      return V.getValue();
+    break;
+  }
   }
   return visitCallBase(*II);
 }
 
 // Fence instruction simplification
-Instruction *InstCombiner::visitFenceInst(FenceInst &FI) {
+Instruction *InstCombinerImpl::visitFenceInst(FenceInst &FI) {
   // Remove identical consecutive fences.
   Instruction *Next = FI.getNextNonDebugInstruction();
   if (auto *NFI = dyn_cast<FenceInst>(Next))
@@ -4375,12 +1571,12 @@ Instruction *InstCombiner::visitFenceInst(FenceInst &FI) {
 }
 
 // InvokeInst simplification
-Instruction *InstCombiner::visitInvokeInst(InvokeInst &II) {
+Instruction *InstCombinerImpl::visitInvokeInst(InvokeInst &II) {
   return visitCallBase(II);
 }
 
 // CallBrInst simplification
-Instruction *InstCombiner::visitCallBrInst(CallBrInst &CBI) {
+Instruction *InstCombinerImpl::visitCallBrInst(CallBrInst &CBI) {
   return visitCallBase(CBI);
 }
 
@@ -4420,7 +1616,7 @@ static bool isSafeToEliminateVarargsCast(const CallBase &Call,
   return true;
 }
 
-Instruction *InstCombiner::tryOptimizeCall(CallInst *CI) {
+Instruction *InstCombinerImpl::tryOptimizeCall(CallInst *CI) {
   if (!CI->getCalledFunction()) return nullptr;
 
   auto InstCombineRAUW = [this](Instruction *From, Value *With) {
@@ -4577,7 +1773,7 @@ static void annotateAnyAllocSite(CallBase &Call, const TargetLibraryInfo *TLI) {
 }
 
 /// Improvements for call, callbr and invoke instructions.
-Instruction *InstCombiner::visitCallBase(CallBase &Call) {
+Instruction *InstCombinerImpl::visitCallBase(CallBase &Call) {
   if (isAllocationFn(&Call, &TLI))
     annotateAnyAllocSite(Call, &TLI);
 
@@ -4728,7 +1924,7 @@ Instruction *InstCombiner::visitCallBase(CallBase &Call) {
 
 /// If the callee is a constexpr cast of a function, attempt to move the cast to
 /// the arguments of the call/callbr/invoke.
-bool InstCombiner::transformConstExprCastCall(CallBase &Call) {
+bool InstCombinerImpl::transformConstExprCastCall(CallBase &Call) {
   auto *Callee =
       dyn_cast<Function>(Call.getCalledOperand()->stripPointerCasts());
   if (!Callee)
@@ -5012,8 +2208,8 @@ bool InstCombiner::transformConstExprCastCall(CallBase &Call) {
 /// Turn a call to a function created by init_trampoline / adjust_trampoline
 /// intrinsic pair into a direct call to the underlying function.
 Instruction *
-InstCombiner::transformCallThroughTrampoline(CallBase &Call,
-                                             IntrinsicInst &Tramp) {
+InstCombinerImpl::transformCallThroughTrampoline(CallBase &Call,
+                                                 IntrinsicInst &Tramp) {
   Value *Callee = Call.getCalledOperand();
   Type *CalleeTy = Callee->getType();
   FunctionType *FTy = Call.getFunctionType();
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
index 3639edb5df4d1..3e923f7caab78 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
@@ -14,10 +14,11 @@
 #include "llvm/ADT/SetVector.h"
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
-#include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DIBuilder.h"
+#include "llvm/IR/DataLayout.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/Support/KnownBits.h"
+#include "llvm/Transforms/InstCombine/InstCombiner.h"
 #include <numeric>
 using namespace llvm;
 using namespace PatternMatch;
@@ -81,8 +82,8 @@ static Value *decomposeSimpleLinearExpr(Value *Val, unsigned &Scale,
 
 /// If we find a cast of an allocation instruction, try to eliminate the cast by
 /// moving the type information into the alloc.
-Instruction *InstCombiner::PromoteCastOfAllocation(BitCastInst &CI,
-                                                   AllocaInst &AI) {
+Instruction *InstCombinerImpl::PromoteCastOfAllocation(BitCastInst &CI,
+                                                       AllocaInst &AI) {
   PointerType *PTy = cast<PointerType>(CI.getType());
 
   IRBuilderBase::InsertPointGuard Guard(Builder);
@@ -160,8 +161,8 @@ Instruction *InstCombiner::PromoteCastOfAllocation(BitCastInst &CI,
 
 /// Given an expression that CanEvaluateTruncated or CanEvaluateSExtd returns
 /// true for, actually insert the code to evaluate the expression.
-Value *InstCombiner::EvaluateInDifferentType(Value *V, Type *Ty,
-                                             bool isSigned) {
+Value *InstCombinerImpl::EvaluateInDifferentType(Value *V, Type *Ty,
+                                                 bool isSigned) {
   if (Constant *C = dyn_cast<Constant>(V)) {
     C = ConstantExpr::getIntegerCast(C, Ty, isSigned /*Sext or ZExt*/);
     // If we got a constantexpr back, try to simplify it with DL info.
@@ -229,8 +230,9 @@ Value *InstCombiner::EvaluateInDifferentType(Value *V, Type *Ty,
   return InsertNewInstWith(Res, *I);
 }
 
-Instruction::CastOps InstCombiner::isEliminableCastPair(const CastInst *CI1,
-                                                        const CastInst *CI2) {
+Instruction::CastOps
+InstCombinerImpl::isEliminableCastPair(const CastInst *CI1,
+                                       const CastInst *CI2) {
   Type *SrcTy = CI1->getSrcTy();
   Type *MidTy = CI1->getDestTy();
   Type *DstTy = CI2->getDestTy();
@@ -257,7 +259,7 @@ Instruction::CastOps InstCombiner::isEliminableCastPair(const CastInst *CI1,
 }
 
 /// Implement the transforms common to all CastInst visitors.
-Instruction *InstCombiner::commonCastTransforms(CastInst &CI) {
+Instruction *InstCombinerImpl::commonCastTransforms(CastInst &CI) {
   Value *Src = CI.getOperand(0);
 
   // Try to eliminate a cast of a cast.
@@ -342,7 +344,7 @@ static bool canNotEvaluateInType(Value *V, Type *Ty) {
 ///
 /// This function works on both vectors and scalars.
 ///
-static bool canEvaluateTruncated(Value *V, Type *Ty, InstCombiner &IC,
+static bool canEvaluateTruncated(Value *V, Type *Ty, InstCombinerImpl &IC,
                                  Instruction *CxtI) {
   if (canAlwaysEvaluateInType(V, Ty))
     return true;
@@ -459,7 +461,8 @@ static bool canEvaluateTruncated(Value *V, Type *Ty, InstCombiner &IC,
 ///   trunc (lshr (bitcast <4 x i32> %X to i128), 32) to i32
 ///   --->
 ///   extractelement <4 x i32> %X, 1
-static Instruction *foldVecTruncToExtElt(TruncInst &Trunc, InstCombiner &IC) {
+static Instruction *foldVecTruncToExtElt(TruncInst &Trunc,
+                                         InstCombinerImpl &IC) {
   Value *TruncOp = Trunc.getOperand(0);
   Type *DestType = Trunc.getType();
   if (!TruncOp->hasOneUse() || !isa<IntegerType>(DestType))
@@ -498,7 +501,7 @@ static Instruction *foldVecTruncToExtElt(TruncInst &Trunc, InstCombiner &IC) {
 
 /// Rotate left/right may occur in a wider type than necessary because of type
 /// promotion rules. Try to narrow the inputs and convert to funnel shift.
-Instruction *InstCombiner::narrowRotate(TruncInst &Trunc) {
+Instruction *InstCombinerImpl::narrowRotate(TruncInst &Trunc) {
   assert((isa<VectorType>(Trunc.getSrcTy()) ||
           shouldChangeType(Trunc.getSrcTy(), Trunc.getType())) &&
          "Don't narrow to an illegal scalar type");
@@ -582,7 +585,7 @@ Instruction *InstCombiner::narrowRotate(TruncInst &Trunc) {
 /// Try to narrow the width of math or bitwise logic instructions by pulling a
 /// truncate ahead of binary operators.
 /// TODO: Transforms for truncated shifts should be moved into here.
-Instruction *InstCombiner::narrowBinOp(TruncInst &Trunc) {
+Instruction *InstCombinerImpl::narrowBinOp(TruncInst &Trunc) {
   Type *SrcTy = Trunc.getSrcTy();
   Type *DestTy = Trunc.getType();
   if (!isa<VectorType>(SrcTy) && !shouldChangeType(SrcTy, DestTy))
@@ -687,7 +690,7 @@ static Instruction *shrinkInsertElt(CastInst &Trunc,
   return nullptr;
 }
 
-Instruction *InstCombiner::visitTrunc(TruncInst &Trunc) {
+Instruction *InstCombinerImpl::visitTrunc(TruncInst &Trunc) {
   if (Instruction *Result = commonCastTransforms(Trunc))
     return Result;
 
@@ -862,6 +865,20 @@ Instruction *InstCombiner::visitTrunc(TruncInst &Trunc) {
   if (Instruction *I = foldVecTruncToExtElt(Trunc, *this))
     return I;
 
+  // FIXME: This is temporary work-around for a problem reported here:
+  // https://github.com/KhronosGroup/SPIRV-LLVM-Translator/issues/645
+  //
+  // InstCombine canonical form for this pattern
+  // ```
+  //   // Example (little endian):
+  //   //   trunc (extractelement <4 x i64> %X, 0) to i32
+  //   //   --->
+  //   //   extractelement <8 x i32> (bitcast <4 x i64> %X to <8 x i32>), i32 0
+  // ```
+  // can't be lowered by SPIR-V translator to "standard" format.
+  if (StringRef(Trunc.getModule()->getTargetTriple()).startswith("spir"))
+    return nullptr;
+
   // Whenever an element is extracted from a vector, and then truncated,
   // canonicalize by converting it to a bitcast followed by an
   // extractelement.
@@ -894,8 +911,8 @@ Instruction *InstCombiner::visitTrunc(TruncInst &Trunc) {
   return nullptr;
 }
 
-Instruction *InstCombiner::transformZExtICmp(ICmpInst *Cmp, ZExtInst &Zext,
-                                             bool DoTransform) {
+Instruction *InstCombinerImpl::transformZExtICmp(ICmpInst *Cmp, ZExtInst &Zext,
+                                                 bool DoTransform) {
   // If we are just checking for a icmp eq of a single bit and zext'ing it
   // to an integer, then shift the bit to the appropriate place and then
   // cast to integer to avoid the comparison.
@@ -1031,7 +1048,7 @@ Instruction *InstCombiner::transformZExtICmp(ICmpInst *Cmp, ZExtInst &Zext,
 ///
 /// This function works on both vectors and scalars.
 static bool canEvaluateZExtd(Value *V, Type *Ty, unsigned &BitsToClear,
-                             InstCombiner &IC, Instruction *CxtI) {
+                             InstCombinerImpl &IC, Instruction *CxtI) {
   BitsToClear = 0;
   if (canAlwaysEvaluateInType(V, Ty))
     return true;
@@ -1136,7 +1153,7 @@ static bool canEvaluateZExtd(Value *V, Type *Ty, unsigned &BitsToClear,
   }
 }
 
-Instruction *InstCombiner::visitZExt(ZExtInst &CI) {
+Instruction *InstCombinerImpl::visitZExt(ZExtInst &CI) {
   // If this zero extend is only used by a truncate, let the truncate be
   // eliminated before we try to optimize this zext.
   if (CI.hasOneUse() && isa<TruncInst>(CI.user_back()))
@@ -1274,7 +1291,8 @@ Instruction *InstCombiner::visitZExt(ZExtInst &CI) {
 }
 
 /// Transform (sext icmp) to bitwise / integer operations to eliminate the icmp.
-Instruction *InstCombiner::transformSExtICmp(ICmpInst *ICI, Instruction &CI) {
+Instruction *InstCombinerImpl::transformSExtICmp(ICmpInst *ICI,
+                                                 Instruction &CI) {
   Value *Op0 = ICI->getOperand(0), *Op1 = ICI->getOperand(1);
   ICmpInst::Predicate Pred = ICI->getPredicate();
 
@@ -1410,7 +1428,7 @@ static bool canEvaluateSExtd(Value *V, Type *Ty) {
   return false;
 }
 
-Instruction *InstCombiner::visitSExt(SExtInst &CI) {
+Instruction *InstCombinerImpl::visitSExt(SExtInst &CI) {
   // If this sign extend is only used by a truncate, let the truncate be
   // eliminated before we try to optimize this sext.
   if (CI.hasOneUse() && isa<TruncInst>(CI.user_back()))
@@ -1497,7 +1515,6 @@ Instruction *InstCombiner::visitSExt(SExtInst &CI) {
   return nullptr;
 }
 
-
 /// Return a Constant* for the specified floating-point constant if it fits
 /// in the specified FP type without changing its value.
 static bool fitsInFPType(ConstantFP *CFP, const fltSemantics &Sem) {
@@ -1616,7 +1633,7 @@ static bool isKnownExactCastIntToFP(CastInst &I) {
   return false;
 }
 
-Instruction *InstCombiner::visitFPTrunc(FPTruncInst &FPT) {
+Instruction *InstCombinerImpl::visitFPTrunc(FPTruncInst &FPT) {
   if (Instruction *I = commonCastTransforms(FPT))
     return I;
 
@@ -1800,7 +1817,7 @@ Instruction *InstCombiner::visitFPTrunc(FPTruncInst &FPT) {
   return nullptr;
 }
 
-Instruction *InstCombiner::visitFPExt(CastInst &FPExt) {
+Instruction *InstCombinerImpl::visitFPExt(CastInst &FPExt) {
   // If the source operand is a cast from integer to FP and known exact, then
   // cast the integer operand directly to the destination type.
   Type *Ty = FPExt.getType();
@@ -1818,7 +1835,7 @@ Instruction *InstCombiner::visitFPExt(CastInst &FPExt) {
 /// This is safe if the intermediate type has enough bits in its mantissa to
 /// accurately represent all values of X.  For example, this won't work with
 /// i64 -> float -> i64.
-Instruction *InstCombiner::foldItoFPtoI(CastInst &FI) {
+Instruction *InstCombinerImpl::foldItoFPtoI(CastInst &FI) {
   if (!isa<UIToFPInst>(FI.getOperand(0)) && !isa<SIToFPInst>(FI.getOperand(0)))
     return nullptr;
 
@@ -1858,29 +1875,29 @@ Instruction *InstCombiner::foldItoFPtoI(CastInst &FI) {
   return replaceInstUsesWith(FI, X);
 }
 
-Instruction *InstCombiner::visitFPToUI(FPToUIInst &FI) {
+Instruction *InstCombinerImpl::visitFPToUI(FPToUIInst &FI) {
   if (Instruction *I = foldItoFPtoI(FI))
     return I;
 
   return commonCastTransforms(FI);
 }
 
-Instruction *InstCombiner::visitFPToSI(FPToSIInst &FI) {
+Instruction *InstCombinerImpl::visitFPToSI(FPToSIInst &FI) {
   if (Instruction *I = foldItoFPtoI(FI))
     return I;
 
   return commonCastTransforms(FI);
 }
 
-Instruction *InstCombiner::visitUIToFP(CastInst &CI) {
+Instruction *InstCombinerImpl::visitUIToFP(CastInst &CI) {
   return commonCastTransforms(CI);
 }
 
-Instruction *InstCombiner::visitSIToFP(CastInst &CI) {
+Instruction *InstCombinerImpl::visitSIToFP(CastInst &CI) {
   return commonCastTransforms(CI);
 }
 
-Instruction *InstCombiner::visitIntToPtr(IntToPtrInst &CI) {
+Instruction *InstCombinerImpl::visitIntToPtr(IntToPtrInst &CI) {
   // If the source integer type is not the intptr_t type for this target, do a
   // trunc or zext to the intptr_t type, then inttoptr of it.  This allows the
   // cast to be exposed to other transforms.
@@ -1903,7 +1920,7 @@ Instruction *InstCombiner::visitIntToPtr(IntToPtrInst &CI) {
 }
 
 /// Implement the transforms for cast of pointer (bitcast/ptrtoint)
-Instruction *InstCombiner::commonPointerCastTransforms(CastInst &CI) {
+Instruction *InstCombinerImpl::commonPointerCastTransforms(CastInst &CI) {
   Value *Src = CI.getOperand(0);
 
   if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Src)) {
@@ -1925,7 +1942,7 @@ Instruction *InstCombiner::commonPointerCastTransforms(CastInst &CI) {
   return commonCastTransforms(CI);
 }
 
-Instruction *InstCombiner::visitPtrToInt(PtrToIntInst &CI) {
+Instruction *InstCombinerImpl::visitPtrToInt(PtrToIntInst &CI) {
   // If the destination integer type is not the intptr_t type for this target,
   // do a ptrtoint to intptr_t then do a trunc or zext.  This allows the cast
   // to be exposed to other transforms.
@@ -1963,9 +1980,9 @@ Instruction *InstCombiner::visitPtrToInt(PtrToIntInst &CI) {
 /// Try to replace it with a shuffle (and vector/vector bitcast) if possible.
 ///
 /// The source and destination vector types may have different element types.
-static Instruction *optimizeVectorResizeWithIntegerBitCasts(Value *InVal,
-                                                            VectorType *DestTy,
-                                                            InstCombiner &IC) {
+static Instruction *
+optimizeVectorResizeWithIntegerBitCasts(Value *InVal, VectorType *DestTy,
+                                        InstCombinerImpl &IC) {
   // We can only do this optimization if the output is a multiple of the input
   // element size, or the input is a multiple of the output element size.
   // Convert the input type to have the same element type as the output.
@@ -2165,7 +2182,7 @@ static bool collectInsertionElements(Value *V, unsigned Shift,
 ///
 /// Into two insertelements that do "buildvector{%inc, %inc5}".
 static Value *optimizeIntegerToVectorInsertions(BitCastInst &CI,
-                                                InstCombiner &IC) {
+                                                InstCombinerImpl &IC) {
   VectorType *DestVecTy = cast<VectorType>(CI.getType());
   Value *IntInput = CI.getOperand(0);
 
@@ -2194,7 +2211,7 @@ static Value *optimizeIntegerToVectorInsertions(BitCastInst &CI,
 /// vectors better than bitcasts of scalars because vector registers are
 /// usually not type-specific like scalar integer or scalar floating-point.
 static Instruction *canonicalizeBitCastExtElt(BitCastInst &BitCast,
-                                              InstCombiner &IC) {
+                                              InstCombinerImpl &IC) {
   // TODO: Create and use a pattern matcher for ExtractElementInst.
   auto *ExtElt = dyn_cast<ExtractElementInst>(BitCast.getOperand(0));
   if (!ExtElt || !ExtElt->hasOneUse())
@@ -2320,7 +2337,8 @@ static bool hasStoreUsersOnly(CastInst &CI) {
 ///
 /// All the related PHI nodes can be replaced by new PHI nodes with type A.
 /// The uses of \p CI can be changed to the new PHI node corresponding to \p PN.
-Instruction *InstCombiner::optimizeBitCastFromPhi(CastInst &CI, PHINode *PN) {
+Instruction *InstCombinerImpl::optimizeBitCastFromPhi(CastInst &CI,
+                                                      PHINode *PN) {
   // BitCast used by Store can be handled in InstCombineLoadStoreAlloca.cpp.
   if (hasStoreUsersOnly(CI))
     return nullptr;
@@ -2484,7 +2502,7 @@ Instruction *InstCombiner::optimizeBitCastFromPhi(CastInst &CI, PHINode *PN) {
   return RetVal;
 }
 
-Instruction *InstCombiner::visitBitCast(BitCastInst &CI) {
+Instruction *InstCombinerImpl::visitBitCast(BitCastInst &CI) {
   // If the operands are integer typed then apply the integer transforms,
   // otherwise just apply the common ones.
   Value *Src = CI.getOperand(0);
@@ -2666,7 +2684,7 @@ Instruction *InstCombiner::visitBitCast(BitCastInst &CI) {
   return commonCastTransforms(CI);
 }
 
-Instruction *InstCombiner::visitAddrSpaceCast(AddrSpaceCastInst &CI) {
+Instruction *InstCombinerImpl::visitAddrSpaceCast(AddrSpaceCastInst &CI) {
   // If the destination pointer element type is not the same as the source's
   // first do a bitcast to the destination type, and then the addrspacecast.
   // This allows the cast to be exposed to other transforms.
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index f1233b62445d0..2b8732fdb80d7 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -24,6 +24,7 @@
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/KnownBits.h"
+#include "llvm/Transforms/InstCombine/InstCombiner.h"
 
 using namespace llvm;
 using namespace PatternMatch;
@@ -142,10 +143,10 @@ static void computeUnsignedMinMaxValuesFromKnownBits(const KnownBits &Known,
 ///
 /// If AndCst is non-null, then the loaded value is masked with that constant
 /// before doing the comparison. This handles cases like "A[i]&4 == 0".
-Instruction *InstCombiner::foldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP,
-                                                        GlobalVariable *GV,
-                                                        CmpInst &ICI,
-                                                        ConstantInt *AndCst) {
+Instruction *
+InstCombinerImpl::foldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP,
+                                               GlobalVariable *GV, CmpInst &ICI,
+                                               ConstantInt *AndCst) {
   Constant *Init = GV->getInitializer();
   if (!isa<ConstantArray>(Init) && !isa<ConstantDataArray>(Init))
     return nullptr;
@@ -422,7 +423,7 @@ Instruction *InstCombiner::foldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP,
 ///
 /// If we can't emit an optimized form for this expression, this returns null.
 ///
-static Value *evaluateGEPOffsetExpression(User *GEP, InstCombiner &IC,
+static Value *evaluateGEPOffsetExpression(User *GEP, InstCombinerImpl &IC,
                                           const DataLayout &DL) {
   gep_type_iterator GTI = gep_type_begin(GEP);
 
@@ -841,9 +842,9 @@ static Instruction *transformToIndexedCompare(GEPOperator *GEPLHS, Value *RHS,
 
 /// Fold comparisons between a GEP instruction and something else. At this point
 /// we know that the GEP is on the LHS of the comparison.
-Instruction *InstCombiner::foldGEPICmp(GEPOperator *GEPLHS, Value *RHS,
-                                       ICmpInst::Predicate Cond,
-                                       Instruction &I) {
+Instruction *InstCombinerImpl::foldGEPICmp(GEPOperator *GEPLHS, Value *RHS,
+                                           ICmpInst::Predicate Cond,
+                                           Instruction &I) {
   // Don't transform signed compares of GEPs into index compares. Even if the
   // GEP is inbounds, the final add of the base pointer can have signed overflow
   // and would change the result of the icmp.
@@ -1021,9 +1022,9 @@ Instruction *InstCombiner::foldGEPICmp(GEPOperator *GEPLHS, Value *RHS,
   return transformToIndexedCompare(GEPLHS, RHS, Cond, DL);
 }
 
-Instruction *InstCombiner::foldAllocaCmp(ICmpInst &ICI,
-                                         const AllocaInst *Alloca,
-                                         const Value *Other) {
+Instruction *InstCombinerImpl::foldAllocaCmp(ICmpInst &ICI,
+                                             const AllocaInst *Alloca,
+                                             const Value *Other) {
   assert(ICI.isEquality() && "Cannot fold non-equality comparison.");
 
   // It would be tempting to fold away comparisons between allocas and any
@@ -1099,8 +1100,8 @@ Instruction *InstCombiner::foldAllocaCmp(ICmpInst &ICI,
 }
 
 /// Fold "icmp pred (X+C), X".
-Instruction *InstCombiner::foldICmpAddOpConst(Value *X, const APInt &C,
-                                              ICmpInst::Predicate Pred) {
+Instruction *InstCombinerImpl::foldICmpAddOpConst(Value *X, const APInt &C,
+                                                  ICmpInst::Predicate Pred) {
   // From this point on, we know that (X+C <= X) --> (X+C < X) because C != 0,
   // so the values can never be equal.  Similarly for all other "or equals"
   // operators.
@@ -1149,9 +1150,9 @@ Instruction *InstCombiner::foldICmpAddOpConst(Value *X, const APInt &C,
 /// Handle "(icmp eq/ne (ashr/lshr AP2, A), AP1)" ->
 /// (icmp eq/ne A, Log2(AP2/AP1)) ->
 /// (icmp eq/ne A, Log2(AP2) - Log2(AP1)).
-Instruction *InstCombiner::foldICmpShrConstConst(ICmpInst &I, Value *A,
-                                                 const APInt &AP1,
-                                                 const APInt &AP2) {
+Instruction *InstCombinerImpl::foldICmpShrConstConst(ICmpInst &I, Value *A,
+                                                     const APInt &AP1,
+                                                     const APInt &AP2) {
   assert(I.isEquality() && "Cannot fold icmp gt/lt");
 
   auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) {
@@ -1208,9 +1209,9 @@ Instruction *InstCombiner::foldICmpShrConstConst(ICmpInst &I, Value *A,
 
 /// Handle "(icmp eq/ne (shl AP2, A), AP1)" ->
 /// (icmp eq/ne A, TrailingZeros(AP1) - TrailingZeros(AP2)).
-Instruction *InstCombiner::foldICmpShlConstConst(ICmpInst &I, Value *A,
-                                                 const APInt &AP1,
-                                                 const APInt &AP2) {
+Instruction *InstCombinerImpl::foldICmpShlConstConst(ICmpInst &I, Value *A,
+                                                     const APInt &AP1,
+                                                     const APInt &AP2) {
   assert(I.isEquality() && "Cannot fold icmp gt/lt");
 
   auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) {
@@ -1254,7 +1255,7 @@ Instruction *InstCombiner::foldICmpShlConstConst(ICmpInst &I, Value *A,
 ///
 static Instruction *processUGT_ADDCST_ADD(ICmpInst &I, Value *A, Value *B,
                                           ConstantInt *CI2, ConstantInt *CI1,
-                                          InstCombiner &IC) {
+                                          InstCombinerImpl &IC) {
   // The transformation we're trying to do here is to transform this into an
   // llvm.sadd.with.overflow.  To do this, we have to replace the original add
   // with a narrower add, and discard the add-with-constant that is part of the
@@ -1340,7 +1341,7 @@ static Instruction *processUGT_ADDCST_ADD(ICmpInst &I, Value *A, Value *B,
 ///   icmp eq/ne (urem/srem %x, %y), 0
 /// iff %y is a power-of-two, we can replace this with a bit test:
 ///   icmp eq/ne (and %x, (add %y, -1)), 0
-Instruction *InstCombiner::foldIRemByPowerOfTwoToBitTest(ICmpInst &I) {
+Instruction *InstCombinerImpl::foldIRemByPowerOfTwoToBitTest(ICmpInst &I) {
   // This fold is only valid for equality predicates.
   if (!I.isEquality())
     return nullptr;
@@ -1359,7 +1360,7 @@ Instruction *InstCombiner::foldIRemByPowerOfTwoToBitTest(ICmpInst &I) {
 
 /// Fold equality-comparison between zero and any (maybe truncated) right-shift
 /// by one-less-than-bitwidth into a sign test on the original value.
-Instruction *InstCombiner::foldSignBitTest(ICmpInst &I) {
+Instruction *InstCombinerImpl::foldSignBitTest(ICmpInst &I) {
   Instruction *Val;
   ICmpInst::Predicate Pred;
   if (!I.isEquality() || !match(&I, m_ICmp(Pred, m_Instruction(Val), m_Zero())))
@@ -1390,7 +1391,7 @@ Instruction *InstCombiner::foldSignBitTest(ICmpInst &I) {
 }
 
 // Handle  icmp pred X, 0
-Instruction *InstCombiner::foldICmpWithZero(ICmpInst &Cmp) {
+Instruction *InstCombinerImpl::foldICmpWithZero(ICmpInst &Cmp) {
   CmpInst::Predicate Pred = Cmp.getPredicate();
   if (!match(Cmp.getOperand(1), m_Zero()))
     return nullptr;
@@ -1431,7 +1432,7 @@ Instruction *InstCombiner::foldICmpWithZero(ICmpInst &Cmp) {
 /// should be moved to some other helper and extended as noted below (it is also
 /// possible that code has been made unnecessary - do we canonicalize IR to
 /// overflow/saturating intrinsics or not?).
-Instruction *InstCombiner::foldICmpWithConstant(ICmpInst &Cmp) {
+Instruction *InstCombinerImpl::foldICmpWithConstant(ICmpInst &Cmp) {
   // Match the following pattern, which is a common idiom when writing
   // overflow-safe integer arithmetic functions. The source performs an addition
   // in wider type and explicitly checks for overflow using comparisons against
@@ -1477,7 +1478,7 @@ Instruction *InstCombiner::foldICmpWithConstant(ICmpInst &Cmp) {
 }
 
 /// Canonicalize icmp instructions based on dominating conditions.
-Instruction *InstCombiner::foldICmpWithDominatingICmp(ICmpInst &Cmp) {
+Instruction *InstCombinerImpl::foldICmpWithDominatingICmp(ICmpInst &Cmp) {
   // This is a cheap/incomplete check for dominance - just match a single
   // predecessor with a conditional branch.
   BasicBlock *CmpBB = Cmp.getParent();
@@ -1547,9 +1548,9 @@ Instruction *InstCombiner::foldICmpWithDominatingICmp(ICmpInst &Cmp) {
 }
 
 /// Fold icmp (trunc X, Y), C.
-Instruction *InstCombiner::foldICmpTruncConstant(ICmpInst &Cmp,
-                                                 TruncInst *Trunc,
-                                                 const APInt &C) {
+Instruction *InstCombinerImpl::foldICmpTruncConstant(ICmpInst &Cmp,
+                                                     TruncInst *Trunc,
+                                                     const APInt &C) {
   ICmpInst::Predicate Pred = Cmp.getPredicate();
   Value *X = Trunc->getOperand(0);
   if (C.isOneValue() && C.getBitWidth() > 1) {
@@ -1580,9 +1581,9 @@ Instruction *InstCombiner::foldICmpTruncConstant(ICmpInst &Cmp,
 }
 
 /// Fold icmp (xor X, Y), C.
-Instruction *InstCombiner::foldICmpXorConstant(ICmpInst &Cmp,
-                                               BinaryOperator *Xor,
-                                               const APInt &C) {
+Instruction *InstCombinerImpl::foldICmpXorConstant(ICmpInst &Cmp,
+                                                   BinaryOperator *Xor,
+                                                   const APInt &C) {
   Value *X = Xor->getOperand(0);
   Value *Y = Xor->getOperand(1);
   const APInt *XorC;
@@ -1649,8 +1650,10 @@ Instruction *InstCombiner::foldICmpXorConstant(ICmpInst &Cmp,
 }
 
 /// Fold icmp (and (sh X, Y), C2), C1.
-Instruction *InstCombiner::foldICmpAndShift(ICmpInst &Cmp, BinaryOperator *And,
-                                            const APInt &C1, const APInt &C2) {
+Instruction *InstCombinerImpl::foldICmpAndShift(ICmpInst &Cmp,
+                                                BinaryOperator *And,
+                                                const APInt &C1,
+                                                const APInt &C2) {
   BinaryOperator *Shift = dyn_cast<BinaryOperator>(And->getOperand(0));
   if (!Shift || !Shift->isShift())
     return nullptr;
@@ -1733,9 +1736,9 @@ Instruction *InstCombiner::foldICmpAndShift(ICmpInst &Cmp, BinaryOperator *And,
 }
 
 /// Fold icmp (and X, C2), C1.
-Instruction *InstCombiner::foldICmpAndConstConst(ICmpInst &Cmp,
-                                                 BinaryOperator *And,
-                                                 const APInt &C1) {
+Instruction *InstCombinerImpl::foldICmpAndConstConst(ICmpInst &Cmp,
+                                                     BinaryOperator *And,
+                                                     const APInt &C1) {
   bool isICMP_NE = Cmp.getPredicate() == ICmpInst::ICMP_NE;
 
   // For vectors: icmp ne (and X, 1), 0 --> trunc X to N x i1
@@ -1841,9 +1844,9 @@ Instruction *InstCombiner::foldICmpAndConstConst(ICmpInst &Cmp,
 }
 
 /// Fold icmp (and X, Y), C.
-Instruction *InstCombiner::foldICmpAndConstant(ICmpInst &Cmp,
-                                               BinaryOperator *And,
-                                               const APInt &C) {
+Instruction *InstCombinerImpl::foldICmpAndConstant(ICmpInst &Cmp,
+                                                   BinaryOperator *And,
+                                                   const APInt &C) {
   if (Instruction *I = foldICmpAndConstConst(Cmp, And, C))
     return I;
 
@@ -1895,8 +1898,9 @@ Instruction *InstCombiner::foldICmpAndConstant(ICmpInst &Cmp,
 }
 
 /// Fold icmp (or X, Y), C.
-Instruction *InstCombiner::foldICmpOrConstant(ICmpInst &Cmp, BinaryOperator *Or,
-                                              const APInt &C) {
+Instruction *InstCombinerImpl::foldICmpOrConstant(ICmpInst &Cmp,
+                                                  BinaryOperator *Or,
+                                                  const APInt &C) {
   ICmpInst::Predicate Pred = Cmp.getPredicate();
   if (C.isOneValue()) {
     // icmp slt signum(V) 1 --> icmp slt V, 1
@@ -1960,9 +1964,9 @@ Instruction *InstCombiner::foldICmpOrConstant(ICmpInst &Cmp, BinaryOperator *Or,
 }
 
 /// Fold icmp (mul X, Y), C.
-Instruction *InstCombiner::foldICmpMulConstant(ICmpInst &Cmp,
-                                               BinaryOperator *Mul,
-                                               const APInt &C) {
+Instruction *InstCombinerImpl::foldICmpMulConstant(ICmpInst &Cmp,
+                                                   BinaryOperator *Mul,
+                                                   const APInt &C) {
   const APInt *MulC;
   if (!match(Mul->getOperand(1), m_APInt(MulC)))
     return nullptr;
@@ -2043,9 +2047,9 @@ static Instruction *foldICmpShlOne(ICmpInst &Cmp, Instruction *Shl,
 }
 
 /// Fold icmp (shl X, Y), C.
-Instruction *InstCombiner::foldICmpShlConstant(ICmpInst &Cmp,
-                                               BinaryOperator *Shl,
-                                               const APInt &C) {
+Instruction *InstCombinerImpl::foldICmpShlConstant(ICmpInst &Cmp,
+                                                   BinaryOperator *Shl,
+                                                   const APInt &C) {
   const APInt *ShiftVal;
   if (Cmp.isEquality() && match(Shl->getOperand(0), m_APInt(ShiftVal)))
     return foldICmpShlConstConst(Cmp, Shl->getOperand(1), C, *ShiftVal);
@@ -2183,9 +2187,9 @@ Instruction *InstCombiner::foldICmpShlConstant(ICmpInst &Cmp,
 }
 
 /// Fold icmp ({al}shr X, Y), C.
-Instruction *InstCombiner::foldICmpShrConstant(ICmpInst &Cmp,
-                                               BinaryOperator *Shr,
-                                               const APInt &C) {
+Instruction *InstCombinerImpl::foldICmpShrConstant(ICmpInst &Cmp,
+                                                   BinaryOperator *Shr,
+                                                   const APInt &C) {
   // An exact shr only shifts out zero bits, so:
   // icmp eq/ne (shr X, Y), 0 --> icmp eq/ne X, 0
   Value *X = Shr->getOperand(0);
@@ -2276,9 +2280,9 @@ Instruction *InstCombiner::foldICmpShrConstant(ICmpInst &Cmp,
   return nullptr;
 }
 
-Instruction *InstCombiner::foldICmpSRemConstant(ICmpInst &Cmp,
-                                                BinaryOperator *SRem,
-                                                const APInt &C) {
+Instruction *InstCombinerImpl::foldICmpSRemConstant(ICmpInst &Cmp,
+                                                    BinaryOperator *SRem,
+                                                    const APInt &C) {
   // Match an 'is positive' or 'is negative' comparison of remainder by a
   // constant power-of-2 value:
   // (X % pow2C) sgt/slt 0
@@ -2315,9 +2319,9 @@ Instruction *InstCombiner::foldICmpSRemConstant(ICmpInst &Cmp,
 }
 
 /// Fold icmp (udiv X, Y), C.
-Instruction *InstCombiner::foldICmpUDivConstant(ICmpInst &Cmp,
-                                                BinaryOperator *UDiv,
-                                                const APInt &C) {
+Instruction *InstCombinerImpl::foldICmpUDivConstant(ICmpInst &Cmp,
+                                                    BinaryOperator *UDiv,
+                                                    const APInt &C) {
   const APInt *C2;
   if (!match(UDiv->getOperand(0), m_APInt(C2)))
     return nullptr;
@@ -2344,9 +2348,9 @@ Instruction *InstCombiner::foldICmpUDivConstant(ICmpInst &Cmp,
 }
 
 /// Fold icmp ({su}div X, Y), C.
-Instruction *InstCombiner::foldICmpDivConstant(ICmpInst &Cmp,
-                                               BinaryOperator *Div,
-                                               const APInt &C) {
+Instruction *InstCombinerImpl::foldICmpDivConstant(ICmpInst &Cmp,
+                                                   BinaryOperator *Div,
+                                                   const APInt &C) {
   // Fold: icmp pred ([us]div X, C2), C -> range test
   // Fold this div into the comparison, producing a range check.
   // Determine, based on the divide type, what the range is being
@@ -2514,9 +2518,9 @@ Instruction *InstCombiner::foldICmpDivConstant(ICmpInst &Cmp,
 }
 
 /// Fold icmp (sub X, Y), C.
-Instruction *InstCombiner::foldICmpSubConstant(ICmpInst &Cmp,
-                                               BinaryOperator *Sub,
-                                               const APInt &C) {
+Instruction *InstCombinerImpl::foldICmpSubConstant(ICmpInst &Cmp,
+                                                   BinaryOperator *Sub,
+                                                   const APInt &C) {
   Value *X = Sub->getOperand(0), *Y = Sub->getOperand(1);
   ICmpInst::Predicate Pred = Cmp.getPredicate();
   const APInt *C2;
@@ -2576,9 +2580,9 @@ Instruction *InstCombiner::foldICmpSubConstant(ICmpInst &Cmp,
 }
 
 /// Fold icmp (add X, Y), C.
-Instruction *InstCombiner::foldICmpAddConstant(ICmpInst &Cmp,
-                                               BinaryOperator *Add,
-                                               const APInt &C) {
+Instruction *InstCombinerImpl::foldICmpAddConstant(ICmpInst &Cmp,
+                                                   BinaryOperator *Add,
+                                                   const APInt &C) {
   Value *Y = Add->getOperand(1);
   const APInt *C2;
   if (Cmp.isEquality() || !match(Y, m_APInt(C2)))
@@ -2642,10 +2646,10 @@ Instruction *InstCombiner::foldICmpAddConstant(ICmpInst &Cmp,
   return nullptr;
 }
 
-bool InstCombiner::matchThreeWayIntCompare(SelectInst *SI, Value *&LHS,
-                                           Value *&RHS, ConstantInt *&Less,
-                                           ConstantInt *&Equal,
-                                           ConstantInt *&Greater) {
+bool InstCombinerImpl::matchThreeWayIntCompare(SelectInst *SI, Value *&LHS,
+                                               Value *&RHS, ConstantInt *&Less,
+                                               ConstantInt *&Equal,
+                                               ConstantInt *&Greater) {
   // TODO: Generalize this to work with other comparison idioms or ensure
   // they get canonicalized into this form.
 
@@ -2682,7 +2686,8 @@ bool InstCombiner::matchThreeWayIntCompare(SelectInst *SI, Value *&LHS,
   if (PredB == ICmpInst::ICMP_SGT && isa<Constant>(RHS2)) {
     // x sgt C-1  <-->  x sge C  <-->  not(x slt C)
     auto FlippedStrictness =
-        getFlippedStrictnessPredicateAndConstant(PredB, cast<Constant>(RHS2));
+        InstCombiner::getFlippedStrictnessPredicateAndConstant(
+            PredB, cast<Constant>(RHS2));
     if (!FlippedStrictness)
       return false;
     assert(FlippedStrictness->first == ICmpInst::ICMP_SGE && "Sanity check");
@@ -2694,9 +2699,9 @@ bool InstCombiner::matchThreeWayIntCompare(SelectInst *SI, Value *&LHS,
   return PredB == ICmpInst::ICMP_SLT && RHS == RHS2;
 }
 
-Instruction *InstCombiner::foldICmpSelectConstant(ICmpInst &Cmp,
-                                                  SelectInst *Select,
-                                                  ConstantInt *C) {
+Instruction *InstCombinerImpl::foldICmpSelectConstant(ICmpInst &Cmp,
+                                                      SelectInst *Select,
+                                                      ConstantInt *C) {
 
   assert(C && "Cmp RHS should be a constant int!");
   // If we're testing a constant value against the result of a three way
@@ -2794,7 +2799,7 @@ static Instruction *foldICmpBitCast(ICmpInst &Cmp,
     const APInt *C;
     bool TrueIfSigned;
     if (match(Op1, m_APInt(C)) && Bitcast->hasOneUse() &&
-        isSignBitCheck(Pred, *C, TrueIfSigned)) {
+        InstCombiner::isSignBitCheck(Pred, *C, TrueIfSigned)) {
       if (match(BCSrcOp, m_FPExt(m_Value(X))) ||
           match(BCSrcOp, m_FPTrunc(m_Value(X)))) {
         // (bitcast (fpext/fptrunc X)) to iX) < 0 --> (bitcast X to iY) < 0
@@ -2870,7 +2875,7 @@ static Instruction *foldICmpBitCast(ICmpInst &Cmp,
 
 /// Try to fold integer comparisons with a constant operand: icmp Pred X, C
 /// where X is some kind of instruction.
-Instruction *InstCombiner::foldICmpInstWithConstant(ICmpInst &Cmp) {
+Instruction *InstCombinerImpl::foldICmpInstWithConstant(ICmpInst &Cmp) {
   const APInt *C;
   if (!match(Cmp.getOperand(1), m_APInt(C)))
     return nullptr;
@@ -2955,9 +2960,8 @@ Instruction *InstCombiner::foldICmpInstWithConstant(ICmpInst &Cmp) {
 
 /// Fold an icmp equality instruction with binary operator LHS and constant RHS:
 /// icmp eq/ne BO, C.
-Instruction *InstCombiner::foldICmpBinOpEqualityWithConstant(ICmpInst &Cmp,
-                                                             BinaryOperator *BO,
-                                                             const APInt &C) {
+Instruction *InstCombinerImpl::foldICmpBinOpEqualityWithConstant(
+    ICmpInst &Cmp, BinaryOperator *BO, const APInt &C) {
   // TODO: Some of these folds could work with arbitrary constants, but this
   // function is limited to scalar and vector splat constants.
   if (!Cmp.isEquality())
@@ -3072,9 +3076,8 @@ Instruction *InstCombiner::foldICmpBinOpEqualityWithConstant(ICmpInst &Cmp,
 }
 
 /// Fold an equality icmp with LLVM intrinsic and constant operand.
-Instruction *InstCombiner::foldICmpEqIntrinsicWithConstant(ICmpInst &Cmp,
-                                                           IntrinsicInst *II,
-                                                           const APInt &C) {
+Instruction *InstCombinerImpl::foldICmpEqIntrinsicWithConstant(
+    ICmpInst &Cmp, IntrinsicInst *II, const APInt &C) {
   Type *Ty = II->getType();
   unsigned BitWidth = C.getBitWidth();
   switch (II->getIntrinsicID()) {
@@ -3145,9 +3148,9 @@ Instruction *InstCombiner::foldICmpEqIntrinsicWithConstant(ICmpInst &Cmp,
 }
 
 /// Fold an icmp with LLVM intrinsic and constant operand: icmp Pred II, C.
-Instruction *InstCombiner::foldICmpIntrinsicWithConstant(ICmpInst &Cmp,
-                                                         IntrinsicInst *II,
-                                                         const APInt &C) {
+Instruction *InstCombinerImpl::foldICmpIntrinsicWithConstant(ICmpInst &Cmp,
+                                                             IntrinsicInst *II,
+                                                             const APInt &C) {
   if (Cmp.isEquality())
     return foldICmpEqIntrinsicWithConstant(Cmp, II, C);
 
@@ -3204,7 +3207,7 @@ Instruction *InstCombiner::foldICmpIntrinsicWithConstant(ICmpInst &Cmp,
 }
 
 /// Handle icmp with constant (but not simple integer constant) RHS.
-Instruction *InstCombiner::foldICmpInstWithConstantNotInt(ICmpInst &I) {
+Instruction *InstCombinerImpl::foldICmpInstWithConstantNotInt(ICmpInst &I) {
   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
   Constant *RHSC = dyn_cast<Constant>(Op1);
   Instruction *LHSI = dyn_cast<Instruction>(Op0);
@@ -3650,7 +3653,7 @@ foldShiftIntoShiftInAnotherHandOfAndInICmp(ICmpInst &I, const SimplifyQuery SQ,
 ///   @llvm.umul.with.overflow(x, y) plus extraction of overflow bit
 /// Note that the comparison is commutative, while inverted (u>=, ==) predicate
 /// will mean that we are looking for the opposite answer.
-Value *InstCombiner::foldUnsignedMultiplicationOverflowCheck(ICmpInst &I) {
+Value *InstCombinerImpl::foldUnsignedMultiplicationOverflowCheck(ICmpInst &I) {
   ICmpInst::Predicate Pred;
   Value *X, *Y;
   Instruction *Mul;
@@ -3716,7 +3719,8 @@ Value *InstCombiner::foldUnsignedMultiplicationOverflowCheck(ICmpInst &I) {
 /// TODO: A large part of this logic is duplicated in InstSimplify's
 /// simplifyICmpWithBinOp(). We should be able to share that and avoid the code
 /// duplication.
-Instruction *InstCombiner::foldICmpBinOp(ICmpInst &I, const SimplifyQuery &SQ) {
+Instruction *InstCombinerImpl::foldICmpBinOp(ICmpInst &I,
+                                             const SimplifyQuery &SQ) {
   const SimplifyQuery Q = SQ.getWithInstruction(&I);
   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
 
@@ -4170,7 +4174,7 @@ static Instruction *foldICmpWithMinMax(ICmpInst &Cmp) {
   return nullptr;
 }
 
-Instruction *InstCombiner::foldICmpEquality(ICmpInst &I) {
+Instruction *InstCombinerImpl::foldICmpEquality(ICmpInst &I) {
   if (!I.isEquality())
     return nullptr;
 
@@ -4438,7 +4442,7 @@ static Instruction *foldICmpWithZextOrSext(ICmpInst &ICmp,
 }
 
 /// Handle icmp (cast x), (cast or constant).
-Instruction *InstCombiner::foldICmpWithCastOp(ICmpInst &ICmp) {
+Instruction *InstCombinerImpl::foldICmpWithCastOp(ICmpInst &ICmp) {
   auto *CastOp0 = dyn_cast<CastInst>(ICmp.getOperand(0));
   if (!CastOp0)
     return nullptr;
@@ -4493,9 +4497,10 @@ static bool isNeutralValue(Instruction::BinaryOps BinaryOp, Value *RHS) {
   }
 }
 
-OverflowResult InstCombiner::computeOverflow(
-    Instruction::BinaryOps BinaryOp, bool IsSigned,
-    Value *LHS, Value *RHS, Instruction *CxtI) const {
+OverflowResult
+InstCombinerImpl::computeOverflow(Instruction::BinaryOps BinaryOp,
+                                  bool IsSigned, Value *LHS, Value *RHS,
+                                  Instruction *CxtI) const {
   switch (BinaryOp) {
     default:
       llvm_unreachable("Unsupported binary op");
@@ -4517,9 +4522,11 @@ OverflowResult InstCombiner::computeOverflow(
   }
 }
 
-bool InstCombiner::OptimizeOverflowCheck(
-    Instruction::BinaryOps BinaryOp, bool IsSigned, Value *LHS, Value *RHS,
-    Instruction &OrigI, Value *&Result, Constant *&Overflow) {
+bool InstCombinerImpl::OptimizeOverflowCheck(Instruction::BinaryOps BinaryOp,
+                                             bool IsSigned, Value *LHS,
+                                             Value *RHS, Instruction &OrigI,
+                                             Value *&Result,
+                                             Constant *&Overflow) {
   if (OrigI.isCommutative() && isa<Constant>(LHS) && !isa<Constant>(RHS))
     std::swap(LHS, RHS);
 
@@ -4575,7 +4582,8 @@ bool InstCombiner::OptimizeOverflowCheck(
 /// \returns Instruction which must replace the compare instruction, NULL if no
 ///          replacement required.
 static Instruction *processUMulZExtIdiom(ICmpInst &I, Value *MulVal,
-                                         Value *OtherVal, InstCombiner &IC) {
+                                         Value *OtherVal,
+                                         InstCombinerImpl &IC) {
   // Don't bother doing this transformation for pointers, don't do it for
   // vectors.
   if (!isa<IntegerType>(MulVal->getType()))
@@ -4723,7 +4731,7 @@ static Instruction *processUMulZExtIdiom(ICmpInst &I, Value *MulVal,
   Function *F = Intrinsic::getDeclaration(
       I.getModule(), Intrinsic::umul_with_overflow, MulType);
   CallInst *Call = Builder.CreateCall(F, {MulA, MulB}, "umul");
-  IC.Worklist.push(MulInstr);
+  IC.addToWorklist(MulInstr);
 
   // If there are uses of mul result other than the comparison, we know that
   // they are truncation or binary AND. Change them to use result of
@@ -4750,11 +4758,11 @@ static Instruction *processUMulZExtIdiom(ICmpInst &I, Value *MulVal,
       } else {
         llvm_unreachable("Unexpected Binary operation");
       }
-      IC.Worklist.push(cast<Instruction>(U));
+      IC.addToWorklist(cast<Instruction>(U));
     }
   }
   if (isa<Instruction>(OtherVal))
-    IC.Worklist.push(cast<Instruction>(OtherVal));
+    IC.addToWorklist(cast<Instruction>(OtherVal));
 
   // The original icmp gets replaced with the overflow value, maybe inverted
   // depending on predicate.
@@ -4799,7 +4807,7 @@ static APInt getDemandedBitsLHSMask(ICmpInst &I, unsigned BitWidth) {
   // If this is a normal comparison, it demands all bits. If it is a sign bit
   // comparison, it only demands the sign bit.
   bool UnusedBit;
-  if (isSignBitCheck(I.getPredicate(), *RHS, UnusedBit))
+  if (InstCombiner::isSignBitCheck(I.getPredicate(), *RHS, UnusedBit))
     return APInt::getSignMask(BitWidth);
 
   switch (I.getPredicate()) {
@@ -4856,9 +4864,9 @@ static bool swapMayExposeCSEOpportunities(const Value *Op0, const Value *Op1) {
 /// \return true when \p UI is the only use of \p DI in the parent block
 /// and all other uses of \p DI are in blocks dominated by \p DB.
 ///
-bool InstCombiner::dominatesAllUses(const Instruction *DI,
-                                    const Instruction *UI,
-                                    const BasicBlock *DB) const {
+bool InstCombinerImpl::dominatesAllUses(const Instruction *DI,
+                                        const Instruction *UI,
+                                        const BasicBlock *DB) const {
   assert(DI && UI && "Instruction not defined\n");
   // Ignore incomplete definitions.
   if (!DI->getParent())
@@ -4931,9 +4939,9 @@ static bool isChainSelectCmpBranch(const SelectInst *SI) {
 /// major restriction since a NE compare should be 'normalized' to an equal
 /// compare, which usually happens in the combiner and test case
 /// select-cmp-br.ll checks for it.
-bool InstCombiner::replacedSelectWithOperand(SelectInst *SI,
-                                             const ICmpInst *Icmp,
-                                             const unsigned SIOpd) {
+bool InstCombinerImpl::replacedSelectWithOperand(SelectInst *SI,
+                                                 const ICmpInst *Icmp,
+                                                 const unsigned SIOpd) {
   assert((SIOpd == 1 || SIOpd == 2) && "Invalid select operand!");
   if (isChainSelectCmpBranch(SI) && Icmp->getPredicate() == ICmpInst::ICMP_EQ) {
     BasicBlock *Succ = SI->getParent()->getTerminator()->getSuccessor(1);
@@ -4959,7 +4967,7 @@ bool InstCombiner::replacedSelectWithOperand(SelectInst *SI,
 
 /// Try to fold the comparison based on range information we can get by checking
 /// whether bits are known to be zero or one in the inputs.
-Instruction *InstCombiner::foldICmpUsingKnownBits(ICmpInst &I) {
+Instruction *InstCombinerImpl::foldICmpUsingKnownBits(ICmpInst &I) {
   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
   Type *Ty = Op0->getType();
   ICmpInst::Predicate Pred = I.getPredicate();
@@ -5186,8 +5194,8 @@ Instruction *InstCombiner::foldICmpUsingKnownBits(ICmpInst &I) {
 }
 
 llvm::Optional<std::pair<CmpInst::Predicate, Constant *>>
-llvm::getFlippedStrictnessPredicateAndConstant(CmpInst::Predicate Pred,
-                                               Constant *C) {
+InstCombiner::getFlippedStrictnessPredicateAndConstant(CmpInst::Predicate Pred,
+                                                       Constant *C) {
   assert(ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate(Pred) &&
          "Only for relational integer predicates.");
 
@@ -5256,7 +5264,7 @@ llvm::getFlippedStrictnessPredicateAndConstant(CmpInst::Predicate Pred,
 static ICmpInst *canonicalizeCmpWithConstant(ICmpInst &I) {
   ICmpInst::Predicate Pred = I.getPredicate();
   if (ICmpInst::isEquality(Pred) || !ICmpInst::isIntPredicate(Pred) ||
-      isCanonicalPredicate(Pred))
+      InstCombiner::isCanonicalPredicate(Pred))
     return nullptr;
 
   Value *Op0 = I.getOperand(0);
@@ -5265,7 +5273,8 @@ static ICmpInst *canonicalizeCmpWithConstant(ICmpInst &I) {
   if (!Op1C)
     return nullptr;
 
-  auto FlippedStrictness = getFlippedStrictnessPredicateAndConstant(Pred, Op1C);
+  auto FlippedStrictness =
+      InstCombiner::getFlippedStrictnessPredicateAndConstant(Pred, Op1C);
   if (!FlippedStrictness)
     return nullptr;
 
@@ -5277,11 +5286,11 @@ static ICmpInst *canonicalizeCmpWithConstant(ICmpInst &I) {
 static CmpInst *canonicalizeICmpPredicate(CmpInst &I) {
   // Is the predicate already canonical?
   CmpInst::Predicate Pred = I.getPredicate();
-  if (isCanonicalPredicate(Pred))
+  if (InstCombiner::isCanonicalPredicate(Pred))
     return nullptr;
 
   // Can all users be adjusted to predicate inversion?
-  if (!canFreelyInvertAllUsersOf(&I, /*IgnoredUser=*/nullptr))
+  if (!InstCombiner::canFreelyInvertAllUsersOf(&I, /*IgnoredUser=*/nullptr))
     return nullptr;
 
   // Ok, we can canonicalize comparison!
@@ -5510,7 +5519,7 @@ static Instruction *foldICmpOfUAddOv(ICmpInst &I) {
   return ExtractValueInst::Create(UAddOv, 1);
 }
 
-Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
+Instruction *InstCombinerImpl::visitICmpInst(ICmpInst &I) {
   bool Changed = false;
   const SimplifyQuery Q = SQ.getWithInstruction(&I);
   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
@@ -5748,8 +5757,9 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
 }
 
 /// Fold fcmp ([us]itofp x, cst) if possible.
-Instruction *InstCombiner::foldFCmpIntToFPConst(FCmpInst &I, Instruction *LHSI,
-                                                Constant *RHSC) {
+Instruction *InstCombinerImpl::foldFCmpIntToFPConst(FCmpInst &I,
+                                                    Instruction *LHSI,
+                                                    Constant *RHSC) {
   if (!isa<ConstantFP>(RHSC)) return nullptr;
   const APFloat &RHS = cast<ConstantFP>(RHSC)->getValueAPF();
 
@@ -6034,7 +6044,7 @@ static Instruction *foldFCmpReciprocalAndZero(FCmpInst &I, Instruction *LHSI,
 }
 
 /// Optimize fabs(X) compared with zero.
-static Instruction *foldFabsWithFcmpZero(FCmpInst &I, InstCombiner &IC) {
+static Instruction *foldFabsWithFcmpZero(FCmpInst &I, InstCombinerImpl &IC) {
   Value *X;
   if (!match(I.getOperand(0), m_Intrinsic<Intrinsic::fabs>(m_Value(X))) ||
       !match(I.getOperand(1), m_PosZeroFP()))
@@ -6096,7 +6106,7 @@ static Instruction *foldFabsWithFcmpZero(FCmpInst &I, InstCombiner &IC) {
   }
 }
 
-Instruction *InstCombiner::visitFCmpInst(FCmpInst &I) {
+Instruction *InstCombinerImpl::visitFCmpInst(FCmpInst &I) {
   bool Changed = false;
 
   /// Orders the operands of the compare so that they are listed from most
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index ca51f37af4d93..b67ceb75eb722 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -15,40 +15,32 @@
 #ifndef LLVM_LIB_TRANSFORMS_INSTCOMBINE_INSTCOMBINEINTERNAL_H
 #define LLVM_LIB_TRANSFORMS_INSTCOMBINE_INSTCOMBINEINTERNAL_H
 
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/TargetFolder.h"
 #include "llvm/Analysis/ValueTracking.h"
-#include "llvm/IR/Argument.h"
-#include "llvm/IR/BasicBlock.h"
-#include "llvm/IR/Constant.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InstVisitor.h"
-#include "llvm/IR/InstrTypes.h"
-#include "llvm/IR/Instruction.h"
-#include "llvm/IR/IntrinsicInst.h"
-#include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/PatternMatch.h"
-#include "llvm/IR/Use.h"
-#include "llvm/IR/Value.h"
-#include "llvm/Support/Casting.h"
-#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/KnownBits.h"
-#include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/InstCombine/InstCombineWorklist.h"
+#include "llvm/Transforms/InstCombine/InstCombiner.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include <cassert>
-#include <cstdint>
 
 #define DEBUG_TYPE "instcombine"
 
 using namespace llvm::PatternMatch;
 
+// As a default, let's assume that we want to be aggressive,
+// and attempt to traverse with no limits in attempt to sink negation.
+static constexpr unsigned NegatorDefaultMaxDepth = ~0U;
+
+// Let's guesstimate that most often we will end up visiting/producing
+// fairly small number of new instructions.
+static constexpr unsigned NegatorMaxNodesSSO = 16;
+
 namespace llvm {
 
 class AAResults;
@@ -65,305 +57,26 @@ class ProfileSummaryInfo;
 class TargetLibraryInfo;
 class User;
 
-/// Assign a complexity or rank value to LLVM Values. This is used to reduce
-/// the amount of pattern matching needed for compares and commutative
-/// instructions. For example, if we have:
-///   icmp ugt X, Constant
-/// or
-///   xor (add X, Constant), cast Z
-///
-/// We do not have to consider the commuted variants of these patterns because
-/// canonicalization based on complexity guarantees the above ordering.
-///
-/// This routine maps IR values to various complexity ranks:
-///   0 -> undef
-///   1 -> Constants
-///   2 -> Other non-instructions
-///   3 -> Arguments
-///   4 -> Cast and (f)neg/not instructions
-///   5 -> Other instructions
-static inline unsigned getComplexity(Value *V) {
-  if (isa<Instruction>(V)) {
-    if (isa<CastInst>(V) || match(V, m_Neg(m_Value())) ||
-        match(V, m_Not(m_Value())) || match(V, m_FNeg(m_Value())))
-      return 4;
-    return 5;
-  }
-  if (isa<Argument>(V))
-    return 3;
-  return isa<Constant>(V) ? (isa<UndefValue>(V) ? 0 : 1) : 2;
-}
-
-/// Predicate canonicalization reduces the number of patterns that need to be
-/// matched by other transforms. For example, we may swap the operands of a
-/// conditional branch or select to create a compare with a canonical (inverted)
-/// predicate which is then more likely to be matched with other values.
-static inline bool isCanonicalPredicate(CmpInst::Predicate Pred) {
-  switch (Pred) {
-  case CmpInst::ICMP_NE:
-  case CmpInst::ICMP_ULE:
-  case CmpInst::ICMP_SLE:
-  case CmpInst::ICMP_UGE:
-  case CmpInst::ICMP_SGE:
-  // TODO: There are 16 FCMP predicates. Should others be (not) canonical?
-  case CmpInst::FCMP_ONE:
-  case CmpInst::FCMP_OLE:
-  case CmpInst::FCMP_OGE:
-    return false;
-  default:
-    return true;
-  }
-}
-
-/// Given an exploded icmp instruction, return true if the comparison only
-/// checks the sign bit. If it only checks the sign bit, set TrueIfSigned if the
-/// result of the comparison is true when the input value is signed.
-inline bool isSignBitCheck(ICmpInst::Predicate Pred, const APInt &RHS,
-                           bool &TrueIfSigned) {
-  switch (Pred) {
-  case ICmpInst::ICMP_SLT: // True if LHS s< 0
-    TrueIfSigned = true;
-    return RHS.isNullValue();
-  case ICmpInst::ICMP_SLE: // True if LHS s<= -1
-    TrueIfSigned = true;
-    return RHS.isAllOnesValue();
-  case ICmpInst::ICMP_SGT: // True if LHS s> -1
-    TrueIfSigned = false;
-    return RHS.isAllOnesValue();
-  case ICmpInst::ICMP_SGE: // True if LHS s>= 0
-    TrueIfSigned = false;
-    return RHS.isNullValue();
-  case ICmpInst::ICMP_UGT:
-    // True if LHS u> RHS and RHS == sign-bit-mask - 1
-    TrueIfSigned = true;
-    return RHS.isMaxSignedValue();
-  case ICmpInst::ICMP_UGE:
-    // True if LHS u>= RHS and RHS == sign-bit-mask (2^7, 2^15, 2^31, etc)
-    TrueIfSigned = true;
-    return RHS.isMinSignedValue();
-  case ICmpInst::ICMP_ULT:
-    // True if LHS u< RHS and RHS == sign-bit-mask (2^7, 2^15, 2^31, etc)
-    TrueIfSigned = false;
-    return RHS.isMinSignedValue();
-  case ICmpInst::ICMP_ULE:
-    // True if LHS u<= RHS and RHS == sign-bit-mask - 1
-    TrueIfSigned = false;
-    return RHS.isMaxSignedValue();
-  default:
-    return false;
-  }
-}
-
-llvm::Optional<std::pair<CmpInst::Predicate, Constant *>>
-getFlippedStrictnessPredicateAndConstant(CmpInst::Predicate Pred, Constant *C);
-
-/// Return the source operand of a potentially bitcasted value while optionally
-/// checking if it has one use. If there is no bitcast or the one use check is
-/// not met, return the input value itself.
-static inline Value *peekThroughBitcast(Value *V, bool OneUseOnly = false) {
-  if (auto *BitCast = dyn_cast<BitCastInst>(V))
-    if (!OneUseOnly || BitCast->hasOneUse())
-      return BitCast->getOperand(0);
-
-  // V is not a bitcast or V has more than one use and OneUseOnly is true.
-  return V;
-}
-
-/// Add one to a Constant
-static inline Constant *AddOne(Constant *C) {
-  return ConstantExpr::getAdd(C, ConstantInt::get(C->getType(), 1));
-}
-
-/// Subtract one from a Constant
-static inline Constant *SubOne(Constant *C) {
-  return ConstantExpr::getSub(C, ConstantInt::get(C->getType(), 1));
-}
-
-/// Return true if the specified value is free to invert (apply ~ to).
-/// This happens in cases where the ~ can be eliminated.  If WillInvertAllUses
-/// is true, work under the assumption that the caller intends to remove all
-/// uses of V and only keep uses of ~V.
-///
-/// See also: canFreelyInvertAllUsersOf()
-static inline bool isFreeToInvert(Value *V, bool WillInvertAllUses) {
-  // ~(~(X)) -> X.
-  if (match(V, m_Not(m_Value())))
-    return true;
-
-  // Constants can be considered to be not'ed values.
-  if (match(V, m_AnyIntegralConstant()))
-    return true;
-
-  // Compares can be inverted if all of their uses are being modified to use the
-  // ~V.
-  if (isa<CmpInst>(V))
-    return WillInvertAllUses;
-
-  // If `V` is of the form `A + Constant` then `-1 - V` can be folded into `(-1
-  // - Constant) - A` if we are willing to invert all of the uses.
-  if (BinaryOperator *BO = dyn_cast<BinaryOperator>(V))
-    if (BO->getOpcode() == Instruction::Add ||
-        BO->getOpcode() == Instruction::Sub)
-      if (isa<Constant>(BO->getOperand(0)) || isa<Constant>(BO->getOperand(1)))
-        return WillInvertAllUses;
-
-  // Selects with invertible operands are freely invertible
-  if (match(V, m_Select(m_Value(), m_Not(m_Value()), m_Not(m_Value()))))
-    return WillInvertAllUses;
-
-  return false;
-}
-
-/// Given i1 V, can every user of V be freely adapted if V is changed to !V ?
-/// InstCombine's canonicalizeICmpPredicate() must be kept in sync with this fn.
-///
-/// See also: isFreeToInvert()
-static inline bool canFreelyInvertAllUsersOf(Value *V, Value *IgnoredUser) {
-  // Look at every user of V.
-  for (Use &U : V->uses()) {
-    if (U.getUser() == IgnoredUser)
-      continue; // Don't consider this user.
-
-    auto *I = cast<Instruction>(U.getUser());
-    switch (I->getOpcode()) {
-    case Instruction::Select:
-      if (U.getOperandNo() != 0) // Only if the value is used as select cond.
-        return false;
-      break;
-    case Instruction::Br:
-      assert(U.getOperandNo() == 0 && "Must be branching on that value.");
-      break; // Free to invert by swapping true/false values/destinations.
-    case Instruction::Xor: // Can invert 'xor' if it's a 'not', by ignoring it.
-      if (!match(I, m_Not(m_Value())))
-        return false; // Not a 'not'.
-      break;
-    default:
-      return false; // Don't know, likely not freely invertible.
-    }
-    // So far all users were free to invert...
-  }
-  return true; // Can freely invert all users!
-}
-
-/// Some binary operators require special handling to avoid poison and undefined
-/// behavior. If a constant vector has undef elements, replace those undefs with
-/// identity constants if possible because those are always safe to execute.
-/// If no identity constant exists, replace undef with some other safe constant.
-static inline Constant *getSafeVectorConstantForBinop(
-      BinaryOperator::BinaryOps Opcode, Constant *In, bool IsRHSConstant) {
-  auto *InVTy = dyn_cast<VectorType>(In->getType());
-  assert(InVTy && "Not expecting scalars here");
-
-  Type *EltTy = InVTy->getElementType();
-  auto *SafeC = ConstantExpr::getBinOpIdentity(Opcode, EltTy, IsRHSConstant);
-  if (!SafeC) {
-    // TODO: Should this be available as a constant utility function? It is
-    // similar to getBinOpAbsorber().
-    if (IsRHSConstant) {
-      switch (Opcode) {
-      case Instruction::SRem: // X % 1 = 0
-      case Instruction::URem: // X %u 1 = 0
-        SafeC = ConstantInt::get(EltTy, 1);
-        break;
-      case Instruction::FRem: // X % 1.0 (doesn't simplify, but it is safe)
-        SafeC = ConstantFP::get(EltTy, 1.0);
-        break;
-      default:
-        llvm_unreachable("Only rem opcodes have no identity constant for RHS");
-      }
-    } else {
-      switch (Opcode) {
-      case Instruction::Shl:  // 0 << X = 0
-      case Instruction::LShr: // 0 >>u X = 0
-      case Instruction::AShr: // 0 >> X = 0
-      case Instruction::SDiv: // 0 / X = 0
-      case Instruction::UDiv: // 0 /u X = 0
-      case Instruction::SRem: // 0 % X = 0
-      case Instruction::URem: // 0 %u X = 0
-      case Instruction::Sub:  // 0 - X (doesn't simplify, but it is safe)
-      case Instruction::FSub: // 0.0 - X (doesn't simplify, but it is safe)
-      case Instruction::FDiv: // 0.0 / X (doesn't simplify, but it is safe)
-      case Instruction::FRem: // 0.0 % X = 0
-        SafeC = Constant::getNullValue(EltTy);
-        break;
-      default:
-        llvm_unreachable("Expected to find identity constant for opcode");
-      }
-    }
-  }
-  assert(SafeC && "Must have safe constant for binop");
-  unsigned NumElts = InVTy->getNumElements();
-  SmallVector<Constant *, 16> Out(NumElts);
-  for (unsigned i = 0; i != NumElts; ++i) {
-    Constant *C = In->getAggregateElement(i);
-    Out[i] = isa<UndefValue>(C) ? SafeC : C;
-  }
-  return ConstantVector::get(Out);
-}
-
-/// The core instruction combiner logic.
-///
-/// This class provides both the logic to recursively visit instructions and
-/// combine them.
-class LLVM_LIBRARY_VISIBILITY InstCombiner
-    : public InstVisitor<InstCombiner, Instruction *> {
-  // FIXME: These members shouldn't be public.
+class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
+    : public InstCombiner,
+      public InstVisitor<InstCombinerImpl, Instruction *> {
 public:
-  /// A worklist of the instructions that need to be simplified.
-  InstCombineWorklist &Worklist;
-
-  /// An IRBuilder that automatically inserts new instructions into the
-  /// worklist.
-  using BuilderTy = IRBuilder<TargetFolder, IRBuilderCallbackInserter>;
-  BuilderTy &Builder;
-
-private:
-  // Mode in which we are running the combiner.
-  const bool MinimizeSize;
+  InstCombinerImpl(InstCombineWorklist &Worklist, BuilderTy &Builder,
+                   bool MinimizeSize, AAResults *AA, AssumptionCache &AC,
+                   TargetLibraryInfo &TLI, TargetTransformInfo &TTI,
+                   DominatorTree &DT, OptimizationRemarkEmitter &ORE,
+                   BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
+                   const DataLayout &DL, LoopInfo *LI)
+      : InstCombiner(Worklist, Builder, MinimizeSize, AA, AC, TLI, TTI, DT, ORE,
+                     BFI, PSI, DL, LI) {}
 
-  AAResults *AA;
-
-  // Required analyses.
-  AssumptionCache &AC;
-  TargetLibraryInfo &TLI;
-  DominatorTree &DT;
-  const DataLayout &DL;
-  const SimplifyQuery SQ;
-  OptimizationRemarkEmitter &ORE;
-  BlockFrequencyInfo *BFI;
-  ProfileSummaryInfo *PSI;
-
-  // Optional analyses. When non-null, these can both be used to do better
-  // combining and will be updated to reflect any changes.
-  LoopInfo *LI;
-
-  bool MadeIRChange = false;
-
-public:
-  InstCombiner(InstCombineWorklist &Worklist, BuilderTy &Builder,
-               bool MinimizeSize, AAResults *AA,
-               AssumptionCache &AC, TargetLibraryInfo &TLI, DominatorTree &DT,
-               OptimizationRemarkEmitter &ORE, BlockFrequencyInfo *BFI,
-               ProfileSummaryInfo *PSI, const DataLayout &DL, LoopInfo *LI)
-      : Worklist(Worklist), Builder(Builder), MinimizeSize(MinimizeSize),
-        AA(AA), AC(AC), TLI(TLI), DT(DT),
-        DL(DL), SQ(DL, &TLI, &DT, &AC), ORE(ORE), BFI(BFI), PSI(PSI), LI(LI) {}
+  virtual ~InstCombinerImpl() {}
 
   /// Run the combiner over the entire worklist until it is empty.
   ///
   /// \returns true if the IR is changed.
   bool run();
 
-  AssumptionCache &getAssumptionCache() const { return AC; }
-
-  const DataLayout &getDataLayout() const { return DL; }
-
-  DominatorTree &getDominatorTree() const { return DT; }
-
-  LoopInfo *getLoopInfo() const { return LI; }
-
-  TargetLibraryInfo &getTargetLibraryInfo() const { return TLI; }
-
   // Visitation implementation - Implement instruction combining for different
   // instruction types.  The semantics are as follows:
   // Return Value:
@@ -726,7 +439,7 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner
   /// When dealing with an instruction that has side effects or produces a void
   /// value, we can't rely on DCE to delete the instruction. Instead, visit
   /// methods should return the value returned by this function.
-  Instruction *eraseInstFromFunction(Instruction &I) {
+  Instruction *eraseInstFromFunction(Instruction &I) override {
     LLVM_DEBUG(dbgs() << "IC: ERASE " << I << '\n');
     assert(I.use_empty() && "Cannot erase instruction that is used!");
     salvageDebugInfo(I);
@@ -808,10 +521,6 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner
       Instruction::BinaryOps BinaryOp, bool IsSigned,
       Value *LHS, Value *RHS, Instruction *CxtI) const;
 
-  /// Maximum size of array considered when transforming.
-  uint64_t MaxArraySizeForCombine = 0;
-
-private:
   /// Performs a few simplifications for operators which are associative
   /// or commutative.
   bool SimplifyAssociativeOrCommutative(BinaryOperator &I);
@@ -857,7 +566,7 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner
                                  unsigned Depth, Instruction *CxtI);
   bool SimplifyDemandedBits(Instruction *I, unsigned Op,
                             const APInt &DemandedMask, KnownBits &Known,
-                            unsigned Depth = 0);
+                            unsigned Depth = 0) override;
 
   /// Helper routine of SimplifyDemandedUseBits. It computes KnownZero/KnownOne
   /// bits. It also tries to handle simplifications that can be done based on
@@ -877,13 +586,10 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner
   /// demanded bits.
   bool SimplifyDemandedInstructionBits(Instruction &Inst);
 
-  Value *simplifyAMDGCNMemoryIntrinsicDemanded(IntrinsicInst *II,
-                                               APInt DemandedElts,
-                                               int DmaskIdx = -1);
-
-  Value *SimplifyDemandedVectorElts(Value *V, APInt DemandedElts,
-                                    APInt &UndefElts, unsigned Depth = 0,
-                                    bool AllowMultipleUsers = false);
+  virtual Value *
+  SimplifyDemandedVectorElts(Value *V, APInt DemandedElts, APInt &UndefElts,
+                             unsigned Depth = 0,
+                             bool AllowMultipleUsers = false) override;
 
   /// Canonicalize the position of binops relative to shufflevector.
   Instruction *foldVectorBinop(BinaryOperator &Inst);
@@ -1023,18 +729,6 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner
   Value *Descale(Value *Val, APInt Scale, bool &NoSignedWrap);
 };
 
-namespace {
-
-// As a default, let's assume that we want to be aggressive,
-// and attempt to traverse with no limits in attempt to sink negation.
-static constexpr unsigned NegatorDefaultMaxDepth = ~0U;
-
-// Let's guesstimate that most often we will end up visiting/producing
-// fairly small number of new instructions.
-static constexpr unsigned NegatorMaxNodesSSO = 16;
-
-} // namespace
-
 class Negator final {
   /// Top-to-bottom, def-to-use negated instruction tree we produced.
   SmallVector<Instruction *, NegatorMaxNodesSSO> NewInstructions;
@@ -1078,7 +772,7 @@ class Negator final {
   /// Attempt to negate \p Root. Retuns nullptr if negation can't be performed,
   /// otherwise returns negated value.
   LLVM_NODISCARD static Value *Negate(bool LHSIsZero, Value *Root,
-                                      InstCombiner &IC);
+                                      InstCombinerImpl &IC);
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
index dad2f23120bdb..d7c1b3c9c2b65 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
@@ -23,6 +23,7 @@
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/PatternMatch.h"
+#include "llvm/Transforms/InstCombine/InstCombiner.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/Local.h"
 using namespace llvm;
@@ -166,7 +167,8 @@ static bool isDereferenceableForAllocaSize(const Value *V, const AllocaInst *AI,
                                             APInt(64, AllocaSize), DL);
 }
 
-static Instruction *simplifyAllocaArraySize(InstCombiner &IC, AllocaInst &AI) {
+static Instruction *simplifyAllocaArraySize(InstCombinerImpl &IC,
+                                            AllocaInst &AI) {
   // Check for array size of 1 (scalar allocation).
   if (!AI.isArrayAllocation()) {
     // i32 1 is the canonical array size for scalar allocations.
@@ -234,7 +236,7 @@ namespace {
 // instruction.
 class PointerReplacer {
 public:
-  PointerReplacer(InstCombiner &IC) : IC(IC) {}
+  PointerReplacer(InstCombinerImpl &IC) : IC(IC) {}
   void replacePointer(Instruction &I, Value *V);
 
 private:
@@ -244,7 +246,7 @@ class PointerReplacer {
 
   SmallVector<Instruction *, 4> Path;
   MapVector<Value *, Value *> WorkMap;
-  InstCombiner &IC;
+  InstCombinerImpl &IC;
 };
 } // end anonymous namespace
 
@@ -323,7 +325,7 @@ void PointerReplacer::replacePointer(Instruction &I, Value *V) {
   findLoadAndReplace(I);
 }
 
-Instruction *InstCombiner::visitAllocaInst(AllocaInst &AI) {
+Instruction *InstCombinerImpl::visitAllocaInst(AllocaInst &AI) {
   if (auto *I = simplifyAllocaArraySize(*this, AI))
     return I;
 
@@ -421,9 +423,9 @@ static bool isSupportedAtomicType(Type *Ty) {
 /// that pointer type, load it, etc.
 ///
 /// Note that this will create all of the instructions with whatever insert
-/// point the \c InstCombiner currently is using.
-LoadInst *InstCombiner::combineLoadToNewType(LoadInst &LI, Type *NewTy,
-                                             const Twine &Suffix) {
+/// point the \c InstCombinerImpl currently is using.
+LoadInst *InstCombinerImpl::combineLoadToNewType(LoadInst &LI, Type *NewTy,
+                                                 const Twine &Suffix) {
   assert((!LI.isAtomic() || isSupportedAtomicType(NewTy)) &&
          "can't fold an atomic load to requested type");
 
@@ -445,7 +447,8 @@ LoadInst *InstCombiner::combineLoadToNewType(LoadInst &LI, Type *NewTy,
 /// Combine a store to a new type.
 ///
 /// Returns the newly created store instruction.
-static StoreInst *combineStoreToNewValue(InstCombiner &IC, StoreInst &SI, Value *V) {
+static StoreInst *combineStoreToNewValue(InstCombinerImpl &IC, StoreInst &SI,
+                                         Value *V) {
   assert((!SI.isAtomic() || isSupportedAtomicType(V->getType())) &&
          "can't fold an atomic store of requested type");
 
@@ -502,7 +505,7 @@ static StoreInst *combineStoreToNewValue(InstCombiner &IC, StoreInst &SI, Value
 static bool isMinMaxWithLoads(Value *V, Type *&LoadTy) {
   assert(V->getType()->isPointerTy() && "Expected pointer type.");
   // Ignore possible ty* to ixx* bitcast.
-  V = peekThroughBitcast(V);
+  V = InstCombiner::peekThroughBitcast(V);
   // Check that select is select ((cmp load V1, load V2), V1, V2) - minmax
   // pattern.
   CmpInst::Predicate Pred;
@@ -537,7 +540,8 @@ static bool isMinMaxWithLoads(Value *V, Type *&LoadTy) {
 /// or a volatile load. This is debatable, and might be reasonable to change
 /// later. However, it is risky in case some backend or other part of LLVM is
 /// relying on the exact type loaded to select appropriate atomic operations.
-static Instruction *combineLoadToOperationType(InstCombiner &IC, LoadInst &LI) {
+static Instruction *combineLoadToOperationType(InstCombinerImpl &IC,
+                                               LoadInst &LI) {
   // FIXME: We could probably with some care handle both volatile and ordered
   // atomic loads here but it isn't clear that this is important.
   if (!LI.isUnordered())
@@ -563,9 +567,9 @@ static Instruction *combineLoadToOperationType(InstCombiner &IC, LoadInst &LI) {
   if (!Ty->isIntegerTy() && Ty->isSized() && !isa<ScalableVectorType>(Ty) &&
       DL.isLegalInteger(DL.getTypeStoreSizeInBits(Ty)) &&
       DL.typeSizeEqualsStoreSize(Ty) && !DL.isNonIntegralPointerType(Ty) &&
-      !isMinMaxWithLoads(
-          peekThroughBitcast(LI.getPointerOperand(), /*OneUseOnly=*/true),
-          Dummy)) {
+      !isMinMaxWithLoads(InstCombiner::peekThroughBitcast(
+                             LI.getPointerOperand(), /*OneUseOnly=*/true),
+                         Dummy)) {
     if (all_of(LI.users(), [&LI](User *U) {
           auto *SI = dyn_cast<StoreInst>(U);
           return SI && SI->getPointerOperand() != &LI &&
@@ -605,7 +609,7 @@ static Instruction *combineLoadToOperationType(InstCombiner &IC, LoadInst &LI) {
   return nullptr;
 }
 
-static Instruction *unpackLoadToAggregate(InstCombiner &IC, LoadInst &LI) {
+static Instruction *unpackLoadToAggregate(InstCombinerImpl &IC, LoadInst &LI) {
   // FIXME: We could probably with some care handle both volatile and atomic
   // stores here but it isn't clear that this is important.
   if (!LI.isSimple())
@@ -804,8 +808,9 @@ static bool isObjectSizeLessThanOrEq(Value *V, uint64_t MaxSize,
 // not zero. Currently, we only handle the first such index. Also, we could
 // also search through non-zero constant indices if we kept track of the
 // offsets those indices implied.
-static bool canReplaceGEPIdxWithZero(InstCombiner &IC, GetElementPtrInst *GEPI,
-                                     Instruction *MemI, unsigned &Idx) {
+static bool canReplaceGEPIdxWithZero(InstCombinerImpl &IC,
+                                     GetElementPtrInst *GEPI, Instruction *MemI,
+                                     unsigned &Idx) {
   if (GEPI->getNumOperands() < 2)
     return false;
 
@@ -874,7 +879,7 @@ static bool canReplaceGEPIdxWithZero(InstCombiner &IC, GetElementPtrInst *GEPI,
 // access, but the object has only one element, we can assume that the index
 // will always be zero. If we replace the GEP, return it.
 template <typename T>
-static Instruction *replaceGEPIdxWithZero(InstCombiner &IC, Value *Ptr,
+static Instruction *replaceGEPIdxWithZero(InstCombinerImpl &IC, Value *Ptr,
                                           T &MemI) {
   if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(Ptr)) {
     unsigned Idx;
@@ -916,7 +921,7 @@ static bool canSimplifyNullLoadOrGEP(LoadInst &LI, Value *Op) {
   return false;
 }
 
-Instruction *InstCombiner::visitLoadInst(LoadInst &LI) {
+Instruction *InstCombinerImpl::visitLoadInst(LoadInst &LI) {
   Value *Op = LI.getOperand(0);
 
   // Try to canonicalize the loaded type.
@@ -1033,7 +1038,7 @@ Instruction *InstCombiner::visitLoadInst(LoadInst &LI) {
 /// and the layout of a <2 x double> is isomorphic to a [2 x double],
 /// then %V1 can be safely approximated by a conceptual "bitcast" of %U.
 /// Note that %U may contain non-undef values where %V1 has undef.
-static Value *likeBitCastFromVector(InstCombiner &IC, Value *V) {
+static Value *likeBitCastFromVector(InstCombinerImpl &IC, Value *V) {
   Value *U = nullptr;
   while (auto *IV = dyn_cast<InsertValueInst>(V)) {
     auto *E = dyn_cast<ExtractElementInst>(IV->getInsertedValueOperand());
@@ -1094,7 +1099,7 @@ static Value *likeBitCastFromVector(InstCombiner &IC, Value *V) {
 /// the caller must erase the store instruction. We have to let the caller erase
 /// the store instruction as otherwise there is no way to signal whether it was
 /// combined or not: IC.EraseInstFromFunction returns a null pointer.
-static bool combineStoreToValueType(InstCombiner &IC, StoreInst &SI) {
+static bool combineStoreToValueType(InstCombinerImpl &IC, StoreInst &SI) {
   // FIXME: We could probably with some care handle both volatile and ordered
   // atomic stores here but it isn't clear that this is important.
   if (!SI.isUnordered())
@@ -1126,7 +1131,7 @@ static bool combineStoreToValueType(InstCombiner &IC, StoreInst &SI) {
   return false;
 }
 
-static bool unpackStoreToAggregate(InstCombiner &IC, StoreInst &SI) {
+static bool unpackStoreToAggregate(InstCombinerImpl &IC, StoreInst &SI) {
   // FIXME: We could probably with some care handle both volatile and atomic
   // stores here but it isn't clear that this is important.
   if (!SI.isSimple())
@@ -1266,7 +1271,7 @@ static bool equivalentAddressValues(Value *A, Value *B) {
 /// Converts store (bitcast (load (bitcast (select ...)))) to
 /// store (load (select ...)), where select is minmax:
 /// select ((cmp load V1, load V2), V1, V2).
-static bool removeBitcastsFromLoadStoreOnMinMax(InstCombiner &IC,
+static bool removeBitcastsFromLoadStoreOnMinMax(InstCombinerImpl &IC,
                                                 StoreInst &SI) {
   // bitcast?
   if (!match(SI.getPointerOperand(), m_BitCast(m_Value())))
@@ -1296,7 +1301,8 @@ static bool removeBitcastsFromLoadStoreOnMinMax(InstCombiner &IC,
   if (!all_of(LI->users(), [LI, LoadAddr](User *U) {
         auto *SI = dyn_cast<StoreInst>(U);
         return SI && SI->getPointerOperand() != LI &&
-               peekThroughBitcast(SI->getPointerOperand()) != LoadAddr &&
+               InstCombiner::peekThroughBitcast(SI->getPointerOperand()) !=
+                   LoadAddr &&
                !SI->getPointerOperand()->isSwiftError();
       }))
     return false;
@@ -1314,7 +1320,7 @@ static bool removeBitcastsFromLoadStoreOnMinMax(InstCombiner &IC,
   return true;
 }
 
-Instruction *InstCombiner::visitStoreInst(StoreInst &SI) {
+Instruction *InstCombinerImpl::visitStoreInst(StoreInst &SI) {
   Value *Val = SI.getOperand(0);
   Value *Ptr = SI.getOperand(1);
 
@@ -1433,7 +1439,7 @@ Instruction *InstCombiner::visitStoreInst(StoreInst &SI) {
 /// or:
 ///   *P = v1; if () { *P = v2; }
 /// into a phi node with a store in the successor.
-bool InstCombiner::mergeStoreIntoSuccessor(StoreInst &SI) {
+bool InstCombinerImpl::mergeStoreIntoSuccessor(StoreInst &SI) {
   if (!SI.isUnordered())
     return false; // This code has not been audited for volatile/ordered case.
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
index f039989c004ce..172b304b77627 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
@@ -32,6 +32,7 @@
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/KnownBits.h"
 #include "llvm/Transforms/InstCombine/InstCombineWorklist.h"
+#include "llvm/Transforms/InstCombine/InstCombiner.h"
 #include "llvm/Transforms/Utils/BuildLibCalls.h"
 #include <cassert>
 #include <cstddef>
@@ -46,7 +47,7 @@ using namespace PatternMatch;
 /// The specific integer value is used in a context where it is known to be
 /// non-zero.  If this allows us to simplify the computation, do so and return
 /// the new operand, otherwise return null.
-static Value *simplifyValueKnownNonZero(Value *V, InstCombiner &IC,
+static Value *simplifyValueKnownNonZero(Value *V, InstCombinerImpl &IC,
                                         Instruction &CxtI) {
   // If V has multiple uses, then we would have to do more analysis to determine
   // if this is safe.  For example, the use could be in dynamically unreached
@@ -171,7 +172,7 @@ static Value *foldMulSelectToNegate(BinaryOperator &I,
   return nullptr;
 }
 
-Instruction *InstCombiner::visitMul(BinaryOperator &I) {
+Instruction *InstCombinerImpl::visitMul(BinaryOperator &I) {
   if (Value *V = SimplifyMulInst(I.getOperand(0), I.getOperand(1),
                                  SQ.getWithInstruction(&I)))
     return replaceInstUsesWith(I, V);
@@ -423,7 +424,7 @@ Instruction *InstCombiner::visitMul(BinaryOperator &I) {
   return Changed ? &I : nullptr;
 }
 
-Instruction *InstCombiner::foldFPSignBitOps(BinaryOperator &I) {
+Instruction *InstCombinerImpl::foldFPSignBitOps(BinaryOperator &I) {
   BinaryOperator::BinaryOps Opcode = I.getOpcode();
   assert((Opcode == Instruction::FMul || Opcode == Instruction::FDiv) &&
          "Expected fmul or fdiv");
@@ -457,7 +458,7 @@ Instruction *InstCombiner::foldFPSignBitOps(BinaryOperator &I) {
   return nullptr;
 }
 
-Instruction *InstCombiner::visitFMul(BinaryOperator &I) {
+Instruction *InstCombinerImpl::visitFMul(BinaryOperator &I) {
   if (Value *V = SimplifyFMulInst(I.getOperand(0), I.getOperand(1),
                                   I.getFastMathFlags(),
                                   SQ.getWithInstruction(&I)))
@@ -637,7 +638,7 @@ Instruction *InstCombiner::visitFMul(BinaryOperator &I) {
 /// Fold a divide or remainder with a select instruction divisor when one of the
 /// select operands is zero. In that case, we can use the other select operand
 /// because div/rem by zero is undefined.
-bool InstCombiner::simplifyDivRemOfSelectWithZeroOp(BinaryOperator &I) {
+bool InstCombinerImpl::simplifyDivRemOfSelectWithZeroOp(BinaryOperator &I) {
   SelectInst *SI = dyn_cast<SelectInst>(I.getOperand(1));
   if (!SI)
     return false;
@@ -738,7 +739,7 @@ static bool isMultiple(const APInt &C1, const APInt &C2, APInt &Quotient,
 /// instructions (udiv and sdiv). It is called by the visitors to those integer
 /// division instructions.
 /// Common integer divide transforms
-Instruction *InstCombiner::commonIDivTransforms(BinaryOperator &I) {
+Instruction *InstCombinerImpl::commonIDivTransforms(BinaryOperator &I) {
   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
   bool IsSigned = I.getOpcode() == Instruction::SDiv;
   Type *Ty = I.getType();
@@ -874,7 +875,7 @@ namespace {
 
 using FoldUDivOperandCb = Instruction *(*)(Value *Op0, Value *Op1,
                                            const BinaryOperator &I,
-                                           InstCombiner &IC);
+                                           InstCombinerImpl &IC);
 
 /// Used to maintain state for visitUDivOperand().
 struct UDivFoldAction {
@@ -903,7 +904,8 @@ struct UDivFoldAction {
 
 // X udiv 2^C -> X >> C
 static Instruction *foldUDivPow2Cst(Value *Op0, Value *Op1,
-                                    const BinaryOperator &I, InstCombiner &IC) {
+                                    const BinaryOperator &I,
+                                    InstCombinerImpl &IC) {
   Constant *C1 = getLogBase2(Op0->getType(), cast<Constant>(Op1));
   if (!C1)
     llvm_unreachable("Failed to constant fold udiv -> logbase2");
@@ -916,7 +918,7 @@ static Instruction *foldUDivPow2Cst(Value *Op0, Value *Op1,
 // X udiv (C1 << N), where C1 is "1<<C2"  -->  X >> (N+C2)
 // X udiv (zext (C1 << N)), where C1 is "1<<C2"  -->  X >> (N+C2)
 static Instruction *foldUDivShl(Value *Op0, Value *Op1, const BinaryOperator &I,
-                                InstCombiner &IC) {
+                                InstCombinerImpl &IC) {
   Value *ShiftLeft;
   if (!match(Op1, m_ZExt(m_Value(ShiftLeft))))
     ShiftLeft = Op1;
@@ -1010,7 +1012,7 @@ static Instruction *narrowUDivURem(BinaryOperator &I,
   return nullptr;
 }
 
-Instruction *InstCombiner::visitUDiv(BinaryOperator &I) {
+Instruction *InstCombinerImpl::visitUDiv(BinaryOperator &I) {
   if (Value *V = SimplifyUDivInst(I.getOperand(0), I.getOperand(1),
                                   SQ.getWithInstruction(&I)))
     return replaceInstUsesWith(I, V);
@@ -1104,7 +1106,7 @@ Instruction *InstCombiner::visitUDiv(BinaryOperator &I) {
   return nullptr;
 }
 
-Instruction *InstCombiner::visitSDiv(BinaryOperator &I) {
+Instruction *InstCombinerImpl::visitSDiv(BinaryOperator &I) {
   if (Value *V = SimplifySDivInst(I.getOperand(0), I.getOperand(1),
                                   SQ.getWithInstruction(&I)))
     return replaceInstUsesWith(I, V);
@@ -1265,7 +1267,7 @@ static Instruction *foldFDivConstantDividend(BinaryOperator &I) {
   return BinaryOperator::CreateFDivFMF(NewC, X, &I);
 }
 
-Instruction *InstCombiner::visitFDiv(BinaryOperator &I) {
+Instruction *InstCombinerImpl::visitFDiv(BinaryOperator &I) {
   if (Value *V = SimplifyFDivInst(I.getOperand(0), I.getOperand(1),
                                   I.getFastMathFlags(),
                                   SQ.getWithInstruction(&I)))
@@ -1372,7 +1374,7 @@ Instruction *InstCombiner::visitFDiv(BinaryOperator &I) {
 /// instructions (urem and srem). It is called by the visitors to those integer
 /// remainder instructions.
 /// Common integer remainder transforms
-Instruction *InstCombiner::commonIRemTransforms(BinaryOperator &I) {
+Instruction *InstCombinerImpl::commonIRemTransforms(BinaryOperator &I) {
   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
 
   // The RHS is known non-zero.
@@ -1410,7 +1412,7 @@ Instruction *InstCombiner::commonIRemTransforms(BinaryOperator &I) {
   return nullptr;
 }
 
-Instruction *InstCombiner::visitURem(BinaryOperator &I) {
+Instruction *InstCombinerImpl::visitURem(BinaryOperator &I) {
   if (Value *V = SimplifyURemInst(I.getOperand(0), I.getOperand(1),
                                   SQ.getWithInstruction(&I)))
     return replaceInstUsesWith(I, V);
@@ -1461,7 +1463,7 @@ Instruction *InstCombiner::visitURem(BinaryOperator &I) {
   return nullptr;
 }
 
-Instruction *InstCombiner::visitSRem(BinaryOperator &I) {
+Instruction *InstCombinerImpl::visitSRem(BinaryOperator &I) {
   if (Value *V = SimplifySRemInst(I.getOperand(0), I.getOperand(1),
                                   SQ.getWithInstruction(&I)))
     return replaceInstUsesWith(I, V);
@@ -1484,7 +1486,7 @@ Instruction *InstCombiner::visitSRem(BinaryOperator &I) {
   // -X srem Y --> -(X srem Y)
   Value *X, *Y;
   if (match(&I, m_SRem(m_OneUse(m_NSWSub(m_Zero(), m_Value(X))), m_Value(Y))))
-    return BinaryOperator::CreateNSWNeg(Builder.CreateSRem(X, Y)); 
+    return BinaryOperator::CreateNSWNeg(Builder.CreateSRem(X, Y));
 
   // If the sign bits of both operands are zero (i.e. we can prove they are
   // unsigned inputs), turn this into a urem.
@@ -1533,7 +1535,7 @@ Instruction *InstCombiner::visitSRem(BinaryOperator &I) {
   return nullptr;
 }
 
-Instruction *InstCombiner::visitFRem(BinaryOperator &I) {
+Instruction *InstCombinerImpl::visitFRem(BinaryOperator &I) {
   if (Value *V = SimplifyFRemInst(I.getOperand(0), I.getOperand(1),
                                   I.getFastMathFlags(),
                                   SQ.getWithInstruction(&I)))
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp b/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp
index 3fe615ac54391..1c7f00b0edee7 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp
@@ -42,6 +42,7 @@
 #include "llvm/Support/DebugCounter.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/InstCombine/InstCombiner.h"
 #include <functional>
 #include <tuple>
 #include <type_traits>
@@ -430,7 +431,7 @@ LLVM_NODISCARD Optional<Negator::Result> Negator::run(Value *Root) {
 }
 
 LLVM_NODISCARD Value *Negator::Negate(bool LHSIsZero, Value *Root,
-                                      InstCombiner &IC) {
+                                      InstCombinerImpl &IC) {
   ++NegatorTotalNegationsAttempted;
   LLVM_DEBUG(dbgs() << "Negator: attempting to sink negation into " << *Root
                     << "\n");
diff --git a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp
index dfaad1b5f8c3a..ea6adc421954d 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp
@@ -17,6 +17,7 @@
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Transforms/InstCombine/InstCombiner.h"
 #include "llvm/Transforms/Utils/Local.h"
 using namespace llvm;
 using namespace llvm::PatternMatch;
@@ -30,7 +31,7 @@ MaxNumPhis("instcombine-max-num-phis", cl::init(512),
 /// The PHI arguments will be folded into a single operation with a PHI node
 /// as input. The debug location of the single operation will be the merged
 /// locations of the original PHI node arguments.
-void InstCombiner::PHIArgMergedDebugLoc(Instruction *Inst, PHINode &PN) {
+void InstCombinerImpl::PHIArgMergedDebugLoc(Instruction *Inst, PHINode &PN) {
   auto *FirstInst = cast<Instruction>(PN.getIncomingValue(0));
   Inst->setDebugLoc(FirstInst->getDebugLoc());
   // We do not expect a CallInst here, otherwise, N-way merging of DebugLoc
@@ -93,7 +94,7 @@ void InstCombiner::PHIArgMergedDebugLoc(Instruction *Inst, PHINode &PN) {
 //    ptr_val_inc = ...
 //    ...
 //
-Instruction *InstCombiner::FoldIntegerTypedPHI(PHINode &PN) {
+Instruction *InstCombinerImpl::FoldIntegerTypedPHI(PHINode &PN) {
   if (!PN.getType()->isIntegerTy())
     return nullptr;
   if (!PN.hasOneUse())
@@ -292,7 +293,7 @@ Instruction *InstCombiner::FoldIntegerTypedPHI(PHINode &PN) {
 
 /// If we have something like phi [add (a,b), add(a,c)] and if a/b/c and the
 /// adds all have a single use, turn this into a phi and a single binop.
-Instruction *InstCombiner::FoldPHIArgBinOpIntoPHI(PHINode &PN) {
+Instruction *InstCombinerImpl::FoldPHIArgBinOpIntoPHI(PHINode &PN) {
   Instruction *FirstInst = cast<Instruction>(PN.getIncomingValue(0));
   assert(isa<BinaryOperator>(FirstInst) || isa<CmpInst>(FirstInst));
   unsigned Opc = FirstInst->getOpcode();
@@ -385,7 +386,7 @@ Instruction *InstCombiner::FoldPHIArgBinOpIntoPHI(PHINode &PN) {
   return NewBinOp;
 }
 
-Instruction *InstCombiner::FoldPHIArgGEPIntoPHI(PHINode &PN) {
+Instruction *InstCombinerImpl::FoldPHIArgGEPIntoPHI(PHINode &PN) {
   GetElementPtrInst *FirstInst =cast<GetElementPtrInst>(PN.getIncomingValue(0));
 
   SmallVector<Value*, 16> FixedOperands(FirstInst->op_begin(),
@@ -494,7 +495,6 @@ Instruction *InstCombiner::FoldPHIArgGEPIntoPHI(PHINode &PN) {
   return NewGEP;
 }
 
-
 /// Return true if we know that it is safe to sink the load out of the block
 /// that defines it. This means that it must be obvious the value of the load is
 /// not changed from the point of the load to the end of the block it is in.
@@ -540,7 +540,7 @@ static bool isSafeAndProfitableToSinkLoad(LoadInst *L) {
   return true;
 }
 
-Instruction *InstCombiner::FoldPHIArgLoadIntoPHI(PHINode &PN) {
+Instruction *InstCombinerImpl::FoldPHIArgLoadIntoPHI(PHINode &PN) {
   LoadInst *FirstLI = cast<LoadInst>(PN.getIncomingValue(0));
 
   // FIXME: This is overconservative; this transform is allowed in some cases
@@ -654,7 +654,7 @@ Instruction *InstCombiner::FoldPHIArgLoadIntoPHI(PHINode &PN) {
 /// TODO: This function could handle other cast types, but then it might
 /// require special-casing a cast from the 'i1' type. See the comment in
 /// FoldPHIArgOpIntoPHI() about pessimizing illegal integer types.
-Instruction *InstCombiner::FoldPHIArgZextsIntoPHI(PHINode &Phi) {
+Instruction *InstCombinerImpl::FoldPHIArgZextsIntoPHI(PHINode &Phi) {
   // We cannot create a new instruction after the PHI if the terminator is an
   // EHPad because there is no valid insertion point.
   if (Instruction *TI = Phi.getParent()->getTerminator())
@@ -728,7 +728,7 @@ Instruction *InstCombiner::FoldPHIArgZextsIntoPHI(PHINode &Phi) {
 /// If all operands to a PHI node are the same "unary" operator and they all are
 /// only used by the PHI, PHI together their inputs, and do the operation once,
 /// to the result of the PHI.
-Instruction *InstCombiner::FoldPHIArgOpIntoPHI(PHINode &PN) {
+Instruction *InstCombinerImpl::FoldPHIArgOpIntoPHI(PHINode &PN) {
   // We cannot create a new instruction after the PHI if the terminator is an
   // EHPad because there is no valid insertion point.
   if (Instruction *TI = PN.getParent()->getTerminator())
@@ -955,7 +955,7 @@ namespace llvm {
 /// TODO: The user of the trunc may be an bitcast to float/double/vector or an
 /// inttoptr.  We should produce new PHIs in the right type.
 ///
-Instruction *InstCombiner::SliceUpIllegalIntegerPHI(PHINode &FirstPhi) {
+Instruction *InstCombinerImpl::SliceUpIllegalIntegerPHI(PHINode &FirstPhi) {
   // PHIUsers - Keep track of all of the truncated values extracted from a set
   // of PHIs, along with their offset.  These are the things we want to rewrite.
   SmallVector<PHIUsageRecord, 16> PHIUsers;
@@ -1203,7 +1203,7 @@ static Value *SimplifyUsingControlFlow(InstCombiner &Self, PHINode &PN,
 
 // PHINode simplification
 //
-Instruction *InstCombiner::visitPHINode(PHINode &PN) {
+Instruction *InstCombinerImpl::visitPHINode(PHINode &PN) {
   if (Value *V = SimplifyInstruction(&PN, SQ.getWithInstruction(&PN)))
     return replaceInstUsesWith(PN, V);
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index db27711f29b17..e533198bd489d 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -38,6 +38,7 @@
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/KnownBits.h"
 #include "llvm/Transforms/InstCombine/InstCombineWorklist.h"
+#include "llvm/Transforms/InstCombine/InstCombiner.h"
 #include <cassert>
 #include <utility>
 
@@ -57,7 +58,7 @@ static Value *createMinMax(InstCombiner::BuilderTy &Builder,
 /// constant of a binop.
 static Instruction *foldSelectBinOpIdentity(SelectInst &Sel,
                                             const TargetLibraryInfo &TLI,
-                                            InstCombiner &IC) {
+                                            InstCombinerImpl &IC) {
   // The select condition must be an equality compare with a constant operand.
   Value *X;
   Constant *C;
@@ -279,8 +280,8 @@ static APInt getSelectFoldableConstant(BinaryOperator *I) {
 }
 
 /// We have (select c, TI, FI), and we know that TI and FI have the same opcode.
-Instruction *InstCombiner::foldSelectOpOp(SelectInst &SI, Instruction *TI,
-                                          Instruction *FI) {
+Instruction *InstCombinerImpl::foldSelectOpOp(SelectInst &SI, Instruction *TI,
+                                              Instruction *FI) {
   // Don't break up min/max patterns. The hasOneUse checks below prevent that
   // for most cases, but vector min/max with bitcasts can be transformed. If the
   // one-use restrictions are eased for other patterns, we still don't want to
@@ -418,8 +419,8 @@ static bool isSelect01(const APInt &C1I, const APInt &C2I) {
 
 /// Try to fold the select into one of the operands to allow further
 /// optimization.
-Instruction *InstCombiner::foldSelectIntoOp(SelectInst &SI, Value *TrueVal,
-                                            Value *FalseVal) {
+Instruction *InstCombinerImpl::foldSelectIntoOp(SelectInst &SI, Value *TrueVal,
+                                                Value *FalseVal) {
   // See the comment above GetSelectFoldableOperands for a description of the
   // transformation we are doing here.
   if (auto *TVI = dyn_cast<BinaryOperator>(TrueVal)) {
@@ -1024,9 +1025,9 @@ static bool adjustMinMax(SelectInst &Sel, ICmpInst &Cmp) {
 /// select (icmp Pred X, C1), C2, X --> select (icmp Pred' X, C2), X, C2
 /// Note: if C1 != C2, this will change the icmp constant to the existing
 /// constant operand of the select.
-static Instruction *
-canonicalizeMinMaxWithConstant(SelectInst &Sel, ICmpInst &Cmp,
-                               InstCombiner &IC) {
+static Instruction *canonicalizeMinMaxWithConstant(SelectInst &Sel,
+                                                   ICmpInst &Cmp,
+                                                   InstCombinerImpl &IC) {
   if (!Cmp.hasOneUse() || !isa<Constant>(Cmp.getOperand(1)))
     return nullptr;
 
@@ -1070,7 +1071,7 @@ canonicalizeMinMaxWithConstant(SelectInst &Sel, ICmpInst &Cmp,
 /// Canonicalize all these variants to 1 pattern.
 /// This makes CSE more likely.
 static Instruction *canonicalizeAbsNabs(SelectInst &Sel, ICmpInst &Cmp,
-                                        InstCombiner &IC) {
+                                        InstCombinerImpl &IC) {
   if (!Cmp.hasOneUse() || !isa<Constant>(Cmp.getOperand(1)))
     return nullptr;
 
@@ -1253,7 +1254,7 @@ static Instruction *canonicalizeClampLike(SelectInst &Sel0, ICmpInst &Cmp0,
                                   APInt::getAllOnesValue(
                                       C0->getType()->getScalarSizeInBits()))))
       return nullptr; // Can't do, have all-ones element[s].
-    C0 = AddOne(C0);
+    C0 = InstCombiner::AddOne(C0);
     std::swap(X, Sel1);
     break;
   case ICmpInst::Predicate::ICMP_UGE:
@@ -1313,7 +1314,7 @@ static Instruction *canonicalizeClampLike(SelectInst &Sel0, ICmpInst &Cmp0,
                                   APInt::getSignedMaxValue(
                                       C2->getType()->getScalarSizeInBits()))))
       return nullptr; // Can't do, have signed max element[s].
-    C2 = AddOne(C2);
+    C2 = InstCombiner::AddOne(C2);
     LLVM_FALLTHROUGH;
   case ICmpInst::Predicate::ICMP_SGE:
     // Also non-canonical, but here we don't need to change C2,
@@ -1360,7 +1361,7 @@ static Instruction *canonicalizeClampLike(SelectInst &Sel0, ICmpInst &Cmp0,
 // and swap the hands of select.
 static Instruction *
 tryToReuseConstantFromSelectInComparison(SelectInst &Sel, ICmpInst &Cmp,
-                                         InstCombiner &IC) {
+                                         InstCombinerImpl &IC) {
   ICmpInst::Predicate Pred;
   Value *X;
   Constant *C0;
@@ -1375,7 +1376,7 @@ tryToReuseConstantFromSelectInComparison(SelectInst &Sel, ICmpInst &Cmp,
 
   // If comparison predicate is non-canonical, then we certainly won't be able
   // to make it canonical; canonicalizeCmpWithConstant() already tried.
-  if (!isCanonicalPredicate(Pred))
+  if (!InstCombiner::isCanonicalPredicate(Pred))
     return nullptr;
 
   // If the [input] type of comparison and select type are different, lets abort
@@ -1403,7 +1404,8 @@ tryToReuseConstantFromSelectInComparison(SelectInst &Sel, ICmpInst &Cmp,
     return nullptr;
 
   // Check the constant we'd have with flipped-strictness predicate.
-  auto FlippedStrictness = getFlippedStrictnessPredicateAndConstant(Pred, C0);
+  auto FlippedStrictness =
+      InstCombiner::getFlippedStrictnessPredicateAndConstant(Pred, C0);
   if (!FlippedStrictness)
     return nullptr;
 
@@ -1426,8 +1428,8 @@ tryToReuseConstantFromSelectInComparison(SelectInst &Sel, ICmpInst &Cmp,
 }
 
 /// Visit a SelectInst that has an ICmpInst as its first operand.
-Instruction *InstCombiner::foldSelectInstWithICmp(SelectInst &SI,
-                                                  ICmpInst *ICI) {
+Instruction *InstCombinerImpl::foldSelectInstWithICmp(SelectInst &SI,
+                                                      ICmpInst *ICI) {
   if (Value *V = foldSelectValueEquivalence(SI, *ICI, SQ))
     return replaceInstUsesWith(SI, V);
 
@@ -1579,11 +1581,11 @@ static bool canSelectOperandBeMappingIntoPredBlock(const Value *V,
 
 /// We have an SPF (e.g. a min or max) of an SPF of the form:
 ///   SPF2(SPF1(A, B), C)
-Instruction *InstCombiner::foldSPFofSPF(Instruction *Inner,
-                                        SelectPatternFlavor SPF1,
-                                        Value *A, Value *B,
-                                        Instruction &Outer,
-                                        SelectPatternFlavor SPF2, Value *C) {
+Instruction *InstCombinerImpl::foldSPFofSPF(Instruction *Inner,
+                                            SelectPatternFlavor SPF1, Value *A,
+                                            Value *B, Instruction &Outer,
+                                            SelectPatternFlavor SPF2,
+                                            Value *C) {
   if (Outer.getType() != Inner->getType())
     return nullptr;
 
@@ -1900,7 +1902,7 @@ foldOverflowingAddSubSelect(SelectInst &SI, InstCombiner::BuilderTy &Builder) {
   return CallInst::Create(F, {X, Y});
 }
 
-Instruction *InstCombiner::foldSelectExtConst(SelectInst &Sel) {
+Instruction *InstCombinerImpl::foldSelectExtConst(SelectInst &Sel) {
   Constant *C;
   if (!match(Sel.getTrueValue(), m_Constant(C)) &&
       !match(Sel.getFalseValue(), m_Constant(C)))
@@ -2001,8 +2003,8 @@ static Instruction *canonicalizeSelectToShuffle(SelectInst &SI) {
 /// to a vector select by splatting the condition. A splat may get folded with
 /// other operations in IR and having all operands of a select be vector types
 /// is likely better for vector codegen.
-static Instruction *canonicalizeScalarSelectOfVecs(
-    SelectInst &Sel, InstCombiner &IC) {
+static Instruction *canonicalizeScalarSelectOfVecs(SelectInst &Sel,
+                                                   InstCombinerImpl &IC) {
   auto *Ty = dyn_cast<VectorType>(Sel.getType());
   if (!Ty)
     return nullptr;
@@ -2172,7 +2174,7 @@ static Instruction *moveAddAfterMinMax(SelectPatternFlavor SPF, Value *X,
 }
 
 /// Match a sadd_sat or ssub_sat which is using min/max to clamp the value.
-Instruction *InstCombiner::matchSAddSubSat(SelectInst &MinMax1) {
+Instruction *InstCombinerImpl::matchSAddSubSat(SelectInst &MinMax1) {
   Type *Ty = MinMax1.getType();
 
   // We are looking for a tree of:
@@ -2297,7 +2299,8 @@ static Instruction *factorizeMinMaxTree(SelectPatternFlavor SPF, Value *LHS,
 /// funnel shift intrinsic. Example:
 /// rotl32(a, b) --> (b == 0 ? a : ((a >> (32 - b)) | (a << b)))
 ///              --> call llvm.fshl.i32(a, a, b)
-static Instruction *foldSelectRotate(SelectInst &Sel) {
+static Instruction *foldSelectRotate(SelectInst &Sel,
+                                     InstCombiner::BuilderTy &Builder) {
   // The false value of the select must be a rotate of the true value.
   Value *Or0, *Or1;
   if (!match(Sel.getFalseValue(), m_OneUse(m_Or(m_Value(Or0), m_Value(Or1)))))
@@ -2305,8 +2308,10 @@ static Instruction *foldSelectRotate(SelectInst &Sel) {
 
   Value *TVal = Sel.getTrueValue();
   Value *SA0, *SA1;
-  if (!match(Or0, m_OneUse(m_LogicalShift(m_Specific(TVal), m_Value(SA0)))) ||
-      !match(Or1, m_OneUse(m_LogicalShift(m_Specific(TVal), m_Value(SA1)))))
+  if (!match(Or0, m_OneUse(m_LogicalShift(m_Specific(TVal),
+                                          m_ZExtOrSelf(m_Value(SA0))))) ||
+      !match(Or1, m_OneUse(m_LogicalShift(m_Specific(TVal),
+                                          m_ZExtOrSelf(m_Value(SA1))))))
     return nullptr;
 
   auto ShiftOpcode0 = cast<BinaryOperator>(Or0)->getOpcode();
@@ -2344,6 +2349,7 @@ static Instruction *foldSelectRotate(SelectInst &Sel) {
                 (ShAmt == SA1 && ShiftOpcode1 == BinaryOperator::Shl);
   Intrinsic::ID IID = IsFshl ? Intrinsic::fshl : Intrinsic::fshr;
   Function *F = Intrinsic::getDeclaration(Sel.getModule(), IID, Sel.getType());
+  ShAmt = Builder.CreateZExt(ShAmt, Sel.getType());
   return IntrinsicInst::Create(F, { TVal, TVal, ShAmt });
 }
 
@@ -2368,7 +2374,8 @@ static Instruction *foldSelectToCopysign(SelectInst &Sel,
   bool IsTrueIfSignSet;
   ICmpInst::Predicate Pred;
   if (!match(Cond, m_OneUse(m_ICmp(Pred, m_BitCast(m_Value(X)), m_APInt(C)))) ||
-      !isSignBitCheck(Pred, *C, IsTrueIfSignSet) || X->getType() != SelType)
+      !InstCombiner::isSignBitCheck(Pred, *C, IsTrueIfSignSet) ||
+      X->getType() != SelType)
     return nullptr;
 
   // If needed, negate the value that will be the sign argument of the copysign:
@@ -2389,7 +2396,7 @@ static Instruction *foldSelectToCopysign(SelectInst &Sel,
   return CopySign;
 }
 
-Instruction *InstCombiner::foldVectorSelect(SelectInst &Sel) {
+Instruction *InstCombinerImpl::foldVectorSelect(SelectInst &Sel) {
   auto *VecTy = dyn_cast<FixedVectorType>(Sel.getType());
   if (!VecTy)
     return nullptr;
@@ -2519,7 +2526,7 @@ static Instruction *foldSelectToPhi(SelectInst &Sel, const DominatorTree &DT,
   return nullptr;
 }
 
-Instruction *InstCombiner::visitSelectInst(SelectInst &SI) {
+Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) {
   Value *CondVal = SI.getCondition();
   Value *TrueVal = SI.getTrueValue();
   Value *FalseVal = SI.getFalseValue();
@@ -2960,7 +2967,7 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) {
   if (Instruction *Select = foldSelectBinOpIdentity(SI, TLI, *this))
     return Select;
 
-  if (Instruction *Rot = foldSelectRotate(SI))
+  if (Instruction *Rot = foldSelectRotate(SI, Builder))
     return Rot;
 
   if (Instruction *Copysign = foldSelectToCopysign(SI, Builder))
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
index 0a842b4e10475..e6d1b31e8df5f 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
@@ -15,6 +15,7 @@
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/PatternMatch.h"
+#include "llvm/Transforms/InstCombine/InstCombiner.h"
 using namespace llvm;
 using namespace PatternMatch;
 
@@ -31,7 +32,7 @@ using namespace PatternMatch;
 //
 // AnalyzeForSignBitExtraction indicates that we will only analyze whether this
 // pattern has any 2 right-shifts that sum to 1 less than original bit width.
-Value *InstCombiner::reassociateShiftAmtsOfTwoSameDirectionShifts(
+Value *InstCombinerImpl::reassociateShiftAmtsOfTwoSameDirectionShifts(
     BinaryOperator *Sh0, const SimplifyQuery &SQ,
     bool AnalyzeForSignBitExtraction) {
   // Look for a shift of some instruction, ignore zext of shift amount if any.
@@ -360,7 +361,7 @@ static Instruction *foldShiftOfShiftedLogic(BinaryOperator &I,
   return BinaryOperator::Create(LogicInst->getOpcode(), NewShift1, NewShift2);
 }
 
-Instruction *InstCombiner::commonShiftTransforms(BinaryOperator &I) {
+Instruction *InstCombinerImpl::commonShiftTransforms(BinaryOperator &I) {
   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
   assert(Op0->getType() == Op1->getType());
 
@@ -420,8 +421,8 @@ Instruction *InstCombiner::commonShiftTransforms(BinaryOperator &I) {
 /// Return true if we can simplify two logical (either left or right) shifts
 /// that have constant shift amounts: OuterShift (InnerShift X, C1), C2.
 static bool canEvaluateShiftedShift(unsigned OuterShAmt, bool IsOuterShl,
-                                    Instruction *InnerShift, InstCombiner &IC,
-                                    Instruction *CxtI) {
+                                    Instruction *InnerShift,
+                                    InstCombinerImpl &IC, Instruction *CxtI) {
   assert(InnerShift->isLogicalShift() && "Unexpected instruction type");
 
   // We need constant scalar or constant splat shifts.
@@ -472,7 +473,7 @@ static bool canEvaluateShiftedShift(unsigned OuterShAmt, bool IsOuterShl,
 /// where the client will ask if E can be computed shifted right by 64-bits. If
 /// this succeeds, getShiftedValue() will be called to produce the value.
 static bool canEvaluateShifted(Value *V, unsigned NumBits, bool IsLeftShift,
-                               InstCombiner &IC, Instruction *CxtI) {
+                               InstCombinerImpl &IC, Instruction *CxtI) {
   // We can always evaluate constants shifted.
   if (isa<Constant>(V))
     return true;
@@ -608,7 +609,7 @@ static Value *foldShiftedShift(BinaryOperator *InnerShift, unsigned OuterShAmt,
 /// When canEvaluateShifted() returns true for an expression, this function
 /// inserts the new computation that produces the shifted value.
 static Value *getShiftedValue(Value *V, unsigned NumBits, bool isLeftShift,
-                              InstCombiner &IC, const DataLayout &DL) {
+                              InstCombinerImpl &IC, const DataLayout &DL) {
   // We can always evaluate constants shifted.
   if (Constant *C = dyn_cast<Constant>(V)) {
     if (isLeftShift)
@@ -618,7 +619,7 @@ static Value *getShiftedValue(Value *V, unsigned NumBits, bool isLeftShift,
   }
 
   Instruction *I = cast<Instruction>(V);
-  IC.Worklist.push(I);
+  IC.addToWorklist(I);
 
   switch (I->getOpcode()) {
   default: llvm_unreachable("Inconsistency with CanEvaluateShifted");
@@ -672,8 +673,8 @@ static bool canShiftBinOpWithConstantRHS(BinaryOperator &Shift,
   }
 }
 
-Instruction *InstCombiner::FoldShiftByConstant(Value *Op0, Constant *Op1,
-                                               BinaryOperator &I) {
+Instruction *InstCombinerImpl::FoldShiftByConstant(Value *Op0, Constant *Op1,
+                                                   BinaryOperator &I) {
   bool isLeftShift = I.getOpcode() == Instruction::Shl;
 
   const APInt *Op1C;
@@ -915,7 +916,7 @@ Instruction *InstCombiner::FoldShiftByConstant(Value *Op0, Constant *Op1,
   return nullptr;
 }
 
-Instruction *InstCombiner::visitShl(BinaryOperator &I) {
+Instruction *InstCombinerImpl::visitShl(BinaryOperator &I) {
   const SimplifyQuery Q = SQ.getWithInstruction(&I);
 
   if (Value *V = SimplifyShlInst(I.getOperand(0), I.getOperand(1),
@@ -1037,7 +1038,7 @@ Instruction *InstCombiner::visitShl(BinaryOperator &I) {
   return nullptr;
 }
 
-Instruction *InstCombiner::visitLShr(BinaryOperator &I) {
+Instruction *InstCombinerImpl::visitLShr(BinaryOperator &I) {
   if (Value *V = SimplifyLShrInst(I.getOperand(0), I.getOperand(1), I.isExact(),
                                   SQ.getWithInstruction(&I)))
     return replaceInstUsesWith(I, V);
@@ -1167,7 +1168,7 @@ Instruction *InstCombiner::visitLShr(BinaryOperator &I) {
 }
 
 Instruction *
-InstCombiner::foldVariableSignZeroExtensionOfVariableHighBitExtract(
+InstCombinerImpl::foldVariableSignZeroExtensionOfVariableHighBitExtract(
     BinaryOperator &OldAShr) {
   assert(OldAShr.getOpcode() == Instruction::AShr &&
          "Must be called with arithmetic right-shift instruction only.");
@@ -1235,7 +1236,7 @@ InstCombiner::foldVariableSignZeroExtensionOfVariableHighBitExtract(
   return TruncInst::CreateTruncOrBitCast(NewAShr, OldAShr.getType());
 }
 
-Instruction *InstCombiner::visitAShr(BinaryOperator &I) {
+Instruction *InstCombinerImpl::visitAShr(BinaryOperator &I) {
   if (Value *V = SimplifyAShrInst(I.getOperand(0), I.getOperand(1), I.isExact(),
                                   SQ.getWithInstruction(&I)))
     return replaceInstUsesWith(I, V);
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
index 7cfe4c8b5892b..f56d561e62d70 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
@@ -12,29 +12,18 @@
 //===----------------------------------------------------------------------===//
 
 #include "InstCombineInternal.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/IntrinsicInst.h"
-#include "llvm/IR/IntrinsicsAMDGPU.h"
-#include "llvm/IR/IntrinsicsX86.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/Support/KnownBits.h"
+#include "llvm/Transforms/InstCombine/InstCombiner.h"
 
 using namespace llvm;
 using namespace llvm::PatternMatch;
 
 #define DEBUG_TYPE "instcombine"
 
-namespace {
-
-struct AMDGPUImageDMaskIntrinsic {
-  unsigned Intr;
-};
-
-#define GET_AMDGPUImageDMaskIntrinsicTable_IMPL
-#include "InstCombineTables.inc"
-
-} // end anonymous namespace
-
 /// Check to see if the specified operand of the specified instruction is a
 /// constant integer. If so, check to see if there are any bits set in the
 /// constant that are not demanded. If so, shrink the constant and return true.
@@ -63,7 +52,7 @@ static bool ShrinkDemandedConstant(Instruction *I, unsigned OpNo,
 
 /// Inst is an integer instruction that SimplifyDemandedBits knows about. See if
 /// the instruction has any properties that allow us to simplify its operands.
-bool InstCombiner::SimplifyDemandedInstructionBits(Instruction &Inst) {
+bool InstCombinerImpl::SimplifyDemandedInstructionBits(Instruction &Inst) {
   unsigned BitWidth = Inst.getType()->getScalarSizeInBits();
   KnownBits Known(BitWidth);
   APInt DemandedMask(APInt::getAllOnesValue(BitWidth));
@@ -79,22 +68,20 @@ bool InstCombiner::SimplifyDemandedInstructionBits(Instruction &Inst) {
 /// This form of SimplifyDemandedBits simplifies the specified instruction
 /// operand if possible, updating it in place. It returns true if it made any
 /// change and false otherwise.
-bool InstCombiner::SimplifyDemandedBits(Instruction *I, unsigned OpNo,
-                                        const APInt &DemandedMask,
-                                        KnownBits &Known,
-                                        unsigned Depth) {
+bool InstCombinerImpl::SimplifyDemandedBits(Instruction *I, unsigned OpNo,
+                                            const APInt &DemandedMask,
+                                            KnownBits &Known, unsigned Depth) {
   Use &U = I->getOperandUse(OpNo);
   Value *NewVal = SimplifyDemandedUseBits(U.get(), DemandedMask, Known,
                                           Depth, I);
   if (!NewVal) return false;
   if (Instruction* OpInst = dyn_cast<Instruction>(U))
     salvageDebugInfo(*OpInst);
-    
+
   replaceUse(U, NewVal);
   return true;
 }
 
-
 /// This function attempts to replace V with a simpler value based on the
 /// demanded bits. When this function is called, it is known that only the bits
 /// set in DemandedMask of the result of V are ever used downstream.
@@ -118,9 +105,10 @@ bool InstCombiner::SimplifyDemandedBits(Instruction *I, unsigned OpNo,
 /// operands based on the information about what bits are demanded. This returns
 /// some other non-null value if it found out that V is equal to another value
 /// in the context where the specified bits are demanded, but not for all users.
-Value *InstCombiner::SimplifyDemandedUseBits(Value *V, APInt DemandedMask,
-                                             KnownBits &Known, unsigned Depth,
-                                             Instruction *CxtI) {
+Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask,
+                                                 KnownBits &Known,
+                                                 unsigned Depth,
+                                                 Instruction *CxtI) {
   assert(V != nullptr && "Null pointer of Value???");
   assert(Depth <= 6 && "Limit Search Depth");
   uint32_t BitWidth = DemandedMask.getBitWidth();
@@ -728,7 +716,6 @@ Value *InstCombiner::SimplifyDemandedUseBits(Value *V, APInt DemandedMask,
     bool KnownBitsComputed = false;
     if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
       switch (II->getIntrinsicID()) {
-      default: break;
       case Intrinsic::bswap: {
         // If the only bits demanded come from one byte of the bswap result,
         // just shift the input byte into position to eliminate the bswap.
@@ -784,39 +771,14 @@ Value *InstCombiner::SimplifyDemandedUseBits(Value *V, APInt DemandedMask,
         KnownBitsComputed = true;
         break;
       }
-      case Intrinsic::x86_mmx_pmovmskb:
-      case Intrinsic::x86_sse_movmsk_ps:
-      case Intrinsic::x86_sse2_movmsk_pd:
-      case Intrinsic::x86_sse2_pmovmskb_128:
-      case Intrinsic::x86_avx_movmsk_ps_256:
-      case Intrinsic::x86_avx_movmsk_pd_256:
-      case Intrinsic::x86_avx2_pmovmskb: {
-        // MOVMSK copies the vector elements' sign bits to the low bits
-        // and zeros the high bits.
-        unsigned ArgWidth;
-        if (II->getIntrinsicID() == Intrinsic::x86_mmx_pmovmskb) {
-          ArgWidth = 8; // Arg is x86_mmx, but treated as <8 x i8>.
-        } else {
-          auto Arg = II->getArgOperand(0);
-          auto ArgType = cast<VectorType>(Arg->getType());
-          ArgWidth = ArgType->getNumElements();
-        }
-
-        // If we don't need any of low bits then return zero,
-        // we know that DemandedMask is non-zero already.
-        APInt DemandedElts = DemandedMask.zextOrTrunc(ArgWidth);
-        if (DemandedElts.isNullValue())
-          return ConstantInt::getNullValue(VTy);
-
-        // We know that the upper bits are set to zero.
-        Known.Zero.setBitsFrom(ArgWidth);
-        KnownBitsComputed = true;
+      default: {
+        // Handle target specific intrinsics
+        Optional<Value *> V = targetSimplifyDemandedUseBitsIntrinsic(
+            *II, DemandedMask, Known, KnownBitsComputed);
+        if (V.hasValue())
+          return V.getValue();
         break;
       }
-      case Intrinsic::x86_sse42_crc32_64_64:
-        Known.Zero.setBitsFrom(32);
-        KnownBitsComputed = true;
-        break;
       }
     }
 
@@ -836,11 +798,9 @@ Value *InstCombiner::SimplifyDemandedUseBits(Value *V, APInt DemandedMask,
 /// Helper routine of SimplifyDemandedUseBits. It computes Known
 /// bits. It also tries to handle simplifications that can be done based on
 /// DemandedMask, but without modifying the Instruction.
-Value *InstCombiner::SimplifyMultipleUseDemandedBits(Instruction *I,
-                                                     const APInt &DemandedMask,
-                                                     KnownBits &Known,
-                                                     unsigned Depth,
-                                                     Instruction *CxtI) {
+Value *InstCombinerImpl::SimplifyMultipleUseDemandedBits(
+    Instruction *I, const APInt &DemandedMask, KnownBits &Known, unsigned Depth,
+    Instruction *CxtI) {
   unsigned BitWidth = DemandedMask.getBitWidth();
   Type *ITy = I->getType();
 
@@ -940,7 +900,6 @@ Value *InstCombiner::SimplifyMultipleUseDemandedBits(Instruction *I,
   return nullptr;
 }
 
-
 /// Helper routine of SimplifyDemandedUseBits. It tries to simplify
 /// "E1 = (X lsr C1) << C2", where the C1 and C2 are constant, into
 /// "E2 = X << (C2 - C1)" or "E2 = X >> (C1 - C2)", depending on the sign
@@ -958,11 +917,9 @@ Value *InstCombiner::SimplifyMultipleUseDemandedBits(Instruction *I,
 ///
 /// As with SimplifyDemandedUseBits, it returns NULL if the simplification was
 /// not successful.
-Value *
-InstCombiner::simplifyShrShlDemandedBits(Instruction *Shr, const APInt &ShrOp1,
-                                         Instruction *Shl, const APInt &ShlOp1,
-                                         const APInt &DemandedMask,
-                                         KnownBits &Known) {
+Value *InstCombinerImpl::simplifyShrShlDemandedBits(
+    Instruction *Shr, const APInt &ShrOp1, Instruction *Shl,
+    const APInt &ShlOp1, const APInt &DemandedMask, KnownBits &Known) {
   if (!ShlOp1 || !ShrOp1)
     return nullptr; // No-op.
 
@@ -1022,153 +979,6 @@ InstCombiner::simplifyShrShlDemandedBits(Instruction *Shr, const APInt &ShrOp1,
   return nullptr;
 }
 
-/// Implement SimplifyDemandedVectorElts for amdgcn buffer and image intrinsics.
-///
-/// Note: This only supports non-TFE/LWE image intrinsic calls; those have
-///       struct returns.
-Value *InstCombiner::simplifyAMDGCNMemoryIntrinsicDemanded(IntrinsicInst *II,
-                                                           APInt DemandedElts,
-                                                           int DMaskIdx) {
-
-  // FIXME: Allow v3i16/v3f16 in buffer intrinsics when the types are fully supported.
-  if (DMaskIdx < 0 &&
-      II->getType()->getScalarSizeInBits() != 32 &&
-      DemandedElts.getActiveBits() == 3)
-    return nullptr;
-
-  auto *IIVTy = cast<VectorType>(II->getType());
-  unsigned VWidth = IIVTy->getNumElements();
-  if (VWidth == 1)
-    return nullptr;
-
-  IRBuilderBase::InsertPointGuard Guard(Builder);
-  Builder.SetInsertPoint(II);
-
-  // Assume the arguments are unchanged and later override them, if needed.
-  SmallVector<Value *, 16> Args(II->arg_begin(), II->arg_end());
-
-  if (DMaskIdx < 0) {
-    // Buffer case.
-
-    const unsigned ActiveBits = DemandedElts.getActiveBits();
-    const unsigned UnusedComponentsAtFront = DemandedElts.countTrailingZeros();
-
-    // Start assuming the prefix of elements is demanded, but possibly clear
-    // some other bits if there are trailing zeros (unused components at front)
-    // and update offset.
-    DemandedElts = (1 << ActiveBits) - 1;
-
-    if (UnusedComponentsAtFront > 0) {
-      static const unsigned InvalidOffsetIdx = 0xf;
-
-      unsigned OffsetIdx;
-      switch (II->getIntrinsicID()) {
-      case Intrinsic::amdgcn_raw_buffer_load:
-        OffsetIdx = 1;
-        break;
-      case Intrinsic::amdgcn_s_buffer_load:
-        // If resulting type is vec3, there is no point in trimming the
-        // load with updated offset, as the vec3 would most likely be widened to
-        // vec4 anyway during lowering.
-        if (ActiveBits == 4 && UnusedComponentsAtFront == 1)
-          OffsetIdx = InvalidOffsetIdx;
-        else
-          OffsetIdx = 1;
-        break;
-      case Intrinsic::amdgcn_struct_buffer_load:
-        OffsetIdx = 2;
-        break;
-      default:
-        // TODO: handle tbuffer* intrinsics.
-        OffsetIdx = InvalidOffsetIdx;
-        break;
-      }
-
-      if (OffsetIdx != InvalidOffsetIdx) {
-        // Clear demanded bits and update the offset.
-        DemandedElts &= ~((1 << UnusedComponentsAtFront) - 1);
-        auto *Offset = II->getArgOperand(OffsetIdx);
-        unsigned SingleComponentSizeInBits =
-            getDataLayout().getTypeSizeInBits(II->getType()->getScalarType());
-        unsigned OffsetAdd =
-            UnusedComponentsAtFront * SingleComponentSizeInBits / 8;
-        auto *OffsetAddVal = ConstantInt::get(Offset->getType(), OffsetAdd);
-        Args[OffsetIdx] = Builder.CreateAdd(Offset, OffsetAddVal);
-      }
-    }
-  } else {
-    // Image case.
-
-    ConstantInt *DMask = cast<ConstantInt>(II->getArgOperand(DMaskIdx));
-    unsigned DMaskVal = DMask->getZExtValue() & 0xf;
-
-    // Mask off values that are undefined because the dmask doesn't cover them
-    DemandedElts &= (1 << countPopulation(DMaskVal)) - 1;
-
-    unsigned NewDMaskVal = 0;
-    unsigned OrigLoadIdx = 0;
-    for (unsigned SrcIdx = 0; SrcIdx < 4; ++SrcIdx) {
-      const unsigned Bit = 1 << SrcIdx;
-      if (!!(DMaskVal & Bit)) {
-        if (!!DemandedElts[OrigLoadIdx])
-          NewDMaskVal |= Bit;
-        OrigLoadIdx++;
-      }
-    }
-
-    if (DMaskVal != NewDMaskVal)
-      Args[DMaskIdx] = ConstantInt::get(DMask->getType(), NewDMaskVal);
-  }
-
-  unsigned NewNumElts = DemandedElts.countPopulation();
-  if (!NewNumElts)
-    return UndefValue::get(II->getType());
-
-  if (NewNumElts >= VWidth && DemandedElts.isMask()) {
-    if (DMaskIdx >= 0)
-      II->setArgOperand(DMaskIdx, Args[DMaskIdx]);
-    return nullptr;
-  }
-
-  // Validate function argument and return types, extracting overloaded types
-  // along the way.
-  SmallVector<Type *, 6> OverloadTys;
-  if (!Intrinsic::getIntrinsicSignature(II->getCalledFunction(), OverloadTys))
-    return nullptr;
-
-  Module *M = II->getParent()->getParent()->getParent();
-  Type *EltTy = IIVTy->getElementType();
-  Type *NewTy =
-      (NewNumElts == 1) ? EltTy : FixedVectorType::get(EltTy, NewNumElts);
-
-  OverloadTys[0] = NewTy;
-  Function *NewIntrin =
-      Intrinsic::getDeclaration(M, II->getIntrinsicID(), OverloadTys);
-
-  CallInst *NewCall = Builder.CreateCall(NewIntrin, Args);
-  NewCall->takeName(II);
-  NewCall->copyMetadata(*II);
-
-  if (NewNumElts == 1) {
-    return Builder.CreateInsertElement(UndefValue::get(II->getType()), NewCall,
-                                       DemandedElts.countTrailingZeros());
-  }
-
-  SmallVector<int, 8> EltMask;
-  unsigned NewLoadIdx = 0;
-  for (unsigned OrigLoadIdx = 0; OrigLoadIdx < VWidth; ++OrigLoadIdx) {
-    if (!!DemandedElts[OrigLoadIdx])
-      EltMask.push_back(NewLoadIdx++);
-    else
-      EltMask.push_back(NewNumElts);
-  }
-
-  Value *Shuffle =
-      Builder.CreateShuffleVector(NewCall, UndefValue::get(NewTy), EltMask);
-
-  return Shuffle;
-}
-
 /// The specified value produces a vector with any number of elements.
 /// This method analyzes which elements of the operand are undef and returns
 /// that information in UndefElts.
@@ -1182,10 +992,11 @@ Value *InstCombiner::simplifyAMDGCNMemoryIntrinsicDemanded(IntrinsicInst *II,
 /// If the information about demanded elements can be used to simplify the
 /// operation, the operation is simplified, then the resultant value is
 /// returned.  This returns null if no change was made.
-Value *InstCombiner::SimplifyDemandedVectorElts(Value *V, APInt DemandedElts,
-                                                APInt &UndefElts,
-                                                unsigned Depth,
-                                                bool AllowMultipleUsers) {
+Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V,
+                                                    APInt DemandedElts,
+                                                    APInt &UndefElts,
+                                                    unsigned Depth,
+                                                    bool AllowMultipleUsers) {
   // Cannot analyze scalable type. The number of vector elements is not a
   // compile-time constant.
   if (isa<ScalableVectorType>(V->getType()))
@@ -1292,12 +1103,12 @@ Value *InstCombiner::SimplifyDemandedVectorElts(Value *V, APInt DemandedElts,
     };
     if (mayIndexStructType(cast<GetElementPtrInst>(*I)))
       break;
-    
+
     // Conservatively track the demanded elements back through any vector
     // operands we may have.  We know there must be at least one, or we
     // wouldn't have a vector result to get here. Note that we intentionally
     // merge the undef bits here since gepping with either an undef base or
-    // index results in undef. 
+    // index results in undef.
     for (unsigned i = 0; i < I->getNumOperands(); i++) {
       if (isa<UndefValue>(I->getOperand(i))) {
         // If the entire vector is undefined, just return this info.
@@ -1621,227 +1432,19 @@ Value *InstCombiner::SimplifyDemandedVectorElts(Value *V, APInt DemandedElts,
       if (II->getIntrinsicID() == Intrinsic::masked_gather)
         simplifyAndSetOp(II, 0, DemandedPtrs, UndefElts2);
       simplifyAndSetOp(II, 3, DemandedPassThrough, UndefElts3);
-      
+
       // Output elements are undefined if the element from both sources are.
       // TODO: can strengthen via mask as well.
       UndefElts = UndefElts2 & UndefElts3;
       break;
     }
-    case Intrinsic::x86_xop_vfrcz_ss:
-    case Intrinsic::x86_xop_vfrcz_sd:
-      // The instructions for these intrinsics are speced to zero upper bits not
-      // pass them through like other scalar intrinsics. So we shouldn't just
-      // use Arg0 if DemandedElts[0] is clear like we do for other intrinsics.
-      // Instead we should return a zero vector.
-      if (!DemandedElts[0]) {
-        Worklist.push(II);
-        return ConstantAggregateZero::get(II->getType());
-      }
-
-      // Only the lower element is used.
-      DemandedElts = 1;
-      simplifyAndSetOp(II, 0, DemandedElts, UndefElts);
-
-      // Only the lower element is undefined. The high elements are zero.
-      UndefElts = UndefElts[0];
-      break;
-
-    // Unary scalar-as-vector operations that work column-wise.
-    case Intrinsic::x86_sse_rcp_ss:
-    case Intrinsic::x86_sse_rsqrt_ss:
-      simplifyAndSetOp(II, 0, DemandedElts, UndefElts);
-
-      // If lowest element of a scalar op isn't used then use Arg0.
-      if (!DemandedElts[0]) {
-        Worklist.push(II);
-        return II->getArgOperand(0);
-      }
-      // TODO: If only low elt lower SQRT to FSQRT (with rounding/exceptions
-      // checks).
-      break;
-
-    // Binary scalar-as-vector operations that work column-wise. The high
-    // elements come from operand 0. The low element is a function of both
-    // operands.
-    case Intrinsic::x86_sse_min_ss:
-    case Intrinsic::x86_sse_max_ss:
-    case Intrinsic::x86_sse_cmp_ss:
-    case Intrinsic::x86_sse2_min_sd:
-    case Intrinsic::x86_sse2_max_sd:
-    case Intrinsic::x86_sse2_cmp_sd: {
-      simplifyAndSetOp(II, 0, DemandedElts, UndefElts);
-
-      // If lowest element of a scalar op isn't used then use Arg0.
-      if (!DemandedElts[0]) {
-        Worklist.push(II);
-        return II->getArgOperand(0);
-      }
-
-      // Only lower element is used for operand 1.
-      DemandedElts = 1;
-      simplifyAndSetOp(II, 1, DemandedElts, UndefElts2);
-
-      // Lower element is undefined if both lower elements are undefined.
-      // Consider things like undef&0.  The result is known zero, not undef.
-      if (!UndefElts2[0])
-        UndefElts.clearBit(0);
-
-      break;
-    }
-
-    // Binary scalar-as-vector operations that work column-wise. The high
-    // elements come from operand 0 and the low element comes from operand 1.
-    case Intrinsic::x86_sse41_round_ss:
-    case Intrinsic::x86_sse41_round_sd: {
-      // Don't use the low element of operand 0.
-      APInt DemandedElts2 = DemandedElts;
-      DemandedElts2.clearBit(0);
-      simplifyAndSetOp(II, 0, DemandedElts2, UndefElts);
-
-      // If lowest element of a scalar op isn't used then use Arg0.
-      if (!DemandedElts[0]) {
-        Worklist.push(II);
-        return II->getArgOperand(0);
-      }
-
-      // Only lower element is used for operand 1.
-      DemandedElts = 1;
-      simplifyAndSetOp(II, 1, DemandedElts, UndefElts2);
-
-      // Take the high undef elements from operand 0 and take the lower element
-      // from operand 1.
-      UndefElts.clearBit(0);
-      UndefElts |= UndefElts2[0];
-      break;
-    }
-
-    // Three input scalar-as-vector operations that work column-wise. The high
-    // elements come from operand 0 and the low element is a function of all
-    // three inputs.
-    case Intrinsic::x86_avx512_mask_add_ss_round:
-    case Intrinsic::x86_avx512_mask_div_ss_round:
-    case Intrinsic::x86_avx512_mask_mul_ss_round:
-    case Intrinsic::x86_avx512_mask_sub_ss_round:
-    case Intrinsic::x86_avx512_mask_max_ss_round:
-    case Intrinsic::x86_avx512_mask_min_ss_round:
-    case Intrinsic::x86_avx512_mask_add_sd_round:
-    case Intrinsic::x86_avx512_mask_div_sd_round:
-    case Intrinsic::x86_avx512_mask_mul_sd_round:
-    case Intrinsic::x86_avx512_mask_sub_sd_round:
-    case Intrinsic::x86_avx512_mask_max_sd_round:
-    case Intrinsic::x86_avx512_mask_min_sd_round:
-      simplifyAndSetOp(II, 0, DemandedElts, UndefElts);
-
-      // If lowest element of a scalar op isn't used then use Arg0.
-      if (!DemandedElts[0]) {
-        Worklist.push(II);
-        return II->getArgOperand(0);
-      }
-
-      // Only lower element is used for operand 1 and 2.
-      DemandedElts = 1;
-      simplifyAndSetOp(II, 1, DemandedElts, UndefElts2);
-      simplifyAndSetOp(II, 2, DemandedElts, UndefElts3);
-
-      // Lower element is undefined if all three lower elements are undefined.
-      // Consider things like undef&0.  The result is known zero, not undef.
-      if (!UndefElts2[0] || !UndefElts3[0])
-        UndefElts.clearBit(0);
-
-      break;
-
-    case Intrinsic::x86_sse2_packssdw_128:
-    case Intrinsic::x86_sse2_packsswb_128:
-    case Intrinsic::x86_sse2_packuswb_128:
-    case Intrinsic::x86_sse41_packusdw:
-    case Intrinsic::x86_avx2_packssdw:
-    case Intrinsic::x86_avx2_packsswb:
-    case Intrinsic::x86_avx2_packusdw:
-    case Intrinsic::x86_avx2_packuswb:
-    case Intrinsic::x86_avx512_packssdw_512:
-    case Intrinsic::x86_avx512_packsswb_512:
-    case Intrinsic::x86_avx512_packusdw_512:
-    case Intrinsic::x86_avx512_packuswb_512: {
-      auto *Ty0 = II->getArgOperand(0)->getType();
-      unsigned InnerVWidth = cast<VectorType>(Ty0)->getNumElements();
-      assert(VWidth == (InnerVWidth * 2) && "Unexpected input size");
-
-      unsigned NumLanes = Ty0->getPrimitiveSizeInBits() / 128;
-      unsigned VWidthPerLane = VWidth / NumLanes;
-      unsigned InnerVWidthPerLane = InnerVWidth / NumLanes;
-
-      // Per lane, pack the elements of the first input and then the second.
-      // e.g.
-      // v8i16 PACK(v4i32 X, v4i32 Y) - (X[0..3],Y[0..3])
-      // v32i8 PACK(v16i16 X, v16i16 Y) - (X[0..7],Y[0..7]),(X[8..15],Y[8..15])
-      for (int OpNum = 0; OpNum != 2; ++OpNum) {
-        APInt OpDemandedElts(InnerVWidth, 0);
-        for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
-          unsigned LaneIdx = Lane * VWidthPerLane;
-          for (unsigned Elt = 0; Elt != InnerVWidthPerLane; ++Elt) {
-            unsigned Idx = LaneIdx + Elt + InnerVWidthPerLane * OpNum;
-            if (DemandedElts[Idx])
-              OpDemandedElts.setBit((Lane * InnerVWidthPerLane) + Elt);
-          }
-        }
-
-        // Demand elements from the operand.
-        APInt OpUndefElts(InnerVWidth, 0);
-        simplifyAndSetOp(II, OpNum, OpDemandedElts, OpUndefElts);
-
-        // Pack the operand's UNDEF elements, one lane at a time.
-        OpUndefElts = OpUndefElts.zext(VWidth);
-        for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
-          APInt LaneElts = OpUndefElts.lshr(InnerVWidthPerLane * Lane);
-          LaneElts = LaneElts.getLoBits(InnerVWidthPerLane);
-          LaneElts <<= InnerVWidthPerLane * (2 * Lane + OpNum);
-          UndefElts |= LaneElts;
-        }
-      }
-      break;
-    }
-
-    // PSHUFB
-    case Intrinsic::x86_ssse3_pshuf_b_128:
-    case Intrinsic::x86_avx2_pshuf_b:
-    case Intrinsic::x86_avx512_pshuf_b_512:
-    // PERMILVAR
-    case Intrinsic::x86_avx_vpermilvar_ps:
-    case Intrinsic::x86_avx_vpermilvar_ps_256:
-    case Intrinsic::x86_avx512_vpermilvar_ps_512:
-    case Intrinsic::x86_avx_vpermilvar_pd:
-    case Intrinsic::x86_avx_vpermilvar_pd_256:
-    case Intrinsic::x86_avx512_vpermilvar_pd_512:
-    // PERMV
-    case Intrinsic::x86_avx2_permd:
-    case Intrinsic::x86_avx2_permps: {
-      simplifyAndSetOp(II, 1, DemandedElts, UndefElts);
-      break;
-    }
-
-    // SSE4A instructions leave the upper 64-bits of the 128-bit result
-    // in an undefined state.
-    case Intrinsic::x86_sse4a_extrq:
-    case Intrinsic::x86_sse4a_extrqi:
-    case Intrinsic::x86_sse4a_insertq:
-    case Intrinsic::x86_sse4a_insertqi:
-      UndefElts.setHighBits(VWidth / 2);
-      break;
-    case Intrinsic::amdgcn_buffer_load:
-    case Intrinsic::amdgcn_buffer_load_format:
-    case Intrinsic::amdgcn_raw_buffer_load:
-    case Intrinsic::amdgcn_raw_buffer_load_format:
-    case Intrinsic::amdgcn_raw_tbuffer_load:
-    case Intrinsic::amdgcn_s_buffer_load:
-    case Intrinsic::amdgcn_struct_buffer_load:
-    case Intrinsic::amdgcn_struct_buffer_load_format:
-    case Intrinsic::amdgcn_struct_tbuffer_load:
-    case Intrinsic::amdgcn_tbuffer_load:
-      return simplifyAMDGCNMemoryIntrinsicDemanded(II, DemandedElts);
     default: {
-      if (getAMDGPUImageDMaskIntrinsic(II->getIntrinsicID()))
-        return simplifyAMDGCNMemoryIntrinsicDemanded(II, DemandedElts, 0);
-
+      // Handle target specific intrinsics
+      Optional<Value *> V = targetSimplifyDemandedVectorEltsIntrinsic(
+          *II, DemandedElts, UndefElts, UndefElts2, UndefElts3,
+          simplifyAndSetOp);
+      if (V.hasValue())
+        return V.getValue();
       break;
     }
     } // switch on IntrinsicID
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index ff70347569abc..bcda851028933 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -35,6 +35,7 @@
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Transforms/InstCombine/InstCombineWorklist.h"
+#include "llvm/Transforms/InstCombine/InstCombiner.h"
 #include <cassert>
 #include <cstdint>
 #include <iterator>
@@ -85,7 +86,8 @@ static bool cheapToScalarize(Value *V, bool IsConstantExtractIndex) {
 // If we have a PHI node with a vector type that is only used to feed
 // itself and be an operand of extractelement at a constant location,
 // try to replace the PHI of the vector type with a PHI of a scalar type.
-Instruction *InstCombiner::scalarizePHI(ExtractElementInst &EI, PHINode *PN) {
+Instruction *InstCombinerImpl::scalarizePHI(ExtractElementInst &EI,
+                                            PHINode *PN) {
   SmallVector<Instruction *, 2> Extracts;
   // The users we want the PHI to have are:
   // 1) The EI ExtractElement (we already know this)
@@ -321,7 +323,7 @@ static APInt findDemandedEltsByAllUsers(Value *V) {
   return UnionUsedElts;
 }
 
-Instruction *InstCombiner::visitExtractElementInst(ExtractElementInst &EI) {
+Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) {
   Value *SrcVec = EI.getVectorOperand();
   Value *Index = EI.getIndexOperand();
   if (Value *V = SimplifyExtractElementInst(SrcVec, Index,
@@ -531,7 +533,7 @@ static bool collectSingleShuffleElements(Value *V, Value *LHS, Value *RHS,
 /// shufflevector to replace one or more insert/extract pairs.
 static void replaceExtractElements(InsertElementInst *InsElt,
                                    ExtractElementInst *ExtElt,
-                                   InstCombiner &IC) {
+                                   InstCombinerImpl &IC) {
   VectorType *InsVecType = InsElt->getType();
   VectorType *ExtVecType = ExtElt->getVectorOperandType();
   unsigned NumInsElts = InsVecType->getNumElements();
@@ -614,7 +616,7 @@ using ShuffleOps = std::pair<Value *, Value *>;
 
 static ShuffleOps collectShuffleElements(Value *V, SmallVectorImpl<int> &Mask,
                                          Value *PermittedRHS,
-                                         InstCombiner &IC) {
+                                         InstCombinerImpl &IC) {
   assert(V->getType()->isVectorTy() && "Invalid shuffle!");
   unsigned NumElts = cast<FixedVectorType>(V->getType())->getNumElements();
 
@@ -699,7 +701,7 @@ static ShuffleOps collectShuffleElements(Value *V, SmallVectorImpl<int> &Mask,
 /// first one, making the first one redundant.
 /// It should be transformed to:
 ///  %0 = insertvalue { i8, i32 } undef, i8 %y, 0
-Instruction *InstCombiner::visitInsertValueInst(InsertValueInst &I) {
+Instruction *InstCombinerImpl::visitInsertValueInst(InsertValueInst &I) {
   bool IsRedundant = false;
   ArrayRef<unsigned int> FirstIndices = I.getIndices();
 
@@ -1041,7 +1043,7 @@ static Instruction *foldConstantInsEltIntoShuffle(InsertElementInst &InsElt) {
   return nullptr;
 }
 
-Instruction *InstCombiner::visitInsertElementInst(InsertElementInst &IE) {
+Instruction *InstCombinerImpl::visitInsertElementInst(InsertElementInst &IE) {
   Value *VecOp    = IE.getOperand(0);
   Value *ScalarOp = IE.getOperand(1);
   Value *IdxOp    = IE.getOperand(2);
@@ -1525,7 +1527,7 @@ static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf) {
       is_contained(Mask, UndefMaskElem) &&
       (Instruction::isIntDivRem(BOpcode) || Instruction::isShift(BOpcode));
   if (MightCreatePoisonOrUB)
-    NewC = getSafeVectorConstantForBinop(BOpcode, NewC, true);
+    NewC = InstCombiner::getSafeVectorConstantForBinop(BOpcode, NewC, true);
 
   // shuf (bop X, C), X, M --> bop X, C'
   // shuf X, (bop X, C), M --> bop X, C'
@@ -1652,7 +1654,8 @@ static Instruction *foldSelectShuffle(ShuffleVectorInst &Shuf,
       is_contained(Mask, UndefMaskElem) &&
       (Instruction::isIntDivRem(BOpc) || Instruction::isShift(BOpc));
   if (MightCreatePoisonOrUB)
-    NewC = getSafeVectorConstantForBinop(BOpc, NewC, ConstantsAreOp1);
+    NewC = InstCombiner::getSafeVectorConstantForBinop(BOpc, NewC,
+                                                       ConstantsAreOp1);
 
   Value *V;
   if (X == Y) {
@@ -1823,7 +1826,7 @@ static Instruction *foldIdentityExtractShuffle(ShuffleVectorInst &Shuf) {
 /// Try to replace a shuffle with an insertelement or try to replace a shuffle
 /// operand with the operand of an insertelement.
 static Instruction *foldShuffleWithInsert(ShuffleVectorInst &Shuf,
-                                          InstCombiner &IC) {
+                                          InstCombinerImpl &IC) {
   Value *V0 = Shuf.getOperand(0), *V1 = Shuf.getOperand(1);
   SmallVector<int, 16> Mask;
   Shuf.getShuffleMask(Mask);
@@ -1974,7 +1977,7 @@ static Instruction *foldIdentityPaddedShuffles(ShuffleVectorInst &Shuf) {
   return new ShuffleVectorInst(X, Y, NewMask);
 }
 
-Instruction *InstCombiner::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
+Instruction *InstCombinerImpl::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
   Value *LHS = SVI.getOperand(0);
   Value *RHS = SVI.getOperand(1);
   SimplifyQuery ShufQuery = SQ.getWithInstruction(&SVI);
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 8eac8637cb9e7..fdf0aaf9b176d 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -59,6 +59,7 @@
 #include "llvm/Analysis/ProfileSummaryInfo.h"
 #include "llvm/Analysis/TargetFolder.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/BasicBlock.h"
@@ -160,7 +161,41 @@ MaxArraySize("instcombine-maxarray-size", cl::init(1024),
 static cl::opt<unsigned> ShouldLowerDbgDeclare("instcombine-lower-dbg-declare",
                                                cl::Hidden, cl::init(true));
 
-Value *InstCombiner::EmitGEPOffset(User *GEP) {
+Optional<Instruction *>
+InstCombiner::targetInstCombineIntrinsic(IntrinsicInst &II) {
+  // Handle target specific intrinsics
+  if (II.getCalledFunction()->isTargetIntrinsic()) {
+    return TTI.instCombineIntrinsic(*this, II);
+  }
+  return None;
+}
+
+Optional<Value *> InstCombiner::targetSimplifyDemandedUseBitsIntrinsic(
+    IntrinsicInst &II, APInt DemandedMask, KnownBits &Known,
+    bool &KnownBitsComputed) {
+  // Handle target specific intrinsics
+  if (II.getCalledFunction()->isTargetIntrinsic()) {
+    return TTI.simplifyDemandedUseBitsIntrinsic(*this, II, DemandedMask, Known,
+                                                KnownBitsComputed);
+  }
+  return None;
+}
+
+Optional<Value *> InstCombiner::targetSimplifyDemandedVectorEltsIntrinsic(
+    IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts, APInt &UndefElts2,
+    APInt &UndefElts3,
+    std::function<void(Instruction *, unsigned, APInt, APInt &)>
+        SimplifyAndSetOp) {
+  // Handle target specific intrinsics
+  if (II.getCalledFunction()->isTargetIntrinsic()) {
+    return TTI.simplifyDemandedVectorEltsIntrinsic(
+        *this, II, DemandedElts, UndefElts, UndefElts2, UndefElts3,
+        SimplifyAndSetOp);
+  }
+  return None;
+}
+
+Value *InstCombinerImpl::EmitGEPOffset(User *GEP) {
   return llvm::EmitGEPOffset(&Builder, DL, GEP);
 }
 
@@ -173,8 +208,8 @@ Value *InstCombiner::EmitGEPOffset(User *GEP) {
 /// legal to convert to, in order to open up more combining opportunities.
 /// NOTE: this treats i8, i16 and i32 specially, due to them being so common
 /// from frontend languages.
-bool InstCombiner::shouldChangeType(unsigned FromWidth,
-                                    unsigned ToWidth) const {
+bool InstCombinerImpl::shouldChangeType(unsigned FromWidth,
+                                        unsigned ToWidth) const {
   bool FromLegal = FromWidth == 1 || DL.isLegalInteger(FromWidth);
   bool ToLegal = ToWidth == 1 || DL.isLegalInteger(ToWidth);
 
@@ -201,7 +236,7 @@ bool InstCombiner::shouldChangeType(unsigned FromWidth,
 /// to a larger illegal type. i1 is always treated as a legal type because it is
 /// a fundamental type in IR, and there are many specialized optimizations for
 /// i1 types.
-bool InstCombiner::shouldChangeType(Type *From, Type *To) const {
+bool InstCombinerImpl::shouldChangeType(Type *From, Type *To) const {
   // TODO: This could be extended to allow vectors. Datalayout changes might be
   // needed to properly support that.
   if (!From->isIntegerTy() || !To->isIntegerTy())
@@ -269,7 +304,8 @@ static void ClearSubclassDataAfterReassociation(BinaryOperator &I) {
 /// cast to eliminate one of the associative operations:
 /// (op (cast (op X, C2)), C1) --> (cast (op X, op (C1, C2)))
 /// (op (cast (op X, C2)), C1) --> (op (cast X), op (C1, C2))
-static bool simplifyAssocCastAssoc(BinaryOperator *BinOp1, InstCombiner &IC) {
+static bool simplifyAssocCastAssoc(BinaryOperator *BinOp1,
+                                   InstCombinerImpl &IC) {
   auto *Cast = dyn_cast<CastInst>(BinOp1->getOperand(0));
   if (!Cast || !Cast->hasOneUse())
     return false;
@@ -327,7 +363,7 @@ static bool simplifyAssocCastAssoc(BinaryOperator *BinOp1, InstCombiner &IC) {
 ///  5. Transform: "A op (B op C)" ==> "B op (C op A)" if "C op A" simplifies.
 ///  6. Transform: "(A op C1) op (B op C2)" ==> "(A op B) op (C1 op C2)"
 ///     if C1 and C2 are constants.
-bool InstCombiner::SimplifyAssociativeOrCommutative(BinaryOperator &I) {
+bool InstCombinerImpl::SimplifyAssociativeOrCommutative(BinaryOperator &I) {
   Instruction::BinaryOps Opcode = I.getOpcode();
   bool Changed = false;
 
@@ -555,9 +591,10 @@ getBinOpsForFactorization(Instruction::BinaryOps TopOpcode, BinaryOperator *Op,
 
 /// This tries to simplify binary operations by factorizing out common terms
 /// (e. g. "(A*B)+(A*C)" -> "A*(B+C)").
-Value *InstCombiner::tryFactorization(BinaryOperator &I,
-                                      Instruction::BinaryOps InnerOpcode,
-                                      Value *A, Value *B, Value *C, Value *D) {
+Value *InstCombinerImpl::tryFactorization(BinaryOperator &I,
+                                          Instruction::BinaryOps InnerOpcode,
+                                          Value *A, Value *B, Value *C,
+                                          Value *D) {
   assert(A && B && C && D && "All values must be provided");
 
   Value *V = nullptr;
@@ -660,7 +697,7 @@ Value *InstCombiner::tryFactorization(BinaryOperator &I,
 /// (eg "(A*B)+(A*C)" -> "A*(B+C)") or expanding out if this results in
 /// simplifications (eg: "A & (B | C) -> (A&B) | (A&C)" if this is a win).
 /// Returns the simplified value, or null if it didn't simplify.
-Value *InstCombiner::SimplifyUsingDistributiveLaws(BinaryOperator &I) {
+Value *InstCombinerImpl::SimplifyUsingDistributiveLaws(BinaryOperator &I) {
   Value *LHS = I.getOperand(0), *RHS = I.getOperand(1);
   BinaryOperator *Op0 = dyn_cast<BinaryOperator>(LHS);
   BinaryOperator *Op1 = dyn_cast<BinaryOperator>(RHS);
@@ -774,8 +811,9 @@ Value *InstCombiner::SimplifyUsingDistributiveLaws(BinaryOperator &I) {
   return SimplifySelectsFeedingBinaryOp(I, LHS, RHS);
 }
 
-Value *InstCombiner::SimplifySelectsFeedingBinaryOp(BinaryOperator &I,
-                                                    Value *LHS, Value *RHS) {
+Value *InstCombinerImpl::SimplifySelectsFeedingBinaryOp(BinaryOperator &I,
+                                                        Value *LHS,
+                                                        Value *RHS) {
   Value *A, *B, *C, *D, *E, *F;
   bool LHSIsSelect = match(LHS, m_Select(m_Value(A), m_Value(B), m_Value(C)));
   bool RHSIsSelect = match(RHS, m_Select(m_Value(D), m_Value(E), m_Value(F)));
@@ -827,7 +865,7 @@ Value *InstCombiner::SimplifySelectsFeedingBinaryOp(BinaryOperator &I,
 
 /// Given a 'sub' instruction, return the RHS of the instruction if the LHS is a
 /// constant zero (which is the 'negate' form).
-Value *InstCombiner::dyn_castNegVal(Value *V) const {
+Value *InstCombinerImpl::dyn_castNegVal(Value *V) const {
   Value *NegV;
   if (match(V, m_Neg(m_Value(NegV))))
     return NegV;
@@ -888,7 +926,8 @@ static Value *foldOperationIntoSelectOperand(Instruction &I, Value *SO,
   return RI;
 }
 
-Instruction *InstCombiner::FoldOpIntoSelect(Instruction &Op, SelectInst *SI) {
+Instruction *InstCombinerImpl::FoldOpIntoSelect(Instruction &Op,
+                                                SelectInst *SI) {
   // Don't modify shared select instructions.
   if (!SI->hasOneUse())
     return nullptr;
@@ -983,7 +1022,7 @@ static Value *foldOperationIntoPhiValue(BinaryOperator *I, Value *InV,
   return RI;
 }
 
-Instruction *InstCombiner::foldOpIntoPhi(Instruction &I, PHINode *PN) {
+Instruction *InstCombinerImpl::foldOpIntoPhi(Instruction &I, PHINode *PN) {
   unsigned NumPHIValues = PN->getNumIncomingValues();
   if (NumPHIValues == 0)
     return nullptr;
@@ -1125,7 +1164,7 @@ Instruction *InstCombiner::foldOpIntoPhi(Instruction &I, PHINode *PN) {
   return replaceInstUsesWith(I, NewPN);
 }
 
-Instruction *InstCombiner::foldBinOpIntoSelectOrPhi(BinaryOperator &I) {
+Instruction *InstCombinerImpl::foldBinOpIntoSelectOrPhi(BinaryOperator &I) {
   if (!isa<Constant>(I.getOperand(1)))
     return nullptr;
 
@@ -1143,8 +1182,9 @@ Instruction *InstCombiner::foldBinOpIntoSelectOrPhi(BinaryOperator &I) {
 /// is a sequence of GEP indices into the pointed type that will land us at the
 /// specified offset. If so, fill them into NewIndices and return the resultant
 /// element type, otherwise return null.
-Type *InstCombiner::FindElementAtOffset(PointerType *PtrTy, int64_t Offset,
-                                        SmallVectorImpl<Value *> &NewIndices) {
+Type *
+InstCombinerImpl::FindElementAtOffset(PointerType *PtrTy, int64_t Offset,
+                                      SmallVectorImpl<Value *> &NewIndices) {
   Type *Ty = PtrTy->getElementType();
   if (!Ty->isSized())
     return nullptr;
@@ -1213,7 +1253,7 @@ static bool shouldMergeGEPs(GEPOperator &GEP, GEPOperator &Src) {
 
 /// Return a value X such that Val = X * Scale, or null if none.
 /// If the multiplication is known not to overflow, then NoSignedWrap is set.
-Value *InstCombiner::Descale(Value *Val, APInt Scale, bool &NoSignedWrap) {
+Value *InstCombinerImpl::Descale(Value *Val, APInt Scale, bool &NoSignedWrap) {
   assert(isa<IntegerType>(Val->getType()) && "Can only descale integers!");
   assert(cast<IntegerType>(Val->getType())->getBitWidth() ==
          Scale.getBitWidth() && "Scale not compatible with value!");
@@ -1453,7 +1493,7 @@ Value *InstCombiner::Descale(Value *Val, APInt Scale, bool &NoSignedWrap) {
   } while (true);
 }
 
-Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
+Instruction *InstCombinerImpl::foldVectorBinop(BinaryOperator &Inst) {
   // FIXME: some of this is likely fine for scalable vectors
   if (!isa<FixedVectorType>(Inst.getType()))
     return nullptr;
@@ -1675,7 +1715,7 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
 /// Try to narrow the width of a binop if at least 1 operand is an extend of
 /// of a value. This requires a potentially expensive known bits check to make
 /// sure the narrow op does not overflow.
-Instruction *InstCombiner::narrowMathIfNoOverflow(BinaryOperator &BO) {
+Instruction *InstCombinerImpl::narrowMathIfNoOverflow(BinaryOperator &BO) {
   // We need at least one extended operand.
   Value *Op0 = BO.getOperand(0), *Op1 = BO.getOperand(1);
 
@@ -1764,7 +1804,7 @@ static Instruction *foldSelectGEP(GetElementPtrInst &GEP,
   return SelectInst::Create(Cond, NewTrueC, NewFalseC, "", nullptr, Sel);
 }
 
-Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
+Instruction *InstCombinerImpl::visitGetElementPtrInst(GetElementPtrInst &GEP) {
   SmallVector<Value*, 8> Ops(GEP.op_begin(), GEP.op_end());
   Type *GEPType = GEP.getType();
   Type *GEPEltType = GEP.getSourceElementType();
@@ -2516,7 +2556,7 @@ static bool isAllocSiteRemovable(Instruction *AI,
   return true;
 }
 
-Instruction *InstCombiner::visitAllocSite(Instruction &MI) {
+Instruction *InstCombinerImpl::visitAllocSite(Instruction &MI) {
   // If we have a malloc call which is only used in any amount of comparisons to
   // null and free calls, delete the calls and replace the comparisons with true
   // or false as appropriate.
@@ -2675,7 +2715,7 @@ static Instruction *tryToMoveFreeBeforeNullTest(CallInst &FI,
   return &FI;
 }
 
-Instruction *InstCombiner::visitFree(CallInst &FI) {
+Instruction *InstCombinerImpl::visitFree(CallInst &FI) {
   Value *Op = FI.getArgOperand(0);
 
   // free undef -> unreachable.
@@ -2716,7 +2756,7 @@ static bool isMustTailCall(Value *V) {
   return false;
 }
 
-Instruction *InstCombiner::visitReturnInst(ReturnInst &RI) {
+Instruction *InstCombinerImpl::visitReturnInst(ReturnInst &RI) {
   if (RI.getNumOperands() == 0) // ret void
     return nullptr;
 
@@ -2739,7 +2779,7 @@ Instruction *InstCombiner::visitReturnInst(ReturnInst &RI) {
   return nullptr;
 }
 
-Instruction *InstCombiner::visitUnconditionalBranchInst(BranchInst &BI) {
+Instruction *InstCombinerImpl::visitUnconditionalBranchInst(BranchInst &BI) {
   assert(BI.isUnconditional() && "Only for unconditional branches.");
 
   // If this store is the second-to-last instruction in the basic block
@@ -2768,7 +2808,7 @@ Instruction *InstCombiner::visitUnconditionalBranchInst(BranchInst &BI) {
   return nullptr;
 }
 
-Instruction *InstCombiner::visitBranchInst(BranchInst &BI) {
+Instruction *InstCombinerImpl::visitBranchInst(BranchInst &BI) {
   if (BI.isUnconditional())
     return visitUnconditionalBranchInst(BI);
 
@@ -2804,7 +2844,7 @@ Instruction *InstCombiner::visitBranchInst(BranchInst &BI) {
   return nullptr;
 }
 
-Instruction *InstCombiner::visitSwitchInst(SwitchInst &SI) {
+Instruction *InstCombinerImpl::visitSwitchInst(SwitchInst &SI) {
   Value *Cond = SI.getCondition();
   Value *Op0;
   ConstantInt *AddRHS;
@@ -2835,7 +2875,7 @@ Instruction *InstCombiner::visitSwitchInst(SwitchInst &SI) {
   unsigned NewWidth = Known.getBitWidth() - std::max(LeadingKnownZeros, LeadingKnownOnes);
 
   // Shrink the condition operand if the new type is smaller than the old type.
-  // But do not shrink to a non-standard type, because backend can't generate 
+  // But do not shrink to a non-standard type, because backend can't generate
   // good code for that yet.
   // TODO: We can make it aggressive again after fixing PR39569.
   if (NewWidth > 0 && NewWidth < Known.getBitWidth() &&
@@ -2854,7 +2894,7 @@ Instruction *InstCombiner::visitSwitchInst(SwitchInst &SI) {
   return nullptr;
 }
 
-Instruction *InstCombiner::visitExtractValueInst(ExtractValueInst &EV) {
+Instruction *InstCombinerImpl::visitExtractValueInst(ExtractValueInst &EV) {
   Value *Agg = EV.getAggregateOperand();
 
   if (!EV.hasIndices())
@@ -3015,7 +3055,7 @@ static bool shorter_filter(const Value *LHS, const Value *RHS) {
     cast<ArrayType>(RHS->getType())->getNumElements();
 }
 
-Instruction *InstCombiner::visitLandingPadInst(LandingPadInst &LI) {
+Instruction *InstCombinerImpl::visitLandingPadInst(LandingPadInst &LI) {
   // The logic here should be correct for any real-world personality function.
   // However if that turns out not to be true, the offending logic can always
   // be conditioned on the personality function, like the catch-all logic is.
@@ -3324,7 +3364,7 @@ Instruction *InstCombiner::visitLandingPadInst(LandingPadInst &LI) {
   return nullptr;
 }
 
-Instruction *InstCombiner::visitFreeze(FreezeInst &I) {
+Instruction *InstCombinerImpl::visitFreeze(FreezeInst &I) {
   Value *Op0 = I.getOperand(0);
 
   if (Value *V = SimplifyFreezeInst(Op0, SQ.getWithInstruction(&I)))
@@ -3435,7 +3475,7 @@ static bool TryToSinkInstruction(Instruction *I, BasicBlock *DestBlock) {
   return true;
 }
 
-bool InstCombiner::run() {
+bool InstCombinerImpl::run() {
   while (!Worklist.isEmpty()) {
     // Walk deferred instructions in reverse order, and push them to the
     // worklist, which means they'll end up popped from the worklist in-order.
@@ -3718,8 +3758,8 @@ static bool prepareICWorklistFromFunction(Function &F, const DataLayout &DL,
 
 static bool combineInstructionsOverFunction(
     Function &F, InstCombineWorklist &Worklist, AliasAnalysis *AA,
-    AssumptionCache &AC, TargetLibraryInfo &TLI, DominatorTree &DT,
-    OptimizationRemarkEmitter &ORE, BlockFrequencyInfo *BFI,
+    AssumptionCache &AC, TargetLibraryInfo &TLI, TargetTransformInfo &TTI,
+    DominatorTree &DT, OptimizationRemarkEmitter &ORE, BlockFrequencyInfo *BFI,
     ProfileSummaryInfo *PSI, unsigned MaxIterations, LoopInfo *LI) {
   auto &DL = F.getParent()->getDataLayout();
   MaxIterations = std::min(MaxIterations, LimitMaxIterations.getValue());
@@ -3763,8 +3803,8 @@ static bool combineInstructionsOverFunction(
 
     MadeIRChange |= prepareICWorklistFromFunction(F, DL, &TLI, Worklist);
 
-    InstCombiner IC(Worklist, Builder, F.hasMinSize(), AA,
-                    AC, TLI, DT, ORE, BFI, PSI, DL, LI);
+    InstCombinerImpl IC(Worklist, Builder, F.hasMinSize(), AA, AC, TLI, TTI, DT,
+                        ORE, BFI, PSI, DL, LI);
     IC.MaxArraySizeForCombine = MaxArraySize;
 
     if (!IC.run())
@@ -3787,6 +3827,7 @@ PreservedAnalyses InstCombinePass::run(Function &F,
   auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
   auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
   auto &ORE = AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
+  auto &TTI = AM.getResult<TargetIRAnalysis>(F);
 
   auto *LI = AM.getCachedResult<LoopAnalysis>(F);
 
@@ -3797,8 +3838,8 @@ PreservedAnalyses InstCombinePass::run(Function &F,
   auto *BFI = (PSI && PSI->hasProfileSummary()) ?
       &AM.getResult<BlockFrequencyAnalysis>(F) : nullptr;
 
-  if (!combineInstructionsOverFunction(F, Worklist, AA, AC, TLI, DT, ORE, BFI,
-                                       PSI, MaxIterations, LI))
+  if (!combineInstructionsOverFunction(F, Worklist, AA, AC, TLI, TTI, DT, ORE,
+                                       BFI, PSI, MaxIterations, LI))
     // No changes, all analyses are preserved.
     return PreservedAnalyses::all();
 
@@ -3816,6 +3857,7 @@ void InstructionCombiningPass::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addRequired<AAResultsWrapperPass>();
   AU.addRequired<AssumptionCacheTracker>();
   AU.addRequired<TargetLibraryInfoWrapperPass>();
+  AU.addRequired<TargetTransformInfoWrapperPass>();
   AU.addRequired<DominatorTreeWrapperPass>();
   AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
   AU.addPreserved<DominatorTreeWrapperPass>();
@@ -3834,6 +3876,7 @@ bool InstructionCombiningPass::runOnFunction(Function &F) {
   auto AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
   auto &AC = getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
   auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
+  auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
   auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
   auto &ORE = getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
 
@@ -3847,8 +3890,8 @@ bool InstructionCombiningPass::runOnFunction(Function &F) {
       &getAnalysis<LazyBlockFrequencyInfoPass>().getBFI() :
       nullptr;
 
-  return combineInstructionsOverFunction(F, Worklist, AA, AC, TLI, DT, ORE, BFI,
-                                         PSI, MaxIterations, LI);
+  return combineInstructionsOverFunction(F, Worklist, AA, AC, TLI, TTI, DT, ORE,
+                                         BFI, PSI, MaxIterations, LI);
 }
 
 char InstructionCombiningPass::ID = 0;
@@ -3867,6 +3910,7 @@ INITIALIZE_PASS_BEGIN(InstructionCombiningPass, "instcombine",
                       "Combine redundant instructions", false, false)
 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass)
diff --git a/llvm/lib/Transforms/Instrumentation/CFGMST.h b/llvm/lib/Transforms/Instrumentation/CFGMST.h
index 9addb5d1ba938..6580b6d7d73e8 100644
--- a/llvm/lib/Transforms/Instrumentation/CFGMST.h
+++ b/llvm/lib/Transforms/Instrumentation/CFGMST.h
@@ -20,7 +20,6 @@
 #include "llvm/Analysis/BranchProbabilityInfo.h"
 #include "llvm/Analysis/CFG.h"
 #include "llvm/Support/BranchProbability.h"
-#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
@@ -30,9 +29,6 @@
 #define DEBUG_TYPE "cfgmst"
 
 using namespace llvm;
-static cl::opt<bool> PGOInstrumentEntry(
-    "pgo-instrument-entry", cl::init(false), cl::Hidden,
-    cl::desc("Force to instrument function entry basicblock."));
 
 namespace llvm {
 
@@ -107,7 +103,7 @@ template <class Edge, class BBInfo> class CFGMST {
     const BasicBlock *Entry = &(F.getEntryBlock());
     uint64_t EntryWeight = (BFI != nullptr ? BFI->getEntryFreq() : 2);
     // If we want to instrument the entry count, lower the weight to 0.
-    if (PGOInstrumentEntry)
+    if (InstrumentFuncEntry)
       EntryWeight = 0;
     Edge *EntryIncoming = nullptr, *EntryOutgoing = nullptr,
          *ExitOutgoing = nullptr, *ExitIncoming = nullptr;
@@ -282,14 +278,19 @@ template <class Edge, class BBInfo> class CFGMST {
   BranchProbabilityInfo *BPI;
   BlockFrequencyInfo *BFI;
 
+  // If function entry will be always instrumented.
+  bool InstrumentFuncEntry;
+
 public:
-  CFGMST(Function &Func, BranchProbabilityInfo *BPI_ = nullptr,
+  CFGMST(Function &Func, bool InstrumentFuncEntry_,
+         BranchProbabilityInfo *BPI_ = nullptr,
          BlockFrequencyInfo *BFI_ = nullptr)
-      : F(Func), BPI(BPI_), BFI(BFI_) {
+      : F(Func), BPI(BPI_), BFI(BFI_),
+        InstrumentFuncEntry(InstrumentFuncEntry_) {
     buildEdges();
     sortEdgesByWeight();
     computeMinimumSpanningTree();
-    if (PGOInstrumentEntry && (AllEdges.size() > 1))
+    if (AllEdges.size() > 1 && InstrumentFuncEntry)
       std::iter_swap(std::move(AllEdges.begin()),
                      std::move(AllEdges.begin() + AllEdges.size() - 1));
   }
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 0a3519502994b..7b03bbfcdfe4b 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -57,8 +57,6 @@ using namespace llvm;
 
 #define DEBUG_TYPE "instrprof"
 
-// FIXME: These are to be removed after switching to the new memop value
-// profiling.
 // The start and end values of precise value profile range for memory
 // intrinsic sizes
 cl::opt<std::string> MemOPSizeRange(
@@ -74,12 +72,6 @@ cl::opt<unsigned> MemOPSizeLarge(
              "Value of 0 disables the large value profiling."),
     cl::init(8192));
 
-cl::opt<bool> UseOldMemOpValueProf(
-    "use-old-memop-value-prof",
-    cl::desc("Use the old memop value profiling buckets. This is "
-             "transitional and to be removed after switching. "),
-    cl::init(true));
-
 namespace {
 
 cl::opt<bool> DoHashBasedCounterSplit(
@@ -403,19 +395,6 @@ class PGOCounterPromoter {
   BlockFrequencyInfo *BFI;
 };
 
-enum class ValueProfilingCallType {
-  // Individual values are tracked. Currently used for indiret call target
-  // profiling.
-  Default,
-
-  // The old memop size value profiling. FIXME: To be removed after switching to
-  // the new one.
-  OldMemOp,
-
-  // MemOp: the (new) memop size value profiling with extended buckets.
-  MemOp
-};
-
 } // end anonymous namespace
 
 PreservedAnalyses InstrProfiling::run(Module &M, ModuleAnalysisManager &AM) {
@@ -600,9 +579,9 @@ bool InstrProfiling::run(
   return true;
 }
 
-static FunctionCallee getOrInsertValueProfilingCall(
-    Module &M, const TargetLibraryInfo &TLI,
-    ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
+static FunctionCallee
+getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI,
+                              bool IsRange = false) {
   LLVMContext &Ctx = M.getContext();
   auto *ReturnTy = Type::getVoidTy(M.getContext());
 
@@ -610,22 +589,16 @@ static FunctionCallee getOrInsertValueProfilingCall(
   if (auto AK = TLI.getExtAttrForI32Param(false))
     AL = AL.addParamAttribute(M.getContext(), 2, AK);
 
-  if (CallType == ValueProfilingCallType::Default ||
-      CallType == ValueProfilingCallType::MemOp) {
+  if (!IsRange) {
     Type *ParamTypes[] = {
 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
 #include "llvm/ProfileData/InstrProfData.inc"
     };
     auto *ValueProfilingCallTy =
         FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
-    StringRef FuncName = CallType == ValueProfilingCallType::Default
-                             ? getInstrProfValueProfFuncName()
-                             : getInstrProfValueProfMemOpFuncName();
-    return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL);
+    return M.getOrInsertFunction(getInstrProfValueProfFuncName(),
+                                 ValueProfilingCallTy, AL);
   } else {
-    // FIXME: This code is to be removed after switching to the new memop value
-    // profiling.
-    assert(CallType == ValueProfilingCallType::OldMemOp);
     Type *RangeParamTypes[] = {
 #define VALUE_RANGE_PROF 1
 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
@@ -665,8 +638,8 @@ void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
     Index += It->second.NumValueSites[Kind];
 
   IRBuilder<> Builder(Ind);
-  bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() ==
-                      llvm::InstrProfValueKind::IPVK_MemOPSize);
+  bool IsRange = (Ind->getValueKind()->getZExtValue() ==
+                  llvm::InstrProfValueKind::IPVK_MemOPSize);
   CallInst *Call = nullptr;
   auto *TLI = &GetTLI(*Ind->getFunction());
 
@@ -676,19 +649,12 @@ void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
   // WinEHPrepare pass.
   SmallVector<OperandBundleDef, 1> OpBundles;
   Ind->getOperandBundlesAsDefs(OpBundles);
-  if (!IsMemOpSize) {
+  if (!IsRange) {
     Value *Args[3] = {Ind->getTargetValue(),
                       Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
                       Builder.getInt32(Index)};
     Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args,
                               OpBundles);
-  } else if (!UseOldMemOpValueProf) {
-    Value *Args[3] = {Ind->getTargetValue(),
-                      Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
-                      Builder.getInt32(Index)};
-    Call = Builder.CreateCall(
-        getOrInsertValueProfilingCall(*M, *TLI, ValueProfilingCallType::MemOp),
-        Args, OpBundles);
   } else {
     Value *Args[6] = {
         Ind->getTargetValue(),
@@ -697,8 +663,7 @@ void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
         Builder.getInt64(MemOPSizeRangeStart),
         Builder.getInt64(MemOPSizeRangeLast),
         Builder.getInt64(MemOPSizeLarge == 0 ? INT64_MIN : MemOPSizeLarge)};
-    Call = Builder.CreateCall(getOrInsertValueProfilingCall(
-                                  *M, *TLI, ValueProfilingCallType::OldMemOp),
+    Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI, true),
                               Args, OpBundles);
   }
   if (auto AK = TLI->getExtAttrForI32Param(false))
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index fcf7f470b3e10..f4f62a31d89ee 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -572,6 +572,9 @@ class MemorySanitizer {
   /// uninitialized value and returns an updated origin id encoding this info.
   FunctionCallee MsanChainOriginFn;
 
+  /// Run-time helper that paints an origin over a region.
+  FunctionCallee MsanSetOriginFn;
+
   /// MSan runtime replacements for memmove, memcpy and memset.
   FunctionCallee MemmoveFn, MemcpyFn, MemsetFn;
 
@@ -850,6 +853,9 @@ void MemorySanitizer::initializeCallbacks(Module &M) {
   // instrumentation.
   MsanChainOriginFn = M.getOrInsertFunction(
     "__msan_chain_origin", IRB.getInt32Ty(), IRB.getInt32Ty());
+  MsanSetOriginFn =
+      M.getOrInsertFunction("__msan_set_origin", IRB.getVoidTy(),
+                            IRB.getInt8PtrTy(), IntptrTy, IRB.getInt32Ty());
   MemmoveFn = M.getOrInsertFunction(
     "__msan_memmove", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
     IRB.getInt8PtrTy(), IntptrTy);
@@ -1143,11 +1149,11 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     const DataLayout &DL = F.getParent()->getDataLayout();
     const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment);
     unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
-    if (Shadow->getType()->isAggregateType()) {
+    if (Shadow->getType()->isArrayTy()) {
       paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
                   OriginAlignment);
     } else {
-      Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB);
+      Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
       if (auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
         if (ClCheckConstantShadow && !ConstantShadow->isZeroValue())
           paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
@@ -1166,8 +1172,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
                             IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()),
                             Origin});
       } else {
-        Value *Cmp = IRB.CreateICmpNE(
-            ConvertedShadow, getCleanShadow(ConvertedShadow), "_mscmp");
+        Value *Cmp = convertToBool(ConvertedShadow, IRB, "_mscmp");
         Instruction *CheckTerm = SplitBlockAndInsertIfThen(
             Cmp, &*IRB.GetInsertPoint(), false, MS.OriginStoreWeights);
         IRBuilder<> IRBNew(CheckTerm);
@@ -1218,7 +1223,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
                            bool AsCall) {
     IRBuilder<> IRB(OrigIns);
     LLVM_DEBUG(dbgs() << "  SHAD0 : " << *Shadow << "\n");
-    Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB);
+    Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
     LLVM_DEBUG(dbgs() << "  SHAD1 : " << *ConvertedShadow << "\n");
 
     if (auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
@@ -1240,8 +1245,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
                                                 ? Origin
                                                 : (Value *)IRB.getInt32(0)});
     } else {
-      Value *Cmp = IRB.CreateICmpNE(ConvertedShadow,
-                                    getCleanShadow(ConvertedShadow), "_mscmp");
+      Value *Cmp = convertToBool(ConvertedShadow, IRB, "_mscmp");
       Instruction *CheckTerm = SplitBlockAndInsertIfThen(
           Cmp, OrigIns,
           /* Unreachable */ !MS.Recover, MS.ColdCallWeights);
@@ -1385,14 +1389,49 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     return ty;
   }
 
-  /// Convert a shadow value to it's flattened variant.
-  Value *convertToShadowTyNoVec(Value *V, IRBuilder<> &IRB) {
+  /// Extract combined shadow of struct elements as a bool
+  Value *collapseStructShadow(StructType *Struct, Value *Shadow,
+                              IRBuilder<> &IRB) {
+    Value *FalseVal = IRB.getIntN(/* width */ 1, /* value */ 0);
+    Value *Aggregator = FalseVal;
+
+    for (unsigned Idx = 0; Idx < Struct->getNumElements(); Idx++) {
+      // Combine by ORing together each element's bool shadow
+      Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx);
+      Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
+      Value *ShadowBool = convertToBool(ShadowInner, IRB);
+
+      if (Aggregator != FalseVal)
+        Aggregator = IRB.CreateOr(Aggregator, ShadowBool);
+      else
+        Aggregator = ShadowBool;
+    }
+
+    return Aggregator;
+  }
+
+  /// Convert a shadow value to it's flattened variant. The resulting
+  /// shadow may not necessarily have the same bit width as the input
+  /// value, but it will always be comparable to zero.
+  Value *convertShadowToScalar(Value *V, IRBuilder<> &IRB) {
+    if (StructType *Struct = dyn_cast<StructType>(V->getType()))
+      return collapseStructShadow(Struct, V, IRB);
     Type *Ty = V->getType();
     Type *NoVecTy = getShadowTyNoVec(Ty);
     if (Ty == NoVecTy) return V;
     return IRB.CreateBitCast(V, NoVecTy);
   }
 
+  // Convert a scalar value to an i1 by comparing with 0
+  Value *convertToBool(Value *V, IRBuilder<> &IRB, const Twine &name = "") {
+    Type *VTy = V->getType();
+    assert(VTy->isIntegerTy());
+    if (VTy->getIntegerBitWidth() == 1)
+      // Just converting a bool to a bool, so do nothing.
+      return V;
+    return IRB.CreateICmpNE(V, ConstantInt::get(VTy, 0), name);
+  }
+
   /// Compute the integer shadow offset that corresponds to a given
   /// application address.
   ///
@@ -1726,8 +1765,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     if (!InsertChecks) return;
 #ifndef NDEBUG
     Type *ShadowTy = Shadow->getType();
-    assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy)) &&
-           "Can only insert checks for integer and vector shadow types");
+    assert(
+        (isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) ||
+         isa<StructType>(ShadowTy)) &&
+        "Can only insert checks for integer, vector, and struct shadow types");
 #endif
     InstrumentationList.push_back(
         ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
@@ -1769,6 +1810,24 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     llvm_unreachable("Unknown ordering");
   }
 
+  Value *makeAddReleaseOrderingTable(IRBuilder<> &IRB) {
+    constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
+    uint32_t OrderingTable[NumOrderings] = {};
+
+    OrderingTable[(int)AtomicOrderingCABI::relaxed] =
+        OrderingTable[(int)AtomicOrderingCABI::release] =
+            (int)AtomicOrderingCABI::release;
+    OrderingTable[(int)AtomicOrderingCABI::consume] =
+        OrderingTable[(int)AtomicOrderingCABI::acquire] =
+            OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
+                (int)AtomicOrderingCABI::acq_rel;
+    OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
+        (int)AtomicOrderingCABI::seq_cst;
+
+    return ConstantDataVector::get(IRB.getContext(),
+                                   makeArrayRef(OrderingTable, NumOrderings));
+  }
+
   AtomicOrdering addAcquireOrdering(AtomicOrdering a) {
     switch (a) {
       case AtomicOrdering::NotAtomic:
@@ -1786,6 +1845,24 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     llvm_unreachable("Unknown ordering");
   }
 
+  Value *makeAddAcquireOrderingTable(IRBuilder<> &IRB) {
+    constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
+    uint32_t OrderingTable[NumOrderings] = {};
+
+    OrderingTable[(int)AtomicOrderingCABI::relaxed] =
+        OrderingTable[(int)AtomicOrderingCABI::acquire] =
+            OrderingTable[(int)AtomicOrderingCABI::consume] =
+                (int)AtomicOrderingCABI::acquire;
+    OrderingTable[(int)AtomicOrderingCABI::release] =
+        OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
+            (int)AtomicOrderingCABI::acq_rel;
+    OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
+        (int)AtomicOrderingCABI::seq_cst;
+
+    return ConstantDataVector::get(IRB.getContext(),
+                                   makeArrayRef(OrderingTable, NumOrderings));
+  }
+
   // ------------------- Visitors.
   using InstVisitor<MemorySanitizerVisitor>::visit;
   void visit(Instruction &I) {
@@ -2046,7 +2123,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
           Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin);
           // No point in adding something that might result in 0 origin value.
           if (!ConstOrigin || !ConstOrigin->isNullValue()) {
-            Value *FlatShadow = MSV->convertToShadowTyNoVec(OpShadow, IRB);
+            Value *FlatShadow = MSV->convertShadowToScalar(OpShadow, IRB);
             Value *Cond =
                 IRB.CreateICmpNE(FlatShadow, MSV->getCleanShadow(FlatShadow));
             Origin = IRB.CreateSelect(Cond, OpOrigin, Origin);
@@ -3404,6 +3481,60 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     }
   }
 
+  void visitLibAtomicLoad(CallBase &CB) {
+    IRBuilder<> IRB(&CB);
+    Value *Size = CB.getArgOperand(0);
+    Value *SrcPtr = CB.getArgOperand(1);
+    Value *DstPtr = CB.getArgOperand(2);
+    Value *Ordering = CB.getArgOperand(3);
+    // Convert the call to have at least Acquire ordering to make sure
+    // the shadow operations aren't reordered before it.
+    Value *NewOrdering =
+        IRB.CreateExtractElement(makeAddAcquireOrderingTable(IRB), Ordering);
+    CB.setArgOperand(3, NewOrdering);
+
+    IRBuilder<> NextIRB(CB.getNextNode());
+    NextIRB.SetCurrentDebugLocation(CB.getDebugLoc());
+
+    Value *SrcShadowPtr, *SrcOriginPtr;
+    std::tie(SrcShadowPtr, SrcOriginPtr) =
+        getShadowOriginPtr(SrcPtr, NextIRB, NextIRB.getInt8Ty(), Align(1),
+                           /*isStore*/ false);
+    Value *DstShadowPtr =
+        getShadowOriginPtr(DstPtr, NextIRB, NextIRB.getInt8Ty(), Align(1),
+                           /*isStore*/ true)
+            .first;
+
+    NextIRB.CreateMemCpy(DstShadowPtr, Align(1), SrcShadowPtr, Align(1), Size);
+    if (MS.TrackOrigins) {
+      Value *SrcOrigin = NextIRB.CreateAlignedLoad(MS.OriginTy, SrcOriginPtr,
+                                                   kMinOriginAlignment);
+      Value *NewOrigin = updateOrigin(SrcOrigin, NextIRB);
+      NextIRB.CreateCall(MS.MsanSetOriginFn, {DstPtr, Size, NewOrigin});
+    }
+  }
+
+  void visitLibAtomicStore(CallBase &CB) {
+    IRBuilder<> IRB(&CB);
+    Value *Size = CB.getArgOperand(0);
+    Value *DstPtr = CB.getArgOperand(2);
+    Value *Ordering = CB.getArgOperand(3);
+    // Convert the call to have at least Release ordering to make sure
+    // the shadow operations aren't reordered after it.
+    Value *NewOrdering =
+        IRB.CreateExtractElement(makeAddReleaseOrderingTable(IRB), Ordering);
+    CB.setArgOperand(3, NewOrdering);
+
+    Value *DstShadowPtr =
+        getShadowOriginPtr(DstPtr, IRB, IRB.getInt8Ty(), Align(1),
+                           /*isStore*/ true)
+            .first;
+
+    // Atomic store always paints clean shadow/origin. See file header.
+    IRB.CreateMemSet(DstShadowPtr, getCleanShadow(IRB.getInt8Ty()), Size,
+                     Align(1));
+  }
+
   void visitCallBase(CallBase &CB) {
     assert(!CB.getMetadata("nosanitize"));
     if (CB.isInlineAsm()) {
@@ -3417,6 +3548,23 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
         visitInstruction(CB);
       return;
     }
+    LibFunc LF;
+    if (TLI->getLibFunc(CB, LF)) {
+      // libatomic.a functions need to have special handling because there isn't
+      // a good way to intercept them or compile the library with
+      // instrumentation.
+      switch (LF) {
+      case LibFunc_atomic_load:
+        visitLibAtomicLoad(CB);
+        return;
+      case LibFunc_atomic_store:
+        visitLibAtomicStore(CB);
+        return;
+      default:
+        break;
+      }
+    }
+
     if (auto *Call = dyn_cast<CallInst>(&CB)) {
       assert(!isa<IntrinsicInst>(Call) && "intrinsics are handled elsewhere");
 
@@ -3438,6 +3586,12 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       maybeMarkSanitizerLibraryCallNoBuiltin(Call, TLI);
     }
     IRBuilder<> IRB(&CB);
+    bool MayCheckCall = ClEagerChecks;
+    if (Function *Func = CB.getCalledFunction()) {
+      // __sanitizer_unaligned_{load,store} functions may be called by users
+      // and always expects shadows in the TLS. So don't check them.
+      MayCheckCall &= !Func->getName().startswith("__sanitizer_unaligned_");
+    }
 
     unsigned ArgOffset = 0;
     LLVM_DEBUG(dbgs() << "  CallSite: " << CB << "\n");
@@ -3463,7 +3617,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
 
       bool ByVal = CB.paramHasAttr(i, Attribute::ByVal);
       bool NoUndef = CB.paramHasAttr(i, Attribute::NoUndef);
-      bool EagerCheck = ClEagerChecks && !ByVal && NoUndef;
+      bool EagerCheck = MayCheckCall && !ByVal && NoUndef;
 
       if (EagerCheck) {
         insertShadowCheck(A, &CB);
@@ -3519,7 +3673,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     if (isa<CallInst>(CB) && cast<CallInst>(CB).isMustTailCall())
       return;
 
-    if (ClEagerChecks && CB.hasRetAttr(Attribute::NoUndef)) {
+    if (MayCheckCall && CB.hasRetAttr(Attribute::NoUndef)) {
       setShadow(&CB, getCleanShadow(&CB));
       setOrigin(&CB, getCleanOrigin());
       return;
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index dcfc28887a486..67bf264e8eeaf 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -249,6 +249,10 @@ static cl::opt<bool>
                                    "optimization remarks: -{Rpass|"
                                    "pass-remarks}=pgo-instrumentation"));
 
+static cl::opt<bool> PGOInstrumentEntry(
+    "pgo-instrument-entry", cl::init(false), cl::Hidden,
+    cl::desc("Force to instrument function entry basicblock."));
+
 // Command line option to turn on CFG dot dump after profile annotation.
 // Defined in Analysis/BlockFrequencyInfo.cpp:  -pgo-view-counts
 extern cl::opt<PGOViewCountsType> PGOViewCounts;
@@ -425,7 +429,7 @@ class PGOInstrumentationGenCreateVarLegacyPass : public ModulePass {
 private:
   bool runOnModule(Module &M) override {
     createProfileFileNameVar(M, InstrProfileOutput);
-    createIRLevelProfileFlagVar(M, true);
+    createIRLevelProfileFlagVar(M, /* IsCS */ true, PGOInstrumentEntry);
     return false;
   }
   std::string InstrProfileOutput;
@@ -528,6 +532,9 @@ template <class Edge, class BBInfo> class FuncPGOInstrumentation {
   // Is this is context-sensitive instrumentation.
   bool IsCS;
 
+  // If we instrument function entry BB by default.
+  bool InstrumentFuncEntry;
+
   // A map that stores the Comdat group in function F.
   std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers;
 
@@ -572,9 +579,11 @@ template <class Edge, class BBInfo> class FuncPGOInstrumentation {
       Function &Func, TargetLibraryInfo &TLI,
       std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
       bool CreateGlobalVar = false, BranchProbabilityInfo *BPI = nullptr,
-      BlockFrequencyInfo *BFI = nullptr, bool IsCS = false)
+      BlockFrequencyInfo *BFI = nullptr, bool IsCS = false,
+      bool InstrumentFuncEntry = true)
       : F(Func), IsCS(IsCS), ComdatMembers(ComdatMembers), VPC(Func, TLI),
-        ValueSites(IPVK_Last + 1), SIVisitor(Func), MST(F, BPI, BFI) {
+        ValueSites(IPVK_Last + 1), SIVisitor(Func),
+        MST(F, InstrumentFuncEntry, BPI, BFI) {
     // This should be done before CFG hash computation.
     SIVisitor.countSelects(Func);
     ValueSites[IPVK_MemOPSize] = VPC.get(IPVK_MemOPSize);
@@ -845,8 +854,8 @@ static void instrumentOneFunc(
   // later in getInstrBB() to avoid invalidating it.
   SplitIndirectBrCriticalEdges(F, BPI, BFI);
 
-  FuncPGOInstrumentation<PGOEdge, BBInfo> FuncInfo(F, TLI, ComdatMembers, true,
-                                                   BPI, BFI, IsCS);
+  FuncPGOInstrumentation<PGOEdge, BBInfo> FuncInfo(
+      F, TLI, ComdatMembers, true, BPI, BFI, IsCS, PGOInstrumentEntry);
   std::vector<BasicBlock *> InstrumentBBs;
   FuncInfo.getInstrumentBBs(InstrumentBBs);
   unsigned NumCounters =
@@ -1004,9 +1013,10 @@ class PGOUseFunc {
   PGOUseFunc(Function &Func, Module *Modu, TargetLibraryInfo &TLI,
              std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
              BranchProbabilityInfo *BPI, BlockFrequencyInfo *BFIin,
-             ProfileSummaryInfo *PSI, bool IsCS)
+             ProfileSummaryInfo *PSI, bool IsCS, bool InstrumentFuncEntry)
       : F(Func), M(Modu), BFI(BFIin), PSI(PSI),
-        FuncInfo(Func, TLI, ComdatMembers, false, BPI, BFIin, IsCS),
+        FuncInfo(Func, TLI, ComdatMembers, false, BPI, BFIin, IsCS,
+                 InstrumentFuncEntry),
         FreqAttr(FFA_Normal), IsCS(IsCS) {}
 
   // Read counts for the instrumented BB from profile.
@@ -1514,7 +1524,7 @@ static bool InstrumentAllFunctions(
   // For the context-sensitve instrumentation, we should have a separated pass
   // (before LTO/ThinLTO linking) to create these variables.
   if (!IsCS)
-    createIRLevelProfileFlagVar(M, /* IsCS */ false);
+    createIRLevelProfileFlagVar(M, /* IsCS */ false, PGOInstrumentEntry);
   std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers;
   collectComdatMembers(M, ComdatMembers);
 
@@ -1532,7 +1542,7 @@ static bool InstrumentAllFunctions(
 PreservedAnalyses
 PGOInstrumentationGenCreateVar::run(Module &M, ModuleAnalysisManager &AM) {
   createProfileFileNameVar(M, CSInstrName);
-  createIRLevelProfileFlagVar(M, /* IsCS */ true);
+  createIRLevelProfileFlagVar(M, /* IsCS */ true, PGOInstrumentEntry);
   return PreservedAnalyses::all();
 }
 
@@ -1619,6 +1629,12 @@ static bool annotateAllFunctions(
   collectComdatMembers(M, ComdatMembers);
   std::vector<Function *> HotFunctions;
   std::vector<Function *> ColdFunctions;
+
+  // If the profile marked as always instrument the entry BB, do the
+  // same. Note this can be overwritten by the internal option in CFGMST.h
+  bool InstrumentFuncEntry = PGOReader->instrEntryBBEnabled();
+  if (PGOInstrumentEntry.getNumOccurrences() > 0)
+    InstrumentFuncEntry = PGOInstrumentEntry;
   for (auto &F : M) {
     if (F.isDeclaration())
       continue;
@@ -1628,7 +1644,8 @@ static bool annotateAllFunctions(
     // Split indirectbr critical edges here before computing the MST rather than
     // later in getInstrBB() to avoid invalidating it.
     SplitIndirectBrCriticalEdges(F, BPI, BFI);
-    PGOUseFunc Func(F, &M, TLI, ComdatMembers, BPI, BFI, PSI, IsCS);
+    PGOUseFunc Func(F, &M, TLI, ComdatMembers, BPI, BFI, PSI, IsCS,
+                    InstrumentFuncEntry);
     bool AllZeros = false;
     if (!Func.readCounters(PGOReader.get(), AllZeros))
       continue;
diff --git a/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp b/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
index 43a1434ae2d37..2b7b859891dcd 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
@@ -38,8 +38,6 @@
 #include "llvm/Pass.h"
 #include "llvm/PassRegistry.h"
 #include "llvm/ProfileData/InstrProf.h"
-#define INSTR_PROF_VALUE_PROF_MEMOP_API
-#include "llvm/ProfileData/InstrProfData.inc"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
@@ -91,25 +89,17 @@ static cl::opt<bool>
                     cl::desc("Scale the memop size counts using the basic "
                              " block count value"));
 
-// FIXME: These are to be removed after switching to the new memop value
-// profiling.
 // This option sets the rangge of precise profile memop sizes.
 extern cl::opt<std::string> MemOPSizeRange;
 
 // This option sets the value that groups large memop sizes
 extern cl::opt<unsigned> MemOPSizeLarge;
 
-extern cl::opt<bool> UseOldMemOpValueProf;
-
 cl::opt<bool>
     MemOPOptMemcmpBcmp("pgo-memop-optimize-memcmp-bcmp", cl::init(true),
                        cl::Hidden,
                        cl::desc("Size-specialize memcmp and bcmp calls"));
 
-static cl::opt<unsigned>
-    MemOpMaxOptSize("memop-value-prof-max-opt-size", cl::Hidden, cl::init(128),
-                    cl::desc("Optimize the memop size <= this value"));
-
 namespace {
 class PGOMemOPSizeOptLegacyPass : public FunctionPass {
 public:
@@ -279,8 +269,6 @@ class MemOPSizeOpt : public InstVisitor<MemOPSizeOpt> {
   TargetLibraryInfo &TLI;
   bool Changed;
   std::vector<MemOp> WorkList;
-  // FIXME: These are to be removed after switching to the new memop value
-  // profiling.
   // Start of the previse range.
   int64_t PreciseRangeStart;
   // Last value of the previse range.
@@ -289,8 +277,6 @@ class MemOPSizeOpt : public InstVisitor<MemOPSizeOpt> {
   std::unique_ptr<InstrProfValueData[]> ValueDataArray;
   bool perform(MemOp MO);
 
-  // FIXME: This is to be removed after switching to the new memop value
-  // profiling.
   // This kind shows which group the value falls in. For PreciseValue, we have
   // the profile count for that value. LargeGroup groups the values that are in
   // range [LargeValue, +inf). NonLargeGroup groups the rest of values.
@@ -379,11 +365,8 @@ bool MemOPSizeOpt::perform(MemOp MO) {
     if (MemOPScaleCount)
       C = getScaledCount(C, ActualCount, SavedTotalCount);
 
-    if (UseOldMemOpValueProf) {
-      // Only care precise value here.
-      if (getMemOPSizeKind(V) != PreciseValue)
-        continue;
-    } else if (!InstrProfIsSingleValRange(V) || V > MemOpMaxOptSize)
+    // Only care precise value here.
+    if (getMemOPSizeKind(V) != PreciseValue)
       continue;
 
     // ValueCounts are sorted on the count. Break at the first un-profitable
diff --git a/llvm/lib/Transforms/Instrumentation/ValueProfileCollector.cpp b/llvm/lib/Transforms/Instrumentation/ValueProfileCollector.cpp
index cd4f636ff1320..fb6216bb21777 100644
--- a/llvm/lib/Transforms/Instrumentation/ValueProfileCollector.cpp
+++ b/llvm/lib/Transforms/Instrumentation/ValueProfileCollector.cpp
@@ -11,10 +11,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "ValueProfilePlugins.inc"
+#include "llvm/IR/Function.h"
 #include "llvm/IR/InstIterator.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/InitializePasses.h"
-
 #include <cassert>
 
 using namespace llvm;
diff --git a/llvm/lib/Transforms/Instrumentation/ValueProfileCollector.h b/llvm/lib/Transforms/Instrumentation/ValueProfileCollector.h
index c3f549c2e7cc5..584a60ab451e5 100644
--- a/llvm/lib/Transforms/Instrumentation/ValueProfileCollector.h
+++ b/llvm/lib/Transforms/Instrumentation/ValueProfileCollector.h
@@ -17,13 +17,16 @@
 #define LLVM_ANALYSIS_PROFILE_GEN_ANALYSIS_H
 
 #include "llvm/Analysis/TargetLibraryInfo.h"
-#include "llvm/IR/Function.h"
-#include "llvm/IR/PassManager.h"
-#include "llvm/Pass.h"
 #include "llvm/ProfileData/InstrProf.h"
+#include <memory>
+#include <vector>
 
 namespace llvm {
 
+class Function;
+class Instruction;
+class Value;
+
 /// Utility analysis that determines what values are worth profiling.
 /// The actual logic is inside the ValueProfileCollectorImpl, whose job is to
 /// populate the Candidates vector.
diff --git a/llvm/lib/Transforms/Scalar/LoopFuse.cpp b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
index 20edc8699d797..5573af834c597 100644
--- a/llvm/lib/Transforms/Scalar/LoopFuse.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
@@ -46,6 +46,7 @@
 
 #include "llvm/Transforms/Scalar/LoopFuse.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/DependenceAnalysis.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/LoopInfo.h"
@@ -53,6 +54,7 @@
 #include "llvm/Analysis/PostDominators.h"
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Verifier.h"
 #include "llvm/InitializePasses.h"
@@ -64,6 +66,7 @@
 #include "llvm/Transforms/Utils.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/CodeMoverUtils.h"
+#include "llvm/Transforms/Utils/UnrollLoop.h"
 
 using namespace llvm;
 
@@ -114,6 +117,11 @@ static cl::opt<FusionDependenceAnalysisChoice> FusionDependenceAnalysis(
                           "Use all available analyses")),
     cl::Hidden, cl::init(FUSION_DEPENDENCE_ANALYSIS_ALL), cl::ZeroOrMore);
 
+static cl::opt<unsigned> FusionPeelMaxCount(
+    "loop-fusion-peel-max-count", cl::init(0), cl::Hidden,
+    cl::desc("Max number of iterations to be peeled from a loop, such that "
+             "fusion can take place"));
+
 #ifndef NDEBUG
 static cl::opt<bool>
     VerboseFusionDebugging("loop-fusion-verbose-debug",
@@ -157,6 +165,12 @@ struct FusionCandidate {
   bool Valid;
   /// Guard branch of the loop, if it exists
   BranchInst *GuardBranch;
+  /// Peeling Paramaters of the Loop.
+  TTI::PeelingPreferences PP;
+  /// Can you Peel this Loop?
+  bool AbleToPeel;
+  /// Has this loop been Peeled
+  bool Peeled;
 
   /// Dominator and PostDominator trees are needed for the
   /// FusionCandidateCompare function, required by FusionCandidateSet to
@@ -168,11 +182,13 @@ struct FusionCandidate {
   OptimizationRemarkEmitter &ORE;
 
   FusionCandidate(Loop *L, const DominatorTree *DT,
-                  const PostDominatorTree *PDT, OptimizationRemarkEmitter &ORE)
+                  const PostDominatorTree *PDT, OptimizationRemarkEmitter &ORE,
+                  TTI::PeelingPreferences PP)
       : Preheader(L->getLoopPreheader()), Header(L->getHeader()),
         ExitingBlock(L->getExitingBlock()), ExitBlock(L->getExitBlock()),
         Latch(L->getLoopLatch()), L(L), Valid(true),
-        GuardBranch(L->getLoopGuardBranch()), DT(DT), PDT(PDT), ORE(ORE) {
+        GuardBranch(L->getLoopGuardBranch()), PP(PP), AbleToPeel(canPeel(L)),
+        Peeled(false), DT(DT), PDT(PDT), ORE(ORE) {
 
     // Walk over all blocks in the loop and check for conditions that may
     // prevent fusion. For each block, walk over all instructions and collect
@@ -243,6 +259,17 @@ struct FusionCandidate {
       return Preheader;
   }
 
+  /// After Peeling the loop is modified quite a bit, hence all of the Blocks
+  /// need to be updated accordingly.
+  void updateAfterPeeling() {
+    Preheader = L->getLoopPreheader();
+    Header = L->getHeader();
+    ExitingBlock = L->getExitingBlock();
+    ExitBlock = L->getExitBlock();
+    Latch = L->getLoopLatch();
+    verify();
+  }
+
   /// Given a guarded loop, get the successor of the guard that is not in the
   /// loop.
   ///
@@ -254,6 +281,8 @@ struct FusionCandidate {
     assert(GuardBranch && "Only valid on guarded loops.");
     assert(GuardBranch->isConditional() &&
            "Expecting guard to be a conditional branch.");
+    if (Peeled)
+      return GuardBranch->getSuccessor(1);
     return (GuardBranch->getSuccessor(0) == Preheader)
                ? GuardBranch->getSuccessor(1)
                : GuardBranch->getSuccessor(0);
@@ -515,13 +544,17 @@ struct LoopFuser {
   ScalarEvolution &SE;
   PostDominatorTree &PDT;
   OptimizationRemarkEmitter &ORE;
+  AssumptionCache &AC;
+
+  const TargetTransformInfo &TTI;
 
 public:
   LoopFuser(LoopInfo &LI, DominatorTree &DT, DependenceInfo &DI,
             ScalarEvolution &SE, PostDominatorTree &PDT,
-            OptimizationRemarkEmitter &ORE, const DataLayout &DL)
+            OptimizationRemarkEmitter &ORE, const DataLayout &DL,
+            AssumptionCache &AC, const TargetTransformInfo &TTI)
       : LDT(LI), DTU(DT, PDT, DomTreeUpdater::UpdateStrategy::Lazy), LI(LI),
-        DT(DT), DI(DI), SE(SE), PDT(PDT), ORE(ORE) {}
+        DT(DT), DI(DI), SE(SE), PDT(PDT), ORE(ORE), AC(AC), TTI(TTI) {}
 
   /// This is the main entry point for loop fusion. It will traverse the
   /// specified function and collect candidate loops to fuse, starting at the
@@ -606,7 +639,9 @@ struct LoopFuser {
   /// Flow Equivalent sets, sorted by dominance.
   void collectFusionCandidates(const LoopVector &LV) {
     for (Loop *L : LV) {
-      FusionCandidate CurrCand(L, &DT, &PDT, ORE);
+      TTI::PeelingPreferences PP =
+          gatherPeelingPreferences(L, SE, TTI, None, None);
+      FusionCandidate CurrCand(L, &DT, &PDT, ORE, PP);
       if (!CurrCand.isEligibleForFusion(SE))
         continue;
 
@@ -656,33 +691,133 @@ struct LoopFuser {
   /// Determine if two fusion candidates have the same trip count (i.e., they
   /// execute the same number of iterations).
   ///
-  /// Note that for now this method simply returns a boolean value because there
-  /// are no mechanisms in loop fusion to handle different trip counts. In the
-  /// future, this behaviour can be extended to adjust one of the loops to make
-  /// the trip counts equal (e.g., loop peeling). When this is added, this
-  /// interface may need to change to return more information than just a
-  /// boolean value.
-  bool identicalTripCounts(const FusionCandidate &FC0,
-                           const FusionCandidate &FC1) const {
+  /// This function will return a pair of values. The first is a boolean,
+  /// stating whether or not the two candidates are known at compile time to
+  /// have the same TripCount. The second is the difference in the two
+  /// TripCounts. This information can be used later to determine whether or not
+  /// peeling can be performed on either one of the candiates.
+  std::pair<bool, Optional<unsigned>>
+  haveIdenticalTripCounts(const FusionCandidate &FC0,
+                          const FusionCandidate &FC1) const {
+
     const SCEV *TripCount0 = SE.getBackedgeTakenCount(FC0.L);
     if (isa<SCEVCouldNotCompute>(TripCount0)) {
       UncomputableTripCount++;
       LLVM_DEBUG(dbgs() << "Trip count of first loop could not be computed!");
-      return false;
+      return {false, None};
     }
 
     const SCEV *TripCount1 = SE.getBackedgeTakenCount(FC1.L);
     if (isa<SCEVCouldNotCompute>(TripCount1)) {
       UncomputableTripCount++;
       LLVM_DEBUG(dbgs() << "Trip count of second loop could not be computed!");
-      return false;
+      return {false, None};
     }
+
     LLVM_DEBUG(dbgs() << "\tTrip counts: " << *TripCount0 << " & "
                       << *TripCount1 << " are "
                       << (TripCount0 == TripCount1 ? "identical" : "different")
                       << "\n");
 
-    return (TripCount0 == TripCount1);
+    if (TripCount0 == TripCount1)
+      return {true, 0};
+
+    LLVM_DEBUG(dbgs() << "The loops do not have the same tripcount, "
+                         "determining the difference between trip counts\n");
+
+    // Currently only considering loops with a single exit point
+    // and a non-constant trip count.
+    const unsigned TC0 = SE.getSmallConstantTripCount(FC0.L);
+    const unsigned TC1 = SE.getSmallConstantTripCount(FC1.L);
+
+    // If any of the tripcounts are zero that means that loop(s) do not have
+    // a single exit or a constant tripcount.
+    if (TC0 == 0 || TC1 == 0) {
+      LLVM_DEBUG(dbgs() << "Loop(s) do not have a single exit point or do not "
+                           "have a constant number of iterations. Peeling "
+                           "is not benefical\n");
+      return {false, None};
+    }
+
+    Optional<unsigned> Difference = None;
+    int Diff = TC0 - TC1;
+
+    if (Diff > 0)
+      Difference = Diff;
+    else {
+      LLVM_DEBUG(
+          dbgs() << "Difference is less than 0. FC1 (second loop) has more "
+                    "iterations than the first one. Currently not supported\n");
+    }
+
+    LLVM_DEBUG(dbgs() << "Difference in loop trip count is: " << Difference
+                      << "\n");
+
+    return {false, Difference};
+  }
+
+  void peelFusionCandidate(FusionCandidate &FC0, const FusionCandidate &FC1,
+                           unsigned PeelCount) {
+    assert(FC0.AbleToPeel && "Should be able to peel loop");
+
+    LLVM_DEBUG(dbgs() << "Attempting to peel first " << PeelCount
+                      << " iterations of the first loop. \n");
+
+    FC0.Peeled = peelLoop(FC0.L, PeelCount, &LI, &SE, &DT, &AC, true);
+    if (FC0.Peeled) {
+      LLVM_DEBUG(dbgs() << "Done Peeling\n");
+
+#ifndef NDEBUG
+      auto IdenticalTripCount = haveIdenticalTripCounts(FC0, FC1);
+
+      assert(IdenticalTripCount.first && *IdenticalTripCount.second == 0 &&
+             "Loops should have identical trip counts after peeling");
+#endif
+
+      FC0.PP.PeelCount += PeelCount;
+
+      // Peeling does not update the PDT
+      PDT.recalculate(*FC0.Preheader->getParent());
+
+      FC0.updateAfterPeeling();
+
+      // In this case the iterations of the loop are constant, so the first
+      // loop will execute completely (will not jump from one of
+      // the peeled blocks to the second loop). Here we are updating the
+      // branch conditions of each of the peeled blocks, such that it will
+      // branch to its successor which is not the preheader of the second loop
+      // in the case of unguarded loops, or the succesors of the exit block of
+      // the first loop otherwise. Doing this update will ensure that the entry
+      // block of the first loop dominates the entry block of the second loop.
+      BasicBlock *BB =
+          FC0.GuardBranch ? FC0.ExitBlock->getUniqueSuccessor() : FC1.Preheader;
+      if (BB) {
+        SmallVector<DominatorTree::UpdateType, 8> TreeUpdates;
+        SmallVector<Instruction *, 8> WorkList;
+        for (BasicBlock *Pred : predecessors(BB)) {
+          if (Pred != FC0.ExitBlock) {
+            WorkList.emplace_back(Pred->getTerminator());
+            TreeUpdates.emplace_back(
+                DominatorTree::UpdateType(DominatorTree::Delete, Pred, BB));
+          }
+        }
+        // Cannot modify the predecessors inside the above loop as it will cause
+        // the iterators to be nullptrs, causing memory errors.
+        for (Instruction *CurrentBranch: WorkList) {
+          BasicBlock *Succ = CurrentBranch->getSuccessor(0);
+          if (Succ == BB)
+            Succ = CurrentBranch->getSuccessor(1);
+          ReplaceInstWithInst(CurrentBranch, BranchInst::Create(Succ));
+        }
+
+        DTU.applyUpdates(TreeUpdates);
+        DTU.flush();
+      }
+      LLVM_DEBUG(
+          dbgs() << "Sucessfully peeled " << FC0.PP.PeelCount
+                 << " iterations from the first loop.\n"
+                    "Both Loops have the same number of iterations now.\n");
+    }
   }
 
   /// Walk each set of control flow equivalent fusion candidates and attempt to
@@ -716,7 +851,32 @@ struct LoopFuser {
           FC0->verify();
           FC1->verify();
 
-          if (!identicalTripCounts(*FC0, *FC1)) {
+          // Check if the candidates have identical tripcounts (first value of
+          // pair), and if not check the difference in the tripcounts between
+          // the loops (second value of pair). The difference is not equal to
+          // None iff the loops iterate a constant number of times, and have a
+          // single exit.
+          std::pair<bool, Optional<unsigned>> IdenticalTripCountRes =
+              haveIdenticalTripCounts(*FC0, *FC1);
+          bool SameTripCount = IdenticalTripCountRes.first;
+          Optional<unsigned> TCDifference = IdenticalTripCountRes.second;
+
+          // Here we are checking that FC0 (the first loop) can be peeled, and
+          // both loops have different tripcounts.
+          if (FC0->AbleToPeel && !SameTripCount && TCDifference) {
+            if (*TCDifference > FusionPeelMaxCount) {
+              LLVM_DEBUG(dbgs()
+                         << "Difference in loop trip counts: " << *TCDifference
+                         << " is greater than maximum peel count specificed: "
+                         << FusionPeelMaxCount << "\n");
+            } else {
+              // Dependent on peeling being performed on the first loop, and
+              // assuming all other conditions for fusion return true.
+              SameTripCount = true;
+            }
+          }
+
+          if (!SameTripCount) {
             LLVM_DEBUG(dbgs() << "Fusion candidates do not have identical trip "
                                  "counts. Not fusing.\n");
             reportLoopFusion<OptimizationRemarkMissed>(*FC0, *FC1,
@@ -734,7 +894,7 @@ struct LoopFuser {
           // Ensure that FC0 and FC1 have identical guards.
           // If one (or both) are not guarded, this check is not necessary.
           if (FC0->GuardBranch && FC1->GuardBranch &&
-              !haveIdenticalGuards(*FC0, *FC1)) {
+              !haveIdenticalGuards(*FC0, *FC1) && !TCDifference) {
             LLVM_DEBUG(dbgs() << "Fusion candidates do not have identical "
                                  "guards. Not Fusing.\n");
             reportLoopFusion<OptimizationRemarkMissed>(*FC0, *FC1,
@@ -803,13 +963,23 @@ struct LoopFuser {
           LLVM_DEBUG(dbgs() << "\tFusion is performed: " << *FC0 << " and "
                             << *FC1 << "\n");
 
+          FusionCandidate FC0Copy = *FC0;
+          // Peel the loop after determining that fusion is legal. The Loops
+          // will still be safe to fuse after the peeling is performed.
+          bool Peel = TCDifference && *TCDifference > 0;
+          if (Peel)
+            peelFusionCandidate(FC0Copy, *FC1, *TCDifference);
+
           // Report fusion to the Optimization Remarks.
           // Note this needs to be done *before* performFusion because
           // performFusion will change the original loops, making it not
           // possible to identify them after fusion is complete.
-          reportLoopFusion<OptimizationRemark>(*FC0, *FC1, FuseCounter);
+          reportLoopFusion<OptimizationRemark>((Peel ? FC0Copy : *FC0), *FC1,
+                                               FuseCounter);
 
-          FusionCandidate FusedCand(performFusion(*FC0, *FC1), &DT, &PDT, ORE);
+          FusionCandidate FusedCand(
+              performFusion((Peel ? FC0Copy : *FC0), *FC1), &DT, &PDT, ORE,
+              FC0Copy.PP);
           FusedCand.verify();
           assert(FusedCand.isEligibleForFusion(SE) &&
                  "Fused candidate should be eligible for fusion!");
@@ -1086,16 +1256,17 @@ struct LoopFuser {
       return (FC1.GuardBranch->getSuccessor(1) == FC1.Preheader);
   }
 
-  /// Simplify the condition of the latch branch of \p FC to true, when both of
-  /// its successors are the same.
+  /// Modify the latch branch of FC to be unconditional since successors of the
+  /// branch are the same.
   void simplifyLatchBranch(const FusionCandidate &FC) const {
     BranchInst *FCLatchBranch = dyn_cast<BranchInst>(FC.Latch->getTerminator());
     if (FCLatchBranch) {
       assert(FCLatchBranch->isConditional() &&
              FCLatchBranch->getSuccessor(0) == FCLatchBranch->getSuccessor(1) &&
              "Expecting the two successors of FCLatchBranch to be the same");
-      FCLatchBranch->setCondition(
-          llvm::ConstantInt::getTrue(FCLatchBranch->getCondition()->getType()));
+      BranchInst *NewBranch =
+          BranchInst::Create(FCLatchBranch->getSuccessor(0));
+      ReplaceInstWithInst(FCLatchBranch, NewBranch);
     }
   }
 
@@ -1155,7 +1326,8 @@ struct LoopFuser {
     if (FC0.GuardBranch)
       return fuseGuardedLoops(FC0, FC1);
 
-    assert(FC1.Preheader == FC0.ExitBlock);
+    assert(FC1.Preheader ==
+           (FC0.Peeled ? FC0.ExitBlock->getUniqueSuccessor() : FC0.ExitBlock));
     assert(FC1.Preheader->size() == 1 &&
            FC1.Preheader->getSingleSuccessor() == FC1.Header);
 
@@ -1197,12 +1369,27 @@ struct LoopFuser {
     // to FC1.Header? I think this is basically what the three sequences are
     // trying to accomplish; however, doing this directly in the CFG may mean
     // the DT/PDT becomes invalid
-    FC0.ExitingBlock->getTerminator()->replaceUsesOfWith(FC1.Preheader,
-                                                         FC1.Header);
-    TreeUpdates.emplace_back(DominatorTree::UpdateType(
-        DominatorTree::Delete, FC0.ExitingBlock, FC1.Preheader));
-    TreeUpdates.emplace_back(DominatorTree::UpdateType(
-        DominatorTree::Insert, FC0.ExitingBlock, FC1.Header));
+    if (!FC0.Peeled) {
+      FC0.ExitingBlock->getTerminator()->replaceUsesOfWith(FC1.Preheader,
+                                                           FC1.Header);
+      TreeUpdates.emplace_back(DominatorTree::UpdateType(
+          DominatorTree::Delete, FC0.ExitingBlock, FC1.Preheader));
+      TreeUpdates.emplace_back(DominatorTree::UpdateType(
+          DominatorTree::Insert, FC0.ExitingBlock, FC1.Header));
+    } else {
+      TreeUpdates.emplace_back(DominatorTree::UpdateType(
+          DominatorTree::Delete, FC0.ExitBlock, FC1.Preheader));
+
+      // Remove the ExitBlock of the first Loop (also not needed)
+      FC0.ExitingBlock->getTerminator()->replaceUsesOfWith(FC0.ExitBlock,
+                                                           FC1.Header);
+      TreeUpdates.emplace_back(DominatorTree::UpdateType(
+          DominatorTree::Delete, FC0.ExitingBlock, FC0.ExitBlock));
+      FC0.ExitBlock->getTerminator()->eraseFromParent();
+      TreeUpdates.emplace_back(DominatorTree::UpdateType(
+          DominatorTree::Insert, FC0.ExitingBlock, FC1.Header));
+      new UnreachableInst(FC0.ExitBlock->getContext(), FC0.ExitBlock);
+    }
 
     // The pre-header of L1 is not necessary anymore.
     assert(pred_begin(FC1.Preheader) == pred_end(FC1.Preheader));
@@ -1246,7 +1433,7 @@ struct LoopFuser {
     FC0.Latch->getTerminator()->replaceUsesOfWith(FC0.Header, FC1.Header);
     FC1.Latch->getTerminator()->replaceUsesOfWith(FC1.Header, FC0.Header);
 
-    // Change the condition of FC0 latch branch to true, as both successors of
+    // Modify the latch branch of FC0 to be unconditional as both successors of
     // the branch are the same.
     simplifyLatchBranch(FC0);
 
@@ -1268,6 +1455,11 @@ struct LoopFuser {
 
     LI.removeBlock(FC1.Preheader);
     DTU.deleteBB(FC1.Preheader);
+    if (FC0.Peeled) {
+      LI.removeBlock(FC0.ExitBlock);
+      DTU.deleteBB(FC0.ExitBlock);
+    }
+
     DTU.flush();
 
     // Is there a way to keep SE up-to-date so we don't need to forget the loops
@@ -1364,10 +1556,15 @@ struct LoopFuser {
     BasicBlock *FC1GuardBlock = FC1.GuardBranch->getParent();
     BasicBlock *FC0NonLoopBlock = FC0.getNonLoopBlock();
     BasicBlock *FC1NonLoopBlock = FC1.getNonLoopBlock();
+    BasicBlock *FC0ExitBlockSuccessor = FC0.ExitBlock->getUniqueSuccessor();
 
     // Move instructions from the exit block of FC0 to the beginning of the exit
-    // block of FC1.
-    moveInstructionsToTheBeginning(*FC0.ExitBlock, *FC1.ExitBlock, DT, PDT, DI);
+    // block of FC1, in the case that the FC0 loop has not been peeled. In the
+    // case that FC0 loop is peeled, then move the instructions of the successor
+    // of the FC0 Exit block to the beginning of the exit block of FC1.
+    moveInstructionsToTheBeginning(
+        (FC0.Peeled ? *FC0ExitBlockSuccessor : *FC0.ExitBlock), *FC1.ExitBlock,
+        DT, PDT, DI);
 
     // Move instructions from the guard block of FC1 to the end of the guard
     // block of FC0.
@@ -1387,8 +1584,9 @@ struct LoopFuser {
     // for FC1 (where FC1 guard would have gone if FC1 was not executed).
     FC1NonLoopBlock->replacePhiUsesWith(FC1GuardBlock, FC0GuardBlock);
     FC0.GuardBranch->replaceUsesOfWith(FC0NonLoopBlock, FC1NonLoopBlock);
-    FC0.ExitBlock->getTerminator()->replaceUsesOfWith(FC1GuardBlock,
-                                                      FC1.Header);
+
+    BasicBlock *BBToUpdate = FC0.Peeled ? FC0ExitBlockSuccessor : FC0.ExitBlock;
+    BBToUpdate->getTerminator()->replaceUsesOfWith(FC1GuardBlock, FC1.Header);
 
     // The guard of FC1 is not necessary anymore.
     FC1.GuardBranch->eraseFromParent();
@@ -1403,6 +1601,15 @@ struct LoopFuser {
     TreeUpdates.emplace_back(DominatorTree::UpdateType(
         DominatorTree::Insert, FC0GuardBlock, FC1NonLoopBlock));
 
+    if (FC0.Peeled) {
+      // Remove the Block after the ExitBlock of FC0
+      TreeUpdates.emplace_back(DominatorTree::UpdateType(
+          DominatorTree::Delete, FC0ExitBlockSuccessor, FC1GuardBlock));
+      FC0ExitBlockSuccessor->getTerminator()->eraseFromParent();
+      new UnreachableInst(FC0ExitBlockSuccessor->getContext(),
+                          FC0ExitBlockSuccessor);
+    }
+
     assert(pred_begin(FC1GuardBlock) == pred_end(FC1GuardBlock) &&
            "Expecting guard block to have no predecessors");
     assert(succ_begin(FC1GuardBlock) == succ_end(FC1GuardBlock) &&
@@ -1509,7 +1716,7 @@ struct LoopFuser {
     FC0.Latch->getTerminator()->replaceUsesOfWith(FC0.Header, FC1.Header);
     FC1.Latch->getTerminator()->replaceUsesOfWith(FC1.Header, FC0.Header);
 
-    // Change the condition of FC0 latch branch to true, as both successors of
+    // Modify the latch branch of FC0 to be unconditional as both successors of
     // the branch are the same.
     simplifyLatchBranch(FC0);
 
@@ -1540,6 +1747,10 @@ struct LoopFuser {
     LI.removeBlock(FC1GuardBlock);
     LI.removeBlock(FC1.Preheader);
     LI.removeBlock(FC0.ExitBlock);
+    if (FC0.Peeled) {
+      LI.removeBlock(FC0ExitBlockSuccessor);
+      DTU.deleteBB(FC0ExitBlockSuccessor);
+    }
     DTU.deleteBB(FC1GuardBlock);
     DTU.deleteBB(FC1.Preheader);
     DTU.deleteBB(FC0.ExitBlock);
@@ -1606,6 +1817,8 @@ struct LoopFuseLegacy : public FunctionPass {
     AU.addRequired<PostDominatorTreeWrapperPass>();
     AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
     AU.addRequired<DependenceAnalysisWrapperPass>();
+    AU.addRequired<AssumptionCacheTracker>();
+    AU.addRequired<TargetTransformInfoWrapperPass>();
 
     AU.addPreserved<ScalarEvolutionWrapperPass>();
     AU.addPreserved<LoopInfoWrapperPass>();
@@ -1622,9 +1835,12 @@ struct LoopFuseLegacy : public FunctionPass {
     auto &SE = getAnalysis<ScalarEvolutionWrapperPass>().getSE();
     auto &PDT = getAnalysis<PostDominatorTreeWrapperPass>().getPostDomTree();
     auto &ORE = getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
-
+    auto &AC = getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
+    const TargetTransformInfo &TTI =
+        getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
     const DataLayout &DL = F.getParent()->getDataLayout();
-    LoopFuser LF(LI, DT, DI, SE, PDT, ORE, DL);
+
+    LoopFuser LF(LI, DT, DI, SE, PDT, ORE, DL, AC, TTI);
     return LF.fuseLoops(F);
   }
 };
@@ -1637,9 +1853,11 @@ PreservedAnalyses LoopFusePass::run(Function &F, FunctionAnalysisManager &AM) {
   auto &SE = AM.getResult<ScalarEvolutionAnalysis>(F);
   auto &PDT = AM.getResult<PostDominatorTreeAnalysis>(F);
   auto &ORE = AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
-
+  auto &AC = AM.getResult<AssumptionAnalysis>(F);
+  const TargetTransformInfo &TTI = AM.getResult<TargetIRAnalysis>(F);
   const DataLayout &DL = F.getParent()->getDataLayout();
-  LoopFuser LF(LI, DT, DI, SE, PDT, ORE, DL);
+
+  LoopFuser LF(LI, DT, DI, SE, PDT, ORE, DL, AC, TTI);
   bool Changed = LF.fuseLoops(F);
   if (!Changed)
     return PreservedAnalyses::all();
@@ -1662,6 +1880,8 @@ INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(DependenceAnalysisWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
+INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
 INITIALIZE_PASS_END(LoopFuseLegacy, "loop-fusion", "Loop Fusion", false, false)
 
 FunctionPass *llvm::createLoopFusePass() { return new LoopFuseLegacy(); }
diff --git a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
index aaf2840f8ff6f..fc980cf3689fb 100644
--- a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
@@ -295,6 +295,31 @@ static void deleteDeadInstruction(Instruction *I) {
   I->eraseFromParent();
 }
 
+namespace {
+class ExpandedValuesCleaner {
+  SCEVExpander &Expander;
+  TargetLibraryInfo *TLI;
+  SmallVector<Value *, 4> ExpandedValues;
+  bool Commit = false;
+
+public:
+  ExpandedValuesCleaner(SCEVExpander &Expander, TargetLibraryInfo *TLI)
+      : Expander(Expander), TLI(TLI) {}
+
+  void add(Value *V) { ExpandedValues.push_back(V); }
+
+  void commit() { Commit = true; }
+
+  ~ExpandedValuesCleaner() {
+    if (!Commit) {
+      Expander.clear();
+      for (auto *V : ExpandedValues)
+        RecursivelyDeleteTriviallyDeadInstructions(V, TLI);
+    }
+  }
+};
+} // namespace
+
 //===----------------------------------------------------------------------===//
 //
 //          Implementation of LoopIdiomRecognize
@@ -908,10 +933,12 @@ bool LoopIdiomRecognize::processLoopStridedStore(
   BasicBlock *Preheader = CurLoop->getLoopPreheader();
   IRBuilder<> Builder(Preheader->getTerminator());
   SCEVExpander Expander(*SE, *DL, "loop-idiom");
+  ExpandedValuesCleaner EVC(Expander, TLI);
 
   Type *DestInt8PtrTy = Builder.getInt8PtrTy(DestAS);
   Type *IntIdxTy = DL->getIndexType(DestPtr->getType());
 
+  bool Changed = false;
   const SCEV *Start = Ev->getStart();
   // Handle negative strided loops.
   if (NegStride)
@@ -920,7 +947,7 @@ bool LoopIdiomRecognize::processLoopStridedStore(
   // TODO: ideally we should still be able to generate memset if SCEV expander
   // is taught to generate the dependencies at the latest point.
   if (!isSafeToExpand(Start, *SE))
-    return false;
+    return Changed;
 
   // Okay, we have a strided store "p[i]" of a splattable value.  We can turn
   // this into a memset in the loop preheader now if we want.  However, this
@@ -929,16 +956,23 @@ bool LoopIdiomRecognize::processLoopStridedStore(
   // base pointer and checking the region.
   Value *BasePtr =
       Expander.expandCodeFor(Start, DestInt8PtrTy, Preheader->getTerminator());
+  EVC.add(BasePtr);
+
+  // From here on out, conservatively report to the pass manager that we've
+  // changed the IR, even if we later clean up these added instructions. There
+  // may be structural differences e.g. in the order of use lists not accounted
+  // for in just a textual dump of the IR. This is written as a variable, even
+  // though statically all the places this dominates could be replaced with
+  // 'true', with the hope that anyone trying to be clever / "more precise" with
+  // the return value will read this comment, and leave them alone.
+  Changed = true;
+
   if (mayLoopAccessLocation(BasePtr, ModRefInfo::ModRef, CurLoop, BECount,
-                            StoreSize, *AA, Stores)) {
-    Expander.clear();
-    // If we generated new code for the base pointer, clean up.
-    RecursivelyDeleteTriviallyDeadInstructions(BasePtr, TLI);
-    return false;
-  }
+                            StoreSize, *AA, Stores))
+    return Changed;
 
   if (avoidLIRForMultiBlockLoop(/*IsMemset=*/true, IsLoopMemset))
-    return false;
+    return Changed;
 
   // Okay, everything looks good, insert the memset.
 
@@ -948,7 +982,7 @@ bool LoopIdiomRecognize::processLoopStridedStore(
   // TODO: ideally we should still be able to generate memset if SCEV expander
   // is taught to generate the dependencies at the latest point.
   if (!isSafeToExpand(NumBytesS, *SE))
-    return false;
+    return Changed;
 
   Value *NumBytes =
       Expander.expandCodeFor(NumBytesS, IntIdxTy, Preheader->getTerminator());
@@ -1007,34 +1041,10 @@ bool LoopIdiomRecognize::processLoopStridedStore(
   if (MSSAU && VerifyMemorySSA)
     MSSAU->getMemorySSA()->verifyMemorySSA();
   ++NumMemSet;
+  EVC.commit();
   return true;
 }
 
-namespace {
-class ExpandedValuesCleaner {
-  SCEVExpander &Expander;
-  TargetLibraryInfo *TLI;
-  SmallVector<Value *, 4> ExpandedValues;
-  bool Commit = false;
-
-public:
-  ExpandedValuesCleaner(SCEVExpander &Expander, TargetLibraryInfo *TLI)
-      : Expander(Expander), TLI(TLI) {}
-
-  void add(Value *V) { ExpandedValues.push_back(V); }
-
-  void commit() { Commit = true; }
-
-  ~ExpandedValuesCleaner() {
-    if (!Commit) {
-      Expander.clear();
-      for (auto *V : ExpandedValues)
-        RecursivelyDeleteTriviallyDeadInstructions(V, TLI);
-    }
-  }
-};
-} // namespace
-
 /// If the stored value is a strided load in the same loop with the same stride
 /// this may be transformable into a memcpy.  This kicks in for stuff like
 /// for (i) A[i] = B[i];
@@ -1067,6 +1077,7 @@ bool LoopIdiomRecognize::processLoopStoreOfLoopLoad(StoreInst *SI,
 
   ExpandedValuesCleaner EVC(Expander, TLI);
 
+  bool Changed = false;
   const SCEV *StrStart = StoreEv->getStart();
   unsigned StrAS = SI->getPointerAddressSpace();
   Type *IntIdxTy = Builder.getIntNTy(DL->getIndexSizeInBits(StrAS));
@@ -1085,11 +1096,20 @@ bool LoopIdiomRecognize::processLoopStoreOfLoopLoad(StoreInst *SI,
       StrStart, Builder.getInt8PtrTy(StrAS), Preheader->getTerminator());
   EVC.add(StoreBasePtr);
 
+  // From here on out, conservatively report to the pass manager that we've
+  // changed the IR, even if we later clean up these added instructions. There
+  // may be structural differences e.g. in the order of use lists not accounted
+  // for in just a textual dump of the IR. This is written as a variable, even
+  // though statically all the places this dominates could be replaced with
+  // 'true', with the hope that anyone trying to be clever / "more precise" with
+  // the return value will read this comment, and leave them alone.
+  Changed = true;
+
   SmallPtrSet<Instruction *, 1> Stores;
   Stores.insert(SI);
   if (mayLoopAccessLocation(StoreBasePtr, ModRefInfo::ModRef, CurLoop, BECount,
                             StoreSize, *AA, Stores))
-    return false;
+    return Changed;
 
   const SCEV *LdStart = LoadEv->getStart();
   unsigned LdAS = LI->getPointerAddressSpace();
@@ -1106,10 +1126,10 @@ bool LoopIdiomRecognize::processLoopStoreOfLoopLoad(StoreInst *SI,
 
   if (mayLoopAccessLocation(LoadBasePtr, ModRefInfo::Mod, CurLoop, BECount,
                             StoreSize, *AA, Stores))
-    return false;
+    return Changed;
 
   if (avoidLIRForMultiBlockLoop())
-    return false;
+    return Changed;
 
   // Okay, everything is safe, we can transform this!
 
@@ -1133,14 +1153,14 @@ bool LoopIdiomRecognize::processLoopStoreOfLoopLoad(StoreInst *SI,
     const Align StoreAlign = SI->getAlign();
     const Align LoadAlign = LI->getAlign();
     if (StoreAlign < StoreSize || LoadAlign < StoreSize)
-      return false;
+      return Changed;
 
     // If the element.atomic memcpy is not lowered into explicit
     // loads/stores later, then it will be lowered into an element-size
     // specific lib call. If the lib call doesn't exist for our store size, then
     // we shouldn't generate the memcpy.
     if (StoreSize > TTI->getAtomicMemIntrinsicMaxElementSize())
-      return false;
+      return Changed;
 
     // Create the call.
     // Note that unordered atomic loads/stores are *required* by the spec to
diff --git a/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp b/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
index 4412b3079461c..3b70695640414 100644
--- a/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
@@ -308,8 +308,8 @@ class LoadEliminationForLoop {
   /// We need a check if one is a pointer for a candidate load and the other is
   /// a pointer for a possibly intervening store.
   bool needsChecking(unsigned PtrIdx1, unsigned PtrIdx2,
-                     const SmallPtrSet<Value *, 4> &PtrsWrittenOnFwdingPath,
-                     const std::set<Value *> &CandLoadPtrs) {
+                     const SmallPtrSetImpl<Value *> &PtrsWrittenOnFwdingPath,
+                     const SmallPtrSetImpl<Value *> &CandLoadPtrs) {
     Value *Ptr1 =
         LAI.getRuntimePointerChecking()->getPointerInfo(PtrIdx1).PointerValue;
     Value *Ptr2 =
@@ -384,11 +384,9 @@ class LoadEliminationForLoop {
         findPointersWrittenOnForwardingPath(Candidates);
 
     // Collect the pointers of the candidate loads.
-    // FIXME: SmallPtrSet does not work with std::inserter.
-    std::set<Value *> CandLoadPtrs;
-    transform(Candidates,
-                   std::inserter(CandLoadPtrs, CandLoadPtrs.begin()),
-                   std::mem_fn(&StoreToLoadForwardingCandidate::getLoadPtr));
+    SmallPtrSet<Value *, 4> CandLoadPtrs;
+    for (const auto &Candidate : Candidates)
+      CandLoadPtrs.insert(Candidate.getLoadPtr());
 
     const auto &AllChecks = LAI.getRuntimePointerChecking()->getChecks();
     SmallVector<RuntimePointerCheck, 4> Checks;
diff --git a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
index 90314b17b5e25..a9332356b3713 100644
--- a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
@@ -42,6 +42,8 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/LoopUtils.h"
+#include "llvm/Transforms/Utils/MatrixUtils.h"
 
 using namespace llvm;
 using namespace PatternMatch;
@@ -61,6 +63,9 @@ static cl::opt<unsigned> TileSize(
     "fuse-matrix-tile-size", cl::init(4), cl::Hidden,
     cl::desc(
         "Tile size for matrix instruction fusion using square-shaped tiles."));
+static cl::opt<bool> TileUseLoops("fuse-matrix-use-loops", cl::init(false),
+                                  cl::Hidden,
+                                  cl::desc("Generate loop nest for tiling."));
 static cl::opt<bool> ForceFusion(
     "force-fuse-matrix", cl::init(false), cl::Hidden,
     cl::desc("Force matrix instruction fusion even if not profitable."));
@@ -182,10 +187,10 @@ class LowerMatrixIntrinsics {
   Function &Func;
   const DataLayout &DL;
   const TargetTransformInfo &TTI;
-  AliasAnalysis &AA;
-  DominatorTree &DT;
-  LoopInfo &LI;
-  OptimizationRemarkEmitter &ORE;
+  AliasAnalysis *AA;
+  DominatorTree *DT;
+  LoopInfo *LI;
+  OptimizationRemarkEmitter *ORE;
 
   /// Contains estimates of the number of operations (loads, stores, compute) required to lower a matrix operation.
   struct OpInfoTy {
@@ -241,7 +246,7 @@ class LowerMatrixIntrinsics {
 
     void setVector(unsigned i, Value *V) { Vectors[i] = V; }
 
-    Type *getElementType() { return getVectorTy()->getElementType(); }
+    Type *getElementType() const { return getVectorTy()->getElementType(); }
 
     unsigned getNumVectors() const {
       if (isColumnMajor())
@@ -271,7 +276,7 @@ class LowerMatrixIntrinsics {
       return getVectorTy();
     }
 
-    VectorType *getVectorTy() {
+    VectorType *getVectorTy() const {
       return cast<VectorType>(Vectors[0]->getType());
     }
 
@@ -393,8 +398,8 @@ class LowerMatrixIntrinsics {
 
 public:
   LowerMatrixIntrinsics(Function &F, TargetTransformInfo &TTI,
-                        AliasAnalysis &AA, DominatorTree &DT, LoopInfo &LI,
-                        OptimizationRemarkEmitter &ORE)
+                        AliasAnalysis *AA, DominatorTree *DT, LoopInfo *LI,
+                        OptimizationRemarkEmitter *ORE)
       : Func(F), DL(F.getParent()->getDataLayout()), TTI(TTI), AA(AA), DT(DT),
         LI(LI), ORE(ORE) {}
 
@@ -727,8 +732,10 @@ class LowerMatrixIntrinsics {
         Changed |= VisitStore(cast<StoreInst>(Inst), Op1, Op2, Builder);
     }
 
-    RemarkGenerator RemarkGen(Inst2ColumnMatrix, ORE, Func);
-    RemarkGen.emitRemarks();
+    if (ORE) {
+      RemarkGenerator RemarkGen(Inst2ColumnMatrix, *ORE, Func);
+      RemarkGen.emitRemarks();
+    }
 
     for (Instruction *Inst : reverse(ToRemove))
       Inst->eraseFromParent();
@@ -1085,7 +1092,7 @@ class LowerMatrixIntrinsics {
     MemoryLocation StoreLoc = MemoryLocation::get(Store);
     MemoryLocation LoadLoc = MemoryLocation::get(Load);
 
-    AliasResult LdAliased = AA.alias(LoadLoc, StoreLoc);
+    AliasResult LdAliased = AA->alias(LoadLoc, StoreLoc);
 
     // If we can statically determine noalias we're good.
     if (!LdAliased)
@@ -1101,13 +1108,13 @@ class LowerMatrixIntrinsics {
     // as we adjust Check0 and Check1's branches.
     SmallVector<DominatorTree::UpdateType, 4> DTUpdates;
     for (BasicBlock *Succ : successors(Check0))
-      DTUpdates.push_back({DT.Delete, Check0, Succ});
+      DTUpdates.push_back({DT->Delete, Check0, Succ});
 
-    BasicBlock *Check1 = SplitBlock(MatMul->getParent(), MatMul, nullptr, &LI,
+    BasicBlock *Check1 = SplitBlock(MatMul->getParent(), MatMul, nullptr, LI,
                                     nullptr, "alias_cont");
     BasicBlock *Copy =
-        SplitBlock(MatMul->getParent(), MatMul, nullptr, &LI, nullptr, "copy");
-    BasicBlock *Fusion = SplitBlock(MatMul->getParent(), MatMul, nullptr, &LI,
+        SplitBlock(MatMul->getParent(), MatMul, nullptr, LI, nullptr, "copy");
+    BasicBlock *Fusion = SplitBlock(MatMul->getParent(), MatMul, nullptr, LI,
                                     nullptr, "no_alias");
 
     // Check if the loaded memory location begins before the end of the store
@@ -1152,11 +1159,11 @@ class LowerMatrixIntrinsics {
     PHI->addIncoming(NewLd, Copy);
 
     // Adjust DT.
-    DTUpdates.push_back({DT.Insert, Check0, Check1});
-    DTUpdates.push_back({DT.Insert, Check0, Fusion});
-    DTUpdates.push_back({DT.Insert, Check1, Copy});
-    DTUpdates.push_back({DT.Insert, Check1, Fusion});
-    DT.applyUpdates(DTUpdates);
+    DTUpdates.push_back({DT->Insert, Check0, Check1});
+    DTUpdates.push_back({DT->Insert, Check0, Fusion});
+    DTUpdates.push_back({DT->Insert, Check1, Copy});
+    DTUpdates.push_back({DT->Insert, Check1, Fusion});
+    DT->applyUpdates(DTUpdates);
     return PHI;
   }
 
@@ -1202,6 +1209,63 @@ class LowerMatrixIntrinsics {
     return Res;
   }
 
+  void createTiledLoops(CallInst *MatMul, Value *LPtr, ShapeInfo LShape,
+                        Value *RPtr, ShapeInfo RShape, StoreInst *Store,
+                        bool AllowContract) {
+    auto *EltType = cast<VectorType>(MatMul->getType())->getElementType();
+
+    // Create the main tiling loop nest.
+    TileInfo TI(LShape.NumRows, RShape.NumColumns, LShape.NumColumns, TileSize);
+    DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
+    Instruction *InsertI = cast<Instruction>(MatMul);
+    BasicBlock *Start = InsertI->getParent();
+    BasicBlock *End =
+        SplitBlock(InsertI->getParent(), InsertI, DT, LI, nullptr, "continue");
+    IRBuilder<> Builder(MatMul);
+    BasicBlock *InnerBody = TI.CreateTiledLoops(Start, End, Builder, DTU, *LI);
+
+    Type *TileVecTy =
+        FixedVectorType::get(MatMul->getType()->getScalarType(), TileSize);
+    MatrixTy TileResult;
+    // Insert in the inner loop header.
+    Builder.SetInsertPoint(TI.InnerLoopHeader->getTerminator());
+    // Create PHI nodes for the result columns to accumulate across iterations.
+    SmallVector<PHINode *, 4> ColumnPhis;
+    for (unsigned I = 0; I < TileSize; I++) {
+      auto *Phi = Builder.CreatePHI(TileVecTy, 2, "result.vec." + Twine(I));
+      Phi->addIncoming(ConstantAggregateZero::get(TileVecTy),
+                       TI.RowLoopHeader->getSingleSuccessor());
+      TileResult.addVector(Phi);
+      ColumnPhis.push_back(Phi);
+    }
+
+    // Insert in the inner loop body, which computes
+    //   Res += Load(CurrentRow, K) * Load(K, CurrentColumn)
+    Builder.SetInsertPoint(InnerBody->getTerminator());
+    // Load tiles of the operands.
+    MatrixTy A = loadMatrix(LPtr, {}, false, LShape, TI.CurrentRow, TI.CurrentK,
+                            {TileSize, TileSize}, EltType, Builder);
+    MatrixTy B = loadMatrix(RPtr, {}, false, RShape, TI.CurrentK, TI.CurrentCol,
+                            {TileSize, TileSize}, EltType, Builder);
+    emitMatrixMultiply(TileResult, A, B, AllowContract, Builder, true);
+    // Store result after the inner loop is done.
+    Builder.SetInsertPoint(TI.RowLoopLatch->getTerminator());
+    storeMatrix(TileResult, Store->getPointerOperand(), Store->getAlign(),
+                Store->isVolatile(), {LShape.NumRows, RShape.NumColumns},
+                TI.CurrentRow, TI.CurrentCol, EltType, Builder);
+
+    for (unsigned I = 0; I < TileResult.getNumVectors(); I++)
+      ColumnPhis[I]->addIncoming(TileResult.getVector(I), TI.InnerLoopLatch);
+
+    // Force unrolling of a few iterations of the inner loop, to make sure there
+    // is enough work per iteration.
+    // FIXME: The unroller should make this decision directly instead, but
+    // currently the cost-model is not up to the task.
+    unsigned InnerLoopUnrollCount = std::min(10u, LShape.NumColumns / TileSize);
+    addStringMetadataToLoop(LI->getLoopFor(TI.InnerLoopHeader),
+                            "llvm.loop.unroll.count", InnerLoopUnrollCount);
+  }
+
   void emitSIMDTiling(CallInst *MatMul, LoadInst *LoadOp0, LoadInst *LoadOp1,
                       StoreInst *Store,
                       SmallPtrSetImpl<Instruction *> &FusedInsts) {
@@ -1224,28 +1288,34 @@ class LowerMatrixIntrinsics {
 
     bool AllowContract = AllowContractEnabled || (isa<FPMathOperator>(MatMul) &&
                                                   MatMul->hasAllowContract());
-    IRBuilder<> Builder(Store);
-    for (unsigned J = 0; J < C; J += TileSize)
-      for (unsigned I = 0; I < R; I += TileSize) {
-        const unsigned TileR = std::min(R - I, unsigned(TileSize));
-        const unsigned TileC = std::min(C - J, unsigned(TileSize));
-        MatrixTy Res = getZeroMatrix(EltType, TileR, TileC);
-
-        for (unsigned K = 0; K < M; K += TileSize) {
-          const unsigned TileM = std::min(M - K, unsigned(TileSize));
-          MatrixTy A =
-              loadMatrix(APtr, LoadOp0->getAlign(), LoadOp0->isVolatile(),
-                         LShape, Builder.getInt64(I), Builder.getInt64(K),
-                         {TileR, TileM}, EltType, Builder);
-          MatrixTy B =
-              loadMatrix(BPtr, LoadOp1->getAlign(), LoadOp1->isVolatile(),
-                         RShape, Builder.getInt64(K), Builder.getInt64(J),
-                         {TileM, TileC}, EltType, Builder);
-          emitMatrixMultiply(Res, A, B, AllowContract, Builder, true);
+    if (TileUseLoops && (R % TileSize == 0 && C % TileSize == 0))
+      createTiledLoops(MatMul, APtr, LShape, BPtr, RShape, Store,
+                       AllowContract);
+    else {
+      IRBuilder<> Builder(Store);
+      for (unsigned J = 0; J < C; J += TileSize)
+        for (unsigned I = 0; I < R; I += TileSize) {
+          const unsigned TileR = std::min(R - I, unsigned(TileSize));
+          const unsigned TileC = std::min(C - J, unsigned(TileSize));
+          MatrixTy Res = getZeroMatrix(EltType, TileR, TileC);
+
+          for (unsigned K = 0; K < M; K += TileSize) {
+            const unsigned TileM = std::min(M - K, unsigned(TileSize));
+            MatrixTy A =
+                loadMatrix(APtr, LoadOp0->getAlign(), LoadOp0->isVolatile(),
+                           LShape, Builder.getInt64(I), Builder.getInt64(K),
+                           {TileR, TileM}, EltType, Builder);
+            MatrixTy B =
+                loadMatrix(BPtr, LoadOp1->getAlign(), LoadOp1->isVolatile(),
+                           RShape, Builder.getInt64(K), Builder.getInt64(J),
+                           {TileM, TileC}, EltType, Builder);
+            emitMatrixMultiply(Res, A, B, AllowContract, Builder, true);
+          }
+          storeMatrix(Res, CPtr, Store->getAlign(), Store->isVolatile(), {R, M},
+                      Builder.getInt64(I), Builder.getInt64(J), EltType,
+                      Builder);
         }
-        storeMatrix(Res, CPtr, Store->getAlign(), Store->isVolatile(), {R, M},
-                    Builder.getInt64(I), Builder.getInt64(J), EltType, Builder);
-      }
+    }
 
     // Mark eliminated instructions as fused and remove them.
     FusedInsts.insert(Store);
@@ -1272,7 +1342,7 @@ class LowerMatrixIntrinsics {
   void LowerMatrixMultiplyFused(CallInst *MatMul,
                                 SmallPtrSetImpl<Instruction *> &FusedInsts) {
     if (!FuseMatrix || !MatMul->hasOneUse() ||
-        MatrixLayout != MatrixLayoutTy::ColumnMajor)
+        MatrixLayout != MatrixLayoutTy::ColumnMajor || !DT)
       return;
 
     auto *LoadOp0 = dyn_cast<LoadInst>(MatMul->getOperand(0));
@@ -1283,7 +1353,7 @@ class LowerMatrixIntrinsics {
       // we create invalid IR.
       // FIXME: See if we can hoist the store address computation.
       auto *AddrI = dyn_cast<Instruction>(Store->getOperand(1));
-      if (AddrI && (!DT.dominates(AddrI, MatMul)))
+      if (AddrI && (!DT->dominates(AddrI, MatMul)))
         return;
 
       emitSIMDTiling(MatMul, LoadOp0, LoadOp1, Store, FusedInsts);
@@ -1300,6 +1370,8 @@ class LowerMatrixIntrinsics {
 
     const MatrixTy &Lhs = getMatrix(MatMul->getArgOperand(0), LShape, Builder);
     const MatrixTy &Rhs = getMatrix(MatMul->getArgOperand(1), RShape, Builder);
+    assert(Lhs.getElementType() == Rhs.getElementType() &&
+           "Matrix multiply argument element types do not match.");
 
     const unsigned R = LShape.NumRows;
     const unsigned C = RShape.NumColumns;
@@ -1307,6 +1379,8 @@ class LowerMatrixIntrinsics {
 
     // Initialize the output
     MatrixTy Result(R, C, EltType);
+    assert(Lhs.getElementType() == Result.getElementType() &&
+           "Matrix multiply result element type does not match arguments.");
 
     bool AllowContract = AllowContractEnabled || (isa<FPMathOperator>(MatMul) &&
                                                   MatMul->hasAllowContract());
@@ -1868,7 +1942,7 @@ PreservedAnalyses LowerMatrixIntrinsicsPass::run(Function &F,
   auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
   auto &LI = AM.getResult<LoopAnalysis>(F);
 
-  LowerMatrixIntrinsics LMT(F, TTI, AA, DT, LI, ORE);
+  LowerMatrixIntrinsics LMT(F, TTI, &AA, &DT, &LI, &ORE);
   if (LMT.Visit()) {
     PreservedAnalyses PA;
     PA.preserveSet<CFGAnalyses>();
@@ -1894,7 +1968,7 @@ class LowerMatrixIntrinsicsLegacyPass : public FunctionPass {
     auto &AA = getAnalysis<AAResultsWrapperPass>().getAAResults();
     auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
     auto &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
-    LowerMatrixIntrinsics LMT(F, TTI, AA, DT, LI, ORE);
+    LowerMatrixIntrinsics LMT(F, TTI, &AA, &DT, &LI, &ORE);
     bool C = LMT.Visit();
     return C;
   }
@@ -1925,3 +1999,45 @@ INITIALIZE_PASS_END(LowerMatrixIntrinsicsLegacyPass, DEBUG_TYPE, pass_name,
 Pass *llvm::createLowerMatrixIntrinsicsPass() {
   return new LowerMatrixIntrinsicsLegacyPass();
 }
+
+namespace {
+
+/// A lightweight version of the matrix lowering pass that only requires TTI.
+/// Advanced features that require DT, AA or ORE like tiling are disabled. This
+/// is used to lower matrix intrinsics if the main lowering pass is not run, for
+/// example with -O0.
+class LowerMatrixIntrinsicsMinimalLegacyPass : public FunctionPass {
+public:
+  static char ID;
+
+  LowerMatrixIntrinsicsMinimalLegacyPass() : FunctionPass(ID) {
+    initializeLowerMatrixIntrinsicsMinimalLegacyPassPass(
+        *PassRegistry::getPassRegistry());
+  }
+
+  bool runOnFunction(Function &F) override {
+    auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
+    LowerMatrixIntrinsics LMT(F, TTI, nullptr, nullptr, nullptr, nullptr);
+    bool C = LMT.Visit();
+    return C;
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.addRequired<TargetTransformInfoWrapperPass>();
+    AU.setPreservesCFG();
+  }
+};
+} // namespace
+
+static const char pass_name_minimal[] = "Lower the matrix intrinsics (minimal)";
+char LowerMatrixIntrinsicsMinimalLegacyPass::ID = 0;
+INITIALIZE_PASS_BEGIN(LowerMatrixIntrinsicsMinimalLegacyPass,
+                      "lower-matrix-intrinsics-minimal", pass_name_minimal,
+                      false, false)
+INITIALIZE_PASS_END(LowerMatrixIntrinsicsMinimalLegacyPass,
+                    "lower-matrix-intrinsics-minimal", pass_name_minimal, false,
+                    false)
+
+Pass *llvm::createLowerMatrixIntrinsicsMinimalPass() {
+  return new LowerMatrixIntrinsicsMinimalLegacyPass();
+}
diff --git a/llvm/lib/Transforms/Scalar/SCCP.cpp b/llvm/lib/Transforms/Scalar/SCCP.cpp
index 11ac7d7e15847..32dc14e5ec195 100644
--- a/llvm/lib/Transforms/Scalar/SCCP.cpp
+++ b/llvm/lib/Transforms/Scalar/SCCP.cpp
@@ -33,6 +33,7 @@
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/ValueLattice.h"
 #include "llvm/Analysis/ValueLatticeUtils.h"
+#include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/Constants.h"
@@ -2021,9 +2022,47 @@ bool llvm::runIPSCCP(
 
   for (const auto &I : Solver.getTrackedRetVals()) {
     Function *F = I.first;
-    if (isOverdefined(I.second) || F->getReturnType()->isVoidTy())
+    const ValueLatticeElement &ReturnValue = I.second;
+
+    // If there is a known constant range for the return value, add !range
+    // metadata to the function's call sites.
+    if (ReturnValue.isConstantRange() &&
+        !ReturnValue.getConstantRange().isSingleElement()) {
+      // Do not add range metadata if the return value may include undef.
+      if (ReturnValue.isConstantRangeIncludingUndef())
+        continue;
+
+      auto &CR = ReturnValue.getConstantRange();
+      for (User *User : F->users()) {
+        auto *CB = dyn_cast<CallBase>(User);
+        if (!CB || CB->getCalledFunction() != F)
+          continue;
+
+        // Limit to cases where the return value is guaranteed to be neither
+        // poison nor undef. Poison will be outside any range and currently
+        // values outside of the specified range cause immediate undefined
+        // behavior.
+        if (!isGuaranteedNotToBeUndefOrPoison(CB, CB))
+          continue;
+
+        // Do not touch existing metadata for now.
+        // TODO: We should be able to take the intersection of the existing
+        // metadata and the inferred range.
+        if (CB->getMetadata(LLVMContext::MD_range))
+          continue;
+
+        LLVMContext &Context = CB->getParent()->getContext();
+        Metadata *RangeMD[] = {
+            ConstantAsMetadata::get(ConstantInt::get(Context, CR.getLower())),
+            ConstantAsMetadata::get(ConstantInt::get(Context, CR.getUpper()))};
+        CB->setMetadata(LLVMContext::MD_range, MDNode::get(Context, RangeMD));
+      }
       continue;
-    findReturnsToZap(*F, ReturnsToZap, Solver);
+    }
+    if (F->getReturnType()->isVoidTy())
+      continue;
+    if (isConstant(ReturnValue) || ReturnValue.isUnknownOrUndef())
+      findReturnsToZap(*F, ReturnsToZap, Solver);
   }
 
   for (auto F : Solver.getMRVFunctionsTracked()) {
diff --git a/llvm/lib/Transforms/Scalar/Scalar.cpp b/llvm/lib/Transforms/Scalar/Scalar.cpp
index 42f79d89f0a28..a0598447030c0 100644
--- a/llvm/lib/Transforms/Scalar/Scalar.cpp
+++ b/llvm/lib/Transforms/Scalar/Scalar.cpp
@@ -83,6 +83,7 @@ void llvm::initializeScalarOpts(PassRegistry &Registry) {
   initializeLowerExpectIntrinsicPass(Registry);
   initializeLowerGuardIntrinsicLegacyPassPass(Registry);
   initializeLowerMatrixIntrinsicsLegacyPassPass(Registry);
+  initializeLowerMatrixIntrinsicsMinimalLegacyPassPass(Registry);
   initializeLowerWidenableConditionLegacyPassPass(Registry);
   initializeMemCpyOptLegacyPassPass(Registry);
   initializeMergeICmpsLegacyPassPass(Registry);
diff --git a/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp b/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp
index 99055b9918050..db5211df397a8 100644
--- a/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp
+++ b/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp
@@ -62,6 +62,10 @@ static cl::opt<bool> UserForwardSwitchCond(
     "forward-switch-cond", cl::Hidden, cl::init(false),
     cl::desc("Forward switch condition to phi ops (default = false)"));
 
+static cl::opt<bool> UserHoistCommonInsts(
+    "hoist-common-insts", cl::Hidden, cl::init(true),
+    cl::desc("hoist common instructions (default = true)"));
+
 static cl::opt<bool> UserSinkCommonInsts(
     "sink-common-insts", cl::Hidden, cl::init(false),
     cl::desc("Sink common instructions (default = false)"));
@@ -222,6 +226,8 @@ static void applyCommandLineOverridesToOptions(SimplifyCFGOptions &Options) {
     Options.ConvertSwitchToLookupTable = UserSwitchToLookup;
   if (UserKeepLoops.getNumOccurrences())
     Options.NeedCanonicalLoop = UserKeepLoops;
+  if (UserHoistCommonInsts.getNumOccurrences())
+    Options.HoistCommonInsts = UserHoistCommonInsts;
   if (UserSinkCommonInsts.getNumOccurrences())
     Options.SinkCommonInsts = UserSinkCommonInsts;
 }
diff --git a/llvm/lib/Transforms/Utils/CMakeLists.txt b/llvm/lib/Transforms/Utils/CMakeLists.txt
index 5c26767c1cebb..19f655c3a78fb 100644
--- a/llvm/lib/Transforms/Utils/CMakeLists.txt
+++ b/llvm/lib/Transforms/Utils/CMakeLists.txt
@@ -46,6 +46,7 @@ add_llvm_component_library(LLVMTransformUtils
   LowerInvoke.cpp
   LowerMemIntrinsics.cpp
   LowerSwitch.cpp
+  MatrixUtils.cpp
   Mem2Reg.cpp
   MetaRenamer.cpp
   MisExpect.cpp
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index 8cdbb9d356523..91036c510e4d4 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -895,6 +895,7 @@ Function *CodeExtractor::constructFunction(const ValueSet &inputs,
       case Attribute::WriteOnly:
       case Attribute::ZExt:
       case Attribute::ImmArg:
+      case Attribute::ByRef:
       case Attribute::EndAttrKinds:
       case Attribute::EmptyKey:
       case Attribute::TombstoneKey:
diff --git a/llvm/lib/Transforms/Utils/LoopSimplify.cpp b/llvm/lib/Transforms/Utils/LoopSimplify.cpp
index a8445e94e55a0..a2e9198e30207 100644
--- a/llvm/lib/Transforms/Utils/LoopSimplify.cpp
+++ b/llvm/lib/Transforms/Utils/LoopSimplify.cpp
@@ -163,7 +163,7 @@ BasicBlock *llvm::InsertPreheaderForLoop(Loop *L, DominatorTree *DT,
 /// if it's not already in there.  Stop predecessor traversal when we reach
 /// StopBlock.
 static void addBlockAndPredsToSet(BasicBlock *InputBB, BasicBlock *StopBlock,
-                                  std::set<BasicBlock*> &Blocks) {
+                                  SmallPtrSetImpl<BasicBlock *> &Blocks) {
   SmallVector<BasicBlock *, 8> Worklist;
   Worklist.push_back(InputBB);
   do {
@@ -171,8 +171,7 @@ static void addBlockAndPredsToSet(BasicBlock *InputBB, BasicBlock *StopBlock,
     if (Blocks.insert(BB).second && BB != StopBlock)
       // If BB is not already processed and it is not a stop block then
       // insert its predecessor in the work list
-      for (pred_iterator I = pred_begin(BB), E = pred_end(BB); I != E; ++I) {
-        BasicBlock *WBB = *I;
+      for (BasicBlock *WBB : predecessors(BB)) {
         Worklist.push_back(WBB);
       }
   } while (!Worklist.empty());
@@ -308,9 +307,8 @@ static Loop *separateNestedLoop(Loop *L, BasicBlock *Preheader,
 
   // Determine which blocks should stay in L and which should be moved out to
   // the Outer loop now.
-  std::set<BasicBlock*> BlocksInL;
-  for (pred_iterator PI=pred_begin(Header), E = pred_end(Header); PI!=E; ++PI) {
-    BasicBlock *P = *PI;
+  SmallPtrSet<BasicBlock *, 4> BlocksInL;
+  for (BasicBlock *P : predecessors(Header)) {
     if (DT->dominates(Header, P))
       addBlockAndPredsToSet(P, Header, BlocksInL);
   }
diff --git a/llvm/lib/Transforms/Utils/MatrixUtils.cpp b/llvm/lib/Transforms/Utils/MatrixUtils.cpp
new file mode 100644
index 0000000000000..6a137630deeb0
--- /dev/null
+++ b/llvm/lib/Transforms/Utils/MatrixUtils.cpp
@@ -0,0 +1,104 @@
+//===- MatrixUtils.cpp - Utilities to lower matrix intrinsics ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Utilities for generating tiled loops for matrix operations.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Utils/MatrixUtils.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Type.h"
+
+using namespace llvm;
+
+BasicBlock *TileInfo::CreateLoop(BasicBlock *Preheader, BasicBlock *Exit,
+                                 Value *Bound, Value *Step, StringRef Name,
+                                 IRBuilderBase &B, DomTreeUpdater &DTU, Loop *L,
+                                 LoopInfo &LI) {
+  LLVMContext &Ctx = Preheader->getContext();
+  BasicBlock *Header = BasicBlock::Create(
+      Preheader->getContext(), Name + ".header", Preheader->getParent(), Exit);
+  BasicBlock *Body = BasicBlock::Create(Header->getContext(), Name + ".body",
+                                        Header->getParent(), Exit);
+  BasicBlock *Latch = BasicBlock::Create(Header->getContext(), Name + ".latch",
+                                         Header->getParent(), Exit);
+
+  Type *I32Ty = Type::getInt64Ty(Ctx);
+  BranchInst::Create(Body, Header);
+  BranchInst::Create(Latch, Body);
+  PHINode *IV =
+      PHINode::Create(I32Ty, 2, Name + ".iv", Header->getTerminator());
+  IV->addIncoming(ConstantInt::get(I32Ty, 0), Preheader);
+
+  B.SetInsertPoint(Latch);
+  Value *Inc = B.CreateAdd(IV, Step, Name + ".step");
+  Value *Cond = B.CreateICmpNE(Inc, Bound, Name + ".cond");
+  BranchInst::Create(Header, Exit, Cond, Latch);
+  IV->addIncoming(Inc, Latch);
+
+  BranchInst *PreheaderBr = cast<BranchInst>(Preheader->getTerminator());
+  BasicBlock *Tmp = PreheaderBr->getSuccessor(0);
+  PreheaderBr->setSuccessor(0, Header);
+  DTU.applyUpdatesPermissive({
+      {DominatorTree::Delete, Preheader, Tmp},
+      {DominatorTree::Insert, Header, Body},
+      {DominatorTree::Insert, Body, Latch},
+      {DominatorTree::Insert, Latch, Header},
+      {DominatorTree::Insert, Latch, Exit},
+      {DominatorTree::Insert, Preheader, Header},
+  });
+
+  L->addBasicBlockToLoop(Header, LI);
+  L->addBasicBlockToLoop(Body, LI);
+  L->addBasicBlockToLoop(Latch, LI);
+  return Body;
+}
+
+// Creates the following loop nest skeleton:
+//  for C = 0; C < NumColumns; C += TileSize
+//    for R = 0; R < NumRows; R += TileSize
+//      for K = 0; K < Inner ; K += TileSize
+BasicBlock *TileInfo::CreateTiledLoops(BasicBlock *Start, BasicBlock *End,
+                                       IRBuilderBase &B, DomTreeUpdater &DTU,
+                                       LoopInfo &LI) {
+  Loop *ColLoop = LI.AllocateLoop();
+  Loop *RowLoop = LI.AllocateLoop();
+  Loop *InnerLoop = LI.AllocateLoop();
+  RowLoop->addChildLoop(InnerLoop);
+  ColLoop->addChildLoop(RowLoop);
+  if (Loop *ParentL = LI.getLoopFor(Start))
+    ParentL->addChildLoop(ColLoop);
+  else
+    LI.addTopLevelLoop(ColLoop);
+
+  BasicBlock *ColBody =
+      CreateLoop(Start, End, B.getInt64(NumColumns), B.getInt64(TileSize),
+                 "cols", B, DTU, ColLoop, LI);
+  BasicBlock *ColLatch = ColBody->getSingleSuccessor();
+  BasicBlock *RowBody =
+      CreateLoop(ColBody, ColLatch, B.getInt64(NumRows), B.getInt64(TileSize),
+                 "rows", B, DTU, RowLoop, LI);
+  RowLoopLatch = RowBody->getSingleSuccessor();
+
+  BasicBlock *InnerBody =
+      CreateLoop(RowBody, RowLoopLatch, B.getInt64(NumInner),
+                 B.getInt64(TileSize), "inner", B, DTU, InnerLoop, LI);
+  InnerLoopLatch = InnerBody->getSingleSuccessor();
+  ColumnLoopHeader = ColBody->getSinglePredecessor();
+  RowLoopHeader = RowBody->getSinglePredecessor();
+  InnerLoopHeader = InnerBody->getSinglePredecessor();
+  CurrentRow = &*RowLoopHeader->begin();
+  CurrentCol = &*ColumnLoopHeader->begin();
+  CurrentK = &*InnerLoopHeader->begin();
+
+  return InnerBody;
+}
diff --git a/llvm/lib/Transforms/Utils/PredicateInfo.cpp b/llvm/lib/Transforms/Utils/PredicateInfo.cpp
index 280d3a996d508..d9bd77e999daa 100644
--- a/llvm/lib/Transforms/Utils/PredicateInfo.cpp
+++ b/llvm/lib/Transforms/Utils/PredicateInfo.cpp
@@ -837,7 +837,10 @@ Optional<PredicateConstraint> PredicateBase::getConstraint() const {
     }
 
     CmpInst *Cmp = dyn_cast<CmpInst>(Condition);
-    assert(Cmp && "Condition should be a CmpInst");
+    if (!Cmp) {
+      // TODO: Make this an assertion once RenamedOp is fully accurate.
+      return None;
+    }
 
     CmpInst::Predicate Pred;
     Value *OtherOp;
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index c54ae26b53234..3527ca4bbbbcc 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -238,7 +238,6 @@ Value *SCEVExpander::InsertBinop(Instruction::BinaryOps Opcode,
     BO->setHasNoUnsignedWrap();
   if (Flags & SCEV::FlagNSW)
     BO->setHasNoSignedWrap();
-  rememberInstruction(BO);
 
   return BO;
 }
@@ -564,10 +563,7 @@ Value *SCEVExpander::expandAddToGEP(const SCEV *const *op_begin,
     }
 
     // Emit a GEP.
-    Value *GEP = Builder.CreateGEP(Builder.getInt8Ty(), V, Idx, "uglygep");
-    rememberInstruction(GEP);
-
-    return GEP;
+    return Builder.CreateGEP(Builder.getInt8Ty(), V, Idx, "uglygep");
   }
 
   {
@@ -598,7 +594,6 @@ Value *SCEVExpander::expandAddToGEP(const SCEV *const *op_begin,
       Casted = InsertNoopCastOfTo(Casted, PTy);
     Value *GEP = Builder.CreateGEP(OriginalElTy, Casted, GepIndices, "scevgep");
     Ops.push_back(SE.getUnknown(GEP));
-    rememberInstruction(GEP);
   }
 
   return expand(SE.getAddExpr(Ops));
@@ -1073,15 +1068,12 @@ Value *SCEVExpander::expandIVInc(PHINode *PN, Value *StepV, const Loop *L,
       GEPPtrTy = PointerType::get(Type::getInt1Ty(SE.getContext()),
                                   GEPPtrTy->getAddressSpace());
     IncV = expandAddToGEP(SE.getSCEV(StepV), GEPPtrTy, IntTy, PN);
-    if (IncV->getType() != PN->getType()) {
+    if (IncV->getType() != PN->getType())
       IncV = Builder.CreateBitCast(IncV, PN->getType());
-      rememberInstruction(IncV);
-    }
   } else {
     IncV = useSubtract ?
       Builder.CreateSub(PN, StepV, Twine(IVName) + ".iv.next") :
       Builder.CreateAdd(PN, StepV, Twine(IVName) + ".iv.next");
-    rememberInstruction(IncV);
   }
   return IncV;
 }
@@ -1307,7 +1299,6 @@ SCEVExpander::getAddRecExprPHILiterally(const SCEVAddRecExpr *Normalized,
   pred_iterator HPB = pred_begin(Header), HPE = pred_end(Header);
   PHINode *PN = Builder.CreatePHI(ExpandTy, std::distance(HPB, HPE),
                                   Twine(IVName) + ".iv");
-  rememberInstruction(PN);
 
   // Create the step instructions and populate the PHI.
   for (pred_iterator HPI = HPB; HPI != HPE; ++HPI) {
@@ -1454,16 +1445,13 @@ Value *SCEVExpander::expandAddRecExprLiterally(const SCEVAddRecExpr *S) {
     if (ResTy != SE.getEffectiveSCEVType(ResTy))
       Result = InsertNoopCastOfTo(Result, SE.getEffectiveSCEVType(ResTy));
     // Truncate the result.
-    if (TruncTy != Result->getType()) {
+    if (TruncTy != Result->getType())
       Result = Builder.CreateTrunc(Result, TruncTy);
-      rememberInstruction(Result);
-    }
+
     // Invert the result.
-    if (InvertStep) {
+    if (InvertStep)
       Result = Builder.CreateSub(expandCodeFor(Normalized->getStart(), TruncTy),
                                  Result);
-      rememberInstruction(Result);
-    }
   }
 
   // Re-apply any non-loop-dominating scale.
@@ -1472,7 +1460,6 @@ Value *SCEVExpander::expandAddRecExprLiterally(const SCEVAddRecExpr *S) {
     Result = InsertNoopCastOfTo(Result, IntTy);
     Result = Builder.CreateMul(Result,
                                expandCodeFor(PostLoopScale, IntTy));
-    rememberInstruction(Result);
   }
 
   // Re-apply any non-loop-dominating offset.
@@ -1488,7 +1475,6 @@ Value *SCEVExpander::expandAddRecExprLiterally(const SCEVAddRecExpr *S) {
       Result = InsertNoopCastOfTo(Result, IntTy);
       Result = Builder.CreateAdd(Result,
                                  expandCodeFor(PostLoopOffset, IntTy));
-      rememberInstruction(Result);
     }
   }
 
@@ -1648,27 +1634,21 @@ Value *SCEVExpander::visitTruncateExpr(const SCEVTruncateExpr *S) {
   Type *Ty = SE.getEffectiveSCEVType(S->getType());
   Value *V = expandCodeFor(S->getOperand(),
                            SE.getEffectiveSCEVType(S->getOperand()->getType()));
-  Value *I = Builder.CreateTrunc(V, Ty);
-  rememberInstruction(I);
-  return I;
+  return Builder.CreateTrunc(V, Ty);
 }
 
 Value *SCEVExpander::visitZeroExtendExpr(const SCEVZeroExtendExpr *S) {
   Type *Ty = SE.getEffectiveSCEVType(S->getType());
   Value *V = expandCodeFor(S->getOperand(),
                            SE.getEffectiveSCEVType(S->getOperand()->getType()));
-  Value *I = Builder.CreateZExt(V, Ty);
-  rememberInstruction(I);
-  return I;
+  return Builder.CreateZExt(V, Ty);
 }
 
 Value *SCEVExpander::visitSignExtendExpr(const SCEVSignExtendExpr *S) {
   Type *Ty = SE.getEffectiveSCEVType(S->getType());
   Value *V = expandCodeFor(S->getOperand(),
                            SE.getEffectiveSCEVType(S->getOperand()->getType()));
-  Value *I = Builder.CreateSExt(V, Ty);
-  rememberInstruction(I);
-  return I;
+  return Builder.CreateSExt(V, Ty);
 }
 
 Value *SCEVExpander::visitSMaxExpr(const SCEVSMaxExpr *S) {
@@ -1684,9 +1664,7 @@ Value *SCEVExpander::visitSMaxExpr(const SCEVSMaxExpr *S) {
     }
     Value *RHS = expandCodeFor(S->getOperand(i), Ty);
     Value *ICmp = Builder.CreateICmpSGT(LHS, RHS);
-    rememberInstruction(ICmp);
     Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "smax");
-    rememberInstruction(Sel);
     LHS = Sel;
   }
   // In the case of mixed integer and pointer types, cast the
@@ -1709,9 +1687,7 @@ Value *SCEVExpander::visitUMaxExpr(const SCEVUMaxExpr *S) {
     }
     Value *RHS = expandCodeFor(S->getOperand(i), Ty);
     Value *ICmp = Builder.CreateICmpUGT(LHS, RHS);
-    rememberInstruction(ICmp);
     Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "umax");
-    rememberInstruction(Sel);
     LHS = Sel;
   }
   // In the case of mixed integer and pointer types, cast the
@@ -1734,9 +1710,7 @@ Value *SCEVExpander::visitSMinExpr(const SCEVSMinExpr *S) {
     }
     Value *RHS = expandCodeFor(S->getOperand(i), Ty);
     Value *ICmp = Builder.CreateICmpSLT(LHS, RHS);
-    rememberInstruction(ICmp);
     Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "smin");
-    rememberInstruction(Sel);
     LHS = Sel;
   }
   // In the case of mixed integer and pointer types, cast the
@@ -1759,9 +1733,7 @@ Value *SCEVExpander::visitUMinExpr(const SCEVUMinExpr *S) {
     }
     Value *RHS = expandCodeFor(S->getOperand(i), Ty);
     Value *ICmp = Builder.CreateICmpULT(LHS, RHS);
-    rememberInstruction(ICmp);
     Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "umin");
-    rememberInstruction(Sel);
     LHS = Sel;
   }
   // In the case of mixed integer and pointer types, cast the
@@ -2443,8 +2415,7 @@ Value *SCEVExpander::generateOverflowCheck(const SCEVAddRecExpr *AR,
     EndCheck = Builder.CreateOr(EndCheck, BackedgeCheck);
   }
 
-  EndCheck = Builder.CreateOr(EndCheck, OfMul);
-  return EndCheck;
+  return Builder.CreateOr(EndCheck, OfMul);
 }
 
 Value *SCEVExpander::expandWrapPredicate(const SCEVWrapPredicate *Pred,
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index aa015ffe56227..a578bbd3f65b4 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -105,6 +105,10 @@ static cl::opt<bool> DupRet(
     "simplifycfg-dup-ret", cl::Hidden, cl::init(false),
     cl::desc("Duplicate return instructions into unconditional branches"));
 
+static cl::opt<bool>
+    HoistCommon("simplifycfg-hoist-common", cl::Hidden, cl::init(true),
+                cl::desc("Hoist common instructions up to the parent block"));
+
 static cl::opt<bool>
     SinkCommon("simplifycfg-sink-common", cl::Hidden, cl::init(true),
                cl::desc("Sink common instructions down to the end block"));
@@ -2229,6 +2233,12 @@ static bool BlockIsSimpleEnoughToThreadThrough(BasicBlock *BB) {
   for (Instruction &I : BB->instructionsWithoutDebug()) {
     if (Size > MaxSmallBlockSize)
       return false; // Don't clone large BB's.
+
+    // Can't fold blocks that contain noduplicate or convergent calls.
+    if (CallInst *CI = dyn_cast<CallInst>(&I))
+      if (CI->cannotDuplicate() || CI->isConvergent())
+        return false;
+
     // We will delete Phis while threading, so Phis should not be accounted in
     // block's size
     if (!isa<PHINode>(I))
@@ -2270,13 +2280,6 @@ static bool FoldCondBranchOnPHI(BranchInst *BI, const DataLayout &DL,
   if (!BlockIsSimpleEnoughToThreadThrough(BB))
     return false;
 
-  // Can't fold blocks that contain noduplicate or convergent calls.
-  if (any_of(*BB, [](const Instruction &I) {
-        const CallInst *CI = dyn_cast<CallInst>(&I);
-        return CI && (CI->cannotDuplicate() || CI->isConvergent());
-      }))
-    return false;
-
   // Okay, this is a simple enough basic block.  See if any phi values are
   // constants.
   for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
@@ -6063,8 +6066,9 @@ bool SimplifyCFGOpt::simplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder) {
   // can hoist it up to the branching block.
   if (BI->getSuccessor(0)->getSinglePredecessor()) {
     if (BI->getSuccessor(1)->getSinglePredecessor()) {
-      if (HoistThenElseCodeToIf(BI, TTI))
-        return requestResimplify();
+      if (HoistCommon && Options.HoistCommonInsts)
+        if (HoistThenElseCodeToIf(BI, TTI))
+          return requestResimplify();
     } else {
       // If Successor #1 has multiple preds, we may be able to conditionally
       // execute Successor #0 if it branches to Successor #1.
diff --git a/llvm/lib/Transforms/Utils/SizeOpts.cpp b/llvm/lib/Transforms/Utils/SizeOpts.cpp
index e257c5a015f51..beeb60698f045 100644
--- a/llvm/lib/Transforms/Utils/SizeOpts.cpp
+++ b/llvm/lib/Transforms/Utils/SizeOpts.cpp
@@ -43,11 +43,6 @@ cl::opt<bool> PGSOColdCodeOnlyForPartialSamplePGO(
     cl::desc("Apply the profile guided size optimizations only "
              "to cold code under partial-profile sample PGO."));
 
-cl::opt<bool> PGSOIRPassOrTestOnly(
-    "pgso-ir-pass-or-test-only", cl::Hidden, cl::init(false),
-    cl::desc("Apply the profile guided size optimizations only"
-             "to the IR passes or tests."));
-
 cl::opt<bool> ForcePGSO(
     "force-pgso", cl::Hidden, cl::init(false),
     cl::desc("Force the (profiled-guided) size optimizations. "));
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 23613775d896d..120b544808bed 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -20,6 +20,7 @@
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/PatternMatch.h"
+#include "llvm/Transforms/Utils/SizeOpts.h"
 #include "llvm/Transforms/Vectorize/LoopVectorize.h"
 
 using namespace llvm;
@@ -412,7 +413,11 @@ int LoopVectorizationLegality::isConsecutivePtr(Value *Ptr) {
   const ValueToValueMap &Strides =
       getSymbolicStrides() ? *getSymbolicStrides() : ValueToValueMap();
 
-  bool CanAddPredicate = !TheLoop->getHeader()->getParent()->hasOptSize();
+  Function *F = TheLoop->getHeader()->getParent();
+  bool OptForSize = F->hasOptSize() ||
+                    llvm::shouldOptimizeForSize(TheLoop->getHeader(), PSI, BFI,
+                                                PGSOQueryType::IRPass);
+  bool CanAddPredicate = !OptForSize;
   int Stride = getPtrStride(PSE, Ptr, TheLoop, Strides, CanAddPredicate, false);
   if (Stride == 1 || Stride == -1)
     return Stride;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7f13a689cedbd..d207ca250783f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -395,11 +395,19 @@ class InnerLoopVectorizer {
                       const TargetTransformInfo *TTI, AssumptionCache *AC,
                       OptimizationRemarkEmitter *ORE, unsigned VecWidth,
                       unsigned UnrollFactor, LoopVectorizationLegality *LVL,
-                      LoopVectorizationCostModel *CM)
+                      LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI,
+                      ProfileSummaryInfo *PSI)
       : OrigLoop(OrigLoop), PSE(PSE), LI(LI), DT(DT), TLI(TLI), TTI(TTI),
         AC(AC), ORE(ORE), VF(VecWidth), UF(UnrollFactor),
         Builder(PSE.getSE()->getContext()),
-        VectorLoopValueMap(UnrollFactor, VecWidth), Legal(LVL), Cost(CM) {}
+        VectorLoopValueMap(UnrollFactor, VecWidth), Legal(LVL), Cost(CM),
+        BFI(BFI), PSI(PSI) {
+    // Query this against the original loop and save it here because the profile
+    // of the original loop header may change as the transformation happens.
+    OptForSizeBasedOnProfile = llvm::shouldOptimizeForSize(
+        OrigLoop->getHeader(), PSI, BFI, PGSOQueryType::IRPass);
+  }
+
   virtual ~InnerLoopVectorizer() = default;
 
   /// Create a new empty loop. Unlink the old loop and connect the new one.
@@ -779,6 +787,14 @@ class InnerLoopVectorizer {
   // Vector of original scalar PHIs whose corresponding widened PHIs need to be
   // fixed up at the end of vector code generation.
   SmallVector<PHINode *, 8> OrigPHIsToFix;
+
+  /// BFI and PSI are used to check for profile guided size optimizations.
+  BlockFrequencyInfo *BFI;
+  ProfileSummaryInfo *PSI;
+
+  // Whether this loop should be optimized for size based on profile guided size
+  // optimizatios.
+  bool OptForSizeBasedOnProfile;
 };
 
 class InnerLoopUnroller : public InnerLoopVectorizer {
@@ -789,9 +805,10 @@ class InnerLoopUnroller : public InnerLoopVectorizer {
                     const TargetTransformInfo *TTI, AssumptionCache *AC,
                     OptimizationRemarkEmitter *ORE, unsigned UnrollFactor,
                     LoopVectorizationLegality *LVL,
-                    LoopVectorizationCostModel *CM)
+                    LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI,
+                    ProfileSummaryInfo *PSI)
       : InnerLoopVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE, 1,
-                            UnrollFactor, LVL, CM) {}
+                            UnrollFactor, LVL, CM, BFI, PSI) {}
 
 private:
   Value *getBroadcastInstrs(Value *V) override;
@@ -2754,7 +2771,8 @@ void InnerLoopVectorizer::emitSCEVChecks(Loop *L, BasicBlock *Bypass) {
     if (C->isZero())
       return;
 
-  assert(!SCEVCheckBlock->getParent()->hasOptSize() &&
+  assert(!(SCEVCheckBlock->getParent()->hasOptSize() ||
+           OptForSizeBasedOnProfile) &&
          "Cannot SCEV check stride or overflow when optimizing for size");
 
   SCEVCheckBlock->setName("vector.scevcheck");
@@ -2800,7 +2818,7 @@ void InnerLoopVectorizer::emitMemRuntimeChecks(Loop *L, BasicBlock *Bypass) {
   assert(MemRuntimeCheck && "no RT checks generated although RtPtrChecking "
                             "claimed checks are required");
 
-  if (MemCheckBlock->getParent()->hasOptSize()) {
+  if (MemCheckBlock->getParent()->hasOptSize() || OptForSizeBasedOnProfile) {
     assert(Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled &&
            "Cannot emit memory checks when optimizing for size, unless forced "
            "to vectorize.");
@@ -7729,7 +7747,7 @@ static bool processLoopInVPlanNativePath(
   LVP.setBestPlan(VF.Width, 1);
 
   InnerLoopVectorizer LB(L, PSE, LI, DT, TLI, TTI, AC, ORE, VF.Width, 1, LVL,
-                         &CM);
+                         &CM, BFI, PSI);
   LLVM_DEBUG(dbgs() << "Vectorizing outer loop in \""
                     << L->getHeader()->getParent()->getName() << "\"\n");
   LVP.executePlan(LB, DT);
@@ -7793,7 +7811,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
   // Check if it is legal to vectorize the loop.
   LoopVectorizationRequirements Requirements(*ORE);
   LoopVectorizationLegality LVL(L, PSE, DT, TTI, TLI, AA, F, GetLAA, LI, ORE,
-                                &Requirements, &Hints, DB, AC);
+                                &Requirements, &Hints, DB, AC, BFI, PSI);
   if (!LVL.canVectorize(EnableVPlanNativePath)) {
     LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Cannot prove legality.\n");
     Hints.emitRemarkWithHints();
@@ -7993,8 +8011,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
     assert(IC > 1 && "interleave count should not be 1 or 0");
     // If we decided that it is not legal to vectorize the loop, then
     // interleave it.
-    InnerLoopUnroller Unroller(L, PSE, LI, DT, TLI, TTI, AC, ORE, IC, &LVL,
-                               &CM);
+    InnerLoopUnroller Unroller(L, PSE, LI, DT, TLI, TTI, AC, ORE, IC, &LVL, &CM,
+                               BFI, PSI);
     LVP.executePlan(Unroller, DT);
 
     ORE->emit([&]() {
@@ -8006,7 +8024,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
   } else {
     // If we decided that it is *legal* to vectorize the loop, then do it.
     InnerLoopVectorizer LB(L, PSE, LI, DT, TLI, TTI, AC, ORE, VF.Width, IC,
-                           &LVL, &CM);
+                           &LVL, &CM, BFI, PSI);
     LVP.executePlan(LB, DT);
     ++LoopsVectorized;
 
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 0f29e24a26eb4..dc74a46c70b55 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -7,7 +7,7 @@
 # Setting CMake minimum required version should be at the very top of the file
 # if this is the entry point.
 if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
-  cmake_minimum_required(VERSION 3.4.3)
+  cmake_minimum_required(VERSION 3.13.4)
   project(Runtimes C CXX ASM)
 endif()
 
@@ -56,7 +56,7 @@ if(${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR})
     set_property(GLOBAL APPEND PROPERTY SUB_COMPONENTS ${name})
   endfunction()
 
-  cmake_minimum_required(VERSION 3.4.3)
+  cmake_minimum_required(VERSION 3.13.4)
   project(Runtimes C CXX ASM)
 
   find_package(LLVM PATHS "${LLVM_BINARY_DIR}" NO_DEFAULT_PATH NO_CMAKE_FIND_ROOT_PATH)
diff --git a/llvm/test/Analysis/ConstantFolding/ARM/lit.local.cfg b/llvm/test/Analysis/ConstantFolding/ARM/lit.local.cfg
new file mode 100644
index 0000000000000..236e1d3441665
--- /dev/null
+++ b/llvm/test/Analysis/ConstantFolding/ARM/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'ARM' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Analysis/ConstantFolding/ARM/mve-vctp.ll b/llvm/test/Analysis/ConstantFolding/ARM/mve-vctp.ll
new file mode 100644
index 0000000000000..0651a489e35af
--- /dev/null
+++ b/llvm/test/Analysis/ConstantFolding/ARM/mve-vctp.ll
@@ -0,0 +1,267 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instsimplify -S -o - %s | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+
+define <16 x i1> @vctp8_0() {
+; CHECK-LABEL: @vctp8_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <16 x i1> zeroinitializer
+;
+entry:
+  %int = call <16 x i1> @llvm.arm.mve.vctp8(i32 0)
+  ret <16 x i1> %int
+}
+
+define <16 x i1> @vctp8_1() {
+; CHECK-LABEL: @vctp8_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <16 x i1> <i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>
+;
+entry:
+  %int = call <16 x i1> @llvm.arm.mve.vctp8(i32 1)
+  ret <16 x i1> %int
+}
+
+define <16 x i1> @vctp8_8() {
+; CHECK-LABEL: @vctp8_8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>
+;
+entry:
+  %int = call <16 x i1> @llvm.arm.mve.vctp8(i32 8)
+  ret <16 x i1> %int
+}
+
+define <16 x i1> @vctp8_15() {
+; CHECK-LABEL: @vctp8_15(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false>
+;
+entry:
+  %int = call <16 x i1> @llvm.arm.mve.vctp8(i32 15)
+  ret <16 x i1> %int
+}
+
+define <16 x i1> @vctp8_16() {
+; CHECK-LABEL: @vctp8_16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>
+;
+entry:
+  %int = call <16 x i1> @llvm.arm.mve.vctp8(i32 16)
+  ret <16 x i1> %int
+}
+
+define <16 x i1> @vctp8_100() {
+; CHECK-LABEL: @vctp8_100(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>
+;
+entry:
+  %int = call <16 x i1> @llvm.arm.mve.vctp8(i32 100)
+  ret <16 x i1> %int
+}
+
+define <16 x i1> @vctp8_m1() {
+; CHECK-LABEL: @vctp8_m1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>
+;
+entry:
+  %int = call <16 x i1> @llvm.arm.mve.vctp8(i32 -1)
+  ret <16 x i1> %int
+}
+
+
+
+define <8 x i1> @vctp16_0() {
+; CHECK-LABEL: @vctp16_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <8 x i1> zeroinitializer
+;
+entry:
+  %int = call <8 x i1> @llvm.arm.mve.vctp16(i32 0)
+  ret <8 x i1> %int
+}
+
+define <8 x i1> @vctp16_1() {
+; CHECK-LABEL: @vctp16_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <8 x i1> <i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>
+;
+entry:
+  %int = call <8 x i1> @llvm.arm.mve.vctp16(i32 1)
+  ret <8 x i1> %int
+}
+
+define <8 x i1> @vctp16_4() {
+; CHECK-LABEL: @vctp16_4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false>
+;
+entry:
+  %int = call <8 x i1> @llvm.arm.mve.vctp16(i32 4)
+  ret <8 x i1> %int
+}
+
+define <8 x i1> @vctp16_7() {
+; CHECK-LABEL: @vctp16_7(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false>
+;
+entry:
+  %int = call <8 x i1> @llvm.arm.mve.vctp16(i32 7)
+  ret <8 x i1> %int
+}
+
+define <8 x i1> @vctp16_8() {
+; CHECK-LABEL: @vctp16_8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>
+;
+entry:
+  %int = call <8 x i1> @llvm.arm.mve.vctp16(i32 8)
+  ret <8 x i1> %int
+}
+
+define <8 x i1> @vctp16_100() {
+; CHECK-LABEL: @vctp16_100(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>
+;
+entry:
+  %int = call <8 x i1> @llvm.arm.mve.vctp16(i32 100)
+  ret <8 x i1> %int
+}
+
+define <8 x i1> @vctp16_m1() {
+; CHECK-LABEL: @vctp16_m1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>
+;
+entry:
+  %int = call <8 x i1> @llvm.arm.mve.vctp16(i32 -1)
+  ret <8 x i1> %int
+}
+
+
+
+define <4 x i1> @vctp32_0() {
+; CHECK-LABEL: @vctp32_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <4 x i1> zeroinitializer
+;
+entry:
+  %int = call <4 x i1> @llvm.arm.mve.vctp32(i32 0)
+  ret <4 x i1> %int
+}
+
+define <4 x i1> @vctp32_1() {
+; CHECK-LABEL: @vctp32_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <4 x i1> <i1 true, i1 false, i1 false, i1 false>
+;
+entry:
+  %int = call <4 x i1> @llvm.arm.mve.vctp32(i32 1)
+  ret <4 x i1> %int
+}
+
+define <4 x i1> @vctp32_3() {
+; CHECK-LABEL: @vctp32_3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <4 x i1> <i1 true, i1 true, i1 true, i1 false>
+;
+entry:
+  %int = call <4 x i1> @llvm.arm.mve.vctp32(i32 3)
+  ret <4 x i1> %int
+}
+
+define <4 x i1> @vctp32_4() {
+; CHECK-LABEL: @vctp32_4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <4 x i1> <i1 true, i1 true, i1 true, i1 true>
+;
+entry:
+  %int = call <4 x i1> @llvm.arm.mve.vctp32(i32 4)
+  ret <4 x i1> %int
+}
+
+define <4 x i1> @vctp32_100() {
+; CHECK-LABEL: @vctp32_100(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <4 x i1> <i1 true, i1 true, i1 true, i1 true>
+;
+entry:
+  %int = call <4 x i1> @llvm.arm.mve.vctp32(i32 100)
+  ret <4 x i1> %int
+}
+
+define <4 x i1> @vctp32_m1() {
+; CHECK-LABEL: @vctp32_m1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <4 x i1> <i1 true, i1 true, i1 true, i1 true>
+;
+entry:
+  %int = call <4 x i1> @llvm.arm.mve.vctp32(i32 -1)
+  ret <4 x i1> %int
+}
+
+
+
+define <4 x i1> @vctp64_0() {
+; CHECK-LABEL: @vctp64_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <4 x i1> zeroinitializer
+;
+entry:
+  %int = call <4 x i1> @llvm.arm.mve.vctp64(i32 0)
+  ret <4 x i1> %int
+}
+
+define <4 x i1> @vctp64_1() {
+; CHECK-LABEL: @vctp64_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <4 x i1> <i1 true, i1 true, i1 false, i1 false>
+;
+entry:
+  %int = call <4 x i1> @llvm.arm.mve.vctp64(i32 1)
+  ret <4 x i1> %int
+}
+
+define <4 x i1> @vctp64_2() {
+; CHECK-LABEL: @vctp64_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <4 x i1> <i1 true, i1 true, i1 true, i1 true>
+;
+entry:
+  %int = call <4 x i1> @llvm.arm.mve.vctp64(i32 2)
+  ret <4 x i1> %int
+}
+
+define <4 x i1> @vctp64_100() {
+; CHECK-LABEL: @vctp64_100(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <4 x i1> <i1 true, i1 true, i1 true, i1 true>
+;
+entry:
+  %int = call <4 x i1> @llvm.arm.mve.vctp64(i32 100)
+  ret <4 x i1> %int
+}
+
+define <4 x i1> @vctp64_m1() {
+; CHECK-LABEL: @vctp64_m1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <4 x i1> <i1 true, i1 true, i1 true, i1 true>
+;
+entry:
+  %int = call <4 x i1> @llvm.arm.mve.vctp64(i32 -1)
+  ret <4 x i1> %int
+}
+
+
+
+declare <4 x i1> @llvm.arm.mve.vctp64(i32)
+declare <4 x i1> @llvm.arm.mve.vctp32(i32)
+declare <8 x i1> @llvm.arm.mve.vctp16(i32)
+declare <16 x i1> @llvm.arm.mve.vctp8(i32)
diff --git a/llvm/test/Analysis/DivergenceAnalysis/AMDGPU/kernel-args.ll b/llvm/test/Analysis/DivergenceAnalysis/AMDGPU/kernel-args.ll
index d7e050de1f3a2..85712e63d5f36 100644
--- a/llvm/test/Analysis/DivergenceAnalysis/AMDGPU/kernel-args.ll
+++ b/llvm/test/Analysis/DivergenceAnalysis/AMDGPU/kernel-args.ll
@@ -1,16 +1,14 @@
 ; RUN: opt %s -mtriple amdgcn-- -analyze -divergence -use-gpu-divergence-analysis | FileCheck %s
 
 ; CHECK-LABEL: Printing analysis 'Legacy Divergence Analysis' for function 'test_amdgpu_ps':
-; CHECK: DIVERGENT:
-; CHECK-NOT: %arg0
-; CHECK-NOT: %arg1
-; CHECK-NOT: %arg2
-; CHECK: <2 x i32> %arg3
+; CHECK: DIVERGENT:  [4 x <16 x i8>] addrspace(4)* %arg0
+; CHECK-NOT: DIVERGENT
+; CHECK: DIVERGENT:  <2 x i32> %arg3
 ; CHECK: DIVERGENT:  <3 x i32> %arg4
 ; CHECK: DIVERGENT:  float %arg5
 ; CHECK: DIVERGENT:  i32 %arg6
 
-define amdgpu_ps void @test_amdgpu_ps([4 x <16 x i8>] addrspace(2)* byval %arg0, float inreg %arg1, i32 inreg %arg2, <2 x i32> %arg3, <3 x i32> %arg4, float %arg5, i32 %arg6) #0 {
+define amdgpu_ps void @test_amdgpu_ps([4 x <16 x i8>] addrspace(4)* byref([4 x <16 x i8>]) %arg0, float inreg %arg1, i32 inreg %arg2, <2 x i32> %arg3, <3 x i32> %arg4, float %arg5, i32 %arg6) #0 {
   ret void
 }
 
@@ -22,7 +20,7 @@ define amdgpu_ps void @test_amdgpu_ps([4 x <16 x i8>] addrspace(2)* byval %arg0,
 ; CHECK-NOT: %arg4
 ; CHECK-NOT: %arg5
 ; CHECK-NOT: %arg6
-define amdgpu_kernel void @test_amdgpu_kernel([4 x <16 x i8>] addrspace(2)* byval %arg0, float inreg %arg1, i32 inreg %arg2, <2 x i32> %arg3, <3 x i32> %arg4, float %arg5, i32 %arg6) #0 {
+define amdgpu_kernel void @test_amdgpu_kernel([4 x <16 x i8>] addrspace(4)* byref([4 x <16 x i8>]) %arg0, float inreg %arg1, i32 inreg %arg2, <2 x i32> %arg3, <3 x i32> %arg4, float %arg5, i32 %arg6) #0 {
   ret void
 }
 
@@ -34,7 +32,7 @@ define amdgpu_kernel void @test_amdgpu_kernel([4 x <16 x i8>] addrspace(2)* byva
 ; CHECK: DIVERGENT:
 ; CHECK: DIVERGENT:
 ; CHECK: DIVERGENT:
-define void @test_c([4 x <16 x i8>] addrspace(2)* byval %arg0, float inreg %arg1, i32 inreg %arg2, <2 x i32> %arg3, <3 x i32> %arg4, float %arg5, i32 %arg6) #0 {
+define void @test_c([4 x <16 x i8>] addrspace(5)* byval %arg0, float inreg %arg1, i32 inreg %arg2, <2 x i32> %arg3, <3 x i32> %arg4, float %arg5, i32 %arg6) #0 {
   ret void
 }
 
diff --git a/llvm/test/Analysis/FunctionPropertiesAnalysis/matmul.ll b/llvm/test/Analysis/FunctionPropertiesAnalysis/matmul.ll
new file mode 100644
index 0000000000000..506914972a104
--- /dev/null
+++ b/llvm/test/Analysis/FunctionPropertiesAnalysis/matmul.ll
@@ -0,0 +1,140 @@
+; RUN: opt < %s -passes='print<func-properties>' -disable-output 2>&1 | FileCheck %s
+
+define i32 @main() {
+; CHECK-DAG: Printing analysis results of CFA for function 'main':
+
+entry:
+  %retval = alloca i32, align 4
+  %mat1 = alloca [2 x [2 x i32]], align 16
+  %mat2 = alloca [2 x [2 x i32]], align 16
+  %res = alloca [2 x [2 x i32]], align 16
+  %i = alloca i32, align 4
+  %j = alloca i32, align 4
+  store i32 0, i32* %retval, align 4
+  %arraydecay = getelementptr inbounds [2 x [2 x i32]], [2 x [2 x i32]]* %mat1, i64 0, i64 0
+  %arraydecay1 = getelementptr inbounds [2 x [2 x i32]], [2 x [2 x i32]]* %mat2, i64 0, i64 0
+  %arraydecay2 = getelementptr inbounds [2 x [2 x i32]], [2 x [2 x i32]]* %res, i64 0, i64 0
+  call void @multiply([2 x i32]* %arraydecay, [2 x i32]* %arraydecay1, [2 x i32]* %arraydecay2)
+  ret i32 0
+}
+; CHECK-DAG: BasicBlockCount: 1
+; CHECK-DAG: BlocksReachedFromConditionalInstruction: 0
+; CHECK-DAG: Uses: 1
+; CHECK-DAG: DirectCallsToDefinedFunctions: 1
+; CHECK-DAG: LoadInstCount: 0
+; CHECK-DAG: StoreInstCount: 1
+; CHECK-DAG: MaxLoopDepth: 0
+; CHECK-DAG: TopLevelLoopCount: 0
+
+define void @multiply([2 x i32]* %mat1, [2 x i32]* %mat2, [2 x i32]* %res) {
+; CHECK-DAG: Printing analysis results of CFA for function 'multiply':
+entry:
+  %mat1.addr = alloca [2 x i32]*, align 8
+  %mat2.addr = alloca [2 x i32]*, align 8
+  %res.addr = alloca [2 x i32]*, align 8
+  %i = alloca i32, align 4
+  %j = alloca i32, align 4
+  %k = alloca i32, align 4
+  store [2 x i32]* %mat1, [2 x i32]** %mat1.addr, align 8
+  store [2 x i32]* %mat2, [2 x i32]** %mat2.addr, align 8
+  store [2 x i32]* %res, [2 x i32]** %res.addr, align 8
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc24, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 2
+  br i1 %cmp, label %for.body, label %for.end26
+
+for.body:                                         ; preds = %for.cond
+  store i32 0, i32* %j, align 4
+  br label %for.cond1
+
+for.cond1:                                        ; preds = %for.inc21, %for.body
+  %1 = load i32, i32* %j, align 4
+  %cmp2 = icmp slt i32 %1, 2
+  br i1 %cmp2, label %for.body3, label %for.end23
+
+for.body3:                                        ; preds = %for.cond1
+  %2 = load [2 x i32]*, [2 x i32]** %res.addr, align 8
+  %3 = load i32, i32* %i, align 4
+  %idxprom = sext i32 %3 to i64
+  %arrayidx = getelementptr inbounds [2 x i32], [2 x i32]* %2, i64 %idxprom
+  %4 = load i32, i32* %j, align 4
+  %idxprom4 = sext i32 %4 to i64
+  %arrayidx5 = getelementptr inbounds [2 x i32], [2 x i32]* %arrayidx, i64 0, i64 %idxprom4
+  store i32 0, i32* %arrayidx5, align 4
+  store i32 0, i32* %k, align 4
+  br label %for.cond6
+
+for.cond6:                                        ; preds = %for.inc, %for.body3
+  %5 = load i32, i32* %k, align 4
+  %cmp7 = icmp slt i32 %5, 2
+  br i1 %cmp7, label %for.body8, label %for.end
+
+for.body8:                                        ; preds = %for.cond6
+  %6 = load [2 x i32]*, [2 x i32]** %mat1.addr, align 8
+  %7 = load i32, i32* %i, align 4
+  %idxprom9 = sext i32 %7 to i64
+  %arrayidx10 = getelementptr inbounds [2 x i32], [2 x i32]* %6, i64 %idxprom9
+  %8 = load i32, i32* %k, align 4
+  %idxprom11 = sext i32 %8 to i64
+  %arrayidx12 = getelementptr inbounds [2 x i32], [2 x i32]* %arrayidx10, i64 0, i64 %idxprom11
+  %9 = load i32, i32* %arrayidx12, align 4
+  %10 = load [2 x i32]*, [2 x i32]** %mat2.addr, align 8
+  %11 = load i32, i32* %k, align 4
+  %idxprom13 = sext i32 %11 to i64
+  %arrayidx14 = getelementptr inbounds [2 x i32], [2 x i32]* %10, i64 %idxprom13
+  %12 = load i32, i32* %j, align 4
+  %idxprom15 = sext i32 %12 to i64
+  %arrayidx16 = getelementptr inbounds [2 x i32], [2 x i32]* %arrayidx14, i64 0, i64 %idxprom15
+  %13 = load i32, i32* %arrayidx16, align 4
+  %mul = mul nsw i32 %9, %13
+  %14 = load [2 x i32]*, [2 x i32]** %res.addr, align 8
+  %15 = load i32, i32* %i, align 4
+  %idxprom17 = sext i32 %15 to i64
+  %arrayidx18 = getelementptr inbounds [2 x i32], [2 x i32]* %14, i64 %idxprom17
+  %16 = load i32, i32* %j, align 4
+  %idxprom19 = sext i32 %16 to i64
+  %arrayidx20 = getelementptr inbounds [2 x i32], [2 x i32]* %arrayidx18, i64 0, i64 %idxprom19
+  %17 = load i32, i32* %arrayidx20, align 4
+  %add = add nsw i32 %17, %mul
+  store i32 %add, i32* %arrayidx20, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body8
+  %18 = load i32, i32* %k, align 4
+  %inc = add nsw i32 %18, 1
+  store i32 %inc, i32* %k, align 4
+  br label %for.cond6
+
+for.end:                                          ; preds = %for.cond6
+  br label %for.inc21
+
+for.inc21:                                        ; preds = %for.end
+  %19 = load i32, i32* %j, align 4
+  %inc22 = add nsw i32 %19, 1
+  store i32 %inc22, i32* %j, align 4
+  br label %for.cond1
+
+for.end23:                                        ; preds = %for.cond1
+  br label %for.inc24
+
+for.inc24:                                        ; preds = %for.end23
+  %20 = load i32, i32* %i, align 4
+  %inc25 = add nsw i32 %20, 1
+  store i32 %inc25, i32* %i, align 4
+  br label %for.cond
+
+for.end26:                                        ; preds = %for.cond
+  ret void
+}
+
+; CHECK-DAG: BasicBlockCount: 13
+; CHECK-DAG: BlocksReachedFromConditionalInstruction: 6
+; CHECK-DAG: Uses: 2
+; CHECK-DAG: DirectCallsToDefinedFunctions: 0
+; CHECK-DAG: LoadInstCount: 21
+; CHECK-DAG: StoreInstCount: 11
+; CHECK-DAG: MaxLoopDepth: 3
+; CHECK-DAG: TopLevelLoopCount: 1
\ No newline at end of file
diff --git a/llvm/test/Analysis/LegacyDivergenceAnalysis/AMDGPU/kernel-args.ll b/llvm/test/Analysis/LegacyDivergenceAnalysis/AMDGPU/kernel-args.ll
index ddf8e8e1f9bcf..7276634f225f1 100644
--- a/llvm/test/Analysis/LegacyDivergenceAnalysis/AMDGPU/kernel-args.ll
+++ b/llvm/test/Analysis/LegacyDivergenceAnalysis/AMDGPU/kernel-args.ll
@@ -1,16 +1,14 @@
 ; RUN: opt %s -mtriple amdgcn-- -amdgpu-use-legacy-divergence-analysis -analyze -divergence | FileCheck %s
 
 ; CHECK-LABEL: Printing analysis 'Legacy Divergence Analysis' for function 'test_amdgpu_ps':
-; CHECK: DIVERGENT:
-; CHECK-NOT: %arg0
-; CHECK-NOT: %arg1
-; CHECK-NOT: %arg2
-; CHECK: <2 x i32> %arg3
+; CHECK: DIVERGENT:  [4 x <16 x i8>] addrspace(4)* %arg0
+; CHECK-NOT: DIVERGENT
+; CHECK: DIVERGENT:  <2 x i32> %arg3
 ; CHECK: DIVERGENT:  <3 x i32> %arg4
 ; CHECK: DIVERGENT:  float %arg5
 ; CHECK: DIVERGENT:  i32 %arg6
 
-define amdgpu_ps void @test_amdgpu_ps([4 x <16 x i8>] addrspace(4)* byval %arg0, float inreg %arg1, i32 inreg %arg2, <2 x i32> %arg3, <3 x i32> %arg4, float %arg5, i32 %arg6) #0 {
+define amdgpu_ps void @test_amdgpu_ps([4 x <16 x i8>] addrspace(4)* byref([4 x <16 x i8>]) %arg0, float inreg %arg1, i32 inreg %arg2, <2 x i32> %arg3, <3 x i32> %arg4, float %arg5, i32 %arg6) #0 {
   ret void
 }
 
@@ -22,7 +20,7 @@ define amdgpu_ps void @test_amdgpu_ps([4 x <16 x i8>] addrspace(4)* byval %arg0,
 ; CHECK-NOT: %arg4
 ; CHECK-NOT: %arg5
 ; CHECK-NOT: %arg6
-define amdgpu_kernel void @test_amdgpu_kernel([4 x <16 x i8>] addrspace(4)* byval %arg0, float inreg %arg1, i32 inreg %arg2, <2 x i32> %arg3, <3 x i32> %arg4, float %arg5, i32 %arg6) #0 {
+define amdgpu_kernel void @test_amdgpu_kernel([4 x <16 x i8>] addrspace(4)* byref([4 x <16 x i8>]) %arg0, float inreg %arg1, i32 inreg %arg2, <2 x i32> %arg3, <3 x i32> %arg4, float %arg5, i32 %arg6) #0 {
   ret void
 }
 
diff --git a/llvm/test/Analysis/ScalarEvolution/pr46786.ll b/llvm/test/Analysis/ScalarEvolution/pr46786.ll
new file mode 100644
index 0000000000000..9255807f02c59
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/pr46786.ll
@@ -0,0 +1,133 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -analyze -enable-new-pm=0 -scalar-evolution | FileCheck %s
+; RUN: opt < %s -disable-output "-passes=print<scalar-evolution>" 2>&1 | FileCheck %s
+
+source_filename = "input.cpp"
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+
+; Function Attrs: nofree
+define i8* @FSE_decompress_usingDTable(i8* %arg, i32 %arg1, i32 %arg2, i32 %arg3) local_unnamed_addr #0 {
+; CHECK-LABEL: 'FSE_decompress_usingDTable'
+; CHECK-NEXT:  Classifying expressions for: @FSE_decompress_usingDTable
+; CHECK-NEXT:    %i = getelementptr inbounds i8, i8* %arg, i32 %arg2
+; CHECK-NEXT:    --> (%arg2 + %arg)<nsw> U: full-set S: full-set
+; CHECK-NEXT:    %i4 = sub nsw i32 0, %arg1
+; CHECK-NEXT:    --> (-1 * %arg1) U: full-set S: full-set
+; CHECK-NEXT:    %i5 = getelementptr inbounds i8, i8* %i, i32 %i4
+; CHECK-NEXT:    --> ((-1 * %arg1) + %arg2 + %arg) U: full-set S: full-set
+; CHECK-NEXT:    %i7 = select i1 %i6, i32 %arg2, i32 %arg1
+; CHECK-NEXT:    --> ((-1 * %arg) + (((-1 * %arg1) + %arg2 + %arg) umin %arg) + %arg1) U: full-set S: full-set
+; CHECK-NEXT:    %i8 = sub i32 %arg3, %i7
+; CHECK-NEXT:    --> ((-1 * (((-1 * %arg1) + %arg2 + %arg) umin %arg)) + (-1 * %arg1) + %arg3 + %arg) U: full-set S: full-set
+; CHECK-NEXT:    %i9 = getelementptr inbounds i8, i8* %arg, i32 %i8
+; CHECK-NEXT:    --> ((2 * %arg) + (-1 * (((-1 * %arg1) + %arg2 + %arg) umin %arg)) + (-1 * %arg1) + %arg3) U: full-set S: full-set
+; CHECK-NEXT:  Determining loop execution counts for: @FSE_decompress_usingDTable
+;
+bb:
+  %i = getelementptr inbounds i8, i8* %arg, i32 %arg2
+  %i4 = sub nsw i32 0, %arg1
+  %i5 = getelementptr inbounds i8, i8* %i, i32 %i4
+  %i6 = icmp ult i8* %i5, %arg
+  %i7 = select i1 %i6, i32 %arg2, i32 %arg1
+  %i8 = sub i32 %arg3, %i7
+  %i9 = getelementptr inbounds i8, i8* %arg, i32 %i8
+  ret i8* %i9
+}
+
+define i8* @test_01(i8* %p) {
+; CHECK-LABEL: 'test_01'
+; CHECK-NEXT:  Classifying expressions for: @test_01
+; CHECK-NEXT:    %p1 = getelementptr i8, i8* %p, i32 2
+; CHECK-NEXT:    --> (2 + %p) U: full-set S: full-set
+; CHECK-NEXT:    %p2 = getelementptr i8, i8* %p, i32 1
+; CHECK-NEXT:    --> (1 + %p) U: full-set S: full-set
+; CHECK-NEXT:    %index = select i1 %cmp, i32 2, i32 1
+; CHECK-NEXT:    --> ((-1 * %p) + ((1 + %p) umax (2 + %p))) U: full-set S: full-set
+; CHECK-NEXT:    %neg_index = sub i32 0, %index
+; CHECK-NEXT:    --> ((-1 * ((1 + %p) umax (2 + %p))) + %p) U: full-set S: full-set
+; CHECK-NEXT:    %gep = getelementptr i8, i8* %p, i32 %neg_index
+; CHECK-NEXT:    --> ((2 * %p) + (-1 * ((1 + %p) umax (2 + %p)))) U: full-set S: full-set
+; CHECK-NEXT:  Determining loop execution counts for: @test_01
+;
+  %p1 = getelementptr i8, i8* %p, i32 2
+  %p2 = getelementptr i8, i8* %p, i32 1
+  %cmp = icmp ugt i8* %p1, %p2
+  %index = select i1 %cmp, i32 2, i32 1
+  %neg_index = sub i32 0, %index
+  %gep = getelementptr i8, i8* %p, i32 %neg_index
+  ret i8* %gep
+}
+
+define i8* @test_02(i8* %p) {
+; CHECK-LABEL: 'test_02'
+; CHECK-NEXT:  Classifying expressions for: @test_02
+; CHECK-NEXT:    %p1 = getelementptr i8, i8* %p, i32 2
+; CHECK-NEXT:    --> (2 + %p) U: full-set S: full-set
+; CHECK-NEXT:    %p2 = getelementptr i8, i8* %p, i32 1
+; CHECK-NEXT:    --> (1 + %p) U: full-set S: full-set
+; CHECK-NEXT:    %index = select i1 %cmp, i32 2, i32 1
+; CHECK-NEXT:    --> ((-1 * %p) + ((1 + %p) smax (2 + %p))) U: full-set S: full-set
+; CHECK-NEXT:    %neg_index = sub i32 0, %index
+; CHECK-NEXT:    --> ((-1 * ((1 + %p) smax (2 + %p))) + %p) U: full-set S: full-set
+; CHECK-NEXT:    %gep = getelementptr i8, i8* %p, i32 %neg_index
+; CHECK-NEXT:    --> ((2 * %p) + (-1 * ((1 + %p) smax (2 + %p)))) U: full-set S: full-set
+; CHECK-NEXT:  Determining loop execution counts for: @test_02
+;
+  %p1 = getelementptr i8, i8* %p, i32 2
+  %p2 = getelementptr i8, i8* %p, i32 1
+  %cmp = icmp sgt i8* %p1, %p2
+  %index = select i1 %cmp, i32 2, i32 1
+  %neg_index = sub i32 0, %index
+  %gep = getelementptr i8, i8* %p, i32 %neg_index
+  ret i8* %gep
+}
+
+define i8* @test_03(i8* %p) {
+; CHECK-LABEL: 'test_03'
+; CHECK-NEXT:  Classifying expressions for: @test_03
+; CHECK-NEXT:    %p1 = getelementptr i8, i8* %p, i32 2
+; CHECK-NEXT:    --> (2 + %p) U: full-set S: full-set
+; CHECK-NEXT:    %p2 = getelementptr i8, i8* %p, i32 1
+; CHECK-NEXT:    --> (1 + %p) U: full-set S: full-set
+; CHECK-NEXT:    %index = select i1 %cmp, i32 2, i32 1
+; CHECK-NEXT:    --> ((-1 * %p) + ((1 + %p) umin (2 + %p))) U: full-set S: full-set
+; CHECK-NEXT:    %neg_index = sub i32 0, %index
+; CHECK-NEXT:    --> ((-1 * ((1 + %p) umin (2 + %p))) + %p) U: full-set S: full-set
+; CHECK-NEXT:    %gep = getelementptr i8, i8* %p, i32 %neg_index
+; CHECK-NEXT:    --> ((2 * %p) + (-1 * ((1 + %p) umin (2 + %p)))) U: full-set S: full-set
+; CHECK-NEXT:  Determining loop execution counts for: @test_03
+;
+  %p1 = getelementptr i8, i8* %p, i32 2
+  %p2 = getelementptr i8, i8* %p, i32 1
+  %cmp = icmp ult i8* %p1, %p2
+  %index = select i1 %cmp, i32 2, i32 1
+  %neg_index = sub i32 0, %index
+  %gep = getelementptr i8, i8* %p, i32 %neg_index
+  ret i8* %gep
+}
+
+define i8* @test_04(i8* %p) {
+; CHECK-LABEL: 'test_04'
+; CHECK-NEXT:  Classifying expressions for: @test_04
+; CHECK-NEXT:    %p1 = getelementptr i8, i8* %p, i32 2
+; CHECK-NEXT:    --> (2 + %p) U: full-set S: full-set
+; CHECK-NEXT:    %p2 = getelementptr i8, i8* %p, i32 1
+; CHECK-NEXT:    --> (1 + %p) U: full-set S: full-set
+; CHECK-NEXT:    %index = select i1 %cmp, i32 2, i32 1
+; CHECK-NEXT:    --> ((-1 * %p) + ((1 + %p) smin (2 + %p))) U: full-set S: full-set
+; CHECK-NEXT:    %neg_index = sub i32 0, %index
+; CHECK-NEXT:    --> ((-1 * ((1 + %p) smin (2 + %p))) + %p) U: full-set S: full-set
+; CHECK-NEXT:    %gep = getelementptr i8, i8* %p, i32 %neg_index
+; CHECK-NEXT:    --> ((2 * %p) + (-1 * ((1 + %p) smin (2 + %p)))) U: full-set S: full-set
+; CHECK-NEXT:  Determining loop execution counts for: @test_04
+;
+  %p1 = getelementptr i8, i8* %p, i32 2
+  %p2 = getelementptr i8, i8* %p, i32 1
+  %cmp = icmp slt i8* %p1, %p2
+  %index = select i1 %cmp, i32 2, i32 1
+  %neg_index = sub i32 0, %index
+  %gep = getelementptr i8, i8* %p, i32 %neg_index
+  ret i8* %gep
+}
+
+attributes #0 = { nofree }
diff --git a/llvm/test/Assembler/byref-parse-error-0.ll b/llvm/test/Assembler/byref-parse-error-0.ll
new file mode 100644
index 0000000000000..287cd8453a651
--- /dev/null
+++ b/llvm/test/Assembler/byref-parse-error-0.ll
@@ -0,0 +1,6 @@
+; RUN: not llvm-as < %s 2>&1 | FileCheck %s
+
+; CHECK: <stdin>:[[@LINE+1]]:34: error: expected '('{{$}}
+define void @test_byref(i8* byref) {
+  ret void
+}
diff --git a/llvm/test/Assembler/byref-parse-error-1.ll b/llvm/test/Assembler/byref-parse-error-1.ll
new file mode 100644
index 0000000000000..3f189d340c9ed
--- /dev/null
+++ b/llvm/test/Assembler/byref-parse-error-1.ll
@@ -0,0 +1,6 @@
+; RUN: not llvm-as < %s 2>&1 | FileCheck %s
+
+; CHECK: <stdin>:[[@LINE+1]]:35: error: expected type{{$}}
+define void @test_byref(i8* byref() {
+  ret void
+}
diff --git a/llvm/test/Assembler/byref-parse-error-10.ll b/llvm/test/Assembler/byref-parse-error-10.ll
new file mode 100644
index 0000000000000..55997e5ffe542
--- /dev/null
+++ b/llvm/test/Assembler/byref-parse-error-10.ll
@@ -0,0 +1,6 @@
+; RUN: not llvm-as < %s 2>&1 | FileCheck %s
+
+; CHECK: <stdin>:[[@LINE+1]]:27: error: invalid use of parameter-only attribute on a function{{$}}
+define void @test_byref() byref(4) {
+  ret void
+}
diff --git a/llvm/test/Assembler/byref-parse-error-2.ll b/llvm/test/Assembler/byref-parse-error-2.ll
new file mode 100644
index 0000000000000..6037c3ac3c73d
--- /dev/null
+++ b/llvm/test/Assembler/byref-parse-error-2.ll
@@ -0,0 +1,7 @@
+; RUN: not llvm-as < %s 2>&1 | FileCheck %s
+
+; CHECK: <stdin>:[[@LINE+1]]:35: error: expected type{{$}}
+define void @test_byref(i8* byref()) {
+  ret void
+}
+
diff --git a/llvm/test/Assembler/byref-parse-error-3.ll b/llvm/test/Assembler/byref-parse-error-3.ll
new file mode 100644
index 0000000000000..b86eab4e69c3e
--- /dev/null
+++ b/llvm/test/Assembler/byref-parse-error-3.ll
@@ -0,0 +1,6 @@
+; RUN: not llvm-as < %s 2>&1 | FileCheck %s
+
+; CHECK: <stdin>:[[@LINE+1]]:35: error: expected type{{$}}
+define void @test_byref(i8* byref(-1)) {
+  ret void
+}
diff --git a/llvm/test/Assembler/byref-parse-error-4.ll b/llvm/test/Assembler/byref-parse-error-4.ll
new file mode 100644
index 0000000000000..6b6a6c25715cf
--- /dev/null
+++ b/llvm/test/Assembler/byref-parse-error-4.ll
@@ -0,0 +1,6 @@
+; RUN: not llvm-as < %s 2>&1 | FileCheck %s
+
+; CHECK: <stdin>:[[@LINE+1]]:35: error: expected type{{$}}
+define void @test_byref(i8* byref(0)) {
+  ret void
+}
diff --git a/llvm/test/Assembler/byref-parse-error-5.ll b/llvm/test/Assembler/byref-parse-error-5.ll
new file mode 100644
index 0000000000000..c1a154805f41b
--- /dev/null
+++ b/llvm/test/Assembler/byref-parse-error-5.ll
@@ -0,0 +1,6 @@
+; RUN: not llvm-as < %s 2>&1 | FileCheck %s
+
+; CHECK: <stdin>:[[@LINE+1]]:8: error: invalid use of parameter-only attribute{{$}}
+define byref i8* @test_byref() {
+  ret void
+}
diff --git a/llvm/test/Assembler/byref-parse-error-6.ll b/llvm/test/Assembler/byref-parse-error-6.ll
new file mode 100644
index 0000000000000..c5ecf83596e21
--- /dev/null
+++ b/llvm/test/Assembler/byref-parse-error-6.ll
@@ -0,0 +1,6 @@
+; RUN: not llvm-as < %s 2>&1 | FileCheck %s
+
+; CHECK: <stdin>:[[@LINE+1]]:8: error: invalid use of parameter-only attribute{{$}}
+define byref 8 i8* @test_byref() {
+  ret void
+}
diff --git a/llvm/test/Assembler/byref-parse-error-7.ll b/llvm/test/Assembler/byref-parse-error-7.ll
new file mode 100644
index 0000000000000..9ac60af5348dc
--- /dev/null
+++ b/llvm/test/Assembler/byref-parse-error-7.ll
@@ -0,0 +1,6 @@
+; RUN: not llvm-as < %s 2>&1 | FileCheck %s
+
+; CHECK: <stdin>:[[@LINE+1]]:8: error: invalid use of parameter-only attribute{{$}}
+define byref(8) i8* @test_byref() {
+  ret void
+}
diff --git a/llvm/test/Assembler/byref-parse-error-8.ll b/llvm/test/Assembler/byref-parse-error-8.ll
new file mode 100644
index 0000000000000..8f86b9f8afc42
--- /dev/null
+++ b/llvm/test/Assembler/byref-parse-error-8.ll
@@ -0,0 +1,6 @@
+; RUN: not llvm-as < %s 2>&1 | FileCheck %s
+
+; CHECK: <stdin>:[[@LINE+1]]:27: error: invalid use of parameter-only attribute on a function{{$}}
+define void @test_byref() byref {
+  ret void
+}
diff --git a/llvm/test/Assembler/byref-parse-error-9.ll b/llvm/test/Assembler/byref-parse-error-9.ll
new file mode 100644
index 0000000000000..bc707062a2e8d
--- /dev/null
+++ b/llvm/test/Assembler/byref-parse-error-9.ll
@@ -0,0 +1,6 @@
+; RUN: not llvm-as < %s 2>&1 | FileCheck %s
+
+; CHECK: <stdin>:[[@LINE+1]]:27: error: invalid use of parameter-only attribute on a function{{$}}
+define void @test_byref() byref=4 {
+  ret void
+}
diff --git a/llvm/test/Assembler/thinlto-bad-summary1.ll b/llvm/test/Assembler/thinlto-bad-summary1.ll
index f76dc969d6ecb..8ff5e06b189a6 100644
--- a/llvm/test/Assembler/thinlto-bad-summary1.ll
+++ b/llvm/test/Assembler/thinlto-bad-summary1.ll
@@ -2,7 +2,7 @@
 ; summary type label.
 ; RUN: not opt %s 2>&1 | FileCheck %s
 
-; CHECK: error: Expected 'gv', 'module', or 'typeid' at the start of summary entry
+; CHECK: error: Expected 'gv', 'module', 'typeid', 'flags' or 'blockcount' at the start of summary entry
 
 ; ModuleID = 'thinlto-function-summary-callgraph.ll'
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Assembler/thinlto-blockcount-summary.ll b/llvm/test/Assembler/thinlto-blockcount-summary.ll
new file mode 100644
index 0000000000000..0e5b618e95fae
--- /dev/null
+++ b/llvm/test/Assembler/thinlto-blockcount-summary.ll
@@ -0,0 +1,10 @@
+; Test that we can parse blockcount summaries.
+; RUN: opt %s -o /dev/null
+
+define void @main() {
+entry:
+    ret void
+}
+
+^0 = module: (path: "{{.*}}thinlto-bad-summary5.ll", hash: (0, 0, 0, 0, 0))
+^1 = blockcount: 1234
diff --git a/llvm/test/Assembler/thinlto-flags-summary.ll b/llvm/test/Assembler/thinlto-flags-summary.ll
new file mode 100644
index 0000000000000..cada38d74738c
--- /dev/null
+++ b/llvm/test/Assembler/thinlto-flags-summary.ll
@@ -0,0 +1,10 @@
+; Test that we can parse flag summaries.
+; RUN: opt %s -o /dev/null
+
+define void @main() {
+entry:
+    ret void
+}
+
+^0 = module: (path: "{{.*}}thinlto-flags-summary.ll", hash: (0, 0, 0, 0, 0))
+^1 = flags: 8
diff --git a/llvm/test/Assembler/thinlto-summary.ll b/llvm/test/Assembler/thinlto-summary.ll
index d018363b3b59f..dfcc62669d9a6 100644
--- a/llvm/test/Assembler/thinlto-summary.ll
+++ b/llvm/test/Assembler/thinlto-summary.ll
@@ -63,6 +63,8 @@
 ; Test the other kinds of type test resoultions
 ^27 = typeid: (name: "_ZTS1D", summary: (typeTestRes: (kind: byteArray, sizeM1BitWidth: 0)))
 ^28 = typeid: (name: "_ZTS1E", summary: (typeTestRes: (kind: unsat, sizeM1BitWidth: 0)))
+^29 = flags: 8
+^30 = blockcount: 1888
 
 ; Make sure we get back from llvm-dis essentially what we put in via llvm-as.
 ; CHECK: ^0 = module: (path: "thinlto-summary1.o", hash: (1369602428, 2747878711, 259090915, 2507395659, 1141468049))
@@ -94,6 +96,8 @@
 ; CHECK: ^26 = typeid: (name: "_ZTS1A", summary: (typeTestRes: (kind: allOnes, sizeM1BitWidth: 7), wpdResolutions: ((offset: 0, wpdRes: (kind: branchFunnel)), (offset: 8, wpdRes: (kind: singleImpl, singleImplName: "_ZN1A1nEi")), (offset: 16, wpdRes: (kind: indir, resByArg: (args: (1, 2), byArg: (kind: indir, byte: 2, bit: 3), args: (3), byArg: (kind: uniformRetVal, info: 1), args: (4), byArg: (kind: uniqueRetVal, info: 1), args: (5), byArg: (kind: virtualConstProp))))))) ; guid = 7004155349499253778
 ; CHECK: ^27 = typeid: (name: "_ZTS1D", summary: (typeTestRes: (kind: byteArray, sizeM1BitWidth: 0))) ; guid = 9614786172484273522
 ; CHECK: ^28 = typeid: (name: "_ZTS1E", summary: (typeTestRes: (kind: unsat, sizeM1BitWidth: 0))) ; guid = 17437243864166745132
+; CHECK: ^29 = flags: 8
+; CHECK: ^30 = blockcount: 1888
 
 ; Make sure parsing of a non-summary entry containing a ":" does not fail
 ; after summary parsing, which handles colons differently.
diff --git a/llvm/test/Bindings/Go/lit.local.cfg b/llvm/test/Bindings/Go/lit.local.cfg
index e32737079e4a0..91a0ad89ab4a2 100644
--- a/llvm/test/Bindings/Go/lit.local.cfg
+++ b/llvm/test/Bindings/Go/lit.local.cfg
@@ -9,7 +9,7 @@ if not 'go' in config.root.llvm_bindings:
 if not config.root.include_go_tests:
     config.unsupported = True
 
-if config.have_tf_aot:
+if config.have_tf_aot or config.have_tf_api:
     config.unsupported = True
 
 def find_executable(executable, path=None):
diff --git a/llvm/test/Bitcode/Inputs/branch-weight.bc b/llvm/test/Bitcode/Inputs/branch-weight.bc
new file mode 100644
index 0000000000000..fbb77a39b1dea
Binary files /dev/null and b/llvm/test/Bitcode/Inputs/branch-weight.bc differ
diff --git a/llvm/test/Bitcode/Inputs/linker-options.bc b/llvm/test/Bitcode/Inputs/linker-options.bc
new file mode 100644
index 0000000000000..cbbfb12d9379b
Binary files /dev/null and b/llvm/test/Bitcode/Inputs/linker-options.bc differ
diff --git a/llvm/test/Bitcode/allocated.ll b/llvm/test/Bitcode/allocated.ll
new file mode 100644
index 0000000000000..060006b788465
--- /dev/null
+++ b/llvm/test/Bitcode/allocated.ll
@@ -0,0 +1,32 @@
+;; This test checks dataLocation field of DICompositeType
+
+; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
+
+;; Test whether DW_AT_data_location is generated.
+; CHECK:  !DICompositeType(tag: DW_TAG_array_type, baseType: !{{[0-9]+}}, size: 32, align: 32, elements: !{{[0-9]+}}, dataLocation: !{{[0-9]+}}, allocated: !{{[0-9]+}})
+; CHECK:  !DICompositeType(tag: DW_TAG_array_type, baseType: !{{[0-9]+}}, size: 32, align: 32, elements: !{{[0-9]+}}, dataLocation: !DIExpression(DW_OP_push_object_address, DW_OP_deref), allocated: !DIExpression(DW_OP_push_object_address, DW_OP_deref))
+
+; ModuleID = 'allocated.f90'
+source_filename = "/dir/allocated.ll"
+
+!llvm.module.flags = !{!0, !1}
+!llvm.dbg.cu = !{!2}
+
+!0 = !{i32 2, !"Dwarf Version", i32 4}
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+!2 = distinct !DICompileUnit(language: DW_LANG_Fortran90, file: !3, producer: " F90 Flang - 1.5 2017-05-01", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, retainedTypes: !5, globals: !4, imports: !4)
+!3 = !DIFile(filename: "allocated.f90", directory: "/dir")
+!4 = !{}
+!5 = !{!6, !17}
+!6 = !DICompositeType(tag: DW_TAG_array_type, baseType: !7, size: 32, align: 32, elements: !8, dataLocation: !10, allocated: !15)
+!7 = !DIBasicType(name: "integer", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !{!9}
+!9 = !DISubrange(count: 19, lowerBound: 2)
+!10 = distinct !DILocalVariable(scope: !11, file: !3, type: !14, flags: DIFlagArtificial)
+!11 = distinct !DISubprogram(name: "main", scope: !2, file: !3, line: 1, type: !12, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagMainSubprogram, unit: !2)
+!12 = !DISubroutineType(cc: DW_CC_program, types: !13)
+!13 = !{null}
+!14 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 32, align: 32)
+!15 = distinct !DILocalVariable(scope: !11, file: !3, type: !16, flags: DIFlagArtificial)
+!16 = !DIBasicType(name: "logical", size: 32, align: 32, encoding: DW_ATE_signed)
+!17 = !DICompositeType(tag: DW_TAG_array_type, baseType: !7, size: 32, align: 32, elements: !8, dataLocation: !DIExpression(DW_OP_push_object_address, DW_OP_deref), allocated: !DIExpression(DW_OP_push_object_address, DW_OP_deref))
diff --git a/llvm/test/Bitcode/associated.ll b/llvm/test/Bitcode/associated.ll
new file mode 100644
index 0000000000000..fd11c0e83c15f
--- /dev/null
+++ b/llvm/test/Bitcode/associated.ll
@@ -0,0 +1,32 @@
+;; This test checks dataLocation field of DICompositeType
+
+; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
+
+;; Test whether DW_AT_data_location is generated.
+; CHECK:  !DICompositeType(tag: DW_TAG_array_type, baseType: !{{[0-9]+}}, size: 32, align: 32, elements: !{{[0-9]+}}, dataLocation: !{{[0-9]+}}, associated: !{{[0-9]+}})
+; CHECK:  !DICompositeType(tag: DW_TAG_array_type, baseType: !{{[0-9]+}}, size: 32, align: 32, elements: !{{[0-9]+}}, dataLocation: !DIExpression(DW_OP_push_object_address, DW_OP_deref), associated: !DIExpression(DW_OP_push_object_address, DW_OP_deref))
+
+; ModuleID = 'associated.f90'
+source_filename = "/dir/associated.ll"
+
+!llvm.module.flags = !{!0, !1}
+!llvm.dbg.cu = !{!2}
+
+!0 = !{i32 2, !"Dwarf Version", i32 4}
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+!2 = distinct !DICompileUnit(language: DW_LANG_Fortran90, file: !3, producer: " F90 Flang - 1.5 2017-05-01", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, retainedTypes: !5, globals: !4, imports: !4)
+!3 = !DIFile(filename: "associated.f90", directory: "/dir")
+!4 = !{}
+!5 = !{!6, !17}
+!6 = !DICompositeType(tag: DW_TAG_array_type, baseType: !7, size: 32, align: 32, elements: !8, dataLocation: !10, associated: !15)
+!7 = !DIBasicType(name: "integer", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !{!9}
+!9 = !DISubrange(count: 19, lowerBound: 2)
+!10 = distinct !DILocalVariable(scope: !11, file: !3, type: !14, flags: DIFlagArtificial)
+!11 = distinct !DISubprogram(name: "main", scope: !2, file: !3, line: 1, type: !12, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagMainSubprogram, unit: !2)
+!12 = !DISubroutineType(cc: DW_CC_program, types: !13)
+!13 = !{null}
+!14 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 32, align: 32)
+!15 = distinct !DILocalVariable(scope: !11, file: !3, type: !16, flags: DIFlagArtificial)
+!16 = !DIBasicType(name: "logical", size: 32, align: 32, encoding: DW_ATE_signed)
+!17 = !DICompositeType(tag: DW_TAG_array_type, baseType: !7, size: 32, align: 32, elements: !8, dataLocation: !DIExpression(DW_OP_push_object_address, DW_OP_deref), associated: !DIExpression(DW_OP_push_object_address, DW_OP_deref))
diff --git a/llvm/test/Bitcode/attributes.ll b/llvm/test/Bitcode/attributes.ll
index fbbe4a80f31e0..409528230a7da 100644
--- a/llvm/test/Bitcode/attributes.ll
+++ b/llvm/test/Bitcode/attributes.ll
@@ -392,6 +392,12 @@ define noundef i32 @f66(i32 noundef %a)
   ret i32 %a
 }
 
+; CHECK: define void @f67(i32* byref(i32) %a)
+define void @f67(i32* byref(i32) %a)
+{
+  ret void
+}
+
 ; CHECK: attributes #0 = { noreturn }
 ; CHECK: attributes #1 = { nounwind }
 ; CHECK: attributes #2 = { readnone }
diff --git a/llvm/test/Bitcode/branch-weight.ll b/llvm/test/Bitcode/branch-weight.ll
new file mode 100644
index 0000000000000..d93d79e939221
--- /dev/null
+++ b/llvm/test/Bitcode/branch-weight.ll
@@ -0,0 +1,19 @@
+;; Test strip branch_weight if operand number doesn't match.
+;; Input bitcode is generated from:
+;; define void @test(i1 %0) {
+;;   br i1 %0, label %2, label %3, !prof !0
+;; 2:
+;;   br i1 %0, label %4, label %3, !prof !1
+;; 3:
+;;   unreachable
+;; 4:
+;;   ret void
+;; }
+;;!0 = !{!"branch_weights", i32 1, i32 2}
+;;!1 = !{!"branch_weights", i32 1, i32 2, i32 3}
+
+; RUN: llvm-dis %S/Inputs/branch-weight.bc -o - | FileCheck %s
+; CHECK: !prof !0
+; CHECK: !0 = !{!"branch_weights", i32 1, i32 2}
+; CHECK-NOT: !prof !1
+; CHECK-NOT: !1 = !{!"branch_weights", i32 1, i32 2, i32 3}
diff --git a/llvm/test/Bitcode/upgrade-linker-options-2.ll b/llvm/test/Bitcode/upgrade-linker-options-2.ll
new file mode 100644
index 0000000000000..b87ed41ef5154
--- /dev/null
+++ b/llvm/test/Bitcode/upgrade-linker-options-2.ll
@@ -0,0 +1,12 @@
+;; Test upgrade linker option doesn't create duplicated linker options.
+;; Inputs is generated from IR and checked in as bitcode as it will get rejected by verifier.
+;; define void @test() {
+;;   ret void
+;; }
+;; !llvm.module.flags = !{!0}
+;; !0 = !{i32 6, !"Linker Options", !1}
+;; !1 = !{!2}
+;; !2 = !{!"-framework", !"Foundation"}
+
+; RUN: llvm-dis %S/Inputs/linker-options.bc -o - | FileCheck %s
+; CHECK: !llvm.linker.options = !{!2}
diff --git a/llvm/test/CMakeLists.txt b/llvm/test/CMakeLists.txt
index b083184a05b4a..a26989213944c 100644
--- a/llvm/test/CMakeLists.txt
+++ b/llvm/test/CMakeLists.txt
@@ -1,12 +1,12 @@
 llvm_canonicalize_cmake_booleans(
   BUILD_SHARED_LIBS
   HAVE_LIBXAR
-  HAVE_LIBZ
   HAVE_OCAMLOPT
   HAVE_OCAML_OUNIT
   LLVM_ENABLE_DIA_SDK
   LLVM_ENABLE_FFI
   LLVM_ENABLE_THREADS
+  LLVM_ENABLE_ZLIB
   LLVM_INCLUDE_GO_TESTS
   LLVM_LIBXML2_ENABLED
   LLVM_LINK_LLVM_DYLIB
@@ -53,6 +53,7 @@ set(LLVM_TEST_DEPENDS
           bugpoint
           count
           file-table-tform
+          extract
           llc
           lli
           lli-child-target
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
index cf596c98d4628..8ac3e50ea0b74 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
@@ -103,7 +103,7 @@ define fp128 @test_quad_dump() {
   ret fp128 0xL00000000000000004000000000000000
 }
 
-; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to legalize instruction: %2:_(p0) = G_EXTRACT_VECTOR_ELT %0:_(<2 x p0>), %3:_(s64) (in function: vector_of_pointers_extractelement)
+; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to legalize instruction: %2:_(p0) = G_EXTRACT_VECTOR_ELT %{{[0-9]+}}:_(<2 x p0>), %{{[0-9]+}}:_(s64) (in function: vector_of_pointers_extractelement)
 ; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for vector_of_pointers_extractelement
 ; FALLBACK-WITH-REPORT-OUT-LABEL: vector_of_pointers_extractelement:
 @var = global <2 x i16*> zeroinitializer
@@ -136,7 +136,7 @@ end:
   br label %block
 }
 
-; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to legalize instruction: %3:_(s96) = G_ADD %2:_, %2:_ (in function: nonpow2_add_narrowing)
+; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to legalize instruction: %{{[0-9]+}}:_(s96) = G_ADD %{{[0-9]+}}:_, %{{[0-9]+}}:_ (in function: nonpow2_add_narrowing)
 ; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for nonpow2_add_narrowing
 ; FALLBACK-WITH-REPORT-OUT-LABEL: nonpow2_add_narrowing:
 define void @nonpow2_add_narrowing() {
@@ -147,7 +147,7 @@ define void @nonpow2_add_narrowing() {
   ret void
 }
 
-; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to legalize instruction: %5:_(s96) = G_INSERT %18:_, %16:_(s32), 64 (in function: nonpow2_or_narrowing)
+; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to legalize instruction: %{{[0-9]+}}:_(s96) = G_INSERT %{{[0-9]+}}:_, %{{[0-9]+}}:_(s32), 64 (in function: nonpow2_or_narrowing)
 ; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for nonpow2_or_narrowing
 ; FALLBACK-WITH-REPORT-OUT-LABEL: nonpow2_or_narrowing:
 define void @nonpow2_or_narrowing() {
@@ -172,7 +172,7 @@ define void @nonpow2_load_narrowing() {
 ; Currently can't handle vector lengths that aren't an exact multiple of
 ; natively supported vector lengths. Test that the fall-back works for those.
 ; FALLBACK-WITH-REPORT-ERR-G_IMPLICIT_DEF-LEGALIZABLE: (FIXME: this is what is expected once we can legalize non-pow-of-2 G_IMPLICIT_DEF) remark: <unknown>:0:0: unable to legalize instruction: %1:_(<7 x s64>) = G_ADD %0, %0 (in function: nonpow2_vector_add_fewerelements
-; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to legalize instruction: %2:_(s64) = G_EXTRACT_VECTOR_ELT %1:_(<7 x s64>), %3:_(s64) (in function: nonpow2_vector_add_fewerelements)
+; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to legalize instruction: %{{[0-9]+}}:_(s64) = G_EXTRACT_VECTOR_ELT %{{[0-9]+}}:_(<7 x s64>), %{{[0-9]+}}:_(s64) (in function: nonpow2_vector_add_fewerelements)
 ; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for nonpow2_vector_add_fewerelements
 ; FALLBACK-WITH-REPORT-OUT-LABEL: nonpow2_vector_add_fewerelements:
 define void @nonpow2_vector_add_fewerelements() {
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
index 7e806e5d0ed95..8975b2d0bc9ae 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
@@ -1266,6 +1266,17 @@ define float @test_pow_intrin(float %l, float %r) {
   ret float %res
 }
 
+declare float @llvm.powi.f32(float, i32)
+define float @test_powi_intrin(float %l, i32 %r) {
+; CHECK-LABEL: name: test_powi_intrin
+; CHECK: [[LHS:%[0-9]+]]:_(s32) = COPY $s0
+; CHECK: [[RHS:%[0-9]+]]:_(s32) = COPY $w0
+; CHECK: [[RES:%[0-9]+]]:_(s32) = nnan ninf nsz arcp contract afn reassoc G_FPOWI [[LHS]], [[RHS]]
+; CHECK: $s0 = COPY [[RES]]
+  %res = call nnan ninf nsz arcp contract afn reassoc float @llvm.powi.f32(float %l, i32 %r)
+  ret float %res
+}
+
 declare float @llvm.fma.f32(float, float, float)
 define float @test_fma_intrin(float %a, float %b, float %c) {
 ; CHECK-LABEL: name: test_fma_intrin
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-sext-trunc-sextload.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-sext-trunc-sextload.mir
index 483547ac0511c..616973c04ac4a 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-sext-trunc-sextload.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-sext-trunc-sextload.mir
@@ -47,35 +47,17 @@ tracksRegLiveness: true
 body: |
   bb.0.entry:
     liveins: $x0
-    ; Here we're trying to extend from a larger width than was extended in the load.
+    ; Here we're trying to extend from a smaller value than was extended in the load.
     ; CHECK-LABEL: name: test_combine_sext_of_sextload_not_matching
     ; CHECK: liveins: $x0
     ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
     ; CHECK: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p0) :: (load 2)
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[SEXTLOAD]](s32)
-    ; CHECK: $w0 = COPY [[COPY1]](s32)
-    %0:_(p0) = COPY $x0
-    %1:_(s32) = G_SEXTLOAD %0:_(p0) :: (load 2)
-    %2:_(s32) = G_SEXT_INREG %1:_(s32), 24
-    $w0 = COPY %2(s32)
-...
----
-name:            test_combine_sext_of_sextload_not_enough_src_signbits
-legalized:       true
-tracksRegLiveness: true
-body: |
-  bb.0.entry:
-    liveins: $x0
-    ; Here we're trying to extend from a smaller width than was extended in the load.
-    ; Don't perform the combine.
-    ; CHECK-LABEL: name: test_combine_sext_of_sextload_not_enough_src_signbits
-    ; CHECK: liveins: $x0
-    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
-    ; CHECK: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p0) :: (load 2)
-    ; CHECK: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[SEXTLOAD]], 8
+    ; CHECK: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY1]], 24
     ; CHECK: $w0 = COPY [[SEXT_INREG]](s32)
     %0:_(p0) = COPY $x0
     %1:_(s32) = G_SEXTLOAD %0:_(p0) :: (load 2)
-    %2:_(s32) = G_SEXT_INREG %1:_(s32), 8
-    $w0 = COPY %2(s32)
+    %2:_(s32) = COPY %1:_(s32)
+    %3:_(s32) = G_SEXT_INREG %2:_(s32), 24
+    $w0 = COPY %3(s32)
 ...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/darwin-tls-call-clobber.ll b/llvm/test/CodeGen/AArch64/GlobalISel/darwin-tls-call-clobber.ll
new file mode 100644
index 0000000000000..cbeac5d85fc4e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/darwin-tls-call-clobber.ll
@@ -0,0 +1,207 @@
+; RUN: llc -mtriple aarch64-apple-darwin -O0 -global-isel -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-ios13.0.0"
+
+@t_val = thread_local global i32 0, align 4
+@.str = private unnamed_addr constant [5 x i8] c"str1\00", align 1
+@str1 = global i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i32 0, i32 0), align 8
+@.str.1 = private unnamed_addr constant [5 x i8] c"str2\00", align 1
+@str2 = global i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str.1, i32 0, i32 0), align 8
+@.str.2 = private unnamed_addr constant [5 x i8] c"str3\00", align 1
+@str3 = global i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str.2, i32 0, i32 0), align 8
+@.str.3 = private unnamed_addr constant [5 x i8] c"str4\00", align 1
+@str4 = global i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str.3, i32 0, i32 0), align 8
+@.str.4 = private unnamed_addr constant [5 x i8] c"str5\00", align 1
+@str5 = global i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str.4, i32 0, i32 0), align 8
+@.str.5 = private unnamed_addr constant [5 x i8] c"str6\00", align 1
+@str6 = global i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str.5, i32 0, i32 0), align 8
+@.str.6 = private unnamed_addr constant [5 x i8] c"str7\00", align 1
+@str7 = global i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str.6, i32 0, i32 0), align 8
+@.str.7 = private unnamed_addr constant [5 x i8] c"str8\00", align 1
+@str8 = global i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str.7, i32 0, i32 0), align 8
+@.str.8 = private unnamed_addr constant [5 x i8] c"str9\00", align 1
+@str9 = global i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str.8, i32 0, i32 0), align 8
+@.str.9 = private unnamed_addr constant [6 x i8] c"str10\00", align 1
+@str10 = global i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str.9, i32 0, i32 0), align 8
+@.str.10 = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+@.str.11 = private unnamed_addr constant [4 x i8] c"%d\0A\00", align 1
+@.str.12 = private unnamed_addr constant [4 x i8] c"xyz\00", align 1
+
+
+; This test checks that we don't re-use the register for the variable descriptor
+; for the second ldr.
+; CHECK:        adrp	x[[PTR1:[0-9]+]], _t_val@TLVPPAGE
+; CHECK:	ldr	x[[PTR1]], [x[[PTR1]], _t_val@TLVPPAGEOFF]
+; CHECK:	ldr	x[[FPTR:[0-9]+]], [x[[PTR1]]]
+; CHECK:        mov	x0, x[[PTR1]]
+; CHECK:        blr     x[[FPTR]]
+
+define void @_Z4funcPKc(i8* %id) {
+entry:
+  %id.addr = alloca i8*, align 8
+  store i8* %id, i8** %id.addr, align 8
+  %0 = load i8*, i8** %id.addr, align 8
+  %1 = load i8*, i8** @str1, align 8
+  %cmp = icmp eq i8* %0, %1
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %2 = load i8*, i8** @str1, align 8
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.10, i64 0, i64 0), i8* %2)
+  %3 = load i8*, i8** @str2, align 8
+  %call1 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.10, i64 0, i64 0), i8* %3)
+  %4 = load i8*, i8** @str3, align 8
+  %call2 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.10, i64 0, i64 0), i8* %4)
+  %5 = load i8*, i8** @str4, align 8
+  %call3 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.10, i64 0, i64 0), i8* %5)
+  %6 = load i8*, i8** @str5, align 8
+  %call4 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.10, i64 0, i64 0), i8* %6)
+  %7 = load i8*, i8** @str6, align 8
+  %call5 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.10, i64 0, i64 0), i8* %7)
+  %8 = load i8*, i8** @str7, align 8
+  %call6 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.10, i64 0, i64 0), i8* %8)
+  %9 = load i8*, i8** @str8, align 8
+  %call7 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.10, i64 0, i64 0), i8* %9)
+  %10 = load i8*, i8** @str9, align 8
+  %call8 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.10, i64 0, i64 0), i8* %10)
+  %11 = load i8*, i8** @str10, align 8
+  %call9 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.10, i64 0, i64 0), i8* %11)
+  %12 = load i32, i32* @t_val, align 4
+  %call10 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.11, i64 0, i64 0), i32 %12)
+  br label %if.end56
+
+if.else:                                          ; preds = %entry
+  %13 = load i8*, i8** %id.addr, align 8
+  %14 = load i8*, i8** @str2, align 8
+  %cmp11 = icmp eq i8* %13, %14
+  br i1 %cmp11, label %if.then12, label %if.else24
+
+if.then12:                                        ; preds = %if.else
+  %15 = load i8*, i8** @str1, align 8
+  %call13 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.10, i64 0, i64 0), i8* %15)
+  %16 = load i8*, i8** @str2, align 8
+  %call14 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.10, i64 0, i64 0), i8* %16)
+  %17 = load i8*, i8** @str3, align 8
+  %call15 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.10, i64 0, i64 0), i8* %17)
+  %18 = load i8*, i8** @str4, align 8
+  %call16 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.10, i64 0, i64 0), i8* %18)
+  %19 = load i8*, i8** @str5, align 8
+  %call17 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.10, i64 0, i64 0), i8* %19)
+  %20 = load i8*, i8** @str6, align 8
+  %call18 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.10, i64 0, i64 0), i8* %20)
+  %21 = load i8*, i8** @str7, align 8
+  %call19 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.10, i64 0, i64 0), i8* %21)
+  %22 = load i8*, i8** @str8, align 8
+  %call20 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.10, i64 0, i64 0), i8* %22)
+  %23 = load i8*, i8** @str9, align 8
+  %call21 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.10, i64 0, i64 0), i8* %23)
+  %24 = load i8*, i8** @str10, align 8
+  %call22 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.10, i64 0, i64 0), i8* %24)
+  %25 = load i32, i32* @t_val, align 4
+  %call23 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.11, i64 0, i64 0), i32 %25)
+  br label %if.end55
+
+if.else24:                                        ; preds = %if.else
+  %26 = load i8*, i8** %id.addr, align 8
+  %27 = load i8*, i8** @str3, align 8
+  %cmp25 = icmp eq i8* %26, %27
+  br i1 %cmp25, label %if.then26, label %if.else27
+
+if.then26:                                        ; preds = %if.else24
+  br label %if.end54
+
+if.else27:                                        ; preds = %if.else24
+  %28 = load i8*, i8** %id.addr, align 8
+  %29 = load i8*, i8** @str4, align 8
+  %cmp28 = icmp eq i8* %28, %29
+  br i1 %cmp28, label %if.then29, label %if.else30
+
+if.then29:                                        ; preds = %if.else27
+  br label %if.end53
+
+if.else30:                                        ; preds = %if.else27
+  %30 = load i8*, i8** %id.addr, align 8
+  %31 = load i8*, i8** @str5, align 8
+  %cmp31 = icmp eq i8* %30, %31
+  br i1 %cmp31, label %if.then32, label %if.else33
+
+if.then32:                                        ; preds = %if.else30
+  br label %if.end52
+
+if.else33:                                        ; preds = %if.else30
+  %32 = load i8*, i8** %id.addr, align 8
+  %33 = load i8*, i8** @str6, align 8
+  %cmp34 = icmp eq i8* %32, %33
+  br i1 %cmp34, label %if.then35, label %if.else36
+
+if.then35:                                        ; preds = %if.else33
+  br label %if.end51
+
+if.else36:                                        ; preds = %if.else33
+  %34 = load i8*, i8** %id.addr, align 8
+  %35 = load i8*, i8** @str7, align 8
+  %cmp37 = icmp eq i8* %34, %35
+  br i1 %cmp37, label %if.then38, label %if.else39
+
+if.then38:                                        ; preds = %if.else36
+  br label %if.end50
+
+if.else39:                                        ; preds = %if.else36
+  %36 = load i8*, i8** %id.addr, align 8
+  %37 = load i8*, i8** @str8, align 8
+  %cmp40 = icmp eq i8* %36, %37
+  br i1 %cmp40, label %if.then41, label %if.else42
+
+if.then41:                                        ; preds = %if.else39
+  br label %if.end49
+
+if.else42:                                        ; preds = %if.else39
+  %38 = load i8*, i8** %id.addr, align 8
+  %39 = load i8*, i8** @str9, align 8
+  %cmp43 = icmp eq i8* %38, %39
+  br i1 %cmp43, label %if.then44, label %if.else45
+
+if.then44:                                        ; preds = %if.else42
+  br label %if.end48
+
+if.else45:                                        ; preds = %if.else42
+  %40 = load i8*, i8** %id.addr, align 8
+  %41 = load i8*, i8** @str10, align 8
+  %cmp46 = icmp eq i8* %40, %41
+  br i1 %cmp46, label %if.then47, label %if.end
+
+if.then47:                                        ; preds = %if.else45
+  br label %if.end
+
+if.end:                                           ; preds = %if.then47, %if.else45
+  br label %if.end48
+
+if.end48:                                         ; preds = %if.end, %if.then44
+  br label %if.end49
+
+if.end49:                                         ; preds = %if.end48, %if.then41
+  br label %if.end50
+
+if.end50:                                         ; preds = %if.end49, %if.then38
+  br label %if.end51
+
+if.end51:                                         ; preds = %if.end50, %if.then35
+  br label %if.end52
+
+if.end52:                                         ; preds = %if.end51, %if.then32
+  br label %if.end53
+
+if.end53:                                         ; preds = %if.end52, %if.then29
+  br label %if.end54
+
+if.end54:                                         ; preds = %if.end53, %if.then26
+  br label %if.end55
+
+if.end55:                                         ; preds = %if.end54, %if.then12
+  br label %if.end56
+
+if.end56:                                         ; preds = %if.end55, %if.then
+  ret void
+}
+declare i32 @printf(i8*, ...)
+
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-inline-asm.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-inline-asm.ll
index f8b23ef84721e..b7b2f0c981429 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-inline-asm.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-inline-asm.ll
@@ -154,7 +154,7 @@ define void @test_input_register_imm() {
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 42
   ; CHECK:   [[COPY:%[0-9]+]]:gpr64common = COPY [[C]](s64)
-  ; CHECK:   INLINEASM &"mov x0, $0", 1 /* sideeffect attdialect */, 9 /* reguse */, [[COPY]]
+  ; CHECK:   INLINEASM &"mov x0, $0", 1 /* sideeffect attdialect */, 1441801 /* reguse:GPR64common */, [[COPY]]
   ; CHECK:   RET_ReallyLR
   call void asm sideeffect "mov x0, $0", "r"(i64 42)
   ret void
@@ -188,7 +188,7 @@ define zeroext i8 @test_input_register(i8* %src) nounwind {
   ; CHECK:   liveins: $x0
   ; CHECK:   [[COPY:%[0-9]+]]:_(p0) = COPY $x0
   ; CHECK:   [[COPY1:%[0-9]+]]:gpr64common = COPY [[COPY]](p0)
-  ; CHECK:   INLINEASM &"ldtrb ${0:w}, [$1]", 0 /* attdialect */, 655370 /* regdef:GPR32common */, def %1, 9 /* reguse */, [[COPY1]]
+  ; CHECK:   INLINEASM &"ldtrb ${0:w}, [$1]", 0 /* attdialect */, 655370 /* regdef:GPR32common */, def %1, 1441801 /* reguse:GPR64common */, [[COPY1]]
   ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY %1
   ; CHECK:   [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY2]](s32)
   ; CHECK:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC]](s8)
@@ -218,7 +218,7 @@ define i16 @test_anyext_input() {
   ; CHECK:   [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 1
   ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[C]](s16)
   ; CHECK:   [[COPY:%[0-9]+]]:gpr32common = COPY [[ANYEXT]](s32)
-  ; CHECK:   INLINEASM &"", 1 /* sideeffect attdialect */, 655370 /* regdef:GPR32common */, def %0, 9 /* reguse */, [[COPY]]
+  ; CHECK:   INLINEASM &"", 1 /* sideeffect attdialect */, 655370 /* regdef:GPR32common */, def %0, 655369 /* reguse:GPR32common */, [[COPY]]
   ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY %0
   ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
   ; CHECK:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index 35b41b8aaa871..dcd310fba7e9e 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -366,6 +366,9 @@
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. the first uncovered type index: 1, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: G_FPOWI (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
+# DEBUG-NEXT:.. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT:.. imm index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: G_FEXP (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. the first uncovered type index: 1, OK
diff --git a/llvm/test/CodeGen/AArch64/arm64-tls-darwin.ll b/llvm/test/CodeGen/AArch64/arm64-tls-darwin.ll
index b90848bc9faec..06d214b1b0acf 100644
--- a/llvm/test/CodeGen/AArch64/arm64-tls-darwin.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-tls-darwin.ll
@@ -9,8 +9,8 @@
 define i8 @get_var() {
 ; CHECK-LABEL: get_var:
 ; CHECK: adrp x[[TLVPDESC_SLOT_HI:[0-9]+]], _var@TLVPPAGE
-; CHECK: ldr x0, [x[[TLVPDESC_SLOT_HI]], _var@TLVPPAGEOFF]
-; CHECK: ldr [[TLV_GET_ADDR:x[0-9]+]], [x0]
+ ; CHECK: ldr x[[PTR:[0-9]+]], [x[[TLVPDESC_SLOT_HI]], _var@TLVPPAGEOFF]
+ ; CHECK: ldr [[TLV_GET_ADDR:x[0-9]+]], [x[[PTR]]]
 ; CHECK: blr [[TLV_GET_ADDR]]
 ; CHECK: ldrb w0, [x0]
 
diff --git a/llvm/test/CodeGen/AArch64/arm64-vhadd.ll b/llvm/test/CodeGen/AArch64/arm64-vhadd.ll
index a5d223cc8aefb..9b351f288d174 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vhadd.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vhadd.ll
@@ -425,6 +425,96 @@ define void @testLowerToSRHADD4s(<4 x i32> %src1, <4 x i32> %src2, <4 x i32>* %d
   ret void
 }
 
+define void @testLowerToSHADD8b(<8 x i8> %src1, <8 x i8> %src2, <8 x i8>* %dest) nounwind {
+; CHECK-LABEL: testLowerToSHADD8b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shadd.8b v0, v0, v1
+; CHECK-NEXT:    str d0, [x0]
+; CHECK-NEXT:    ret
+  %sextsrc1 = sext <8 x i8> %src1 to <8 x i16>
+  %sextsrc2 = sext <8 x i8> %src2 to <8 x i16>
+  %add = add <8 x i16> %sextsrc1, %sextsrc2
+  %resulti16 = lshr <8 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %result = trunc <8 x i16> %resulti16 to <8 x i8>
+  store <8 x i8> %result, <8 x i8>* %dest, align 8
+  ret void
+}
+
+define void @testLowerToSHADD4h(<4 x i16> %src1, <4 x i16> %src2, <4 x i16>* %dest) nounwind {
+; CHECK-LABEL: testLowerToSHADD4h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shadd.4h v0, v0, v1
+; CHECK-NEXT:    str d0, [x0]
+; CHECK-NEXT:    ret
+  %sextsrc1 = sext <4 x i16> %src1 to <4 x i32>
+  %sextsrc2 = sext <4 x i16> %src2 to <4 x i32>
+  %add = add <4 x i32> %sextsrc1, %sextsrc2
+  %resulti16 = lshr <4 x i32> %add, <i32 1, i32 1, i32 1, i32 1>
+  %result = trunc <4 x i32> %resulti16 to <4 x i16>
+  store <4 x i16> %result, <4 x i16>* %dest, align 8
+  ret void
+}
+
+define void @testLowerToSHADD2s(<2 x i32> %src1, <2 x i32> %src2, <2 x i32>* %dest) nounwind {
+; CHECK-LABEL: testLowerToSHADD2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shadd.2s v0, v0, v1
+; CHECK-NEXT:    str d0, [x0]
+; CHECK-NEXT:    ret
+  %sextsrc1 = sext <2 x i32> %src1 to <2 x i64>
+  %sextsrc2 = sext <2 x i32> %src2 to <2 x i64>
+  %add = add <2 x i64> %sextsrc1, %sextsrc2
+  %resulti16 = lshr <2 x i64> %add, <i64 1, i64 1>
+  %result = trunc <2 x i64> %resulti16 to <2 x i32>
+  store <2 x i32> %result, <2 x i32>* %dest, align 8
+  ret void
+}
+
+define void @testLowerToSHADD16b(<16 x i8> %src1, <16 x i8> %src2, <16 x i8>* %dest) nounwind {
+; CHECK-LABEL: testLowerToSHADD16b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shadd.16b v0, v0, v1
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
+  %sextsrc1 = sext <16 x i8> %src1 to <16 x i16>
+  %sextsrc2 = sext <16 x i8> %src2 to <16 x i16>
+  %add = add <16 x i16> %sextsrc1, %sextsrc2
+  %resulti16 = lshr <16 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %result = trunc <16 x i16> %resulti16 to <16 x i8>
+  store <16 x i8> %result, <16 x i8>* %dest, align 16
+  ret void
+}
+
+define void @testLowerToSHADD8h(<8 x i16> %src1, <8 x i16> %src2, <8 x i16>* %dest) nounwind {
+; CHECK-LABEL: testLowerToSHADD8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shadd.8h v0, v0, v1
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
+  %sextsrc1 = sext <8 x i16> %src1 to <8 x i32>
+  %sextsrc2 = sext <8 x i16> %src2 to <8 x i32>
+  %add = add <8 x i32> %sextsrc1, %sextsrc2
+  %resulti16 = lshr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %result = trunc <8 x i32> %resulti16 to <8 x i16>
+  store <8 x i16> %result, <8 x i16>* %dest, align 16
+  ret void
+}
+
+define void @testLowerToSHADD4s(<4 x i32> %src1, <4 x i32> %src2, <4 x i32>* %dest) nounwind {
+; CHECK-LABEL: testLowerToSHADD4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shadd.4s v0, v0, v1
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
+  %sextsrc1 = sext <4 x i32> %src1 to <4 x i64>
+  %sextsrc2 = sext <4 x i32> %src2 to <4 x i64>
+  %add = add <4 x i64> %sextsrc1, %sextsrc2
+  %resulti16 = lshr <4 x i64> %add, <i64 1, i64 1, i64 1, i64 1>
+  %result = trunc <4 x i64> %resulti16 to <4 x i32>
+  store <4 x i32> %result, <4 x i32>* %dest, align 16
+  ret void
+}
+
 define void @testLowerToURHADD8b(<8 x i8> %src1, <8 x i8> %src2, <8 x i8>* %dest) nounwind {
 ; CHECK-LABEL: testLowerToURHADD8b:
 ; CHECK:       // %bb.0:
@@ -521,6 +611,96 @@ define void @testLowerToURHADD4s(<4 x i32> %src1, <4 x i32> %src2, <4 x i32>* %d
   ret void
 }
 
+define void @testLowerToUHADD8b(<8 x i8> %src1, <8 x i8> %src2, <8 x i8>* %dest) nounwind {
+; CHECK-LABEL: testLowerToUHADD8b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uhadd.8b v0, v0, v1
+; CHECK-NEXT:    str d0, [x0]
+; CHECK-NEXT:    ret
+  %zextsrc1 = zext <8 x i8> %src1 to <8 x i16>
+  %zextsrc2 = zext <8 x i8> %src2 to <8 x i16>
+  %add = add <8 x i16> %zextsrc1, %zextsrc2
+  %resulti16 = lshr <8 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %result = trunc <8 x i16> %resulti16 to <8 x i8>
+  store <8 x i8> %result, <8 x i8>* %dest, align 8
+  ret void
+}
+
+define void @testLowerToUHADD4h(<4 x i16> %src1, <4 x i16> %src2, <4 x i16>* %dest) nounwind {
+; CHECK-LABEL: testLowerToUHADD4h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uhadd.4h v0, v0, v1
+; CHECK-NEXT:    str d0, [x0]
+; CHECK-NEXT:    ret
+  %zextsrc1 = zext <4 x i16> %src1 to <4 x i32>
+  %zextsrc2 = zext <4 x i16> %src2 to <4 x i32>
+  %add = add <4 x i32> %zextsrc1, %zextsrc2
+  %resulti16 = lshr <4 x i32> %add, <i32 1, i32 1, i32 1, i32 1>
+  %result = trunc <4 x i32> %resulti16 to <4 x i16>
+  store <4 x i16> %result, <4 x i16>* %dest, align 8
+  ret void
+}
+
+define void @testLowerToUHADD2s(<2 x i32> %src1, <2 x i32> %src2, <2 x i32>* %dest) nounwind {
+; CHECK-LABEL: testLowerToUHADD2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uhadd.2s v0, v0, v1
+; CHECK-NEXT:    str d0, [x0]
+; CHECK-NEXT:    ret
+  %zextsrc1 = zext <2 x i32> %src1 to <2 x i64>
+  %zextsrc2 = zext <2 x i32> %src2 to <2 x i64>
+  %add = add <2 x i64> %zextsrc1, %zextsrc2
+  %resulti16 = lshr <2 x i64> %add, <i64 1, i64 1>
+  %result = trunc <2 x i64> %resulti16 to <2 x i32>
+  store <2 x i32> %result, <2 x i32>* %dest, align 8
+  ret void
+}
+
+define void @testLowerToUHADD16b(<16 x i8> %src1, <16 x i8> %src2, <16 x i8>* %dest) nounwind {
+; CHECK-LABEL: testLowerToUHADD16b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uhadd.16b v0, v0, v1
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
+  %zextsrc1 = zext <16 x i8> %src1 to <16 x i16>
+  %zextsrc2 = zext <16 x i8> %src2 to <16 x i16>
+  %add = add <16 x i16> %zextsrc1, %zextsrc2
+  %resulti16 = lshr <16 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %result = trunc <16 x i16> %resulti16 to <16 x i8>
+  store <16 x i8> %result, <16 x i8>* %dest, align 16
+  ret void
+}
+
+define void @testLowerToUHADD8h(<8 x i16> %src1, <8 x i16> %src2, <8 x i16>* %dest) nounwind {
+; CHECK-LABEL: testLowerToUHADD8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uhadd.8h v0, v0, v1
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
+  %zextsrc1 = zext <8 x i16> %src1 to <8 x i32>
+  %zextsrc2 = zext <8 x i16> %src2 to <8 x i32>
+  %add = add <8 x i32> %zextsrc1, %zextsrc2
+  %resulti16 = lshr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %result = trunc <8 x i32> %resulti16 to <8 x i16>
+  store <8 x i16> %result, <8 x i16>* %dest, align 16
+  ret void
+}
+
+define void @testLowerToUHADD4s(<4 x i32> %src1, <4 x i32> %src2, <4 x i32>* %dest) nounwind {
+; CHECK-LABEL: testLowerToUHADD4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uhadd.4s v0, v0, v1
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
+  %zextsrc1 = zext <4 x i32> %src1 to <4 x i64>
+  %zextsrc2 = zext <4 x i32> %src2 to <4 x i64>
+  %add = add <4 x i64> %zextsrc1, %zextsrc2
+  %resulti16 = lshr <4 x i64> %add, <i64 1, i64 1, i64 1, i64 1>
+  %result = trunc <4 x i64> %resulti16 to <4 x i32>
+  store <4 x i32> %result, <4 x i32>* %dest, align 16
+  ret void
+}
+
 declare <8 x i8>  @llvm.aarch64.neon.srhadd.v8i8(<8 x i8>, <8 x i8>) nounwind readnone
 declare <4 x i16> @llvm.aarch64.neon.srhadd.v4i16(<4 x i16>, <4 x i16>) nounwind readnone
 declare <2 x i32> @llvm.aarch64.neon.srhadd.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
diff --git a/llvm/test/CodeGen/AArch64/chkstk.ll b/llvm/test/CodeGen/AArch64/chkstk.ll
index 9689a3b9b588a..22768d8199685 100644
--- a/llvm/test/CodeGen/AArch64/chkstk.ll
+++ b/llvm/test/CodeGen/AArch64/chkstk.ll
@@ -1,11 +1,11 @@
 ; RUN: llc -mtriple=aarch64-windows -verify-machineinstrs %s -o - \
 ; RUN:  | FileCheck -check-prefix CHECK-DEFAULT-CODE-MODEL %s
-; RUN: llc -mtriple=aarch64-windows -print-machineinstrs=prologepilog %s -o - 2>&1 \
+; RUN: llc < %s -mtriple=aarch64-windows -stop-after=prologepilog \
 ; RUN:  | FileCheck -check-prefix CHECK-REGSTATE %s
 
 ; RUN: llc -mtriple=aarch64-windows -verify-machineinstrs -code-model=large %s -o - \
 ; RUN:  | FileCheck -check-prefix CHECK-LARGE-CODE-MODEL %s
-; RUN: llc -mtriple=aarch64-windows -print-machineinstrs=prologepilog -code-model=large %s -o - 2>&1 \
+; RUN: llc < %s -mtriple=aarch64-windows -stop-after=prologepilog -code-model=large \
 ; RUN:  | FileCheck -check-prefix CHECK-REGSTATE-LARGE %s
 
 define void @check_watermark() {
diff --git a/llvm/test/CodeGen/AArch64/windows-trap1.ll b/llvm/test/CodeGen/AArch64/debugtrap.ll
similarity index 65%
rename from llvm/test/CodeGen/AArch64/windows-trap1.ll
rename to llvm/test/CodeGen/AArch64/debugtrap.ll
index 6f6fa3a6609ac..ec90272363217 100644
--- a/llvm/test/CodeGen/AArch64/windows-trap1.ll
+++ b/llvm/test/CodeGen/AArch64/debugtrap.ll
@@ -1,6 +1,9 @@
 ; RUN: llc -mtriple=aarch64-windows %s -o -| FileCheck %s
 ; RUN: llc -mtriple=aarch64-windows -fast-isel %s -o - | FileCheck %s
 ; RUN: llc -mtriple=aarch64-windows -global-isel %s -o - | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu %s -o -| FileCheck %s
+; RUN: llc -mtriple=arm64-apple-ios -global-isel %s -o - | FileCheck %s
+; RUN: llc -mtriple=arm64-apple-macosx -fast-isel %s -o - | FileCheck %s
 
 ; CHECK-LABEL: test1:
 ; CHECK: brk #0xf000
diff --git a/llvm/test/CodeGen/AArch64/framelayout-scavengingslot.mir b/llvm/test/CodeGen/AArch64/framelayout-scavengingslot.mir
new file mode 100644
index 0000000000000..d1252435f874a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/framelayout-scavengingslot.mir
@@ -0,0 +1,27 @@
+# RUN: llc -mtriple=aarch64-none-linux-gnu -run-pass=prologepilog %s -o - | FileCheck %s
+---
+# This test verifies that the emergency scavenging slot is located near
+# the SP when the stack is realigned.
+name: LateScavengingSlotRealignment
+# CHECK-LABEL: name: LateScavengingSlotRealignment
+# CHECK: bb.0:
+# CHECK:      STRXui killed $[[SCRATCH:x[0-9]+]], $sp, 3
+# CHECK-NEXT: $[[SCRATCH]] = ADDXri $sp, 40, 0
+# CHECK-NEXT: STRXui $x0, killed $[[SCRATCH]], 4095
+# CHECK-NEXT: $[[SCRATCH]] = LDRXui $sp, 3
+# CHECK: bb.1:
+tracksRegLiveness: true
+frameInfo:
+  isFrameAddressTaken: true
+stack:
+  - { id: 0, size:    16, alignment: 16 }
+  - { id: 1, size: 32768, alignment: 32 }
+body: |
+  bb.0:
+    liveins: $x0, $x8
+    STRXui $x0, %stack.0, 0
+    B %bb.1
+  bb.1:
+    liveins: $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8, $x9, $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17, $x18, $x19, $x20, $x21, $x22, $x23, $x24, $x25, $x26, $x27, $x28, $lr
+    RET_ReallyLR implicit $x19, implicit $x20, implicit $x21, implicit $x22, implicit $x23, implicit $x24, implicit $x25, implicit $x26, implicit $x27, implicit $x28, implicit $lr
+...
diff --git a/llvm/test/CodeGen/AArch64/framelayout-sve-basepointer.mir b/llvm/test/CodeGen/AArch64/framelayout-sve-basepointer.mir
new file mode 100644
index 0000000000000..a366744d8fa2b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/framelayout-sve-basepointer.mir
@@ -0,0 +1,23 @@
+# RUN: llc -mtriple=aarch64-none-linux-gnu -run-pass=prologepilog -mattr=+sve %s -o - | FileCheck %s
+---
+# This test verifies that the basepointer is available in presence of SVE stack objects.
+name: hasBasepointer
+# CHECK-LABEL: name: hasBasepointer
+# CHECK: bb.0:
+# CHECK:      $sp = frame-setup ADDVL_XXI $sp, -1
+# CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
+# CHECK-NEXT: $x19 = ADDXri $sp, 0, 0
+# CHECK:      STRXui $x0, $x19, 0
+tracksRegLiveness: true
+frameInfo:
+  isFrameAddressTaken: true
+stack:
+  - { id: 0, type: variable-sized,  alignment: 1 }
+  - { id: 1, name: '', size: 16, alignment: 8 }
+  - { id: 2, stack-id: sve-vec, size: 16, alignment: 16 }
+body: |
+  bb.0:
+    liveins: $x0
+    STRXui $x0, %stack.1, 0
+    RET_ReallyLR
+...
diff --git a/llvm/test/CodeGen/AArch64/framelayout-sve-scavengingslot.mir b/llvm/test/CodeGen/AArch64/framelayout-sve-scavengingslot.mir
new file mode 100644
index 0000000000000..2ee6007db289a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/framelayout-sve-scavengingslot.mir
@@ -0,0 +1,28 @@
+# RUN: llc -mtriple=aarch64-none-linux-gnu -run-pass=prologepilog -mattr=+sve %s -o - | FileCheck %s
+---
+# This test verifies that the emergency scavenging slot is located near the SP/BP.
+name: LateScavengingSlot
+# CHECK-LABEL: name: LateScavengingSlot
+# CHECK: bb.0:
+# CHECK:      $sp = frame-setup ADDVL_XXI $sp, -1
+# CHECK-NEXT: $sp = frame-setup SUBXri $sp, 8, 12
+# CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
+# CHECK:      STRXui killed $[[SCRATCH:x[0-9]+]], $sp, 1
+# CHECK-NEXT: $[[SCRATCH]] = ADDVL_XXI $fp, -1
+# CHECK-NEXT: STRXui $x0, killed $[[SCRATCH]], 0
+# CHECK: bb.1:
+tracksRegLiveness: true
+frameInfo:
+  isFrameAddressTaken: true
+stack:
+  - { id: 0, name: '', size: 32761, alignment: 8 }
+  - { id: 1, stack-id: sve-vec, size: 16, alignment: 16 }
+body: |
+  bb.0:
+    liveins: $x0, $x8
+    STRXui $x0, %stack.1, 0
+    B %bb.1
+  bb.1:
+    liveins: $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8, $x9, $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17, $x18, $x19, $x20, $x21, $x22, $x23, $x24, $x25, $x26, $x27, $x28, $lr
+    RET_ReallyLR implicit $x19, implicit $x20, implicit $x21, implicit $x22, implicit $x23, implicit $x24, implicit $x25, implicit $x26, implicit $x27, implicit $x28, implicit $lr
+...
diff --git a/llvm/test/CodeGen/AArch64/framelayout-sve.mir b/llvm/test/CodeGen/AArch64/framelayout-sve.mir
index 046357b860b3b..7903df64863bf 100644
--- a/llvm/test/CodeGen/AArch64/framelayout-sve.mir
+++ b/llvm/test/CodeGen/AArch64/framelayout-sve.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=aarch64-none-linux-gnu -run-pass=prologepilog %s -o - | FileCheck %s
+# RUN: llc -mattr=+sve -mtriple=aarch64-none-linux-gnu -run-pass=prologepilog %s -o - | FileCheck %s
 #
 # Test allocation and deallocation of SVE objects on the stack,
 # as well as using a combination of scalable and non-scalable
@@ -30,7 +30,7 @@
   define void @test_address_sve_fp() nounwind { entry: unreachable }
   define void @test_stack_arg_sve() nounwind { entry: unreachable }
   define void @test_address_sve_out_of_range() nounwind { entry: unreachable }
-  define void @test_address_gpr_vla_nobp() nounwind { entry: unreachable }
+  define void @test_address_gpr_vla() nounwind { entry: unreachable }
   define aarch64_sve_vector_pcs void @save_restore_pregs_sve() nounwind { entry: unreachable }
   define aarch64_sve_vector_pcs void @save_restore_zregs_sve() nounwind { entry: unreachable }
   define aarch64_sve_vector_pcs void @save_restore_sve() nounwind { entry: unreachable }
@@ -335,23 +335,23 @@ body:             |
     RET_ReallyLR
 ---
 ...
-# Test that non-SVE objects are accessed from FP when there is no BP,
-# but the SP cannot be used because of variable-length arrays.
+# Test that non-SVE objects are accessed from BP when there are
+# variable length arrays, because it will be more expensive to
+# access from the FP when there are also SVE objects on the stack.
 #
 # +----------+ <- FP
 # | %fstack.0|  // 16 scalable bytes
 # +----------+ <- @FP - 16 scalable bytes
 # | %stack.0 |  // 16 bytes
-# +----------+ <- @FP - 16 scalable bytes - 16b
+# +----------+ <- @BP
 # : %stack.1 :  // variable length
 # +----------+ <- SP
 
-# CHECK-LABEL: name: test_address_gpr_vla_nobp
-# CHECK:      bb.0.entry:
-# CHECK:      $[[TMP:x[0-9]+]] = ADDVL_XXI $fp, -1
-# CHECK-NEXT: STURXi $xzr, killed $[[TMP]], -16
-# CHECK:      RET_ReallyLR
-name:            test_address_gpr_vla_nobp
+# CHECK-LABEL: name: test_address_gpr_vla
+# CHECK: bb.0.entry:
+# CHECK: STRXui $xzr, $x19, 0
+# CHECK: RET_ReallyLR
+name:            test_address_gpr_vla
 frameInfo:
   maxAlignment:  16
 fixedStack:
diff --git a/llvm/test/CodeGen/AArch64/max-jump-table.ll b/llvm/test/CodeGen/AArch64/max-jump-table.ll
index 2d0dbbf241fcb..d01924a9a5427 100644
--- a/llvm/test/CodeGen/AArch64/max-jump-table.ll
+++ b/llvm/test/CodeGen/AArch64/max-jump-table.ll
@@ -1,8 +1,8 @@
-; RUN: llc %s -O2 -print-machineinstrs -mtriple=aarch64-linux-gnu -jump-table-density=40                         -o /dev/null 2> %t; FileCheck %s --check-prefixes=CHECK,CHECK0  < %t
-; RUN: llc %s -O2 -print-machineinstrs -mtriple=aarch64-linux-gnu -jump-table-density=40 -max-jump-table-size=4  -o /dev/null 2> %t; FileCheck %s --check-prefixes=CHECK,CHECK4  < %t
-; RUN: llc %s -O2 -print-machineinstrs -mtriple=aarch64-linux-gnu -jump-table-density=40 -max-jump-table-size=8  -o /dev/null 2> %t; FileCheck %s --check-prefixes=CHECK,CHECK8  < %t
-; RUN: llc %s -O2 -print-machineinstrs -mtriple=aarch64-linux-gnu -jump-table-density=40 -max-jump-table-size=16 -o /dev/null 2> %t; FileCheck %s --check-prefixes=CHECK,CHECK16 < %t
-; RUN: llc %s -O2 -print-machineinstrs -mtriple=aarch64-linux-gnu -jump-table-density=40 -mcpu=exynos-m3         -o /dev/null 2> %t; FileCheck %s --check-prefixes=CHECK,CHECKM3 < %t
+; RUN: llc %s -O2 -print-after-isel -mtriple=aarch64-linux-gnu -jump-table-density=40                         -o /dev/null 2> %t; FileCheck %s --check-prefixes=CHECK,CHECK0  < %t
+; RUN: llc %s -O2 -print-after-isel -mtriple=aarch64-linux-gnu -jump-table-density=40 -max-jump-table-size=4  -o /dev/null 2> %t; FileCheck %s --check-prefixes=CHECK,CHECK4  < %t
+; RUN: llc %s -O2 -print-after-isel -mtriple=aarch64-linux-gnu -jump-table-density=40 -max-jump-table-size=8  -o /dev/null 2> %t; FileCheck %s --check-prefixes=CHECK,CHECK8  < %t
+; RUN: llc %s -O2 -print-after-isel -mtriple=aarch64-linux-gnu -jump-table-density=40 -max-jump-table-size=16 -o /dev/null 2> %t; FileCheck %s --check-prefixes=CHECK,CHECK16 < %t
+; RUN: llc %s -O2 -print-after-isel -mtriple=aarch64-linux-gnu -jump-table-density=40 -mcpu=exynos-m3         -o /dev/null 2> %t; FileCheck %s --check-prefixes=CHECK,CHECKM3 < %t
 
 declare void @ext(i32, i32)
 
diff --git a/llvm/test/CodeGen/AArch64/min-jump-table.ll b/llvm/test/CodeGen/AArch64/min-jump-table.ll
index b5b7ff0777ad4..bf6bddf7b9c4f 100644
--- a/llvm/test/CodeGen/AArch64/min-jump-table.ll
+++ b/llvm/test/CodeGen/AArch64/min-jump-table.ll
@@ -1,7 +1,7 @@
-; RUN: llc %s -O2 -print-machineinstrs -mtriple=aarch64-linux-gnu -jump-table-density=40 -min-jump-table-entries=0 -o /dev/null 2> %t; FileCheck %s --check-prefixes=CHECK,CHECK0  < %t
-; RUN: llc %s -O2 -print-machineinstrs -mtriple=aarch64-linux-gnu -jump-table-density=40 -min-jump-table-entries=2 -o /dev/null 2> %t; FileCheck %s --check-prefixes=CHECK,CHECK2  < %t
-; RUN: llc %s -O2 -print-machineinstrs -mtriple=aarch64-linux-gnu -jump-table-density=40 -min-jump-table-entries=4 -o /dev/null 2> %t; FileCheck %s --check-prefixes=CHECK,CHECK4  < %t
-; RUN: llc %s -O2 -print-machineinstrs -mtriple=aarch64-linux-gnu -jump-table-density=40 -min-jump-table-entries=8 -o /dev/null 2> %t; FileCheck %s --check-prefixes=CHECK,CHECK8  < %t
+; RUN: llc %s -O2 -print-after-isel -mtriple=aarch64-linux-gnu -jump-table-density=40 -min-jump-table-entries=0 -o /dev/null 2> %t; FileCheck %s --check-prefixes=CHECK,CHECK0  < %t
+; RUN: llc %s -O2 -print-after-isel -mtriple=aarch64-linux-gnu -jump-table-density=40 -min-jump-table-entries=2 -o /dev/null 2> %t; FileCheck %s --check-prefixes=CHECK,CHECK2  < %t
+; RUN: llc %s -O2 -print-after-isel -mtriple=aarch64-linux-gnu -jump-table-density=40 -min-jump-table-entries=4 -o /dev/null 2> %t; FileCheck %s --check-prefixes=CHECK,CHECK4  < %t
+; RUN: llc %s -O2 -print-after-isel -mtriple=aarch64-linux-gnu -jump-table-density=40 -min-jump-table-entries=8 -o /dev/null 2> %t; FileCheck %s --check-prefixes=CHECK,CHECK8  < %t
 
 declare void @ext(i32, i32)
 
diff --git a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-0.ll b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-0.ll
index d7a2c62cf826e..798cd72c5be60 100644
--- a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-0.ll
+++ b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-0.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -mtriple=aarch64-linux %s               -o - | \
 ; RUN:   FileCheck %s --check-prefix=ASM
 ; RUN: llc -mtriple=aarch64-linux %s -filetype=obj -o - |  \
-; RUN:   llvm-readelf --notes | FileCheck %s --check-prefix=OBJ
+; RUN:   llvm-readelf --notes - | FileCheck %s --check-prefix=OBJ
 @x = common dso_local global i32 0, align 4
 
 attributes #0 = { "branch-target-enforcement" }
diff --git a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-1.ll b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-1.ll
index 92973ea73ff3d..747836cab832d 100644
--- a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-1.ll
+++ b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-1.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -mtriple=aarch64-linux %s               -o - | \
 ; RUN:   FileCheck %s --check-prefix=ASM
 ; RUN: llc -mtriple=aarch64-linux %s -filetype=obj -o - |  \
-; RUN:   llvm-readelf --notes | FileCheck %s --check-prefix=OBJ
+; RUN:   llvm-readelf --notes - | FileCheck %s --check-prefix=OBJ
 
 define dso_local i32 @f() #0 {
 entry:
diff --git a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-2.ll b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-2.ll
index 826dd934bd956..71b310932199b 100644
--- a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-2.ll
+++ b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-2.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -mtriple=aarch64-linux %s               -o - | \
 ; RUN:   FileCheck %s --check-prefix=ASM
 ; RUN: llc -mtriple=aarch64-linux %s -filetype=obj -o - |  \
-; RUN:   llvm-readelf --notes | FileCheck %s --check-prefix=OBJ
+; RUN:   llvm-readelf --notes - | FileCheck %s --check-prefix=OBJ
 
 define dso_local i32 @f() #0 {
 entry:
diff --git a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-3.ll b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-3.ll
index c25b23aa1f5a3..ad222e8b4e48c 100644
--- a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-3.ll
+++ b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-3.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -mtriple=aarch64-linux %s               -o - | \
 ; RUN:   FileCheck %s --check-prefix=ASM
 ; RUN: llc -mtriple=aarch64-linux %s -filetype=obj -o - |  \
-; RUN:   llvm-readelf --notes | FileCheck %s --check-prefix=OBJ
+; RUN:   llvm-readelf --notes - | FileCheck %s --check-prefix=OBJ
 
 define dso_local i32 @f() #0 {
 entry:
diff --git a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-4.ll b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-4.ll
index 9667288afc5c1..83da5305e249f 100644
--- a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-4.ll
+++ b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-4.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -mtriple=aarch64-linux %s               -o - | \
 ; RUN:   FileCheck %s --check-prefix=ASM
 ; RUN: llc -mtriple=aarch64-linux %s -filetype=obj -o - |  \
-; RUN:   llvm-readelf --notes | FileCheck %s --check-prefix=OBJ
+; RUN:   llvm-readelf --notes - | FileCheck %s --check-prefix=OBJ
 
 define dso_local i32 @f() #0 {
 entry:
diff --git a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-5.ll b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-5.ll
index f7d1b0e287d56..083f8b38f6762 100644
--- a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-5.ll
+++ b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-5.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -mtriple=aarch64-linux %s               -o - 2>&1 | \
 ; RUN:   FileCheck %s --check-prefix=ASM
 ; RUN: llc -mtriple=aarch64-linux %s -filetype=obj -o -      |  \
-; RUN:   llvm-readelf --notes | FileCheck %s --check-prefix=OBJ
+; RUN:   llvm-readelf --notes - | FileCheck %s --check-prefix=OBJ
 
 define dso_local i32 @f() #0 {
 entry:
diff --git a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-6.ll b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-6.ll
index b57a23b97b2c3..8fb550bcef489 100644
--- a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-6.ll
+++ b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-6.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -mtriple=aarch64-linux %s               -o - | \
 ; RUN:   FileCheck %s --check-prefix=ASM
 ; RUN: llc -mtriple=aarch64-linux %s -filetype=obj -o - |  \
-; RUN:   llvm-readelf -S | FileCheck %s --check-prefix=OBJ
+; RUN:   llvm-readelf -S - | FileCheck %s --check-prefix=OBJ
 
 define dso_local i32 @f() #0 {
 entry:
diff --git a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-7.ll b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-7.ll
index b663dafe44c3e..6768252feff1e 100644
--- a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-7.ll
+++ b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-7.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -mtriple=aarch64-linux %s               -o - 2>&1 | \
 ; RUN:   FileCheck %s --check-prefix=ASM
 ; RUN: llc -mtriple=aarch64-linux %s -filetype=obj -o -      |  \
-; RUN:   llvm-readelf -S | FileCheck %s --check-prefix=OBJ
+; RUN:   llvm-readelf -S - | FileCheck %s --check-prefix=OBJ
 
 define dso_local i32 @f() #0 {
 entry:
diff --git a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-8.ll b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-8.ll
index e27141afdf70e..44545b413ad46 100644
--- a/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-8.ll
+++ b/llvm/test/CodeGen/AArch64/note-gnu-property-pac-bti-8.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -mtriple=aarch64-linux %s               -o - | \
 ; RUN:   FileCheck %s --check-prefix=ASM
 ; RUN: llc -mtriple=aarch64-linux %s -filetype=obj -o - | \
-; RUN:   llvm-readelf --notes | FileCheck %s --check-prefix=OBJ
+; RUN:   llvm-readelf --notes - | FileCheck %s --check-prefix=OBJ
 
 define dso_local i32 @f() #0 {
 entry:
diff --git a/llvm/test/CodeGen/AArch64/partial-pipeline-execution.ll b/llvm/test/CodeGen/AArch64/partial-pipeline-execution.ll
index 82cb0a35f2a58..067da9123fab8 100644
--- a/llvm/test/CodeGen/AArch64/partial-pipeline-execution.ll
+++ b/llvm/test/CodeGen/AArch64/partial-pipeline-execution.ll
@@ -7,6 +7,11 @@
 ; optimize the loop differently
 ; RUN: cmp %t.s %t2.s
 
+; Check that we can do the same with unreachable-mbb-elimination pass
+; RUN: llc -O3 -stop-after=unreachable-mbb-elimination %s -o %t-mbb-elim.mir
+; RUN: llc -O3 -start-after=unreachable-mbb-elimination %t-mbb-elim.mir -o %t3.s
+; RUN: cmp %t.s %t3.s
+
 ; ModuleID = 'loop.c'
 source_filename = "loop.c"
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
diff --git a/llvm/test/CodeGen/AArch64/sve-calling-convention.ll b/llvm/test/CodeGen/AArch64/sve-calling-convention.ll
index 767a3cd8acfe9..f95e749ad5ee9 100644
--- a/llvm/test/CodeGen/AArch64/sve-calling-convention.ll
+++ b/llvm/test/CodeGen/AArch64/sve-calling-convention.ll
@@ -1,4 +1,5 @@
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -stop-after=finalize-isel < %s 2>%t | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -stop-after=prologepilog < %s 2>%t | FileCheck %s --check-prefix=CHECKCSR
 ; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
 
 ; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it.
@@ -123,3 +124,25 @@ define <vscale x 4 x i1> @sve_signature_pred_caller(<vscale x 4 x i1> %arg1, <vs
   %res = call <vscale x 4 x i1> @sve_signature_pred(<vscale x 4 x i1> %arg2, <vscale x 4 x i1> %arg1)
   ret <vscale x 4 x i1> %res
 }
+
+; Test that functions returning or taking SVE arguments use the correct
+; callee-saved set when using the default C calling convention (as opposed
+; to aarch64_sve_vector_pcs)
+
+; CHECKCSR-LABEL: name: sve_signature_vec_ret_callee
+; CHECKCSR: callee-saved-register: '$z8'
+; CHECKCSR: callee-saved-register: '$p4'
+; CHECKCSR: RET_ReallyLR
+define <vscale x 4 x i32> @sve_signature_vec_ret_callee() nounwind {
+  call void asm sideeffect "nop", "~{z8},~{p4}"()
+  ret <vscale x 4 x i32> zeroinitializer
+}
+
+; CHECKCSR-LABEL: name: sve_signature_vec_arg_callee
+; CHECKCSR: callee-saved-register: '$z8'
+; CHECKCSR: callee-saved-register: '$p4'
+; CHECKCSR: RET_ReallyLR
+define void @sve_signature_vec_arg_callee(<vscale x 4 x i32> %v) nounwind {
+  call void asm sideeffect "nop", "~{z8},~{p4}"()
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-copy-zprpair.mir b/llvm/test/CodeGen/AArch64/sve-copy-zprpair.mir
new file mode 100644
index 0000000000000..83a0b5dd1c14a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-copy-zprpair.mir
@@ -0,0 +1,78 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -run-pass=postrapseudos -simplify-mir -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name:            copy_zpr2
+alignment:       4
+tracksRegLiveness: true
+liveins:
+  - { reg: '$z0_z1' }
+frameInfo:
+  maxCallFrameSize: 0
+body:             |
+  bb.0:
+    liveins: $z0_z1
+    ; CHECK-LABEL: name: copy_zpr2
+    ; CHECK: liveins: $z0_z1
+    ; CHECK: $z2 = ORR_ZZZ $z1, $z1
+    ; CHECK: $z1 = ORR_ZZZ $z0, $z0
+    ; CHECK: $z0 = ORR_ZZZ $z1, $z1
+    ; CHECK: $z1 = ORR_ZZZ $z2, $z2
+    ; CHECK: RET_ReallyLR
+    $z1_z2 = COPY $z0_z1
+    $z0_z1 = COPY $z1_z2
+    RET_ReallyLR
+
+...
+---
+name:            copy_zpr3
+alignment:       4
+tracksRegLiveness: true
+liveins:
+  - { reg: '$z0_z1_z2' }
+frameInfo:
+  maxCallFrameSize: 0
+body:             |
+  bb.0:
+    liveins: $z0_z1_z2
+    ; CHECK-LABEL: name: copy_zpr3
+    ; CHECK: liveins: $z0_z1_z2
+    ; CHECK: $z3 = ORR_ZZZ $z2, $z2
+    ; CHECK: $z2 = ORR_ZZZ $z1, $z1
+    ; CHECK: $z1 = ORR_ZZZ $z0, $z0
+    ; CHECK: $z0 = ORR_ZZZ $z1, $z1
+    ; CHECK: $z1 = ORR_ZZZ $z2, $z2
+    ; CHECK: $z2 = ORR_ZZZ $z3, $z3
+    ; CHECK: RET_ReallyLR
+    $z1_z2_z3 = COPY $z0_z1_z2
+    $z0_z1_z2 = COPY $z1_z2_z3
+    RET_ReallyLR
+
+...
+---
+name:            copy_zpr4
+alignment:       4
+tracksRegLiveness: true
+liveins:
+  - { reg: '$z0_z1_z2_z3' }
+frameInfo:
+  maxCallFrameSize: 0
+body:             |
+  bb.0:
+    liveins: $z0_z1_z2_z3
+    ; CHECK-LABEL: name: copy_zpr4
+    ; CHECK: liveins: $z0_z1_z2_z3
+    ; CHECK: $z4 = ORR_ZZZ $z3, $z3
+    ; CHECK: $z3 = ORR_ZZZ $z2, $z2
+    ; CHECK: $z2 = ORR_ZZZ $z1, $z1
+    ; CHECK: $z1 = ORR_ZZZ $z0, $z0
+    ; CHECK: $z0 = ORR_ZZZ $z1, $z1
+    ; CHECK: $z1 = ORR_ZZZ $z2, $z2
+    ; CHECK: $z2 = ORR_ZZZ $z3, $z3
+    ; CHECK: $z3 = ORR_ZZZ $z4, $z4
+    ; CHECK: RET_ReallyLR
+    $z1_z2_z3_z4 = COPY $z0_z1_z2_z3
+    $z0_z1_z2_z3 = COPY $z1_z2_z3_z4
+    RET_ReallyLR
+
+...
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-arith.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-arith.ll
index 9c32748e3b8ef..242fabe44f793 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-arith.ll
@@ -25,6 +25,10 @@ target triple = "aarch64-unknown-linux-gnu"
 ; Don't use SVE when its registers are no bigger than NEON.
 ; NO_SVE-NOT: ptrue
 
+;
+; FADD
+;
+
 ; Don't use SVE for 64-bit vectors.
 define <4 x half> @fadd_v4f16(<4 x half> %op1, <4 x half> %op2) #0 {
 ; CHECK-LABEL: @fadd_v4f16
@@ -112,7 +116,7 @@ define void @fadd_v64f16(<64 x half>* %a, <64 x half>* %b) #0 {
   ret void
 }
 
-; NOTE: Check lines only cover the first VBYTES because the add_v#f16 tests
+; NOTE: Check lines only cover the first VBYTES because the fadd_v#f16 tests
 ; already cover the general legalisation cases.
 define void @fadd_v128f16(<128 x half>* %a, <128 x half>* %b) #0 {
 ; CHECK-LABEL: @fadd_v128f16
@@ -162,7 +166,7 @@ define void @fadd_v8f32(<8 x float>* %a, <8 x float>* %b) #0 {
   ret void
 }
 
-; NOTE: Check lines only cover the first VBYTES because the add_v#f16 tests
+; NOTE: Check lines only cover the first VBYTES because the fadd_v#f16 tests
 ; already cover the general legalisation cases.
 define void @fadd_v16f32(<16 x float>* %a, <16 x float>* %b) #0 {
 ; CHECK-LABEL: @fadd_v16f32
@@ -179,7 +183,7 @@ define void @fadd_v16f32(<16 x float>* %a, <16 x float>* %b) #0 {
   ret void
 }
 
-; NOTE: Check lines only cover the first VBYTES because the add_v#f16 tests
+; NOTE: Check lines only cover the first VBYTES because the fadd_v#f16 tests
 ; already cover the general legalisation cases.
 define void @fadd_v32f32(<32 x float>* %a, <32 x float>* %b) #0 {
 ; CHECK-LABEL: @fadd_v32f32
@@ -196,7 +200,7 @@ define void @fadd_v32f32(<32 x float>* %a, <32 x float>* %b) #0 {
   ret void
 }
 
-; NOTE: Check lines only cover the first VBYTES because the add_v#f16 tests
+; NOTE: Check lines only cover the first VBYTES because the fadd_v#f16 tests
 ; already cover the general legalisation cases.
 define void @fadd_v64f32(<64 x float>* %a, <64 x float>* %b) #0 {
 ; CHECK-LABEL: @fadd_v64f32
@@ -246,7 +250,7 @@ define void @fadd_v4f64(<4 x double>* %a, <4 x double>* %b) #0 {
   ret void
 }
 
-; NOTE: Check lines only cover the first VBYTES because the add_v#f16 tests
+; NOTE: Check lines only cover the first VBYTES because the fadd_v#f16 tests
 ; already cover the general legalisation cases.
 define void @fadd_v8f64(<8 x double>* %a, <8 x double>* %b) #0 {
 ; CHECK-LABEL: @fadd_v8f64
@@ -263,7 +267,7 @@ define void @fadd_v8f64(<8 x double>* %a, <8 x double>* %b) #0 {
   ret void
 }
 
-; NOTE: Check lines only cover the first VBYTES because the add_v#f16 tests
+; NOTE: Check lines only cover the first VBYTES because the fadd_v#f16 tests
 ; already cover the general legalisation cases.
 define void @fadd_v16f64(<16 x double>* %a, <16 x double>* %b) #0 {
 ; CHECK-LABEL: @fadd_v16f64
@@ -280,7 +284,7 @@ define void @fadd_v16f64(<16 x double>* %a, <16 x double>* %b) #0 {
   ret void
 }
 
-; NOTE: Check lines only cover the first VBYTES because the add_v#f16 tests
+; NOTE: Check lines only cover the first VBYTES because the fadd_v#f16 tests
 ; already cover the general legalisation cases.
 define void @fadd_v32f64(<32 x double>* %a, <32 x double>* %b) #0 {
 ; CHECK-LABEL: @fadd_v32f64
@@ -297,4 +301,1004 @@ define void @fadd_v32f64(<32 x double>* %a, <32 x double>* %b) #0 {
   ret void
 }
 
-attributes #0 = { "target-features"="+sve" }
+;
+; NOTE: Tests beyond this point only have CHECK lines to validate the first
+; VBYTES because the fadd tests already validate the legalisation code paths.
+;
+
+;
+; FDIV
+;
+
+; Don't use SVE for 64-bit vectors.
+define <4 x half> @fdiv_v4f16(<4 x half> %op1, <4 x half> %op2) #0 {
+; CHECK-LABEL: @fdiv_v4f16
+; CHECK: fdiv v0.4h, v0.4h, v1.4h
+; CHECK: ret
+  %res = fdiv <4 x half> %op1, %op2
+  ret <4 x half> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <8 x half> @fdiv_v8f16(<8 x half> %op1, <8 x half> %op2) #0 {
+; CHECK-LABEL: @fdiv_v8f16
+; CHECK: fdiv v0.8h, v0.8h, v1.8h
+; CHECK: ret
+  %res = fdiv <8 x half> %op1, %op2
+  ret <8 x half> %res
+}
+
+define void @fdiv_v16f16(<16 x half>* %a, <16 x half>* %b) #0 {
+; CHECK-LABEL: @fdiv_v16f16
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),16)]]
+; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
+; CHECK: fdiv [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <16 x half>, <16 x half>* %a
+  %op2 = load <16 x half>, <16 x half>* %b
+  %res = fdiv <16 x half> %op1, %op2
+  store <16 x half> %res, <16 x half>* %a
+  ret void
+}
+
+define void @fdiv_v32f16(<32 x half>* %a, <32 x half>* %b) #0 {
+; CHECK-LABEL: @fdiv_v32f16
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),32)]]
+; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
+; CHECK: fdiv [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <32 x half>, <32 x half>* %a
+  %op2 = load <32 x half>, <32 x half>* %b
+  %res = fdiv <32 x half> %op1, %op2
+  store <32 x half> %res, <32 x half>* %a
+  ret void
+}
+
+define void @fdiv_v64f16(<64 x half>* %a, <64 x half>* %b) #0 {
+; CHECK-LABEL: @fdiv_v64f16
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),64)]]
+; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
+; CHECK: fdiv [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <64 x half>, <64 x half>* %a
+  %op2 = load <64 x half>, <64 x half>* %b
+  %res = fdiv <64 x half> %op1, %op2
+  store <64 x half> %res, <64 x half>* %a
+  ret void
+}
+
+define void @fdiv_v128f16(<128 x half>* %a, <128 x half>* %b) #0 {
+; CHECK-LABEL: @fdiv_v128f16
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),128)]]
+; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
+; CHECK: fdiv [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <128 x half>, <128 x half>* %a
+  %op2 = load <128 x half>, <128 x half>* %b
+  %res = fdiv <128 x half> %op1, %op2
+  store <128 x half> %res, <128 x half>* %a
+  ret void
+}
+
+; Don't use SVE for 64-bit vectors.
+define <2 x float> @fdiv_v2f32(<2 x float> %op1, <2 x float> %op2) #0 {
+; CHECK-LABEL: @fdiv_v2f32
+; CHECK: fdiv v0.2s, v0.2s, v1.2s
+; CHECK: ret
+  %res = fdiv <2 x float> %op1, %op2
+  ret <2 x float> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <4 x float> @fdiv_v4f32(<4 x float> %op1, <4 x float> %op2) #0 {
+; CHECK-LABEL: @fdiv_v4f32
+; CHECK: fdiv v0.4s, v0.4s, v1.4s
+; CHECK: ret
+  %res = fdiv <4 x float> %op1, %op2
+  ret <4 x float> %res
+}
+
+define void @fdiv_v8f32(<8 x float>* %a, <8 x float>* %b) #0 {
+; CHECK-LABEL: @fdiv_v8f32
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),8)]]
+; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
+; CHECK: fdiv [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <8 x float>, <8 x float>* %a
+  %op2 = load <8 x float>, <8 x float>* %b
+  %res = fdiv <8 x float> %op1, %op2
+  store <8 x float> %res, <8 x float>* %a
+  ret void
+}
+
+define void @fdiv_v16f32(<16 x float>* %a, <16 x float>* %b) #0 {
+; CHECK-LABEL: @fdiv_v16f32
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),16)]]
+; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
+; CHECK: fdiv [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <16 x float>, <16 x float>* %a
+  %op2 = load <16 x float>, <16 x float>* %b
+  %res = fdiv <16 x float> %op1, %op2
+  store <16 x float> %res, <16 x float>* %a
+  ret void
+}
+
+define void @fdiv_v32f32(<32 x float>* %a, <32 x float>* %b) #0 {
+; CHECK-LABEL: @fdiv_v32f32
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),32)]]
+; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
+; CHECK: fdiv [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <32 x float>, <32 x float>* %a
+  %op2 = load <32 x float>, <32 x float>* %b
+  %res = fdiv <32 x float> %op1, %op2
+  store <32 x float> %res, <32 x float>* %a
+  ret void
+}
+
+define void @fdiv_v64f32(<64 x float>* %a, <64 x float>* %b) #0 {
+; CHECK-LABEL: @fdiv_v64f32
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),64)]]
+; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
+; CHECK: fdiv [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <64 x float>, <64 x float>* %a
+  %op2 = load <64 x float>, <64 x float>* %b
+  %res = fdiv <64 x float> %op1, %op2
+  store <64 x float> %res, <64 x float>* %a
+  ret void
+}
+
+; Don't use SVE for 64-bit vectors.
+define <1 x double> @fdiv_v1f64(<1 x double> %op1, <1 x double> %op2) #0 {
+; CHECK-LABEL: @fdiv_v1f64
+; CHECK: fdiv d0, d0, d1
+; CHECK: ret
+  %res = fdiv <1 x double> %op1, %op2
+  ret <1 x double> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <2 x double> @fdiv_v2f64(<2 x double> %op1, <2 x double> %op2) #0 {
+; CHECK-LABEL: @fdiv_v2f64
+; CHECK: fdiv v0.2d, v0.2d, v1.2d
+; CHECK: ret
+  %res = fdiv <2 x double> %op1, %op2
+  ret <2 x double> %res
+}
+
+define void @fdiv_v4f64(<4 x double>* %a, <4 x double>* %b) #0 {
+; CHECK-LABEL: @fdiv_v4f64
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),4)]]
+; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
+; CHECK: fdiv [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <4 x double>, <4 x double>* %a
+  %op2 = load <4 x double>, <4 x double>* %b
+  %res = fdiv <4 x double> %op1, %op2
+  store <4 x double> %res, <4 x double>* %a
+  ret void
+}
+
+define void @fdiv_v8f64(<8 x double>* %a, <8 x double>* %b) #0 {
+; CHECK-LABEL: @fdiv_v8f64
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),8)]]
+; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
+; CHECK: fdiv [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <8 x double>, <8 x double>* %a
+  %op2 = load <8 x double>, <8 x double>* %b
+  %res = fdiv <8 x double> %op1, %op2
+  store <8 x double> %res, <8 x double>* %a
+  ret void
+}
+
+define void @fdiv_v16f64(<16 x double>* %a, <16 x double>* %b) #0 {
+; CHECK-LABEL: @fdiv_v16f64
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),16)]]
+; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
+; CHECK: fdiv [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <16 x double>, <16 x double>* %a
+  %op2 = load <16 x double>, <16 x double>* %b
+  %res = fdiv <16 x double> %op1, %op2
+  store <16 x double> %res, <16 x double>* %a
+  ret void
+}
+
+define void @fdiv_v32f64(<32 x double>* %a, <32 x double>* %b) #0 {
+; CHECK-LABEL: @fdiv_v32f64
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),32)]]
+; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
+; CHECK: fdiv [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <32 x double>, <32 x double>* %a
+  %op2 = load <32 x double>, <32 x double>* %b
+  %res = fdiv <32 x double> %op1, %op2
+  store <32 x double> %res, <32 x double>* %a
+  ret void
+}
+
+;
+; FMA
+;
+
+; Don't use SVE for 64-bit vectors.
+define <4 x half> @fma_v4f16(<4 x half> %op1, <4 x half> %op2, <4 x half> %op3) #0 {
+; CHECK-LABEL: @fma_v4f16
+; CHECK: fmla v2.4h, v1.4h, v0.4h
+; CHECK: ret
+  %res = call <4 x half> @llvm.fma.v4f16(<4 x half> %op1, <4 x half> %op2, <4 x half> %op3)
+  ret <4 x half> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <8 x half> @fma_v8f16(<8 x half> %op1, <8 x half> %op2, <8 x half> %op3) #0 {
+; CHECK-LABEL: @fma_v8f16
+; CHECK: fmla v2.8h, v1.8h, v0.8h
+; CHECK: ret
+  %res = call <8 x half> @llvm.fma.v8f16(<8 x half> %op1, <8 x half> %op2, <8 x half> %op3)
+  ret <8 x half> %res
+}
+
+define void @fma_v16f16(<16 x half>* %a, <16 x half>* %b, <16 x half>* %c) #0 {
+; CHECK-LABEL: @fma_v16f16
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),16)]]
+; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
+; CHECK-DAG: ld1h { [[OP3:z[0-9]+]].h }, [[PG]]/z, [x2]
+; CHECK: fmla [[OP3]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: st1h { [[OP3]].h }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <16 x half>, <16 x half>* %a
+  %op2 = load <16 x half>, <16 x half>* %b
+  %op3 = load <16 x half>, <16 x half>* %c
+  %res = call <16 x half> @llvm.fma.v16f16(<16 x half> %op1, <16 x half> %op2, <16 x half> %op3)
+  store <16 x half> %res, <16 x half>* %a
+  ret void
+}
+
+define void @fma_v32f16(<32 x half>* %a, <32 x half>* %b, <32 x half>* %c) #0 {
+; CHECK-LABEL: @fma_v32f16
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),32)]]
+; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
+; CHECK-DAG: ld1h { [[OP3:z[0-9]+]].h }, [[PG]]/z, [x2]
+; CHECK: fmla [[OP3]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: st1h { [[OP3]].h }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <32 x half>, <32 x half>* %a
+  %op2 = load <32 x half>, <32 x half>* %b
+  %op3 = load <32 x half>, <32 x half>* %c
+  %res = call <32 x half> @llvm.fma.v32f16(<32 x half> %op1, <32 x half> %op2, <32 x half> %op3)
+  store <32 x half> %res, <32 x half>* %a
+  ret void
+}
+
+define void @fma_v64f16(<64 x half>* %a, <64 x half>* %b, <64 x half>* %c) #0 {
+; CHECK-LABEL: @fma_v64f16
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),64)]]
+; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
+; CHECK-DAG: ld1h { [[OP3:z[0-9]+]].h }, [[PG]]/z, [x2]
+; CHECK: fmla [[OP3]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: st1h { [[OP3]].h }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <64 x half>, <64 x half>* %a
+  %op2 = load <64 x half>, <64 x half>* %b
+  %op3 = load <64 x half>, <64 x half>* %c
+  %res = call <64 x half> @llvm.fma.v64f16(<64 x half> %op1, <64 x half> %op2, <64 x half> %op3)
+  store <64 x half> %res, <64 x half>* %a
+  ret void
+}
+
+define void @fma_v128f16(<128 x half>* %a, <128 x half>* %b, <128 x half>* %c) #0 {
+; CHECK-LABEL: @fma_v128f16
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),128)]]
+; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
+; CHECK-DAG: ld1h { [[OP3:z[0-9]+]].h }, [[PG]]/z, [x2]
+; CHECK: fmla [[OP3]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: st1h { [[OP3]].h }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <128 x half>, <128 x half>* %a
+  %op2 = load <128 x half>, <128 x half>* %b
+  %op3 = load <128 x half>, <128 x half>* %c
+  %res = call <128 x half> @llvm.fma.v128f16(<128 x half> %op1, <128 x half> %op2, <128 x half> %op3)
+  store <128 x half> %res, <128 x half>* %a
+  ret void
+}
+
+; Don't use SVE for 64-bit vectors.
+define <2 x float> @fma_v2f32(<2 x float> %op1, <2 x float> %op2, <2 x float> %op3) #0 {
+; CHECK-LABEL: @fma_v2f32
+; CHECK: fmla v2.2s, v1.2s, v0.2s
+; CHECK: ret
+  %res = call <2 x float> @llvm.fma.v2f32(<2 x float> %op1, <2 x float> %op2, <2 x float> %op3)
+  ret <2 x float> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <4 x float> @fma_v4f32(<4 x float> %op1, <4 x float> %op2, <4 x float> %op3) #0 {
+; CHECK-LABEL: @fma_v4f32
+; CHECK: fmla v2.4s, v1.4s, v0.4s
+; CHECK: ret
+  %res = call <4 x float> @llvm.fma.v4f32(<4 x float> %op1, <4 x float> %op2, <4 x float> %op3)
+  ret <4 x float> %res
+}
+
+define void @fma_v8f32(<8 x float>* %a, <8 x float>* %b, <8 x float>* %c) #0 {
+; CHECK-LABEL: @fma_v8f32
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),8)]]
+; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
+; CHECK-DAG: ld1w { [[OP3:z[0-9]+]].s }, [[PG]]/z, [x2]
+; CHECK: fmla [[OP3]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: st1w { [[OP3]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <8 x float>, <8 x float>* %a
+  %op2 = load <8 x float>, <8 x float>* %b
+  %op3 = load <8 x float>, <8 x float>* %c
+  %res = call <8 x float> @llvm.fma.v8f32(<8 x float> %op1, <8 x float> %op2, <8 x float> %op3)
+  store <8 x float> %res, <8 x float>* %a
+  ret void
+}
+
+define void @fma_v16f32(<16 x float>* %a, <16 x float>* %b, <16 x float>* %c) #0 {
+; CHECK-LABEL: @fma_v16f32
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),16)]]
+; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
+; CHECK-DAG: ld1w { [[OP3:z[0-9]+]].s }, [[PG]]/z, [x2]
+; CHECK: fmla [[OP3]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: st1w { [[OP3]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <16 x float>, <16 x float>* %a
+  %op2 = load <16 x float>, <16 x float>* %b
+  %op3 = load <16 x float>, <16 x float>* %c
+  %res = call <16 x float> @llvm.fma.v16f32(<16 x float> %op1, <16 x float> %op2, <16 x float> %op3)
+  store <16 x float> %res, <16 x float>* %a
+  ret void
+}
+
+define void @fma_v32f32(<32 x float>* %a, <32 x float>* %b, <32 x float>* %c) #0 {
+; CHECK-LABEL: @fma_v32f32
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),32)]]
+; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
+; CHECK-DAG: ld1w { [[OP3:z[0-9]+]].s }, [[PG]]/z, [x2]
+; CHECK: fmla [[OP3]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: st1w { [[OP3]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <32 x float>, <32 x float>* %a
+  %op2 = load <32 x float>, <32 x float>* %b
+  %op3 = load <32 x float>, <32 x float>* %c
+  %res = call <32 x float> @llvm.fma.v32f32(<32 x float> %op1, <32 x float> %op2, <32 x float> %op3)
+  store <32 x float> %res, <32 x float>* %a
+  ret void
+}
+
+define void @fma_v64f32(<64 x float>* %a, <64 x float>* %b, <64 x float>* %c) #0 {
+; CHECK-LABEL: @fma_v64f32
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),64)]]
+; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
+; CHECK-DAG: ld1w { [[OP3:z[0-9]+]].s }, [[PG]]/z, [x2]
+; CHECK: fmla [[OP3]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: st1w { [[OP3]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <64 x float>, <64 x float>* %a
+  %op2 = load <64 x float>, <64 x float>* %b
+  %op3 = load <64 x float>, <64 x float>* %c
+  %res = call <64 x float> @llvm.fma.v64f32(<64 x float> %op1, <64 x float> %op2, <64 x float> %op3)
+  store <64 x float> %res, <64 x float>* %a
+  ret void
+}
+
+; Don't use SVE for 64-bit vectors.
+define <1 x double> @fma_v1f64(<1 x double> %op1, <1 x double> %op2, <1 x double> %op3) #0 {
+; CHECK-LABEL: @fma_v1f64
+; CHECK: fmadd d0, d0, d1, d2
+; CHECK: ret
+  %res = call <1 x double> @llvm.fma.v1f64(<1 x double> %op1, <1 x double> %op2, <1 x double> %op3)
+  ret <1 x double> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <2 x double> @fma_v2f64(<2 x double> %op1, <2 x double> %op2, <2 x double> %op3) #0 {
+; CHECK-LABEL: @fma_v2f64
+; CHECK: fmla v2.2d, v1.2d, v0.2d
+; CHECK: ret
+  %res = call <2 x double> @llvm.fma.v2f64(<2 x double> %op1, <2 x double> %op2, <2 x double> %op3)
+  ret <2 x double> %res
+}
+
+define void @fma_v4f64(<4 x double>* %a, <4 x double>* %b, <4 x double>* %c) #0 {
+; CHECK-LABEL: @fma_v4f64
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),4)]]
+; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
+; CHECK-DAG: ld1d { [[OP3:z[0-9]+]].d }, [[PG]]/z, [x2]
+; CHECK: fmla [[OP3]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: st1d { [[OP3]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <4 x double>, <4 x double>* %a
+  %op2 = load <4 x double>, <4 x double>* %b
+  %op3 = load <4 x double>, <4 x double>* %c
+  %res = call <4 x double> @llvm.fma.v4f64(<4 x double> %op1, <4 x double> %op2, <4 x double> %op3)
+  store <4 x double> %res, <4 x double>* %a
+  ret void
+}
+
+define void @fma_v8f64(<8 x double>* %a, <8 x double>* %b, <8 x double>* %c) #0 {
+; CHECK-LABEL: @fma_v8f64
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),8)]]
+; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
+; CHECK-DAG: ld1d { [[OP3:z[0-9]+]].d }, [[PG]]/z, [x2]
+; CHECK: fmla [[OP3]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: st1d { [[OP3]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <8 x double>, <8 x double>* %a
+  %op2 = load <8 x double>, <8 x double>* %b
+  %op3 = load <8 x double>, <8 x double>* %c
+  %res = call <8 x double> @llvm.fma.v8f64(<8 x double> %op1, <8 x double> %op2, <8 x double> %op3)
+  store <8 x double> %res, <8 x double>* %a
+  ret void
+}
+
+define void @fma_v16f64(<16 x double>* %a, <16 x double>* %b, <16 x double>* %c) #0 {
+; CHECK-LABEL: @fma_v16f64
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),16)]]
+; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
+; CHECK-DAG: ld1d { [[OP3:z[0-9]+]].d }, [[PG]]/z, [x2]
+; CHECK: fmla [[OP3]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: st1d { [[OP3]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <16 x double>, <16 x double>* %a
+  %op2 = load <16 x double>, <16 x double>* %b
+  %op3 = load <16 x double>, <16 x double>* %c
+  %res = call <16 x double> @llvm.fma.v16f64(<16 x double> %op1, <16 x double> %op2, <16 x double> %op3)
+  store <16 x double> %res, <16 x double>* %a
+  ret void
+}
+
+define void @fma_v32f64(<32 x double>* %a, <32 x double>* %b, <32 x double>* %c) #0 {
+; CHECK-LABEL: @fma_v32f64
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),32)]]
+; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
+; CHECK-DAG: ld1d { [[OP3:z[0-9]+]].d }, [[PG]]/z, [x2]
+; CHECK: fmla [[OP3]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: st1d { [[OP3]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <32 x double>, <32 x double>* %a
+  %op2 = load <32 x double>, <32 x double>* %b
+  %op3 = load <32 x double>, <32 x double>* %c
+  %res = call <32 x double> @llvm.fma.v32f64(<32 x double> %op1, <32 x double> %op2, <32 x double> %op3)
+  store <32 x double> %res, <32 x double>* %a
+  ret void
+}
+
+;
+; FMUL
+;
+
+; Don't use SVE for 64-bit vectors.
+define <4 x half> @fmul_v4f16(<4 x half> %op1, <4 x half> %op2) #0 {
+; CHECK-LABEL: @fmul_v4f16
+; CHECK: fmul v0.4h, v0.4h, v1.4h
+; CHECK: ret
+  %res = fmul <4 x half> %op1, %op2
+  ret <4 x half> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <8 x half> @fmul_v8f16(<8 x half> %op1, <8 x half> %op2) #0 {
+; CHECK-LABEL: @fmul_v8f16
+; CHECK: fmul v0.8h, v0.8h, v1.8h
+; CHECK: ret
+  %res = fmul <8 x half> %op1, %op2
+  ret <8 x half> %res
+}
+
+define void @fmul_v16f16(<16 x half>* %a, <16 x half>* %b) #0 {
+; CHECK-LABEL: @fmul_v16f16
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),16)]]
+; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
+; CHECK: fmul [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <16 x half>, <16 x half>* %a
+  %op2 = load <16 x half>, <16 x half>* %b
+  %res = fmul <16 x half> %op1, %op2
+  store <16 x half> %res, <16 x half>* %a
+  ret void
+}
+
+define void @fmul_v32f16(<32 x half>* %a, <32 x half>* %b) #0 {
+; CHECK-LABEL: @fmul_v32f16
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),32)]]
+; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
+; CHECK: fmul [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <32 x half>, <32 x half>* %a
+  %op2 = load <32 x half>, <32 x half>* %b
+  %res = fmul <32 x half> %op1, %op2
+  store <32 x half> %res, <32 x half>* %a
+  ret void
+}
+
+define void @fmul_v64f16(<64 x half>* %a, <64 x half>* %b) #0 {
+; CHECK-LABEL: @fmul_v64f16
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),64)]]
+; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
+; CHECK: fmul [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <64 x half>, <64 x half>* %a
+  %op2 = load <64 x half>, <64 x half>* %b
+  %res = fmul <64 x half> %op1, %op2
+  store <64 x half> %res, <64 x half>* %a
+  ret void
+}
+
+define void @fmul_v128f16(<128 x half>* %a, <128 x half>* %b) #0 {
+; CHECK-LABEL: @fmul_v128f16
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),128)]]
+; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
+; CHECK: fmul [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <128 x half>, <128 x half>* %a
+  %op2 = load <128 x half>, <128 x half>* %b
+  %res = fmul <128 x half> %op1, %op2
+  store <128 x half> %res, <128 x half>* %a
+  ret void
+}
+
+; Don't use SVE for 64-bit vectors.
+define <2 x float> @fmul_v2f32(<2 x float> %op1, <2 x float> %op2) #0 {
+; CHECK-LABEL: @fmul_v2f32
+; CHECK: fmul v0.2s, v0.2s, v1.2s
+; CHECK: ret
+  %res = fmul <2 x float> %op1, %op2
+  ret <2 x float> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <4 x float> @fmul_v4f32(<4 x float> %op1, <4 x float> %op2) #0 {
+; CHECK-LABEL: @fmul_v4f32
+; CHECK: fmul v0.4s, v0.4s, v1.4s
+; CHECK: ret
+  %res = fmul <4 x float> %op1, %op2
+  ret <4 x float> %res
+}
+
+define void @fmul_v8f32(<8 x float>* %a, <8 x float>* %b) #0 {
+; CHECK-LABEL: @fmul_v8f32
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),8)]]
+; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
+; CHECK: fmul [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <8 x float>, <8 x float>* %a
+  %op2 = load <8 x float>, <8 x float>* %b
+  %res = fmul <8 x float> %op1, %op2
+  store <8 x float> %res, <8 x float>* %a
+  ret void
+}
+
+define void @fmul_v16f32(<16 x float>* %a, <16 x float>* %b) #0 {
+; CHECK-LABEL: @fmul_v16f32
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),16)]]
+; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
+; CHECK: fmul [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <16 x float>, <16 x float>* %a
+  %op2 = load <16 x float>, <16 x float>* %b
+  %res = fmul <16 x float> %op1, %op2
+  store <16 x float> %res, <16 x float>* %a
+  ret void
+}
+
+define void @fmul_v32f32(<32 x float>* %a, <32 x float>* %b) #0 {
+; CHECK-LABEL: @fmul_v32f32
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),32)]]
+; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
+; CHECK: fmul [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <32 x float>, <32 x float>* %a
+  %op2 = load <32 x float>, <32 x float>* %b
+  %res = fmul <32 x float> %op1, %op2
+  store <32 x float> %res, <32 x float>* %a
+  ret void
+}
+
+define void @fmul_v64f32(<64 x float>* %a, <64 x float>* %b) #0 {
+; CHECK-LABEL: @fmul_v64f32
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),64)]]
+; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
+; CHECK: fmul [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <64 x float>, <64 x float>* %a
+  %op2 = load <64 x float>, <64 x float>* %b
+  %res = fmul <64 x float> %op1, %op2
+  store <64 x float> %res, <64 x float>* %a
+  ret void
+}
+
+; Don't use SVE for 64-bit vectors.
+define <1 x double> @fmul_v1f64(<1 x double> %op1, <1 x double> %op2) #0 {
+; CHECK-LABEL: @fmul_v1f64
+; CHECK: fmul d0, d0, d1
+; CHECK: ret
+  %res = fmul <1 x double> %op1, %op2
+  ret <1 x double> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <2 x double> @fmul_v2f64(<2 x double> %op1, <2 x double> %op2) #0 {
+; CHECK-LABEL: @fmul_v2f64
+; CHECK: fmul v0.2d, v0.2d, v1.2d
+; CHECK: ret
+  %res = fmul <2 x double> %op1, %op2
+  ret <2 x double> %res
+}
+
+define void @fmul_v4f64(<4 x double>* %a, <4 x double>* %b) #0 {
+; CHECK-LABEL: @fmul_v4f64
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),4)]]
+; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
+; CHECK: fmul [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <4 x double>, <4 x double>* %a
+  %op2 = load <4 x double>, <4 x double>* %b
+  %res = fmul <4 x double> %op1, %op2
+  store <4 x double> %res, <4 x double>* %a
+  ret void
+}
+
+define void @fmul_v8f64(<8 x double>* %a, <8 x double>* %b) #0 {
+; CHECK-LABEL: @fmul_v8f64
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),8)]]
+; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
+; CHECK: fmul [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <8 x double>, <8 x double>* %a
+  %op2 = load <8 x double>, <8 x double>* %b
+  %res = fmul <8 x double> %op1, %op2
+  store <8 x double> %res, <8 x double>* %a
+  ret void
+}
+
+define void @fmul_v16f64(<16 x double>* %a, <16 x double>* %b) #0 {
+; CHECK-LABEL: @fmul_v16f64
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),16)]]
+; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
+; CHECK: fmul [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <16 x double>, <16 x double>* %a
+  %op2 = load <16 x double>, <16 x double>* %b
+  %res = fmul <16 x double> %op1, %op2
+  store <16 x double> %res, <16 x double>* %a
+  ret void
+}
+
+define void @fmul_v32f64(<32 x double>* %a, <32 x double>* %b) #0 {
+; CHECK-LABEL: @fmul_v32f64
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),32)]]
+; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
+; CHECK: fmul [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <32 x double>, <32 x double>* %a
+  %op2 = load <32 x double>, <32 x double>* %b
+  %res = fmul <32 x double> %op1, %op2
+  store <32 x double> %res, <32 x double>* %a
+  ret void
+}
+
+;
+; FSUB
+;
+
+; Don't use SVE for 64-bit vectors.
+define <4 x half> @fsub_v4f16(<4 x half> %op1, <4 x half> %op2) #0 {
+; CHECK-LABEL: @fsub_v4f16
+; CHECK: fsub v0.4h, v0.4h, v1.4h
+; CHECK: ret
+  %res = fsub <4 x half> %op1, %op2
+  ret <4 x half> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <8 x half> @fsub_v8f16(<8 x half> %op1, <8 x half> %op2) #0 {
+; CHECK-LABEL: @fsub_v8f16
+; CHECK: fsub v0.8h, v0.8h, v1.8h
+; CHECK: ret
+  %res = fsub <8 x half> %op1, %op2
+  ret <8 x half> %res
+}
+
+define void @fsub_v16f16(<16 x half>* %a, <16 x half>* %b) #0 {
+; CHECK-LABEL: @fsub_v16f16
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),16)]]
+; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
+; CHECK: fsub [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <16 x half>, <16 x half>* %a
+  %op2 = load <16 x half>, <16 x half>* %b
+  %res = fsub <16 x half> %op1, %op2
+  store <16 x half> %res, <16 x half>* %a
+  ret void
+}
+
+define void @fsub_v32f16(<32 x half>* %a, <32 x half>* %b) #0 {
+; CHECK-LABEL: @fsub_v32f16
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),32)]]
+; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
+; CHECK: fsub [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <32 x half>, <32 x half>* %a
+  %op2 = load <32 x half>, <32 x half>* %b
+  %res = fsub <32 x half> %op1, %op2
+  store <32 x half> %res, <32 x half>* %a
+  ret void
+}
+
+define void @fsub_v64f16(<64 x half>* %a, <64 x half>* %b) #0 {
+; CHECK-LABEL: @fsub_v64f16
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),64)]]
+; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
+; CHECK: fsub [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <64 x half>, <64 x half>* %a
+  %op2 = load <64 x half>, <64 x half>* %b
+  %res = fsub <64 x half> %op1, %op2
+  store <64 x half> %res, <64 x half>* %a
+  ret void
+}
+
+define void @fsub_v128f16(<128 x half>* %a, <128 x half>* %b) #0 {
+; CHECK-LABEL: @fsub_v128f16
+; CHECK: ptrue [[PG:p[0-9]+]].h, vl[[#min(div(VBYTES,2),128)]]
+; CHECK-DAG: ld1h { [[OP1:z[0-9]+]].h }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1h { [[OP2:z[0-9]+]].h }, [[PG]]/z, [x1]
+; CHECK: fsub [[RES:z[0-9]+]].h, [[PG]]/m, [[OP1]].h, [[OP2]].h
+; CHECK: st1h { [[RES]].h }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <128 x half>, <128 x half>* %a
+  %op2 = load <128 x half>, <128 x half>* %b
+  %res = fsub <128 x half> %op1, %op2
+  store <128 x half> %res, <128 x half>* %a
+  ret void
+}
+
+; Don't use SVE for 64-bit vectors.
+define <2 x float> @fsub_v2f32(<2 x float> %op1, <2 x float> %op2) #0 {
+; CHECK-LABEL: @fsub_v2f32
+; CHECK: fsub v0.2s, v0.2s, v1.2s
+; CHECK: ret
+  %res = fsub <2 x float> %op1, %op2
+  ret <2 x float> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <4 x float> @fsub_v4f32(<4 x float> %op1, <4 x float> %op2) #0 {
+; CHECK-LABEL: @fsub_v4f32
+; CHECK: fsub v0.4s, v0.4s, v1.4s
+; CHECK: ret
+  %res = fsub <4 x float> %op1, %op2
+  ret <4 x float> %res
+}
+
+define void @fsub_v8f32(<8 x float>* %a, <8 x float>* %b) #0 {
+; CHECK-LABEL: @fsub_v8f32
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),8)]]
+; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
+; CHECK: fsub [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <8 x float>, <8 x float>* %a
+  %op2 = load <8 x float>, <8 x float>* %b
+  %res = fsub <8 x float> %op1, %op2
+  store <8 x float> %res, <8 x float>* %a
+  ret void
+}
+
+define void @fsub_v16f32(<16 x float>* %a, <16 x float>* %b) #0 {
+; CHECK-LABEL: @fsub_v16f32
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),16)]]
+; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
+; CHECK: fsub [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <16 x float>, <16 x float>* %a
+  %op2 = load <16 x float>, <16 x float>* %b
+  %res = fsub <16 x float> %op1, %op2
+  store <16 x float> %res, <16 x float>* %a
+  ret void
+}
+
+define void @fsub_v32f32(<32 x float>* %a, <32 x float>* %b) #0 {
+; CHECK-LABEL: @fsub_v32f32
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),32)]]
+; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
+; CHECK: fsub [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <32 x float>, <32 x float>* %a
+  %op2 = load <32 x float>, <32 x float>* %b
+  %res = fsub <32 x float> %op1, %op2
+  store <32 x float> %res, <32 x float>* %a
+  ret void
+}
+
+define void @fsub_v64f32(<64 x float>* %a, <64 x float>* %b) #0 {
+; CHECK-LABEL: @fsub_v64f32
+; CHECK: ptrue [[PG:p[0-9]+]].s, vl[[#min(div(VBYTES,4),64)]]
+; CHECK-DAG: ld1w { [[OP1:z[0-9]+]].s }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1w { [[OP2:z[0-9]+]].s }, [[PG]]/z, [x1]
+; CHECK: fsub [[RES:z[0-9]+]].s, [[PG]]/m, [[OP1]].s, [[OP2]].s
+; CHECK: st1w { [[RES]].s }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <64 x float>, <64 x float>* %a
+  %op2 = load <64 x float>, <64 x float>* %b
+  %res = fsub <64 x float> %op1, %op2
+  store <64 x float> %res, <64 x float>* %a
+  ret void
+}
+
+; Don't use SVE for 64-bit vectors.
+define <1 x double> @fsub_v1f64(<1 x double> %op1, <1 x double> %op2) #0 {
+; CHECK-LABEL: @fsub_v1f64
+; CHECK: fsub d0, d0, d1
+; CHECK: ret
+  %res = fsub <1 x double> %op1, %op2
+  ret <1 x double> %res
+}
+
+; Don't use SVE for 128-bit vectors.
+define <2 x double> @fsub_v2f64(<2 x double> %op1, <2 x double> %op2) #0 {
+; CHECK-LABEL: @fsub_v2f64
+; CHECK: fsub v0.2d, v0.2d, v1.2d
+; CHECK: ret
+  %res = fsub <2 x double> %op1, %op2
+  ret <2 x double> %res
+}
+
+define void @fsub_v4f64(<4 x double>* %a, <4 x double>* %b) #0 {
+; CHECK-LABEL: @fsub_v4f64
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),4)]]
+; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
+; CHECK: fsub [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <4 x double>, <4 x double>* %a
+  %op2 = load <4 x double>, <4 x double>* %b
+  %res = fsub <4 x double> %op1, %op2
+  store <4 x double> %res, <4 x double>* %a
+  ret void
+}
+
+define void @fsub_v8f64(<8 x double>* %a, <8 x double>* %b) #0 {
+; CHECK-LABEL: @fsub_v8f64
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),8)]]
+; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
+; CHECK: fsub [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <8 x double>, <8 x double>* %a
+  %op2 = load <8 x double>, <8 x double>* %b
+  %res = fsub <8 x double> %op1, %op2
+  store <8 x double> %res, <8 x double>* %a
+  ret void
+}
+
+define void @fsub_v16f64(<16 x double>* %a, <16 x double>* %b) #0 {
+; CHECK-LABEL: @fsub_v16f64
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),16)]]
+; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
+; CHECK: fsub [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <16 x double>, <16 x double>* %a
+  %op2 = load <16 x double>, <16 x double>* %b
+  %res = fsub <16 x double> %op1, %op2
+  store <16 x double> %res, <16 x double>* %a
+  ret void
+}
+
+define void @fsub_v32f64(<32 x double>* %a, <32 x double>* %b) #0 {
+; CHECK-LABEL: @fsub_v32f64
+; CHECK: ptrue [[PG:p[0-9]+]].d, vl[[#min(div(VBYTES,8),32)]]
+; CHECK-DAG: ld1d { [[OP1:z[0-9]+]].d }, [[PG]]/z, [x0]
+; CHECK-DAG: ld1d { [[OP2:z[0-9]+]].d }, [[PG]]/z, [x1]
+; CHECK: fsub [[RES:z[0-9]+]].d, [[PG]]/m, [[OP1]].d, [[OP2]].d
+; CHECK: st1d { [[RES]].d }, [[PG]], [x0]
+; CHECK: ret
+  %op1 = load <32 x double>, <32 x double>* %a
+  %op2 = load <32 x double>, <32 x double>* %b
+  %res = fsub <32 x double> %op1, %op2
+  store <32 x double> %res, <32 x double>* %a
+  ret void
+}
+
+attributes #0 = { "target-features"="+sve" }
+
+declare <4 x half> @llvm.fma.v4f16(<4 x half>, <4 x half>, <4 x half>)
+declare <8 x half> @llvm.fma.v8f16(<8 x half>, <8 x half>, <8 x half>)
+declare <16 x half> @llvm.fma.v16f16(<16 x half>, <16 x half>, <16 x half>)
+declare <32 x half> @llvm.fma.v32f16(<32 x half>, <32 x half>, <32 x half>)
+declare <64 x half> @llvm.fma.v64f16(<64 x half>, <64 x half>, <64 x half>)
+declare <128 x half> @llvm.fma.v128f16(<128 x half>, <128 x half>, <128 x half>)
+declare <2 x float> @llvm.fma.v2f32(<2 x float>, <2 x float>, <2 x float>)
+declare <4 x float> @llvm.fma.v4f32(<4 x float>, <4 x float>, <4 x float>)
+declare <8 x float> @llvm.fma.v8f32(<8 x float>, <8 x float>, <8 x float>)
+declare <16 x float> @llvm.fma.v16f32(<16 x float>, <16 x float>, <16 x float>)
+declare <32 x float> @llvm.fma.v32f32(<32 x float>, <32 x float>, <32 x float>)
+declare <64 x float> @llvm.fma.v64f32(<64 x float>, <64 x float>, <64 x float>)
+declare <1 x double> @llvm.fma.v1f64(<1 x double>, <1 x double>, <1 x double>)
+declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>)
+declare <4 x double> @llvm.fma.v4f64(<4 x double>, <4 x double>, <4 x double>)
+declare <8 x double> @llvm.fma.v8f64(<8 x double>, <8 x double>, <8 x double>)
+declare <16 x double> @llvm.fma.v16f64(<16 x double>, <16 x double>, <16 x double>)
+declare <32 x double> @llvm.fma.v32f64(<32 x double>, <32 x double>, <32 x double>)
diff --git a/llvm/test/CodeGen/AArch64/sve-trunc.ll b/llvm/test/CodeGen/AArch64/sve-trunc.ll
index 876003a3962c6..46d152bbf7acb 100644
--- a/llvm/test/CodeGen/AArch64/sve-trunc.ll
+++ b/llvm/test/CodeGen/AArch64/sve-trunc.ll
@@ -59,3 +59,132 @@ entry:
   %out = trunc <vscale x 2 x i64> %in to <vscale x 2 x i32>
   ret <vscale x 2 x i32> %out
 }
+
+; Truncating to i1 requires convert it to a cmp
+
+define <vscale x 2 x i1> @trunc_i64toi1(<vscale x 2 x i64> %in) {
+; CHECK-LABEL: trunc_i64toi1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    and z0.d, z0.d, #0x1
+; CHECK-NEXT:    cmpne p0.d, p0/z, z0.d, #0
+; CHECK-NEXT:    ret
+entry:
+  %out = trunc <vscale x 2 x i64> %in to <vscale x 2 x i1>
+  ret <vscale x 2 x i1> %out
+}
+
+define <vscale x 4 x i1> @trunc_i64toi1_split(<vscale x 4 x i64> %in) {
+; CHECK-LABEL: trunc_i64toi1_split:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    and z1.d, z1.d, #0x1
+; CHECK-NEXT:    and z0.d, z0.d, #0x1
+; CHECK-NEXT:    cmpne p1.d, p0/z, z1.d, #0
+; CHECK-NEXT:    cmpne p0.d, p0/z, z0.d, #0
+; CHECK-NEXT:    uzp1 p0.s, p0.s, p1.s
+; CHECK-NEXT:    ret
+entry:
+  %out = trunc <vscale x 4 x i64> %in to <vscale x 4 x i1>
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 8 x i1> @trunc_i64toi1_split2(<vscale x 8 x i64> %in) {
+; CHECK-LABEL: trunc_i64toi1_split2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    and z3.d, z3.d, #0x1
+; CHECK-NEXT:    and z2.d, z2.d, #0x1
+; CHECK-NEXT:    and z1.d, z1.d, #0x1
+; CHECK-NEXT:    and z0.d, z0.d, #0x1
+; CHECK-NEXT:    cmpne p1.d, p0/z, z3.d, #0
+; CHECK-NEXT:    cmpne p2.d, p0/z, z2.d, #0
+; CHECK-NEXT:    uzp1 p1.s, p2.s, p1.s
+; CHECK-NEXT:    cmpne p2.d, p0/z, z1.d, #0
+; CHECK-NEXT:    cmpne p0.d, p0/z, z0.d, #0
+; CHECK-NEXT:    uzp1 p0.s, p0.s, p2.s
+; CHECK-NEXT:    uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT:    ret
+entry:
+  %out = trunc <vscale x 8 x i64> %in to <vscale x 8 x i1>
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 16 x i1> @trunc_i64toi1_split3(<vscale x 16 x i64> %in) {
+; CHECK-LABEL: trunc_i64toi1_split3:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset p4, -16
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    and z7.d, z7.d, #0x1
+; CHECK-NEXT:    and z6.d, z6.d, #0x1
+; CHECK-NEXT:    and z5.d, z5.d, #0x1
+; CHECK-NEXT:    and z4.d, z4.d, #0x1
+; CHECK-NEXT:    and z3.d, z3.d, #0x1
+; CHECK-NEXT:    and z2.d, z2.d, #0x1
+; CHECK-NEXT:    cmpne p1.d, p0/z, z7.d, #0
+; CHECK-NEXT:    cmpne p2.d, p0/z, z6.d, #0
+; CHECK-NEXT:    cmpne p3.d, p0/z, z5.d, #0
+; CHECK-NEXT:    cmpne p4.d, p0/z, z4.d, #0
+; CHECK-NEXT:    and z1.d, z1.d, #0x1
+; CHECK-NEXT:    and z0.d, z0.d, #0x1
+; CHECK-NEXT:    uzp1 p1.s, p2.s, p1.s
+; CHECK-NEXT:    cmpne p2.d, p0/z, z3.d, #0
+; CHECK-NEXT:    uzp1 p3.s, p4.s, p3.s
+; CHECK-NEXT:    cmpne p4.d, p0/z, z2.d, #0
+; CHECK-NEXT:    uzp1 p2.s, p4.s, p2.s
+; CHECK-NEXT:    cmpne p4.d, p0/z, z1.d, #0
+; CHECK-NEXT:    cmpne p0.d, p0/z, z0.d, #0
+; CHECK-NEXT:    uzp1 p0.s, p0.s, p4.s
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    uzp1 p1.h, p3.h, p1.h
+; CHECK-NEXT:    uzp1 p0.h, p0.h, p2.h
+; CHECK-NEXT:    uzp1 p0.b, p0.b, p1.b
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %out = trunc <vscale x 16 x i64> %in to <vscale x 16 x i1>
+  ret <vscale x 16 x i1> %out
+}
+
+
+define <vscale x 4 x i1> @trunc_i32toi1(<vscale x 4 x i32> %in) {
+; CHECK-LABEL: trunc_i32toi1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    and z0.s, z0.s, #0x1
+; CHECK-NEXT:    cmpne p0.s, p0/z, z0.s, #0
+; CHECK-NEXT:    ret
+entry:
+  %out = trunc <vscale x 4 x i32> %in to <vscale x 4 x i1>
+  ret <vscale x 4 x i1> %out
+}
+
+define <vscale x 8 x i1> @trunc_i16toi1(<vscale x 8 x i16> %in) {
+; CHECK-LABEL: trunc_i16toi1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    and z0.h, z0.h, #0x1
+; CHECK-NEXT:    cmpne p0.h, p0/z, z0.h, #0
+; CHECK-NEXT:    ret
+entry:
+  %out = trunc <vscale x 8 x i16> %in to <vscale x 8 x i1>
+  ret <vscale x 8 x i1> %out
+}
+
+define <vscale x 16 x i1> @trunc_i8toi1(<vscale x 16 x i8> %in) {
+; CHECK-LABEL: trunc_i8toi1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    and z0.b, z0.b, #0x1
+; CHECK-NEXT:    cmpne p0.b, p0/z, z0.b, #0
+; CHECK-NEXT:    ret
+entry:
+  %out = trunc <vscale x 16 x i8> %in to <vscale x 16 x i1>
+  ret <vscale x 16 x i1> %out
+}
diff --git a/llvm/test/CodeGen/AArch64/win64-jumptable.ll b/llvm/test/CodeGen/AArch64/win64-jumptable.ll
index 6a9752687aaf5..5a89c23908183 100644
--- a/llvm/test/CodeGen/AArch64/win64-jumptable.ll
+++ b/llvm/test/CodeGen/AArch64/win64-jumptable.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -o - %s -mtriple=aarch64-windows -aarch64-enable-compress-jump-tables=0 | FileCheck %s
-; RUN: llc -o - %s -mtriple=aarch64-windows -aarch64-enable-compress-jump-tables=0 -filetype=obj | llvm-readobj -unwind | FileCheck %s -check-prefix=UNWIND
+; RUN: llc -o - %s -mtriple=aarch64-windows -aarch64-enable-compress-jump-tables=0 -filetype=obj | llvm-readobj -unwind - | FileCheck %s -check-prefix=UNWIND
 
 define void @f(i32 %x) {
 entry:
diff --git a/llvm/test/CodeGen/AArch64/wineh-mingw.ll b/llvm/test/CodeGen/AArch64/wineh-mingw.ll
index 3e95789a3d4a9..ff1a55711b9ea 100644
--- a/llvm/test/CodeGen/AArch64/wineh-mingw.ll
+++ b/llvm/test/CodeGen/AArch64/wineh-mingw.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s -mtriple=aarch64-pc-mingw32 | FileCheck %s -check-prefix=WINEH
-; RUN: llc < %s -mtriple=aarch64-pc-mingw32 -filetype=obj | llvm-readobj -S | FileCheck %s -check-prefix=WINEH-SECTIONS
+; RUN: llc < %s -mtriple=aarch64-pc-mingw32 -filetype=obj | llvm-readobj -S - | FileCheck %s -check-prefix=WINEH-SECTIONS
 
 ; Check emission of eh handler and handler data
 declare i32 @_d_eh_personality(i32, i32, i64, i8*, i8*)
diff --git a/llvm/test/CodeGen/AArch64/wineh1.mir b/llvm/test/CodeGen/AArch64/wineh1.mir
index e468d7f0b8280..deaff395ca69f 100644
--- a/llvm/test/CodeGen/AArch64/wineh1.mir
+++ b/llvm/test/CodeGen/AArch64/wineh1.mir
@@ -1,5 +1,5 @@
 # RUN: llc -o - %s -mtriple=aarch64-windows -start-after=prologepilog -filetype=obj  \
-# RUN:   | llvm-readobj --unwind | FileCheck %s
+# RUN:   | llvm-readobj --unwind - | FileCheck %s
 # RUN: llc -o - %s -mtriple=aarch64-windows -run-pass=aarch64-ldst-opt \
 # RUN:   | FileCheck %s --check-prefix=CHECK-LDSTOPT
 # This test case checks the basic validity of the .xdata section.  It's
diff --git a/llvm/test/CodeGen/AArch64/wineh2.mir b/llvm/test/CodeGen/AArch64/wineh2.mir
index aa78ee7b93bd4..79d022df26923 100644
--- a/llvm/test/CodeGen/AArch64/wineh2.mir
+++ b/llvm/test/CodeGen/AArch64/wineh2.mir
@@ -1,5 +1,5 @@
 # RUN: llc -o - %s -mtriple=aarch64-windows -start-after=prologepilog \
-# RUN:    -filetype=obj | llvm-readobj --unwind | FileCheck %s
+# RUN:    -filetype=obj | llvm-readobj --unwind - | FileCheck %s
 # Test that the pre/post increment save of a flating point register is correct.
 
 # CHECK:        ExceptionData {
diff --git a/llvm/test/CodeGen/AArch64/wineh3.mir b/llvm/test/CodeGen/AArch64/wineh3.mir
index 82f65eac457cb..6cbe7f42dc5ec 100644
--- a/llvm/test/CodeGen/AArch64/wineh3.mir
+++ b/llvm/test/CodeGen/AArch64/wineh3.mir
@@ -1,5 +1,5 @@
 # RUN: llc -o - %s -mtriple=aarch64-windows -start-after=prologepilog \
-# RUN:    -filetype=obj | llvm-readobj --unwind | FileCheck %s
+# RUN:    -filetype=obj | llvm-readobj --unwind - | FileCheck %s
 # Test that the register pairing of both general purpose and floating point
 # registers is correctly saved in the .xdata section, as well as the pre/post
 # increment of floating point register pairs.
diff --git a/llvm/test/CodeGen/AArch64/wineh4.mir b/llvm/test/CodeGen/AArch64/wineh4.mir
index 50c8bf1b770ef..c007671a83c80 100644
--- a/llvm/test/CodeGen/AArch64/wineh4.mir
+++ b/llvm/test/CodeGen/AArch64/wineh4.mir
@@ -1,6 +1,6 @@
 # RUN: llc -o - %s -mtriple=aarch64-windows -start-after=prologepilog \
 # RUN:   -disable-branch-fold  -filetype=obj \
-# RUN: | llvm-readobj --unwind | FileCheck %s
+# RUN: | llvm-readobj --unwind - | FileCheck %s
 # Check that identical multiple epilgoues are correctly shared in .xdata.
 
 # CHECK:        ExceptionData {
diff --git a/llvm/test/CodeGen/AArch64/wineh5.mir b/llvm/test/CodeGen/AArch64/wineh5.mir
index b81a847497496..1280997fbb3c4 100644
--- a/llvm/test/CodeGen/AArch64/wineh5.mir
+++ b/llvm/test/CodeGen/AArch64/wineh5.mir
@@ -1,5 +1,5 @@
 # RUN: llc -o - %s -mtriple=aarch64-windows -start-after=prologepilog \
-# RUN:    -filetype=obj | llvm-readobj --unwind | FileCheck %s
+# RUN:    -filetype=obj | llvm-readobj --unwind - | FileCheck %s
 
 # Check that that the large stack allocation is correctly represented in .xdata.
 
diff --git a/llvm/test/CodeGen/AArch64/wineh6.mir b/llvm/test/CodeGen/AArch64/wineh6.mir
index bff09cba626f3..3ea7c0f20d45c 100644
--- a/llvm/test/CodeGen/AArch64/wineh6.mir
+++ b/llvm/test/CodeGen/AArch64/wineh6.mir
@@ -1,5 +1,5 @@
 # RUN: llc -o - %s -mtriple=aarch64-windows -start-after=prologepilog \
-# RUN:    -filetype=obj | llvm-readobj --unwind | FileCheck %s
+# RUN:    -filetype=obj | llvm-readobj --unwind - | FileCheck %s
 # Check save_fplr_x, set_fp, alloc_s
 
 # CHECK: 	ExceptionData {
diff --git a/llvm/test/CodeGen/AArch64/wineh7.mir b/llvm/test/CodeGen/AArch64/wineh7.mir
index 040ab61f916df..c445cbfd6b005 100644
--- a/llvm/test/CodeGen/AArch64/wineh7.mir
+++ b/llvm/test/CodeGen/AArch64/wineh7.mir
@@ -1,5 +1,5 @@
 # RUN: llc -o - %s -mtriple=aarch64-windows -start-after=prologepilog \
-# RUN:   -filetype=obj  | llvm-readobj --unwind | FileCheck %s
+# RUN:   -filetype=obj  | llvm-readobj --unwind - | FileCheck %s
 # Check AddFP
 
 # CHECK:	 ExceptionData {
diff --git a/llvm/test/CodeGen/AArch64/wineh8.mir b/llvm/test/CodeGen/AArch64/wineh8.mir
index 5b56300b1bc6f..9a37ed760ad88 100644
--- a/llvm/test/CodeGen/AArch64/wineh8.mir
+++ b/llvm/test/CodeGen/AArch64/wineh8.mir
@@ -1,6 +1,6 @@
 # RUN: llc -o - %s -mtriple=aarch64-windows -start-after=prologepilog \
 # RUN:   -disable-branch-fold  -filetype=obj \
-# RUN: | llvm-readobj --unwind | FileCheck %s
+# RUN: | llvm-readobj --unwind - | FileCheck %s
 # Check that non-identical multiple epilgoues are correctly shared in .xdata.
 
 # CHECK:        ExceptionData {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ptrmask.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ptrmask.mir
index 37c94f8bea635..bbb885c705ed6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ptrmask.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ptrmask.mir
@@ -337,31 +337,6 @@ body: |
 
 ...
 
----
-name:  ptrmask_p0_s32_sgpr_sgpr_sgpr
-legalized:       true
-regBankSelected: true
-
-body: |
-  bb.0:
-    liveins: $sgpr0_sgpr1, $sgpr2_sgpr3
-
-    ; CHECK-LABEL: name: ptrmask_p0_s32_sgpr_sgpr_sgpr
-    ; CHECK: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
-    ; CHECK: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
-    ; CHECK: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[COPY]].sub0
-    ; CHECK: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[COPY]].sub1
-    ; CHECK: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY2]], [[COPY1]], implicit-def $scc
-    ; CHECK: [[S_AND_B32_1:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY3]], [[COPY1]], implicit-def $scc
-    ; CHECK: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_AND_B32_]], %subreg.sub0, [[S_AND_B32_1]], %subreg.sub1
-    ; CHECK: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
-    %0:sgpr(p0) = COPY $sgpr0_sgpr1
-    %1:sgpr(s32) = COPY $sgpr2
-    %2:sgpr(p0) = G_PTRMASK %0, %1
-    S_ENDPGM 0, implicit %2
-
-...
-
 ---
 name:  ptrmask_p0_s64_sgpr_sgpr_clearhi1
 legalized:       true
@@ -756,55 +731,6 @@ body: |
 
 ...
 
----
-name:  ptrmask_p0_s32_vgpr_vgpr_vgpr
-legalized:       true
-regBankSelected: true
-
-body: |
-  bb.0:
-    liveins: $vgpr0_vgpr1, $vgpr2
-
-    ; CHECK-LABEL: name: ptrmask_p0_s32_vgpr_vgpr_vgpr
-    ; CHECK: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
-    ; CHECK: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-    ; CHECK: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
-    ; CHECK: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
-    ; CHECK: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY2]], [[COPY1]], implicit $exec
-    ; CHECK: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY3]], [[COPY1]], implicit $exec
-    ; CHECK: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_AND_B32_e64_]], %subreg.sub0, [[V_AND_B32_e64_1]], %subreg.sub1
-    ; CHECK: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
-    %0:vgpr(p0) = COPY $vgpr0_vgpr1
-    %1:vgpr(s32) = COPY $vgpr2
-    %2:vgpr(p0) = G_PTRMASK %0, %1
-    S_ENDPGM 0, implicit %2
-
-...
-
----
-name:  ptrmask_p0_s32_vgpr_vgpr_vgpr_0xffffffff
-legalized:       true
-regBankSelected: true
-
-body: |
-  bb.0:
-    liveins: $vgpr0_vgpr1, $vgpr2
-
-    ; CHECK-LABEL: name: ptrmask_p0_s32_vgpr_vgpr_vgpr_0xffffffff
-    ; CHECK: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
-    ; CHECK: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294967295, implicit $exec
-    ; CHECK: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
-    ; CHECK: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1
-    ; CHECK: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY2]], [[V_MOV_B32_e32_]], implicit $exec
-    ; CHECK: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[V_AND_B32_e64_]], %subreg.sub1
-    ; CHECK: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
-    %0:vgpr(p0) = COPY $vgpr0_vgpr1
-    %1:vgpr(s32) = G_CONSTANT i32 -1
-    %2:vgpr(p0) = G_PTRMASK %0, %1
-    S_ENDPGM 0, implicit %2
-
-...
-
 ---
 name:  ptrmask_p0_s64_vgpr_vgpr_clearlo1
 legalized:       true
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_kernel.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_kernel.ll
index 76a6f1732102b..eda521cce827e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_kernel.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_kernel.ll
@@ -1265,3 +1265,467 @@ define amdgpu_kernel void @unused_i32_arg(i32 addrspace(1)* nocapture %out, i32
 entry:
   ret void
 }
+
+; Byref pointers should only be treated as offsets from kernarg
+define amdgpu_kernel void @byref_constant_i8_arg(i32 addrspace(1)* nocapture %out, i8 addrspace(4)* byref(i8) %in.byref) {
+  ; HSA-VI-LABEL: name: byref_constant_i8_arg
+  ; HSA-VI: bb.1 (%ir-block.0):
+  ; HSA-VI:   liveins: $sgpr4_sgpr5
+  ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
+  ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; HSA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
+  ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; HSA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[PTR_ADD1]](p4) :: (load 1 from %ir.in.byref, addrspace 4)
+  ; HSA-VI:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD1]](s8)
+  ; HSA-VI:   G_STORE [[ZEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
+  ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: byref_constant_i8_arg
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[PTR_ADD1]](p4) :: (load 1 from %ir.in.byref, addrspace 4)
+  ; LEGACY-MESA-VI:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD1]](s8)
+  ; LEGACY-MESA-VI:   G_STORE [[ZEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
+  %in = load i8, i8 addrspace(4)* %in.byref
+  %ext = zext i8 %in to i32
+  store i32 %ext, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @byref_constant_i16_arg(i32 addrspace(1)* nocapture %out, i16 addrspace(4)* byref(i16) align 2 %in.byref) {
+  ; HSA-VI-LABEL: name: byref_constant_i16_arg
+  ; HSA-VI: bb.1 (%ir-block.0):
+  ; HSA-VI:   liveins: $sgpr4_sgpr5
+  ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
+  ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; HSA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
+  ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; HSA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s16) = G_LOAD [[PTR_ADD1]](p4) :: (load 2 from %ir.in.byref, addrspace 4)
+  ; HSA-VI:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD1]](s16)
+  ; HSA-VI:   G_STORE [[ZEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
+  ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: byref_constant_i16_arg
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s16) = G_LOAD [[PTR_ADD1]](p4) :: (load 2 from %ir.in.byref, addrspace 4)
+  ; LEGACY-MESA-VI:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD1]](s16)
+  ; LEGACY-MESA-VI:   G_STORE [[ZEXT]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
+  %in = load i16, i16 addrspace(4)* %in.byref
+  %ext = zext i16 %in to i32
+  store i32 %ext, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @byref_constant_i32_arg(i32 addrspace(1)* nocapture %out, i32 addrspace(4)* byref(i32) align 4 %in.byref, i32 %after.offset) {
+  ; HSA-VI-LABEL: name: byref_constant_i32_arg
+  ; HSA-VI: bb.1 (%ir-block.0):
+  ; HSA-VI:   liveins: $sgpr4_sgpr5
+  ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
+  ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; HSA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
+  ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; HSA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
+  ; HSA-VI:   [[PTR_ADD2:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C2]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p4) :: (dereferenceable invariant load 4, addrspace 4)
+  ; HSA-VI:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p4) :: (load 4 from %ir.in.byref, addrspace 4)
+  ; HSA-VI:   G_STORE [[LOAD2]](s32), [[LOAD]](p1) :: (volatile store 4 into %ir.out, addrspace 1)
+  ; HSA-VI:   G_STORE [[LOAD1]](s32), [[LOAD]](p1) :: (volatile store 4 into %ir.out, addrspace 1)
+  ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: byref_constant_i32_arg
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+  ; LEGACY-MESA-VI:   [[PTR_ADD2:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C2]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p4) :: (dereferenceable invariant load 4, align 16, addrspace 4)
+  ; LEGACY-MESA-VI:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p4) :: (load 4 from %ir.in.byref, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD2]](s32), [[LOAD]](p1) :: (volatile store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](s32), [[LOAD]](p1) :: (volatile store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
+  %in = load i32, i32 addrspace(4)* %in.byref
+  store volatile i32 %in, i32 addrspace(1)* %out, align 4
+  store volatile i32 %after.offset, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @byref_constant_v4i32_arg(<4 x i32> addrspace(1)* nocapture %out, <4 x i32> addrspace(4)* byref(<4 x i32>) align(16) %in.byref, i32 %after.offset) {
+  ; HSA-VI-LABEL: name: byref_constant_v4i32_arg
+  ; HSA-VI: bb.1 (%ir-block.0):
+  ; HSA-VI:   liveins: $sgpr4_sgpr5
+  ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
+  ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; HSA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
+  ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+  ; HSA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
+  ; HSA-VI:   [[PTR_ADD2:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C2]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p4) :: (dereferenceable invariant load 4, align 16, addrspace 4)
+  ; HSA-VI:   [[LOAD2:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[PTR_ADD1]](p4) :: (load 16 from %ir.in.byref, addrspace 4)
+  ; HSA-VI:   G_STORE [[LOAD2]](<4 x s32>), [[LOAD]](p1) :: (volatile store 16 into %ir.out, align 4, addrspace 1)
+  ; HSA-VI:   G_STORE [[LOAD1]](s32), [[LOAD]](p1) :: (volatile store 4 into %ir.out.cast, addrspace 1)
+  ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: byref_constant_v4i32_arg
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 52
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 68
+  ; LEGACY-MESA-VI:   [[PTR_ADD2:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C2]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p4) :: (dereferenceable invariant load 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[LOAD2:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[PTR_ADD1]](p4) :: (load 16 from %ir.in.byref, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD2]](<4 x s32>), [[LOAD]](p1) :: (volatile store 16 into %ir.out, align 4, addrspace 1)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](s32), [[LOAD]](p1) :: (volatile store 4 into %ir.out.cast, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
+  %in = load <4 x i32>, <4 x i32> addrspace(4)* %in.byref
+  store volatile <4 x i32> %in, <4 x i32> addrspace(1)* %out, align 4
+  %out.cast = bitcast <4 x i32> addrspace(1)* %out to i32 addrspace(1)*
+  store volatile i32 %after.offset, i32 addrspace(1)* %out.cast, align 4
+  ret void
+}
+
+define amdgpu_kernel void @byref_align_constant_i32_arg(i32 addrspace(1)* nocapture %out, i32 addrspace(4)* byref(i32) align(256) %in.byref, i32 %after.offset) {
+  ; HSA-VI-LABEL: name: byref_align_constant_i32_arg
+  ; HSA-VI: bb.1 (%ir-block.0):
+  ; HSA-VI:   liveins: $sgpr4_sgpr5
+  ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
+  ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; HSA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
+  ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 256
+  ; HSA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 260
+  ; HSA-VI:   [[PTR_ADD2:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C2]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p4) :: (dereferenceable invariant load 4, addrspace 4)
+  ; HSA-VI:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p4) :: (load 4 from %ir.in.byref, addrspace 4)
+  ; HSA-VI:   G_STORE [[LOAD2]](s32), [[LOAD]](p1) :: (volatile store 4 into %ir.out, addrspace 1)
+  ; HSA-VI:   G_STORE [[LOAD1]](s32), [[LOAD]](p1) :: (volatile store 4 into %ir.out, addrspace 1)
+  ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: byref_align_constant_i32_arg
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 292
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 296
+  ; LEGACY-MESA-VI:   [[PTR_ADD2:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C2]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p4) :: (dereferenceable invariant load 4, align 8, addrspace 4)
+  ; LEGACY-MESA-VI:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p4) :: (load 4 from %ir.in.byref, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD2]](s32), [[LOAD]](p1) :: (volatile store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](s32), [[LOAD]](p1) :: (volatile store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
+  %in = load i32, i32 addrspace(4)* %in.byref
+  store volatile i32 %in, i32 addrspace(1)* %out, align 4
+  store volatile i32 %after.offset, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @byref_natural_align_constant_v16i32_arg(i32 addrspace(1)* nocapture %out, i8, <16 x i32> addrspace(4)* byref(<16 x i32>) align(64) %in.byref, i32 %after.offset) {
+  ; HSA-VI-LABEL: name: byref_natural_align_constant_v16i32_arg
+  ; HSA-VI: bb.1 (%ir-block.1):
+  ; HSA-VI:   liveins: $sgpr4_sgpr5
+  ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
+  ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; HSA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
+  ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
+  ; HSA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 128
+  ; HSA-VI:   [[PTR_ADD2:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C2]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p4) :: (dereferenceable invariant load 4, align 16, addrspace 4)
+  ; HSA-VI:   [[LOAD2:%[0-9]+]]:_(<16 x s32>) = G_LOAD [[PTR_ADD1]](p4) :: (load 64 from %ir.in.byref, addrspace 4)
+  ; HSA-VI:   G_STORE [[LOAD2]](<16 x s32>), [[LOAD]](p1) :: (volatile store 64 into %ir.cast.out, align 4, addrspace 1)
+  ; HSA-VI:   G_STORE [[LOAD1]](s32), [[LOAD]](p1) :: (volatile store 4 into %ir.out, addrspace 1)
+  ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: byref_natural_align_constant_v16i32_arg
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.1):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 100
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 164
+  ; LEGACY-MESA-VI:   [[PTR_ADD2:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C2]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p4) :: (dereferenceable invariant load 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[LOAD2:%[0-9]+]]:_(<16 x s32>) = G_LOAD [[PTR_ADD1]](p4) :: (load 64 from %ir.in.byref, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD2]](<16 x s32>), [[LOAD]](p1) :: (volatile store 64 into %ir.cast.out, align 4, addrspace 1)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](s32), [[LOAD]](p1) :: (volatile store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
+  %in = load <16 x i32>, <16 x i32> addrspace(4)* %in.byref
+  %cast.out = bitcast i32 addrspace(1)* %out to <16 x i32> addrspace(1)*
+  store volatile <16 x i32> %in, <16 x i32> addrspace(1)* %cast.out, align 4
+  store volatile i32 %after.offset, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; Also accept byref kernel arguments with other global address spaces.
+define amdgpu_kernel void @byref_global_i32_arg(i32 addrspace(1)* nocapture %out, i32 addrspace(1)* byref(i32) align(4) %in.byref) {
+  ; HSA-VI-LABEL: name: byref_global_i32_arg
+  ; HSA-VI: bb.1 (%ir-block.0):
+  ; HSA-VI:   liveins: $sgpr4_sgpr5
+  ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
+  ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; HSA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
+  ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; HSA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[ADDRSPACE_CAST:%[0-9]+]]:_(p1) = G_ADDRSPACE_CAST [[PTR_ADD1]](p4)
+  ; HSA-VI:   [[COPY1:%[0-9]+]]:_(p1) = COPY [[ADDRSPACE_CAST]](p1)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[COPY1]](p1) :: (load 4 from %ir.1, addrspace 1)
+  ; HSA-VI:   G_STORE [[LOAD1]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
+  ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: byref_global_i32_arg
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[ADDRSPACE_CAST:%[0-9]+]]:_(p1) = G_ADDRSPACE_CAST [[PTR_ADD1]](p4)
+  ; LEGACY-MESA-VI:   [[COPY1:%[0-9]+]]:_(p1) = COPY [[ADDRSPACE_CAST]](p1)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[COPY1]](p1) :: (load 4 from %ir.1, addrspace 1)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
+  %in = load i32, i32 addrspace(1)* %in.byref
+  store i32 %in, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @byref_flat_i32_arg(i32 addrspace(1)* nocapture %out, i32* byref(i32) align(4) %in.byref) {
+  ; HSA-VI-LABEL: name: byref_flat_i32_arg
+  ; HSA-VI: bb.1 (%ir-block.0):
+  ; HSA-VI:   liveins: $sgpr4_sgpr5
+  ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
+  ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; HSA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
+  ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; HSA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[ADDRSPACE_CAST:%[0-9]+]]:_(p0) = G_ADDRSPACE_CAST [[PTR_ADD1]](p4)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[ADDRSPACE_CAST]](p0) :: (load 4 from %ir.in.byref)
+  ; HSA-VI:   G_STORE [[LOAD1]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
+  ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: byref_flat_i32_arg
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[ADDRSPACE_CAST:%[0-9]+]]:_(p0) = G_ADDRSPACE_CAST [[PTR_ADD1]](p4)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[ADDRSPACE_CAST]](p0) :: (load 4 from %ir.in.byref)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
+  %in = load i32, i32* %in.byref
+  store i32 %in, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @byref_constant_32bit_i32_arg(i32 addrspace(1)* nocapture %out, i32 addrspace(6)* byref(i32) align(4) %in.byref) {
+  ; HSA-VI-LABEL: name: byref_constant_32bit_i32_arg
+  ; HSA-VI: bb.1 (%ir-block.0):
+  ; HSA-VI:   liveins: $sgpr4_sgpr5
+  ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
+  ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; HSA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
+  ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; HSA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[ADDRSPACE_CAST:%[0-9]+]]:_(p6) = G_ADDRSPACE_CAST [[PTR_ADD1]](p4)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[ADDRSPACE_CAST]](p6) :: (load 4 from %ir.in.byref, addrspace 6)
+  ; HSA-VI:   G_STORE [[LOAD1]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
+  ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: byref_constant_32bit_i32_arg
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[ADDRSPACE_CAST:%[0-9]+]]:_(p6) = G_ADDRSPACE_CAST [[PTR_ADD1]](p4)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[ADDRSPACE_CAST]](p6) :: (load 4 from %ir.in.byref, addrspace 6)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
+  %in = load i32, i32 addrspace(6)* %in.byref
+  store i32 %in, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @byref_unknown_as_i32_arg(i32 addrspace(1)* nocapture %out, i32 addrspace(999)* byref(i32) align(4) %in.byref) {
+  ; HSA-VI-LABEL: name: byref_unknown_as_i32_arg
+  ; HSA-VI: bb.1 (%ir-block.0):
+  ; HSA-VI:   liveins: $sgpr4_sgpr5
+  ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
+  ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; HSA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
+  ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; HSA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[ADDRSPACE_CAST:%[0-9]+]]:_(p999) = G_ADDRSPACE_CAST [[PTR_ADD1]](p4)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[ADDRSPACE_CAST]](p999) :: (load 4 from %ir.in.byref, addrspace 999)
+  ; HSA-VI:   G_STORE [[LOAD1]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
+  ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: byref_unknown_as_i32_arg
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[ADDRSPACE_CAST:%[0-9]+]]:_(p999) = G_ADDRSPACE_CAST [[PTR_ADD1]](p4)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[ADDRSPACE_CAST]](p999) :: (load 4 from %ir.in.byref, addrspace 999)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
+  %in = load i32, i32 addrspace(999)* %in.byref
+  store i32 %in, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; Invalid, but should not crash.
+define amdgpu_kernel void @byref_local_i32_arg(i32 addrspace(1)* nocapture %out, i32 addrspace(3)* byref(i32) align(4) %in.byref) {
+  ; HSA-VI-LABEL: name: byref_local_i32_arg
+  ; HSA-VI: bb.1 (%ir-block.0):
+  ; HSA-VI:   liveins: $sgpr4_sgpr5
+  ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
+  ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; HSA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
+  ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; HSA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[ADDRSPACE_CAST:%[0-9]+]]:_(p3) = G_ADDRSPACE_CAST [[PTR_ADD1]](p4)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[ADDRSPACE_CAST]](p3) :: (load 4 from %ir.in.byref, addrspace 3)
+  ; HSA-VI:   G_STORE [[LOAD1]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
+  ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: byref_local_i32_arg
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[ADDRSPACE_CAST:%[0-9]+]]:_(p3) = G_ADDRSPACE_CAST [[PTR_ADD1]](p4)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[ADDRSPACE_CAST]](p3) :: (load 4 from %ir.in.byref, addrspace 3)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](s32), [[LOAD]](p1) :: (store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
+  %in = load i32, i32 addrspace(3)* %in.byref
+  store i32 %in, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @multi_byref_constant_i32_arg(i32 addrspace(1)* nocapture %out, i32 addrspace(4)* byref(i32) align(4) %in0.byref, i32 addrspace(4)* byref(i32) align(4) %in1.byref, i32 %after.offset) {
+  ; HSA-VI-LABEL: name: multi_byref_constant_i32_arg
+  ; HSA-VI: bb.1 (%ir-block.0):
+  ; HSA-VI:   liveins: $sgpr4_sgpr5
+  ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
+  ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; HSA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 16, addrspace 4)
+  ; HSA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; HSA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; HSA-VI:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
+  ; HSA-VI:   [[PTR_ADD2:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C2]](s64)
+  ; HSA-VI:   [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+  ; HSA-VI:   [[PTR_ADD3:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C3]](s64)
+  ; HSA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p4) :: (dereferenceable invariant load 4, align 16, addrspace 4)
+  ; HSA-VI:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p4) :: (load 4 from %ir.in0.byref, addrspace 4)
+  ; HSA-VI:   [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p4) :: (load 4 from %ir.in1.byref, addrspace 4)
+  ; HSA-VI:   G_STORE [[LOAD2]](s32), [[LOAD]](p1) :: (volatile store 4 into %ir.out, addrspace 1)
+  ; HSA-VI:   G_STORE [[LOAD3]](s32), [[LOAD]](p1) :: (volatile store 4 into %ir.out, addrspace 1)
+  ; HSA-VI:   G_STORE [[LOAD1]](s32), [[LOAD]](p1) :: (volatile store 4 into %ir.out, addrspace 1)
+  ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: multi_byref_constant_i32_arg
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8, align 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; LEGACY-MESA-VI:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C1]](s64)
+  ; LEGACY-MESA-VI:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+  ; LEGACY-MESA-VI:   [[PTR_ADD2:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C2]](s64)
+  ; LEGACY-MESA-VI:   [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 52
+  ; LEGACY-MESA-VI:   [[PTR_ADD3:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C3]](s64)
+  ; LEGACY-MESA-VI:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p4) :: (dereferenceable invariant load 4, addrspace 4)
+  ; LEGACY-MESA-VI:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p4) :: (load 4 from %ir.in0.byref, addrspace 4)
+  ; LEGACY-MESA-VI:   [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p4) :: (load 4 from %ir.in1.byref, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD2]](s32), [[LOAD]](p1) :: (volatile store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD3]](s32), [[LOAD]](p1) :: (volatile store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD1]](s32), [[LOAD]](p1) :: (volatile store 4 into %ir.out, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
+  %in0 = load i32, i32 addrspace(4)* %in0.byref
+  %in1 = load i32, i32 addrspace(4)* %in1.byref
+  store volatile i32 %in0, i32 addrspace(1)* %out, align 4
+  store volatile i32 %in1, i32 addrspace(1)* %out, align 4
+  store volatile i32 %after.offset, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @byref_constant_i32_arg_offset0(i32 addrspace(4)* byref(i32) align(4) %in.byref) {
+  ; HSA-VI-LABEL: name: byref_constant_i32_arg_offset0
+  ; HSA-VI: bb.1 (%ir-block.0):
+  ; HSA-VI:   liveins: $sgpr4_sgpr5
+  ; HSA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr4_sgpr5
+  ; HSA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; HSA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; HSA-VI:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; HSA-VI:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p4) :: (load 4 from %ir.in.byref, addrspace 4)
+  ; HSA-VI:   G_STORE [[LOAD]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; HSA-VI:   S_ENDPGM 0
+  ; LEGACY-MESA-VI-LABEL: name: byref_constant_i32_arg_offset0
+  ; LEGACY-MESA-VI: bb.1 (%ir-block.0):
+  ; LEGACY-MESA-VI:   liveins: $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[COPY:%[0-9]+]]:_(p4) = COPY $sgpr0_sgpr1
+  ; LEGACY-MESA-VI:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; LEGACY-MESA-VI:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+  ; LEGACY-MESA-VI:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; LEGACY-MESA-VI:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p4) :: (load 4 from %ir.in.byref, addrspace 4)
+  ; LEGACY-MESA-VI:   G_STORE [[LOAD]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; LEGACY-MESA-VI:   S_ENDPGM 0
+  %in = load i32, i32 addrspace(4)* %in.byref
+  store i32 %in, i32 addrspace(1)* undef, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-implicit-args.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-implicit-args.ll
new file mode 100644
index 0000000000000..aa0850a5dbe5b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-implicit-args.ll
@@ -0,0 +1,1247 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -amdgpu-fixed-function-abi -stop-after=irtranslator -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -verify-machineinstrs -o - %s | FileCheck -enable-var-scope -check-prefix=GFX900 %s
+; RUN: llc -global-isel -amdgpu-fixed-function-abi -stop-after=irtranslator -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx908 -verify-machineinstrs -o - %s | FileCheck -enable-var-scope -check-prefix=GFX908 %s
+
+; Workitem IDs are passed to the kernel differently for gfx908
+
+declare hidden void @external_void_func_void() #0
+declare hidden void @external_void_func_i32(i32) #0
+declare hidden void @external_void_func_v32i32(<32 x i32>) #0
+
+define amdgpu_kernel void @test_call_external_void_func_i32([17 x i8]) #0 {
+  ; GFX900-LABEL: name: test_call_external_void_func_i32
+  ; GFX900: bb.1 (%ir-block.1):
+  ; GFX900:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GFX900:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GFX900:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GFX900:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GFX900:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GFX900:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GFX900:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GFX900:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GFX900:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GFX900:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GFX900:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GFX900:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
+  ; GFX900:   [[INT:%[0-9]+]]:_(p4) = G_INTRINSIC intrinsic(@llvm.amdgcn.kernarg.segment.ptr)
+  ; GFX900:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GFX900:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i32
+  ; GFX900:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GFX900:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GFX900:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GFX900:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 20
+  ; GFX900:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; GFX900:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GFX900:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GFX900:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GFX900:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GFX900:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GFX900:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GFX900:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GFX900:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; GFX900:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GFX900:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GFX900:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GFX900:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; GFX900:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GFX900:   $vgpr0 = COPY [[C]](s32)
+  ; GFX900:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GFX900:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GFX900:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GFX900:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GFX900:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GFX900:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GFX900:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GFX900:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GFX900:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GFX900:   $vgpr31 = COPY [[OR1]](s32)
+  ; GFX900:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; GFX900:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GFX900:   S_ENDPGM 0
+  ; GFX908-LABEL: name: test_call_external_void_func_i32
+  ; GFX908: bb.1 (%ir-block.1):
+  ; GFX908:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GFX908:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GFX908:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GFX908:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GFX908:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GFX908:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GFX908:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GFX908:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GFX908:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GFX908:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GFX908:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GFX908:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
+  ; GFX908:   [[INT:%[0-9]+]]:_(p4) = G_INTRINSIC intrinsic(@llvm.amdgcn.kernarg.segment.ptr)
+  ; GFX908:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GFX908:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i32
+  ; GFX908:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GFX908:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GFX908:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GFX908:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 20
+  ; GFX908:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; GFX908:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GFX908:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GFX908:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GFX908:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GFX908:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GFX908:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GFX908:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GFX908:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; GFX908:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GFX908:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GFX908:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GFX908:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; GFX908:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GFX908:   $vgpr0 = COPY [[C]](s32)
+  ; GFX908:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GFX908:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GFX908:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GFX908:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GFX908:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GFX908:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GFX908:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GFX908:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GFX908:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GFX908:   $vgpr31 = COPY [[OR1]](s32)
+  ; GFX908:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; GFX908:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GFX908:   S_ENDPGM 0
+  call void @external_void_func_i32(i32 42)
+  ret void
+}
+
+define void @test_func_call_external_void_func_i32() #0 {
+  ; GFX900-LABEL: name: test_func_call_external_void_func_i32
+  ; GFX900: bb.1 (%ir-block.0):
+  ; GFX900:   liveins: $sgpr12, $sgpr13, $sgpr14, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11, $sgpr30_sgpr31
+  ; GFX900:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr31
+  ; GFX900:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GFX900:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr13
+  ; GFX900:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr12
+  ; GFX900:   [[COPY4:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GFX900:   [[COPY5:%[0-9]+]]:sgpr_64 = COPY $sgpr8_sgpr9
+  ; GFX900:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GFX900:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GFX900:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; GFX900:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 99
+  ; GFX900:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GFX900:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i32
+  ; GFX900:   [[COPY9:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GFX900:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY6]]
+  ; GFX900:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY5]]
+  ; GFX900:   [[COPY12:%[0-9]+]]:_(s64) = COPY [[COPY4]]
+  ; GFX900:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GFX900:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY2]]
+  ; GFX900:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY1]]
+  ; GFX900:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GFX900:   $vgpr0 = COPY [[C]](s32)
+  ; GFX900:   [[COPY17:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX900:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY17]](<4 x s32>)
+  ; GFX900:   $sgpr4_sgpr5 = COPY [[COPY9]](p4)
+  ; GFX900:   $sgpr6_sgpr7 = COPY [[COPY10]](p4)
+  ; GFX900:   $sgpr8_sgpr9 = COPY [[COPY11]](p4)
+  ; GFX900:   $sgpr10_sgpr11 = COPY [[COPY12]](s64)
+  ; GFX900:   $sgpr12 = COPY [[COPY13]](s32)
+  ; GFX900:   $sgpr13 = COPY [[COPY14]](s32)
+  ; GFX900:   $sgpr14 = COPY [[COPY15]](s32)
+  ; GFX900:   $vgpr31 = COPY [[COPY16]](s32)
+  ; GFX900:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; GFX900:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GFX900:   [[COPY18:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY8]]
+  ; GFX900:   S_SETPC_B64_return [[COPY18]]
+  ; GFX908-LABEL: name: test_func_call_external_void_func_i32
+  ; GFX908: bb.1 (%ir-block.0):
+  ; GFX908:   liveins: $sgpr12, $sgpr13, $sgpr14, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11, $sgpr30_sgpr31
+  ; GFX908:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr31
+  ; GFX908:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GFX908:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr13
+  ; GFX908:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr12
+  ; GFX908:   [[COPY4:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GFX908:   [[COPY5:%[0-9]+]]:sgpr_64 = COPY $sgpr8_sgpr9
+  ; GFX908:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GFX908:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GFX908:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; GFX908:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 99
+  ; GFX908:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GFX908:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i32
+  ; GFX908:   [[COPY9:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GFX908:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY6]]
+  ; GFX908:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY5]]
+  ; GFX908:   [[COPY12:%[0-9]+]]:_(s64) = COPY [[COPY4]]
+  ; GFX908:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GFX908:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY2]]
+  ; GFX908:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY1]]
+  ; GFX908:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GFX908:   $vgpr0 = COPY [[C]](s32)
+  ; GFX908:   [[COPY17:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX908:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY17]](<4 x s32>)
+  ; GFX908:   $sgpr4_sgpr5 = COPY [[COPY9]](p4)
+  ; GFX908:   $sgpr6_sgpr7 = COPY [[COPY10]](p4)
+  ; GFX908:   $sgpr8_sgpr9 = COPY [[COPY11]](p4)
+  ; GFX908:   $sgpr10_sgpr11 = COPY [[COPY12]](s64)
+  ; GFX908:   $sgpr12 = COPY [[COPY13]](s32)
+  ; GFX908:   $sgpr13 = COPY [[COPY14]](s32)
+  ; GFX908:   $sgpr14 = COPY [[COPY15]](s32)
+  ; GFX908:   $vgpr31 = COPY [[COPY16]](s32)
+  ; GFX908:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; GFX908:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GFX908:   [[COPY18:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY8]]
+  ; GFX908:   S_SETPC_B64_return [[COPY18]]
+  call void @external_void_func_i32(i32 99)
+  ret void
+}
+
+; Explicit argument is split between registers ad the stack due to v31
+; being used for workitem IDs.
+define amdgpu_kernel void @test_call_external_void_func_v32i32([17 x i8]) #0 {
+  ; GFX900-LABEL: name: test_call_external_void_func_v32i32
+  ; GFX900: bb.1 (%ir-block.1):
+  ; GFX900:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GFX900:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GFX900:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GFX900:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GFX900:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GFX900:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GFX900:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GFX900:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GFX900:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GFX900:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GFX900:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GFX900:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX900:   [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32)
+  ; GFX900:   [[INT:%[0-9]+]]:_(p4) = G_INTRINSIC intrinsic(@llvm.amdgcn.kernarg.segment.ptr)
+  ; GFX900:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32), [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32), [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<32 x s32>)
+  ; GFX900:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GFX900:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v32i32
+  ; GFX900:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GFX900:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GFX900:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GFX900:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 20
+  ; GFX900:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; GFX900:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GFX900:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GFX900:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GFX900:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GFX900:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GFX900:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GFX900:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GFX900:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; GFX900:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GFX900:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GFX900:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GFX900:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; GFX900:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GFX900:   $vgpr0 = COPY [[UV]](s32)
+  ; GFX900:   $vgpr1 = COPY [[UV1]](s32)
+  ; GFX900:   $vgpr2 = COPY [[UV2]](s32)
+  ; GFX900:   $vgpr3 = COPY [[UV3]](s32)
+  ; GFX900:   $vgpr4 = COPY [[UV4]](s32)
+  ; GFX900:   $vgpr5 = COPY [[UV5]](s32)
+  ; GFX900:   $vgpr6 = COPY [[UV6]](s32)
+  ; GFX900:   $vgpr7 = COPY [[UV7]](s32)
+  ; GFX900:   $vgpr8 = COPY [[UV8]](s32)
+  ; GFX900:   $vgpr9 = COPY [[UV9]](s32)
+  ; GFX900:   $vgpr10 = COPY [[UV10]](s32)
+  ; GFX900:   $vgpr11 = COPY [[UV11]](s32)
+  ; GFX900:   $vgpr12 = COPY [[UV12]](s32)
+  ; GFX900:   $vgpr13 = COPY [[UV13]](s32)
+  ; GFX900:   $vgpr14 = COPY [[UV14]](s32)
+  ; GFX900:   $vgpr15 = COPY [[UV15]](s32)
+  ; GFX900:   $vgpr16 = COPY [[UV16]](s32)
+  ; GFX900:   $vgpr17 = COPY [[UV17]](s32)
+  ; GFX900:   $vgpr18 = COPY [[UV18]](s32)
+  ; GFX900:   $vgpr19 = COPY [[UV19]](s32)
+  ; GFX900:   $vgpr20 = COPY [[UV20]](s32)
+  ; GFX900:   $vgpr21 = COPY [[UV21]](s32)
+  ; GFX900:   $vgpr22 = COPY [[UV22]](s32)
+  ; GFX900:   $vgpr23 = COPY [[UV23]](s32)
+  ; GFX900:   $vgpr24 = COPY [[UV24]](s32)
+  ; GFX900:   $vgpr25 = COPY [[UV25]](s32)
+  ; GFX900:   $vgpr26 = COPY [[UV26]](s32)
+  ; GFX900:   $vgpr27 = COPY [[UV27]](s32)
+  ; GFX900:   $vgpr28 = COPY [[UV28]](s32)
+  ; GFX900:   $vgpr29 = COPY [[UV29]](s32)
+  ; GFX900:   $vgpr30 = COPY [[UV30]](s32)
+  ; GFX900:   [[COPY20:%[0-9]+]]:_(p5) = COPY $sp_reg
+  ; GFX900:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX900:   [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY20]], [[C4]](s32)
+  ; GFX900:   G_STORE [[UV31]](s32), [[PTR_ADD1]](p5) :: (store 4 into stack, align 16, addrspace 5)
+  ; GFX900:   [[COPY21:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GFX900:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY21]](<4 x s32>)
+  ; GFX900:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GFX900:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GFX900:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GFX900:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GFX900:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GFX900:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GFX900:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GFX900:   $vgpr31 = COPY [[OR1]](s32)
+  ; GFX900:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v32i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15, implicit $vgpr16, implicit $vgpr17, implicit $vgpr18, implicit $vgpr19, implicit $vgpr20, implicit $vgpr21, implicit $vgpr22, implicit $vgpr23, implicit $vgpr24, implicit $vgpr25, implicit $vgpr26, implicit $vgpr27, implicit $vgpr28, implicit $vgpr29, implicit $vgpr30, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; GFX900:   ADJCALLSTACKDOWN 0, 4, implicit-def $scc
+  ; GFX900:   S_ENDPGM 0
+  ; GFX908-LABEL: name: test_call_external_void_func_v32i32
+  ; GFX908: bb.1 (%ir-block.1):
+  ; GFX908:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GFX908:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GFX908:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GFX908:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GFX908:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GFX908:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GFX908:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GFX908:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GFX908:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GFX908:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GFX908:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GFX908:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX908:   [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32)
+  ; GFX908:   [[INT:%[0-9]+]]:_(p4) = G_INTRINSIC intrinsic(@llvm.amdgcn.kernarg.segment.ptr)
+  ; GFX908:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32), [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32), [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<32 x s32>)
+  ; GFX908:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GFX908:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v32i32
+  ; GFX908:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GFX908:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GFX908:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GFX908:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 20
+  ; GFX908:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; GFX908:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GFX908:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GFX908:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GFX908:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GFX908:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GFX908:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GFX908:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GFX908:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; GFX908:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GFX908:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GFX908:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GFX908:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; GFX908:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GFX908:   $vgpr0 = COPY [[UV]](s32)
+  ; GFX908:   $vgpr1 = COPY [[UV1]](s32)
+  ; GFX908:   $vgpr2 = COPY [[UV2]](s32)
+  ; GFX908:   $vgpr3 = COPY [[UV3]](s32)
+  ; GFX908:   $vgpr4 = COPY [[UV4]](s32)
+  ; GFX908:   $vgpr5 = COPY [[UV5]](s32)
+  ; GFX908:   $vgpr6 = COPY [[UV6]](s32)
+  ; GFX908:   $vgpr7 = COPY [[UV7]](s32)
+  ; GFX908:   $vgpr8 = COPY [[UV8]](s32)
+  ; GFX908:   $vgpr9 = COPY [[UV9]](s32)
+  ; GFX908:   $vgpr10 = COPY [[UV10]](s32)
+  ; GFX908:   $vgpr11 = COPY [[UV11]](s32)
+  ; GFX908:   $vgpr12 = COPY [[UV12]](s32)
+  ; GFX908:   $vgpr13 = COPY [[UV13]](s32)
+  ; GFX908:   $vgpr14 = COPY [[UV14]](s32)
+  ; GFX908:   $vgpr15 = COPY [[UV15]](s32)
+  ; GFX908:   $vgpr16 = COPY [[UV16]](s32)
+  ; GFX908:   $vgpr17 = COPY [[UV17]](s32)
+  ; GFX908:   $vgpr18 = COPY [[UV18]](s32)
+  ; GFX908:   $vgpr19 = COPY [[UV19]](s32)
+  ; GFX908:   $vgpr20 = COPY [[UV20]](s32)
+  ; GFX908:   $vgpr21 = COPY [[UV21]](s32)
+  ; GFX908:   $vgpr22 = COPY [[UV22]](s32)
+  ; GFX908:   $vgpr23 = COPY [[UV23]](s32)
+  ; GFX908:   $vgpr24 = COPY [[UV24]](s32)
+  ; GFX908:   $vgpr25 = COPY [[UV25]](s32)
+  ; GFX908:   $vgpr26 = COPY [[UV26]](s32)
+  ; GFX908:   $vgpr27 = COPY [[UV27]](s32)
+  ; GFX908:   $vgpr28 = COPY [[UV28]](s32)
+  ; GFX908:   $vgpr29 = COPY [[UV29]](s32)
+  ; GFX908:   $vgpr30 = COPY [[UV30]](s32)
+  ; GFX908:   [[COPY20:%[0-9]+]]:_(p5) = COPY $sp_reg
+  ; GFX908:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX908:   [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY20]], [[C4]](s32)
+  ; GFX908:   G_STORE [[UV31]](s32), [[PTR_ADD1]](p5) :: (store 4 into stack, align 16, addrspace 5)
+  ; GFX908:   [[COPY21:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GFX908:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY21]](<4 x s32>)
+  ; GFX908:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GFX908:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GFX908:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GFX908:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GFX908:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GFX908:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GFX908:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GFX908:   $vgpr31 = COPY [[OR1]](s32)
+  ; GFX908:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v32i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15, implicit $vgpr16, implicit $vgpr17, implicit $vgpr18, implicit $vgpr19, implicit $vgpr20, implicit $vgpr21, implicit $vgpr22, implicit $vgpr23, implicit $vgpr24, implicit $vgpr25, implicit $vgpr26, implicit $vgpr27, implicit $vgpr28, implicit $vgpr29, implicit $vgpr30, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; GFX908:   ADJCALLSTACKDOWN 0, 4, implicit-def $scc
+  ; GFX908:   S_ENDPGM 0
+  call void @external_void_func_v32i32(<32 x i32> zeroinitializer)
+  ret void
+}
+
+define void @test_func_call_external_void_func_v32i32([17 x i8]) #0 {
+  ; GFX900-LABEL: name: test_func_call_external_void_func_v32i32
+  ; GFX900: bb.1 (%ir-block.1):
+  ; GFX900:   liveins: $sgpr12, $sgpr13, $sgpr14, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11, $sgpr30_sgpr31
+  ; GFX900:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr31
+  ; GFX900:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GFX900:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr13
+  ; GFX900:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr12
+  ; GFX900:   [[COPY4:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GFX900:   [[COPY5:%[0-9]+]]:sgpr_64 = COPY $sgpr8_sgpr9
+  ; GFX900:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GFX900:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GFX900:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX900:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY8]](s32)
+  ; GFX900:   [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC]](s16)
+  ; GFX900:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX900:   [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
+  ; GFX900:   [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC2]](s16)
+  ; GFX900:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX900:   [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[COPY10]](s32)
+  ; GFX900:   [[TRUNC5:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC4]](s16)
+  ; GFX900:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX900:   [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[COPY11]](s32)
+  ; GFX900:   [[TRUNC7:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC6]](s16)
+  ; GFX900:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX900:   [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[COPY12]](s32)
+  ; GFX900:   [[TRUNC9:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC8]](s16)
+  ; GFX900:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; GFX900:   [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[COPY13]](s32)
+  ; GFX900:   [[TRUNC11:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC10]](s16)
+  ; GFX900:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; GFX900:   [[TRUNC12:%[0-9]+]]:_(s16) = G_TRUNC [[COPY14]](s32)
+  ; GFX900:   [[TRUNC13:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC12]](s16)
+  ; GFX900:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; GFX900:   [[TRUNC14:%[0-9]+]]:_(s16) = G_TRUNC [[COPY15]](s32)
+  ; GFX900:   [[TRUNC15:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC14]](s16)
+  ; GFX900:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; GFX900:   [[TRUNC16:%[0-9]+]]:_(s16) = G_TRUNC [[COPY16]](s32)
+  ; GFX900:   [[TRUNC17:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC16]](s16)
+  ; GFX900:   [[COPY17:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; GFX900:   [[TRUNC18:%[0-9]+]]:_(s16) = G_TRUNC [[COPY17]](s32)
+  ; GFX900:   [[TRUNC19:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC18]](s16)
+  ; GFX900:   [[COPY18:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; GFX900:   [[TRUNC20:%[0-9]+]]:_(s16) = G_TRUNC [[COPY18]](s32)
+  ; GFX900:   [[TRUNC21:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC20]](s16)
+  ; GFX900:   [[COPY19:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; GFX900:   [[TRUNC22:%[0-9]+]]:_(s16) = G_TRUNC [[COPY19]](s32)
+  ; GFX900:   [[TRUNC23:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC22]](s16)
+  ; GFX900:   [[COPY20:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; GFX900:   [[TRUNC24:%[0-9]+]]:_(s16) = G_TRUNC [[COPY20]](s32)
+  ; GFX900:   [[TRUNC25:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC24]](s16)
+  ; GFX900:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; GFX900:   [[TRUNC26:%[0-9]+]]:_(s16) = G_TRUNC [[COPY21]](s32)
+  ; GFX900:   [[TRUNC27:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC26]](s16)
+  ; GFX900:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; GFX900:   [[TRUNC28:%[0-9]+]]:_(s16) = G_TRUNC [[COPY22]](s32)
+  ; GFX900:   [[TRUNC29:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC28]](s16)
+  ; GFX900:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; GFX900:   [[TRUNC30:%[0-9]+]]:_(s16) = G_TRUNC [[COPY23]](s32)
+  ; GFX900:   [[TRUNC31:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC30]](s16)
+  ; GFX900:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr16
+  ; GFX900:   [[TRUNC32:%[0-9]+]]:_(s16) = G_TRUNC [[COPY24]](s32)
+  ; GFX900:   [[TRUNC33:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC32]](s16)
+  ; GFX900:   [[COPY25:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; GFX900:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX900:   [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32)
+  ; GFX900:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32), [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32), [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<32 x s32>)
+  ; GFX900:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GFX900:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v32i32
+  ; GFX900:   [[COPY26:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GFX900:   [[COPY27:%[0-9]+]]:_(p4) = COPY [[COPY6]]
+  ; GFX900:   [[COPY28:%[0-9]+]]:_(p4) = COPY [[COPY5]]
+  ; GFX900:   [[COPY29:%[0-9]+]]:_(s64) = COPY [[COPY4]]
+  ; GFX900:   [[COPY30:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GFX900:   [[COPY31:%[0-9]+]]:_(s32) = COPY [[COPY2]]
+  ; GFX900:   [[COPY32:%[0-9]+]]:_(s32) = COPY [[COPY1]]
+  ; GFX900:   [[COPY33:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GFX900:   $vgpr0 = COPY [[UV]](s32)
+  ; GFX900:   $vgpr1 = COPY [[UV1]](s32)
+  ; GFX900:   $vgpr2 = COPY [[UV2]](s32)
+  ; GFX900:   $vgpr3 = COPY [[UV3]](s32)
+  ; GFX900:   $vgpr4 = COPY [[UV4]](s32)
+  ; GFX900:   $vgpr5 = COPY [[UV5]](s32)
+  ; GFX900:   $vgpr6 = COPY [[UV6]](s32)
+  ; GFX900:   $vgpr7 = COPY [[UV7]](s32)
+  ; GFX900:   $vgpr8 = COPY [[UV8]](s32)
+  ; GFX900:   $vgpr9 = COPY [[UV9]](s32)
+  ; GFX900:   $vgpr10 = COPY [[UV10]](s32)
+  ; GFX900:   $vgpr11 = COPY [[UV11]](s32)
+  ; GFX900:   $vgpr12 = COPY [[UV12]](s32)
+  ; GFX900:   $vgpr13 = COPY [[UV13]](s32)
+  ; GFX900:   $vgpr14 = COPY [[UV14]](s32)
+  ; GFX900:   $vgpr15 = COPY [[UV15]](s32)
+  ; GFX900:   $vgpr16 = COPY [[UV16]](s32)
+  ; GFX900:   $vgpr17 = COPY [[UV17]](s32)
+  ; GFX900:   $vgpr18 = COPY [[UV18]](s32)
+  ; GFX900:   $vgpr19 = COPY [[UV19]](s32)
+  ; GFX900:   $vgpr20 = COPY [[UV20]](s32)
+  ; GFX900:   $vgpr21 = COPY [[UV21]](s32)
+  ; GFX900:   $vgpr22 = COPY [[UV22]](s32)
+  ; GFX900:   $vgpr23 = COPY [[UV23]](s32)
+  ; GFX900:   $vgpr24 = COPY [[UV24]](s32)
+  ; GFX900:   $vgpr25 = COPY [[UV25]](s32)
+  ; GFX900:   $vgpr26 = COPY [[UV26]](s32)
+  ; GFX900:   $vgpr27 = COPY [[UV27]](s32)
+  ; GFX900:   $vgpr28 = COPY [[UV28]](s32)
+  ; GFX900:   $vgpr29 = COPY [[UV29]](s32)
+  ; GFX900:   $vgpr30 = COPY [[UV30]](s32)
+  ; GFX900:   [[COPY34:%[0-9]+]]:_(p5) = COPY $sgpr32
+  ; GFX900:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX900:   [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY34]], [[C1]](s32)
+  ; GFX900:   G_STORE [[UV31]](s32), [[PTR_ADD]](p5) :: (store 4 into stack, align 16, addrspace 5)
+  ; GFX900:   [[COPY35:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX900:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY35]](<4 x s32>)
+  ; GFX900:   $sgpr4_sgpr5 = COPY [[COPY26]](p4)
+  ; GFX900:   $sgpr6_sgpr7 = COPY [[COPY27]](p4)
+  ; GFX900:   $sgpr8_sgpr9 = COPY [[COPY28]](p4)
+  ; GFX900:   $sgpr10_sgpr11 = COPY [[COPY29]](s64)
+  ; GFX900:   $sgpr12 = COPY [[COPY30]](s32)
+  ; GFX900:   $sgpr13 = COPY [[COPY31]](s32)
+  ; GFX900:   $sgpr14 = COPY [[COPY32]](s32)
+  ; GFX900:   $vgpr31 = COPY [[COPY33]](s32)
+  ; GFX900:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v32i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15, implicit $vgpr16, implicit $vgpr17, implicit $vgpr18, implicit $vgpr19, implicit $vgpr20, implicit $vgpr21, implicit $vgpr22, implicit $vgpr23, implicit $vgpr24, implicit $vgpr25, implicit $vgpr26, implicit $vgpr27, implicit $vgpr28, implicit $vgpr29, implicit $vgpr30, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; GFX900:   ADJCALLSTACKDOWN 0, 4, implicit-def $scc
+  ; GFX900:   [[COPY36:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY25]]
+  ; GFX900:   S_SETPC_B64_return [[COPY36]]
+  ; GFX908-LABEL: name: test_func_call_external_void_func_v32i32
+  ; GFX908: bb.1 (%ir-block.1):
+  ; GFX908:   liveins: $sgpr12, $sgpr13, $sgpr14, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11, $sgpr30_sgpr31
+  ; GFX908:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr31
+  ; GFX908:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GFX908:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr13
+  ; GFX908:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr12
+  ; GFX908:   [[COPY4:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GFX908:   [[COPY5:%[0-9]+]]:sgpr_64 = COPY $sgpr8_sgpr9
+  ; GFX908:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GFX908:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GFX908:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX908:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY8]](s32)
+  ; GFX908:   [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC]](s16)
+  ; GFX908:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX908:   [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
+  ; GFX908:   [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC2]](s16)
+  ; GFX908:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX908:   [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[COPY10]](s32)
+  ; GFX908:   [[TRUNC5:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC4]](s16)
+  ; GFX908:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX908:   [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[COPY11]](s32)
+  ; GFX908:   [[TRUNC7:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC6]](s16)
+  ; GFX908:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX908:   [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[COPY12]](s32)
+  ; GFX908:   [[TRUNC9:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC8]](s16)
+  ; GFX908:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; GFX908:   [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[COPY13]](s32)
+  ; GFX908:   [[TRUNC11:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC10]](s16)
+  ; GFX908:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; GFX908:   [[TRUNC12:%[0-9]+]]:_(s16) = G_TRUNC [[COPY14]](s32)
+  ; GFX908:   [[TRUNC13:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC12]](s16)
+  ; GFX908:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; GFX908:   [[TRUNC14:%[0-9]+]]:_(s16) = G_TRUNC [[COPY15]](s32)
+  ; GFX908:   [[TRUNC15:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC14]](s16)
+  ; GFX908:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; GFX908:   [[TRUNC16:%[0-9]+]]:_(s16) = G_TRUNC [[COPY16]](s32)
+  ; GFX908:   [[TRUNC17:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC16]](s16)
+  ; GFX908:   [[COPY17:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; GFX908:   [[TRUNC18:%[0-9]+]]:_(s16) = G_TRUNC [[COPY17]](s32)
+  ; GFX908:   [[TRUNC19:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC18]](s16)
+  ; GFX908:   [[COPY18:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; GFX908:   [[TRUNC20:%[0-9]+]]:_(s16) = G_TRUNC [[COPY18]](s32)
+  ; GFX908:   [[TRUNC21:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC20]](s16)
+  ; GFX908:   [[COPY19:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; GFX908:   [[TRUNC22:%[0-9]+]]:_(s16) = G_TRUNC [[COPY19]](s32)
+  ; GFX908:   [[TRUNC23:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC22]](s16)
+  ; GFX908:   [[COPY20:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; GFX908:   [[TRUNC24:%[0-9]+]]:_(s16) = G_TRUNC [[COPY20]](s32)
+  ; GFX908:   [[TRUNC25:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC24]](s16)
+  ; GFX908:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; GFX908:   [[TRUNC26:%[0-9]+]]:_(s16) = G_TRUNC [[COPY21]](s32)
+  ; GFX908:   [[TRUNC27:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC26]](s16)
+  ; GFX908:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; GFX908:   [[TRUNC28:%[0-9]+]]:_(s16) = G_TRUNC [[COPY22]](s32)
+  ; GFX908:   [[TRUNC29:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC28]](s16)
+  ; GFX908:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; GFX908:   [[TRUNC30:%[0-9]+]]:_(s16) = G_TRUNC [[COPY23]](s32)
+  ; GFX908:   [[TRUNC31:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC30]](s16)
+  ; GFX908:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr16
+  ; GFX908:   [[TRUNC32:%[0-9]+]]:_(s16) = G_TRUNC [[COPY24]](s32)
+  ; GFX908:   [[TRUNC33:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC32]](s16)
+  ; GFX908:   [[COPY25:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; GFX908:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX908:   [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32)
+  ; GFX908:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32), [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32), [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<32 x s32>)
+  ; GFX908:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GFX908:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v32i32
+  ; GFX908:   [[COPY26:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GFX908:   [[COPY27:%[0-9]+]]:_(p4) = COPY [[COPY6]]
+  ; GFX908:   [[COPY28:%[0-9]+]]:_(p4) = COPY [[COPY5]]
+  ; GFX908:   [[COPY29:%[0-9]+]]:_(s64) = COPY [[COPY4]]
+  ; GFX908:   [[COPY30:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GFX908:   [[COPY31:%[0-9]+]]:_(s32) = COPY [[COPY2]]
+  ; GFX908:   [[COPY32:%[0-9]+]]:_(s32) = COPY [[COPY1]]
+  ; GFX908:   [[COPY33:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GFX908:   $vgpr0 = COPY [[UV]](s32)
+  ; GFX908:   $vgpr1 = COPY [[UV1]](s32)
+  ; GFX908:   $vgpr2 = COPY [[UV2]](s32)
+  ; GFX908:   $vgpr3 = COPY [[UV3]](s32)
+  ; GFX908:   $vgpr4 = COPY [[UV4]](s32)
+  ; GFX908:   $vgpr5 = COPY [[UV5]](s32)
+  ; GFX908:   $vgpr6 = COPY [[UV6]](s32)
+  ; GFX908:   $vgpr7 = COPY [[UV7]](s32)
+  ; GFX908:   $vgpr8 = COPY [[UV8]](s32)
+  ; GFX908:   $vgpr9 = COPY [[UV9]](s32)
+  ; GFX908:   $vgpr10 = COPY [[UV10]](s32)
+  ; GFX908:   $vgpr11 = COPY [[UV11]](s32)
+  ; GFX908:   $vgpr12 = COPY [[UV12]](s32)
+  ; GFX908:   $vgpr13 = COPY [[UV13]](s32)
+  ; GFX908:   $vgpr14 = COPY [[UV14]](s32)
+  ; GFX908:   $vgpr15 = COPY [[UV15]](s32)
+  ; GFX908:   $vgpr16 = COPY [[UV16]](s32)
+  ; GFX908:   $vgpr17 = COPY [[UV17]](s32)
+  ; GFX908:   $vgpr18 = COPY [[UV18]](s32)
+  ; GFX908:   $vgpr19 = COPY [[UV19]](s32)
+  ; GFX908:   $vgpr20 = COPY [[UV20]](s32)
+  ; GFX908:   $vgpr21 = COPY [[UV21]](s32)
+  ; GFX908:   $vgpr22 = COPY [[UV22]](s32)
+  ; GFX908:   $vgpr23 = COPY [[UV23]](s32)
+  ; GFX908:   $vgpr24 = COPY [[UV24]](s32)
+  ; GFX908:   $vgpr25 = COPY [[UV25]](s32)
+  ; GFX908:   $vgpr26 = COPY [[UV26]](s32)
+  ; GFX908:   $vgpr27 = COPY [[UV27]](s32)
+  ; GFX908:   $vgpr28 = COPY [[UV28]](s32)
+  ; GFX908:   $vgpr29 = COPY [[UV29]](s32)
+  ; GFX908:   $vgpr30 = COPY [[UV30]](s32)
+  ; GFX908:   [[COPY34:%[0-9]+]]:_(p5) = COPY $sgpr32
+  ; GFX908:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX908:   [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY34]], [[C1]](s32)
+  ; GFX908:   G_STORE [[UV31]](s32), [[PTR_ADD]](p5) :: (store 4 into stack, align 16, addrspace 5)
+  ; GFX908:   [[COPY35:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX908:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY35]](<4 x s32>)
+  ; GFX908:   $sgpr4_sgpr5 = COPY [[COPY26]](p4)
+  ; GFX908:   $sgpr6_sgpr7 = COPY [[COPY27]](p4)
+  ; GFX908:   $sgpr8_sgpr9 = COPY [[COPY28]](p4)
+  ; GFX908:   $sgpr10_sgpr11 = COPY [[COPY29]](s64)
+  ; GFX908:   $sgpr12 = COPY [[COPY30]](s32)
+  ; GFX908:   $sgpr13 = COPY [[COPY31]](s32)
+  ; GFX908:   $sgpr14 = COPY [[COPY32]](s32)
+  ; GFX908:   $vgpr31 = COPY [[COPY33]](s32)
+  ; GFX908:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v32i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15, implicit $vgpr16, implicit $vgpr17, implicit $vgpr18, implicit $vgpr19, implicit $vgpr20, implicit $vgpr21, implicit $vgpr22, implicit $vgpr23, implicit $vgpr24, implicit $vgpr25, implicit $vgpr26, implicit $vgpr27, implicit $vgpr28, implicit $vgpr29, implicit $vgpr30, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; GFX908:   ADJCALLSTACKDOWN 0, 4, implicit-def $scc
+  ; GFX908:   [[COPY36:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY25]]
+  ; GFX908:   S_SETPC_B64_return [[COPY36]]
+  call void @external_void_func_v32i32(<32 x i32> zeroinitializer)
+  ret void
+}
+
+; FIXME: Should fold out parts with known 0 id.
+
+define amdgpu_kernel void @test_only_workitem_id_x() #0 !reqd_work_group_size !0 {
+  ; GFX900-LABEL: name: test_only_workitem_id_x
+  ; GFX900: bb.1 (%ir-block.0):
+  ; GFX900:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GFX900:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GFX900:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GFX900:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GFX900:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GFX900:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GFX900:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GFX900:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GFX900:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GFX900:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GFX900:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GFX900:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
+  ; GFX900:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GFX900:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i32
+  ; GFX900:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GFX900:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GFX900:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GFX900:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GFX900:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; GFX900:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GFX900:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GFX900:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GFX900:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GFX900:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GFX900:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GFX900:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GFX900:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; GFX900:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GFX900:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GFX900:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GFX900:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; GFX900:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GFX900:   $vgpr0 = COPY [[C]](s32)
+  ; GFX900:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GFX900:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GFX900:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GFX900:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GFX900:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GFX900:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GFX900:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GFX900:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GFX900:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GFX900:   $vgpr31 = COPY [[OR1]](s32)
+  ; GFX900:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; GFX900:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GFX900:   S_ENDPGM 0
+  ; GFX908-LABEL: name: test_only_workitem_id_x
+  ; GFX908: bb.1 (%ir-block.0):
+  ; GFX908:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GFX908:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GFX908:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GFX908:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GFX908:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GFX908:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GFX908:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GFX908:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GFX908:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GFX908:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GFX908:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GFX908:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
+  ; GFX908:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GFX908:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i32
+  ; GFX908:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GFX908:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GFX908:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GFX908:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GFX908:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; GFX908:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GFX908:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GFX908:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GFX908:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GFX908:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GFX908:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GFX908:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GFX908:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; GFX908:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GFX908:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GFX908:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GFX908:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; GFX908:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GFX908:   $vgpr0 = COPY [[C]](s32)
+  ; GFX908:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GFX908:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GFX908:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GFX908:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GFX908:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GFX908:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GFX908:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GFX908:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GFX908:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GFX908:   $vgpr31 = COPY [[OR1]](s32)
+  ; GFX908:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; GFX908:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GFX908:   S_ENDPGM 0
+  call void @external_void_func_i32(i32 42)
+  ret void
+}
+
+define amdgpu_kernel void @test_only_workitem_id_y() #0 !reqd_work_group_size !1 {
+  ; GFX900-LABEL: name: test_only_workitem_id_y
+  ; GFX900: bb.1 (%ir-block.0):
+  ; GFX900:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GFX900:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GFX900:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GFX900:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GFX900:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GFX900:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GFX900:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GFX900:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GFX900:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GFX900:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GFX900:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GFX900:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
+  ; GFX900:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GFX900:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i32
+  ; GFX900:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GFX900:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GFX900:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GFX900:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GFX900:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; GFX900:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GFX900:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GFX900:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GFX900:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GFX900:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GFX900:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GFX900:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GFX900:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; GFX900:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GFX900:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GFX900:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GFX900:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; GFX900:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GFX900:   $vgpr0 = COPY [[C]](s32)
+  ; GFX900:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GFX900:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GFX900:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GFX900:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GFX900:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GFX900:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GFX900:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GFX900:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GFX900:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GFX900:   $vgpr31 = COPY [[OR1]](s32)
+  ; GFX900:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; GFX900:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GFX900:   S_ENDPGM 0
+  ; GFX908-LABEL: name: test_only_workitem_id_y
+  ; GFX908: bb.1 (%ir-block.0):
+  ; GFX908:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GFX908:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GFX908:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GFX908:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GFX908:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GFX908:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GFX908:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GFX908:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GFX908:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GFX908:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GFX908:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GFX908:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
+  ; GFX908:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GFX908:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i32
+  ; GFX908:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GFX908:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GFX908:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GFX908:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GFX908:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; GFX908:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GFX908:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GFX908:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GFX908:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GFX908:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GFX908:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GFX908:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GFX908:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; GFX908:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GFX908:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GFX908:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GFX908:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; GFX908:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GFX908:   $vgpr0 = COPY [[C]](s32)
+  ; GFX908:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GFX908:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GFX908:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GFX908:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GFX908:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GFX908:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GFX908:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GFX908:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GFX908:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GFX908:   $vgpr31 = COPY [[OR1]](s32)
+  ; GFX908:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; GFX908:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GFX908:   S_ENDPGM 0
+  call void @external_void_func_i32(i32 42)
+  ret void
+}
+
+define amdgpu_kernel void @test_only_workitem_id_z() #0 !reqd_work_group_size !2 {
+  ; GFX900-LABEL: name: test_only_workitem_id_z
+  ; GFX900: bb.1 (%ir-block.0):
+  ; GFX900:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GFX900:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GFX900:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GFX900:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GFX900:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GFX900:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GFX900:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GFX900:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GFX900:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GFX900:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GFX900:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GFX900:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
+  ; GFX900:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GFX900:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i32
+  ; GFX900:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GFX900:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GFX900:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GFX900:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GFX900:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; GFX900:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GFX900:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GFX900:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GFX900:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GFX900:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GFX900:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GFX900:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GFX900:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; GFX900:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GFX900:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GFX900:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GFX900:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; GFX900:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GFX900:   $vgpr0 = COPY [[C]](s32)
+  ; GFX900:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GFX900:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GFX900:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GFX900:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GFX900:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GFX900:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GFX900:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GFX900:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GFX900:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GFX900:   $vgpr31 = COPY [[OR1]](s32)
+  ; GFX900:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; GFX900:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GFX900:   S_ENDPGM 0
+  ; GFX908-LABEL: name: test_only_workitem_id_z
+  ; GFX908: bb.1 (%ir-block.0):
+  ; GFX908:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GFX908:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GFX908:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GFX908:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GFX908:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GFX908:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GFX908:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GFX908:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GFX908:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GFX908:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GFX908:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GFX908:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
+  ; GFX908:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GFX908:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i32
+  ; GFX908:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GFX908:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GFX908:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GFX908:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GFX908:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; GFX908:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GFX908:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GFX908:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GFX908:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GFX908:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GFX908:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GFX908:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GFX908:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; GFX908:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GFX908:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GFX908:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GFX908:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; GFX908:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GFX908:   $vgpr0 = COPY [[C]](s32)
+  ; GFX908:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GFX908:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GFX908:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GFX908:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GFX908:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GFX908:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GFX908:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GFX908:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GFX908:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GFX908:   $vgpr31 = COPY [[OR1]](s32)
+  ; GFX908:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; GFX908:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GFX908:   S_ENDPGM 0
+  call void @external_void_func_i32(i32 42)
+  ret void
+}
+
+define amdgpu_kernel void @test_only_workitem_id_xy() #0 !reqd_work_group_size !3 {
+  ; GFX900-LABEL: name: test_only_workitem_id_xy
+  ; GFX900: bb.1 (%ir-block.0):
+  ; GFX900:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GFX900:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GFX900:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GFX900:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GFX900:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GFX900:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GFX900:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GFX900:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GFX900:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GFX900:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GFX900:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GFX900:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
+  ; GFX900:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GFX900:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i32
+  ; GFX900:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GFX900:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GFX900:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GFX900:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GFX900:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; GFX900:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GFX900:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GFX900:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GFX900:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GFX900:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GFX900:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GFX900:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GFX900:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; GFX900:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GFX900:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GFX900:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GFX900:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; GFX900:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GFX900:   $vgpr0 = COPY [[C]](s32)
+  ; GFX900:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GFX900:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GFX900:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GFX900:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GFX900:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GFX900:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GFX900:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GFX900:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GFX900:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GFX900:   $vgpr31 = COPY [[OR1]](s32)
+  ; GFX900:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; GFX900:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GFX900:   S_ENDPGM 0
+  ; GFX908-LABEL: name: test_only_workitem_id_xy
+  ; GFX908: bb.1 (%ir-block.0):
+  ; GFX908:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GFX908:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GFX908:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GFX908:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GFX908:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GFX908:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GFX908:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GFX908:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GFX908:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GFX908:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GFX908:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GFX908:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
+  ; GFX908:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GFX908:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i32
+  ; GFX908:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GFX908:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GFX908:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GFX908:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GFX908:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; GFX908:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GFX908:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GFX908:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GFX908:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GFX908:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GFX908:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GFX908:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GFX908:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; GFX908:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GFX908:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GFX908:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GFX908:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; GFX908:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GFX908:   $vgpr0 = COPY [[C]](s32)
+  ; GFX908:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GFX908:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GFX908:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GFX908:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GFX908:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GFX908:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GFX908:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GFX908:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GFX908:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GFX908:   $vgpr31 = COPY [[OR1]](s32)
+  ; GFX908:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; GFX908:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GFX908:   S_ENDPGM 0
+  call void @external_void_func_i32(i32 42)
+  ret void
+}
+
+define amdgpu_kernel void @test_only_workitem_id_yz() #0 !reqd_work_group_size !4 {
+  ; GFX900-LABEL: name: test_only_workitem_id_yz
+  ; GFX900: bb.1 (%ir-block.0):
+  ; GFX900:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GFX900:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GFX900:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GFX900:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GFX900:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GFX900:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GFX900:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GFX900:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GFX900:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GFX900:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GFX900:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GFX900:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
+  ; GFX900:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GFX900:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i32
+  ; GFX900:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GFX900:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GFX900:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GFX900:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GFX900:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; GFX900:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GFX900:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GFX900:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GFX900:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GFX900:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GFX900:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GFX900:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GFX900:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; GFX900:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GFX900:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GFX900:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GFX900:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; GFX900:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GFX900:   $vgpr0 = COPY [[C]](s32)
+  ; GFX900:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GFX900:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GFX900:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GFX900:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GFX900:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GFX900:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GFX900:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GFX900:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GFX900:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GFX900:   $vgpr31 = COPY [[OR1]](s32)
+  ; GFX900:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; GFX900:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GFX900:   S_ENDPGM 0
+  ; GFX908-LABEL: name: test_only_workitem_id_yz
+  ; GFX908: bb.1 (%ir-block.0):
+  ; GFX908:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GFX908:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GFX908:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GFX908:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GFX908:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GFX908:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GFX908:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GFX908:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GFX908:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GFX908:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GFX908:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GFX908:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
+  ; GFX908:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GFX908:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i32
+  ; GFX908:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GFX908:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GFX908:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GFX908:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GFX908:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; GFX908:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GFX908:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GFX908:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GFX908:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GFX908:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GFX908:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GFX908:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GFX908:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; GFX908:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GFX908:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GFX908:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GFX908:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; GFX908:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GFX908:   $vgpr0 = COPY [[C]](s32)
+  ; GFX908:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GFX908:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GFX908:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GFX908:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GFX908:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GFX908:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GFX908:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GFX908:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GFX908:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GFX908:   $vgpr31 = COPY [[OR1]](s32)
+  ; GFX908:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; GFX908:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GFX908:   S_ENDPGM 0
+  call void @external_void_func_i32(i32 42)
+  ret void
+}
+
+define amdgpu_kernel void @test_only_workitem_id_xz() #0 !reqd_work_group_size !5 {
+  ; GFX900-LABEL: name: test_only_workitem_id_xz
+  ; GFX900: bb.1 (%ir-block.0):
+  ; GFX900:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GFX900:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GFX900:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GFX900:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GFX900:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GFX900:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GFX900:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GFX900:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GFX900:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GFX900:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GFX900:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GFX900:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
+  ; GFX900:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GFX900:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i32
+  ; GFX900:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GFX900:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GFX900:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GFX900:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GFX900:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; GFX900:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GFX900:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GFX900:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GFX900:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GFX900:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GFX900:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GFX900:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GFX900:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; GFX900:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GFX900:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GFX900:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GFX900:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; GFX900:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GFX900:   $vgpr0 = COPY [[C]](s32)
+  ; GFX900:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GFX900:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GFX900:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GFX900:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GFX900:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GFX900:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GFX900:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GFX900:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GFX900:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GFX900:   $vgpr31 = COPY [[OR1]](s32)
+  ; GFX900:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; GFX900:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GFX900:   S_ENDPGM 0
+  ; GFX908-LABEL: name: test_only_workitem_id_xz
+  ; GFX908: bb.1 (%ir-block.0):
+  ; GFX908:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GFX908:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GFX908:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GFX908:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GFX908:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GFX908:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GFX908:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GFX908:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GFX908:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GFX908:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GFX908:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GFX908:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
+  ; GFX908:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GFX908:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i32
+  ; GFX908:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GFX908:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GFX908:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GFX908:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GFX908:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; GFX908:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GFX908:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GFX908:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GFX908:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GFX908:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GFX908:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GFX908:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GFX908:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; GFX908:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GFX908:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GFX908:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GFX908:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; GFX908:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GFX908:   $vgpr0 = COPY [[C]](s32)
+  ; GFX908:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GFX908:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GFX908:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GFX908:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GFX908:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GFX908:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GFX908:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GFX908:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GFX908:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GFX908:   $vgpr31 = COPY [[OR1]](s32)
+  ; GFX908:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; GFX908:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GFX908:   S_ENDPGM 0
+  call void @external_void_func_i32(i32 42)
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x() #1
+declare i32 @llvm.amdgcn.workitem.id.y() #1
+declare i32 @llvm.amdgcn.workitem.id.z() #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone speculatable willreturn }
+
+!0 = !{i32 64, i32 1, i32 1}
+!1 = !{i32 1, i32 64, i32 1}
+!2 = !{i32 1, i32 1, i32 64}
+!3 = !{i32 32, i32 2, i32 1}
+!4 = !{i32 1, i32 32, i32 2}
+!5 = !{i32 32, i32 1, i32 2}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-return-values.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-return-values.ll
new file mode 100644
index 0000000000000..1136afafbd4aa
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-return-values.ll
@@ -0,0 +1,2509 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -amdgpu-fixed-function-abi -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -stop-after=irtranslator -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+declare i1 @external_i1_func_void() #0
+declare zeroext i1 @external_i1_zeroext_func_void() #0
+declare signext i1 @external_i1_signext_func_void() #0
+
+declare i8 @external_i8_func_void() #0
+declare zeroext i8 @external_i8_zeroext_func_void() #0
+declare signext i8 @external_i8_signext_func_void() #0
+
+declare i16 @external_i16_func_void() #0
+declare <2 x i16> @external_v2i16_func_void() #0
+declare <3 x i16> @external_v3i16_func_void() #0
+declare <4 x i16> @external_v4i16_func_void() #0
+declare zeroext i16 @external_i16_zeroext_func_void() #0
+declare signext i16 @external_i16_signext_func_void() #0
+
+declare i48 @external_i48_func_void() #0
+declare zeroext i48 @external_i48_zeroext_func_void() #0
+declare signext i48 @external_i48_signext_func_void() #0
+
+declare i32 @external_i32_func_void() #0
+declare i64 @external_i64_func_void() #0
+declare half @external_f16_func_void() #0
+declare float @external_f32_func_void() #0
+declare double @external_f64_func_void() #0
+
+declare i8 addrspace(1)* @external_p1_func_void() #0
+declare <2 x i8 addrspace(1)*> @external_v2p1_func_void() #0
+
+declare i8 addrspace(3)* @external_p3_func_void() #0
+declare <2 x i8 addrspace(3)*> @external_v2p3_func_void() #0
+
+declare <2 x half> @external_v2f16_func_void() #0
+declare <3 x half> @external_v3f16_func_void() #0
+declare <4 x half> @external_v4f16_func_void() #0
+declare <3 x float> @external_v3f32_func_void() #0
+declare <5 x float> @external_v5f32_func_void() #0
+declare <2 x double> @external_v2f64_func_void() #0
+
+declare <2 x i32> @external_v2i32_func_void() #0
+declare <3 x i32> @external_v3i32_func_void() #0
+declare <4 x i32> @external_v4i32_func_void() #0
+declare <5 x i32> @external_v5i32_func_void() #0
+declare <8 x i32> @external_v8i32_func_void() #0
+declare <16 x i32> @external_v16i32_func_void() #0
+declare <32 x i32> @external_v32i32_func_void() #0
+declare { <32 x i32>, i32 } @external_v32i32_i32_func_void() #0
+
+declare { i32, i64 } @external_i32_i64_func_void() #0
+
+declare [2 x i32] @external_a2i32_func_void() #0
+declare [5 x i8] @external_a5i8_func_void() #0
+
+; return value and argument
+declare hidden i32 @external_i32_func_i32(i32) #0
+
+
+define amdgpu_kernel void @test_call_external_i32_func_i32_imm(i32 addrspace(1)* %out) #0 {
+  ; GCN-LABEL: name: test_call_external_i32_func_i32_imm
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
+  ; GCN:   [[INT:%[0-9]+]]:_(p4) = G_INTRINSIC intrinsic(@llvm.amdgcn.kernarg.segment.ptr)
+  ; GCN:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[INT]](p4) :: (dereferenceable invariant load 8 from %ir.out.kernarg.offset.cast, align 16, addrspace 4)
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_i32_func_i32
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   $vgpr0 = COPY [[C]](s32)
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_i32_func_i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   G_STORE [[COPY21]](s32), [[LOAD]](p1) :: (volatile store 4 into %ir.out.load, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call i32 @external_i32_func_i32(i32 42)
+  store volatile i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_i1_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_i1_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_i1_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_i1_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[TRUNC:%[0-9]+]]:_(s1) = G_TRUNC [[COPY21]](s32)
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   G_STORE [[TRUNC]](s1), [[DEF]](p1) :: (volatile store 1 into `i1 addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call i1 @external_i1_func_void()
+  store volatile i1 %val, i1 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_i1_zeroext_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_i1_zeroext_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_i1_zeroext_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_i1_zeroext_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[TRUNC:%[0-9]+]]:_(s1) = G_TRUNC [[COPY21]](s32)
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC]](s1)
+  ; GCN:   G_STORE [[ZEXT]](s32), [[DEF]](p1) :: (volatile store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call i1 @external_i1_zeroext_func_void()
+  %val.ext = zext i1 %val to i32
+  store volatile i32 %val.ext, i32 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_i1_signext_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_i1_signext_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_i1_signext_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_i1_signext_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[TRUNC:%[0-9]+]]:_(s1) = G_TRUNC [[COPY21]](s32)
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[TRUNC]](s1)
+  ; GCN:   G_STORE [[SEXT]](s32), [[DEF]](p1) :: (volatile store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call i1 @external_i1_signext_func_void()
+  %val.ext = sext i1 %val to i32
+  store volatile i32 %val.ext, i32 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_i8_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_i8_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_i8_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_i8_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY21]](s32)
+  ; GCN:   [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC]](s16)
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   G_STORE [[TRUNC1]](s8), [[DEF]](p1) :: (volatile store 1 into `i8 addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call i8 @external_i8_func_void()
+  store volatile i8 %val, i8 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_i8_zeroext_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_i8_zeroext_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_i8_zeroext_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_i8_zeroext_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY21]](s32)
+  ; GCN:   [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC]](s16)
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC1]](s8)
+  ; GCN:   G_STORE [[ZEXT]](s32), [[DEF]](p1) :: (volatile store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call i8 @external_i8_zeroext_func_void()
+  %val.ext = zext i8 %val to i32
+  store volatile i32 %val.ext, i32 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_i8_signext_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_i8_signext_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_i8_signext_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_i8_signext_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY21]](s32)
+  ; GCN:   [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC]](s16)
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[TRUNC1]](s8)
+  ; GCN:   G_STORE [[SEXT]](s32), [[DEF]](p1) :: (volatile store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call i8 @external_i8_signext_func_void()
+  %val.ext = sext i8 %val to i32
+  store volatile i32 %val.ext, i32 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_i16_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_i16_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_i16_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_i16_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY21]](s32)
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   G_STORE [[TRUNC]](s16), [[DEF]](p1) :: (volatile store 2 into `i16 addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call i16 @external_i16_func_void()
+  store volatile i16 %val, i16 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_i16_zeroext_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_i16_zeroext_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_i16_zeroext_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_i16_zeroext_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY21]](s32)
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC]](s16)
+  ; GCN:   G_STORE [[ZEXT]](s32), [[DEF]](p1) :: (volatile store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call i16 @external_i16_zeroext_func_void()
+  %val.ext = zext i16 %val to i32
+  store volatile i32 %val.ext, i32 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_i16_signext_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_i16_signext_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_i16_signext_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_i16_signext_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY21]](s32)
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[TRUNC]](s16)
+  ; GCN:   G_STORE [[SEXT]](s32), [[DEF]](p1) :: (volatile store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call i16 @external_i16_signext_func_void()
+  %val.ext = sext i16 %val to i32
+  store volatile i32 %val.ext, i32 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_i32_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_i32_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_i32_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_i32_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   G_STORE [[COPY21]](s32), [[DEF]](p1) :: (volatile store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call i32 @external_i32_func_void()
+  store volatile i32 %val, i32 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_i48_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_i48_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_i48_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_i48_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0, implicit-def $vgpr1
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY21]](s32), [[COPY22]](s32)
+  ; GCN:   [[TRUNC:%[0-9]+]]:_(s48) = G_TRUNC [[MV]](s64)
+  ; GCN:   G_STORE [[TRUNC]](s48), [[DEF]](p1) :: (volatile store 6 into `i48 addrspace(1)* undef`, align 8, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call i48 @external_i48_func_void()
+  store volatile i48 %val, i48 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_i48_zeroext_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_i48_zeroext_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_i48_zeroext_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_i48_zeroext_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0, implicit-def $vgpr1
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY21]](s32), [[COPY22]](s32)
+  ; GCN:   [[TRUNC:%[0-9]+]]:_(s48) = G_TRUNC [[MV]](s64)
+  ; GCN:   [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC]](s48)
+  ; GCN:   G_STORE [[ZEXT]](s64), [[DEF]](p1) :: (volatile store 8 into `i64 addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call i48 @external_i48_zeroext_func_void()
+  %ext = zext i48 %val to i64
+  store volatile i64 %ext, i64 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_i48_signext_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_i48_signext_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_i48_signext_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_i48_signext_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0, implicit-def $vgpr1
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY21]](s32), [[COPY22]](s32)
+  ; GCN:   [[TRUNC:%[0-9]+]]:_(s48) = G_TRUNC [[MV]](s64)
+  ; GCN:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[TRUNC]](s48)
+  ; GCN:   G_STORE [[SEXT]](s64), [[DEF]](p1) :: (volatile store 8 into `i64 addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call i48 @external_i48_signext_func_void()
+  %ext = sext i48 %val to i64
+  store volatile i64 %ext, i64 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_i64_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_i64_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_i64_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_i64_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0, implicit-def $vgpr1
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY21]](s32), [[COPY22]](s32)
+  ; GCN:   G_STORE [[MV]](s64), [[DEF]](p1) :: (volatile store 8 into `i64 addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call i64 @external_i64_func_void()
+  store volatile i64 %val, i64 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_p1_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_p1_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_p1_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_p1_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0, implicit-def $vgpr1
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY21]](s32), [[COPY22]](s32)
+  ; GCN:   G_STORE [[MV]](p1), [[DEF]](p1) :: (volatile store 8 into `i8 addrspace(1)* addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call i8 addrspace(1)* @external_p1_func_void()
+  store volatile i8 addrspace(1)* %val, i8 addrspace(1)* addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_v2p1_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_v2p1_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_v2p1_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_v2p1_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0, implicit-def $vgpr1, implicit-def $vgpr2, implicit-def $vgpr3
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GCN:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GCN:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY21]](s32), [[COPY22]](s32)
+  ; GCN:   [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY23]](s32), [[COPY24]](s32)
+  ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p1>) = G_BUILD_VECTOR [[MV]](p1), [[MV1]](p1)
+  ; GCN:   G_STORE [[BUILD_VECTOR]](<2 x p1>), [[DEF]](p1) :: (volatile store 16 into `<2 x i8 addrspace(1)*> addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call <2 x i8 addrspace(1)*> @external_v2p1_func_void()
+  store volatile <2 x i8 addrspace(1)*> %val, <2 x i8 addrspace(1)*> addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_p3_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_p3_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p3) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_p3_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_p3_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0
+  ; GCN:   [[COPY21:%[0-9]+]]:_(p3) = COPY $vgpr0
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   G_STORE [[COPY21]](p3), [[DEF]](p3) :: (volatile store 4 into `i8 addrspace(3)* addrspace(3)* undef`, addrspace 3)
+  ; GCN:   S_ENDPGM 0
+  %val = call i8 addrspace(3)* @external_p3_func_void()
+  store volatile i8 addrspace(3)* %val, i8 addrspace(3)* addrspace(3)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_v2p3_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_v2p3_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p3) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_v2p3_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_v2p3_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0, implicit-def $vgpr1
+  ; GCN:   [[COPY21:%[0-9]+]]:_(p3) = COPY $vgpr0
+  ; GCN:   [[COPY22:%[0-9]+]]:_(p3) = COPY $vgpr1
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[COPY21]](p3), [[COPY22]](p3)
+  ; GCN:   G_STORE [[BUILD_VECTOR]](<2 x p3>), [[DEF]](p3) :: (volatile store 8 into `<2 x i8 addrspace(3)*> addrspace(3)* undef`, addrspace 3)
+  ; GCN:   S_ENDPGM 0
+  %val = call <2 x i8 addrspace(3)*> @external_v2p3_func_void()
+  store volatile <2 x i8 addrspace(3)*> %val, <2 x i8 addrspace(3)*> addrspace(3)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_f16_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_f16_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_f16_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_f16_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY21]](s32)
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   G_STORE [[TRUNC]](s16), [[DEF]](p1) :: (volatile store 2 into `half addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call half @external_f16_func_void()
+  store volatile half %val, half addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_f32_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_f32_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_f32_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_f32_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   G_STORE [[COPY21]](s32), [[DEF]](p1) :: (volatile store 4 into `float addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call float @external_f32_func_void()
+  store volatile float %val, float addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_f64_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_f64_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_f64_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_f64_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0, implicit-def $vgpr1
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY21]](s32), [[COPY22]](s32)
+  ; GCN:   G_STORE [[MV]](s64), [[DEF]](p1) :: (volatile store 8 into `double addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call double @external_f64_func_void()
+  store volatile double %val, double addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_v2f64_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_v2f64_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_v2f64_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_v2f64_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0, implicit-def $vgpr1, implicit-def $vgpr2, implicit-def $vgpr3
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GCN:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GCN:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY21]](s32), [[COPY22]](s32)
+  ; GCN:   [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY23]](s32), [[COPY24]](s32)
+  ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
+  ; GCN:   G_STORE [[BUILD_VECTOR]](<2 x s64>), [[DEF]](p1) :: (volatile store 16 into `<2 x double> addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call <2 x double> @external_v2f64_func_void()
+  store volatile <2 x double> %val, <2 x double> addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_v2i32_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_v2i32_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_v2i32_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_v2i32_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0, implicit-def $vgpr1
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY21]](s32), [[COPY22]](s32)
+  ; GCN:   G_STORE [[BUILD_VECTOR]](<2 x s32>), [[DEF]](p1) :: (volatile store 8 into `<2 x i32> addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call <2 x i32> @external_v2i32_func_void()
+  store volatile <2 x i32> %val, <2 x i32> addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_v3i32_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_v3i32_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_v3i32_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_v3i32_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0, implicit-def $vgpr1, implicit-def $vgpr2
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GCN:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[COPY21]](s32), [[COPY22]](s32), [[COPY23]](s32)
+  ; GCN:   G_STORE [[BUILD_VECTOR]](<3 x s32>), [[DEF]](p1) :: (volatile store 12 into `<3 x i32> addrspace(1)* undef`, align 8, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call <3 x i32> @external_v3i32_func_void()
+  store volatile <3 x i32> %val, <3 x i32> addrspace(1)* undef, align 8
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_v4i32_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_v4i32_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_v4i32_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_v4i32_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0, implicit-def $vgpr1, implicit-def $vgpr2, implicit-def $vgpr3
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GCN:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GCN:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY21]](s32), [[COPY22]](s32), [[COPY23]](s32), [[COPY24]](s32)
+  ; GCN:   G_STORE [[BUILD_VECTOR]](<4 x s32>), [[DEF]](p1) :: (volatile store 16 into `<4 x i32> addrspace(1)* undef`, align 8, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call <4 x i32> @external_v4i32_func_void()
+  store volatile <4 x i32> %val, <4 x i32> addrspace(1)* undef, align 8
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_v5i32_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_v5i32_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_v5i32_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_v5i32_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0, implicit-def $vgpr1, implicit-def $vgpr2, implicit-def $vgpr3, implicit-def $vgpr4
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GCN:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GCN:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GCN:   [[COPY25:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[COPY21]](s32), [[COPY22]](s32), [[COPY23]](s32), [[COPY24]](s32), [[COPY25]](s32)
+  ; GCN:   G_STORE [[BUILD_VECTOR]](<5 x s32>), [[DEF]](p1) :: (volatile store 20 into `<5 x i32> addrspace(1)* undef`, align 8, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call <5 x i32> @external_v5i32_func_void()
+  store volatile <5 x i32> %val, <5 x i32> addrspace(1)* undef, align 8
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_v8i32_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_v8i32_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_v8i32_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_v8i32_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0, implicit-def $vgpr1, implicit-def $vgpr2, implicit-def $vgpr3, implicit-def $vgpr4, implicit-def $vgpr5, implicit-def $vgpr6, implicit-def $vgpr7
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GCN:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GCN:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GCN:   [[COPY25:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GCN:   [[COPY26:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; GCN:   [[COPY27:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; GCN:   [[COPY28:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY21]](s32), [[COPY22]](s32), [[COPY23]](s32), [[COPY24]](s32), [[COPY25]](s32), [[COPY26]](s32), [[COPY27]](s32), [[COPY28]](s32)
+  ; GCN:   G_STORE [[BUILD_VECTOR]](<8 x s32>), [[DEF]](p1) :: (volatile store 32 into `<8 x i32> addrspace(1)* undef`, align 8, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call <8 x i32> @external_v8i32_func_void()
+  store volatile <8 x i32> %val, <8 x i32> addrspace(1)* undef, align 8
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_v16i32_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_v16i32_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_v16i32_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_v16i32_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0, implicit-def $vgpr1, implicit-def $vgpr2, implicit-def $vgpr3, implicit-def $vgpr4, implicit-def $vgpr5, implicit-def $vgpr6, implicit-def $vgpr7, implicit-def $vgpr8, implicit-def $vgpr9, implicit-def $vgpr10, implicit-def $vgpr11, implicit-def $vgpr12, implicit-def $vgpr13, implicit-def $vgpr14, implicit-def $vgpr15
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GCN:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GCN:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GCN:   [[COPY25:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GCN:   [[COPY26:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; GCN:   [[COPY27:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; GCN:   [[COPY28:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; GCN:   [[COPY29:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; GCN:   [[COPY30:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; GCN:   [[COPY31:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; GCN:   [[COPY32:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; GCN:   [[COPY33:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; GCN:   [[COPY34:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; GCN:   [[COPY35:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; GCN:   [[COPY36:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<16 x s32>) = G_BUILD_VECTOR [[COPY21]](s32), [[COPY22]](s32), [[COPY23]](s32), [[COPY24]](s32), [[COPY25]](s32), [[COPY26]](s32), [[COPY27]](s32), [[COPY28]](s32), [[COPY29]](s32), [[COPY30]](s32), [[COPY31]](s32), [[COPY32]](s32), [[COPY33]](s32), [[COPY34]](s32), [[COPY35]](s32), [[COPY36]](s32)
+  ; GCN:   G_STORE [[BUILD_VECTOR]](<16 x s32>), [[DEF]](p1) :: (volatile store 64 into `<16 x i32> addrspace(1)* undef`, align 8, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call <16 x i32> @external_v16i32_func_void()
+  store volatile <16 x i32> %val, <16 x i32> addrspace(1)* undef, align 8
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_v32i32_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_v32i32_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_v32i32_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_v32i32_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0, implicit-def $vgpr1, implicit-def $vgpr2, implicit-def $vgpr3, implicit-def $vgpr4, implicit-def $vgpr5, implicit-def $vgpr6, implicit-def $vgpr7, implicit-def $vgpr8, implicit-def $vgpr9, implicit-def $vgpr10, implicit-def $vgpr11, implicit-def $vgpr12, implicit-def $vgpr13, implicit-def $vgpr14, implicit-def $vgpr15, implicit-def $vgpr16, implicit-def $vgpr17, implicit-def $vgpr18, implicit-def $vgpr19, implicit-def $vgpr20, implicit-def $vgpr21, implicit-def $vgpr22, implicit-def $vgpr23, implicit-def $vgpr24, implicit-def $vgpr25, implicit-def $vgpr26, implicit-def $vgpr27, implicit-def $vgpr28, implicit-def $vgpr29, implicit-def $vgpr30, implicit-def $vgpr31
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GCN:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GCN:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GCN:   [[COPY25:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GCN:   [[COPY26:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; GCN:   [[COPY27:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; GCN:   [[COPY28:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; GCN:   [[COPY29:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; GCN:   [[COPY30:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; GCN:   [[COPY31:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; GCN:   [[COPY32:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; GCN:   [[COPY33:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; GCN:   [[COPY34:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; GCN:   [[COPY35:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; GCN:   [[COPY36:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; GCN:   [[COPY37:%[0-9]+]]:_(s32) = COPY $vgpr16
+  ; GCN:   [[COPY38:%[0-9]+]]:_(s32) = COPY $vgpr17
+  ; GCN:   [[COPY39:%[0-9]+]]:_(s32) = COPY $vgpr18
+  ; GCN:   [[COPY40:%[0-9]+]]:_(s32) = COPY $vgpr19
+  ; GCN:   [[COPY41:%[0-9]+]]:_(s32) = COPY $vgpr20
+  ; GCN:   [[COPY42:%[0-9]+]]:_(s32) = COPY $vgpr21
+  ; GCN:   [[COPY43:%[0-9]+]]:_(s32) = COPY $vgpr22
+  ; GCN:   [[COPY44:%[0-9]+]]:_(s32) = COPY $vgpr23
+  ; GCN:   [[COPY45:%[0-9]+]]:_(s32) = COPY $vgpr24
+  ; GCN:   [[COPY46:%[0-9]+]]:_(s32) = COPY $vgpr25
+  ; GCN:   [[COPY47:%[0-9]+]]:_(s32) = COPY $vgpr26
+  ; GCN:   [[COPY48:%[0-9]+]]:_(s32) = COPY $vgpr27
+  ; GCN:   [[COPY49:%[0-9]+]]:_(s32) = COPY $vgpr28
+  ; GCN:   [[COPY50:%[0-9]+]]:_(s32) = COPY $vgpr29
+  ; GCN:   [[COPY51:%[0-9]+]]:_(s32) = COPY $vgpr30
+  ; GCN:   [[COPY52:%[0-9]+]]:_(s32) = COPY $vgpr31
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s32>) = G_BUILD_VECTOR [[COPY21]](s32), [[COPY22]](s32), [[COPY23]](s32), [[COPY24]](s32), [[COPY25]](s32), [[COPY26]](s32), [[COPY27]](s32), [[COPY28]](s32), [[COPY29]](s32), [[COPY30]](s32), [[COPY31]](s32), [[COPY32]](s32), [[COPY33]](s32), [[COPY34]](s32), [[COPY35]](s32), [[COPY36]](s32), [[COPY37]](s32), [[COPY38]](s32), [[COPY39]](s32), [[COPY40]](s32), [[COPY41]](s32), [[COPY42]](s32), [[COPY43]](s32), [[COPY44]](s32), [[COPY45]](s32), [[COPY46]](s32), [[COPY47]](s32), [[COPY48]](s32), [[COPY49]](s32), [[COPY50]](s32), [[COPY51]](s32), [[COPY52]](s32)
+  ; GCN:   G_STORE [[BUILD_VECTOR]](<32 x s32>), [[DEF]](p1) :: (volatile store 128 into `<32 x i32> addrspace(1)* undef`, align 8, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call <32 x i32> @external_v32i32_func_void()
+  store volatile <32 x i32> %val, <32 x i32> addrspace(1)* undef, align 8
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_v2i16_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_v2i16_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_v2i16_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_v2i16_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0
+  ; GCN:   [[COPY21:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   G_STORE [[COPY21]](<2 x s16>), [[DEF]](p1) :: (volatile store 4 into `<2 x i16> addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call <2 x i16> @external_v2i16_func_void()
+  store volatile <2 x i16> %val, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_v3i16_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_v3i16_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_v3i16_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_v3i16_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0, implicit-def $vgpr1
+  ; GCN:   [[COPY21:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+  ; GCN:   [[COPY22:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[DEF1:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
+  ; GCN:   [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[COPY21]](<2 x s16>), [[COPY22]](<2 x s16>), [[DEF1]](<2 x s16>)
+  ; GCN:   [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<6 x s16>)
+  ; GCN:   G_STORE [[UV]](<3 x s16>), [[DEF]](p1) :: (volatile store 6 into `<3 x i16> addrspace(1)* undef`, align 8, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call <3 x i16> @external_v3i16_func_void()
+  store volatile <3 x i16> %val, <3 x i16> addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_v4i16_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_v4i16_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_v4i16_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_v4i16_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0, implicit-def $vgpr1
+  ; GCN:   [[COPY21:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+  ; GCN:   [[COPY22:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[COPY21]](<2 x s16>), [[COPY22]](<2 x s16>)
+  ; GCN:   G_STORE [[CONCAT_VECTORS]](<4 x s16>), [[DEF]](p1) :: (volatile store 8 into `<4 x i16> addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call <4 x i16> @external_v4i16_func_void()
+  store volatile <4 x i16> %val, <4 x i16> addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_v2f16_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_v2f16_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_v2f16_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_v2f16_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0
+  ; GCN:   [[COPY21:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   G_STORE [[COPY21]](<2 x s16>), [[DEF]](p1) :: (volatile store 4 into `<2 x half> addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call <2 x half> @external_v2f16_func_void()
+  store volatile <2 x half> %val, <2 x half> addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_v3f16_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_v3f16_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_v3f16_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_v3f16_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0, implicit-def $vgpr1
+  ; GCN:   [[COPY21:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+  ; GCN:   [[COPY22:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[DEF1:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
+  ; GCN:   [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[COPY21]](<2 x s16>), [[COPY22]](<2 x s16>), [[DEF1]](<2 x s16>)
+  ; GCN:   [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<6 x s16>)
+  ; GCN:   G_STORE [[UV]](<3 x s16>), [[DEF]](p1) :: (volatile store 6 into `<3 x half> addrspace(1)* undef`, align 8, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call <3 x half> @external_v3f16_func_void()
+  store volatile <3 x half> %val, <3 x half> addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_v4f16_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_v4f16_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_v4f16_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_v4f16_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0, implicit-def $vgpr1
+  ; GCN:   [[COPY21:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+  ; GCN:   [[COPY22:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[COPY21]](<2 x s16>), [[COPY22]](<2 x s16>)
+  ; GCN:   G_STORE [[CONCAT_VECTORS]](<4 x s16>), [[DEF]](p1) :: (volatile store 8 into `<4 x half> addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call <4 x half> @external_v4f16_func_void()
+  store volatile <4 x half> %val, <4 x half> addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_v3f32_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_v3f32_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_v3f32_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_v3f32_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0, implicit-def $vgpr1, implicit-def $vgpr2
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GCN:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[COPY21]](s32), [[COPY22]](s32), [[COPY23]](s32)
+  ; GCN:   G_STORE [[BUILD_VECTOR]](<3 x s32>), [[DEF]](p1) :: (volatile store 12 into `<3 x float> addrspace(1)* undef`, align 16, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call <3 x float> @external_v3f32_func_void()
+  store volatile <3 x float> %val, <3 x float> addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_v5f32_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_v5f32_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_v5f32_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_v5f32_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0, implicit-def $vgpr1, implicit-def $vgpr2, implicit-def $vgpr3, implicit-def $vgpr4
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GCN:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GCN:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GCN:   [[COPY25:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[COPY21]](s32), [[COPY22]](s32), [[COPY23]](s32), [[COPY24]](s32), [[COPY25]](s32)
+  ; GCN:   G_STORE [[BUILD_VECTOR]](<5 x s32>), [[DEF]](p1) :: (volatile store 20 into `<5 x float> addrspace(1)* undef`, align 32, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call <5 x float> @external_v5f32_func_void()
+  store volatile <5 x float> %val, <5 x float> addrspace(1)* undef
+  ret void
+}
+
+
+define amdgpu_kernel void @test_call_external_i32_i64_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_i32_i64_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p1) = COPY [[DEF]](p1)
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_i32_i64_func_void
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY13:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY13]], [[C]](s64)
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY18]], [[SHL]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY20]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY21:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY21]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY12]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY14]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY16]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY17]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_i32_i64_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0, implicit-def $vgpr1, implicit-def $vgpr2
+  ; GCN:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GCN:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY23]](s32), [[COPY24]](s32)
+  ; GCN:   G_STORE [[COPY22]](s32), [[DEF]](p1) :: (volatile store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; GCN:   G_STORE [[MV]](s64), [[COPY10]](p1) :: (volatile store 8 into `i64 addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call { i32, i64 } @external_i32_i64_func_void()
+  %val.0 = extractvalue { i32, i64 } %val, 0
+  %val.1 = extractvalue { i32, i64 } %val, 1
+  store volatile i32 %val.0, i32 addrspace(1)* undef
+  store volatile i64 %val.1, i64 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_a2i32_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_a2i32_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_a2i32_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_a2i32_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0, implicit-def $vgpr1
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   G_STORE [[COPY21]](s32), [[DEF]](p1) :: (volatile store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; GCN:   G_STORE [[COPY22]](s32), [[DEF]](p1) :: (volatile store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call [2 x i32] @external_a2i32_func_void()
+  %val.0 = extractvalue [2 x i32] %val, 0
+  %val.1 = extractvalue [2 x i32] %val, 1
+  store volatile i32 %val.0, i32 addrspace(1)* undef
+  store volatile i32 %val.1, i32 addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_a5i8_func_void() #0 {
+  ; GCN-LABEL: name: test_call_external_a5i8_func_void
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_a5i8_func_void
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; GCN:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY14]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY16]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_a5i8_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31, implicit-def $vgpr0, implicit-def $vgpr1, implicit-def $vgpr2, implicit-def $vgpr3, implicit-def $vgpr4
+  ; GCN:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GCN:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY21]](s32)
+  ; GCN:   [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC]](s16)
+  ; GCN:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GCN:   [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY22]](s32)
+  ; GCN:   [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC2]](s16)
+  ; GCN:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GCN:   [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[COPY23]](s32)
+  ; GCN:   [[TRUNC5:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC4]](s16)
+  ; GCN:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GCN:   [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[COPY24]](s32)
+  ; GCN:   [[TRUNC7:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC6]](s16)
+  ; GCN:   [[COPY25:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GCN:   [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[COPY25]](s32)
+  ; GCN:   [[TRUNC9:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC8]](s16)
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   G_STORE [[TRUNC1]](s8), [[DEF]](p1) :: (volatile store 1 into `i8 addrspace(1)* undef`, addrspace 1)
+  ; GCN:   G_STORE [[TRUNC3]](s8), [[DEF]](p1) :: (volatile store 1 into `i8 addrspace(1)* undef`, addrspace 1)
+  ; GCN:   G_STORE [[TRUNC5]](s8), [[DEF]](p1) :: (volatile store 1 into `i8 addrspace(1)* undef`, addrspace 1)
+  ; GCN:   G_STORE [[TRUNC7]](s8), [[DEF]](p1) :: (volatile store 1 into `i8 addrspace(1)* undef`, addrspace 1)
+  ; GCN:   G_STORE [[TRUNC9]](s8), [[DEF]](p1) :: (volatile store 1 into `i8 addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %val = call [5 x i8] @external_a5i8_func_void()
+  %val.0 = extractvalue [5 x i8] %val, 0
+  %val.1 = extractvalue [5 x i8] %val, 1
+  %val.2 = extractvalue [5 x i8] %val, 2
+  %val.3 = extractvalue [5 x i8] %val, 3
+  %val.4 = extractvalue [5 x i8] %val, 4
+  store volatile i8 %val.0, i8 addrspace(1)* undef
+  store volatile i8 %val.1, i8 addrspace(1)* undef
+  store volatile i8 %val.2, i8 addrspace(1)* undef
+  store volatile i8 %val.3, i8 addrspace(1)* undef
+  store volatile i8 %val.4, i8 addrspace(1)* undef
+  ret void
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind noinline }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-sret.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-sret.ll
new file mode 100644
index 0000000000000..d53cfe688f53c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-sret.ll
@@ -0,0 +1,85 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -amdgpu-fixed-function-abi -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -stop-after=irtranslator -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+declare hidden void @external_void_func_sret_struct_i8_i32_byval_struct_i8_i32({ i8, i32 } addrspace(5)* sret, { i8, i32 } addrspace(5)* byval) #0
+
+define amdgpu_kernel void @test_call_external_void_func_sret_struct_i8_i32_byval_struct_i8_i32(i32) #0 {
+  ; GCN-LABEL: name: test_call_external_void_func_sret_struct_i8_i32_byval_struct_i8_i32
+  ; GCN: bb.1 (%ir-block.1):
+  ; GCN:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GCN:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; GCN:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; GCN:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; GCN:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; GCN:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; GCN:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; GCN:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; GCN:   [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 3
+  ; GCN:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+  ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; GCN:   [[COPY10:%[0-9]+]]:_(p1) = COPY [[DEF]](p1)
+  ; GCN:   [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %stack.0.in.val
+  ; GCN:   [[FRAME_INDEX1:%[0-9]+]]:_(p5) = G_FRAME_INDEX %stack.1.out.val
+  ; GCN:   [[INT:%[0-9]+]]:_(p4) = G_INTRINSIC intrinsic(@llvm.amdgcn.kernarg.segment.ptr)
+  ; GCN:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; GCN:   [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C2]](s32)
+  ; GCN:   G_STORE [[C]](s8), [[FRAME_INDEX]](p5) :: (store 1 into %ir.in.gep01, addrspace 5)
+  ; GCN:   G_STORE [[C1]](s32), [[PTR_ADD]](p5) :: (store 4 into %ir.in.gep1, addrspace 5)
+  ; GCN:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; GCN:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_sret_struct_i8_i32_byval_struct_i8_i32
+  ; GCN:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; GCN:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; GCN:   [[COPY13:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; GCN:   [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; GCN:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY13]], [[C3]](s64)
+  ; GCN:   [[COPY14:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; GCN:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; GCN:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; GCN:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; GCN:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; GCN:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; GCN:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GCN:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C4]](s32)
+  ; GCN:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY18]], [[SHL]]
+  ; GCN:   [[COPY20:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; GCN:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; GCN:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY20]], [[C5]](s32)
+  ; GCN:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; GCN:   $vgpr0 = COPY [[FRAME_INDEX1]](p5)
+  ; GCN:   $vgpr1 = COPY [[FRAME_INDEX]](p5)
+  ; GCN:   [[COPY21:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; GCN:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY21]](<4 x s32>)
+  ; GCN:   $sgpr4_sgpr5 = COPY [[COPY11]](p4)
+  ; GCN:   $sgpr6_sgpr7 = COPY [[COPY12]](p4)
+  ; GCN:   $sgpr8_sgpr9 = COPY [[PTR_ADD1]](p4)
+  ; GCN:   $sgpr10_sgpr11 = COPY [[COPY14]](s64)
+  ; GCN:   $sgpr12 = COPY [[COPY15]](s32)
+  ; GCN:   $sgpr13 = COPY [[COPY16]](s32)
+  ; GCN:   $sgpr14 = COPY [[COPY17]](s32)
+  ; GCN:   $vgpr31 = COPY [[OR1]](s32)
+  ; GCN:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_sret_struct_i8_i32_byval_struct_i8_i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; GCN:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; GCN:   [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX1]], [[C2]](s32)
+  ; GCN:   [[LOAD:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX1]](p5) :: (dereferenceable load 1 from %ir.out.gep02, addrspace 5)
+  ; GCN:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (dereferenceable load 4 from %ir.out.gep1, addrspace 5)
+  ; GCN:   G_STORE [[LOAD]](s8), [[DEF]](p1) :: (volatile store 1 into `i8 addrspace(1)* undef`, addrspace 1)
+  ; GCN:   G_STORE [[LOAD1]](s32), [[COPY10]](p1) :: (volatile store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+  ; GCN:   S_ENDPGM 0
+  %in.val = alloca { i8, i32 }, align 4, addrspace(5)
+  %out.val = alloca { i8, i32 }, align 4, addrspace(5)
+  %in.gep0 = getelementptr inbounds { i8, i32 }, { i8, i32 } addrspace(5)* %in.val, i32 0, i32 0
+  %in.gep1 = getelementptr inbounds { i8, i32 }, { i8, i32 } addrspace(5)* %in.val, i32 0, i32 1
+  store i8 3, i8 addrspace(5)* %in.gep0
+  store i32 8, i32 addrspace(5)* %in.gep1
+  call void @external_void_func_sret_struct_i8_i32_byval_struct_i8_i32({ i8, i32 } addrspace(5)* %out.val, { i8, i32 } addrspace(5)* %in.val)
+  %out.gep0 = getelementptr inbounds { i8, i32 }, { i8, i32 } addrspace(5)* %out.val, i32 0, i32 0
+  %out.gep1 = getelementptr inbounds { i8, i32 }, { i8, i32 } addrspace(5)* %out.val, i32 0, i32 1
+  %out.val0 = load i8, i8 addrspace(5)* %out.gep0
+  %out.val1 = load i32, i32 addrspace(5)* %out.gep1
+  store volatile i8 %out.val0, i8 addrspace(1)* undef
+  store volatile i32 %out.val1, i32 addrspace(1)* undef
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll
new file mode 100644
index 0000000000000..6b29697ca086e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll
@@ -0,0 +1,4934 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -amdgpu-fixed-function-abi -stop-after=irtranslator -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -verify-machineinstrs -o - %s | FileCheck -enable-var-scope %s
+
+declare hidden void @external_void_func_void() #0
+
+declare hidden void @external_void_func_empty_struct({}, i32) #0
+declare hidden void @external_void_func_empty_array([0 x i8], i32) #0
+
+declare hidden void @external_void_func_i1(i1) #0
+declare hidden void @external_void_func_i1_signext(i1 signext) #0
+declare hidden void @external_void_func_i1_zeroext(i1 zeroext) #0
+
+declare hidden void @external_void_func_i8(i8) #0
+declare hidden void @external_void_func_i8_signext(i8 signext) #0
+declare hidden void @external_void_func_i8_zeroext(i8 zeroext) #0
+
+declare hidden void @external_void_func_i16(i16) #0
+declare hidden void @external_void_func_i16_signext(i16 signext) #0
+declare hidden void @external_void_func_i16_zeroext(i16 zeroext) #0
+
+declare hidden void @external_void_func_i32(i32) #0
+declare hidden void @external_void_func_i64(i64) #0
+declare hidden void @external_void_func_v2i64(<2 x i64>) #0
+declare hidden void @external_void_func_v3i64(<3 x i64>) #0
+declare hidden void @external_void_func_v4i64(<4 x i64>) #0
+
+
+declare hidden void @external_void_func_i48(i48) #0
+declare hidden void @external_void_func_i48_signext(i48 signext) #0
+declare hidden void @external_void_func_i48_zeroext(i48 zeroext) #0
+
+declare hidden void @external_void_func_p0(i8*) #0
+declare hidden void @external_void_func_v2p0(<2 x i8*>) #0
+
+declare hidden void @external_void_func_f16(half) #0
+declare hidden void @external_void_func_f32(float) #0
+declare hidden void @external_void_func_f64(double) #0
+declare hidden void @external_void_func_v2f32(<2 x float>) #0
+declare hidden void @external_void_func_v2f64(<2 x double>) #0
+declare hidden void @external_void_func_v3f32(<3 x float>) #0
+declare hidden void @external_void_func_v3f64(<3 x double>) #0
+declare hidden void @external_void_func_v5f32(<5 x float>) #0
+
+declare hidden void @external_void_func_v2i16(<2 x i16>) #0
+declare hidden void @external_void_func_v2f16(<2 x half>) #0
+declare hidden void @external_void_func_v3i16(<3 x i16>) #0
+declare hidden void @external_void_func_v3f16(<3 x half>) #0
+declare hidden void @external_void_func_v4i16(<4 x i16>) #0
+declare hidden void @external_void_func_v4f16(<4 x half>) #0
+declare hidden void @external_void_func_v5i16(<5 x i16>) #0
+declare hidden void @external_void_func_v7i16(<7 x i16>) #0
+declare hidden void @external_void_func_v63i16(<63 x i16>) #0
+declare hidden void @external_void_func_v65i16(<65 x i16>) #0
+declare hidden void @external_void_func_v66i16(<66 x i16>) #0
+
+declare hidden void @external_void_func_v2i32(<2 x i32>) #0
+declare hidden void @external_void_func_v3i32(<3 x i32>) #0
+declare hidden void @external_void_func_v3i32_i32(<3 x i32>, i32) #0
+declare hidden void @external_void_func_v4i32(<4 x i32>) #0
+declare hidden void @external_void_func_v5i32(<5 x i32>) #0
+declare hidden void @external_void_func_v8i32(<8 x i32>) #0
+declare hidden void @external_void_func_v16i32(<16 x i32>) #0
+declare hidden void @external_void_func_v32i32(<32 x i32>) #0
+declare hidden void @external_void_func_v32i32_i32(<32 x i32>, i32) #0
+declare hidden void @external_void_func_v32i32_p3_p5(<32 x i32>, i8 addrspace(3)*, i8 addrspace(5)*) #0
+declare hidden void @external_void_func_v32i32_i8_i8_i16(<32 x i32>, i8, i8, i16) #0
+
+; Structs
+declare hidden void @external_void_func_struct_i8_i32({ i8, i32 }) #0
+declare hidden void @external_void_func_byval_struct_i8_i32({ i8, i32 } addrspace(5)* byval) #0
+declare hidden void @external_void_func_sret_struct_i8_i32_byval_struct_i8_i32({ i8, i32 } addrspace(5)* sret, { i8, i32 } addrspace(5)* byval) #0
+
+declare hidden void @external_void_func_v2i8(<2 x i8>) #0
+declare hidden void @external_void_func_v3i8(<3 x i8>) #0
+declare hidden void @external_void_func_v4i8(<4 x i8>) #0
+declare hidden void @external_void_func_v8i8(<8 x i8>) #0
+declare hidden void @external_void_func_v16i8(<16 x i8>) #0
+
+declare hidden void @byval_align16_f64_arg(<32 x i32>, double addrspace(5)* byval align 16) #0
+declare hidden void @stack_passed_f64_arg(<32 x i32>, double) #0
+declare hidden void @external_void_func_12xv3i32(<3 x i32>, <3 x i32>, <3 x i32>, <3 x i32>,
+    <3 x i32>, <3 x i32>, <3 x i32>, <3 x i32>, <3 x i32>, <3 x i32>, <3 x i32>, <3 x i32>) #0
+declare hidden void @external_void_func_8xv5i32(<5 x i32>, <5 x i32>, <5 x i32>, <5 x i32>,
+    <5 x i32>, <5 x i32>, <5 x i32>, <5 x i32>) #0
+declare hidden void @external_void_func_12xv3f32(<3 x float>, <3 x float>, <3 x float>, <3 x float>,
+    <3 x float>, <3 x float>, <3 x float>, <3 x float>, <3 x float>, <3 x float>, <3 x float>, <3 x float>) #0
+declare hidden void @external_void_func_8xv5f32(<5 x float>, <5 x float>, <5 x float>, <5 x float>,
+    <5 x float>, <5 x float>, <5 x float>, <5 x float>) #0
+
+define amdgpu_kernel void @test_call_external_void_func_void() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_void
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  call void @external_void_func_void()
+  ret void
+}
+
+define void @test_func_call_external_void_func_void() #0 {
+  ; CHECK-LABEL: name: test_func_call_external_void_func_void
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr12, $sgpr13, $sgpr14, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr31
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr13
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr12
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_64 = COPY $sgpr8_sgpr9
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_void
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY6]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY5]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s64) = COPY [[COPY4]]
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY2]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY1]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY17]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY9]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY12]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY13]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY15]](s32)
+  ; CHECK:   $vgpr31 = COPY [[COPY16]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_void, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   [[COPY18:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY8]]
+  ; CHECK:   S_SETPC_B64_return [[COPY18]]
+  call void @external_void_func_void()
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_empty_struct() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_empty_struct
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 23
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_empty_struct
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[C]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_empty_struct, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  call void @external_void_func_empty_struct({} zeroinitializer, i32 23)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_empty_array() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_empty_array
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 23
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_empty_array
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[C]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_empty_array, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  call void @external_void_func_empty_array([0 x i8] zeroinitializer, i32 23)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_i1_imm() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_i1_imm
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[C]](s1)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i1
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i1, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  call void @external_void_func_i1(i1 true)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_i1_signext(i32) #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_i1_signext
+  ; CHECK: bb.1 (%ir-block.1):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[INT:%[0-9]+]]:_(p4) = G_INTRINSIC intrinsic(@llvm.amdgcn.kernarg.segment.ptr)
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s1) = G_LOAD [[DEF]](p1) :: (volatile load 1 from `i1 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[LOAD]](s1)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i1_signext
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[SEXT]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i1_signext, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %var = load volatile i1, i1 addrspace(1)* undef
+  call void @external_void_func_i1_signext(i1 signext %var)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_i1_zeroext(i32) #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_i1_zeroext
+  ; CHECK: bb.1 (%ir-block.1):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[INT:%[0-9]+]]:_(p4) = G_INTRINSIC intrinsic(@llvm.amdgcn.kernarg.segment.ptr)
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s1) = G_LOAD [[DEF]](p1) :: (volatile load 1 from `i1 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD]](s1)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i1_zeroext
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[ZEXT]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i1_zeroext, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %var = load volatile i1, i1 addrspace(1)* undef
+  call void @external_void_func_i1_zeroext(i1 zeroext %var)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_i8_imm(i32) #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_i8_imm
+  ; CHECK: bb.1 (%ir-block.1):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 123
+  ; CHECK:   [[INT:%[0-9]+]]:_(p4) = G_INTRINSIC intrinsic(@llvm.amdgcn.kernarg.segment.ptr)
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[C]](s8)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i8
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i8, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  call void @external_void_func_i8(i8 123)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_i8_signext(i32) #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_i8_signext
+  ; CHECK: bb.1 (%ir-block.1):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[INT:%[0-9]+]]:_(p4) = G_INTRINSIC intrinsic(@llvm.amdgcn.kernarg.segment.ptr)
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s8) = G_LOAD [[DEF]](p1) :: (volatile load 1 from `i8 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[LOAD]](s8)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i8_signext
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[SEXT]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i8_signext, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %var = load volatile i8, i8 addrspace(1)* undef
+  call void @external_void_func_i8_signext(i8 signext %var)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_i8_zeroext(i32) #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_i8_zeroext
+  ; CHECK: bb.1 (%ir-block.1):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[INT:%[0-9]+]]:_(p4) = G_INTRINSIC intrinsic(@llvm.amdgcn.kernarg.segment.ptr)
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s8) = G_LOAD [[DEF]](p1) :: (volatile load 1 from `i8 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD]](s8)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i8_zeroext
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[ZEXT]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i8_zeroext, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %var = load volatile i8, i8 addrspace(1)* undef
+  call void @external_void_func_i8_zeroext(i8 zeroext %var)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_i16_imm() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_i16_imm
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 123
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[C]](s16)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i16
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i16, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  call void @external_void_func_i16(i16 123)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_i16_signext(i32) #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_i16_signext
+  ; CHECK: bb.1 (%ir-block.1):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[INT:%[0-9]+]]:_(p4) = G_INTRINSIC intrinsic(@llvm.amdgcn.kernarg.segment.ptr)
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s16) = G_LOAD [[DEF]](p1) :: (volatile load 2 from `i16 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[LOAD]](s16)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i16_signext
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[SEXT]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i16_signext, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %var = load volatile i16, i16 addrspace(1)* undef
+  call void @external_void_func_i16_signext(i16 signext %var)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_i16_zeroext(i32) #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_i16_zeroext
+  ; CHECK: bb.1 (%ir-block.1):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[INT:%[0-9]+]]:_(p4) = G_INTRINSIC intrinsic(@llvm.amdgcn.kernarg.segment.ptr)
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s16) = G_LOAD [[DEF]](p1) :: (volatile load 2 from `i16 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD]](s16)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i16_zeroext
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[ZEXT]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i16_zeroext, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %var = load volatile i16, i16 addrspace(1)* undef
+  call void @external_void_func_i16_zeroext(i16 zeroext %var)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_i32_imm(i32) #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_i32_imm
+  ; CHECK: bb.1 (%ir-block.1):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
+  ; CHECK:   [[INT:%[0-9]+]]:_(p4) = G_INTRINSIC intrinsic(@llvm.amdgcn.kernarg.segment.ptr)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i32
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[C]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  call void @external_void_func_i32(i32 42)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_i64_imm() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_i64_imm
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 123
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C]](s64)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i64
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i64, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  call void @external_void_func_i64(i64 123)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v2i64() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v2i64
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[C]](p1) :: (load 16 from `<2 x i64> addrspace(1)* null`, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD]](<2 x s64>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v2i64
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v2i64, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %val = load <2 x i64>, <2 x i64> addrspace(1)* null
+  call void @external_void_func_v2i64(<2 x i64> %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v2i64_imm() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v2i64_imm
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8589934593
+  ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 17179869187
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C1]](s64)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<2 x s64>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v2i64
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C2]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C3]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C4]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v2i64, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  call void @external_void_func_v2i64(<2 x i64> <i64 8589934593, i64 17179869187>)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_i48(i32) #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_i48
+  ; CHECK: bb.1 (%ir-block.1):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[INT:%[0-9]+]]:_(p4) = G_INTRINSIC intrinsic(@llvm.amdgcn.kernarg.segment.ptr)
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s48) = G_LOAD [[DEF]](p1) :: (volatile load 6 from `i48 addrspace(1)* undef`, align 8, addrspace 1)
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[LOAD]](s48)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ANYEXT]](s64)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i48
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i48, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %var = load volatile i48, i48 addrspace(1)* undef
+  call void @external_void_func_i48(i48 %var)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_i48_signext(i32) #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_i48_signext
+  ; CHECK: bb.1 (%ir-block.1):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[INT:%[0-9]+]]:_(p4) = G_INTRINSIC intrinsic(@llvm.amdgcn.kernarg.segment.ptr)
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s48) = G_LOAD [[DEF]](p1) :: (volatile load 6 from `i48 addrspace(1)* undef`, align 8, addrspace 1)
+  ; CHECK:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[LOAD]](s48)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[SEXT]](s64)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i48_signext
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i48_signext, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %var = load volatile i48, i48 addrspace(1)* undef
+  call void @external_void_func_i48_signext(i48 signext %var)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_i48_zeroext(i32) #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_i48_zeroext
+  ; CHECK: bb.1 (%ir-block.1):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[INT:%[0-9]+]]:_(p4) = G_INTRINSIC intrinsic(@llvm.amdgcn.kernarg.segment.ptr)
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s48) = G_LOAD [[DEF]](p1) :: (volatile load 6 from `i48 addrspace(1)* undef`, align 8, addrspace 1)
+  ; CHECK:   [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[LOAD]](s48)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ZEXT]](s64)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_i48_zeroext
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_i48_zeroext, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %var = load volatile i48, i48 addrspace(1)* undef
+  call void @external_void_func_i48_zeroext(i48 zeroext %var)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_p0_imm(i8* %arg) #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_p0_imm
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[INT:%[0-9]+]]:_(p4) = G_INTRINSIC intrinsic(@llvm.amdgcn.kernarg.segment.ptr)
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[INT]](p4) :: (dereferenceable invariant load 8 from %ir.arg.kernarg.offset.cast, align 16, addrspace 4)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD]](p0)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_p0
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_p0, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  call void @external_void_func_p0(i8* %arg)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v2p0() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v2p0
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<2 x p0>) = G_LOAD [[C]](p1) :: (load 16 from `<2 x i8*> addrspace(1)* null`, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD]](<2 x p0>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v2p0
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v2p0, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %val = load <2 x i8*>, <2 x i8*> addrspace(1)* null
+  call void @external_void_func_v2p0(<2 x i8*> %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v3i64() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v3i64
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+  ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8589934593
+  ; CHECK:   [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C1]](s64), [[DEF]](s64)
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[C]](p1) :: (load 16 from `<2 x i64> addrspace(1)* null`, addrspace 1)
+  ; CHECK:   [[SHUF:%[0-9]+]]:_(<3 x s64>) = G_SHUFFLE_VECTOR [[LOAD]](<2 x s64>), [[BUILD_VECTOR]], shufflemask(0, 1, 2)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[SHUF]](<3 x s64>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v3i64
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C2]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C3]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C4]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](s32)
+  ; CHECK:   $vgpr5 = COPY [[UV5]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v3i64, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %load = load <2 x i64>, <2 x i64> addrspace(1)* null
+  %val = shufflevector <2 x i64> %load, <2 x i64> <i64 8589934593, i64 undef>, <3 x i32> <i32 0, i32 1, i32 2>
+
+  call void @external_void_func_v3i64(<3 x i64> %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v4i64() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v4i64
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+  ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8589934593
+  ; CHECK:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 17179869187
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C1]](s64), [[C2]](s64)
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[C]](p1) :: (load 16 from `<2 x i64> addrspace(1)* null`, addrspace 1)
+  ; CHECK:   [[SHUF:%[0-9]+]]:_(<4 x s64>) = G_SHUFFLE_VECTOR [[LOAD]](<2 x s64>), [[BUILD_VECTOR]], shufflemask(0, 1, 2, 3)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[SHUF]](<4 x s64>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v4i64
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C3]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C4]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C5]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](s32)
+  ; CHECK:   $vgpr5 = COPY [[UV5]](s32)
+  ; CHECK:   $vgpr6 = COPY [[UV6]](s32)
+  ; CHECK:   $vgpr7 = COPY [[UV7]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v4i64, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %load = load <2 x i64>, <2 x i64> addrspace(1)* null
+  %val = shufflevector <2 x i64> %load, <2 x i64> <i64 8589934593, i64 17179869187>, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  call void @external_void_func_v4i64(<4 x i64> %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_f16_imm() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_f16_imm
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(s16) = G_FCONSTANT half 0xH4400
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_f16
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[C]](s16)
+  ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_f16, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  call void @external_void_func_f16(half 4.0)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_f32_imm() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_f32_imm
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 4.000000e+00
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_f32
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[C]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_f32, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  call void @external_void_func_f32(float 4.0)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v2f32_imm() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v2f32_imm
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 2.000000e+00
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C1]](s32)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<2 x s32>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v2f32
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C2]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C3]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C4]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v2f32, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  call void @external_void_func_v2f32(<2 x float> <float 1.0, float 2.0>)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v3f32_imm() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v3f32_imm
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 2.000000e+00
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 4.000000e+00
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C1]](s32), [[C2]](s32)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<3 x s32>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v3f32
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C3]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C4]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C5]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v3f32, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  call void @external_void_func_v3f32(<3 x float> <float 1.0, float 2.0, float 4.0>)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v5f32_imm() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v5f32_imm
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 2.000000e+00
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 4.000000e+00
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float -1.000000e+00
+  ; CHECK:   [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 5.000000e-01
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C1]](s32), [[C2]](s32), [[C3]](s32), [[C4]](s32)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<5 x s32>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v5f32
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C5]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C6]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C7]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v5f32, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  call void @external_void_func_v5f32(<5 x float> <float 1.0, float 2.0, float 4.0, float -1.0, float 0.5>)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_f64_imm() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_f64_imm
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double 4.000000e+00
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C]](s64)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_f64
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_f64, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  call void @external_void_func_f64(double 4.0)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v2f64_imm() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v2f64_imm
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double 2.000000e+00
+  ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_FCONSTANT double 4.000000e+00
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C1]](s64)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<2 x s64>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v2f64
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C2]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C3]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C4]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v2f64, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  call void @external_void_func_v2f64(<2 x double> <double 2.0, double 4.0>)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v3f64_imm() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v3f64_imm
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double 2.000000e+00
+  ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_FCONSTANT double 4.000000e+00
+  ; CHECK:   [[C2:%[0-9]+]]:_(s64) = G_FCONSTANT double 8.000000e+00
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C1]](s64), [[C2]](s64)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<3 x s64>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v3f64
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C3]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C4]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C5]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](s32)
+  ; CHECK:   $vgpr5 = COPY [[UV5]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v3f64, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  call void @external_void_func_v3f64(<3 x double> <double 2.0, double 4.0, double 8.0>)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v2i16() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v2i16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[DEF]](p1) :: (load 4 from `<2 x i16> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v2i16
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[LOAD]](<2 x s16>)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v2i16, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %val = load <2 x i16>, <2 x i16> addrspace(1)* undef
+  call void @external_void_func_v2i16(<2 x i16> %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v3i16() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v3i16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<3 x s16>) = G_LOAD [[DEF]](p1) :: (load 6 from `<3 x i16> addrspace(1)* undef`, align 8, addrspace 1)
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+  ; CHECK:   [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[LOAD]](<3 x s16>), 0
+  ; CHECK:   [[EXTRACT:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<4 x s16>), 0
+  ; CHECK:   [[EXTRACT1:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<4 x s16>), 32
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v3i16
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[EXTRACT]](<2 x s16>)
+  ; CHECK:   $vgpr1 = COPY [[EXTRACT1]](<2 x s16>)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v3i16, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %val = load <3 x i16>, <3 x i16> addrspace(1)* undef
+  call void @external_void_func_v3i16(<3 x i16> %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v3f16() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v3f16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<3 x s16>) = G_LOAD [[DEF]](p1) :: (load 6 from `<3 x half> addrspace(1)* undef`, align 8, addrspace 1)
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+  ; CHECK:   [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[LOAD]](<3 x s16>), 0
+  ; CHECK:   [[EXTRACT:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<4 x s16>), 0
+  ; CHECK:   [[EXTRACT1:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<4 x s16>), 32
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v3f16
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[EXTRACT]](<2 x s16>)
+  ; CHECK:   $vgpr1 = COPY [[EXTRACT1]](<2 x s16>)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v3f16, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %val = load <3 x half>, <3 x half> addrspace(1)* undef
+  call void @external_void_func_v3f16(<3 x half> %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v4i16() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v4i16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[DEF]](p1) :: (load 8 from `<4 x i16> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[LOAD]](<4 x s16>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v4i16
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](<2 x s16>)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](<2 x s16>)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v4i16, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %val = load <4 x i16>, <4 x i16> addrspace(1)* undef
+  call void @external_void_func_v4i16(<4 x i16> %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v4i16_imm() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v4i16_imm
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 1
+  ; CHECK:   [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 2
+  ; CHECK:   [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 3
+  ; CHECK:   [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 4
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C1]](s16), [[C2]](s16), [[C3]](s16)
+  ; CHECK:   [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s16>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v4i16
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C4]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C5]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C6]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](<2 x s16>)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](<2 x s16>)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v4i16, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  call void @external_void_func_v4i16(<4 x i16> <i16 1, i16 2, i16 3, i16 4>)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v5i16() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v5i16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<5 x s16>) = G_LOAD [[DEF]](p1) :: (load 10 from `<5 x i16> addrspace(1)* undef`, align 16, addrspace 1)
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(<6 x s16>) = G_IMPLICIT_DEF
+  ; CHECK:   [[INSERT:%[0-9]+]]:_(<6 x s16>) = G_INSERT [[DEF1]], [[LOAD]](<5 x s16>), 0
+  ; CHECK:   [[EXTRACT:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<6 x s16>), 0
+  ; CHECK:   [[EXTRACT1:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<6 x s16>), 32
+  ; CHECK:   [[EXTRACT2:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<6 x s16>), 64
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v5i16
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[EXTRACT]](<2 x s16>)
+  ; CHECK:   $vgpr1 = COPY [[EXTRACT1]](<2 x s16>)
+  ; CHECK:   $vgpr2 = COPY [[EXTRACT2]](<2 x s16>)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v5i16, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %val = load <5 x i16>, <5 x i16> addrspace(1)* undef
+  call void @external_void_func_v5i16(<5 x i16> %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v7i16() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v7i16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<7 x s16>) = G_LOAD [[DEF]](p1) :: (load 14 from `<7 x i16> addrspace(1)* undef`, align 16, addrspace 1)
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(<8 x s16>) = G_IMPLICIT_DEF
+  ; CHECK:   [[INSERT:%[0-9]+]]:_(<8 x s16>) = G_INSERT [[DEF1]], [[LOAD]](<7 x s16>), 0
+  ; CHECK:   [[EXTRACT:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<8 x s16>), 0
+  ; CHECK:   [[EXTRACT1:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<8 x s16>), 32
+  ; CHECK:   [[EXTRACT2:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<8 x s16>), 64
+  ; CHECK:   [[EXTRACT3:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<8 x s16>), 96
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v7i16
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[EXTRACT]](<2 x s16>)
+  ; CHECK:   $vgpr1 = COPY [[EXTRACT1]](<2 x s16>)
+  ; CHECK:   $vgpr2 = COPY [[EXTRACT2]](<2 x s16>)
+  ; CHECK:   $vgpr3 = COPY [[EXTRACT3]](<2 x s16>)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v7i16, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %val = load <7 x i16>, <7 x i16> addrspace(1)* undef
+  call void @external_void_func_v7i16(<7 x i16> %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v63i16() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v63i16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<63 x s16>) = G_LOAD [[DEF]](p1) :: (load 126 from `<63 x i16> addrspace(1)* undef`, align 128, addrspace 1)
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(<64 x s16>) = G_IMPLICIT_DEF
+  ; CHECK:   [[INSERT:%[0-9]+]]:_(<64 x s16>) = G_INSERT [[DEF1]], [[LOAD]](<63 x s16>), 0
+  ; CHECK:   [[EXTRACT:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 0
+  ; CHECK:   [[EXTRACT1:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 32
+  ; CHECK:   [[EXTRACT2:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 64
+  ; CHECK:   [[EXTRACT3:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 96
+  ; CHECK:   [[EXTRACT4:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 128
+  ; CHECK:   [[EXTRACT5:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 160
+  ; CHECK:   [[EXTRACT6:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 192
+  ; CHECK:   [[EXTRACT7:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 224
+  ; CHECK:   [[EXTRACT8:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 256
+  ; CHECK:   [[EXTRACT9:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 288
+  ; CHECK:   [[EXTRACT10:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 320
+  ; CHECK:   [[EXTRACT11:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 352
+  ; CHECK:   [[EXTRACT12:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 384
+  ; CHECK:   [[EXTRACT13:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 416
+  ; CHECK:   [[EXTRACT14:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 448
+  ; CHECK:   [[EXTRACT15:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 480
+  ; CHECK:   [[EXTRACT16:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 512
+  ; CHECK:   [[EXTRACT17:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 544
+  ; CHECK:   [[EXTRACT18:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 576
+  ; CHECK:   [[EXTRACT19:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 608
+  ; CHECK:   [[EXTRACT20:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 640
+  ; CHECK:   [[EXTRACT21:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 672
+  ; CHECK:   [[EXTRACT22:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 704
+  ; CHECK:   [[EXTRACT23:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 736
+  ; CHECK:   [[EXTRACT24:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 768
+  ; CHECK:   [[EXTRACT25:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 800
+  ; CHECK:   [[EXTRACT26:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 832
+  ; CHECK:   [[EXTRACT27:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 864
+  ; CHECK:   [[EXTRACT28:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 896
+  ; CHECK:   [[EXTRACT29:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 928
+  ; CHECK:   [[EXTRACT30:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 960
+  ; CHECK:   [[EXTRACT31:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<64 x s16>), 992
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v63i16
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[EXTRACT]](<2 x s16>)
+  ; CHECK:   $vgpr1 = COPY [[EXTRACT1]](<2 x s16>)
+  ; CHECK:   $vgpr2 = COPY [[EXTRACT2]](<2 x s16>)
+  ; CHECK:   $vgpr3 = COPY [[EXTRACT3]](<2 x s16>)
+  ; CHECK:   $vgpr4 = COPY [[EXTRACT4]](<2 x s16>)
+  ; CHECK:   $vgpr5 = COPY [[EXTRACT5]](<2 x s16>)
+  ; CHECK:   $vgpr6 = COPY [[EXTRACT6]](<2 x s16>)
+  ; CHECK:   $vgpr7 = COPY [[EXTRACT7]](<2 x s16>)
+  ; CHECK:   $vgpr8 = COPY [[EXTRACT8]](<2 x s16>)
+  ; CHECK:   $vgpr9 = COPY [[EXTRACT9]](<2 x s16>)
+  ; CHECK:   $vgpr10 = COPY [[EXTRACT10]](<2 x s16>)
+  ; CHECK:   $vgpr11 = COPY [[EXTRACT11]](<2 x s16>)
+  ; CHECK:   $vgpr12 = COPY [[EXTRACT12]](<2 x s16>)
+  ; CHECK:   $vgpr13 = COPY [[EXTRACT13]](<2 x s16>)
+  ; CHECK:   $vgpr14 = COPY [[EXTRACT14]](<2 x s16>)
+  ; CHECK:   $vgpr15 = COPY [[EXTRACT15]](<2 x s16>)
+  ; CHECK:   $vgpr16 = COPY [[EXTRACT16]](<2 x s16>)
+  ; CHECK:   $vgpr17 = COPY [[EXTRACT17]](<2 x s16>)
+  ; CHECK:   $vgpr18 = COPY [[EXTRACT18]](<2 x s16>)
+  ; CHECK:   $vgpr19 = COPY [[EXTRACT19]](<2 x s16>)
+  ; CHECK:   $vgpr20 = COPY [[EXTRACT20]](<2 x s16>)
+  ; CHECK:   $vgpr21 = COPY [[EXTRACT21]](<2 x s16>)
+  ; CHECK:   $vgpr22 = COPY [[EXTRACT22]](<2 x s16>)
+  ; CHECK:   $vgpr23 = COPY [[EXTRACT23]](<2 x s16>)
+  ; CHECK:   $vgpr24 = COPY [[EXTRACT24]](<2 x s16>)
+  ; CHECK:   $vgpr25 = COPY [[EXTRACT25]](<2 x s16>)
+  ; CHECK:   $vgpr26 = COPY [[EXTRACT26]](<2 x s16>)
+  ; CHECK:   $vgpr27 = COPY [[EXTRACT27]](<2 x s16>)
+  ; CHECK:   $vgpr28 = COPY [[EXTRACT28]](<2 x s16>)
+  ; CHECK:   $vgpr29 = COPY [[EXTRACT29]](<2 x s16>)
+  ; CHECK:   $vgpr30 = COPY [[EXTRACT30]](<2 x s16>)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(p5) = COPY $sp_reg
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK:   [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY20]], [[C3]](s32)
+  ; CHECK:   G_STORE [[EXTRACT31]](<2 x s16>), [[PTR_ADD1]](p5) :: (store 4 into stack, align 16, addrspace 5)
+  ; CHECK:   [[COPY21:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY21]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v63i16, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15, implicit $vgpr16, implicit $vgpr17, implicit $vgpr18, implicit $vgpr19, implicit $vgpr20, implicit $vgpr21, implicit $vgpr22, implicit $vgpr23, implicit $vgpr24, implicit $vgpr25, implicit $vgpr26, implicit $vgpr27, implicit $vgpr28, implicit $vgpr29, implicit $vgpr30, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 4, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %val = load <63 x i16>, <63 x i16> addrspace(1)* undef
+  call void @external_void_func_v63i16(<63 x i16> %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v65i16() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v65i16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<65 x s16>) = G_LOAD [[DEF]](p1) :: (load 130 from `<65 x i16> addrspace(1)* undef`, align 256, addrspace 1)
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(<66 x s16>) = G_IMPLICIT_DEF
+  ; CHECK:   [[INSERT:%[0-9]+]]:_(<66 x s16>) = G_INSERT [[DEF1]], [[LOAD]](<65 x s16>), 0
+  ; CHECK:   [[EXTRACT:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 0
+  ; CHECK:   [[EXTRACT1:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 32
+  ; CHECK:   [[EXTRACT2:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 64
+  ; CHECK:   [[EXTRACT3:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 96
+  ; CHECK:   [[EXTRACT4:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 128
+  ; CHECK:   [[EXTRACT5:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 160
+  ; CHECK:   [[EXTRACT6:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 192
+  ; CHECK:   [[EXTRACT7:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 224
+  ; CHECK:   [[EXTRACT8:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 256
+  ; CHECK:   [[EXTRACT9:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 288
+  ; CHECK:   [[EXTRACT10:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 320
+  ; CHECK:   [[EXTRACT11:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 352
+  ; CHECK:   [[EXTRACT12:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 384
+  ; CHECK:   [[EXTRACT13:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 416
+  ; CHECK:   [[EXTRACT14:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 448
+  ; CHECK:   [[EXTRACT15:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 480
+  ; CHECK:   [[EXTRACT16:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 512
+  ; CHECK:   [[EXTRACT17:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 544
+  ; CHECK:   [[EXTRACT18:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 576
+  ; CHECK:   [[EXTRACT19:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 608
+  ; CHECK:   [[EXTRACT20:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 640
+  ; CHECK:   [[EXTRACT21:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 672
+  ; CHECK:   [[EXTRACT22:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 704
+  ; CHECK:   [[EXTRACT23:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 736
+  ; CHECK:   [[EXTRACT24:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 768
+  ; CHECK:   [[EXTRACT25:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 800
+  ; CHECK:   [[EXTRACT26:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 832
+  ; CHECK:   [[EXTRACT27:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 864
+  ; CHECK:   [[EXTRACT28:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 896
+  ; CHECK:   [[EXTRACT29:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 928
+  ; CHECK:   [[EXTRACT30:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 960
+  ; CHECK:   [[EXTRACT31:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 992
+  ; CHECK:   [[EXTRACT32:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[INSERT]](<66 x s16>), 1024
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v65i16
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[EXTRACT]](<2 x s16>)
+  ; CHECK:   $vgpr1 = COPY [[EXTRACT1]](<2 x s16>)
+  ; CHECK:   $vgpr2 = COPY [[EXTRACT2]](<2 x s16>)
+  ; CHECK:   $vgpr3 = COPY [[EXTRACT3]](<2 x s16>)
+  ; CHECK:   $vgpr4 = COPY [[EXTRACT4]](<2 x s16>)
+  ; CHECK:   $vgpr5 = COPY [[EXTRACT5]](<2 x s16>)
+  ; CHECK:   $vgpr6 = COPY [[EXTRACT6]](<2 x s16>)
+  ; CHECK:   $vgpr7 = COPY [[EXTRACT7]](<2 x s16>)
+  ; CHECK:   $vgpr8 = COPY [[EXTRACT8]](<2 x s16>)
+  ; CHECK:   $vgpr9 = COPY [[EXTRACT9]](<2 x s16>)
+  ; CHECK:   $vgpr10 = COPY [[EXTRACT10]](<2 x s16>)
+  ; CHECK:   $vgpr11 = COPY [[EXTRACT11]](<2 x s16>)
+  ; CHECK:   $vgpr12 = COPY [[EXTRACT12]](<2 x s16>)
+  ; CHECK:   $vgpr13 = COPY [[EXTRACT13]](<2 x s16>)
+  ; CHECK:   $vgpr14 = COPY [[EXTRACT14]](<2 x s16>)
+  ; CHECK:   $vgpr15 = COPY [[EXTRACT15]](<2 x s16>)
+  ; CHECK:   $vgpr16 = COPY [[EXTRACT16]](<2 x s16>)
+  ; CHECK:   $vgpr17 = COPY [[EXTRACT17]](<2 x s16>)
+  ; CHECK:   $vgpr18 = COPY [[EXTRACT18]](<2 x s16>)
+  ; CHECK:   $vgpr19 = COPY [[EXTRACT19]](<2 x s16>)
+  ; CHECK:   $vgpr20 = COPY [[EXTRACT20]](<2 x s16>)
+  ; CHECK:   $vgpr21 = COPY [[EXTRACT21]](<2 x s16>)
+  ; CHECK:   $vgpr22 = COPY [[EXTRACT22]](<2 x s16>)
+  ; CHECK:   $vgpr23 = COPY [[EXTRACT23]](<2 x s16>)
+  ; CHECK:   $vgpr24 = COPY [[EXTRACT24]](<2 x s16>)
+  ; CHECK:   $vgpr25 = COPY [[EXTRACT25]](<2 x s16>)
+  ; CHECK:   $vgpr26 = COPY [[EXTRACT26]](<2 x s16>)
+  ; CHECK:   $vgpr27 = COPY [[EXTRACT27]](<2 x s16>)
+  ; CHECK:   $vgpr28 = COPY [[EXTRACT28]](<2 x s16>)
+  ; CHECK:   $vgpr29 = COPY [[EXTRACT29]](<2 x s16>)
+  ; CHECK:   $vgpr30 = COPY [[EXTRACT30]](<2 x s16>)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(p5) = COPY $sp_reg
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK:   [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY20]], [[C3]](s32)
+  ; CHECK:   G_STORE [[EXTRACT31]](<2 x s16>), [[PTR_ADD1]](p5) :: (store 4 into stack, align 16, addrspace 5)
+  ; CHECK:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; CHECK:   [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY20]], [[C4]](s32)
+  ; CHECK:   G_STORE [[EXTRACT32]](<2 x s16>), [[PTR_ADD2]](p5) :: (store 4 into stack + 4, addrspace 5)
+  ; CHECK:   [[COPY21:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY21]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v65i16, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15, implicit $vgpr16, implicit $vgpr17, implicit $vgpr18, implicit $vgpr19, implicit $vgpr20, implicit $vgpr21, implicit $vgpr22, implicit $vgpr23, implicit $vgpr24, implicit $vgpr25, implicit $vgpr26, implicit $vgpr27, implicit $vgpr28, implicit $vgpr29, implicit $vgpr30, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 8, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %val = load <65 x i16>, <65 x i16> addrspace(1)* undef
+  call void @external_void_func_v65i16(<65 x i16> %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v66i16() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v66i16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<66 x s16>) = G_LOAD [[DEF]](p1) :: (load 132 from `<66 x i16> addrspace(1)* undef`, align 256, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>), [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>), [[UV6:%[0-9]+]]:_(<2 x s16>), [[UV7:%[0-9]+]]:_(<2 x s16>), [[UV8:%[0-9]+]]:_(<2 x s16>), [[UV9:%[0-9]+]]:_(<2 x s16>), [[UV10:%[0-9]+]]:_(<2 x s16>), [[UV11:%[0-9]+]]:_(<2 x s16>), [[UV12:%[0-9]+]]:_(<2 x s16>), [[UV13:%[0-9]+]]:_(<2 x s16>), [[UV14:%[0-9]+]]:_(<2 x s16>), [[UV15:%[0-9]+]]:_(<2 x s16>), [[UV16:%[0-9]+]]:_(<2 x s16>), [[UV17:%[0-9]+]]:_(<2 x s16>), [[UV18:%[0-9]+]]:_(<2 x s16>), [[UV19:%[0-9]+]]:_(<2 x s16>), [[UV20:%[0-9]+]]:_(<2 x s16>), [[UV21:%[0-9]+]]:_(<2 x s16>), [[UV22:%[0-9]+]]:_(<2 x s16>), [[UV23:%[0-9]+]]:_(<2 x s16>), [[UV24:%[0-9]+]]:_(<2 x s16>), [[UV25:%[0-9]+]]:_(<2 x s16>), [[UV26:%[0-9]+]]:_(<2 x s16>), [[UV27:%[0-9]+]]:_(<2 x s16>), [[UV28:%[0-9]+]]:_(<2 x s16>), [[UV29:%[0-9]+]]:_(<2 x s16>), [[UV30:%[0-9]+]]:_(<2 x s16>), [[UV31:%[0-9]+]]:_(<2 x s16>), [[UV32:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[LOAD]](<66 x s16>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v66i16
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](<2 x s16>)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](<2 x s16>)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](<2 x s16>)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](<2 x s16>)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](<2 x s16>)
+  ; CHECK:   $vgpr5 = COPY [[UV5]](<2 x s16>)
+  ; CHECK:   $vgpr6 = COPY [[UV6]](<2 x s16>)
+  ; CHECK:   $vgpr7 = COPY [[UV7]](<2 x s16>)
+  ; CHECK:   $vgpr8 = COPY [[UV8]](<2 x s16>)
+  ; CHECK:   $vgpr9 = COPY [[UV9]](<2 x s16>)
+  ; CHECK:   $vgpr10 = COPY [[UV10]](<2 x s16>)
+  ; CHECK:   $vgpr11 = COPY [[UV11]](<2 x s16>)
+  ; CHECK:   $vgpr12 = COPY [[UV12]](<2 x s16>)
+  ; CHECK:   $vgpr13 = COPY [[UV13]](<2 x s16>)
+  ; CHECK:   $vgpr14 = COPY [[UV14]](<2 x s16>)
+  ; CHECK:   $vgpr15 = COPY [[UV15]](<2 x s16>)
+  ; CHECK:   $vgpr16 = COPY [[UV16]](<2 x s16>)
+  ; CHECK:   $vgpr17 = COPY [[UV17]](<2 x s16>)
+  ; CHECK:   $vgpr18 = COPY [[UV18]](<2 x s16>)
+  ; CHECK:   $vgpr19 = COPY [[UV19]](<2 x s16>)
+  ; CHECK:   $vgpr20 = COPY [[UV20]](<2 x s16>)
+  ; CHECK:   $vgpr21 = COPY [[UV21]](<2 x s16>)
+  ; CHECK:   $vgpr22 = COPY [[UV22]](<2 x s16>)
+  ; CHECK:   $vgpr23 = COPY [[UV23]](<2 x s16>)
+  ; CHECK:   $vgpr24 = COPY [[UV24]](<2 x s16>)
+  ; CHECK:   $vgpr25 = COPY [[UV25]](<2 x s16>)
+  ; CHECK:   $vgpr26 = COPY [[UV26]](<2 x s16>)
+  ; CHECK:   $vgpr27 = COPY [[UV27]](<2 x s16>)
+  ; CHECK:   $vgpr28 = COPY [[UV28]](<2 x s16>)
+  ; CHECK:   $vgpr29 = COPY [[UV29]](<2 x s16>)
+  ; CHECK:   $vgpr30 = COPY [[UV30]](<2 x s16>)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(p5) = COPY $sp_reg
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK:   [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY20]], [[C3]](s32)
+  ; CHECK:   G_STORE [[UV31]](<2 x s16>), [[PTR_ADD1]](p5) :: (store 4 into stack, align 16, addrspace 5)
+  ; CHECK:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; CHECK:   [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY20]], [[C4]](s32)
+  ; CHECK:   G_STORE [[UV32]](<2 x s16>), [[PTR_ADD2]](p5) :: (store 4 into stack + 4, addrspace 5)
+  ; CHECK:   [[COPY21:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY21]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v66i16, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15, implicit $vgpr16, implicit $vgpr17, implicit $vgpr18, implicit $vgpr19, implicit $vgpr20, implicit $vgpr21, implicit $vgpr22, implicit $vgpr23, implicit $vgpr24, implicit $vgpr25, implicit $vgpr26, implicit $vgpr27, implicit $vgpr28, implicit $vgpr29, implicit $vgpr30, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 8, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %val = load <66 x i16>, <66 x i16> addrspace(1)* undef
+  call void @external_void_func_v66i16(<66 x i16> %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v2f16() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v2f16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[DEF]](p1) :: (load 4 from `<2 x half> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v2f16
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[LOAD]](<2 x s16>)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v2f16, csr_amdgpu_highregs, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %val = load <2 x half>, <2 x half> addrspace(1)* undef
+  call void @external_void_func_v2f16(<2 x half> %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v2i32() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v2i32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[DEF]](p1) :: (load 8 from `<2 x i32> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD]](<2 x s32>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v2i32
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v2i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %val = load <2 x i32>, <2 x i32> addrspace(1)* undef
+  call void @external_void_func_v2i32(<2 x i32> %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v2i32_imm() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v2i32_imm
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C1]](s32)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<2 x s32>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v2i32
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C2]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C3]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C4]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v2i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  call void @external_void_func_v2i32(<2 x i32> <i32 1, i32 2>)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v3i32_imm(i32) #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v3i32_imm
+  ; CHECK: bb.1 (%ir-block.1):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C1]](s32), [[C2]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:_(p4) = G_INTRINSIC intrinsic(@llvm.amdgcn.kernarg.segment.ptr)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<3 x s32>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v3i32
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C3]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C4]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C5]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v3i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  call void @external_void_func_v3i32(<3 x i32> <i32 3, i32 4, i32 5>)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v3i32_i32(i32) #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v3i32_i32
+  ; CHECK: bb.1 (%ir-block.1):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C1]](s32), [[C2]](s32)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
+  ; CHECK:   [[INT:%[0-9]+]]:_(p4) = G_INTRINSIC intrinsic(@llvm.amdgcn.kernarg.segment.ptr)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<3 x s32>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v3i32_i32
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C4]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C5]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C6]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[C3]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v3i32_i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  call void @external_void_func_v3i32_i32(<3 x i32> <i32 3, i32 4, i32 5>, i32 6)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v4i32() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v4i32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[DEF]](p1) :: (load 16 from `<4 x i32> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD]](<4 x s32>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v4i32
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v4i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %val = load <4 x i32>, <4 x i32> addrspace(1)* undef
+  call void @external_void_func_v4i32(<4 x i32> %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v4i32_imm() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v4i32_imm
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C1]](s32), [[C2]](s32), [[C3]](s32)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v4i32
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C4]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C5]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C6]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v4i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  call void @external_void_func_v4i32(<4 x i32> <i32 1, i32 2, i32 3, i32 4>)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v5i32_imm() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v5i32_imm
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; CHECK:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C1]](s32), [[C2]](s32), [[C3]](s32), [[C4]](s32)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<5 x s32>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v5i32
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C5]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C6]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C7]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v5i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  call void @external_void_func_v5i32(<5 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5>)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v8i32() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v8i32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (load 8 from `<8 x i32> addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(<8 x s32>) = G_LOAD [[LOAD]](p1) :: (load 32 from %ir.ptr, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD1]](<8 x s32>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v8i32
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](s32)
+  ; CHECK:   $vgpr5 = COPY [[UV5]](s32)
+  ; CHECK:   $vgpr6 = COPY [[UV6]](s32)
+  ; CHECK:   $vgpr7 = COPY [[UV7]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v8i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %ptr = load <8 x i32> addrspace(1)*, <8 x i32> addrspace(1)* addrspace(4)* undef
+  %val = load <8 x i32>, <8 x i32> addrspace(1)* %ptr
+  call void @external_void_func_v8i32(<8 x i32> %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v8i32_imm() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v8i32_imm
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; CHECK:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+  ; CHECK:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
+  ; CHECK:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
+  ; CHECK:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C1]](s32), [[C2]](s32), [[C3]](s32), [[C4]](s32), [[C5]](s32), [[C6]](s32), [[C7]](s32)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<8 x s32>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v8i32
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C8:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C8]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C9]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C10]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](s32)
+  ; CHECK:   $vgpr5 = COPY [[UV5]](s32)
+  ; CHECK:   $vgpr6 = COPY [[UV6]](s32)
+  ; CHECK:   $vgpr7 = COPY [[UV7]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v8i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  call void @external_void_func_v8i32(<8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v16i32() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v16i32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (load 8 from `<16 x i32> addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(<16 x s32>) = G_LOAD [[LOAD]](p1) :: (load 64 from %ir.ptr, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD1]](<16 x s32>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v16i32
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](s32)
+  ; CHECK:   $vgpr5 = COPY [[UV5]](s32)
+  ; CHECK:   $vgpr6 = COPY [[UV6]](s32)
+  ; CHECK:   $vgpr7 = COPY [[UV7]](s32)
+  ; CHECK:   $vgpr8 = COPY [[UV8]](s32)
+  ; CHECK:   $vgpr9 = COPY [[UV9]](s32)
+  ; CHECK:   $vgpr10 = COPY [[UV10]](s32)
+  ; CHECK:   $vgpr11 = COPY [[UV11]](s32)
+  ; CHECK:   $vgpr12 = COPY [[UV12]](s32)
+  ; CHECK:   $vgpr13 = COPY [[UV13]](s32)
+  ; CHECK:   $vgpr14 = COPY [[UV14]](s32)
+  ; CHECK:   $vgpr15 = COPY [[UV15]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v16i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %ptr = load <16 x i32> addrspace(1)*, <16 x i32> addrspace(1)* addrspace(4)* undef
+  %val = load <16 x i32>, <16 x i32> addrspace(1)* %ptr
+  call void @external_void_func_v16i32(<16 x i32> %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v32i32() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v32i32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (load 8 from `<32 x i32> addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(<32 x s32>) = G_LOAD [[LOAD]](p1) :: (load 128 from %ir.ptr, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32), [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32), [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD1]](<32 x s32>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v32i32
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](s32)
+  ; CHECK:   $vgpr5 = COPY [[UV5]](s32)
+  ; CHECK:   $vgpr6 = COPY [[UV6]](s32)
+  ; CHECK:   $vgpr7 = COPY [[UV7]](s32)
+  ; CHECK:   $vgpr8 = COPY [[UV8]](s32)
+  ; CHECK:   $vgpr9 = COPY [[UV9]](s32)
+  ; CHECK:   $vgpr10 = COPY [[UV10]](s32)
+  ; CHECK:   $vgpr11 = COPY [[UV11]](s32)
+  ; CHECK:   $vgpr12 = COPY [[UV12]](s32)
+  ; CHECK:   $vgpr13 = COPY [[UV13]](s32)
+  ; CHECK:   $vgpr14 = COPY [[UV14]](s32)
+  ; CHECK:   $vgpr15 = COPY [[UV15]](s32)
+  ; CHECK:   $vgpr16 = COPY [[UV16]](s32)
+  ; CHECK:   $vgpr17 = COPY [[UV17]](s32)
+  ; CHECK:   $vgpr18 = COPY [[UV18]](s32)
+  ; CHECK:   $vgpr19 = COPY [[UV19]](s32)
+  ; CHECK:   $vgpr20 = COPY [[UV20]](s32)
+  ; CHECK:   $vgpr21 = COPY [[UV21]](s32)
+  ; CHECK:   $vgpr22 = COPY [[UV22]](s32)
+  ; CHECK:   $vgpr23 = COPY [[UV23]](s32)
+  ; CHECK:   $vgpr24 = COPY [[UV24]](s32)
+  ; CHECK:   $vgpr25 = COPY [[UV25]](s32)
+  ; CHECK:   $vgpr26 = COPY [[UV26]](s32)
+  ; CHECK:   $vgpr27 = COPY [[UV27]](s32)
+  ; CHECK:   $vgpr28 = COPY [[UV28]](s32)
+  ; CHECK:   $vgpr29 = COPY [[UV29]](s32)
+  ; CHECK:   $vgpr30 = COPY [[UV30]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(p5) = COPY $sp_reg
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK:   [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY20]], [[C3]](s32)
+  ; CHECK:   G_STORE [[UV31]](s32), [[PTR_ADD1]](p5) :: (store 4 into stack, align 16, addrspace 5)
+  ; CHECK:   [[COPY21:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY21]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v32i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15, implicit $vgpr16, implicit $vgpr17, implicit $vgpr18, implicit $vgpr19, implicit $vgpr20, implicit $vgpr21, implicit $vgpr22, implicit $vgpr23, implicit $vgpr24, implicit $vgpr25, implicit $vgpr26, implicit $vgpr27, implicit $vgpr28, implicit $vgpr29, implicit $vgpr30, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 4, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %ptr = load <32 x i32> addrspace(1)*, <32 x i32> addrspace(1)* addrspace(4)* undef
+  %val = load <32 x i32>, <32 x i32> addrspace(1)* %ptr
+  call void @external_void_func_v32i32(<32 x i32> %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v32i32_i32(i32) #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v32i32_i32
+  ; CHECK: bb.1 (%ir-block.1):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[INT:%[0-9]+]]:_(p4) = G_INTRINSIC intrinsic(@llvm.amdgcn.kernarg.segment.ptr)
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (load 8 from `<32 x i32> addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(<32 x s32>) = G_LOAD [[LOAD]](p1) :: (load 128 from %ir.ptr0, addrspace 1)
+  ; CHECK:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[DEF1]](p1) :: (load 4 from `i32 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32), [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32), [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD1]](<32 x s32>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v32i32_i32
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](s32)
+  ; CHECK:   $vgpr5 = COPY [[UV5]](s32)
+  ; CHECK:   $vgpr6 = COPY [[UV6]](s32)
+  ; CHECK:   $vgpr7 = COPY [[UV7]](s32)
+  ; CHECK:   $vgpr8 = COPY [[UV8]](s32)
+  ; CHECK:   $vgpr9 = COPY [[UV9]](s32)
+  ; CHECK:   $vgpr10 = COPY [[UV10]](s32)
+  ; CHECK:   $vgpr11 = COPY [[UV11]](s32)
+  ; CHECK:   $vgpr12 = COPY [[UV12]](s32)
+  ; CHECK:   $vgpr13 = COPY [[UV13]](s32)
+  ; CHECK:   $vgpr14 = COPY [[UV14]](s32)
+  ; CHECK:   $vgpr15 = COPY [[UV15]](s32)
+  ; CHECK:   $vgpr16 = COPY [[UV16]](s32)
+  ; CHECK:   $vgpr17 = COPY [[UV17]](s32)
+  ; CHECK:   $vgpr18 = COPY [[UV18]](s32)
+  ; CHECK:   $vgpr19 = COPY [[UV19]](s32)
+  ; CHECK:   $vgpr20 = COPY [[UV20]](s32)
+  ; CHECK:   $vgpr21 = COPY [[UV21]](s32)
+  ; CHECK:   $vgpr22 = COPY [[UV22]](s32)
+  ; CHECK:   $vgpr23 = COPY [[UV23]](s32)
+  ; CHECK:   $vgpr24 = COPY [[UV24]](s32)
+  ; CHECK:   $vgpr25 = COPY [[UV25]](s32)
+  ; CHECK:   $vgpr26 = COPY [[UV26]](s32)
+  ; CHECK:   $vgpr27 = COPY [[UV27]](s32)
+  ; CHECK:   $vgpr28 = COPY [[UV28]](s32)
+  ; CHECK:   $vgpr29 = COPY [[UV29]](s32)
+  ; CHECK:   $vgpr30 = COPY [[UV30]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(p5) = COPY $sp_reg
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK:   [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY20]], [[C3]](s32)
+  ; CHECK:   G_STORE [[UV31]](s32), [[PTR_ADD1]](p5) :: (store 4 into stack, align 16, addrspace 5)
+  ; CHECK:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; CHECK:   [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY20]], [[C4]](s32)
+  ; CHECK:   G_STORE [[LOAD2]](s32), [[PTR_ADD2]](p5) :: (store 4 into stack + 4, addrspace 5)
+  ; CHECK:   [[COPY21:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY21]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v32i32_i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15, implicit $vgpr16, implicit $vgpr17, implicit $vgpr18, implicit $vgpr19, implicit $vgpr20, implicit $vgpr21, implicit $vgpr22, implicit $vgpr23, implicit $vgpr24, implicit $vgpr25, implicit $vgpr26, implicit $vgpr27, implicit $vgpr28, implicit $vgpr29, implicit $vgpr30, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 8, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %ptr0 = load <32 x i32> addrspace(1)*, <32 x i32> addrspace(1)* addrspace(4)* undef
+  %val0 = load <32 x i32>, <32 x i32> addrspace(1)* %ptr0
+  %val1 = load i32, i32 addrspace(1)* undef
+  call void @external_void_func_v32i32_i32(<32 x i32> %val0, i32 %val1)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v32i32_i8_i8_i16() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v32i32_i8_i8_i16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p1) = COPY [[DEF1]](p1)
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (load 8 from `<32 x i32> addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(<32 x s32>) = G_LOAD [[LOAD]](p1) :: (load 128 from %ir.ptr0, addrspace 1)
+  ; CHECK:   [[LOAD2:%[0-9]+]]:_(s8) = G_LOAD [[DEF1]](p1) :: (load 1 from `i8 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[LOAD3:%[0-9]+]]:_(s16) = G_LOAD [[COPY10]](p1) :: (load 2 from `i16 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32), [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32), [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD1]](<32 x s32>)
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LOAD2]](s8)
+  ; CHECK:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[LOAD3]](s16)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v32i32_i8_i8_i16
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY13]], [[C]](s64)
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY18]], [[SHL]]
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY20]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](s32)
+  ; CHECK:   $vgpr5 = COPY [[UV5]](s32)
+  ; CHECK:   $vgpr6 = COPY [[UV6]](s32)
+  ; CHECK:   $vgpr7 = COPY [[UV7]](s32)
+  ; CHECK:   $vgpr8 = COPY [[UV8]](s32)
+  ; CHECK:   $vgpr9 = COPY [[UV9]](s32)
+  ; CHECK:   $vgpr10 = COPY [[UV10]](s32)
+  ; CHECK:   $vgpr11 = COPY [[UV11]](s32)
+  ; CHECK:   $vgpr12 = COPY [[UV12]](s32)
+  ; CHECK:   $vgpr13 = COPY [[UV13]](s32)
+  ; CHECK:   $vgpr14 = COPY [[UV14]](s32)
+  ; CHECK:   $vgpr15 = COPY [[UV15]](s32)
+  ; CHECK:   $vgpr16 = COPY [[UV16]](s32)
+  ; CHECK:   $vgpr17 = COPY [[UV17]](s32)
+  ; CHECK:   $vgpr18 = COPY [[UV18]](s32)
+  ; CHECK:   $vgpr19 = COPY [[UV19]](s32)
+  ; CHECK:   $vgpr20 = COPY [[UV20]](s32)
+  ; CHECK:   $vgpr21 = COPY [[UV21]](s32)
+  ; CHECK:   $vgpr22 = COPY [[UV22]](s32)
+  ; CHECK:   $vgpr23 = COPY [[UV23]](s32)
+  ; CHECK:   $vgpr24 = COPY [[UV24]](s32)
+  ; CHECK:   $vgpr25 = COPY [[UV25]](s32)
+  ; CHECK:   $vgpr26 = COPY [[UV26]](s32)
+  ; CHECK:   $vgpr27 = COPY [[UV27]](s32)
+  ; CHECK:   $vgpr28 = COPY [[UV28]](s32)
+  ; CHECK:   $vgpr29 = COPY [[UV29]](s32)
+  ; CHECK:   $vgpr30 = COPY [[UV30]](s32)
+  ; CHECK:   [[COPY21:%[0-9]+]]:_(p5) = COPY $sp_reg
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK:   [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY21]], [[C3]](s32)
+  ; CHECK:   G_STORE [[UV31]](s32), [[PTR_ADD1]](p5) :: (store 4 into stack, align 16, addrspace 5)
+  ; CHECK:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; CHECK:   [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY21]], [[C4]](s32)
+  ; CHECK:   G_STORE [[ANYEXT]](s32), [[PTR_ADD2]](p5) :: (store 4 into stack + 4, addrspace 5)
+  ; CHECK:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+  ; CHECK:   [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY21]], [[C5]](s32)
+  ; CHECK:   G_STORE [[ANYEXT]](s32), [[PTR_ADD3]](p5) :: (store 4 into stack + 8, align 8, addrspace 5)
+  ; CHECK:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+  ; CHECK:   [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY21]], [[C6]](s32)
+  ; CHECK:   G_STORE [[ANYEXT1]](s32), [[PTR_ADD4]](p5) :: (store 4 into stack + 12, addrspace 5)
+  ; CHECK:   [[COPY22:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY22]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY12]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY14]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY16]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY17]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v32i32_i8_i8_i16, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15, implicit $vgpr16, implicit $vgpr17, implicit $vgpr18, implicit $vgpr19, implicit $vgpr20, implicit $vgpr21, implicit $vgpr22, implicit $vgpr23, implicit $vgpr24, implicit $vgpr25, implicit $vgpr26, implicit $vgpr27, implicit $vgpr28, implicit $vgpr29, implicit $vgpr30, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 16, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %ptr0 = load <32 x i32> addrspace(1)*, <32 x i32> addrspace(1)* addrspace(4)* undef
+  %val0 = load <32 x i32>, <32 x i32> addrspace(1)* %ptr0
+  %val1 = load i8, i8 addrspace(1)* undef
+  %val2 = load i8, i8 addrspace(1)* undef
+  %val3 = load i16, i16 addrspace(1)* undef
+  call void @external_void_func_v32i32_i8_i8_i16(<32 x i32> %val0, i8 %val1, i8 %val2, i16 %val3)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v32i32_p3_p5() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v32i32_p3_p5
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p1) = COPY [[DEF1]](p1)
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (load 8 from `<32 x i32> addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(<32 x s32>) = G_LOAD [[LOAD]](p1) :: (load 128 from %ir.ptr0, addrspace 1)
+  ; CHECK:   [[LOAD2:%[0-9]+]]:_(p3) = G_LOAD [[DEF1]](p1) :: (load 4 from `i8 addrspace(3)* addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[LOAD3:%[0-9]+]]:_(p5) = G_LOAD [[COPY10]](p1) :: (load 4 from `i8 addrspace(5)* addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32), [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32), [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD1]](<32 x s32>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v32i32_p3_p5
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY13]], [[C]](s64)
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY18]], [[SHL]]
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY20]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](s32)
+  ; CHECK:   $vgpr5 = COPY [[UV5]](s32)
+  ; CHECK:   $vgpr6 = COPY [[UV6]](s32)
+  ; CHECK:   $vgpr7 = COPY [[UV7]](s32)
+  ; CHECK:   $vgpr8 = COPY [[UV8]](s32)
+  ; CHECK:   $vgpr9 = COPY [[UV9]](s32)
+  ; CHECK:   $vgpr10 = COPY [[UV10]](s32)
+  ; CHECK:   $vgpr11 = COPY [[UV11]](s32)
+  ; CHECK:   $vgpr12 = COPY [[UV12]](s32)
+  ; CHECK:   $vgpr13 = COPY [[UV13]](s32)
+  ; CHECK:   $vgpr14 = COPY [[UV14]](s32)
+  ; CHECK:   $vgpr15 = COPY [[UV15]](s32)
+  ; CHECK:   $vgpr16 = COPY [[UV16]](s32)
+  ; CHECK:   $vgpr17 = COPY [[UV17]](s32)
+  ; CHECK:   $vgpr18 = COPY [[UV18]](s32)
+  ; CHECK:   $vgpr19 = COPY [[UV19]](s32)
+  ; CHECK:   $vgpr20 = COPY [[UV20]](s32)
+  ; CHECK:   $vgpr21 = COPY [[UV21]](s32)
+  ; CHECK:   $vgpr22 = COPY [[UV22]](s32)
+  ; CHECK:   $vgpr23 = COPY [[UV23]](s32)
+  ; CHECK:   $vgpr24 = COPY [[UV24]](s32)
+  ; CHECK:   $vgpr25 = COPY [[UV25]](s32)
+  ; CHECK:   $vgpr26 = COPY [[UV26]](s32)
+  ; CHECK:   $vgpr27 = COPY [[UV27]](s32)
+  ; CHECK:   $vgpr28 = COPY [[UV28]](s32)
+  ; CHECK:   $vgpr29 = COPY [[UV29]](s32)
+  ; CHECK:   $vgpr30 = COPY [[UV30]](s32)
+  ; CHECK:   [[COPY21:%[0-9]+]]:_(p5) = COPY $sp_reg
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK:   [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY21]], [[C3]](s32)
+  ; CHECK:   G_STORE [[UV31]](s32), [[PTR_ADD1]](p5) :: (store 4 into stack, align 16, addrspace 5)
+  ; CHECK:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; CHECK:   [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY21]], [[C4]](s32)
+  ; CHECK:   G_STORE [[LOAD2]](p3), [[PTR_ADD2]](p5) :: (store 4 into stack + 4, addrspace 5)
+  ; CHECK:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+  ; CHECK:   [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY21]], [[C5]](s32)
+  ; CHECK:   G_STORE [[LOAD3]](p5), [[PTR_ADD3]](p5) :: (store 4 into stack + 8, align 8, addrspace 5)
+  ; CHECK:   [[COPY22:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY22]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY12]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY14]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY16]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY17]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v32i32_p3_p5, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15, implicit $vgpr16, implicit $vgpr17, implicit $vgpr18, implicit $vgpr19, implicit $vgpr20, implicit $vgpr21, implicit $vgpr22, implicit $vgpr23, implicit $vgpr24, implicit $vgpr25, implicit $vgpr26, implicit $vgpr27, implicit $vgpr28, implicit $vgpr29, implicit $vgpr30, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 12, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %ptr0 = load <32 x i32> addrspace(1)*, <32 x i32> addrspace(1)* addrspace(4)* undef
+  %val0 = load <32 x i32>, <32 x i32> addrspace(1)* %ptr0
+  %val1 = load i8 addrspace(3)*, i8 addrspace(3)* addrspace(1)* undef
+  %val2 = load i8 addrspace(5)*, i8 addrspace(5)* addrspace(1)* undef
+  call void @external_void_func_v32i32_p3_p5(<32 x i32> %val0, i8 addrspace(3)* %val1, i8 addrspace(5)* %val2)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_struct_i8_i32() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_struct_i8_i32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (load 8 from `{ i8, i32 } addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[LOAD]](p1) :: (load 1 from %ir.ptr0, align 4, addrspace 1)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[LOAD]], [[C]](s64)
+  ; CHECK:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p1) :: (load 4 from %ir.ptr0 + 4, addrspace 1)
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LOAD1]](s8)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_struct_i8_i32
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   $vgpr1 = COPY [[LOAD2]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD1]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_struct_i8_i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %ptr0 = load { i8, i32 } addrspace(1)*, { i8, i32 } addrspace(1)* addrspace(4)* undef
+  %val = load { i8, i32 }, { i8, i32 } addrspace(1)* %ptr0
+  call void @external_void_func_struct_i8_i32({ i8, i32 } %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_byval_struct_i8_i32() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_byval_struct_i8_i32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 3
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+  ; CHECK:   [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %stack.0.val
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[FRAME_INDEX]], [[C2]](s32)
+  ; CHECK:   G_STORE [[C]](s8), [[FRAME_INDEX]](p5) :: (store 1 into %ir.gep01, addrspace 5)
+  ; CHECK:   G_STORE [[C1]](s32), [[PTR_ADD]](p5) :: (store 4 into %ir.gep1, addrspace 5)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_byval_struct_i8_i32
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C3]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C4]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C5]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(p5) = COPY $sp_reg
+  ; CHECK:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK:   [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY20]], [[C6]](s32)
+  ; CHECK:   G_STORE [[FRAME_INDEX]](p5), [[PTR_ADD2]](p5) :: (store 4 into stack, align 16, addrspace 5)
+  ; CHECK:   [[COPY21:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY21]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD1]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_byval_struct_i8_i32, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 8, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %val = alloca { i8, i32 }, align 4, addrspace(5)
+  %gep0 = getelementptr inbounds { i8, i32 }, { i8, i32 } addrspace(5)* %val, i32 0, i32 0
+  %gep1 = getelementptr inbounds { i8, i32 }, { i8, i32 } addrspace(5)* %val, i32 0, i32 1
+  store i8 3, i8 addrspace(5)* %gep0
+  store i32 8, i32 addrspace(5)* %gep1
+  call void @external_void_func_byval_struct_i8_i32({ i8, i32 } addrspace(5)* byval({ i8, i32 }) %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v2i8() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v2i8
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (load 8 from `<2 x i8> addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(<2 x s8>) = G_LOAD [[LOAD]](p1) :: (load 2 from %ir.ptr, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[LOAD1]](<2 x s8>)
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[UV]](s8)
+  ; CHECK:   [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[UV1]](s8)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v2i8
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT]](s16)
+  ; CHECK:   $vgpr0 = COPY [[ANYEXT2]](s32)
+  ; CHECK:   [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT1]](s16)
+  ; CHECK:   $vgpr1 = COPY [[ANYEXT3]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v2i8, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %ptr = load <2 x i8> addrspace(1)*, <2 x i8> addrspace(1)* addrspace(4)* undef
+  %val = load <2 x i8>, <2 x i8> addrspace(1)* %ptr
+  call void @external_void_func_v2i8(<2 x i8> %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v3i8() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v3i8
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (load 8 from `<3 x i8> addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[LOAD]](p1) :: (load 3 from %ir.ptr, align 4, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8), [[UV2:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[LOAD1]](<3 x s8>)
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[UV]](s8)
+  ; CHECK:   [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[UV1]](s8)
+  ; CHECK:   [[ANYEXT2:%[0-9]+]]:_(s16) = G_ANYEXT [[UV2]](s8)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v3i8
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT]](s16)
+  ; CHECK:   $vgpr0 = COPY [[ANYEXT3]](s32)
+  ; CHECK:   [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT1]](s16)
+  ; CHECK:   $vgpr1 = COPY [[ANYEXT4]](s32)
+  ; CHECK:   [[ANYEXT5:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT2]](s16)
+  ; CHECK:   $vgpr2 = COPY [[ANYEXT5]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v3i8, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %ptr = load <3 x i8> addrspace(1)*, <3 x i8> addrspace(1)* addrspace(4)* undef
+  %val = load <3 x i8>, <3 x i8> addrspace(1)* %ptr
+  call void @external_void_func_v3i8(<3 x i8> %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v4i8() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v4i8
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (load 8 from `<4 x i8> addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(<4 x s8>) = G_LOAD [[LOAD]](p1) :: (load 4 from %ir.ptr, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8), [[UV2:%[0-9]+]]:_(s8), [[UV3:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[LOAD1]](<4 x s8>)
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[UV]](s8)
+  ; CHECK:   [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[UV1]](s8)
+  ; CHECK:   [[ANYEXT2:%[0-9]+]]:_(s16) = G_ANYEXT [[UV2]](s8)
+  ; CHECK:   [[ANYEXT3:%[0-9]+]]:_(s16) = G_ANYEXT [[UV3]](s8)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v4i8
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT]](s16)
+  ; CHECK:   $vgpr0 = COPY [[ANYEXT4]](s32)
+  ; CHECK:   [[ANYEXT5:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT1]](s16)
+  ; CHECK:   $vgpr1 = COPY [[ANYEXT5]](s32)
+  ; CHECK:   [[ANYEXT6:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT2]](s16)
+  ; CHECK:   $vgpr2 = COPY [[ANYEXT6]](s32)
+  ; CHECK:   [[ANYEXT7:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT3]](s16)
+  ; CHECK:   $vgpr3 = COPY [[ANYEXT7]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v4i8, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %ptr = load <4 x i8> addrspace(1)*, <4 x i8> addrspace(1)* addrspace(4)* undef
+  %val = load <4 x i8>, <4 x i8> addrspace(1)* %ptr
+  call void @external_void_func_v4i8(<4 x i8> %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v8i8() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v8i8
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (load 8 from `<8 x i8> addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(<8 x s8>) = G_LOAD [[LOAD]](p1) :: (load 8 from %ir.ptr, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8), [[UV2:%[0-9]+]]:_(s8), [[UV3:%[0-9]+]]:_(s8), [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8), [[UV6:%[0-9]+]]:_(s8), [[UV7:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[LOAD1]](<8 x s8>)
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[UV]](s8)
+  ; CHECK:   [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[UV1]](s8)
+  ; CHECK:   [[ANYEXT2:%[0-9]+]]:_(s16) = G_ANYEXT [[UV2]](s8)
+  ; CHECK:   [[ANYEXT3:%[0-9]+]]:_(s16) = G_ANYEXT [[UV3]](s8)
+  ; CHECK:   [[ANYEXT4:%[0-9]+]]:_(s16) = G_ANYEXT [[UV4]](s8)
+  ; CHECK:   [[ANYEXT5:%[0-9]+]]:_(s16) = G_ANYEXT [[UV5]](s8)
+  ; CHECK:   [[ANYEXT6:%[0-9]+]]:_(s16) = G_ANYEXT [[UV6]](s8)
+  ; CHECK:   [[ANYEXT7:%[0-9]+]]:_(s16) = G_ANYEXT [[UV7]](s8)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v8i8
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   [[ANYEXT8:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT]](s16)
+  ; CHECK:   $vgpr0 = COPY [[ANYEXT8]](s32)
+  ; CHECK:   [[ANYEXT9:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT1]](s16)
+  ; CHECK:   $vgpr1 = COPY [[ANYEXT9]](s32)
+  ; CHECK:   [[ANYEXT10:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT2]](s16)
+  ; CHECK:   $vgpr2 = COPY [[ANYEXT10]](s32)
+  ; CHECK:   [[ANYEXT11:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT3]](s16)
+  ; CHECK:   $vgpr3 = COPY [[ANYEXT11]](s32)
+  ; CHECK:   [[ANYEXT12:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT4]](s16)
+  ; CHECK:   $vgpr4 = COPY [[ANYEXT12]](s32)
+  ; CHECK:   [[ANYEXT13:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT5]](s16)
+  ; CHECK:   $vgpr5 = COPY [[ANYEXT13]](s32)
+  ; CHECK:   [[ANYEXT14:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT6]](s16)
+  ; CHECK:   $vgpr6 = COPY [[ANYEXT14]](s32)
+  ; CHECK:   [[ANYEXT15:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT7]](s16)
+  ; CHECK:   $vgpr7 = COPY [[ANYEXT15]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v8i8, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %ptr = load <8 x i8> addrspace(1)*, <8 x i8> addrspace(1)* addrspace(4)* undef
+  %val = load <8 x i8>, <8 x i8> addrspace(1)* %ptr
+  call void @external_void_func_v8i8(<8 x i8> %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_external_void_func_v16i8() #0 {
+  ; CHECK-LABEL: name: test_call_external_void_func_v16i8
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[DEF]](p4) :: (load 8 from `<16 x i8> addrspace(1)* addrspace(4)* undef`, addrspace 4)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(<16 x s8>) = G_LOAD [[LOAD]](p1) :: (load 16 from %ir.ptr, addrspace 1)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8), [[UV2:%[0-9]+]]:_(s8), [[UV3:%[0-9]+]]:_(s8), [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8), [[UV6:%[0-9]+]]:_(s8), [[UV7:%[0-9]+]]:_(s8), [[UV8:%[0-9]+]]:_(s8), [[UV9:%[0-9]+]]:_(s8), [[UV10:%[0-9]+]]:_(s8), [[UV11:%[0-9]+]]:_(s8), [[UV12:%[0-9]+]]:_(s8), [[UV13:%[0-9]+]]:_(s8), [[UV14:%[0-9]+]]:_(s8), [[UV15:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[LOAD1]](<16 x s8>)
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[UV]](s8)
+  ; CHECK:   [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[UV1]](s8)
+  ; CHECK:   [[ANYEXT2:%[0-9]+]]:_(s16) = G_ANYEXT [[UV2]](s8)
+  ; CHECK:   [[ANYEXT3:%[0-9]+]]:_(s16) = G_ANYEXT [[UV3]](s8)
+  ; CHECK:   [[ANYEXT4:%[0-9]+]]:_(s16) = G_ANYEXT [[UV4]](s8)
+  ; CHECK:   [[ANYEXT5:%[0-9]+]]:_(s16) = G_ANYEXT [[UV5]](s8)
+  ; CHECK:   [[ANYEXT6:%[0-9]+]]:_(s16) = G_ANYEXT [[UV6]](s8)
+  ; CHECK:   [[ANYEXT7:%[0-9]+]]:_(s16) = G_ANYEXT [[UV7]](s8)
+  ; CHECK:   [[ANYEXT8:%[0-9]+]]:_(s16) = G_ANYEXT [[UV8]](s8)
+  ; CHECK:   [[ANYEXT9:%[0-9]+]]:_(s16) = G_ANYEXT [[UV9]](s8)
+  ; CHECK:   [[ANYEXT10:%[0-9]+]]:_(s16) = G_ANYEXT [[UV10]](s8)
+  ; CHECK:   [[ANYEXT11:%[0-9]+]]:_(s16) = G_ANYEXT [[UV11]](s8)
+  ; CHECK:   [[ANYEXT12:%[0-9]+]]:_(s16) = G_ANYEXT [[UV12]](s8)
+  ; CHECK:   [[ANYEXT13:%[0-9]+]]:_(s16) = G_ANYEXT [[UV13]](s8)
+  ; CHECK:   [[ANYEXT14:%[0-9]+]]:_(s16) = G_ANYEXT [[UV14]](s8)
+  ; CHECK:   [[ANYEXT15:%[0-9]+]]:_(s16) = G_ANYEXT [[UV15]](s8)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_v16i8
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   [[ANYEXT16:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT]](s16)
+  ; CHECK:   $vgpr0 = COPY [[ANYEXT16]](s32)
+  ; CHECK:   [[ANYEXT17:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT1]](s16)
+  ; CHECK:   $vgpr1 = COPY [[ANYEXT17]](s32)
+  ; CHECK:   [[ANYEXT18:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT2]](s16)
+  ; CHECK:   $vgpr2 = COPY [[ANYEXT18]](s32)
+  ; CHECK:   [[ANYEXT19:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT3]](s16)
+  ; CHECK:   $vgpr3 = COPY [[ANYEXT19]](s32)
+  ; CHECK:   [[ANYEXT20:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT4]](s16)
+  ; CHECK:   $vgpr4 = COPY [[ANYEXT20]](s32)
+  ; CHECK:   [[ANYEXT21:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT5]](s16)
+  ; CHECK:   $vgpr5 = COPY [[ANYEXT21]](s32)
+  ; CHECK:   [[ANYEXT22:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT6]](s16)
+  ; CHECK:   $vgpr6 = COPY [[ANYEXT22]](s32)
+  ; CHECK:   [[ANYEXT23:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT7]](s16)
+  ; CHECK:   $vgpr7 = COPY [[ANYEXT23]](s32)
+  ; CHECK:   [[ANYEXT24:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT8]](s16)
+  ; CHECK:   $vgpr8 = COPY [[ANYEXT24]](s32)
+  ; CHECK:   [[ANYEXT25:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT9]](s16)
+  ; CHECK:   $vgpr9 = COPY [[ANYEXT25]](s32)
+  ; CHECK:   [[ANYEXT26:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT10]](s16)
+  ; CHECK:   $vgpr10 = COPY [[ANYEXT26]](s32)
+  ; CHECK:   [[ANYEXT27:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT11]](s16)
+  ; CHECK:   $vgpr11 = COPY [[ANYEXT27]](s32)
+  ; CHECK:   [[ANYEXT28:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT12]](s16)
+  ; CHECK:   $vgpr12 = COPY [[ANYEXT28]](s32)
+  ; CHECK:   [[ANYEXT29:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT13]](s16)
+  ; CHECK:   $vgpr13 = COPY [[ANYEXT29]](s32)
+  ; CHECK:   [[ANYEXT30:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT14]](s16)
+  ; CHECK:   $vgpr14 = COPY [[ANYEXT30]](s32)
+  ; CHECK:   [[ANYEXT31:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT15]](s16)
+  ; CHECK:   $vgpr15 = COPY [[ANYEXT31]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_v16i8, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  %ptr = load <16 x i8> addrspace(1)*, <16 x i8> addrspace(1)* addrspace(4)* undef
+  %val = load <16 x i8>, <16 x i8> addrspace(1)* %ptr
+  call void @external_void_func_v16i8(<16 x i8> %val)
+  ret void
+}
+
+define amdgpu_kernel void @stack_passed_arg_alignment_v32i32_f64(<32 x i32> %val, double %tmp) #0 {
+  ; CHECK-LABEL: name: stack_passed_arg_alignment_v32i32_f64
+  ; CHECK: bb.1.entry:
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[INT:%[0-9]+]]:_(p4) = G_INTRINSIC intrinsic(@llvm.amdgcn.kernarg.segment.ptr)
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(<32 x s32>) = G_LOAD [[INT]](p4) :: (dereferenceable invariant load 128 from %ir.val.kernarg.offset.cast, align 16, addrspace 4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 128
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[INT]], [[C]](s64)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load 8 from %ir.tmp.kernarg.offset.cast, align 16, addrspace 4)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32), [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32), [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD]](<32 x s32>)
+  ; CHECK:   [[UV32:%[0-9]+]]:_(s32), [[UV33:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD1]](s64)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @stack_passed_f64_arg
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 136
+  ; CHECK:   [[PTR_ADD1:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C1]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C2]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](s32)
+  ; CHECK:   $vgpr5 = COPY [[UV5]](s32)
+  ; CHECK:   $vgpr6 = COPY [[UV6]](s32)
+  ; CHECK:   $vgpr7 = COPY [[UV7]](s32)
+  ; CHECK:   $vgpr8 = COPY [[UV8]](s32)
+  ; CHECK:   $vgpr9 = COPY [[UV9]](s32)
+  ; CHECK:   $vgpr10 = COPY [[UV10]](s32)
+  ; CHECK:   $vgpr11 = COPY [[UV11]](s32)
+  ; CHECK:   $vgpr12 = COPY [[UV12]](s32)
+  ; CHECK:   $vgpr13 = COPY [[UV13]](s32)
+  ; CHECK:   $vgpr14 = COPY [[UV14]](s32)
+  ; CHECK:   $vgpr15 = COPY [[UV15]](s32)
+  ; CHECK:   $vgpr16 = COPY [[UV16]](s32)
+  ; CHECK:   $vgpr17 = COPY [[UV17]](s32)
+  ; CHECK:   $vgpr18 = COPY [[UV18]](s32)
+  ; CHECK:   $vgpr19 = COPY [[UV19]](s32)
+  ; CHECK:   $vgpr20 = COPY [[UV20]](s32)
+  ; CHECK:   $vgpr21 = COPY [[UV21]](s32)
+  ; CHECK:   $vgpr22 = COPY [[UV22]](s32)
+  ; CHECK:   $vgpr23 = COPY [[UV23]](s32)
+  ; CHECK:   $vgpr24 = COPY [[UV24]](s32)
+  ; CHECK:   $vgpr25 = COPY [[UV25]](s32)
+  ; CHECK:   $vgpr26 = COPY [[UV26]](s32)
+  ; CHECK:   $vgpr27 = COPY [[UV27]](s32)
+  ; CHECK:   $vgpr28 = COPY [[UV28]](s32)
+  ; CHECK:   $vgpr29 = COPY [[UV29]](s32)
+  ; CHECK:   $vgpr30 = COPY [[UV30]](s32)
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(p5) = COPY $sp_reg
+  ; CHECK:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK:   [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY20]], [[C4]](s32)
+  ; CHECK:   G_STORE [[UV31]](s32), [[PTR_ADD2]](p5) :: (store 4 into stack, align 16, addrspace 5)
+  ; CHECK:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; CHECK:   [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY20]], [[C5]](s32)
+  ; CHECK:   G_STORE [[UV32]](s32), [[PTR_ADD3]](p5) :: (store 4 into stack + 4, addrspace 5)
+  ; CHECK:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+  ; CHECK:   [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY20]], [[C6]](s32)
+  ; CHECK:   G_STORE [[UV33]](s32), [[PTR_ADD4]](p5) :: (store 4 into stack + 8, align 8, addrspace 5)
+  ; CHECK:   [[COPY21:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY21]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD1]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @stack_passed_f64_arg, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15, implicit $vgpr16, implicit $vgpr17, implicit $vgpr18, implicit $vgpr19, implicit $vgpr20, implicit $vgpr21, implicit $vgpr22, implicit $vgpr23, implicit $vgpr24, implicit $vgpr25, implicit $vgpr26, implicit $vgpr27, implicit $vgpr28, implicit $vgpr29, implicit $vgpr30, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 12, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+entry:
+  call void @stack_passed_f64_arg(<32 x i32> %val, double %tmp)
+  ret void
+}
+
+define void @stack_12xv3i32() #0 {
+  ; CHECK-LABEL: name: stack_12xv3i32
+  ; CHECK: bb.1.entry:
+  ; CHECK:   liveins: $sgpr12, $sgpr13, $sgpr14, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr31
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr13
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr12
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_64 = COPY $sgpr8_sgpr9
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32), [[C]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C1]](s32), [[C1]](s32), [[C1]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; CHECK:   [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C2]](s32), [[C2]](s32), [[C2]](s32)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+  ; CHECK:   [[BUILD_VECTOR3:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C3]](s32), [[C3]](s32), [[C3]](s32)
+  ; CHECK:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; CHECK:   [[BUILD_VECTOR4:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C4]](s32), [[C4]](s32), [[C4]](s32)
+  ; CHECK:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+  ; CHECK:   [[BUILD_VECTOR5:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C5]](s32), [[C5]](s32), [[C5]](s32)
+  ; CHECK:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
+  ; CHECK:   [[BUILD_VECTOR6:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C6]](s32), [[C6]](s32), [[C6]](s32)
+  ; CHECK:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
+  ; CHECK:   [[BUILD_VECTOR7:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C7]](s32), [[C7]](s32), [[C7]](s32)
+  ; CHECK:   [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+  ; CHECK:   [[BUILD_VECTOR8:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C8]](s32), [[C8]](s32), [[C8]](s32)
+  ; CHECK:   [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 9
+  ; CHECK:   [[BUILD_VECTOR9:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C9]](s32), [[C9]](s32), [[C9]](s32)
+  ; CHECK:   [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 11
+  ; CHECK:   [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+  ; CHECK:   [[BUILD_VECTOR10:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C10]](s32), [[C11]](s32), [[C12]](s32)
+  ; CHECK:   [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 13
+  ; CHECK:   [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 14
+  ; CHECK:   [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+  ; CHECK:   [[BUILD_VECTOR11:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C13]](s32), [[C14]](s32), [[C15]](s32)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<3 x s32>)
+  ; CHECK:   [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<3 x s32>)
+  ; CHECK:   [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR2]](<3 x s32>)
+  ; CHECK:   [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR3]](<3 x s32>)
+  ; CHECK:   [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR4]](<3 x s32>)
+  ; CHECK:   [[UV15:%[0-9]+]]:_(s32), [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR5]](<3 x s32>)
+  ; CHECK:   [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR6]](<3 x s32>)
+  ; CHECK:   [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR7]](<3 x s32>)
+  ; CHECK:   [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR8]](<3 x s32>)
+  ; CHECK:   [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR9]](<3 x s32>)
+  ; CHECK:   [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32), [[UV32:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR10]](<3 x s32>)
+  ; CHECK:   [[UV33:%[0-9]+]]:_(s32), [[UV34:%[0-9]+]]:_(s32), [[UV35:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR11]](<3 x s32>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_12xv3i32
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY6]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY5]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s64) = COPY [[COPY4]]
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY2]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY1]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](s32)
+  ; CHECK:   $vgpr5 = COPY [[UV5]](s32)
+  ; CHECK:   $vgpr6 = COPY [[UV6]](s32)
+  ; CHECK:   $vgpr7 = COPY [[UV7]](s32)
+  ; CHECK:   $vgpr8 = COPY [[UV8]](s32)
+  ; CHECK:   $vgpr9 = COPY [[UV9]](s32)
+  ; CHECK:   $vgpr10 = COPY [[UV10]](s32)
+  ; CHECK:   $vgpr11 = COPY [[UV11]](s32)
+  ; CHECK:   $vgpr12 = COPY [[UV12]](s32)
+  ; CHECK:   $vgpr13 = COPY [[UV13]](s32)
+  ; CHECK:   $vgpr14 = COPY [[UV14]](s32)
+  ; CHECK:   $vgpr15 = COPY [[UV15]](s32)
+  ; CHECK:   $vgpr16 = COPY [[UV16]](s32)
+  ; CHECK:   $vgpr17 = COPY [[UV17]](s32)
+  ; CHECK:   $vgpr18 = COPY [[UV18]](s32)
+  ; CHECK:   $vgpr19 = COPY [[UV19]](s32)
+  ; CHECK:   $vgpr20 = COPY [[UV20]](s32)
+  ; CHECK:   $vgpr21 = COPY [[UV21]](s32)
+  ; CHECK:   $vgpr22 = COPY [[UV22]](s32)
+  ; CHECK:   $vgpr23 = COPY [[UV23]](s32)
+  ; CHECK:   $vgpr24 = COPY [[UV24]](s32)
+  ; CHECK:   $vgpr25 = COPY [[UV25]](s32)
+  ; CHECK:   $vgpr26 = COPY [[UV26]](s32)
+  ; CHECK:   $vgpr27 = COPY [[UV27]](s32)
+  ; CHECK:   $vgpr28 = COPY [[UV28]](s32)
+  ; CHECK:   $vgpr29 = COPY [[UV29]](s32)
+  ; CHECK:   $vgpr30 = COPY [[UV30]](s32)
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(p5) = COPY $sgpr32
+  ; CHECK:   [[C16:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C16]](s32)
+  ; CHECK:   G_STORE [[UV31]](s32), [[PTR_ADD]](p5) :: (store 4 into stack, align 16, addrspace 5)
+  ; CHECK:   [[C17:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; CHECK:   [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C17]](s32)
+  ; CHECK:   G_STORE [[UV32]](s32), [[PTR_ADD1]](p5) :: (store 4 into stack + 4, addrspace 5)
+  ; CHECK:   [[C18:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+  ; CHECK:   [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C18]](s32)
+  ; CHECK:   G_STORE [[UV33]](s32), [[PTR_ADD2]](p5) :: (store 4 into stack + 8, align 8, addrspace 5)
+  ; CHECK:   [[C19:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+  ; CHECK:   [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C19]](s32)
+  ; CHECK:   G_STORE [[UV34]](s32), [[PTR_ADD3]](p5) :: (store 4 into stack + 12, addrspace 5)
+  ; CHECK:   [[C20:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; CHECK:   [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C20]](s32)
+  ; CHECK:   G_STORE [[UV35]](s32), [[PTR_ADD4]](p5) :: (store 4 into stack + 16, align 16, addrspace 5)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY18]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY9]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY12]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY13]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY15]](s32)
+  ; CHECK:   $vgpr31 = COPY [[COPY16]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_12xv3i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15, implicit $vgpr16, implicit $vgpr17, implicit $vgpr18, implicit $vgpr19, implicit $vgpr20, implicit $vgpr21, implicit $vgpr22, implicit $vgpr23, implicit $vgpr24, implicit $vgpr25, implicit $vgpr26, implicit $vgpr27, implicit $vgpr28, implicit $vgpr29, implicit $vgpr30, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 20, implicit-def $scc
+  ; CHECK:   [[COPY19:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY8]]
+  ; CHECK:   S_SETPC_B64_return [[COPY19]]
+entry:
+  call void @external_void_func_12xv3i32(
+      <3 x i32> <i32 0, i32 0, i32 0>,
+      <3 x i32> <i32 1, i32 1, i32 1>,
+      <3 x i32> <i32 2, i32 2, i32 2>,
+      <3 x i32> <i32 3, i32 3, i32 3>,
+      <3 x i32> <i32 4, i32 4, i32 4>,
+      <3 x i32> <i32 5, i32 5, i32 5>,
+      <3 x i32> <i32 6, i32 6, i32 6>,
+      <3 x i32> <i32 7, i32 7, i32 7>,
+      <3 x i32> <i32 8, i32 8, i32 8>,
+      <3 x i32> <i32 9, i32 9, i32 9>,
+      <3 x i32> <i32 10, i32 11, i32 12>,
+      <3 x i32> <i32 13, i32 14, i32 15>)
+  ret void
+}
+
+define void @stack_12xv3f32() #0 {
+  ; CHECK-LABEL: name: stack_12xv3f32
+  ; CHECK: bb.1.entry:
+  ; CHECK:   liveins: $sgpr12, $sgpr13, $sgpr14, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr31
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr13
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr12
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_64 = COPY $sgpr8_sgpr9
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32), [[C]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C1]](s32), [[C1]](s32), [[C1]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 2.000000e+00
+  ; CHECK:   [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C2]](s32), [[C2]](s32), [[C2]](s32)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 3.000000e+00
+  ; CHECK:   [[BUILD_VECTOR3:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C3]](s32), [[C3]](s32), [[C3]](s32)
+  ; CHECK:   [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 4.000000e+00
+  ; CHECK:   [[BUILD_VECTOR4:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C4]](s32), [[C4]](s32), [[C4]](s32)
+  ; CHECK:   [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 5.000000e+00
+  ; CHECK:   [[BUILD_VECTOR5:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C5]](s32), [[C5]](s32), [[C5]](s32)
+  ; CHECK:   [[C6:%[0-9]+]]:_(s32) = G_FCONSTANT float 6.000000e+00
+  ; CHECK:   [[BUILD_VECTOR6:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C6]](s32), [[C6]](s32), [[C6]](s32)
+  ; CHECK:   [[C7:%[0-9]+]]:_(s32) = G_FCONSTANT float 7.000000e+00
+  ; CHECK:   [[BUILD_VECTOR7:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C7]](s32), [[C7]](s32), [[C7]](s32)
+  ; CHECK:   [[C8:%[0-9]+]]:_(s32) = G_FCONSTANT float 8.000000e+00
+  ; CHECK:   [[BUILD_VECTOR8:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C8]](s32), [[C8]](s32), [[C8]](s32)
+  ; CHECK:   [[C9:%[0-9]+]]:_(s32) = G_FCONSTANT float 9.000000e+00
+  ; CHECK:   [[BUILD_VECTOR9:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C9]](s32), [[C9]](s32), [[C9]](s32)
+  ; CHECK:   [[C10:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+01
+  ; CHECK:   [[C11:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.100000e+01
+  ; CHECK:   [[C12:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.200000e+01
+  ; CHECK:   [[BUILD_VECTOR10:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C10]](s32), [[C11]](s32), [[C12]](s32)
+  ; CHECK:   [[C13:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.300000e+01
+  ; CHECK:   [[C14:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.400000e+01
+  ; CHECK:   [[C15:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.500000e+01
+  ; CHECK:   [[BUILD_VECTOR11:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[C13]](s32), [[C14]](s32), [[C15]](s32)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<3 x s32>)
+  ; CHECK:   [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<3 x s32>)
+  ; CHECK:   [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR2]](<3 x s32>)
+  ; CHECK:   [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR3]](<3 x s32>)
+  ; CHECK:   [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR4]](<3 x s32>)
+  ; CHECK:   [[UV15:%[0-9]+]]:_(s32), [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR5]](<3 x s32>)
+  ; CHECK:   [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR6]](<3 x s32>)
+  ; CHECK:   [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR7]](<3 x s32>)
+  ; CHECK:   [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR8]](<3 x s32>)
+  ; CHECK:   [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR9]](<3 x s32>)
+  ; CHECK:   [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32), [[UV32:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR10]](<3 x s32>)
+  ; CHECK:   [[UV33:%[0-9]+]]:_(s32), [[UV34:%[0-9]+]]:_(s32), [[UV35:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR11]](<3 x s32>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_12xv3f32
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY6]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY5]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s64) = COPY [[COPY4]]
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY2]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY1]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](s32)
+  ; CHECK:   $vgpr5 = COPY [[UV5]](s32)
+  ; CHECK:   $vgpr6 = COPY [[UV6]](s32)
+  ; CHECK:   $vgpr7 = COPY [[UV7]](s32)
+  ; CHECK:   $vgpr8 = COPY [[UV8]](s32)
+  ; CHECK:   $vgpr9 = COPY [[UV9]](s32)
+  ; CHECK:   $vgpr10 = COPY [[UV10]](s32)
+  ; CHECK:   $vgpr11 = COPY [[UV11]](s32)
+  ; CHECK:   $vgpr12 = COPY [[UV12]](s32)
+  ; CHECK:   $vgpr13 = COPY [[UV13]](s32)
+  ; CHECK:   $vgpr14 = COPY [[UV14]](s32)
+  ; CHECK:   $vgpr15 = COPY [[UV15]](s32)
+  ; CHECK:   $vgpr16 = COPY [[UV16]](s32)
+  ; CHECK:   $vgpr17 = COPY [[UV17]](s32)
+  ; CHECK:   $vgpr18 = COPY [[UV18]](s32)
+  ; CHECK:   $vgpr19 = COPY [[UV19]](s32)
+  ; CHECK:   $vgpr20 = COPY [[UV20]](s32)
+  ; CHECK:   $vgpr21 = COPY [[UV21]](s32)
+  ; CHECK:   $vgpr22 = COPY [[UV22]](s32)
+  ; CHECK:   $vgpr23 = COPY [[UV23]](s32)
+  ; CHECK:   $vgpr24 = COPY [[UV24]](s32)
+  ; CHECK:   $vgpr25 = COPY [[UV25]](s32)
+  ; CHECK:   $vgpr26 = COPY [[UV26]](s32)
+  ; CHECK:   $vgpr27 = COPY [[UV27]](s32)
+  ; CHECK:   $vgpr28 = COPY [[UV28]](s32)
+  ; CHECK:   $vgpr29 = COPY [[UV29]](s32)
+  ; CHECK:   $vgpr30 = COPY [[UV30]](s32)
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(p5) = COPY $sgpr32
+  ; CHECK:   [[C16:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C16]](s32)
+  ; CHECK:   G_STORE [[UV31]](s32), [[PTR_ADD]](p5) :: (store 4 into stack, align 16, addrspace 5)
+  ; CHECK:   [[C17:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; CHECK:   [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C17]](s32)
+  ; CHECK:   G_STORE [[UV32]](s32), [[PTR_ADD1]](p5) :: (store 4 into stack + 4, addrspace 5)
+  ; CHECK:   [[C18:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+  ; CHECK:   [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C18]](s32)
+  ; CHECK:   G_STORE [[UV33]](s32), [[PTR_ADD2]](p5) :: (store 4 into stack + 8, align 8, addrspace 5)
+  ; CHECK:   [[C19:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+  ; CHECK:   [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C19]](s32)
+  ; CHECK:   G_STORE [[UV34]](s32), [[PTR_ADD3]](p5) :: (store 4 into stack + 12, addrspace 5)
+  ; CHECK:   [[C20:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; CHECK:   [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C20]](s32)
+  ; CHECK:   G_STORE [[UV35]](s32), [[PTR_ADD4]](p5) :: (store 4 into stack + 16, align 16, addrspace 5)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY18]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY9]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY12]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY13]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY15]](s32)
+  ; CHECK:   $vgpr31 = COPY [[COPY16]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_12xv3f32, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15, implicit $vgpr16, implicit $vgpr17, implicit $vgpr18, implicit $vgpr19, implicit $vgpr20, implicit $vgpr21, implicit $vgpr22, implicit $vgpr23, implicit $vgpr24, implicit $vgpr25, implicit $vgpr26, implicit $vgpr27, implicit $vgpr28, implicit $vgpr29, implicit $vgpr30, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 20, implicit-def $scc
+  ; CHECK:   [[COPY19:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY8]]
+  ; CHECK:   S_SETPC_B64_return [[COPY19]]
+entry:
+  call void @external_void_func_12xv3f32(
+      <3 x float> <float 0.0, float 0.0, float 0.0>,
+      <3 x float> <float 1.0, float 1.0, float 1.0>,
+      <3 x float> <float 2.0, float 2.0, float 2.0>,
+      <3 x float> <float 3.0, float 3.0, float 3.0>,
+      <3 x float> <float 4.0, float 4.0, float 4.0>,
+      <3 x float> <float 5.0, float 5.0, float 5.0>,
+      <3 x float> <float 6.0, float 6.0, float 6.0>,
+      <3 x float> <float 7.0, float 7.0, float 7.0>,
+      <3 x float> <float 8.0, float 8.0, float 8.0>,
+      <3 x float> <float 9.0, float 9.0, float 9.0>,
+      <3 x float> <float 10.0, float 11.0, float 12.0>,
+      <3 x float> <float 13.0, float 14.0, float 15.0>)
+  ret void
+}
+
+define void @stack_8xv5i32() #0 {
+  ; CHECK-LABEL: name: stack_8xv5i32
+  ; CHECK: bb.1.entry:
+  ; CHECK:   liveins: $sgpr12, $sgpr13, $sgpr14, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr31
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr13
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr12
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_64 = COPY $sgpr8_sgpr9
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[C1]](s32), [[C1]](s32), [[C1]](s32), [[C1]](s32), [[C1]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; CHECK:   [[BUILD_VECTOR2:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[C2]](s32), [[C2]](s32), [[C2]](s32), [[C2]](s32), [[C2]](s32)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+  ; CHECK:   [[BUILD_VECTOR3:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[C3]](s32), [[C3]](s32), [[C3]](s32), [[C3]](s32), [[C3]](s32)
+  ; CHECK:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; CHECK:   [[BUILD_VECTOR4:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[C4]](s32), [[C4]](s32), [[C4]](s32), [[C4]](s32), [[C4]](s32)
+  ; CHECK:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+  ; CHECK:   [[BUILD_VECTOR5:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[C5]](s32), [[C5]](s32), [[C5]](s32), [[C5]](s32), [[C5]](s32)
+  ; CHECK:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
+  ; CHECK:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
+  ; CHECK:   [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+  ; CHECK:   [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 9
+  ; CHECK:   [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[BUILD_VECTOR6:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[C6]](s32), [[C7]](s32), [[C8]](s32), [[C9]](s32), [[C10]](s32)
+  ; CHECK:   [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 11
+  ; CHECK:   [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+  ; CHECK:   [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 13
+  ; CHECK:   [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 14
+  ; CHECK:   [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+  ; CHECK:   [[BUILD_VECTOR7:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[C11]](s32), [[C12]](s32), [[C13]](s32), [[C14]](s32), [[C15]](s32)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<5 x s32>)
+  ; CHECK:   [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<5 x s32>)
+  ; CHECK:   [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR2]](<5 x s32>)
+  ; CHECK:   [[UV15:%[0-9]+]]:_(s32), [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR3]](<5 x s32>)
+  ; CHECK:   [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR4]](<5 x s32>)
+  ; CHECK:   [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR5]](<5 x s32>)
+  ; CHECK:   [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32), [[UV32:%[0-9]+]]:_(s32), [[UV33:%[0-9]+]]:_(s32), [[UV34:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR6]](<5 x s32>)
+  ; CHECK:   [[UV35:%[0-9]+]]:_(s32), [[UV36:%[0-9]+]]:_(s32), [[UV37:%[0-9]+]]:_(s32), [[UV38:%[0-9]+]]:_(s32), [[UV39:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR7]](<5 x s32>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_8xv5i32
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY6]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY5]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s64) = COPY [[COPY4]]
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY2]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY1]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](s32)
+  ; CHECK:   $vgpr5 = COPY [[UV5]](s32)
+  ; CHECK:   $vgpr6 = COPY [[UV6]](s32)
+  ; CHECK:   $vgpr7 = COPY [[UV7]](s32)
+  ; CHECK:   $vgpr8 = COPY [[UV8]](s32)
+  ; CHECK:   $vgpr9 = COPY [[UV9]](s32)
+  ; CHECK:   $vgpr10 = COPY [[UV10]](s32)
+  ; CHECK:   $vgpr11 = COPY [[UV11]](s32)
+  ; CHECK:   $vgpr12 = COPY [[UV12]](s32)
+  ; CHECK:   $vgpr13 = COPY [[UV13]](s32)
+  ; CHECK:   $vgpr14 = COPY [[UV14]](s32)
+  ; CHECK:   $vgpr15 = COPY [[UV15]](s32)
+  ; CHECK:   $vgpr16 = COPY [[UV16]](s32)
+  ; CHECK:   $vgpr17 = COPY [[UV17]](s32)
+  ; CHECK:   $vgpr18 = COPY [[UV18]](s32)
+  ; CHECK:   $vgpr19 = COPY [[UV19]](s32)
+  ; CHECK:   $vgpr20 = COPY [[UV20]](s32)
+  ; CHECK:   $vgpr21 = COPY [[UV21]](s32)
+  ; CHECK:   $vgpr22 = COPY [[UV22]](s32)
+  ; CHECK:   $vgpr23 = COPY [[UV23]](s32)
+  ; CHECK:   $vgpr24 = COPY [[UV24]](s32)
+  ; CHECK:   $vgpr25 = COPY [[UV25]](s32)
+  ; CHECK:   $vgpr26 = COPY [[UV26]](s32)
+  ; CHECK:   $vgpr27 = COPY [[UV27]](s32)
+  ; CHECK:   $vgpr28 = COPY [[UV28]](s32)
+  ; CHECK:   $vgpr29 = COPY [[UV29]](s32)
+  ; CHECK:   $vgpr30 = COPY [[UV30]](s32)
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(p5) = COPY $sgpr32
+  ; CHECK:   [[C16:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C16]](s32)
+  ; CHECK:   G_STORE [[UV31]](s32), [[PTR_ADD]](p5) :: (store 4 into stack, align 16, addrspace 5)
+  ; CHECK:   [[C17:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; CHECK:   [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C17]](s32)
+  ; CHECK:   G_STORE [[UV32]](s32), [[PTR_ADD1]](p5) :: (store 4 into stack + 4, addrspace 5)
+  ; CHECK:   [[C18:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+  ; CHECK:   [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C18]](s32)
+  ; CHECK:   G_STORE [[UV33]](s32), [[PTR_ADD2]](p5) :: (store 4 into stack + 8, align 8, addrspace 5)
+  ; CHECK:   [[C19:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+  ; CHECK:   [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C19]](s32)
+  ; CHECK:   G_STORE [[UV34]](s32), [[PTR_ADD3]](p5) :: (store 4 into stack + 12, addrspace 5)
+  ; CHECK:   [[C20:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; CHECK:   [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C20]](s32)
+  ; CHECK:   G_STORE [[UV35]](s32), [[PTR_ADD4]](p5) :: (store 4 into stack + 16, align 16, addrspace 5)
+  ; CHECK:   [[C21:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C21]](s32)
+  ; CHECK:   G_STORE [[UV36]](s32), [[PTR_ADD5]](p5) :: (store 4 into stack + 20, addrspace 5)
+  ; CHECK:   [[C22:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+  ; CHECK:   [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C22]](s32)
+  ; CHECK:   G_STORE [[UV37]](s32), [[PTR_ADD6]](p5) :: (store 4 into stack + 24, align 8, addrspace 5)
+  ; CHECK:   [[C23:%[0-9]+]]:_(s32) = G_CONSTANT i32 28
+  ; CHECK:   [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C23]](s32)
+  ; CHECK:   G_STORE [[UV38]](s32), [[PTR_ADD7]](p5) :: (store 4 into stack + 28, addrspace 5)
+  ; CHECK:   [[C24:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+  ; CHECK:   [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C24]](s32)
+  ; CHECK:   G_STORE [[UV39]](s32), [[PTR_ADD8]](p5) :: (store 4 into stack + 32, align 16, addrspace 5)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY18]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY9]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY12]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY13]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY15]](s32)
+  ; CHECK:   $vgpr31 = COPY [[COPY16]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_8xv5i32, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15, implicit $vgpr16, implicit $vgpr17, implicit $vgpr18, implicit $vgpr19, implicit $vgpr20, implicit $vgpr21, implicit $vgpr22, implicit $vgpr23, implicit $vgpr24, implicit $vgpr25, implicit $vgpr26, implicit $vgpr27, implicit $vgpr28, implicit $vgpr29, implicit $vgpr30, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 36, implicit-def $scc
+  ; CHECK:   [[COPY19:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY8]]
+  ; CHECK:   S_SETPC_B64_return [[COPY19]]
+entry:
+  call void @external_void_func_8xv5i32(
+      <5 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0>,
+      <5 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1>,
+      <5 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2>,
+      <5 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3>,
+      <5 x i32> <i32 4, i32 4, i32 4, i32 4, i32 4>,
+      <5 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5>,
+      <5 x i32> <i32 6, i32 7, i32 8, i32 9, i32 10>,
+      <5 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15>)
+  ret void
+}
+
+define void @stack_8xv5f32() #0 {
+  ; CHECK-LABEL: name: stack_8xv5f32
+  ; CHECK: bb.1.entry:
+  ; CHECK:   liveins: $sgpr12, $sgpr13, $sgpr14, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr31
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr13
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr12
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_64 = COPY $sgpr8_sgpr9
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[C1]](s32), [[C1]](s32), [[C1]](s32), [[C1]](s32), [[C1]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 2.000000e+00
+  ; CHECK:   [[BUILD_VECTOR2:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[C2]](s32), [[C2]](s32), [[C2]](s32), [[C2]](s32), [[C2]](s32)
+  ; CHECK:   [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 3.000000e+00
+  ; CHECK:   [[BUILD_VECTOR3:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[C3]](s32), [[C3]](s32), [[C3]](s32), [[C3]](s32), [[C3]](s32)
+  ; CHECK:   [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 4.000000e+00
+  ; CHECK:   [[BUILD_VECTOR4:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[C4]](s32), [[C4]](s32), [[C4]](s32), [[C4]](s32), [[C4]](s32)
+  ; CHECK:   [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 5.000000e+00
+  ; CHECK:   [[BUILD_VECTOR5:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[C5]](s32), [[C5]](s32), [[C5]](s32), [[C5]](s32), [[C5]](s32)
+  ; CHECK:   [[C6:%[0-9]+]]:_(s32) = G_FCONSTANT float 6.000000e+00
+  ; CHECK:   [[C7:%[0-9]+]]:_(s32) = G_FCONSTANT float 7.000000e+00
+  ; CHECK:   [[C8:%[0-9]+]]:_(s32) = G_FCONSTANT float 8.000000e+00
+  ; CHECK:   [[C9:%[0-9]+]]:_(s32) = G_FCONSTANT float 9.000000e+00
+  ; CHECK:   [[C10:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+01
+  ; CHECK:   [[BUILD_VECTOR6:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[C6]](s32), [[C7]](s32), [[C8]](s32), [[C9]](s32), [[C10]](s32)
+  ; CHECK:   [[C11:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.100000e+01
+  ; CHECK:   [[C12:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.200000e+01
+  ; CHECK:   [[C13:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.300000e+01
+  ; CHECK:   [[C14:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.400000e+01
+  ; CHECK:   [[C15:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.500000e+01
+  ; CHECK:   [[BUILD_VECTOR7:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[C11]](s32), [[C12]](s32), [[C13]](s32), [[C14]](s32), [[C15]](s32)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<5 x s32>)
+  ; CHECK:   [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<5 x s32>)
+  ; CHECK:   [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR2]](<5 x s32>)
+  ; CHECK:   [[UV15:%[0-9]+]]:_(s32), [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR3]](<5 x s32>)
+  ; CHECK:   [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR4]](<5 x s32>)
+  ; CHECK:   [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR5]](<5 x s32>)
+  ; CHECK:   [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32), [[UV32:%[0-9]+]]:_(s32), [[UV33:%[0-9]+]]:_(s32), [[UV34:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR6]](<5 x s32>)
+  ; CHECK:   [[UV35:%[0-9]+]]:_(s32), [[UV36:%[0-9]+]]:_(s32), [[UV37:%[0-9]+]]:_(s32), [[UV38:%[0-9]+]]:_(s32), [[UV39:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BUILD_VECTOR7]](<5 x s32>)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[GV:%[0-9]+]]:sreg_64(p0) = G_GLOBAL_VALUE @external_void_func_8xv5f32
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY6]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY5]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s64) = COPY [[COPY4]]
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY2]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY1]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   $vgpr0 = COPY [[UV]](s32)
+  ; CHECK:   $vgpr1 = COPY [[UV1]](s32)
+  ; CHECK:   $vgpr2 = COPY [[UV2]](s32)
+  ; CHECK:   $vgpr3 = COPY [[UV3]](s32)
+  ; CHECK:   $vgpr4 = COPY [[UV4]](s32)
+  ; CHECK:   $vgpr5 = COPY [[UV5]](s32)
+  ; CHECK:   $vgpr6 = COPY [[UV6]](s32)
+  ; CHECK:   $vgpr7 = COPY [[UV7]](s32)
+  ; CHECK:   $vgpr8 = COPY [[UV8]](s32)
+  ; CHECK:   $vgpr9 = COPY [[UV9]](s32)
+  ; CHECK:   $vgpr10 = COPY [[UV10]](s32)
+  ; CHECK:   $vgpr11 = COPY [[UV11]](s32)
+  ; CHECK:   $vgpr12 = COPY [[UV12]](s32)
+  ; CHECK:   $vgpr13 = COPY [[UV13]](s32)
+  ; CHECK:   $vgpr14 = COPY [[UV14]](s32)
+  ; CHECK:   $vgpr15 = COPY [[UV15]](s32)
+  ; CHECK:   $vgpr16 = COPY [[UV16]](s32)
+  ; CHECK:   $vgpr17 = COPY [[UV17]](s32)
+  ; CHECK:   $vgpr18 = COPY [[UV18]](s32)
+  ; CHECK:   $vgpr19 = COPY [[UV19]](s32)
+  ; CHECK:   $vgpr20 = COPY [[UV20]](s32)
+  ; CHECK:   $vgpr21 = COPY [[UV21]](s32)
+  ; CHECK:   $vgpr22 = COPY [[UV22]](s32)
+  ; CHECK:   $vgpr23 = COPY [[UV23]](s32)
+  ; CHECK:   $vgpr24 = COPY [[UV24]](s32)
+  ; CHECK:   $vgpr25 = COPY [[UV25]](s32)
+  ; CHECK:   $vgpr26 = COPY [[UV26]](s32)
+  ; CHECK:   $vgpr27 = COPY [[UV27]](s32)
+  ; CHECK:   $vgpr28 = COPY [[UV28]](s32)
+  ; CHECK:   $vgpr29 = COPY [[UV29]](s32)
+  ; CHECK:   $vgpr30 = COPY [[UV30]](s32)
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(p5) = COPY $sgpr32
+  ; CHECK:   [[C16:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C16]](s32)
+  ; CHECK:   G_STORE [[UV31]](s32), [[PTR_ADD]](p5) :: (store 4 into stack, align 16, addrspace 5)
+  ; CHECK:   [[C17:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; CHECK:   [[PTR_ADD1:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C17]](s32)
+  ; CHECK:   G_STORE [[UV32]](s32), [[PTR_ADD1]](p5) :: (store 4 into stack + 4, addrspace 5)
+  ; CHECK:   [[C18:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+  ; CHECK:   [[PTR_ADD2:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C18]](s32)
+  ; CHECK:   G_STORE [[UV33]](s32), [[PTR_ADD2]](p5) :: (store 4 into stack + 8, align 8, addrspace 5)
+  ; CHECK:   [[C19:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+  ; CHECK:   [[PTR_ADD3:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C19]](s32)
+  ; CHECK:   G_STORE [[UV34]](s32), [[PTR_ADD3]](p5) :: (store 4 into stack + 12, addrspace 5)
+  ; CHECK:   [[C20:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; CHECK:   [[PTR_ADD4:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C20]](s32)
+  ; CHECK:   G_STORE [[UV35]](s32), [[PTR_ADD4]](p5) :: (store 4 into stack + 16, align 16, addrspace 5)
+  ; CHECK:   [[C21:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[PTR_ADD5:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C21]](s32)
+  ; CHECK:   G_STORE [[UV36]](s32), [[PTR_ADD5]](p5) :: (store 4 into stack + 20, addrspace 5)
+  ; CHECK:   [[C22:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+  ; CHECK:   [[PTR_ADD6:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C22]](s32)
+  ; CHECK:   G_STORE [[UV37]](s32), [[PTR_ADD6]](p5) :: (store 4 into stack + 24, align 8, addrspace 5)
+  ; CHECK:   [[C23:%[0-9]+]]:_(s32) = G_CONSTANT i32 28
+  ; CHECK:   [[PTR_ADD7:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C23]](s32)
+  ; CHECK:   G_STORE [[UV38]](s32), [[PTR_ADD7]](p5) :: (store 4 into stack + 28, addrspace 5)
+  ; CHECK:   [[C24:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+  ; CHECK:   [[PTR_ADD8:%[0-9]+]]:_(p5) = G_PTR_ADD [[COPY17]], [[C24]](s32)
+  ; CHECK:   G_STORE [[UV39]](s32), [[PTR_ADD8]](p5) :: (store 4 into stack + 32, align 16, addrspace 5)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY18]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY9]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY12]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY13]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY15]](s32)
+  ; CHECK:   $vgpr31 = COPY [[COPY16]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[GV]](p0), @external_void_func_8xv5f32, csr_amdgpu_highregs, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit $vgpr8, implicit $vgpr9, implicit $vgpr10, implicit $vgpr11, implicit $vgpr12, implicit $vgpr13, implicit $vgpr14, implicit $vgpr15, implicit $vgpr16, implicit $vgpr17, implicit $vgpr18, implicit $vgpr19, implicit $vgpr20, implicit $vgpr21, implicit $vgpr22, implicit $vgpr23, implicit $vgpr24, implicit $vgpr25, implicit $vgpr26, implicit $vgpr27, implicit $vgpr28, implicit $vgpr29, implicit $vgpr30, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 36, implicit-def $scc
+  ; CHECK:   [[COPY19:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY8]]
+  ; CHECK:   S_SETPC_B64_return [[COPY19]]
+entry:
+  call void @external_void_func_8xv5f32(
+      <5 x float> <float 0.0, float 0.0, float 0.0, float 0.0, float 0.0>,
+      <5 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>,
+      <5 x float> <float 2.0, float 2.0, float 2.0, float 2.0, float 2.0>,
+      <5 x float> <float 3.0, float 3.0, float 3.0, float 3.0, float 3.0>,
+      <5 x float> <float 4.0, float 4.0, float 4.0, float 4.0, float 4.0>,
+      <5 x float> <float 5.0, float 5.0, float 5.0, float 5.0, float 5.0>,
+      <5 x float> <float 6.0, float 7.0, float 8.0, float 9.0, float 10.0>,
+      <5 x float> <float 11.0, float 12.0, float 13.0, float 14.0, float 15.0>)
+  ret void
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind noinline }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll
index 340392ea8d464..28f60ca7528db 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll
@@ -1795,6 +1795,62 @@ define void @void_func_v32i32_i1_i8_i16(<32 x i32> %arg0, i1 %arg1, i8 %arg2, i1
   ret void
 }
 
+define void @void_func_v32i32_p3_p5_i16(<32 x i32> %arg0, i8 addrspace(3)* %arg1, i8 addrspace(5)* %arg2) #0 {
+  ; CHECK-LABEL: name: void_func_v32i32_p3_p5_i16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21, $vgpr22, $vgpr23, $vgpr24, $vgpr25, $vgpr26, $vgpr27, $vgpr28, $vgpr29, $vgpr30, $vgpr31, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr16
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY $vgpr17
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY $vgpr18
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY $vgpr19
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(s32) = COPY $vgpr20
+  ; CHECK:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr21
+  ; CHECK:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr22
+  ; CHECK:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr23
+  ; CHECK:   [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr24
+  ; CHECK:   [[COPY25:%[0-9]+]]:_(s32) = COPY $vgpr25
+  ; CHECK:   [[COPY26:%[0-9]+]]:_(s32) = COPY $vgpr26
+  ; CHECK:   [[COPY27:%[0-9]+]]:_(s32) = COPY $vgpr27
+  ; CHECK:   [[COPY28:%[0-9]+]]:_(s32) = COPY $vgpr28
+  ; CHECK:   [[COPY29:%[0-9]+]]:_(s32) = COPY $vgpr29
+  ; CHECK:   [[COPY30:%[0-9]+]]:_(s32) = COPY $vgpr30
+  ; CHECK:   [[COPY31:%[0-9]+]]:_(s32) = COPY $vgpr31
+  ; CHECK:   [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.1
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(p3) = G_LOAD [[FRAME_INDEX]](p5) :: (invariant load 4 from %fixed-stack.1, align 16, addrspace 5)
+  ; CHECK:   [[FRAME_INDEX1:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.0
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(p5) = G_LOAD [[FRAME_INDEX1]](p5) :: (invariant load 4 from %fixed-stack.0, addrspace 5)
+  ; CHECK:   [[COPY32:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32), [[COPY17]](s32), [[COPY18]](s32), [[COPY19]](s32), [[COPY20]](s32), [[COPY21]](s32), [[COPY22]](s32), [[COPY23]](s32), [[COPY24]](s32), [[COPY25]](s32), [[COPY26]](s32), [[COPY27]](s32), [[COPY28]](s32), [[COPY29]](s32), [[COPY30]](s32), [[COPY31]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[COPY33:%[0-9]+]]:_(p1) = COPY [[DEF]](p1)
+  ; CHECK:   [[COPY34:%[0-9]+]]:_(p1) = COPY [[DEF]](p1)
+  ; CHECK:   G_STORE [[BUILD_VECTOR]](<32 x s32>), [[DEF]](p1) :: (volatile store 128 into `<32 x i32> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[LOAD]](p3), [[COPY33]](p1) :: (volatile store 4 into `i8 addrspace(3)* addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[LOAD1]](p5), [[COPY34]](p1) :: (volatile store 4 into `i8 addrspace(5)* addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   [[COPY35:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY32]]
+  ; CHECK:   S_SETPC_B64_return [[COPY35]]
+  store volatile <32 x i32> %arg0, <32 x i32> addrspace(1)* undef
+  store volatile i8 addrspace(3)* %arg1, i8 addrspace(3)* addrspace(1)* undef
+  store volatile i8 addrspace(5)* %arg2, i8 addrspace(5)* addrspace(1)* undef
+  ret void
+}
+
 define void @void_func_v32i32_v2i32_v2f32(<32 x i32> %arg0, <2 x i32> %arg1, <2 x float> %arg2) #0 {
   ; CHECK-LABEL: name: void_func_v32i32_v2i32_v2f32
   ; CHECK: bb.1 (%ir-block.0):
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-indirect-call.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-indirect-call.ll
new file mode 100644
index 0000000000000..96eb596dabc51
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-indirect-call.ll
@@ -0,0 +1,54 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -amdgpu-fixed-function-abi -stop-after=irtranslator -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -verify-machineinstrs -o - %s | FileCheck -enable-var-scope %s
+
+define amdgpu_kernel void @test_indirect_call_sgpr_ptr(void()* %fptr) {
+  ; CHECK-LABEL: name: test_indirect_call_sgpr_ptr
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $vgpr1, $vgpr2, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr16
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr15
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr14
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr_64 = COPY $sgpr10_sgpr11
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
+  ; CHECK:   [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
+  ; CHECK:   [[INT:%[0-9]+]]:_(p4) = G_INTRINSIC intrinsic(@llvm.amdgcn.kernarg.segment.ptr)
+  ; CHECK:   [[LOAD:%[0-9]+]]:sreg_64(p0) = G_LOAD [[INT]](p4) :: (dereferenceable invariant load 8 from %ir.fptr.kernarg.offset.cast, align 16, addrspace 4)
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK:   [[COPY10:%[0-9]+]]:_(p4) = COPY [[COPY8]]
+  ; CHECK:   [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
+  ; CHECK:   [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p4) = G_PTR_ADD [[COPY12]], [[C]](s64)
+  ; CHECK:   [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
+  ; CHECK:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
+  ; CHECK:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
+  ; CHECK:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
+  ; CHECK:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+  ; CHECK:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; CHECK:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY18]], [[C1]](s32)
+  ; CHECK:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY17]], [[SHL]]
+  ; CHECK:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
+  ; CHECK:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C2]](s32)
+  ; CHECK:   [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+  ; CHECK:   [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
+  ; CHECK:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
+  ; CHECK:   $sgpr4_sgpr5 = COPY [[COPY10]](p4)
+  ; CHECK:   $sgpr6_sgpr7 = COPY [[COPY11]](p4)
+  ; CHECK:   $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
+  ; CHECK:   $sgpr10_sgpr11 = COPY [[COPY13]](s64)
+  ; CHECK:   $sgpr12 = COPY [[COPY14]](s32)
+  ; CHECK:   $sgpr13 = COPY [[COPY15]](s32)
+  ; CHECK:   $sgpr14 = COPY [[COPY16]](s32)
+  ; CHECK:   $vgpr31 = COPY [[OR1]](s32)
+  ; CHECK:   $sgpr30_sgpr31 = SI_CALL [[LOAD]](p0), 0, csr_amdgpu_highregs, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $vgpr31
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK:   S_ENDPGM 0
+  call void %fptr()
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll
index 7dc247de3687e..07b8dced57132 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll
@@ -174,7 +174,7 @@ define amdgpu_kernel void @test_input_vgpr_imm() {
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY [[C]](s32)
-  ; CHECK:   INLINEASM &"v_mov_b32 v0, $0", 1 /* sideeffect attdialect */, 9 /* reguse */, [[COPY]]
+  ; CHECK:   INLINEASM &"v_mov_b32 v0, $0", 1 /* sideeffect attdialect */, 1835017 /* reguse:VGPR_32 */, [[COPY]]
   ; CHECK:   S_ENDPGM 0
   call void asm sideeffect "v_mov_b32 v0, $0", "v"(i32 42)
   ret void
@@ -185,7 +185,7 @@ define amdgpu_kernel void @test_input_sgpr_imm() {
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
   ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY [[C]](s32)
-  ; CHECK:   INLINEASM &"s_mov_b32 s0, $0", 1 /* sideeffect attdialect */, 9 /* reguse */, [[COPY]]
+  ; CHECK:   INLINEASM &"s_mov_b32 s0, $0", 1 /* sideeffect attdialect */, 1966089 /* reguse:SReg_32 */, [[COPY]]
   ; CHECK:   S_ENDPGM 0
   call void asm sideeffect "s_mov_b32 s0, $0", "s"(i32 42)
   ret void
@@ -209,7 +209,7 @@ define float @test_input_vgpr(i32 %src) nounwind {
   ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
   ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]](s32)
-  ; CHECK:   INLINEASM &"v_add_f32 $0, 1.0, $1", 0 /* attdialect */, 1835018 /* regdef:VGPR_32 */, def %2, 9 /* reguse */, [[COPY2]]
+  ; CHECK:   INLINEASM &"v_add_f32 $0, 1.0, $1", 0 /* attdialect */, 1835018 /* regdef:VGPR_32 */, def %2, 1835017 /* reguse:VGPR_32 */, [[COPY2]]
   ; CHECK:   [[COPY3:%[0-9]+]]:_(s32) = COPY %2
   ; CHECK:   $vgpr0 = COPY [[COPY3]](s32)
   ; CHECK:   [[COPY4:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
@@ -264,7 +264,7 @@ define i32 @test_sgpr_matching_constraint() nounwind {
   ; CHECK:   [[COPY2:%[0-9]+]]:_(s32) = COPY %3
   ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32 = COPY [[COPY1]](s32)
   ; CHECK:   [[COPY4:%[0-9]+]]:sreg_32 = COPY [[COPY2]](s32)
-  ; CHECK:   INLINEASM &"s_add_u32 $0, $1, $2", 0 /* attdialect */, 1966090 /* regdef:SReg_32 */, def %5, 9 /* reguse */, [[COPY3]], 2147483657 /* reguse tiedto:$0 */, [[COPY4]](tied-def 3)
+  ; CHECK:   INLINEASM &"s_add_u32 $0, $1, $2", 0 /* attdialect */, 1966090 /* regdef:SReg_32 */, def %5, 1966089 /* reguse:SReg_32 */, [[COPY3]], 2147483657 /* reguse tiedto:$0 */, [[COPY4]](tied-def 3)
   ; CHECK:   [[COPY5:%[0-9]+]]:_(s32) = COPY %5
   ; CHECK:   $vgpr0 = COPY [[COPY5]](s32)
   ; CHECK:   [[COPY6:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-and.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-and.mir
index 5aaf09a26509e..3d26cefc4f55d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-and.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-and.mir
@@ -443,31 +443,29 @@ body: |
     ; CHECK: [[DEF1:%[0-9]+]]:_(<3 x s8>) = G_IMPLICIT_DEF
     ; CHECK: [[DEF2:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
     ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF2]], [[DEF]](<3 x s8>), 0
-    ; CHECK: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF2]](<4 x s8>)
-    ; CHECK: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[DEF1]](<3 x s8>), 0
-    ; CHECK: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT1]](<4 x s16>)
+    ; CHECK: [[INSERT1:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF2]], [[DEF1]](<3 x s8>), 0
     ; CHECK: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8), [[UV2:%[0-9]+]]:_(s8), [[UV3:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[INSERT]](<4 x s8>)
-    ; CHECK: [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8), [[UV6:%[0-9]+]]:_(s8), [[UV7:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[TRUNC]](<4 x s8>)
-    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s8)
-    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV4]](s8)
-    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[ANYEXT2]]
-    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[AND]](s32)
-    ; CHECK: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s8)
-    ; CHECK: [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[UV5]](s8)
-    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ANYEXT3]], [[ANYEXT4]]
-    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[AND1]](s32)
-    ; CHECK: [[ANYEXT5:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s8)
-    ; CHECK: [[ANYEXT6:%[0-9]+]]:_(s32) = G_ANYEXT [[UV6]](s8)
-    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ANYEXT5]], [[ANYEXT6]]
-    ; CHECK: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[AND2]](s32)
-    ; CHECK: [[ANYEXT7:%[0-9]+]]:_(s32) = G_ANYEXT [[UV3]](s8)
-    ; CHECK: [[ANYEXT8:%[0-9]+]]:_(s32) = G_ANYEXT [[UV7]](s8)
-    ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ANYEXT7]], [[ANYEXT8]]
-    ; CHECK: [[TRUNC4:%[0-9]+]]:_(s8) = G_TRUNC [[AND3]](s32)
-    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s8>) = G_BUILD_VECTOR [[TRUNC1]](s8), [[TRUNC2]](s8), [[TRUNC3]](s8), [[TRUNC4]](s8)
+    ; CHECK: [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8), [[UV6:%[0-9]+]]:_(s8), [[UV7:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[INSERT1]](<4 x s8>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s8)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV4]](s8)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[ANYEXT1]]
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[AND]](s32)
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s8)
+    ; CHECK: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[UV5]](s8)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[ANYEXT3]]
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[AND1]](s32)
+    ; CHECK: [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s8)
+    ; CHECK: [[ANYEXT5:%[0-9]+]]:_(s32) = G_ANYEXT [[UV6]](s8)
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ANYEXT4]], [[ANYEXT5]]
+    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[AND2]](s32)
+    ; CHECK: [[ANYEXT6:%[0-9]+]]:_(s32) = G_ANYEXT [[UV3]](s8)
+    ; CHECK: [[ANYEXT7:%[0-9]+]]:_(s32) = G_ANYEXT [[UV7]](s8)
+    ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ANYEXT6]], [[ANYEXT7]]
+    ; CHECK: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[AND3]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s8>) = G_BUILD_VECTOR [[TRUNC]](s8), [[TRUNC1]](s8), [[TRUNC2]](s8), [[TRUNC3]](s8)
     ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s8>) = G_EXTRACT [[BUILD_VECTOR]](<4 x s8>), 0
-    ; CHECK: [[ANYEXT9:%[0-9]+]]:_(<3 x s32>) = G_ANYEXT [[EXTRACT]](<3 x s8>)
-    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[ANYEXT9]](<3 x s32>)
+    ; CHECK: [[ANYEXT8:%[0-9]+]]:_(<3 x s32>) = G_ANYEXT [[EXTRACT]](<3 x s8>)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[ANYEXT8]](<3 x s32>)
     %0:_(<3 x s8>) = G_IMPLICIT_DEF
     %1:_(<3 x s8>) = G_IMPLICIT_DEF
     %2:_(<3 x s8>) = G_AND %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fceil.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fceil.mir
index 39537fef6471a..0ab5583f20c2a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fceil.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fceil.mir
@@ -79,7 +79,7 @@ body: |
     ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
     ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 11
-    ; SI: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[C]](s32), [[C1]](s32)
+    ; SI: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[UV1]](s32), [[C]](s32), [[C1]](s32)
     ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1023
     ; SI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[INT]], [[C2]]
     ; SI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
@@ -96,14 +96,13 @@ body: |
     ; SI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[SUB]](s32), [[C7]]
     ; SI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[MV]], [[AND1]]
     ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[COPY]], [[SELECT]]
-    ; SI: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s64) = G_INTRINSIC_TRUNC [[COPY]]
     ; SI: [[C8:%[0-9]+]]:_(s64) = G_FCONSTANT double 0.000000e+00
     ; SI: [[C9:%[0-9]+]]:_(s64) = G_FCONSTANT double 1.000000e+00
     ; SI: [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[COPY]](s64), [[C8]]
-    ; SI: [[FCMP1:%[0-9]+]]:_(s1) = G_FCMP floatpred(one), [[COPY]](s64), [[INTRINSIC_TRUNC]]
+    ; SI: [[FCMP1:%[0-9]+]]:_(s1) = G_FCMP floatpred(one), [[COPY]](s64), [[SELECT1]]
     ; SI: [[AND2:%[0-9]+]]:_(s1) = G_AND [[FCMP]], [[FCMP1]]
-    ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[AND2]](s1), [[C9]], [[C8]]
-    ; SI: [[FADD:%[0-9]+]]:_(s64) = G_FADD [[INTRINSIC_TRUNC]], [[SELECT1]]
+    ; SI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[AND2]](s1), [[C9]], [[C8]]
+    ; SI: [[FADD:%[0-9]+]]:_(s64) = G_FADD [[SELECT1]], [[SELECT2]]
     ; SI: [[FCEIL:%[0-9]+]]:_(s64) = G_FCEIL [[COPY]]
     ; SI: $vgpr0_vgpr1 = COPY [[FCEIL]](s64)
     ; CI-LABEL: name: test_fceil_s64
@@ -197,7 +196,7 @@ body: |
     ; SI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
     ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 11
-    ; SI: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[C]](s32), [[C1]](s32)
+    ; SI: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[UV3]](s32), [[C]](s32), [[C1]](s32)
     ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1023
     ; SI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[INT]], [[C2]]
     ; SI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
@@ -214,17 +213,16 @@ body: |
     ; SI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[SUB]](s32), [[C7]]
     ; SI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[MV]], [[AND1]]
     ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV]], [[SELECT]]
-    ; SI: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s64) = G_INTRINSIC_TRUNC [[UV]]
     ; SI: [[C8:%[0-9]+]]:_(s64) = G_FCONSTANT double 0.000000e+00
     ; SI: [[C9:%[0-9]+]]:_(s64) = G_FCONSTANT double 1.000000e+00
     ; SI: [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[UV]](s64), [[C8]]
-    ; SI: [[FCMP1:%[0-9]+]]:_(s1) = G_FCMP floatpred(one), [[UV]](s64), [[INTRINSIC_TRUNC]]
+    ; SI: [[FCMP1:%[0-9]+]]:_(s1) = G_FCMP floatpred(one), [[UV]](s64), [[SELECT1]]
     ; SI: [[AND2:%[0-9]+]]:_(s1) = G_AND [[FCMP]], [[FCMP1]]
-    ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[AND2]](s1), [[C9]], [[C8]]
-    ; SI: [[FADD:%[0-9]+]]:_(s64) = G_FADD [[INTRINSIC_TRUNC]], [[SELECT1]]
+    ; SI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[AND2]](s1), [[C9]], [[C8]]
+    ; SI: [[FADD:%[0-9]+]]:_(s64) = G_FADD [[SELECT1]], [[SELECT2]]
     ; SI: [[FCEIL:%[0-9]+]]:_(s64) = G_FCEIL [[UV]]
     ; SI: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
-    ; SI: [[INT1:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[C]](s32), [[C1]](s32)
+    ; SI: [[INT1:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[UV5]](s32), [[C]](s32), [[C1]](s32)
     ; SI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[INT1]], [[C2]]
     ; SI: [[AND3:%[0-9]+]]:_(s32) = G_AND [[UV5]], [[C3]]
     ; SI: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C5]](s32), [[AND3]](s32)
@@ -233,14 +231,13 @@ body: |
     ; SI: [[AND4:%[0-9]+]]:_(s64) = G_AND [[UV1]], [[XOR1]]
     ; SI: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SUB1]](s32), [[C5]]
     ; SI: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[SUB1]](s32), [[C7]]
-    ; SI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[MV1]], [[AND4]]
-    ; SI: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV1]], [[SELECT2]]
-    ; SI: [[INTRINSIC_TRUNC1:%[0-9]+]]:_(s64) = G_INTRINSIC_TRUNC [[UV1]]
+    ; SI: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[MV1]], [[AND4]]
+    ; SI: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV1]], [[SELECT3]]
     ; SI: [[FCMP2:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[UV1]](s64), [[C8]]
-    ; SI: [[FCMP3:%[0-9]+]]:_(s1) = G_FCMP floatpred(one), [[UV1]](s64), [[INTRINSIC_TRUNC1]]
+    ; SI: [[FCMP3:%[0-9]+]]:_(s1) = G_FCMP floatpred(one), [[UV1]](s64), [[SELECT4]]
     ; SI: [[AND5:%[0-9]+]]:_(s1) = G_AND [[FCMP2]], [[FCMP3]]
-    ; SI: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[AND5]](s1), [[C9]], [[C8]]
-    ; SI: [[FADD:%[0-9]+]]:_(s64) = G_FADD [[INTRINSIC_TRUNC1]], [[SELECT3]]
+    ; SI: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[AND5]](s1), [[C9]], [[C8]]
+    ; SI: [[FADD:%[0-9]+]]:_(s64) = G_FADD [[SELECT4]], [[SELECT5]]
     ; SI: [[FCEIL1:%[0-9]+]]:_(s64) = G_FCEIL [[UV1]]
     ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FCEIL]](s64), [[FCEIL1]](s64)
     ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpext.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpext.mir
index e946212c3f67c..c6ba0b60d99dd 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpext.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpext.mir
@@ -192,3 +192,45 @@ body: |
     %1:_(<4 x s64>) = G_FPEXT %0
     $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY %1
 ...
+
+---
+name: test_fpext_f16_to_f64
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_fpext_f16_to_f64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; CHECK: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
+    ; CHECK: [[FPEXT1:%[0-9]+]]:_(s64) = G_FPEXT [[FPEXT]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[FPEXT1]](s64)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_TRUNC %0
+    %2:_(s64) = G_FPEXT %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_fpext_v2f16_to_v2f64
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_fpext_v2f16_to_v2f64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; CHECK: [[FPEXT:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC]](s16)
+    ; CHECK: [[FPEXT1:%[0-9]+]]:_(s64) = G_FPEXT [[FPEXT]](s32)
+    ; CHECK: [[FPEXT2:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC1]](s16)
+    ; CHECK: [[FPEXT3:%[0-9]+]]:_(s64) = G_FPEXT [[FPEXT2]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FPEXT1]](s64), [[FPEXT3]](s64)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s64>) = nnan G_FPEXT %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpowi.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpowi.mir
new file mode 100644
index 0000000000000..c0f44e83495d1
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpowi.mir
@@ -0,0 +1,70 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer -o - %s | FileCheck -check-prefix=GFX6 %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -run-pass=legalizer -o - %s | FileCheck -check-prefix=GFX9 %s
+
+---
+name: test_fpowi_s16_s32_flags
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; GFX6-LABEL: name: test_fpowi_s16_s32_flags
+    ; GFX6: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX6: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX6: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
+    ; GFX6: [[SITOFP:%[0-9]+]]:_(s32) = G_SITOFP [[COPY1]](s32)
+    ; GFX6: [[FLOG2_:%[0-9]+]]:_(s32) = nnan G_FLOG2 [[FPEXT]]
+    ; GFX6: [[INT:%[0-9]+]]:_(s32) = nnan G_INTRINSIC intrinsic(@llvm.amdgcn.fmul.legacy), [[FLOG2_]](s32), [[SITOFP]](s32)
+    ; GFX6: [[FEXP2_:%[0-9]+]]:_(s32) = nnan G_FEXP2 [[INT]]
+    ; GFX6: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FEXP2_]](s32)
+    ; GFX6: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC]](s16)
+    ; GFX6: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_fpowi_s16_s32_flags
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[SITOFP:%[0-9]+]]:_(s16) = G_SITOFP [[COPY1]](s32)
+    ; GFX9: [[FLOG2_:%[0-9]+]]:_(s16) = nnan G_FLOG2 [[TRUNC]]
+    ; GFX9: [[FPEXT:%[0-9]+]]:_(s32) = nnan G_FPEXT [[FLOG2_]](s16)
+    ; GFX9: [[FPEXT1:%[0-9]+]]:_(s32) = nnan G_FPEXT [[SITOFP]](s16)
+    ; GFX9: [[INT:%[0-9]+]]:_(s32) = nnan G_INTRINSIC intrinsic(@llvm.amdgcn.fmul.legacy), [[FPEXT]](s32), [[FPEXT1]](s32)
+    ; GFX9: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[INT]](s32)
+    ; GFX9: [[FEXP2_:%[0-9]+]]:_(s16) = nnan G_FEXP2 [[FPTRUNC]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FEXP2_]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %0
+    %3:_(s16) = nnan G_FPOWI %2, %1
+    %4:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
+
+---
+name: test_fpowi_s32_s32_flags
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; GFX6-LABEL: name: test_fpowi_s32_s32_flags
+    ; GFX6: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX6: [[SITOFP:%[0-9]+]]:_(s32) = G_SITOFP [[COPY1]](s32)
+    ; GFX6: [[FLOG2_:%[0-9]+]]:_(s32) = nnan G_FLOG2 [[COPY]]
+    ; GFX6: [[INT:%[0-9]+]]:_(s32) = nnan G_INTRINSIC intrinsic(@llvm.amdgcn.fmul.legacy), [[FLOG2_]](s32), [[SITOFP]](s32)
+    ; GFX6: [[FEXP2_:%[0-9]+]]:_(s32) = nnan G_FEXP2 [[INT]]
+    ; GFX6: $vgpr0 = COPY [[FEXP2_]](s32)
+    ; GFX9-LABEL: name: test_fpowi_s32_s32_flags
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[SITOFP:%[0-9]+]]:_(s32) = G_SITOFP [[COPY1]](s32)
+    ; GFX9: [[FLOG2_:%[0-9]+]]:_(s32) = nnan G_FLOG2 [[COPY]]
+    ; GFX9: [[INT:%[0-9]+]]:_(s32) = nnan G_INTRINSIC intrinsic(@llvm.amdgcn.fmul.legacy), [[FLOG2_]](s32), [[SITOFP]](s32)
+    ; GFX9: [[FEXP2_:%[0-9]+]]:_(s32) = nnan G_FEXP2 [[INT]]
+    ; GFX9: $vgpr0 = COPY [[FEXP2_]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = nnan G_FPOWI %0, %1
+    $vgpr0 = COPY %2
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptosi.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptosi.mir
index 81056ce92ea0b..74f6f01a45489 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptosi.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptosi.mir
@@ -171,7 +171,7 @@ body: |
     ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
     ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 11
-    ; SI: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[C]](s32), [[C1]](s32)
+    ; SI: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[UV1]](s32), [[C]](s32), [[C1]](s32)
     ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1023
     ; SI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[INT]], [[C2]]
     ; SI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
@@ -188,18 +188,17 @@ body: |
     ; SI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[SUB]](s32), [[C7]]
     ; SI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[MV]], [[AND1]]
     ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[COPY]], [[SELECT]]
-    ; SI: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s64) = G_INTRINSIC_TRUNC [[COPY]]
     ; SI: [[C8:%[0-9]+]]:_(s64) = G_FCONSTANT double 0x3DF0000000000000
     ; SI: [[C9:%[0-9]+]]:_(s64) = G_FCONSTANT double 0xC1F0000000000000
-    ; SI: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[INTRINSIC_TRUNC]], [[C8]]
+    ; SI: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[SELECT1]], [[C8]]
     ; SI: [[INT1:%[0-9]+]]:_(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.fract), [[FMUL]](s64)
     ; SI: [[C10:%[0-9]+]]:_(s64) = G_FCONSTANT double 0x3FEFFFFFFFFFFFFF
     ; SI: [[FMINNUM_IEEE:%[0-9]+]]:_(s64) = G_FMINNUM_IEEE [[INT1]], [[C10]]
     ; SI: [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ord), [[FMUL]](s64), [[FMUL]]
-    ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[FCMP]](s1), [[FMUL]], [[FMINNUM_IEEE]]
-    ; SI: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[SELECT1]]
+    ; SI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[FCMP]](s1), [[FMUL]], [[FMINNUM_IEEE]]
+    ; SI: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[SELECT2]]
     ; SI: [[FADD:%[0-9]+]]:_(s64) = G_FADD [[FMUL]], [[FNEG]]
-    ; SI: [[FMA:%[0-9]+]]:_(s64) = G_FMA [[FADD]], [[C9]], [[INTRINSIC_TRUNC]]
+    ; SI: [[FMA:%[0-9]+]]:_(s64) = G_FMA [[FADD]], [[C9]], [[SELECT1]]
     ; SI: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[FADD]](s64)
     ; SI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMA]](s64)
     ; SI: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[FPTOUI]](s32), [[FPTOSI]](s32)
@@ -232,7 +231,7 @@ body: |
     ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
     ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 11
-    ; SI: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[C]](s32), [[C1]](s32)
+    ; SI: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[UV1]](s32), [[C]](s32), [[C1]](s32)
     ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1023
     ; SI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[INT]], [[C2]]
     ; SI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
@@ -249,16 +248,15 @@ body: |
     ; SI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[SUB]](s32), [[C7]]
     ; SI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[MV]], [[AND1]]
     ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[COPY]], [[SELECT]]
-    ; SI: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s64) = nnan G_INTRINSIC_TRUNC [[COPY]]
     ; SI: [[C8:%[0-9]+]]:_(s64) = G_FCONSTANT double 0x3DF0000000000000
     ; SI: [[C9:%[0-9]+]]:_(s64) = G_FCONSTANT double 0xC1F0000000000000
-    ; SI: [[FMUL:%[0-9]+]]:_(s64) = nnan G_FMUL [[INTRINSIC_TRUNC]], [[C8]]
+    ; SI: [[FMUL:%[0-9]+]]:_(s64) = nnan G_FMUL [[SELECT1]], [[C8]]
     ; SI: [[INT1:%[0-9]+]]:_(s64) = nnan G_INTRINSIC intrinsic(@llvm.amdgcn.fract), [[FMUL]](s64)
     ; SI: [[C10:%[0-9]+]]:_(s64) = G_FCONSTANT double 0x3FEFFFFFFFFFFFFF
     ; SI: [[FMINNUM_IEEE:%[0-9]+]]:_(s64) = nnan G_FMINNUM_IEEE [[INT1]], [[C10]]
     ; SI: [[FNEG:%[0-9]+]]:_(s64) = nnan G_FNEG [[FMINNUM_IEEE]]
     ; SI: [[FADD:%[0-9]+]]:_(s64) = nnan G_FADD [[FMUL]], [[FNEG]]
-    ; SI: [[FMA:%[0-9]+]]:_(s64) = nnan G_FMA [[FADD]], [[C9]], [[INTRINSIC_TRUNC]]
+    ; SI: [[FMA:%[0-9]+]]:_(s64) = nnan G_FMA [[FADD]], [[C9]], [[SELECT1]]
     ; SI: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[FADD]](s64)
     ; SI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMA]](s64)
     ; SI: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[FPTOUI]](s32), [[FPTOSI]](s32)
@@ -292,7 +290,7 @@ body: |
     ; SI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
     ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 11
-    ; SI: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[C]](s32), [[C1]](s32)
+    ; SI: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[UV3]](s32), [[C]](s32), [[C1]](s32)
     ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1023
     ; SI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[INT]], [[C2]]
     ; SI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
@@ -309,23 +307,22 @@ body: |
     ; SI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[SUB]](s32), [[C7]]
     ; SI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[MV]], [[AND1]]
     ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV]], [[SELECT]]
-    ; SI: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s64) = G_INTRINSIC_TRUNC [[UV]]
     ; SI: [[C8:%[0-9]+]]:_(s64) = G_FCONSTANT double 0x3DF0000000000000
     ; SI: [[C9:%[0-9]+]]:_(s64) = G_FCONSTANT double 0xC1F0000000000000
-    ; SI: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[INTRINSIC_TRUNC]], [[C8]]
+    ; SI: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[SELECT1]], [[C8]]
     ; SI: [[INT1:%[0-9]+]]:_(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.fract), [[FMUL]](s64)
     ; SI: [[C10:%[0-9]+]]:_(s64) = G_FCONSTANT double 0x3FEFFFFFFFFFFFFF
     ; SI: [[FMINNUM_IEEE:%[0-9]+]]:_(s64) = G_FMINNUM_IEEE [[INT1]], [[C10]]
     ; SI: [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ord), [[FMUL]](s64), [[FMUL]]
-    ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[FCMP]](s1), [[FMUL]], [[FMINNUM_IEEE]]
-    ; SI: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[SELECT1]]
+    ; SI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[FCMP]](s1), [[FMUL]], [[FMINNUM_IEEE]]
+    ; SI: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[SELECT2]]
     ; SI: [[FADD:%[0-9]+]]:_(s64) = G_FADD [[FMUL]], [[FNEG]]
-    ; SI: [[FMA:%[0-9]+]]:_(s64) = G_FMA [[FADD]], [[C9]], [[INTRINSIC_TRUNC]]
+    ; SI: [[FMA:%[0-9]+]]:_(s64) = G_FMA [[FADD]], [[C9]], [[SELECT1]]
     ; SI: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[FADD]](s64)
     ; SI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FMA]](s64)
     ; SI: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[FPTOUI]](s32), [[FPTOSI]](s32)
     ; SI: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
-    ; SI: [[INT2:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[C]](s32), [[C1]](s32)
+    ; SI: [[INT2:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[UV5]](s32), [[C]](s32), [[C1]](s32)
     ; SI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[INT2]], [[C2]]
     ; SI: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV5]], [[C3]]
     ; SI: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C5]](s32), [[AND2]](s32)
@@ -334,17 +331,16 @@ body: |
     ; SI: [[AND3:%[0-9]+]]:_(s64) = G_AND [[UV1]], [[XOR1]]
     ; SI: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SUB1]](s32), [[C5]]
     ; SI: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[SUB1]](s32), [[C7]]
-    ; SI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[MV2]], [[AND3]]
-    ; SI: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV1]], [[SELECT2]]
-    ; SI: [[INTRINSIC_TRUNC1:%[0-9]+]]:_(s64) = G_INTRINSIC_TRUNC [[UV1]]
-    ; SI: [[FMUL1:%[0-9]+]]:_(s64) = G_FMUL [[INTRINSIC_TRUNC1]], [[C8]]
+    ; SI: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[MV2]], [[AND3]]
+    ; SI: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV1]], [[SELECT3]]
+    ; SI: [[FMUL1:%[0-9]+]]:_(s64) = G_FMUL [[SELECT4]], [[C8]]
     ; SI: [[INT3:%[0-9]+]]:_(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.fract), [[FMUL1]](s64)
     ; SI: [[FMINNUM_IEEE1:%[0-9]+]]:_(s64) = G_FMINNUM_IEEE [[INT3]], [[C10]]
     ; SI: [[FCMP1:%[0-9]+]]:_(s1) = G_FCMP floatpred(ord), [[FMUL1]](s64), [[FMUL1]]
-    ; SI: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[FCMP1]](s1), [[FMUL1]], [[FMINNUM_IEEE1]]
-    ; SI: [[FNEG1:%[0-9]+]]:_(s64) = G_FNEG [[SELECT3]]
+    ; SI: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[FCMP1]](s1), [[FMUL1]], [[FMINNUM_IEEE1]]
+    ; SI: [[FNEG1:%[0-9]+]]:_(s64) = G_FNEG [[SELECT5]]
     ; SI: [[FADD1:%[0-9]+]]:_(s64) = G_FADD [[FMUL1]], [[FNEG1]]
-    ; SI: [[FMA1:%[0-9]+]]:_(s64) = G_FMA [[FADD1]], [[C9]], [[INTRINSIC_TRUNC1]]
+    ; SI: [[FMA1:%[0-9]+]]:_(s64) = G_FMA [[FADD1]], [[C9]], [[SELECT4]]
     ; SI: [[FPTOSI1:%[0-9]+]]:_(s32) = G_FPTOSI [[FADD1]](s64)
     ; SI: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMA1]](s64)
     ; SI: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[FPTOUI1]](s32), [[FPTOSI1]](s32)
@@ -587,3 +583,64 @@ body: |
     %1:_(<2 x s64>) = G_FPTOSI %0
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
 ...
+
+---
+name: test_fptosi_s16_to_s64
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; SI-LABEL: name: test_fptosi_s16_to_s64
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; SI: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[TRUNC]](s16)
+    ; SI: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[FPTOSI]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[SEXT]](s64)
+    ; VI-LABEL: name: test_fptosi_s16_to_s64
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[TRUNC]](s16)
+    ; VI: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[FPTOSI]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[SEXT]](s64)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_TRUNC %0
+    %2:_(s64) = G_FPTOSI %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_fptosi_v2s16_to_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; SI-LABEL: name: test_fptosi_v2s16_to_v2s64
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; SI: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[TRUNC]](s16)
+    ; SI: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[FPTOSI]](s32)
+    ; SI: [[FPTOSI1:%[0-9]+]]:_(s32) = G_FPTOSI [[TRUNC1]](s16)
+    ; SI: [[SEXT1:%[0-9]+]]:_(s64) = G_SEXT [[FPTOSI1]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SEXT]](s64), [[SEXT1]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; VI-LABEL: name: test_fptosi_v2s16_to_v2s64
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; VI: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[TRUNC]](s16)
+    ; VI: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[FPTOSI]](s32)
+    ; VI: [[FPTOSI1:%[0-9]+]]:_(s32) = G_FPTOSI [[TRUNC1]](s16)
+    ; VI: [[SEXT1:%[0-9]+]]:_(s64) = G_SEXT [[FPTOSI1]](s32)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SEXT]](s64), [[SEXT1]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s64>) = G_FPTOSI %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptoui.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptoui.mir
index b895a1a09662a..87482c0b48d8f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptoui.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptoui.mir
@@ -171,7 +171,7 @@ body: |
     ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
     ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 11
-    ; SI: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[C]](s32), [[C1]](s32)
+    ; SI: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[UV1]](s32), [[C]](s32), [[C1]](s32)
     ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1023
     ; SI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[INT]], [[C2]]
     ; SI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
@@ -188,18 +188,17 @@ body: |
     ; SI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[SUB]](s32), [[C7]]
     ; SI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[MV]], [[AND1]]
     ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[COPY]], [[SELECT]]
-    ; SI: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s64) = G_INTRINSIC_TRUNC [[COPY]]
     ; SI: [[C8:%[0-9]+]]:_(s64) = G_FCONSTANT double 0x3DF0000000000000
     ; SI: [[C9:%[0-9]+]]:_(s64) = G_FCONSTANT double 0xC1F0000000000000
-    ; SI: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[INTRINSIC_TRUNC]], [[C8]]
+    ; SI: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[SELECT1]], [[C8]]
     ; SI: [[INT1:%[0-9]+]]:_(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.fract), [[FMUL]](s64)
     ; SI: [[C10:%[0-9]+]]:_(s64) = G_FCONSTANT double 0x3FEFFFFFFFFFFFFF
     ; SI: [[FMINNUM_IEEE:%[0-9]+]]:_(s64) = G_FMINNUM_IEEE [[INT1]], [[C10]]
     ; SI: [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ord), [[FMUL]](s64), [[FMUL]]
-    ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[FCMP]](s1), [[FMUL]], [[FMINNUM_IEEE]]
-    ; SI: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[SELECT1]]
+    ; SI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[FCMP]](s1), [[FMUL]], [[FMINNUM_IEEE]]
+    ; SI: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[SELECT2]]
     ; SI: [[FADD:%[0-9]+]]:_(s64) = G_FADD [[FMUL]], [[FNEG]]
-    ; SI: [[FMA:%[0-9]+]]:_(s64) = G_FMA [[FADD]], [[C9]], [[INTRINSIC_TRUNC]]
+    ; SI: [[FMA:%[0-9]+]]:_(s64) = G_FMA [[FADD]], [[C9]], [[SELECT1]]
     ; SI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD]](s64)
     ; SI: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMA]](s64)
     ; SI: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[FPTOUI1]](s32), [[FPTOUI]](s32)
@@ -232,7 +231,7 @@ body: |
     ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
     ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 11
-    ; SI: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[C]](s32), [[C1]](s32)
+    ; SI: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[UV1]](s32), [[C]](s32), [[C1]](s32)
     ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1023
     ; SI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[INT]], [[C2]]
     ; SI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
@@ -249,16 +248,15 @@ body: |
     ; SI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[SUB]](s32), [[C7]]
     ; SI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[MV]], [[AND1]]
     ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[COPY]], [[SELECT]]
-    ; SI: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s64) = nnan G_INTRINSIC_TRUNC [[COPY]]
     ; SI: [[C8:%[0-9]+]]:_(s64) = G_FCONSTANT double 0x3DF0000000000000
     ; SI: [[C9:%[0-9]+]]:_(s64) = G_FCONSTANT double 0xC1F0000000000000
-    ; SI: [[FMUL:%[0-9]+]]:_(s64) = nnan G_FMUL [[INTRINSIC_TRUNC]], [[C8]]
+    ; SI: [[FMUL:%[0-9]+]]:_(s64) = nnan G_FMUL [[SELECT1]], [[C8]]
     ; SI: [[INT1:%[0-9]+]]:_(s64) = nnan G_INTRINSIC intrinsic(@llvm.amdgcn.fract), [[FMUL]](s64)
     ; SI: [[C10:%[0-9]+]]:_(s64) = G_FCONSTANT double 0x3FEFFFFFFFFFFFFF
     ; SI: [[FMINNUM_IEEE:%[0-9]+]]:_(s64) = nnan G_FMINNUM_IEEE [[INT1]], [[C10]]
     ; SI: [[FNEG:%[0-9]+]]:_(s64) = nnan G_FNEG [[FMINNUM_IEEE]]
     ; SI: [[FADD:%[0-9]+]]:_(s64) = nnan G_FADD [[FMUL]], [[FNEG]]
-    ; SI: [[FMA:%[0-9]+]]:_(s64) = nnan G_FMA [[FADD]], [[C9]], [[INTRINSIC_TRUNC]]
+    ; SI: [[FMA:%[0-9]+]]:_(s64) = nnan G_FMA [[FADD]], [[C9]], [[SELECT1]]
     ; SI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD]](s64)
     ; SI: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMA]](s64)
     ; SI: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[FPTOUI1]](s32), [[FPTOUI]](s32)
@@ -292,7 +290,7 @@ body: |
     ; SI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
     ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 11
-    ; SI: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[C]](s32), [[C1]](s32)
+    ; SI: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[UV3]](s32), [[C]](s32), [[C1]](s32)
     ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1023
     ; SI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[INT]], [[C2]]
     ; SI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
@@ -309,23 +307,22 @@ body: |
     ; SI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[SUB]](s32), [[C7]]
     ; SI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[MV]], [[AND1]]
     ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV]], [[SELECT]]
-    ; SI: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s64) = G_INTRINSIC_TRUNC [[UV]]
     ; SI: [[C8:%[0-9]+]]:_(s64) = G_FCONSTANT double 0x3DF0000000000000
     ; SI: [[C9:%[0-9]+]]:_(s64) = G_FCONSTANT double 0xC1F0000000000000
-    ; SI: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[INTRINSIC_TRUNC]], [[C8]]
+    ; SI: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[SELECT1]], [[C8]]
     ; SI: [[INT1:%[0-9]+]]:_(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.fract), [[FMUL]](s64)
     ; SI: [[C10:%[0-9]+]]:_(s64) = G_FCONSTANT double 0x3FEFFFFFFFFFFFFF
     ; SI: [[FMINNUM_IEEE:%[0-9]+]]:_(s64) = G_FMINNUM_IEEE [[INT1]], [[C10]]
     ; SI: [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ord), [[FMUL]](s64), [[FMUL]]
-    ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[FCMP]](s1), [[FMUL]], [[FMINNUM_IEEE]]
-    ; SI: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[SELECT1]]
+    ; SI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[FCMP]](s1), [[FMUL]], [[FMINNUM_IEEE]]
+    ; SI: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[SELECT2]]
     ; SI: [[FADD:%[0-9]+]]:_(s64) = G_FADD [[FMUL]], [[FNEG]]
-    ; SI: [[FMA:%[0-9]+]]:_(s64) = G_FMA [[FADD]], [[C9]], [[INTRINSIC_TRUNC]]
+    ; SI: [[FMA:%[0-9]+]]:_(s64) = G_FMA [[FADD]], [[C9]], [[SELECT1]]
     ; SI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD]](s64)
     ; SI: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[FMA]](s64)
     ; SI: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[FPTOUI1]](s32), [[FPTOUI]](s32)
     ; SI: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
-    ; SI: [[INT2:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[C]](s32), [[C1]](s32)
+    ; SI: [[INT2:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[UV5]](s32), [[C]](s32), [[C1]](s32)
     ; SI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[INT2]], [[C2]]
     ; SI: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV5]], [[C3]]
     ; SI: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C5]](s32), [[AND2]](s32)
@@ -334,17 +331,16 @@ body: |
     ; SI: [[AND3:%[0-9]+]]:_(s64) = G_AND [[UV1]], [[XOR1]]
     ; SI: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SUB1]](s32), [[C5]]
     ; SI: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[SUB1]](s32), [[C7]]
-    ; SI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[MV2]], [[AND3]]
-    ; SI: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV1]], [[SELECT2]]
-    ; SI: [[INTRINSIC_TRUNC1:%[0-9]+]]:_(s64) = G_INTRINSIC_TRUNC [[UV1]]
-    ; SI: [[FMUL1:%[0-9]+]]:_(s64) = G_FMUL [[INTRINSIC_TRUNC1]], [[C8]]
+    ; SI: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[MV2]], [[AND3]]
+    ; SI: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV1]], [[SELECT3]]
+    ; SI: [[FMUL1:%[0-9]+]]:_(s64) = G_FMUL [[SELECT4]], [[C8]]
     ; SI: [[INT3:%[0-9]+]]:_(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.fract), [[FMUL1]](s64)
     ; SI: [[FMINNUM_IEEE1:%[0-9]+]]:_(s64) = G_FMINNUM_IEEE [[INT3]], [[C10]]
     ; SI: [[FCMP1:%[0-9]+]]:_(s1) = G_FCMP floatpred(ord), [[FMUL1]](s64), [[FMUL1]]
-    ; SI: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[FCMP1]](s1), [[FMUL1]], [[FMINNUM_IEEE1]]
-    ; SI: [[FNEG1:%[0-9]+]]:_(s64) = G_FNEG [[SELECT3]]
+    ; SI: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[FCMP1]](s1), [[FMUL1]], [[FMINNUM_IEEE1]]
+    ; SI: [[FNEG1:%[0-9]+]]:_(s64) = G_FNEG [[SELECT5]]
     ; SI: [[FADD1:%[0-9]+]]:_(s64) = G_FADD [[FMUL1]], [[FNEG1]]
-    ; SI: [[FMA1:%[0-9]+]]:_(s64) = G_FMA [[FADD1]], [[C9]], [[INTRINSIC_TRUNC1]]
+    ; SI: [[FMA1:%[0-9]+]]:_(s64) = G_FMA [[FADD1]], [[C9]], [[SELECT4]]
     ; SI: [[FPTOUI2:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s64)
     ; SI: [[FPTOUI3:%[0-9]+]]:_(s32) = G_FPTOUI [[FMA1]](s64)
     ; SI: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[FPTOUI3]](s32), [[FPTOUI2]](s32)
@@ -757,3 +753,64 @@ body: |
     %1:_(<2 x s64>) = G_FPTOUI %0
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
 ...
+
+---
+name: test_fptoui_s16_to_s64
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; SI-LABEL: name: test_fptoui_s16_to_s64
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; SI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[TRUNC]](s16)
+    ; SI: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[FPTOUI]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[ZEXT]](s64)
+    ; VI-LABEL: name: test_fptoui_s16_to_s64
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[TRUNC]](s16)
+    ; VI: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[FPTOUI]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[ZEXT]](s64)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_TRUNC %0
+    %2:_(s64) = G_FPTOUI %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_fptoui_v2s16_to_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; SI-LABEL: name: test_fptoui_v2s16_to_v2s64
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; SI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; SI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[TRUNC]](s16)
+    ; SI: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[FPTOUI]](s32)
+    ; SI: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[TRUNC1]](s16)
+    ; SI: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[FPTOUI1]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[ZEXT]](s64), [[ZEXT1]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; VI-LABEL: name: test_fptoui_v2s16_to_v2s64
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; VI: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; VI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[TRUNC]](s16)
+    ; VI: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[FPTOUI]](s32)
+    ; VI: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[TRUNC1]](s16)
+    ; VI: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[FPTOUI1]](s32)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[ZEXT]](s64), [[ZEXT1]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s64>) = G_FPTOUI %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-freeze.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-freeze.mir
index e1faf04cbca28..460d23b441a0b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-freeze.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-freeze.mir
@@ -486,15 +486,13 @@ body: |
     ; CHECK-LABEL: name: test_freeze_v3s1
     ; CHECK: [[DEF:%[0-9]+]]:_(<3 x s1>) = G_IMPLICIT_DEF
     ; CHECK: [[DEF1:%[0-9]+]]:_(<4 x s1>) = G_IMPLICIT_DEF
-    ; CHECK: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF1]](<4 x s1>)
-    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[DEF]](<3 x s1>), 0
-    ; CHECK: [[TRUNC:%[0-9]+]]:_(<4 x s1>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(<4 x s32>) = G_ANYEXT [[TRUNC]](<4 x s1>)
-    ; CHECK: [[FREEZE:%[0-9]+]]:_(<4 x s32>) = G_FREEZE [[ANYEXT1]]
-    ; CHECK: [[TRUNC1:%[0-9]+]]:_(<4 x s1>) = G_TRUNC [[FREEZE]](<4 x s32>)
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s1>) = G_EXTRACT [[TRUNC1]](<4 x s1>), 0
-    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(<3 x s32>) = G_ANYEXT [[EXTRACT]](<3 x s1>)
-    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[ANYEXT2]](<3 x s32>)
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s1>) = G_INSERT [[DEF1]], [[DEF]](<3 x s1>), 0
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(<4 x s32>) = G_ANYEXT [[INSERT]](<4 x s1>)
+    ; CHECK: [[FREEZE:%[0-9]+]]:_(<4 x s32>) = G_FREEZE [[ANYEXT]]
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(<4 x s1>) = G_TRUNC [[FREEZE]](<4 x s32>)
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s1>) = G_EXTRACT [[TRUNC]](<4 x s1>), 0
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(<3 x s32>) = G_ANYEXT [[EXTRACT]](<3 x s1>)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[ANYEXT1]](<3 x s32>)
     %0:_(<3 x s1>) = G_IMPLICIT_DEF
     %1:_(<3 x s1>) = G_FREEZE %0
     %2:_(<3 x s32>) = G_ANYEXT %1
@@ -528,15 +526,13 @@ body: |
     ; CHECK: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
     ; CHECK: [[TRUNC:%[0-9]+]]:_(<3 x s8>) = G_TRUNC [[COPY]](<3 x s32>)
     ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; CHECK: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[TRUNC]](<3 x s8>), 0
-    ; CHECK: [[TRUNC1:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(<4 x s32>) = G_ANYEXT [[TRUNC1]](<4 x s8>)
-    ; CHECK: [[FREEZE:%[0-9]+]]:_(<4 x s32>) = G_FREEZE [[ANYEXT1]]
-    ; CHECK: [[TRUNC2:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[FREEZE]](<4 x s32>)
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s8>) = G_EXTRACT [[TRUNC2]](<4 x s8>), 0
-    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(<3 x s32>) = G_ANYEXT [[EXTRACT]](<3 x s8>)
-    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[ANYEXT2]](<3 x s32>)
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[TRUNC]](<3 x s8>), 0
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(<4 x s32>) = G_ANYEXT [[INSERT]](<4 x s8>)
+    ; CHECK: [[FREEZE:%[0-9]+]]:_(<4 x s32>) = G_FREEZE [[ANYEXT]]
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[FREEZE]](<4 x s32>)
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s8>) = G_EXTRACT [[TRUNC1]](<4 x s8>), 0
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(<3 x s32>) = G_ANYEXT [[EXTRACT]](<3 x s8>)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[ANYEXT1]](<3 x s32>)
     %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
     %1:_(<3 x s8>) = G_TRUNC %0
     %2:_(<3 x s8>) = G_FREEZE %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-frint.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-frint.mir
index d923411697a21..48f712f7076f9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-frint.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-frint.mir
@@ -67,8 +67,7 @@ body: |
     ; SI: [[FABS:%[0-9]+]]:_(s64) = G_FABS [[COPY]]
     ; SI: [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[FABS]](s64), [[C2]]
     ; SI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[FCMP]](s1), [[COPY]], [[FADD1]]
-    ; SI: [[FRINT:%[0-9]+]]:_(s64) = G_FRINT [[COPY]]
-    ; SI: $vgpr0_vgpr1 = COPY [[FRINT]](s64)
+    ; SI: $vgpr0_vgpr1 = COPY [[SELECT]](s64)
     ; CI-LABEL: name: test_frint_s64
     ; CI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; CI: [[FRINT:%[0-9]+]]:_(s64) = G_FRINT [[COPY]]
@@ -140,7 +139,6 @@ body: |
     ; SI: [[FABS:%[0-9]+]]:_(s64) = G_FABS [[UV]]
     ; SI: [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[FABS]](s64), [[C2]]
     ; SI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[FCMP]](s1), [[UV]], [[FADD1]]
-    ; SI: [[FRINT:%[0-9]+]]:_(s64) = G_FRINT [[UV]]
     ; SI: [[AND1:%[0-9]+]]:_(s64) = G_AND [[UV1]], [[C]]
     ; SI: [[OR1:%[0-9]+]]:_(s64) = G_OR [[C1]], [[AND1]]
     ; SI: [[FADD2:%[0-9]+]]:_(s64) = G_FADD [[UV1]], [[OR1]]
@@ -148,9 +146,8 @@ body: |
     ; SI: [[FADD3:%[0-9]+]]:_(s64) = G_FADD [[FADD2]], [[FNEG1]]
     ; SI: [[FABS1:%[0-9]+]]:_(s64) = G_FABS [[UV1]]
     ; SI: [[FCMP1:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[FABS1]](s64), [[C2]]
-    ; SI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[FCMP1]](s1), [[UV1]], [[FADD3]]
-    ; SI: [[FRINT1:%[0-9]+]]:_(s64) = G_FRINT [[UV1]]
-    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FRINT]](s64), [[FRINT1]](s64)
+    ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[FCMP1]](s1), [[UV1]], [[FADD3]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT]](s64), [[SELECT1]](s64)
     ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     ; CI-LABEL: name: test_frint_v2s64
     ; CI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-round.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-round.mir
index 005025bca5032..463deb7a565dd 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-round.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-round.mir
@@ -126,7 +126,7 @@ body: |
     ; GFX6: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
     ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
     ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 11
-    ; GFX6: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[C]](s32), [[C1]](s32)
+    ; GFX6: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[UV1]](s32), [[C]](s32), [[C1]](s32)
     ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1023
     ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[INT]], [[C2]]
     ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
@@ -143,8 +143,7 @@ body: |
     ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[SUB]](s32), [[C7]]
     ; GFX6: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[MV]], [[AND1]]
     ; GFX6: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[COPY]], [[SELECT]]
-    ; GFX6: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s64) = G_INTRINSIC_TRUNC [[COPY]]
-    ; GFX6: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[INTRINSIC_TRUNC]]
+    ; GFX6: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[SELECT1]]
     ; GFX6: [[FADD:%[0-9]+]]:_(s64) = G_FADD [[COPY]], [[FNEG]]
     ; GFX6: [[FABS:%[0-9]+]]:_(s64) = G_FABS [[FADD]]
     ; GFX6: [[C8:%[0-9]+]]:_(s64) = G_FCONSTANT double 0.000000e+00
@@ -154,8 +153,8 @@ body: |
     ; GFX6: [[AND2:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C10]]
     ; GFX6: [[OR:%[0-9]+]]:_(s64) = G_OR [[C11]], [[AND2]]
     ; GFX6: [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(oge), [[FABS]](s64), [[C9]]
-    ; GFX6: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[FCMP]](s1), [[OR]], [[C8]]
-    ; GFX6: [[FADD1:%[0-9]+]]:_(s64) = G_FADD [[INTRINSIC_TRUNC]], [[SELECT1]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[FCMP]](s1), [[OR]], [[C8]]
+    ; GFX6: [[FADD1:%[0-9]+]]:_(s64) = G_FADD [[SELECT1]], [[SELECT2]]
     ; GFX6: $vgpr0_vgpr1 = COPY [[FADD1]](s64)
     ; GFX8-LABEL: name: test_intrinsic_round_s64
     ; GFX8: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
@@ -292,7 +291,7 @@ body: |
     ; GFX6: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
     ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
     ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 11
-    ; GFX6: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[C]](s32), [[C1]](s32)
+    ; GFX6: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[UV3]](s32), [[C]](s32), [[C1]](s32)
     ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1023
     ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[INT]], [[C2]]
     ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
@@ -309,8 +308,7 @@ body: |
     ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[SUB]](s32), [[C7]]
     ; GFX6: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[MV]], [[AND1]]
     ; GFX6: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV]], [[SELECT]]
-    ; GFX6: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s64) = G_INTRINSIC_TRUNC [[UV]]
-    ; GFX6: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[INTRINSIC_TRUNC]]
+    ; GFX6: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[SELECT1]]
     ; GFX6: [[FADD:%[0-9]+]]:_(s64) = G_FADD [[UV]], [[FNEG]]
     ; GFX6: [[FABS:%[0-9]+]]:_(s64) = G_FABS [[FADD]]
     ; GFX6: [[C8:%[0-9]+]]:_(s64) = G_FCONSTANT double 0.000000e+00
@@ -320,10 +318,10 @@ body: |
     ; GFX6: [[AND2:%[0-9]+]]:_(s64) = G_AND [[UV]], [[C10]]
     ; GFX6: [[OR:%[0-9]+]]:_(s64) = G_OR [[C11]], [[AND2]]
     ; GFX6: [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(oge), [[FABS]](s64), [[C9]]
-    ; GFX6: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[FCMP]](s1), [[OR]], [[C8]]
-    ; GFX6: [[FADD1:%[0-9]+]]:_(s64) = G_FADD [[INTRINSIC_TRUNC]], [[SELECT1]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[FCMP]](s1), [[OR]], [[C8]]
+    ; GFX6: [[FADD1:%[0-9]+]]:_(s64) = G_FADD [[SELECT1]], [[SELECT2]]
     ; GFX6: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
-    ; GFX6: [[INT1:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[C]](s32), [[C1]](s32)
+    ; GFX6: [[INT1:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[UV5]](s32), [[C]](s32), [[C1]](s32)
     ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[INT1]], [[C2]]
     ; GFX6: [[AND3:%[0-9]+]]:_(s32) = G_AND [[UV5]], [[C3]]
     ; GFX6: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C5]](s32), [[AND3]](s32)
@@ -332,17 +330,16 @@ body: |
     ; GFX6: [[AND4:%[0-9]+]]:_(s64) = G_AND [[UV1]], [[XOR1]]
     ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SUB1]](s32), [[C5]]
     ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[SUB1]](s32), [[C7]]
-    ; GFX6: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[MV1]], [[AND4]]
-    ; GFX6: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV1]], [[SELECT2]]
-    ; GFX6: [[INTRINSIC_TRUNC1:%[0-9]+]]:_(s64) = G_INTRINSIC_TRUNC [[UV1]]
-    ; GFX6: [[FNEG1:%[0-9]+]]:_(s64) = G_FNEG [[INTRINSIC_TRUNC1]]
+    ; GFX6: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[MV1]], [[AND4]]
+    ; GFX6: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV1]], [[SELECT3]]
+    ; GFX6: [[FNEG1:%[0-9]+]]:_(s64) = G_FNEG [[SELECT4]]
     ; GFX6: [[FADD2:%[0-9]+]]:_(s64) = G_FADD [[UV1]], [[FNEG1]]
     ; GFX6: [[FABS1:%[0-9]+]]:_(s64) = G_FABS [[FADD2]]
     ; GFX6: [[AND5:%[0-9]+]]:_(s64) = G_AND [[UV1]], [[C10]]
     ; GFX6: [[OR1:%[0-9]+]]:_(s64) = G_OR [[C11]], [[AND5]]
     ; GFX6: [[FCMP1:%[0-9]+]]:_(s1) = G_FCMP floatpred(oge), [[FABS1]](s64), [[C9]]
-    ; GFX6: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[FCMP1]](s1), [[OR1]], [[C8]]
-    ; GFX6: [[FADD3:%[0-9]+]]:_(s64) = G_FADD [[INTRINSIC_TRUNC1]], [[SELECT3]]
+    ; GFX6: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[FCMP1]](s1), [[OR1]], [[C8]]
+    ; GFX6: [[FADD3:%[0-9]+]]:_(s64) = G_FADD [[SELECT4]], [[SELECT5]]
     ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FADD1]](s64), [[FADD3]](s64)
     ; GFX6: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     ; GFX8-LABEL: name: test_intrinsic_round_v2s64
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-trunc.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-trunc.mir
index ad3d02eea0ef6..ed38f54ddd626 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-trunc.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-trunc.mir
@@ -79,7 +79,7 @@ body: |
     ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
     ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 11
-    ; SI: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[C]](s32), [[C1]](s32)
+    ; SI: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[UV1]](s32), [[C]](s32), [[C1]](s32)
     ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1023
     ; SI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[INT]], [[C2]]
     ; SI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
@@ -96,8 +96,7 @@ body: |
     ; SI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[SUB]](s32), [[C7]]
     ; SI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[MV]], [[AND1]]
     ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[COPY]], [[SELECT]]
-    ; SI: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s64) = G_INTRINSIC_TRUNC [[COPY]]
-    ; SI: $vgpr0_vgpr1 = COPY [[INTRINSIC_TRUNC]](s64)
+    ; SI: $vgpr0_vgpr1 = COPY [[SELECT1]](s64)
     ; CI-LABEL: name: test_intrinsic_trunc_s64
     ; CI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; CI: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s64) = G_INTRINSIC_TRUNC [[COPY]]
@@ -189,7 +188,7 @@ body: |
     ; SI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
     ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
     ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 11
-    ; SI: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[C]](s32), [[C1]](s32)
+    ; SI: [[INT:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[UV3]](s32), [[C]](s32), [[C1]](s32)
     ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1023
     ; SI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[INT]], [[C2]]
     ; SI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
@@ -206,9 +205,8 @@ body: |
     ; SI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[SUB]](s32), [[C7]]
     ; SI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[MV]], [[AND1]]
     ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV]], [[SELECT]]
-    ; SI: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s64) = G_INTRINSIC_TRUNC [[UV]]
     ; SI: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
-    ; SI: [[INT1:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[C]](s32), [[C1]](s32)
+    ; SI: [[INT1:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ubfe), [[UV5]](s32), [[C]](s32), [[C1]](s32)
     ; SI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[INT1]], [[C2]]
     ; SI: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV5]], [[C3]]
     ; SI: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C5]](s32), [[AND2]](s32)
@@ -217,10 +215,9 @@ body: |
     ; SI: [[AND3:%[0-9]+]]:_(s64) = G_AND [[UV1]], [[XOR1]]
     ; SI: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SUB1]](s32), [[C5]]
     ; SI: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[SUB1]](s32), [[C7]]
-    ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[MV1]], [[AND3]]
-    ; SI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV1]], [[SELECT1]]
-    ; SI: [[INTRINSIC_TRUNC1:%[0-9]+]]:_(s64) = G_INTRINSIC_TRUNC [[UV1]]
-    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[INTRINSIC_TRUNC]](s64), [[INTRINSIC_TRUNC1]](s64)
+    ; SI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[MV1]], [[AND3]]
+    ; SI: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV1]], [[SELECT2]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT1]](s64), [[SELECT3]](s64)
     ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     ; CI-LABEL: name: test_intrinsic_trunc_v2s64
     ; CI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-constant.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-constant.mir
index 3233c2742d884..d91886d1626da 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-constant.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-constant.mir
@@ -4331,42 +4331,32 @@ body: |
     ; CI: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
     ; CI: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p4) :: (load 3, align 4, addrspace 4)
     ; CI: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; CI: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; CI: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; CI: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; CI: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; VI-LABEL: name: test_load_constant_v3s8_align4
     ; VI: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
     ; VI: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p4) :: (load 3, align 4, addrspace 4)
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; VI: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; VI: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; VI: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; VI: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; GFX9-LABEL: name: test_load_constant_v3s8_align4
     ; GFX9: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
     ; GFX9: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p4) :: (load 3, align 4, addrspace 4)
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; GFX9: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; GFX9: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; GFX9: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; CI-MESA-LABEL: name: test_load_constant_v3s8_align4
     ; CI-MESA: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
     ; CI-MESA: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p4) :: (load 3, align 4, addrspace 4)
     ; CI-MESA: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; CI-MESA: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; CI-MESA: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; CI-MESA: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; CI-MESA: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; GFX9-MESA-LABEL: name: test_load_constant_v3s8_align4
     ; GFX9-MESA: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
     ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p4) :: (load 3, align 4, addrspace 4)
     ; GFX9-MESA: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; GFX9-MESA: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; GFX9-MESA: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; GFX9-MESA: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; GFX9-MESA: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     %0:_(p4) = COPY $vgpr0_vgpr1
     %1:_(<3 x s8>) = G_LOAD %0 :: (load 3, align 4, addrspace 4)
     %2:_(<4 x s8>) = G_IMPLICIT_DEF
@@ -4384,42 +4374,32 @@ body: |
     ; CI: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
     ; CI: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p4) :: (load 2, align 1, addrspace 4)
     ; CI: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; CI: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; CI: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; CI: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; CI: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; VI-LABEL: name: test_load_constant_v3s8_align1
     ; VI: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
     ; VI: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p4) :: (load 2, align 1, addrspace 4)
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; VI: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; VI: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; VI: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; VI: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; GFX9-LABEL: name: test_load_constant_v3s8_align1
     ; GFX9: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
     ; GFX9: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p4) :: (load 2, align 1, addrspace 4)
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; GFX9: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; GFX9: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; GFX9: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; CI-MESA-LABEL: name: test_load_constant_v3s8_align1
     ; CI-MESA: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
     ; CI-MESA: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p4) :: (load 2, align 1, addrspace 4)
     ; CI-MESA: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; CI-MESA: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; CI-MESA: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; CI-MESA: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; CI-MESA: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; GFX9-MESA-LABEL: name: test_load_constant_v3s8_align1
     ; GFX9-MESA: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
     ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p4) :: (load 2, align 1, addrspace 4)
     ; GFX9-MESA: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; GFX9-MESA: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; GFX9-MESA: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; GFX9-MESA: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; GFX9-MESA: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     %0:_(p4) = COPY $vgpr0_vgpr1
     %1:_(<3 x s8>) = G_LOAD %0 :: (load 2, align 1, addrspace 4)
     %2:_(<4 x s8>) = G_IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir
index 9dbbede334ac5..a3f43848e8feb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir
@@ -4351,42 +4351,32 @@ body: |
     ; CI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
     ; CI: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p0) :: (load 3, align 4)
     ; CI: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; CI: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; CI: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; CI: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; CI: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; VI-LABEL: name: test_load_flat_v3s8_align4
     ; VI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
     ; VI: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p0) :: (load 3, align 4)
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; VI: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; VI: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; VI: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; VI: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; GFX9-LABEL: name: test_load_flat_v3s8_align4
     ; GFX9: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
     ; GFX9: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p0) :: (load 3, align 4)
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; GFX9: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; GFX9: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; GFX9: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; CI-MESA-LABEL: name: test_load_flat_v3s8_align4
     ; CI-MESA: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
     ; CI-MESA: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p0) :: (load 3, align 4)
     ; CI-MESA: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; CI-MESA: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; CI-MESA: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; CI-MESA: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; CI-MESA: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; GFX9-MESA-LABEL: name: test_load_flat_v3s8_align4
     ; GFX9-MESA: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
     ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p0) :: (load 3, align 4)
     ; GFX9-MESA: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; GFX9-MESA: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; GFX9-MESA: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; GFX9-MESA: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; GFX9-MESA: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     %0:_(p0) = COPY $vgpr0_vgpr1
     %1:_(<3 x s8>) = G_LOAD %0 :: (load 3, align 4, addrspace 0)
     %2:_(<4 x s8>) = G_IMPLICIT_DEF
@@ -4404,42 +4394,32 @@ body: |
     ; CI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
     ; CI: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p0) :: (load 2, align 1)
     ; CI: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; CI: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; CI: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; CI: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; CI: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; VI-LABEL: name: test_load_flat_v3s8_align1
     ; VI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
     ; VI: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p0) :: (load 2, align 1)
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; VI: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; VI: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; VI: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; VI: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; GFX9-LABEL: name: test_load_flat_v3s8_align1
     ; GFX9: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
     ; GFX9: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p0) :: (load 2, align 1)
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; GFX9: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; GFX9: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; GFX9: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; CI-MESA-LABEL: name: test_load_flat_v3s8_align1
     ; CI-MESA: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
     ; CI-MESA: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p0) :: (load 2, align 1)
     ; CI-MESA: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; CI-MESA: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; CI-MESA: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; CI-MESA: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; CI-MESA: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; GFX9-MESA-LABEL: name: test_load_flat_v3s8_align1
     ; GFX9-MESA: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
     ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p0) :: (load 2, align 1)
     ; GFX9-MESA: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; GFX9-MESA: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; GFX9-MESA: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; GFX9-MESA: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; GFX9-MESA: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     %0:_(p0) = COPY $vgpr0_vgpr1
     %1:_(<3 x s8>) = G_LOAD %0 :: (load 2, align 1, addrspace 0)
     %2:_(<4 x s8>) = G_IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-global.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-global.mir
index 275a4ec274c09..a8e3aa5c8c7ce 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-global.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-global.mir
@@ -4134,50 +4134,38 @@ body: |
     ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p1) :: (load 3, align 4, addrspace 1)
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; SI: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; SI: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; SI: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; SI: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; CI-HSA-LABEL: name: test_load_global_v3s8_align4
     ; CI-HSA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; CI-HSA: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p1) :: (load 3, align 4, addrspace 1)
     ; CI-HSA: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; CI-HSA: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; CI-HSA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; CI-HSA: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; CI-HSA: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; CI-HSA: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; CI-HSA: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; CI-MESA-LABEL: name: test_load_global_v3s8_align4
     ; CI-MESA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; CI-MESA: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p1) :: (load 3, align 4, addrspace 1)
     ; CI-MESA: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; CI-MESA: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; CI-MESA: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; CI-MESA: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; CI-MESA: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; VI-LABEL: name: test_load_global_v3s8_align4
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p1) :: (load 3, align 4, addrspace 1)
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; VI: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; VI: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; VI: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; VI: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; GFX9-HSA-LABEL: name: test_load_global_v3s8_align4
     ; GFX9-HSA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; GFX9-HSA: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p1) :: (load 3, align 4, addrspace 1)
     ; GFX9-HSA: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; GFX9-HSA: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; GFX9-HSA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; GFX9-HSA: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; GFX9-HSA: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; GFX9-HSA: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; GFX9-HSA: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; GFX9-MESA-LABEL: name: test_load_global_v3s8_align4
     ; GFX9-MESA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p1) :: (load 3, align 4, addrspace 1)
     ; GFX9-MESA: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; GFX9-MESA: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; GFX9-MESA: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; GFX9-MESA: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; GFX9-MESA: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<3 x s8>) = G_LOAD %0 :: (load 3, align 4, addrspace 1)
     %2:_(<4 x s8>) = G_IMPLICIT_DEF
@@ -4196,50 +4184,38 @@ body: |
     ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p1) :: (load 3, align 1, addrspace 1)
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; SI: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; SI: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; SI: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; SI: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; CI-HSA-LABEL: name: test_load_global_v3s8_align1
     ; CI-HSA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; CI-HSA: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p1) :: (load 3, align 1, addrspace 1)
     ; CI-HSA: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; CI-HSA: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; CI-HSA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; CI-HSA: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; CI-HSA: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; CI-HSA: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; CI-HSA: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; CI-MESA-LABEL: name: test_load_global_v3s8_align1
     ; CI-MESA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; CI-MESA: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p1) :: (load 3, align 1, addrspace 1)
     ; CI-MESA: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; CI-MESA: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; CI-MESA: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; CI-MESA: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; CI-MESA: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; CI-MESA: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; VI-LABEL: name: test_load_global_v3s8_align1
     ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p1) :: (load 3, align 1, addrspace 1)
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; VI: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; VI: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; VI: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; VI: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; GFX9-HSA-LABEL: name: test_load_global_v3s8_align1
     ; GFX9-HSA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; GFX9-HSA: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p1) :: (load 3, align 1, addrspace 1)
     ; GFX9-HSA: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; GFX9-HSA: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; GFX9-HSA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; GFX9-HSA: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; GFX9-HSA: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; GFX9-HSA: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; GFX9-HSA: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; GFX9-MESA-LABEL: name: test_load_global_v3s8_align1
     ; GFX9-MESA: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; GFX9-MESA: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p1) :: (load 3, align 1, addrspace 1)
     ; GFX9-MESA: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; GFX9-MESA: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; GFX9-MESA: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; GFX9-MESA: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; GFX9-MESA: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; GFX9-MESA: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<3 x s8>) = G_LOAD %0 :: (load 3, align 1, addrspace 1)
     %2:_(<4 x s8>) = G_IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir
index a465c05d921aa..f5a852bd98f36 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir
@@ -5095,42 +5095,32 @@ body: |
     ; SI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
     ; SI: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p3) :: (load 3, align 4, addrspace 1)
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; SI: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; SI: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; SI: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; SI: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; CI-LABEL: name: test_load_local_v3s8_align4
     ; CI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
     ; CI: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p3) :: (load 3, align 4, addrspace 1)
     ; CI: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; CI: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; CI: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; CI: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; CI: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; CI-DS128-LABEL: name: test_load_local_v3s8_align4
     ; CI-DS128: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
     ; CI-DS128: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p3) :: (load 3, align 4, addrspace 1)
     ; CI-DS128: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; CI-DS128: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; CI-DS128: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; CI-DS128: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; CI-DS128: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; CI-DS128: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; CI-DS128: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; VI-LABEL: name: test_load_local_v3s8_align4
     ; VI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
     ; VI: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p3) :: (load 3, align 4, addrspace 1)
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; VI: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; VI: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; VI: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; VI: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; GFX9-LABEL: name: test_load_local_v3s8_align4
     ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
     ; GFX9: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p3) :: (load 3, align 4, addrspace 1)
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; GFX9: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; GFX9: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; GFX9: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     %0:_(p3) = COPY $vgpr0
     %1:_(<3 x s8>) = G_LOAD %0 :: (load 3, addrspace 1, align 4)
     %2:_(<4 x s8>) = G_IMPLICIT_DEF
@@ -5148,42 +5138,32 @@ body: |
     ; SI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
     ; SI: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p3) :: (load 3, align 1, addrspace 3)
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; SI: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; SI: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; SI: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; SI: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; CI-LABEL: name: test_load_local_v3s8_align1
     ; CI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
     ; CI: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p3) :: (load 3, align 1, addrspace 3)
     ; CI: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; CI: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; CI: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; CI: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; CI: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; CI-DS128-LABEL: name: test_load_local_v3s8_align1
     ; CI-DS128: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
     ; CI-DS128: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p3) :: (load 3, align 1, addrspace 3)
     ; CI-DS128: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; CI-DS128: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; CI-DS128: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; CI-DS128: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; CI-DS128: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; CI-DS128: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; CI-DS128: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; VI-LABEL: name: test_load_local_v3s8_align1
     ; VI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
     ; VI: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p3) :: (load 3, align 1, addrspace 3)
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; VI: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; VI: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; VI: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; VI: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; GFX9-LABEL: name: test_load_local_v3s8_align1
     ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
     ; GFX9: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p3) :: (load 3, align 1, addrspace 3)
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; GFX9: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; GFX9: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; GFX9: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     %0:_(p3) = COPY $vgpr0
     %1:_(<3 x s8>) = G_LOAD %0 :: (load 3, align 1, addrspace 3)
     %2:_(<4 x s8>) = G_IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir
index 9c43f7ae5453e..58f13f172d9ab 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir
@@ -4222,34 +4222,26 @@ body: |
     ; SI: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
     ; SI: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p5) :: (load 3, align 4, addrspace 1)
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; SI: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; SI: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; SI: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; SI: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; CI-LABEL: name: test_load_private_v3s8_align4
     ; CI: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
     ; CI: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p5) :: (load 3, align 4, addrspace 1)
     ; CI: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; CI: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; CI: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; CI: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; CI: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; VI-LABEL: name: test_load_private_v3s8_align4
     ; VI: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
     ; VI: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p5) :: (load 3, align 4, addrspace 1)
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; VI: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; VI: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; VI: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; VI: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; GFX9-LABEL: name: test_load_private_v3s8_align4
     ; GFX9: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
     ; GFX9: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p5) :: (load 3, align 4, addrspace 1)
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; GFX9: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; GFX9: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; GFX9: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     %0:_(p5) = COPY $vgpr0
     %1:_(<3 x s8>) = G_LOAD %0 :: (load 3, addrspace 1, align 4)
     %2:_(<4 x s8>) = G_IMPLICIT_DEF
@@ -4267,34 +4259,26 @@ body: |
     ; SI: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
     ; SI: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p5) :: (load 3, align 1, addrspace 5)
     ; SI: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; SI: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; SI: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; SI: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; SI: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; CI-LABEL: name: test_load_private_v3s8_align1
     ; CI: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
     ; CI: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p5) :: (load 3, align 1, addrspace 5)
     ; CI: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; CI: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; CI: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; CI: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; CI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; CI: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; VI-LABEL: name: test_load_private_v3s8_align1
     ; VI: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
     ; VI: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p5) :: (load 3, align 1, addrspace 5)
     ; VI: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; VI: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; VI: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; VI: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; VI: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     ; GFX9-LABEL: name: test_load_private_v3s8_align1
     ; GFX9: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
     ; GFX9: [[LOAD:%[0-9]+]]:_(<3 x s8>) = G_LOAD [[COPY]](p5) :: (load 3, align 1, addrspace 5)
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[LOAD]](<3 x s8>), 0
-    ; GFX9: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT]](<4 x s16>)
-    ; GFX9: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[LOAD]](<3 x s8>), 0
+    ; GFX9: $vgpr0 = COPY [[INSERT]](<4 x s8>)
     %0:_(p5) = COPY $vgpr0
     %1:_(<3 x s8>) = G_LOAD %0 :: (load 3, align 1, addrspace 5)
     %2:_(<4 x s8>) = G_IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-or.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-or.mir
index 3fb4e15d434d6..b4a5c48eb2e68 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-or.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-or.mir
@@ -443,31 +443,29 @@ body: |
     ; CHECK: [[DEF1:%[0-9]+]]:_(<3 x s8>) = G_IMPLICIT_DEF
     ; CHECK: [[DEF2:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
     ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF2]], [[DEF]](<3 x s8>), 0
-    ; CHECK: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF2]](<4 x s8>)
-    ; CHECK: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[DEF1]](<3 x s8>), 0
-    ; CHECK: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT1]](<4 x s16>)
+    ; CHECK: [[INSERT1:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF2]], [[DEF1]](<3 x s8>), 0
     ; CHECK: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8), [[UV2:%[0-9]+]]:_(s8), [[UV3:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[INSERT]](<4 x s8>)
-    ; CHECK: [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8), [[UV6:%[0-9]+]]:_(s8), [[UV7:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[TRUNC]](<4 x s8>)
-    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s8)
-    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV4]](s8)
-    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[ANYEXT1]], [[ANYEXT2]]
-    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[OR]](s32)
-    ; CHECK: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s8)
-    ; CHECK: [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[UV5]](s8)
-    ; CHECK: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ANYEXT3]], [[ANYEXT4]]
-    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[OR1]](s32)
-    ; CHECK: [[ANYEXT5:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s8)
-    ; CHECK: [[ANYEXT6:%[0-9]+]]:_(s32) = G_ANYEXT [[UV6]](s8)
-    ; CHECK: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ANYEXT5]], [[ANYEXT6]]
-    ; CHECK: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[OR2]](s32)
-    ; CHECK: [[ANYEXT7:%[0-9]+]]:_(s32) = G_ANYEXT [[UV3]](s8)
-    ; CHECK: [[ANYEXT8:%[0-9]+]]:_(s32) = G_ANYEXT [[UV7]](s8)
-    ; CHECK: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ANYEXT7]], [[ANYEXT8]]
-    ; CHECK: [[TRUNC4:%[0-9]+]]:_(s8) = G_TRUNC [[OR3]](s32)
-    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s8>) = G_BUILD_VECTOR [[TRUNC1]](s8), [[TRUNC2]](s8), [[TRUNC3]](s8), [[TRUNC4]](s8)
+    ; CHECK: [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8), [[UV6:%[0-9]+]]:_(s8), [[UV7:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[INSERT1]](<4 x s8>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s8)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV4]](s8)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[ANYEXT]], [[ANYEXT1]]
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[OR]](s32)
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s8)
+    ; CHECK: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[UV5]](s8)
+    ; CHECK: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ANYEXT2]], [[ANYEXT3]]
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[OR1]](s32)
+    ; CHECK: [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s8)
+    ; CHECK: [[ANYEXT5:%[0-9]+]]:_(s32) = G_ANYEXT [[UV6]](s8)
+    ; CHECK: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ANYEXT4]], [[ANYEXT5]]
+    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[OR2]](s32)
+    ; CHECK: [[ANYEXT6:%[0-9]+]]:_(s32) = G_ANYEXT [[UV3]](s8)
+    ; CHECK: [[ANYEXT7:%[0-9]+]]:_(s32) = G_ANYEXT [[UV7]](s8)
+    ; CHECK: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ANYEXT6]], [[ANYEXT7]]
+    ; CHECK: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[OR3]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s8>) = G_BUILD_VECTOR [[TRUNC]](s8), [[TRUNC1]](s8), [[TRUNC2]](s8), [[TRUNC3]](s8)
     ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s8>) = G_EXTRACT [[BUILD_VECTOR]](<4 x s8>), 0
-    ; CHECK: [[ANYEXT9:%[0-9]+]]:_(<3 x s32>) = G_ANYEXT [[EXTRACT]](<3 x s8>)
-    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[ANYEXT9]](<3 x s32>)
+    ; CHECK: [[ANYEXT8:%[0-9]+]]:_(<3 x s32>) = G_ANYEXT [[EXTRACT]](<3 x s8>)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[ANYEXT8]](<3 x s32>)
     %0:_(<3 x s8>) = G_IMPLICIT_DEF
     %1:_(<3 x s8>) = G_IMPLICIT_DEF
     %2:_(<3 x s8>) = G_OR %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ptrmask.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ptrmask.mir
index fe819107ce6e6..3cf5aeef84318 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ptrmask.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ptrmask.mir
@@ -10,10 +10,10 @@ body: |
     ; CHECK-LABEL: name: ptrmask_p1_s16
     ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
-    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
-    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
-    ; CHECK: [[PTRMASK:%[0-9]+]]:_(p1) = G_PTRMASK [[COPY]], [[AND]](s32)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[COPY1]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s64) = G_AND [[ANYEXT]], [[C]]
+    ; CHECK: [[PTRMASK:%[0-9]+]]:_(p1) = G_PTRMASK [[COPY]], [[AND]](s64)
     ; CHECK: $vgpr0_vgpr1 = COPY [[PTRMASK]](p1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s32) = COPY $vgpr2
@@ -31,7 +31,8 @@ body: |
     ; CHECK-LABEL: name: ptrmask_p1_s32
     ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
-    ; CHECK: [[PTRMASK:%[0-9]+]]:_(p1) = G_PTRMASK [[COPY]], [[COPY1]](s32)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY1]](s32)
+    ; CHECK: [[PTRMASK:%[0-9]+]]:_(p1) = G_PTRMASK [[COPY]], [[ZEXT]](s64)
     ; CHECK: $vgpr0_vgpr1 = COPY [[PTRMASK]](p1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s32) = COPY $vgpr2
@@ -83,10 +84,10 @@ body: |
     ; CHECK-LABEL: name: ptrmask_p0_s16
     ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
-    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
-    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
-    ; CHECK: [[PTRMASK:%[0-9]+]]:_(p0) = G_PTRMASK [[COPY]], [[AND]](s32)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[COPY1]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s64) = G_AND [[ANYEXT]], [[C]]
+    ; CHECK: [[PTRMASK:%[0-9]+]]:_(p0) = G_PTRMASK [[COPY]], [[AND]](s64)
     ; CHECK: $vgpr0_vgpr1 = COPY [[PTRMASK]](p0)
     %0:_(p0) = COPY $vgpr0_vgpr1
     %1:_(s32) = COPY $vgpr2
@@ -104,7 +105,8 @@ body: |
     ; CHECK-LABEL: name: ptrmask_p0_s32
     ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
-    ; CHECK: [[PTRMASK:%[0-9]+]]:_(p0) = G_PTRMASK [[COPY]], [[COPY1]](s32)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY1]](s32)
+    ; CHECK: [[PTRMASK:%[0-9]+]]:_(p0) = G_PTRMASK [[COPY]], [[ZEXT]](s64)
     ; CHECK: $vgpr0_vgpr1 = COPY [[PTRMASK]](p0)
     %0:_(p0) = COPY $vgpr0_vgpr1
     %1:_(s32) = COPY $vgpr2
@@ -194,7 +196,8 @@ body: |
     ; CHECK-LABEL: name: ptrmask_p3_s64
     ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr1_vgpr2
-    ; CHECK: [[PTRMASK:%[0-9]+]]:_(p3) = G_PTRMASK [[COPY]], [[COPY1]](s64)
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK: [[PTRMASK:%[0-9]+]]:_(p3) = G_PTRMASK [[COPY]], [[TRUNC]](s32)
     ; CHECK: $vgpr0 = COPY [[PTRMASK]](p3)
     %0:_(p3) = COPY $vgpr0
     %1:_(s64) = COPY $vgpr1_vgpr2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddsat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddsat.mir
index 8b3fbdaa73eba..28a8efad1d102 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddsat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddsat.mir
@@ -12,51 +12,66 @@ body: |
     ; GFX6-LABEL: name: saddsat_s7
     ; GFX6: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX6: [[TRUNC:%[0-9]+]]:_(s7) = G_TRUNC [[COPY]](s32)
-    ; GFX6: [[TRUNC1:%[0-9]+]]:_(s7) = G_TRUNC [[COPY1]](s32)
-    ; GFX6: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC]](s7)
-    ; GFX6: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC1]](s7)
-    ; GFX6: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 9
-    ; GFX6: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX6: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX6: [[SADDSAT:%[0-9]+]]:_(s16) = G_SADDSAT [[SHL]], [[SHL1]]
-    ; GFX6: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[C]](s16)
-    ; GFX6: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[SADDSAT]](s16)
-    ; GFX6: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SEXT]], [[ZEXT]](s32)
-    ; GFX6: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[ASHR]](s32)
-    ; GFX6: [[TRUNC3:%[0-9]+]]:_(s7) = G_TRUNC [[TRUNC2]](s16)
-    ; GFX6: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC3]](s7)
-    ; GFX6: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 25
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX6: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SHL]], [[C3]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMAX]]
+    ; GFX6: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SHL]], [[C3]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMIN]]
+    ; GFX6: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB1]], [[SHL1]]
+    ; GFX6: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SHL]], [[SMIN1]]
+    ; GFX6: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[ADD]], [[C]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ASHR]](s32)
+    ; GFX6: $vgpr0 = COPY [[COPY4]](s32)
     ; GFX8-LABEL: name: saddsat_s7
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX8: [[TRUNC:%[0-9]+]]:_(s7) = G_TRUNC [[COPY]](s32)
-    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s7) = G_TRUNC [[COPY1]](s32)
-    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC]](s7)
-    ; GFX8: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC1]](s7)
+    ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
     ; GFX8: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 9
-    ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX8: [[SADDSAT:%[0-9]+]]:_(s16) = G_SADDSAT [[SHL]], [[SHL1]]
-    ; GFX8: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SADDSAT]], [[C]](s16)
-    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s7) = G_TRUNC [[ASHR]](s16)
-    ; GFX8: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC2]](s7)
-    ; GFX8: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
+    ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
+    ; GFX8: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
+    ; GFX8: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
+    ; GFX8: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 0
+    ; GFX8: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[SHL]], [[C3]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[C1]], [[SMAX]]
+    ; GFX8: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[SHL]], [[C3]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[C2]], [[SMIN]]
+    ; GFX8: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB1]], [[SHL1]]
+    ; GFX8: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[SHL]], [[SMIN1]]
+    ; GFX8: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[ADD]], [[C]](s16)
+    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ASHR]](s16)
+    ; GFX8: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: saddsat_s7
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX9: [[TRUNC:%[0-9]+]]:_(s7) = G_TRUNC [[COPY]](s32)
-    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s7) = G_TRUNC [[COPY1]](s32)
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC]](s7)
-    ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC1]](s7)
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
     ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 9
-    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX9: [[SADDSAT:%[0-9]+]]:_(s16) = G_SADDSAT [[SHL]], [[SHL1]]
-    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SADDSAT]], [[C]](s16)
-    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s7) = G_TRUNC [[ASHR]](s16)
-    ; GFX9: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC2]](s7)
-    ; GFX9: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
+    ; GFX9: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
+    ; GFX9: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
+    ; GFX9: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 0
+    ; GFX9: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[SHL]], [[C3]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[C1]], [[SMAX]]
+    ; GFX9: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[SHL]], [[C3]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[C2]], [[SMIN]]
+    ; GFX9: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB1]], [[SHL1]]
+    ; GFX9: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[SHL]], [[SMIN1]]
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[ADD]], [[C]](s16)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ASHR]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s7) = G_TRUNC %0
@@ -75,51 +90,66 @@ body: |
     ; GFX6-LABEL: name: saddsat_s8
     ; GFX6: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX6: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
-    ; GFX6: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32)
-    ; GFX6: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC]](s8)
-    ; GFX6: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC1]](s8)
-    ; GFX6: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX6: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX6: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX6: [[SADDSAT:%[0-9]+]]:_(s16) = G_SADDSAT [[SHL]], [[SHL1]]
-    ; GFX6: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[C]](s16)
-    ; GFX6: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[SADDSAT]](s16)
-    ; GFX6: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SEXT]], [[ZEXT]](s32)
-    ; GFX6: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[ASHR]](s32)
-    ; GFX6: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC2]](s16)
-    ; GFX6: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC3]](s8)
-    ; GFX6: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX6: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SHL]], [[C3]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMAX]]
+    ; GFX6: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SHL]], [[C3]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMIN]]
+    ; GFX6: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB1]], [[SHL1]]
+    ; GFX6: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SHL]], [[SMIN1]]
+    ; GFX6: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[ADD]], [[C]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ASHR]](s32)
+    ; GFX6: $vgpr0 = COPY [[COPY4]](s32)
     ; GFX8-LABEL: name: saddsat_s8
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX8: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
-    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32)
-    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC]](s8)
-    ; GFX8: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC1]](s8)
+    ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
     ; GFX8: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX8: [[SADDSAT:%[0-9]+]]:_(s16) = G_SADDSAT [[SHL]], [[SHL1]]
-    ; GFX8: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SADDSAT]], [[C]](s16)
-    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[ASHR]](s16)
-    ; GFX8: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC2]](s8)
-    ; GFX8: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
+    ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
+    ; GFX8: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
+    ; GFX8: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
+    ; GFX8: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 0
+    ; GFX8: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[SHL]], [[C3]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[C1]], [[SMAX]]
+    ; GFX8: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[SHL]], [[C3]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[C2]], [[SMIN]]
+    ; GFX8: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB1]], [[SHL1]]
+    ; GFX8: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[SHL]], [[SMIN1]]
+    ; GFX8: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[ADD]], [[C]](s16)
+    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ASHR]](s16)
+    ; GFX8: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: saddsat_s8
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX9: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
-    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32)
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC]](s8)
-    ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC1]](s8)
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
     ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX9: [[SADDSAT:%[0-9]+]]:_(s16) = G_SADDSAT [[SHL]], [[SHL1]]
-    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SADDSAT]], [[C]](s16)
-    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[ASHR]](s16)
-    ; GFX9: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC2]](s8)
-    ; GFX9: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
+    ; GFX9: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
+    ; GFX9: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
+    ; GFX9: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 0
+    ; GFX9: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[SHL]], [[C3]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[C1]], [[SMAX]]
+    ; GFX9: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[SHL]], [[C3]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[C2]], [[SMIN]]
+    ; GFX9: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB1]], [[SHL1]]
+    ; GFX9: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[SHL]], [[SMIN1]]
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[ADD]], [[C]](s16)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ASHR]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s8) = G_TRUNC %0
@@ -138,75 +168,152 @@ body: |
     ; GFX6-LABEL: name: saddsat_v2s8
     ; GFX6: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX6: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
-    ; GFX6: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX6: [[BITCAST:%[0-9]+]]:_(<2 x s8>) = G_BITCAST [[TRUNC]](s16)
-    ; GFX6: [[BITCAST1:%[0-9]+]]:_(<2 x s8>) = G_BITCAST [[TRUNC1]](s16)
-    ; GFX6: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BITCAST]](<2 x s8>)
-    ; GFX6: [[UV2:%[0-9]+]]:_(s8), [[UV3:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BITCAST1]](<2 x s8>)
-    ; GFX6: [[SADDSAT:%[0-9]+]]:_(s8) = G_SADDSAT [[UV]], [[UV2]]
-    ; GFX6: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[UV1]](s8)
-    ; GFX6: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[UV3]](s8)
-    ; GFX6: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX6: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX6: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX6: [[SADDSAT1:%[0-9]+]]:_(s16) = G_SADDSAT [[SHL]], [[SHL1]]
-    ; GFX6: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[C]](s16)
-    ; GFX6: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[SADDSAT1]](s16)
-    ; GFX6: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SEXT]], [[ZEXT]](s32)
-    ; GFX6: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[ASHR]](s32)
-    ; GFX6: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC2]](s16)
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s8>) = G_BUILD_VECTOR [[SADDSAT]](s8), [[TRUNC3]](s8)
-    ; GFX6: [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<2 x s8>)
-    ; GFX6: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV4]](s8), [[UV5]](s8)
-    ; GFX6: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
-    ; GFX6: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX6: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX6: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; GFX6: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
+    ; GFX6: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C1]](s32)
+    ; GFX6: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C2]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C2]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C2]](s32)
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+    ; GFX6: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; GFX6: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX6: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SHL]], [[C5]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[SMAX]]
+    ; GFX6: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SHL]], [[C5]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C4]], [[SMIN]]
+    ; GFX6: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB1]], [[SHL1]]
+    ; GFX6: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SHL]], [[SMIN1]]
+    ; GFX6: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[ADD]], [[C2]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX6: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; GFX6: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY4]], [[C2]](s32)
+    ; GFX6: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[COPY5]], [[C2]](s32)
+    ; GFX6: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[SHL2]], [[C5]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[SMAX2]]
+    ; GFX6: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[SHL2]], [[C5]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C4]], [[SMIN2]]
+    ; GFX6: [[SMAX3:%[0-9]+]]:_(s32) = G_SMAX [[SUB3]], [[SHL3]]
+    ; GFX6: [[SMIN3:%[0-9]+]]:_(s32) = G_SMIN [[SMAX3]], [[SUB2]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SHL2]], [[SMIN3]]
+    ; GFX6: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[ADD1]], [[C2]](s32)
+    ; GFX6: [[C6:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX6: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[ASHR]](s32)
+    ; GFX6: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C6]]
+    ; GFX6: [[COPY6:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; GFX6: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; GFX6: [[COPY7:%[0-9]+]]:_(s32) = COPY [[ASHR1]](s32)
+    ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C7]]
+    ; GFX6: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY6]](s32)
+    ; GFX6: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL4]](s32)
+    ; GFX6: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; GFX6: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; GFX6: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX8-LABEL: name: saddsat_v2s8
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX8: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; GFX8: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
+    ; GFX8: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C1]](s32)
+    ; GFX8: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C2]](s32)
     ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
     ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX8: [[BITCAST:%[0-9]+]]:_(<2 x s8>) = G_BITCAST [[TRUNC]](s16)
-    ; GFX8: [[BITCAST1:%[0-9]+]]:_(<2 x s8>) = G_BITCAST [[TRUNC1]](s16)
-    ; GFX8: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BITCAST]](<2 x s8>)
-    ; GFX8: [[UV2:%[0-9]+]]:_(s8), [[UV3:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BITCAST1]](<2 x s8>)
-    ; GFX8: [[SADDSAT:%[0-9]+]]:_(s8) = G_SADDSAT [[UV]], [[UV2]]
-    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[UV1]](s8)
-    ; GFX8: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[UV3]](s8)
-    ; GFX8: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX8: [[SADDSAT1:%[0-9]+]]:_(s16) = G_SADDSAT [[SHL]], [[SHL1]]
-    ; GFX8: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SADDSAT1]], [[C]](s16)
-    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[ASHR]](s16)
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s8>) = G_BUILD_VECTOR [[SADDSAT]](s8), [[TRUNC2]](s8)
-    ; GFX8: [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<2 x s8>)
-    ; GFX8: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV4]](s8), [[UV5]](s8)
-    ; GFX8: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
-    ; GFX8: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX8: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C3]](s16)
+    ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C3]](s16)
+    ; GFX8: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
+    ; GFX8: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
+    ; GFX8: [[C6:%[0-9]+]]:_(s16) = G_CONSTANT i16 0
+    ; GFX8: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[SHL]], [[C6]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[C4]], [[SMAX]]
+    ; GFX8: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[SHL]], [[C6]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[C5]], [[SMIN]]
+    ; GFX8: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB1]], [[SHL1]]
+    ; GFX8: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[SHL]], [[SMIN1]]
+    ; GFX8: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[ADD]], [[C3]](s16)
+    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX8: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+    ; GFX8: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[C3]](s16)
+    ; GFX8: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[TRUNC3]], [[C3]](s16)
+    ; GFX8: [[SMAX2:%[0-9]+]]:_(s16) = G_SMAX [[SHL2]], [[C6]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[C4]], [[SMAX2]]
+    ; GFX8: [[SMIN2:%[0-9]+]]:_(s16) = G_SMIN [[SHL2]], [[C6]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s16) = G_SUB [[C5]], [[SMIN2]]
+    ; GFX8: [[SMAX3:%[0-9]+]]:_(s16) = G_SMAX [[SUB3]], [[SHL3]]
+    ; GFX8: [[SMIN3:%[0-9]+]]:_(s16) = G_SMIN [[SMAX3]], [[SUB2]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[SHL2]], [[SMIN3]]
+    ; GFX8: [[ASHR1:%[0-9]+]]:_(s16) = G_ASHR [[ADD1]], [[C3]](s16)
+    ; GFX8: [[C7:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s16) = COPY [[ASHR]](s16)
+    ; GFX8: [[AND:%[0-9]+]]:_(s16) = G_AND [[COPY2]], [[C7]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s16) = COPY [[ASHR1]](s16)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s16) = G_AND [[COPY3]], [[C7]]
+    ; GFX8: [[SHL4:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C3]](s16)
+    ; GFX8: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL4]]
+    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; GFX8: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: saddsat_v2s8
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; GFX9: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
+    ; GFX9: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C1]](s32)
+    ; GFX9: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C2]](s32)
     ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
     ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX9: [[BITCAST:%[0-9]+]]:_(<2 x s8>) = G_BITCAST [[TRUNC]](s16)
-    ; GFX9: [[BITCAST1:%[0-9]+]]:_(<2 x s8>) = G_BITCAST [[TRUNC1]](s16)
-    ; GFX9: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BITCAST]](<2 x s8>)
-    ; GFX9: [[UV2:%[0-9]+]]:_(s8), [[UV3:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BITCAST1]](<2 x s8>)
-    ; GFX9: [[SADDSAT:%[0-9]+]]:_(s8) = G_SADDSAT [[UV]], [[UV2]]
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[UV1]](s8)
-    ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[UV3]](s8)
-    ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX9: [[SADDSAT1:%[0-9]+]]:_(s16) = G_SADDSAT [[SHL]], [[SHL1]]
-    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SADDSAT1]], [[C]](s16)
-    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[ASHR]](s16)
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s8>) = G_BUILD_VECTOR [[SADDSAT]](s8), [[TRUNC2]](s8)
-    ; GFX9: [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<2 x s8>)
-    ; GFX9: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV4]](s8), [[UV5]](s8)
-    ; GFX9: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
-    ; GFX9: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX9: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C3]](s16)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C3]](s16)
+    ; GFX9: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
+    ; GFX9: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
+    ; GFX9: [[C6:%[0-9]+]]:_(s16) = G_CONSTANT i16 0
+    ; GFX9: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[SHL]], [[C6]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[C4]], [[SMAX]]
+    ; GFX9: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[SHL]], [[C6]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[C5]], [[SMIN]]
+    ; GFX9: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB1]], [[SHL1]]
+    ; GFX9: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[SHL]], [[SMIN1]]
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[ADD]], [[C3]](s16)
+    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX9: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+    ; GFX9: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[C3]](s16)
+    ; GFX9: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[TRUNC3]], [[C3]](s16)
+    ; GFX9: [[SMAX2:%[0-9]+]]:_(s16) = G_SMAX [[SHL2]], [[C6]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[C4]], [[SMAX2]]
+    ; GFX9: [[SMIN2:%[0-9]+]]:_(s16) = G_SMIN [[SHL2]], [[C6]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s16) = G_SUB [[C5]], [[SMIN2]]
+    ; GFX9: [[SMAX3:%[0-9]+]]:_(s16) = G_SMAX [[SUB3]], [[SHL3]]
+    ; GFX9: [[SMIN3:%[0-9]+]]:_(s16) = G_SMIN [[SMAX3]], [[SUB2]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[SHL2]], [[SMIN3]]
+    ; GFX9: [[ASHR1:%[0-9]+]]:_(s16) = G_ASHR [[ADD1]], [[C3]](s16)
+    ; GFX9: [[C7:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s16) = COPY [[ASHR]](s16)
+    ; GFX9: [[AND:%[0-9]+]]:_(s16) = G_AND [[COPY2]], [[C7]]
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s16) = COPY [[ASHR1]](s16)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s16) = G_AND [[COPY3]], [[C7]]
+    ; GFX9: [[SHL4:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C3]](s16)
+    ; GFX9: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL4]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s16) = G_TRUNC %0
@@ -228,26 +335,57 @@ body: |
     ; GFX6-LABEL: name: saddsat_s16
     ; GFX6: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX6: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
-    ; GFX6: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX6: [[SADDSAT:%[0-9]+]]:_(s16) = G_SADDSAT [[TRUNC]], [[TRUNC1]]
-    ; GFX6: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SADDSAT]](s16)
-    ; GFX6: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX6: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SHL]], [[C3]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMAX]]
+    ; GFX6: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SHL]], [[C3]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMIN]]
+    ; GFX6: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB1]], [[SHL1]]
+    ; GFX6: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SHL]], [[SMIN1]]
+    ; GFX6: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[ADD]], [[C]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ASHR]](s32)
+    ; GFX6: $vgpr0 = COPY [[COPY4]](s32)
     ; GFX8-LABEL: name: saddsat_s16
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
     ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX8: [[SADDSAT:%[0-9]+]]:_(s16) = G_SADDSAT [[TRUNC]], [[TRUNC1]]
-    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SADDSAT]](s16)
+    ; GFX8: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
+    ; GFX8: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
+    ; GFX8: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 0
+    ; GFX8: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC]], [[C2]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[C]], [[SMAX]]
+    ; GFX8: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC]], [[C2]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[C1]], [[SMIN]]
+    ; GFX8: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB1]], [[TRUNC1]]
+    ; GFX8: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[TRUNC]], [[SMIN1]]
+    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ADD]](s16)
     ; GFX8: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: saddsat_s16
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
     ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX9: [[SADDSAT:%[0-9]+]]:_(s16) = G_SADDSAT [[TRUNC]], [[TRUNC1]]
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SADDSAT]](s16)
+    ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
+    ; GFX9: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
+    ; GFX9: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 0
+    ; GFX9: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC]], [[C2]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[C]], [[SMAX]]
+    ; GFX9: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC]], [[C2]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[C1]], [[SMIN]]
+    ; GFX9: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB1]], [[TRUNC1]]
+    ; GFX9: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[TRUNC]], [[SMIN1]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ADD]](s16)
     ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
@@ -267,30 +405,99 @@ body: |
     ; GFX6-LABEL: name: saddsat_v2s16
     ; GFX6: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
-    ; GFX6: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
-    ; GFX6: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
-    ; GFX6: [[SADDSAT:%[0-9]+]]:_(s16) = G_SADDSAT [[UV]], [[UV2]]
-    ; GFX6: [[SADDSAT1:%[0-9]+]]:_(s16) = G_SADDSAT [[UV1]], [[UV3]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[SADDSAT]](s16), [[SADDSAT1]](s16)
-    ; GFX6: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX6: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX6: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; GFX6: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX6: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SHL]], [[C3]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMAX]]
+    ; GFX6: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SHL]], [[C3]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMIN]]
+    ; GFX6: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB1]], [[SHL1]]
+    ; GFX6: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SHL]], [[SMIN1]]
+    ; GFX6: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[ADD]], [[C]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX6: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX6: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY4]], [[C]](s32)
+    ; GFX6: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[COPY5]], [[C]](s32)
+    ; GFX6: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[SHL2]], [[C3]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMAX2]]
+    ; GFX6: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[SHL2]], [[C3]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMIN2]]
+    ; GFX6: [[SMAX3:%[0-9]+]]:_(s32) = G_SMAX [[SUB3]], [[SHL3]]
+    ; GFX6: [[SMIN3:%[0-9]+]]:_(s32) = G_SMIN [[SMAX3]], [[SUB2]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SHL2]], [[SMIN3]]
+    ; GFX6: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[ADD1]], [[C]](s32)
+    ; GFX6: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX6: [[COPY6:%[0-9]+]]:_(s32) = COPY [[ASHR]](s32)
+    ; GFX6: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C4]]
+    ; GFX6: [[COPY7:%[0-9]+]]:_(s32) = COPY [[ASHR1]](s32)
+    ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C4]]
+    ; GFX6: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX6: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL4]]
+    ; GFX6: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX6: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
     ; GFX8-LABEL: name: saddsat_v2s16
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
-    ; GFX8: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
-    ; GFX8: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
-    ; GFX8: [[SADDSAT:%[0-9]+]]:_(s16) = G_SADDSAT [[UV]], [[UV2]]
-    ; GFX8: [[SADDSAT1:%[0-9]+]]:_(s16) = G_SADDSAT [[UV1]], [[UV3]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[SADDSAT]](s16), [[SADDSAT1]](s16)
-    ; GFX8: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX8: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX8: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
+    ; GFX8: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
+    ; GFX8: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 0
+    ; GFX8: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC]], [[C3]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[C1]], [[SMAX]]
+    ; GFX8: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC]], [[C3]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[C2]], [[SMIN]]
+    ; GFX8: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB1]], [[TRUNC2]]
+    ; GFX8: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[TRUNC]], [[SMIN1]]
+    ; GFX8: [[SMAX2:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC1]], [[C3]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[C1]], [[SMAX2]]
+    ; GFX8: [[SMIN2:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC1]], [[C3]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s16) = G_SUB [[C2]], [[SMIN2]]
+    ; GFX8: [[SMAX3:%[0-9]+]]:_(s16) = G_SMAX [[SUB3]], [[TRUNC3]]
+    ; GFX8: [[SMIN3:%[0-9]+]]:_(s16) = G_SMIN [[SMAX3]], [[SUB2]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[TRUNC1]], [[SMIN3]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
+    ; GFX8: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX8: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
     ; GFX9-LABEL: name: saddsat_v2s16
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
-    ; GFX9: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
-    ; GFX9: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
-    ; GFX9: [[SADDSAT:%[0-9]+]]:_(s16) = G_SADDSAT [[UV]], [[UV2]]
-    ; GFX9: [[SADDSAT1:%[0-9]+]]:_(s16) = G_SADDSAT [[UV1]], [[UV3]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[SADDSAT]](s16), [[SADDSAT1]](s16)
-    ; GFX9: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767
+    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C]](s32), [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -32768
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C1]](s32), [[C1]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[BUILD_VECTOR_TRUNC2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C2]](s32), [[C2]](s32)
+    ; GFX9: [[SMAX:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[COPY]], [[BUILD_VECTOR_TRUNC2]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC]], [[SMAX]]
+    ; GFX9: [[SMIN:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[COPY]], [[BUILD_VECTOR_TRUNC2]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC1]], [[SMIN]]
+    ; GFX9: [[SMAX1:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[SUB1]], [[COPY1]]
+    ; GFX9: [[SMIN1:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(<2 x s16>) = G_ADD [[COPY]], [[SMIN1]]
+    ; GFX9: $vgpr0 = COPY [[ADD]](<2 x s16>)
     %0:_(<2 x s16>) = COPY $vgpr0
     %1:_(<2 x s16>) = COPY $vgpr1
     %2:_(<2 x s16>) = G_SADDSAT %0, %1
@@ -306,42 +513,197 @@ body: |
     ; GFX6-LABEL: name: saddsat_v3s16
     ; GFX6: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     ; GFX6: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
-    ; GFX6: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UV]](<3 x s16>)
-    ; GFX6: [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UV1]](<3 x s16>)
-    ; GFX6: [[SADDSAT:%[0-9]+]]:_(s16) = G_SADDSAT [[UV2]], [[UV5]]
-    ; GFX6: [[SADDSAT1:%[0-9]+]]:_(s16) = G_SADDSAT [[UV3]], [[UV6]]
-    ; GFX6: [[SADDSAT2:%[0-9]+]]:_(s16) = G_SADDSAT [[UV4]], [[UV7]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[SADDSAT]](s16), [[SADDSAT1]](s16), [[SADDSAT2]](s16)
     ; GFX6: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX6: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX6: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<3 x s16>), [[EXTRACT]](<3 x s16>)
-    ; GFX6: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
+    ; GFX6: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
+    ; GFX6: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX6: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX6: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX6: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX6: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV1]](<3 x s16>), 0
+    ; GFX6: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; GFX6: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
+    ; GFX6: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; GFX6: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
+    ; GFX6: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX6: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SHL]], [[C3]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMAX]]
+    ; GFX6: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SHL]], [[C3]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMIN]]
+    ; GFX6: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB1]], [[SHL1]]
+    ; GFX6: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SHL]], [[SMIN1]]
+    ; GFX6: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[ADD]], [[C]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; GFX6: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+    ; GFX6: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[COPY4]], [[C]](s32)
+    ; GFX6: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[SHL2]], [[C3]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMAX2]]
+    ; GFX6: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[SHL2]], [[C3]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMIN2]]
+    ; GFX6: [[SMAX3:%[0-9]+]]:_(s32) = G_SMAX [[SUB3]], [[SHL3]]
+    ; GFX6: [[SMIN3:%[0-9]+]]:_(s32) = G_SMIN [[SMAX3]], [[SUB2]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SHL2]], [[SMIN3]]
+    ; GFX6: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[ADD1]], [[C]](s32)
+    ; GFX6: [[COPY5:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; GFX6: [[COPY6:%[0-9]+]]:_(s32) = COPY [[BITCAST3]](s32)
+    ; GFX6: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[COPY5]], [[C]](s32)
+    ; GFX6: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[COPY6]], [[C]](s32)
+    ; GFX6: [[SMAX4:%[0-9]+]]:_(s32) = G_SMAX [[SHL4]], [[C3]]
+    ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMAX4]]
+    ; GFX6: [[SMIN4:%[0-9]+]]:_(s32) = G_SMIN [[SHL4]], [[C3]]
+    ; GFX6: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMIN4]]
+    ; GFX6: [[SMAX5:%[0-9]+]]:_(s32) = G_SMAX [[SUB5]], [[SHL5]]
+    ; GFX6: [[SMIN5:%[0-9]+]]:_(s32) = G_SMIN [[SMAX5]], [[SUB4]]
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SHL4]], [[SMIN5]]
+    ; GFX6: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[ADD2]], [[C]](s32)
+    ; GFX6: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX6: [[COPY7:%[0-9]+]]:_(s32) = COPY [[ASHR]](s32)
+    ; GFX6: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C4]]
+    ; GFX6: [[COPY8:%[0-9]+]]:_(s32) = COPY [[ASHR1]](s32)
+    ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY8]], [[C4]]
+    ; GFX6: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX6: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL6]]
+    ; GFX6: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX6: [[COPY9:%[0-9]+]]:_(s32) = COPY [[ASHR2]](s32)
+    ; GFX6: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C4]]
+    ; GFX6: [[COPY10:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
+    ; GFX6: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[COPY10]], [[C]](s32)
+    ; GFX6: [[OR1:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL7]]
+    ; GFX6: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
+    ; GFX6: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; GFX6: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
+    ; GFX6: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX6: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
+    ; GFX6: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS1]](<6 x s16>)
     ; GFX8-LABEL: name: saddsat_v3s16
     ; GFX8: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     ; GFX8: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
-    ; GFX8: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UV]](<3 x s16>)
-    ; GFX8: [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UV1]](<3 x s16>)
-    ; GFX8: [[SADDSAT:%[0-9]+]]:_(s16) = G_SADDSAT [[UV2]], [[UV5]]
-    ; GFX8: [[SADDSAT1:%[0-9]+]]:_(s16) = G_SADDSAT [[UV3]], [[UV6]]
-    ; GFX8: [[SADDSAT2:%[0-9]+]]:_(s16) = G_SADDSAT [[UV4]], [[UV7]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[SADDSAT]](s16), [[SADDSAT1]](s16), [[SADDSAT2]](s16)
     ; GFX8: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX8: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX8: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<3 x s16>), [[EXTRACT]](<3 x s16>)
-    ; GFX8: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
+    ; GFX8: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
+    ; GFX8: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX8: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX8: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV1]](<3 x s16>), 0
+    ; GFX8: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; GFX8: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
+    ; GFX8: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
+    ; GFX8: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; GFX8: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+    ; GFX8: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
+    ; GFX8: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+    ; GFX8: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
+    ; GFX8: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
+    ; GFX8: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 0
+    ; GFX8: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC]], [[C3]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[C1]], [[SMAX]]
+    ; GFX8: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC]], [[C3]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[C2]], [[SMIN]]
+    ; GFX8: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB1]], [[TRUNC3]]
+    ; GFX8: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[TRUNC]], [[SMIN1]]
+    ; GFX8: [[SMAX2:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC1]], [[C3]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[C1]], [[SMAX2]]
+    ; GFX8: [[SMIN2:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC1]], [[C3]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s16) = G_SUB [[C2]], [[SMIN2]]
+    ; GFX8: [[SMAX3:%[0-9]+]]:_(s16) = G_SMAX [[SUB3]], [[TRUNC4]]
+    ; GFX8: [[SMIN3:%[0-9]+]]:_(s16) = G_SMIN [[SMAX3]], [[SUB2]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[TRUNC1]], [[SMIN3]]
+    ; GFX8: [[SMAX4:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC2]], [[C3]]
+    ; GFX8: [[SUB4:%[0-9]+]]:_(s16) = G_SUB [[C1]], [[SMAX4]]
+    ; GFX8: [[SMIN4:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC2]], [[C3]]
+    ; GFX8: [[SUB5:%[0-9]+]]:_(s16) = G_SUB [[C2]], [[SMIN4]]
+    ; GFX8: [[SMAX5:%[0-9]+]]:_(s16) = G_SMAX [[SUB5]], [[TRUNC5]]
+    ; GFX8: [[SMIN5:%[0-9]+]]:_(s16) = G_SMIN [[SMAX5]], [[SUB4]]
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s16) = G_ADD [[TRUNC2]], [[SMIN5]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
+    ; GFX8: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ADD2]](s16)
+    ; GFX8: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C]](s32)
+    ; GFX8: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
+    ; GFX8: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
+    ; GFX8: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; GFX8: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
+    ; GFX8: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX8: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
+    ; GFX8: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS1]](<6 x s16>)
     ; GFX9-LABEL: name: saddsat_v3s16
     ; GFX9: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
-    ; GFX9: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UV]](<3 x s16>)
-    ; GFX9: [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UV1]](<3 x s16>)
-    ; GFX9: [[SADDSAT:%[0-9]+]]:_(s16) = G_SADDSAT [[UV2]], [[UV5]]
-    ; GFX9: [[SADDSAT1:%[0-9]+]]:_(s16) = G_SADDSAT [[UV3]], [[UV6]]
-    ; GFX9: [[SADDSAT2:%[0-9]+]]:_(s16) = G_SADDSAT [[UV4]], [[UV7]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[SADDSAT]](s16), [[SADDSAT1]](s16), [[SADDSAT2]](s16)
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<3 x s16>), [[EXTRACT]](<3 x s16>)
-    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
+    ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY1]](s32), [[COPY2]](s32)
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; GFX9: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY3]](s32), [[DEF1]](s32)
+    ; GFX9: [[DEF2:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV1]](<3 x s16>), 0
+    ; GFX9: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; GFX9: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; GFX9: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
+    ; GFX9: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+    ; GFX9: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY4]](s32), [[COPY5]](s32)
+    ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[BITCAST3]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY6]](s32), [[DEF1]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767
+    ; GFX9: [[BUILD_VECTOR_TRUNC4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C1]](s32), [[C1]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -32768
+    ; GFX9: [[BUILD_VECTOR_TRUNC5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C2]](s32), [[C2]](s32)
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[BUILD_VECTOR_TRUNC6:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C3]](s32), [[C3]](s32)
+    ; GFX9: [[SMAX:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[BUILD_VECTOR_TRUNC]], [[BUILD_VECTOR_TRUNC6]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC4]], [[SMAX]]
+    ; GFX9: [[SMIN:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[BUILD_VECTOR_TRUNC]], [[BUILD_VECTOR_TRUNC6]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC5]], [[SMIN]]
+    ; GFX9: [[SMAX1:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[SUB1]], [[BUILD_VECTOR_TRUNC2]]
+    ; GFX9: [[SMIN1:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(<2 x s16>) = G_ADD [[BUILD_VECTOR_TRUNC]], [[SMIN1]]
+    ; GFX9: [[BUILD_VECTOR_TRUNC7:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C1]](s32), [[C1]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC8:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C2]](s32), [[C2]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC9:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C3]](s32), [[C3]](s32)
+    ; GFX9: [[SMAX2:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[BUILD_VECTOR_TRUNC1]], [[BUILD_VECTOR_TRUNC9]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC7]], [[SMAX2]]
+    ; GFX9: [[SMIN2:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[BUILD_VECTOR_TRUNC1]], [[BUILD_VECTOR_TRUNC9]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC8]], [[SMIN2]]
+    ; GFX9: [[SMAX3:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[SUB3]], [[BUILD_VECTOR_TRUNC3]]
+    ; GFX9: [[SMIN3:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[SMAX3]], [[SUB2]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(<2 x s16>) = G_ADD [[BUILD_VECTOR_TRUNC1]], [[SMIN3]]
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[ADD]](<2 x s16>), [[ADD1]](<2 x s16>), [[DEF2]](<2 x s16>)
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<6 x s16>), 0
+    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
+    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS1]](<6 x s16>)
     %0:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     %1:_(<3 x s16>), %2:_(<3 x s16>) = G_UNMERGE_VALUES %0
     %3:_(<3 x s16>) = G_SADDSAT %1, %2
@@ -359,36 +721,180 @@ body: |
     ; GFX6-LABEL: name: saddsat_v4s16
     ; GFX6: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
     ; GFX6: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
-    ; GFX6: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
-    ; GFX6: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
-    ; GFX6: [[SADDSAT:%[0-9]+]]:_(s16) = G_SADDSAT [[UV]], [[UV4]]
-    ; GFX6: [[SADDSAT1:%[0-9]+]]:_(s16) = G_SADDSAT [[UV1]], [[UV5]]
-    ; GFX6: [[SADDSAT2:%[0-9]+]]:_(s16) = G_SADDSAT [[UV2]], [[UV6]]
-    ; GFX6: [[SADDSAT3:%[0-9]+]]:_(s16) = G_SADDSAT [[UV3]], [[UV7]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[SADDSAT]](s16), [[SADDSAT1]](s16), [[SADDSAT2]](s16), [[SADDSAT3]](s16)
-    ; GFX6: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; GFX6: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; GFX6: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX6: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX6: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX6: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; GFX6: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX6: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; GFX6: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX6: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX6: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SHL]], [[C3]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMAX]]
+    ; GFX6: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SHL]], [[C3]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMIN]]
+    ; GFX6: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB1]], [[SHL1]]
+    ; GFX6: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SHL]], [[SMIN1]]
+    ; GFX6: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[ADD]], [[C]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX6: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; GFX6: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY4]], [[C]](s32)
+    ; GFX6: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[COPY5]], [[C]](s32)
+    ; GFX6: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[SHL2]], [[C3]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMAX2]]
+    ; GFX6: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[SHL2]], [[C3]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMIN2]]
+    ; GFX6: [[SMAX3:%[0-9]+]]:_(s32) = G_SMAX [[SUB3]], [[SHL3]]
+    ; GFX6: [[SMIN3:%[0-9]+]]:_(s32) = G_SMIN [[SMAX3]], [[SUB2]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SHL2]], [[SMIN3]]
+    ; GFX6: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[ADD1]], [[C]](s32)
+    ; GFX6: [[COPY6:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; GFX6: [[COPY7:%[0-9]+]]:_(s32) = COPY [[BITCAST3]](s32)
+    ; GFX6: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[COPY6]], [[C]](s32)
+    ; GFX6: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[COPY7]], [[C]](s32)
+    ; GFX6: [[SMAX4:%[0-9]+]]:_(s32) = G_SMAX [[SHL4]], [[C3]]
+    ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMAX4]]
+    ; GFX6: [[SMIN4:%[0-9]+]]:_(s32) = G_SMIN [[SHL4]], [[C3]]
+    ; GFX6: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMIN4]]
+    ; GFX6: [[SMAX5:%[0-9]+]]:_(s32) = G_SMAX [[SUB5]], [[SHL5]]
+    ; GFX6: [[SMIN5:%[0-9]+]]:_(s32) = G_SMIN [[SMAX5]], [[SUB4]]
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SHL4]], [[SMIN5]]
+    ; GFX6: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[ADD2]], [[C]](s32)
+    ; GFX6: [[COPY8:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX6: [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; GFX6: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[COPY8]], [[C]](s32)
+    ; GFX6: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[COPY9]], [[C]](s32)
+    ; GFX6: [[SMAX6:%[0-9]+]]:_(s32) = G_SMAX [[SHL6]], [[C3]]
+    ; GFX6: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMAX6]]
+    ; GFX6: [[SMIN6:%[0-9]+]]:_(s32) = G_SMIN [[SHL6]], [[C3]]
+    ; GFX6: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[SMIN6]]
+    ; GFX6: [[SMAX7:%[0-9]+]]:_(s32) = G_SMAX [[SUB7]], [[SHL7]]
+    ; GFX6: [[SMIN7:%[0-9]+]]:_(s32) = G_SMIN [[SMAX7]], [[SUB6]]
+    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SHL6]], [[SMIN7]]
+    ; GFX6: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[ADD3]], [[C]](s32)
+    ; GFX6: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX6: [[COPY10:%[0-9]+]]:_(s32) = COPY [[ASHR]](s32)
+    ; GFX6: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY10]], [[C4]]
+    ; GFX6: [[COPY11:%[0-9]+]]:_(s32) = COPY [[ASHR1]](s32)
+    ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY11]], [[C4]]
+    ; GFX6: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX6: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL8]]
+    ; GFX6: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX6: [[COPY12:%[0-9]+]]:_(s32) = COPY [[ASHR2]](s32)
+    ; GFX6: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY12]], [[C4]]
+    ; GFX6: [[COPY13:%[0-9]+]]:_(s32) = COPY [[ASHR3]](s32)
+    ; GFX6: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY13]], [[C4]]
+    ; GFX6: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
+    ; GFX6: [[OR1:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL9]]
+    ; GFX6: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
+    ; GFX6: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; GFX6: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
     ; GFX8-LABEL: name: saddsat_v4s16
     ; GFX8: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
     ; GFX8: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
-    ; GFX8: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
-    ; GFX8: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
-    ; GFX8: [[SADDSAT:%[0-9]+]]:_(s16) = G_SADDSAT [[UV]], [[UV4]]
-    ; GFX8: [[SADDSAT1:%[0-9]+]]:_(s16) = G_SADDSAT [[UV1]], [[UV5]]
-    ; GFX8: [[SADDSAT2:%[0-9]+]]:_(s16) = G_SADDSAT [[UV2]], [[UV6]]
-    ; GFX8: [[SADDSAT3:%[0-9]+]]:_(s16) = G_SADDSAT [[UV3]], [[UV7]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[SADDSAT]](s16), [[SADDSAT1]](s16), [[SADDSAT2]](s16), [[SADDSAT3]](s16)
-    ; GFX8: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; GFX8: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX8: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX8: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+    ; GFX8: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; GFX8: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX8: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
+    ; GFX8: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; GFX8: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+    ; GFX8: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX8: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+    ; GFX8: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; GFX8: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
+    ; GFX8: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
+    ; GFX8: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 0
+    ; GFX8: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC]], [[C3]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[C1]], [[SMAX]]
+    ; GFX8: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC]], [[C3]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[C2]], [[SMIN]]
+    ; GFX8: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB1]], [[TRUNC4]]
+    ; GFX8: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[TRUNC]], [[SMIN1]]
+    ; GFX8: [[SMAX2:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC1]], [[C3]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[C1]], [[SMAX2]]
+    ; GFX8: [[SMIN2:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC1]], [[C3]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s16) = G_SUB [[C2]], [[SMIN2]]
+    ; GFX8: [[SMAX3:%[0-9]+]]:_(s16) = G_SMAX [[SUB3]], [[TRUNC5]]
+    ; GFX8: [[SMIN3:%[0-9]+]]:_(s16) = G_SMIN [[SMAX3]], [[SUB2]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[TRUNC1]], [[SMIN3]]
+    ; GFX8: [[SMAX4:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC2]], [[C3]]
+    ; GFX8: [[SUB4:%[0-9]+]]:_(s16) = G_SUB [[C1]], [[SMAX4]]
+    ; GFX8: [[SMIN4:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC2]], [[C3]]
+    ; GFX8: [[SUB5:%[0-9]+]]:_(s16) = G_SUB [[C2]], [[SMIN4]]
+    ; GFX8: [[SMAX5:%[0-9]+]]:_(s16) = G_SMAX [[SUB5]], [[TRUNC6]]
+    ; GFX8: [[SMIN5:%[0-9]+]]:_(s16) = G_SMIN [[SMAX5]], [[SUB4]]
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s16) = G_ADD [[TRUNC2]], [[SMIN5]]
+    ; GFX8: [[SMAX6:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC3]], [[C3]]
+    ; GFX8: [[SUB6:%[0-9]+]]:_(s16) = G_SUB [[C1]], [[SMAX6]]
+    ; GFX8: [[SMIN6:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC3]], [[C3]]
+    ; GFX8: [[SUB7:%[0-9]+]]:_(s16) = G_SUB [[C2]], [[SMIN6]]
+    ; GFX8: [[SMAX7:%[0-9]+]]:_(s16) = G_SMAX [[SUB7]], [[TRUNC7]]
+    ; GFX8: [[SMIN7:%[0-9]+]]:_(s16) = G_SMIN [[SMAX7]], [[SUB6]]
+    ; GFX8: [[ADD3:%[0-9]+]]:_(s16) = G_ADD [[TRUNC3]], [[SMIN7]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
+    ; GFX8: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ADD2]](s16)
+    ; GFX8: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[ADD3]](s16)
+    ; GFX8: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+    ; GFX8: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
+    ; GFX8: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
+    ; GFX8: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; GFX8: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
     ; GFX9-LABEL: name: saddsat_v4s16
     ; GFX9: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
-    ; GFX9: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
-    ; GFX9: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
-    ; GFX9: [[SADDSAT:%[0-9]+]]:_(s16) = G_SADDSAT [[UV]], [[UV4]]
-    ; GFX9: [[SADDSAT1:%[0-9]+]]:_(s16) = G_SADDSAT [[UV1]], [[UV5]]
-    ; GFX9: [[SADDSAT2:%[0-9]+]]:_(s16) = G_SADDSAT [[UV2]], [[UV6]]
-    ; GFX9: [[SADDSAT3:%[0-9]+]]:_(s16) = G_SADDSAT [[UV3]], [[UV7]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[SADDSAT]](s16), [[SADDSAT1]](s16), [[SADDSAT2]](s16), [[SADDSAT3]](s16)
-    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767
+    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C]](s32), [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -32768
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C1]](s32), [[C1]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[BUILD_VECTOR_TRUNC2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C2]](s32), [[C2]](s32)
+    ; GFX9: [[SMAX:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[UV]], [[BUILD_VECTOR_TRUNC2]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC]], [[SMAX]]
+    ; GFX9: [[SMIN:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[UV]], [[BUILD_VECTOR_TRUNC2]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC1]], [[SMIN]]
+    ; GFX9: [[SMAX1:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[SUB1]], [[UV2]]
+    ; GFX9: [[SMIN1:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(<2 x s16>) = G_ADD [[UV]], [[SMIN1]]
+    ; GFX9: [[BUILD_VECTOR_TRUNC3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C]](s32), [[C]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C1]](s32), [[C1]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C2]](s32), [[C2]](s32)
+    ; GFX9: [[SMAX2:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[UV1]], [[BUILD_VECTOR_TRUNC5]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC3]], [[SMAX2]]
+    ; GFX9: [[SMIN2:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[UV1]], [[BUILD_VECTOR_TRUNC5]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC4]], [[SMIN2]]
+    ; GFX9: [[SMAX3:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[SUB3]], [[UV3]]
+    ; GFX9: [[SMIN3:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[SMAX3]], [[SUB2]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(<2 x s16>) = G_ADD [[UV1]], [[SMIN3]]
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[ADD]](<2 x s16>), [[ADD1]](<2 x s16>)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
     %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
     %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
     %2:_(<4 x s16>) = G_SADDSAT %0, %1
@@ -404,18 +910,45 @@ body: |
     ; GFX6-LABEL: name: saddsat_s32
     ; GFX6: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX6: [[SADDSAT:%[0-9]+]]:_(s32) = G_SADDSAT [[COPY]], [[COPY1]]
-    ; GFX6: $vgpr0 = COPY [[SADDSAT]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX6: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[COPY]], [[C2]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C]], [[SMAX]]
+    ; GFX6: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[COPY]], [[C2]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMIN]]
+    ; GFX6: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB1]], [[COPY1]]
+    ; GFX6: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[SMIN1]]
+    ; GFX6: $vgpr0 = COPY [[ADD]](s32)
     ; GFX8-LABEL: name: saddsat_s32
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX8: [[SADDSAT:%[0-9]+]]:_(s32) = G_SADDSAT [[COPY]], [[COPY1]]
-    ; GFX8: $vgpr0 = COPY [[SADDSAT]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[COPY]], [[C2]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C]], [[SMAX]]
+    ; GFX8: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[COPY]], [[C2]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMIN]]
+    ; GFX8: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB1]], [[COPY1]]
+    ; GFX8: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[SMIN1]]
+    ; GFX8: $vgpr0 = COPY [[ADD]](s32)
     ; GFX9-LABEL: name: saddsat_s32
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX9: [[SADDSAT:%[0-9]+]]:_(s32) = G_SADDSAT [[COPY]], [[COPY1]]
-    ; GFX9: $vgpr0 = COPY [[SADDSAT]](s32)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[COPY]], [[C2]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C]], [[SMAX]]
+    ; GFX9: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[COPY]], [[C2]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMIN]]
+    ; GFX9: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB1]], [[COPY1]]
+    ; GFX9: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[SMIN1]]
+    ; GFX9: $vgpr0 = COPY [[ADD]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = G_SADDSAT %0, %1
@@ -433,27 +966,72 @@ body: |
     ; GFX6: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
     ; GFX6: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
     ; GFX6: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
-    ; GFX6: [[SADDSAT:%[0-9]+]]:_(s32) = G_SADDSAT [[UV]], [[UV2]]
-    ; GFX6: [[SADDSAT1:%[0-9]+]]:_(s32) = G_SADDSAT [[UV1]], [[UV3]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SADDSAT]](s32), [[SADDSAT1]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX6: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[UV]], [[C2]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C]], [[SMAX]]
+    ; GFX6: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[UV]], [[C2]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMIN]]
+    ; GFX6: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB1]], [[UV2]]
+    ; GFX6: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[UV]], [[SMIN1]]
+    ; GFX6: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[UV1]], [[C2]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[C]], [[SMAX2]]
+    ; GFX6: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[UV1]], [[C2]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMIN2]]
+    ; GFX6: [[SMAX3:%[0-9]+]]:_(s32) = G_SMAX [[SUB3]], [[UV3]]
+    ; GFX6: [[SMIN3:%[0-9]+]]:_(s32) = G_SMIN [[SMAX3]], [[SUB2]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[SMIN3]]
+    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[ADD]](s32), [[ADD1]](s32)
     ; GFX6: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ; GFX8-LABEL: name: saddsat_v2s32
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
     ; GFX8: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
     ; GFX8: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
-    ; GFX8: [[SADDSAT:%[0-9]+]]:_(s32) = G_SADDSAT [[UV]], [[UV2]]
-    ; GFX8: [[SADDSAT1:%[0-9]+]]:_(s32) = G_SADDSAT [[UV1]], [[UV3]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SADDSAT]](s32), [[SADDSAT1]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[UV]], [[C2]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C]], [[SMAX]]
+    ; GFX8: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[UV]], [[C2]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMIN]]
+    ; GFX8: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB1]], [[UV2]]
+    ; GFX8: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[UV]], [[SMIN1]]
+    ; GFX8: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[UV1]], [[C2]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[C]], [[SMAX2]]
+    ; GFX8: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[UV1]], [[C2]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMIN2]]
+    ; GFX8: [[SMAX3:%[0-9]+]]:_(s32) = G_SMAX [[SUB3]], [[UV3]]
+    ; GFX8: [[SMIN3:%[0-9]+]]:_(s32) = G_SMIN [[SMAX3]], [[SUB2]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[SMIN3]]
+    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[ADD]](s32), [[ADD1]](s32)
     ; GFX8: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ; GFX9-LABEL: name: saddsat_v2s32
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
     ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
     ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
-    ; GFX9: [[SADDSAT:%[0-9]+]]:_(s32) = G_SADDSAT [[UV]], [[UV2]]
-    ; GFX9: [[SADDSAT1:%[0-9]+]]:_(s32) = G_SADDSAT [[UV1]], [[UV3]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SADDSAT]](s32), [[SADDSAT1]](s32)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[UV]], [[C2]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C]], [[SMAX]]
+    ; GFX9: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[UV]], [[C2]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMIN]]
+    ; GFX9: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB1]], [[UV2]]
+    ; GFX9: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[UV]], [[SMIN1]]
+    ; GFX9: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[UV1]], [[C2]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[C]], [[SMAX2]]
+    ; GFX9: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[UV1]], [[C2]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C1]], [[SMIN2]]
+    ; GFX9: [[SMAX3:%[0-9]+]]:_(s32) = G_SMAX [[SUB3]], [[UV3]]
+    ; GFX9: [[SMIN3:%[0-9]+]]:_(s32) = G_SMIN [[SMAX3]], [[SUB2]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[SMIN3]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[ADD]](s32), [[ADD1]](s32)
     ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
     %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
@@ -470,18 +1048,69 @@ body: |
     ; GFX6-LABEL: name: saddsat_s64
     ; GFX6: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX6: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX6: [[SADDSAT:%[0-9]+]]:_(s64) = G_SADDSAT [[COPY]], [[COPY1]]
-    ; GFX6: $vgpr0_vgpr1 = COPY [[SADDSAT]](s64)
+    ; GFX6: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX6: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX6: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV]], [[UV2]]
+    ; GFX6: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV1]], [[UV3]], [[UADDO1]]
+    ; GFX6: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV]](s64), [[COPY]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[COPY1]](s64), [[C]]
+    ; GFX6: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]]
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX6: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MV]], [[C1]](s32)
+    ; GFX6: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -9223372036854775808
+    ; GFX6: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX6: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64)
+    ; GFX6: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX6: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO3]]
+    ; GFX6: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[MV]]
+    ; GFX6: $vgpr0_vgpr1 = COPY [[SELECT]](s64)
     ; GFX8-LABEL: name: saddsat_s64
     ; GFX8: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX8: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX8: [[SADDSAT:%[0-9]+]]:_(s64) = G_SADDSAT [[COPY]], [[COPY1]]
-    ; GFX8: $vgpr0_vgpr1 = COPY [[SADDSAT]](s64)
+    ; GFX8: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX8: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX8: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV]], [[UV2]]
+    ; GFX8: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV1]], [[UV3]], [[UADDO1]]
+    ; GFX8: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV]](s64), [[COPY]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[COPY1]](s64), [[C]]
+    ; GFX8: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]]
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX8: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MV]], [[C1]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -9223372036854775808
+    ; GFX8: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX8: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64)
+    ; GFX8: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX8: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO3]]
+    ; GFX8: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[MV]]
+    ; GFX8: $vgpr0_vgpr1 = COPY [[SELECT]](s64)
     ; GFX9-LABEL: name: saddsat_s64
     ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX9: [[SADDSAT:%[0-9]+]]:_(s64) = G_SADDSAT [[COPY]], [[COPY1]]
-    ; GFX9: $vgpr0_vgpr1 = COPY [[SADDSAT]](s64)
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX9: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV]], [[UV2]]
+    ; GFX9: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV1]], [[UV3]], [[UADDO1]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX9: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV]](s64), [[COPY]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[COPY1]](s64), [[C]]
+    ; GFX9: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MV]], [[C1]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -9223372036854775808
+    ; GFX9: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX9: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64)
+    ; GFX9: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX9: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO3]]
+    ; GFX9: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[MV]]
+    ; GFX9: $vgpr0_vgpr1 = COPY [[SELECT]](s64)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s64) = COPY $vgpr2_vgpr3
     %2:_(s64) = G_SADDSAT %0, %1
@@ -499,27 +1128,120 @@ body: |
     ; GFX6: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX6: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
     ; GFX6: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
-    ; GFX6: [[SADDSAT:%[0-9]+]]:_(s64) = G_SADDSAT [[UV]], [[UV2]]
-    ; GFX6: [[SADDSAT1:%[0-9]+]]:_(s64) = G_SADDSAT [[UV1]], [[UV3]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SADDSAT]](s64), [[SADDSAT1]](s64)
+    ; GFX6: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX6: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX6: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX6: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO1]]
+    ; GFX6: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV]](s64), [[UV]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[UV2]](s64), [[C]]
+    ; GFX6: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]]
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX6: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MV]], [[C1]](s32)
+    ; GFX6: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -9223372036854775808
+    ; GFX6: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX6: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64)
+    ; GFX6: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV8]], [[UV10]]
+    ; GFX6: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV9]], [[UV11]], [[UADDO3]]
+    ; GFX6: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[MV]]
+    ; GFX6: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX6: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX6: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[UV12]], [[UV14]]
+    ; GFX6: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[UV13]], [[UV15]], [[UADDO5]]
+    ; GFX6: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO4]](s32), [[UADDE4]](s32)
+    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV2]](s64), [[UV1]]
+    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[UV3]](s64), [[C]]
+    ; GFX6: [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[ICMP3]], [[ICMP2]]
+    ; GFX6: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[MV2]], [[C1]](s32)
+    ; GFX6: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64)
+    ; GFX6: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64)
+    ; GFX6: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UV16]], [[UV18]]
+    ; GFX6: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[UV17]], [[UV19]], [[UADDO7]]
+    ; GFX6: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO6]](s32), [[UADDE6]](s32)
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[XOR1]](s1), [[MV3]], [[MV2]]
+    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT]](s64), [[SELECT1]](s64)
     ; GFX6: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     ; GFX8-LABEL: name: saddsat_v2s64
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX8: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
     ; GFX8: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
-    ; GFX8: [[SADDSAT:%[0-9]+]]:_(s64) = G_SADDSAT [[UV]], [[UV2]]
-    ; GFX8: [[SADDSAT1:%[0-9]+]]:_(s64) = G_SADDSAT [[UV1]], [[UV3]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SADDSAT]](s64), [[SADDSAT1]](s64)
+    ; GFX8: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX8: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX8: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX8: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO1]]
+    ; GFX8: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV]](s64), [[UV]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[UV2]](s64), [[C]]
+    ; GFX8: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]]
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX8: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MV]], [[C1]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -9223372036854775808
+    ; GFX8: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX8: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64)
+    ; GFX8: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV8]], [[UV10]]
+    ; GFX8: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV9]], [[UV11]], [[UADDO3]]
+    ; GFX8: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[MV]]
+    ; GFX8: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX8: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX8: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[UV12]], [[UV14]]
+    ; GFX8: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[UV13]], [[UV15]], [[UADDO5]]
+    ; GFX8: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO4]](s32), [[UADDE4]](s32)
+    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV2]](s64), [[UV1]]
+    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[UV3]](s64), [[C]]
+    ; GFX8: [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[ICMP3]], [[ICMP2]]
+    ; GFX8: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[MV2]], [[C1]](s32)
+    ; GFX8: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64)
+    ; GFX8: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64)
+    ; GFX8: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UV16]], [[UV18]]
+    ; GFX8: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[UV17]], [[UV19]], [[UADDO7]]
+    ; GFX8: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO6]](s32), [[UADDE6]](s32)
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[XOR1]](s1), [[MV3]], [[MV2]]
+    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT]](s64), [[SELECT1]](s64)
     ; GFX8: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     ; GFX9-LABEL: name: saddsat_v2s64
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
     ; GFX9: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
-    ; GFX9: [[SADDSAT:%[0-9]+]]:_(s64) = G_SADDSAT [[UV]], [[UV2]]
-    ; GFX9: [[SADDSAT1:%[0-9]+]]:_(s64) = G_SADDSAT [[UV1]], [[UV3]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SADDSAT]](s64), [[SADDSAT1]](s64)
+    ; GFX9: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX9: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX9: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX9: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO1]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX9: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV]](s64), [[UV]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[UV2]](s64), [[C]]
+    ; GFX9: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MV]], [[C1]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -9223372036854775808
+    ; GFX9: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX9: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64)
+    ; GFX9: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV8]], [[UV10]]
+    ; GFX9: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV9]], [[UV11]], [[UADDO3]]
+    ; GFX9: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[MV]]
+    ; GFX9: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX9: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX9: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[UV12]], [[UV14]]
+    ; GFX9: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[UV13]], [[UV15]], [[UADDO5]]
+    ; GFX9: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO4]](s32), [[UADDE4]](s32)
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV2]](s64), [[UV1]]
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[UV3]](s64), [[C]]
+    ; GFX9: [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[ICMP3]], [[ICMP2]]
+    ; GFX9: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[MV2]], [[C1]](s32)
+    ; GFX9: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64)
+    ; GFX9: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64)
+    ; GFX9: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UV16]], [[UV18]]
+    ; GFX9: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[UV17]], [[UV19]], [[UADDO7]]
+    ; GFX9: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO6]](s32), [[UADDE6]](s32)
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[XOR1]](s1), [[MV3]], [[MV2]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT]](s64), [[SELECT1]](s64)
     ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     %1:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir
index 5f5697f5fddc8..88689ae03616f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir
@@ -271,31 +271,29 @@ body: |
     ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[C]]
     ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
     ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[TRUNC]](<3 x s8>), 0
-    ; CHECK: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF]](<4 x s8>)
-    ; CHECK: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[TRUNC1]](<3 x s8>), 0
-    ; CHECK: [[TRUNC2:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT1]](<4 x s16>)
+    ; CHECK: [[INSERT1:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF]], [[TRUNC1]](<3 x s8>), 0
     ; CHECK: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8), [[UV2:%[0-9]+]]:_(s8), [[UV3:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[INSERT]](<4 x s8>)
-    ; CHECK: [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8), [[UV6:%[0-9]+]]:_(s8), [[UV7:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[TRUNC2]](<4 x s8>)
-    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[UV]](s8)
-    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s16) = G_ANYEXT [[UV4]](s8)
-    ; CHECK: [[SELECT:%[0-9]+]]:_(s16) = G_SELECT [[ICMP]](s1), [[ANYEXT1]], [[ANYEXT2]]
-    ; CHECK: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[SELECT]](s16)
-    ; CHECK: [[ANYEXT3:%[0-9]+]]:_(s16) = G_ANYEXT [[UV1]](s8)
-    ; CHECK: [[ANYEXT4:%[0-9]+]]:_(s16) = G_ANYEXT [[UV5]](s8)
-    ; CHECK: [[SELECT1:%[0-9]+]]:_(s16) = G_SELECT [[ICMP]](s1), [[ANYEXT3]], [[ANYEXT4]]
-    ; CHECK: [[TRUNC4:%[0-9]+]]:_(s8) = G_TRUNC [[SELECT1]](s16)
-    ; CHECK: [[ANYEXT5:%[0-9]+]]:_(s16) = G_ANYEXT [[UV2]](s8)
-    ; CHECK: [[ANYEXT6:%[0-9]+]]:_(s16) = G_ANYEXT [[UV6]](s8)
-    ; CHECK: [[SELECT2:%[0-9]+]]:_(s16) = G_SELECT [[ICMP]](s1), [[ANYEXT5]], [[ANYEXT6]]
-    ; CHECK: [[TRUNC5:%[0-9]+]]:_(s8) = G_TRUNC [[SELECT2]](s16)
-    ; CHECK: [[ANYEXT7:%[0-9]+]]:_(s16) = G_ANYEXT [[UV3]](s8)
-    ; CHECK: [[ANYEXT8:%[0-9]+]]:_(s16) = G_ANYEXT [[UV7]](s8)
-    ; CHECK: [[SELECT3:%[0-9]+]]:_(s16) = G_SELECT [[ICMP]](s1), [[ANYEXT7]], [[ANYEXT8]]
-    ; CHECK: [[TRUNC6:%[0-9]+]]:_(s8) = G_TRUNC [[SELECT3]](s16)
-    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s8>) = G_BUILD_VECTOR [[TRUNC3]](s8), [[TRUNC4]](s8), [[TRUNC5]](s8), [[TRUNC6]](s8)
+    ; CHECK: [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8), [[UV6:%[0-9]+]]:_(s8), [[UV7:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[INSERT1]](<4 x s8>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[UV]](s8)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[UV4]](s8)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s16) = G_SELECT [[ICMP]](s1), [[ANYEXT]], [[ANYEXT1]]
+    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[SELECT]](s16)
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s16) = G_ANYEXT [[UV1]](s8)
+    ; CHECK: [[ANYEXT3:%[0-9]+]]:_(s16) = G_ANYEXT [[UV5]](s8)
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(s16) = G_SELECT [[ICMP]](s1), [[ANYEXT2]], [[ANYEXT3]]
+    ; CHECK: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[SELECT1]](s16)
+    ; CHECK: [[ANYEXT4:%[0-9]+]]:_(s16) = G_ANYEXT [[UV2]](s8)
+    ; CHECK: [[ANYEXT5:%[0-9]+]]:_(s16) = G_ANYEXT [[UV6]](s8)
+    ; CHECK: [[SELECT2:%[0-9]+]]:_(s16) = G_SELECT [[ICMP]](s1), [[ANYEXT4]], [[ANYEXT5]]
+    ; CHECK: [[TRUNC4:%[0-9]+]]:_(s8) = G_TRUNC [[SELECT2]](s16)
+    ; CHECK: [[ANYEXT6:%[0-9]+]]:_(s16) = G_ANYEXT [[UV3]](s8)
+    ; CHECK: [[ANYEXT7:%[0-9]+]]:_(s16) = G_ANYEXT [[UV7]](s8)
+    ; CHECK: [[SELECT3:%[0-9]+]]:_(s16) = G_SELECT [[ICMP]](s1), [[ANYEXT6]], [[ANYEXT7]]
+    ; CHECK: [[TRUNC5:%[0-9]+]]:_(s8) = G_TRUNC [[SELECT3]](s16)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s8>) = G_BUILD_VECTOR [[TRUNC2]](s8), [[TRUNC3]](s8), [[TRUNC4]](s8), [[TRUNC5]](s8)
     ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s8>) = G_EXTRACT [[BUILD_VECTOR]](<4 x s8>), 0
-    ; CHECK: [[ANYEXT9:%[0-9]+]]:_(<3 x s32>) = G_ANYEXT [[EXTRACT]](<3 x s8>)
-    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[ANYEXT9]](<3 x s32>)
+    ; CHECK: [[ANYEXT8:%[0-9]+]]:_(<3 x s32>) = G_ANYEXT [[EXTRACT]](<3 x s8>)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[ANYEXT8]](<3 x s32>)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = G_CONSTANT i32 0
     %2:_(<3 x s32>) = COPY $vgpr1_vgpr2_vgpr3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubsat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubsat.mir
index 31f119c13e5e1..40eb12034c97b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubsat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubsat.mir
@@ -12,51 +12,66 @@ body: |
     ; GFX6-LABEL: name: ssubsat_s7
     ; GFX6: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX6: [[TRUNC:%[0-9]+]]:_(s7) = G_TRUNC [[COPY]](s32)
-    ; GFX6: [[TRUNC1:%[0-9]+]]:_(s7) = G_TRUNC [[COPY1]](s32)
-    ; GFX6: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC]](s7)
-    ; GFX6: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC1]](s7)
-    ; GFX6: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 9
-    ; GFX6: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX6: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX6: [[SSUBSAT:%[0-9]+]]:_(s16) = G_SSUBSAT [[SHL]], [[SHL1]]
-    ; GFX6: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[C]](s16)
-    ; GFX6: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[SSUBSAT]](s16)
-    ; GFX6: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SEXT]], [[ZEXT]](s32)
-    ; GFX6: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[ASHR]](s32)
-    ; GFX6: [[TRUNC3:%[0-9]+]]:_(s7) = G_TRUNC [[TRUNC2]](s16)
-    ; GFX6: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC3]](s7)
-    ; GFX6: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 25
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX6: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SHL]], [[C3]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SMAX]], [[C1]]
+    ; GFX6: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SHL]], [[C3]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SMIN]], [[C2]]
+    ; GFX6: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB]], [[SHL1]]
+    ; GFX6: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SHL]], [[SMIN1]]
+    ; GFX6: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SUB2]], [[C]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ASHR]](s32)
+    ; GFX6: $vgpr0 = COPY [[COPY4]](s32)
     ; GFX8-LABEL: name: ssubsat_s7
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX8: [[TRUNC:%[0-9]+]]:_(s7) = G_TRUNC [[COPY]](s32)
-    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s7) = G_TRUNC [[COPY1]](s32)
-    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC]](s7)
-    ; GFX8: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC1]](s7)
+    ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
     ; GFX8: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 9
-    ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX8: [[SSUBSAT:%[0-9]+]]:_(s16) = G_SSUBSAT [[SHL]], [[SHL1]]
-    ; GFX8: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SSUBSAT]], [[C]](s16)
-    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s7) = G_TRUNC [[ASHR]](s16)
-    ; GFX8: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC2]](s7)
-    ; GFX8: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
+    ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
+    ; GFX8: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
+    ; GFX8: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
+    ; GFX8: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; GFX8: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[SHL]], [[C3]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SMAX]], [[C1]]
+    ; GFX8: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[SHL]], [[C3]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[SMIN]], [[C2]]
+    ; GFX8: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB]], [[SHL1]]
+    ; GFX8: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[SHL]], [[SMIN1]]
+    ; GFX8: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SUB2]], [[C]](s16)
+    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ASHR]](s16)
+    ; GFX8: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: ssubsat_s7
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX9: [[TRUNC:%[0-9]+]]:_(s7) = G_TRUNC [[COPY]](s32)
-    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s7) = G_TRUNC [[COPY1]](s32)
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC]](s7)
-    ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC1]](s7)
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
     ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 9
-    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX9: [[SSUBSAT:%[0-9]+]]:_(s16) = G_SSUBSAT [[SHL]], [[SHL1]]
-    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SSUBSAT]], [[C]](s16)
-    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s7) = G_TRUNC [[ASHR]](s16)
-    ; GFX9: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC2]](s7)
-    ; GFX9: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
+    ; GFX9: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
+    ; GFX9: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
+    ; GFX9: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; GFX9: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[SHL]], [[C3]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SMAX]], [[C1]]
+    ; GFX9: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[SHL]], [[C3]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[SMIN]], [[C2]]
+    ; GFX9: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB]], [[SHL1]]
+    ; GFX9: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[SHL]], [[SMIN1]]
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SUB2]], [[C]](s16)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ASHR]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s7) = G_TRUNC %0
@@ -75,51 +90,66 @@ body: |
     ; GFX6-LABEL: name: ssubsat_s8
     ; GFX6: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX6: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
-    ; GFX6: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32)
-    ; GFX6: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC]](s8)
-    ; GFX6: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC1]](s8)
-    ; GFX6: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX6: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX6: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX6: [[SSUBSAT:%[0-9]+]]:_(s16) = G_SSUBSAT [[SHL]], [[SHL1]]
-    ; GFX6: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[C]](s16)
-    ; GFX6: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[SSUBSAT]](s16)
-    ; GFX6: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SEXT]], [[ZEXT]](s32)
-    ; GFX6: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[ASHR]](s32)
-    ; GFX6: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC2]](s16)
-    ; GFX6: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC3]](s8)
-    ; GFX6: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX6: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SHL]], [[C3]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SMAX]], [[C1]]
+    ; GFX6: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SHL]], [[C3]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SMIN]], [[C2]]
+    ; GFX6: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB]], [[SHL1]]
+    ; GFX6: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SHL]], [[SMIN1]]
+    ; GFX6: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SUB2]], [[C]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ASHR]](s32)
+    ; GFX6: $vgpr0 = COPY [[COPY4]](s32)
     ; GFX8-LABEL: name: ssubsat_s8
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX8: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
-    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32)
-    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC]](s8)
-    ; GFX8: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC1]](s8)
+    ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
     ; GFX8: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX8: [[SSUBSAT:%[0-9]+]]:_(s16) = G_SSUBSAT [[SHL]], [[SHL1]]
-    ; GFX8: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SSUBSAT]], [[C]](s16)
-    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[ASHR]](s16)
-    ; GFX8: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC2]](s8)
-    ; GFX8: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
+    ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
+    ; GFX8: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
+    ; GFX8: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
+    ; GFX8: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; GFX8: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[SHL]], [[C3]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SMAX]], [[C1]]
+    ; GFX8: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[SHL]], [[C3]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[SMIN]], [[C2]]
+    ; GFX8: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB]], [[SHL1]]
+    ; GFX8: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[SHL]], [[SMIN1]]
+    ; GFX8: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SUB2]], [[C]](s16)
+    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ASHR]](s16)
+    ; GFX8: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: ssubsat_s8
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX9: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
-    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32)
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC]](s8)
-    ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC1]](s8)
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
     ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX9: [[SSUBSAT:%[0-9]+]]:_(s16) = G_SSUBSAT [[SHL]], [[SHL1]]
-    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SSUBSAT]], [[C]](s16)
-    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[ASHR]](s16)
-    ; GFX9: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC2]](s8)
-    ; GFX9: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
+    ; GFX9: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
+    ; GFX9: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
+    ; GFX9: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; GFX9: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[SHL]], [[C3]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SMAX]], [[C1]]
+    ; GFX9: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[SHL]], [[C3]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[SMIN]], [[C2]]
+    ; GFX9: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB]], [[SHL1]]
+    ; GFX9: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[SHL]], [[SMIN1]]
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SUB2]], [[C]](s16)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ASHR]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s8) = G_TRUNC %0
@@ -138,75 +168,152 @@ body: |
     ; GFX6-LABEL: name: ssubsat_v2s8
     ; GFX6: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX6: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
-    ; GFX6: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX6: [[BITCAST:%[0-9]+]]:_(<2 x s8>) = G_BITCAST [[TRUNC]](s16)
-    ; GFX6: [[BITCAST1:%[0-9]+]]:_(<2 x s8>) = G_BITCAST [[TRUNC1]](s16)
-    ; GFX6: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BITCAST]](<2 x s8>)
-    ; GFX6: [[UV2:%[0-9]+]]:_(s8), [[UV3:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BITCAST1]](<2 x s8>)
-    ; GFX6: [[SSUBSAT:%[0-9]+]]:_(s8) = G_SSUBSAT [[UV]], [[UV2]]
-    ; GFX6: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[UV1]](s8)
-    ; GFX6: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[UV3]](s8)
-    ; GFX6: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX6: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX6: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX6: [[SSUBSAT1:%[0-9]+]]:_(s16) = G_SSUBSAT [[SHL]], [[SHL1]]
-    ; GFX6: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[C]](s16)
-    ; GFX6: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[SSUBSAT1]](s16)
-    ; GFX6: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SEXT]], [[ZEXT]](s32)
-    ; GFX6: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[ASHR]](s32)
-    ; GFX6: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC2]](s16)
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s8>) = G_BUILD_VECTOR [[SSUBSAT]](s8), [[TRUNC3]](s8)
-    ; GFX6: [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<2 x s8>)
-    ; GFX6: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV4]](s8), [[UV5]](s8)
-    ; GFX6: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
-    ; GFX6: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX6: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX6: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; GFX6: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
+    ; GFX6: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C1]](s32)
+    ; GFX6: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C2]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C2]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C2]](s32)
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+    ; GFX6: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; GFX6: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX6: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SHL]], [[C5]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SMAX]], [[C3]]
+    ; GFX6: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SHL]], [[C5]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SMIN]], [[C4]]
+    ; GFX6: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB]], [[SHL1]]
+    ; GFX6: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SHL]], [[SMIN1]]
+    ; GFX6: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SUB2]], [[C2]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX6: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; GFX6: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY4]], [[C2]](s32)
+    ; GFX6: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[COPY5]], [[C2]](s32)
+    ; GFX6: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[SHL2]], [[C5]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SMAX2]], [[C3]]
+    ; GFX6: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[SHL2]], [[C5]]
+    ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[SMIN2]], [[C4]]
+    ; GFX6: [[SMAX3:%[0-9]+]]:_(s32) = G_SMAX [[SUB3]], [[SHL3]]
+    ; GFX6: [[SMIN3:%[0-9]+]]:_(s32) = G_SMIN [[SMAX3]], [[SUB4]]
+    ; GFX6: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[SHL2]], [[SMIN3]]
+    ; GFX6: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SUB5]], [[C2]](s32)
+    ; GFX6: [[C6:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX6: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[ASHR]](s32)
+    ; GFX6: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C6]]
+    ; GFX6: [[COPY6:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; GFX6: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; GFX6: [[COPY7:%[0-9]+]]:_(s32) = COPY [[ASHR1]](s32)
+    ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C7]]
+    ; GFX6: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY6]](s32)
+    ; GFX6: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL4]](s32)
+    ; GFX6: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; GFX6: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; GFX6: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX8-LABEL: name: ssubsat_v2s8
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX8: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; GFX8: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
+    ; GFX8: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C1]](s32)
+    ; GFX8: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C2]](s32)
     ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
     ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX8: [[BITCAST:%[0-9]+]]:_(<2 x s8>) = G_BITCAST [[TRUNC]](s16)
-    ; GFX8: [[BITCAST1:%[0-9]+]]:_(<2 x s8>) = G_BITCAST [[TRUNC1]](s16)
-    ; GFX8: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BITCAST]](<2 x s8>)
-    ; GFX8: [[UV2:%[0-9]+]]:_(s8), [[UV3:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BITCAST1]](<2 x s8>)
-    ; GFX8: [[SSUBSAT:%[0-9]+]]:_(s8) = G_SSUBSAT [[UV]], [[UV2]]
-    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[UV1]](s8)
-    ; GFX8: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[UV3]](s8)
-    ; GFX8: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX8: [[SSUBSAT1:%[0-9]+]]:_(s16) = G_SSUBSAT [[SHL]], [[SHL1]]
-    ; GFX8: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SSUBSAT1]], [[C]](s16)
-    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[ASHR]](s16)
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s8>) = G_BUILD_VECTOR [[SSUBSAT]](s8), [[TRUNC2]](s8)
-    ; GFX8: [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<2 x s8>)
-    ; GFX8: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV4]](s8), [[UV5]](s8)
-    ; GFX8: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
-    ; GFX8: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX8: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C3]](s16)
+    ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C3]](s16)
+    ; GFX8: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
+    ; GFX8: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
+    ; GFX8: [[C6:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; GFX8: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[SHL]], [[C6]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SMAX]], [[C4]]
+    ; GFX8: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[SHL]], [[C6]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[SMIN]], [[C5]]
+    ; GFX8: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB]], [[SHL1]]
+    ; GFX8: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[SHL]], [[SMIN1]]
+    ; GFX8: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SUB2]], [[C3]](s16)
+    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX8: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+    ; GFX8: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[C3]](s16)
+    ; GFX8: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[TRUNC3]], [[C3]](s16)
+    ; GFX8: [[SMAX2:%[0-9]+]]:_(s16) = G_SMAX [[SHL2]], [[C6]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s16) = G_SUB [[SMAX2]], [[C4]]
+    ; GFX8: [[SMIN2:%[0-9]+]]:_(s16) = G_SMIN [[SHL2]], [[C6]]
+    ; GFX8: [[SUB4:%[0-9]+]]:_(s16) = G_SUB [[SMIN2]], [[C5]]
+    ; GFX8: [[SMAX3:%[0-9]+]]:_(s16) = G_SMAX [[SUB3]], [[SHL3]]
+    ; GFX8: [[SMIN3:%[0-9]+]]:_(s16) = G_SMIN [[SMAX3]], [[SUB4]]
+    ; GFX8: [[SUB5:%[0-9]+]]:_(s16) = G_SUB [[SHL2]], [[SMIN3]]
+    ; GFX8: [[ASHR1:%[0-9]+]]:_(s16) = G_ASHR [[SUB5]], [[C3]](s16)
+    ; GFX8: [[C7:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s16) = COPY [[ASHR]](s16)
+    ; GFX8: [[AND:%[0-9]+]]:_(s16) = G_AND [[COPY2]], [[C7]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s16) = COPY [[ASHR1]](s16)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s16) = G_AND [[COPY3]], [[C7]]
+    ; GFX8: [[SHL4:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C3]](s16)
+    ; GFX8: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL4]]
+    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; GFX8: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: ssubsat_v2s8
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; GFX9: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
+    ; GFX9: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C1]](s32)
+    ; GFX9: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C2]](s32)
     ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
     ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX9: [[BITCAST:%[0-9]+]]:_(<2 x s8>) = G_BITCAST [[TRUNC]](s16)
-    ; GFX9: [[BITCAST1:%[0-9]+]]:_(<2 x s8>) = G_BITCAST [[TRUNC1]](s16)
-    ; GFX9: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BITCAST]](<2 x s8>)
-    ; GFX9: [[UV2:%[0-9]+]]:_(s8), [[UV3:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BITCAST1]](<2 x s8>)
-    ; GFX9: [[SSUBSAT:%[0-9]+]]:_(s8) = G_SSUBSAT [[UV]], [[UV2]]
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[UV1]](s8)
-    ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[UV3]](s8)
-    ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX9: [[SSUBSAT1:%[0-9]+]]:_(s16) = G_SSUBSAT [[SHL]], [[SHL1]]
-    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SSUBSAT1]], [[C]](s16)
-    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[ASHR]](s16)
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s8>) = G_BUILD_VECTOR [[SSUBSAT]](s8), [[TRUNC2]](s8)
-    ; GFX9: [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<2 x s8>)
-    ; GFX9: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV4]](s8), [[UV5]](s8)
-    ; GFX9: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
-    ; GFX9: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX9: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C3]](s16)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C3]](s16)
+    ; GFX9: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
+    ; GFX9: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
+    ; GFX9: [[C6:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; GFX9: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[SHL]], [[C6]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SMAX]], [[C4]]
+    ; GFX9: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[SHL]], [[C6]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[SMIN]], [[C5]]
+    ; GFX9: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB]], [[SHL1]]
+    ; GFX9: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[SHL]], [[SMIN1]]
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SUB2]], [[C3]](s16)
+    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX9: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+    ; GFX9: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[C3]](s16)
+    ; GFX9: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[TRUNC3]], [[C3]](s16)
+    ; GFX9: [[SMAX2:%[0-9]+]]:_(s16) = G_SMAX [[SHL2]], [[C6]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s16) = G_SUB [[SMAX2]], [[C4]]
+    ; GFX9: [[SMIN2:%[0-9]+]]:_(s16) = G_SMIN [[SHL2]], [[C6]]
+    ; GFX9: [[SUB4:%[0-9]+]]:_(s16) = G_SUB [[SMIN2]], [[C5]]
+    ; GFX9: [[SMAX3:%[0-9]+]]:_(s16) = G_SMAX [[SUB3]], [[SHL3]]
+    ; GFX9: [[SMIN3:%[0-9]+]]:_(s16) = G_SMIN [[SMAX3]], [[SUB4]]
+    ; GFX9: [[SUB5:%[0-9]+]]:_(s16) = G_SUB [[SHL2]], [[SMIN3]]
+    ; GFX9: [[ASHR1:%[0-9]+]]:_(s16) = G_ASHR [[SUB5]], [[C3]](s16)
+    ; GFX9: [[C7:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s16) = COPY [[ASHR]](s16)
+    ; GFX9: [[AND:%[0-9]+]]:_(s16) = G_AND [[COPY2]], [[C7]]
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s16) = COPY [[ASHR1]](s16)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s16) = G_AND [[COPY3]], [[C7]]
+    ; GFX9: [[SHL4:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C3]](s16)
+    ; GFX9: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL4]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s16) = G_TRUNC %0
@@ -228,26 +335,57 @@ body: |
     ; GFX6-LABEL: name: ssubsat_s16
     ; GFX6: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX6: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
-    ; GFX6: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX6: [[SSUBSAT:%[0-9]+]]:_(s16) = G_SSUBSAT [[TRUNC]], [[TRUNC1]]
-    ; GFX6: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SSUBSAT]](s16)
-    ; GFX6: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX6: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SHL]], [[C3]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SMAX]], [[C1]]
+    ; GFX6: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SHL]], [[C3]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SMIN]], [[C2]]
+    ; GFX6: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB]], [[SHL1]]
+    ; GFX6: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SHL]], [[SMIN1]]
+    ; GFX6: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SUB2]], [[C]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ASHR]](s32)
+    ; GFX6: $vgpr0 = COPY [[COPY4]](s32)
     ; GFX8-LABEL: name: ssubsat_s16
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
     ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX8: [[SSUBSAT:%[0-9]+]]:_(s16) = G_SSUBSAT [[TRUNC]], [[TRUNC1]]
-    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SSUBSAT]](s16)
+    ; GFX8: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
+    ; GFX8: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
+    ; GFX8: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; GFX8: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC]], [[C2]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SMAX]], [[C]]
+    ; GFX8: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC]], [[C2]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[SMIN]], [[C1]]
+    ; GFX8: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB]], [[TRUNC1]]
+    ; GFX8: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[TRUNC]], [[SMIN1]]
+    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SUB2]](s16)
     ; GFX8: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: ssubsat_s16
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
     ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX9: [[SSUBSAT:%[0-9]+]]:_(s16) = G_SSUBSAT [[TRUNC]], [[TRUNC1]]
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SSUBSAT]](s16)
+    ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
+    ; GFX9: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
+    ; GFX9: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; GFX9: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC]], [[C2]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SMAX]], [[C]]
+    ; GFX9: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC]], [[C2]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[SMIN]], [[C1]]
+    ; GFX9: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB]], [[TRUNC1]]
+    ; GFX9: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[TRUNC]], [[SMIN1]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SUB2]](s16)
     ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
@@ -267,30 +405,99 @@ body: |
     ; GFX6-LABEL: name: ssubsat_v2s16
     ; GFX6: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
-    ; GFX6: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
-    ; GFX6: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
-    ; GFX6: [[SSUBSAT:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV]], [[UV2]]
-    ; GFX6: [[SSUBSAT1:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV1]], [[UV3]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[SSUBSAT]](s16), [[SSUBSAT1]](s16)
-    ; GFX6: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX6: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX6: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; GFX6: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX6: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SHL]], [[C3]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SMAX]], [[C1]]
+    ; GFX6: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SHL]], [[C3]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SMIN]], [[C2]]
+    ; GFX6: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB]], [[SHL1]]
+    ; GFX6: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SHL]], [[SMIN1]]
+    ; GFX6: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SUB2]], [[C]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX6: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX6: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY4]], [[C]](s32)
+    ; GFX6: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[COPY5]], [[C]](s32)
+    ; GFX6: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[SHL2]], [[C3]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SMAX2]], [[C1]]
+    ; GFX6: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[SHL2]], [[C3]]
+    ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[SMIN2]], [[C2]]
+    ; GFX6: [[SMAX3:%[0-9]+]]:_(s32) = G_SMAX [[SUB3]], [[SHL3]]
+    ; GFX6: [[SMIN3:%[0-9]+]]:_(s32) = G_SMIN [[SMAX3]], [[SUB4]]
+    ; GFX6: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[SHL2]], [[SMIN3]]
+    ; GFX6: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SUB5]], [[C]](s32)
+    ; GFX6: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX6: [[COPY6:%[0-9]+]]:_(s32) = COPY [[ASHR]](s32)
+    ; GFX6: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C4]]
+    ; GFX6: [[COPY7:%[0-9]+]]:_(s32) = COPY [[ASHR1]](s32)
+    ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C4]]
+    ; GFX6: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX6: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL4]]
+    ; GFX6: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX6: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
     ; GFX8-LABEL: name: ssubsat_v2s16
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
-    ; GFX8: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
-    ; GFX8: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
-    ; GFX8: [[SSUBSAT:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV]], [[UV2]]
-    ; GFX8: [[SSUBSAT1:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV1]], [[UV3]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[SSUBSAT]](s16), [[SSUBSAT1]](s16)
-    ; GFX8: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX8: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX8: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
+    ; GFX8: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
+    ; GFX8: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; GFX8: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC]], [[C3]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SMAX]], [[C1]]
+    ; GFX8: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC]], [[C3]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[SMIN]], [[C2]]
+    ; GFX8: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB]], [[TRUNC2]]
+    ; GFX8: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[TRUNC]], [[SMIN1]]
+    ; GFX8: [[SMAX2:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC1]], [[C3]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s16) = G_SUB [[SMAX2]], [[C1]]
+    ; GFX8: [[SMIN2:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC1]], [[C3]]
+    ; GFX8: [[SUB4:%[0-9]+]]:_(s16) = G_SUB [[SMIN2]], [[C2]]
+    ; GFX8: [[SMAX3:%[0-9]+]]:_(s16) = G_SMAX [[SUB3]], [[TRUNC3]]
+    ; GFX8: [[SMIN3:%[0-9]+]]:_(s16) = G_SMIN [[SMAX3]], [[SUB4]]
+    ; GFX8: [[SUB5:%[0-9]+]]:_(s16) = G_SUB [[TRUNC1]], [[SMIN3]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SUB2]](s16)
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SUB5]](s16)
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
+    ; GFX8: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX8: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
     ; GFX9-LABEL: name: ssubsat_v2s16
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
-    ; GFX9: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
-    ; GFX9: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
-    ; GFX9: [[SSUBSAT:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV]], [[UV2]]
-    ; GFX9: [[SSUBSAT1:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV1]], [[UV3]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[SSUBSAT]](s16), [[SSUBSAT1]](s16)
-    ; GFX9: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767
+    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C]](s32), [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -32768
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C1]](s32), [[C1]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX9: [[BUILD_VECTOR_TRUNC2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C2]](s32), [[C2]](s32)
+    ; GFX9: [[SMAX:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[COPY]], [[BUILD_VECTOR_TRUNC2]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(<2 x s16>) = G_SUB [[SMAX]], [[BUILD_VECTOR_TRUNC]]
+    ; GFX9: [[SMIN:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[COPY]], [[BUILD_VECTOR_TRUNC2]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(<2 x s16>) = G_SUB [[SMIN]], [[BUILD_VECTOR_TRUNC1]]
+    ; GFX9: [[SMAX1:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[SUB]], [[COPY1]]
+    ; GFX9: [[SMIN1:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(<2 x s16>) = G_SUB [[COPY]], [[SMIN1]]
+    ; GFX9: $vgpr0 = COPY [[SUB2]](<2 x s16>)
     %0:_(<2 x s16>) = COPY $vgpr0
     %1:_(<2 x s16>) = COPY $vgpr1
     %2:_(<2 x s16>) = G_SSUBSAT %0, %1
@@ -306,42 +513,197 @@ body: |
     ; GFX6-LABEL: name: ssubsat_v3s16
     ; GFX6: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     ; GFX6: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
-    ; GFX6: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UV]](<3 x s16>)
-    ; GFX6: [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UV1]](<3 x s16>)
-    ; GFX6: [[SSUBSAT:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV2]], [[UV5]]
-    ; GFX6: [[SSUBSAT1:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV3]], [[UV6]]
-    ; GFX6: [[SSUBSAT2:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV4]], [[UV7]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[SSUBSAT]](s16), [[SSUBSAT1]](s16), [[SSUBSAT2]](s16)
     ; GFX6: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX6: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX6: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<3 x s16>), [[EXTRACT]](<3 x s16>)
-    ; GFX6: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
+    ; GFX6: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
+    ; GFX6: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX6: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX6: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX6: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX6: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV1]](<3 x s16>), 0
+    ; GFX6: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; GFX6: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
+    ; GFX6: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; GFX6: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
+    ; GFX6: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX6: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SHL]], [[C3]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SMAX]], [[C1]]
+    ; GFX6: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SHL]], [[C3]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SMIN]], [[C2]]
+    ; GFX6: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB]], [[SHL1]]
+    ; GFX6: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SHL]], [[SMIN1]]
+    ; GFX6: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SUB2]], [[C]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; GFX6: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+    ; GFX6: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[COPY4]], [[C]](s32)
+    ; GFX6: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[SHL2]], [[C3]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SMAX2]], [[C1]]
+    ; GFX6: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[SHL2]], [[C3]]
+    ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[SMIN2]], [[C2]]
+    ; GFX6: [[SMAX3:%[0-9]+]]:_(s32) = G_SMAX [[SUB3]], [[SHL3]]
+    ; GFX6: [[SMIN3:%[0-9]+]]:_(s32) = G_SMIN [[SMAX3]], [[SUB4]]
+    ; GFX6: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[SHL2]], [[SMIN3]]
+    ; GFX6: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SUB5]], [[C]](s32)
+    ; GFX6: [[COPY5:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; GFX6: [[COPY6:%[0-9]+]]:_(s32) = COPY [[BITCAST3]](s32)
+    ; GFX6: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[COPY5]], [[C]](s32)
+    ; GFX6: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[COPY6]], [[C]](s32)
+    ; GFX6: [[SMAX4:%[0-9]+]]:_(s32) = G_SMAX [[SHL4]], [[C3]]
+    ; GFX6: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SMAX4]], [[C1]]
+    ; GFX6: [[SMIN4:%[0-9]+]]:_(s32) = G_SMIN [[SHL4]], [[C3]]
+    ; GFX6: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[SMIN4]], [[C2]]
+    ; GFX6: [[SMAX5:%[0-9]+]]:_(s32) = G_SMAX [[SUB6]], [[SHL5]]
+    ; GFX6: [[SMIN5:%[0-9]+]]:_(s32) = G_SMIN [[SMAX5]], [[SUB7]]
+    ; GFX6: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SHL4]], [[SMIN5]]
+    ; GFX6: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SUB8]], [[C]](s32)
+    ; GFX6: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX6: [[COPY7:%[0-9]+]]:_(s32) = COPY [[ASHR]](s32)
+    ; GFX6: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C4]]
+    ; GFX6: [[COPY8:%[0-9]+]]:_(s32) = COPY [[ASHR1]](s32)
+    ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY8]], [[C4]]
+    ; GFX6: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX6: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL6]]
+    ; GFX6: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX6: [[COPY9:%[0-9]+]]:_(s32) = COPY [[ASHR2]](s32)
+    ; GFX6: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C4]]
+    ; GFX6: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX6: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C5]], [[C]](s32)
+    ; GFX6: [[OR1:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL7]]
+    ; GFX6: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
+    ; GFX6: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; GFX6: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
+    ; GFX6: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX6: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
+    ; GFX6: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS1]](<6 x s16>)
     ; GFX8-LABEL: name: ssubsat_v3s16
     ; GFX8: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     ; GFX8: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
-    ; GFX8: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UV]](<3 x s16>)
-    ; GFX8: [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UV1]](<3 x s16>)
-    ; GFX8: [[SSUBSAT:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV2]], [[UV5]]
-    ; GFX8: [[SSUBSAT1:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV3]], [[UV6]]
-    ; GFX8: [[SSUBSAT2:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV4]], [[UV7]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[SSUBSAT]](s16), [[SSUBSAT1]](s16), [[SSUBSAT2]](s16)
     ; GFX8: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX8: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX8: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<3 x s16>), [[EXTRACT]](<3 x s16>)
-    ; GFX8: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
+    ; GFX8: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
+    ; GFX8: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX8: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX8: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV1]](<3 x s16>), 0
+    ; GFX8: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; GFX8: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
+    ; GFX8: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
+    ; GFX8: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; GFX8: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+    ; GFX8: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
+    ; GFX8: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+    ; GFX8: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
+    ; GFX8: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
+    ; GFX8: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; GFX8: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC]], [[C3]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SMAX]], [[C1]]
+    ; GFX8: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC]], [[C3]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[SMIN]], [[C2]]
+    ; GFX8: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB]], [[TRUNC3]]
+    ; GFX8: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[TRUNC]], [[SMIN1]]
+    ; GFX8: [[SMAX2:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC1]], [[C3]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s16) = G_SUB [[SMAX2]], [[C1]]
+    ; GFX8: [[SMIN2:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC1]], [[C3]]
+    ; GFX8: [[SUB4:%[0-9]+]]:_(s16) = G_SUB [[SMIN2]], [[C2]]
+    ; GFX8: [[SMAX3:%[0-9]+]]:_(s16) = G_SMAX [[SUB3]], [[TRUNC4]]
+    ; GFX8: [[SMIN3:%[0-9]+]]:_(s16) = G_SMIN [[SMAX3]], [[SUB4]]
+    ; GFX8: [[SUB5:%[0-9]+]]:_(s16) = G_SUB [[TRUNC1]], [[SMIN3]]
+    ; GFX8: [[SMAX4:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC2]], [[C3]]
+    ; GFX8: [[SUB6:%[0-9]+]]:_(s16) = G_SUB [[SMAX4]], [[C1]]
+    ; GFX8: [[SMIN4:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC2]], [[C3]]
+    ; GFX8: [[SUB7:%[0-9]+]]:_(s16) = G_SUB [[SMIN4]], [[C2]]
+    ; GFX8: [[SMAX5:%[0-9]+]]:_(s16) = G_SMAX [[SUB6]], [[TRUNC5]]
+    ; GFX8: [[SMIN5:%[0-9]+]]:_(s16) = G_SMIN [[SMAX5]], [[SUB7]]
+    ; GFX8: [[SUB8:%[0-9]+]]:_(s16) = G_SUB [[TRUNC2]], [[SMIN5]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SUB2]](s16)
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SUB5]](s16)
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
+    ; GFX8: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SUB8]](s16)
+    ; GFX8: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C]](s32)
+    ; GFX8: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
+    ; GFX8: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
+    ; GFX8: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; GFX8: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
+    ; GFX8: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX8: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
+    ; GFX8: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS1]](<6 x s16>)
     ; GFX9-LABEL: name: ssubsat_v3s16
     ; GFX9: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
-    ; GFX9: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UV]](<3 x s16>)
-    ; GFX9: [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UV1]](<3 x s16>)
-    ; GFX9: [[SSUBSAT:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV2]], [[UV5]]
-    ; GFX9: [[SSUBSAT1:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV3]], [[UV6]]
-    ; GFX9: [[SSUBSAT2:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV4]], [[UV7]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[SSUBSAT]](s16), [[SSUBSAT1]](s16), [[SSUBSAT2]](s16)
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<3 x s16>), [[EXTRACT]](<3 x s16>)
-    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
+    ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY1]](s32), [[COPY2]](s32)
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; GFX9: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY3]](s32), [[DEF1]](s32)
+    ; GFX9: [[DEF2:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV1]](<3 x s16>), 0
+    ; GFX9: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; GFX9: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; GFX9: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
+    ; GFX9: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+    ; GFX9: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY4]](s32), [[COPY5]](s32)
+    ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[BITCAST3]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY6]](s32), [[DEF1]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767
+    ; GFX9: [[BUILD_VECTOR_TRUNC4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C1]](s32), [[C1]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -32768
+    ; GFX9: [[BUILD_VECTOR_TRUNC5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C2]](s32), [[C2]](s32)
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX9: [[BUILD_VECTOR_TRUNC6:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C3]](s32), [[C3]](s32)
+    ; GFX9: [[SMAX:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[BUILD_VECTOR_TRUNC]], [[BUILD_VECTOR_TRUNC6]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(<2 x s16>) = G_SUB [[SMAX]], [[BUILD_VECTOR_TRUNC4]]
+    ; GFX9: [[SMIN:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[BUILD_VECTOR_TRUNC]], [[BUILD_VECTOR_TRUNC6]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(<2 x s16>) = G_SUB [[SMIN]], [[BUILD_VECTOR_TRUNC5]]
+    ; GFX9: [[SMAX1:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[SUB]], [[BUILD_VECTOR_TRUNC2]]
+    ; GFX9: [[SMIN1:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC]], [[SMIN1]]
+    ; GFX9: [[BUILD_VECTOR_TRUNC7:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C1]](s32), [[C1]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC8:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C2]](s32), [[C2]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC9:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C3]](s32), [[C3]](s32)
+    ; GFX9: [[SMAX2:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[BUILD_VECTOR_TRUNC1]], [[BUILD_VECTOR_TRUNC9]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(<2 x s16>) = G_SUB [[SMAX2]], [[BUILD_VECTOR_TRUNC7]]
+    ; GFX9: [[SMIN2:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[BUILD_VECTOR_TRUNC1]], [[BUILD_VECTOR_TRUNC9]]
+    ; GFX9: [[SUB4:%[0-9]+]]:_(<2 x s16>) = G_SUB [[SMIN2]], [[BUILD_VECTOR_TRUNC8]]
+    ; GFX9: [[SMAX3:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[SUB3]], [[BUILD_VECTOR_TRUNC3]]
+    ; GFX9: [[SMIN3:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[SMAX3]], [[SUB4]]
+    ; GFX9: [[SUB5:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC1]], [[SMIN3]]
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[SUB2]](<2 x s16>), [[SUB5]](<2 x s16>), [[DEF2]](<2 x s16>)
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<6 x s16>), 0
+    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
+    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS1]](<6 x s16>)
     %0:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     %1:_(<3 x s16>), %2:_(<3 x s16>) = G_UNMERGE_VALUES %0
     %3:_(<3 x s16>) = G_SSUBSAT %1, %2
@@ -359,36 +721,180 @@ body: |
     ; GFX6-LABEL: name: ssubsat_v4s16
     ; GFX6: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
     ; GFX6: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
-    ; GFX6: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
-    ; GFX6: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
-    ; GFX6: [[SSUBSAT:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV]], [[UV4]]
-    ; GFX6: [[SSUBSAT1:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV1]], [[UV5]]
-    ; GFX6: [[SSUBSAT2:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV2]], [[UV6]]
-    ; GFX6: [[SSUBSAT3:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV3]], [[UV7]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[SSUBSAT]](s16), [[SSUBSAT1]](s16), [[SSUBSAT2]](s16), [[SSUBSAT3]](s16)
-    ; GFX6: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; GFX6: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; GFX6: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX6: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX6: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX6: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; GFX6: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX6: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; GFX6: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX6: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX6: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[SHL]], [[C3]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SMAX]], [[C1]]
+    ; GFX6: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[SHL]], [[C3]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SMIN]], [[C2]]
+    ; GFX6: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB]], [[SHL1]]
+    ; GFX6: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SHL]], [[SMIN1]]
+    ; GFX6: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SUB2]], [[C]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX6: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; GFX6: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY4]], [[C]](s32)
+    ; GFX6: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[COPY5]], [[C]](s32)
+    ; GFX6: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[SHL2]], [[C3]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SMAX2]], [[C1]]
+    ; GFX6: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[SHL2]], [[C3]]
+    ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[SMIN2]], [[C2]]
+    ; GFX6: [[SMAX3:%[0-9]+]]:_(s32) = G_SMAX [[SUB3]], [[SHL3]]
+    ; GFX6: [[SMIN3:%[0-9]+]]:_(s32) = G_SMIN [[SMAX3]], [[SUB4]]
+    ; GFX6: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[SHL2]], [[SMIN3]]
+    ; GFX6: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SUB5]], [[C]](s32)
+    ; GFX6: [[COPY6:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; GFX6: [[COPY7:%[0-9]+]]:_(s32) = COPY [[BITCAST3]](s32)
+    ; GFX6: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[COPY6]], [[C]](s32)
+    ; GFX6: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[COPY7]], [[C]](s32)
+    ; GFX6: [[SMAX4:%[0-9]+]]:_(s32) = G_SMAX [[SHL4]], [[C3]]
+    ; GFX6: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SMAX4]], [[C1]]
+    ; GFX6: [[SMIN4:%[0-9]+]]:_(s32) = G_SMIN [[SHL4]], [[C3]]
+    ; GFX6: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[SMIN4]], [[C2]]
+    ; GFX6: [[SMAX5:%[0-9]+]]:_(s32) = G_SMAX [[SUB6]], [[SHL5]]
+    ; GFX6: [[SMIN5:%[0-9]+]]:_(s32) = G_SMIN [[SMAX5]], [[SUB7]]
+    ; GFX6: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SHL4]], [[SMIN5]]
+    ; GFX6: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SUB8]], [[C]](s32)
+    ; GFX6: [[COPY8:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX6: [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; GFX6: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[COPY8]], [[C]](s32)
+    ; GFX6: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[COPY9]], [[C]](s32)
+    ; GFX6: [[SMAX6:%[0-9]+]]:_(s32) = G_SMAX [[SHL6]], [[C3]]
+    ; GFX6: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[SMAX6]], [[C1]]
+    ; GFX6: [[SMIN6:%[0-9]+]]:_(s32) = G_SMIN [[SHL6]], [[C3]]
+    ; GFX6: [[SUB10:%[0-9]+]]:_(s32) = G_SUB [[SMIN6]], [[C2]]
+    ; GFX6: [[SMAX7:%[0-9]+]]:_(s32) = G_SMAX [[SUB9]], [[SHL7]]
+    ; GFX6: [[SMIN7:%[0-9]+]]:_(s32) = G_SMIN [[SMAX7]], [[SUB10]]
+    ; GFX6: [[SUB11:%[0-9]+]]:_(s32) = G_SUB [[SHL6]], [[SMIN7]]
+    ; GFX6: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[SUB11]], [[C]](s32)
+    ; GFX6: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX6: [[COPY10:%[0-9]+]]:_(s32) = COPY [[ASHR]](s32)
+    ; GFX6: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY10]], [[C4]]
+    ; GFX6: [[COPY11:%[0-9]+]]:_(s32) = COPY [[ASHR1]](s32)
+    ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY11]], [[C4]]
+    ; GFX6: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX6: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL8]]
+    ; GFX6: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX6: [[COPY12:%[0-9]+]]:_(s32) = COPY [[ASHR2]](s32)
+    ; GFX6: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY12]], [[C4]]
+    ; GFX6: [[COPY13:%[0-9]+]]:_(s32) = COPY [[ASHR3]](s32)
+    ; GFX6: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY13]], [[C4]]
+    ; GFX6: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
+    ; GFX6: [[OR1:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL9]]
+    ; GFX6: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
+    ; GFX6: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; GFX6: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
     ; GFX8-LABEL: name: ssubsat_v4s16
     ; GFX8: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
     ; GFX8: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
-    ; GFX8: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
-    ; GFX8: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
-    ; GFX8: [[SSUBSAT:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV]], [[UV4]]
-    ; GFX8: [[SSUBSAT1:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV1]], [[UV5]]
-    ; GFX8: [[SSUBSAT2:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV2]], [[UV6]]
-    ; GFX8: [[SSUBSAT3:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV3]], [[UV7]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[SSUBSAT]](s16), [[SSUBSAT1]](s16), [[SSUBSAT2]](s16), [[SSUBSAT3]](s16)
-    ; GFX8: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; GFX8: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX8: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX8: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+    ; GFX8: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; GFX8: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX8: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
+    ; GFX8: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; GFX8: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+    ; GFX8: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX8: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+    ; GFX8: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; GFX8: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767
+    ; GFX8: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 -32768
+    ; GFX8: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; GFX8: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC]], [[C3]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SMAX]], [[C1]]
+    ; GFX8: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC]], [[C3]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[SMIN]], [[C2]]
+    ; GFX8: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[SUB]], [[TRUNC4]]
+    ; GFX8: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[TRUNC]], [[SMIN1]]
+    ; GFX8: [[SMAX2:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC1]], [[C3]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s16) = G_SUB [[SMAX2]], [[C1]]
+    ; GFX8: [[SMIN2:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC1]], [[C3]]
+    ; GFX8: [[SUB4:%[0-9]+]]:_(s16) = G_SUB [[SMIN2]], [[C2]]
+    ; GFX8: [[SMAX3:%[0-9]+]]:_(s16) = G_SMAX [[SUB3]], [[TRUNC5]]
+    ; GFX8: [[SMIN3:%[0-9]+]]:_(s16) = G_SMIN [[SMAX3]], [[SUB4]]
+    ; GFX8: [[SUB5:%[0-9]+]]:_(s16) = G_SUB [[TRUNC1]], [[SMIN3]]
+    ; GFX8: [[SMAX4:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC2]], [[C3]]
+    ; GFX8: [[SUB6:%[0-9]+]]:_(s16) = G_SUB [[SMAX4]], [[C1]]
+    ; GFX8: [[SMIN4:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC2]], [[C3]]
+    ; GFX8: [[SUB7:%[0-9]+]]:_(s16) = G_SUB [[SMIN4]], [[C2]]
+    ; GFX8: [[SMAX5:%[0-9]+]]:_(s16) = G_SMAX [[SUB6]], [[TRUNC6]]
+    ; GFX8: [[SMIN5:%[0-9]+]]:_(s16) = G_SMIN [[SMAX5]], [[SUB7]]
+    ; GFX8: [[SUB8:%[0-9]+]]:_(s16) = G_SUB [[TRUNC2]], [[SMIN5]]
+    ; GFX8: [[SMAX6:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC3]], [[C3]]
+    ; GFX8: [[SUB9:%[0-9]+]]:_(s16) = G_SUB [[SMAX6]], [[C1]]
+    ; GFX8: [[SMIN6:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC3]], [[C3]]
+    ; GFX8: [[SUB10:%[0-9]+]]:_(s16) = G_SUB [[SMIN6]], [[C2]]
+    ; GFX8: [[SMAX7:%[0-9]+]]:_(s16) = G_SMAX [[SUB9]], [[TRUNC7]]
+    ; GFX8: [[SMIN7:%[0-9]+]]:_(s16) = G_SMIN [[SMAX7]], [[SUB10]]
+    ; GFX8: [[SUB11:%[0-9]+]]:_(s16) = G_SUB [[TRUNC3]], [[SMIN7]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SUB2]](s16)
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SUB5]](s16)
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
+    ; GFX8: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SUB8]](s16)
+    ; GFX8: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[SUB11]](s16)
+    ; GFX8: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+    ; GFX8: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
+    ; GFX8: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
+    ; GFX8: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; GFX8: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
     ; GFX9-LABEL: name: ssubsat_v4s16
     ; GFX9: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
-    ; GFX9: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
-    ; GFX9: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
-    ; GFX9: [[SSUBSAT:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV]], [[UV4]]
-    ; GFX9: [[SSUBSAT1:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV1]], [[UV5]]
-    ; GFX9: [[SSUBSAT2:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV2]], [[UV6]]
-    ; GFX9: [[SSUBSAT3:%[0-9]+]]:_(s16) = G_SSUBSAT [[UV3]], [[UV7]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[SSUBSAT]](s16), [[SSUBSAT1]](s16), [[SSUBSAT2]](s16), [[SSUBSAT3]](s16)
-    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767
+    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C]](s32), [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -32768
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C1]](s32), [[C1]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX9: [[BUILD_VECTOR_TRUNC2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C2]](s32), [[C2]](s32)
+    ; GFX9: [[SMAX:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[UV]], [[BUILD_VECTOR_TRUNC2]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(<2 x s16>) = G_SUB [[SMAX]], [[BUILD_VECTOR_TRUNC]]
+    ; GFX9: [[SMIN:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[UV]], [[BUILD_VECTOR_TRUNC2]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(<2 x s16>) = G_SUB [[SMIN]], [[BUILD_VECTOR_TRUNC1]]
+    ; GFX9: [[SMAX1:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[SUB]], [[UV2]]
+    ; GFX9: [[SMIN1:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(<2 x s16>) = G_SUB [[UV]], [[SMIN1]]
+    ; GFX9: [[BUILD_VECTOR_TRUNC3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C]](s32), [[C]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C1]](s32), [[C1]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C2]](s32), [[C2]](s32)
+    ; GFX9: [[SMAX2:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[UV1]], [[BUILD_VECTOR_TRUNC5]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(<2 x s16>) = G_SUB [[SMAX2]], [[BUILD_VECTOR_TRUNC3]]
+    ; GFX9: [[SMIN2:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[UV1]], [[BUILD_VECTOR_TRUNC5]]
+    ; GFX9: [[SUB4:%[0-9]+]]:_(<2 x s16>) = G_SUB [[SMIN2]], [[BUILD_VECTOR_TRUNC4]]
+    ; GFX9: [[SMAX3:%[0-9]+]]:_(<2 x s16>) = G_SMAX [[SUB3]], [[UV3]]
+    ; GFX9: [[SMIN3:%[0-9]+]]:_(<2 x s16>) = G_SMIN [[SMAX3]], [[SUB4]]
+    ; GFX9: [[SUB5:%[0-9]+]]:_(<2 x s16>) = G_SUB [[UV1]], [[SMIN3]]
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[SUB2]](<2 x s16>), [[SUB5]](<2 x s16>)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
     %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
     %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
     %2:_(<4 x s16>) = G_SSUBSAT %0, %1
@@ -404,18 +910,45 @@ body: |
     ; GFX6-LABEL: name: ssubsat_s32
     ; GFX6: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX6: [[SSUBSAT:%[0-9]+]]:_(s32) = G_SSUBSAT [[COPY]], [[COPY1]]
-    ; GFX6: $vgpr0 = COPY [[SSUBSAT]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX6: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[COPY]], [[C2]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SMAX]], [[C]]
+    ; GFX6: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[COPY]], [[C2]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SMIN]], [[C1]]
+    ; GFX6: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB]], [[COPY1]]
+    ; GFX6: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[SMIN1]]
+    ; GFX6: $vgpr0 = COPY [[SUB2]](s32)
     ; GFX8-LABEL: name: ssubsat_s32
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX8: [[SSUBSAT:%[0-9]+]]:_(s32) = G_SSUBSAT [[COPY]], [[COPY1]]
-    ; GFX8: $vgpr0 = COPY [[SSUBSAT]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX8: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[COPY]], [[C2]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SMAX]], [[C]]
+    ; GFX8: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[COPY]], [[C2]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SMIN]], [[C1]]
+    ; GFX8: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB]], [[COPY1]]
+    ; GFX8: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[SMIN1]]
+    ; GFX8: $vgpr0 = COPY [[SUB2]](s32)
     ; GFX9-LABEL: name: ssubsat_s32
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX9: [[SSUBSAT:%[0-9]+]]:_(s32) = G_SSUBSAT [[COPY]], [[COPY1]]
-    ; GFX9: $vgpr0 = COPY [[SSUBSAT]](s32)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX9: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[COPY]], [[C2]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SMAX]], [[C]]
+    ; GFX9: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[COPY]], [[C2]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SMIN]], [[C1]]
+    ; GFX9: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB]], [[COPY1]]
+    ; GFX9: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[SMIN1]]
+    ; GFX9: $vgpr0 = COPY [[SUB2]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = G_SSUBSAT %0, %1
@@ -433,27 +966,72 @@ body: |
     ; GFX6: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
     ; GFX6: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
     ; GFX6: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
-    ; GFX6: [[SSUBSAT:%[0-9]+]]:_(s32) = G_SSUBSAT [[UV]], [[UV2]]
-    ; GFX6: [[SSUBSAT1:%[0-9]+]]:_(s32) = G_SSUBSAT [[UV1]], [[UV3]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SSUBSAT]](s32), [[SSUBSAT1]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX6: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[UV]], [[C2]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SMAX]], [[C]]
+    ; GFX6: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[UV]], [[C2]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SMIN]], [[C1]]
+    ; GFX6: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB]], [[UV2]]
+    ; GFX6: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[UV]], [[SMIN1]]
+    ; GFX6: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[UV1]], [[C2]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SMAX2]], [[C]]
+    ; GFX6: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[UV1]], [[C2]]
+    ; GFX6: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[SMIN2]], [[C1]]
+    ; GFX6: [[SMAX3:%[0-9]+]]:_(s32) = G_SMAX [[SUB3]], [[UV3]]
+    ; GFX6: [[SMIN3:%[0-9]+]]:_(s32) = G_SMIN [[SMAX3]], [[SUB4]]
+    ; GFX6: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[UV1]], [[SMIN3]]
+    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SUB2]](s32), [[SUB5]](s32)
     ; GFX6: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ; GFX8-LABEL: name: ssubsat_v2s32
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
     ; GFX8: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
     ; GFX8: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
-    ; GFX8: [[SSUBSAT:%[0-9]+]]:_(s32) = G_SSUBSAT [[UV]], [[UV2]]
-    ; GFX8: [[SSUBSAT1:%[0-9]+]]:_(s32) = G_SSUBSAT [[UV1]], [[UV3]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SSUBSAT]](s32), [[SSUBSAT1]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX8: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[UV]], [[C2]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SMAX]], [[C]]
+    ; GFX8: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[UV]], [[C2]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SMIN]], [[C1]]
+    ; GFX8: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB]], [[UV2]]
+    ; GFX8: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[UV]], [[SMIN1]]
+    ; GFX8: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[UV1]], [[C2]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SMAX2]], [[C]]
+    ; GFX8: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[UV1]], [[C2]]
+    ; GFX8: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[SMIN2]], [[C1]]
+    ; GFX8: [[SMAX3:%[0-9]+]]:_(s32) = G_SMAX [[SUB3]], [[UV3]]
+    ; GFX8: [[SMIN3:%[0-9]+]]:_(s32) = G_SMIN [[SMAX3]], [[SUB4]]
+    ; GFX8: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[UV1]], [[SMIN3]]
+    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SUB2]](s32), [[SUB5]](s32)
     ; GFX8: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ; GFX9-LABEL: name: ssubsat_v2s32
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
     ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
     ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
-    ; GFX9: [[SSUBSAT:%[0-9]+]]:_(s32) = G_SSUBSAT [[UV]], [[UV2]]
-    ; GFX9: [[SSUBSAT1:%[0-9]+]]:_(s32) = G_SSUBSAT [[UV1]], [[UV3]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SSUBSAT]](s32), [[SSUBSAT1]](s32)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2147483647
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -2147483648
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX9: [[SMAX:%[0-9]+]]:_(s32) = G_SMAX [[UV]], [[C2]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SMAX]], [[C]]
+    ; GFX9: [[SMIN:%[0-9]+]]:_(s32) = G_SMIN [[UV]], [[C2]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SMIN]], [[C1]]
+    ; GFX9: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SUB]], [[UV2]]
+    ; GFX9: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SMAX1]], [[SUB1]]
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[UV]], [[SMIN1]]
+    ; GFX9: [[SMAX2:%[0-9]+]]:_(s32) = G_SMAX [[UV1]], [[C2]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SMAX2]], [[C]]
+    ; GFX9: [[SMIN2:%[0-9]+]]:_(s32) = G_SMIN [[UV1]], [[C2]]
+    ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[SMIN2]], [[C1]]
+    ; GFX9: [[SMAX3:%[0-9]+]]:_(s32) = G_SMAX [[SUB3]], [[UV3]]
+    ; GFX9: [[SMIN3:%[0-9]+]]:_(s32) = G_SMIN [[SMAX3]], [[SUB4]]
+    ; GFX9: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[UV1]], [[SMIN3]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SUB2]](s32), [[SUB5]](s32)
     ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
     %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
@@ -470,18 +1048,69 @@ body: |
     ; GFX6-LABEL: name: ssubsat_s64
     ; GFX6: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX6: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX6: [[SSUBSAT:%[0-9]+]]:_(s64) = G_SSUBSAT [[COPY]], [[COPY1]]
-    ; GFX6: $vgpr0_vgpr1 = COPY [[SSUBSAT]](s64)
+    ; GFX6: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX6: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX6: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV]], [[UV2]]
+    ; GFX6: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV1]], [[UV3]], [[USUBO1]]
+    ; GFX6: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO]](s32), [[USUBE]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV]](s64), [[COPY]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[COPY1]](s64), [[C]]
+    ; GFX6: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]]
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX6: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MV]], [[C1]](s32)
+    ; GFX6: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -9223372036854775808
+    ; GFX6: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX6: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64)
+    ; GFX6: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX6: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO1]]
+    ; GFX6: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[MV]]
+    ; GFX6: $vgpr0_vgpr1 = COPY [[SELECT]](s64)
     ; GFX8-LABEL: name: ssubsat_s64
     ; GFX8: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX8: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX8: [[SSUBSAT:%[0-9]+]]:_(s64) = G_SSUBSAT [[COPY]], [[COPY1]]
-    ; GFX8: $vgpr0_vgpr1 = COPY [[SSUBSAT]](s64)
+    ; GFX8: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX8: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX8: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV]], [[UV2]]
+    ; GFX8: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV1]], [[UV3]], [[USUBO1]]
+    ; GFX8: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO]](s32), [[USUBE]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV]](s64), [[COPY]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[COPY1]](s64), [[C]]
+    ; GFX8: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]]
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX8: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MV]], [[C1]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -9223372036854775808
+    ; GFX8: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX8: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64)
+    ; GFX8: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX8: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO1]]
+    ; GFX8: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[MV]]
+    ; GFX8: $vgpr0_vgpr1 = COPY [[SELECT]](s64)
     ; GFX9-LABEL: name: ssubsat_s64
     ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX9: [[SSUBSAT:%[0-9]+]]:_(s64) = G_SSUBSAT [[COPY]], [[COPY1]]
-    ; GFX9: $vgpr0_vgpr1 = COPY [[SSUBSAT]](s64)
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX9: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV]], [[UV2]]
+    ; GFX9: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV1]], [[UV3]], [[USUBO1]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO]](s32), [[USUBE]](s32)
+    ; GFX9: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV]](s64), [[COPY]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[COPY1]](s64), [[C]]
+    ; GFX9: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MV]], [[C1]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -9223372036854775808
+    ; GFX9: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX9: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64)
+    ; GFX9: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX9: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO1]]
+    ; GFX9: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[MV]]
+    ; GFX9: $vgpr0_vgpr1 = COPY [[SELECT]](s64)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s64) = COPY $vgpr2_vgpr3
     %2:_(s64) = G_SSUBSAT %0, %1
@@ -499,27 +1128,120 @@ body: |
     ; GFX6: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX6: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
     ; GFX6: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
-    ; GFX6: [[SSUBSAT:%[0-9]+]]:_(s64) = G_SSUBSAT [[UV]], [[UV2]]
-    ; GFX6: [[SSUBSAT1:%[0-9]+]]:_(s64) = G_SSUBSAT [[UV1]], [[UV3]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SSUBSAT]](s64), [[SSUBSAT1]](s64)
+    ; GFX6: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX6: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX6: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV4]], [[UV6]]
+    ; GFX6: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV5]], [[UV7]], [[USUBO1]]
+    ; GFX6: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO]](s32), [[USUBE]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV]](s64), [[UV]]
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[UV2]](s64), [[C]]
+    ; GFX6: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]]
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX6: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MV]], [[C1]](s32)
+    ; GFX6: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -9223372036854775808
+    ; GFX6: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX6: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64)
+    ; GFX6: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV8]], [[UV10]]
+    ; GFX6: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV9]], [[UV11]], [[UADDO1]]
+    ; GFX6: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[MV]]
+    ; GFX6: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX6: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX6: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV12]], [[UV14]]
+    ; GFX6: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV13]], [[UV15]], [[USUBO3]]
+    ; GFX6: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO2]](s32), [[USUBE2]](s32)
+    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV2]](s64), [[UV1]]
+    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[UV3]](s64), [[C]]
+    ; GFX6: [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[ICMP3]], [[ICMP2]]
+    ; GFX6: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[MV2]], [[C1]](s32)
+    ; GFX6: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64)
+    ; GFX6: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64)
+    ; GFX6: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV16]], [[UV18]]
+    ; GFX6: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV17]], [[UV19]], [[UADDO3]]
+    ; GFX6: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[XOR1]](s1), [[MV3]], [[MV2]]
+    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT]](s64), [[SELECT1]](s64)
     ; GFX6: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     ; GFX8-LABEL: name: ssubsat_v2s64
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX8: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
     ; GFX8: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
-    ; GFX8: [[SSUBSAT:%[0-9]+]]:_(s64) = G_SSUBSAT [[UV]], [[UV2]]
-    ; GFX8: [[SSUBSAT1:%[0-9]+]]:_(s64) = G_SSUBSAT [[UV1]], [[UV3]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SSUBSAT]](s64), [[SSUBSAT1]](s64)
+    ; GFX8: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX8: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX8: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV4]], [[UV6]]
+    ; GFX8: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV5]], [[UV7]], [[USUBO1]]
+    ; GFX8: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO]](s32), [[USUBE]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV]](s64), [[UV]]
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[UV2]](s64), [[C]]
+    ; GFX8: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]]
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX8: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MV]], [[C1]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -9223372036854775808
+    ; GFX8: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX8: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64)
+    ; GFX8: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV8]], [[UV10]]
+    ; GFX8: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV9]], [[UV11]], [[UADDO1]]
+    ; GFX8: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[MV]]
+    ; GFX8: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX8: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX8: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV12]], [[UV14]]
+    ; GFX8: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV13]], [[UV15]], [[USUBO3]]
+    ; GFX8: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO2]](s32), [[USUBE2]](s32)
+    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV2]](s64), [[UV1]]
+    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[UV3]](s64), [[C]]
+    ; GFX8: [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[ICMP3]], [[ICMP2]]
+    ; GFX8: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[MV2]], [[C1]](s32)
+    ; GFX8: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64)
+    ; GFX8: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64)
+    ; GFX8: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV16]], [[UV18]]
+    ; GFX8: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV17]], [[UV19]], [[UADDO3]]
+    ; GFX8: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[XOR1]](s1), [[MV3]], [[MV2]]
+    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT]](s64), [[SELECT1]](s64)
     ; GFX8: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     ; GFX9-LABEL: name: ssubsat_v2s64
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
     ; GFX9: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
-    ; GFX9: [[SSUBSAT:%[0-9]+]]:_(s64) = G_SSUBSAT [[UV]], [[UV2]]
-    ; GFX9: [[SSUBSAT1:%[0-9]+]]:_(s64) = G_SSUBSAT [[UV1]], [[UV3]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SSUBSAT]](s64), [[SSUBSAT1]](s64)
+    ; GFX9: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX9: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX9: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV4]], [[UV6]]
+    ; GFX9: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV5]], [[UV7]], [[USUBO1]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO]](s32), [[USUBE]](s32)
+    ; GFX9: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV]](s64), [[UV]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[UV2]](s64), [[C]]
+    ; GFX9: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MV]], [[C1]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -9223372036854775808
+    ; GFX9: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR]](s64)
+    ; GFX9: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64)
+    ; GFX9: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV8]], [[UV10]]
+    ; GFX9: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV9]], [[UV11]], [[UADDO1]]
+    ; GFX9: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[XOR]](s1), [[MV1]], [[MV]]
+    ; GFX9: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX9: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX9: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV12]], [[UV14]]
+    ; GFX9: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV13]], [[UV15]], [[USUBO3]]
+    ; GFX9: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO2]](s32), [[USUBE2]](s32)
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[MV2]](s64), [[UV1]]
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[UV3]](s64), [[C]]
+    ; GFX9: [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[ICMP3]], [[ICMP2]]
+    ; GFX9: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[MV2]], [[C1]](s32)
+    ; GFX9: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ASHR1]](s64)
+    ; GFX9: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C2]](s64)
+    ; GFX9: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV16]], [[UV18]]
+    ; GFX9: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV17]], [[UV19]], [[UADDO3]]
+    ; GFX9: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[XOR1]](s1), [[MV3]], [[MV2]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT]](s64), [[SELECT1]](s64)
     ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     %1:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddsat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddsat.mir
index e080bde81b3a5..690bf34482ddd 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddsat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddsat.mir
@@ -12,51 +12,48 @@ body: |
     ; GFX6-LABEL: name: uaddsat_s7
     ; GFX6: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX6: [[TRUNC:%[0-9]+]]:_(s7) = G_TRUNC [[COPY]](s32)
-    ; GFX6: [[TRUNC1:%[0-9]+]]:_(s7) = G_TRUNC [[COPY1]](s32)
-    ; GFX6: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC]](s7)
-    ; GFX6: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC1]](s7)
-    ; GFX6: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 9
-    ; GFX6: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX6: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX6: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[SHL]], [[SHL1]]
-    ; GFX6: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[C]](s16)
-    ; GFX6: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT]](s16)
-    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[ZEXT1]], [[ZEXT]](s32)
-    ; GFX6: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; GFX6: [[TRUNC3:%[0-9]+]]:_(s7) = G_TRUNC [[TRUNC2]](s16)
-    ; GFX6: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC3]](s7)
-    ; GFX6: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 25
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX6: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[SHL]], [[C1]]
+    ; GFX6: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[XOR]], [[SHL1]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SHL]], [[UMIN]]
+    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX6: $vgpr0 = COPY [[COPY4]](s32)
     ; GFX8-LABEL: name: uaddsat_s7
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX8: [[TRUNC:%[0-9]+]]:_(s7) = G_TRUNC [[COPY]](s32)
-    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s7) = G_TRUNC [[COPY1]](s32)
-    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC]](s7)
-    ; GFX8: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC1]](s7)
+    ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
     ; GFX8: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 9
-    ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX8: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[SHL]], [[SHL1]]
-    ; GFX8: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[UADDSAT]], [[C]](s16)
-    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s7) = G_TRUNC [[LSHR]](s16)
-    ; GFX8: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC2]](s7)
-    ; GFX8: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
+    ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
+    ; GFX8: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; GFX8: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[SHL]], [[C1]]
+    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[XOR]], [[SHL1]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[SHL]], [[UMIN]]
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[ADD]], [[C]](s16)
+    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
+    ; GFX8: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: uaddsat_s7
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX9: [[TRUNC:%[0-9]+]]:_(s7) = G_TRUNC [[COPY]](s32)
-    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s7) = G_TRUNC [[COPY1]](s32)
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC]](s7)
-    ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC1]](s7)
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
     ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 9
-    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX9: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[SHL]], [[SHL1]]
-    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[UADDSAT]], [[C]](s16)
-    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s7) = G_TRUNC [[LSHR]](s16)
-    ; GFX9: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC2]](s7)
-    ; GFX9: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
+    ; GFX9: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; GFX9: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[SHL]], [[C1]]
+    ; GFX9: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[XOR]], [[SHL1]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[SHL]], [[UMIN]]
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[ADD]], [[C]](s16)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s7) = G_TRUNC %0
@@ -75,51 +72,48 @@ body: |
     ; GFX6-LABEL: name: uaddsat_s8
     ; GFX6: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX6: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
-    ; GFX6: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32)
-    ; GFX6: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC]](s8)
-    ; GFX6: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC1]](s8)
-    ; GFX6: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX6: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX6: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX6: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[SHL]], [[SHL1]]
-    ; GFX6: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[C]](s16)
-    ; GFX6: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT]](s16)
-    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[ZEXT1]], [[ZEXT]](s32)
-    ; GFX6: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; GFX6: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC2]](s16)
-    ; GFX6: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC3]](s8)
-    ; GFX6: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX6: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[SHL]], [[C1]]
+    ; GFX6: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[XOR]], [[SHL1]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SHL]], [[UMIN]]
+    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX6: $vgpr0 = COPY [[COPY4]](s32)
     ; GFX8-LABEL: name: uaddsat_s8
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX8: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
-    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32)
-    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC]](s8)
-    ; GFX8: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC1]](s8)
+    ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
     ; GFX8: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX8: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[SHL]], [[SHL1]]
-    ; GFX8: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[UADDSAT]], [[C]](s16)
-    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR]](s16)
-    ; GFX8: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC2]](s8)
-    ; GFX8: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
+    ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
+    ; GFX8: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; GFX8: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[SHL]], [[C1]]
+    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[XOR]], [[SHL1]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[SHL]], [[UMIN]]
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[ADD]], [[C]](s16)
+    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
+    ; GFX8: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: uaddsat_s8
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX9: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
-    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32)
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC]](s8)
-    ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC1]](s8)
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
     ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX9: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[SHL]], [[SHL1]]
-    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[UADDSAT]], [[C]](s16)
-    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR]](s16)
-    ; GFX9: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC2]](s8)
-    ; GFX9: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
+    ; GFX9: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; GFX9: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[SHL]], [[C1]]
+    ; GFX9: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[XOR]], [[SHL1]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[SHL]], [[UMIN]]
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[ADD]], [[C]](s16)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s8) = G_TRUNC %0
@@ -138,75 +132,122 @@ body: |
     ; GFX6-LABEL: name: uaddsat_v2s8
     ; GFX6: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX6: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
-    ; GFX6: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX6: [[BITCAST:%[0-9]+]]:_(<2 x s8>) = G_BITCAST [[TRUNC]](s16)
-    ; GFX6: [[BITCAST1:%[0-9]+]]:_(<2 x s8>) = G_BITCAST [[TRUNC1]](s16)
-    ; GFX6: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BITCAST]](<2 x s8>)
-    ; GFX6: [[UV2:%[0-9]+]]:_(s8), [[UV3:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BITCAST1]](<2 x s8>)
-    ; GFX6: [[UADDSAT:%[0-9]+]]:_(s8) = G_UADDSAT [[UV]], [[UV2]]
-    ; GFX6: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[UV1]](s8)
-    ; GFX6: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[UV3]](s8)
-    ; GFX6: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX6: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX6: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX6: [[UADDSAT1:%[0-9]+]]:_(s16) = G_UADDSAT [[SHL]], [[SHL1]]
-    ; GFX6: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[C]](s16)
-    ; GFX6: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT1]](s16)
-    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[ZEXT1]], [[ZEXT]](s32)
-    ; GFX6: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; GFX6: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC2]](s16)
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s8>) = G_BUILD_VECTOR [[UADDSAT]](s8), [[TRUNC3]](s8)
-    ; GFX6: [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<2 x s8>)
-    ; GFX6: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV4]](s8), [[UV5]](s8)
-    ; GFX6: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
-    ; GFX6: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX6: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX6: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; GFX6: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
+    ; GFX6: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C1]](s32)
+    ; GFX6: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C2]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C2]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C2]](s32)
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX6: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[SHL]], [[C3]]
+    ; GFX6: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[XOR]], [[SHL1]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SHL]], [[UMIN]]
+    ; GFX6: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C2]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX6: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; GFX6: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY4]], [[C2]](s32)
+    ; GFX6: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[COPY5]], [[C2]](s32)
+    ; GFX6: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[SHL2]], [[C3]]
+    ; GFX6: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[XOR1]], [[SHL3]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SHL2]], [[UMIN1]]
+    ; GFX6: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C2]](s32)
+    ; GFX6: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX6: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR6]](s32)
+    ; GFX6: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C4]]
+    ; GFX6: [[COPY6:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; GFX6: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; GFX6: [[COPY7:%[0-9]+]]:_(s32) = COPY [[LSHR7]](s32)
+    ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C5]]
+    ; GFX6: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY6]](s32)
+    ; GFX6: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL4]](s32)
+    ; GFX6: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; GFX6: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; GFX6: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX8-LABEL: name: uaddsat_v2s8
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX8: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; GFX8: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
+    ; GFX8: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C1]](s32)
+    ; GFX8: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C2]](s32)
     ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
     ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX8: [[BITCAST:%[0-9]+]]:_(<2 x s8>) = G_BITCAST [[TRUNC]](s16)
-    ; GFX8: [[BITCAST1:%[0-9]+]]:_(<2 x s8>) = G_BITCAST [[TRUNC1]](s16)
-    ; GFX8: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BITCAST]](<2 x s8>)
-    ; GFX8: [[UV2:%[0-9]+]]:_(s8), [[UV3:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BITCAST1]](<2 x s8>)
-    ; GFX8: [[UADDSAT:%[0-9]+]]:_(s8) = G_UADDSAT [[UV]], [[UV2]]
-    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[UV1]](s8)
-    ; GFX8: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[UV3]](s8)
-    ; GFX8: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX8: [[UADDSAT1:%[0-9]+]]:_(s16) = G_UADDSAT [[SHL]], [[SHL1]]
-    ; GFX8: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[UADDSAT1]], [[C]](s16)
-    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR]](s16)
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s8>) = G_BUILD_VECTOR [[UADDSAT]](s8), [[TRUNC2]](s8)
-    ; GFX8: [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<2 x s8>)
-    ; GFX8: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV4]](s8), [[UV5]](s8)
-    ; GFX8: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
-    ; GFX8: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX8: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C3]](s16)
+    ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C3]](s16)
+    ; GFX8: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; GFX8: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[SHL]], [[C4]]
+    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[XOR]], [[SHL1]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[SHL]], [[UMIN]]
+    ; GFX8: [[LSHR6:%[0-9]+]]:_(s16) = G_LSHR [[ADD]], [[C3]](s16)
+    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX8: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+    ; GFX8: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[C3]](s16)
+    ; GFX8: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[TRUNC3]], [[C3]](s16)
+    ; GFX8: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[SHL2]], [[C4]]
+    ; GFX8: [[UMIN1:%[0-9]+]]:_(s16) = G_UMIN [[XOR1]], [[SHL3]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[SHL2]], [[UMIN1]]
+    ; GFX8: [[LSHR7:%[0-9]+]]:_(s16) = G_LSHR [[ADD1]], [[C3]](s16)
+    ; GFX8: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s16) = COPY [[LSHR6]](s16)
+    ; GFX8: [[AND:%[0-9]+]]:_(s16) = G_AND [[COPY2]], [[C5]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s16) = COPY [[LSHR7]](s16)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s16) = G_AND [[COPY3]], [[C5]]
+    ; GFX8: [[SHL4:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C3]](s16)
+    ; GFX8: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL4]]
+    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; GFX8: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: uaddsat_v2s8
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; GFX9: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
+    ; GFX9: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C1]](s32)
+    ; GFX9: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C2]](s32)
     ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
     ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX9: [[BITCAST:%[0-9]+]]:_(<2 x s8>) = G_BITCAST [[TRUNC]](s16)
-    ; GFX9: [[BITCAST1:%[0-9]+]]:_(<2 x s8>) = G_BITCAST [[TRUNC1]](s16)
-    ; GFX9: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BITCAST]](<2 x s8>)
-    ; GFX9: [[UV2:%[0-9]+]]:_(s8), [[UV3:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BITCAST1]](<2 x s8>)
-    ; GFX9: [[UADDSAT:%[0-9]+]]:_(s8) = G_UADDSAT [[UV]], [[UV2]]
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[UV1]](s8)
-    ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[UV3]](s8)
-    ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX9: [[UADDSAT1:%[0-9]+]]:_(s16) = G_UADDSAT [[SHL]], [[SHL1]]
-    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[UADDSAT1]], [[C]](s16)
-    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR]](s16)
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s8>) = G_BUILD_VECTOR [[UADDSAT]](s8), [[TRUNC2]](s8)
-    ; GFX9: [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<2 x s8>)
-    ; GFX9: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV4]](s8), [[UV5]](s8)
-    ; GFX9: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
-    ; GFX9: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX9: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C3]](s16)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C3]](s16)
+    ; GFX9: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; GFX9: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[SHL]], [[C4]]
+    ; GFX9: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[XOR]], [[SHL1]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[SHL]], [[UMIN]]
+    ; GFX9: [[LSHR6:%[0-9]+]]:_(s16) = G_LSHR [[ADD]], [[C3]](s16)
+    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX9: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+    ; GFX9: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[C3]](s16)
+    ; GFX9: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[TRUNC3]], [[C3]](s16)
+    ; GFX9: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[SHL2]], [[C4]]
+    ; GFX9: [[UMIN1:%[0-9]+]]:_(s16) = G_UMIN [[XOR1]], [[SHL3]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[SHL2]], [[UMIN1]]
+    ; GFX9: [[LSHR7:%[0-9]+]]:_(s16) = G_LSHR [[ADD1]], [[C3]](s16)
+    ; GFX9: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s16) = COPY [[LSHR6]](s16)
+    ; GFX9: [[AND:%[0-9]+]]:_(s16) = G_AND [[COPY2]], [[C5]]
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s16) = COPY [[LSHR7]](s16)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s16) = G_AND [[COPY3]], [[C5]]
+    ; GFX9: [[SHL4:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C3]](s16)
+    ; GFX9: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL4]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s16) = G_TRUNC %0
@@ -228,26 +269,39 @@ body: |
     ; GFX6-LABEL: name: uaddsat_s16
     ; GFX6: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX6: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
-    ; GFX6: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX6: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC]], [[TRUNC1]]
-    ; GFX6: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UADDSAT]](s16)
-    ; GFX6: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX6: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[SHL]], [[C1]]
+    ; GFX6: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[XOR]], [[SHL1]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SHL]], [[UMIN]]
+    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX6: $vgpr0 = COPY [[COPY4]](s32)
     ; GFX8-LABEL: name: uaddsat_s16
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
     ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX8: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC]], [[TRUNC1]]
-    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UADDSAT]](s16)
+    ; GFX8: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; GFX8: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[TRUNC]], [[C]]
+    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[XOR]], [[TRUNC1]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[TRUNC]], [[UMIN]]
+    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ADD]](s16)
     ; GFX8: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: uaddsat_s16
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
     ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX9: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC]], [[TRUNC1]]
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UADDSAT]](s16)
+    ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; GFX9: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[TRUNC]], [[C]]
+    ; GFX9: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[XOR]], [[TRUNC1]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[TRUNC]], [[UMIN]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ADD]](s16)
     ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
@@ -267,30 +321,71 @@ body: |
     ; GFX6-LABEL: name: uaddsat_v2s16
     ; GFX6: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
-    ; GFX6: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
-    ; GFX6: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
-    ; GFX6: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[UV]], [[UV2]]
-    ; GFX6: [[UADDSAT1:%[0-9]+]]:_(s16) = G_UADDSAT [[UV1]], [[UV3]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[UADDSAT]](s16), [[UADDSAT1]](s16)
-    ; GFX6: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX6: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX6: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; GFX6: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX6: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[SHL]], [[C1]]
+    ; GFX6: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[XOR]], [[SHL1]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SHL]], [[UMIN]]
+    ; GFX6: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX6: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX6: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY4]], [[C]](s32)
+    ; GFX6: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[COPY5]], [[C]](s32)
+    ; GFX6: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[SHL2]], [[C1]]
+    ; GFX6: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[XOR1]], [[SHL3]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SHL2]], [[UMIN1]]
+    ; GFX6: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C]](s32)
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX6: [[COPY6:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; GFX6: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C2]]
+    ; GFX6: [[COPY7:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C2]]
+    ; GFX6: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX6: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL4]]
+    ; GFX6: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX6: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
     ; GFX8-LABEL: name: uaddsat_v2s16
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
-    ; GFX8: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
-    ; GFX8: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
-    ; GFX8: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[UV]], [[UV2]]
-    ; GFX8: [[UADDSAT1:%[0-9]+]]:_(s16) = G_UADDSAT [[UV1]], [[UV3]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[UADDSAT]](s16), [[UADDSAT1]](s16)
-    ; GFX8: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX8: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX8: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; GFX8: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[TRUNC]], [[C1]]
+    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[XOR]], [[TRUNC2]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[TRUNC]], [[UMIN]]
+    ; GFX8: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[TRUNC1]], [[C1]]
+    ; GFX8: [[UMIN1:%[0-9]+]]:_(s16) = G_UMIN [[XOR1]], [[TRUNC3]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[TRUNC1]], [[UMIN1]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
+    ; GFX8: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX8: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
     ; GFX9-LABEL: name: uaddsat_v2s16
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
-    ; GFX9: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
-    ; GFX9: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
-    ; GFX9: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[UV]], [[UV2]]
-    ; GFX9: [[UADDSAT1:%[0-9]+]]:_(s16) = G_UADDSAT [[UV1]], [[UV3]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[UADDSAT]](s16), [[UADDSAT1]](s16)
-    ; GFX9: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C]](s32), [[C]](s32)
+    ; GFX9: [[XOR:%[0-9]+]]:_(<2 x s16>) = G_XOR [[COPY]], [[BUILD_VECTOR_TRUNC]]
+    ; GFX9: [[UMIN:%[0-9]+]]:_(<2 x s16>) = G_UMIN [[XOR]], [[COPY1]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(<2 x s16>) = G_ADD [[COPY]], [[UMIN]]
+    ; GFX9: $vgpr0 = COPY [[ADD]](<2 x s16>)
     %0:_(<2 x s16>) = COPY $vgpr0
     %1:_(<2 x s16>) = COPY $vgpr1
     %2:_(<2 x s16>) = G_UADDSAT %0, %1
@@ -306,42 +401,155 @@ body: |
     ; GFX6-LABEL: name: uaddsat_v3s16
     ; GFX6: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     ; GFX6: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
-    ; GFX6: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UV]](<3 x s16>)
-    ; GFX6: [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UV1]](<3 x s16>)
-    ; GFX6: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[UV2]], [[UV5]]
-    ; GFX6: [[UADDSAT1:%[0-9]+]]:_(s16) = G_UADDSAT [[UV3]], [[UV6]]
-    ; GFX6: [[UADDSAT2:%[0-9]+]]:_(s16) = G_UADDSAT [[UV4]], [[UV7]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[UADDSAT]](s16), [[UADDSAT1]](s16), [[UADDSAT2]](s16)
     ; GFX6: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX6: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX6: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<3 x s16>), [[EXTRACT]](<3 x s16>)
-    ; GFX6: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
+    ; GFX6: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
+    ; GFX6: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX6: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX6: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX6: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX6: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV1]](<3 x s16>), 0
+    ; GFX6: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; GFX6: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
+    ; GFX6: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; GFX6: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
+    ; GFX6: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX6: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[SHL]], [[C1]]
+    ; GFX6: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[XOR]], [[SHL1]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SHL]], [[UMIN]]
+    ; GFX6: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; GFX6: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+    ; GFX6: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[COPY4]], [[C]](s32)
+    ; GFX6: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[SHL2]], [[C1]]
+    ; GFX6: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[XOR1]], [[SHL3]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SHL2]], [[UMIN1]]
+    ; GFX6: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C]](s32)
+    ; GFX6: [[COPY5:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; GFX6: [[COPY6:%[0-9]+]]:_(s32) = COPY [[BITCAST3]](s32)
+    ; GFX6: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[COPY5]], [[C]](s32)
+    ; GFX6: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[COPY6]], [[C]](s32)
+    ; GFX6: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SHL4]], [[C1]]
+    ; GFX6: [[UMIN2:%[0-9]+]]:_(s32) = G_UMIN [[XOR2]], [[SHL5]]
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SHL4]], [[UMIN2]]
+    ; GFX6: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[ADD2]], [[C]](s32)
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX6: [[COPY7:%[0-9]+]]:_(s32) = COPY [[LSHR4]](s32)
+    ; GFX6: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C2]]
+    ; GFX6: [[COPY8:%[0-9]+]]:_(s32) = COPY [[LSHR5]](s32)
+    ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY8]], [[C2]]
+    ; GFX6: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX6: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL6]]
+    ; GFX6: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX6: [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR6]](s32)
+    ; GFX6: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C2]]
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX6: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C]](s32)
+    ; GFX6: [[OR1:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL7]]
+    ; GFX6: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
+    ; GFX6: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; GFX6: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
+    ; GFX6: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX6: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
+    ; GFX6: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS1]](<6 x s16>)
     ; GFX8-LABEL: name: uaddsat_v3s16
     ; GFX8: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     ; GFX8: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
-    ; GFX8: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UV]](<3 x s16>)
-    ; GFX8: [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UV1]](<3 x s16>)
-    ; GFX8: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[UV2]], [[UV5]]
-    ; GFX8: [[UADDSAT1:%[0-9]+]]:_(s16) = G_UADDSAT [[UV3]], [[UV6]]
-    ; GFX8: [[UADDSAT2:%[0-9]+]]:_(s16) = G_UADDSAT [[UV4]], [[UV7]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[UADDSAT]](s16), [[UADDSAT1]](s16), [[UADDSAT2]](s16)
     ; GFX8: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX8: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX8: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<3 x s16>), [[EXTRACT]](<3 x s16>)
-    ; GFX8: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
+    ; GFX8: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
+    ; GFX8: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX8: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX8: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV1]](<3 x s16>), 0
+    ; GFX8: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; GFX8: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
+    ; GFX8: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
+    ; GFX8: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; GFX8: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+    ; GFX8: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
+    ; GFX8: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+    ; GFX8: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; GFX8: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[TRUNC]], [[C1]]
+    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[XOR]], [[TRUNC3]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[TRUNC]], [[UMIN]]
+    ; GFX8: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[TRUNC1]], [[C1]]
+    ; GFX8: [[UMIN1:%[0-9]+]]:_(s16) = G_UMIN [[XOR1]], [[TRUNC4]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[TRUNC1]], [[UMIN1]]
+    ; GFX8: [[XOR2:%[0-9]+]]:_(s16) = G_XOR [[TRUNC2]], [[C1]]
+    ; GFX8: [[UMIN2:%[0-9]+]]:_(s16) = G_UMIN [[XOR2]], [[TRUNC5]]
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s16) = G_ADD [[TRUNC2]], [[UMIN2]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
+    ; GFX8: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ADD2]](s16)
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
+    ; GFX8: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
+    ; GFX8: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
+    ; GFX8: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; GFX8: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
+    ; GFX8: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX8: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
+    ; GFX8: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS1]](<6 x s16>)
     ; GFX9-LABEL: name: uaddsat_v3s16
     ; GFX9: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
-    ; GFX9: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UV]](<3 x s16>)
-    ; GFX9: [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UV1]](<3 x s16>)
-    ; GFX9: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[UV2]], [[UV5]]
-    ; GFX9: [[UADDSAT1:%[0-9]+]]:_(s16) = G_UADDSAT [[UV3]], [[UV6]]
-    ; GFX9: [[UADDSAT2:%[0-9]+]]:_(s16) = G_UADDSAT [[UV4]], [[UV7]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[UADDSAT]](s16), [[UADDSAT1]](s16), [[UADDSAT2]](s16)
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<3 x s16>), [[EXTRACT]](<3 x s16>)
-    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
+    ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY1]](s32), [[COPY2]](s32)
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; GFX9: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY3]](s32), [[DEF1]](s32)
+    ; GFX9: [[DEF2:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV1]](<3 x s16>), 0
+    ; GFX9: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; GFX9: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; GFX9: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
+    ; GFX9: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+    ; GFX9: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY4]](s32), [[COPY5]](s32)
+    ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[BITCAST3]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY6]](s32), [[DEF1]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX9: [[BUILD_VECTOR_TRUNC4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C1]](s32), [[C1]](s32)
+    ; GFX9: [[XOR:%[0-9]+]]:_(<2 x s16>) = G_XOR [[BUILD_VECTOR_TRUNC]], [[BUILD_VECTOR_TRUNC4]]
+    ; GFX9: [[UMIN:%[0-9]+]]:_(<2 x s16>) = G_UMIN [[XOR]], [[BUILD_VECTOR_TRUNC2]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(<2 x s16>) = G_ADD [[BUILD_VECTOR_TRUNC]], [[UMIN]]
+    ; GFX9: [[BUILD_VECTOR_TRUNC5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C1]](s32), [[C1]](s32)
+    ; GFX9: [[XOR1:%[0-9]+]]:_(<2 x s16>) = G_XOR [[BUILD_VECTOR_TRUNC1]], [[BUILD_VECTOR_TRUNC5]]
+    ; GFX9: [[UMIN1:%[0-9]+]]:_(<2 x s16>) = G_UMIN [[XOR1]], [[BUILD_VECTOR_TRUNC3]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(<2 x s16>) = G_ADD [[BUILD_VECTOR_TRUNC1]], [[UMIN1]]
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[ADD]](<2 x s16>), [[ADD1]](<2 x s16>), [[DEF2]](<2 x s16>)
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<6 x s16>), 0
+    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
+    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS1]](<6 x s16>)
     %0:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     %1:_(<3 x s16>), %2:_(<3 x s16>) = G_UNMERGE_VALUES %0
     %3:_(<3 x s16>) = G_UADDSAT %1, %2
@@ -359,36 +567,130 @@ body: |
     ; GFX6-LABEL: name: uaddsat_v4s16
     ; GFX6: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
     ; GFX6: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
-    ; GFX6: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
-    ; GFX6: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
-    ; GFX6: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[UV]], [[UV4]]
-    ; GFX6: [[UADDSAT1:%[0-9]+]]:_(s16) = G_UADDSAT [[UV1]], [[UV5]]
-    ; GFX6: [[UADDSAT2:%[0-9]+]]:_(s16) = G_UADDSAT [[UV2]], [[UV6]]
-    ; GFX6: [[UADDSAT3:%[0-9]+]]:_(s16) = G_UADDSAT [[UV3]], [[UV7]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[UADDSAT]](s16), [[UADDSAT1]](s16), [[UADDSAT2]](s16), [[UADDSAT3]](s16)
-    ; GFX6: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; GFX6: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; GFX6: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX6: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX6: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX6: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; GFX6: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX6: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; GFX6: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX6: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX6: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[SHL]], [[C1]]
+    ; GFX6: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[XOR]], [[SHL1]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[SHL]], [[UMIN]]
+    ; GFX6: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX6: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; GFX6: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY4]], [[C]](s32)
+    ; GFX6: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[COPY5]], [[C]](s32)
+    ; GFX6: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[SHL2]], [[C1]]
+    ; GFX6: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[XOR1]], [[SHL3]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SHL2]], [[UMIN1]]
+    ; GFX6: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C]](s32)
+    ; GFX6: [[COPY6:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; GFX6: [[COPY7:%[0-9]+]]:_(s32) = COPY [[BITCAST3]](s32)
+    ; GFX6: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[COPY6]], [[C]](s32)
+    ; GFX6: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[COPY7]], [[C]](s32)
+    ; GFX6: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[SHL4]], [[C1]]
+    ; GFX6: [[UMIN2:%[0-9]+]]:_(s32) = G_UMIN [[XOR2]], [[SHL5]]
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SHL4]], [[UMIN2]]
+    ; GFX6: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[ADD2]], [[C]](s32)
+    ; GFX6: [[COPY8:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX6: [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; GFX6: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[COPY8]], [[C]](s32)
+    ; GFX6: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[COPY9]], [[C]](s32)
+    ; GFX6: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[SHL6]], [[C1]]
+    ; GFX6: [[UMIN3:%[0-9]+]]:_(s32) = G_UMIN [[XOR3]], [[SHL7]]
+    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SHL6]], [[UMIN3]]
+    ; GFX6: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[ADD3]], [[C]](s32)
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX6: [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR4]](s32)
+    ; GFX6: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY10]], [[C2]]
+    ; GFX6: [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR5]](s32)
+    ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY11]], [[C2]]
+    ; GFX6: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX6: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL8]]
+    ; GFX6: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX6: [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR6]](s32)
+    ; GFX6: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY12]], [[C2]]
+    ; GFX6: [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR7]](s32)
+    ; GFX6: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY13]], [[C2]]
+    ; GFX6: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
+    ; GFX6: [[OR1:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL9]]
+    ; GFX6: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
+    ; GFX6: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; GFX6: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
     ; GFX8-LABEL: name: uaddsat_v4s16
     ; GFX8: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
     ; GFX8: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
-    ; GFX8: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
-    ; GFX8: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
-    ; GFX8: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[UV]], [[UV4]]
-    ; GFX8: [[UADDSAT1:%[0-9]+]]:_(s16) = G_UADDSAT [[UV1]], [[UV5]]
-    ; GFX8: [[UADDSAT2:%[0-9]+]]:_(s16) = G_UADDSAT [[UV2]], [[UV6]]
-    ; GFX8: [[UADDSAT3:%[0-9]+]]:_(s16) = G_UADDSAT [[UV3]], [[UV7]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[UADDSAT]](s16), [[UADDSAT1]](s16), [[UADDSAT2]](s16), [[UADDSAT3]](s16)
-    ; GFX8: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; GFX8: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX8: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX8: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+    ; GFX8: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; GFX8: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX8: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
+    ; GFX8: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; GFX8: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+    ; GFX8: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX8: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+    ; GFX8: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; GFX8: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -1
+    ; GFX8: [[XOR:%[0-9]+]]:_(s16) = G_XOR [[TRUNC]], [[C1]]
+    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[XOR]], [[TRUNC4]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[TRUNC]], [[UMIN]]
+    ; GFX8: [[XOR1:%[0-9]+]]:_(s16) = G_XOR [[TRUNC1]], [[C1]]
+    ; GFX8: [[UMIN1:%[0-9]+]]:_(s16) = G_UMIN [[XOR1]], [[TRUNC5]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[TRUNC1]], [[UMIN1]]
+    ; GFX8: [[XOR2:%[0-9]+]]:_(s16) = G_XOR [[TRUNC2]], [[C1]]
+    ; GFX8: [[UMIN2:%[0-9]+]]:_(s16) = G_UMIN [[XOR2]], [[TRUNC6]]
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s16) = G_ADD [[TRUNC2]], [[UMIN2]]
+    ; GFX8: [[XOR3:%[0-9]+]]:_(s16) = G_XOR [[TRUNC3]], [[C1]]
+    ; GFX8: [[UMIN3:%[0-9]+]]:_(s16) = G_UMIN [[XOR3]], [[TRUNC7]]
+    ; GFX8: [[ADD3:%[0-9]+]]:_(s16) = G_ADD [[TRUNC3]], [[UMIN3]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
+    ; GFX8: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ADD2]](s16)
+    ; GFX8: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[ADD3]](s16)
+    ; GFX8: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+    ; GFX8: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
+    ; GFX8: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
+    ; GFX8: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; GFX8: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
     ; GFX9-LABEL: name: uaddsat_v4s16
     ; GFX9: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
-    ; GFX9: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
-    ; GFX9: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
-    ; GFX9: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[UV]], [[UV4]]
-    ; GFX9: [[UADDSAT1:%[0-9]+]]:_(s16) = G_UADDSAT [[UV1]], [[UV5]]
-    ; GFX9: [[UADDSAT2:%[0-9]+]]:_(s16) = G_UADDSAT [[UV2]], [[UV6]]
-    ; GFX9: [[UADDSAT3:%[0-9]+]]:_(s16) = G_UADDSAT [[UV3]], [[UV7]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[UADDSAT]](s16), [[UADDSAT1]](s16), [[UADDSAT2]](s16), [[UADDSAT3]](s16)
-    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C]](s32), [[C]](s32)
+    ; GFX9: [[XOR:%[0-9]+]]:_(<2 x s16>) = G_XOR [[UV]], [[BUILD_VECTOR_TRUNC]]
+    ; GFX9: [[UMIN:%[0-9]+]]:_(<2 x s16>) = G_UMIN [[XOR]], [[UV2]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(<2 x s16>) = G_ADD [[UV]], [[UMIN]]
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[C]](s32), [[C]](s32)
+    ; GFX9: [[XOR1:%[0-9]+]]:_(<2 x s16>) = G_XOR [[UV1]], [[BUILD_VECTOR_TRUNC1]]
+    ; GFX9: [[UMIN1:%[0-9]+]]:_(<2 x s16>) = G_UMIN [[XOR1]], [[UV3]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(<2 x s16>) = G_ADD [[UV1]], [[UMIN1]]
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[ADD]](<2 x s16>), [[ADD1]](<2 x s16>)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
     %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
     %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
     %2:_(<4 x s16>) = G_UADDSAT %0, %1
@@ -404,18 +706,27 @@ body: |
     ; GFX6-LABEL: name: uaddsat_s32
     ; GFX6: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX6: [[UADDSAT:%[0-9]+]]:_(s32) = G_UADDSAT [[COPY]], [[COPY1]]
-    ; GFX6: $vgpr0 = COPY [[UADDSAT]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX6: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY]], [[C]]
+    ; GFX6: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[XOR]], [[COPY1]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[UMIN]]
+    ; GFX6: $vgpr0 = COPY [[ADD]](s32)
     ; GFX8-LABEL: name: uaddsat_s32
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX8: [[UADDSAT:%[0-9]+]]:_(s32) = G_UADDSAT [[COPY]], [[COPY1]]
-    ; GFX8: $vgpr0 = COPY [[UADDSAT]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX8: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY]], [[C]]
+    ; GFX8: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[XOR]], [[COPY1]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[UMIN]]
+    ; GFX8: $vgpr0 = COPY [[ADD]](s32)
     ; GFX9-LABEL: name: uaddsat_s32
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX9: [[UADDSAT:%[0-9]+]]:_(s32) = G_UADDSAT [[COPY]], [[COPY1]]
-    ; GFX9: $vgpr0 = COPY [[UADDSAT]](s32)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX9: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY]], [[C]]
+    ; GFX9: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[XOR]], [[COPY1]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[UMIN]]
+    ; GFX9: $vgpr0 = COPY [[ADD]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = G_UADDSAT %0, %1
@@ -433,27 +744,42 @@ body: |
     ; GFX6: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
     ; GFX6: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
     ; GFX6: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
-    ; GFX6: [[UADDSAT:%[0-9]+]]:_(s32) = G_UADDSAT [[UV]], [[UV2]]
-    ; GFX6: [[UADDSAT1:%[0-9]+]]:_(s32) = G_UADDSAT [[UV1]], [[UV3]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[UADDSAT]](s32), [[UADDSAT1]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX6: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[UV]], [[C]]
+    ; GFX6: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[XOR]], [[UV2]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[UV]], [[UMIN]]
+    ; GFX6: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[UV1]], [[C]]
+    ; GFX6: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[XOR1]], [[UV3]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[UMIN1]]
+    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[ADD]](s32), [[ADD1]](s32)
     ; GFX6: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ; GFX8-LABEL: name: uaddsat_v2s32
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
     ; GFX8: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
     ; GFX8: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
-    ; GFX8: [[UADDSAT:%[0-9]+]]:_(s32) = G_UADDSAT [[UV]], [[UV2]]
-    ; GFX8: [[UADDSAT1:%[0-9]+]]:_(s32) = G_UADDSAT [[UV1]], [[UV3]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[UADDSAT]](s32), [[UADDSAT1]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX8: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[UV]], [[C]]
+    ; GFX8: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[XOR]], [[UV2]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[UV]], [[UMIN]]
+    ; GFX8: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[UV1]], [[C]]
+    ; GFX8: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[XOR1]], [[UV3]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[UMIN1]]
+    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[ADD]](s32), [[ADD1]](s32)
     ; GFX8: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ; GFX9-LABEL: name: uaddsat_v2s32
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
     ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
     ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
-    ; GFX9: [[UADDSAT:%[0-9]+]]:_(s32) = G_UADDSAT [[UV]], [[UV2]]
-    ; GFX9: [[UADDSAT1:%[0-9]+]]:_(s32) = G_UADDSAT [[UV1]], [[UV3]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[UADDSAT]](s32), [[UADDSAT1]](s32)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; GFX9: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[UV]], [[C]]
+    ; GFX9: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[XOR]], [[UV2]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[UV]], [[UMIN]]
+    ; GFX9: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[UV1]], [[C]]
+    ; GFX9: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[XOR1]], [[UV3]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[UMIN1]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[ADD]](s32), [[ADD1]](s32)
     ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
     %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
@@ -470,18 +796,39 @@ body: |
     ; GFX6-LABEL: name: uaddsat_s64
     ; GFX6: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX6: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX6: [[UADDSAT:%[0-9]+]]:_(s64) = G_UADDSAT [[COPY]], [[COPY1]]
-    ; GFX6: $vgpr0_vgpr1 = COPY [[UADDSAT]](s64)
+    ; GFX6: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX6: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX6: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV]], [[UV2]]
+    ; GFX6: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV1]], [[UV3]], [[UADDO1]]
+    ; GFX6: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[MV]](s64), [[COPY1]]
+    ; GFX6: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[C]], [[MV]]
+    ; GFX6: $vgpr0_vgpr1 = COPY [[SELECT]](s64)
     ; GFX8-LABEL: name: uaddsat_s64
     ; GFX8: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX8: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX8: [[UADDSAT:%[0-9]+]]:_(s64) = G_UADDSAT [[COPY]], [[COPY1]]
-    ; GFX8: $vgpr0_vgpr1 = COPY [[UADDSAT]](s64)
+    ; GFX8: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX8: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX8: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV]], [[UV2]]
+    ; GFX8: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV1]], [[UV3]], [[UADDO1]]
+    ; GFX8: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[MV]](s64), [[COPY1]]
+    ; GFX8: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[C]], [[MV]]
+    ; GFX8: $vgpr0_vgpr1 = COPY [[SELECT]](s64)
     ; GFX9-LABEL: name: uaddsat_s64
     ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX9: [[UADDSAT:%[0-9]+]]:_(s64) = G_UADDSAT [[COPY]], [[COPY1]]
-    ; GFX9: $vgpr0_vgpr1 = COPY [[UADDSAT]](s64)
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX9: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV]], [[UV2]]
+    ; GFX9: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV1]], [[UV3]], [[UADDO1]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[MV]](s64), [[COPY1]]
+    ; GFX9: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[C]], [[MV]]
+    ; GFX9: $vgpr0_vgpr1 = COPY [[SELECT]](s64)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s64) = COPY $vgpr2_vgpr3
     %2:_(s64) = G_UADDSAT %0, %1
@@ -499,27 +846,66 @@ body: |
     ; GFX6: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX6: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
     ; GFX6: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
-    ; GFX6: [[UADDSAT:%[0-9]+]]:_(s64) = G_UADDSAT [[UV]], [[UV2]]
-    ; GFX6: [[UADDSAT1:%[0-9]+]]:_(s64) = G_UADDSAT [[UV1]], [[UV3]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[UADDSAT]](s64), [[UADDSAT1]](s64)
+    ; GFX6: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX6: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX6: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX6: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO1]]
+    ; GFX6: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[MV]](s64), [[UV2]]
+    ; GFX6: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[C]], [[MV]]
+    ; GFX6: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX6: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX6: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV8]], [[UV10]]
+    ; GFX6: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV9]], [[UV11]], [[UADDO3]]
+    ; GFX6: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[MV1]](s64), [[UV3]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[C]], [[MV1]]
+    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT]](s64), [[SELECT1]](s64)
     ; GFX6: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     ; GFX8-LABEL: name: uaddsat_v2s64
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX8: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
     ; GFX8: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
-    ; GFX8: [[UADDSAT:%[0-9]+]]:_(s64) = G_UADDSAT [[UV]], [[UV2]]
-    ; GFX8: [[UADDSAT1:%[0-9]+]]:_(s64) = G_UADDSAT [[UV1]], [[UV3]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[UADDSAT]](s64), [[UADDSAT1]](s64)
+    ; GFX8: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX8: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX8: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX8: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO1]]
+    ; GFX8: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[MV]](s64), [[UV2]]
+    ; GFX8: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[C]], [[MV]]
+    ; GFX8: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX8: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX8: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV8]], [[UV10]]
+    ; GFX8: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV9]], [[UV11]], [[UADDO3]]
+    ; GFX8: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[MV1]](s64), [[UV3]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[C]], [[MV1]]
+    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT]](s64), [[SELECT1]](s64)
     ; GFX8: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     ; GFX9-LABEL: name: uaddsat_v2s64
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
     ; GFX9: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
-    ; GFX9: [[UADDSAT:%[0-9]+]]:_(s64) = G_UADDSAT [[UV]], [[UV2]]
-    ; GFX9: [[UADDSAT1:%[0-9]+]]:_(s64) = G_UADDSAT [[UV1]], [[UV3]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[UADDSAT]](s64), [[UADDSAT1]](s64)
+    ; GFX9: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX9: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX9: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[UV4]], [[UV6]]
+    ; GFX9: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[UV5]], [[UV7]], [[UADDO1]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO]](s32), [[UADDE]](s32)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[MV]](s64), [[UV2]]
+    ; GFX9: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[C]], [[MV]]
+    ; GFX9: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX9: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX9: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UV8]], [[UV10]]
+    ; GFX9: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[UV9]], [[UV11]], [[UADDO3]]
+    ; GFX9: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO2]](s32), [[UADDE2]](s32)
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[MV1]](s64), [[UV3]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[C]], [[MV1]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT]](s64), [[SELECT1]](s64)
     ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     %1:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubsat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubsat.mir
index 56a1f1baded55..356bb38456ea6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubsat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubsat.mir
@@ -12,51 +12,42 @@ body: |
     ; GFX6-LABEL: name: usubsat_s7
     ; GFX6: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX6: [[TRUNC:%[0-9]+]]:_(s7) = G_TRUNC [[COPY]](s32)
-    ; GFX6: [[TRUNC1:%[0-9]+]]:_(s7) = G_TRUNC [[COPY1]](s32)
-    ; GFX6: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC]](s7)
-    ; GFX6: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC1]](s7)
-    ; GFX6: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 9
-    ; GFX6: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX6: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX6: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[SHL]], [[SHL1]]
-    ; GFX6: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[C]](s16)
-    ; GFX6: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT]](s16)
-    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[ZEXT1]], [[ZEXT]](s32)
-    ; GFX6: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; GFX6: [[TRUNC3:%[0-9]+]]:_(s7) = G_TRUNC [[TRUNC2]](s16)
-    ; GFX6: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC3]](s7)
-    ; GFX6: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 25
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+    ; GFX6: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[SHL]], [[SHL1]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SHL]], [[UMIN]]
+    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX6: $vgpr0 = COPY [[COPY4]](s32)
     ; GFX8-LABEL: name: usubsat_s7
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX8: [[TRUNC:%[0-9]+]]:_(s7) = G_TRUNC [[COPY]](s32)
-    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s7) = G_TRUNC [[COPY1]](s32)
-    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC]](s7)
-    ; GFX8: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC1]](s7)
+    ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
     ; GFX8: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 9
-    ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX8: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[SHL]], [[SHL1]]
-    ; GFX8: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[USUBSAT]], [[C]](s16)
-    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s7) = G_TRUNC [[LSHR]](s16)
-    ; GFX8: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC2]](s7)
-    ; GFX8: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
+    ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
+    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[SHL]], [[SHL1]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SHL]], [[UMIN]]
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[SUB]], [[C]](s16)
+    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
+    ; GFX8: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: usubsat_s7
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX9: [[TRUNC:%[0-9]+]]:_(s7) = G_TRUNC [[COPY]](s32)
-    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s7) = G_TRUNC [[COPY1]](s32)
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC]](s7)
-    ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC1]](s7)
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
     ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 9
-    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX9: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[SHL]], [[SHL1]]
-    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[USUBSAT]], [[C]](s16)
-    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s7) = G_TRUNC [[LSHR]](s16)
-    ; GFX9: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC2]](s7)
-    ; GFX9: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
+    ; GFX9: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[SHL]], [[SHL1]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SHL]], [[UMIN]]
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[SUB]], [[C]](s16)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s7) = G_TRUNC %0
@@ -75,51 +66,42 @@ body: |
     ; GFX6-LABEL: name: usubsat_s8
     ; GFX6: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX6: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
-    ; GFX6: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32)
-    ; GFX6: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC]](s8)
-    ; GFX6: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC1]](s8)
-    ; GFX6: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX6: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX6: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX6: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[SHL]], [[SHL1]]
-    ; GFX6: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[C]](s16)
-    ; GFX6: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT]](s16)
-    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[ZEXT1]], [[ZEXT]](s32)
-    ; GFX6: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; GFX6: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC2]](s16)
-    ; GFX6: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC3]](s8)
-    ; GFX6: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+    ; GFX6: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[SHL]], [[SHL1]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SHL]], [[UMIN]]
+    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX6: $vgpr0 = COPY [[COPY4]](s32)
     ; GFX8-LABEL: name: usubsat_s8
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX8: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
-    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32)
-    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC]](s8)
-    ; GFX8: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC1]](s8)
+    ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
     ; GFX8: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX8: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[SHL]], [[SHL1]]
-    ; GFX8: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[USUBSAT]], [[C]](s16)
-    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR]](s16)
-    ; GFX8: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC2]](s8)
-    ; GFX8: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
+    ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
+    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[SHL]], [[SHL1]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SHL]], [[UMIN]]
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[SUB]], [[C]](s16)
+    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
+    ; GFX8: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: usubsat_s8
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX9: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
-    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32)
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC]](s8)
-    ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[TRUNC1]](s8)
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
     ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX9: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[SHL]], [[SHL1]]
-    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[USUBSAT]], [[C]](s16)
-    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR]](s16)
-    ; GFX9: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC2]](s8)
-    ; GFX9: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C]](s16)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C]](s16)
+    ; GFX9: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[SHL]], [[SHL1]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SHL]], [[UMIN]]
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[SUB]], [[C]](s16)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s8) = G_TRUNC %0
@@ -138,75 +120,113 @@ body: |
     ; GFX6-LABEL: name: usubsat_v2s8
     ; GFX6: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX6: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
-    ; GFX6: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX6: [[BITCAST:%[0-9]+]]:_(<2 x s8>) = G_BITCAST [[TRUNC]](s16)
-    ; GFX6: [[BITCAST1:%[0-9]+]]:_(<2 x s8>) = G_BITCAST [[TRUNC1]](s16)
-    ; GFX6: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BITCAST]](<2 x s8>)
-    ; GFX6: [[UV2:%[0-9]+]]:_(s8), [[UV3:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BITCAST1]](<2 x s8>)
-    ; GFX6: [[USUBSAT:%[0-9]+]]:_(s8) = G_USUBSAT [[UV]], [[UV2]]
-    ; GFX6: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[UV1]](s8)
-    ; GFX6: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[UV3]](s8)
-    ; GFX6: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX6: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX6: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX6: [[USUBSAT1:%[0-9]+]]:_(s16) = G_USUBSAT [[SHL]], [[SHL1]]
-    ; GFX6: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[C]](s16)
-    ; GFX6: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT1]](s16)
-    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[ZEXT1]], [[ZEXT]](s32)
-    ; GFX6: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-    ; GFX6: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC2]](s16)
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s8>) = G_BUILD_VECTOR [[USUBSAT]](s8), [[TRUNC3]](s8)
-    ; GFX6: [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<2 x s8>)
-    ; GFX6: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV4]](s8), [[UV5]](s8)
-    ; GFX6: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
-    ; GFX6: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX6: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX6: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; GFX6: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
+    ; GFX6: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C1]](s32)
+    ; GFX6: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C2]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C2]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C2]](s32)
+    ; GFX6: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[SHL]], [[SHL1]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SHL]], [[UMIN]]
+    ; GFX6: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C2]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX6: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; GFX6: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY4]], [[C2]](s32)
+    ; GFX6: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[COPY5]], [[C2]](s32)
+    ; GFX6: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[SHL2]], [[SHL3]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SHL2]], [[UMIN1]]
+    ; GFX6: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[SUB1]], [[C2]](s32)
+    ; GFX6: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX6: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR6]](s32)
+    ; GFX6: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C3]]
+    ; GFX6: [[COPY6:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; GFX6: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; GFX6: [[COPY7:%[0-9]+]]:_(s32) = COPY [[LSHR7]](s32)
+    ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C4]]
+    ; GFX6: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[COPY6]](s32)
+    ; GFX6: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL4]](s32)
+    ; GFX6: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[TRUNC1]]
+    ; GFX6: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; GFX6: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX8-LABEL: name: usubsat_v2s8
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX8: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; GFX8: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
+    ; GFX8: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C1]](s32)
+    ; GFX8: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C2]](s32)
     ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
     ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX8: [[BITCAST:%[0-9]+]]:_(<2 x s8>) = G_BITCAST [[TRUNC]](s16)
-    ; GFX8: [[BITCAST1:%[0-9]+]]:_(<2 x s8>) = G_BITCAST [[TRUNC1]](s16)
-    ; GFX8: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BITCAST]](<2 x s8>)
-    ; GFX8: [[UV2:%[0-9]+]]:_(s8), [[UV3:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BITCAST1]](<2 x s8>)
-    ; GFX8: [[USUBSAT:%[0-9]+]]:_(s8) = G_USUBSAT [[UV]], [[UV2]]
-    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[UV1]](s8)
-    ; GFX8: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[UV3]](s8)
-    ; GFX8: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX8: [[USUBSAT1:%[0-9]+]]:_(s16) = G_USUBSAT [[SHL]], [[SHL1]]
-    ; GFX8: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[USUBSAT1]], [[C]](s16)
-    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR]](s16)
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s8>) = G_BUILD_VECTOR [[USUBSAT]](s8), [[TRUNC2]](s8)
-    ; GFX8: [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<2 x s8>)
-    ; GFX8: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV4]](s8), [[UV5]](s8)
-    ; GFX8: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
-    ; GFX8: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX8: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX8: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C3]](s16)
+    ; GFX8: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C3]](s16)
+    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[SHL]], [[SHL1]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SHL]], [[UMIN]]
+    ; GFX8: [[LSHR6:%[0-9]+]]:_(s16) = G_LSHR [[SUB]], [[C3]](s16)
+    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX8: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+    ; GFX8: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[C3]](s16)
+    ; GFX8: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[TRUNC3]], [[C3]](s16)
+    ; GFX8: [[UMIN1:%[0-9]+]]:_(s16) = G_UMIN [[SHL2]], [[SHL3]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[SHL2]], [[UMIN1]]
+    ; GFX8: [[LSHR7:%[0-9]+]]:_(s16) = G_LSHR [[SUB1]], [[C3]](s16)
+    ; GFX8: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s16) = COPY [[LSHR6]](s16)
+    ; GFX8: [[AND:%[0-9]+]]:_(s16) = G_AND [[COPY2]], [[C4]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s16) = COPY [[LSHR7]](s16)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s16) = G_AND [[COPY3]], [[C4]]
+    ; GFX8: [[SHL4:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C3]](s16)
+    ; GFX8: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL4]]
+    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; GFX8: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: usubsat_v2s8
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C1]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C2]](s32)
+    ; GFX9: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
+    ; GFX9: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C1]](s32)
+    ; GFX9: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C2]](s32)
     ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
     ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX9: [[BITCAST:%[0-9]+]]:_(<2 x s8>) = G_BITCAST [[TRUNC]](s16)
-    ; GFX9: [[BITCAST1:%[0-9]+]]:_(<2 x s8>) = G_BITCAST [[TRUNC1]](s16)
-    ; GFX9: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BITCAST]](<2 x s8>)
-    ; GFX9: [[UV2:%[0-9]+]]:_(s8), [[UV3:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BITCAST1]](<2 x s8>)
-    ; GFX9: [[USUBSAT:%[0-9]+]]:_(s8) = G_USUBSAT [[UV]], [[UV2]]
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[UV1]](s8)
-    ; GFX9: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[UV3]](s8)
-    ; GFX9: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT]], [[C]](s16)
-    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[ANYEXT1]], [[C]](s16)
-    ; GFX9: [[USUBSAT1:%[0-9]+]]:_(s16) = G_USUBSAT [[SHL]], [[SHL1]]
-    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[USUBSAT1]], [[C]](s16)
-    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[LSHR]](s16)
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s8>) = G_BUILD_VECTOR [[USUBSAT]](s8), [[TRUNC2]](s8)
-    ; GFX9: [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<2 x s8>)
-    ; GFX9: [[MV:%[0-9]+]]:_(s16) = G_MERGE_VALUES [[UV4]](s8), [[UV5]](s8)
-    ; GFX9: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[MV]](s16)
-    ; GFX9: $vgpr0 = COPY [[ANYEXT2]](s32)
+    ; GFX9: [[C3:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[C3]](s16)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[C3]](s16)
+    ; GFX9: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[SHL]], [[SHL1]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[SHL]], [[UMIN]]
+    ; GFX9: [[LSHR6:%[0-9]+]]:_(s16) = G_LSHR [[SUB]], [[C3]](s16)
+    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX9: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+    ; GFX9: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[C3]](s16)
+    ; GFX9: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[TRUNC3]], [[C3]](s16)
+    ; GFX9: [[UMIN1:%[0-9]+]]:_(s16) = G_UMIN [[SHL2]], [[SHL3]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[SHL2]], [[UMIN1]]
+    ; GFX9: [[LSHR7:%[0-9]+]]:_(s16) = G_LSHR [[SUB1]], [[C3]](s16)
+    ; GFX9: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s16) = COPY [[LSHR6]](s16)
+    ; GFX9: [[AND:%[0-9]+]]:_(s16) = G_AND [[COPY2]], [[C4]]
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s16) = COPY [[LSHR7]](s16)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s16) = G_AND [[COPY3]], [[C4]]
+    ; GFX9: [[SHL4:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C3]](s16)
+    ; GFX9: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL4]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[OR]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s16) = G_TRUNC %0
@@ -228,26 +248,33 @@ body: |
     ; GFX6-LABEL: name: usubsat_s16
     ; GFX6: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX6: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
-    ; GFX6: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX6: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC]], [[TRUNC1]]
-    ; GFX6: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[USUBSAT]](s16)
-    ; GFX6: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+    ; GFX6: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[SHL]], [[SHL1]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SHL]], [[UMIN]]
+    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX6: $vgpr0 = COPY [[COPY4]](s32)
     ; GFX8-LABEL: name: usubsat_s16
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
     ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX8: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC]], [[TRUNC1]]
-    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[USUBSAT]](s16)
+    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC]], [[TRUNC1]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[TRUNC]], [[UMIN]]
+    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SUB]](s16)
     ; GFX8: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX9-LABEL: name: usubsat_s16
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
     ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-    ; GFX9: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC]], [[TRUNC1]]
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[USUBSAT]](s16)
+    ; GFX9: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC]], [[TRUNC1]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[TRUNC]], [[UMIN]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SUB]](s16)
     ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
@@ -267,30 +294,62 @@ body: |
     ; GFX6-LABEL: name: usubsat_v2s16
     ; GFX6: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
-    ; GFX6: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
-    ; GFX6: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
-    ; GFX6: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[UV]], [[UV2]]
-    ; GFX6: [[USUBSAT1:%[0-9]+]]:_(s16) = G_USUBSAT [[UV1]], [[UV3]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[USUBSAT]](s16), [[USUBSAT1]](s16)
-    ; GFX6: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX6: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX6: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; GFX6: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+    ; GFX6: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[SHL]], [[SHL1]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SHL]], [[UMIN]]
+    ; GFX6: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX6: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX6: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY4]], [[C]](s32)
+    ; GFX6: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[COPY5]], [[C]](s32)
+    ; GFX6: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[SHL2]], [[SHL3]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SHL2]], [[UMIN1]]
+    ; GFX6: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[SUB1]], [[C]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX6: [[COPY6:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; GFX6: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C1]]
+    ; GFX6: [[COPY7:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C1]]
+    ; GFX6: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX6: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL4]]
+    ; GFX6: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX6: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
     ; GFX8-LABEL: name: usubsat_v2s16
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
-    ; GFX8: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
-    ; GFX8: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
-    ; GFX8: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[UV]], [[UV2]]
-    ; GFX8: [[USUBSAT1:%[0-9]+]]:_(s16) = G_USUBSAT [[UV1]], [[UV3]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[USUBSAT]](s16), [[USUBSAT1]](s16)
-    ; GFX8: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX8: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX8: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC]], [[TRUNC2]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[TRUNC]], [[UMIN]]
+    ; GFX8: [[UMIN1:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC1]], [[TRUNC3]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[TRUNC1]], [[UMIN1]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SUB]](s16)
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SUB1]](s16)
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
+    ; GFX8: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX8: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
     ; GFX9-LABEL: name: usubsat_v2s16
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
-    ; GFX9: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
-    ; GFX9: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
-    ; GFX9: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[UV]], [[UV2]]
-    ; GFX9: [[USUBSAT1:%[0-9]+]]:_(s16) = G_USUBSAT [[UV1]], [[UV3]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[USUBSAT]](s16), [[USUBSAT1]](s16)
-    ; GFX9: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX9: [[UMIN:%[0-9]+]]:_(<2 x s16>) = G_UMIN [[COPY]], [[COPY1]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(<2 x s16>) = G_SUB [[COPY]], [[UMIN]]
+    ; GFX9: $vgpr0 = COPY [[SUB]](<2 x s16>)
     %0:_(<2 x s16>) = COPY $vgpr0
     %1:_(<2 x s16>) = COPY $vgpr1
     %2:_(<2 x s16>) = G_USUBSAT %0, %1
@@ -306,42 +365,142 @@ body: |
     ; GFX6-LABEL: name: usubsat_v3s16
     ; GFX6: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     ; GFX6: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
-    ; GFX6: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UV]](<3 x s16>)
-    ; GFX6: [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UV1]](<3 x s16>)
-    ; GFX6: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[UV2]], [[UV5]]
-    ; GFX6: [[USUBSAT1:%[0-9]+]]:_(s16) = G_USUBSAT [[UV3]], [[UV6]]
-    ; GFX6: [[USUBSAT2:%[0-9]+]]:_(s16) = G_USUBSAT [[UV4]], [[UV7]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[USUBSAT]](s16), [[USUBSAT1]](s16), [[USUBSAT2]](s16)
     ; GFX6: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX6: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX6: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<3 x s16>), [[EXTRACT]](<3 x s16>)
-    ; GFX6: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
+    ; GFX6: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
+    ; GFX6: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX6: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX6: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX6: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX6: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV1]](<3 x s16>), 0
+    ; GFX6: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; GFX6: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
+    ; GFX6: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; GFX6: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
+    ; GFX6: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; GFX6: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[SHL]], [[SHL1]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SHL]], [[UMIN]]
+    ; GFX6: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; GFX6: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+    ; GFX6: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[COPY4]], [[C]](s32)
+    ; GFX6: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[SHL2]], [[SHL3]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SHL2]], [[UMIN1]]
+    ; GFX6: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[SUB1]], [[C]](s32)
+    ; GFX6: [[COPY5:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; GFX6: [[COPY6:%[0-9]+]]:_(s32) = COPY [[BITCAST3]](s32)
+    ; GFX6: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[COPY5]], [[C]](s32)
+    ; GFX6: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[COPY6]], [[C]](s32)
+    ; GFX6: [[UMIN2:%[0-9]+]]:_(s32) = G_UMIN [[SHL4]], [[SHL5]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SHL4]], [[UMIN2]]
+    ; GFX6: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[SUB2]], [[C]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX6: [[COPY7:%[0-9]+]]:_(s32) = COPY [[LSHR4]](s32)
+    ; GFX6: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C1]]
+    ; GFX6: [[COPY8:%[0-9]+]]:_(s32) = COPY [[LSHR5]](s32)
+    ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY8]], [[C1]]
+    ; GFX6: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX6: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL6]]
+    ; GFX6: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX6: [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR6]](s32)
+    ; GFX6: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C1]]
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX6: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
+    ; GFX6: [[OR1:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL7]]
+    ; GFX6: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
+    ; GFX6: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; GFX6: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
+    ; GFX6: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX6: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
+    ; GFX6: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS1]](<6 x s16>)
     ; GFX8-LABEL: name: usubsat_v3s16
     ; GFX8: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     ; GFX8: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
-    ; GFX8: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UV]](<3 x s16>)
-    ; GFX8: [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UV1]](<3 x s16>)
-    ; GFX8: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[UV2]], [[UV5]]
-    ; GFX8: [[USUBSAT1:%[0-9]+]]:_(s16) = G_USUBSAT [[UV3]], [[UV6]]
-    ; GFX8: [[USUBSAT2:%[0-9]+]]:_(s16) = G_USUBSAT [[UV4]], [[UV7]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[USUBSAT]](s16), [[USUBSAT1]](s16), [[USUBSAT2]](s16)
     ; GFX8: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX8: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX8: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<3 x s16>), [[EXTRACT]](<3 x s16>)
-    ; GFX8: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
+    ; GFX8: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
+    ; GFX8: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX8: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX8: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV1]](<3 x s16>), 0
+    ; GFX8: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; GFX8: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
+    ; GFX8: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
+    ; GFX8: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; GFX8: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+    ; GFX8: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
+    ; GFX8: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+    ; GFX8: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC]], [[TRUNC3]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[TRUNC]], [[UMIN]]
+    ; GFX8: [[UMIN1:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC1]], [[TRUNC4]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[TRUNC1]], [[UMIN1]]
+    ; GFX8: [[UMIN2:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC2]], [[TRUNC5]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[TRUNC2]], [[UMIN2]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SUB]](s16)
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SUB1]](s16)
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
+    ; GFX8: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SUB2]](s16)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+    ; GFX8: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
+    ; GFX8: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
+    ; GFX8: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; GFX8: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<4 x s16>), 0
+    ; GFX8: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX8: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
+    ; GFX8: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS1]](<6 x s16>)
     ; GFX9-LABEL: name: usubsat_v3s16
     ; GFX9: [[COPY:%[0-9]+]]:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     ; GFX9: [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[COPY]](<6 x s16>)
-    ; GFX9: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UV]](<3 x s16>)
-    ; GFX9: [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[UV1]](<3 x s16>)
-    ; GFX9: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[UV2]], [[UV5]]
-    ; GFX9: [[USUBSAT1:%[0-9]+]]:_(s16) = G_USUBSAT [[UV3]], [[UV6]]
-    ; GFX9: [[USUBSAT2:%[0-9]+]]:_(s16) = G_USUBSAT [[UV4]], [[UV7]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[USUBSAT]](s16), [[USUBSAT1]](s16), [[USUBSAT2]](s16)
     ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
-    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<3 x s16>), [[EXTRACT]](<3 x s16>)
-    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV]](<3 x s16>), 0
+    ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT]](<4 x s16>)
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX9: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY1]](s32), [[COPY2]](s32)
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; GFX9: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; GFX9: [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY3]](s32), [[DEF1]](s32)
+    ; GFX9: [[DEF2:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[UV1]](<3 x s16>), 0
+    ; GFX9: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[INSERT1]](<4 x s16>)
+    ; GFX9: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV4]](<2 x s16>)
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; GFX9: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV5]](<2 x s16>)
+    ; GFX9: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+    ; GFX9: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY4]](s32), [[COPY5]](s32)
+    ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[BITCAST3]](s32)
+    ; GFX9: [[BUILD_VECTOR_TRUNC3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY6]](s32), [[DEF1]](s32)
+    ; GFX9: [[UMIN:%[0-9]+]]:_(<2 x s16>) = G_UMIN [[BUILD_VECTOR_TRUNC]], [[BUILD_VECTOR_TRUNC2]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC]], [[UMIN]]
+    ; GFX9: [[UMIN1:%[0-9]+]]:_(<2 x s16>) = G_UMIN [[BUILD_VECTOR_TRUNC1]], [[BUILD_VECTOR_TRUNC3]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(<2 x s16>) = G_SUB [[BUILD_VECTOR_TRUNC1]], [[UMIN1]]
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[SUB]](<2 x s16>), [[SUB1]](<2 x s16>), [[DEF2]](<2 x s16>)
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[CONCAT_VECTORS]](<6 x s16>), 0
+    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[EXTRACT]](<3 x s16>), [[EXTRACT1]](<3 x s16>)
+    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS1]](<6 x s16>)
     %0:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
     %1:_(<3 x s16>), %2:_(<3 x s16>) = G_UNMERGE_VALUES %0
     %3:_(<3 x s16>) = G_USUBSAT %1, %2
@@ -359,36 +518,115 @@ body: |
     ; GFX6-LABEL: name: usubsat_v4s16
     ; GFX6: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
     ; GFX6: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
-    ; GFX6: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
-    ; GFX6: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
-    ; GFX6: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[UV]], [[UV4]]
-    ; GFX6: [[USUBSAT1:%[0-9]+]]:_(s16) = G_USUBSAT [[UV1]], [[UV5]]
-    ; GFX6: [[USUBSAT2:%[0-9]+]]:_(s16) = G_USUBSAT [[UV2]], [[UV6]]
-    ; GFX6: [[USUBSAT3:%[0-9]+]]:_(s16) = G_USUBSAT [[UV3]], [[UV7]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[USUBSAT]](s16), [[USUBSAT1]](s16), [[USUBSAT2]](s16), [[USUBSAT3]](s16)
-    ; GFX6: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; GFX6: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; GFX6: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX6: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX6: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX6: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX6: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; GFX6: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX6: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; GFX6: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX6: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY [[BITCAST]](s32)
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s32) = COPY [[BITCAST2]](s32)
+    ; GFX6: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; GFX6: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C]](s32)
+    ; GFX6: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[SHL]], [[SHL1]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[SHL]], [[UMIN]]
+    ; GFX6: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C]](s32)
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; GFX6: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LSHR2]](s32)
+    ; GFX6: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY4]], [[C]](s32)
+    ; GFX6: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[COPY5]], [[C]](s32)
+    ; GFX6: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[SHL2]], [[SHL3]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SHL2]], [[UMIN1]]
+    ; GFX6: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[SUB1]], [[C]](s32)
+    ; GFX6: [[COPY6:%[0-9]+]]:_(s32) = COPY [[BITCAST1]](s32)
+    ; GFX6: [[COPY7:%[0-9]+]]:_(s32) = COPY [[BITCAST3]](s32)
+    ; GFX6: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[COPY6]], [[C]](s32)
+    ; GFX6: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[COPY7]], [[C]](s32)
+    ; GFX6: [[UMIN2:%[0-9]+]]:_(s32) = G_UMIN [[SHL4]], [[SHL5]]
+    ; GFX6: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SHL4]], [[UMIN2]]
+    ; GFX6: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[SUB2]], [[C]](s32)
+    ; GFX6: [[COPY8:%[0-9]+]]:_(s32) = COPY [[LSHR1]](s32)
+    ; GFX6: [[COPY9:%[0-9]+]]:_(s32) = COPY [[LSHR3]](s32)
+    ; GFX6: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[COPY8]], [[C]](s32)
+    ; GFX6: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[COPY9]], [[C]](s32)
+    ; GFX6: [[UMIN3:%[0-9]+]]:_(s32) = G_UMIN [[SHL6]], [[SHL7]]
+    ; GFX6: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SHL6]], [[UMIN3]]
+    ; GFX6: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[SUB3]], [[C]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX6: [[COPY10:%[0-9]+]]:_(s32) = COPY [[LSHR4]](s32)
+    ; GFX6: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY10]], [[C1]]
+    ; GFX6: [[COPY11:%[0-9]+]]:_(s32) = COPY [[LSHR5]](s32)
+    ; GFX6: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY11]], [[C1]]
+    ; GFX6: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+    ; GFX6: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL8]]
+    ; GFX6: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX6: [[COPY12:%[0-9]+]]:_(s32) = COPY [[LSHR6]](s32)
+    ; GFX6: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY12]], [[C1]]
+    ; GFX6: [[COPY13:%[0-9]+]]:_(s32) = COPY [[LSHR7]](s32)
+    ; GFX6: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY13]], [[C1]]
+    ; GFX6: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C]](s32)
+    ; GFX6: [[OR1:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL9]]
+    ; GFX6: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
+    ; GFX6: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; GFX6: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
     ; GFX8-LABEL: name: usubsat_v4s16
     ; GFX8: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
     ; GFX8: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
-    ; GFX8: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
-    ; GFX8: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
-    ; GFX8: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[UV]], [[UV4]]
-    ; GFX8: [[USUBSAT1:%[0-9]+]]:_(s16) = G_USUBSAT [[UV1]], [[UV5]]
-    ; GFX8: [[USUBSAT2:%[0-9]+]]:_(s16) = G_USUBSAT [[UV2]], [[UV6]]
-    ; GFX8: [[USUBSAT3:%[0-9]+]]:_(s16) = G_USUBSAT [[UV3]], [[UV7]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[USUBSAT]](s16), [[USUBSAT1]](s16), [[USUBSAT2]](s16), [[USUBSAT3]](s16)
-    ; GFX8: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; GFX8: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; GFX8: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV]](<2 x s16>)
+    ; GFX8: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX8: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; GFX8: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
+    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST1]](s32)
+    ; GFX8: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+    ; GFX8: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+    ; GFX8: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; GFX8: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[UV2]](<2 x s16>)
+    ; GFX8: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
+    ; GFX8: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
+    ; GFX8: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+    ; GFX8: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[UV3]](<2 x s16>)
+    ; GFX8: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST3]](s32)
+    ; GFX8: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C]](s32)
+    ; GFX8: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+    ; GFX8: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC]], [[TRUNC4]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[TRUNC]], [[UMIN]]
+    ; GFX8: [[UMIN1:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC1]], [[TRUNC5]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[TRUNC1]], [[UMIN1]]
+    ; GFX8: [[UMIN2:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC2]], [[TRUNC6]]
+    ; GFX8: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[TRUNC2]], [[UMIN2]]
+    ; GFX8: [[UMIN3:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC3]], [[TRUNC7]]
+    ; GFX8: [[SUB3:%[0-9]+]]:_(s16) = G_SUB [[TRUNC3]], [[UMIN3]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SUB]](s16)
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SUB1]](s16)
+    ; GFX8: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+    ; GFX8: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
+    ; GFX8: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
+    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SUB2]](s16)
+    ; GFX8: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[SUB3]](s16)
+    ; GFX8: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+    ; GFX8: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
+    ; GFX8: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
+    ; GFX8: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
+    ; GFX8: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
     ; GFX9-LABEL: name: usubsat_v4s16
     ; GFX9: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
-    ; GFX9: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
-    ; GFX9: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
-    ; GFX9: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[UV]], [[UV4]]
-    ; GFX9: [[USUBSAT1:%[0-9]+]]:_(s16) = G_USUBSAT [[UV1]], [[UV5]]
-    ; GFX9: [[USUBSAT2:%[0-9]+]]:_(s16) = G_USUBSAT [[UV2]], [[UV6]]
-    ; GFX9: [[USUBSAT3:%[0-9]+]]:_(s16) = G_USUBSAT [[UV3]], [[UV7]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[USUBSAT]](s16), [[USUBSAT1]](s16), [[USUBSAT2]](s16), [[USUBSAT3]](s16)
-    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; GFX9: [[UMIN:%[0-9]+]]:_(<2 x s16>) = G_UMIN [[UV]], [[UV2]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(<2 x s16>) = G_SUB [[UV]], [[UMIN]]
+    ; GFX9: [[UMIN1:%[0-9]+]]:_(<2 x s16>) = G_UMIN [[UV1]], [[UV3]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(<2 x s16>) = G_SUB [[UV1]], [[UMIN1]]
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[SUB]](<2 x s16>), [[SUB1]](<2 x s16>)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
     %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
     %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
     %2:_(<4 x s16>) = G_USUBSAT %0, %1
@@ -404,18 +642,21 @@ body: |
     ; GFX6-LABEL: name: usubsat_s32
     ; GFX6: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX6: [[USUBSAT:%[0-9]+]]:_(s32) = G_USUBSAT [[COPY]], [[COPY1]]
-    ; GFX6: $vgpr0 = COPY [[USUBSAT]](s32)
+    ; GFX6: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[COPY]], [[COPY1]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[UMIN]]
+    ; GFX6: $vgpr0 = COPY [[SUB]](s32)
     ; GFX8-LABEL: name: usubsat_s32
     ; GFX8: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX8: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX8: [[USUBSAT:%[0-9]+]]:_(s32) = G_USUBSAT [[COPY]], [[COPY1]]
-    ; GFX8: $vgpr0 = COPY [[USUBSAT]](s32)
+    ; GFX8: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[COPY]], [[COPY1]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[UMIN]]
+    ; GFX8: $vgpr0 = COPY [[SUB]](s32)
     ; GFX9-LABEL: name: usubsat_s32
     ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; GFX9: [[USUBSAT:%[0-9]+]]:_(s32) = G_USUBSAT [[COPY]], [[COPY1]]
-    ; GFX9: $vgpr0 = COPY [[USUBSAT]](s32)
+    ; GFX9: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[COPY]], [[COPY1]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[UMIN]]
+    ; GFX9: $vgpr0 = COPY [[SUB]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = G_USUBSAT %0, %1
@@ -433,27 +674,33 @@ body: |
     ; GFX6: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
     ; GFX6: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
     ; GFX6: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
-    ; GFX6: [[USUBSAT:%[0-9]+]]:_(s32) = G_USUBSAT [[UV]], [[UV2]]
-    ; GFX6: [[USUBSAT1:%[0-9]+]]:_(s32) = G_USUBSAT [[UV1]], [[UV3]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[USUBSAT]](s32), [[USUBSAT1]](s32)
+    ; GFX6: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[UV]], [[UV2]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV]], [[UMIN]]
+    ; GFX6: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[UV1]], [[UV3]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[UV1]], [[UMIN1]]
+    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SUB]](s32), [[SUB1]](s32)
     ; GFX6: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ; GFX8-LABEL: name: usubsat_v2s32
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
     ; GFX8: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
     ; GFX8: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
-    ; GFX8: [[USUBSAT:%[0-9]+]]:_(s32) = G_USUBSAT [[UV]], [[UV2]]
-    ; GFX8: [[USUBSAT1:%[0-9]+]]:_(s32) = G_USUBSAT [[UV1]], [[UV3]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[USUBSAT]](s32), [[USUBSAT1]](s32)
+    ; GFX8: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[UV]], [[UV2]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV]], [[UMIN]]
+    ; GFX8: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[UV1]], [[UV3]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[UV1]], [[UMIN1]]
+    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SUB]](s32), [[SUB1]](s32)
     ; GFX8: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ; GFX9-LABEL: name: usubsat_v2s32
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
     ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
     ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
-    ; GFX9: [[USUBSAT:%[0-9]+]]:_(s32) = G_USUBSAT [[UV]], [[UV2]]
-    ; GFX9: [[USUBSAT1:%[0-9]+]]:_(s32) = G_USUBSAT [[UV1]], [[UV3]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[USUBSAT]](s32), [[USUBSAT1]](s32)
+    ; GFX9: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[UV]], [[UV2]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV]], [[UMIN]]
+    ; GFX9: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[UV1]], [[UV3]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[UV1]], [[UMIN1]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SUB]](s32), [[SUB1]](s32)
     ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
     %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
@@ -470,18 +717,39 @@ body: |
     ; GFX6-LABEL: name: usubsat_s64
     ; GFX6: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX6: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX6: [[USUBSAT:%[0-9]+]]:_(s64) = G_USUBSAT [[COPY]], [[COPY1]]
-    ; GFX6: $vgpr0_vgpr1 = COPY [[USUBSAT]](s64)
+    ; GFX6: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX6: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX6: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV]], [[UV2]]
+    ; GFX6: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV1]], [[UV3]], [[USUBO1]]
+    ; GFX6: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO]](s32), [[USUBE]](s32)
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[COPY]](s64), [[COPY1]]
+    ; GFX6: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[C]], [[MV]]
+    ; GFX6: $vgpr0_vgpr1 = COPY [[SELECT]](s64)
     ; GFX8-LABEL: name: usubsat_s64
     ; GFX8: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX8: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX8: [[USUBSAT:%[0-9]+]]:_(s64) = G_USUBSAT [[COPY]], [[COPY1]]
-    ; GFX8: $vgpr0_vgpr1 = COPY [[USUBSAT]](s64)
+    ; GFX8: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX8: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX8: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV]], [[UV2]]
+    ; GFX8: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV1]], [[UV3]], [[USUBO1]]
+    ; GFX8: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO]](s32), [[USUBE]](s32)
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[COPY]](s64), [[COPY1]]
+    ; GFX8: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[C]], [[MV]]
+    ; GFX8: $vgpr0_vgpr1 = COPY [[SELECT]](s64)
     ; GFX9-LABEL: name: usubsat_s64
     ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX9: [[USUBSAT:%[0-9]+]]:_(s64) = G_USUBSAT [[COPY]], [[COPY1]]
-    ; GFX9: $vgpr0_vgpr1 = COPY [[USUBSAT]](s64)
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX9: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV]], [[UV2]]
+    ; GFX9: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV1]], [[UV3]], [[USUBO1]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO]](s32), [[USUBE]](s32)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[COPY]](s64), [[COPY1]]
+    ; GFX9: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[C]], [[MV]]
+    ; GFX9: $vgpr0_vgpr1 = COPY [[SELECT]](s64)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s64) = COPY $vgpr2_vgpr3
     %2:_(s64) = G_USUBSAT %0, %1
@@ -499,27 +767,66 @@ body: |
     ; GFX6: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX6: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
     ; GFX6: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
-    ; GFX6: [[USUBSAT:%[0-9]+]]:_(s64) = G_USUBSAT [[UV]], [[UV2]]
-    ; GFX6: [[USUBSAT1:%[0-9]+]]:_(s64) = G_USUBSAT [[UV1]], [[UV3]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[USUBSAT]](s64), [[USUBSAT1]](s64)
+    ; GFX6: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX6: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX6: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV4]], [[UV6]]
+    ; GFX6: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV5]], [[UV7]], [[USUBO1]]
+    ; GFX6: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO]](s32), [[USUBE]](s32)
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[UV]](s64), [[UV2]]
+    ; GFX6: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[C]], [[MV]]
+    ; GFX6: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX6: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX6: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV8]], [[UV10]]
+    ; GFX6: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV9]], [[UV11]], [[USUBO3]]
+    ; GFX6: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO2]](s32), [[USUBE2]](s32)
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[UV1]](s64), [[UV3]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[C]], [[MV1]]
+    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT]](s64), [[SELECT1]](s64)
     ; GFX6: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     ; GFX8-LABEL: name: usubsat_v2s64
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX8: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
     ; GFX8: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
-    ; GFX8: [[USUBSAT:%[0-9]+]]:_(s64) = G_USUBSAT [[UV]], [[UV2]]
-    ; GFX8: [[USUBSAT1:%[0-9]+]]:_(s64) = G_USUBSAT [[UV1]], [[UV3]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[USUBSAT]](s64), [[USUBSAT1]](s64)
+    ; GFX8: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX8: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX8: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV4]], [[UV6]]
+    ; GFX8: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV5]], [[UV7]], [[USUBO1]]
+    ; GFX8: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO]](s32), [[USUBE]](s32)
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[UV]](s64), [[UV2]]
+    ; GFX8: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[C]], [[MV]]
+    ; GFX8: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX8: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX8: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV8]], [[UV10]]
+    ; GFX8: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV9]], [[UV11]], [[USUBO3]]
+    ; GFX8: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO2]](s32), [[USUBE2]](s32)
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[UV1]](s64), [[UV3]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[C]], [[MV1]]
+    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT]](s64), [[SELECT1]](s64)
     ; GFX8: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     ; GFX9-LABEL: name: usubsat_v2s64
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
     ; GFX9: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
-    ; GFX9: [[USUBSAT:%[0-9]+]]:_(s64) = G_USUBSAT [[UV]], [[UV2]]
-    ; GFX9: [[USUBSAT1:%[0-9]+]]:_(s64) = G_USUBSAT [[UV1]], [[UV3]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[USUBSAT]](s64), [[USUBSAT1]](s64)
+    ; GFX9: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX9: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX9: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV4]], [[UV6]]
+    ; GFX9: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV5]], [[UV7]], [[USUBO1]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO]](s32), [[USUBE]](s32)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[UV]](s64), [[UV2]]
+    ; GFX9: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[C]], [[MV]]
+    ; GFX9: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX9: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX9: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV8]], [[UV10]]
+    ; GFX9: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV9]], [[UV11]], [[USUBO3]]
+    ; GFX9: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO2]](s32), [[USUBE2]](s32)
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[UV1]](s64), [[UV3]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[C]], [[MV1]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT]](s64), [[SELECT1]](s64)
     ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     %1:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-xor.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-xor.mir
index b612916c1891f..110ed17400d26 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-xor.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-xor.mir
@@ -443,31 +443,29 @@ body: |
     ; CHECK: [[DEF1:%[0-9]+]]:_(<3 x s8>) = G_IMPLICIT_DEF
     ; CHECK: [[DEF2:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
     ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF2]], [[DEF]](<3 x s8>), 0
-    ; CHECK: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[DEF2]](<4 x s8>)
-    ; CHECK: [[INSERT1:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[ANYEXT]], [[DEF1]](<3 x s8>), 0
-    ; CHECK: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[INSERT1]](<4 x s16>)
+    ; CHECK: [[INSERT1:%[0-9]+]]:_(<4 x s8>) = G_INSERT [[DEF2]], [[DEF1]](<3 x s8>), 0
     ; CHECK: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8), [[UV2:%[0-9]+]]:_(s8), [[UV3:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[INSERT]](<4 x s8>)
-    ; CHECK: [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8), [[UV6:%[0-9]+]]:_(s8), [[UV7:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[TRUNC]](<4 x s8>)
-    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s8)
-    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV4]](s8)
-    ; CHECK: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[ANYEXT1]], [[ANYEXT2]]
-    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[XOR]](s32)
-    ; CHECK: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s8)
-    ; CHECK: [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[UV5]](s8)
-    ; CHECK: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ANYEXT3]], [[ANYEXT4]]
-    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[XOR1]](s32)
-    ; CHECK: [[ANYEXT5:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s8)
-    ; CHECK: [[ANYEXT6:%[0-9]+]]:_(s32) = G_ANYEXT [[UV6]](s8)
-    ; CHECK: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[ANYEXT5]], [[ANYEXT6]]
-    ; CHECK: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[XOR2]](s32)
-    ; CHECK: [[ANYEXT7:%[0-9]+]]:_(s32) = G_ANYEXT [[UV3]](s8)
-    ; CHECK: [[ANYEXT8:%[0-9]+]]:_(s32) = G_ANYEXT [[UV7]](s8)
-    ; CHECK: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[ANYEXT7]], [[ANYEXT8]]
-    ; CHECK: [[TRUNC4:%[0-9]+]]:_(s8) = G_TRUNC [[XOR3]](s32)
-    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s8>) = G_BUILD_VECTOR [[TRUNC1]](s8), [[TRUNC2]](s8), [[TRUNC3]](s8), [[TRUNC4]](s8)
+    ; CHECK: [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8), [[UV6:%[0-9]+]]:_(s8), [[UV7:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[INSERT1]](<4 x s8>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s8)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV4]](s8)
+    ; CHECK: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[ANYEXT]], [[ANYEXT1]]
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[XOR]](s32)
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s8)
+    ; CHECK: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[UV5]](s8)
+    ; CHECK: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ANYEXT2]], [[ANYEXT3]]
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[XOR1]](s32)
+    ; CHECK: [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s8)
+    ; CHECK: [[ANYEXT5:%[0-9]+]]:_(s32) = G_ANYEXT [[UV6]](s8)
+    ; CHECK: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[ANYEXT4]], [[ANYEXT5]]
+    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[XOR2]](s32)
+    ; CHECK: [[ANYEXT6:%[0-9]+]]:_(s32) = G_ANYEXT [[UV3]](s8)
+    ; CHECK: [[ANYEXT7:%[0-9]+]]:_(s32) = G_ANYEXT [[UV7]](s8)
+    ; CHECK: [[XOR3:%[0-9]+]]:_(s32) = G_XOR [[ANYEXT6]], [[ANYEXT7]]
+    ; CHECK: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[XOR3]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s8>) = G_BUILD_VECTOR [[TRUNC]](s8), [[TRUNC1]](s8), [[TRUNC2]](s8), [[TRUNC3]](s8)
     ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s8>) = G_EXTRACT [[BUILD_VECTOR]](<4 x s8>), 0
-    ; CHECK: [[ANYEXT9:%[0-9]+]]:_(<3 x s32>) = G_ANYEXT [[EXTRACT]](<3 x s8>)
-    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[ANYEXT9]](<3 x s32>)
+    ; CHECK: [[ANYEXT8:%[0-9]+]]:_(<3 x s32>) = G_ANYEXT [[EXTRACT]](<3 x s8>)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[ANYEXT8]](<3 x s32>)
     %0:_(<3 x s8>) = G_IMPLICIT_DEF
     %1:_(<3 x s8>) = G_IMPLICIT_DEF
     %2:_(<3 x s8>) = G_XOR %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.set.inactive.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.set.inactive.ll
new file mode 100644
index 0000000000000..5ca821922bd22
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.set.inactive.ll
@@ -0,0 +1 @@
+; RUN: llc -global-isel -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %S/../llvm.amdgcn.set.inactive.ll |  FileCheck -check-prefix=SI -check-prefix=GCN %S/../llvm.amdgcn.set.inactive.ll
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.powi.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.powi.ll
new file mode 100644
index 0000000000000..7298f40488e75
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.powi.ll
@@ -0,0 +1,181 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii < %s | FileCheck -check-prefixes=GCN,GFX7 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji < %s | FileCheck -check-prefixes=GCN,GFX8 %s
+
+define i16 @v_powi_f16(i16 %l, i32 %r) {
+; GFX7-LABEL: v_powi_f16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX7-NEXT:    v_cvt_f32_i32_e32 v1, v1
+; GFX7-NEXT:    v_log_f32_e32 v0, v0
+; GFX7-NEXT:    v_mul_legacy_f32_e32 v0, v0, v1
+; GFX7-NEXT:    v_exp_f32_e32 v0, v0
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_powi_f16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cvt_f32_i32_e32 v1, v1
+; GFX8-NEXT:    v_log_f16_e32 v0, v0
+; GFX8-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX8-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX8-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX8-NEXT:    v_mul_legacy_f32_e32 v0, v0, v1
+; GFX8-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX8-NEXT:    v_exp_f16_e32 v0, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+  %l.cast = bitcast i16 %l to half
+  %res = call half @llvm.powi.f16(half %l.cast, i32 %r)
+  %res.cast = bitcast half %res to i16
+  ret i16 %res.cast
+}
+
+define float @v_powi_f32(float %l, i32 %r) {
+; GCN-LABEL: v_powi_f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, v1
+; GCN-NEXT:    v_log_f32_e32 v0, v0
+; GCN-NEXT:    v_mul_legacy_f32_e32 v0, v0, v1
+; GCN-NEXT:    v_exp_f32_e32 v0, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %res = call float @llvm.powi.f32(float %l, i32 %r)
+  ret float %res
+}
+
+define float @v_powi_0_f32(float %l) {
+; GCN-LABEL: v_powi_0_f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, 1.0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %res = call float @llvm.powi.f32(float %l, i32 0)
+  ret float %res
+}
+
+define float @v_powi_1_f32(float %l) {
+; GCN-LABEL: v_powi_1_f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %res = call float @llvm.powi.f32(float %l, i32 1)
+  ret float %res
+}
+
+define float @v_powi_neg1_f32(float %l) {
+; GCN-LABEL: v_powi_neg1_f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, -1
+; GCN-NEXT:    v_log_f32_e32 v0, v0
+; GCN-NEXT:    v_mul_legacy_f32_e32 v0, v0, v1
+; GCN-NEXT:    v_exp_f32_e32 v0, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %res = call float @llvm.powi.f32(float %l, i32 -1)
+  ret float %res
+}
+
+define float @v_powi_2_f32(float %l) {
+; GCN-LABEL: v_powi_2_f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_log_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_ubyte0_e32 v1, 2
+; GCN-NEXT:    v_mul_legacy_f32_e32 v0, v0, v1
+; GCN-NEXT:    v_exp_f32_e32 v0, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %res = call float @llvm.powi.f32(float %l, i32 2)
+  ret float %res
+}
+
+define float @v_powi_neg2_f32(float %l) {
+; GCN-LABEL: v_powi_neg2_f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, -2
+; GCN-NEXT:    v_log_f32_e32 v0, v0
+; GCN-NEXT:    v_mul_legacy_f32_e32 v0, v0, v1
+; GCN-NEXT:    v_exp_f32_e32 v0, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %res = call float @llvm.powi.f32(float %l, i32 -2)
+  ret float %res
+}
+
+define float @v_powi_4_f32(float %l) {
+; GCN-LABEL: v_powi_4_f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_log_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_ubyte0_e32 v1, 4
+; GCN-NEXT:    v_mul_legacy_f32_e32 v0, v0, v1
+; GCN-NEXT:    v_exp_f32_e32 v0, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %res = call float @llvm.powi.f32(float %l, i32 4)
+  ret float %res
+}
+
+define float @v_powi_8_f32(float %l) {
+; GCN-LABEL: v_powi_8_f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_log_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_ubyte0_e32 v1, 8
+; GCN-NEXT:    v_mul_legacy_f32_e32 v0, v0, v1
+; GCN-NEXT:    v_exp_f32_e32 v0, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %res = call float @llvm.powi.f32(float %l, i32 8)
+  ret float %res
+}
+
+define float @v_powi_16_f32(float %l) {
+; GCN-LABEL: v_powi_16_f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_log_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_ubyte0_e32 v1, 16
+; GCN-NEXT:    v_mul_legacy_f32_e32 v0, v0, v1
+; GCN-NEXT:    v_exp_f32_e32 v0, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %res = call float @llvm.powi.f32(float %l, i32 16)
+  ret float %res
+}
+
+define float @v_powi_128_f32(float %l) {
+; GCN-LABEL: v_powi_128_f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_log_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_f32_ubyte0_e32 v1, 0x80
+; GCN-NEXT:    v_mul_legacy_f32_e32 v0, v0, v1
+; GCN-NEXT:    v_exp_f32_e32 v0, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %res = call float @llvm.powi.f32(float %l, i32 128)
+  ret float %res
+}
+
+define float @v_powi_neg128_f32(float %l) {
+; GCN-LABEL: v_powi_neg128_f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, 0xffffff80
+; GCN-NEXT:    v_log_f32_e32 v0, v0
+; GCN-NEXT:    v_mul_legacy_f32_e32 v0, v0, v1
+; GCN-NEXT:    v_exp_f32_e32 v0, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %res = call float @llvm.powi.f32(float %l, i32 -128)
+  ret float %res
+}
+
+; FIXME: f64 broken
+; define double @v_powi_f64(double %l, i32 %r) {
+;   %res = call double @llvm.powi.f64(double %l, i32 %r)
+;   ret double %res
+; }
+
+declare half @llvm.powi.f16(half, i32) #0
+declare float @llvm.powi.f32(float, i32) #0
+declare double @llvm.powi.f64(double, i32) #0
+
+attributes #0 = { nounwind readnone speculatable willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/minmaxabs.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/minmaxabs.ll
new file mode 100644
index 0000000000000..0f84f12a12574
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/minmaxabs.ll
@@ -0,0 +1,104 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-unknown-amdhsa < %s | FileCheck %s
+
+declare i32 @llvm.umin.i32(i32, i32)
+declare i32 @llvm.umax.i32(i32, i32)
+declare i32 @llvm.smin.i32(i32, i32)
+declare i32 @llvm.smax.i32(i32, i32)
+
+declare <4 x i32> @llvm.umin.v4i32(<4 x i32>, <4 x i32>)
+declare <4 x i32> @llvm.umax.v4i32(<4 x i32>, <4 x i32>)
+declare <4 x i32> @llvm.smin.v4i32(<4 x i32>, <4 x i32>)
+declare <4 x i32> @llvm.smax.v4i32(<4 x i32>, <4 x i32>)
+
+define i32 @test_umin_i32(i32 %a, i32 %b) {
+; CHECK-LABEL: test_umin_i32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_min_u32_e32 v0, v0, v1
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %r = call i32 @llvm.umin.i32(i32 %a, i32 %b)
+  ret i32 %r
+}
+
+define i32 @test_umax_i32(i32 %a, i32 %b) {
+; CHECK-LABEL: test_umax_i32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_max_u32_e32 v0, v0, v1
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %r = call i32 @llvm.umax.i32(i32 %a, i32 %b)
+  ret i32 %r
+}
+
+define i32 @test_smin_i32(i32 %a, i32 %b) {
+; CHECK-LABEL: test_smin_i32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_min_i32_e32 v0, v0, v1
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %r = call i32 @llvm.smin.i32(i32 %a, i32 %b)
+  ret i32 %r
+}
+
+define i32 @test_smax_i32(i32 %a, i32 %b) {
+; CHECK-LABEL: test_smax_i32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_max_i32_e32 v0, v0, v1
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %r = call i32 @llvm.smax.i32(i32 %a, i32 %b)
+  ret i32 %r
+}
+
+define <4 x i32> @test_umin_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_umin_v4i32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_min_u32_e32 v0, v0, v4
+; CHECK-NEXT:    v_min_u32_e32 v1, v1, v5
+; CHECK-NEXT:    v_min_u32_e32 v2, v2, v6
+; CHECK-NEXT:    v_min_u32_e32 v3, v3, v7
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %r = call <4 x i32> @llvm.umin.v4i32(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @test_umax_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_umax_v4i32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_max_u32_e32 v0, v0, v4
+; CHECK-NEXT:    v_max_u32_e32 v1, v1, v5
+; CHECK-NEXT:    v_max_u32_e32 v2, v2, v6
+; CHECK-NEXT:    v_max_u32_e32 v3, v3, v7
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %r = call <4 x i32> @llvm.umax.v4i32(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @test_smin_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_smin_v4i32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_min_i32_e32 v0, v0, v4
+; CHECK-NEXT:    v_min_i32_e32 v1, v1, v5
+; CHECK-NEXT:    v_min_i32_e32 v2, v2, v6
+; CHECK-NEXT:    v_min_i32_e32 v3, v3, v7
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %r = call <4 x i32> @llvm.smin.v4i32(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @test_smax_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_smax_v4i32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_max_i32_e32 v0, v0, v4
+; CHECK-NEXT:    v_max_i32_e32 v1, v1, v5
+; CHECK-NEXT:    v_max_i32_e32 v2, v2, v6
+; CHECK-NEXT:    v_max_i32_e32 v3, v3, v7
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %r = call <4 x i32> @llvm.smax.v4i32(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %r
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
new file mode 100644
index 0000000000000..80ff9bb2b5752
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
@@ -0,0 +1,9901 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=tahiti -o - %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=fiji -o - %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -o - %s | FileCheck -check-prefix=GFX10 %s
+
+define i7 @v_saddsat_i7(i7 %lhs, i7 %rhs) {
+; GFX6-LABEL: v_saddsat_i7:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 25, v0
+; GFX6-NEXT:    v_min_i32_e32 v3, 0, v0
+; GFX6-NEXT:    v_max_i32_e32 v2, 0, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 25, v1
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, 0x80000000, v3
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, 0x7fffffff, v2
+; GFX6-NEXT:    v_max_i32_e32 v1, v3, v1
+; GFX6-NEXT:    v_min_i32_e32 v1, v1, v2
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 25, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_i7:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 9, v0
+; GFX8-NEXT:    v_min_i16_e32 v3, 0, v0
+; GFX8-NEXT:    v_max_i16_e32 v2, 0, v0
+; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 9, v1
+; GFX8-NEXT:    v_sub_u16_e32 v3, 0x8000, v3
+; GFX8-NEXT:    v_sub_u16_e32 v2, 0x7fff, v2
+; GFX8-NEXT:    v_max_i16_e32 v1, v3, v1
+; GFX8-NEXT:    v_min_i16_e32 v1, v1, v2
+; GFX8-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_ashrrev_i16_e32 v0, 9, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_i7:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 9, v0
+; GFX9-NEXT:    v_min_i16_e32 v3, 0, v0
+; GFX9-NEXT:    v_max_i16_e32 v2, 0, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 9, v1
+; GFX9-NEXT:    v_sub_u16_e32 v3, 0x8000, v3
+; GFX9-NEXT:    v_sub_u16_e32 v2, 0x7fff, v2
+; GFX9-NEXT:    v_max_i16_e32 v1, v3, v1
+; GFX9-NEXT:    v_min_i16_e32 v1, v1, v2
+; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_ashrrev_i16_e32 v0, 9, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_saddsat_i7:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 9, v0
+; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 9, v1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_min_i16_e64 v2, v0, 0
+; GFX10-NEXT:    v_max_i16_e64 v3, v0, 0
+; GFX10-NEXT:    v_sub_nc_u16_e64 v2, 0x8000, v2
+; GFX10-NEXT:    v_sub_nc_u16_e64 v3, 0x7fff, v3
+; GFX10-NEXT:    v_max_i16_e64 v1, v2, v1
+; GFX10-NEXT:    v_min_i16_e64 v1, v1, v3
+; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    v_ashrrev_i16_e64 v0, 9, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i7 @llvm.sadd.sat.i7(i7 %lhs, i7 %rhs)
+  ret i7 %result
+}
+
+define amdgpu_ps i7 @s_saddsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
+; GFX6-LABEL: s_saddsat_i7:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 25
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 25
+; GFX6-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX6-NEXT:    s_cselect_b32 s2, s0, 0
+; GFX6-NEXT:    s_sub_i32 s2, 0x7fffffff, s2
+; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX6-NEXT:    s_cselect_b32 s3, s0, 0
+; GFX6-NEXT:    s_sub_i32 s3, 0x80000000, s3
+; GFX6-NEXT:    s_cmp_gt_i32 s3, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX6-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX6-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX6-NEXT:    s_add_i32 s0, s0, s1
+; GFX6-NEXT:    s_ashr_i32 s0, s0, 25
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_saddsat_i7:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_bfe_u32 s2, 9, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX8-NEXT:    s_sext_i32_i16 s3, s0
+; GFX8-NEXT:    s_sext_i32_i16 s4, 0
+; GFX8-NEXT:    s_cmp_gt_i32 s3, s4
+; GFX8-NEXT:    s_cselect_b32 s5, s3, s4
+; GFX8-NEXT:    s_sub_i32 s5, 0x7fff, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX8-NEXT:    s_sub_i32 s3, 0x8000, s3
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_cmp_gt_i32 s3, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_sext_i32_i16 s3, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX8-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX8-NEXT:    s_add_i32 s0, s0, s1
+; GFX8-NEXT:    s_sext_i32_i16 s0, s0
+; GFX8-NEXT:    s_ashr_i32 s0, s0, s2
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_saddsat_i7:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_bfe_u32 s2, 9, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX9-NEXT:    s_sext_i32_i16 s3, s0
+; GFX9-NEXT:    s_sext_i32_i16 s4, 0
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s4
+; GFX9-NEXT:    s_cselect_b32 s5, s3, s4
+; GFX9-NEXT:    s_sub_i32 s5, 0x7fff, s5
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX9-NEXT:    s_sub_i32 s3, 0x8000, s3
+; GFX9-NEXT:    s_sext_i32_i16 s3, s3
+; GFX9-NEXT:    s_sext_i32_i16 s1, s1
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX9-NEXT:    s_sext_i32_i16 s1, s1
+; GFX9-NEXT:    s_sext_i32_i16 s3, s5
+; GFX9-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX9-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX9-NEXT:    s_add_i32 s0, s0, s1
+; GFX9-NEXT:    s_sext_i32_i16 s0, s0
+; GFX9-NEXT:    s_ashr_i32 s0, s0, s2
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_saddsat_i7:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_bfe_u32 s2, 9, 0x100000
+; GFX10-NEXT:    s_sext_i32_i16 s4, 0
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX10-NEXT:    s_sext_i32_i16 s3, s0
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_cmp_gt_i32 s3, s4
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s5, s3, s4
+; GFX10-NEXT:    s_sub_i32 s5, 0x7fff, s5
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX10-NEXT:    s_sub_i32 s3, 0x8000, s3
+; GFX10-NEXT:    s_sext_i32_i16 s3, s3
+; GFX10-NEXT:    s_cmp_gt_i32 s3, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX10-NEXT:    s_sext_i32_i16 s3, s5
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX10-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX10-NEXT:    s_add_i32 s0, s0, s1
+; GFX10-NEXT:    s_sext_i32_i16 s0, s0
+; GFX10-NEXT:    s_ashr_i32 s0, s0, s2
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i7 @llvm.sadd.sat.i7(i7 %lhs, i7 %rhs)
+  ret i7 %result
+}
+
+define i8 @v_saddsat_i8(i8 %lhs, i8 %rhs) {
+; GFX6-LABEL: v_saddsat_i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
+; GFX6-NEXT:    v_min_i32_e32 v3, 0, v0
+; GFX6-NEXT:    v_max_i32_e32 v2, 0, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, 0x80000000, v3
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, 0x7fffffff, v2
+; GFX6-NEXT:    v_max_i32_e32 v1, v3, v1
+; GFX6-NEXT:    v_min_i32_e32 v1, v1, v2
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 24, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX8-NEXT:    v_min_i16_e32 v3, 0, v0
+; GFX8-NEXT:    v_max_i16_e32 v2, 0, v0
+; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX8-NEXT:    v_sub_u16_e32 v3, 0x8000, v3
+; GFX8-NEXT:    v_sub_u16_e32 v2, 0x7fff, v2
+; GFX8-NEXT:    v_max_i16_e32 v1, v3, v1
+; GFX8-NEXT:    v_min_i16_e32 v1, v1, v2
+; GFX8-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_ashrrev_i16_e32 v0, 8, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    v_min_i16_e32 v3, 0, v0
+; GFX9-NEXT:    v_max_i16_e32 v2, 0, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-NEXT:    v_sub_u16_e32 v3, 0x8000, v3
+; GFX9-NEXT:    v_sub_u16_e32 v2, 0x7fff, v2
+; GFX9-NEXT:    v_max_i16_e32 v1, v3, v1
+; GFX9-NEXT:    v_min_i16_e32 v1, v1, v2
+; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_ashrrev_i16_e32 v0, 8, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_saddsat_i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 8, v0
+; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 8, v1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_min_i16_e64 v2, v0, 0
+; GFX10-NEXT:    v_max_i16_e64 v3, v0, 0
+; GFX10-NEXT:    v_sub_nc_u16_e64 v2, 0x8000, v2
+; GFX10-NEXT:    v_sub_nc_u16_e64 v3, 0x7fff, v3
+; GFX10-NEXT:    v_max_i16_e64 v1, v2, v1
+; GFX10-NEXT:    v_min_i16_e64 v1, v1, v3
+; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    v_ashrrev_i16_e64 v0, 8, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i8 @llvm.sadd.sat.i8(i8 %lhs, i8 %rhs)
+  ret i8 %result
+}
+
+define amdgpu_ps i8 @s_saddsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
+; GFX6-LABEL: s_saddsat_i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 24
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX6-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX6-NEXT:    s_cselect_b32 s2, s0, 0
+; GFX6-NEXT:    s_sub_i32 s2, 0x7fffffff, s2
+; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX6-NEXT:    s_cselect_b32 s3, s0, 0
+; GFX6-NEXT:    s_sub_i32 s3, 0x80000000, s3
+; GFX6-NEXT:    s_cmp_gt_i32 s3, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX6-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX6-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX6-NEXT:    s_add_i32 s0, s0, s1
+; GFX6-NEXT:    s_ashr_i32 s0, s0, 24
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_saddsat_i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_bfe_u32 s2, 8, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX8-NEXT:    s_sext_i32_i16 s3, s0
+; GFX8-NEXT:    s_sext_i32_i16 s4, 0
+; GFX8-NEXT:    s_cmp_gt_i32 s3, s4
+; GFX8-NEXT:    s_cselect_b32 s5, s3, s4
+; GFX8-NEXT:    s_sub_i32 s5, 0x7fff, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX8-NEXT:    s_sub_i32 s3, 0x8000, s3
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_cmp_gt_i32 s3, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_sext_i32_i16 s3, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX8-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX8-NEXT:    s_add_i32 s0, s0, s1
+; GFX8-NEXT:    s_sext_i32_i16 s0, s0
+; GFX8-NEXT:    s_ashr_i32 s0, s0, s2
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_saddsat_i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_bfe_u32 s2, 8, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX9-NEXT:    s_sext_i32_i16 s3, s0
+; GFX9-NEXT:    s_sext_i32_i16 s4, 0
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s4
+; GFX9-NEXT:    s_cselect_b32 s5, s3, s4
+; GFX9-NEXT:    s_sub_i32 s5, 0x7fff, s5
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX9-NEXT:    s_sub_i32 s3, 0x8000, s3
+; GFX9-NEXT:    s_sext_i32_i16 s3, s3
+; GFX9-NEXT:    s_sext_i32_i16 s1, s1
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX9-NEXT:    s_sext_i32_i16 s1, s1
+; GFX9-NEXT:    s_sext_i32_i16 s3, s5
+; GFX9-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX9-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX9-NEXT:    s_add_i32 s0, s0, s1
+; GFX9-NEXT:    s_sext_i32_i16 s0, s0
+; GFX9-NEXT:    s_ashr_i32 s0, s0, s2
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_saddsat_i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_bfe_u32 s2, 8, 0x100000
+; GFX10-NEXT:    s_sext_i32_i16 s4, 0
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX10-NEXT:    s_sext_i32_i16 s3, s0
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_cmp_gt_i32 s3, s4
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s5, s3, s4
+; GFX10-NEXT:    s_sub_i32 s5, 0x7fff, s5
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX10-NEXT:    s_sub_i32 s3, 0x8000, s3
+; GFX10-NEXT:    s_sext_i32_i16 s3, s3
+; GFX10-NEXT:    s_cmp_gt_i32 s3, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX10-NEXT:    s_sext_i32_i16 s3, s5
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX10-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX10-NEXT:    s_add_i32 s0, s0, s1
+; GFX10-NEXT:    s_sext_i32_i16 s0, s0
+; GFX10-NEXT:    s_ashr_i32 s0, s0, s2
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i8 @llvm.sadd.sat.i8(i8 %lhs, i8 %rhs)
+  ret i8 %result
+}
+
+define i16 @v_saddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
+; GFX6-LABEL: v_saddsat_v2i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 8, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v5, 0, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 8, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, s5, v5
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    v_max_i32_e32 v4, 0, v0
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, s4, v4
+; GFX6-NEXT:    v_max_i32_e32 v1, v5, v1
+; GFX6-NEXT:    v_min_i32_e32 v1, v1, v4
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v2
+; GFX6-NEXT:    v_min_i32_e32 v4, 0, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 24, v3
+; GFX6-NEXT:    v_max_i32_e32 v3, 0, v1
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, s5, v4
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, s4, v3
+; GFX6-NEXT:    v_max_i32_e32 v2, v4, v2
+; GFX6-NEXT:    v_min_i32_e32 v2, v2, v3
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 24, v1
+; GFX6-NEXT:    v_mov_b32_e32 v2, 0xff
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 24, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, v1, v2
+; GFX6-NEXT:    v_and_b32_e32 v0, v0, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_v2i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v2, 8
+; GFX8-NEXT:    v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX8-NEXT:    s_mov_b32 s5, 0x8000
+; GFX8-NEXT:    v_min_i16_e32 v5, 0, v0
+; GFX8-NEXT:    v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX8-NEXT:    v_sub_u16_e32 v5, s5, v5
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX8-NEXT:    v_max_i16_e32 v4, 0, v0
+; GFX8-NEXT:    v_sub_u16_e32 v4, s4, v4
+; GFX8-NEXT:    v_max_i16_e32 v1, v5, v1
+; GFX8-NEXT:    v_min_i16_e32 v1, v1, v4
+; GFX8-NEXT:    v_min_i16_e32 v4, 0, v3
+; GFX8-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_max_i16_e32 v1, 0, v3
+; GFX8-NEXT:    v_sub_u16_e32 v4, s5, v4
+; GFX8-NEXT:    v_sub_u16_e32 v1, s4, v1
+; GFX8-NEXT:    v_max_i16_e32 v2, v4, v2
+; GFX8-NEXT:    v_min_i16_e32 v1, v2, v1
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0xff
+; GFX8-NEXT:    v_add_u16_e32 v1, v3, v1
+; GFX8-NEXT:    v_and_b32_sdwa v0, sext(v0), v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v1, sext(v1), v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_v2i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    s_mov_b32 s5, 0x8000
+; GFX9-NEXT:    v_min_i16_e32 v5, 0, v0
+; GFX9-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-NEXT:    v_sub_u16_e32 v5, s5, v5
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    v_max_i16_e32 v4, 0, v0
+; GFX9-NEXT:    v_sub_u16_e32 v4, s4, v4
+; GFX9-NEXT:    v_max_i16_e32 v1, v5, v1
+; GFX9-NEXT:    v_min_i16_e32 v1, v1, v4
+; GFX9-NEXT:    v_min_i16_e32 v4, 0, v2
+; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_max_i16_e32 v1, 0, v2
+; GFX9-NEXT:    v_sub_u16_e32 v4, s5, v4
+; GFX9-NEXT:    v_sub_u16_e32 v1, s4, v1
+; GFX9-NEXT:    v_max_i16_e32 v3, v4, v3
+; GFX9-NEXT:    v_min_i16_e32 v1, v3, v1
+; GFX9-NEXT:    s_movk_i32 s4, 0xff
+; GFX9-NEXT:    v_add_u16_e32 v1, v2, v1
+; GFX9-NEXT:    v_and_b32_sdwa v0, sext(v0), s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v1, sext(v1), s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_saddsat_v2i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_mov_b32 s4, 8
+; GFX10-NEXT:    v_lshlrev_b16_e64 v2, 8, v0
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v0, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    s_mov_b32 s5, 0x8000
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 8, v1
+; GFX10-NEXT:    v_min_i16_e64 v4, v2, 0
+; GFX10-NEXT:    v_min_i16_e64 v5, v0, 0
+; GFX10-NEXT:    v_max_i16_e64 v6, v2, 0
+; GFX10-NEXT:    v_max_i16_e64 v7, v0, 0
+; GFX10-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX10-NEXT:    v_sub_nc_u16_e64 v4, s5, v4
+; GFX10-NEXT:    v_sub_nc_u16_e64 v5, s5, v5
+; GFX10-NEXT:    v_sub_nc_u16_e64 v6, s4, v6
+; GFX10-NEXT:    v_sub_nc_u16_e64 v7, s4, v7
+; GFX10-NEXT:    s_movk_i32 s4, 0xff
+; GFX10-NEXT:    v_max_i16_e64 v1, v4, v1
+; GFX10-NEXT:    v_max_i16_e64 v10, v5, v3
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_min_i16_e64 v1, v1, v6
+; GFX10-NEXT:    v_min_i16_e64 v3, v10, v7
+; GFX10-NEXT:    v_add_nc_u16_e64 v1, v2, v1
+; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v3
+; GFX10-NEXT:    v_and_b32_sdwa v1, sext(v1), s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v0, sext(v0), s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %lhs = bitcast i16 %lhs.arg to <2 x i8>
+  %rhs = bitcast i16 %rhs.arg to <2 x i8>
+  %result = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> %lhs, <2 x i8> %rhs)
+  %cast.result = bitcast <2 x i8> %result to i16
+  ret i16 %cast.result
+}
+
+define amdgpu_ps i16 @s_saddsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
+; GFX6-LABEL: s_saddsat_v2i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX6-NEXT:    s_lshr_b32 s3, s1, 8
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 24
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX6-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    s_cselect_b32 s6, s0, 0
+; GFX6-NEXT:    s_sub_i32 s6, s4, s6
+; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_cselect_b32 s7, s0, 0
+; GFX6-NEXT:    s_sub_i32 s7, s5, s7
+; GFX6-NEXT:    s_cmp_gt_i32 s7, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s7, s1
+; GFX6-NEXT:    s_cmp_lt_i32 s1, s6
+; GFX6-NEXT:    s_cselect_b32 s1, s1, s6
+; GFX6-NEXT:    s_add_i32 s0, s0, s1
+; GFX6-NEXT:    s_lshl_b32 s1, s2, 24
+; GFX6-NEXT:    s_ashr_i32 s0, s0, 24
+; GFX6-NEXT:    s_lshl_b32 s2, s3, 24
+; GFX6-NEXT:    s_cmp_gt_i32 s1, 0
+; GFX6-NEXT:    s_cselect_b32 s3, s1, 0
+; GFX6-NEXT:    s_sub_i32 s3, s4, s3
+; GFX6-NEXT:    s_cmp_lt_i32 s1, 0
+; GFX6-NEXT:    s_cselect_b32 s4, s1, 0
+; GFX6-NEXT:    s_sub_i32 s4, s5, s4
+; GFX6-NEXT:    s_cmp_gt_i32 s4, s2
+; GFX6-NEXT:    s_cselect_b32 s2, s4, s2
+; GFX6-NEXT:    s_cmp_lt_i32 s2, s3
+; GFX6-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX6-NEXT:    s_add_i32 s1, s1, s2
+; GFX6-NEXT:    s_movk_i32 s2, 0xff
+; GFX6-NEXT:    s_ashr_i32 s1, s1, 24
+; GFX6-NEXT:    s_and_b32 s1, s1, s2
+; GFX6-NEXT:    s_and_b32 s0, s0, s2
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_saddsat_v2i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_bfe_u32 s4, 8, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s4
+; GFX8-NEXT:    s_lshr_b32 s3, s1, 8
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s4
+; GFX8-NEXT:    s_sext_i32_i16 s7, s0
+; GFX8-NEXT:    s_sext_i32_i16 s8, 0
+; GFX8-NEXT:    s_cmp_gt_i32 s7, s8
+; GFX8-NEXT:    s_movk_i32 s5, 0x7fff
+; GFX8-NEXT:    s_cselect_b32 s9, s7, s8
+; GFX8-NEXT:    s_sub_i32 s9, s5, s9
+; GFX8-NEXT:    s_cmp_lt_i32 s7, s8
+; GFX8-NEXT:    s_mov_b32 s6, 0x8000
+; GFX8-NEXT:    s_cselect_b32 s7, s7, s8
+; GFX8-NEXT:    s_sub_i32 s7, s6, s7
+; GFX8-NEXT:    s_sext_i32_i16 s7, s7
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_cmp_gt_i32 s7, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s7, s1
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_sext_i32_i16 s7, s9
+; GFX8-NEXT:    s_cmp_lt_i32 s1, s7
+; GFX8-NEXT:    s_cselect_b32 s1, s1, s7
+; GFX8-NEXT:    s_add_i32 s0, s0, s1
+; GFX8-NEXT:    s_sext_i32_i16 s0, s0
+; GFX8-NEXT:    s_lshl_b32 s1, s2, s4
+; GFX8-NEXT:    s_lshl_b32 s2, s3, s4
+; GFX8-NEXT:    s_ashr_i32 s0, s0, s4
+; GFX8-NEXT:    s_sext_i32_i16 s3, s1
+; GFX8-NEXT:    s_cmp_gt_i32 s3, s8
+; GFX8-NEXT:    s_cselect_b32 s7, s3, s8
+; GFX8-NEXT:    s_sub_i32 s5, s5, s7
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s8
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s8
+; GFX8-NEXT:    s_sub_i32 s3, s6, s3
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_sext_i32_i16 s2, s2
+; GFX8-NEXT:    s_cmp_gt_i32 s3, s2
+; GFX8-NEXT:    s_cselect_b32 s2, s3, s2
+; GFX8-NEXT:    s_sext_i32_i16 s2, s2
+; GFX8-NEXT:    s_sext_i32_i16 s3, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s2, s3
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX8-NEXT:    s_add_i32 s1, s1, s2
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_movk_i32 s2, 0xff
+; GFX8-NEXT:    s_ashr_i32 s1, s1, s4
+; GFX8-NEXT:    s_and_b32 s1, s1, s2
+; GFX8-NEXT:    s_and_b32 s0, s0, s2
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s4
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_saddsat_v2i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_bfe_u32 s4, 8, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s4
+; GFX9-NEXT:    s_lshr_b32 s3, s1, 8
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s4
+; GFX9-NEXT:    s_sext_i32_i16 s7, s0
+; GFX9-NEXT:    s_sext_i32_i16 s8, 0
+; GFX9-NEXT:    s_cmp_gt_i32 s7, s8
+; GFX9-NEXT:    s_movk_i32 s5, 0x7fff
+; GFX9-NEXT:    s_cselect_b32 s9, s7, s8
+; GFX9-NEXT:    s_sub_i32 s9, s5, s9
+; GFX9-NEXT:    s_cmp_lt_i32 s7, s8
+; GFX9-NEXT:    s_mov_b32 s6, 0x8000
+; GFX9-NEXT:    s_cselect_b32 s7, s7, s8
+; GFX9-NEXT:    s_sub_i32 s7, s6, s7
+; GFX9-NEXT:    s_sext_i32_i16 s7, s7
+; GFX9-NEXT:    s_sext_i32_i16 s1, s1
+; GFX9-NEXT:    s_cmp_gt_i32 s7, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s7, s1
+; GFX9-NEXT:    s_sext_i32_i16 s1, s1
+; GFX9-NEXT:    s_sext_i32_i16 s7, s9
+; GFX9-NEXT:    s_cmp_lt_i32 s1, s7
+; GFX9-NEXT:    s_cselect_b32 s1, s1, s7
+; GFX9-NEXT:    s_add_i32 s0, s0, s1
+; GFX9-NEXT:    s_sext_i32_i16 s0, s0
+; GFX9-NEXT:    s_lshl_b32 s1, s2, s4
+; GFX9-NEXT:    s_lshl_b32 s2, s3, s4
+; GFX9-NEXT:    s_ashr_i32 s0, s0, s4
+; GFX9-NEXT:    s_sext_i32_i16 s3, s1
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s8
+; GFX9-NEXT:    s_cselect_b32 s7, s3, s8
+; GFX9-NEXT:    s_sub_i32 s5, s5, s7
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s8
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s8
+; GFX9-NEXT:    s_sub_i32 s3, s6, s3
+; GFX9-NEXT:    s_sext_i32_i16 s3, s3
+; GFX9-NEXT:    s_sext_i32_i16 s2, s2
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s2
+; GFX9-NEXT:    s_cselect_b32 s2, s3, s2
+; GFX9-NEXT:    s_sext_i32_i16 s2, s2
+; GFX9-NEXT:    s_sext_i32_i16 s3, s5
+; GFX9-NEXT:    s_cmp_lt_i32 s2, s3
+; GFX9-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX9-NEXT:    s_add_i32 s1, s1, s2
+; GFX9-NEXT:    s_sext_i32_i16 s1, s1
+; GFX9-NEXT:    s_movk_i32 s2, 0xff
+; GFX9-NEXT:    s_ashr_i32 s1, s1, s4
+; GFX9-NEXT:    s_and_b32 s1, s1, s2
+; GFX9-NEXT:    s_and_b32 s0, s0, s2
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s4
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_saddsat_v2i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_bfe_u32 s2, 8, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s3, s0, 8
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX10-NEXT:    s_sext_i32_i16 s6, 0
+; GFX10-NEXT:    s_sext_i32_i16 s5, s0
+; GFX10-NEXT:    s_lshr_b32 s4, s1, 8
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX10-NEXT:    s_cmp_gt_i32 s5, s6
+; GFX10-NEXT:    s_movk_i32 s7, 0x7fff
+; GFX10-NEXT:    s_cselect_b32 s8, s5, s6
+; GFX10-NEXT:    s_mov_b32 s9, 0x8000
+; GFX10-NEXT:    s_sub_i32 s8, s7, s8
+; GFX10-NEXT:    s_cmp_lt_i32 s5, s6
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_sub_i32 s5, s9, s5
+; GFX10-NEXT:    s_sext_i32_i16 s5, s5
+; GFX10-NEXT:    s_cmp_gt_i32 s5, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s5, s1
+; GFX10-NEXT:    s_sext_i32_i16 s5, s8
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_cmp_lt_i32 s1, s5
+; GFX10-NEXT:    s_cselect_b32 s1, s1, s5
+; GFX10-NEXT:    s_lshl_b32 s3, s3, s2
+; GFX10-NEXT:    s_add_i32 s0, s0, s1
+; GFX10-NEXT:    s_lshl_b32 s1, s4, s2
+; GFX10-NEXT:    s_sext_i32_i16 s0, s0
+; GFX10-NEXT:    s_sext_i32_i16 s4, s3
+; GFX10-NEXT:    s_ashr_i32 s0, s0, s2
+; GFX10-NEXT:    s_cmp_gt_i32 s4, s6
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_cselect_b32 s5, s4, s6
+; GFX10-NEXT:    s_sub_i32 s5, s7, s5
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s6
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX10-NEXT:    s_sub_i32 s4, s9, s4
+; GFX10-NEXT:    s_sext_i32_i16 s4, s4
+; GFX10-NEXT:    s_cmp_gt_i32 s4, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s4, s1
+; GFX10-NEXT:    s_sext_i32_i16 s4, s5
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_cmp_lt_i32 s1, s4
+; GFX10-NEXT:    s_cselect_b32 s1, s1, s4
+; GFX10-NEXT:    s_add_i32 s3, s3, s1
+; GFX10-NEXT:    s_sext_i32_i16 s1, s3
+; GFX10-NEXT:    s_movk_i32 s3, 0xff
+; GFX10-NEXT:    s_ashr_i32 s1, s1, s2
+; GFX10-NEXT:    s_and_b32 s0, s0, s3
+; GFX10-NEXT:    s_and_b32 s1, s1, s3
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %lhs = bitcast i16 %lhs.arg to <2 x i8>
+  %rhs = bitcast i16 %rhs.arg to <2 x i8>
+  %result = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> %lhs, <2 x i8> %rhs)
+  %cast.result = bitcast <2 x i8> %result to i16
+  ret i16 %cast.result
+}
+
+define i32 @v_saddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
+; GFX6-LABEL: v_saddsat_v4i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 8, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 24, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v10, 0, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v5, 8, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
+; GFX6-NEXT:    v_sub_i32_e32 v10, vcc, s5, v10
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    v_max_i32_e32 v8, 0, v0
+; GFX6-NEXT:    v_sub_i32_e32 v8, vcc, s4, v8
+; GFX6-NEXT:    v_max_i32_e32 v1, v10, v1
+; GFX6-NEXT:    v_min_i32_e32 v1, v1, v8
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v2
+; GFX6-NEXT:    v_min_i32_e32 v8, 0, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 24, v5
+; GFX6-NEXT:    v_max_i32_e32 v5, 0, v1
+; GFX6-NEXT:    v_sub_i32_e32 v8, vcc, s5, v8
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, s4, v5
+; GFX6-NEXT:    v_max_i32_e32 v2, v8, v2
+; GFX6-NEXT:    v_min_i32_e32 v2, v2, v5
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 24, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 24, v6
+; GFX6-NEXT:    v_min_i32_e32 v6, 0, v2
+; GFX6-NEXT:    v_sub_i32_e32 v6, vcc, s5, v6
+; GFX6-NEXT:    v_bfrev_b32_e32 v9, -2
+; GFX6-NEXT:    v_max_i32_e32 v5, 0, v2
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, v9, v5
+; GFX6-NEXT:    v_max_i32_e32 v3, v6, v3
+; GFX6-NEXT:    v_min_i32_e32 v3, v3, v5
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 24, v4
+; GFX6-NEXT:    v_mov_b32_e32 v11, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v6, 0, v3
+; GFX6-NEXT:    v_max_i32_e32 v5, 0, v3
+; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 24, v1
+; GFX6-NEXT:    s_movk_i32 s4, 0xff
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 24, v7
+; GFX6-NEXT:    v_sub_i32_e32 v6, vcc, v11, v6
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 24, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, v9, v5
+; GFX6-NEXT:    v_max_i32_e32 v4, v6, v4
+; GFX6-NEXT:    v_min_i32_e32 v4, v4, v5
+; GFX6-NEXT:    v_ashrrev_i32_e32 v2, 24, v2
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_ashrrev_i32_e32 v3, 24, v3
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_v4i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v2, 8
+; GFX8-NEXT:    v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 24, v0
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX8-NEXT:    s_mov_b32 s5, 0x8000
+; GFX8-NEXT:    v_min_i16_e32 v10, 0, v0
+; GFX8-NEXT:    v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
+; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX8-NEXT:    v_sub_u16_e32 v10, s5, v10
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX8-NEXT:    v_max_i16_e32 v8, 0, v0
+; GFX8-NEXT:    v_sub_u16_e32 v8, s4, v8
+; GFX8-NEXT:    v_max_i16_e32 v1, v10, v1
+; GFX8-NEXT:    v_min_i16_e32 v1, v1, v8
+; GFX8-NEXT:    v_min_i16_e32 v8, 0, v3
+; GFX8-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_max_i16_e32 v1, 0, v3
+; GFX8-NEXT:    v_sub_u16_e32 v8, s5, v8
+; GFX8-NEXT:    v_sub_u16_e32 v1, s4, v1
+; GFX8-NEXT:    v_max_i16_e32 v2, v8, v2
+; GFX8-NEXT:    v_min_i16_e32 v1, v2, v1
+; GFX8-NEXT:    v_lshlrev_b16_e32 v2, 8, v4
+; GFX8-NEXT:    v_add_u16_e32 v1, v3, v1
+; GFX8-NEXT:    v_lshlrev_b16_e32 v3, 8, v6
+; GFX8-NEXT:    v_min_i16_e32 v6, 0, v2
+; GFX8-NEXT:    v_sub_u16_e32 v6, s5, v6
+; GFX8-NEXT:    v_mov_b32_e32 v9, 0x7fff
+; GFX8-NEXT:    v_max_i16_e32 v4, 0, v2
+; GFX8-NEXT:    v_sub_u16_e32 v4, v9, v4
+; GFX8-NEXT:    v_max_i16_e32 v3, v6, v3
+; GFX8-NEXT:    v_min_i16_e32 v3, v3, v4
+; GFX8-NEXT:    v_add_u16_e32 v2, v2, v3
+; GFX8-NEXT:    v_lshlrev_b16_e32 v3, 8, v5
+; GFX8-NEXT:    v_min_i16_e32 v6, 0, v3
+; GFX8-NEXT:    v_max_i16_e32 v5, 0, v3
+; GFX8-NEXT:    v_lshlrev_b16_e32 v4, 8, v7
+; GFX8-NEXT:    v_sub_u16_e32 v6, 0x8000, v6
+; GFX8-NEXT:    v_sub_u16_e32 v5, v9, v5
+; GFX8-NEXT:    v_max_i16_e32 v4, v6, v4
+; GFX8-NEXT:    v_min_i16_e32 v4, v4, v5
+; GFX8-NEXT:    v_add_u16_e32 v3, v3, v4
+; GFX8-NEXT:    v_mov_b32_e32 v4, 0xff
+; GFX8-NEXT:    v_and_b32_sdwa v1, sext(v1), v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v0, sext(v0), v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_and_b32_sdwa v1, sext(v2), v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_and_b32_sdwa v1, sext(v3), v4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_v4i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 24, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    s_mov_b32 s5, 0x8000
+; GFX9-NEXT:    v_min_i16_e32 v10, 0, v0
+; GFX9-NEXT:    v_lshrrev_b32_sdwa v5, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-NEXT:    v_sub_u16_e32 v10, s5, v10
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    v_max_i16_e32 v8, 0, v0
+; GFX9-NEXT:    v_sub_u16_e32 v8, s4, v8
+; GFX9-NEXT:    v_max_i16_e32 v1, v10, v1
+; GFX9-NEXT:    v_min_i16_e32 v1, v1, v8
+; GFX9-NEXT:    v_min_i16_e32 v8, 0, v2
+; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_max_i16_e32 v1, 0, v2
+; GFX9-NEXT:    v_sub_u16_e32 v8, s5, v8
+; GFX9-NEXT:    v_sub_u16_e32 v1, s4, v1
+; GFX9-NEXT:    v_max_i16_e32 v5, v8, v5
+; GFX9-NEXT:    v_min_i16_e32 v1, v5, v1
+; GFX9-NEXT:    v_add_u16_e32 v1, v2, v1
+; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v3
+; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v6
+; GFX9-NEXT:    v_min_i16_e32 v6, 0, v2
+; GFX9-NEXT:    v_sub_u16_e32 v6, s5, v6
+; GFX9-NEXT:    v_mov_b32_e32 v9, 0x7fff
+; GFX9-NEXT:    v_max_i16_e32 v5, 0, v2
+; GFX9-NEXT:    v_sub_u16_e32 v5, v9, v5
+; GFX9-NEXT:    v_max_i16_e32 v3, v6, v3
+; GFX9-NEXT:    v_min_i16_e32 v3, v3, v5
+; GFX9-NEXT:    v_add_u16_e32 v2, v2, v3
+; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v4
+; GFX9-NEXT:    v_min_i16_e32 v6, 0, v3
+; GFX9-NEXT:    v_max_i16_e32 v5, 0, v3
+; GFX9-NEXT:    v_lshlrev_b16_e32 v4, 8, v7
+; GFX9-NEXT:    v_sub_u16_e32 v6, 0x8000, v6
+; GFX9-NEXT:    s_movk_i32 s4, 0xff
+; GFX9-NEXT:    v_sub_u16_e32 v5, v9, v5
+; GFX9-NEXT:    v_max_i16_e32 v4, v6, v4
+; GFX9-NEXT:    v_and_b32_sdwa v1, sext(v1), s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_min_i16_e32 v4, v4, v5
+; GFX9-NEXT:    v_ashrrev_i16_e32 v0, 8, v0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX9-NEXT:    v_add_u16_e32 v3, v3, v4
+; GFX9-NEXT:    v_and_or_b32 v0, v0, s4, v1
+; GFX9-NEXT:    v_and_b32_sdwa v1, sext(v2), s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v2, sext(v3), s4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_saddsat_v4i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_lshlrev_b16_e64 v4, 8, v0
+; GFX10-NEXT:    s_mov_b32 s4, 8
+; GFX10-NEXT:    s_mov_b32 s5, 16
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s5, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_min_i16_e64 v8, v4, 0
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v6, s5, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    s_mov_b32 s5, 0x8000
+; GFX10-NEXT:    v_min_i16_e64 v9, v2, 0
+; GFX10-NEXT:    v_lshlrev_b16_e64 v7, 8, v1
+; GFX10-NEXT:    v_sub_nc_u16_e64 v8, s5, v8
+; GFX10-NEXT:    v_max_i16_e64 v10, v4, 0
+; GFX10-NEXT:    s_mov_b32 s6, 24
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v5, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX10-NEXT:    v_sub_nc_u16_e64 v15, s5, v9
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v0, s6, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_max_i16_e64 v11, v2, 0
+; GFX10-NEXT:    v_max_i16_e64 v7, v8, v7
+; GFX10-NEXT:    v_sub_nc_u16_e64 v10, s4, v10
+; GFX10-NEXT:    v_max_i16_e64 v5, v15, v5
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v1, s6, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_sub_nc_u16_e64 v8, s4, v11
+; GFX10-NEXT:    v_min_i16_e64 v11, v3, 0
+; GFX10-NEXT:    v_min_i16_e64 v7, v7, v10
+; GFX10-NEXT:    v_min_i16_e64 v10, v0, 0
+; GFX10-NEXT:    v_mov_b32_e32 v9, 0x7fff
+; GFX10-NEXT:    v_min_i16_e64 v5, v5, v8
+; GFX10-NEXT:    v_sub_nc_u16_e64 v11, s5, v11
+; GFX10-NEXT:    v_max_i16_e64 v8, v3, 0
+; GFX10-NEXT:    v_sub_nc_u16_e64 v10, 0x8000, v10
+; GFX10-NEXT:    v_max_i16_e64 v12, v0, 0
+; GFX10-NEXT:    v_add_nc_u16_e64 v2, v2, v5
+; GFX10-NEXT:    v_max_i16_e64 v6, v11, v6
+; GFX10-NEXT:    v_sub_nc_u16_e64 v5, v9, v8
+; GFX10-NEXT:    v_max_i16_e64 v1, v10, v1
+; GFX10-NEXT:    v_sub_nc_u16_e64 v8, v9, v12
+; GFX10-NEXT:    s_movk_i32 s4, 0xff
+; GFX10-NEXT:    v_add_nc_u16_e64 v4, v4, v7
+; GFX10-NEXT:    v_and_b32_sdwa v2, sext(v2), s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_min_i16_e64 v5, v6, v5
+; GFX10-NEXT:    v_min_i16_e64 v1, v1, v8
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_ashrrev_i16_e64 v4, 8, v4
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX10-NEXT:    v_add_nc_u16_e64 v3, v3, v5
+; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    v_and_or_b32 v1, v4, s4, v2
+; GFX10-NEXT:    v_and_b32_sdwa v2, sext(v3), s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v0, sext(v0), s4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_or3_b32 v0, v1, v2, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %lhs = bitcast i32 %lhs.arg to <4 x i8>
+  %rhs = bitcast i32 %rhs.arg to <4 x i8>
+  %result = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> %lhs, <4 x i8> %rhs)
+  %cast.result = bitcast <4 x i8> %result to i32
+  ret i32 %cast.result
+}
+
+define amdgpu_ps i32 @s_saddsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
+; GFX6-LABEL: s_saddsat_v4i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX6-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX6-NEXT:    s_lshr_b32 s4, s0, 24
+; GFX6-NEXT:    s_lshr_b32 s5, s1, 8
+; GFX6-NEXT:    s_lshr_b32 s6, s1, 16
+; GFX6-NEXT:    s_lshr_b32 s7, s1, 24
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 24
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX6-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX6-NEXT:    s_brev_b32 s8, -2
+; GFX6-NEXT:    s_cselect_b32 s10, s0, 0
+; GFX6-NEXT:    s_sub_i32 s10, s8, s10
+; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX6-NEXT:    s_mov_b32 s9, 0x80000000
+; GFX6-NEXT:    s_cselect_b32 s11, s0, 0
+; GFX6-NEXT:    s_sub_i32 s11, s9, s11
+; GFX6-NEXT:    s_cmp_gt_i32 s11, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s11, s1
+; GFX6-NEXT:    s_cmp_lt_i32 s1, s10
+; GFX6-NEXT:    s_cselect_b32 s1, s1, s10
+; GFX6-NEXT:    s_add_i32 s0, s0, s1
+; GFX6-NEXT:    s_lshl_b32 s1, s2, 24
+; GFX6-NEXT:    s_ashr_i32 s0, s0, 24
+; GFX6-NEXT:    s_lshl_b32 s2, s5, 24
+; GFX6-NEXT:    s_cmp_gt_i32 s1, 0
+; GFX6-NEXT:    s_cselect_b32 s5, s1, 0
+; GFX6-NEXT:    s_sub_i32 s5, s8, s5
+; GFX6-NEXT:    s_cmp_lt_i32 s1, 0
+; GFX6-NEXT:    s_cselect_b32 s10, s1, 0
+; GFX6-NEXT:    s_sub_i32 s10, s9, s10
+; GFX6-NEXT:    s_cmp_gt_i32 s10, s2
+; GFX6-NEXT:    s_cselect_b32 s2, s10, s2
+; GFX6-NEXT:    s_cmp_lt_i32 s2, s5
+; GFX6-NEXT:    s_cselect_b32 s2, s2, s5
+; GFX6-NEXT:    s_add_i32 s1, s1, s2
+; GFX6-NEXT:    s_lshl_b32 s2, s3, 24
+; GFX6-NEXT:    s_ashr_i32 s1, s1, 24
+; GFX6-NEXT:    s_lshl_b32 s3, s6, 24
+; GFX6-NEXT:    s_cmp_gt_i32 s2, 0
+; GFX6-NEXT:    s_cselect_b32 s5, s2, 0
+; GFX6-NEXT:    s_sub_i32 s5, s8, s5
+; GFX6-NEXT:    s_cmp_lt_i32 s2, 0
+; GFX6-NEXT:    s_cselect_b32 s6, s2, 0
+; GFX6-NEXT:    s_sub_i32 s6, s9, s6
+; GFX6-NEXT:    s_cmp_gt_i32 s6, s3
+; GFX6-NEXT:    s_cselect_b32 s3, s6, s3
+; GFX6-NEXT:    s_cmp_lt_i32 s3, s5
+; GFX6-NEXT:    s_cselect_b32 s3, s3, s5
+; GFX6-NEXT:    s_add_i32 s2, s2, s3
+; GFX6-NEXT:    s_lshl_b32 s3, s4, 24
+; GFX6-NEXT:    s_ashr_i32 s2, s2, 24
+; GFX6-NEXT:    s_lshl_b32 s4, s7, 24
+; GFX6-NEXT:    s_cmp_gt_i32 s3, 0
+; GFX6-NEXT:    s_cselect_b32 s5, s3, 0
+; GFX6-NEXT:    s_sub_i32 s5, s8, s5
+; GFX6-NEXT:    s_cmp_lt_i32 s3, 0
+; GFX6-NEXT:    s_cselect_b32 s6, s3, 0
+; GFX6-NEXT:    s_sub_i32 s6, s9, s6
+; GFX6-NEXT:    s_cmp_gt_i32 s6, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s6, s4
+; GFX6-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX6-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX6-NEXT:    s_add_i32 s3, s3, s4
+; GFX6-NEXT:    s_movk_i32 s4, 0xff
+; GFX6-NEXT:    s_and_b32 s1, s1, s4
+; GFX6-NEXT:    s_and_b32 s0, s0, s4
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_and_b32 s1, s2, s4
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_ashr_i32 s3, s3, 24
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_and_b32 s1, s3, s4
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_saddsat_v4i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_bfe_u32 s8, 8, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX8-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s4, s0, 24
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s8
+; GFX8-NEXT:    s_lshr_b32 s5, s1, 8
+; GFX8-NEXT:    s_lshr_b32 s6, s1, 16
+; GFX8-NEXT:    s_lshr_b32 s7, s1, 24
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s8
+; GFX8-NEXT:    s_sext_i32_i16 s11, s0
+; GFX8-NEXT:    s_sext_i32_i16 s12, 0
+; GFX8-NEXT:    s_cmp_gt_i32 s11, s12
+; GFX8-NEXT:    s_movk_i32 s9, 0x7fff
+; GFX8-NEXT:    s_cselect_b32 s13, s11, s12
+; GFX8-NEXT:    s_sub_i32 s13, s9, s13
+; GFX8-NEXT:    s_cmp_lt_i32 s11, s12
+; GFX8-NEXT:    s_mov_b32 s10, 0x8000
+; GFX8-NEXT:    s_cselect_b32 s11, s11, s12
+; GFX8-NEXT:    s_sub_i32 s11, s10, s11
+; GFX8-NEXT:    s_sext_i32_i16 s11, s11
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_cmp_gt_i32 s11, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s11, s1
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_sext_i32_i16 s11, s13
+; GFX8-NEXT:    s_cmp_lt_i32 s1, s11
+; GFX8-NEXT:    s_cselect_b32 s1, s1, s11
+; GFX8-NEXT:    s_add_i32 s0, s0, s1
+; GFX8-NEXT:    s_sext_i32_i16 s0, s0
+; GFX8-NEXT:    s_lshl_b32 s1, s2, s8
+; GFX8-NEXT:    s_lshl_b32 s2, s5, s8
+; GFX8-NEXT:    s_ashr_i32 s0, s0, s8
+; GFX8-NEXT:    s_sext_i32_i16 s5, s1
+; GFX8-NEXT:    s_cmp_gt_i32 s5, s12
+; GFX8-NEXT:    s_cselect_b32 s11, s5, s12
+; GFX8-NEXT:    s_sub_i32 s11, s9, s11
+; GFX8-NEXT:    s_cmp_lt_i32 s5, s12
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s12
+; GFX8-NEXT:    s_sub_i32 s5, s10, s5
+; GFX8-NEXT:    s_sext_i32_i16 s5, s5
+; GFX8-NEXT:    s_sext_i32_i16 s2, s2
+; GFX8-NEXT:    s_cmp_gt_i32 s5, s2
+; GFX8-NEXT:    s_cselect_b32 s2, s5, s2
+; GFX8-NEXT:    s_sext_i32_i16 s2, s2
+; GFX8-NEXT:    s_sext_i32_i16 s5, s11
+; GFX8-NEXT:    s_cmp_lt_i32 s2, s5
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s5
+; GFX8-NEXT:    s_add_i32 s1, s1, s2
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_lshl_b32 s2, s3, s8
+; GFX8-NEXT:    s_lshl_b32 s3, s6, s8
+; GFX8-NEXT:    s_ashr_i32 s1, s1, s8
+; GFX8-NEXT:    s_sext_i32_i16 s5, s2
+; GFX8-NEXT:    s_cmp_gt_i32 s5, s12
+; GFX8-NEXT:    s_cselect_b32 s6, s5, s12
+; GFX8-NEXT:    s_sub_i32 s6, s9, s6
+; GFX8-NEXT:    s_cmp_lt_i32 s5, s12
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s12
+; GFX8-NEXT:    s_sub_i32 s5, s10, s5
+; GFX8-NEXT:    s_sext_i32_i16 s5, s5
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_cmp_gt_i32 s5, s3
+; GFX8-NEXT:    s_cselect_b32 s3, s5, s3
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_sext_i32_i16 s5, s6
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s5
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s5
+; GFX8-NEXT:    s_add_i32 s2, s2, s3
+; GFX8-NEXT:    s_sext_i32_i16 s2, s2
+; GFX8-NEXT:    s_lshl_b32 s3, s4, s8
+; GFX8-NEXT:    s_lshl_b32 s4, s7, s8
+; GFX8-NEXT:    s_ashr_i32 s2, s2, s8
+; GFX8-NEXT:    s_sext_i32_i16 s5, s3
+; GFX8-NEXT:    s_cmp_gt_i32 s5, s12
+; GFX8-NEXT:    s_cselect_b32 s6, s5, s12
+; GFX8-NEXT:    s_sub_i32 s6, s9, s6
+; GFX8-NEXT:    s_cmp_lt_i32 s5, s12
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s12
+; GFX8-NEXT:    s_sub_i32 s5, s10, s5
+; GFX8-NEXT:    s_sext_i32_i16 s5, s5
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_cmp_gt_i32 s5, s4
+; GFX8-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_sext_i32_i16 s5, s6
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX8-NEXT:    s_add_i32 s3, s3, s4
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_and_b32 s1, s1, s4
+; GFX8-NEXT:    s_and_b32 s0, s0, s4
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    s_and_b32 s1, s2, s4
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX8-NEXT:    s_ashr_i32 s3, s3, s8
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    s_and_b32 s1, s3, s4
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_saddsat_v4i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_bfe_u32 s8, 8, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX9-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s4, s0, 24
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s8
+; GFX9-NEXT:    s_lshr_b32 s5, s1, 8
+; GFX9-NEXT:    s_lshr_b32 s6, s1, 16
+; GFX9-NEXT:    s_lshr_b32 s7, s1, 24
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s8
+; GFX9-NEXT:    s_sext_i32_i16 s11, s0
+; GFX9-NEXT:    s_sext_i32_i16 s12, 0
+; GFX9-NEXT:    s_cmp_gt_i32 s11, s12
+; GFX9-NEXT:    s_movk_i32 s9, 0x7fff
+; GFX9-NEXT:    s_cselect_b32 s13, s11, s12
+; GFX9-NEXT:    s_sub_i32 s13, s9, s13
+; GFX9-NEXT:    s_cmp_lt_i32 s11, s12
+; GFX9-NEXT:    s_mov_b32 s10, 0x8000
+; GFX9-NEXT:    s_cselect_b32 s11, s11, s12
+; GFX9-NEXT:    s_sub_i32 s11, s10, s11
+; GFX9-NEXT:    s_sext_i32_i16 s11, s11
+; GFX9-NEXT:    s_sext_i32_i16 s1, s1
+; GFX9-NEXT:    s_cmp_gt_i32 s11, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s11, s1
+; GFX9-NEXT:    s_sext_i32_i16 s1, s1
+; GFX9-NEXT:    s_sext_i32_i16 s11, s13
+; GFX9-NEXT:    s_cmp_lt_i32 s1, s11
+; GFX9-NEXT:    s_cselect_b32 s1, s1, s11
+; GFX9-NEXT:    s_add_i32 s0, s0, s1
+; GFX9-NEXT:    s_sext_i32_i16 s0, s0
+; GFX9-NEXT:    s_lshl_b32 s1, s2, s8
+; GFX9-NEXT:    s_lshl_b32 s2, s5, s8
+; GFX9-NEXT:    s_ashr_i32 s0, s0, s8
+; GFX9-NEXT:    s_sext_i32_i16 s5, s1
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s12
+; GFX9-NEXT:    s_cselect_b32 s11, s5, s12
+; GFX9-NEXT:    s_sub_i32 s11, s9, s11
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s12
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s12
+; GFX9-NEXT:    s_sub_i32 s5, s10, s5
+; GFX9-NEXT:    s_sext_i32_i16 s5, s5
+; GFX9-NEXT:    s_sext_i32_i16 s2, s2
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s2
+; GFX9-NEXT:    s_cselect_b32 s2, s5, s2
+; GFX9-NEXT:    s_sext_i32_i16 s2, s2
+; GFX9-NEXT:    s_sext_i32_i16 s5, s11
+; GFX9-NEXT:    s_cmp_lt_i32 s2, s5
+; GFX9-NEXT:    s_cselect_b32 s2, s2, s5
+; GFX9-NEXT:    s_add_i32 s1, s1, s2
+; GFX9-NEXT:    s_sext_i32_i16 s1, s1
+; GFX9-NEXT:    s_lshl_b32 s2, s3, s8
+; GFX9-NEXT:    s_lshl_b32 s3, s6, s8
+; GFX9-NEXT:    s_ashr_i32 s1, s1, s8
+; GFX9-NEXT:    s_sext_i32_i16 s5, s2
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s12
+; GFX9-NEXT:    s_cselect_b32 s6, s5, s12
+; GFX9-NEXT:    s_sub_i32 s6, s9, s6
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s12
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s12
+; GFX9-NEXT:    s_sub_i32 s5, s10, s5
+; GFX9-NEXT:    s_sext_i32_i16 s5, s5
+; GFX9-NEXT:    s_sext_i32_i16 s3, s3
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s3
+; GFX9-NEXT:    s_cselect_b32 s3, s5, s3
+; GFX9-NEXT:    s_sext_i32_i16 s3, s3
+; GFX9-NEXT:    s_sext_i32_i16 s5, s6
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s5
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s5
+; GFX9-NEXT:    s_add_i32 s2, s2, s3
+; GFX9-NEXT:    s_sext_i32_i16 s2, s2
+; GFX9-NEXT:    s_lshl_b32 s3, s4, s8
+; GFX9-NEXT:    s_lshl_b32 s4, s7, s8
+; GFX9-NEXT:    s_ashr_i32 s2, s2, s8
+; GFX9-NEXT:    s_sext_i32_i16 s5, s3
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s12
+; GFX9-NEXT:    s_cselect_b32 s6, s5, s12
+; GFX9-NEXT:    s_sub_i32 s6, s9, s6
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s12
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s12
+; GFX9-NEXT:    s_sub_i32 s5, s10, s5
+; GFX9-NEXT:    s_sext_i32_i16 s5, s5
+; GFX9-NEXT:    s_sext_i32_i16 s4, s4
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s4
+; GFX9-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX9-NEXT:    s_sext_i32_i16 s4, s4
+; GFX9-NEXT:    s_sext_i32_i16 s5, s6
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX9-NEXT:    s_add_i32 s3, s3, s4
+; GFX9-NEXT:    s_movk_i32 s4, 0xff
+; GFX9-NEXT:    s_and_b32 s1, s1, s4
+; GFX9-NEXT:    s_and_b32 s0, s0, s4
+; GFX9-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX9-NEXT:    s_sext_i32_i16 s3, s3
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    s_and_b32 s1, s2, s4
+; GFX9-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX9-NEXT:    s_ashr_i32 s3, s3, s8
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    s_and_b32 s1, s3, s4
+; GFX9-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_saddsat_v4i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_bfe_u32 s6, 8, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX10-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX10-NEXT:    s_lshr_b32 s4, s0, 24
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s6
+; GFX10-NEXT:    s_sext_i32_i16 s10, 0
+; GFX10-NEXT:    s_sext_i32_i16 s9, s0
+; GFX10-NEXT:    s_lshr_b32 s5, s1, 8
+; GFX10-NEXT:    s_lshr_b32 s7, s1, 16
+; GFX10-NEXT:    s_lshr_b32 s8, s1, 24
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s6
+; GFX10-NEXT:    s_cmp_gt_i32 s9, s10
+; GFX10-NEXT:    s_movk_i32 s11, 0x7fff
+; GFX10-NEXT:    s_cselect_b32 s12, s9, s10
+; GFX10-NEXT:    s_mov_b32 s13, 0x8000
+; GFX10-NEXT:    s_sub_i32 s12, s11, s12
+; GFX10-NEXT:    s_cmp_lt_i32 s9, s10
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_cselect_b32 s9, s9, s10
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_sub_i32 s9, s13, s9
+; GFX10-NEXT:    s_sext_i32_i16 s9, s9
+; GFX10-NEXT:    s_cmp_gt_i32 s9, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s9, s1
+; GFX10-NEXT:    s_sext_i32_i16 s9, s12
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_cmp_lt_i32 s1, s9
+; GFX10-NEXT:    s_cselect_b32 s1, s1, s9
+; GFX10-NEXT:    s_lshl_b32 s5, s5, s6
+; GFX10-NEXT:    s_add_i32 s0, s0, s1
+; GFX10-NEXT:    s_lshl_b32 s1, s2, s6
+; GFX10-NEXT:    s_sext_i32_i16 s0, s0
+; GFX10-NEXT:    s_sext_i32_i16 s2, s1
+; GFX10-NEXT:    s_ashr_i32 s0, s0, s6
+; GFX10-NEXT:    s_cmp_gt_i32 s2, s10
+; GFX10-NEXT:    s_sext_i32_i16 s5, s5
+; GFX10-NEXT:    s_cselect_b32 s9, s2, s10
+; GFX10-NEXT:    s_sub_i32 s9, s11, s9
+; GFX10-NEXT:    s_cmp_lt_i32 s2, s10
+; GFX10-NEXT:    s_cselect_b32 s2, s2, s10
+; GFX10-NEXT:    s_sub_i32 s2, s13, s2
+; GFX10-NEXT:    s_sext_i32_i16 s2, s2
+; GFX10-NEXT:    s_cmp_gt_i32 s2, s5
+; GFX10-NEXT:    s_cselect_b32 s2, s2, s5
+; GFX10-NEXT:    s_sext_i32_i16 s5, s9
+; GFX10-NEXT:    s_sext_i32_i16 s2, s2
+; GFX10-NEXT:    s_cmp_lt_i32 s2, s5
+; GFX10-NEXT:    s_cselect_b32 s2, s2, s5
+; GFX10-NEXT:    s_lshl_b32 s3, s3, s6
+; GFX10-NEXT:    s_add_i32 s1, s1, s2
+; GFX10-NEXT:    s_sext_i32_i16 s5, s3
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_lshl_b32 s2, s7, s6
+; GFX10-NEXT:    s_ashr_i32 s1, s1, s6
+; GFX10-NEXT:    s_cmp_gt_i32 s5, s10
+; GFX10-NEXT:    s_sext_i32_i16 s2, s2
+; GFX10-NEXT:    s_cselect_b32 s7, s5, s10
+; GFX10-NEXT:    s_sub_i32 s7, s11, s7
+; GFX10-NEXT:    s_cmp_lt_i32 s5, s10
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s10
+; GFX10-NEXT:    s_sub_i32 s5, s13, s5
+; GFX10-NEXT:    s_sext_i32_i16 s5, s5
+; GFX10-NEXT:    s_cmp_gt_i32 s5, s2
+; GFX10-NEXT:    s_cselect_b32 s2, s5, s2
+; GFX10-NEXT:    s_sext_i32_i16 s5, s7
+; GFX10-NEXT:    s_sext_i32_i16 s2, s2
+; GFX10-NEXT:    s_cmp_lt_i32 s2, s5
+; GFX10-NEXT:    s_cselect_b32 s2, s2, s5
+; GFX10-NEXT:    s_lshl_b32 s4, s4, s6
+; GFX10-NEXT:    s_add_i32 s3, s3, s2
+; GFX10-NEXT:    s_sext_i32_i16 s5, s4
+; GFX10-NEXT:    s_sext_i32_i16 s3, s3
+; GFX10-NEXT:    s_lshl_b32 s2, s8, s6
+; GFX10-NEXT:    s_ashr_i32 s3, s3, s6
+; GFX10-NEXT:    s_cmp_gt_i32 s5, s10
+; GFX10-NEXT:    s_sext_i32_i16 s2, s2
+; GFX10-NEXT:    s_cselect_b32 s7, s5, s10
+; GFX10-NEXT:    s_sub_i32 s7, s11, s7
+; GFX10-NEXT:    s_cmp_lt_i32 s5, s10
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s10
+; GFX10-NEXT:    s_sub_i32 s5, s13, s5
+; GFX10-NEXT:    s_sext_i32_i16 s5, s5
+; GFX10-NEXT:    s_cmp_gt_i32 s5, s2
+; GFX10-NEXT:    s_cselect_b32 s2, s5, s2
+; GFX10-NEXT:    s_sext_i32_i16 s5, s7
+; GFX10-NEXT:    s_sext_i32_i16 s2, s2
+; GFX10-NEXT:    s_movk_i32 s7, 0xff
+; GFX10-NEXT:    s_cmp_lt_i32 s2, s5
+; GFX10-NEXT:    s_cselect_b32 s2, s2, s5
+; GFX10-NEXT:    s_and_b32 s1, s1, s7
+; GFX10-NEXT:    s_add_i32 s4, s4, s2
+; GFX10-NEXT:    s_and_b32 s2, s3, s7
+; GFX10-NEXT:    s_sext_i32_i16 s3, s4
+; GFX10-NEXT:    s_and_b32 s0, s0, s7
+; GFX10-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX10-NEXT:    s_ashr_i32 s3, s3, s6
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    s_lshl_b32 s1, s2, 16
+; GFX10-NEXT:    s_and_b32 s2, s3, s7
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    s_lshl_b32 s1, s2, 24
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %lhs = bitcast i32 %lhs.arg to <4 x i8>
+  %rhs = bitcast i32 %rhs.arg to <4 x i8>
+  %result = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> %lhs, <4 x i8> %rhs)
+  %cast.result = bitcast <4 x i8> %result to i32
+  ret i32 %cast.result
+}
+
+define i24 @v_saddsat_i24(i24 %lhs, i24 %rhs) {
+; GFX6-LABEL: v_saddsat_i24:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX6-NEXT:    v_min_i32_e32 v3, 0, v0
+; GFX6-NEXT:    v_max_i32_e32 v2, 0, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, 0x80000000, v3
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, 0x7fffffff, v2
+; GFX6-NEXT:    v_max_i32_e32 v1, v3, v1
+; GFX6-NEXT:    v_min_i32_e32 v1, v1, v2
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 8, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_i24:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v0, v1
+; GFX8-NEXT:    v_bfe_i32 v3, v2, 0, 24
+; GFX8-NEXT:    v_bfe_i32 v0, v0, 0, 24
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v3, v0
+; GFX8-NEXT:    v_bfe_i32 v0, v1, 0, 24
+; GFX8-NEXT:    s_bfe_i32 s6, 0, 0x180000
+; GFX8-NEXT:    v_cmp_gt_i32_e64 s[6:7], s6, v0
+; GFX8-NEXT:    v_ashrrev_i32_e32 v0, 23, v3
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 0xff800000, v0
+; GFX8-NEXT:    s_xor_b64 vcc, s[6:7], s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_i24:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX9-NEXT:    v_min_i32_e32 v3, 0, v0
+; GFX9-NEXT:    v_max_i32_e32 v2, 0, v0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX9-NEXT:    v_sub_u32_e32 v3, 0x80000000, v3
+; GFX9-NEXT:    v_sub_u32_e32 v2, 0x7fffffff, v2
+; GFX9-NEXT:    v_max_i32_e32 v1, v3, v1
+; GFX9-NEXT:    v_min_i32_e32 v1, v1, v2
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 8, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_saddsat_i24:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_min_i32_e32 v2, 0, v0
+; GFX10-NEXT:    v_max_i32_e32 v3, 0, v0
+; GFX10-NEXT:    v_sub_nc_u32_e32 v2, 0x80000000, v2
+; GFX10-NEXT:    v_sub_nc_u32_e32 v3, 0x7fffffff, v3
+; GFX10-NEXT:    v_max_i32_e32 v1, v2, v1
+; GFX10-NEXT:    v_min_i32_e32 v1, v1, v3
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT:    v_ashrrev_i32_e32 v0, 8, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i24 @llvm.sadd.sat.i24(i24 %lhs, i24 %rhs)
+  ret i24 %result
+}
+
+define amdgpu_ps i24 @s_saddsat_i24(i24 inreg %lhs, i24 inreg %rhs) {
+; GFX6-LABEL: s_saddsat_i24:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 8
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX6-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX6-NEXT:    s_cselect_b32 s2, s0, 0
+; GFX6-NEXT:    s_sub_i32 s2, 0x7fffffff, s2
+; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX6-NEXT:    s_cselect_b32 s3, s0, 0
+; GFX6-NEXT:    s_sub_i32 s3, 0x80000000, s3
+; GFX6-NEXT:    s_cmp_gt_i32 s3, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX6-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX6-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX6-NEXT:    s_add_i32 s0, s0, s1
+; GFX6-NEXT:    s_ashr_i32 s0, s0, 8
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_saddsat_i24:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_add_i32 s2, s0, s1
+; GFX8-NEXT:    s_bfe_i32 s3, s2, 0x180000
+; GFX8-NEXT:    s_bfe_i32 s0, s0, 0x180000
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s0
+; GFX8-NEXT:    s_cselect_b32 s0, 1, 0
+; GFX8-NEXT:    s_bfe_i32 s1, s1, 0x180000
+; GFX8-NEXT:    s_bfe_i32 s4, 0, 0x180000
+; GFX8-NEXT:    s_cmp_lt_i32 s1, s4
+; GFX8-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX8-NEXT:    s_xor_b32 s0, s1, s0
+; GFX8-NEXT:    s_ashr_i32 s1, s3, 23
+; GFX8-NEXT:    s_add_i32 s1, s1, 0xff800000
+; GFX8-NEXT:    s_and_b32 s0, s0, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s0, 0
+; GFX8-NEXT:    s_cselect_b32 s0, s1, s2
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_saddsat_i24:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 8
+; GFX9-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX9-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX9-NEXT:    s_cselect_b32 s2, s0, 0
+; GFX9-NEXT:    s_sub_i32 s2, 0x7fffffff, s2
+; GFX9-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX9-NEXT:    s_cselect_b32 s3, s0, 0
+; GFX9-NEXT:    s_sub_i32 s3, 0x80000000, s3
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX9-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX9-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX9-NEXT:    s_add_i32 s0, s0, s1
+; GFX9-NEXT:    s_ashr_i32 s0, s0, 8
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_saddsat_i24:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_lshl_b32 s0, s0, 8
+; GFX10-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX10-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s2, s0, 0
+; GFX10-NEXT:    s_sub_i32 s2, 0x7fffffff, s2
+; GFX10-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX10-NEXT:    s_cselect_b32 s3, s0, 0
+; GFX10-NEXT:    s_sub_i32 s3, 0x80000000, s3
+; GFX10-NEXT:    s_cmp_gt_i32 s3, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX10-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX10-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX10-NEXT:    s_add_i32 s0, s0, s1
+; GFX10-NEXT:    s_ashr_i32 s0, s0, 8
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i24 @llvm.sadd.sat.i24(i24 %lhs, i24 %rhs)
+  ret i24 %result
+}
+
+define i32 @v_saddsat_i32(i32 %lhs, i32 %rhs) {
+; GFX6-LABEL: v_saddsat_i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_min_i32_e32 v3, 0, v0
+; GFX6-NEXT:    v_max_i32_e32 v2, 0, v0
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, 0x80000000, v3
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, 0x7fffffff, v2
+; GFX6-NEXT:    v_max_i32_e32 v1, v3, v1
+; GFX6-NEXT:    v_min_i32_e32 v1, v1, v2
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_min_i32_e32 v3, 0, v0
+; GFX8-NEXT:    v_max_i32_e32 v2, 0, v0
+; GFX8-NEXT:    v_sub_u32_e32 v3, vcc, 0x80000000, v3
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, 0x7fffffff, v2
+; GFX8-NEXT:    v_max_i32_e32 v1, v3, v1
+; GFX8-NEXT:    v_min_i32_e32 v1, v1, v2
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_min_i32_e32 v3, 0, v0
+; GFX9-NEXT:    v_max_i32_e32 v2, 0, v0
+; GFX9-NEXT:    v_sub_u32_e32 v3, 0x80000000, v3
+; GFX9-NEXT:    v_sub_u32_e32 v2, 0x7fffffff, v2
+; GFX9-NEXT:    v_max_i32_e32 v1, v3, v1
+; GFX9-NEXT:    v_min_i32_e32 v1, v1, v2
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_saddsat_i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_min_i32_e32 v2, 0, v0
+; GFX10-NEXT:    v_max_i32_e32 v3, 0, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_sub_nc_u32_e32 v2, 0x80000000, v2
+; GFX10-NEXT:    v_sub_nc_u32_e32 v3, 0x7fffffff, v3
+; GFX10-NEXT:    v_max_i32_e32 v1, v2, v1
+; GFX10-NEXT:    v_min_i32_e32 v1, v1, v3
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i32 @llvm.sadd.sat.i32(i32 %lhs, i32 %rhs)
+  ret i32 %result
+}
+
+define amdgpu_ps i32 @s_saddsat_i32(i32 inreg %lhs, i32 inreg %rhs) {
+; GCN-LABEL: s_saddsat_i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_cmp_gt_i32 s0, 0
+; GCN-NEXT:    ; implicit-def: $vcc_hi
+; GCN-NEXT:    s_cselect_b32 s2, s0, 0
+; GCN-NEXT:    s_sub_i32 s2, 0x7fffffff, s2
+; GCN-NEXT:    s_cmp_lt_i32 s0, 0
+; GCN-NEXT:    s_cselect_b32 s3, s0, 0
+; GCN-NEXT:    s_sub_i32 s3, 0x80000000, s3
+; GCN-NEXT:    s_cmp_gt_i32 s3, s1
+; GCN-NEXT:    s_cselect_b32 s1, s3, s1
+; GCN-NEXT:    s_cmp_lt_i32 s1, s2
+; GCN-NEXT:    s_cselect_b32 s1, s1, s2
+; GCN-NEXT:    s_add_i32 s0, s0, s1
+; GCN-NEXT:    ; return to shader part epilog
+; GFX6-LABEL: s_saddsat_i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX6-NEXT:    s_cselect_b32 s2, s0, 0
+; GFX6-NEXT:    s_sub_i32 s2, 0x7fffffff, s2
+; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX6-NEXT:    s_cselect_b32 s3, s0, 0
+; GFX6-NEXT:    s_sub_i32 s3, 0x80000000, s3
+; GFX6-NEXT:    s_cmp_gt_i32 s3, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX6-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX6-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX6-NEXT:    s_add_i32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_saddsat_i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX8-NEXT:    s_cselect_b32 s2, s0, 0
+; GFX8-NEXT:    s_sub_i32 s2, 0x7fffffff, s2
+; GFX8-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX8-NEXT:    s_cselect_b32 s3, s0, 0
+; GFX8-NEXT:    s_sub_i32 s3, 0x80000000, s3
+; GFX8-NEXT:    s_cmp_gt_i32 s3, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX8-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX8-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX8-NEXT:    s_add_i32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_saddsat_i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX9-NEXT:    s_cselect_b32 s2, s0, 0
+; GFX9-NEXT:    s_sub_i32 s2, 0x7fffffff, s2
+; GFX9-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX9-NEXT:    s_cselect_b32 s3, s0, 0
+; GFX9-NEXT:    s_sub_i32 s3, 0x80000000, s3
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX9-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX9-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX9-NEXT:    s_add_i32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_saddsat_i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s2, s0, 0
+; GFX10-NEXT:    s_sub_i32 s2, 0x7fffffff, s2
+; GFX10-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX10-NEXT:    s_cselect_b32 s3, s0, 0
+; GFX10-NEXT:    s_sub_i32 s3, 0x80000000, s3
+; GFX10-NEXT:    s_cmp_gt_i32 s3, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX10-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX10-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX10-NEXT:    s_add_i32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i32 @llvm.sadd.sat.i32(i32 %lhs, i32 %rhs)
+  ret i32 %result
+}
+
+define amdgpu_ps float @saddsat_i32_sv(i32 inreg %lhs, i32 %rhs) {
+; GFX6-LABEL: saddsat_i32_sv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX6-NEXT:    s_cselect_b32 s1, s0, 0
+; GFX6-NEXT:    s_sub_i32 s1, 0x7fffffff, s1
+; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX6-NEXT:    s_cselect_b32 s2, s0, 0
+; GFX6-NEXT:    s_sub_i32 s2, 0x80000000, s2
+; GFX6-NEXT:    v_max_i32_e32 v0, s2, v0
+; GFX6-NEXT:    v_min_i32_e32 v0, s1, v0
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: saddsat_i32_sv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX8-NEXT:    s_cselect_b32 s1, s0, 0
+; GFX8-NEXT:    s_sub_i32 s1, 0x7fffffff, s1
+; GFX8-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX8-NEXT:    s_cselect_b32 s2, s0, 0
+; GFX8-NEXT:    s_sub_i32 s2, 0x80000000, s2
+; GFX8-NEXT:    v_max_i32_e32 v0, s2, v0
+; GFX8-NEXT:    v_min_i32_e32 v0, s1, v0
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: saddsat_i32_sv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX9-NEXT:    s_cselect_b32 s1, s0, 0
+; GFX9-NEXT:    s_sub_i32 s1, 0x7fffffff, s1
+; GFX9-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX9-NEXT:    s_cselect_b32 s2, s0, 0
+; GFX9-NEXT:    s_sub_i32 s2, 0x80000000, s2
+; GFX9-NEXT:    v_max_i32_e32 v0, s2, v0
+; GFX9-NEXT:    v_min_i32_e32 v0, s1, v0
+; GFX9-NEXT:    v_add_u32_e32 v0, s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: saddsat_i32_sv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s1, s0, 0
+; GFX10-NEXT:    s_sub_i32 s1, 0x7fffffff, s1
+; GFX10-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX10-NEXT:    s_cselect_b32 s2, s0, 0
+; GFX10-NEXT:    s_sub_i32 s2, 0x80000000, s2
+; GFX10-NEXT:    v_max_i32_e32 v0, s2, v0
+; GFX10-NEXT:    v_min_i32_e32 v0, s1, v0
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i32 @llvm.sadd.sat.i32(i32 %lhs, i32 %rhs)
+  %cast = bitcast i32 %result to float
+  ret float %cast
+}
+
+define amdgpu_ps float @saddsat_i32_vs(i32 %lhs, i32 inreg %rhs) {
+; GFX6-LABEL: saddsat_i32_vs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_min_i32_e32 v2, 0, v0
+; GFX6-NEXT:    v_max_i32_e32 v1, 0, v0
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, 0x80000000, v2
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, 0x7fffffff, v1
+; GFX6-NEXT:    v_max_i32_e32 v2, s0, v2
+; GFX6-NEXT:    v_min_i32_e32 v1, v2, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: saddsat_i32_vs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_min_i32_e32 v2, 0, v0
+; GFX8-NEXT:    v_max_i32_e32 v1, 0, v0
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, 0x80000000, v2
+; GFX8-NEXT:    v_sub_u32_e32 v1, vcc, 0x7fffffff, v1
+; GFX8-NEXT:    v_max_i32_e32 v2, s0, v2
+; GFX8-NEXT:    v_min_i32_e32 v1, v2, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: saddsat_i32_vs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_min_i32_e32 v2, 0, v0
+; GFX9-NEXT:    v_max_i32_e32 v1, 0, v0
+; GFX9-NEXT:    v_sub_u32_e32 v2, 0x80000000, v2
+; GFX9-NEXT:    v_sub_u32_e32 v1, 0x7fffffff, v1
+; GFX9-NEXT:    v_max_i32_e32 v2, s0, v2
+; GFX9-NEXT:    v_min_i32_e32 v1, v2, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: saddsat_i32_vs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_min_i32_e32 v1, 0, v0
+; GFX10-NEXT:    v_max_i32_e32 v2, 0, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_sub_nc_u32_e32 v1, 0x80000000, v1
+; GFX10-NEXT:    v_sub_nc_u32_e32 v2, 0x7fffffff, v2
+; GFX10-NEXT:    v_max_i32_e32 v1, s0, v1
+; GFX10-NEXT:    v_min_i32_e32 v1, v1, v2
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i32 @llvm.sadd.sat.i32(i32 %lhs, i32 %rhs)
+  %cast = bitcast i32 %result to float
+  ret float %cast
+}
+
+define <2 x i32> @v_saddsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
+; GFX6-LABEL: v_saddsat_v2i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v5, 0, v0
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, s5, v5
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    v_max_i32_e32 v4, 0, v0
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, s4, v4
+; GFX6-NEXT:    v_max_i32_e32 v2, v5, v2
+; GFX6-NEXT:    v_min_i32_e32 v2, v2, v4
+; GFX6-NEXT:    v_min_i32_e32 v4, 0, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT:    v_max_i32_e32 v2, 0, v1
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, s5, v4
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, s4, v2
+; GFX6-NEXT:    v_max_i32_e32 v3, v4, v3
+; GFX6-NEXT:    v_min_i32_e32 v2, v3, v2
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_v2i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX8-NEXT:    v_min_i32_e32 v5, 0, v0
+; GFX8-NEXT:    v_sub_u32_e32 v5, vcc, s5, v5
+; GFX8-NEXT:    s_brev_b32 s4, -2
+; GFX8-NEXT:    v_max_i32_e32 v4, 0, v0
+; GFX8-NEXT:    v_sub_u32_e32 v4, vcc, s4, v4
+; GFX8-NEXT:    v_max_i32_e32 v2, v5, v2
+; GFX8-NEXT:    v_min_i32_e32 v2, v2, v4
+; GFX8-NEXT:    v_min_i32_e32 v4, 0, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT:    v_max_i32_e32 v2, 0, v1
+; GFX8-NEXT:    v_sub_u32_e32 v4, vcc, s5, v4
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, s4, v2
+; GFX8-NEXT:    v_max_i32_e32 v3, v4, v3
+; GFX8-NEXT:    v_min_i32_e32 v2, v3, v2
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_v2i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX9-NEXT:    v_min_i32_e32 v5, 0, v0
+; GFX9-NEXT:    v_sub_u32_e32 v5, s5, v5
+; GFX9-NEXT:    s_brev_b32 s4, -2
+; GFX9-NEXT:    v_max_i32_e32 v4, 0, v0
+; GFX9-NEXT:    v_sub_u32_e32 v4, s4, v4
+; GFX9-NEXT:    v_max_i32_e32 v2, v5, v2
+; GFX9-NEXT:    v_min_i32_e32 v2, v2, v4
+; GFX9-NEXT:    v_min_i32_e32 v4, 0, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v2
+; GFX9-NEXT:    v_max_i32_e32 v2, 0, v1
+; GFX9-NEXT:    v_sub_u32_e32 v4, s5, v4
+; GFX9-NEXT:    v_sub_u32_e32 v2, s4, v2
+; GFX9-NEXT:    v_max_i32_e32 v3, v4, v3
+; GFX9-NEXT:    v_min_i32_e32 v2, v3, v2
+; GFX9-NEXT:    v_add_u32_e32 v1, v1, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_saddsat_v2i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_min_i32_e32 v4, 0, v0
+; GFX10-NEXT:    v_min_i32_e32 v5, 0, v1
+; GFX10-NEXT:    s_mov_b32 s4, 0x80000000
+; GFX10-NEXT:    v_max_i32_e32 v6, 0, v0
+; GFX10-NEXT:    v_max_i32_e32 v7, 0, v1
+; GFX10-NEXT:    v_sub_nc_u32_e32 v4, s4, v4
+; GFX10-NEXT:    v_sub_nc_u32_e32 v5, s4, v5
+; GFX10-NEXT:    s_brev_b32 s4, -2
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_sub_nc_u32_e32 v6, s4, v6
+; GFX10-NEXT:    v_max_i32_e32 v11, v4, v2
+; GFX10-NEXT:    v_sub_nc_u32_e32 v7, s4, v7
+; GFX10-NEXT:    v_max_i32_e32 v10, v5, v3
+; GFX10-NEXT:    v_min_i32_e32 v2, v11, v6
+; GFX10-NEXT:    v_min_i32_e32 v3, v10, v7
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v2
+; GFX10-NEXT:    v_add_nc_u32_e32 v1, v1, v3
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> %lhs, <2 x i32> %rhs)
+  ret <2 x i32> %result
+}
+
+define amdgpu_ps <2 x i32> @s_saddsat_v2i32(<2 x i32> inreg %lhs, <2 x i32> inreg %rhs) {
+; GFX6-LABEL: s_saddsat_v2i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    s_cselect_b32 s6, s0, 0
+; GFX6-NEXT:    s_sub_i32 s6, s4, s6
+; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_cselect_b32 s7, s0, 0
+; GFX6-NEXT:    s_sub_i32 s7, s5, s7
+; GFX6-NEXT:    s_cmp_gt_i32 s7, s2
+; GFX6-NEXT:    s_cselect_b32 s2, s7, s2
+; GFX6-NEXT:    s_cmp_lt_i32 s2, s6
+; GFX6-NEXT:    s_cselect_b32 s2, s2, s6
+; GFX6-NEXT:    s_add_i32 s0, s0, s2
+; GFX6-NEXT:    s_cmp_gt_i32 s1, 0
+; GFX6-NEXT:    s_cselect_b32 s2, s1, 0
+; GFX6-NEXT:    s_sub_i32 s2, s4, s2
+; GFX6-NEXT:    s_cmp_lt_i32 s1, 0
+; GFX6-NEXT:    s_cselect_b32 s4, s1, 0
+; GFX6-NEXT:    s_sub_i32 s4, s5, s4
+; GFX6-NEXT:    s_cmp_gt_i32 s4, s3
+; GFX6-NEXT:    s_cselect_b32 s3, s4, s3
+; GFX6-NEXT:    s_cmp_lt_i32 s3, s2
+; GFX6-NEXT:    s_cselect_b32 s2, s3, s2
+; GFX6-NEXT:    s_add_i32 s1, s1, s2
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_saddsat_v2i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX8-NEXT:    s_brev_b32 s4, -2
+; GFX8-NEXT:    s_cselect_b32 s6, s0, 0
+; GFX8-NEXT:    s_sub_i32 s6, s4, s6
+; GFX8-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX8-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX8-NEXT:    s_cselect_b32 s7, s0, 0
+; GFX8-NEXT:    s_sub_i32 s7, s5, s7
+; GFX8-NEXT:    s_cmp_gt_i32 s7, s2
+; GFX8-NEXT:    s_cselect_b32 s2, s7, s2
+; GFX8-NEXT:    s_cmp_lt_i32 s2, s6
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s6
+; GFX8-NEXT:    s_add_i32 s0, s0, s2
+; GFX8-NEXT:    s_cmp_gt_i32 s1, 0
+; GFX8-NEXT:    s_cselect_b32 s2, s1, 0
+; GFX8-NEXT:    s_sub_i32 s2, s4, s2
+; GFX8-NEXT:    s_cmp_lt_i32 s1, 0
+; GFX8-NEXT:    s_cselect_b32 s4, s1, 0
+; GFX8-NEXT:    s_sub_i32 s4, s5, s4
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s3
+; GFX8-NEXT:    s_cselect_b32 s3, s4, s3
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s2
+; GFX8-NEXT:    s_cselect_b32 s2, s3, s2
+; GFX8-NEXT:    s_add_i32 s1, s1, s2
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_saddsat_v2i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX9-NEXT:    s_brev_b32 s4, -2
+; GFX9-NEXT:    s_cselect_b32 s6, s0, 0
+; GFX9-NEXT:    s_sub_i32 s6, s4, s6
+; GFX9-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX9-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX9-NEXT:    s_cselect_b32 s7, s0, 0
+; GFX9-NEXT:    s_sub_i32 s7, s5, s7
+; GFX9-NEXT:    s_cmp_gt_i32 s7, s2
+; GFX9-NEXT:    s_cselect_b32 s2, s7, s2
+; GFX9-NEXT:    s_cmp_lt_i32 s2, s6
+; GFX9-NEXT:    s_cselect_b32 s2, s2, s6
+; GFX9-NEXT:    s_add_i32 s0, s0, s2
+; GFX9-NEXT:    s_cmp_gt_i32 s1, 0
+; GFX9-NEXT:    s_cselect_b32 s2, s1, 0
+; GFX9-NEXT:    s_sub_i32 s2, s4, s2
+; GFX9-NEXT:    s_cmp_lt_i32 s1, 0
+; GFX9-NEXT:    s_cselect_b32 s4, s1, 0
+; GFX9-NEXT:    s_sub_i32 s4, s5, s4
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s3
+; GFX9-NEXT:    s_cselect_b32 s3, s4, s3
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s2
+; GFX9-NEXT:    s_cselect_b32 s2, s3, s2
+; GFX9-NEXT:    s_add_i32 s1, s1, s2
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_saddsat_v2i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX10-NEXT:    s_brev_b32 s4, -2
+; GFX10-NEXT:    s_cselect_b32 s5, s0, 0
+; GFX10-NEXT:    s_mov_b32 s6, 0x80000000
+; GFX10-NEXT:    s_sub_i32 s5, s4, s5
+; GFX10-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s7, s0, 0
+; GFX10-NEXT:    s_sub_i32 s7, s6, s7
+; GFX10-NEXT:    s_cmp_gt_i32 s7, s2
+; GFX10-NEXT:    s_cselect_b32 s2, s7, s2
+; GFX10-NEXT:    s_cmp_lt_i32 s2, s5
+; GFX10-NEXT:    s_cselect_b32 s2, s2, s5
+; GFX10-NEXT:    s_add_i32 s0, s0, s2
+; GFX10-NEXT:    s_cmp_gt_i32 s1, 0
+; GFX10-NEXT:    s_cselect_b32 s2, s1, 0
+; GFX10-NEXT:    s_sub_i32 s2, s4, s2
+; GFX10-NEXT:    s_cmp_lt_i32 s1, 0
+; GFX10-NEXT:    s_cselect_b32 s4, s1, 0
+; GFX10-NEXT:    s_sub_i32 s4, s6, s4
+; GFX10-NEXT:    s_cmp_gt_i32 s4, s3
+; GFX10-NEXT:    s_cselect_b32 s3, s4, s3
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s2
+; GFX10-NEXT:    s_cselect_b32 s2, s3, s2
+; GFX10-NEXT:    s_add_i32 s1, s1, s2
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> %lhs, <2 x i32> %rhs)
+  ret <2 x i32> %result
+}
+
+define <3 x i32> @v_saddsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
+; GFX6-LABEL: v_saddsat_v3i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v7, 0, v0
+; GFX6-NEXT:    v_sub_i32_e32 v7, vcc, s5, v7
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    v_max_i32_e32 v6, 0, v0
+; GFX6-NEXT:    v_sub_i32_e32 v6, vcc, s4, v6
+; GFX6-NEXT:    v_max_i32_e32 v3, v7, v3
+; GFX6-NEXT:    v_min_i32_e32 v3, v3, v6
+; GFX6-NEXT:    v_min_i32_e32 v6, 0, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
+; GFX6-NEXT:    v_max_i32_e32 v3, 0, v1
+; GFX6-NEXT:    v_sub_i32_e32 v6, vcc, s5, v6
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, s4, v3
+; GFX6-NEXT:    v_max_i32_e32 v4, v6, v4
+; GFX6-NEXT:    v_min_i32_e32 v3, v4, v3
+; GFX6-NEXT:    v_min_i32_e32 v4, 0, v2
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, s5, v4
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GFX6-NEXT:    v_max_i32_e32 v3, 0, v2
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, s4, v3
+; GFX6-NEXT:    v_max_i32_e32 v4, v4, v5
+; GFX6-NEXT:    v_min_i32_e32 v3, v4, v3
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_v3i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX8-NEXT:    v_min_i32_e32 v7, 0, v0
+; GFX8-NEXT:    v_sub_u32_e32 v7, vcc, s5, v7
+; GFX8-NEXT:    s_brev_b32 s4, -2
+; GFX8-NEXT:    v_max_i32_e32 v6, 0, v0
+; GFX8-NEXT:    v_sub_u32_e32 v6, vcc, s4, v6
+; GFX8-NEXT:    v_max_i32_e32 v3, v7, v3
+; GFX8-NEXT:    v_min_i32_e32 v3, v3, v6
+; GFX8-NEXT:    v_min_i32_e32 v6, 0, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v3
+; GFX8-NEXT:    v_max_i32_e32 v3, 0, v1
+; GFX8-NEXT:    v_sub_u32_e32 v6, vcc, s5, v6
+; GFX8-NEXT:    v_sub_u32_e32 v3, vcc, s4, v3
+; GFX8-NEXT:    v_max_i32_e32 v4, v6, v4
+; GFX8-NEXT:    v_min_i32_e32 v3, v4, v3
+; GFX8-NEXT:    v_min_i32_e32 v4, 0, v2
+; GFX8-NEXT:    v_sub_u32_e32 v4, vcc, s5, v4
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v3
+; GFX8-NEXT:    v_max_i32_e32 v3, 0, v2
+; GFX8-NEXT:    v_sub_u32_e32 v3, vcc, s4, v3
+; GFX8-NEXT:    v_max_i32_e32 v4, v4, v5
+; GFX8-NEXT:    v_min_i32_e32 v3, v4, v3
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v3
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_v3i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX9-NEXT:    v_min_i32_e32 v7, 0, v0
+; GFX9-NEXT:    v_sub_u32_e32 v7, s5, v7
+; GFX9-NEXT:    s_brev_b32 s4, -2
+; GFX9-NEXT:    v_max_i32_e32 v6, 0, v0
+; GFX9-NEXT:    v_sub_u32_e32 v6, s4, v6
+; GFX9-NEXT:    v_max_i32_e32 v3, v7, v3
+; GFX9-NEXT:    v_min_i32_e32 v3, v3, v6
+; GFX9-NEXT:    v_min_i32_e32 v6, 0, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v3
+; GFX9-NEXT:    v_max_i32_e32 v3, 0, v1
+; GFX9-NEXT:    v_sub_u32_e32 v6, s5, v6
+; GFX9-NEXT:    v_sub_u32_e32 v3, s4, v3
+; GFX9-NEXT:    v_max_i32_e32 v4, v6, v4
+; GFX9-NEXT:    v_min_i32_e32 v3, v4, v3
+; GFX9-NEXT:    v_min_i32_e32 v4, 0, v2
+; GFX9-NEXT:    v_sub_u32_e32 v4, s5, v4
+; GFX9-NEXT:    v_add_u32_e32 v1, v1, v3
+; GFX9-NEXT:    v_max_i32_e32 v3, 0, v2
+; GFX9-NEXT:    v_sub_u32_e32 v3, s4, v3
+; GFX9-NEXT:    v_max_i32_e32 v4, v4, v5
+; GFX9-NEXT:    v_min_i32_e32 v3, v4, v3
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v3
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_saddsat_v3i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_min_i32_e32 v7, 0, v0
+; GFX10-NEXT:    v_min_i32_e32 v8, 0, v1
+; GFX10-NEXT:    v_min_i32_e32 v9, 0, v2
+; GFX10-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX10-NEXT:    v_max_i32_e32 v6, 0, v0
+; GFX10-NEXT:    v_sub_nc_u32_e32 v14, s5, v7
+; GFX10-NEXT:    v_sub_nc_u32_e32 v15, s5, v8
+; GFX10-NEXT:    v_sub_nc_u32_e32 v19, s5, v9
+; GFX10-NEXT:    v_max_i32_e32 v10, 0, v1
+; GFX10-NEXT:    v_max_i32_e32 v11, 0, v2
+; GFX10-NEXT:    s_brev_b32 s4, -2
+; GFX10-NEXT:    v_max_i32_e32 v3, v14, v3
+; GFX10-NEXT:    v_sub_nc_u32_e32 v6, s4, v6
+; GFX10-NEXT:    v_sub_nc_u32_e32 v7, s4, v10
+; GFX10-NEXT:    v_max_i32_e32 v4, v15, v4
+; GFX10-NEXT:    v_sub_nc_u32_e32 v8, s4, v11
+; GFX10-NEXT:    v_max_i32_e32 v5, v19, v5
+; GFX10-NEXT:    v_min_i32_e32 v3, v3, v6
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_min_i32_e32 v4, v4, v7
+; GFX10-NEXT:    v_min_i32_e32 v5, v5, v8
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v3
+; GFX10-NEXT:    v_add_nc_u32_e32 v1, v1, v4
+; GFX10-NEXT:    v_add_nc_u32_e32 v2, v2, v5
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <3 x i32> @llvm.sadd.sat.v3i32(<3 x i32> %lhs, <3 x i32> %rhs)
+  ret <3 x i32> %result
+}
+
+define amdgpu_ps <3 x i32> @s_saddsat_v3i32(<3 x i32> inreg %lhs, <3 x i32> inreg %rhs) {
+; GFX6-LABEL: s_saddsat_v3i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX6-NEXT:    s_brev_b32 s6, -2
+; GFX6-NEXT:    s_cselect_b32 s8, s0, 0
+; GFX6-NEXT:    s_sub_i32 s8, s6, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX6-NEXT:    s_mov_b32 s7, 0x80000000
+; GFX6-NEXT:    s_cselect_b32 s9, s0, 0
+; GFX6-NEXT:    s_sub_i32 s9, s7, s9
+; GFX6-NEXT:    s_cmp_gt_i32 s9, s3
+; GFX6-NEXT:    s_cselect_b32 s3, s9, s3
+; GFX6-NEXT:    s_cmp_lt_i32 s3, s8
+; GFX6-NEXT:    s_cselect_b32 s3, s3, s8
+; GFX6-NEXT:    s_add_i32 s0, s0, s3
+; GFX6-NEXT:    s_cmp_gt_i32 s1, 0
+; GFX6-NEXT:    s_cselect_b32 s3, s1, 0
+; GFX6-NEXT:    s_sub_i32 s3, s6, s3
+; GFX6-NEXT:    s_cmp_lt_i32 s1, 0
+; GFX6-NEXT:    s_cselect_b32 s8, s1, 0
+; GFX6-NEXT:    s_sub_i32 s8, s7, s8
+; GFX6-NEXT:    s_cmp_gt_i32 s8, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s8, s4
+; GFX6-NEXT:    s_cmp_lt_i32 s4, s3
+; GFX6-NEXT:    s_cselect_b32 s3, s4, s3
+; GFX6-NEXT:    s_add_i32 s1, s1, s3
+; GFX6-NEXT:    s_cmp_gt_i32 s2, 0
+; GFX6-NEXT:    s_cselect_b32 s3, s2, 0
+; GFX6-NEXT:    s_sub_i32 s3, s6, s3
+; GFX6-NEXT:    s_cmp_lt_i32 s2, 0
+; GFX6-NEXT:    s_cselect_b32 s4, s2, 0
+; GFX6-NEXT:    s_sub_i32 s4, s7, s4
+; GFX6-NEXT:    s_cmp_gt_i32 s4, s5
+; GFX6-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX6-NEXT:    s_cmp_lt_i32 s4, s3
+; GFX6-NEXT:    s_cselect_b32 s3, s4, s3
+; GFX6-NEXT:    s_add_i32 s2, s2, s3
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_saddsat_v3i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX8-NEXT:    s_brev_b32 s6, -2
+; GFX8-NEXT:    s_cselect_b32 s8, s0, 0
+; GFX8-NEXT:    s_sub_i32 s8, s6, s8
+; GFX8-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX8-NEXT:    s_mov_b32 s7, 0x80000000
+; GFX8-NEXT:    s_cselect_b32 s9, s0, 0
+; GFX8-NEXT:    s_sub_i32 s9, s7, s9
+; GFX8-NEXT:    s_cmp_gt_i32 s9, s3
+; GFX8-NEXT:    s_cselect_b32 s3, s9, s3
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s8
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s8
+; GFX8-NEXT:    s_add_i32 s0, s0, s3
+; GFX8-NEXT:    s_cmp_gt_i32 s1, 0
+; GFX8-NEXT:    s_cselect_b32 s3, s1, 0
+; GFX8-NEXT:    s_sub_i32 s3, s6, s3
+; GFX8-NEXT:    s_cmp_lt_i32 s1, 0
+; GFX8-NEXT:    s_cselect_b32 s8, s1, 0
+; GFX8-NEXT:    s_sub_i32 s8, s7, s8
+; GFX8-NEXT:    s_cmp_gt_i32 s8, s4
+; GFX8-NEXT:    s_cselect_b32 s4, s8, s4
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s3
+; GFX8-NEXT:    s_cselect_b32 s3, s4, s3
+; GFX8-NEXT:    s_add_i32 s1, s1, s3
+; GFX8-NEXT:    s_cmp_gt_i32 s2, 0
+; GFX8-NEXT:    s_cselect_b32 s3, s2, 0
+; GFX8-NEXT:    s_sub_i32 s3, s6, s3
+; GFX8-NEXT:    s_cmp_lt_i32 s2, 0
+; GFX8-NEXT:    s_cselect_b32 s4, s2, 0
+; GFX8-NEXT:    s_sub_i32 s4, s7, s4
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s5
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s3
+; GFX8-NEXT:    s_cselect_b32 s3, s4, s3
+; GFX8-NEXT:    s_add_i32 s2, s2, s3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_saddsat_v3i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX9-NEXT:    s_brev_b32 s6, -2
+; GFX9-NEXT:    s_cselect_b32 s8, s0, 0
+; GFX9-NEXT:    s_sub_i32 s8, s6, s8
+; GFX9-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX9-NEXT:    s_mov_b32 s7, 0x80000000
+; GFX9-NEXT:    s_cselect_b32 s9, s0, 0
+; GFX9-NEXT:    s_sub_i32 s9, s7, s9
+; GFX9-NEXT:    s_cmp_gt_i32 s9, s3
+; GFX9-NEXT:    s_cselect_b32 s3, s9, s3
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s8
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s8
+; GFX9-NEXT:    s_add_i32 s0, s0, s3
+; GFX9-NEXT:    s_cmp_gt_i32 s1, 0
+; GFX9-NEXT:    s_cselect_b32 s3, s1, 0
+; GFX9-NEXT:    s_sub_i32 s3, s6, s3
+; GFX9-NEXT:    s_cmp_lt_i32 s1, 0
+; GFX9-NEXT:    s_cselect_b32 s8, s1, 0
+; GFX9-NEXT:    s_sub_i32 s8, s7, s8
+; GFX9-NEXT:    s_cmp_gt_i32 s8, s4
+; GFX9-NEXT:    s_cselect_b32 s4, s8, s4
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s3
+; GFX9-NEXT:    s_cselect_b32 s3, s4, s3
+; GFX9-NEXT:    s_add_i32 s1, s1, s3
+; GFX9-NEXT:    s_cmp_gt_i32 s2, 0
+; GFX9-NEXT:    s_cselect_b32 s3, s2, 0
+; GFX9-NEXT:    s_sub_i32 s3, s6, s3
+; GFX9-NEXT:    s_cmp_lt_i32 s2, 0
+; GFX9-NEXT:    s_cselect_b32 s4, s2, 0
+; GFX9-NEXT:    s_sub_i32 s4, s7, s4
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s5
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s3
+; GFX9-NEXT:    s_cselect_b32 s3, s4, s3
+; GFX9-NEXT:    s_add_i32 s2, s2, s3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_saddsat_v3i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX10-NEXT:    s_brev_b32 s6, -2
+; GFX10-NEXT:    s_cselect_b32 s7, s0, 0
+; GFX10-NEXT:    s_mov_b32 s8, 0x80000000
+; GFX10-NEXT:    s_sub_i32 s7, s6, s7
+; GFX10-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s9, s0, 0
+; GFX10-NEXT:    s_sub_i32 s9, s8, s9
+; GFX10-NEXT:    s_cmp_gt_i32 s9, s3
+; GFX10-NEXT:    s_cselect_b32 s3, s9, s3
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s7
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s7
+; GFX10-NEXT:    s_add_i32 s0, s0, s3
+; GFX10-NEXT:    s_cmp_gt_i32 s1, 0
+; GFX10-NEXT:    s_cselect_b32 s3, s1, 0
+; GFX10-NEXT:    s_sub_i32 s3, s6, s3
+; GFX10-NEXT:    s_cmp_lt_i32 s1, 0
+; GFX10-NEXT:    s_cselect_b32 s7, s1, 0
+; GFX10-NEXT:    s_sub_i32 s7, s8, s7
+; GFX10-NEXT:    s_cmp_gt_i32 s7, s4
+; GFX10-NEXT:    s_cselect_b32 s4, s7, s4
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s3
+; GFX10-NEXT:    s_cselect_b32 s3, s4, s3
+; GFX10-NEXT:    s_add_i32 s1, s1, s3
+; GFX10-NEXT:    s_cmp_gt_i32 s2, 0
+; GFX10-NEXT:    s_cselect_b32 s3, s2, 0
+; GFX10-NEXT:    s_sub_i32 s3, s6, s3
+; GFX10-NEXT:    s_cmp_lt_i32 s2, 0
+; GFX10-NEXT:    s_cselect_b32 s4, s2, 0
+; GFX10-NEXT:    s_sub_i32 s4, s8, s4
+; GFX10-NEXT:    s_cmp_gt_i32 s4, s5
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s3
+; GFX10-NEXT:    s_cselect_b32 s3, s4, s3
+; GFX10-NEXT:    s_add_i32 s2, s2, s3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <3 x i32> @llvm.sadd.sat.v3i32(<3 x i32> %lhs, <3 x i32> %rhs)
+  ret <3 x i32> %result
+}
+
+define <4 x i32> @v_saddsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
+; GFX6-LABEL: v_saddsat_v4i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v9, 0, v0
+; GFX6-NEXT:    v_sub_i32_e32 v9, vcc, s5, v9
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    v_max_i32_e32 v8, 0, v0
+; GFX6-NEXT:    v_sub_i32_e32 v8, vcc, s4, v8
+; GFX6-NEXT:    v_max_i32_e32 v4, v9, v4
+; GFX6-NEXT:    v_min_i32_e32 v4, v4, v8
+; GFX6-NEXT:    v_min_i32_e32 v8, 0, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; GFX6-NEXT:    v_max_i32_e32 v4, 0, v1
+; GFX6-NEXT:    v_sub_i32_e32 v8, vcc, s5, v8
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, s4, v4
+; GFX6-NEXT:    v_max_i32_e32 v5, v8, v5
+; GFX6-NEXT:    v_min_i32_e32 v4, v5, v4
+; GFX6-NEXT:    v_min_i32_e32 v5, 0, v2
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, s5, v5
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
+; GFX6-NEXT:    v_max_i32_e32 v4, 0, v2
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, s4, v4
+; GFX6-NEXT:    v_max_i32_e32 v5, v5, v6
+; GFX6-NEXT:    v_min_i32_e32 v4, v5, v4
+; GFX6-NEXT:    v_min_i32_e32 v5, 0, v3
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, 0x80000000, v5
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; GFX6-NEXT:    v_max_i32_e32 v4, 0, v3
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, 0x7fffffff, v4
+; GFX6-NEXT:    v_max_i32_e32 v5, v5, v7
+; GFX6-NEXT:    v_min_i32_e32 v4, v5, v4
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_v4i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX8-NEXT:    v_min_i32_e32 v9, 0, v0
+; GFX8-NEXT:    v_sub_u32_e32 v9, vcc, s5, v9
+; GFX8-NEXT:    s_brev_b32 s4, -2
+; GFX8-NEXT:    v_max_i32_e32 v8, 0, v0
+; GFX8-NEXT:    v_sub_u32_e32 v8, vcc, s4, v8
+; GFX8-NEXT:    v_max_i32_e32 v4, v9, v4
+; GFX8-NEXT:    v_min_i32_e32 v4, v4, v8
+; GFX8-NEXT:    v_min_i32_e32 v8, 0, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v4
+; GFX8-NEXT:    v_max_i32_e32 v4, 0, v1
+; GFX8-NEXT:    v_sub_u32_e32 v8, vcc, s5, v8
+; GFX8-NEXT:    v_sub_u32_e32 v4, vcc, s4, v4
+; GFX8-NEXT:    v_max_i32_e32 v5, v8, v5
+; GFX8-NEXT:    v_min_i32_e32 v4, v5, v4
+; GFX8-NEXT:    v_min_i32_e32 v5, 0, v2
+; GFX8-NEXT:    v_sub_u32_e32 v5, vcc, s5, v5
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v4
+; GFX8-NEXT:    v_max_i32_e32 v4, 0, v2
+; GFX8-NEXT:    v_sub_u32_e32 v4, vcc, s4, v4
+; GFX8-NEXT:    v_max_i32_e32 v5, v5, v6
+; GFX8-NEXT:    v_min_i32_e32 v4, v5, v4
+; GFX8-NEXT:    v_min_i32_e32 v5, 0, v3
+; GFX8-NEXT:    v_sub_u32_e32 v5, vcc, 0x80000000, v5
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v4
+; GFX8-NEXT:    v_max_i32_e32 v4, 0, v3
+; GFX8-NEXT:    v_sub_u32_e32 v4, vcc, 0x7fffffff, v4
+; GFX8-NEXT:    v_max_i32_e32 v5, v5, v7
+; GFX8-NEXT:    v_min_i32_e32 v4, v5, v4
+; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v4
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_v4i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX9-NEXT:    v_min_i32_e32 v9, 0, v0
+; GFX9-NEXT:    v_sub_u32_e32 v9, s5, v9
+; GFX9-NEXT:    s_brev_b32 s4, -2
+; GFX9-NEXT:    v_max_i32_e32 v8, 0, v0
+; GFX9-NEXT:    v_sub_u32_e32 v8, s4, v8
+; GFX9-NEXT:    v_max_i32_e32 v4, v9, v4
+; GFX9-NEXT:    v_min_i32_e32 v4, v4, v8
+; GFX9-NEXT:    v_min_i32_e32 v8, 0, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v4
+; GFX9-NEXT:    v_max_i32_e32 v4, 0, v1
+; GFX9-NEXT:    v_sub_u32_e32 v8, s5, v8
+; GFX9-NEXT:    v_sub_u32_e32 v4, s4, v4
+; GFX9-NEXT:    v_max_i32_e32 v5, v8, v5
+; GFX9-NEXT:    v_min_i32_e32 v4, v5, v4
+; GFX9-NEXT:    v_min_i32_e32 v5, 0, v2
+; GFX9-NEXT:    v_sub_u32_e32 v5, s5, v5
+; GFX9-NEXT:    v_add_u32_e32 v1, v1, v4
+; GFX9-NEXT:    v_max_i32_e32 v4, 0, v2
+; GFX9-NEXT:    v_sub_u32_e32 v4, s4, v4
+; GFX9-NEXT:    v_max_i32_e32 v5, v5, v6
+; GFX9-NEXT:    v_min_i32_e32 v4, v5, v4
+; GFX9-NEXT:    v_min_i32_e32 v5, 0, v3
+; GFX9-NEXT:    v_sub_u32_e32 v5, 0x80000000, v5
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v4
+; GFX9-NEXT:    v_max_i32_e32 v4, 0, v3
+; GFX9-NEXT:    v_sub_u32_e32 v4, 0x7fffffff, v4
+; GFX9-NEXT:    v_max_i32_e32 v5, v5, v7
+; GFX9-NEXT:    v_min_i32_e32 v4, v5, v4
+; GFX9-NEXT:    v_add_u32_e32 v3, v3, v4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_saddsat_v4i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_min_i32_e32 v8, 0, v0
+; GFX10-NEXT:    s_mov_b32 s4, 0x80000000
+; GFX10-NEXT:    v_min_i32_e32 v11, 0, v1
+; GFX10-NEXT:    v_min_i32_e32 v12, 0, v3
+; GFX10-NEXT:    v_max_i32_e32 v9, 0, v0
+; GFX10-NEXT:    v_sub_nc_u32_e32 v15, s4, v8
+; GFX10-NEXT:    v_min_i32_e32 v8, 0, v2
+; GFX10-NEXT:    v_sub_nc_u32_e32 v11, s4, v11
+; GFX10-NEXT:    v_sub_nc_u32_e32 v12, 0x80000000, v12
+; GFX10-NEXT:    v_max_i32_e32 v10, 0, v1
+; GFX10-NEXT:    v_max_i32_e32 v13, 0, v2
+; GFX10-NEXT:    v_sub_nc_u32_e32 v8, s4, v8
+; GFX10-NEXT:    v_max_i32_e32 v14, 0, v3
+; GFX10-NEXT:    s_brev_b32 s5, -2
+; GFX10-NEXT:    v_max_i32_e32 v5, v11, v5
+; GFX10-NEXT:    v_sub_nc_u32_e32 v10, s5, v10
+; GFX10-NEXT:    v_max_i32_e32 v6, v8, v6
+; GFX10-NEXT:    v_sub_nc_u32_e32 v11, s5, v13
+; GFX10-NEXT:    v_sub_nc_u32_e32 v9, s5, v9
+; GFX10-NEXT:    v_max_i32_e32 v4, v15, v4
+; GFX10-NEXT:    v_sub_nc_u32_e32 v8, 0x7fffffff, v14
+; GFX10-NEXT:    v_max_i32_e32 v7, v12, v7
+; GFX10-NEXT:    v_min_i32_e32 v11, v6, v11
+; GFX10-NEXT:    v_min_i32_e32 v19, v5, v10
+; GFX10-NEXT:    v_min_i32_e32 v15, v4, v9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_min_i32_e32 v6, v7, v8
+; GFX10-NEXT:    v_add_nc_u32_e32 v2, v2, v11
+; GFX10-NEXT:    v_add_nc_u32_e32 v1, v1, v19
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v15
+; GFX10-NEXT:    v_add_nc_u32_e32 v3, v3, v6
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %lhs, <4 x i32> %rhs)
+  ret <4 x i32> %result
+}
+
+define amdgpu_ps <4 x i32> @s_saddsat_v4i32(<4 x i32> inreg %lhs, <4 x i32> inreg %rhs) {
+; GFX6-LABEL: s_saddsat_v4i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX6-NEXT:    s_brev_b32 s8, -2
+; GFX6-NEXT:    s_cselect_b32 s10, s0, 0
+; GFX6-NEXT:    s_sub_i32 s10, s8, s10
+; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX6-NEXT:    s_mov_b32 s9, 0x80000000
+; GFX6-NEXT:    s_cselect_b32 s11, s0, 0
+; GFX6-NEXT:    s_sub_i32 s11, s9, s11
+; GFX6-NEXT:    s_cmp_gt_i32 s11, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s11, s4
+; GFX6-NEXT:    s_cmp_lt_i32 s4, s10
+; GFX6-NEXT:    s_cselect_b32 s4, s4, s10
+; GFX6-NEXT:    s_add_i32 s0, s0, s4
+; GFX6-NEXT:    s_cmp_gt_i32 s1, 0
+; GFX6-NEXT:    s_cselect_b32 s4, s1, 0
+; GFX6-NEXT:    s_sub_i32 s4, s8, s4
+; GFX6-NEXT:    s_cmp_lt_i32 s1, 0
+; GFX6-NEXT:    s_cselect_b32 s10, s1, 0
+; GFX6-NEXT:    s_sub_i32 s10, s9, s10
+; GFX6-NEXT:    s_cmp_gt_i32 s10, s5
+; GFX6-NEXT:    s_cselect_b32 s5, s10, s5
+; GFX6-NEXT:    s_cmp_lt_i32 s5, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX6-NEXT:    s_add_i32 s1, s1, s4
+; GFX6-NEXT:    s_cmp_gt_i32 s2, 0
+; GFX6-NEXT:    s_cselect_b32 s4, s2, 0
+; GFX6-NEXT:    s_sub_i32 s4, s8, s4
+; GFX6-NEXT:    s_cmp_lt_i32 s2, 0
+; GFX6-NEXT:    s_cselect_b32 s5, s2, 0
+; GFX6-NEXT:    s_sub_i32 s5, s9, s5
+; GFX6-NEXT:    s_cmp_gt_i32 s5, s6
+; GFX6-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX6-NEXT:    s_cmp_lt_i32 s5, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX6-NEXT:    s_add_i32 s2, s2, s4
+; GFX6-NEXT:    s_cmp_gt_i32 s3, 0
+; GFX6-NEXT:    s_cselect_b32 s4, s3, 0
+; GFX6-NEXT:    s_sub_i32 s4, s8, s4
+; GFX6-NEXT:    s_cmp_lt_i32 s3, 0
+; GFX6-NEXT:    s_cselect_b32 s5, s3, 0
+; GFX6-NEXT:    s_sub_i32 s5, s9, s5
+; GFX6-NEXT:    s_cmp_gt_i32 s5, s7
+; GFX6-NEXT:    s_cselect_b32 s5, s5, s7
+; GFX6-NEXT:    s_cmp_lt_i32 s5, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX6-NEXT:    s_add_i32 s3, s3, s4
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_saddsat_v4i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX8-NEXT:    s_brev_b32 s8, -2
+; GFX8-NEXT:    s_cselect_b32 s10, s0, 0
+; GFX8-NEXT:    s_sub_i32 s10, s8, s10
+; GFX8-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX8-NEXT:    s_mov_b32 s9, 0x80000000
+; GFX8-NEXT:    s_cselect_b32 s11, s0, 0
+; GFX8-NEXT:    s_sub_i32 s11, s9, s11
+; GFX8-NEXT:    s_cmp_gt_i32 s11, s4
+; GFX8-NEXT:    s_cselect_b32 s4, s11, s4
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s10
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s10
+; GFX8-NEXT:    s_add_i32 s0, s0, s4
+; GFX8-NEXT:    s_cmp_gt_i32 s1, 0
+; GFX8-NEXT:    s_cselect_b32 s4, s1, 0
+; GFX8-NEXT:    s_sub_i32 s4, s8, s4
+; GFX8-NEXT:    s_cmp_lt_i32 s1, 0
+; GFX8-NEXT:    s_cselect_b32 s10, s1, 0
+; GFX8-NEXT:    s_sub_i32 s10, s9, s10
+; GFX8-NEXT:    s_cmp_gt_i32 s10, s5
+; GFX8-NEXT:    s_cselect_b32 s5, s10, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s5, s4
+; GFX8-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX8-NEXT:    s_add_i32 s1, s1, s4
+; GFX8-NEXT:    s_cmp_gt_i32 s2, 0
+; GFX8-NEXT:    s_cselect_b32 s4, s2, 0
+; GFX8-NEXT:    s_sub_i32 s4, s8, s4
+; GFX8-NEXT:    s_cmp_lt_i32 s2, 0
+; GFX8-NEXT:    s_cselect_b32 s5, s2, 0
+; GFX8-NEXT:    s_sub_i32 s5, s9, s5
+; GFX8-NEXT:    s_cmp_gt_i32 s5, s6
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX8-NEXT:    s_cmp_lt_i32 s5, s4
+; GFX8-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX8-NEXT:    s_add_i32 s2, s2, s4
+; GFX8-NEXT:    s_cmp_gt_i32 s3, 0
+; GFX8-NEXT:    s_cselect_b32 s4, s3, 0
+; GFX8-NEXT:    s_sub_i32 s4, s8, s4
+; GFX8-NEXT:    s_cmp_lt_i32 s3, 0
+; GFX8-NEXT:    s_cselect_b32 s5, s3, 0
+; GFX8-NEXT:    s_sub_i32 s5, s9, s5
+; GFX8-NEXT:    s_cmp_gt_i32 s5, s7
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s7
+; GFX8-NEXT:    s_cmp_lt_i32 s5, s4
+; GFX8-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX8-NEXT:    s_add_i32 s3, s3, s4
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_saddsat_v4i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX9-NEXT:    s_brev_b32 s8, -2
+; GFX9-NEXT:    s_cselect_b32 s10, s0, 0
+; GFX9-NEXT:    s_sub_i32 s10, s8, s10
+; GFX9-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX9-NEXT:    s_mov_b32 s9, 0x80000000
+; GFX9-NEXT:    s_cselect_b32 s11, s0, 0
+; GFX9-NEXT:    s_sub_i32 s11, s9, s11
+; GFX9-NEXT:    s_cmp_gt_i32 s11, s4
+; GFX9-NEXT:    s_cselect_b32 s4, s11, s4
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s10
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s10
+; GFX9-NEXT:    s_add_i32 s0, s0, s4
+; GFX9-NEXT:    s_cmp_gt_i32 s1, 0
+; GFX9-NEXT:    s_cselect_b32 s4, s1, 0
+; GFX9-NEXT:    s_sub_i32 s4, s8, s4
+; GFX9-NEXT:    s_cmp_lt_i32 s1, 0
+; GFX9-NEXT:    s_cselect_b32 s10, s1, 0
+; GFX9-NEXT:    s_sub_i32 s10, s9, s10
+; GFX9-NEXT:    s_cmp_gt_i32 s10, s5
+; GFX9-NEXT:    s_cselect_b32 s5, s10, s5
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s4
+; GFX9-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX9-NEXT:    s_add_i32 s1, s1, s4
+; GFX9-NEXT:    s_cmp_gt_i32 s2, 0
+; GFX9-NEXT:    s_cselect_b32 s4, s2, 0
+; GFX9-NEXT:    s_sub_i32 s4, s8, s4
+; GFX9-NEXT:    s_cmp_lt_i32 s2, 0
+; GFX9-NEXT:    s_cselect_b32 s5, s2, 0
+; GFX9-NEXT:    s_sub_i32 s5, s9, s5
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s6
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s4
+; GFX9-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX9-NEXT:    s_add_i32 s2, s2, s4
+; GFX9-NEXT:    s_cmp_gt_i32 s3, 0
+; GFX9-NEXT:    s_cselect_b32 s4, s3, 0
+; GFX9-NEXT:    s_sub_i32 s4, s8, s4
+; GFX9-NEXT:    s_cmp_lt_i32 s3, 0
+; GFX9-NEXT:    s_cselect_b32 s5, s3, 0
+; GFX9-NEXT:    s_sub_i32 s5, s9, s5
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s7
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s7
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s4
+; GFX9-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX9-NEXT:    s_add_i32 s3, s3, s4
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_saddsat_v4i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX10-NEXT:    s_brev_b32 s8, -2
+; GFX10-NEXT:    s_cselect_b32 s9, s0, 0
+; GFX10-NEXT:    s_mov_b32 s10, 0x80000000
+; GFX10-NEXT:    s_sub_i32 s9, s8, s9
+; GFX10-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s11, s0, 0
+; GFX10-NEXT:    s_sub_i32 s11, s10, s11
+; GFX10-NEXT:    s_cmp_gt_i32 s11, s4
+; GFX10-NEXT:    s_cselect_b32 s4, s11, s4
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s9
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s9
+; GFX10-NEXT:    s_add_i32 s0, s0, s4
+; GFX10-NEXT:    s_cmp_gt_i32 s1, 0
+; GFX10-NEXT:    s_cselect_b32 s4, s1, 0
+; GFX10-NEXT:    s_sub_i32 s4, s8, s4
+; GFX10-NEXT:    s_cmp_lt_i32 s1, 0
+; GFX10-NEXT:    s_cselect_b32 s9, s1, 0
+; GFX10-NEXT:    s_sub_i32 s9, s10, s9
+; GFX10-NEXT:    s_cmp_gt_i32 s9, s5
+; GFX10-NEXT:    s_cselect_b32 s5, s9, s5
+; GFX10-NEXT:    s_cmp_lt_i32 s5, s4
+; GFX10-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX10-NEXT:    s_add_i32 s1, s1, s4
+; GFX10-NEXT:    s_cmp_gt_i32 s2, 0
+; GFX10-NEXT:    s_cselect_b32 s4, s2, 0
+; GFX10-NEXT:    s_sub_i32 s4, s8, s4
+; GFX10-NEXT:    s_cmp_lt_i32 s2, 0
+; GFX10-NEXT:    s_cselect_b32 s5, s2, 0
+; GFX10-NEXT:    s_sub_i32 s5, s10, s5
+; GFX10-NEXT:    s_cmp_gt_i32 s5, s6
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX10-NEXT:    s_cmp_lt_i32 s5, s4
+; GFX10-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX10-NEXT:    s_add_i32 s2, s2, s4
+; GFX10-NEXT:    s_cmp_gt_i32 s3, 0
+; GFX10-NEXT:    s_cselect_b32 s4, s3, 0
+; GFX10-NEXT:    s_sub_i32 s4, s8, s4
+; GFX10-NEXT:    s_cmp_lt_i32 s3, 0
+; GFX10-NEXT:    s_cselect_b32 s5, s3, 0
+; GFX10-NEXT:    s_sub_i32 s5, s10, s5
+; GFX10-NEXT:    s_cmp_gt_i32 s5, s7
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s7
+; GFX10-NEXT:    s_cmp_lt_i32 s5, s4
+; GFX10-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX10-NEXT:    s_add_i32 s3, s3, s4
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %lhs, <4 x i32> %rhs)
+  ret <4 x i32> %result
+}
+
+define <5 x i32> @v_saddsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
+; GFX6-LABEL: v_saddsat_v5i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v12, 0, v0
+; GFX6-NEXT:    v_sub_i32_e32 v12, vcc, s5, v12
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    v_max_i32_e32 v10, 0, v0
+; GFX6-NEXT:    v_sub_i32_e32 v10, vcc, s4, v10
+; GFX6-NEXT:    v_max_i32_e32 v5, v12, v5
+; GFX6-NEXT:    v_min_i32_e32 v5, v5, v10
+; GFX6-NEXT:    v_min_i32_e32 v10, 0, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v5
+; GFX6-NEXT:    v_max_i32_e32 v5, 0, v1
+; GFX6-NEXT:    v_sub_i32_e32 v10, vcc, s5, v10
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, s4, v5
+; GFX6-NEXT:    v_max_i32_e32 v6, v10, v6
+; GFX6-NEXT:    v_min_i32_e32 v5, v6, v5
+; GFX6-NEXT:    v_min_i32_e32 v6, 0, v2
+; GFX6-NEXT:    v_sub_i32_e32 v6, vcc, s5, v6
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v5
+; GFX6-NEXT:    v_max_i32_e32 v5, 0, v2
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, s4, v5
+; GFX6-NEXT:    v_max_i32_e32 v6, v6, v7
+; GFX6-NEXT:    v_min_i32_e32 v5, v6, v5
+; GFX6-NEXT:    v_mov_b32_e32 v13, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v6, 0, v3
+; GFX6-NEXT:    v_sub_i32_e32 v6, vcc, v13, v6
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GFX6-NEXT:    v_bfrev_b32_e32 v11, -2
+; GFX6-NEXT:    v_max_i32_e32 v5, 0, v3
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, v11, v5
+; GFX6-NEXT:    v_max_i32_e32 v6, v6, v8
+; GFX6-NEXT:    v_min_i32_e32 v5, v6, v5
+; GFX6-NEXT:    v_min_i32_e32 v6, 0, v4
+; GFX6-NEXT:    v_sub_i32_e32 v6, vcc, v13, v6
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GFX6-NEXT:    v_max_i32_e32 v5, 0, v4
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, v11, v5
+; GFX6-NEXT:    v_max_i32_e32 v6, v6, v9
+; GFX6-NEXT:    v_min_i32_e32 v5, v6, v5
+; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_v5i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX8-NEXT:    v_min_i32_e32 v12, 0, v0
+; GFX8-NEXT:    v_sub_u32_e32 v12, vcc, s5, v12
+; GFX8-NEXT:    s_brev_b32 s4, -2
+; GFX8-NEXT:    v_max_i32_e32 v10, 0, v0
+; GFX8-NEXT:    v_sub_u32_e32 v10, vcc, s4, v10
+; GFX8-NEXT:    v_max_i32_e32 v5, v12, v5
+; GFX8-NEXT:    v_min_i32_e32 v5, v5, v10
+; GFX8-NEXT:    v_min_i32_e32 v10, 0, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v5
+; GFX8-NEXT:    v_max_i32_e32 v5, 0, v1
+; GFX8-NEXT:    v_sub_u32_e32 v10, vcc, s5, v10
+; GFX8-NEXT:    v_sub_u32_e32 v5, vcc, s4, v5
+; GFX8-NEXT:    v_max_i32_e32 v6, v10, v6
+; GFX8-NEXT:    v_min_i32_e32 v5, v6, v5
+; GFX8-NEXT:    v_min_i32_e32 v6, 0, v2
+; GFX8-NEXT:    v_sub_u32_e32 v6, vcc, s5, v6
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v5
+; GFX8-NEXT:    v_max_i32_e32 v5, 0, v2
+; GFX8-NEXT:    v_sub_u32_e32 v5, vcc, s4, v5
+; GFX8-NEXT:    v_max_i32_e32 v6, v6, v7
+; GFX8-NEXT:    v_min_i32_e32 v5, v6, v5
+; GFX8-NEXT:    v_mov_b32_e32 v13, 0x80000000
+; GFX8-NEXT:    v_min_i32_e32 v6, 0, v3
+; GFX8-NEXT:    v_sub_u32_e32 v6, vcc, v13, v6
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v5
+; GFX8-NEXT:    v_bfrev_b32_e32 v11, -2
+; GFX8-NEXT:    v_max_i32_e32 v5, 0, v3
+; GFX8-NEXT:    v_sub_u32_e32 v5, vcc, v11, v5
+; GFX8-NEXT:    v_max_i32_e32 v6, v6, v8
+; GFX8-NEXT:    v_min_i32_e32 v5, v6, v5
+; GFX8-NEXT:    v_min_i32_e32 v6, 0, v4
+; GFX8-NEXT:    v_sub_u32_e32 v6, vcc, v13, v6
+; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v5
+; GFX8-NEXT:    v_max_i32_e32 v5, 0, v4
+; GFX8-NEXT:    v_sub_u32_e32 v5, vcc, v11, v5
+; GFX8-NEXT:    v_max_i32_e32 v6, v6, v9
+; GFX8-NEXT:    v_min_i32_e32 v5, v6, v5
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v4, v5
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_v5i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX9-NEXT:    v_min_i32_e32 v12, 0, v0
+; GFX9-NEXT:    v_sub_u32_e32 v12, s5, v12
+; GFX9-NEXT:    s_brev_b32 s4, -2
+; GFX9-NEXT:    v_max_i32_e32 v10, 0, v0
+; GFX9-NEXT:    v_sub_u32_e32 v10, s4, v10
+; GFX9-NEXT:    v_max_i32_e32 v5, v12, v5
+; GFX9-NEXT:    v_min_i32_e32 v5, v5, v10
+; GFX9-NEXT:    v_min_i32_e32 v10, 0, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v5
+; GFX9-NEXT:    v_max_i32_e32 v5, 0, v1
+; GFX9-NEXT:    v_sub_u32_e32 v10, s5, v10
+; GFX9-NEXT:    v_sub_u32_e32 v5, s4, v5
+; GFX9-NEXT:    v_max_i32_e32 v6, v10, v6
+; GFX9-NEXT:    v_min_i32_e32 v5, v6, v5
+; GFX9-NEXT:    v_min_i32_e32 v6, 0, v2
+; GFX9-NEXT:    v_sub_u32_e32 v6, s5, v6
+; GFX9-NEXT:    v_add_u32_e32 v1, v1, v5
+; GFX9-NEXT:    v_max_i32_e32 v5, 0, v2
+; GFX9-NEXT:    v_sub_u32_e32 v5, s4, v5
+; GFX9-NEXT:    v_max_i32_e32 v6, v6, v7
+; GFX9-NEXT:    v_min_i32_e32 v5, v6, v5
+; GFX9-NEXT:    v_mov_b32_e32 v13, 0x80000000
+; GFX9-NEXT:    v_min_i32_e32 v6, 0, v3
+; GFX9-NEXT:    v_sub_u32_e32 v6, v13, v6
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v5
+; GFX9-NEXT:    v_bfrev_b32_e32 v11, -2
+; GFX9-NEXT:    v_max_i32_e32 v5, 0, v3
+; GFX9-NEXT:    v_sub_u32_e32 v5, v11, v5
+; GFX9-NEXT:    v_max_i32_e32 v6, v6, v8
+; GFX9-NEXT:    v_min_i32_e32 v5, v6, v5
+; GFX9-NEXT:    v_min_i32_e32 v6, 0, v4
+; GFX9-NEXT:    v_sub_u32_e32 v6, v13, v6
+; GFX9-NEXT:    v_add_u32_e32 v3, v3, v5
+; GFX9-NEXT:    v_max_i32_e32 v5, 0, v4
+; GFX9-NEXT:    v_sub_u32_e32 v5, v11, v5
+; GFX9-NEXT:    v_max_i32_e32 v6, v6, v9
+; GFX9-NEXT:    v_min_i32_e32 v5, v6, v5
+; GFX9-NEXT:    v_add_u32_e32 v4, v4, v5
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_saddsat_v5i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_min_i32_e32 v13, 0, v1
+; GFX10-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX10-NEXT:    v_min_i32_e32 v10, 0, v0
+; GFX10-NEXT:    v_min_i32_e32 v16, 0, v2
+; GFX10-NEXT:    v_mov_b32_e32 v15, 0x80000000
+; GFX10-NEXT:    v_sub_nc_u32_e32 v13, s5, v13
+; GFX10-NEXT:    v_min_i32_e32 v17, 0, v4
+; GFX10-NEXT:    v_sub_nc_u32_e32 v10, s5, v10
+; GFX10-NEXT:    v_sub_nc_u32_e32 v16, s5, v16
+; GFX10-NEXT:    v_max_i32_e32 v11, 0, v0
+; GFX10-NEXT:    v_max_i32_e32 v23, v13, v6
+; GFX10-NEXT:    v_min_i32_e32 v13, 0, v3
+; GFX10-NEXT:    v_max_i32_e32 v5, v10, v5
+; GFX10-NEXT:    v_bfrev_b32_e32 v12, -2
+; GFX10-NEXT:    v_max_i32_e32 v14, 0, v1
+; GFX10-NEXT:    v_max_i32_e32 v10, 0, v2
+; GFX10-NEXT:    v_sub_nc_u32_e32 v13, v15, v13
+; GFX10-NEXT:    v_sub_nc_u32_e32 v15, v15, v17
+; GFX10-NEXT:    v_max_i32_e32 v18, 0, v3
+; GFX10-NEXT:    v_max_i32_e32 v19, 0, v4
+; GFX10-NEXT:    s_brev_b32 s4, -2
+; GFX10-NEXT:    v_max_i32_e32 v7, v16, v7
+; GFX10-NEXT:    v_sub_nc_u32_e32 v11, s4, v11
+; GFX10-NEXT:    v_sub_nc_u32_e32 v14, s4, v14
+; GFX10-NEXT:    v_sub_nc_u32_e32 v10, s4, v10
+; GFX10-NEXT:    v_sub_nc_u32_e32 v16, v12, v18
+; GFX10-NEXT:    v_max_i32_e32 v27, v13, v8
+; GFX10-NEXT:    v_sub_nc_u32_e32 v12, v12, v19
+; GFX10-NEXT:    v_max_i32_e32 v9, v15, v9
+; GFX10-NEXT:    v_min_i32_e32 v5, v5, v11
+; GFX10-NEXT:    v_min_i32_e32 v6, v23, v14
+; GFX10-NEXT:    v_min_i32_e32 v7, v7, v10
+; GFX10-NEXT:    v_min_i32_e32 v8, v27, v16
+; GFX10-NEXT:    v_min_i32_e32 v9, v9, v12
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v5
+; GFX10-NEXT:    v_add_nc_u32_e32 v1, v1, v6
+; GFX10-NEXT:    v_add_nc_u32_e32 v2, v2, v7
+; GFX10-NEXT:    v_add_nc_u32_e32 v3, v3, v8
+; GFX10-NEXT:    v_add_nc_u32_e32 v4, v4, v9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <5 x i32> @llvm.sadd.sat.v5i32(<5 x i32> %lhs, <5 x i32> %rhs)
+  ret <5 x i32> %result
+}
+
+define amdgpu_ps <5 x i32> @s_saddsat_v5i32(<5 x i32> inreg %lhs, <5 x i32> inreg %rhs) {
+; GFX6-LABEL: s_saddsat_v5i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX6-NEXT:    s_brev_b32 s10, -2
+; GFX6-NEXT:    s_cselect_b32 s12, s0, 0
+; GFX6-NEXT:    s_sub_i32 s12, s10, s12
+; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX6-NEXT:    s_mov_b32 s11, 0x80000000
+; GFX6-NEXT:    s_cselect_b32 s13, s0, 0
+; GFX6-NEXT:    s_sub_i32 s13, s11, s13
+; GFX6-NEXT:    s_cmp_gt_i32 s13, s5
+; GFX6-NEXT:    s_cselect_b32 s5, s13, s5
+; GFX6-NEXT:    s_cmp_lt_i32 s5, s12
+; GFX6-NEXT:    s_cselect_b32 s5, s5, s12
+; GFX6-NEXT:    s_add_i32 s0, s0, s5
+; GFX6-NEXT:    s_cmp_gt_i32 s1, 0
+; GFX6-NEXT:    s_cselect_b32 s5, s1, 0
+; GFX6-NEXT:    s_sub_i32 s5, s10, s5
+; GFX6-NEXT:    s_cmp_lt_i32 s1, 0
+; GFX6-NEXT:    s_cselect_b32 s12, s1, 0
+; GFX6-NEXT:    s_sub_i32 s12, s11, s12
+; GFX6-NEXT:    s_cmp_gt_i32 s12, s6
+; GFX6-NEXT:    s_cselect_b32 s6, s12, s6
+; GFX6-NEXT:    s_cmp_lt_i32 s6, s5
+; GFX6-NEXT:    s_cselect_b32 s5, s6, s5
+; GFX6-NEXT:    s_add_i32 s1, s1, s5
+; GFX6-NEXT:    s_cmp_gt_i32 s2, 0
+; GFX6-NEXT:    s_cselect_b32 s5, s2, 0
+; GFX6-NEXT:    s_sub_i32 s5, s10, s5
+; GFX6-NEXT:    s_cmp_lt_i32 s2, 0
+; GFX6-NEXT:    s_cselect_b32 s6, s2, 0
+; GFX6-NEXT:    s_sub_i32 s6, s11, s6
+; GFX6-NEXT:    s_cmp_gt_i32 s6, s7
+; GFX6-NEXT:    s_cselect_b32 s6, s6, s7
+; GFX6-NEXT:    s_cmp_lt_i32 s6, s5
+; GFX6-NEXT:    s_cselect_b32 s5, s6, s5
+; GFX6-NEXT:    s_add_i32 s2, s2, s5
+; GFX6-NEXT:    s_cmp_gt_i32 s3, 0
+; GFX6-NEXT:    s_cselect_b32 s5, s3, 0
+; GFX6-NEXT:    s_sub_i32 s5, s10, s5
+; GFX6-NEXT:    s_cmp_lt_i32 s3, 0
+; GFX6-NEXT:    s_cselect_b32 s6, s3, 0
+; GFX6-NEXT:    s_sub_i32 s6, s11, s6
+; GFX6-NEXT:    s_cmp_gt_i32 s6, s8
+; GFX6-NEXT:    s_cselect_b32 s6, s6, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s6, s5
+; GFX6-NEXT:    s_cselect_b32 s5, s6, s5
+; GFX6-NEXT:    s_add_i32 s3, s3, s5
+; GFX6-NEXT:    s_cmp_gt_i32 s4, 0
+; GFX6-NEXT:    s_cselect_b32 s5, s4, 0
+; GFX6-NEXT:    s_sub_i32 s5, s10, s5
+; GFX6-NEXT:    s_cmp_lt_i32 s4, 0
+; GFX6-NEXT:    s_cselect_b32 s6, s4, 0
+; GFX6-NEXT:    s_sub_i32 s6, s11, s6
+; GFX6-NEXT:    s_cmp_gt_i32 s6, s9
+; GFX6-NEXT:    s_cselect_b32 s6, s6, s9
+; GFX6-NEXT:    s_cmp_lt_i32 s6, s5
+; GFX6-NEXT:    s_cselect_b32 s5, s6, s5
+; GFX6-NEXT:    s_add_i32 s4, s4, s5
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_saddsat_v5i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX8-NEXT:    s_brev_b32 s10, -2
+; GFX8-NEXT:    s_cselect_b32 s12, s0, 0
+; GFX8-NEXT:    s_sub_i32 s12, s10, s12
+; GFX8-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX8-NEXT:    s_mov_b32 s11, 0x80000000
+; GFX8-NEXT:    s_cselect_b32 s13, s0, 0
+; GFX8-NEXT:    s_sub_i32 s13, s11, s13
+; GFX8-NEXT:    s_cmp_gt_i32 s13, s5
+; GFX8-NEXT:    s_cselect_b32 s5, s13, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s5, s12
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s12
+; GFX8-NEXT:    s_add_i32 s0, s0, s5
+; GFX8-NEXT:    s_cmp_gt_i32 s1, 0
+; GFX8-NEXT:    s_cselect_b32 s5, s1, 0
+; GFX8-NEXT:    s_sub_i32 s5, s10, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s1, 0
+; GFX8-NEXT:    s_cselect_b32 s12, s1, 0
+; GFX8-NEXT:    s_sub_i32 s12, s11, s12
+; GFX8-NEXT:    s_cmp_gt_i32 s12, s6
+; GFX8-NEXT:    s_cselect_b32 s6, s12, s6
+; GFX8-NEXT:    s_cmp_lt_i32 s6, s5
+; GFX8-NEXT:    s_cselect_b32 s5, s6, s5
+; GFX8-NEXT:    s_add_i32 s1, s1, s5
+; GFX8-NEXT:    s_cmp_gt_i32 s2, 0
+; GFX8-NEXT:    s_cselect_b32 s5, s2, 0
+; GFX8-NEXT:    s_sub_i32 s5, s10, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s2, 0
+; GFX8-NEXT:    s_cselect_b32 s6, s2, 0
+; GFX8-NEXT:    s_sub_i32 s6, s11, s6
+; GFX8-NEXT:    s_cmp_gt_i32 s6, s7
+; GFX8-NEXT:    s_cselect_b32 s6, s6, s7
+; GFX8-NEXT:    s_cmp_lt_i32 s6, s5
+; GFX8-NEXT:    s_cselect_b32 s5, s6, s5
+; GFX8-NEXT:    s_add_i32 s2, s2, s5
+; GFX8-NEXT:    s_cmp_gt_i32 s3, 0
+; GFX8-NEXT:    s_cselect_b32 s5, s3, 0
+; GFX8-NEXT:    s_sub_i32 s5, s10, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s3, 0
+; GFX8-NEXT:    s_cselect_b32 s6, s3, 0
+; GFX8-NEXT:    s_sub_i32 s6, s11, s6
+; GFX8-NEXT:    s_cmp_gt_i32 s6, s8
+; GFX8-NEXT:    s_cselect_b32 s6, s6, s8
+; GFX8-NEXT:    s_cmp_lt_i32 s6, s5
+; GFX8-NEXT:    s_cselect_b32 s5, s6, s5
+; GFX8-NEXT:    s_add_i32 s3, s3, s5
+; GFX8-NEXT:    s_cmp_gt_i32 s4, 0
+; GFX8-NEXT:    s_cselect_b32 s5, s4, 0
+; GFX8-NEXT:    s_sub_i32 s5, s10, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s4, 0
+; GFX8-NEXT:    s_cselect_b32 s6, s4, 0
+; GFX8-NEXT:    s_sub_i32 s6, s11, s6
+; GFX8-NEXT:    s_cmp_gt_i32 s6, s9
+; GFX8-NEXT:    s_cselect_b32 s6, s6, s9
+; GFX8-NEXT:    s_cmp_lt_i32 s6, s5
+; GFX8-NEXT:    s_cselect_b32 s5, s6, s5
+; GFX8-NEXT:    s_add_i32 s4, s4, s5
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_saddsat_v5i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX9-NEXT:    s_brev_b32 s10, -2
+; GFX9-NEXT:    s_cselect_b32 s12, s0, 0
+; GFX9-NEXT:    s_sub_i32 s12, s10, s12
+; GFX9-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX9-NEXT:    s_mov_b32 s11, 0x80000000
+; GFX9-NEXT:    s_cselect_b32 s13, s0, 0
+; GFX9-NEXT:    s_sub_i32 s13, s11, s13
+; GFX9-NEXT:    s_cmp_gt_i32 s13, s5
+; GFX9-NEXT:    s_cselect_b32 s5, s13, s5
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s12
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s12
+; GFX9-NEXT:    s_add_i32 s0, s0, s5
+; GFX9-NEXT:    s_cmp_gt_i32 s1, 0
+; GFX9-NEXT:    s_cselect_b32 s5, s1, 0
+; GFX9-NEXT:    s_sub_i32 s5, s10, s5
+; GFX9-NEXT:    s_cmp_lt_i32 s1, 0
+; GFX9-NEXT:    s_cselect_b32 s12, s1, 0
+; GFX9-NEXT:    s_sub_i32 s12, s11, s12
+; GFX9-NEXT:    s_cmp_gt_i32 s12, s6
+; GFX9-NEXT:    s_cselect_b32 s6, s12, s6
+; GFX9-NEXT:    s_cmp_lt_i32 s6, s5
+; GFX9-NEXT:    s_cselect_b32 s5, s6, s5
+; GFX9-NEXT:    s_add_i32 s1, s1, s5
+; GFX9-NEXT:    s_cmp_gt_i32 s2, 0
+; GFX9-NEXT:    s_cselect_b32 s5, s2, 0
+; GFX9-NEXT:    s_sub_i32 s5, s10, s5
+; GFX9-NEXT:    s_cmp_lt_i32 s2, 0
+; GFX9-NEXT:    s_cselect_b32 s6, s2, 0
+; GFX9-NEXT:    s_sub_i32 s6, s11, s6
+; GFX9-NEXT:    s_cmp_gt_i32 s6, s7
+; GFX9-NEXT:    s_cselect_b32 s6, s6, s7
+; GFX9-NEXT:    s_cmp_lt_i32 s6, s5
+; GFX9-NEXT:    s_cselect_b32 s5, s6, s5
+; GFX9-NEXT:    s_add_i32 s2, s2, s5
+; GFX9-NEXT:    s_cmp_gt_i32 s3, 0
+; GFX9-NEXT:    s_cselect_b32 s5, s3, 0
+; GFX9-NEXT:    s_sub_i32 s5, s10, s5
+; GFX9-NEXT:    s_cmp_lt_i32 s3, 0
+; GFX9-NEXT:    s_cselect_b32 s6, s3, 0
+; GFX9-NEXT:    s_sub_i32 s6, s11, s6
+; GFX9-NEXT:    s_cmp_gt_i32 s6, s8
+; GFX9-NEXT:    s_cselect_b32 s6, s6, s8
+; GFX9-NEXT:    s_cmp_lt_i32 s6, s5
+; GFX9-NEXT:    s_cselect_b32 s5, s6, s5
+; GFX9-NEXT:    s_add_i32 s3, s3, s5
+; GFX9-NEXT:    s_cmp_gt_i32 s4, 0
+; GFX9-NEXT:    s_cselect_b32 s5, s4, 0
+; GFX9-NEXT:    s_sub_i32 s5, s10, s5
+; GFX9-NEXT:    s_cmp_lt_i32 s4, 0
+; GFX9-NEXT:    s_cselect_b32 s6, s4, 0
+; GFX9-NEXT:    s_sub_i32 s6, s11, s6
+; GFX9-NEXT:    s_cmp_gt_i32 s6, s9
+; GFX9-NEXT:    s_cselect_b32 s6, s6, s9
+; GFX9-NEXT:    s_cmp_lt_i32 s6, s5
+; GFX9-NEXT:    s_cselect_b32 s5, s6, s5
+; GFX9-NEXT:    s_add_i32 s4, s4, s5
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_saddsat_v5i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX10-NEXT:    s_brev_b32 s10, -2
+; GFX10-NEXT:    s_cselect_b32 s11, s0, 0
+; GFX10-NEXT:    s_mov_b32 s12, 0x80000000
+; GFX10-NEXT:    s_sub_i32 s11, s10, s11
+; GFX10-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s13, s0, 0
+; GFX10-NEXT:    s_sub_i32 s13, s12, s13
+; GFX10-NEXT:    s_cmp_gt_i32 s13, s5
+; GFX10-NEXT:    s_cselect_b32 s5, s13, s5
+; GFX10-NEXT:    s_cmp_lt_i32 s5, s11
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s11
+; GFX10-NEXT:    s_add_i32 s0, s0, s5
+; GFX10-NEXT:    s_cmp_gt_i32 s1, 0
+; GFX10-NEXT:    s_cselect_b32 s5, s1, 0
+; GFX10-NEXT:    s_sub_i32 s5, s10, s5
+; GFX10-NEXT:    s_cmp_lt_i32 s1, 0
+; GFX10-NEXT:    s_cselect_b32 s11, s1, 0
+; GFX10-NEXT:    s_sub_i32 s11, s12, s11
+; GFX10-NEXT:    s_cmp_gt_i32 s11, s6
+; GFX10-NEXT:    s_cselect_b32 s6, s11, s6
+; GFX10-NEXT:    s_cmp_lt_i32 s6, s5
+; GFX10-NEXT:    s_cselect_b32 s5, s6, s5
+; GFX10-NEXT:    s_add_i32 s1, s1, s5
+; GFX10-NEXT:    s_cmp_gt_i32 s2, 0
+; GFX10-NEXT:    s_cselect_b32 s5, s2, 0
+; GFX10-NEXT:    s_sub_i32 s5, s10, s5
+; GFX10-NEXT:    s_cmp_lt_i32 s2, 0
+; GFX10-NEXT:    s_cselect_b32 s6, s2, 0
+; GFX10-NEXT:    s_sub_i32 s6, s12, s6
+; GFX10-NEXT:    s_cmp_gt_i32 s6, s7
+; GFX10-NEXT:    s_cselect_b32 s6, s6, s7
+; GFX10-NEXT:    s_cmp_lt_i32 s6, s5
+; GFX10-NEXT:    s_cselect_b32 s5, s6, s5
+; GFX10-NEXT:    s_add_i32 s2, s2, s5
+; GFX10-NEXT:    s_cmp_gt_i32 s3, 0
+; GFX10-NEXT:    s_cselect_b32 s5, s3, 0
+; GFX10-NEXT:    s_sub_i32 s5, s10, s5
+; GFX10-NEXT:    s_cmp_lt_i32 s3, 0
+; GFX10-NEXT:    s_cselect_b32 s6, s3, 0
+; GFX10-NEXT:    s_sub_i32 s6, s12, s6
+; GFX10-NEXT:    s_cmp_gt_i32 s6, s8
+; GFX10-NEXT:    s_cselect_b32 s6, s6, s8
+; GFX10-NEXT:    s_cmp_lt_i32 s6, s5
+; GFX10-NEXT:    s_cselect_b32 s5, s6, s5
+; GFX10-NEXT:    s_add_i32 s3, s3, s5
+; GFX10-NEXT:    s_cmp_gt_i32 s4, 0
+; GFX10-NEXT:    s_cselect_b32 s5, s4, 0
+; GFX10-NEXT:    s_sub_i32 s5, s10, s5
+; GFX10-NEXT:    s_cmp_lt_i32 s4, 0
+; GFX10-NEXT:    s_cselect_b32 s6, s4, 0
+; GFX10-NEXT:    s_sub_i32 s6, s12, s6
+; GFX10-NEXT:    s_cmp_gt_i32 s6, s9
+; GFX10-NEXT:    s_cselect_b32 s6, s6, s9
+; GFX10-NEXT:    s_cmp_lt_i32 s6, s5
+; GFX10-NEXT:    s_cselect_b32 s5, s6, s5
+; GFX10-NEXT:    s_add_i32 s4, s4, s5
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <5 x i32> @llvm.sadd.sat.v5i32(<5 x i32> %lhs, <5 x i32> %rhs)
+  ret <5 x i32> %result
+}
+
+define <16 x i32> @v_saddsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
+; GFX6-LABEL: v_saddsat_v16i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s4, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v32, 0, v0
+; GFX6-NEXT:    v_sub_i32_e32 v32, vcc, s4, v32
+; GFX6-NEXT:    v_max_i32_e32 v16, v32, v16
+; GFX6-NEXT:    s_brev_b32 s5, -2
+; GFX6-NEXT:    v_max_i32_e32 v32, 0, v0
+; GFX6-NEXT:    v_sub_i32_e32 v32, vcc, s5, v32
+; GFX6-NEXT:    v_min_i32_e32 v16, v16, v32
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v16
+; GFX6-NEXT:    v_min_i32_e32 v16, 0, v1
+; GFX6-NEXT:    v_sub_i32_e32 v16, vcc, s4, v16
+; GFX6-NEXT:    v_max_i32_e32 v16, v16, v17
+; GFX6-NEXT:    v_max_i32_e32 v17, 0, v1
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, s5, v17
+; GFX6-NEXT:    v_min_i32_e32 v16, v16, v17
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v16
+; GFX6-NEXT:    v_min_i32_e32 v16, 0, v2
+; GFX6-NEXT:    v_sub_i32_e32 v16, vcc, s4, v16
+; GFX6-NEXT:    v_max_i32_e32 v17, 0, v2
+; GFX6-NEXT:    v_max_i32_e32 v16, v16, v18
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, s5, v17
+; GFX6-NEXT:    v_min_i32_e32 v16, v16, v17
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v16
+; GFX6-NEXT:    v_mov_b32_e32 v16, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v17, 0, v3
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v16, v17
+; GFX6-NEXT:    v_max_i32_e32 v17, v17, v19
+; GFX6-NEXT:    v_bfrev_b32_e32 v18, -2
+; GFX6-NEXT:    v_max_i32_e32 v19, 0, v3
+; GFX6-NEXT:    v_sub_i32_e32 v19, vcc, v18, v19
+; GFX6-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v17
+; GFX6-NEXT:    v_min_i32_e32 v17, 0, v4
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v16, v17
+; GFX6-NEXT:    v_max_i32_e32 v19, 0, v4
+; GFX6-NEXT:    v_max_i32_e32 v17, v17, v20
+; GFX6-NEXT:    v_sub_i32_e32 v19, vcc, v18, v19
+; GFX6-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v4, v17
+; GFX6-NEXT:    v_min_i32_e32 v17, 0, v5
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v16, v17
+; GFX6-NEXT:    v_max_i32_e32 v19, 0, v5
+; GFX6-NEXT:    v_max_i32_e32 v17, v17, v21
+; GFX6-NEXT:    v_sub_i32_e32 v19, vcc, v18, v19
+; GFX6-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX6-NEXT:    v_add_i32_e32 v5, vcc, v5, v17
+; GFX6-NEXT:    v_min_i32_e32 v17, 0, v6
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v16, v17
+; GFX6-NEXT:    v_max_i32_e32 v19, 0, v6
+; GFX6-NEXT:    v_max_i32_e32 v17, v17, v22
+; GFX6-NEXT:    v_sub_i32_e32 v19, vcc, v18, v19
+; GFX6-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX6-NEXT:    v_add_i32_e32 v6, vcc, v6, v17
+; GFX6-NEXT:    v_min_i32_e32 v17, 0, v7
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v16, v17
+; GFX6-NEXT:    v_max_i32_e32 v19, 0, v7
+; GFX6-NEXT:    v_max_i32_e32 v17, v17, v23
+; GFX6-NEXT:    v_sub_i32_e32 v19, vcc, v18, v19
+; GFX6-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX6-NEXT:    v_add_i32_e32 v7, vcc, v7, v17
+; GFX6-NEXT:    v_min_i32_e32 v17, 0, v8
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v16, v17
+; GFX6-NEXT:    v_max_i32_e32 v19, 0, v8
+; GFX6-NEXT:    v_max_i32_e32 v17, v17, v24
+; GFX6-NEXT:    v_sub_i32_e32 v19, vcc, v18, v19
+; GFX6-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX6-NEXT:    v_add_i32_e32 v8, vcc, v8, v17
+; GFX6-NEXT:    v_min_i32_e32 v17, 0, v9
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v16, v17
+; GFX6-NEXT:    v_max_i32_e32 v19, 0, v9
+; GFX6-NEXT:    v_max_i32_e32 v17, v17, v25
+; GFX6-NEXT:    v_sub_i32_e32 v19, vcc, v18, v19
+; GFX6-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX6-NEXT:    v_add_i32_e32 v9, vcc, v9, v17
+; GFX6-NEXT:    v_min_i32_e32 v17, 0, v10
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v16, v17
+; GFX6-NEXT:    v_max_i32_e32 v19, 0, v10
+; GFX6-NEXT:    v_max_i32_e32 v17, v17, v26
+; GFX6-NEXT:    v_sub_i32_e32 v19, vcc, v18, v19
+; GFX6-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX6-NEXT:    v_add_i32_e32 v10, vcc, v10, v17
+; GFX6-NEXT:    v_min_i32_e32 v17, 0, v11
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v16, v17
+; GFX6-NEXT:    v_max_i32_e32 v19, 0, v11
+; GFX6-NEXT:    v_max_i32_e32 v17, v17, v27
+; GFX6-NEXT:    v_sub_i32_e32 v19, vcc, v18, v19
+; GFX6-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX6-NEXT:    v_add_i32_e32 v11, vcc, v11, v17
+; GFX6-NEXT:    v_min_i32_e32 v17, 0, v12
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v16, v17
+; GFX6-NEXT:    v_max_i32_e32 v19, 0, v12
+; GFX6-NEXT:    v_max_i32_e32 v17, v17, v28
+; GFX6-NEXT:    v_sub_i32_e32 v19, vcc, v18, v19
+; GFX6-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX6-NEXT:    v_add_i32_e32 v12, vcc, v12, v17
+; GFX6-NEXT:    v_min_i32_e32 v17, 0, v13
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v16, v17
+; GFX6-NEXT:    v_max_i32_e32 v19, 0, v13
+; GFX6-NEXT:    v_max_i32_e32 v17, v17, v29
+; GFX6-NEXT:    v_sub_i32_e32 v19, vcc, v18, v19
+; GFX6-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX6-NEXT:    v_add_i32_e32 v13, vcc, v13, v17
+; GFX6-NEXT:    v_min_i32_e32 v17, 0, v14
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v16, v17
+; GFX6-NEXT:    v_max_i32_e32 v19, 0, v14
+; GFX6-NEXT:    v_max_i32_e32 v17, v17, v30
+; GFX6-NEXT:    v_sub_i32_e32 v19, vcc, v18, v19
+; GFX6-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX6-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; GFX6-NEXT:    v_max_i32_e32 v17, 0, v15
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v18, v17
+; GFX6-NEXT:    v_min_i32_e32 v18, 0, v15
+; GFX6-NEXT:    v_sub_i32_e32 v16, vcc, v16, v18
+; GFX6-NEXT:    v_max_i32_e32 v16, v16, v31
+; GFX6-NEXT:    v_min_i32_e32 v16, v16, v17
+; GFX6-NEXT:    v_add_i32_e32 v15, vcc, v15, v16
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_v16i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s4, 0x80000000
+; GFX8-NEXT:    v_min_i32_e32 v32, 0, v0
+; GFX8-NEXT:    v_sub_u32_e32 v32, vcc, s4, v32
+; GFX8-NEXT:    v_max_i32_e32 v16, v32, v16
+; GFX8-NEXT:    s_brev_b32 s5, -2
+; GFX8-NEXT:    v_max_i32_e32 v32, 0, v0
+; GFX8-NEXT:    v_sub_u32_e32 v32, vcc, s5, v32
+; GFX8-NEXT:    v_min_i32_e32 v16, v16, v32
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v16
+; GFX8-NEXT:    v_min_i32_e32 v16, 0, v1
+; GFX8-NEXT:    v_sub_u32_e32 v16, vcc, s4, v16
+; GFX8-NEXT:    v_max_i32_e32 v16, v16, v17
+; GFX8-NEXT:    v_max_i32_e32 v17, 0, v1
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, s5, v17
+; GFX8-NEXT:    v_min_i32_e32 v16, v16, v17
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v16
+; GFX8-NEXT:    v_min_i32_e32 v16, 0, v2
+; GFX8-NEXT:    v_sub_u32_e32 v16, vcc, s4, v16
+; GFX8-NEXT:    v_max_i32_e32 v17, 0, v2
+; GFX8-NEXT:    v_max_i32_e32 v16, v16, v18
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, s5, v17
+; GFX8-NEXT:    v_min_i32_e32 v16, v16, v17
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v16
+; GFX8-NEXT:    v_mov_b32_e32 v16, 0x80000000
+; GFX8-NEXT:    v_min_i32_e32 v17, 0, v3
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v16, v17
+; GFX8-NEXT:    v_max_i32_e32 v17, v17, v19
+; GFX8-NEXT:    v_bfrev_b32_e32 v18, -2
+; GFX8-NEXT:    v_max_i32_e32 v19, 0, v3
+; GFX8-NEXT:    v_sub_u32_e32 v19, vcc, v18, v19
+; GFX8-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v17
+; GFX8-NEXT:    v_min_i32_e32 v17, 0, v4
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v16, v17
+; GFX8-NEXT:    v_max_i32_e32 v19, 0, v4
+; GFX8-NEXT:    v_max_i32_e32 v17, v17, v20
+; GFX8-NEXT:    v_sub_u32_e32 v19, vcc, v18, v19
+; GFX8-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v4, v17
+; GFX8-NEXT:    v_min_i32_e32 v17, 0, v5
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v16, v17
+; GFX8-NEXT:    v_max_i32_e32 v19, 0, v5
+; GFX8-NEXT:    v_max_i32_e32 v17, v17, v21
+; GFX8-NEXT:    v_sub_u32_e32 v19, vcc, v18, v19
+; GFX8-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v17
+; GFX8-NEXT:    v_min_i32_e32 v17, 0, v6
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v16, v17
+; GFX8-NEXT:    v_max_i32_e32 v19, 0, v6
+; GFX8-NEXT:    v_max_i32_e32 v17, v17, v22
+; GFX8-NEXT:    v_sub_u32_e32 v19, vcc, v18, v19
+; GFX8-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v6, v17
+; GFX8-NEXT:    v_min_i32_e32 v17, 0, v7
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v16, v17
+; GFX8-NEXT:    v_max_i32_e32 v19, 0, v7
+; GFX8-NEXT:    v_max_i32_e32 v17, v17, v23
+; GFX8-NEXT:    v_sub_u32_e32 v19, vcc, v18, v19
+; GFX8-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v17
+; GFX8-NEXT:    v_min_i32_e32 v17, 0, v8
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v16, v17
+; GFX8-NEXT:    v_max_i32_e32 v19, 0, v8
+; GFX8-NEXT:    v_max_i32_e32 v17, v17, v24
+; GFX8-NEXT:    v_sub_u32_e32 v19, vcc, v18, v19
+; GFX8-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX8-NEXT:    v_add_u32_e32 v8, vcc, v8, v17
+; GFX8-NEXT:    v_min_i32_e32 v17, 0, v9
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v16, v17
+; GFX8-NEXT:    v_max_i32_e32 v19, 0, v9
+; GFX8-NEXT:    v_max_i32_e32 v17, v17, v25
+; GFX8-NEXT:    v_sub_u32_e32 v19, vcc, v18, v19
+; GFX8-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX8-NEXT:    v_add_u32_e32 v9, vcc, v9, v17
+; GFX8-NEXT:    v_min_i32_e32 v17, 0, v10
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v16, v17
+; GFX8-NEXT:    v_max_i32_e32 v19, 0, v10
+; GFX8-NEXT:    v_max_i32_e32 v17, v17, v26
+; GFX8-NEXT:    v_sub_u32_e32 v19, vcc, v18, v19
+; GFX8-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX8-NEXT:    v_add_u32_e32 v10, vcc, v10, v17
+; GFX8-NEXT:    v_min_i32_e32 v17, 0, v11
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v16, v17
+; GFX8-NEXT:    v_max_i32_e32 v19, 0, v11
+; GFX8-NEXT:    v_max_i32_e32 v17, v17, v27
+; GFX8-NEXT:    v_sub_u32_e32 v19, vcc, v18, v19
+; GFX8-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX8-NEXT:    v_add_u32_e32 v11, vcc, v11, v17
+; GFX8-NEXT:    v_min_i32_e32 v17, 0, v12
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v16, v17
+; GFX8-NEXT:    v_max_i32_e32 v19, 0, v12
+; GFX8-NEXT:    v_max_i32_e32 v17, v17, v28
+; GFX8-NEXT:    v_sub_u32_e32 v19, vcc, v18, v19
+; GFX8-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX8-NEXT:    v_add_u32_e32 v12, vcc, v12, v17
+; GFX8-NEXT:    v_min_i32_e32 v17, 0, v13
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v16, v17
+; GFX8-NEXT:    v_max_i32_e32 v19, 0, v13
+; GFX8-NEXT:    v_max_i32_e32 v17, v17, v29
+; GFX8-NEXT:    v_sub_u32_e32 v19, vcc, v18, v19
+; GFX8-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX8-NEXT:    v_add_u32_e32 v13, vcc, v13, v17
+; GFX8-NEXT:    v_min_i32_e32 v17, 0, v14
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v16, v17
+; GFX8-NEXT:    v_max_i32_e32 v19, 0, v14
+; GFX8-NEXT:    v_max_i32_e32 v17, v17, v30
+; GFX8-NEXT:    v_sub_u32_e32 v19, vcc, v18, v19
+; GFX8-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX8-NEXT:    v_add_u32_e32 v14, vcc, v14, v17
+; GFX8-NEXT:    v_max_i32_e32 v17, 0, v15
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v18, v17
+; GFX8-NEXT:    v_min_i32_e32 v18, 0, v15
+; GFX8-NEXT:    v_sub_u32_e32 v16, vcc, v16, v18
+; GFX8-NEXT:    v_max_i32_e32 v16, v16, v31
+; GFX8-NEXT:    v_min_i32_e32 v16, v16, v17
+; GFX8-NEXT:    v_add_u32_e32 v15, vcc, v15, v16
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_v16i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s4, 0x80000000
+; GFX9-NEXT:    v_min_i32_e32 v32, 0, v0
+; GFX9-NEXT:    v_sub_u32_e32 v32, s4, v32
+; GFX9-NEXT:    v_max_i32_e32 v16, v32, v16
+; GFX9-NEXT:    s_brev_b32 s5, -2
+; GFX9-NEXT:    v_max_i32_e32 v32, 0, v0
+; GFX9-NEXT:    v_sub_u32_e32 v32, s5, v32
+; GFX9-NEXT:    v_min_i32_e32 v16, v16, v32
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v16
+; GFX9-NEXT:    v_min_i32_e32 v16, 0, v1
+; GFX9-NEXT:    v_sub_u32_e32 v16, s4, v16
+; GFX9-NEXT:    v_max_i32_e32 v16, v16, v17
+; GFX9-NEXT:    v_max_i32_e32 v17, 0, v1
+; GFX9-NEXT:    v_sub_u32_e32 v17, s5, v17
+; GFX9-NEXT:    v_min_i32_e32 v16, v16, v17
+; GFX9-NEXT:    v_add_u32_e32 v1, v1, v16
+; GFX9-NEXT:    v_min_i32_e32 v16, 0, v2
+; GFX9-NEXT:    v_sub_u32_e32 v16, s4, v16
+; GFX9-NEXT:    v_max_i32_e32 v17, 0, v2
+; GFX9-NEXT:    v_max_i32_e32 v16, v16, v18
+; GFX9-NEXT:    v_sub_u32_e32 v17, s5, v17
+; GFX9-NEXT:    v_min_i32_e32 v16, v16, v17
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v16
+; GFX9-NEXT:    v_mov_b32_e32 v16, 0x80000000
+; GFX9-NEXT:    v_min_i32_e32 v17, 0, v3
+; GFX9-NEXT:    v_sub_u32_e32 v17, v16, v17
+; GFX9-NEXT:    v_max_i32_e32 v17, v17, v19
+; GFX9-NEXT:    v_bfrev_b32_e32 v18, -2
+; GFX9-NEXT:    v_max_i32_e32 v19, 0, v3
+; GFX9-NEXT:    v_sub_u32_e32 v19, v18, v19
+; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX9-NEXT:    v_add_u32_e32 v3, v3, v17
+; GFX9-NEXT:    v_min_i32_e32 v17, 0, v4
+; GFX9-NEXT:    v_sub_u32_e32 v17, v16, v17
+; GFX9-NEXT:    v_max_i32_e32 v19, 0, v4
+; GFX9-NEXT:    v_max_i32_e32 v17, v17, v20
+; GFX9-NEXT:    v_sub_u32_e32 v19, v18, v19
+; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX9-NEXT:    v_add_u32_e32 v4, v4, v17
+; GFX9-NEXT:    v_min_i32_e32 v17, 0, v5
+; GFX9-NEXT:    v_sub_u32_e32 v17, v16, v17
+; GFX9-NEXT:    v_max_i32_e32 v19, 0, v5
+; GFX9-NEXT:    v_max_i32_e32 v17, v17, v21
+; GFX9-NEXT:    v_sub_u32_e32 v19, v18, v19
+; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX9-NEXT:    v_add_u32_e32 v5, v5, v17
+; GFX9-NEXT:    v_min_i32_e32 v17, 0, v6
+; GFX9-NEXT:    v_sub_u32_e32 v17, v16, v17
+; GFX9-NEXT:    v_max_i32_e32 v19, 0, v6
+; GFX9-NEXT:    v_max_i32_e32 v17, v17, v22
+; GFX9-NEXT:    v_sub_u32_e32 v19, v18, v19
+; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX9-NEXT:    v_add_u32_e32 v6, v6, v17
+; GFX9-NEXT:    v_min_i32_e32 v17, 0, v7
+; GFX9-NEXT:    v_sub_u32_e32 v17, v16, v17
+; GFX9-NEXT:    v_max_i32_e32 v19, 0, v7
+; GFX9-NEXT:    v_max_i32_e32 v17, v17, v23
+; GFX9-NEXT:    v_sub_u32_e32 v19, v18, v19
+; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX9-NEXT:    v_add_u32_e32 v7, v7, v17
+; GFX9-NEXT:    v_min_i32_e32 v17, 0, v8
+; GFX9-NEXT:    v_sub_u32_e32 v17, v16, v17
+; GFX9-NEXT:    v_max_i32_e32 v19, 0, v8
+; GFX9-NEXT:    v_max_i32_e32 v17, v17, v24
+; GFX9-NEXT:    v_sub_u32_e32 v19, v18, v19
+; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX9-NEXT:    v_add_u32_e32 v8, v8, v17
+; GFX9-NEXT:    v_min_i32_e32 v17, 0, v9
+; GFX9-NEXT:    v_sub_u32_e32 v17, v16, v17
+; GFX9-NEXT:    v_max_i32_e32 v19, 0, v9
+; GFX9-NEXT:    v_max_i32_e32 v17, v17, v25
+; GFX9-NEXT:    v_sub_u32_e32 v19, v18, v19
+; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX9-NEXT:    v_add_u32_e32 v9, v9, v17
+; GFX9-NEXT:    v_min_i32_e32 v17, 0, v10
+; GFX9-NEXT:    v_sub_u32_e32 v17, v16, v17
+; GFX9-NEXT:    v_max_i32_e32 v19, 0, v10
+; GFX9-NEXT:    v_max_i32_e32 v17, v17, v26
+; GFX9-NEXT:    v_sub_u32_e32 v19, v18, v19
+; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX9-NEXT:    v_add_u32_e32 v10, v10, v17
+; GFX9-NEXT:    v_min_i32_e32 v17, 0, v11
+; GFX9-NEXT:    v_sub_u32_e32 v17, v16, v17
+; GFX9-NEXT:    v_max_i32_e32 v19, 0, v11
+; GFX9-NEXT:    v_max_i32_e32 v17, v17, v27
+; GFX9-NEXT:    v_sub_u32_e32 v19, v18, v19
+; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX9-NEXT:    v_add_u32_e32 v11, v11, v17
+; GFX9-NEXT:    v_min_i32_e32 v17, 0, v12
+; GFX9-NEXT:    v_sub_u32_e32 v17, v16, v17
+; GFX9-NEXT:    v_max_i32_e32 v19, 0, v12
+; GFX9-NEXT:    v_max_i32_e32 v17, v17, v28
+; GFX9-NEXT:    v_sub_u32_e32 v19, v18, v19
+; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX9-NEXT:    v_add_u32_e32 v12, v12, v17
+; GFX9-NEXT:    v_min_i32_e32 v17, 0, v13
+; GFX9-NEXT:    v_sub_u32_e32 v17, v16, v17
+; GFX9-NEXT:    v_max_i32_e32 v19, 0, v13
+; GFX9-NEXT:    v_max_i32_e32 v17, v17, v29
+; GFX9-NEXT:    v_sub_u32_e32 v19, v18, v19
+; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX9-NEXT:    v_add_u32_e32 v13, v13, v17
+; GFX9-NEXT:    v_min_i32_e32 v17, 0, v14
+; GFX9-NEXT:    v_sub_u32_e32 v17, v16, v17
+; GFX9-NEXT:    v_max_i32_e32 v19, 0, v14
+; GFX9-NEXT:    v_max_i32_e32 v17, v17, v30
+; GFX9-NEXT:    v_sub_u32_e32 v19, v18, v19
+; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX9-NEXT:    v_add_u32_e32 v14, v14, v17
+; GFX9-NEXT:    v_max_i32_e32 v17, 0, v15
+; GFX9-NEXT:    v_sub_u32_e32 v17, v18, v17
+; GFX9-NEXT:    v_min_i32_e32 v18, 0, v15
+; GFX9-NEXT:    v_sub_u32_e32 v16, v16, v18
+; GFX9-NEXT:    v_max_i32_e32 v16, v16, v31
+; GFX9-NEXT:    v_min_i32_e32 v16, v16, v17
+; GFX9-NEXT:    v_add_u32_e32 v15, v15, v16
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_saddsat_v16i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_min_i32_e32 v32, 0, v0
+; GFX10-NEXT:    s_mov_b32 s4, 0x80000000
+; GFX10-NEXT:    v_max_i32_e32 v33, 0, v0
+; GFX10-NEXT:    s_brev_b32 s5, -2
+; GFX10-NEXT:    v_min_i32_e32 v36, 0, v2
+; GFX10-NEXT:    v_sub_nc_u32_e32 v35, s4, v32
+; GFX10-NEXT:    v_min_i32_e32 v32, 0, v1
+; GFX10-NEXT:    v_sub_nc_u32_e32 v33, s5, v33
+; GFX10-NEXT:    v_max_i32_e32 v37, 0, v1
+; GFX10-NEXT:    v_sub_nc_u32_e32 v36, s4, v36
+; GFX10-NEXT:    v_max_i32_e32 v16, v35, v16
+; GFX10-NEXT:    v_sub_nc_u32_e32 v32, s4, v32
+; GFX10-NEXT:    v_mov_b32_e32 v35, 0x80000000
+; GFX10-NEXT:    v_min_i32_e32 v38, 0, v3
+; GFX10-NEXT:    v_max_i32_e32 v18, v36, v18
+; GFX10-NEXT:    v_min_i32_e32 v16, v16, v33
+; GFX10-NEXT:    v_max_i32_e32 v33, 0, v2
+; GFX10-NEXT:    v_max_i32_e32 v39, v32, v17
+; GFX10-NEXT:    v_sub_nc_u32_e32 v36, v35, v38
+; GFX10-NEXT:    v_sub_nc_u32_e32 v37, s5, v37
+; GFX10-NEXT:    v_bfrev_b32_e32 v34, -2
+; GFX10-NEXT:    v_sub_nc_u32_e32 v32, s5, v33
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v16
+; GFX10-NEXT:    v_max_i32_e32 v33, 0, v3
+; GFX10-NEXT:    v_min_i32_e32 v39, v39, v37
+; GFX10-NEXT:    v_max_i32_e32 v19, v36, v19
+; GFX10-NEXT:    v_min_i32_e32 v16, v18, v32
+; GFX10-NEXT:    v_min_i32_e32 v32, 0, v6
+; GFX10-NEXT:    v_sub_nc_u32_e32 v18, v34, v33
+; GFX10-NEXT:    v_min_i32_e32 v38, 0, v5
+; GFX10-NEXT:    v_max_i32_e32 v17, 0, v4
+; GFX10-NEXT:    v_add_nc_u32_e32 v2, v2, v16
+; GFX10-NEXT:    v_min_i32_e32 v16, 0, v4
+; GFX10-NEXT:    v_min_i32_e32 v18, v19, v18
+; GFX10-NEXT:    v_sub_nc_u32_e32 v19, v35, v38
+; GFX10-NEXT:    v_add_nc_u32_e32 v1, v1, v39
+; GFX10-NEXT:    v_sub_nc_u32_e32 v32, v35, v32
+; GFX10-NEXT:    v_sub_nc_u32_e32 v39, v35, v16
+; GFX10-NEXT:    v_max_i32_e32 v33, 0, v5
+; GFX10-NEXT:    v_max_i32_e32 v36, 0, v6
+; GFX10-NEXT:    v_max_i32_e32 v19, v19, v21
+; GFX10-NEXT:    v_add_nc_u32_e32 v3, v3, v18
+; GFX10-NEXT:    v_max_i32_e32 v16, v39, v20
+; GFX10-NEXT:    v_sub_nc_u32_e32 v17, v34, v17
+; GFX10-NEXT:    v_sub_nc_u32_e32 v20, v34, v33
+; GFX10-NEXT:    v_sub_nc_u32_e32 v21, v34, v36
+; GFX10-NEXT:    v_max_i32_e32 v22, v32, v22
+; GFX10-NEXT:    v_min_i32_e32 v18, 0, v7
+; GFX10-NEXT:    v_min_i32_e32 v39, v16, v17
+; GFX10-NEXT:    v_min_i32_e32 v38, v19, v20
+; GFX10-NEXT:    v_max_i32_e32 v16, 0, v7
+; GFX10-NEXT:    v_min_i32_e32 v19, v22, v21
+; GFX10-NEXT:    v_sub_nc_u32_e32 v17, v35, v18
+; GFX10-NEXT:    v_min_i32_e32 v18, 0, v8
+; GFX10-NEXT:    v_min_i32_e32 v20, 0, v9
+; GFX10-NEXT:    v_sub_nc_u32_e32 v16, v34, v16
+; GFX10-NEXT:    v_add_nc_u32_e32 v6, v6, v19
+; GFX10-NEXT:    v_max_i32_e32 v19, 0, v8
+; GFX10-NEXT:    v_max_i32_e32 v17, v17, v23
+; GFX10-NEXT:    v_sub_nc_u32_e32 v18, v35, v18
+; GFX10-NEXT:    v_min_i32_e32 v22, 0, v10
+; GFX10-NEXT:    v_max_i32_e32 v21, 0, v9
+; GFX10-NEXT:    v_sub_nc_u32_e32 v20, v35, v20
+; GFX10-NEXT:    v_add_nc_u32_e32 v4, v4, v39
+; GFX10-NEXT:    v_max_i32_e32 v18, v18, v24
+; GFX10-NEXT:    v_sub_nc_u32_e32 v39, v35, v22
+; GFX10-NEXT:    v_min_i32_e32 v16, v17, v16
+; GFX10-NEXT:    v_sub_nc_u32_e32 v19, v34, v19
+; GFX10-NEXT:    v_max_i32_e32 v23, 0, v10
+; GFX10-NEXT:    v_max_i32_e32 v20, v20, v25
+; GFX10-NEXT:    v_sub_nc_u32_e32 v21, v34, v21
+; GFX10-NEXT:    v_add_nc_u32_e32 v7, v7, v16
+; GFX10-NEXT:    v_min_i32_e32 v17, v18, v19
+; GFX10-NEXT:    v_min_i32_e32 v16, 0, v11
+; GFX10-NEXT:    v_sub_nc_u32_e32 v18, v34, v23
+; GFX10-NEXT:    v_max_i32_e32 v19, v39, v26
+; GFX10-NEXT:    v_min_i32_e32 v22, 0, v12
+; GFX10-NEXT:    v_min_i32_e32 v20, v20, v21
+; GFX10-NEXT:    v_sub_nc_u32_e32 v16, v35, v16
+; GFX10-NEXT:    v_min_i32_e32 v26, 0, v15
+; GFX10-NEXT:    v_add_nc_u32_e32 v8, v8, v17
+; GFX10-NEXT:    v_min_i32_e32 v17, v19, v18
+; GFX10-NEXT:    v_sub_nc_u32_e32 v19, v35, v22
+; GFX10-NEXT:    v_min_i32_e32 v22, 0, v14
+; GFX10-NEXT:    v_min_i32_e32 v21, 0, v13
+; GFX10-NEXT:    v_max_i32_e32 v24, 0, v14
+; GFX10-NEXT:    v_max_i32_e32 v25, 0, v15
+; GFX10-NEXT:    v_add_nc_u32_e32 v9, v9, v20
+; GFX10-NEXT:    v_max_i32_e32 v20, 0, v13
+; GFX10-NEXT:    v_sub_nc_u32_e32 v39, v35, v22
+; GFX10-NEXT:    v_max_i32_e32 v23, 0, v11
+; GFX10-NEXT:    v_add_nc_u32_e32 v10, v10, v17
+; GFX10-NEXT:    v_max_i32_e32 v17, 0, v12
+; GFX10-NEXT:    v_max_i32_e32 v16, v16, v27
+; GFX10-NEXT:    v_sub_nc_u32_e32 v27, v35, v21
+; GFX10-NEXT:    v_sub_nc_u32_e32 v26, v35, v26
+; GFX10-NEXT:    v_sub_nc_u32_e32 v18, v34, v23
+; GFX10-NEXT:    v_sub_nc_u32_e32 v17, v34, v17
+; GFX10-NEXT:    v_max_i32_e32 v19, v19, v28
+; GFX10-NEXT:    v_sub_nc_u32_e32 v20, v34, v20
+; GFX10-NEXT:    v_max_i32_e32 v21, v27, v29
+; GFX10-NEXT:    v_sub_nc_u32_e32 v24, v34, v24
+; GFX10-NEXT:    v_max_i32_e32 v22, v39, v30
+; GFX10-NEXT:    v_sub_nc_u32_e32 v25, v34, v25
+; GFX10-NEXT:    v_max_i32_e32 v23, v26, v31
+; GFX10-NEXT:    v_min_i32_e32 v16, v16, v18
+; GFX10-NEXT:    v_min_i32_e32 v17, v19, v17
+; GFX10-NEXT:    v_min_i32_e32 v18, v21, v20
+; GFX10-NEXT:    v_min_i32_e32 v19, v22, v24
+; GFX10-NEXT:    v_min_i32_e32 v20, v23, v25
+; GFX10-NEXT:    v_add_nc_u32_e32 v5, v5, v38
+; GFX10-NEXT:    v_add_nc_u32_e32 v11, v11, v16
+; GFX10-NEXT:    v_add_nc_u32_e32 v12, v12, v17
+; GFX10-NEXT:    v_add_nc_u32_e32 v13, v13, v18
+; GFX10-NEXT:    v_add_nc_u32_e32 v14, v14, v19
+; GFX10-NEXT:    v_add_nc_u32_e32 v15, v15, v20
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
+  ret <16 x i32> %result
+}
+
+define amdgpu_ps <16 x i32> @s_saddsat_v16i32(<16 x i32> inreg %lhs, <16 x i32> inreg %rhs) {
+; GFX6-LABEL: s_saddsat_v16i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX6-NEXT:    s_brev_b32 s32, -2
+; GFX6-NEXT:    s_cselect_b32 s34, s0, 0
+; GFX6-NEXT:    s_sub_i32 s34, s32, s34
+; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX6-NEXT:    s_mov_b32 s33, 0x80000000
+; GFX6-NEXT:    s_cselect_b32 s35, s0, 0
+; GFX6-NEXT:    s_sub_i32 s35, s33, s35
+; GFX6-NEXT:    s_cmp_gt_i32 s35, s16
+; GFX6-NEXT:    s_cselect_b32 s16, s35, s16
+; GFX6-NEXT:    s_cmp_lt_i32 s16, s34
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s34
+; GFX6-NEXT:    s_add_i32 s0, s0, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s1, 0
+; GFX6-NEXT:    s_cselect_b32 s16, s1, 0
+; GFX6-NEXT:    s_sub_i32 s16, s32, s16
+; GFX6-NEXT:    s_cmp_lt_i32 s1, 0
+; GFX6-NEXT:    s_cselect_b32 s34, s1, 0
+; GFX6-NEXT:    s_sub_i32 s34, s33, s34
+; GFX6-NEXT:    s_cmp_gt_i32 s34, s17
+; GFX6-NEXT:    s_cselect_b32 s17, s34, s17
+; GFX6-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX6-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX6-NEXT:    s_add_i32 s1, s1, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s2, 0
+; GFX6-NEXT:    s_cselect_b32 s16, s2, 0
+; GFX6-NEXT:    s_sub_i32 s16, s32, s16
+; GFX6-NEXT:    s_cmp_lt_i32 s2, 0
+; GFX6-NEXT:    s_cselect_b32 s17, s2, 0
+; GFX6-NEXT:    s_sub_i32 s17, s33, s17
+; GFX6-NEXT:    s_cmp_gt_i32 s17, s18
+; GFX6-NEXT:    s_cselect_b32 s17, s17, s18
+; GFX6-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX6-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX6-NEXT:    s_add_i32 s2, s2, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s3, 0
+; GFX6-NEXT:    s_cselect_b32 s16, s3, 0
+; GFX6-NEXT:    s_sub_i32 s16, s32, s16
+; GFX6-NEXT:    s_cmp_lt_i32 s3, 0
+; GFX6-NEXT:    s_cselect_b32 s17, s3, 0
+; GFX6-NEXT:    s_sub_i32 s17, s33, s17
+; GFX6-NEXT:    s_cmp_gt_i32 s17, s19
+; GFX6-NEXT:    s_cselect_b32 s17, s17, s19
+; GFX6-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX6-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX6-NEXT:    s_add_i32 s3, s3, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s4, 0
+; GFX6-NEXT:    s_cselect_b32 s16, s4, 0
+; GFX6-NEXT:    s_sub_i32 s16, s32, s16
+; GFX6-NEXT:    s_cmp_lt_i32 s4, 0
+; GFX6-NEXT:    s_cselect_b32 s17, s4, 0
+; GFX6-NEXT:    s_sub_i32 s17, s33, s17
+; GFX6-NEXT:    s_cmp_gt_i32 s17, s20
+; GFX6-NEXT:    s_cselect_b32 s17, s17, s20
+; GFX6-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX6-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX6-NEXT:    s_add_i32 s4, s4, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s5, 0
+; GFX6-NEXT:    s_cselect_b32 s16, s5, 0
+; GFX6-NEXT:    s_sub_i32 s16, s32, s16
+; GFX6-NEXT:    s_cmp_lt_i32 s5, 0
+; GFX6-NEXT:    s_cselect_b32 s17, s5, 0
+; GFX6-NEXT:    s_sub_i32 s17, s33, s17
+; GFX6-NEXT:    s_cmp_gt_i32 s17, s21
+; GFX6-NEXT:    s_cselect_b32 s17, s17, s21
+; GFX6-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX6-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX6-NEXT:    s_add_i32 s5, s5, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s6, 0
+; GFX6-NEXT:    s_cselect_b32 s16, s6, 0
+; GFX6-NEXT:    s_sub_i32 s16, s32, s16
+; GFX6-NEXT:    s_cmp_lt_i32 s6, 0
+; GFX6-NEXT:    s_cselect_b32 s17, s6, 0
+; GFX6-NEXT:    s_sub_i32 s17, s33, s17
+; GFX6-NEXT:    s_cmp_gt_i32 s17, s22
+; GFX6-NEXT:    s_cselect_b32 s17, s17, s22
+; GFX6-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX6-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX6-NEXT:    s_add_i32 s6, s6, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s7, 0
+; GFX6-NEXT:    s_cselect_b32 s16, s7, 0
+; GFX6-NEXT:    s_sub_i32 s16, s32, s16
+; GFX6-NEXT:    s_cmp_lt_i32 s7, 0
+; GFX6-NEXT:    s_cselect_b32 s17, s7, 0
+; GFX6-NEXT:    s_sub_i32 s17, s33, s17
+; GFX6-NEXT:    s_cmp_gt_i32 s17, s23
+; GFX6-NEXT:    s_cselect_b32 s17, s17, s23
+; GFX6-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX6-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX6-NEXT:    s_add_i32 s7, s7, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s8, 0
+; GFX6-NEXT:    s_cselect_b32 s16, s8, 0
+; GFX6-NEXT:    s_sub_i32 s16, s32, s16
+; GFX6-NEXT:    s_cmp_lt_i32 s8, 0
+; GFX6-NEXT:    s_cselect_b32 s17, s8, 0
+; GFX6-NEXT:    s_sub_i32 s17, s33, s17
+; GFX6-NEXT:    s_cmp_gt_i32 s17, s24
+; GFX6-NEXT:    s_cselect_b32 s17, s17, s24
+; GFX6-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX6-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX6-NEXT:    s_add_i32 s8, s8, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s9, 0
+; GFX6-NEXT:    s_cselect_b32 s16, s9, 0
+; GFX6-NEXT:    s_sub_i32 s16, s32, s16
+; GFX6-NEXT:    s_cmp_lt_i32 s9, 0
+; GFX6-NEXT:    s_cselect_b32 s17, s9, 0
+; GFX6-NEXT:    s_sub_i32 s17, s33, s17
+; GFX6-NEXT:    s_cmp_gt_i32 s17, s25
+; GFX6-NEXT:    s_cselect_b32 s17, s17, s25
+; GFX6-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX6-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX6-NEXT:    s_add_i32 s9, s9, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s10, 0
+; GFX6-NEXT:    s_cselect_b32 s16, s10, 0
+; GFX6-NEXT:    s_sub_i32 s16, s32, s16
+; GFX6-NEXT:    s_cmp_lt_i32 s10, 0
+; GFX6-NEXT:    s_cselect_b32 s17, s10, 0
+; GFX6-NEXT:    s_sub_i32 s17, s33, s17
+; GFX6-NEXT:    s_cmp_gt_i32 s17, s26
+; GFX6-NEXT:    s_cselect_b32 s17, s17, s26
+; GFX6-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX6-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX6-NEXT:    s_add_i32 s10, s10, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s11, 0
+; GFX6-NEXT:    s_cselect_b32 s16, s11, 0
+; GFX6-NEXT:    s_sub_i32 s16, s32, s16
+; GFX6-NEXT:    s_cmp_lt_i32 s11, 0
+; GFX6-NEXT:    s_cselect_b32 s17, s11, 0
+; GFX6-NEXT:    s_sub_i32 s17, s33, s17
+; GFX6-NEXT:    s_cmp_gt_i32 s17, s27
+; GFX6-NEXT:    s_cselect_b32 s17, s17, s27
+; GFX6-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX6-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX6-NEXT:    s_add_i32 s11, s11, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s12, 0
+; GFX6-NEXT:    s_cselect_b32 s16, s12, 0
+; GFX6-NEXT:    s_sub_i32 s16, s32, s16
+; GFX6-NEXT:    s_cmp_lt_i32 s12, 0
+; GFX6-NEXT:    s_cselect_b32 s17, s12, 0
+; GFX6-NEXT:    s_sub_i32 s17, s33, s17
+; GFX6-NEXT:    s_cmp_gt_i32 s17, s28
+; GFX6-NEXT:    s_cselect_b32 s17, s17, s28
+; GFX6-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX6-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX6-NEXT:    s_add_i32 s12, s12, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s13, 0
+; GFX6-NEXT:    s_cselect_b32 s16, s13, 0
+; GFX6-NEXT:    s_sub_i32 s16, s32, s16
+; GFX6-NEXT:    s_cmp_lt_i32 s13, 0
+; GFX6-NEXT:    s_cselect_b32 s17, s13, 0
+; GFX6-NEXT:    s_sub_i32 s17, s33, s17
+; GFX6-NEXT:    s_cmp_gt_i32 s17, s29
+; GFX6-NEXT:    s_cselect_b32 s17, s17, s29
+; GFX6-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX6-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX6-NEXT:    s_add_i32 s13, s13, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s14, 0
+; GFX6-NEXT:    s_cselect_b32 s16, s14, 0
+; GFX6-NEXT:    s_sub_i32 s16, s32, s16
+; GFX6-NEXT:    s_cmp_lt_i32 s14, 0
+; GFX6-NEXT:    s_cselect_b32 s17, s14, 0
+; GFX6-NEXT:    s_sub_i32 s17, s33, s17
+; GFX6-NEXT:    s_cmp_gt_i32 s17, s30
+; GFX6-NEXT:    s_cselect_b32 s17, s17, s30
+; GFX6-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX6-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX6-NEXT:    s_add_i32 s14, s14, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s15, 0
+; GFX6-NEXT:    s_cselect_b32 s16, s15, 0
+; GFX6-NEXT:    s_sub_i32 s16, s32, s16
+; GFX6-NEXT:    s_cmp_lt_i32 s15, 0
+; GFX6-NEXT:    s_cselect_b32 s17, s15, 0
+; GFX6-NEXT:    s_sub_i32 s17, s33, s17
+; GFX6-NEXT:    s_cmp_gt_i32 s17, s31
+; GFX6-NEXT:    s_cselect_b32 s17, s17, s31
+; GFX6-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX6-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX6-NEXT:    s_add_i32 s15, s15, s16
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_saddsat_v16i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX8-NEXT:    s_brev_b32 s32, -2
+; GFX8-NEXT:    s_cselect_b32 s34, s0, 0
+; GFX8-NEXT:    s_sub_i32 s34, s32, s34
+; GFX8-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX8-NEXT:    s_mov_b32 s33, 0x80000000
+; GFX8-NEXT:    s_cselect_b32 s35, s0, 0
+; GFX8-NEXT:    s_sub_i32 s35, s33, s35
+; GFX8-NEXT:    s_cmp_gt_i32 s35, s16
+; GFX8-NEXT:    s_cselect_b32 s16, s35, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s16, s34
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s34
+; GFX8-NEXT:    s_add_i32 s0, s0, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s1, 0
+; GFX8-NEXT:    s_cselect_b32 s16, s1, 0
+; GFX8-NEXT:    s_sub_i32 s16, s32, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s1, 0
+; GFX8-NEXT:    s_cselect_b32 s34, s1, 0
+; GFX8-NEXT:    s_sub_i32 s34, s33, s34
+; GFX8-NEXT:    s_cmp_gt_i32 s34, s17
+; GFX8-NEXT:    s_cselect_b32 s17, s34, s17
+; GFX8-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX8-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX8-NEXT:    s_add_i32 s1, s1, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s2, 0
+; GFX8-NEXT:    s_cselect_b32 s16, s2, 0
+; GFX8-NEXT:    s_sub_i32 s16, s32, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s2, 0
+; GFX8-NEXT:    s_cselect_b32 s17, s2, 0
+; GFX8-NEXT:    s_sub_i32 s17, s33, s17
+; GFX8-NEXT:    s_cmp_gt_i32 s17, s18
+; GFX8-NEXT:    s_cselect_b32 s17, s17, s18
+; GFX8-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX8-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX8-NEXT:    s_add_i32 s2, s2, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s3, 0
+; GFX8-NEXT:    s_cselect_b32 s16, s3, 0
+; GFX8-NEXT:    s_sub_i32 s16, s32, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s3, 0
+; GFX8-NEXT:    s_cselect_b32 s17, s3, 0
+; GFX8-NEXT:    s_sub_i32 s17, s33, s17
+; GFX8-NEXT:    s_cmp_gt_i32 s17, s19
+; GFX8-NEXT:    s_cselect_b32 s17, s17, s19
+; GFX8-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX8-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX8-NEXT:    s_add_i32 s3, s3, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s4, 0
+; GFX8-NEXT:    s_cselect_b32 s16, s4, 0
+; GFX8-NEXT:    s_sub_i32 s16, s32, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s4, 0
+; GFX8-NEXT:    s_cselect_b32 s17, s4, 0
+; GFX8-NEXT:    s_sub_i32 s17, s33, s17
+; GFX8-NEXT:    s_cmp_gt_i32 s17, s20
+; GFX8-NEXT:    s_cselect_b32 s17, s17, s20
+; GFX8-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX8-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX8-NEXT:    s_add_i32 s4, s4, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s5, 0
+; GFX8-NEXT:    s_cselect_b32 s16, s5, 0
+; GFX8-NEXT:    s_sub_i32 s16, s32, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s5, 0
+; GFX8-NEXT:    s_cselect_b32 s17, s5, 0
+; GFX8-NEXT:    s_sub_i32 s17, s33, s17
+; GFX8-NEXT:    s_cmp_gt_i32 s17, s21
+; GFX8-NEXT:    s_cselect_b32 s17, s17, s21
+; GFX8-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX8-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX8-NEXT:    s_add_i32 s5, s5, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s6, 0
+; GFX8-NEXT:    s_cselect_b32 s16, s6, 0
+; GFX8-NEXT:    s_sub_i32 s16, s32, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s6, 0
+; GFX8-NEXT:    s_cselect_b32 s17, s6, 0
+; GFX8-NEXT:    s_sub_i32 s17, s33, s17
+; GFX8-NEXT:    s_cmp_gt_i32 s17, s22
+; GFX8-NEXT:    s_cselect_b32 s17, s17, s22
+; GFX8-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX8-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX8-NEXT:    s_add_i32 s6, s6, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s7, 0
+; GFX8-NEXT:    s_cselect_b32 s16, s7, 0
+; GFX8-NEXT:    s_sub_i32 s16, s32, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s7, 0
+; GFX8-NEXT:    s_cselect_b32 s17, s7, 0
+; GFX8-NEXT:    s_sub_i32 s17, s33, s17
+; GFX8-NEXT:    s_cmp_gt_i32 s17, s23
+; GFX8-NEXT:    s_cselect_b32 s17, s17, s23
+; GFX8-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX8-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX8-NEXT:    s_add_i32 s7, s7, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s8, 0
+; GFX8-NEXT:    s_cselect_b32 s16, s8, 0
+; GFX8-NEXT:    s_sub_i32 s16, s32, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s8, 0
+; GFX8-NEXT:    s_cselect_b32 s17, s8, 0
+; GFX8-NEXT:    s_sub_i32 s17, s33, s17
+; GFX8-NEXT:    s_cmp_gt_i32 s17, s24
+; GFX8-NEXT:    s_cselect_b32 s17, s17, s24
+; GFX8-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX8-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX8-NEXT:    s_add_i32 s8, s8, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s9, 0
+; GFX8-NEXT:    s_cselect_b32 s16, s9, 0
+; GFX8-NEXT:    s_sub_i32 s16, s32, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s9, 0
+; GFX8-NEXT:    s_cselect_b32 s17, s9, 0
+; GFX8-NEXT:    s_sub_i32 s17, s33, s17
+; GFX8-NEXT:    s_cmp_gt_i32 s17, s25
+; GFX8-NEXT:    s_cselect_b32 s17, s17, s25
+; GFX8-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX8-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX8-NEXT:    s_add_i32 s9, s9, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s10, 0
+; GFX8-NEXT:    s_cselect_b32 s16, s10, 0
+; GFX8-NEXT:    s_sub_i32 s16, s32, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s10, 0
+; GFX8-NEXT:    s_cselect_b32 s17, s10, 0
+; GFX8-NEXT:    s_sub_i32 s17, s33, s17
+; GFX8-NEXT:    s_cmp_gt_i32 s17, s26
+; GFX8-NEXT:    s_cselect_b32 s17, s17, s26
+; GFX8-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX8-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX8-NEXT:    s_add_i32 s10, s10, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s11, 0
+; GFX8-NEXT:    s_cselect_b32 s16, s11, 0
+; GFX8-NEXT:    s_sub_i32 s16, s32, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s11, 0
+; GFX8-NEXT:    s_cselect_b32 s17, s11, 0
+; GFX8-NEXT:    s_sub_i32 s17, s33, s17
+; GFX8-NEXT:    s_cmp_gt_i32 s17, s27
+; GFX8-NEXT:    s_cselect_b32 s17, s17, s27
+; GFX8-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX8-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX8-NEXT:    s_add_i32 s11, s11, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s12, 0
+; GFX8-NEXT:    s_cselect_b32 s16, s12, 0
+; GFX8-NEXT:    s_sub_i32 s16, s32, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s12, 0
+; GFX8-NEXT:    s_cselect_b32 s17, s12, 0
+; GFX8-NEXT:    s_sub_i32 s17, s33, s17
+; GFX8-NEXT:    s_cmp_gt_i32 s17, s28
+; GFX8-NEXT:    s_cselect_b32 s17, s17, s28
+; GFX8-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX8-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX8-NEXT:    s_add_i32 s12, s12, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s13, 0
+; GFX8-NEXT:    s_cselect_b32 s16, s13, 0
+; GFX8-NEXT:    s_sub_i32 s16, s32, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s13, 0
+; GFX8-NEXT:    s_cselect_b32 s17, s13, 0
+; GFX8-NEXT:    s_sub_i32 s17, s33, s17
+; GFX8-NEXT:    s_cmp_gt_i32 s17, s29
+; GFX8-NEXT:    s_cselect_b32 s17, s17, s29
+; GFX8-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX8-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX8-NEXT:    s_add_i32 s13, s13, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s14, 0
+; GFX8-NEXT:    s_cselect_b32 s16, s14, 0
+; GFX8-NEXT:    s_sub_i32 s16, s32, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s14, 0
+; GFX8-NEXT:    s_cselect_b32 s17, s14, 0
+; GFX8-NEXT:    s_sub_i32 s17, s33, s17
+; GFX8-NEXT:    s_cmp_gt_i32 s17, s30
+; GFX8-NEXT:    s_cselect_b32 s17, s17, s30
+; GFX8-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX8-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX8-NEXT:    s_add_i32 s14, s14, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s15, 0
+; GFX8-NEXT:    s_cselect_b32 s16, s15, 0
+; GFX8-NEXT:    s_sub_i32 s16, s32, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s15, 0
+; GFX8-NEXT:    s_cselect_b32 s17, s15, 0
+; GFX8-NEXT:    s_sub_i32 s17, s33, s17
+; GFX8-NEXT:    s_cmp_gt_i32 s17, s31
+; GFX8-NEXT:    s_cselect_b32 s17, s17, s31
+; GFX8-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX8-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX8-NEXT:    s_add_i32 s15, s15, s16
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_saddsat_v16i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX9-NEXT:    s_brev_b32 s32, -2
+; GFX9-NEXT:    s_cselect_b32 s34, s0, 0
+; GFX9-NEXT:    s_sub_i32 s34, s32, s34
+; GFX9-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX9-NEXT:    s_mov_b32 s33, 0x80000000
+; GFX9-NEXT:    s_cselect_b32 s35, s0, 0
+; GFX9-NEXT:    s_sub_i32 s35, s33, s35
+; GFX9-NEXT:    s_cmp_gt_i32 s35, s16
+; GFX9-NEXT:    s_cselect_b32 s16, s35, s16
+; GFX9-NEXT:    s_cmp_lt_i32 s16, s34
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s34
+; GFX9-NEXT:    s_add_i32 s0, s0, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s1, 0
+; GFX9-NEXT:    s_cselect_b32 s16, s1, 0
+; GFX9-NEXT:    s_sub_i32 s16, s32, s16
+; GFX9-NEXT:    s_cmp_lt_i32 s1, 0
+; GFX9-NEXT:    s_cselect_b32 s34, s1, 0
+; GFX9-NEXT:    s_sub_i32 s34, s33, s34
+; GFX9-NEXT:    s_cmp_gt_i32 s34, s17
+; GFX9-NEXT:    s_cselect_b32 s17, s34, s17
+; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX9-NEXT:    s_add_i32 s1, s1, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s2, 0
+; GFX9-NEXT:    s_cselect_b32 s16, s2, 0
+; GFX9-NEXT:    s_sub_i32 s16, s32, s16
+; GFX9-NEXT:    s_cmp_lt_i32 s2, 0
+; GFX9-NEXT:    s_cselect_b32 s17, s2, 0
+; GFX9-NEXT:    s_sub_i32 s17, s33, s17
+; GFX9-NEXT:    s_cmp_gt_i32 s17, s18
+; GFX9-NEXT:    s_cselect_b32 s17, s17, s18
+; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX9-NEXT:    s_add_i32 s2, s2, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s3, 0
+; GFX9-NEXT:    s_cselect_b32 s16, s3, 0
+; GFX9-NEXT:    s_sub_i32 s16, s32, s16
+; GFX9-NEXT:    s_cmp_lt_i32 s3, 0
+; GFX9-NEXT:    s_cselect_b32 s17, s3, 0
+; GFX9-NEXT:    s_sub_i32 s17, s33, s17
+; GFX9-NEXT:    s_cmp_gt_i32 s17, s19
+; GFX9-NEXT:    s_cselect_b32 s17, s17, s19
+; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX9-NEXT:    s_add_i32 s3, s3, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s4, 0
+; GFX9-NEXT:    s_cselect_b32 s16, s4, 0
+; GFX9-NEXT:    s_sub_i32 s16, s32, s16
+; GFX9-NEXT:    s_cmp_lt_i32 s4, 0
+; GFX9-NEXT:    s_cselect_b32 s17, s4, 0
+; GFX9-NEXT:    s_sub_i32 s17, s33, s17
+; GFX9-NEXT:    s_cmp_gt_i32 s17, s20
+; GFX9-NEXT:    s_cselect_b32 s17, s17, s20
+; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX9-NEXT:    s_add_i32 s4, s4, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s5, 0
+; GFX9-NEXT:    s_cselect_b32 s16, s5, 0
+; GFX9-NEXT:    s_sub_i32 s16, s32, s16
+; GFX9-NEXT:    s_cmp_lt_i32 s5, 0
+; GFX9-NEXT:    s_cselect_b32 s17, s5, 0
+; GFX9-NEXT:    s_sub_i32 s17, s33, s17
+; GFX9-NEXT:    s_cmp_gt_i32 s17, s21
+; GFX9-NEXT:    s_cselect_b32 s17, s17, s21
+; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX9-NEXT:    s_add_i32 s5, s5, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s6, 0
+; GFX9-NEXT:    s_cselect_b32 s16, s6, 0
+; GFX9-NEXT:    s_sub_i32 s16, s32, s16
+; GFX9-NEXT:    s_cmp_lt_i32 s6, 0
+; GFX9-NEXT:    s_cselect_b32 s17, s6, 0
+; GFX9-NEXT:    s_sub_i32 s17, s33, s17
+; GFX9-NEXT:    s_cmp_gt_i32 s17, s22
+; GFX9-NEXT:    s_cselect_b32 s17, s17, s22
+; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX9-NEXT:    s_add_i32 s6, s6, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s7, 0
+; GFX9-NEXT:    s_cselect_b32 s16, s7, 0
+; GFX9-NEXT:    s_sub_i32 s16, s32, s16
+; GFX9-NEXT:    s_cmp_lt_i32 s7, 0
+; GFX9-NEXT:    s_cselect_b32 s17, s7, 0
+; GFX9-NEXT:    s_sub_i32 s17, s33, s17
+; GFX9-NEXT:    s_cmp_gt_i32 s17, s23
+; GFX9-NEXT:    s_cselect_b32 s17, s17, s23
+; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX9-NEXT:    s_add_i32 s7, s7, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s8, 0
+; GFX9-NEXT:    s_cselect_b32 s16, s8, 0
+; GFX9-NEXT:    s_sub_i32 s16, s32, s16
+; GFX9-NEXT:    s_cmp_lt_i32 s8, 0
+; GFX9-NEXT:    s_cselect_b32 s17, s8, 0
+; GFX9-NEXT:    s_sub_i32 s17, s33, s17
+; GFX9-NEXT:    s_cmp_gt_i32 s17, s24
+; GFX9-NEXT:    s_cselect_b32 s17, s17, s24
+; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX9-NEXT:    s_add_i32 s8, s8, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s9, 0
+; GFX9-NEXT:    s_cselect_b32 s16, s9, 0
+; GFX9-NEXT:    s_sub_i32 s16, s32, s16
+; GFX9-NEXT:    s_cmp_lt_i32 s9, 0
+; GFX9-NEXT:    s_cselect_b32 s17, s9, 0
+; GFX9-NEXT:    s_sub_i32 s17, s33, s17
+; GFX9-NEXT:    s_cmp_gt_i32 s17, s25
+; GFX9-NEXT:    s_cselect_b32 s17, s17, s25
+; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX9-NEXT:    s_add_i32 s9, s9, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s10, 0
+; GFX9-NEXT:    s_cselect_b32 s16, s10, 0
+; GFX9-NEXT:    s_sub_i32 s16, s32, s16
+; GFX9-NEXT:    s_cmp_lt_i32 s10, 0
+; GFX9-NEXT:    s_cselect_b32 s17, s10, 0
+; GFX9-NEXT:    s_sub_i32 s17, s33, s17
+; GFX9-NEXT:    s_cmp_gt_i32 s17, s26
+; GFX9-NEXT:    s_cselect_b32 s17, s17, s26
+; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX9-NEXT:    s_add_i32 s10, s10, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s11, 0
+; GFX9-NEXT:    s_cselect_b32 s16, s11, 0
+; GFX9-NEXT:    s_sub_i32 s16, s32, s16
+; GFX9-NEXT:    s_cmp_lt_i32 s11, 0
+; GFX9-NEXT:    s_cselect_b32 s17, s11, 0
+; GFX9-NEXT:    s_sub_i32 s17, s33, s17
+; GFX9-NEXT:    s_cmp_gt_i32 s17, s27
+; GFX9-NEXT:    s_cselect_b32 s17, s17, s27
+; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX9-NEXT:    s_add_i32 s11, s11, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s12, 0
+; GFX9-NEXT:    s_cselect_b32 s16, s12, 0
+; GFX9-NEXT:    s_sub_i32 s16, s32, s16
+; GFX9-NEXT:    s_cmp_lt_i32 s12, 0
+; GFX9-NEXT:    s_cselect_b32 s17, s12, 0
+; GFX9-NEXT:    s_sub_i32 s17, s33, s17
+; GFX9-NEXT:    s_cmp_gt_i32 s17, s28
+; GFX9-NEXT:    s_cselect_b32 s17, s17, s28
+; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX9-NEXT:    s_add_i32 s12, s12, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s13, 0
+; GFX9-NEXT:    s_cselect_b32 s16, s13, 0
+; GFX9-NEXT:    s_sub_i32 s16, s32, s16
+; GFX9-NEXT:    s_cmp_lt_i32 s13, 0
+; GFX9-NEXT:    s_cselect_b32 s17, s13, 0
+; GFX9-NEXT:    s_sub_i32 s17, s33, s17
+; GFX9-NEXT:    s_cmp_gt_i32 s17, s29
+; GFX9-NEXT:    s_cselect_b32 s17, s17, s29
+; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX9-NEXT:    s_add_i32 s13, s13, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s14, 0
+; GFX9-NEXT:    s_cselect_b32 s16, s14, 0
+; GFX9-NEXT:    s_sub_i32 s16, s32, s16
+; GFX9-NEXT:    s_cmp_lt_i32 s14, 0
+; GFX9-NEXT:    s_cselect_b32 s17, s14, 0
+; GFX9-NEXT:    s_sub_i32 s17, s33, s17
+; GFX9-NEXT:    s_cmp_gt_i32 s17, s30
+; GFX9-NEXT:    s_cselect_b32 s17, s17, s30
+; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX9-NEXT:    s_add_i32 s14, s14, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s15, 0
+; GFX9-NEXT:    s_cselect_b32 s16, s15, 0
+; GFX9-NEXT:    s_sub_i32 s16, s32, s16
+; GFX9-NEXT:    s_cmp_lt_i32 s15, 0
+; GFX9-NEXT:    s_cselect_b32 s17, s15, 0
+; GFX9-NEXT:    s_sub_i32 s17, s33, s17
+; GFX9-NEXT:    s_cmp_gt_i32 s17, s31
+; GFX9-NEXT:    s_cselect_b32 s17, s17, s31
+; GFX9-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX9-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX9-NEXT:    s_add_i32 s15, s15, s16
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_saddsat_v16i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX10-NEXT:    s_brev_b32 s32, -2
+; GFX10-NEXT:    s_cselect_b32 s33, s0, 0
+; GFX10-NEXT:    s_mov_b32 s34, 0x80000000
+; GFX10-NEXT:    s_sub_i32 s46, s32, s33
+; GFX10-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s35, s0, 0
+; GFX10-NEXT:    s_sub_i32 s35, s34, s35
+; GFX10-NEXT:    s_cmp_gt_i32 s35, s16
+; GFX10-NEXT:    s_cselect_b32 s16, s35, s16
+; GFX10-NEXT:    s_cmp_lt_i32 s16, s46
+; GFX10-NEXT:    s_cselect_b32 s46, s16, s46
+; GFX10-NEXT:    s_add_i32 s0, s0, s46
+; GFX10-NEXT:    s_cmp_gt_i32 s1, 0
+; GFX10-NEXT:    s_cselect_b32 s46, s1, 0
+; GFX10-NEXT:    s_sub_i32 s16, s32, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s1, 0
+; GFX10-NEXT:    s_cselect_b32 s33, s1, 0
+; GFX10-NEXT:    s_sub_i32 s46, s34, s33
+; GFX10-NEXT:    s_cmp_gt_i32 s46, s17
+; GFX10-NEXT:    s_cselect_b32 s17, s46, s17
+; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX10-NEXT:    s_cselect_b32 s46, s17, s16
+; GFX10-NEXT:    s_add_i32 s1, s1, s46
+; GFX10-NEXT:    s_cmp_gt_i32 s2, 0
+; GFX10-NEXT:    s_cselect_b32 s46, s2, 0
+; GFX10-NEXT:    s_sub_i32 s16, s32, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s2, 0
+; GFX10-NEXT:    s_cselect_b32 s17, s2, 0
+; GFX10-NEXT:    s_sub_i32 s17, s34, s17
+; GFX10-NEXT:    s_cmp_gt_i32 s17, s18
+; GFX10-NEXT:    s_cselect_b32 s17, s17, s18
+; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX10-NEXT:    s_add_i32 s2, s2, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s3, 0
+; GFX10-NEXT:    s_cselect_b32 s46, s3, 0
+; GFX10-NEXT:    s_sub_i32 s16, s32, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s3, 0
+; GFX10-NEXT:    s_cselect_b32 s17, s3, 0
+; GFX10-NEXT:    s_sub_i32 s17, s34, s17
+; GFX10-NEXT:    s_cmp_gt_i32 s17, s19
+; GFX10-NEXT:    s_cselect_b32 s17, s17, s19
+; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX10-NEXT:    s_add_i32 s3, s3, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s4, 0
+; GFX10-NEXT:    s_cselect_b32 s46, s4, 0
+; GFX10-NEXT:    s_sub_i32 s16, s32, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s4, 0
+; GFX10-NEXT:    s_cselect_b32 s17, s4, 0
+; GFX10-NEXT:    s_sub_i32 s17, s34, s17
+; GFX10-NEXT:    s_cmp_gt_i32 s17, s20
+; GFX10-NEXT:    s_cselect_b32 s17, s17, s20
+; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX10-NEXT:    s_add_i32 s4, s4, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s5, 0
+; GFX10-NEXT:    s_cselect_b32 s46, s5, 0
+; GFX10-NEXT:    s_sub_i32 s16, s32, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s5, 0
+; GFX10-NEXT:    s_cselect_b32 s17, s5, 0
+; GFX10-NEXT:    s_sub_i32 s17, s34, s17
+; GFX10-NEXT:    s_cmp_gt_i32 s17, s21
+; GFX10-NEXT:    s_cselect_b32 s17, s17, s21
+; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX10-NEXT:    s_add_i32 s5, s5, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s6, 0
+; GFX10-NEXT:    s_cselect_b32 s46, s6, 0
+; GFX10-NEXT:    s_sub_i32 s16, s32, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s6, 0
+; GFX10-NEXT:    s_cselect_b32 s17, s6, 0
+; GFX10-NEXT:    s_sub_i32 s17, s34, s17
+; GFX10-NEXT:    s_cmp_gt_i32 s17, s22
+; GFX10-NEXT:    s_cselect_b32 s17, s17, s22
+; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX10-NEXT:    s_add_i32 s6, s6, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s7, 0
+; GFX10-NEXT:    s_cselect_b32 s46, s7, 0
+; GFX10-NEXT:    s_sub_i32 s16, s32, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s7, 0
+; GFX10-NEXT:    s_cselect_b32 s17, s7, 0
+; GFX10-NEXT:    s_sub_i32 s17, s34, s17
+; GFX10-NEXT:    s_cmp_gt_i32 s17, s23
+; GFX10-NEXT:    s_cselect_b32 s17, s17, s23
+; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX10-NEXT:    s_add_i32 s7, s7, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s8, 0
+; GFX10-NEXT:    s_cselect_b32 s46, s8, 0
+; GFX10-NEXT:    s_sub_i32 s16, s32, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s8, 0
+; GFX10-NEXT:    s_cselect_b32 s17, s8, 0
+; GFX10-NEXT:    s_sub_i32 s17, s34, s17
+; GFX10-NEXT:    s_cmp_gt_i32 s17, s24
+; GFX10-NEXT:    s_cselect_b32 s17, s17, s24
+; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX10-NEXT:    s_add_i32 s8, s8, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s9, 0
+; GFX10-NEXT:    s_cselect_b32 s46, s9, 0
+; GFX10-NEXT:    s_sub_i32 s16, s32, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s9, 0
+; GFX10-NEXT:    s_cselect_b32 s17, s9, 0
+; GFX10-NEXT:    s_sub_i32 s17, s34, s17
+; GFX10-NEXT:    s_cmp_gt_i32 s17, s25
+; GFX10-NEXT:    s_cselect_b32 s17, s17, s25
+; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX10-NEXT:    s_add_i32 s9, s9, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s10, 0
+; GFX10-NEXT:    s_cselect_b32 s46, s10, 0
+; GFX10-NEXT:    s_sub_i32 s16, s32, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s10, 0
+; GFX10-NEXT:    s_cselect_b32 s17, s10, 0
+; GFX10-NEXT:    s_sub_i32 s17, s34, s17
+; GFX10-NEXT:    s_cmp_gt_i32 s17, s26
+; GFX10-NEXT:    s_cselect_b32 s17, s17, s26
+; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX10-NEXT:    s_add_i32 s10, s10, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s11, 0
+; GFX10-NEXT:    s_cselect_b32 s46, s11, 0
+; GFX10-NEXT:    s_sub_i32 s16, s32, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s11, 0
+; GFX10-NEXT:    s_cselect_b32 s17, s11, 0
+; GFX10-NEXT:    s_sub_i32 s17, s34, s17
+; GFX10-NEXT:    s_cmp_gt_i32 s17, s27
+; GFX10-NEXT:    s_cselect_b32 s17, s17, s27
+; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX10-NEXT:    s_add_i32 s11, s11, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s12, 0
+; GFX10-NEXT:    s_cselect_b32 s46, s12, 0
+; GFX10-NEXT:    s_sub_i32 s16, s32, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s12, 0
+; GFX10-NEXT:    s_cselect_b32 s17, s12, 0
+; GFX10-NEXT:    s_sub_i32 s17, s34, s17
+; GFX10-NEXT:    s_cmp_gt_i32 s17, s28
+; GFX10-NEXT:    s_cselect_b32 s17, s17, s28
+; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX10-NEXT:    s_add_i32 s12, s12, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s13, 0
+; GFX10-NEXT:    s_cselect_b32 s46, s13, 0
+; GFX10-NEXT:    s_sub_i32 s16, s32, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s13, 0
+; GFX10-NEXT:    s_cselect_b32 s17, s13, 0
+; GFX10-NEXT:    s_sub_i32 s17, s34, s17
+; GFX10-NEXT:    s_cmp_gt_i32 s17, s29
+; GFX10-NEXT:    s_cselect_b32 s17, s17, s29
+; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX10-NEXT:    s_add_i32 s13, s13, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s14, 0
+; GFX10-NEXT:    s_cselect_b32 s46, s14, 0
+; GFX10-NEXT:    s_sub_i32 s16, s32, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s14, 0
+; GFX10-NEXT:    s_cselect_b32 s17, s14, 0
+; GFX10-NEXT:    s_sub_i32 s17, s34, s17
+; GFX10-NEXT:    s_cmp_gt_i32 s17, s30
+; GFX10-NEXT:    s_cselect_b32 s17, s17, s30
+; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX10-NEXT:    s_add_i32 s14, s14, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s15, 0
+; GFX10-NEXT:    s_cselect_b32 s30, s15, 0
+; GFX10-NEXT:    s_sub_i32 s16, s32, s30
+; GFX10-NEXT:    s_cmp_lt_i32 s15, 0
+; GFX10-NEXT:    s_cselect_b32 s17, s15, 0
+; GFX10-NEXT:    s_sub_i32 s17, s34, s17
+; GFX10-NEXT:    s_cmp_gt_i32 s17, s31
+; GFX10-NEXT:    s_cselect_b32 s17, s17, s31
+; GFX10-NEXT:    s_cmp_lt_i32 s17, s16
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s16
+; GFX10-NEXT:    s_add_i32 s15, s15, s16
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
+  ret <16 x i32> %result
+}
+
+define i16 @v_saddsat_i16(i16 %lhs, i16 %rhs) {
+; GFX6-LABEL: v_saddsat_i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_min_i32_e32 v3, 0, v0
+; GFX6-NEXT:    v_max_i32_e32 v2, 0, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, 0x80000000, v3
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, 0x7fffffff, v2
+; GFX6-NEXT:    v_max_i32_e32 v1, v3, v1
+; GFX6-NEXT:    v_min_i32_e32 v1, v1, v2
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_min_i16_e32 v3, 0, v0
+; GFX8-NEXT:    v_max_i16_e32 v2, 0, v0
+; GFX8-NEXT:    v_sub_u16_e32 v3, 0x8000, v3
+; GFX8-NEXT:    v_sub_u16_e32 v2, 0x7fff, v2
+; GFX8-NEXT:    v_max_i16_e32 v1, v3, v1
+; GFX8-NEXT:    v_min_i16_e32 v1, v1, v2
+; GFX8-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_min_i16_e32 v3, 0, v0
+; GFX9-NEXT:    v_max_i16_e32 v2, 0, v0
+; GFX9-NEXT:    v_sub_u16_e32 v3, 0x8000, v3
+; GFX9-NEXT:    v_sub_u16_e32 v2, 0x7fff, v2
+; GFX9-NEXT:    v_max_i16_e32 v1, v3, v1
+; GFX9-NEXT:    v_min_i16_e32 v1, v1, v2
+; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_saddsat_i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_min_i16_e64 v2, v0, 0
+; GFX10-NEXT:    v_max_i16_e64 v3, v0, 0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_sub_nc_u16_e64 v2, 0x8000, v2
+; GFX10-NEXT:    v_sub_nc_u16_e64 v3, 0x7fff, v3
+; GFX10-NEXT:    v_max_i16_e64 v1, v2, v1
+; GFX10-NEXT:    v_min_i16_e64 v1, v1, v3
+; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i16 @llvm.sadd.sat.i16(i16 %lhs, i16 %rhs)
+  ret i16 %result
+}
+
+define amdgpu_ps i16 @s_saddsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
+; GFX6-LABEL: s_saddsat_i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX6-NEXT:    s_cselect_b32 s2, s0, 0
+; GFX6-NEXT:    s_sub_i32 s2, 0x7fffffff, s2
+; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX6-NEXT:    s_cselect_b32 s3, s0, 0
+; GFX6-NEXT:    s_sub_i32 s3, 0x80000000, s3
+; GFX6-NEXT:    s_cmp_gt_i32 s3, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX6-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX6-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX6-NEXT:    s_add_i32 s0, s0, s1
+; GFX6-NEXT:    s_ashr_i32 s0, s0, 16
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_saddsat_i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_sext_i32_i16 s2, s0
+; GFX8-NEXT:    s_sext_i32_i16 s3, 0
+; GFX8-NEXT:    s_cmp_gt_i32 s2, s3
+; GFX8-NEXT:    s_cselect_b32 s4, s2, s3
+; GFX8-NEXT:    s_sub_i32 s4, 0x7fff, s4
+; GFX8-NEXT:    s_cmp_lt_i32 s2, s3
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX8-NEXT:    s_sub_i32 s2, 0x8000, s2
+; GFX8-NEXT:    s_sext_i32_i16 s2, s2
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_cmp_gt_i32 s2, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_sext_i32_i16 s2, s4
+; GFX8-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX8-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX8-NEXT:    s_add_i32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_saddsat_i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_sext_i32_i16 s2, s0
+; GFX9-NEXT:    s_sext_i32_i16 s3, 0
+; GFX9-NEXT:    s_cmp_gt_i32 s2, s3
+; GFX9-NEXT:    s_cselect_b32 s4, s2, s3
+; GFX9-NEXT:    s_sub_i32 s4, 0x7fff, s4
+; GFX9-NEXT:    s_cmp_lt_i32 s2, s3
+; GFX9-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX9-NEXT:    s_sub_i32 s2, 0x8000, s2
+; GFX9-NEXT:    s_sext_i32_i16 s2, s2
+; GFX9-NEXT:    s_sext_i32_i16 s1, s1
+; GFX9-NEXT:    s_cmp_gt_i32 s2, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX9-NEXT:    s_sext_i32_i16 s1, s1
+; GFX9-NEXT:    s_sext_i32_i16 s2, s4
+; GFX9-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX9-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX9-NEXT:    s_add_i32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_saddsat_i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_sext_i32_i16 s2, 0
+; GFX10-NEXT:    s_sext_i32_i16 s3, s0
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_cmp_gt_i32 s3, s2
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s4, s3, s2
+; GFX10-NEXT:    s_sub_i32 s4, 0x7fff, s4
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s2
+; GFX10-NEXT:    s_cselect_b32 s2, s3, s2
+; GFX10-NEXT:    s_sub_i32 s2, 0x8000, s2
+; GFX10-NEXT:    s_sext_i32_i16 s2, s2
+; GFX10-NEXT:    s_cmp_gt_i32 s2, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX10-NEXT:    s_sext_i32_i16 s2, s4
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX10-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX10-NEXT:    s_add_i32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i16 @llvm.sadd.sat.i16(i16 %lhs, i16 %rhs)
+  ret i16 %result
+}
+
+define amdgpu_ps half @saddsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
+; GFX6-LABEL: saddsat_i16_sv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX6-NEXT:    s_cselect_b32 s1, s0, 0
+; GFX6-NEXT:    s_sub_i32 s1, 0x7fffffff, s1
+; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX6-NEXT:    s_cselect_b32 s2, s0, 0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    s_sub_i32 s2, 0x80000000, s2
+; GFX6-NEXT:    v_max_i32_e32 v0, s2, v0
+; GFX6-NEXT:    v_min_i32_e32 v0, s1, v0
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: saddsat_i16_sv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_sext_i32_i16 s1, s0
+; GFX8-NEXT:    s_sext_i32_i16 s2, 0
+; GFX8-NEXT:    s_cmp_gt_i32 s1, s2
+; GFX8-NEXT:    s_cselect_b32 s3, s1, s2
+; GFX8-NEXT:    s_sub_i32 s3, 0x7fff, s3
+; GFX8-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX8-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX8-NEXT:    s_sub_i32 s1, 0x8000, s1
+; GFX8-NEXT:    v_max_i16_e32 v0, s1, v0
+; GFX8-NEXT:    v_min_i16_e32 v0, s3, v0
+; GFX8-NEXT:    v_add_u16_e32 v0, s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: saddsat_i16_sv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_sext_i32_i16 s1, s0
+; GFX9-NEXT:    s_sext_i32_i16 s2, 0
+; GFX9-NEXT:    s_cmp_gt_i32 s1, s2
+; GFX9-NEXT:    s_cselect_b32 s3, s1, s2
+; GFX9-NEXT:    s_sub_i32 s3, 0x7fff, s3
+; GFX9-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX9-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX9-NEXT:    s_sub_i32 s1, 0x8000, s1
+; GFX9-NEXT:    v_max_i16_e32 v0, s1, v0
+; GFX9-NEXT:    v_min_i16_e32 v0, s3, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: saddsat_i16_sv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_sext_i32_i16 s1, s0
+; GFX10-NEXT:    s_sext_i32_i16 s2, 0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cmp_gt_i32 s1, s2
+; GFX10-NEXT:    s_cselect_b32 s3, s1, s2
+; GFX10-NEXT:    s_sub_i32 s3, 0x7fff, s3
+; GFX10-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX10-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX10-NEXT:    s_sub_i32 s1, 0x8000, s1
+; GFX10-NEXT:    v_max_i16_e64 v0, s1, v0
+; GFX10-NEXT:    v_min_i16_e64 v0, v0, s3
+; GFX10-NEXT:    v_add_nc_u16_e64 v0, s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i16 @llvm.sadd.sat.i16(i16 %lhs, i16 %rhs)
+  %cast = bitcast i16 %result to half
+  ret half %cast
+}
+
+define amdgpu_ps half @saddsat_i16_vs(i16 %lhs, i16 inreg %rhs) {
+; GFX6-LABEL: saddsat_i16_vs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_min_i32_e32 v2, 0, v0
+; GFX6-NEXT:    v_max_i32_e32 v1, 0, v0
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, 0x80000000, v2
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, 0x7fffffff, v1
+; GFX6-NEXT:    v_max_i32_e32 v2, s0, v2
+; GFX6-NEXT:    v_min_i32_e32 v1, v2, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: saddsat_i16_vs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_min_i16_e32 v2, 0, v0
+; GFX8-NEXT:    v_max_i16_e32 v1, 0, v0
+; GFX8-NEXT:    v_sub_u16_e32 v2, 0x8000, v2
+; GFX8-NEXT:    v_sub_u16_e32 v1, 0x7fff, v1
+; GFX8-NEXT:    v_max_i16_e32 v2, s0, v2
+; GFX8-NEXT:    v_min_i16_e32 v1, v2, v1
+; GFX8-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: saddsat_i16_vs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_min_i16_e32 v2, 0, v0
+; GFX9-NEXT:    v_max_i16_e32 v1, 0, v0
+; GFX9-NEXT:    v_sub_u16_e32 v2, 0x8000, v2
+; GFX9-NEXT:    v_sub_u16_e32 v1, 0x7fff, v1
+; GFX9-NEXT:    v_max_i16_e32 v2, s0, v2
+; GFX9-NEXT:    v_min_i16_e32 v1, v2, v1
+; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: saddsat_i16_vs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_min_i16_e64 v1, v0, 0
+; GFX10-NEXT:    v_max_i16_e64 v2, v0, 0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_sub_nc_u16_e64 v1, 0x8000, v1
+; GFX10-NEXT:    v_sub_nc_u16_e64 v2, 0x7fff, v2
+; GFX10-NEXT:    v_max_i16_e64 v1, v1, s0
+; GFX10-NEXT:    v_min_i16_e64 v1, v1, v2
+; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i16 @llvm.sadd.sat.i16(i16 %lhs, i16 %rhs)
+  %cast = bitcast i16 %result to half
+  ret half %cast
+}
+
+define <2 x i16> @v_saddsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
+; GFX6-LABEL: v_saddsat_v2i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v5, 0, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, s5, v5
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    v_max_i32_e32 v4, 0, v0
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, s4, v4
+; GFX6-NEXT:    v_max_i32_e32 v2, v5, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_min_i32_e32 v2, v2, v4
+; GFX6-NEXT:    v_min_i32_e32 v4, 0, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
+; GFX6-NEXT:    v_max_i32_e32 v3, 0, v1
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, s5, v4
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, s4, v3
+; GFX6-NEXT:    v_max_i32_e32 v2, v4, v2
+; GFX6-NEXT:    v_min_i32_e32 v2, v2, v3
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_v2i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s5, 0x8000
+; GFX8-NEXT:    v_min_i16_e32 v4, 0, v0
+; GFX8-NEXT:    v_sub_u16_e32 v4, s5, v4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX8-NEXT:    v_max_i16_e32 v3, 0, v0
+; GFX8-NEXT:    v_min_i16_e32 v5, 0, v2
+; GFX8-NEXT:    v_sub_u16_e32 v3, s4, v3
+; GFX8-NEXT:    v_max_i16_e32 v4, v4, v1
+; GFX8-NEXT:    v_min_i16_e32 v3, v4, v3
+; GFX8-NEXT:    v_max_i16_e32 v4, 0, v2
+; GFX8-NEXT:    v_sub_u16_e32 v5, s5, v5
+; GFX8-NEXT:    v_sub_u16_e32 v4, s4, v4
+; GFX8-NEXT:    v_max_i16_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_min_i16_e32 v1, v1, v4
+; GFX8-NEXT:    v_add_u16_e32 v0, v0, v3
+; GFX8-NEXT:    v_add_u16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, 0, 0
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
+; GFX9-NEXT:    v_pk_min_i16 v3, v0, s6
+; GFX9-NEXT:    v_pk_sub_i16 v3, s5, v3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
+; GFX9-NEXT:    v_pk_max_i16 v2, v0, s6
+; GFX9-NEXT:    v_pk_sub_i16 v2, s4, v2
+; GFX9-NEXT:    v_pk_max_i16 v1, v3, v1
+; GFX9-NEXT:    v_pk_min_i16 v1, v1, v2
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_saddsat_v2i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, 0, 0
+; GFX10-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX10-NEXT:    v_pk_min_i16 v2, v0, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
+; GFX10-NEXT:    v_pk_max_i16 v3, v0, s4
+; GFX10-NEXT:    s_movk_i32 s6, 0x7fff
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_pk_sub_i16 v2, s5, v2
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s6, s6
+; GFX10-NEXT:    v_pk_sub_i16 v3, s4, v3
+; GFX10-NEXT:    v_pk_max_i16 v1, v2, v1
+; GFX10-NEXT:    v_pk_min_i16 v1, v1, v3
+; GFX10-NEXT:    v_pk_add_u16 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
+  ret <2 x i16> %result
+}
+
+define amdgpu_ps i32 @s_saddsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs) {
+; GFX6-LABEL: s_saddsat_v2i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    s_cselect_b32 s6, s0, 0
+; GFX6-NEXT:    s_sub_i32 s6, s4, s6
+; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_cselect_b32 s7, s0, 0
+; GFX6-NEXT:    s_sub_i32 s7, s5, s7
+; GFX6-NEXT:    s_cmp_gt_i32 s7, s2
+; GFX6-NEXT:    s_cselect_b32 s2, s7, s2
+; GFX6-NEXT:    s_cmp_lt_i32 s2, s6
+; GFX6-NEXT:    s_cselect_b32 s2, s2, s6
+; GFX6-NEXT:    s_add_i32 s0, s0, s2
+; GFX6-NEXT:    s_ashr_i32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s2, s3, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s1, 0
+; GFX6-NEXT:    s_cselect_b32 s3, s1, 0
+; GFX6-NEXT:    s_sub_i32 s3, s4, s3
+; GFX6-NEXT:    s_cmp_lt_i32 s1, 0
+; GFX6-NEXT:    s_cselect_b32 s4, s1, 0
+; GFX6-NEXT:    s_sub_i32 s4, s5, s4
+; GFX6-NEXT:    s_cmp_gt_i32 s4, s2
+; GFX6-NEXT:    s_cselect_b32 s2, s4, s2
+; GFX6-NEXT:    s_cmp_lt_i32 s2, s3
+; GFX6-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX6-NEXT:    s_add_i32 s1, s1, s2
+; GFX6-NEXT:    s_mov_b32 s2, 0xffff
+; GFX6-NEXT:    s_ashr_i32 s1, s1, 16
+; GFX6-NEXT:    s_and_b32 s1, s1, s2
+; GFX6-NEXT:    s_and_b32 s0, s0, s2
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_saddsat_v2i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX8-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX8-NEXT:    s_sext_i32_i16 s6, s0
+; GFX8-NEXT:    s_sext_i32_i16 s7, 0
+; GFX8-NEXT:    s_cmp_gt_i32 s6, s7
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX8-NEXT:    s_cselect_b32 s8, s6, s7
+; GFX8-NEXT:    s_sub_i32 s8, s4, s8
+; GFX8-NEXT:    s_cmp_lt_i32 s6, s7
+; GFX8-NEXT:    s_mov_b32 s5, 0x8000
+; GFX8-NEXT:    s_cselect_b32 s6, s6, s7
+; GFX8-NEXT:    s_sub_i32 s6, s5, s6
+; GFX8-NEXT:    s_sext_i32_i16 s6, s6
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_cmp_gt_i32 s6, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s6, s1
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_sext_i32_i16 s6, s8
+; GFX8-NEXT:    s_cmp_lt_i32 s1, s6
+; GFX8-NEXT:    s_cselect_b32 s1, s1, s6
+; GFX8-NEXT:    s_add_i32 s0, s0, s1
+; GFX8-NEXT:    s_sext_i32_i16 s1, s2
+; GFX8-NEXT:    s_cmp_gt_i32 s1, s7
+; GFX8-NEXT:    s_cselect_b32 s6, s1, s7
+; GFX8-NEXT:    s_sub_i32 s4, s4, s6
+; GFX8-NEXT:    s_cmp_lt_i32 s1, s7
+; GFX8-NEXT:    s_cselect_b32 s1, s1, s7
+; GFX8-NEXT:    s_sub_i32 s1, s5, s1
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_cmp_gt_i32 s1, s3
+; GFX8-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_sext_i32_i16 s3, s4
+; GFX8-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX8-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX8-NEXT:    s_add_i32 s2, s2, s1
+; GFX8-NEXT:    s_bfe_u32 s1, s2, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_saddsat_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, 0, 0
+; GFX9-NEXT:    s_sext_i32_i16 s7, s4
+; GFX9-NEXT:    s_sext_i32_i16 s5, s0
+; GFX9-NEXT:    s_ashr_i32 s6, s0, 16
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s7
+; GFX9-NEXT:    s_cselect_b32 s8, s5, s7
+; GFX9-NEXT:    s_cmp_gt_i32 s6, s4
+; GFX9-NEXT:    s_movk_i32 s2, 0x7fff
+; GFX9-NEXT:    s_cselect_b32 s9, s6, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s8, s8, s9
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s2
+; GFX9-NEXT:    s_lshr_b32 s9, s2, 16
+; GFX9-NEXT:    s_lshr_b32 s10, s8, 16
+; GFX9-NEXT:    s_sub_i32 s2, s2, s8
+; GFX9-NEXT:    s_sub_i32 s8, s9, s10
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s7
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s7
+; GFX9-NEXT:    s_cmp_lt_i32 s6, s4
+; GFX9-NEXT:    s_mov_b32 s3, 0xffff8000
+; GFX9-NEXT:    s_cselect_b32 s4, s6, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s5, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s3
+; GFX9-NEXT:    s_lshr_b32 s5, s3, 16
+; GFX9-NEXT:    s_lshr_b32 s6, s4, 16
+; GFX9-NEXT:    s_sub_i32 s3, s3, s4
+; GFX9-NEXT:    s_sub_i32 s4, s5, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
+; GFX9-NEXT:    s_sext_i32_i16 s4, s3
+; GFX9-NEXT:    s_sext_i32_i16 s5, s1
+; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX9-NEXT:    s_ashr_i32 s1, s1, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s5
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s4, s1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s8
+; GFX9-NEXT:    s_sext_i32_i16 s3, s1
+; GFX9-NEXT:    s_sext_i32_i16 s4, s2
+; GFX9-NEXT:    s_ashr_i32 s1, s1, 16
+; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX9-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX9-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s3, s1
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX9-NEXT:    s_add_i32 s0, s0, s1
+; GFX9-NEXT:    s_add_i32 s2, s2, s3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_saddsat_v2i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, 0, 0
+; GFX10-NEXT:    s_sext_i32_i16 s3, s0
+; GFX10-NEXT:    s_sext_i32_i16 s5, s2
+; GFX10-NEXT:    s_ashr_i32 s4, s0, 16
+; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s3, s5
+; GFX10-NEXT:    s_movk_i32 s7, 0x7fff
+; GFX10-NEXT:    s_cselect_b32 s6, s3, s5
+; GFX10-NEXT:    s_cmp_gt_i32 s4, s2
+; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s7, s7
+; GFX10-NEXT:    s_cselect_b32 s8, s4, s2
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s8
+; GFX10-NEXT:    s_lshr_b32 s8, s7, 16
+; GFX10-NEXT:    s_lshr_b32 s9, s6, 16
+; GFX10-NEXT:    s_sub_i32 s6, s7, s6
+; GFX10-NEXT:    s_sub_i32 s7, s8, s9
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s5
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s5
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s2
+; GFX10-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX10-NEXT:    s_cselect_b32 s2, s4, s2
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s5, s5
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s3, s2
+; GFX10-NEXT:    s_lshr_b32 s3, s4, 16
+; GFX10-NEXT:    s_lshr_b32 s5, s2, 16
+; GFX10-NEXT:    s_sub_i32 s2, s4, s2
+; GFX10-NEXT:    s_sub_i32 s3, s3, s5
+; GFX10-NEXT:    s_sext_i32_i16 s4, s1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
+; GFX10-NEXT:    s_ashr_i32 s1, s1, 16
+; GFX10-NEXT:    s_sext_i32_i16 s3, s2
+; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s3, s4
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX10-NEXT:    s_cmp_gt_i32 s2, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s6, s7
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s3, s1
+; GFX10-NEXT:    s_sext_i32_i16 s4, s2
+; GFX10-NEXT:    s_sext_i32_i16 s3, s1
+; GFX10-NEXT:    s_ashr_i32 s1, s1, 16
+; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX10-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX10-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX10-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s3, s1
+; GFX10-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX10-NEXT:    s_add_i32 s0, s0, s1
+; GFX10-NEXT:    s_add_i32 s2, s2, s3
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
+  %cast = bitcast <2 x i16> %result to i32
+  ret i32 %cast
+}
+
+define amdgpu_ps float @saddsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
+; GFX6-LABEL: saddsat_v2i16_sv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX6-NEXT:    s_brev_b32 s2, -2
+; GFX6-NEXT:    s_cselect_b32 s4, s0, 0
+; GFX6-NEXT:    s_sub_i32 s4, s2, s4
+; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX6-NEXT:    s_mov_b32 s3, 0x80000000
+; GFX6-NEXT:    s_cselect_b32 s5, s0, 0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    s_sub_i32 s5, s3, s5
+; GFX6-NEXT:    v_max_i32_e32 v0, s5, v0
+; GFX6-NEXT:    v_min_i32_e32 v0, s4, v0
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; GFX6-NEXT:    s_lshl_b32 s0, s1, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX6-NEXT:    s_cselect_b32 s1, s0, 0
+; GFX6-NEXT:    s_sub_i32 s1, s2, s1
+; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX6-NEXT:    s_cselect_b32 s2, s0, 0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    s_sub_i32 s2, s3, s2
+; GFX6-NEXT:    v_max_i32_e32 v1, s2, v1
+; GFX6-NEXT:    v_min_i32_e32 v1, s1, v1
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, s0, v1
+; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
+; GFX6-NEXT:    s_mov_b32 s0, 0xffff
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, s0, v1
+; GFX6-NEXT:    v_and_b32_e32 v0, s0, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: saddsat_v2i16_sv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_lshr_b32 s1, s0, 16
+; GFX8-NEXT:    s_sext_i32_i16 s4, s0
+; GFX8-NEXT:    s_sext_i32_i16 s5, 0
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s5
+; GFX8-NEXT:    s_movk_i32 s2, 0x7fff
+; GFX8-NEXT:    s_cselect_b32 s6, s4, s5
+; GFX8-NEXT:    s_sub_i32 s6, s2, s6
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX8-NEXT:    s_mov_b32 s3, 0x8000
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX8-NEXT:    s_sub_i32 s4, s3, s4
+; GFX8-NEXT:    v_max_i16_e32 v1, s4, v0
+; GFX8-NEXT:    s_sext_i32_i16 s4, s1
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s5
+; GFX8-NEXT:    v_min_i16_e32 v1, s6, v1
+; GFX8-NEXT:    s_cselect_b32 s6, s4, s5
+; GFX8-NEXT:    s_sub_i32 s2, s2, s6
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX8-NEXT:    s_sub_i32 s3, s3, s4
+; GFX8-NEXT:    v_mov_b32_e32 v2, s3
+; GFX8-NEXT:    v_max_i16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_min_i16_e32 v0, s2, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    v_add_u16_e32 v1, s0, v1
+; GFX8-NEXT:    v_add_u16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: saddsat_v2i16_sv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, 0, 0
+; GFX9-NEXT:    s_sext_i32_i16 s6, s3
+; GFX9-NEXT:    s_sext_i32_i16 s4, s0
+; GFX9-NEXT:    s_ashr_i32 s5, s0, 16
+; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s6
+; GFX9-NEXT:    s_cselect_b32 s7, s4, s6
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s3
+; GFX9-NEXT:    s_movk_i32 s1, 0x7fff
+; GFX9-NEXT:    s_cselect_b32 s8, s5, s3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s7, s7, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s1
+; GFX9-NEXT:    s_lshr_b32 s8, s1, 16
+; GFX9-NEXT:    s_lshr_b32 s9, s7, 16
+; GFX9-NEXT:    s_sub_i32 s1, s1, s7
+; GFX9-NEXT:    s_sub_i32 s7, s8, s9
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s6
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s3
+; GFX9-NEXT:    s_mov_b32 s2, 0xffff8000
+; GFX9-NEXT:    s_cselect_b32 s3, s5, s3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s4, s3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s2
+; GFX9-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX9-NEXT:    s_lshr_b32 s5, s3, 16
+; GFX9-NEXT:    s_sub_i32 s2, s2, s3
+; GFX9-NEXT:    s_sub_i32 s3, s4, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s7
+; GFX9-NEXT:    v_pk_max_i16 v0, s2, v0
+; GFX9-NEXT:    v_pk_min_i16 v0, v0, s1
+; GFX9-NEXT:    v_pk_add_u16 v0, s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: saddsat_v2i16_sv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, 0, 0
+; GFX10-NEXT:    s_sext_i32_i16 s2, s0
+; GFX10-NEXT:    s_sext_i32_i16 s4, s1
+; GFX10-NEXT:    s_ashr_i32 s3, s0, 16
+; GFX10-NEXT:    s_ashr_i32 s1, s1, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s2, s4
+; GFX10-NEXT:    s_movk_i32 s6, 0x7fff
+; GFX10-NEXT:    s_cselect_b32 s5, s2, s4
+; GFX10-NEXT:    s_cmp_gt_i32 s3, s1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s6
+; GFX10-NEXT:    s_cselect_b32 s7, s3, s1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s7
+; GFX10-NEXT:    s_lshr_b32 s7, s6, 16
+; GFX10-NEXT:    s_lshr_b32 s8, s5, 16
+; GFX10-NEXT:    s_sub_i32 s5, s6, s5
+; GFX10-NEXT:    s_sub_i32 s6, s7, s8
+; GFX10-NEXT:    s_cmp_lt_i32 s2, s4
+; GFX10-NEXT:    s_cselect_b32 s2, s2, s4
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s1
+; GFX10-NEXT:    s_mov_b32 s4, 0xffff8000
+; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s4, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s2, s1
+; GFX10-NEXT:    s_lshr_b32 s2, s3, 16
+; GFX10-NEXT:    s_lshr_b32 s4, s1, 16
+; GFX10-NEXT:    s_sub_i32 s1, s3, s1
+; GFX10-NEXT:    s_sub_i32 s2, s2, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
+; GFX10-NEXT:    v_pk_max_i16 v0, s1, v0
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s5, s6
+; GFX10-NEXT:    v_pk_min_i16 v0, v0, s1
+; GFX10-NEXT:    v_pk_add_u16 v0, s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
+  %cast = bitcast <2 x i16> %result to float
+  ret float %cast
+}
+
+define amdgpu_ps float @saddsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
+; GFX6-LABEL: saddsat_v2i16_vs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    s_mov_b32 s3, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v3, 0, v0
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, s3, v3
+; GFX6-NEXT:    s_brev_b32 s2, -2
+; GFX6-NEXT:    v_max_i32_e32 v2, 0, v0
+; GFX6-NEXT:    v_max_i32_e32 v3, s0, v3
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, s2, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_min_i32_e32 v2, v3, v2
+; GFX6-NEXT:    v_min_i32_e32 v3, 0, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT:    v_max_i32_e32 v2, 0, v1
+; GFX6-NEXT:    s_lshl_b32 s0, s1, 16
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, s3, v3
+; GFX6-NEXT:    v_max_i32_e32 v3, s0, v3
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, s2, v2
+; GFX6-NEXT:    v_min_i32_e32 v2, v3, v2
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
+; GFX6-NEXT:    s_mov_b32 s0, 0xffff
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, s0, v1
+; GFX6-NEXT:    v_and_b32_e32 v0, s0, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: saddsat_v2i16_vs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s3, 0x8000
+; GFX8-NEXT:    v_min_i16_e32 v3, 0, v0
+; GFX8-NEXT:    v_sub_u16_e32 v3, s3, v3
+; GFX8-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; GFX8-NEXT:    s_movk_i32 s2, 0x7fff
+; GFX8-NEXT:    v_max_i16_e32 v2, 0, v0
+; GFX8-NEXT:    v_min_i16_e32 v4, 0, v1
+; GFX8-NEXT:    v_sub_u16_e32 v2, s2, v2
+; GFX8-NEXT:    v_max_i16_e32 v3, s0, v3
+; GFX8-NEXT:    v_min_i16_e32 v2, v3, v2
+; GFX8-NEXT:    v_max_i16_e32 v3, 0, v1
+; GFX8-NEXT:    s_lshr_b32 s1, s0, 16
+; GFX8-NEXT:    v_sub_u16_e32 v4, s3, v4
+; GFX8-NEXT:    v_sub_u16_e32 v3, s2, v3
+; GFX8-NEXT:    v_max_i16_e32 v4, s1, v4
+; GFX8-NEXT:    v_min_i16_e32 v3, v4, v3
+; GFX8-NEXT:    v_add_u16_e32 v0, v0, v2
+; GFX8-NEXT:    v_add_u16_sdwa v1, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: saddsat_v2i16_vs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s2, 0xffff8000
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, 0, 0
+; GFX9-NEXT:    s_movk_i32 s1, 0x7fff
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s2
+; GFX9-NEXT:    v_pk_min_i16 v2, v0, s3
+; GFX9-NEXT:    v_pk_sub_i16 v2, s2, v2
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s1
+; GFX9-NEXT:    v_pk_max_i16 v1, v0, s3
+; GFX9-NEXT:    v_pk_sub_i16 v1, s1, v1
+; GFX9-NEXT:    v_pk_max_i16 v2, v2, s0
+; GFX9-NEXT:    v_pk_min_i16 v1, v2, v1
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: saddsat_v2i16_vs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, 0, 0
+; GFX10-NEXT:    s_mov_b32 s2, 0xffff8000
+; GFX10-NEXT:    v_pk_min_i16 v1, v0, s1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s2
+; GFX10-NEXT:    v_pk_max_i16 v2, v0, s1
+; GFX10-NEXT:    s_movk_i32 s3, 0x7fff
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_pk_sub_i16 v1, s2, v1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s3, s3
+; GFX10-NEXT:    v_pk_sub_i16 v2, s1, v2
+; GFX10-NEXT:    v_pk_max_i16 v1, v1, s0
+; GFX10-NEXT:    v_pk_min_i16 v1, v1, v2
+; GFX10-NEXT:    v_pk_add_u16 v0, v0, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
+  %cast = bitcast <2 x i16> %result to float
+  ret float %cast
+}
+
+; FIXME: v3i16 insert/extract
+; define <3 x i16> @v_saddsat_v3i16(<3 x i16> %lhs, <3 x i16> %rhs) {
+;   %result = call <3 x i16> @llvm.sadd.sat.v3i16(<3 x i16> %lhs, <3 x i16> %rhs)
+;   ret <3 x i16> %result
+; }
+
+; define amdgpu_ps <3 x i16> @s_saddsat_v3i16(<3 x i16> inreg %lhs, <3 x i16> inreg %rhs) {
+;   %result = call <3 x i16> @llvm.sadd.sat.v3i16(<3 x i16> %lhs, <3 x i16> %rhs)
+;   ret <3 x i16> %result
+; }
+
+define <2 x float> @v_saddsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
+; GFX6-LABEL: v_saddsat_v4i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v10, 0, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT:    v_sub_i32_e32 v10, vcc, s5, v10
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    v_max_i32_e32 v8, 0, v0
+; GFX6-NEXT:    v_sub_i32_e32 v8, vcc, s4, v8
+; GFX6-NEXT:    v_max_i32_e32 v4, v10, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_min_i32_e32 v4, v4, v8
+; GFX6-NEXT:    v_min_i32_e32 v8, 0, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
+; GFX6-NEXT:    v_max_i32_e32 v5, 0, v1
+; GFX6-NEXT:    v_sub_i32_e32 v8, vcc, s5, v8
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, s4, v5
+; GFX6-NEXT:    v_max_i32_e32 v4, v8, v4
+; GFX6-NEXT:    v_min_i32_e32 v4, v4, v5
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v6
+; GFX6-NEXT:    v_min_i32_e32 v6, 0, v2
+; GFX6-NEXT:    v_sub_i32_e32 v6, vcc, s5, v6
+; GFX6-NEXT:    v_bfrev_b32_e32 v9, -2
+; GFX6-NEXT:    v_max_i32_e32 v5, 0, v2
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, v9, v5
+; GFX6-NEXT:    v_max_i32_e32 v4, v6, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_min_i32_e32 v4, v4, v5
+; GFX6-NEXT:    v_mov_b32_e32 v11, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v6, 0, v3
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; GFX6-NEXT:    v_max_i32_e32 v5, 0, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v7
+; GFX6-NEXT:    v_sub_i32_e32 v6, vcc, v11, v6
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, v9, v5
+; GFX6-NEXT:    v_max_i32_e32 v4, v6, v4
+; GFX6-NEXT:    v_min_i32_e32 v4, v4, v5
+; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
+; GFX6-NEXT:    v_ashrrev_i32_e32 v3, 16, v3
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_v4i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s5, 0x8000
+; GFX8-NEXT:    v_min_i16_e32 v7, 0, v0
+; GFX8-NEXT:    v_sub_u16_e32 v7, s5, v7
+; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX8-NEXT:    v_max_i16_e32 v6, 0, v0
+; GFX8-NEXT:    v_min_i16_e32 v8, 0, v4
+; GFX8-NEXT:    v_sub_u16_e32 v6, s4, v6
+; GFX8-NEXT:    v_max_i16_e32 v7, v7, v2
+; GFX8-NEXT:    v_min_i16_e32 v6, v7, v6
+; GFX8-NEXT:    v_max_i16_e32 v7, 0, v4
+; GFX8-NEXT:    v_sub_u16_e32 v8, s5, v8
+; GFX8-NEXT:    v_max_i16_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_min_i16_e32 v8, 0, v1
+; GFX8-NEXT:    v_sub_u16_e32 v7, s4, v7
+; GFX8-NEXT:    v_min_i16_e32 v2, v2, v7
+; GFX8-NEXT:    v_max_i16_e32 v7, 0, v1
+; GFX8-NEXT:    v_sub_u16_e32 v8, s5, v8
+; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
+; GFX8-NEXT:    v_min_i16_e32 v9, 0, v5
+; GFX8-NEXT:    v_sub_u16_e32 v7, s4, v7
+; GFX8-NEXT:    v_max_i16_e32 v8, v8, v3
+; GFX8-NEXT:    v_min_i16_e32 v7, v8, v7
+; GFX8-NEXT:    v_max_i16_e32 v8, 0, v5
+; GFX8-NEXT:    v_sub_u16_e32 v9, s5, v9
+; GFX8-NEXT:    v_sub_u16_e32 v8, s4, v8
+; GFX8-NEXT:    v_max_i16_sdwa v3, v9, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_min_i16_e32 v3, v3, v8
+; GFX8-NEXT:    v_add_u16_e32 v0, v0, v6
+; GFX8-NEXT:    v_add_u16_sdwa v2, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT:    v_add_u16_e32 v1, v1, v7
+; GFX8-NEXT:    v_add_u16_sdwa v2, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_v4i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, 0, 0
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
+; GFX9-NEXT:    v_pk_min_i16 v5, v0, s6
+; GFX9-NEXT:    v_pk_sub_i16 v5, s5, v5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
+; GFX9-NEXT:    v_pk_max_i16 v4, v0, s6
+; GFX9-NEXT:    v_pk_sub_i16 v4, s4, v4
+; GFX9-NEXT:    v_pk_max_i16 v2, v5, v2
+; GFX9-NEXT:    v_pk_min_i16 v2, v2, v4
+; GFX9-NEXT:    v_pk_min_i16 v4, v1, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, v2
+; GFX9-NEXT:    v_pk_max_i16 v2, v1, s6
+; GFX9-NEXT:    v_pk_sub_i16 v4, s5, v4
+; GFX9-NEXT:    v_pk_sub_i16 v2, s4, v2
+; GFX9-NEXT:    v_pk_max_i16 v3, v4, v3
+; GFX9-NEXT:    v_pk_min_i16 v2, v3, v2
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_saddsat_v4i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, 0, 0
+; GFX10-NEXT:    s_mov_b32 s6, 0xffff8000
+; GFX10-NEXT:    v_pk_min_i16 v4, v0, s5
+; GFX10-NEXT:    v_pk_min_i16 v5, v1, s5
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s6
+; GFX10-NEXT:    v_pk_max_i16 v6, v0, s5
+; GFX10-NEXT:    v_pk_max_i16 v7, v1, s5
+; GFX10-NEXT:    v_pk_sub_i16 v4, s6, v4
+; GFX10-NEXT:    v_pk_sub_i16 v5, s6, v5
+; GFX10-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
+; GFX10-NEXT:    v_pk_max_i16 v11, v4, v2
+; GFX10-NEXT:    v_pk_sub_i16 v6, s4, v6
+; GFX10-NEXT:    v_pk_sub_i16 v4, s4, v7
+; GFX10-NEXT:    v_pk_max_i16 v3, v5, v3
+; GFX10-NEXT:    v_pk_min_i16 v2, v11, v6
+; GFX10-NEXT:    v_pk_min_i16 v3, v3, v4
+; GFX10-NEXT:    v_pk_add_u16 v0, v0, v2
+; GFX10-NEXT:    v_pk_add_u16 v1, v1, v3
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> %lhs, <4 x i16> %rhs)
+  %cast = bitcast <4 x i16> %result to <2 x float>
+  ret <2 x float> %cast
+}
+
+define amdgpu_ps <2 x i32> @s_saddsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inreg %rhs) {
+; GFX6-LABEL: s_saddsat_v4i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX6-NEXT:    s_brev_b32 s8, -2
+; GFX6-NEXT:    s_cselect_b32 s10, s0, 0
+; GFX6-NEXT:    s_sub_i32 s10, s8, s10
+; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX6-NEXT:    s_mov_b32 s9, 0x80000000
+; GFX6-NEXT:    s_cselect_b32 s11, s0, 0
+; GFX6-NEXT:    s_sub_i32 s11, s9, s11
+; GFX6-NEXT:    s_cmp_gt_i32 s11, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s11, s4
+; GFX6-NEXT:    s_cmp_lt_i32 s4, s10
+; GFX6-NEXT:    s_cselect_b32 s4, s4, s10
+; GFX6-NEXT:    s_add_i32 s0, s0, s4
+; GFX6-NEXT:    s_ashr_i32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s4, s5, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s1, 0
+; GFX6-NEXT:    s_cselect_b32 s5, s1, 0
+; GFX6-NEXT:    s_sub_i32 s5, s8, s5
+; GFX6-NEXT:    s_cmp_lt_i32 s1, 0
+; GFX6-NEXT:    s_cselect_b32 s10, s1, 0
+; GFX6-NEXT:    s_sub_i32 s10, s9, s10
+; GFX6-NEXT:    s_cmp_gt_i32 s10, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s10, s4
+; GFX6-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX6-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX6-NEXT:    s_add_i32 s1, s1, s4
+; GFX6-NEXT:    s_ashr_i32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_lshl_b32 s4, s6, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s2, 0
+; GFX6-NEXT:    s_cselect_b32 s5, s2, 0
+; GFX6-NEXT:    s_sub_i32 s5, s8, s5
+; GFX6-NEXT:    s_cmp_lt_i32 s2, 0
+; GFX6-NEXT:    s_cselect_b32 s6, s2, 0
+; GFX6-NEXT:    s_sub_i32 s6, s9, s6
+; GFX6-NEXT:    s_cmp_gt_i32 s6, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s6, s4
+; GFX6-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX6-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX6-NEXT:    s_add_i32 s2, s2, s4
+; GFX6-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX6-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX6-NEXT:    s_lshl_b32 s4, s7, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s3, 0
+; GFX6-NEXT:    s_cselect_b32 s5, s3, 0
+; GFX6-NEXT:    s_sub_i32 s5, s8, s5
+; GFX6-NEXT:    s_cmp_lt_i32 s3, 0
+; GFX6-NEXT:    s_cselect_b32 s6, s3, 0
+; GFX6-NEXT:    s_sub_i32 s6, s9, s6
+; GFX6-NEXT:    s_cmp_gt_i32 s6, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s6, s4
+; GFX6-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX6-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX6-NEXT:    s_add_i32 s3, s3, s4
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    s_and_b32 s1, s1, s4
+; GFX6-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX6-NEXT:    s_and_b32 s0, s0, s4
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_and_b32 s1, s2, s4
+; GFX6-NEXT:    s_and_b32 s2, s3, s4
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_or_b32 s1, s1, s2
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_saddsat_v4i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_lshr_b32 s6, s2, 16
+; GFX8-NEXT:    s_lshr_b32 s7, s3, 16
+; GFX8-NEXT:    s_lshr_b32 s4, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s5, s1, 16
+; GFX8-NEXT:    s_sext_i32_i16 s10, s0
+; GFX8-NEXT:    s_sext_i32_i16 s11, 0
+; GFX8-NEXT:    s_cmp_gt_i32 s10, s11
+; GFX8-NEXT:    s_movk_i32 s8, 0x7fff
+; GFX8-NEXT:    s_cselect_b32 s12, s10, s11
+; GFX8-NEXT:    s_sub_i32 s12, s8, s12
+; GFX8-NEXT:    s_cmp_lt_i32 s10, s11
+; GFX8-NEXT:    s_mov_b32 s9, 0x8000
+; GFX8-NEXT:    s_cselect_b32 s10, s10, s11
+; GFX8-NEXT:    s_sub_i32 s10, s9, s10
+; GFX8-NEXT:    s_sext_i32_i16 s10, s10
+; GFX8-NEXT:    s_sext_i32_i16 s2, s2
+; GFX8-NEXT:    s_cmp_gt_i32 s10, s2
+; GFX8-NEXT:    s_cselect_b32 s2, s10, s2
+; GFX8-NEXT:    s_sext_i32_i16 s2, s2
+; GFX8-NEXT:    s_sext_i32_i16 s10, s12
+; GFX8-NEXT:    s_cmp_lt_i32 s2, s10
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s10
+; GFX8-NEXT:    s_add_i32 s0, s0, s2
+; GFX8-NEXT:    s_sext_i32_i16 s2, s4
+; GFX8-NEXT:    s_cmp_gt_i32 s2, s11
+; GFX8-NEXT:    s_cselect_b32 s10, s2, s11
+; GFX8-NEXT:    s_sub_i32 s10, s8, s10
+; GFX8-NEXT:    s_cmp_lt_i32 s2, s11
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s11
+; GFX8-NEXT:    s_sub_i32 s2, s9, s2
+; GFX8-NEXT:    s_sext_i32_i16 s2, s2
+; GFX8-NEXT:    s_sext_i32_i16 s6, s6
+; GFX8-NEXT:    s_cmp_gt_i32 s2, s6
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s6
+; GFX8-NEXT:    s_sext_i32_i16 s2, s2
+; GFX8-NEXT:    s_sext_i32_i16 s6, s10
+; GFX8-NEXT:    s_cmp_lt_i32 s2, s6
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s6
+; GFX8-NEXT:    s_add_i32 s4, s4, s2
+; GFX8-NEXT:    s_sext_i32_i16 s2, s1
+; GFX8-NEXT:    s_cmp_gt_i32 s2, s11
+; GFX8-NEXT:    s_cselect_b32 s6, s2, s11
+; GFX8-NEXT:    s_sub_i32 s6, s8, s6
+; GFX8-NEXT:    s_cmp_lt_i32 s2, s11
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s11
+; GFX8-NEXT:    s_sub_i32 s2, s9, s2
+; GFX8-NEXT:    s_sext_i32_i16 s2, s2
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_cmp_gt_i32 s2, s3
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX8-NEXT:    s_sext_i32_i16 s3, s6
+; GFX8-NEXT:    s_sext_i32_i16 s2, s2
+; GFX8-NEXT:    s_cmp_lt_i32 s2, s3
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX8-NEXT:    s_add_i32 s1, s1, s2
+; GFX8-NEXT:    s_sext_i32_i16 s2, s5
+; GFX8-NEXT:    s_cmp_gt_i32 s2, s11
+; GFX8-NEXT:    s_cselect_b32 s3, s2, s11
+; GFX8-NEXT:    s_sub_i32 s3, s8, s3
+; GFX8-NEXT:    s_cmp_lt_i32 s2, s11
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s11
+; GFX8-NEXT:    s_sub_i32 s2, s9, s2
+; GFX8-NEXT:    s_sext_i32_i16 s2, s2
+; GFX8-NEXT:    s_sext_i32_i16 s6, s7
+; GFX8-NEXT:    s_cmp_gt_i32 s2, s6
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s6
+; GFX8-NEXT:    s_sext_i32_i16 s2, s2
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_cmp_lt_i32 s2, s3
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX8-NEXT:    s_add_i32 s5, s5, s2
+; GFX8-NEXT:    s_bfe_u32 s2, s4, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX8-NEXT:    s_or_b32 s0, s0, s2
+; GFX8-NEXT:    s_bfe_u32 s2, s5, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX8-NEXT:    s_or_b32 s1, s1, s2
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_saddsat_v4i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, 0, 0
+; GFX9-NEXT:    s_sext_i32_i16 s9, s6
+; GFX9-NEXT:    s_sext_i32_i16 s7, s0
+; GFX9-NEXT:    s_ashr_i32 s8, s0, 16
+; GFX9-NEXT:    s_ashr_i32 s6, s6, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s7, s9
+; GFX9-NEXT:    s_cselect_b32 s10, s7, s9
+; GFX9-NEXT:    s_cmp_gt_i32 s8, s6
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    s_cselect_b32 s11, s8, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s11
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
+; GFX9-NEXT:    s_lshr_b32 s12, s10, 16
+; GFX9-NEXT:    s_lshr_b32 s11, s4, 16
+; GFX9-NEXT:    s_sub_i32 s10, s4, s10
+; GFX9-NEXT:    s_sub_i32 s12, s11, s12
+; GFX9-NEXT:    s_cmp_lt_i32 s7, s9
+; GFX9-NEXT:    s_cselect_b32 s7, s7, s9
+; GFX9-NEXT:    s_cmp_lt_i32 s8, s6
+; GFX9-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX9-NEXT:    s_cselect_b32 s8, s8, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s7, s7, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s12
+; GFX9-NEXT:    s_lshr_b32 s12, s7, 16
+; GFX9-NEXT:    s_lshr_b32 s8, s5, 16
+; GFX9-NEXT:    s_sub_i32 s7, s5, s7
+; GFX9-NEXT:    s_sub_i32 s12, s8, s12
+; GFX9-NEXT:    s_pack_ll_b32_b16 s7, s7, s12
+; GFX9-NEXT:    s_sext_i32_i16 s12, s7
+; GFX9-NEXT:    s_sext_i32_i16 s13, s2
+; GFX9-NEXT:    s_ashr_i32 s7, s7, 16
+; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s12, s13
+; GFX9-NEXT:    s_cselect_b32 s12, s12, s13
+; GFX9-NEXT:    s_cmp_gt_i32 s7, s2
+; GFX9-NEXT:    s_cselect_b32 s2, s7, s2
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s12, s2
+; GFX9-NEXT:    s_sext_i32_i16 s7, s2
+; GFX9-NEXT:    s_sext_i32_i16 s12, s10
+; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX9-NEXT:    s_ashr_i32 s10, s10, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s7, s12
+; GFX9-NEXT:    s_cselect_b32 s7, s7, s12
+; GFX9-NEXT:    s_cmp_lt_i32 s2, s10
+; GFX9-NEXT:    s_cselect_b32 s2, s2, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s7, s2
+; GFX9-NEXT:    s_lshr_b32 s7, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s10, s2, 16
+; GFX9-NEXT:    s_add_i32 s0, s0, s2
+; GFX9-NEXT:    s_add_i32 s7, s7, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s7
+; GFX9-NEXT:    s_sext_i32_i16 s2, s1
+; GFX9-NEXT:    s_ashr_i32 s7, s1, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s2, s9
+; GFX9-NEXT:    s_cselect_b32 s10, s2, s9
+; GFX9-NEXT:    s_cmp_gt_i32 s7, s6
+; GFX9-NEXT:    s_cselect_b32 s12, s7, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s12
+; GFX9-NEXT:    s_lshr_b32 s12, s10, 16
+; GFX9-NEXT:    s_sub_i32 s4, s4, s10
+; GFX9-NEXT:    s_sub_i32 s10, s11, s12
+; GFX9-NEXT:    s_cmp_lt_i32 s2, s9
+; GFX9-NEXT:    s_cselect_b32 s2, s2, s9
+; GFX9-NEXT:    s_cmp_lt_i32 s7, s6
+; GFX9-NEXT:    s_cselect_b32 s6, s7, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s6
+; GFX9-NEXT:    s_lshr_b32 s6, s2, 16
+; GFX9-NEXT:    s_sub_i32 s2, s5, s2
+; GFX9-NEXT:    s_sub_i32 s5, s8, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s5
+; GFX9-NEXT:    s_sext_i32_i16 s5, s2
+; GFX9-NEXT:    s_sext_i32_i16 s6, s3
+; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s6
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX9-NEXT:    s_cmp_gt_i32 s2, s3
+; GFX9-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s5, s2
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s10
+; GFX9-NEXT:    s_sext_i32_i16 s3, s2
+; GFX9-NEXT:    s_sext_i32_i16 s5, s4
+; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s5
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s5
+; GFX9-NEXT:    s_cmp_lt_i32 s2, s4
+; GFX9-NEXT:    s_cselect_b32 s2, s2, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s3, s2
+; GFX9-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX9-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX9-NEXT:    s_add_i32 s1, s1, s2
+; GFX9-NEXT:    s_add_i32 s3, s3, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_saddsat_v4i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, 0, 0
+; GFX10-NEXT:    s_sext_i32_i16 s5, s0
+; GFX10-NEXT:    s_sext_i32_i16 s7, s4
+; GFX10-NEXT:    s_ashr_i32 s6, s0, 16
+; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s5, s7
+; GFX10-NEXT:    s_movk_i32 s9, 0x7fff
+; GFX10-NEXT:    s_cselect_b32 s8, s5, s7
+; GFX10-NEXT:    s_cmp_gt_i32 s6, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s9
+; GFX10-NEXT:    s_cselect_b32 s10, s6, s4
+; GFX10-NEXT:    s_mov_b32 s12, 0xffff8000
+; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s10
+; GFX10-NEXT:    s_lshr_b32 s10, s9, 16
+; GFX10-NEXT:    s_lshr_b32 s11, s8, 16
+; GFX10-NEXT:    s_sub_i32 s8, s9, s8
+; GFX10-NEXT:    s_sub_i32 s11, s10, s11
+; GFX10-NEXT:    s_cmp_lt_i32 s5, s7
+; GFX10-NEXT:    s_pack_ll_b32_b16 s12, s12, s12
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s7
+; GFX10-NEXT:    s_cmp_lt_i32 s6, s4
+; GFX10-NEXT:    s_sext_i32_i16 s14, s2
+; GFX10-NEXT:    s_cselect_b32 s6, s6, s4
+; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
+; GFX10-NEXT:    s_lshr_b32 s6, s12, 16
+; GFX10-NEXT:    s_lshr_b32 s13, s5, 16
+; GFX10-NEXT:    s_sub_i32 s5, s12, s5
+; GFX10-NEXT:    s_sub_i32 s13, s6, s13
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s13
+; GFX10-NEXT:    s_sext_i32_i16 s13, s5
+; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s13, s14
+; GFX10-NEXT:    s_cselect_b32 s13, s13, s14
+; GFX10-NEXT:    s_cmp_gt_i32 s5, s2
+; GFX10-NEXT:    s_cselect_b32 s2, s5, s2
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s8, s11
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s13, s2
+; GFX10-NEXT:    s_sext_i32_i16 s11, s5
+; GFX10-NEXT:    s_sext_i32_i16 s8, s2
+; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s8, s11
+; GFX10-NEXT:    s_cselect_b32 s8, s8, s11
+; GFX10-NEXT:    s_cmp_lt_i32 s2, s5
+; GFX10-NEXT:    s_cselect_b32 s2, s2, s5
+; GFX10-NEXT:    s_lshr_b32 s5, s0, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s8, s2
+; GFX10-NEXT:    s_lshr_b32 s8, s2, 16
+; GFX10-NEXT:    s_add_i32 s0, s0, s2
+; GFX10-NEXT:    s_sext_i32_i16 s2, s1
+; GFX10-NEXT:    s_add_i32 s5, s5, s8
+; GFX10-NEXT:    s_ashr_i32 s8, s1, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s2, s7
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s5
+; GFX10-NEXT:    s_cselect_b32 s11, s2, s7
+; GFX10-NEXT:    s_cmp_gt_i32 s8, s4
+; GFX10-NEXT:    s_cselect_b32 s13, s8, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s11, s11, s13
+; GFX10-NEXT:    s_lshr_b32 s13, s11, 16
+; GFX10-NEXT:    s_sub_i32 s9, s9, s11
+; GFX10-NEXT:    s_sub_i32 s10, s10, s13
+; GFX10-NEXT:    s_cmp_lt_i32 s2, s7
+; GFX10-NEXT:    s_cselect_b32 s2, s2, s7
+; GFX10-NEXT:    s_cmp_lt_i32 s8, s4
+; GFX10-NEXT:    s_cselect_b32 s4, s8, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
+; GFX10-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX10-NEXT:    s_sub_i32 s2, s12, s2
+; GFX10-NEXT:    s_sub_i32 s4, s6, s4
+; GFX10-NEXT:    s_sext_i32_i16 s6, s3
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
+; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX10-NEXT:    s_sext_i32_i16 s4, s2
+; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s4, s6
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX10-NEXT:    s_cmp_gt_i32 s2, s3
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s9, s10
+; GFX10-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX10-NEXT:    s_sext_i32_i16 s3, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s4, s2
+; GFX10-NEXT:    s_ashr_i32 s4, s6, 16
+; GFX10-NEXT:    s_sext_i32_i16 s6, s2
+; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s6, s3
+; GFX10-NEXT:    s_cselect_b32 s3, s6, s3
+; GFX10-NEXT:    s_cmp_lt_i32 s2, s4
+; GFX10-NEXT:    s_cselect_b32 s2, s2, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s3, s2
+; GFX10-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX10-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX10-NEXT:    s_add_i32 s1, s1, s2
+; GFX10-NEXT:    s_add_i32 s3, s3, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> %lhs, <4 x i16> %rhs)
+  %cast = bitcast <4 x i16> %result to <2 x i32>
+  ret <2 x i32> %cast
+}
+
+; FIXME
+; define <5 x i16> @v_saddsat_v5i16(<5 x i16> %lhs, <5 x i16> %rhs) {
+;   %result = call <5 x i16> @llvm.sadd.sat.v5i16(<5 x i16> %lhs, <5 x i16> %rhs)
+;   ret <5 x i16> %result
+; }
+
+; define amdgpu_ps <5 x i16> @s_saddsat_v5i16(<5 x i16> inreg %lhs, <5 x i16> inreg %rhs) {
+;   %result = call <5 x i16> @llvm.sadd.sat.v5i16(<5 x i16> %lhs, <5 x i16> %rhs)
+;   ret <5 x i16> %result
+; }
+
+define <3 x float> @v_saddsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
+; GFX6-LABEL: v_saddsat_v6i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v14, 0, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GFX6-NEXT:    v_sub_i32_e32 v14, vcc, s5, v14
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    v_max_i32_e32 v12, 0, v0
+; GFX6-NEXT:    v_sub_i32_e32 v12, vcc, s4, v12
+; GFX6-NEXT:    v_max_i32_e32 v6, v14, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_min_i32_e32 v6, v6, v12
+; GFX6-NEXT:    v_min_i32_e32 v12, 0, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
+; GFX6-NEXT:    v_max_i32_e32 v7, 0, v1
+; GFX6-NEXT:    v_sub_i32_e32 v12, vcc, s5, v12
+; GFX6-NEXT:    v_sub_i32_e32 v7, vcc, s4, v7
+; GFX6-NEXT:    v_max_i32_e32 v6, v12, v6
+; GFX6-NEXT:    v_min_i32_e32 v6, v6, v7
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v8
+; GFX6-NEXT:    v_min_i32_e32 v8, 0, v2
+; GFX6-NEXT:    v_sub_i32_e32 v8, vcc, s5, v8
+; GFX6-NEXT:    v_bfrev_b32_e32 v13, -2
+; GFX6-NEXT:    v_max_i32_e32 v7, 0, v2
+; GFX6-NEXT:    v_sub_i32_e32 v7, vcc, v13, v7
+; GFX6-NEXT:    v_max_i32_e32 v6, v8, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_min_i32_e32 v6, v6, v7
+; GFX6-NEXT:    v_mov_b32_e32 v15, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v8, 0, v3
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v6
+; GFX6-NEXT:    v_max_i32_e32 v7, 0, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v9
+; GFX6-NEXT:    v_sub_i32_e32 v8, vcc, v15, v8
+; GFX6-NEXT:    v_sub_i32_e32 v7, vcc, v13, v7
+; GFX6-NEXT:    v_max_i32_e32 v6, v8, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT:    v_min_i32_e32 v6, v6, v7
+; GFX6-NEXT:    v_min_i32_e32 v8, 0, v4
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
+; GFX6-NEXT:    v_max_i32_e32 v7, 0, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v10
+; GFX6-NEXT:    v_sub_i32_e32 v8, vcc, v15, v8
+; GFX6-NEXT:    v_sub_i32_e32 v7, vcc, v13, v7
+; GFX6-NEXT:    v_max_i32_e32 v6, v8, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GFX6-NEXT:    v_min_i32_e32 v6, v6, v7
+; GFX6-NEXT:    v_min_i32_e32 v8, 0, v5
+; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; GFX6-NEXT:    v_max_i32_e32 v7, 0, v5
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v11
+; GFX6-NEXT:    v_sub_i32_e32 v8, vcc, v15, v8
+; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    v_sub_i32_e32 v7, vcc, v13, v7
+; GFX6-NEXT:    v_max_i32_e32 v6, v8, v6
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_min_i32_e32 v6, v6, v7
+; GFX6-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; GFX6-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
+; GFX6-NEXT:    v_ashrrev_i32_e32 v3, 16, v3
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v3
+; GFX6-NEXT:    v_ashrrev_i32_e32 v5, 16, v5
+; GFX6-NEXT:    v_and_b32_e32 v3, s4, v5
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_ashrrev_i32_e32 v4, 16, v4
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_v6i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s5, 0x8000
+; GFX8-NEXT:    v_min_i16_e32 v11, 0, v0
+; GFX8-NEXT:    v_sub_u16_e32 v11, s5, v11
+; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX8-NEXT:    v_max_i16_e32 v9, 0, v0
+; GFX8-NEXT:    v_min_i16_e32 v13, 0, v6
+; GFX8-NEXT:    v_sub_u16_e32 v9, s4, v9
+; GFX8-NEXT:    v_max_i16_e32 v11, v11, v3
+; GFX8-NEXT:    v_min_i16_e32 v9, v11, v9
+; GFX8-NEXT:    v_max_i16_e32 v11, 0, v6
+; GFX8-NEXT:    v_sub_u16_e32 v13, s5, v13
+; GFX8-NEXT:    v_max_i16_sdwa v3, v13, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_min_i16_e32 v13, 0, v1
+; GFX8-NEXT:    v_sub_u16_e32 v11, s4, v11
+; GFX8-NEXT:    v_min_i16_e32 v3, v3, v11
+; GFX8-NEXT:    v_max_i16_e32 v11, 0, v1
+; GFX8-NEXT:    v_sub_u16_e32 v13, s5, v13
+; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v1
+; GFX8-NEXT:    v_min_i16_e32 v14, 0, v7
+; GFX8-NEXT:    v_sub_u16_e32 v11, s4, v11
+; GFX8-NEXT:    v_max_i16_e32 v13, v13, v4
+; GFX8-NEXT:    v_min_i16_e32 v11, v13, v11
+; GFX8-NEXT:    v_max_i16_e32 v13, 0, v7
+; GFX8-NEXT:    v_sub_u16_e32 v14, s5, v14
+; GFX8-NEXT:    v_max_i16_sdwa v4, v14, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_u16_e32 v13, s4, v13
+; GFX8-NEXT:    v_mov_b32_e32 v12, 0x8000
+; GFX8-NEXT:    v_min_i16_e32 v14, 0, v2
+; GFX8-NEXT:    v_sub_u16_e32 v14, v12, v14
+; GFX8-NEXT:    v_min_i16_e32 v4, v4, v13
+; GFX8-NEXT:    v_mov_b32_e32 v10, 0x7fff
+; GFX8-NEXT:    v_max_i16_e32 v13, 0, v2
+; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
+; GFX8-NEXT:    v_sub_u16_e32 v13, v10, v13
+; GFX8-NEXT:    v_max_i16_e32 v14, v14, v5
+; GFX8-NEXT:    v_min_i16_e32 v13, v14, v13
+; GFX8-NEXT:    v_max_i16_e32 v14, 0, v8
+; GFX8-NEXT:    v_sub_u16_e32 v10, v10, v14
+; GFX8-NEXT:    v_min_i16_e32 v14, 0, v8
+; GFX8-NEXT:    v_sub_u16_e32 v12, v12, v14
+; GFX8-NEXT:    v_max_i16_sdwa v5, v12, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_add_u16_e32 v0, v0, v9
+; GFX8-NEXT:    v_add_u16_sdwa v3, v6, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX8-NEXT:    v_min_i16_e32 v5, v5, v10
+; GFX8-NEXT:    v_add_u16_e32 v1, v1, v11
+; GFX8-NEXT:    v_add_u16_sdwa v3, v7, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v3
+; GFX8-NEXT:    v_add_u16_e32 v2, v2, v13
+; GFX8-NEXT:    v_add_u16_sdwa v3, v8, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_v6i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, 0, 0
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
+; GFX9-NEXT:    v_pk_min_i16 v7, v0, s6
+; GFX9-NEXT:    v_pk_sub_i16 v7, s5, v7
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
+; GFX9-NEXT:    v_pk_max_i16 v6, v0, s6
+; GFX9-NEXT:    v_pk_sub_i16 v6, s4, v6
+; GFX9-NEXT:    v_pk_max_i16 v3, v7, v3
+; GFX9-NEXT:    v_pk_min_i16 v3, v3, v6
+; GFX9-NEXT:    v_pk_min_i16 v6, v1, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, v3
+; GFX9-NEXT:    v_pk_max_i16 v3, v1, s6
+; GFX9-NEXT:    v_pk_sub_i16 v6, s5, v6
+; GFX9-NEXT:    v_pk_sub_i16 v3, s4, v3
+; GFX9-NEXT:    v_pk_max_i16 v4, v6, v4
+; GFX9-NEXT:    v_pk_min_i16 v3, v4, v3
+; GFX9-NEXT:    v_pk_min_i16 v4, v2, s6
+; GFX9-NEXT:    v_pk_sub_i16 v4, s5, v4
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, v3
+; GFX9-NEXT:    v_pk_max_i16 v3, v2, s6
+; GFX9-NEXT:    v_pk_sub_i16 v3, s4, v3
+; GFX9-NEXT:    v_pk_max_i16 v4, v4, v5
+; GFX9-NEXT:    v_pk_min_i16 v3, v4, v3
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, v3
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_saddsat_v6i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, 0, 0
+; GFX10-NEXT:    s_mov_b32 s6, 0xffff8000
+; GFX10-NEXT:    v_pk_min_i16 v7, v0, s5
+; GFX10-NEXT:    v_pk_min_i16 v8, v1, s5
+; GFX10-NEXT:    v_pk_min_i16 v9, v2, s5
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s6
+; GFX10-NEXT:    v_pk_max_i16 v6, v0, s5
+; GFX10-NEXT:    v_pk_sub_i16 v14, s6, v7
+; GFX10-NEXT:    v_pk_sub_i16 v15, s6, v8
+; GFX10-NEXT:    v_pk_sub_i16 v19, s6, v9
+; GFX10-NEXT:    v_pk_max_i16 v10, v1, s5
+; GFX10-NEXT:    v_pk_max_i16 v11, v2, s5
+; GFX10-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX10-NEXT:    v_pk_max_i16 v3, v14, v3
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
+; GFX10-NEXT:    v_pk_max_i16 v4, v15, v4
+; GFX10-NEXT:    v_pk_sub_i16 v6, s4, v6
+; GFX10-NEXT:    v_pk_sub_i16 v7, s4, v10
+; GFX10-NEXT:    v_pk_sub_i16 v8, s4, v11
+; GFX10-NEXT:    v_pk_max_i16 v5, v19, v5
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_pk_min_i16 v3, v3, v6
+; GFX10-NEXT:    v_pk_min_i16 v4, v4, v7
+; GFX10-NEXT:    v_pk_min_i16 v5, v5, v8
+; GFX10-NEXT:    v_pk_add_u16 v0, v0, v3
+; GFX10-NEXT:    v_pk_add_u16 v1, v1, v4
+; GFX10-NEXT:    v_pk_add_u16 v2, v2, v5
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <6 x i16> @llvm.sadd.sat.v6i16(<6 x i16> %lhs, <6 x i16> %rhs)
+  %cast = bitcast <6 x i16> %result to <3 x float>
+  ret <3 x float> %cast
+}
+
+define amdgpu_ps <3 x i32> @s_saddsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inreg %rhs) {
+; GFX6-LABEL: s_saddsat_v6i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s6, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX6-NEXT:    s_brev_b32 s12, -2
+; GFX6-NEXT:    s_cselect_b32 s14, s0, 0
+; GFX6-NEXT:    s_sub_i32 s14, s12, s14
+; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX6-NEXT:    s_mov_b32 s13, 0x80000000
+; GFX6-NEXT:    s_cselect_b32 s15, s0, 0
+; GFX6-NEXT:    s_sub_i32 s15, s13, s15
+; GFX6-NEXT:    s_cmp_gt_i32 s15, s6
+; GFX6-NEXT:    s_cselect_b32 s6, s15, s6
+; GFX6-NEXT:    s_cmp_lt_i32 s6, s14
+; GFX6-NEXT:    s_cselect_b32 s6, s6, s14
+; GFX6-NEXT:    s_add_i32 s0, s0, s6
+; GFX6-NEXT:    s_ashr_i32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s7, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s1, 0
+; GFX6-NEXT:    s_cselect_b32 s7, s1, 0
+; GFX6-NEXT:    s_sub_i32 s7, s12, s7
+; GFX6-NEXT:    s_cmp_lt_i32 s1, 0
+; GFX6-NEXT:    s_cselect_b32 s14, s1, 0
+; GFX6-NEXT:    s_sub_i32 s14, s13, s14
+; GFX6-NEXT:    s_cmp_gt_i32 s14, s6
+; GFX6-NEXT:    s_cselect_b32 s6, s14, s6
+; GFX6-NEXT:    s_cmp_lt_i32 s6, s7
+; GFX6-NEXT:    s_cselect_b32 s6, s6, s7
+; GFX6-NEXT:    s_add_i32 s1, s1, s6
+; GFX6-NEXT:    s_ashr_i32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s8, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s2, 0
+; GFX6-NEXT:    s_cselect_b32 s7, s2, 0
+; GFX6-NEXT:    s_sub_i32 s7, s12, s7
+; GFX6-NEXT:    s_cmp_lt_i32 s2, 0
+; GFX6-NEXT:    s_cselect_b32 s8, s2, 0
+; GFX6-NEXT:    s_sub_i32 s8, s13, s8
+; GFX6-NEXT:    s_cmp_gt_i32 s8, s6
+; GFX6-NEXT:    s_cselect_b32 s6, s8, s6
+; GFX6-NEXT:    s_cmp_lt_i32 s6, s7
+; GFX6-NEXT:    s_cselect_b32 s6, s6, s7
+; GFX6-NEXT:    s_add_i32 s2, s2, s6
+; GFX6-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX6-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s9, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s3, 0
+; GFX6-NEXT:    s_cselect_b32 s7, s3, 0
+; GFX6-NEXT:    s_sub_i32 s7, s12, s7
+; GFX6-NEXT:    s_cmp_lt_i32 s3, 0
+; GFX6-NEXT:    s_cselect_b32 s8, s3, 0
+; GFX6-NEXT:    s_sub_i32 s8, s13, s8
+; GFX6-NEXT:    s_cmp_gt_i32 s8, s6
+; GFX6-NEXT:    s_cselect_b32 s6, s8, s6
+; GFX6-NEXT:    s_cmp_lt_i32 s6, s7
+; GFX6-NEXT:    s_cselect_b32 s6, s6, s7
+; GFX6-NEXT:    s_add_i32 s3, s3, s6
+; GFX6-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX6-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s10, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s4, 0
+; GFX6-NEXT:    s_cselect_b32 s7, s4, 0
+; GFX6-NEXT:    s_sub_i32 s7, s12, s7
+; GFX6-NEXT:    s_cmp_lt_i32 s4, 0
+; GFX6-NEXT:    s_cselect_b32 s8, s4, 0
+; GFX6-NEXT:    s_sub_i32 s8, s13, s8
+; GFX6-NEXT:    s_cmp_gt_i32 s8, s6
+; GFX6-NEXT:    s_cselect_b32 s6, s8, s6
+; GFX6-NEXT:    s_cmp_lt_i32 s6, s7
+; GFX6-NEXT:    s_cselect_b32 s6, s6, s7
+; GFX6-NEXT:    s_add_i32 s4, s4, s6
+; GFX6-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX6-NEXT:    s_lshl_b32 s5, s5, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s11, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s5, 0
+; GFX6-NEXT:    s_cselect_b32 s7, s5, 0
+; GFX6-NEXT:    s_sub_i32 s7, s12, s7
+; GFX6-NEXT:    s_cmp_lt_i32 s5, 0
+; GFX6-NEXT:    s_cselect_b32 s8, s5, 0
+; GFX6-NEXT:    s_sub_i32 s8, s13, s8
+; GFX6-NEXT:    s_cmp_gt_i32 s8, s6
+; GFX6-NEXT:    s_cselect_b32 s6, s8, s6
+; GFX6-NEXT:    s_cmp_lt_i32 s6, s7
+; GFX6-NEXT:    s_cselect_b32 s6, s6, s7
+; GFX6-NEXT:    s_add_i32 s5, s5, s6
+; GFX6-NEXT:    s_mov_b32 s6, 0xffff
+; GFX6-NEXT:    s_and_b32 s1, s1, s6
+; GFX6-NEXT:    s_and_b32 s0, s0, s6
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_and_b32 s1, s2, s6
+; GFX6-NEXT:    s_and_b32 s2, s3, s6
+; GFX6-NEXT:    s_ashr_i32 s5, s5, 16
+; GFX6-NEXT:    s_and_b32 s3, s5, s6
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_or_b32 s1, s1, s2
+; GFX6-NEXT:    s_and_b32 s2, s4, s6
+; GFX6-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX6-NEXT:    s_or_b32 s2, s2, s3
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_saddsat_v6i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_lshr_b32 s9, s3, 16
+; GFX8-NEXT:    s_lshr_b32 s10, s4, 16
+; GFX8-NEXT:    s_lshr_b32 s11, s5, 16
+; GFX8-NEXT:    s_lshr_b32 s6, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s7, s1, 16
+; GFX8-NEXT:    s_lshr_b32 s8, s2, 16
+; GFX8-NEXT:    s_sext_i32_i16 s14, s0
+; GFX8-NEXT:    s_sext_i32_i16 s15, 0
+; GFX8-NEXT:    s_cmp_gt_i32 s14, s15
+; GFX8-NEXT:    s_movk_i32 s12, 0x7fff
+; GFX8-NEXT:    s_cselect_b32 s16, s14, s15
+; GFX8-NEXT:    s_sub_i32 s16, s12, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s14, s15
+; GFX8-NEXT:    s_mov_b32 s13, 0x8000
+; GFX8-NEXT:    s_cselect_b32 s14, s14, s15
+; GFX8-NEXT:    s_sub_i32 s14, s13, s14
+; GFX8-NEXT:    s_sext_i32_i16 s14, s14
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_cmp_gt_i32 s14, s3
+; GFX8-NEXT:    s_cselect_b32 s3, s14, s3
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_sext_i32_i16 s14, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s14
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s14
+; GFX8-NEXT:    s_add_i32 s0, s0, s3
+; GFX8-NEXT:    s_sext_i32_i16 s3, s6
+; GFX8-NEXT:    s_cmp_gt_i32 s3, s15
+; GFX8-NEXT:    s_cselect_b32 s14, s3, s15
+; GFX8-NEXT:    s_sub_i32 s14, s12, s14
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s15
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s15
+; GFX8-NEXT:    s_sub_i32 s3, s13, s3
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_sext_i32_i16 s9, s9
+; GFX8-NEXT:    s_cmp_gt_i32 s3, s9
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s9
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_sext_i32_i16 s9, s14
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s9
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s9
+; GFX8-NEXT:    s_add_i32 s6, s6, s3
+; GFX8-NEXT:    s_sext_i32_i16 s3, s1
+; GFX8-NEXT:    s_cmp_gt_i32 s3, s15
+; GFX8-NEXT:    s_cselect_b32 s9, s3, s15
+; GFX8-NEXT:    s_sub_i32 s9, s12, s9
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s15
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s15
+; GFX8-NEXT:    s_sub_i32 s3, s13, s3
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_cmp_gt_i32 s3, s4
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX8-NEXT:    s_sext_i32_i16 s4, s9
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX8-NEXT:    s_add_i32 s1, s1, s3
+; GFX8-NEXT:    s_sext_i32_i16 s3, s7
+; GFX8-NEXT:    s_cmp_gt_i32 s3, s15
+; GFX8-NEXT:    s_cselect_b32 s4, s3, s15
+; GFX8-NEXT:    s_sub_i32 s4, s12, s4
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s15
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s15
+; GFX8-NEXT:    s_sub_i32 s3, s13, s3
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_sext_i32_i16 s9, s10
+; GFX8-NEXT:    s_cmp_gt_i32 s3, s9
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s9
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX8-NEXT:    s_add_i32 s7, s7, s3
+; GFX8-NEXT:    s_sext_i32_i16 s3, s2
+; GFX8-NEXT:    s_cmp_gt_i32 s3, s15
+; GFX8-NEXT:    s_cselect_b32 s4, s3, s15
+; GFX8-NEXT:    s_sub_i32 s4, s12, s4
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s15
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s15
+; GFX8-NEXT:    s_sub_i32 s3, s13, s3
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_sext_i32_i16 s5, s5
+; GFX8-NEXT:    s_cmp_gt_i32 s3, s5
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s5
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX8-NEXT:    s_add_i32 s2, s2, s3
+; GFX8-NEXT:    s_sext_i32_i16 s3, s8
+; GFX8-NEXT:    s_cmp_gt_i32 s3, s15
+; GFX8-NEXT:    s_cselect_b32 s4, s3, s15
+; GFX8-NEXT:    s_sub_i32 s4, s12, s4
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s15
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s15
+; GFX8-NEXT:    s_sub_i32 s3, s13, s3
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_sext_i32_i16 s5, s11
+; GFX8-NEXT:    s_cmp_gt_i32 s3, s5
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s5
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX8-NEXT:    s_add_i32 s8, s8, s3
+; GFX8-NEXT:    s_bfe_u32 s3, s6, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX8-NEXT:    s_or_b32 s0, s0, s3
+; GFX8-NEXT:    s_bfe_u32 s3, s7, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX8-NEXT:    s_or_b32 s1, s1, s3
+; GFX8-NEXT:    s_bfe_u32 s3, s8, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX8-NEXT:    s_or_b32 s2, s2, s3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_saddsat_v6i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_pack_ll_b32_b16 s8, 0, 0
+; GFX9-NEXT:    s_sext_i32_i16 s11, s8
+; GFX9-NEXT:    s_sext_i32_i16 s9, s0
+; GFX9-NEXT:    s_ashr_i32 s10, s0, 16
+; GFX9-NEXT:    s_ashr_i32 s8, s8, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s9, s11
+; GFX9-NEXT:    s_cselect_b32 s12, s9, s11
+; GFX9-NEXT:    s_cmp_gt_i32 s10, s8
+; GFX9-NEXT:    s_movk_i32 s6, 0x7fff
+; GFX9-NEXT:    s_cselect_b32 s13, s10, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s12, s12, s13
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s6
+; GFX9-NEXT:    s_lshr_b32 s14, s12, 16
+; GFX9-NEXT:    s_lshr_b32 s13, s6, 16
+; GFX9-NEXT:    s_sub_i32 s12, s6, s12
+; GFX9-NEXT:    s_sub_i32 s14, s13, s14
+; GFX9-NEXT:    s_cmp_lt_i32 s9, s11
+; GFX9-NEXT:    s_cselect_b32 s9, s9, s11
+; GFX9-NEXT:    s_cmp_lt_i32 s10, s8
+; GFX9-NEXT:    s_mov_b32 s7, 0xffff8000
+; GFX9-NEXT:    s_cselect_b32 s10, s10, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s7, s7, s7
+; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s12, s12, s14
+; GFX9-NEXT:    s_lshr_b32 s14, s9, 16
+; GFX9-NEXT:    s_lshr_b32 s10, s7, 16
+; GFX9-NEXT:    s_sub_i32 s9, s7, s9
+; GFX9-NEXT:    s_sub_i32 s14, s10, s14
+; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s14
+; GFX9-NEXT:    s_sext_i32_i16 s14, s9
+; GFX9-NEXT:    s_sext_i32_i16 s15, s3
+; GFX9-NEXT:    s_ashr_i32 s9, s9, 16
+; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s14, s15
+; GFX9-NEXT:    s_cselect_b32 s14, s14, s15
+; GFX9-NEXT:    s_cmp_gt_i32 s9, s3
+; GFX9-NEXT:    s_cselect_b32 s3, s9, s3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s14, s3
+; GFX9-NEXT:    s_sext_i32_i16 s9, s3
+; GFX9-NEXT:    s_sext_i32_i16 s14, s12
+; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX9-NEXT:    s_ashr_i32 s12, s12, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s9, s14
+; GFX9-NEXT:    s_cselect_b32 s9, s9, s14
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s12
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s12
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s9, s3
+; GFX9-NEXT:    s_lshr_b32 s9, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s12, s3, 16
+; GFX9-NEXT:    s_add_i32 s0, s0, s3
+; GFX9-NEXT:    s_add_i32 s9, s9, s12
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s9
+; GFX9-NEXT:    s_sext_i32_i16 s3, s1
+; GFX9-NEXT:    s_ashr_i32 s9, s1, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s11
+; GFX9-NEXT:    s_cselect_b32 s12, s3, s11
+; GFX9-NEXT:    s_cmp_gt_i32 s9, s8
+; GFX9-NEXT:    s_cselect_b32 s14, s9, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s12, s12, s14
+; GFX9-NEXT:    s_lshr_b32 s14, s12, 16
+; GFX9-NEXT:    s_sub_i32 s12, s6, s12
+; GFX9-NEXT:    s_sub_i32 s14, s13, s14
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s11
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s11
+; GFX9-NEXT:    s_cmp_lt_i32 s9, s8
+; GFX9-NEXT:    s_cselect_b32 s9, s9, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s9
+; GFX9-NEXT:    s_lshr_b32 s9, s3, 16
+; GFX9-NEXT:    s_sub_i32 s3, s7, s3
+; GFX9-NEXT:    s_sub_i32 s9, s10, s9
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s9
+; GFX9-NEXT:    s_pack_ll_b32_b16 s12, s12, s14
+; GFX9-NEXT:    s_sext_i32_i16 s9, s3
+; GFX9-NEXT:    s_sext_i32_i16 s14, s4
+; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s9, s14
+; GFX9-NEXT:    s_cselect_b32 s9, s9, s14
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s4
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s9, s3
+; GFX9-NEXT:    s_sext_i32_i16 s4, s3
+; GFX9-NEXT:    s_sext_i32_i16 s9, s12
+; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX9-NEXT:    s_ashr_i32 s12, s12, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s9
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s9
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s12
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s12
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s4, s3
+; GFX9-NEXT:    s_lshr_b32 s4, s1, 16
+; GFX9-NEXT:    s_lshr_b32 s9, s3, 16
+; GFX9-NEXT:    s_add_i32 s1, s1, s3
+; GFX9-NEXT:    s_add_i32 s4, s4, s9
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
+; GFX9-NEXT:    s_sext_i32_i16 s3, s2
+; GFX9-NEXT:    s_ashr_i32 s4, s2, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s11
+; GFX9-NEXT:    s_cselect_b32 s9, s3, s11
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s8
+; GFX9-NEXT:    s_cselect_b32 s12, s4, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s12
+; GFX9-NEXT:    s_lshr_b32 s12, s9, 16
+; GFX9-NEXT:    s_sub_i32 s6, s6, s9
+; GFX9-NEXT:    s_sub_i32 s9, s13, s12
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s11
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s11
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s8
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
+; GFX9-NEXT:    s_lshr_b32 s4, s3, 16
+; GFX9-NEXT:    s_sub_i32 s3, s7, s3
+; GFX9-NEXT:    s_sub_i32 s4, s10, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
+; GFX9-NEXT:    s_sext_i32_i16 s4, s3
+; GFX9-NEXT:    s_sext_i32_i16 s7, s5
+; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX9-NEXT:    s_ashr_i32 s5, s5, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s7
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s7
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s5
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s4, s3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s9
+; GFX9-NEXT:    s_sext_i32_i16 s4, s3
+; GFX9-NEXT:    s_sext_i32_i16 s5, s6
+; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX9-NEXT:    s_ashr_i32 s6, s6, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s6
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s4, s3
+; GFX9-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX9-NEXT:    s_lshr_b32 s5, s3, 16
+; GFX9-NEXT:    s_add_i32 s2, s2, s3
+; GFX9-NEXT:    s_add_i32 s4, s4, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_saddsat_v6i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, 0, 0
+; GFX10-NEXT:    s_sext_i32_i16 s7, s0
+; GFX10-NEXT:    s_sext_i32_i16 s9, s6
+; GFX10-NEXT:    s_ashr_i32 s8, s0, 16
+; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s7, s9
+; GFX10-NEXT:    s_movk_i32 s11, 0x7fff
+; GFX10-NEXT:    s_cselect_b32 s10, s7, s9
+; GFX10-NEXT:    s_cmp_gt_i32 s8, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s11, s11, s11
+; GFX10-NEXT:    s_cselect_b32 s12, s8, s6
+; GFX10-NEXT:    s_mov_b32 s14, 0xffff8000
+; GFX10-NEXT:    s_pack_ll_b32_b16 s10, s10, s12
+; GFX10-NEXT:    s_lshr_b32 s12, s11, 16
+; GFX10-NEXT:    s_lshr_b32 s13, s10, 16
+; GFX10-NEXT:    s_sub_i32 s10, s11, s10
+; GFX10-NEXT:    s_sub_i32 s13, s12, s13
+; GFX10-NEXT:    s_cmp_lt_i32 s7, s9
+; GFX10-NEXT:    s_pack_ll_b32_b16 s14, s14, s14
+; GFX10-NEXT:    s_cselect_b32 s7, s7, s9
+; GFX10-NEXT:    s_cmp_lt_i32 s8, s6
+; GFX10-NEXT:    s_sext_i32_i16 s16, s3
+; GFX10-NEXT:    s_cselect_b32 s8, s8, s6
+; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s7, s8
+; GFX10-NEXT:    s_lshr_b32 s8, s14, 16
+; GFX10-NEXT:    s_lshr_b32 s15, s7, 16
+; GFX10-NEXT:    s_sub_i32 s7, s14, s7
+; GFX10-NEXT:    s_sub_i32 s15, s8, s15
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s7, s15
+; GFX10-NEXT:    s_sext_i32_i16 s15, s7
+; GFX10-NEXT:    s_ashr_i32 s7, s7, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s15, s16
+; GFX10-NEXT:    s_cselect_b32 s15, s15, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s7, s3
+; GFX10-NEXT:    s_sext_i32_i16 s16, s4
+; GFX10-NEXT:    s_cselect_b32 s3, s7, s3
+; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s10, s13
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s15, s3
+; GFX10-NEXT:    s_sext_i32_i16 s13, s7
+; GFX10-NEXT:    s_sext_i32_i16 s10, s3
+; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX10-NEXT:    s_ashr_i32 s7, s7, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s10, s13
+; GFX10-NEXT:    s_cselect_b32 s10, s10, s13
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s7
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s7
+; GFX10-NEXT:    s_lshr_b32 s7, s0, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s10, s3
+; GFX10-NEXT:    s_lshr_b32 s10, s3, 16
+; GFX10-NEXT:    s_add_i32 s0, s0, s3
+; GFX10-NEXT:    s_sext_i32_i16 s3, s1
+; GFX10-NEXT:    s_add_i32 s7, s7, s10
+; GFX10-NEXT:    s_ashr_i32 s10, s1, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s3, s9
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s7
+; GFX10-NEXT:    s_cselect_b32 s13, s3, s9
+; GFX10-NEXT:    s_cmp_gt_i32 s10, s6
+; GFX10-NEXT:    s_cselect_b32 s15, s10, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s13, s13, s15
+; GFX10-NEXT:    s_lshr_b32 s15, s13, 16
+; GFX10-NEXT:    s_sub_i32 s13, s11, s13
+; GFX10-NEXT:    s_sub_i32 s15, s12, s15
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s9
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s9
+; GFX10-NEXT:    s_cmp_lt_i32 s10, s6
+; GFX10-NEXT:    s_cselect_b32 s10, s10, s6
+; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s10
+; GFX10-NEXT:    s_lshr_b32 s10, s3, 16
+; GFX10-NEXT:    s_sub_i32 s3, s14, s3
+; GFX10-NEXT:    s_sub_i32 s10, s8, s10
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s10
+; GFX10-NEXT:    s_sext_i32_i16 s10, s3
+; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s10, s16
+; GFX10-NEXT:    s_cselect_b32 s10, s10, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s3, s4
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s13, s15
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s10, s3
+; GFX10-NEXT:    s_sext_i32_i16 s13, s4
+; GFX10-NEXT:    s_sext_i32_i16 s10, s3
+; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s10, s13
+; GFX10-NEXT:    s_cselect_b32 s10, s10, s13
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX10-NEXT:    s_sext_i32_i16 s4, s2
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s10, s3
+; GFX10-NEXT:    s_lshr_b32 s10, s1, 16
+; GFX10-NEXT:    s_lshr_b32 s13, s3, 16
+; GFX10-NEXT:    s_add_i32 s1, s1, s3
+; GFX10-NEXT:    s_add_i32 s10, s10, s13
+; GFX10-NEXT:    s_ashr_i32 s3, s2, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s4, s9
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s10
+; GFX10-NEXT:    s_cselect_b32 s13, s4, s9
+; GFX10-NEXT:    s_cmp_gt_i32 s3, s6
+; GFX10-NEXT:    s_cselect_b32 s15, s3, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s13, s13, s15
+; GFX10-NEXT:    s_lshr_b32 s15, s13, 16
+; GFX10-NEXT:    s_sub_i32 s11, s11, s13
+; GFX10-NEXT:    s_sub_i32 s12, s12, s15
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s9
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s9
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s6
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s6
+; GFX10-NEXT:    s_sext_i32_i16 s6, s5
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s4, s3
+; GFX10-NEXT:    s_lshr_b32 s4, s3, 16
+; GFX10-NEXT:    s_sub_i32 s3, s14, s3
+; GFX10-NEXT:    s_sub_i32 s4, s8, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
+; GFX10-NEXT:    s_ashr_i32 s4, s5, 16
+; GFX10-NEXT:    s_sext_i32_i16 s5, s3
+; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s5, s6
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX10-NEXT:    s_cmp_gt_i32 s3, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s11, s12
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX10-NEXT:    s_sext_i32_i16 s4, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s5, s3
+; GFX10-NEXT:    s_ashr_i32 s5, s6, 16
+; GFX10-NEXT:    s_sext_i32_i16 s6, s3
+; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s6, s4
+; GFX10-NEXT:    s_cselect_b32 s4, s6, s4
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s5
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s5
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s4, s3
+; GFX10-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX10-NEXT:    s_lshr_b32 s5, s3, 16
+; GFX10-NEXT:    s_add_i32 s2, s2, s3
+; GFX10-NEXT:    s_add_i32 s4, s4, s5
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <6 x i16> @llvm.sadd.sat.v6i16(<6 x i16> %lhs, <6 x i16> %rhs)
+  %cast = bitcast <6 x i16> %result to <3 x i32>
+  ret <3 x i32> %cast
+}
+
+define <4 x float> @v_saddsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
+; GFX6-LABEL: v_saddsat_v8i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v18, 0, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GFX6-NEXT:    v_sub_i32_e32 v18, vcc, s5, v18
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    v_max_i32_e32 v16, 0, v0
+; GFX6-NEXT:    v_sub_i32_e32 v16, vcc, s4, v16
+; GFX6-NEXT:    v_max_i32_e32 v8, v18, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_min_i32_e32 v8, v8, v16
+; GFX6-NEXT:    v_min_i32_e32 v16, 0, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v9
+; GFX6-NEXT:    v_max_i32_e32 v9, 0, v1
+; GFX6-NEXT:    v_sub_i32_e32 v16, vcc, s5, v16
+; GFX6-NEXT:    v_sub_i32_e32 v9, vcc, s4, v9
+; GFX6-NEXT:    v_max_i32_e32 v8, v16, v8
+; GFX6-NEXT:    v_min_i32_e32 v8, v8, v9
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v10
+; GFX6-NEXT:    v_min_i32_e32 v10, 0, v2
+; GFX6-NEXT:    v_sub_i32_e32 v10, vcc, s5, v10
+; GFX6-NEXT:    v_bfrev_b32_e32 v17, -2
+; GFX6-NEXT:    v_max_i32_e32 v9, 0, v2
+; GFX6-NEXT:    v_sub_i32_e32 v9, vcc, v17, v9
+; GFX6-NEXT:    v_max_i32_e32 v8, v10, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_min_i32_e32 v8, v8, v9
+; GFX6-NEXT:    v_mov_b32_e32 v19, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v10, 0, v3
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v8
+; GFX6-NEXT:    v_max_i32_e32 v9, 0, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v11
+; GFX6-NEXT:    v_sub_i32_e32 v10, vcc, v19, v10
+; GFX6-NEXT:    v_sub_i32_e32 v9, vcc, v17, v9
+; GFX6-NEXT:    v_max_i32_e32 v8, v10, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT:    v_min_i32_e32 v8, v8, v9
+; GFX6-NEXT:    v_min_i32_e32 v10, 0, v4
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v8
+; GFX6-NEXT:    v_max_i32_e32 v9, 0, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v12
+; GFX6-NEXT:    v_sub_i32_e32 v10, vcc, v19, v10
+; GFX6-NEXT:    v_sub_i32_e32 v9, vcc, v17, v9
+; GFX6-NEXT:    v_max_i32_e32 v8, v10, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GFX6-NEXT:    v_min_i32_e32 v8, v8, v9
+; GFX6-NEXT:    v_min_i32_e32 v10, 0, v5
+; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GFX6-NEXT:    v_max_i32_e32 v9, 0, v5
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v13
+; GFX6-NEXT:    v_sub_i32_e32 v10, vcc, v19, v10
+; GFX6-NEXT:    v_sub_i32_e32 v9, vcc, v17, v9
+; GFX6-NEXT:    v_max_i32_e32 v8, v10, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GFX6-NEXT:    v_min_i32_e32 v8, v8, v9
+; GFX6-NEXT:    v_min_i32_e32 v10, 0, v6
+; GFX6-NEXT:    v_add_i32_e32 v5, vcc, v5, v8
+; GFX6-NEXT:    v_max_i32_e32 v9, 0, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v14
+; GFX6-NEXT:    v_sub_i32_e32 v10, vcc, v19, v10
+; GFX6-NEXT:    v_sub_i32_e32 v9, vcc, v17, v9
+; GFX6-NEXT:    v_max_i32_e32 v8, v10, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GFX6-NEXT:    v_min_i32_e32 v8, v8, v9
+; GFX6-NEXT:    v_min_i32_e32 v10, 0, v7
+; GFX6-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; GFX6-NEXT:    v_max_i32_e32 v9, 0, v7
+; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v15
+; GFX6-NEXT:    v_sub_i32_e32 v10, vcc, v19, v10
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_sub_i32_e32 v9, vcc, v17, v9
+; GFX6-NEXT:    v_max_i32_e32 v8, v10, v8
+; GFX6-NEXT:    v_min_i32_e32 v8, v8, v9
+; GFX6-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
+; GFX6-NEXT:    v_ashrrev_i32_e32 v3, 16, v3
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v3
+; GFX6-NEXT:    v_ashrrev_i32_e32 v5, 16, v5
+; GFX6-NEXT:    v_and_b32_e32 v3, s4, v5
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_ashrrev_i32_e32 v4, 16, v4
+; GFX6-NEXT:    v_ashrrev_i32_e32 v7, 16, v7
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v4
+; GFX6-NEXT:    v_and_b32_e32 v4, s4, v7
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_ashrrev_i32_e32 v6, 16, v6
+; GFX6-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX6-NEXT:    v_and_b32_e32 v3, s4, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT:    v_or_b32_e32 v3, v3, v4
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_v8i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s5, 0x8000
+; GFX8-NEXT:    v_min_i16_e32 v14, 0, v0
+; GFX8-NEXT:    v_sub_u16_e32 v14, s5, v14
+; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 16, v0
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX8-NEXT:    v_max_i16_e32 v12, 0, v0
+; GFX8-NEXT:    v_min_i16_e32 v16, 0, v8
+; GFX8-NEXT:    v_sub_u16_e32 v12, s4, v12
+; GFX8-NEXT:    v_max_i16_e32 v14, v14, v4
+; GFX8-NEXT:    v_min_i16_e32 v12, v14, v12
+; GFX8-NEXT:    v_max_i16_e32 v14, 0, v8
+; GFX8-NEXT:    v_sub_u16_e32 v16, s5, v16
+; GFX8-NEXT:    v_max_i16_sdwa v4, v16, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_min_i16_e32 v16, 0, v1
+; GFX8-NEXT:    v_sub_u16_e32 v14, s4, v14
+; GFX8-NEXT:    v_min_i16_e32 v4, v4, v14
+; GFX8-NEXT:    v_max_i16_e32 v14, 0, v1
+; GFX8-NEXT:    v_sub_u16_e32 v16, s5, v16
+; GFX8-NEXT:    v_lshrrev_b32_e32 v9, 16, v1
+; GFX8-NEXT:    v_min_i16_e32 v17, 0, v9
+; GFX8-NEXT:    v_sub_u16_e32 v14, s4, v14
+; GFX8-NEXT:    v_max_i16_e32 v16, v16, v5
+; GFX8-NEXT:    v_min_i16_e32 v14, v16, v14
+; GFX8-NEXT:    v_max_i16_e32 v16, 0, v9
+; GFX8-NEXT:    v_sub_u16_e32 v17, s5, v17
+; GFX8-NEXT:    v_max_i16_sdwa v5, v17, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_u16_e32 v16, s4, v16
+; GFX8-NEXT:    v_mov_b32_e32 v15, 0x8000
+; GFX8-NEXT:    v_min_i16_e32 v17, 0, v2
+; GFX8-NEXT:    v_sub_u16_e32 v17, v15, v17
+; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
+; GFX8-NEXT:    v_min_i16_e32 v5, v5, v16
+; GFX8-NEXT:    v_mov_b32_e32 v13, 0x7fff
+; GFX8-NEXT:    v_max_i16_e32 v16, 0, v2
+; GFX8-NEXT:    v_min_i16_e32 v18, 0, v10
+; GFX8-NEXT:    v_sub_u16_e32 v16, v13, v16
+; GFX8-NEXT:    v_max_i16_e32 v17, v17, v6
+; GFX8-NEXT:    v_min_i16_e32 v16, v17, v16
+; GFX8-NEXT:    v_max_i16_e32 v17, 0, v10
+; GFX8-NEXT:    v_sub_u16_e32 v18, v15, v18
+; GFX8-NEXT:    v_max_i16_sdwa v6, v18, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_min_i16_e32 v18, 0, v3
+; GFX8-NEXT:    v_sub_u16_e32 v17, v13, v17
+; GFX8-NEXT:    v_sub_u16_e32 v18, v15, v18
+; GFX8-NEXT:    v_min_i16_e32 v6, v6, v17
+; GFX8-NEXT:    v_max_i16_e32 v17, 0, v3
+; GFX8-NEXT:    v_lshrrev_b32_e32 v11, 16, v3
+; GFX8-NEXT:    v_sub_u16_e32 v17, v13, v17
+; GFX8-NEXT:    v_max_i16_e32 v18, v18, v7
+; GFX8-NEXT:    v_min_i16_e32 v17, v18, v17
+; GFX8-NEXT:    v_max_i16_e32 v18, 0, v11
+; GFX8-NEXT:    v_sub_u16_e32 v13, v13, v18
+; GFX8-NEXT:    v_min_i16_e32 v18, 0, v11
+; GFX8-NEXT:    v_sub_u16_e32 v15, v15, v18
+; GFX8-NEXT:    v_add_u16_e32 v0, v0, v12
+; GFX8-NEXT:    v_add_u16_sdwa v4, v8, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_i16_sdwa v7, v15, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT:    v_add_u16_e32 v1, v1, v14
+; GFX8-NEXT:    v_add_u16_sdwa v4, v9, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v4
+; GFX8-NEXT:    v_min_i16_e32 v7, v7, v13
+; GFX8-NEXT:    v_add_u16_e32 v2, v2, v16
+; GFX8-NEXT:    v_add_u16_sdwa v4, v10, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v2, v2, v4
+; GFX8-NEXT:    v_add_u16_e32 v3, v3, v17
+; GFX8-NEXT:    v_add_u16_sdwa v4, v11, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v3, v3, v4
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_v8i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, 0, 0
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
+; GFX9-NEXT:    v_pk_min_i16 v9, v0, s6
+; GFX9-NEXT:    v_pk_sub_i16 v9, s5, v9
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
+; GFX9-NEXT:    v_pk_max_i16 v8, v0, s6
+; GFX9-NEXT:    v_pk_sub_i16 v8, s4, v8
+; GFX9-NEXT:    v_pk_max_i16 v4, v9, v4
+; GFX9-NEXT:    v_pk_min_i16 v4, v4, v8
+; GFX9-NEXT:    v_pk_min_i16 v8, v1, s6
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, v4
+; GFX9-NEXT:    v_pk_max_i16 v4, v1, s6
+; GFX9-NEXT:    v_pk_sub_i16 v8, s5, v8
+; GFX9-NEXT:    v_pk_sub_i16 v4, s4, v4
+; GFX9-NEXT:    v_pk_max_i16 v5, v8, v5
+; GFX9-NEXT:    v_pk_min_i16 v4, v5, v4
+; GFX9-NEXT:    v_pk_min_i16 v5, v2, s6
+; GFX9-NEXT:    v_pk_sub_i16 v5, s5, v5
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, v4
+; GFX9-NEXT:    v_pk_max_i16 v4, v2, s6
+; GFX9-NEXT:    v_pk_sub_i16 v4, s4, v4
+; GFX9-NEXT:    v_pk_max_i16 v5, v5, v6
+; GFX9-NEXT:    v_pk_min_i16 v4, v5, v4
+; GFX9-NEXT:    v_pk_min_i16 v5, v3, s6
+; GFX9-NEXT:    v_pk_sub_i16 v5, s5, v5
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, v4
+; GFX9-NEXT:    v_pk_max_i16 v4, v3, s6
+; GFX9-NEXT:    v_pk_sub_i16 v4, s4, v4
+; GFX9-NEXT:    v_pk_max_i16 v5, v5, v7
+; GFX9-NEXT:    v_pk_min_i16 v4, v5, v4
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, v4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_saddsat_v8i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, 0, 0
+; GFX10-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX10-NEXT:    v_pk_min_i16 v8, v0, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
+; GFX10-NEXT:    v_pk_min_i16 v11, v1, s4
+; GFX10-NEXT:    v_pk_min_i16 v12, v3, s4
+; GFX10-NEXT:    v_pk_max_i16 v9, v0, s4
+; GFX10-NEXT:    v_pk_sub_i16 v15, s5, v8
+; GFX10-NEXT:    v_pk_min_i16 v8, v2, s4
+; GFX10-NEXT:    v_pk_sub_i16 v11, s5, v11
+; GFX10-NEXT:    v_pk_sub_i16 v12, s5, v12
+; GFX10-NEXT:    v_pk_max_i16 v10, v1, s4
+; GFX10-NEXT:    v_pk_max_i16 v13, v2, s4
+; GFX10-NEXT:    v_pk_sub_i16 v8, s5, v8
+; GFX10-NEXT:    v_pk_max_i16 v14, v3, s4
+; GFX10-NEXT:    s_movk_i32 s6, 0x7fff
+; GFX10-NEXT:    v_pk_max_i16 v4, v15, v4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s6
+; GFX10-NEXT:    v_pk_max_i16 v5, v11, v5
+; GFX10-NEXT:    v_pk_sub_i16 v9, s6, v9
+; GFX10-NEXT:    v_pk_sub_i16 v10, s6, v10
+; GFX10-NEXT:    v_pk_max_i16 v6, v8, v6
+; GFX10-NEXT:    v_pk_sub_i16 v11, s6, v13
+; GFX10-NEXT:    v_pk_sub_i16 v8, s6, v14
+; GFX10-NEXT:    v_pk_max_i16 v7, v12, v7
+; GFX10-NEXT:    v_pk_min_i16 v15, v4, v9
+; GFX10-NEXT:    v_pk_min_i16 v19, v5, v10
+; GFX10-NEXT:    v_pk_min_i16 v11, v6, v11
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_pk_min_i16 v6, v7, v8
+; GFX10-NEXT:    v_pk_add_u16 v0, v0, v15
+; GFX10-NEXT:    v_pk_add_u16 v1, v1, v19
+; GFX10-NEXT:    v_pk_add_u16 v2, v2, v11
+; GFX10-NEXT:    v_pk_add_u16 v3, v3, v6
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %lhs, <8 x i16> %rhs)
+  %cast = bitcast <8 x i16> %result to <4 x float>
+  ret <4 x float> %cast
+}
+
+define amdgpu_ps <4 x i32> @s_saddsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inreg %rhs) {
+; GFX6-LABEL: s_saddsat_v8i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s8, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s0, 0
+; GFX6-NEXT:    s_brev_b32 s16, -2
+; GFX6-NEXT:    s_cselect_b32 s18, s0, 0
+; GFX6-NEXT:    s_sub_i32 s18, s16, s18
+; GFX6-NEXT:    s_cmp_lt_i32 s0, 0
+; GFX6-NEXT:    s_mov_b32 s17, 0x80000000
+; GFX6-NEXT:    s_cselect_b32 s19, s0, 0
+; GFX6-NEXT:    s_sub_i32 s19, s17, s19
+; GFX6-NEXT:    s_cmp_gt_i32 s19, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s19, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s8, s18
+; GFX6-NEXT:    s_cselect_b32 s8, s8, s18
+; GFX6-NEXT:    s_add_i32 s0, s0, s8
+; GFX6-NEXT:    s_ashr_i32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s9, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s1, 0
+; GFX6-NEXT:    s_cselect_b32 s9, s1, 0
+; GFX6-NEXT:    s_sub_i32 s9, s16, s9
+; GFX6-NEXT:    s_cmp_lt_i32 s1, 0
+; GFX6-NEXT:    s_cselect_b32 s18, s1, 0
+; GFX6-NEXT:    s_sub_i32 s18, s17, s18
+; GFX6-NEXT:    s_cmp_gt_i32 s18, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s18, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s8, s9
+; GFX6-NEXT:    s_cselect_b32 s8, s8, s9
+; GFX6-NEXT:    s_add_i32 s1, s1, s8
+; GFX6-NEXT:    s_ashr_i32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s10, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s2, 0
+; GFX6-NEXT:    s_cselect_b32 s9, s2, 0
+; GFX6-NEXT:    s_sub_i32 s9, s16, s9
+; GFX6-NEXT:    s_cmp_lt_i32 s2, 0
+; GFX6-NEXT:    s_cselect_b32 s10, s2, 0
+; GFX6-NEXT:    s_sub_i32 s10, s17, s10
+; GFX6-NEXT:    s_cmp_gt_i32 s10, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s10, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s8, s9
+; GFX6-NEXT:    s_cselect_b32 s8, s8, s9
+; GFX6-NEXT:    s_add_i32 s2, s2, s8
+; GFX6-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX6-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s11, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s3, 0
+; GFX6-NEXT:    s_cselect_b32 s9, s3, 0
+; GFX6-NEXT:    s_sub_i32 s9, s16, s9
+; GFX6-NEXT:    s_cmp_lt_i32 s3, 0
+; GFX6-NEXT:    s_cselect_b32 s10, s3, 0
+; GFX6-NEXT:    s_sub_i32 s10, s17, s10
+; GFX6-NEXT:    s_cmp_gt_i32 s10, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s10, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s8, s9
+; GFX6-NEXT:    s_cselect_b32 s8, s8, s9
+; GFX6-NEXT:    s_add_i32 s3, s3, s8
+; GFX6-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX6-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s12, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s4, 0
+; GFX6-NEXT:    s_cselect_b32 s9, s4, 0
+; GFX6-NEXT:    s_sub_i32 s9, s16, s9
+; GFX6-NEXT:    s_cmp_lt_i32 s4, 0
+; GFX6-NEXT:    s_cselect_b32 s10, s4, 0
+; GFX6-NEXT:    s_sub_i32 s10, s17, s10
+; GFX6-NEXT:    s_cmp_gt_i32 s10, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s10, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s8, s9
+; GFX6-NEXT:    s_cselect_b32 s8, s8, s9
+; GFX6-NEXT:    s_add_i32 s4, s4, s8
+; GFX6-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX6-NEXT:    s_lshl_b32 s5, s5, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s13, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s5, 0
+; GFX6-NEXT:    s_cselect_b32 s9, s5, 0
+; GFX6-NEXT:    s_sub_i32 s9, s16, s9
+; GFX6-NEXT:    s_cmp_lt_i32 s5, 0
+; GFX6-NEXT:    s_cselect_b32 s10, s5, 0
+; GFX6-NEXT:    s_sub_i32 s10, s17, s10
+; GFX6-NEXT:    s_cmp_gt_i32 s10, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s10, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s8, s9
+; GFX6-NEXT:    s_cselect_b32 s8, s8, s9
+; GFX6-NEXT:    s_add_i32 s5, s5, s8
+; GFX6-NEXT:    s_ashr_i32 s5, s5, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s6, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s14, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s6, 0
+; GFX6-NEXT:    s_cselect_b32 s9, s6, 0
+; GFX6-NEXT:    s_sub_i32 s9, s16, s9
+; GFX6-NEXT:    s_cmp_lt_i32 s6, 0
+; GFX6-NEXT:    s_cselect_b32 s10, s6, 0
+; GFX6-NEXT:    s_sub_i32 s10, s17, s10
+; GFX6-NEXT:    s_cmp_gt_i32 s10, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s10, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s8, s9
+; GFX6-NEXT:    s_cselect_b32 s8, s8, s9
+; GFX6-NEXT:    s_add_i32 s6, s6, s8
+; GFX6-NEXT:    s_ashr_i32 s6, s6, 16
+; GFX6-NEXT:    s_lshl_b32 s7, s7, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s15, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s7, 0
+; GFX6-NEXT:    s_cselect_b32 s9, s7, 0
+; GFX6-NEXT:    s_sub_i32 s9, s16, s9
+; GFX6-NEXT:    s_cmp_lt_i32 s7, 0
+; GFX6-NEXT:    s_cselect_b32 s10, s7, 0
+; GFX6-NEXT:    s_sub_i32 s10, s17, s10
+; GFX6-NEXT:    s_cmp_gt_i32 s10, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s10, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s8, s9
+; GFX6-NEXT:    s_cselect_b32 s8, s8, s9
+; GFX6-NEXT:    s_add_i32 s7, s7, s8
+; GFX6-NEXT:    s_mov_b32 s8, 0xffff
+; GFX6-NEXT:    s_and_b32 s1, s1, s8
+; GFX6-NEXT:    s_and_b32 s0, s0, s8
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_and_b32 s1, s2, s8
+; GFX6-NEXT:    s_and_b32 s2, s3, s8
+; GFX6-NEXT:    s_and_b32 s3, s5, s8
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_ashr_i32 s7, s7, 16
+; GFX6-NEXT:    s_or_b32 s1, s1, s2
+; GFX6-NEXT:    s_and_b32 s2, s4, s8
+; GFX6-NEXT:    s_and_b32 s4, s7, s8
+; GFX6-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX6-NEXT:    s_or_b32 s2, s2, s3
+; GFX6-NEXT:    s_and_b32 s3, s6, s8
+; GFX6-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX6-NEXT:    s_or_b32 s3, s3, s4
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_saddsat_v8i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_lshr_b32 s12, s4, 16
+; GFX8-NEXT:    s_lshr_b32 s13, s5, 16
+; GFX8-NEXT:    s_lshr_b32 s14, s6, 16
+; GFX8-NEXT:    s_lshr_b32 s8, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s9, s1, 16
+; GFX8-NEXT:    s_lshr_b32 s10, s2, 16
+; GFX8-NEXT:    s_lshr_b32 s11, s3, 16
+; GFX8-NEXT:    s_lshr_b32 s15, s7, 16
+; GFX8-NEXT:    s_sext_i32_i16 s18, s0
+; GFX8-NEXT:    s_sext_i32_i16 s19, 0
+; GFX8-NEXT:    s_cmp_gt_i32 s18, s19
+; GFX8-NEXT:    s_movk_i32 s16, 0x7fff
+; GFX8-NEXT:    s_cselect_b32 s20, s18, s19
+; GFX8-NEXT:    s_sub_i32 s20, s16, s20
+; GFX8-NEXT:    s_cmp_lt_i32 s18, s19
+; GFX8-NEXT:    s_mov_b32 s17, 0x8000
+; GFX8-NEXT:    s_cselect_b32 s18, s18, s19
+; GFX8-NEXT:    s_sub_i32 s18, s17, s18
+; GFX8-NEXT:    s_sext_i32_i16 s18, s18
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_cmp_gt_i32 s18, s4
+; GFX8-NEXT:    s_cselect_b32 s4, s18, s4
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_sext_i32_i16 s18, s20
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s18
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s18
+; GFX8-NEXT:    s_add_i32 s0, s0, s4
+; GFX8-NEXT:    s_sext_i32_i16 s4, s8
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s19
+; GFX8-NEXT:    s_cselect_b32 s18, s4, s19
+; GFX8-NEXT:    s_sub_i32 s18, s16, s18
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s19
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s19
+; GFX8-NEXT:    s_sub_i32 s4, s17, s4
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_sext_i32_i16 s12, s12
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s12
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s12
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_sext_i32_i16 s12, s18
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s12
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s12
+; GFX8-NEXT:    s_add_i32 s8, s8, s4
+; GFX8-NEXT:    s_sext_i32_i16 s4, s1
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s19
+; GFX8-NEXT:    s_cselect_b32 s12, s4, s19
+; GFX8-NEXT:    s_sub_i32 s12, s16, s12
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s19
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s19
+; GFX8-NEXT:    s_sub_i32 s4, s17, s4
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_sext_i32_i16 s5, s5
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s5
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX8-NEXT:    s_sext_i32_i16 s5, s12
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX8-NEXT:    s_add_i32 s1, s1, s4
+; GFX8-NEXT:    s_sext_i32_i16 s4, s9
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s19
+; GFX8-NEXT:    s_cselect_b32 s5, s4, s19
+; GFX8-NEXT:    s_sub_i32 s5, s16, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s19
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s19
+; GFX8-NEXT:    s_sub_i32 s4, s17, s4
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_sext_i32_i16 s12, s13
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s12
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s12
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_sext_i32_i16 s5, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX8-NEXT:    s_add_i32 s9, s9, s4
+; GFX8-NEXT:    s_sext_i32_i16 s4, s2
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s19
+; GFX8-NEXT:    s_cselect_b32 s5, s4, s19
+; GFX8-NEXT:    s_sub_i32 s5, s16, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s19
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s19
+; GFX8-NEXT:    s_sub_i32 s4, s17, s4
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_sext_i32_i16 s6, s6
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s6
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_sext_i32_i16 s5, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX8-NEXT:    s_add_i32 s2, s2, s4
+; GFX8-NEXT:    s_sext_i32_i16 s4, s10
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s19
+; GFX8-NEXT:    s_cselect_b32 s5, s4, s19
+; GFX8-NEXT:    s_sub_i32 s5, s16, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s19
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s19
+; GFX8-NEXT:    s_sub_i32 s4, s17, s4
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_sext_i32_i16 s6, s14
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s6
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_sext_i32_i16 s5, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX8-NEXT:    s_add_i32 s10, s10, s4
+; GFX8-NEXT:    s_sext_i32_i16 s4, s3
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s19
+; GFX8-NEXT:    s_cselect_b32 s5, s4, s19
+; GFX8-NEXT:    s_sub_i32 s5, s16, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s19
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s19
+; GFX8-NEXT:    s_sub_i32 s4, s17, s4
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_sext_i32_i16 s6, s7
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s6
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_sext_i32_i16 s5, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX8-NEXT:    s_add_i32 s3, s3, s4
+; GFX8-NEXT:    s_sext_i32_i16 s4, s11
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s19
+; GFX8-NEXT:    s_cselect_b32 s5, s4, s19
+; GFX8-NEXT:    s_sub_i32 s5, s16, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s19
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s19
+; GFX8-NEXT:    s_sub_i32 s4, s17, s4
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_sext_i32_i16 s6, s15
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s6
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_sext_i32_i16 s5, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX8-NEXT:    s_add_i32 s11, s11, s4
+; GFX8-NEXT:    s_bfe_u32 s4, s8, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX8-NEXT:    s_or_b32 s0, s0, s4
+; GFX8-NEXT:    s_bfe_u32 s4, s9, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX8-NEXT:    s_or_b32 s1, s1, s4
+; GFX8-NEXT:    s_bfe_u32 s4, s10, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX8-NEXT:    s_or_b32 s2, s2, s4
+; GFX8-NEXT:    s_bfe_u32 s4, s11, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX8-NEXT:    s_or_b32 s3, s3, s4
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_saddsat_v8i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_pack_ll_b32_b16 s10, 0, 0
+; GFX9-NEXT:    s_sext_i32_i16 s13, s10
+; GFX9-NEXT:    s_sext_i32_i16 s11, s0
+; GFX9-NEXT:    s_ashr_i32 s12, s0, 16
+; GFX9-NEXT:    s_ashr_i32 s10, s10, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s11, s13
+; GFX9-NEXT:    s_cselect_b32 s14, s11, s13
+; GFX9-NEXT:    s_cmp_gt_i32 s12, s10
+; GFX9-NEXT:    s_movk_i32 s8, 0x7fff
+; GFX9-NEXT:    s_cselect_b32 s15, s12, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s14, s14, s15
+; GFX9-NEXT:    s_pack_ll_b32_b16 s8, s8, s8
+; GFX9-NEXT:    s_lshr_b32 s16, s14, 16
+; GFX9-NEXT:    s_lshr_b32 s15, s8, 16
+; GFX9-NEXT:    s_sub_i32 s14, s8, s14
+; GFX9-NEXT:    s_sub_i32 s16, s15, s16
+; GFX9-NEXT:    s_cmp_lt_i32 s11, s13
+; GFX9-NEXT:    s_cselect_b32 s11, s11, s13
+; GFX9-NEXT:    s_cmp_lt_i32 s12, s10
+; GFX9-NEXT:    s_mov_b32 s9, 0xffff8000
+; GFX9-NEXT:    s_cselect_b32 s12, s12, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s9
+; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s12
+; GFX9-NEXT:    s_pack_ll_b32_b16 s14, s14, s16
+; GFX9-NEXT:    s_lshr_b32 s16, s11, 16
+; GFX9-NEXT:    s_lshr_b32 s12, s9, 16
+; GFX9-NEXT:    s_sub_i32 s11, s9, s11
+; GFX9-NEXT:    s_sub_i32 s16, s12, s16
+; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s16
+; GFX9-NEXT:    s_sext_i32_i16 s16, s11
+; GFX9-NEXT:    s_sext_i32_i16 s17, s4
+; GFX9-NEXT:    s_ashr_i32 s11, s11, 16
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s16, s17
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX9-NEXT:    s_cmp_gt_i32 s11, s4
+; GFX9-NEXT:    s_cselect_b32 s4, s11, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s16, s4
+; GFX9-NEXT:    s_sext_i32_i16 s11, s4
+; GFX9-NEXT:    s_sext_i32_i16 s16, s14
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX9-NEXT:    s_ashr_i32 s14, s14, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s11, s16
+; GFX9-NEXT:    s_cselect_b32 s11, s11, s16
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s14
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s14
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s11, s4
+; GFX9-NEXT:    s_lshr_b32 s11, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s14, s4, 16
+; GFX9-NEXT:    s_add_i32 s0, s0, s4
+; GFX9-NEXT:    s_add_i32 s11, s11, s14
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s11
+; GFX9-NEXT:    s_sext_i32_i16 s4, s1
+; GFX9-NEXT:    s_ashr_i32 s11, s1, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s13
+; GFX9-NEXT:    s_cselect_b32 s14, s4, s13
+; GFX9-NEXT:    s_cmp_gt_i32 s11, s10
+; GFX9-NEXT:    s_cselect_b32 s16, s11, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s14, s14, s16
+; GFX9-NEXT:    s_lshr_b32 s16, s14, 16
+; GFX9-NEXT:    s_sub_i32 s14, s8, s14
+; GFX9-NEXT:    s_sub_i32 s16, s15, s16
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s13
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s13
+; GFX9-NEXT:    s_cmp_lt_i32 s11, s10
+; GFX9-NEXT:    s_cselect_b32 s11, s11, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s11
+; GFX9-NEXT:    s_lshr_b32 s11, s4, 16
+; GFX9-NEXT:    s_sub_i32 s4, s9, s4
+; GFX9-NEXT:    s_sub_i32 s11, s12, s11
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s11
+; GFX9-NEXT:    s_pack_ll_b32_b16 s14, s14, s16
+; GFX9-NEXT:    s_sext_i32_i16 s11, s4
+; GFX9-NEXT:    s_sext_i32_i16 s16, s5
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX9-NEXT:    s_ashr_i32 s5, s5, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s11, s16
+; GFX9-NEXT:    s_cselect_b32 s11, s11, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s5
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s11, s4
+; GFX9-NEXT:    s_sext_i32_i16 s5, s4
+; GFX9-NEXT:    s_sext_i32_i16 s11, s14
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX9-NEXT:    s_ashr_i32 s14, s14, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s11
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s11
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s14
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s14
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s5, s4
+; GFX9-NEXT:    s_lshr_b32 s5, s1, 16
+; GFX9-NEXT:    s_lshr_b32 s11, s4, 16
+; GFX9-NEXT:    s_add_i32 s1, s1, s4
+; GFX9-NEXT:    s_add_i32 s5, s5, s11
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s5
+; GFX9-NEXT:    s_sext_i32_i16 s4, s2
+; GFX9-NEXT:    s_ashr_i32 s5, s2, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s13
+; GFX9-NEXT:    s_cselect_b32 s11, s4, s13
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s10
+; GFX9-NEXT:    s_cselect_b32 s14, s5, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s14
+; GFX9-NEXT:    s_lshr_b32 s14, s11, 16
+; GFX9-NEXT:    s_sub_i32 s11, s8, s11
+; GFX9-NEXT:    s_sub_i32 s14, s15, s14
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s13
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s13
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s10
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
+; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
+; GFX9-NEXT:    s_sub_i32 s4, s9, s4
+; GFX9-NEXT:    s_sub_i32 s5, s12, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s14
+; GFX9-NEXT:    s_sext_i32_i16 s5, s4
+; GFX9-NEXT:    s_sext_i32_i16 s14, s6
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX9-NEXT:    s_ashr_i32 s6, s6, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s14
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s14
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s6
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s5, s4
+; GFX9-NEXT:    s_sext_i32_i16 s5, s4
+; GFX9-NEXT:    s_sext_i32_i16 s6, s11
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX9-NEXT:    s_ashr_i32 s11, s11, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s6
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s11
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s11
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s5, s4
+; GFX9-NEXT:    s_lshr_b32 s5, s2, 16
+; GFX9-NEXT:    s_lshr_b32 s6, s4, 16
+; GFX9-NEXT:    s_add_i32 s2, s2, s4
+; GFX9-NEXT:    s_add_i32 s5, s5, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s5
+; GFX9-NEXT:    s_sext_i32_i16 s4, s3
+; GFX9-NEXT:    s_ashr_i32 s5, s3, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s13
+; GFX9-NEXT:    s_cselect_b32 s6, s4, s13
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s10
+; GFX9-NEXT:    s_cselect_b32 s11, s5, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s11
+; GFX9-NEXT:    s_lshr_b32 s11, s6, 16
+; GFX9-NEXT:    s_sub_i32 s6, s8, s6
+; GFX9-NEXT:    s_sub_i32 s8, s15, s11
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s13
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s13
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s10
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
+; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
+; GFX9-NEXT:    s_sub_i32 s4, s9, s4
+; GFX9-NEXT:    s_sub_i32 s5, s12, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s8
+; GFX9-NEXT:    s_sext_i32_i16 s5, s4
+; GFX9-NEXT:    s_sext_i32_i16 s8, s7
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX9-NEXT:    s_ashr_i32 s7, s7, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s8
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s8
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s7
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s7
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s5, s4
+; GFX9-NEXT:    s_sext_i32_i16 s5, s4
+; GFX9-NEXT:    s_sext_i32_i16 s7, s6
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX9-NEXT:    s_ashr_i32 s6, s6, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s7
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s7
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s6
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s5, s4
+; GFX9-NEXT:    s_lshr_b32 s5, s3, 16
+; GFX9-NEXT:    s_lshr_b32 s6, s4, 16
+; GFX9-NEXT:    s_add_i32 s3, s3, s4
+; GFX9-NEXT:    s_add_i32 s5, s5, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s5
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_saddsat_v8i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_pack_ll_b32_b16 s8, 0, 0
+; GFX10-NEXT:    s_sext_i32_i16 s9, s0
+; GFX10-NEXT:    s_sext_i32_i16 s11, s8
+; GFX10-NEXT:    s_ashr_i32 s10, s0, 16
+; GFX10-NEXT:    s_ashr_i32 s8, s8, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s9, s11
+; GFX10-NEXT:    s_movk_i32 s13, 0x7fff
+; GFX10-NEXT:    s_cselect_b32 s12, s9, s11
+; GFX10-NEXT:    s_cmp_gt_i32 s10, s8
+; GFX10-NEXT:    s_pack_ll_b32_b16 s13, s13, s13
+; GFX10-NEXT:    s_cselect_b32 s14, s10, s8
+; GFX10-NEXT:    s_mov_b32 s16, 0xffff8000
+; GFX10-NEXT:    s_pack_ll_b32_b16 s12, s12, s14
+; GFX10-NEXT:    s_lshr_b32 s14, s13, 16
+; GFX10-NEXT:    s_lshr_b32 s15, s12, 16
+; GFX10-NEXT:    s_sub_i32 s12, s13, s12
+; GFX10-NEXT:    s_sub_i32 s15, s14, s15
+; GFX10-NEXT:    s_cmp_lt_i32 s9, s11
+; GFX10-NEXT:    s_pack_ll_b32_b16 s16, s16, s16
+; GFX10-NEXT:    s_cselect_b32 s9, s9, s11
+; GFX10-NEXT:    s_cmp_lt_i32 s10, s8
+; GFX10-NEXT:    s_sext_i32_i16 s18, s4
+; GFX10-NEXT:    s_cselect_b32 s10, s10, s8
+; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
+; GFX10-NEXT:    s_lshr_b32 s10, s16, 16
+; GFX10-NEXT:    s_lshr_b32 s17, s9, 16
+; GFX10-NEXT:    s_sub_i32 s9, s16, s9
+; GFX10-NEXT:    s_sub_i32 s17, s10, s17
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s17
+; GFX10-NEXT:    s_sext_i32_i16 s17, s9
+; GFX10-NEXT:    s_ashr_i32 s9, s9, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s17, s18
+; GFX10-NEXT:    s_cselect_b32 s17, s17, s18
+; GFX10-NEXT:    s_cmp_gt_i32 s9, s4
+; GFX10-NEXT:    s_sext_i32_i16 s18, s5
+; GFX10-NEXT:    s_cselect_b32 s4, s9, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s12, s15
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s17, s4
+; GFX10-NEXT:    s_sext_i32_i16 s15, s9
+; GFX10-NEXT:    s_sext_i32_i16 s12, s4
+; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX10-NEXT:    s_ashr_i32 s9, s9, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s12, s15
+; GFX10-NEXT:    s_cselect_b32 s12, s12, s15
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s9
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s9
+; GFX10-NEXT:    s_lshr_b32 s9, s0, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s12, s4
+; GFX10-NEXT:    s_lshr_b32 s12, s4, 16
+; GFX10-NEXT:    s_add_i32 s0, s0, s4
+; GFX10-NEXT:    s_sext_i32_i16 s4, s1
+; GFX10-NEXT:    s_add_i32 s9, s9, s12
+; GFX10-NEXT:    s_ashr_i32 s12, s1, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s4, s11
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s9
+; GFX10-NEXT:    s_cselect_b32 s15, s4, s11
+; GFX10-NEXT:    s_cmp_gt_i32 s12, s8
+; GFX10-NEXT:    s_cselect_b32 s17, s12, s8
+; GFX10-NEXT:    s_pack_ll_b32_b16 s15, s15, s17
+; GFX10-NEXT:    s_lshr_b32 s17, s15, 16
+; GFX10-NEXT:    s_sub_i32 s15, s13, s15
+; GFX10-NEXT:    s_sub_i32 s17, s14, s17
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s11
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s11
+; GFX10-NEXT:    s_cmp_lt_i32 s12, s8
+; GFX10-NEXT:    s_cselect_b32 s12, s12, s8
+; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s12
+; GFX10-NEXT:    s_lshr_b32 s12, s4, 16
+; GFX10-NEXT:    s_sub_i32 s4, s16, s4
+; GFX10-NEXT:    s_sub_i32 s12, s10, s12
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s12
+; GFX10-NEXT:    s_sext_i32_i16 s12, s4
+; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s12, s18
+; GFX10-NEXT:    s_cselect_b32 s12, s12, s18
+; GFX10-NEXT:    s_cmp_gt_i32 s4, s5
+; GFX10-NEXT:    s_sext_i32_i16 s18, s6
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s15, s17
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s12, s4
+; GFX10-NEXT:    s_sext_i32_i16 s15, s5
+; GFX10-NEXT:    s_sext_i32_i16 s12, s4
+; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s12, s15
+; GFX10-NEXT:    s_cselect_b32 s12, s12, s15
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX10-NEXT:    s_lshr_b32 s5, s1, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s12, s4
+; GFX10-NEXT:    s_lshr_b32 s12, s4, 16
+; GFX10-NEXT:    s_add_i32 s1, s1, s4
+; GFX10-NEXT:    s_sext_i32_i16 s4, s2
+; GFX10-NEXT:    s_add_i32 s5, s5, s12
+; GFX10-NEXT:    s_ashr_i32 s12, s2, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s4, s11
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s5
+; GFX10-NEXT:    s_cselect_b32 s15, s4, s11
+; GFX10-NEXT:    s_cmp_gt_i32 s12, s8
+; GFX10-NEXT:    s_cselect_b32 s17, s12, s8
+; GFX10-NEXT:    s_pack_ll_b32_b16 s15, s15, s17
+; GFX10-NEXT:    s_lshr_b32 s17, s15, 16
+; GFX10-NEXT:    s_sub_i32 s15, s13, s15
+; GFX10-NEXT:    s_sub_i32 s17, s14, s17
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s11
+; GFX10-NEXT:    s_pack_ll_b32_b16 s15, s15, s17
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s11
+; GFX10-NEXT:    s_cmp_lt_i32 s12, s8
+; GFX10-NEXT:    s_cselect_b32 s12, s12, s8
+; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s12
+; GFX10-NEXT:    s_lshr_b32 s12, s4, 16
+; GFX10-NEXT:    s_sub_i32 s4, s16, s4
+; GFX10-NEXT:    s_sub_i32 s12, s10, s12
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s12
+; GFX10-NEXT:    s_sext_i32_i16 s12, s4
+; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s12, s18
+; GFX10-NEXT:    s_cselect_b32 s12, s12, s18
+; GFX10-NEXT:    s_cmp_gt_i32 s4, s6
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX10-NEXT:    s_sext_i32_i16 s6, s15
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s12, s4
+; GFX10-NEXT:    s_ashr_i32 s12, s15, 16
+; GFX10-NEXT:    s_sext_i32_i16 s15, s4
+; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s15, s6
+; GFX10-NEXT:    s_cselect_b32 s6, s15, s6
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s12
+; GFX10-NEXT:    s_sext_i32_i16 s15, s3
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s12
+; GFX10-NEXT:    s_lshr_b32 s12, s2, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s6, s4
+; GFX10-NEXT:    s_ashr_i32 s6, s3, 16
+; GFX10-NEXT:    s_lshr_b32 s17, s4, 16
+; GFX10-NEXT:    s_add_i32 s2, s2, s4
+; GFX10-NEXT:    s_add_i32 s12, s12, s17
+; GFX10-NEXT:    s_cmp_gt_i32 s15, s11
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s12
+; GFX10-NEXT:    s_cselect_b32 s4, s15, s11
+; GFX10-NEXT:    s_cmp_gt_i32 s6, s8
+; GFX10-NEXT:    s_cselect_b32 s17, s6, s8
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s17
+; GFX10-NEXT:    s_lshr_b32 s17, s4, 16
+; GFX10-NEXT:    s_sub_i32 s4, s13, s4
+; GFX10-NEXT:    s_sub_i32 s13, s14, s17
+; GFX10-NEXT:    s_cmp_lt_i32 s15, s11
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s13
+; GFX10-NEXT:    s_cselect_b32 s11, s15, s11
+; GFX10-NEXT:    s_cmp_lt_i32 s6, s8
+; GFX10-NEXT:    s_cselect_b32 s6, s6, s8
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s11, s6
+; GFX10-NEXT:    s_lshr_b32 s8, s6, 16
+; GFX10-NEXT:    s_sub_i32 s6, s16, s6
+; GFX10-NEXT:    s_sub_i32 s8, s10, s8
+; GFX10-NEXT:    s_sext_i32_i16 s10, s7
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s8
+; GFX10-NEXT:    s_ashr_i32 s7, s7, 16
+; GFX10-NEXT:    s_sext_i32_i16 s8, s6
+; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s8, s10
+; GFX10-NEXT:    s_cselect_b32 s8, s8, s10
+; GFX10-NEXT:    s_cmp_gt_i32 s6, s7
+; GFX10-NEXT:    s_cselect_b32 s6, s6, s7
+; GFX10-NEXT:    s_sext_i32_i16 s7, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s8, s6
+; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX10-NEXT:    s_sext_i32_i16 s8, s6
+; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s8, s7
+; GFX10-NEXT:    s_cselect_b32 s7, s8, s7
+; GFX10-NEXT:    s_cmp_lt_i32 s6, s4
+; GFX10-NEXT:    s_cselect_b32 s4, s6, s4
+; GFX10-NEXT:    s_lshr_b32 s5, s3, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s7, s4
+; GFX10-NEXT:    s_lshr_b32 s6, s4, 16
+; GFX10-NEXT:    s_add_i32 s3, s3, s4
+; GFX10-NEXT:    s_add_i32 s5, s5, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s5
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %lhs, <8 x i16> %rhs)
+  %cast = bitcast <8 x i16> %result to <4 x i32>
+  ret <4 x i32> %cast
+}
+
+; FIXME: i48 broken because i48 add broken
+; define i48 @v_saddsat_i48(i48 %lhs, i48 %rhs) {
+;   %result = call i48 @llvm.sadd.sat.i48(i48 %lhs, i48 %rhs)
+;   ret i48 %result
+; }
+
+; define amdgpu_ps i48 @s_saddsat_i48(i48 inreg %lhs, i48 inreg %rhs) {
+;   %result = call i48 @llvm.sadd.sat.i48(i48 %lhs, i48 %rhs)
+;   ret i48 %result
+; }
+
+; define amdgpu_ps <2 x float> @saddsat_i48_sv(i48 inreg %lhs, i48 %rhs) {
+;   %result = call i48 @llvm.sadd.sat.i48(i48 %lhs, i48 %rhs)
+;   %ext.result = zext i48 %result to i64
+;   %cast = bitcast i64 %ext.result to <2 x float>
+;   ret <2 x float> %cast
+; }
+
+; define amdgpu_ps <2 x float> @saddsat_i48_vs(i48 %lhs, i48 inreg %rhs) {
+;   %result = call i48 @llvm.sadd.sat.i48(i48 %lhs, i48 %rhs)
+;   %ext.result = zext i48 %result to i64
+;   %cast = bitcast i64 %ext.result to <2 x float>
+;   ret <2 x float> %cast
+; }
+
+define i64 @v_saddsat_i64(i64 %lhs, i64 %rhs) {
+; GFX6-LABEL: v_saddsat_i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v0, v2
+; GFX6-NEXT:    v_addc_u32_e32 v5, vcc, v1, v3, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, v[4:5], v[0:1]
+; GFX6-NEXT:    v_cmp_gt_i64_e64 s[4:5], 0, v[2:3]
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 31, v5
+; GFX6-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX6-NEXT:    v_add_i32_e64 v2, s[6:7], 0, v0
+; GFX6-NEXT:    v_addc_u32_e64 v1, s[6:7], v0, v1, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v4, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v0, v2
+; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, v1, v3, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, v[4:5], v[0:1]
+; GFX8-NEXT:    v_cmp_gt_i64_e64 s[4:5], 0, v[2:3]
+; GFX8-NEXT:    v_ashrrev_i32_e32 v0, 31, v5
+; GFX8-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX8-NEXT:    v_add_u32_e64 v2, s[6:7], 0, v0
+; GFX8-NEXT:    v_addc_u32_e64 v1, s[6:7], v0, v1, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v0, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v1, v3, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, v[4:5], v[0:1]
+; GFX9-NEXT:    v_cmp_gt_i64_e64 s[4:5], 0, v[2:3]
+; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 31, v5
+; GFX9-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX9-NEXT:    v_add_co_u32_e64 v2, s[6:7], 0, v0
+; GFX9-NEXT:    v_addc_co_u32_e64 v1, s[6:7], v0, v1, s[6:7]
+; GFX9-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_saddsat_i64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v10, vcc_lo, v0, v2
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v11, vcc_lo, v1, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_i64_e32 vcc_lo, 0, v[2:3]
+; GFX10-NEXT:    v_ashrrev_i32_e32 v6, 31, v11
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s4, v[10:11], v[0:1]
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s5, v6, 0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s5, 0x80000000, v6, s5
+; GFX10-NEXT:    s_xor_b32 vcc_lo, vcc_lo, s4
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v11, v1, vcc_lo
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i64 @llvm.sadd.sat.i64(i64 %lhs, i64 %rhs)
+  ret i64 %result
+}
+
+define amdgpu_ps i64 @s_saddsat_i64(i64 inreg %lhs, i64 inreg %rhs) {
+; GFX6-LABEL: s_saddsat_i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_add_u32 s4, s0, s2
+; GFX6-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX6-NEXT:    s_and_b32 s5, s5, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s5, 0
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_addc_u32 s5, s1, s3
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
+; GFX6-NEXT:    v_cmp_lt_i64_e64 s[0:1], s[2:3], 0
+; GFX6-NEXT:    s_ashr_i32 s2, s5, 31
+; GFX6-NEXT:    s_xor_b64 vcc, s[0:1], vcc
+; GFX6-NEXT:    s_add_u32 s0, s2, 0
+; GFX6-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX6-NEXT:    s_and_b32 s1, s1, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX6-NEXT:    s_addc_u32 s1, s2, 0x80000000
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    v_mov_b32_e32 v2, s1
+; GFX6-NEXT:    v_mov_b32_e32 v3, s5
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_saddsat_i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_add_u32 s4, s0, s2
+; GFX8-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX8-NEXT:    s_and_b32 s5, s5, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    s_addc_u32 s5, s1, s3
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
+; GFX8-NEXT:    v_cmp_lt_i64_e64 s[0:1], s[2:3], 0
+; GFX8-NEXT:    s_ashr_i32 s2, s5, 31
+; GFX8-NEXT:    s_xor_b64 vcc, s[0:1], vcc
+; GFX8-NEXT:    s_add_u32 s0, s2, 0
+; GFX8-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX8-NEXT:    s_and_b32 s1, s1, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX8-NEXT:    s_addc_u32 s1, s2, 0x80000000
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    v_mov_b32_e32 v3, s5
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_saddsat_i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_add_u32 s4, s0, s2
+; GFX9-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX9-NEXT:    s_and_b32 s5, s5, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s5, 0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    s_addc_u32 s5, s1, s3
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
+; GFX9-NEXT:    v_cmp_lt_i64_e64 s[0:1], s[2:3], 0
+; GFX9-NEXT:    s_ashr_i32 s2, s5, 31
+; GFX9-NEXT:    s_xor_b64 vcc, s[0:1], vcc
+; GFX9-NEXT:    s_add_u32 s0, s2, 0
+; GFX9-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX9-NEXT:    s_and_b32 s1, s1, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX9-NEXT:    s_addc_u32 s1, s2, 0x80000000
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_saddsat_i64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s4, s0, s2
+; GFX10-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX10-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-NEXT:    s_and_b32 s5, s5, 1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cmp_lg_u32 s5, 0
+; GFX10-NEXT:    s_addc_u32 s5, s1, s3
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s0, s[4:5], s[0:1]
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s1, s[2:3], 0
+; GFX10-NEXT:    s_ashr_i32 s2, s5, 31
+; GFX10-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-NEXT:    s_xor_b32 s3, s1, s0
+; GFX10-NEXT:    s_add_u32 s0, s2, 0
+; GFX10-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, s0, s3
+; GFX10-NEXT:    s_and_b32 s1, s1, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX10-NEXT:    s_addc_u32 s1, s2, 0x80000000
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s1, s3
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i64 @llvm.sadd.sat.i64(i64 %lhs, i64 %rhs)
+  ret i64 %result
+}
+
+define amdgpu_ps <2 x float> @saddsat_i64_sv(i64 inreg %lhs, i64 %rhs) {
+; GFX6-LABEL: saddsat_i64_sv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v3, s1
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, s0, v0
+; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v3, v1, vcc
+; GFX6-NEXT:    v_cmp_gt_i64_e32 vcc, s[0:1], v[2:3]
+; GFX6-NEXT:    v_cmp_gt_i64_e64 s[0:1], 0, v[0:1]
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 31, v3
+; GFX6-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX6-NEXT:    v_add_i32_e64 v4, s[2:3], 0, v0
+; GFX6-NEXT:    v_addc_u32_e64 v1, s[2:3], v0, v1, s[2:3]
+; GFX6-NEXT:    s_xor_b64 vcc, s[0:1], vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: saddsat_i64_sv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s0, v0
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, v3, v1, vcc
+; GFX8-NEXT:    v_cmp_gt_i64_e32 vcc, s[0:1], v[2:3]
+; GFX8-NEXT:    v_cmp_gt_i64_e64 s[0:1], 0, v[0:1]
+; GFX8-NEXT:    v_ashrrev_i32_e32 v0, 31, v3
+; GFX8-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX8-NEXT:    v_add_u32_e64 v4, s[2:3], 0, v0
+; GFX8-NEXT:    v_addc_u32_e64 v1, s[2:3], v0, v1, s[2:3]
+; GFX8-NEXT:    s_xor_b64 vcc, s[0:1], vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: saddsat_i64_sv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v1, vcc
+; GFX9-NEXT:    v_cmp_gt_i64_e32 vcc, s[0:1], v[2:3]
+; GFX9-NEXT:    v_cmp_gt_i64_e64 s[0:1], 0, v[0:1]
+; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 31, v3
+; GFX9-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX9-NEXT:    v_add_co_u32_e64 v4, s[2:3], 0, v0
+; GFX9-NEXT:    v_addc_co_u32_e64 v1, s[2:3], v0, v1, s[2:3]
+; GFX9-NEXT:    s_xor_b64 vcc, s[0:1], vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: saddsat_i64_sv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_add_co_u32_e64 v2, vcc_lo, s0, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, s1, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_i64_e32 vcc_lo, 0, v[0:1]
+; GFX10-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
+; GFX10-NEXT:    v_cmp_gt_i64_e64 s0, s[0:1], v[2:3]
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s1, v4, 0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s1, 0x80000000, v4, s1
+; GFX10-NEXT:    s_xor_b32 vcc_lo, vcc_lo, s0
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i64 @llvm.sadd.sat.i64(i64 %lhs, i64 %rhs)
+  %cast = bitcast i64 %result to <2 x float>
+  ret <2 x float> %cast
+}
+
+define amdgpu_ps <2 x float> @saddsat_i64_vs(i64 %lhs, i64 inreg %rhs) {
+; GFX6-LABEL: saddsat_i64_vs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v3, s1
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, s0, v0
+; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v1, v3, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, v[2:3], v[0:1]
+; GFX6-NEXT:    v_cmp_lt_i64_e64 s[2:3], s[0:1], 0
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 31, v3
+; GFX6-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX6-NEXT:    v_add_i32_e64 v4, s[0:1], 0, v0
+; GFX6-NEXT:    v_addc_u32_e64 v1, s[0:1], v0, v1, s[0:1]
+; GFX6-NEXT:    s_xor_b64 vcc, s[2:3], vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: saddsat_i64_vs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s0, v0
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, v1, v3, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, v[2:3], v[0:1]
+; GFX8-NEXT:    v_cmp_lt_i64_e64 s[2:3], s[0:1], 0
+; GFX8-NEXT:    v_ashrrev_i32_e32 v0, 31, v3
+; GFX8-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX8-NEXT:    v_add_u32_e64 v4, s[0:1], 0, v0
+; GFX8-NEXT:    v_addc_u32_e64 v1, s[0:1], v0, v1, s[0:1]
+; GFX8-NEXT:    s_xor_b64 vcc, s[2:3], vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: saddsat_i64_vs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v1, v3, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, v[2:3], v[0:1]
+; GFX9-NEXT:    v_cmp_lt_i64_e64 s[2:3], s[0:1], 0
+; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 31, v3
+; GFX9-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX9-NEXT:    v_add_co_u32_e64 v4, s[0:1], 0, v0
+; GFX9-NEXT:    v_addc_co_u32_e64 v1, s[0:1], v0, v1, s[0:1]
+; GFX9-NEXT:    s_xor_b64 vcc, s[2:3], vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: saddsat_i64_vs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_add_co_u32_e64 v2, vcc_lo, v0, s0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, s1, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s1, s[0:1], 0
+; GFX10-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
+; GFX10-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, v4, 0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, 0x80000000, v4, s0
+; GFX10-NEXT:    s_xor_b32 vcc_lo, s1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i64 @llvm.sadd.sat.i64(i64 %lhs, i64 %rhs)
+  %cast = bitcast i64 %result to <2 x float>
+  ret <2 x float> %cast
+}
+
+define <2 x i64> @v_saddsat_v2i64(<2 x i64> %lhs, <2 x i64> %rhs) {
+; GFX6-LABEL: v_saddsat_v2i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_add_i32_e32 v8, vcc, v0, v4
+; GFX6-NEXT:    v_addc_u32_e32 v9, vcc, v1, v5, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, v[8:9], v[0:1]
+; GFX6-NEXT:    v_cmp_gt_i64_e64 s[4:5], 0, v[4:5]
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 31, v9
+; GFX6-NEXT:    s_brev_b32 s8, 1
+; GFX6-NEXT:    v_mov_b32_e32 v1, s8
+; GFX6-NEXT:    v_add_i32_e64 v4, s[6:7], 0, v0
+; GFX6-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX6-NEXT:    v_addc_u32_e64 v1, s[6:7], v0, v1, s[6:7]
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v8, v4, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
+; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v2, v6
+; GFX6-NEXT:    v_addc_u32_e32 v5, vcc, v3, v7, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, v[4:5], v[2:3]
+; GFX6-NEXT:    v_cmp_gt_i64_e64 s[4:5], 0, v[6:7]
+; GFX6-NEXT:    v_ashrrev_i32_e32 v2, 31, v5
+; GFX6-NEXT:    v_mov_b32_e32 v3, s8
+; GFX6-NEXT:    v_add_i32_e64 v6, s[6:7], 0, v2
+; GFX6-NEXT:    v_addc_u32_e64 v3, s[6:7], v2, v3, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v4, v6, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_v2i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v8, vcc, v0, v4
+; GFX8-NEXT:    v_addc_u32_e32 v9, vcc, v1, v5, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, v[8:9], v[0:1]
+; GFX8-NEXT:    v_cmp_gt_i64_e64 s[4:5], 0, v[4:5]
+; GFX8-NEXT:    v_ashrrev_i32_e32 v0, 31, v9
+; GFX8-NEXT:    s_brev_b32 s8, 1
+; GFX8-NEXT:    v_mov_b32_e32 v1, s8
+; GFX8-NEXT:    v_add_u32_e64 v4, s[6:7], 0, v0
+; GFX8-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX8-NEXT:    v_addc_u32_e64 v1, s[6:7], v0, v1, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v8, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v2, v6
+; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, v3, v7, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, v[4:5], v[2:3]
+; GFX8-NEXT:    v_cmp_gt_i64_e64 s[4:5], 0, v[6:7]
+; GFX8-NEXT:    v_ashrrev_i32_e32 v2, 31, v5
+; GFX8-NEXT:    v_mov_b32_e32 v3, s8
+; GFX8-NEXT:    v_add_u32_e64 v6, s[6:7], 0, v2
+; GFX8-NEXT:    v_addc_u32_e64 v3, s[6:7], v2, v3, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v4, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_v2i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, v1, v5, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, v[8:9], v[0:1]
+; GFX9-NEXT:    v_cmp_gt_i64_e64 s[4:5], 0, v[4:5]
+; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 31, v9
+; GFX9-NEXT:    s_brev_b32 s8, 1
+; GFX9-NEXT:    v_mov_b32_e32 v1, s8
+; GFX9-NEXT:    v_add_co_u32_e64 v4, s[6:7], 0, v0
+; GFX9-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX9-NEXT:    v_addc_co_u32_e64 v1, s[6:7], v0, v1, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v8, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v2, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v3, v7, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, v[4:5], v[2:3]
+; GFX9-NEXT:    v_cmp_gt_i64_e64 s[4:5], 0, v[6:7]
+; GFX9-NEXT:    v_ashrrev_i32_e32 v2, 31, v5
+; GFX9-NEXT:    v_mov_b32_e32 v3, s8
+; GFX9-NEXT:    v_add_co_u32_e64 v6, s[6:7], 0, v2
+; GFX9-NEXT:    v_addc_co_u32_e64 v3, s[6:7], v2, v3, s[6:7]
+; GFX9-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_saddsat_v2i64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_mov_b32_e32 v9, v0
+; GFX10-NEXT:    v_mov_b32_e32 v10, v1
+; GFX10-NEXT:    v_mov_b32_e32 v13, v2
+; GFX10-NEXT:    v_mov_b32_e32 v14, v3
+; GFX10-NEXT:    v_cmp_gt_i64_e64 s4, 0, v[4:5]
+; GFX10-NEXT:    v_add_co_u32_e64 v19, vcc_lo, v9, v4
+; GFX10-NEXT:    s_brev_b32 s8, 1
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v20, vcc_lo, v10, v5, vcc_lo
+; GFX10-NEXT:    v_add_co_u32_e64 v23, vcc_lo, v13, v6
+; GFX10-NEXT:    v_cmp_gt_i64_e64 s6, 0, v[6:7]
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v24, vcc_lo, v14, v7, vcc_lo
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_ashrrev_i32_e32 v0, 31, v20
+; GFX10-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[19:20], v[9:10]
+; GFX10-NEXT:    v_ashrrev_i32_e32 v1, 31, v24
+; GFX10-NEXT:    v_add_co_u32_e64 v4, s5, v0, 0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v5, s5, s8, v0, s5
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s5, v[23:24], v[13:14]
+; GFX10-NEXT:    v_add_co_u32_e64 v2, s7, v1, 0
+; GFX10-NEXT:    s_xor_b32 vcc_lo, s4, vcc_lo
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, s7, s8, v1, s7
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v19, v4, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v20, v5, vcc_lo
+; GFX10-NEXT:    s_xor_b32 vcc_lo, s6, s5
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v23, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v24, v3, vcc_lo
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> %lhs, <2 x i64> %rhs)
+  ret <2 x i64> %result
+}
+
+define amdgpu_ps <2 x i64> @s_saddsat_v2i64(<2 x i64> inreg %lhs, <2 x i64> inreg %rhs) {
+; GFX6-LABEL: s_saddsat_v2i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_add_u32 s8, s0, s4
+; GFX6-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX6-NEXT:    s_and_b32 s9, s9, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_addc_u32 s9, s1, s5
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_cmp_lt_i64_e64 s[0:1], s[4:5], 0
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, s[8:9], v[0:1]
+; GFX6-NEXT:    s_ashr_i32 s4, s9, 31
+; GFX6-NEXT:    s_xor_b64 vcc, s[0:1], vcc
+; GFX6-NEXT:    s_add_u32 s0, s4, 0
+; GFX6-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX6-NEXT:    s_and_b32 s1, s1, 1
+; GFX6-NEXT:    s_brev_b32 s5, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX6-NEXT:    s_addc_u32 s1, s4, s5
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_add_u32 s0, s2, s6
+; GFX6-NEXT:    v_mov_b32_e32 v2, s1
+; GFX6-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX6-NEXT:    s_and_b32 s1, s1, 1
+; GFX6-NEXT:    v_mov_b32_e32 v0, s8
+; GFX6-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, v0, v1, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    s_addc_u32 s1, s3, s7
+; GFX6-NEXT:    v_mov_b32_e32 v3, s9
+; GFX6-NEXT:    v_mov_b32_e32 v1, s3
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, s[0:1], v[0:1]
+; GFX6-NEXT:    v_cmp_lt_i64_e64 s[2:3], s[6:7], 0
+; GFX6-NEXT:    s_ashr_i32 s4, s1, 31
+; GFX6-NEXT:    s_xor_b64 vcc, s[2:3], vcc
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_add_u32 s0, s4, 0
+; GFX6-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX6-NEXT:    s_and_b32 s2, s2, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX6-NEXT:    s_addc_u32 s3, s4, s5
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    v_mov_b32_e32 v5, s1
+; GFX6-NEXT:    v_mov_b32_e32 v3, s3
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX6-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX6-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s3, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_saddsat_v2i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_add_u32 s8, s0, s4
+; GFX8-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX8-NEXT:    s_and_b32 s9, s9, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    s_addc_u32 s9, s1, s5
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_cmp_lt_i64_e64 s[0:1], s[4:5], 0
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[8:9], v[0:1]
+; GFX8-NEXT:    s_ashr_i32 s4, s9, 31
+; GFX8-NEXT:    s_xor_b64 vcc, s[0:1], vcc
+; GFX8-NEXT:    s_add_u32 s0, s4, 0
+; GFX8-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX8-NEXT:    s_and_b32 s1, s1, 1
+; GFX8-NEXT:    s_brev_b32 s5, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX8-NEXT:    s_addc_u32 s1, s4, s5
+; GFX8-NEXT:    v_mov_b32_e32 v1, s0
+; GFX8-NEXT:    s_add_u32 s0, s2, s6
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX8-NEXT:    s_and_b32 s1, s1, 1
+; GFX8-NEXT:    v_mov_b32_e32 v0, s8
+; GFX8-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v0, v1, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    s_addc_u32 s1, s3, s7
+; GFX8-NEXT:    v_mov_b32_e32 v3, s9
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[0:1], v[0:1]
+; GFX8-NEXT:    v_cmp_lt_i64_e64 s[2:3], s[6:7], 0
+; GFX8-NEXT:    s_ashr_i32 s4, s1, 31
+; GFX8-NEXT:    s_xor_b64 vcc, s[2:3], vcc
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    s_add_u32 s0, s4, 0
+; GFX8-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX8-NEXT:    s_and_b32 s2, s2, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX8-NEXT:    s_addc_u32 s3, s4, s5
+; GFX8-NEXT:    v_mov_b32_e32 v1, s0
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v3, s3
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_saddsat_v2i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_add_u32 s8, s0, s4
+; GFX9-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX9-NEXT:    s_and_b32 s9, s9, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    s_addc_u32 s9, s1, s5
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_cmp_lt_i64_e64 s[0:1], s[4:5], 0
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, s[8:9], v[0:1]
+; GFX9-NEXT:    s_ashr_i32 s4, s9, 31
+; GFX9-NEXT:    s_xor_b64 vcc, s[0:1], vcc
+; GFX9-NEXT:    s_add_u32 s0, s4, 0
+; GFX9-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX9-NEXT:    s_and_b32 s1, s1, 1
+; GFX9-NEXT:    s_brev_b32 s5, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX9-NEXT:    s_addc_u32 s1, s4, s5
+; GFX9-NEXT:    v_mov_b32_e32 v1, s0
+; GFX9-NEXT:    s_add_u32 s0, s2, s6
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX9-NEXT:    s_and_b32 s1, s1, 1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s8
+; GFX9-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v0, v1, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    s_addc_u32 s1, s3, s7
+; GFX9-NEXT:    v_mov_b32_e32 v3, s9
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, s[0:1], v[0:1]
+; GFX9-NEXT:    v_cmp_lt_i64_e64 s[2:3], s[6:7], 0
+; GFX9-NEXT:    s_ashr_i32 s4, s1, 31
+; GFX9-NEXT:    s_xor_b64 vcc, s[2:3], vcc
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    s_add_u32 s0, s4, 0
+; GFX9-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX9-NEXT:    s_and_b32 s2, s2, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX9-NEXT:    s_addc_u32 s3, s4, s5
+; GFX9-NEXT:    v_mov_b32_e32 v1, s0
+; GFX9-NEXT:    v_mov_b32_e32 v5, s1
+; GFX9-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_saddsat_v2i64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s8, s0, s4
+; GFX10-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s4, s[4:5], 0
+; GFX10-NEXT:    s_and_b32 s9, s9, 1
+; GFX10-NEXT:    v_mov_b32_e32 v0, s8
+; GFX10-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX10-NEXT:    s_brev_b32 s10, 1
+; GFX10-NEXT:    s_addc_u32 s9, s1, s5
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s0, s[8:9], s[0:1]
+; GFX10-NEXT:    s_ashr_i32 s1, s9, 31
+; GFX10-NEXT:    v_mov_b32_e32 v1, s9
+; GFX10-NEXT:    s_xor_b32 s8, s4, s0
+; GFX10-NEXT:    s_add_u32 s0, s1, 0
+; GFX10-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, s0, s8
+; GFX10-NEXT:    s_and_b32 s4, s4, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX10-NEXT:    s_addc_u32 s1, s1, s10
+; GFX10-NEXT:    s_add_u32 s4, s2, s6
+; GFX10-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s1, s8
+; GFX10-NEXT:    s_and_b32 s5, s5, 1
+; GFX10-NEXT:    v_mov_b32_e32 v2, s4
+; GFX10-NEXT:    s_cmp_lg_u32 s5, 0
+; GFX10-NEXT:    s_addc_u32 s5, s3, s7
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s2, s[4:5], s[2:3]
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s3, s[6:7], 0
+; GFX10-NEXT:    s_ashr_i32 s1, s5, 31
+; GFX10-NEXT:    v_mov_b32_e32 v3, s5
+; GFX10-NEXT:    s_xor_b32 s2, s3, s2
+; GFX10-NEXT:    s_add_u32 s0, s1, 0
+; GFX10-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s0, s2
+; GFX10-NEXT:    s_and_b32 s3, s3, 1
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    s_cmp_lg_u32 s3, 0
+; GFX10-NEXT:    s_addc_u32 s1, s1, s10
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, s1, s2
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> %lhs, <2 x i64> %rhs)
+  ret <2 x i64> %result
+}
+
+define amdgpu_ps i128 @s_saddsat_i128(i128 inreg %lhs, i128 inreg %rhs) {
+; GFX6-LABEL: s_saddsat_i128:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_add_u32 s8, s0, s4
+; GFX6-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX6-NEXT:    s_and_b32 s9, s9, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX6-NEXT:    s_addc_u32 s9, s1, s5
+; GFX6-NEXT:    s_cselect_b32 s10, 1, 0
+; GFX6-NEXT:    s_and_b32 s10, s10, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX6-NEXT:    s_addc_u32 s10, s2, s6
+; GFX6-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX6-NEXT:    v_mov_b32_e32 v3, s1
+; GFX6-NEXT:    s_and_b32 s11, s11, 1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[8:9], v[2:3]
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    s_addc_u32 s11, s3, s7
+; GFX6-NEXT:    v_mov_b32_e32 v1, s3
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, s[10:11], v[0:1]
+; GFX6-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[4:5], 0
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, s[10:11], v[0:1]
+; GFX6-NEXT:    s_movk_i32 s2, 0x7f
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; GFX6-NEXT:    v_cmp_lt_i64_e64 s[0:1], s[6:7], 0
+; GFX6-NEXT:    v_cmp_eq_u64_e64 vcc, s[6:7], 0
+; GFX6-NEXT:    s_sub_i32 s6, s2, 64
+; GFX6-NEXT:    s_sub_i32 s4, 64, s2
+; GFX6-NEXT:    s_cmp_lt_u32 s2, 64
+; GFX6-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s2, 0
+; GFX6-NEXT:    s_cselect_b32 s13, 1, 0
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX6-NEXT:    s_ashr_i64 s[0:1], s[10:11], s2
+; GFX6-NEXT:    s_lshr_b64 s[2:3], s[8:9], s2
+; GFX6-NEXT:    s_lshl_b64 s[4:5], s[10:11], s4
+; GFX6-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX6-NEXT:    s_ashr_i32 s4, s11, 31
+; GFX6-NEXT:    s_ashr_i64 s[6:7], s[10:11], s6
+; GFX6-NEXT:    s_and_b32 s12, s12, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX6-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[6:7]
+; GFX6-NEXT:    s_and_b32 s6, s13, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s6, 0
+; GFX6-NEXT:    s_cselect_b64 s[2:3], s[8:9], s[2:3]
+; GFX6-NEXT:    s_mov_b32 s5, s4
+; GFX6-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[4:5]
+; GFX6-NEXT:    s_add_u32 s2, s2, 0
+; GFX6-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX6-NEXT:    s_and_b32 s4, s4, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX6-NEXT:    s_addc_u32 s3, s3, 0
+; GFX6-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX6-NEXT:    s_and_b32 s4, s4, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX6-NEXT:    s_addc_u32 s0, s0, 0
+; GFX6-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX6-NEXT:    v_xor_b32_e32 v0, v1, v0
+; GFX6-NEXT:    s_and_b32 s4, s4, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX6-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX6-NEXT:    s_addc_u32 s1, s1, 0x80000000
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v2, s3
+; GFX6-NEXT:    v_mov_b32_e32 v3, s8
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v4, s9
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v3, v1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v4, v2, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    v_mov_b32_e32 v3, s1
+; GFX6-NEXT:    v_mov_b32_e32 v4, s10
+; GFX6-NEXT:    v_mov_b32_e32 v5, s11
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX6-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX6-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_saddsat_i128:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_add_u32 s8, s0, s4
+; GFX8-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX8-NEXT:    s_and_b32 s9, s9, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX8-NEXT:    s_addc_u32 s9, s1, s5
+; GFX8-NEXT:    s_cselect_b32 s10, 1, 0
+; GFX8-NEXT:    s_and_b32 s10, s10, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX8-NEXT:    s_addc_u32 s10, s2, s6
+; GFX8-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX8-NEXT:    s_and_b32 s11, s11, 1
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:    s_addc_u32 s11, s3, s7
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[8:9], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    s_cmp_eq_u64 s[10:11], s[2:3]
+; GFX8-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[10:11], v[0:1]
+; GFX8-NEXT:    s_and_b32 s0, 1, s2
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[4:5], 0
+; GFX8-NEXT:    s_cmp_eq_u64 s[6:7], 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; GFX8-NEXT:    v_cmp_lt_i64_e64 s[0:1], s[6:7], 0
+; GFX8-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX8-NEXT:    s_and_b32 s0, 1, s2
+; GFX8-NEXT:    s_movk_i32 s2, 0x7f
+; GFX8-NEXT:    s_sub_i32 s6, s2, 64
+; GFX8-NEXT:    s_sub_i32 s4, 64, s2
+; GFX8-NEXT:    s_cmp_lt_u32 s2, 64
+; GFX8-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s2, 0
+; GFX8-NEXT:    s_cselect_b32 s13, 1, 0
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    s_ashr_i64 s[0:1], s[10:11], s2
+; GFX8-NEXT:    s_lshr_b64 s[2:3], s[8:9], s2
+; GFX8-NEXT:    s_lshl_b64 s[4:5], s[10:11], s4
+; GFX8-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX8-NEXT:    s_ashr_i32 s4, s11, 31
+; GFX8-NEXT:    s_ashr_i64 s[6:7], s[10:11], s6
+; GFX8-NEXT:    s_and_b32 s12, s12, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX8-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[6:7]
+; GFX8-NEXT:    s_and_b32 s6, s13, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s6, 0
+; GFX8-NEXT:    s_cselect_b64 s[2:3], s[8:9], s[2:3]
+; GFX8-NEXT:    s_mov_b32 s5, s4
+; GFX8-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[4:5]
+; GFX8-NEXT:    s_add_u32 s2, s2, 0
+; GFX8-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX8-NEXT:    s_and_b32 s4, s4, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX8-NEXT:    s_addc_u32 s3, s3, 0
+; GFX8-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX8-NEXT:    s_and_b32 s4, s4, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX8-NEXT:    s_addc_u32 s0, s0, 0
+; GFX8-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX8-NEXT:    v_xor_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_and_b32 s4, s4, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX8-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX8-NEXT:    s_addc_u32 s1, s1, 0x80000000
+; GFX8-NEXT:    v_mov_b32_e32 v1, s2
+; GFX8-NEXT:    v_mov_b32_e32 v2, s3
+; GFX8-NEXT:    v_mov_b32_e32 v3, s8
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s9
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v4, v2, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s10
+; GFX8-NEXT:    v_mov_b32_e32 v5, s11
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_saddsat_i128:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_add_u32 s8, s0, s4
+; GFX9-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX9-NEXT:    s_and_b32 s9, s9, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX9-NEXT:    s_addc_u32 s9, s1, s5
+; GFX9-NEXT:    s_cselect_b32 s10, 1, 0
+; GFX9-NEXT:    s_and_b32 s10, s10, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX9-NEXT:    s_addc_u32 s10, s2, s6
+; GFX9-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX9-NEXT:    s_and_b32 s11, s11, 1
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s0
+; GFX9-NEXT:    s_addc_u32 s11, s3, s7
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[8:9], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    s_cmp_eq_u64 s[10:11], s[2:3]
+; GFX9-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, s[10:11], v[0:1]
+; GFX9-NEXT:    s_and_b32 s0, 1, s2
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[4:5], 0
+; GFX9-NEXT:    s_cmp_eq_u64 s[6:7], 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; GFX9-NEXT:    v_cmp_lt_i64_e64 s[0:1], s[6:7], 0
+; GFX9-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX9-NEXT:    s_and_b32 s0, 1, s2
+; GFX9-NEXT:    s_movk_i32 s2, 0x7f
+; GFX9-NEXT:    s_sub_i32 s6, s2, 64
+; GFX9-NEXT:    s_sub_i32 s4, 64, s2
+; GFX9-NEXT:    s_cmp_lt_u32 s2, 64
+; GFX9-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s2, 0
+; GFX9-NEXT:    s_cselect_b32 s13, 1, 0
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    s_ashr_i64 s[0:1], s[10:11], s2
+; GFX9-NEXT:    s_lshr_b64 s[2:3], s[8:9], s2
+; GFX9-NEXT:    s_lshl_b64 s[4:5], s[10:11], s4
+; GFX9-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX9-NEXT:    s_ashr_i32 s4, s11, 31
+; GFX9-NEXT:    s_ashr_i64 s[6:7], s[10:11], s6
+; GFX9-NEXT:    s_and_b32 s12, s12, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX9-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[6:7]
+; GFX9-NEXT:    s_and_b32 s6, s13, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s6, 0
+; GFX9-NEXT:    s_cselect_b64 s[2:3], s[8:9], s[2:3]
+; GFX9-NEXT:    s_mov_b32 s5, s4
+; GFX9-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[4:5]
+; GFX9-NEXT:    s_add_u32 s2, s2, 0
+; GFX9-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX9-NEXT:    s_and_b32 s4, s4, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX9-NEXT:    s_addc_u32 s3, s3, 0
+; GFX9-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX9-NEXT:    s_and_b32 s4, s4, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX9-NEXT:    s_addc_u32 s0, s0, 0
+; GFX9-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    v_xor_b32_e32 v0, v1, v0
+; GFX9-NEXT:    s_and_b32 s4, s4, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    s_addc_u32 s1, s1, 0x80000000
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-NEXT:    v_mov_b32_e32 v3, s8
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    v_mov_b32_e32 v4, s9
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v2, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v2, s0
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_mov_b32_e32 v4, s10
+; GFX9-NEXT:    v_mov_b32_e32 v5, s11
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_saddsat_i128:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s8, s0, s4
+; GFX10-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX10-NEXT:    s_movk_i32 s12, 0x7f
+; GFX10-NEXT:    s_and_b32 s9, s9, 1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX10-NEXT:    s_addc_u32 s9, s1, s5
+; GFX10-NEXT:    s_cselect_b32 s10, 1, 0
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s0, s[8:9], s[0:1]
+; GFX10-NEXT:    s_and_b32 s10, s10, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX10-NEXT:    s_addc_u32 s10, s2, s6
+; GFX10-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX10-NEXT:    s_and_b32 s11, s11, 1
+; GFX10-NEXT:    v_mov_b32_e32 v3, s10
+; GFX10-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX10-NEXT:    s_addc_u32 s11, s3, s7
+; GFX10-NEXT:    s_cmp_eq_u64 s[10:11], s[2:3]
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s1, s[10:11], s[2:3]
+; GFX10-NEXT:    s_cselect_b32 s0, 1, 0
+; GFX10-NEXT:    v_mov_b32_e32 v4, s11
+; GFX10-NEXT:    s_and_b32 s0, 1, s0
+; GFX10-NEXT:    s_cmp_eq_u64 s[6:7], 0
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s0
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s0, s[4:5], 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s1
+; GFX10-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX10-NEXT:    s_sub_i32 s13, s12, 64
+; GFX10-NEXT:    s_and_b32 s14, 1, s1
+; GFX10-NEXT:    s_sub_i32 s2, 64, s12
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s0, s[6:7], 0
+; GFX10-NEXT:    s_cmp_lt_u32 s12, 64
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s14
+; GFX10-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s12, 0
+; GFX10-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s0
+; GFX10-NEXT:    s_lshr_b64 s[0:1], s[8:9], s12
+; GFX10-NEXT:    s_lshl_b64 s[2:3], s[10:11], s2
+; GFX10-NEXT:    s_ashr_i64 s[4:5], s[10:11], s12
+; GFX10-NEXT:    s_and_b32 s12, s15, 1
+; GFX10-NEXT:    s_or_b64 s[0:1], s[0:1], s[2:3]
+; GFX10-NEXT:    s_ashr_i32 s2, s11, 31
+; GFX10-NEXT:    s_ashr_i64 s[6:7], s[10:11], s13
+; GFX10-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX10-NEXT:    s_mov_b32 s3, s2
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[6:7]
+; GFX10-NEXT:    s_and_b32 s6, s16, 1
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc_lo
+; GFX10-NEXT:    s_cmp_lg_u32 s6, 0
+; GFX10-NEXT:    v_mov_b32_e32 v2, s9
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[8:9], s[0:1]
+; GFX10-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX10-NEXT:    v_xor_b32_e32 v0, v1, v0
+; GFX10-NEXT:    s_cselect_b64 s[2:3], s[4:5], s[2:3]
+; GFX10-NEXT:    s_add_u32 s0, s0, 0
+; GFX10-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s8
+; GFX10-NEXT:    s_and_b32 s4, s4, 1
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX10-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX10-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX10-NEXT:    s_and_b32 s4, s4, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX10-NEXT:    s_addc_u32 s2, s2, 0
+; GFX10-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v1, s0, vcc_lo
+; GFX10-NEXT:    s_and_b32 s4, s4, 1
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v2, s1, vcc_lo
+; GFX10-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v3, s2, vcc_lo
+; GFX10-NEXT:    s_addc_u32 s3, s3, 0x80000000
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v4, s3, vcc_lo
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i128 @llvm.sadd.sat.i128(i128 %lhs, i128 %rhs)
+  ret i128 %result
+}
+
+define amdgpu_ps <4 x float> @saddsat_i128_sv(i128 inreg %lhs, i128 %rhs) {
+; GFX6-LABEL: saddsat_i128_sv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v5, s1
+; GFX6-NEXT:    v_add_i32_e32 v4, vcc, s0, v0
+; GFX6-NEXT:    v_addc_u32_e32 v5, vcc, v5, v1, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v6, s2
+; GFX6-NEXT:    v_mov_b32_e32 v7, s3
+; GFX6-NEXT:    v_addc_u32_e32 v6, vcc, v6, v2, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v7, vcc, v7, v3, vcc
+; GFX6-NEXT:    v_cmp_gt_u64_e32 vcc, s[0:1], v[4:5]
+; GFX6-NEXT:    s_movk_i32 s0, 0x7f
+; GFX6-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_gt_i64_e32 vcc, s[2:3], v[6:7]
+; GFX6-NEXT:    s_sub_i32 s1, s0, 64
+; GFX6-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, s[2:3], v[6:7]
+; GFX6-NEXT:    s_sub_i32 s2, 64, s0
+; GFX6-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
+; GFX6-NEXT:    v_cmp_gt_u64_e32 vcc, 0, v[0:1]
+; GFX6-NEXT:    s_cmp_lt_u32 s0, 64
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[2:3]
+; GFX6-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX6-NEXT:    v_lshl_b64 v[2:3], v[6:7], s2
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX6-NEXT:    v_xor_b32_e32 v10, v0, v8
+; GFX6-NEXT:    v_lshr_b64 v[0:1], v[4:5], s0
+; GFX6-NEXT:    s_cmp_eq_u32 s0, 0
+; GFX6-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX6-NEXT:    v_ashr_i64 v[8:9], v[6:7], s0
+; GFX6-NEXT:    s_and_b32 s0, 1, s3
+; GFX6-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX6-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX6-NEXT:    v_ashr_i64 v[0:1], v[6:7], s1
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX6-NEXT:    s_and_b32 s0, 1, s4
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX6-NEXT:    s_and_b32 s0, 1, s3
+; GFX6-NEXT:    v_ashrrev_i32_e32 v11, 31, v7
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v11, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v11, v9, vcc
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, 0, v0
+; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX6-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v3, v8, vcc
+; GFX6-NEXT:    v_and_b32_e32 v8, 1, v10
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: saddsat_i128_sv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, s0, v0
+; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, v5, v1, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v6, s2
+; GFX8-NEXT:    v_mov_b32_e32 v7, s3
+; GFX8-NEXT:    v_addc_u32_e32 v6, vcc, v6, v2, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v7, vcc, v7, v3, vcc
+; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[0:1], v[4:5]
+; GFX8-NEXT:    s_movk_i32 s0, 0x7f
+; GFX8-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_gt_i64_e32 vcc, s[2:3], v[6:7]
+; GFX8-NEXT:    s_sub_i32 s1, s0, 64
+; GFX8-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, s[2:3], v[6:7]
+; GFX8-NEXT:    s_sub_i32 s2, 64, s0
+; GFX8-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
+; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, 0, v[0:1]
+; GFX8-NEXT:    s_cmp_lt_u32 s0, 64
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[2:3]
+; GFX8-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX8-NEXT:    v_lshlrev_b64 v[2:3], s2, v[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT:    v_xor_b32_e32 v10, v0, v8
+; GFX8-NEXT:    v_lshrrev_b64 v[0:1], s0, v[4:5]
+; GFX8-NEXT:    s_cmp_eq_u32 s0, 0
+; GFX8-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX8-NEXT:    v_ashrrev_i64 v[8:9], s0, v[6:7]
+; GFX8-NEXT:    s_and_b32 s0, 1, s3
+; GFX8-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX8-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX8-NEXT:    v_ashrrev_i64 v[0:1], s1, v[6:7]
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    s_and_b32 s0, 1, s4
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    s_and_b32 s0, 1, s3
+; GFX8-NEXT:    v_ashrrev_i32_e32 v11, 31, v7
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v11, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v11, v9, vcc
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 0, v0
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX8-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, v3, v8, vcc
+; GFX8-NEXT:    v_and_b32_e32 v8, 1, v10
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: saddsat_i128_sv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v5, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, s0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v5, v1, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v6, s2
+; GFX9-NEXT:    v_mov_b32_e32 v7, s3
+; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, v6, v2, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, v7, v3, vcc
+; GFX9-NEXT:    v_cmp_gt_u64_e32 vcc, s[0:1], v[4:5]
+; GFX9-NEXT:    s_movk_i32 s0, 0x7f
+; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_gt_i64_e32 vcc, s[2:3], v[6:7]
+; GFX9-NEXT:    s_sub_i32 s1, s0, 64
+; GFX9-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, s[2:3], v[6:7]
+; GFX9-NEXT:    s_sub_i32 s2, 64, s0
+; GFX9-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
+; GFX9-NEXT:    v_cmp_gt_u64_e32 vcc, 0, v[0:1]
+; GFX9-NEXT:    s_cmp_lt_u32 s0, 64
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[2:3]
+; GFX9-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX9-NEXT:    v_lshlrev_b64 v[2:3], s2, v[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX9-NEXT:    v_xor_b32_e32 v10, v0, v8
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], s0, v[4:5]
+; GFX9-NEXT:    s_cmp_eq_u32 s0, 0
+; GFX9-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX9-NEXT:    v_ashrrev_i64 v[8:9], s0, v[6:7]
+; GFX9-NEXT:    s_and_b32 s0, 1, s3
+; GFX9-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX9-NEXT:    v_ashrrev_i64 v[0:1], s1, v[6:7]
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    s_and_b32 s0, 1, s4
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    s_and_b32 s0, 1, s3
+; GFX9-NEXT:    v_ashrrev_i32_e32 v11, 31, v7
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v11, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v11, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v8, vcc
+; GFX9-NEXT:    v_and_b32_e32 v8, 1, v10
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: saddsat_i128_sv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_add_co_u32_e64 v4, vcc_lo, s0, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, s1, v1, vcc_lo
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, s2, v2, vcc_lo
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, s3, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[0:1], v[4:5]
+; GFX10-NEXT:    s_movk_i32 s0, 0x7f
+; GFX10-NEXT:    s_sub_i32 s1, 64, s0
+; GFX10-NEXT:    v_lshrrev_b64 v[15:16], s0, v[4:5]
+; GFX10-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_i64_e32 vcc_lo, s[2:3], v[6:7]
+; GFX10-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[2:3], v[6:7]
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v9, v8, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_u64_e32 vcc_lo, 0, v[0:1]
+; GFX10-NEXT:    v_lshlrev_b64 v[8:9], s1, v[6:7]
+; GFX10-NEXT:    s_sub_i32 s1, s0, 64
+; GFX10-NEXT:    s_cmp_lt_u32 s0, 64
+; GFX10-NEXT:    v_ashrrev_i64 v[0:1], s0, v[6:7]
+; GFX10-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_i64_e32 vcc_lo, 0, v[2:3]
+; GFX10-NEXT:    v_or_b32_e32 v8, v15, v8
+; GFX10-NEXT:    v_or_b32_e32 v9, v16, v9
+; GFX10-NEXT:    v_ashrrev_i32_e32 v15, 31, v7
+; GFX10-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[2:3]
+; GFX10-NEXT:    v_ashrrev_i64 v[2:3], s1, v[6:7]
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v12, v11, vcc_lo
+; GFX10-NEXT:    s_cselect_b32 vcc_lo, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s0, 0
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v9, vcc_lo
+; GFX10-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc_lo
+; GFX10-NEXT:    s_and_b32 s0, 1, s1
+; GFX10-NEXT:    s_and_b32 s1, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s0
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s0, 0, s1
+; GFX10-NEXT:    v_xor_b32_e32 v9, v11, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v15, v0, s0
+; GFX10-NEXT:    v_and_b32_e32 v8, 1, v9
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v15, v1, s0
+; GFX10-NEXT:    v_add_co_u32_e64 v2, vcc_lo, v2, 0
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s0, 0, v8
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, 0, v0, vcc_lo
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, 0x80000000, v1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v4, v2, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v5, v3, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v6, v8, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i128 @llvm.sadd.sat.i128(i128 %lhs, i128 %rhs)
+  %cast = bitcast i128 %result to <4 x float>
+  ret <4 x float> %cast
+}
+
+define amdgpu_ps <4 x float> @saddsat_i128_vs(i128 %lhs, i128 inreg %rhs) {
+; GFX6-LABEL: saddsat_i128_vs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v5, s1
+; GFX6-NEXT:    v_add_i32_e32 v4, vcc, s0, v0
+; GFX6-NEXT:    v_addc_u32_e32 v5, vcc, v1, v5, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v6, s2
+; GFX6-NEXT:    v_mov_b32_e32 v7, s3
+; GFX6-NEXT:    v_addc_u32_e32 v6, vcc, v2, v6, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v7, vcc, v3, v7, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, v[4:5], v[0:1]
+; GFX6-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[0:1], 0
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, v[6:7], v[2:3]
+; GFX6-NEXT:    v_ashrrev_i32_e32 v11, 31, v7
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; GFX6-NEXT:    v_cmp_lt_i64_e64 s[0:1], s[2:3], 0
+; GFX6-NEXT:    v_cmp_eq_u64_e64 vcc, s[2:3], 0
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX6-NEXT:    s_movk_i32 s0, 0x7f
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX6-NEXT:    s_sub_i32 s2, 64, s0
+; GFX6-NEXT:    s_sub_i32 s1, s0, 64
+; GFX6-NEXT:    v_xor_b32_e32 v10, v1, v0
+; GFX6-NEXT:    s_cmp_lt_u32 s0, 64
+; GFX6-NEXT:    v_lshr_b64 v[0:1], v[4:5], s0
+; GFX6-NEXT:    v_lshl_b64 v[2:3], v[6:7], s2
+; GFX6-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s0, 0
+; GFX6-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX6-NEXT:    v_ashr_i64 v[8:9], v[6:7], s0
+; GFX6-NEXT:    s_and_b32 s0, 1, s3
+; GFX6-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX6-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX6-NEXT:    v_ashr_i64 v[0:1], v[6:7], s1
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX6-NEXT:    s_and_b32 s0, 1, s4
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX6-NEXT:    s_and_b32 s0, 1, s3
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v11, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v11, v9, vcc
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, 0, v0
+; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX6-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v3, v8, vcc
+; GFX6-NEXT:    v_and_b32_e32 v8, 1, v10
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: saddsat_i128_vs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, s0, v0
+; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, v1, v5, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v6, s2
+; GFX8-NEXT:    v_mov_b32_e32 v7, s3
+; GFX8-NEXT:    v_addc_u32_e32 v6, vcc, v2, v6, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v7, vcc, v3, v7, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[4:5], v[0:1]
+; GFX8-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[0:1], 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, v[6:7], v[2:3]
+; GFX8-NEXT:    s_cmp_eq_u64 s[2:3], 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX8-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; GFX8-NEXT:    v_cmp_lt_i64_e64 s[0:1], s[2:3], 0
+; GFX8-NEXT:    v_ashrrev_i32_e32 v11, 31, v7
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX8-NEXT:    s_and_b32 s0, 1, s4
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    s_movk_i32 s0, 0x7f
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX8-NEXT:    s_sub_i32 s2, 64, s0
+; GFX8-NEXT:    s_sub_i32 s1, s0, 64
+; GFX8-NEXT:    v_xor_b32_e32 v10, v1, v0
+; GFX8-NEXT:    s_cmp_lt_u32 s0, 64
+; GFX8-NEXT:    v_lshrrev_b64 v[0:1], s0, v[4:5]
+; GFX8-NEXT:    v_lshlrev_b64 v[2:3], s2, v[6:7]
+; GFX8-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s0, 0
+; GFX8-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX8-NEXT:    v_ashrrev_i64 v[8:9], s0, v[6:7]
+; GFX8-NEXT:    s_and_b32 s0, 1, s3
+; GFX8-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX8-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX8-NEXT:    v_ashrrev_i64 v[0:1], s1, v[6:7]
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    s_and_b32 s0, 1, s4
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    s_and_b32 s0, 1, s3
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v11, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v11, v9, vcc
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 0, v0
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX8-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, v3, v8, vcc
+; GFX8-NEXT:    v_and_b32_e32 v8, 1, v10
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: saddsat_i128_vs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v5, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, s0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v1, v5, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v6, s2
+; GFX9-NEXT:    v_mov_b32_e32 v7, s3
+; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, v2, v6, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, v3, v7, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[4:5], v[0:1]
+; GFX9-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[0:1], 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, v[6:7], v[2:3]
+; GFX9-NEXT:    s_cmp_eq_u64 s[2:3], 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX9-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; GFX9-NEXT:    v_cmp_lt_i64_e64 s[0:1], s[2:3], 0
+; GFX9-NEXT:    v_ashrrev_i32_e32 v11, 31, v7
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX9-NEXT:    s_and_b32 s0, 1, s4
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    s_movk_i32 s0, 0x7f
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    s_sub_i32 s2, 64, s0
+; GFX9-NEXT:    s_sub_i32 s1, s0, 64
+; GFX9-NEXT:    v_xor_b32_e32 v10, v1, v0
+; GFX9-NEXT:    s_cmp_lt_u32 s0, 64
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], s0, v[4:5]
+; GFX9-NEXT:    v_lshlrev_b64 v[2:3], s2, v[6:7]
+; GFX9-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s0, 0
+; GFX9-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX9-NEXT:    v_ashrrev_i64 v[8:9], s0, v[6:7]
+; GFX9-NEXT:    s_and_b32 s0, 1, s3
+; GFX9-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX9-NEXT:    v_ashrrev_i64 v[0:1], s1, v[6:7]
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    s_and_b32 s0, 1, s4
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    s_and_b32 s0, 1, s3
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v11, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v11, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v8, vcc
+; GFX9-NEXT:    v_and_b32_e32 v8, 1, v10
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: saddsat_i128_vs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_mov_b32_e32 v5, v0
+; GFX10-NEXT:    v_mov_b32_e32 v6, v1
+; GFX10-NEXT:    v_mov_b32_e32 v9, v2
+; GFX10-NEXT:    v_mov_b32_e32 v10, v3
+; GFX10-NEXT:    s_cmp_eq_u64 s[2:3], 0
+; GFX10-NEXT:    v_add_co_u32_e64 v15, vcc_lo, v5, s0
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s0, s[0:1], 0
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v16, vcc_lo, s1, v6, vcc_lo
+; GFX10-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v19, vcc_lo, s2, v9, vcc_lo
+; GFX10-NEXT:    s_and_b32 s1, 1, s4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v20, vcc_lo, s3, v10, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[15:16], v[5:6]
+; GFX10-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s0
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s0, s[2:3], 0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_ashrrev_i32_e32 v7, 31, v20
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[19:20], v[9:10]
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[19:20], v[9:10]
+; GFX10-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s0
+; GFX10-NEXT:    s_movk_i32 s0, 0x7f
+; GFX10-NEXT:    s_sub_i32 s2, 64, s0
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v1, v0, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s1
+; GFX10-NEXT:    v_lshrrev_b64 v[0:1], s0, v[15:16]
+; GFX10-NEXT:    v_lshlrev_b64 v[2:3], s2, v[19:20]
+; GFX10-NEXT:    s_sub_i32 s1, s0, 64
+; GFX10-NEXT:    s_cmp_lt_u32 s0, 64
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v9, v8, vcc_lo
+; GFX10-NEXT:    s_cselect_b32 vcc_lo, 1, 0
+; GFX10-NEXT:    v_ashrrev_i64 v[8:9], s1, v[19:20]
+; GFX10-NEXT:    s_cmp_eq_u32 s0, 0
+; GFX10-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX10-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX10-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX10-NEXT:    v_ashrrev_i64 v[0:1], s0, v[19:20]
+; GFX10-NEXT:    s_and_b32 s0, 1, s1
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc_lo
+; GFX10-NEXT:    s_and_b32 s1, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s0
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s0, 0, s1
+; GFX10-NEXT:    v_xor_b32_e32 v9, v11, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v15, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v16, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v7, v0, s0
+; GFX10-NEXT:    v_and_b32_e32 v8, 1, v9
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v7, v1, s0
+; GFX10-NEXT:    v_add_co_u32_e64 v2, vcc_lo, v2, 0
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s0, 0, v8
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, 0, v0, vcc_lo
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, 0x80000000, v1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v15, v2, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v16, v3, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v19, v8, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v20, v9, s0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i128 @llvm.sadd.sat.i128(i128 %lhs, i128 %rhs)
+  %cast = bitcast i128 %result to <4 x float>
+  ret <4 x float> %cast
+}
+
+define <2 x i128> @v_saddsat_v2i128(<2 x i128> %lhs, <2 x i128> %rhs) {
+; GFX6-LABEL: v_saddsat_v2i128:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_add_i32_e32 v16, vcc, v0, v8
+; GFX6-NEXT:    v_addc_u32_e32 v17, vcc, v1, v9, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v18, vcc, v2, v10, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v19, vcc, v3, v11, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, v[16:17], v[0:1]
+; GFX6-NEXT:    s_movk_i32 s6, 0x7f
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, v[18:19], v[2:3]
+; GFX6-NEXT:    s_sub_i32 s7, s6, 64
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, v[18:19], v[2:3]
+; GFX6-NEXT:    s_sub_i32 s8, 64, s6
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX6-NEXT:    v_cmp_gt_u64_e32 vcc, 0, v[8:9]
+; GFX6-NEXT:    s_cmp_lt_u32 s6, 64
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[10:11]
+; GFX6-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[10:11]
+; GFX6-NEXT:    s_cmp_eq_u32 s6, 0
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX6-NEXT:    v_xor_b32_e32 v10, v1, v0
+; GFX6-NEXT:    v_lshr_b64 v[0:1], v[16:17], s6
+; GFX6-NEXT:    v_lshl_b64 v[2:3], v[18:19], s8
+; GFX6-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX6-NEXT:    s_and_b32 s4, 1, s4
+; GFX6-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX6-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX6-NEXT:    v_ashr_i64 v[0:1], v[18:19], s7
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    s_and_b32 s4, 1, s5
+; GFX6-NEXT:    v_ashr_i64 v[8:9], v[18:19], s6
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, s4
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX6-NEXT:    v_ashrrev_i32_e32 v11, 31, v19
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v0, v16, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v1, v17, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v11, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v11, v9, vcc
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, 0, v0
+; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX6-NEXT:    s_brev_b32 s4, 1
+; GFX6-NEXT:    v_mov_b32_e32 v8, s4
+; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v3, v8, vcc
+; GFX6-NEXT:    v_and_b32_e32 v8, 1, v10
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v18, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v19, v3, vcc
+; GFX6-NEXT:    v_add_i32_e32 v8, vcc, v4, v12
+; GFX6-NEXT:    v_addc_u32_e32 v9, vcc, v5, v13, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v10, vcc, v6, v14, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v11, vcc, v7, v15, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
+; GFX6-NEXT:    s_cmp_lt_u32 s6, 64
+; GFX6-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, v[10:11], v[6:7]
+; GFX6-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX6-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, v[10:11], v[6:7]
+; GFX6-NEXT:    s_cmp_eq_u32 s6, 0
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
+; GFX6-NEXT:    v_cmp_gt_u64_e32 vcc, 0, v[12:13]
+; GFX6-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX6-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[14:15]
+; GFX6-NEXT:    v_ashr_i64 v[12:13], v[10:11], s6
+; GFX6-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[14:15]
+; GFX6-NEXT:    v_ashrrev_i32_e32 v15, 31, v11
+; GFX6-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
+; GFX6-NEXT:    v_xor_b32_e32 v14, v5, v4
+; GFX6-NEXT:    v_lshr_b64 v[4:5], v[8:9], s6
+; GFX6-NEXT:    v_lshl_b64 v[6:7], v[10:11], s8
+; GFX6-NEXT:    s_and_b32 s6, 1, s5
+; GFX6-NEXT:    v_or_b32_e32 v6, v4, v6
+; GFX6-NEXT:    v_or_b32_e32 v7, v5, v7
+; GFX6-NEXT:    v_ashr_i64 v[4:5], v[10:11], s7
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s6
+; GFX6-NEXT:    s_and_b32 s6, 1, s9
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s6
+; GFX6-NEXT:    s_and_b32 s5, 1, s5
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, v4, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v5, v5, v9, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s5
+; GFX6-NEXT:    v_cndmask_b32_e32 v6, v15, v12, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v7, v15, v13, vcc
+; GFX6-NEXT:    v_add_i32_e32 v4, vcc, 0, v4
+; GFX6-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v12, s4
+; GFX6-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v7, vcc, v7, v12, vcc
+; GFX6-NEXT:    v_and_b32_e32 v12, 1, v14
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v5, v9, v5, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v6, v10, v6, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v7, v11, v7, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_saddsat_v2i128:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v16, vcc, v0, v8
+; GFX8-NEXT:    v_addc_u32_e32 v17, vcc, v1, v9, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v18, vcc, v2, v10, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v19, vcc, v3, v11, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[16:17], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s6, 0x7f
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, v[18:19], v[2:3]
+; GFX8-NEXT:    s_sub_i32 s7, s6, 64
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[18:19], v[2:3]
+; GFX8-NEXT:    s_sub_i32 s8, 64, s6
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, 0, v[8:9]
+; GFX8-NEXT:    s_cmp_lt_u32 s6, 64
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[10:11]
+; GFX8-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[10:11]
+; GFX8-NEXT:    s_cmp_eq_u32 s6, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX8-NEXT:    v_xor_b32_e32 v10, v1, v0
+; GFX8-NEXT:    v_lshrrev_b64 v[0:1], s6, v[16:17]
+; GFX8-NEXT:    v_lshlrev_b64 v[2:3], s8, v[18:19]
+; GFX8-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX8-NEXT:    s_and_b32 s4, 1, s4
+; GFX8-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX8-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX8-NEXT:    v_ashrrev_i64 v[0:1], s7, v[18:19]
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    s_and_b32 s4, 1, s5
+; GFX8-NEXT:    v_ashrrev_i64 v[8:9], s6, v[18:19]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, s4
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX8-NEXT:    v_ashrrev_i32_e32 v11, 31, v19
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v16, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v17, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v11, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v11, v9, vcc
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 0, v0
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT:    s_brev_b32 s4, 1
+; GFX8-NEXT:    v_mov_b32_e32 v8, s4
+; GFX8-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, v3, v8, vcc
+; GFX8-NEXT:    v_and_b32_e32 v8, 1, v10
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v18, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v19, v3, vcc
+; GFX8-NEXT:    v_add_u32_e32 v8, vcc, v4, v12
+; GFX8-NEXT:    v_addc_u32_e32 v9, vcc, v5, v13, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v10, vcc, v6, v14, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v11, vcc, v7, v15, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
+; GFX8-NEXT:    s_cmp_lt_u32 s6, 64
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, v[10:11], v[6:7]
+; GFX8-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[10:11], v[6:7]
+; GFX8-NEXT:    s_cmp_eq_u32 s6, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
+; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, 0, v[12:13]
+; GFX8-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[14:15]
+; GFX8-NEXT:    v_ashrrev_i64 v[12:13], s6, v[10:11]
+; GFX8-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[14:15]
+; GFX8-NEXT:    v_ashrrev_i32_e32 v15, 31, v11
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
+; GFX8-NEXT:    v_xor_b32_e32 v14, v5, v4
+; GFX8-NEXT:    v_lshrrev_b64 v[4:5], s6, v[8:9]
+; GFX8-NEXT:    v_lshlrev_b64 v[6:7], s8, v[10:11]
+; GFX8-NEXT:    s_and_b32 s6, 1, s5
+; GFX8-NEXT:    v_or_b32_e32 v6, v4, v6
+; GFX8-NEXT:    v_or_b32_e32 v7, v5, v7
+; GFX8-NEXT:    v_ashrrev_i64 v[4:5], s7, v[10:11]
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s6
+; GFX8-NEXT:    s_and_b32 s6, 1, s9
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s6
+; GFX8-NEXT:    s_and_b32 s5, 1, s5
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v9, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s5
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v15, v12, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v15, v13, vcc
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 0, v4
+; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v12, s4
+; GFX8-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v7, vcc, v7, v12, vcc
+; GFX8-NEXT:    v_and_b32_e32 v12, 1, v14
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v9, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v10, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v11, v7, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_saddsat_v2i128:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, v0, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v17, vcc, v1, v9, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v18, vcc, v2, v10, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v19, vcc, v3, v11, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[16:17], v[0:1]
+; GFX9-NEXT:    s_movk_i32 s6, 0x7f
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, v[18:19], v[2:3]
+; GFX9-NEXT:    s_sub_i32 s7, s6, 64
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[18:19], v[2:3]
+; GFX9-NEXT:    s_sub_i32 s8, 64, s6
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX9-NEXT:    v_cmp_gt_u64_e32 vcc, 0, v[8:9]
+; GFX9-NEXT:    s_cmp_lt_u32 s6, 64
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[10:11]
+; GFX9-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[10:11]
+; GFX9-NEXT:    s_cmp_eq_u32 s6, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    v_xor_b32_e32 v10, v1, v0
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], s6, v[16:17]
+; GFX9-NEXT:    v_lshlrev_b64 v[2:3], s8, v[18:19]
+; GFX9-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX9-NEXT:    s_and_b32 s4, 1, s4
+; GFX9-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX9-NEXT:    v_ashrrev_i64 v[0:1], s7, v[18:19]
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    s_and_b32 s4, 1, s5
+; GFX9-NEXT:    v_ashrrev_i64 v[8:9], s6, v[18:19]
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, s4
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT:    v_ashrrev_i32_e32 v11, 31, v19
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v16, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v17, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v11, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v11, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_brev_b32 s4, 1
+; GFX9-NEXT:    v_mov_b32_e32 v8, s4
+; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v8, vcc
+; GFX9-NEXT:    v_and_b32_e32 v8, 1, v10
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v18, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v19, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v4, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, v5, v13, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v10, vcc, v6, v14, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, v7, v15, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
+; GFX9-NEXT:    s_cmp_lt_u32 s6, 64
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, v[10:11], v[6:7]
+; GFX9-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[10:11], v[6:7]
+; GFX9-NEXT:    s_cmp_eq_u32 s6, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
+; GFX9-NEXT:    v_cmp_gt_u64_e32 vcc, 0, v[12:13]
+; GFX9-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[14:15]
+; GFX9-NEXT:    v_ashrrev_i64 v[12:13], s6, v[10:11]
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[14:15]
+; GFX9-NEXT:    v_ashrrev_i32_e32 v15, 31, v11
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
+; GFX9-NEXT:    v_xor_b32_e32 v14, v5, v4
+; GFX9-NEXT:    v_lshrrev_b64 v[4:5], s6, v[8:9]
+; GFX9-NEXT:    v_lshlrev_b64 v[6:7], s8, v[10:11]
+; GFX9-NEXT:    s_and_b32 s6, 1, s5
+; GFX9-NEXT:    v_or_b32_e32 v6, v4, v6
+; GFX9-NEXT:    v_or_b32_e32 v7, v5, v7
+; GFX9-NEXT:    v_ashrrev_i64 v[4:5], s7, v[10:11]
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s6
+; GFX9-NEXT:    s_and_b32 s6, 1, s9
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s6
+; GFX9-NEXT:    s_and_b32 s5, 1, s5
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v9, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s5
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, v15, v12, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v7, v15, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v12, s4
+; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, 0, v6, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, v7, v12, vcc
+; GFX9-NEXT:    v_and_b32_e32 v12, 1, v14
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v9, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, v10, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v7, v11, v7, vcc
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_saddsat_v2i128:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_mov_b32_e32 v22, v0
+; GFX10-NEXT:    v_mov_b32_e32 v23, v1
+; GFX10-NEXT:    v_mov_b32_e32 v20, v2
+; GFX10-NEXT:    v_mov_b32_e32 v21, v3
+; GFX10-NEXT:    s_movk_i32 s5, 0x7f
+; GFX10-NEXT:    v_add_co_u32_e64 v16, vcc_lo, v22, v8
+; GFX10-NEXT:    s_sub_i32 s6, 64, s5
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v17, vcc_lo, v23, v9, vcc_lo
+; GFX10-NEXT:    s_sub_i32 s7, s5, 64
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v18, vcc_lo, v20, v10, vcc_lo
+; GFX10-NEXT:    s_cmp_lt_u32 s5, 64
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v19, vcc_lo, v21, v11, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[16:17], v[22:23]
+; GFX10-NEXT:    v_mov_b32_e32 v26, v4
+; GFX10-NEXT:    v_mov_b32_e32 v27, v5
+; GFX10-NEXT:    v_mov_b32_e32 v24, v6
+; GFX10-NEXT:    v_lshlrev_b64 v[2:3], s6, v[18:19]
+; GFX10-NEXT:    v_mov_b32_e32 v25, v7
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[18:19], v[20:21]
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[18:19], v[20:21]
+; GFX10-NEXT:    v_cndmask_b32_e32 v20, v1, v0, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_u64_e32 vcc_lo, 0, v[8:9]
+; GFX10-NEXT:    v_lshrrev_b64 v[0:1], s5, v[16:17]
+; GFX10-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_i64_e32 vcc_lo, 0, v[10:11]
+; GFX10-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX10-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX10-NEXT:    v_ashrrev_i64 v[0:1], s5, v[18:19]
+; GFX10-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[10:11]
+; GFX10-NEXT:    v_ashrrev_i32_e32 v11, 31, v19
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v9, v8, vcc_lo
+; GFX10-NEXT:    s_cselect_b32 vcc_lo, 1, 0
+; GFX10-NEXT:    v_ashrrev_i64 v[8:9], s7, v[18:19]
+; GFX10-NEXT:    s_cmp_eq_u32 s5, 0
+; GFX10-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX10-NEXT:    s_and_b32 s8, 1, vcc_lo
+; GFX10-NEXT:    s_and_b32 s4, 1, s4
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s4
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s4, 0, s8
+; GFX10-NEXT:    v_xor_b32_e32 v9, v10, v20
+; GFX10-NEXT:    s_brev_b32 s8, 1
+; GFX10-NEXT:    s_cmp_lt_u32 s5, 64
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v16, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v17, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v11, v0, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v11, v1, s4
+; GFX10-NEXT:    v_and_b32_e32 v8, 1, v9
+; GFX10-NEXT:    v_add_co_u32_e64 v2, vcc_lo, v2, 0
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v20, vcc_lo, 0, v0, vcc_lo
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v21, vcc_lo, s8, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v8
+; GFX10-NEXT:    v_add_co_u32_e64 v8, s4, v26, v12
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v9, s4, v27, v13, s4
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v10, s4, v24, v14, s4
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v16, v2, vcc_lo
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v11, s4, v25, v15, s4
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s4, v[8:9], v[26:27]
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v17, v3, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v18, v20, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b64 v[3:4], s5, v[8:9]
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, 0, 1, s4
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s4, v[10:11], v[24:25]
+; GFX10-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s4
+; GFX10-NEXT:    v_cmp_gt_u64_e64 s4, 0, v[12:13]
+; GFX10-NEXT:    v_lshlrev_b64 v[12:13], s6, v[10:11]
+; GFX10-NEXT:    v_cndmask_b32_e64 v17, 0, 1, s4
+; GFX10-NEXT:    v_cmp_gt_i64_e64 s4, 0, v[14:15]
+; GFX10-NEXT:    v_or_b32_e32 v12, v3, v12
+; GFX10-NEXT:    v_or_b32_e32 v13, v4, v13
+; GFX10-NEXT:    v_ashrrev_i64 v[3:4], s5, v[10:11]
+; GFX10-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s4
+; GFX10-NEXT:    v_cmp_eq_u64_e64 s4, v[10:11], v[24:25]
+; GFX10-NEXT:    v_cndmask_b32_e64 v7, v16, v5, s4
+; GFX10-NEXT:    v_cmp_eq_u64_e64 s4, 0, v[14:15]
+; GFX10-NEXT:    v_ashrrev_i64 v[5:6], s7, v[10:11]
+; GFX10-NEXT:    v_cndmask_b32_e64 v14, v18, v17, s4
+; GFX10-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s5, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, v12, s4
+; GFX10-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, v13, s4
+; GFX10-NEXT:    s_and_b32 s5, 1, s6
+; GFX10-NEXT:    s_and_b32 s6, 1, s4
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s4, 0, s5
+; GFX10-NEXT:    v_xor_b32_e32 v7, v14, v7
+; GFX10-NEXT:    v_ashrrev_i32_e32 v18, 31, v11
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s5, 0, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, v8, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, v9, s4
+; GFX10-NEXT:    v_and_b32_e32 v7, 1, v7
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v18, v3, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, v18, v4, s5
+; GFX10-NEXT:    v_add_co_u32_e64 v5, s4, v5, 0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v6, s4, 0, v6, s4
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s5, 0, v7
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v7, s4, 0, v3, s4
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v19, v21, vcc_lo
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v12, s4, s8, v4, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, v8, v5, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v9, v6, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v10, v7, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v7, v11, v12, s5
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i128> @llvm.sadd.sat.v2i128(<2 x i128> %lhs, <2 x i128> %rhs)
+  ret <2 x i128> %result
+}
+
+define amdgpu_ps <2 x i128> @s_saddsat_v2i128(<2 x i128> inreg %lhs, <2 x i128> inreg %rhs) {
+; GFX6-LABEL: s_saddsat_v2i128:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_add_u32 s16, s0, s8
+; GFX6-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX6-NEXT:    s_and_b32 s17, s17, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX6-NEXT:    s_addc_u32 s17, s1, s9
+; GFX6-NEXT:    s_cselect_b32 s18, 1, 0
+; GFX6-NEXT:    s_and_b32 s18, s18, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s18, 0
+; GFX6-NEXT:    s_addc_u32 s18, s2, s10
+; GFX6-NEXT:    s_cselect_b32 s19, 1, 0
+; GFX6-NEXT:    v_mov_b32_e32 v3, s1
+; GFX6-NEXT:    s_and_b32 s19, s19, 1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[16:17], v[2:3]
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    s_addc_u32 s19, s3, s11
+; GFX6-NEXT:    s_movk_i32 s20, 0x7f
+; GFX6-NEXT:    v_mov_b32_e32 v1, s3
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, s[18:19], v[0:1]
+; GFX6-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[8:9], 0
+; GFX6-NEXT:    s_sub_i32 s21, s20, 64
+; GFX6-NEXT:    s_sub_i32 s22, 64, s20
+; GFX6-NEXT:    s_cmp_lt_u32 s20, 64
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, s[18:19], v[0:1]
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; GFX6-NEXT:    s_cselect_b32 s23, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s20, 0
+; GFX6-NEXT:    v_cmp_lt_i64_e64 s[0:1], s[10:11], 0
+; GFX6-NEXT:    s_cselect_b32 s24, 1, 0
+; GFX6-NEXT:    s_lshr_b64 s[2:3], s[16:17], s20
+; GFX6-NEXT:    s_lshl_b64 s[8:9], s[18:19], s22
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX6-NEXT:    v_cmp_eq_u64_e64 vcc, s[10:11], 0
+; GFX6-NEXT:    s_or_b64 s[2:3], s[2:3], s[8:9]
+; GFX6-NEXT:    s_ashr_i32 s8, s19, 31
+; GFX6-NEXT:    s_ashr_i64 s[0:1], s[18:19], s20
+; GFX6-NEXT:    s_ashr_i64 s[10:11], s[18:19], s21
+; GFX6-NEXT:    s_and_b32 s23, s23, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s23, 0
+; GFX6-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[10:11]
+; GFX6-NEXT:    s_and_b32 s10, s24, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX6-NEXT:    s_cselect_b64 s[2:3], s[16:17], s[2:3]
+; GFX6-NEXT:    s_cmp_lg_u32 s23, 0
+; GFX6-NEXT:    s_mov_b32 s9, s8
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[8:9]
+; GFX6-NEXT:    s_add_u32 s2, s2, 0
+; GFX6-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX6-NEXT:    s_and_b32 s8, s8, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX6-NEXT:    s_addc_u32 s3, s3, 0
+; GFX6-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX6-NEXT:    s_and_b32 s8, s8, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX6-NEXT:    s_addc_u32 s0, s0, 0
+; GFX6-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX6-NEXT:    s_and_b32 s8, s8, 1
+; GFX6-NEXT:    v_xor_b32_e32 v0, v1, v0
+; GFX6-NEXT:    s_brev_b32 s23, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX6-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX6-NEXT:    s_addc_u32 s1, s1, s23
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v3, s16
+; GFX6-NEXT:    s_add_u32 s0, s4, s12
+; GFX6-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX6-NEXT:    s_and_b32 s1, s1, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX6-NEXT:    s_addc_u32 s1, s5, s13
+; GFX6-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX6-NEXT:    s_and_b32 s2, s2, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX6-NEXT:    v_mov_b32_e32 v2, s3
+; GFX6-NEXT:    v_mov_b32_e32 v4, s17
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v2, s18
+; GFX6-NEXT:    v_mov_b32_e32 v3, s19
+; GFX6-NEXT:    s_addc_u32 s2, s6, s14
+; GFX6-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX6-NEXT:    v_cndmask_b32_e32 v6, v2, v0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v7, v3, v1, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v2, s4
+; GFX6-NEXT:    s_and_b32 s3, s3, 1
+; GFX6-NEXT:    v_mov_b32_e32 v3, s5
+; GFX6-NEXT:    s_cmp_lg_u32 s3, 0
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX6-NEXT:    v_mov_b32_e32 v0, s6
+; GFX6-NEXT:    s_addc_u32 s3, s7, s15
+; GFX6-NEXT:    v_mov_b32_e32 v1, s7
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
+; GFX6-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[12:13], 0
+; GFX6-NEXT:    s_cmp_lt_u32 s20, 64
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, s[2:3], v[0:1]
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[4:5]
+; GFX6-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s20, 0
+; GFX6-NEXT:    v_cmp_lt_i64_e64 s[4:5], s[14:15], 0
+; GFX6-NEXT:    s_cselect_b32 s13, 1, 0
+; GFX6-NEXT:    s_lshr_b64 s[6:7], s[0:1], s20
+; GFX6-NEXT:    s_lshl_b64 s[8:9], s[2:3], s22
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[4:5]
+; GFX6-NEXT:    s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX6-NEXT:    s_ashr_i32 s8, s3, 31
+; GFX6-NEXT:    s_ashr_i64 s[4:5], s[2:3], s20
+; GFX6-NEXT:    s_ashr_i64 s[10:11], s[2:3], s21
+; GFX6-NEXT:    s_and_b32 s12, s12, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX6-NEXT:    s_cselect_b64 s[6:7], s[6:7], s[10:11]
+; GFX6-NEXT:    s_and_b32 s10, s13, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX6-NEXT:    s_cselect_b64 s[6:7], s[0:1], s[6:7]
+; GFX6-NEXT:    s_mov_b32 s9, s8
+; GFX6-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX6-NEXT:    s_cselect_b64 s[4:5], s[4:5], s[8:9]
+; GFX6-NEXT:    s_add_u32 s6, s6, 0
+; GFX6-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX6-NEXT:    s_and_b32 s8, s8, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX6-NEXT:    s_addc_u32 s7, s7, 0
+; GFX6-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX6-NEXT:    s_and_b32 s8, s8, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX6-NEXT:    v_cmp_eq_u64_e64 vcc, s[14:15], 0
+; GFX6-NEXT:    s_addc_u32 s4, s4, 0
+; GFX6-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX6-NEXT:    v_xor_b32_e32 v0, v1, v0
+; GFX6-NEXT:    s_and_b32 s8, s8, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX6-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX6-NEXT:    v_mov_b32_e32 v3, s0
+; GFX6-NEXT:    v_mov_b32_e32 v8, s1
+; GFX6-NEXT:    s_addc_u32 s5, s5, s23
+; GFX6-NEXT:    v_mov_b32_e32 v1, s6
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v2, s7
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v3, v1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v8, v2, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v2, s4
+; GFX6-NEXT:    v_mov_b32_e32 v8, s2
+; GFX6-NEXT:    v_mov_b32_e32 v3, s5
+; GFX6-NEXT:    v_mov_b32_e32 v9, s3
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v5
+; GFX6-NEXT:    v_readfirstlane_b32 s1, v4
+; GFX6-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX6-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX6-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX6-NEXT:    v_readfirstlane_b32 s6, v2
+; GFX6-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_saddsat_v2i128:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_add_u32 s16, s0, s8
+; GFX8-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX8-NEXT:    s_and_b32 s17, s17, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX8-NEXT:    s_addc_u32 s17, s1, s9
+; GFX8-NEXT:    s_cselect_b32 s18, 1, 0
+; GFX8-NEXT:    s_and_b32 s18, s18, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s18, 0
+; GFX8-NEXT:    s_addc_u32 s18, s2, s10
+; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
+; GFX8-NEXT:    s_and_b32 s19, s19, 1
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:    s_addc_u32 s19, s3, s11
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[16:17], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    s_cmp_eq_u64 s[18:19], s[2:3]
+; GFX8-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[18:19], v[0:1]
+; GFX8-NEXT:    s_and_b32 s0, 1, s2
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[8:9], 0
+; GFX8-NEXT:    s_cmp_eq_u64 s[10:11], 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; GFX8-NEXT:    v_cmp_lt_i64_e64 s[0:1], s[10:11], 0
+; GFX8-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX8-NEXT:    s_movk_i32 s20, 0x7f
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX8-NEXT:    s_and_b32 s0, 1, s2
+; GFX8-NEXT:    s_sub_i32 s21, s20, 64
+; GFX8-NEXT:    s_sub_i32 s22, 64, s20
+; GFX8-NEXT:    s_cmp_lt_u32 s20, 64
+; GFX8-NEXT:    s_cselect_b32 s23, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s20, 0
+; GFX8-NEXT:    s_cselect_b32 s24, 1, 0
+; GFX8-NEXT:    s_lshr_b64 s[2:3], s[16:17], s20
+; GFX8-NEXT:    s_lshl_b64 s[8:9], s[18:19], s22
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    s_or_b64 s[2:3], s[2:3], s[8:9]
+; GFX8-NEXT:    s_ashr_i32 s8, s19, 31
+; GFX8-NEXT:    s_ashr_i64 s[0:1], s[18:19], s20
+; GFX8-NEXT:    s_ashr_i64 s[10:11], s[18:19], s21
+; GFX8-NEXT:    s_and_b32 s23, s23, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s23, 0
+; GFX8-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[10:11]
+; GFX8-NEXT:    s_and_b32 s10, s24, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX8-NEXT:    s_cselect_b64 s[2:3], s[16:17], s[2:3]
+; GFX8-NEXT:    s_cmp_lg_u32 s23, 0
+; GFX8-NEXT:    s_mov_b32 s9, s8
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[8:9]
+; GFX8-NEXT:    s_add_u32 s2, s2, 0
+; GFX8-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX8-NEXT:    s_and_b32 s8, s8, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX8-NEXT:    s_addc_u32 s3, s3, 0
+; GFX8-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX8-NEXT:    s_and_b32 s8, s8, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX8-NEXT:    s_addc_u32 s0, s0, 0
+; GFX8-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX8-NEXT:    s_and_b32 s8, s8, 1
+; GFX8-NEXT:    v_xor_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_brev_b32 s23, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX8-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX8-NEXT:    s_addc_u32 s1, s1, s23
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s16
+; GFX8-NEXT:    s_add_u32 s0, s4, s12
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX8-NEXT:    s_and_b32 s1, s1, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX8-NEXT:    s_addc_u32 s1, s5, s13
+; GFX8-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX8-NEXT:    s_and_b32 s2, s2, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s3
+; GFX8-NEXT:    v_mov_b32_e32 v4, s17
+; GFX8-NEXT:    s_addc_u32 s2, s6, s14
+; GFX8-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v2, s18
+; GFX8-NEXT:    v_mov_b32_e32 v3, s19
+; GFX8-NEXT:    s_and_b32 s3, s3, 1
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v2, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v3, v1, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    s_cmp_lg_u32 s3, 0
+; GFX8-NEXT:    v_mov_b32_e32 v3, s5
+; GFX8-NEXT:    s_addc_u32 s3, s7, s15
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s6
+; GFX8-NEXT:    v_mov_b32_e32 v1, s7
+; GFX8-NEXT:    s_cmp_eq_u64 s[2:3], s[6:7]
+; GFX8-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
+; GFX8-NEXT:    s_and_b32 s4, 1, s6
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[12:13], 0
+; GFX8-NEXT:    s_cmp_eq_u64 s[14:15], 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[4:5]
+; GFX8-NEXT:    v_cmp_lt_i64_e64 s[4:5], s[14:15], 0
+; GFX8-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[4:5]
+; GFX8-NEXT:    s_and_b32 s4, 1, s6
+; GFX8-NEXT:    s_cmp_lt_u32 s20, 64
+; GFX8-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s20, 0
+; GFX8-NEXT:    s_cselect_b32 s13, 1, 0
+; GFX8-NEXT:    s_lshr_b64 s[6:7], s[0:1], s20
+; GFX8-NEXT:    s_lshl_b64 s[8:9], s[2:3], s22
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX8-NEXT:    s_ashr_i32 s8, s3, 31
+; GFX8-NEXT:    s_ashr_i64 s[4:5], s[2:3], s20
+; GFX8-NEXT:    s_ashr_i64 s[10:11], s[2:3], s21
+; GFX8-NEXT:    s_and_b32 s12, s12, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX8-NEXT:    s_cselect_b64 s[6:7], s[6:7], s[10:11]
+; GFX8-NEXT:    s_and_b32 s10, s13, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX8-NEXT:    s_cselect_b64 s[6:7], s[0:1], s[6:7]
+; GFX8-NEXT:    s_mov_b32 s9, s8
+; GFX8-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX8-NEXT:    s_cselect_b64 s[4:5], s[4:5], s[8:9]
+; GFX8-NEXT:    s_add_u32 s6, s6, 0
+; GFX8-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX8-NEXT:    s_and_b32 s8, s8, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX8-NEXT:    s_addc_u32 s7, s7, 0
+; GFX8-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX8-NEXT:    s_and_b32 s8, s8, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX8-NEXT:    s_addc_u32 s4, s4, 0
+; GFX8-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX8-NEXT:    v_xor_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_and_b32 s8, s8, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX8-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v3, s0
+; GFX8-NEXT:    v_mov_b32_e32 v8, s1
+; GFX8-NEXT:    s_addc_u32 s5, s5, s23
+; GFX8-NEXT:    v_mov_b32_e32 v1, s6
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s7
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v8, v2, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_mov_b32_e32 v8, s2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s5
+; GFX8-NEXT:    v_mov_b32_e32 v9, s3
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v5
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v4
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX8-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s6, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_saddsat_v2i128:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_add_u32 s16, s0, s8
+; GFX9-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX9-NEXT:    s_and_b32 s17, s17, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX9-NEXT:    s_addc_u32 s17, s1, s9
+; GFX9-NEXT:    s_cselect_b32 s18, 1, 0
+; GFX9-NEXT:    s_and_b32 s18, s18, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s18, 0
+; GFX9-NEXT:    s_addc_u32 s18, s2, s10
+; GFX9-NEXT:    s_cselect_b32 s19, 1, 0
+; GFX9-NEXT:    s_and_b32 s19, s19, 1
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s0
+; GFX9-NEXT:    s_addc_u32 s19, s3, s11
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[16:17], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    s_cmp_eq_u64 s[18:19], s[2:3]
+; GFX9-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, s[18:19], v[0:1]
+; GFX9-NEXT:    s_and_b32 s0, 1, s2
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[8:9], 0
+; GFX9-NEXT:    s_cmp_eq_u64 s[10:11], 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; GFX9-NEXT:    v_cmp_lt_i64_e64 s[0:1], s[10:11], 0
+; GFX9-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX9-NEXT:    s_movk_i32 s20, 0x7f
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX9-NEXT:    s_and_b32 s0, 1, s2
+; GFX9-NEXT:    s_sub_i32 s21, s20, 64
+; GFX9-NEXT:    s_sub_i32 s22, 64, s20
+; GFX9-NEXT:    s_cmp_lt_u32 s20, 64
+; GFX9-NEXT:    s_cselect_b32 s23, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s20, 0
+; GFX9-NEXT:    s_cselect_b32 s24, 1, 0
+; GFX9-NEXT:    s_lshr_b64 s[2:3], s[16:17], s20
+; GFX9-NEXT:    s_lshl_b64 s[8:9], s[18:19], s22
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    s_or_b64 s[2:3], s[2:3], s[8:9]
+; GFX9-NEXT:    s_ashr_i32 s8, s19, 31
+; GFX9-NEXT:    s_ashr_i64 s[0:1], s[18:19], s20
+; GFX9-NEXT:    s_ashr_i64 s[10:11], s[18:19], s21
+; GFX9-NEXT:    s_and_b32 s23, s23, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s23, 0
+; GFX9-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[10:11]
+; GFX9-NEXT:    s_and_b32 s10, s24, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX9-NEXT:    s_cselect_b64 s[2:3], s[16:17], s[2:3]
+; GFX9-NEXT:    s_cmp_lg_u32 s23, 0
+; GFX9-NEXT:    s_mov_b32 s9, s8
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[8:9]
+; GFX9-NEXT:    s_add_u32 s2, s2, 0
+; GFX9-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX9-NEXT:    s_and_b32 s8, s8, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX9-NEXT:    s_addc_u32 s3, s3, 0
+; GFX9-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX9-NEXT:    s_and_b32 s8, s8, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX9-NEXT:    s_addc_u32 s0, s0, 0
+; GFX9-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    s_and_b32 s8, s8, 1
+; GFX9-NEXT:    v_xor_b32_e32 v0, v1, v0
+; GFX9-NEXT:    s_brev_b32 s23, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    s_addc_u32 s1, s1, s23
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    v_mov_b32_e32 v3, s16
+; GFX9-NEXT:    s_add_u32 s0, s4, s12
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX9-NEXT:    s_and_b32 s1, s1, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX9-NEXT:    s_addc_u32 s1, s5, s13
+; GFX9-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX9-NEXT:    s_and_b32 s2, s2, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-NEXT:    v_mov_b32_e32 v4, s17
+; GFX9-NEXT:    s_addc_u32 s2, s6, s14
+; GFX9-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v2, s18
+; GFX9-NEXT:    v_mov_b32_e32 v3, s19
+; GFX9-NEXT:    s_and_b32 s3, s3, 1
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, v2, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v7, v3, v1, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    s_cmp_lg_u32 s3, 0
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
+; GFX9-NEXT:    s_addc_u32 s3, s7, s15
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v0, s6
+; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    s_cmp_eq_u64 s[2:3], s[6:7]
+; GFX9-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
+; GFX9-NEXT:    s_and_b32 s4, 1, s6
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[12:13], 0
+; GFX9-NEXT:    s_cmp_eq_u64 s[14:15], 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[4:5]
+; GFX9-NEXT:    v_cmp_lt_i64_e64 s[4:5], s[14:15], 0
+; GFX9-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[4:5]
+; GFX9-NEXT:    s_and_b32 s4, 1, s6
+; GFX9-NEXT:    s_cmp_lt_u32 s20, 64
+; GFX9-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s20, 0
+; GFX9-NEXT:    s_cselect_b32 s13, 1, 0
+; GFX9-NEXT:    s_lshr_b64 s[6:7], s[0:1], s20
+; GFX9-NEXT:    s_lshl_b64 s[8:9], s[2:3], s22
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX9-NEXT:    s_ashr_i32 s8, s3, 31
+; GFX9-NEXT:    s_ashr_i64 s[4:5], s[2:3], s20
+; GFX9-NEXT:    s_ashr_i64 s[10:11], s[2:3], s21
+; GFX9-NEXT:    s_and_b32 s12, s12, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX9-NEXT:    s_cselect_b64 s[6:7], s[6:7], s[10:11]
+; GFX9-NEXT:    s_and_b32 s10, s13, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX9-NEXT:    s_cselect_b64 s[6:7], s[0:1], s[6:7]
+; GFX9-NEXT:    s_mov_b32 s9, s8
+; GFX9-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX9-NEXT:    s_cselect_b64 s[4:5], s[4:5], s[8:9]
+; GFX9-NEXT:    s_add_u32 s6, s6, 0
+; GFX9-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX9-NEXT:    s_and_b32 s8, s8, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX9-NEXT:    s_addc_u32 s7, s7, 0
+; GFX9-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX9-NEXT:    s_and_b32 s8, s8, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX9-NEXT:    s_addc_u32 s4, s4, 0
+; GFX9-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    v_xor_b32_e32 v0, v1, v0
+; GFX9-NEXT:    s_and_b32 s8, s8, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v3, s0
+; GFX9-NEXT:    v_mov_b32_e32 v8, s1
+; GFX9-NEXT:    s_addc_u32 s5, s5, s23
+; GFX9-NEXT:    v_mov_b32_e32 v1, s6
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s7
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v8, v2, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v8, s2
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
+; GFX9-NEXT:    v_mov_b32_e32 v9, s3
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v5
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v4
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX9-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s6, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_saddsat_v2i128:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s28, s0, s8
+; GFX10-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX10-NEXT:    s_mov_b32 s46, s0
+; GFX10-NEXT:    s_and_b32 s17, s17, 1
+; GFX10-NEXT:    s_mov_b32 s47, s1
+; GFX10-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_addc_u32 s29, s1, s9
+; GFX10-NEXT:    s_cselect_b32 s18, 1, 0
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s0, s[28:29], s[46:47]
+; GFX10-NEXT:    s_and_b32 s18, s18, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s18, 0
+; GFX10-NEXT:    s_addc_u32 s30, s2, s10
+; GFX10-NEXT:    s_cselect_b32 s19, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX10-NEXT:    s_and_b32 s19, s19, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX10-NEXT:    s_addc_u32 s31, s3, s11
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s0, s[30:31], s[2:3]
+; GFX10-NEXT:    s_cmp_eq_u64 s[30:31], s[2:3]
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s2, s[8:9], 0
+; GFX10-NEXT:    s_cselect_b32 s20, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX10-NEXT:    s_and_b32 s0, 1, s20
+; GFX10-NEXT:    s_cmp_eq_u64 s[10:11], 0
+; GFX10-NEXT:    s_movk_i32 s20, 0x7f
+; GFX10-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s2
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s2, s[10:11], 0
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s0
+; GFX10-NEXT:    s_and_b32 s1, 1, s1
+; GFX10-NEXT:    s_sub_i32 s21, s20, 64
+; GFX10-NEXT:    s_sub_i32 s22, 64, s20
+; GFX10-NEXT:    s_cmp_lt_u32 s20, 64
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX10-NEXT:    s_cselect_b32 s10, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s20, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, 0, 1, s2
+; GFX10-NEXT:    s_cselect_b32 s23, 1, 0
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s1
+; GFX10-NEXT:    s_lshr_b64 s[0:1], s[28:29], s20
+; GFX10-NEXT:    s_lshl_b64 s[2:3], s[30:31], s22
+; GFX10-NEXT:    s_and_b32 s24, s10, 1
+; GFX10-NEXT:    s_or_b64 s[0:1], s[0:1], s[2:3]
+; GFX10-NEXT:    s_ashr_i32 s2, s31, 31
+; GFX10-NEXT:    s_ashr_i64 s[8:9], s[30:31], s20
+; GFX10-NEXT:    s_ashr_i64 s[10:11], s[30:31], s21
+; GFX10-NEXT:    s_cmp_lg_u32 s24, 0
+; GFX10-NEXT:    s_mov_b32 s3, s2
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[10:11]
+; GFX10-NEXT:    s_and_b32 s10, s23, 1
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v3, v2, vcc_lo
+; GFX10-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX10-NEXT:    v_mov_b32_e32 v2, s29
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[28:29], s[0:1]
+; GFX10-NEXT:    s_cmp_lg_u32 s24, 0
+; GFX10-NEXT:    v_xor_b32_e32 v0, v1, v0
+; GFX10-NEXT:    s_cselect_b64 s[2:3], s[8:9], s[2:3]
+; GFX10-NEXT:    s_add_u32 s0, s0, 0
+; GFX10-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s28
+; GFX10-NEXT:    s_and_b32 s8, s8, 1
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX10-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX10-NEXT:    s_brev_b32 s23, 1
+; GFX10-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX10-NEXT:    s_and_b32 s8, s8, 1
+; GFX10-NEXT:    v_mov_b32_e32 v3, s31
+; GFX10-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX10-NEXT:    s_addc_u32 s2, s2, 0
+; GFX10-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v1, s0, vcc_lo
+; GFX10-NEXT:    s_and_b32 s8, s8, 1
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v2, s1, vcc_lo
+; GFX10-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX10-NEXT:    v_mov_b32_e32 v2, s30
+; GFX10-NEXT:    s_addc_u32 s3, s3, s23
+; GFX10-NEXT:    s_add_u32 s0, s4, s12
+; GFX10-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, s3, vcc_lo
+; GFX10-NEXT:    s_and_b32 s1, s1, 1
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s2, vcc_lo
+; GFX10-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX10-NEXT:    s_addc_u32 s1, s5, s13
+; GFX10-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s3, s[0:1], s[4:5]
+; GFX10-NEXT:    s_and_b32 s8, s8, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX10-NEXT:    s_addc_u32 s8, s6, s14
+; GFX10-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s3
+; GFX10-NEXT:    s_and_b32 s9, s9, 1
+; GFX10-NEXT:    v_mov_b32_e32 v7, s8
+; GFX10-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX10-NEXT:    s_addc_u32 s9, s7, s15
+; GFX10-NEXT:    s_cmp_eq_u64 s[8:9], s[6:7]
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s3, s[8:9], s[6:7]
+; GFX10-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX10-NEXT:    v_mov_b32_e32 v8, s9
+; GFX10-NEXT:    s_and_b32 s2, 1, s2
+; GFX10-NEXT:    s_cmp_eq_u64 s[14:15], 0
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s2
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s2, s[12:13], 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, 0, 1, s3
+; GFX10-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX10-NEXT:    s_and_b32 s16, 1, s3
+; GFX10-NEXT:    s_cmp_lt_u32 s20, 64
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, 0, 1, s2
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s2, s[14:15], 0
+; GFX10-NEXT:    s_cselect_b32 s10, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s20, 0
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s16
+; GFX10-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX10-NEXT:    s_lshl_b64 s[4:5], s[8:9], s22
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s2
+; GFX10-NEXT:    s_lshr_b64 s[2:3], s[0:1], s20
+; GFX10-NEXT:    s_and_b32 s13, s10, 1
+; GFX10-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX10-NEXT:    s_ashr_i32 s4, s9, 31
+; GFX10-NEXT:    s_ashr_i64 s[6:7], s[8:9], s20
+; GFX10-NEXT:    s_ashr_i64 s[10:11], s[8:9], s21
+; GFX10-NEXT:    s_cmp_lg_u32 s13, 0
+; GFX10-NEXT:    s_mov_b32 s5, s4
+; GFX10-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[10:11]
+; GFX10-NEXT:    s_and_b32 s10, s12, 1
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc_lo
+; GFX10-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX10-NEXT:    v_mov_b32_e32 v6, s1
+; GFX10-NEXT:    s_cselect_b64 s[2:3], s[0:1], s[2:3]
+; GFX10-NEXT:    s_cmp_lg_u32 s13, 0
+; GFX10-NEXT:    v_xor_b32_e32 v4, v5, v4
+; GFX10-NEXT:    s_cselect_b64 s[4:5], s[6:7], s[4:5]
+; GFX10-NEXT:    s_add_u32 s2, s2, 0
+; GFX10-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX10-NEXT:    v_mov_b32_e32 v5, s0
+; GFX10-NEXT:    s_and_b32 s6, s6, 1
+; GFX10-NEXT:    v_and_b32_e32 v4, 1, v4
+; GFX10-NEXT:    s_cmp_lg_u32 s6, 0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    s_addc_u32 s3, s3, 0
+; GFX10-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v4
+; GFX10-NEXT:    s_and_b32 s6, s6, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s6, 0
+; GFX10-NEXT:    s_addc_u32 s4, s4, 0
+; GFX10-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, v5, s2, vcc_lo
+; GFX10-NEXT:    s_and_b32 s6, s6, 1
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v6, s3, vcc_lo
+; GFX10-NEXT:    s_cmp_lg_u32 s6, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v7, s4, vcc_lo
+; GFX10-NEXT:    s_addc_u32 s1, s5, s23
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_cndmask_b32_e64 v7, v8, s1, vcc_lo
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX10-NEXT:    v_readfirstlane_b32 s4, v4
+; GFX10-NEXT:    v_readfirstlane_b32 s5, v5
+; GFX10-NEXT:    v_readfirstlane_b32 s6, v6
+; GFX10-NEXT:    v_readfirstlane_b32 s7, v7
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i128> @llvm.sadd.sat.v2i128(<2 x i128> %lhs, <2 x i128> %rhs)
+  ret <2 x i128> %result
+}
+
+declare i7 @llvm.sadd.sat.i7(i7, i7) #0
+declare i8 @llvm.sadd.sat.i8(i8, i8) #0
+declare <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8>, <2 x i8>) #0
+declare <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8>, <4 x i8>) #0
+
+declare i16 @llvm.sadd.sat.i16(i16, i16) #0
+declare <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16>, <2 x i16>) #0
+declare <3 x i16> @llvm.sadd.sat.v3i16(<3 x i16>, <3 x i16>) #0
+declare <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16>, <4 x i16>) #0
+declare <5 x i16> @llvm.sadd.sat.v5i16(<5 x i16>, <5 x i16>) #0
+declare <6 x i16> @llvm.sadd.sat.v6i16(<6 x i16>, <6 x i16>) #0
+declare <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16>, <8 x i16>) #0
+
+declare i24 @llvm.sadd.sat.i24(i24, i24) #0
+
+declare i32 @llvm.sadd.sat.i32(i32, i32) #0
+declare <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32>, <2 x i32>) #0
+declare <3 x i32> @llvm.sadd.sat.v3i32(<3 x i32>, <3 x i32>) #0
+declare <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32>, <4 x i32>) #0
+declare <5 x i32> @llvm.sadd.sat.v5i32(<5 x i32>, <5 x i32>) #0
+declare <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32>, <16 x i32>) #0
+
+declare i48 @llvm.sadd.sat.i48(i48, i48) #0
+
+declare i64 @llvm.sadd.sat.i64(i64, i64) #0
+declare <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64>, <2 x i64>) #0
+
+declare i128 @llvm.sadd.sat.i128(i128, i128) #0
+declare <2 x i128> @llvm.sadd.sat.v2i128(<2 x i128>, <2 x i128>) #0
+
+attributes #0 = { nounwind readnone speculatable willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
new file mode 100644
index 0000000000000..a7154131e3c07
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
@@ -0,0 +1,9914 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=tahiti -o - %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=fiji -o - %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -o - %s | FileCheck -check-prefix=GFX10 %s
+
+define i7 @v_ssubsat_i7(i7 %lhs, i7 %rhs) {
+; GFX6-LABEL: v_ssubsat_i7:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 25, v0
+; GFX6-NEXT:    v_max_i32_e32 v2, -1, v0
+; GFX6-NEXT:    v_min_i32_e32 v3, -1, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 25, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, 0x7fffffff, v2
+; GFX6-NEXT:    v_subrev_i32_e32 v3, vcc, 0x80000000, v3
+; GFX6-NEXT:    v_max_i32_e32 v1, v2, v1
+; GFX6-NEXT:    v_min_i32_e32 v1, v1, v3
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 25, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_i7:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 9, v0
+; GFX8-NEXT:    s_mov_b32 s4, 0xffff
+; GFX8-NEXT:    v_max_i16_e32 v2, s4, v0
+; GFX8-NEXT:    v_min_i16_e32 v3, s4, v0
+; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 9, v1
+; GFX8-NEXT:    v_subrev_u16_e32 v2, 0x7fff, v2
+; GFX8-NEXT:    v_subrev_u16_e32 v3, 0x8000, v3
+; GFX8-NEXT:    v_max_i16_e32 v1, v2, v1
+; GFX8-NEXT:    v_min_i16_e32 v1, v1, v3
+; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_ashrrev_i16_e32 v0, 9, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_i7:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 9, v0
+; GFX9-NEXT:    s_mov_b32 s4, 0xffff
+; GFX9-NEXT:    v_max_i16_e32 v2, s4, v0
+; GFX9-NEXT:    v_min_i16_e32 v3, s4, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 9, v1
+; GFX9-NEXT:    v_subrev_u16_e32 v2, 0x7fff, v2
+; GFX9-NEXT:    v_subrev_u16_e32 v3, 0x8000, v3
+; GFX9-NEXT:    v_max_i16_e32 v1, v2, v1
+; GFX9-NEXT:    v_min_i16_e32 v1, v1, v3
+; GFX9-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_ashrrev_i16_e32 v0, 9, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_ssubsat_i7:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 9, v0
+; GFX10-NEXT:    s_mov_b32 s4, 0xffff
+; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 9, v1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_max_i16_e64 v2, v0, s4
+; GFX10-NEXT:    v_min_i16_e64 v3, v0, s4
+; GFX10-NEXT:    v_sub_nc_u16_e64 v2, v2, 0x7fff
+; GFX10-NEXT:    v_sub_nc_u16_e64 v3, v3, 0x8000
+; GFX10-NEXT:    v_max_i16_e64 v1, v2, v1
+; GFX10-NEXT:    v_min_i16_e64 v1, v1, v3
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    v_ashrrev_i16_e64 v0, 9, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i7 @llvm.ssub.sat.i7(i7 %lhs, i7 %rhs)
+  ret i7 %result
+}
+
+define amdgpu_ps i7 @s_ssubsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
+; GFX6-LABEL: s_ssubsat_i7:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 25
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 25
+; GFX6-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX6-NEXT:    s_cselect_b32 s2, s0, -1
+; GFX6-NEXT:    s_sub_i32 s2, s2, 0x7fffffff
+; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX6-NEXT:    s_cselect_b32 s3, s0, -1
+; GFX6-NEXT:    s_sub_i32 s3, s3, 0x80000000
+; GFX6-NEXT:    s_cmp_gt_i32 s2, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX6-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX6-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX6-NEXT:    s_sub_i32 s0, s0, s1
+; GFX6-NEXT:    s_ashr_i32 s0, s0, 25
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_ssubsat_i7:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_bfe_u32 s2, 9, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX8-NEXT:    s_sext_i32_i16 s3, s0
+; GFX8-NEXT:    s_sext_i32_i16 s4, 0xffff
+; GFX8-NEXT:    s_cmp_gt_i32 s3, s4
+; GFX8-NEXT:    s_cselect_b32 s5, s3, s4
+; GFX8-NEXT:    s_sub_i32 s5, s5, 0x7fff
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX8-NEXT:    s_sub_i32 s3, s3, 0x8000
+; GFX8-NEXT:    s_sext_i32_i16 s4, s5
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s4, s1
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX8-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX8-NEXT:    s_sub_i32 s0, s0, s1
+; GFX8-NEXT:    s_sext_i32_i16 s0, s0
+; GFX8-NEXT:    s_ashr_i32 s0, s0, s2
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_ssubsat_i7:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_bfe_u32 s2, 9, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX9-NEXT:    s_sext_i32_i16 s3, s0
+; GFX9-NEXT:    s_sext_i32_i16 s4, 0xffff
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s4
+; GFX9-NEXT:    s_cselect_b32 s5, s3, s4
+; GFX9-NEXT:    s_sub_i32 s5, s5, 0x7fff
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX9-NEXT:    s_sub_i32 s3, s3, 0x8000
+; GFX9-NEXT:    s_sext_i32_i16 s4, s5
+; GFX9-NEXT:    s_sext_i32_i16 s1, s1
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s4, s1
+; GFX9-NEXT:    s_sext_i32_i16 s1, s1
+; GFX9-NEXT:    s_sext_i32_i16 s3, s3
+; GFX9-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX9-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX9-NEXT:    s_sub_i32 s0, s0, s1
+; GFX9-NEXT:    s_sext_i32_i16 s0, s0
+; GFX9-NEXT:    s_ashr_i32 s0, s0, s2
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_ssubsat_i7:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_bfe_u32 s2, 9, 0x100000
+; GFX10-NEXT:    s_sext_i32_i16 s4, 0xffff
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX10-NEXT:    s_sext_i32_i16 s3, s0
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_cmp_gt_i32 s3, s4
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s5, s3, s4
+; GFX10-NEXT:    s_sub_i32 s5, s5, 0x7fff
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX10-NEXT:    s_sext_i32_i16 s4, s5
+; GFX10-NEXT:    s_sub_i32 s3, s3, 0x8000
+; GFX10-NEXT:    s_cmp_gt_i32 s4, s1
+; GFX10-NEXT:    s_sext_i32_i16 s3, s3
+; GFX10-NEXT:    s_cselect_b32 s1, s4, s1
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX10-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX10-NEXT:    s_sub_i32 s0, s0, s1
+; GFX10-NEXT:    s_sext_i32_i16 s0, s0
+; GFX10-NEXT:    s_ashr_i32 s0, s0, s2
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i7 @llvm.ssub.sat.i7(i7 %lhs, i7 %rhs)
+  ret i7 %result
+}
+
+define i8 @v_ssubsat_i8(i8 %lhs, i8 %rhs) {
+; GFX6-LABEL: v_ssubsat_i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
+; GFX6-NEXT:    v_max_i32_e32 v2, -1, v0
+; GFX6-NEXT:    v_min_i32_e32 v3, -1, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, 0x7fffffff, v2
+; GFX6-NEXT:    v_subrev_i32_e32 v3, vcc, 0x80000000, v3
+; GFX6-NEXT:    v_max_i32_e32 v1, v2, v1
+; GFX6-NEXT:    v_min_i32_e32 v1, v1, v3
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 24, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX8-NEXT:    s_mov_b32 s4, 0xffff
+; GFX8-NEXT:    v_max_i16_e32 v2, s4, v0
+; GFX8-NEXT:    v_min_i16_e32 v3, s4, v0
+; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX8-NEXT:    v_subrev_u16_e32 v2, 0x7fff, v2
+; GFX8-NEXT:    v_subrev_u16_e32 v3, 0x8000, v3
+; GFX8-NEXT:    v_max_i16_e32 v1, v2, v1
+; GFX8-NEXT:    v_min_i16_e32 v1, v1, v3
+; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_ashrrev_i16_e32 v0, 8, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    s_mov_b32 s4, 0xffff
+; GFX9-NEXT:    v_max_i16_e32 v2, s4, v0
+; GFX9-NEXT:    v_min_i16_e32 v3, s4, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-NEXT:    v_subrev_u16_e32 v2, 0x7fff, v2
+; GFX9-NEXT:    v_subrev_u16_e32 v3, 0x8000, v3
+; GFX9-NEXT:    v_max_i16_e32 v1, v2, v1
+; GFX9-NEXT:    v_min_i16_e32 v1, v1, v3
+; GFX9-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_ashrrev_i16_e32 v0, 8, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_ssubsat_i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 8, v0
+; GFX10-NEXT:    s_mov_b32 s4, 0xffff
+; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 8, v1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_max_i16_e64 v2, v0, s4
+; GFX10-NEXT:    v_min_i16_e64 v3, v0, s4
+; GFX10-NEXT:    v_sub_nc_u16_e64 v2, v2, 0x7fff
+; GFX10-NEXT:    v_sub_nc_u16_e64 v3, v3, 0x8000
+; GFX10-NEXT:    v_max_i16_e64 v1, v2, v1
+; GFX10-NEXT:    v_min_i16_e64 v1, v1, v3
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    v_ashrrev_i16_e64 v0, 8, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i8 @llvm.ssub.sat.i8(i8 %lhs, i8 %rhs)
+  ret i8 %result
+}
+
+define amdgpu_ps i8 @s_ssubsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
+; GFX6-LABEL: s_ssubsat_i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 24
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX6-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX6-NEXT:    s_cselect_b32 s2, s0, -1
+; GFX6-NEXT:    s_sub_i32 s2, s2, 0x7fffffff
+; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX6-NEXT:    s_cselect_b32 s3, s0, -1
+; GFX6-NEXT:    s_sub_i32 s3, s3, 0x80000000
+; GFX6-NEXT:    s_cmp_gt_i32 s2, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX6-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX6-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX6-NEXT:    s_sub_i32 s0, s0, s1
+; GFX6-NEXT:    s_ashr_i32 s0, s0, 24
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_ssubsat_i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_bfe_u32 s2, 8, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX8-NEXT:    s_sext_i32_i16 s3, s0
+; GFX8-NEXT:    s_sext_i32_i16 s4, 0xffff
+; GFX8-NEXT:    s_cmp_gt_i32 s3, s4
+; GFX8-NEXT:    s_cselect_b32 s5, s3, s4
+; GFX8-NEXT:    s_sub_i32 s5, s5, 0x7fff
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX8-NEXT:    s_sub_i32 s3, s3, 0x8000
+; GFX8-NEXT:    s_sext_i32_i16 s4, s5
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s4, s1
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX8-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX8-NEXT:    s_sub_i32 s0, s0, s1
+; GFX8-NEXT:    s_sext_i32_i16 s0, s0
+; GFX8-NEXT:    s_ashr_i32 s0, s0, s2
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_ssubsat_i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_bfe_u32 s2, 8, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX9-NEXT:    s_sext_i32_i16 s3, s0
+; GFX9-NEXT:    s_sext_i32_i16 s4, 0xffff
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s4
+; GFX9-NEXT:    s_cselect_b32 s5, s3, s4
+; GFX9-NEXT:    s_sub_i32 s5, s5, 0x7fff
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX9-NEXT:    s_sub_i32 s3, s3, 0x8000
+; GFX9-NEXT:    s_sext_i32_i16 s4, s5
+; GFX9-NEXT:    s_sext_i32_i16 s1, s1
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s4, s1
+; GFX9-NEXT:    s_sext_i32_i16 s1, s1
+; GFX9-NEXT:    s_sext_i32_i16 s3, s3
+; GFX9-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX9-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX9-NEXT:    s_sub_i32 s0, s0, s1
+; GFX9-NEXT:    s_sext_i32_i16 s0, s0
+; GFX9-NEXT:    s_ashr_i32 s0, s0, s2
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_ssubsat_i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_bfe_u32 s2, 8, 0x100000
+; GFX10-NEXT:    s_sext_i32_i16 s4, 0xffff
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX10-NEXT:    s_sext_i32_i16 s3, s0
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_cmp_gt_i32 s3, s4
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s5, s3, s4
+; GFX10-NEXT:    s_sub_i32 s5, s5, 0x7fff
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX10-NEXT:    s_sext_i32_i16 s4, s5
+; GFX10-NEXT:    s_sub_i32 s3, s3, 0x8000
+; GFX10-NEXT:    s_cmp_gt_i32 s4, s1
+; GFX10-NEXT:    s_sext_i32_i16 s3, s3
+; GFX10-NEXT:    s_cselect_b32 s1, s4, s1
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX10-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX10-NEXT:    s_sub_i32 s0, s0, s1
+; GFX10-NEXT:    s_sext_i32_i16 s0, s0
+; GFX10-NEXT:    s_ashr_i32 s0, s0, s2
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i8 @llvm.ssub.sat.i8(i8 %lhs, i8 %rhs)
+  ret i8 %result
+}
+
+define i16 @v_ssubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
+; GFX6-LABEL: v_ssubsat_v2i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 8, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    v_max_i32_e32 v4, -1, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 8, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v4, vcc, s4, v4
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v5, -1, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v5, vcc, s5, v5
+; GFX6-NEXT:    v_max_i32_e32 v1, v4, v1
+; GFX6-NEXT:    v_min_i32_e32 v1, v1, v5
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 24, v3
+; GFX6-NEXT:    v_max_i32_e32 v3, -1, v1
+; GFX6-NEXT:    v_min_i32_e32 v4, -1, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v3, vcc, s4, v3
+; GFX6-NEXT:    v_subrev_i32_e32 v4, vcc, s5, v4
+; GFX6-NEXT:    v_max_i32_e32 v2, v3, v2
+; GFX6-NEXT:    v_min_i32_e32 v2, v2, v4
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 24, v1
+; GFX6-NEXT:    v_mov_b32_e32 v2, 0xff
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 24, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, v1, v2
+; GFX6-NEXT:    v_and_b32_e32 v0, v0, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_v2i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v2, 8
+; GFX8-NEXT:    v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX8-NEXT:    s_mov_b32 s6, 0xffff
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX8-NEXT:    v_max_i16_e32 v4, s6, v0
+; GFX8-NEXT:    v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX8-NEXT:    v_subrev_u16_e32 v4, s4, v4
+; GFX8-NEXT:    s_mov_b32 s5, 0x8000
+; GFX8-NEXT:    v_min_i16_e32 v5, s6, v0
+; GFX8-NEXT:    v_max_i16_e32 v1, v4, v1
+; GFX8-NEXT:    v_subrev_u16_e32 v5, s5, v5
+; GFX8-NEXT:    v_min_i16_e32 v1, v1, v5
+; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_max_i16_e32 v1, s6, v3
+; GFX8-NEXT:    v_subrev_u16_e32 v1, s4, v1
+; GFX8-NEXT:    v_min_i16_e32 v4, s6, v3
+; GFX8-NEXT:    v_max_i16_e32 v1, v1, v2
+; GFX8-NEXT:    v_subrev_u16_e32 v4, s5, v4
+; GFX8-NEXT:    v_min_i16_e32 v1, v1, v4
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0xff
+; GFX8-NEXT:    v_sub_u16_e32 v1, v3, v1
+; GFX8-NEXT:    v_and_b32_sdwa v0, sext(v0), v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v1, sext(v1), v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_v2i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    s_mov_b32 s6, 0xffff
+; GFX9-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    v_max_i16_e32 v4, s6, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-NEXT:    v_subrev_u16_e32 v4, s4, v4
+; GFX9-NEXT:    s_mov_b32 s5, 0x8000
+; GFX9-NEXT:    v_min_i16_e32 v5, s6, v0
+; GFX9-NEXT:    v_max_i16_e32 v1, v4, v1
+; GFX9-NEXT:    v_subrev_u16_e32 v5, s5, v5
+; GFX9-NEXT:    v_min_i16_e32 v1, v1, v5
+; GFX9-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_max_i16_e32 v1, s6, v2
+; GFX9-NEXT:    v_subrev_u16_e32 v1, s4, v1
+; GFX9-NEXT:    v_min_i16_e32 v4, s6, v2
+; GFX9-NEXT:    v_subrev_u16_e32 v4, s5, v4
+; GFX9-NEXT:    v_max_i16_e32 v1, v1, v3
+; GFX9-NEXT:    v_min_i16_e32 v1, v1, v4
+; GFX9-NEXT:    s_movk_i32 s4, 0xff
+; GFX9-NEXT:    v_sub_u16_e32 v1, v2, v1
+; GFX9-NEXT:    v_and_b32_sdwa v0, sext(v0), s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v1, sext(v1), s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_ssubsat_v2i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_mov_b32 s4, 8
+; GFX10-NEXT:    v_lshlrev_b16_e64 v2, 8, v0
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v0, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    s_mov_b32 s6, 0xffff
+; GFX10-NEXT:    s_movk_i32 s5, 0x7fff
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_max_i16_e64 v4, v2, s6
+; GFX10-NEXT:    v_max_i16_e64 v5, v0, s6
+; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 8, v1
+; GFX10-NEXT:    v_min_i16_e64 v6, v2, s6
+; GFX10-NEXT:    v_min_i16_e64 v7, v0, s6
+; GFX10-NEXT:    v_sub_nc_u16_e64 v4, v4, s5
+; GFX10-NEXT:    v_sub_nc_u16_e64 v5, v5, s5
+; GFX10-NEXT:    s_mov_b32 s4, 0x8000
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_sub_nc_u16_e64 v6, v6, s4
+; GFX10-NEXT:    v_max_i16_e64 v1, v4, v1
+; GFX10-NEXT:    v_sub_nc_u16_e64 v7, v7, s4
+; GFX10-NEXT:    v_max_i16_e64 v10, v5, v3
+; GFX10-NEXT:    s_movk_i32 s4, 0xff
+; GFX10-NEXT:    v_min_i16_e64 v1, v1, v6
+; GFX10-NEXT:    v_min_i16_e64 v3, v10, v7
+; GFX10-NEXT:    v_sub_nc_u16_e64 v1, v2, v1
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v3
+; GFX10-NEXT:    v_and_b32_sdwa v1, sext(v1), s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v0, sext(v0), s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %lhs = bitcast i16 %lhs.arg to <2 x i8>
+  %rhs = bitcast i16 %rhs.arg to <2 x i8>
+  %result = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %lhs, <2 x i8> %rhs)
+  %cast.result = bitcast <2 x i8> %result to i16
+  ret i16 %cast.result
+}
+
+define amdgpu_ps i16 @s_ssubsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
+; GFX6-LABEL: s_ssubsat_v2i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX6-NEXT:    s_lshr_b32 s3, s1, 8
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 24
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX6-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    s_cselect_b32 s6, s0, -1
+; GFX6-NEXT:    s_sub_i32 s6, s6, s4
+; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_cselect_b32 s7, s0, -1
+; GFX6-NEXT:    s_sub_i32 s7, s7, s5
+; GFX6-NEXT:    s_cmp_gt_i32 s6, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s6, s1
+; GFX6-NEXT:    s_cmp_lt_i32 s1, s7
+; GFX6-NEXT:    s_cselect_b32 s1, s1, s7
+; GFX6-NEXT:    s_sub_i32 s0, s0, s1
+; GFX6-NEXT:    s_lshl_b32 s1, s2, 24
+; GFX6-NEXT:    s_ashr_i32 s0, s0, 24
+; GFX6-NEXT:    s_lshl_b32 s2, s3, 24
+; GFX6-NEXT:    s_cmp_gt_i32 s1, -1
+; GFX6-NEXT:    s_cselect_b32 s3, s1, -1
+; GFX6-NEXT:    s_sub_i32 s3, s3, s4
+; GFX6-NEXT:    s_cmp_lt_i32 s1, -1
+; GFX6-NEXT:    s_cselect_b32 s4, s1, -1
+; GFX6-NEXT:    s_sub_i32 s4, s4, s5
+; GFX6-NEXT:    s_cmp_gt_i32 s3, s2
+; GFX6-NEXT:    s_cselect_b32 s2, s3, s2
+; GFX6-NEXT:    s_cmp_lt_i32 s2, s4
+; GFX6-NEXT:    s_cselect_b32 s2, s2, s4
+; GFX6-NEXT:    s_sub_i32 s1, s1, s2
+; GFX6-NEXT:    s_movk_i32 s2, 0xff
+; GFX6-NEXT:    s_ashr_i32 s1, s1, 24
+; GFX6-NEXT:    s_and_b32 s1, s1, s2
+; GFX6-NEXT:    s_and_b32 s0, s0, s2
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_ssubsat_v2i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_bfe_u32 s4, 8, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s4
+; GFX8-NEXT:    s_lshr_b32 s3, s1, 8
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s4
+; GFX8-NEXT:    s_sext_i32_i16 s7, s0
+; GFX8-NEXT:    s_sext_i32_i16 s8, 0xffff
+; GFX8-NEXT:    s_cmp_gt_i32 s7, s8
+; GFX8-NEXT:    s_movk_i32 s5, 0x7fff
+; GFX8-NEXT:    s_cselect_b32 s9, s7, s8
+; GFX8-NEXT:    s_sub_i32 s9, s9, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s7, s8
+; GFX8-NEXT:    s_mov_b32 s6, 0x8000
+; GFX8-NEXT:    s_cselect_b32 s7, s7, s8
+; GFX8-NEXT:    s_sub_i32 s7, s7, s6
+; GFX8-NEXT:    s_sext_i32_i16 s9, s9
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_cmp_gt_i32 s9, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s9, s1
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_sext_i32_i16 s7, s7
+; GFX8-NEXT:    s_cmp_lt_i32 s1, s7
+; GFX8-NEXT:    s_cselect_b32 s1, s1, s7
+; GFX8-NEXT:    s_sub_i32 s0, s0, s1
+; GFX8-NEXT:    s_sext_i32_i16 s0, s0
+; GFX8-NEXT:    s_lshl_b32 s1, s2, s4
+; GFX8-NEXT:    s_lshl_b32 s2, s3, s4
+; GFX8-NEXT:    s_ashr_i32 s0, s0, s4
+; GFX8-NEXT:    s_sext_i32_i16 s3, s1
+; GFX8-NEXT:    s_cmp_gt_i32 s3, s8
+; GFX8-NEXT:    s_cselect_b32 s7, s3, s8
+; GFX8-NEXT:    s_sub_i32 s5, s7, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s8
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s8
+; GFX8-NEXT:    s_sub_i32 s3, s3, s6
+; GFX8-NEXT:    s_sext_i32_i16 s5, s5
+; GFX8-NEXT:    s_sext_i32_i16 s2, s2
+; GFX8-NEXT:    s_cmp_gt_i32 s5, s2
+; GFX8-NEXT:    s_cselect_b32 s2, s5, s2
+; GFX8-NEXT:    s_sext_i32_i16 s2, s2
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_cmp_lt_i32 s2, s3
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX8-NEXT:    s_sub_i32 s1, s1, s2
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_movk_i32 s2, 0xff
+; GFX8-NEXT:    s_ashr_i32 s1, s1, s4
+; GFX8-NEXT:    s_and_b32 s1, s1, s2
+; GFX8-NEXT:    s_and_b32 s0, s0, s2
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s4
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_ssubsat_v2i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_bfe_u32 s4, 8, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s4
+; GFX9-NEXT:    s_lshr_b32 s3, s1, 8
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s4
+; GFX9-NEXT:    s_sext_i32_i16 s7, s0
+; GFX9-NEXT:    s_sext_i32_i16 s8, 0xffff
+; GFX9-NEXT:    s_cmp_gt_i32 s7, s8
+; GFX9-NEXT:    s_movk_i32 s5, 0x7fff
+; GFX9-NEXT:    s_cselect_b32 s9, s7, s8
+; GFX9-NEXT:    s_sub_i32 s9, s9, s5
+; GFX9-NEXT:    s_cmp_lt_i32 s7, s8
+; GFX9-NEXT:    s_mov_b32 s6, 0x8000
+; GFX9-NEXT:    s_cselect_b32 s7, s7, s8
+; GFX9-NEXT:    s_sub_i32 s7, s7, s6
+; GFX9-NEXT:    s_sext_i32_i16 s9, s9
+; GFX9-NEXT:    s_sext_i32_i16 s1, s1
+; GFX9-NEXT:    s_cmp_gt_i32 s9, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s9, s1
+; GFX9-NEXT:    s_sext_i32_i16 s1, s1
+; GFX9-NEXT:    s_sext_i32_i16 s7, s7
+; GFX9-NEXT:    s_cmp_lt_i32 s1, s7
+; GFX9-NEXT:    s_cselect_b32 s1, s1, s7
+; GFX9-NEXT:    s_sub_i32 s0, s0, s1
+; GFX9-NEXT:    s_sext_i32_i16 s0, s0
+; GFX9-NEXT:    s_lshl_b32 s1, s2, s4
+; GFX9-NEXT:    s_lshl_b32 s2, s3, s4
+; GFX9-NEXT:    s_ashr_i32 s0, s0, s4
+; GFX9-NEXT:    s_sext_i32_i16 s3, s1
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s8
+; GFX9-NEXT:    s_cselect_b32 s7, s3, s8
+; GFX9-NEXT:    s_sub_i32 s5, s7, s5
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s8
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s8
+; GFX9-NEXT:    s_sub_i32 s3, s3, s6
+; GFX9-NEXT:    s_sext_i32_i16 s5, s5
+; GFX9-NEXT:    s_sext_i32_i16 s2, s2
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s2
+; GFX9-NEXT:    s_cselect_b32 s2, s5, s2
+; GFX9-NEXT:    s_sext_i32_i16 s2, s2
+; GFX9-NEXT:    s_sext_i32_i16 s3, s3
+; GFX9-NEXT:    s_cmp_lt_i32 s2, s3
+; GFX9-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX9-NEXT:    s_sub_i32 s1, s1, s2
+; GFX9-NEXT:    s_sext_i32_i16 s1, s1
+; GFX9-NEXT:    s_movk_i32 s2, 0xff
+; GFX9-NEXT:    s_ashr_i32 s1, s1, s4
+; GFX9-NEXT:    s_and_b32 s1, s1, s2
+; GFX9-NEXT:    s_and_b32 s0, s0, s2
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s4
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_ssubsat_v2i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_bfe_u32 s2, 8, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s3, s0, 8
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX10-NEXT:    s_sext_i32_i16 s6, 0xffff
+; GFX10-NEXT:    s_sext_i32_i16 s5, s0
+; GFX10-NEXT:    s_lshr_b32 s4, s1, 8
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX10-NEXT:    s_cmp_gt_i32 s5, s6
+; GFX10-NEXT:    s_movk_i32 s7, 0x7fff
+; GFX10-NEXT:    s_cselect_b32 s8, s5, s6
+; GFX10-NEXT:    s_mov_b32 s9, 0x8000
+; GFX10-NEXT:    s_sub_i32 s8, s8, s7
+; GFX10-NEXT:    s_cmp_lt_i32 s5, s6
+; GFX10-NEXT:    s_sext_i32_i16 s8, s8
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_sub_i32 s5, s5, s9
+; GFX10-NEXT:    s_cmp_gt_i32 s8, s1
+; GFX10-NEXT:    s_sext_i32_i16 s5, s5
+; GFX10-NEXT:    s_cselect_b32 s1, s8, s1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_cmp_lt_i32 s1, s5
+; GFX10-NEXT:    s_cselect_b32 s1, s1, s5
+; GFX10-NEXT:    s_lshl_b32 s3, s3, s2
+; GFX10-NEXT:    s_sub_i32 s0, s0, s1
+; GFX10-NEXT:    s_lshl_b32 s1, s4, s2
+; GFX10-NEXT:    s_sext_i32_i16 s0, s0
+; GFX10-NEXT:    s_sext_i32_i16 s4, s3
+; GFX10-NEXT:    s_ashr_i32 s0, s0, s2
+; GFX10-NEXT:    s_cmp_gt_i32 s4, s6
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_cselect_b32 s5, s4, s6
+; GFX10-NEXT:    s_sub_i32 s5, s5, s7
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s6
+; GFX10-NEXT:    s_sext_i32_i16 s5, s5
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX10-NEXT:    s_sub_i32 s4, s4, s9
+; GFX10-NEXT:    s_cmp_gt_i32 s5, s1
+; GFX10-NEXT:    s_sext_i32_i16 s4, s4
+; GFX10-NEXT:    s_cselect_b32 s1, s5, s1
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_cmp_lt_i32 s1, s4
+; GFX10-NEXT:    s_cselect_b32 s1, s1, s4
+; GFX10-NEXT:    s_sub_i32 s1, s3, s1
+; GFX10-NEXT:    s_movk_i32 s3, 0xff
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_and_b32 s0, s0, s3
+; GFX10-NEXT:    s_ashr_i32 s1, s1, s2
+; GFX10-NEXT:    s_and_b32 s1, s1, s3
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %lhs = bitcast i16 %lhs.arg to <2 x i8>
+  %rhs = bitcast i16 %rhs.arg to <2 x i8>
+  %result = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %lhs, <2 x i8> %rhs)
+  %cast.result = bitcast <2 x i8> %result to i16
+  ret i16 %cast.result
+}
+
+define i32 @v_ssubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
+; GFX6-LABEL: v_ssubsat_v4i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 8, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 24, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    v_max_i32_e32 v8, -1, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v5, 8, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v8, vcc, s4, v8
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v10, -1, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v10, vcc, s5, v10
+; GFX6-NEXT:    v_max_i32_e32 v1, v8, v1
+; GFX6-NEXT:    v_min_i32_e32 v1, v1, v10
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 24, v5
+; GFX6-NEXT:    v_max_i32_e32 v5, -1, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v5, vcc, s4, v5
+; GFX6-NEXT:    v_min_i32_e32 v8, -1, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v8, vcc, s5, v8
+; GFX6-NEXT:    v_max_i32_e32 v2, v5, v2
+; GFX6-NEXT:    v_min_i32_e32 v2, v2, v8
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 24, v3
+; GFX6-NEXT:    v_bfrev_b32_e32 v9, -2
+; GFX6-NEXT:    v_max_i32_e32 v5, -1, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 24, v6
+; GFX6-NEXT:    v_min_i32_e32 v6, -1, v2
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, v5, v9
+; GFX6-NEXT:    v_subrev_i32_e32 v6, vcc, s5, v6
+; GFX6-NEXT:    v_max_i32_e32 v3, v5, v3
+; GFX6-NEXT:    v_min_i32_e32 v3, v3, v6
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 24, v4
+; GFX6-NEXT:    v_max_i32_e32 v5, -1, v3
+; GFX6-NEXT:    v_mov_b32_e32 v11, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v6, -1, v3
+; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 24, v1
+; GFX6-NEXT:    s_movk_i32 s4, 0xff
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 24, v7
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, v5, v9
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 24, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_sub_i32_e32 v6, vcc, v6, v11
+; GFX6-NEXT:    v_max_i32_e32 v4, v5, v4
+; GFX6-NEXT:    v_min_i32_e32 v4, v4, v6
+; GFX6-NEXT:    v_ashrrev_i32_e32 v2, 24, v2
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, v3, v4
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_ashrrev_i32_e32 v3, 24, v3
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_v4i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v2, 8
+; GFX8-NEXT:    v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 24, v0
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX8-NEXT:    s_mov_b32 s6, 0xffff
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX8-NEXT:    v_max_i16_e32 v9, s6, v0
+; GFX8-NEXT:    v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
+; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX8-NEXT:    v_subrev_u16_e32 v9, s4, v9
+; GFX8-NEXT:    s_mov_b32 s5, 0x8000
+; GFX8-NEXT:    v_min_i16_e32 v11, s6, v0
+; GFX8-NEXT:    v_max_i16_e32 v1, v9, v1
+; GFX8-NEXT:    v_subrev_u16_e32 v11, s5, v11
+; GFX8-NEXT:    v_min_i16_e32 v1, v1, v11
+; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_max_i16_e32 v1, s6, v3
+; GFX8-NEXT:    v_subrev_u16_e32 v1, s4, v1
+; GFX8-NEXT:    v_min_i16_e32 v9, s6, v3
+; GFX8-NEXT:    v_max_i16_e32 v1, v1, v2
+; GFX8-NEXT:    v_subrev_u16_e32 v9, s5, v9
+; GFX8-NEXT:    v_mov_b32_e32 v8, 0xffff
+; GFX8-NEXT:    v_lshlrev_b16_e32 v2, 8, v4
+; GFX8-NEXT:    v_min_i16_e32 v1, v1, v9
+; GFX8-NEXT:    v_mov_b32_e32 v10, 0x7fff
+; GFX8-NEXT:    v_max_i16_e32 v4, v2, v8
+; GFX8-NEXT:    v_sub_u16_e32 v1, v3, v1
+; GFX8-NEXT:    v_lshlrev_b16_e32 v3, 8, v6
+; GFX8-NEXT:    v_min_i16_e32 v6, v2, v8
+; GFX8-NEXT:    v_sub_u16_e32 v4, v4, v10
+; GFX8-NEXT:    v_max_i16_e32 v3, v4, v3
+; GFX8-NEXT:    v_subrev_u16_e32 v6, s5, v6
+; GFX8-NEXT:    v_min_i16_e32 v3, v3, v6
+; GFX8-NEXT:    v_sub_u16_e32 v2, v2, v3
+; GFX8-NEXT:    v_lshlrev_b16_e32 v3, 8, v5
+; GFX8-NEXT:    v_max_i16_e32 v5, v3, v8
+; GFX8-NEXT:    v_min_i16_e32 v6, v3, v8
+; GFX8-NEXT:    v_lshlrev_b16_e32 v4, 8, v7
+; GFX8-NEXT:    v_sub_u16_e32 v5, v5, v10
+; GFX8-NEXT:    v_subrev_u16_e32 v6, 0x8000, v6
+; GFX8-NEXT:    v_max_i16_e32 v4, v5, v4
+; GFX8-NEXT:    v_min_i16_e32 v4, v4, v6
+; GFX8-NEXT:    v_sub_u16_e32 v3, v3, v4
+; GFX8-NEXT:    v_mov_b32_e32 v4, 0xff
+; GFX8-NEXT:    v_and_b32_sdwa v1, sext(v1), v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v0, sext(v0), v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_and_b32_sdwa v1, sext(v2), v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_and_b32_sdwa v1, sext(v3), v4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_v4i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 24, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    s_mov_b32 s6, 0xffff
+; GFX9-NEXT:    v_lshrrev_b32_sdwa v5, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    v_max_i16_e32 v9, s6, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-NEXT:    v_subrev_u16_e32 v9, s4, v9
+; GFX9-NEXT:    s_mov_b32 s5, 0x8000
+; GFX9-NEXT:    v_min_i16_e32 v11, s6, v0
+; GFX9-NEXT:    v_max_i16_e32 v1, v9, v1
+; GFX9-NEXT:    v_subrev_u16_e32 v11, s5, v11
+; GFX9-NEXT:    v_min_i16_e32 v1, v1, v11
+; GFX9-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_max_i16_e32 v1, s6, v2
+; GFX9-NEXT:    v_subrev_u16_e32 v1, s4, v1
+; GFX9-NEXT:    v_min_i16_e32 v9, s6, v2
+; GFX9-NEXT:    v_subrev_u16_e32 v9, s5, v9
+; GFX9-NEXT:    v_max_i16_e32 v1, v1, v5
+; GFX9-NEXT:    v_min_i16_e32 v1, v1, v9
+; GFX9-NEXT:    v_sub_u16_e32 v1, v2, v1
+; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v3
+; GFX9-NEXT:    v_mov_b32_e32 v8, 0xffff
+; GFX9-NEXT:    v_mov_b32_e32 v10, 0x7fff
+; GFX9-NEXT:    v_max_i16_e32 v5, v2, v8
+; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v6
+; GFX9-NEXT:    v_min_i16_e32 v6, v2, v8
+; GFX9-NEXT:    v_sub_u16_e32 v5, v5, v10
+; GFX9-NEXT:    v_subrev_u16_e32 v6, s5, v6
+; GFX9-NEXT:    v_max_i16_e32 v3, v5, v3
+; GFX9-NEXT:    v_min_i16_e32 v3, v3, v6
+; GFX9-NEXT:    v_sub_u16_e32 v2, v2, v3
+; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v4
+; GFX9-NEXT:    v_max_i16_e32 v5, v3, v8
+; GFX9-NEXT:    v_min_i16_e32 v6, v3, v8
+; GFX9-NEXT:    v_lshlrev_b16_e32 v4, 8, v7
+; GFX9-NEXT:    v_sub_u16_e32 v5, v5, v10
+; GFX9-NEXT:    s_movk_i32 s4, 0xff
+; GFX9-NEXT:    v_subrev_u16_e32 v6, 0x8000, v6
+; GFX9-NEXT:    v_max_i16_e32 v4, v5, v4
+; GFX9-NEXT:    v_and_b32_sdwa v1, sext(v1), s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_min_i16_e32 v4, v4, v6
+; GFX9-NEXT:    v_ashrrev_i16_e32 v0, 8, v0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX9-NEXT:    v_sub_u16_e32 v3, v3, v4
+; GFX9-NEXT:    v_and_or_b32 v0, v0, s4, v1
+; GFX9-NEXT:    v_and_b32_sdwa v1, sext(v2), s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v2, sext(v3), s4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_ssubsat_v4i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_mov_b32 s4, 8
+; GFX10-NEXT:    s_mov_b32 s5, 16
+; GFX10-NEXT:    s_mov_b32 s6, 24
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s5, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v19, s6, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 8, v0
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v5, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    s_mov_b32 s4, 0xffff
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v6, s5, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    s_movk_i32 s5, 0x7fff
+; GFX10-NEXT:    v_max_i16_e64 v8, v0, s4
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v7, s6, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_max_i16_e64 v9, v2, s4
+; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 8, v1
+; GFX10-NEXT:    v_min_i16_e64 v10, v0, s4
+; GFX10-NEXT:    v_sub_nc_u16_e64 v8, v8, s5
+; GFX10-NEXT:    s_mov_b32 s6, 0x8000
+; GFX10-NEXT:    v_sub_nc_u16_e64 v15, v9, s5
+; GFX10-NEXT:    v_min_i16_e64 v11, v2, s4
+; GFX10-NEXT:    v_mov_b32_e32 v12, 0xffff
+; GFX10-NEXT:    v_max_i16_e64 v1, v8, v1
+; GFX10-NEXT:    v_sub_nc_u16_e64 v10, v10, s6
+; GFX10-NEXT:    v_max_i16_e64 v5, v15, v5
+; GFX10-NEXT:    v_sub_nc_u16_e64 v8, v11, s6
+; GFX10-NEXT:    v_mov_b32_e32 v9, 0x7fff
+; GFX10-NEXT:    v_max_i16_e64 v11, v3, v12
+; GFX10-NEXT:    v_min_i16_e64 v1, v1, v10
+; GFX10-NEXT:    v_max_i16_e64 v10, v19, v12
+; GFX10-NEXT:    v_min_i16_e64 v5, v5, v8
+; GFX10-NEXT:    v_min_i16_e64 v8, v3, v12
+; GFX10-NEXT:    v_sub_nc_u16_e64 v11, v11, v9
+; GFX10-NEXT:    v_min_i16_e64 v12, v19, v12
+; GFX10-NEXT:    v_sub_nc_u16_e64 v9, v10, v9
+; GFX10-NEXT:    v_sub_nc_u16_e64 v2, v2, v5
+; GFX10-NEXT:    v_sub_nc_u16_e64 v5, v8, s6
+; GFX10-NEXT:    v_max_i16_e64 v6, v11, v6
+; GFX10-NEXT:    v_sub_nc_u16_e64 v8, v12, 0x8000
+; GFX10-NEXT:    v_max_i16_e64 v7, v9, v7
+; GFX10-NEXT:    s_movk_i32 s4, 0xff
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    v_and_b32_sdwa v1, sext(v2), s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_min_i16_e64 v2, v6, v5
+; GFX10-NEXT:    v_min_i16_e64 v5, v7, v8
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_ashrrev_i16_e64 v0, 8, v0
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX10-NEXT:    v_sub_nc_u16_e64 v2, v3, v2
+; GFX10-NEXT:    v_sub_nc_u16_e64 v3, v19, v5
+; GFX10-NEXT:    v_and_or_b32 v0, v0, s4, v1
+; GFX10-NEXT:    v_and_b32_sdwa v1, sext(v2), s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v2, sext(v3), s4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %lhs = bitcast i32 %lhs.arg to <4 x i8>
+  %rhs = bitcast i32 %rhs.arg to <4 x i8>
+  %result = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> %lhs, <4 x i8> %rhs)
+  %cast.result = bitcast <4 x i8> %result to i32
+  ret i32 %cast.result
+}
+
+define amdgpu_ps i32 @s_ssubsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
+; GFX6-LABEL: s_ssubsat_v4i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX6-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX6-NEXT:    s_lshr_b32 s4, s0, 24
+; GFX6-NEXT:    s_lshr_b32 s5, s1, 8
+; GFX6-NEXT:    s_lshr_b32 s6, s1, 16
+; GFX6-NEXT:    s_lshr_b32 s7, s1, 24
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 24
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX6-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX6-NEXT:    s_brev_b32 s8, -2
+; GFX6-NEXT:    s_cselect_b32 s10, s0, -1
+; GFX6-NEXT:    s_sub_i32 s10, s10, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX6-NEXT:    s_mov_b32 s9, 0x80000000
+; GFX6-NEXT:    s_cselect_b32 s11, s0, -1
+; GFX6-NEXT:    s_sub_i32 s11, s11, s9
+; GFX6-NEXT:    s_cmp_gt_i32 s10, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s10, s1
+; GFX6-NEXT:    s_cmp_lt_i32 s1, s11
+; GFX6-NEXT:    s_cselect_b32 s1, s1, s11
+; GFX6-NEXT:    s_sub_i32 s0, s0, s1
+; GFX6-NEXT:    s_lshl_b32 s1, s2, 24
+; GFX6-NEXT:    s_ashr_i32 s0, s0, 24
+; GFX6-NEXT:    s_lshl_b32 s2, s5, 24
+; GFX6-NEXT:    s_cmp_gt_i32 s1, -1
+; GFX6-NEXT:    s_cselect_b32 s5, s1, -1
+; GFX6-NEXT:    s_sub_i32 s5, s5, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s1, -1
+; GFX6-NEXT:    s_cselect_b32 s10, s1, -1
+; GFX6-NEXT:    s_sub_i32 s10, s10, s9
+; GFX6-NEXT:    s_cmp_gt_i32 s5, s2
+; GFX6-NEXT:    s_cselect_b32 s2, s5, s2
+; GFX6-NEXT:    s_cmp_lt_i32 s2, s10
+; GFX6-NEXT:    s_cselect_b32 s2, s2, s10
+; GFX6-NEXT:    s_sub_i32 s1, s1, s2
+; GFX6-NEXT:    s_lshl_b32 s2, s3, 24
+; GFX6-NEXT:    s_ashr_i32 s1, s1, 24
+; GFX6-NEXT:    s_lshl_b32 s3, s6, 24
+; GFX6-NEXT:    s_cmp_gt_i32 s2, -1
+; GFX6-NEXT:    s_cselect_b32 s5, s2, -1
+; GFX6-NEXT:    s_sub_i32 s5, s5, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s2, -1
+; GFX6-NEXT:    s_cselect_b32 s6, s2, -1
+; GFX6-NEXT:    s_sub_i32 s6, s6, s9
+; GFX6-NEXT:    s_cmp_gt_i32 s5, s3
+; GFX6-NEXT:    s_cselect_b32 s3, s5, s3
+; GFX6-NEXT:    s_cmp_lt_i32 s3, s6
+; GFX6-NEXT:    s_cselect_b32 s3, s3, s6
+; GFX6-NEXT:    s_sub_i32 s2, s2, s3
+; GFX6-NEXT:    s_lshl_b32 s3, s4, 24
+; GFX6-NEXT:    s_ashr_i32 s2, s2, 24
+; GFX6-NEXT:    s_lshl_b32 s4, s7, 24
+; GFX6-NEXT:    s_cmp_gt_i32 s3, -1
+; GFX6-NEXT:    s_cselect_b32 s5, s3, -1
+; GFX6-NEXT:    s_sub_i32 s5, s5, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s3, -1
+; GFX6-NEXT:    s_cselect_b32 s6, s3, -1
+; GFX6-NEXT:    s_sub_i32 s6, s6, s9
+; GFX6-NEXT:    s_cmp_gt_i32 s5, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX6-NEXT:    s_cmp_lt_i32 s4, s6
+; GFX6-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX6-NEXT:    s_sub_i32 s3, s3, s4
+; GFX6-NEXT:    s_movk_i32 s4, 0xff
+; GFX6-NEXT:    s_and_b32 s1, s1, s4
+; GFX6-NEXT:    s_and_b32 s0, s0, s4
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_and_b32 s1, s2, s4
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_ashr_i32 s3, s3, 24
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_and_b32 s1, s3, s4
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_ssubsat_v4i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_bfe_u32 s8, 8, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX8-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s4, s0, 24
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s8
+; GFX8-NEXT:    s_lshr_b32 s5, s1, 8
+; GFX8-NEXT:    s_lshr_b32 s6, s1, 16
+; GFX8-NEXT:    s_lshr_b32 s7, s1, 24
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s8
+; GFX8-NEXT:    s_sext_i32_i16 s11, s0
+; GFX8-NEXT:    s_sext_i32_i16 s12, 0xffff
+; GFX8-NEXT:    s_cmp_gt_i32 s11, s12
+; GFX8-NEXT:    s_movk_i32 s9, 0x7fff
+; GFX8-NEXT:    s_cselect_b32 s13, s11, s12
+; GFX8-NEXT:    s_sub_i32 s13, s13, s9
+; GFX8-NEXT:    s_cmp_lt_i32 s11, s12
+; GFX8-NEXT:    s_mov_b32 s10, 0x8000
+; GFX8-NEXT:    s_cselect_b32 s11, s11, s12
+; GFX8-NEXT:    s_sub_i32 s11, s11, s10
+; GFX8-NEXT:    s_sext_i32_i16 s13, s13
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_cmp_gt_i32 s13, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s13, s1
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_sext_i32_i16 s11, s11
+; GFX8-NEXT:    s_cmp_lt_i32 s1, s11
+; GFX8-NEXT:    s_cselect_b32 s1, s1, s11
+; GFX8-NEXT:    s_sub_i32 s0, s0, s1
+; GFX8-NEXT:    s_sext_i32_i16 s0, s0
+; GFX8-NEXT:    s_lshl_b32 s1, s2, s8
+; GFX8-NEXT:    s_lshl_b32 s2, s5, s8
+; GFX8-NEXT:    s_ashr_i32 s0, s0, s8
+; GFX8-NEXT:    s_sext_i32_i16 s5, s1
+; GFX8-NEXT:    s_cmp_gt_i32 s5, s12
+; GFX8-NEXT:    s_cselect_b32 s11, s5, s12
+; GFX8-NEXT:    s_sub_i32 s11, s11, s9
+; GFX8-NEXT:    s_cmp_lt_i32 s5, s12
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s12
+; GFX8-NEXT:    s_sub_i32 s5, s5, s10
+; GFX8-NEXT:    s_sext_i32_i16 s11, s11
+; GFX8-NEXT:    s_sext_i32_i16 s2, s2
+; GFX8-NEXT:    s_cmp_gt_i32 s11, s2
+; GFX8-NEXT:    s_cselect_b32 s2, s11, s2
+; GFX8-NEXT:    s_sext_i32_i16 s2, s2
+; GFX8-NEXT:    s_sext_i32_i16 s5, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s2, s5
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s5
+; GFX8-NEXT:    s_sub_i32 s1, s1, s2
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_lshl_b32 s2, s3, s8
+; GFX8-NEXT:    s_lshl_b32 s3, s6, s8
+; GFX8-NEXT:    s_ashr_i32 s1, s1, s8
+; GFX8-NEXT:    s_sext_i32_i16 s5, s2
+; GFX8-NEXT:    s_cmp_gt_i32 s5, s12
+; GFX8-NEXT:    s_cselect_b32 s6, s5, s12
+; GFX8-NEXT:    s_sub_i32 s6, s6, s9
+; GFX8-NEXT:    s_cmp_lt_i32 s5, s12
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s12
+; GFX8-NEXT:    s_sub_i32 s5, s5, s10
+; GFX8-NEXT:    s_sext_i32_i16 s6, s6
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_cmp_gt_i32 s6, s3
+; GFX8-NEXT:    s_cselect_b32 s3, s6, s3
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_sext_i32_i16 s5, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s5
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s5
+; GFX8-NEXT:    s_sub_i32 s2, s2, s3
+; GFX8-NEXT:    s_sext_i32_i16 s2, s2
+; GFX8-NEXT:    s_lshl_b32 s3, s4, s8
+; GFX8-NEXT:    s_lshl_b32 s4, s7, s8
+; GFX8-NEXT:    s_ashr_i32 s2, s2, s8
+; GFX8-NEXT:    s_sext_i32_i16 s5, s3
+; GFX8-NEXT:    s_cmp_gt_i32 s5, s12
+; GFX8-NEXT:    s_cselect_b32 s6, s5, s12
+; GFX8-NEXT:    s_sub_i32 s6, s6, s9
+; GFX8-NEXT:    s_cmp_lt_i32 s5, s12
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s12
+; GFX8-NEXT:    s_sub_i32 s5, s5, s10
+; GFX8-NEXT:    s_sext_i32_i16 s6, s6
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_cmp_gt_i32 s6, s4
+; GFX8-NEXT:    s_cselect_b32 s4, s6, s4
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_sext_i32_i16 s5, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX8-NEXT:    s_sub_i32 s3, s3, s4
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_and_b32 s1, s1, s4
+; GFX8-NEXT:    s_and_b32 s0, s0, s4
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    s_and_b32 s1, s2, s4
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX8-NEXT:    s_ashr_i32 s3, s3, s8
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    s_and_b32 s1, s3, s4
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_ssubsat_v4i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_bfe_u32 s8, 8, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX9-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s4, s0, 24
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s8
+; GFX9-NEXT:    s_lshr_b32 s5, s1, 8
+; GFX9-NEXT:    s_lshr_b32 s6, s1, 16
+; GFX9-NEXT:    s_lshr_b32 s7, s1, 24
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s8
+; GFX9-NEXT:    s_sext_i32_i16 s11, s0
+; GFX9-NEXT:    s_sext_i32_i16 s12, 0xffff
+; GFX9-NEXT:    s_cmp_gt_i32 s11, s12
+; GFX9-NEXT:    s_movk_i32 s9, 0x7fff
+; GFX9-NEXT:    s_cselect_b32 s13, s11, s12
+; GFX9-NEXT:    s_sub_i32 s13, s13, s9
+; GFX9-NEXT:    s_cmp_lt_i32 s11, s12
+; GFX9-NEXT:    s_mov_b32 s10, 0x8000
+; GFX9-NEXT:    s_cselect_b32 s11, s11, s12
+; GFX9-NEXT:    s_sub_i32 s11, s11, s10
+; GFX9-NEXT:    s_sext_i32_i16 s13, s13
+; GFX9-NEXT:    s_sext_i32_i16 s1, s1
+; GFX9-NEXT:    s_cmp_gt_i32 s13, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s13, s1
+; GFX9-NEXT:    s_sext_i32_i16 s1, s1
+; GFX9-NEXT:    s_sext_i32_i16 s11, s11
+; GFX9-NEXT:    s_cmp_lt_i32 s1, s11
+; GFX9-NEXT:    s_cselect_b32 s1, s1, s11
+; GFX9-NEXT:    s_sub_i32 s0, s0, s1
+; GFX9-NEXT:    s_sext_i32_i16 s0, s0
+; GFX9-NEXT:    s_lshl_b32 s1, s2, s8
+; GFX9-NEXT:    s_lshl_b32 s2, s5, s8
+; GFX9-NEXT:    s_ashr_i32 s0, s0, s8
+; GFX9-NEXT:    s_sext_i32_i16 s5, s1
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s12
+; GFX9-NEXT:    s_cselect_b32 s11, s5, s12
+; GFX9-NEXT:    s_sub_i32 s11, s11, s9
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s12
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s12
+; GFX9-NEXT:    s_sub_i32 s5, s5, s10
+; GFX9-NEXT:    s_sext_i32_i16 s11, s11
+; GFX9-NEXT:    s_sext_i32_i16 s2, s2
+; GFX9-NEXT:    s_cmp_gt_i32 s11, s2
+; GFX9-NEXT:    s_cselect_b32 s2, s11, s2
+; GFX9-NEXT:    s_sext_i32_i16 s2, s2
+; GFX9-NEXT:    s_sext_i32_i16 s5, s5
+; GFX9-NEXT:    s_cmp_lt_i32 s2, s5
+; GFX9-NEXT:    s_cselect_b32 s2, s2, s5
+; GFX9-NEXT:    s_sub_i32 s1, s1, s2
+; GFX9-NEXT:    s_sext_i32_i16 s1, s1
+; GFX9-NEXT:    s_lshl_b32 s2, s3, s8
+; GFX9-NEXT:    s_lshl_b32 s3, s6, s8
+; GFX9-NEXT:    s_ashr_i32 s1, s1, s8
+; GFX9-NEXT:    s_sext_i32_i16 s5, s2
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s12
+; GFX9-NEXT:    s_cselect_b32 s6, s5, s12
+; GFX9-NEXT:    s_sub_i32 s6, s6, s9
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s12
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s12
+; GFX9-NEXT:    s_sub_i32 s5, s5, s10
+; GFX9-NEXT:    s_sext_i32_i16 s6, s6
+; GFX9-NEXT:    s_sext_i32_i16 s3, s3
+; GFX9-NEXT:    s_cmp_gt_i32 s6, s3
+; GFX9-NEXT:    s_cselect_b32 s3, s6, s3
+; GFX9-NEXT:    s_sext_i32_i16 s3, s3
+; GFX9-NEXT:    s_sext_i32_i16 s5, s5
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s5
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s5
+; GFX9-NEXT:    s_sub_i32 s2, s2, s3
+; GFX9-NEXT:    s_sext_i32_i16 s2, s2
+; GFX9-NEXT:    s_lshl_b32 s3, s4, s8
+; GFX9-NEXT:    s_lshl_b32 s4, s7, s8
+; GFX9-NEXT:    s_ashr_i32 s2, s2, s8
+; GFX9-NEXT:    s_sext_i32_i16 s5, s3
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s12
+; GFX9-NEXT:    s_cselect_b32 s6, s5, s12
+; GFX9-NEXT:    s_sub_i32 s6, s6, s9
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s12
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s12
+; GFX9-NEXT:    s_sub_i32 s5, s5, s10
+; GFX9-NEXT:    s_sext_i32_i16 s6, s6
+; GFX9-NEXT:    s_sext_i32_i16 s4, s4
+; GFX9-NEXT:    s_cmp_gt_i32 s6, s4
+; GFX9-NEXT:    s_cselect_b32 s4, s6, s4
+; GFX9-NEXT:    s_sext_i32_i16 s4, s4
+; GFX9-NEXT:    s_sext_i32_i16 s5, s5
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX9-NEXT:    s_sub_i32 s3, s3, s4
+; GFX9-NEXT:    s_movk_i32 s4, 0xff
+; GFX9-NEXT:    s_and_b32 s1, s1, s4
+; GFX9-NEXT:    s_and_b32 s0, s0, s4
+; GFX9-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX9-NEXT:    s_sext_i32_i16 s3, s3
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    s_and_b32 s1, s2, s4
+; GFX9-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX9-NEXT:    s_ashr_i32 s3, s3, s8
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    s_and_b32 s1, s3, s4
+; GFX9-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_ssubsat_v4i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_bfe_u32 s6, 8, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX10-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX10-NEXT:    s_lshr_b32 s4, s0, 24
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s6
+; GFX10-NEXT:    s_sext_i32_i16 s10, 0xffff
+; GFX10-NEXT:    s_sext_i32_i16 s9, s0
+; GFX10-NEXT:    s_lshr_b32 s5, s1, 8
+; GFX10-NEXT:    s_lshr_b32 s7, s1, 16
+; GFX10-NEXT:    s_lshr_b32 s8, s1, 24
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s6
+; GFX10-NEXT:    s_cmp_gt_i32 s9, s10
+; GFX10-NEXT:    s_movk_i32 s11, 0x7fff
+; GFX10-NEXT:    s_cselect_b32 s12, s9, s10
+; GFX10-NEXT:    s_mov_b32 s13, 0x8000
+; GFX10-NEXT:    s_sub_i32 s12, s12, s11
+; GFX10-NEXT:    s_cmp_lt_i32 s9, s10
+; GFX10-NEXT:    s_sext_i32_i16 s12, s12
+; GFX10-NEXT:    s_cselect_b32 s9, s9, s10
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_sub_i32 s9, s9, s13
+; GFX10-NEXT:    s_cmp_gt_i32 s12, s1
+; GFX10-NEXT:    s_sext_i32_i16 s9, s9
+; GFX10-NEXT:    s_cselect_b32 s1, s12, s1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_cmp_lt_i32 s1, s9
+; GFX10-NEXT:    s_cselect_b32 s1, s1, s9
+; GFX10-NEXT:    s_lshl_b32 s5, s5, s6
+; GFX10-NEXT:    s_sub_i32 s0, s0, s1
+; GFX10-NEXT:    s_lshl_b32 s1, s2, s6
+; GFX10-NEXT:    s_sext_i32_i16 s0, s0
+; GFX10-NEXT:    s_sext_i32_i16 s2, s1
+; GFX10-NEXT:    s_ashr_i32 s0, s0, s6
+; GFX10-NEXT:    s_cmp_gt_i32 s2, s10
+; GFX10-NEXT:    s_sext_i32_i16 s5, s5
+; GFX10-NEXT:    s_cselect_b32 s9, s2, s10
+; GFX10-NEXT:    s_sub_i32 s9, s9, s11
+; GFX10-NEXT:    s_cmp_lt_i32 s2, s10
+; GFX10-NEXT:    s_sext_i32_i16 s9, s9
+; GFX10-NEXT:    s_cselect_b32 s2, s2, s10
+; GFX10-NEXT:    s_sub_i32 s2, s2, s13
+; GFX10-NEXT:    s_cmp_gt_i32 s9, s5
+; GFX10-NEXT:    s_sext_i32_i16 s2, s2
+; GFX10-NEXT:    s_cselect_b32 s5, s9, s5
+; GFX10-NEXT:    s_sext_i32_i16 s5, s5
+; GFX10-NEXT:    s_cmp_lt_i32 s5, s2
+; GFX10-NEXT:    s_cselect_b32 s2, s5, s2
+; GFX10-NEXT:    s_lshl_b32 s3, s3, s6
+; GFX10-NEXT:    s_sub_i32 s1, s1, s2
+; GFX10-NEXT:    s_sext_i32_i16 s5, s3
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_lshl_b32 s2, s7, s6
+; GFX10-NEXT:    s_ashr_i32 s1, s1, s6
+; GFX10-NEXT:    s_cmp_gt_i32 s5, s10
+; GFX10-NEXT:    s_sext_i32_i16 s2, s2
+; GFX10-NEXT:    s_cselect_b32 s7, s5, s10
+; GFX10-NEXT:    s_sub_i32 s7, s7, s11
+; GFX10-NEXT:    s_cmp_lt_i32 s5, s10
+; GFX10-NEXT:    s_sext_i32_i16 s7, s7
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s10
+; GFX10-NEXT:    s_sub_i32 s5, s5, s13
+; GFX10-NEXT:    s_cmp_gt_i32 s7, s2
+; GFX10-NEXT:    s_sext_i32_i16 s5, s5
+; GFX10-NEXT:    s_cselect_b32 s2, s7, s2
+; GFX10-NEXT:    s_sext_i32_i16 s2, s2
+; GFX10-NEXT:    s_cmp_lt_i32 s2, s5
+; GFX10-NEXT:    s_cselect_b32 s2, s2, s5
+; GFX10-NEXT:    s_lshl_b32 s4, s4, s6
+; GFX10-NEXT:    s_sub_i32 s2, s3, s2
+; GFX10-NEXT:    s_sext_i32_i16 s5, s4
+; GFX10-NEXT:    s_sext_i32_i16 s2, s2
+; GFX10-NEXT:    s_lshl_b32 s3, s8, s6
+; GFX10-NEXT:    s_ashr_i32 s2, s2, s6
+; GFX10-NEXT:    s_cmp_gt_i32 s5, s10
+; GFX10-NEXT:    s_sext_i32_i16 s3, s3
+; GFX10-NEXT:    s_cselect_b32 s7, s5, s10
+; GFX10-NEXT:    s_sub_i32 s7, s7, s11
+; GFX10-NEXT:    s_cmp_lt_i32 s5, s10
+; GFX10-NEXT:    s_sext_i32_i16 s7, s7
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s10
+; GFX10-NEXT:    s_sub_i32 s5, s5, s13
+; GFX10-NEXT:    s_cmp_gt_i32 s7, s3
+; GFX10-NEXT:    s_sext_i32_i16 s5, s5
+; GFX10-NEXT:    s_cselect_b32 s3, s7, s3
+; GFX10-NEXT:    s_movk_i32 s7, 0xff
+; GFX10-NEXT:    s_sext_i32_i16 s3, s3
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s5
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s5
+; GFX10-NEXT:    s_and_b32 s1, s1, s7
+; GFX10-NEXT:    s_sub_i32 s3, s4, s3
+; GFX10-NEXT:    s_and_b32 s0, s0, s7
+; GFX10-NEXT:    s_sext_i32_i16 s3, s3
+; GFX10-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX10-NEXT:    s_and_b32 s2, s2, s7
+; GFX10-NEXT:    s_ashr_i32 s3, s3, s6
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    s_lshl_b32 s1, s2, 16
+; GFX10-NEXT:    s_and_b32 s2, s3, s7
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    s_lshl_b32 s1, s2, 24
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %lhs = bitcast i32 %lhs.arg to <4 x i8>
+  %rhs = bitcast i32 %rhs.arg to <4 x i8>
+  %result = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> %lhs, <4 x i8> %rhs)
+  %cast.result = bitcast <4 x i8> %result to i32
+  ret i32 %cast.result
+}
+
+define i24 @v_ssubsat_i24(i24 %lhs, i24 %rhs) {
+; GFX6-LABEL: v_ssubsat_i24:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX6-NEXT:    v_max_i32_e32 v2, -1, v0
+; GFX6-NEXT:    v_min_i32_e32 v3, -1, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, 0x7fffffff, v2
+; GFX6-NEXT:    v_subrev_i32_e32 v3, vcc, 0x80000000, v3
+; GFX6-NEXT:    v_max_i32_e32 v1, v2, v1
+; GFX6-NEXT:    v_min_i32_e32 v1, v1, v3
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 8, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_i24:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, v0, v1
+; GFX8-NEXT:    v_bfe_i32 v3, v2, 0, 24
+; GFX8-NEXT:    v_bfe_i32 v0, v0, 0, 24
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v3, v0
+; GFX8-NEXT:    v_bfe_i32 v0, v1, 0, 24
+; GFX8-NEXT:    s_bfe_i32 s6, 0, 0x180000
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[6:7], s6, v0
+; GFX8-NEXT:    v_ashrrev_i32_e32 v0, 23, v3
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 0xff800000, v0
+; GFX8-NEXT:    s_xor_b64 vcc, s[6:7], s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_i24:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX9-NEXT:    v_max_i32_e32 v2, -1, v0
+; GFX9-NEXT:    v_min_i32_e32 v3, -1, v0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX9-NEXT:    v_subrev_u32_e32 v2, 0x7fffffff, v2
+; GFX9-NEXT:    v_subrev_u32_e32 v3, 0x80000000, v3
+; GFX9-NEXT:    v_max_i32_e32 v1, v2, v1
+; GFX9-NEXT:    v_min_i32_e32 v1, v1, v3
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 8, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_ssubsat_i24:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_max_i32_e32 v2, -1, v0
+; GFX10-NEXT:    v_min_i32_e32 v3, -1, v0
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v2, 0x7fffffff, v2
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, 0x80000000, v3
+; GFX10-NEXT:    v_max_i32_e32 v1, v2, v1
+; GFX10-NEXT:    v_min_i32_e32 v1, v1, v3
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT:    v_ashrrev_i32_e32 v0, 8, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i24 @llvm.ssub.sat.i24(i24 %lhs, i24 %rhs)
+  ret i24 %result
+}
+
+define amdgpu_ps i24 @s_ssubsat_i24(i24 inreg %lhs, i24 inreg %rhs) {
+; GFX6-LABEL: s_ssubsat_i24:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 8
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX6-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX6-NEXT:    s_cselect_b32 s2, s0, -1
+; GFX6-NEXT:    s_sub_i32 s2, s2, 0x7fffffff
+; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX6-NEXT:    s_cselect_b32 s3, s0, -1
+; GFX6-NEXT:    s_sub_i32 s3, s3, 0x80000000
+; GFX6-NEXT:    s_cmp_gt_i32 s2, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX6-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX6-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX6-NEXT:    s_sub_i32 s0, s0, s1
+; GFX6-NEXT:    s_ashr_i32 s0, s0, 8
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_ssubsat_i24:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_sub_i32 s2, s0, s1
+; GFX8-NEXT:    s_bfe_i32 s3, s2, 0x180000
+; GFX8-NEXT:    s_bfe_i32 s0, s0, 0x180000
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s0
+; GFX8-NEXT:    s_cselect_b32 s0, 1, 0
+; GFX8-NEXT:    s_bfe_i32 s1, s1, 0x180000
+; GFX8-NEXT:    s_bfe_i32 s4, 0, 0x180000
+; GFX8-NEXT:    s_cmp_gt_i32 s1, s4
+; GFX8-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX8-NEXT:    s_xor_b32 s0, s1, s0
+; GFX8-NEXT:    s_ashr_i32 s1, s3, 23
+; GFX8-NEXT:    s_add_i32 s1, s1, 0xff800000
+; GFX8-NEXT:    s_and_b32 s0, s0, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s0, 0
+; GFX8-NEXT:    s_cselect_b32 s0, s1, s2
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_ssubsat_i24:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 8
+; GFX9-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX9-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX9-NEXT:    s_cselect_b32 s2, s0, -1
+; GFX9-NEXT:    s_sub_i32 s2, s2, 0x7fffffff
+; GFX9-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX9-NEXT:    s_cselect_b32 s3, s0, -1
+; GFX9-NEXT:    s_sub_i32 s3, s3, 0x80000000
+; GFX9-NEXT:    s_cmp_gt_i32 s2, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX9-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX9-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX9-NEXT:    s_sub_i32 s0, s0, s1
+; GFX9-NEXT:    s_ashr_i32 s0, s0, 8
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_ssubsat_i24:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_lshl_b32 s0, s0, 8
+; GFX10-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX10-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s2, s0, -1
+; GFX10-NEXT:    s_sub_i32 s2, s2, 0x7fffffff
+; GFX10-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX10-NEXT:    s_cselect_b32 s3, s0, -1
+; GFX10-NEXT:    s_sub_i32 s3, s3, 0x80000000
+; GFX10-NEXT:    s_cmp_gt_i32 s2, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX10-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX10-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX10-NEXT:    s_sub_i32 s0, s0, s1
+; GFX10-NEXT:    s_ashr_i32 s0, s0, 8
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i24 @llvm.ssub.sat.i24(i24 %lhs, i24 %rhs)
+  ret i24 %result
+}
+
+define i32 @v_ssubsat_i32(i32 %lhs, i32 %rhs) {
+; GFX6-LABEL: v_ssubsat_i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_max_i32_e32 v2, -1, v0
+; GFX6-NEXT:    v_min_i32_e32 v3, -1, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, 0x7fffffff, v2
+; GFX6-NEXT:    v_subrev_i32_e32 v3, vcc, 0x80000000, v3
+; GFX6-NEXT:    v_max_i32_e32 v1, v2, v1
+; GFX6-NEXT:    v_min_i32_e32 v1, v1, v3
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_max_i32_e32 v2, -1, v0
+; GFX8-NEXT:    v_min_i32_e32 v3, -1, v0
+; GFX8-NEXT:    v_subrev_u32_e32 v2, vcc, 0x7fffffff, v2
+; GFX8-NEXT:    v_subrev_u32_e32 v3, vcc, 0x80000000, v3
+; GFX8-NEXT:    v_max_i32_e32 v1, v2, v1
+; GFX8-NEXT:    v_min_i32_e32 v1, v1, v3
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_max_i32_e32 v2, -1, v0
+; GFX9-NEXT:    v_min_i32_e32 v3, -1, v0
+; GFX9-NEXT:    v_subrev_u32_e32 v2, 0x7fffffff, v2
+; GFX9-NEXT:    v_subrev_u32_e32 v3, 0x80000000, v3
+; GFX9-NEXT:    v_max_i32_e32 v1, v2, v1
+; GFX9-NEXT:    v_min_i32_e32 v1, v1, v3
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_ssubsat_i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_max_i32_e32 v2, -1, v0
+; GFX10-NEXT:    v_min_i32_e32 v3, -1, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v2, 0x7fffffff, v2
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, 0x80000000, v3
+; GFX10-NEXT:    v_max_i32_e32 v1, v2, v1
+; GFX10-NEXT:    v_min_i32_e32 v1, v1, v3
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i32 @llvm.ssub.sat.i32(i32 %lhs, i32 %rhs)
+  ret i32 %result
+}
+
+define amdgpu_ps i32 @s_ssubsat_i32(i32 inreg %lhs, i32 inreg %rhs) {
+; GFX6-LABEL: s_ssubsat_i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX6-NEXT:    s_cselect_b32 s2, s0, -1
+; GFX6-NEXT:    s_sub_i32 s2, s2, 0x7fffffff
+; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX6-NEXT:    s_cselect_b32 s3, s0, -1
+; GFX6-NEXT:    s_sub_i32 s3, s3, 0x80000000
+; GFX6-NEXT:    s_cmp_gt_i32 s2, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX6-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX6-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX6-NEXT:    s_sub_i32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_ssubsat_i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX8-NEXT:    s_cselect_b32 s2, s0, -1
+; GFX8-NEXT:    s_sub_i32 s2, s2, 0x7fffffff
+; GFX8-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX8-NEXT:    s_cselect_b32 s3, s0, -1
+; GFX8-NEXT:    s_sub_i32 s3, s3, 0x80000000
+; GFX8-NEXT:    s_cmp_gt_i32 s2, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX8-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX8-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX8-NEXT:    s_sub_i32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_ssubsat_i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX9-NEXT:    s_cselect_b32 s2, s0, -1
+; GFX9-NEXT:    s_sub_i32 s2, s2, 0x7fffffff
+; GFX9-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX9-NEXT:    s_cselect_b32 s3, s0, -1
+; GFX9-NEXT:    s_sub_i32 s3, s3, 0x80000000
+; GFX9-NEXT:    s_cmp_gt_i32 s2, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX9-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX9-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX9-NEXT:    s_sub_i32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_ssubsat_i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s2, s0, -1
+; GFX10-NEXT:    s_sub_i32 s2, s2, 0x7fffffff
+; GFX10-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX10-NEXT:    s_cselect_b32 s3, s0, -1
+; GFX10-NEXT:    s_sub_i32 s3, s3, 0x80000000
+; GFX10-NEXT:    s_cmp_gt_i32 s2, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX10-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX10-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX10-NEXT:    s_sub_i32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i32 @llvm.ssub.sat.i32(i32 %lhs, i32 %rhs)
+  ret i32 %result
+}
+
+define amdgpu_ps float @ssubsat_i32_sv(i32 inreg %lhs, i32 %rhs) {
+; GFX6-LABEL: ssubsat_i32_sv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX6-NEXT:    s_cselect_b32 s1, s0, -1
+; GFX6-NEXT:    s_sub_i32 s1, s1, 0x7fffffff
+; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX6-NEXT:    s_cselect_b32 s2, s0, -1
+; GFX6-NEXT:    s_sub_i32 s2, s2, 0x80000000
+; GFX6-NEXT:    v_max_i32_e32 v0, s1, v0
+; GFX6-NEXT:    v_min_i32_e32 v0, s2, v0
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: ssubsat_i32_sv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX8-NEXT:    s_cselect_b32 s1, s0, -1
+; GFX8-NEXT:    s_sub_i32 s1, s1, 0x7fffffff
+; GFX8-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX8-NEXT:    s_cselect_b32 s2, s0, -1
+; GFX8-NEXT:    s_sub_i32 s2, s2, 0x80000000
+; GFX8-NEXT:    v_max_i32_e32 v0, s1, v0
+; GFX8-NEXT:    v_min_i32_e32 v0, s2, v0
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: ssubsat_i32_sv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX9-NEXT:    s_cselect_b32 s1, s0, -1
+; GFX9-NEXT:    s_sub_i32 s1, s1, 0x7fffffff
+; GFX9-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX9-NEXT:    s_cselect_b32 s2, s0, -1
+; GFX9-NEXT:    s_sub_i32 s2, s2, 0x80000000
+; GFX9-NEXT:    v_max_i32_e32 v0, s1, v0
+; GFX9-NEXT:    v_min_i32_e32 v0, s2, v0
+; GFX9-NEXT:    v_sub_u32_e32 v0, s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: ssubsat_i32_sv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s1, s0, -1
+; GFX10-NEXT:    s_sub_i32 s1, s1, 0x7fffffff
+; GFX10-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX10-NEXT:    v_max_i32_e32 v0, s1, v0
+; GFX10-NEXT:    s_cselect_b32 s1, s0, -1
+; GFX10-NEXT:    s_sub_i32 s1, s1, 0x80000000
+; GFX10-NEXT:    v_min_i32_e32 v0, s1, v0
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i32 @llvm.ssub.sat.i32(i32 %lhs, i32 %rhs)
+  %cast = bitcast i32 %result to float
+  ret float %cast
+}
+
+define amdgpu_ps float @ssubsat_i32_vs(i32 %lhs, i32 inreg %rhs) {
+; GFX6-LABEL: ssubsat_i32_vs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_max_i32_e32 v1, -1, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v1, vcc, 0x7fffffff, v1
+; GFX6-NEXT:    v_min_i32_e32 v2, -1, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, 0x80000000, v2
+; GFX6-NEXT:    v_max_i32_e32 v1, s0, v1
+; GFX6-NEXT:    v_min_i32_e32 v1, v1, v2
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: ssubsat_i32_vs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_max_i32_e32 v1, -1, v0
+; GFX8-NEXT:    v_subrev_u32_e32 v1, vcc, 0x7fffffff, v1
+; GFX8-NEXT:    v_min_i32_e32 v2, -1, v0
+; GFX8-NEXT:    v_subrev_u32_e32 v2, vcc, 0x80000000, v2
+; GFX8-NEXT:    v_max_i32_e32 v1, s0, v1
+; GFX8-NEXT:    v_min_i32_e32 v1, v1, v2
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: ssubsat_i32_vs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_max_i32_e32 v1, -1, v0
+; GFX9-NEXT:    v_subrev_u32_e32 v1, 0x7fffffff, v1
+; GFX9-NEXT:    v_min_i32_e32 v2, -1, v0
+; GFX9-NEXT:    v_subrev_u32_e32 v2, 0x80000000, v2
+; GFX9-NEXT:    v_max_i32_e32 v1, s0, v1
+; GFX9-NEXT:    v_min_i32_e32 v1, v1, v2
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: ssubsat_i32_vs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_max_i32_e32 v1, -1, v0
+; GFX10-NEXT:    v_min_i32_e32 v2, -1, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v1, 0x7fffffff, v1
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v2, 0x80000000, v2
+; GFX10-NEXT:    v_max_i32_e32 v1, s0, v1
+; GFX10-NEXT:    v_min_i32_e32 v1, v1, v2
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i32 @llvm.ssub.sat.i32(i32 %lhs, i32 %rhs)
+  %cast = bitcast i32 %result to float
+  ret float %cast
+}
+
+define <2 x i32> @v_ssubsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
+; GFX6-LABEL: v_ssubsat_v2i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    v_max_i32_e32 v4, -1, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v4, vcc, s4, v4
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v5, -1, v0
+; GFX6-NEXT:    v_max_i32_e32 v2, v4, v2
+; GFX6-NEXT:    v_subrev_i32_e32 v5, vcc, s5, v5
+; GFX6-NEXT:    v_min_i32_e32 v2, v2, v5
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT:    v_max_i32_e32 v2, -1, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, s4, v2
+; GFX6-NEXT:    v_min_i32_e32 v4, -1, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v4, vcc, s5, v4
+; GFX6-NEXT:    v_max_i32_e32 v2, v2, v3
+; GFX6-NEXT:    v_min_i32_e32 v2, v2, v4
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_v2i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_brev_b32 s4, -2
+; GFX8-NEXT:    v_max_i32_e32 v4, -1, v0
+; GFX8-NEXT:    v_subrev_u32_e32 v4, vcc, s4, v4
+; GFX8-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX8-NEXT:    v_min_i32_e32 v5, -1, v0
+; GFX8-NEXT:    v_max_i32_e32 v2, v4, v2
+; GFX8-NEXT:    v_subrev_u32_e32 v5, vcc, s5, v5
+; GFX8-NEXT:    v_min_i32_e32 v2, v2, v5
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT:    v_max_i32_e32 v2, -1, v1
+; GFX8-NEXT:    v_subrev_u32_e32 v2, vcc, s4, v2
+; GFX8-NEXT:    v_min_i32_e32 v4, -1, v1
+; GFX8-NEXT:    v_subrev_u32_e32 v4, vcc, s5, v4
+; GFX8-NEXT:    v_max_i32_e32 v2, v2, v3
+; GFX8-NEXT:    v_min_i32_e32 v2, v2, v4
+; GFX8-NEXT:    v_sub_u32_e32 v1, vcc, v1, v2
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_v2i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_brev_b32 s4, -2
+; GFX9-NEXT:    v_max_i32_e32 v4, -1, v0
+; GFX9-NEXT:    v_subrev_u32_e32 v4, s4, v4
+; GFX9-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX9-NEXT:    v_min_i32_e32 v5, -1, v0
+; GFX9-NEXT:    v_max_i32_e32 v2, v4, v2
+; GFX9-NEXT:    v_subrev_u32_e32 v5, s5, v5
+; GFX9-NEXT:    v_min_i32_e32 v2, v2, v5
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v2
+; GFX9-NEXT:    v_max_i32_e32 v2, -1, v1
+; GFX9-NEXT:    v_subrev_u32_e32 v2, s4, v2
+; GFX9-NEXT:    v_min_i32_e32 v4, -1, v1
+; GFX9-NEXT:    v_subrev_u32_e32 v4, s5, v4
+; GFX9-NEXT:    v_max_i32_e32 v2, v2, v3
+; GFX9-NEXT:    v_min_i32_e32 v2, v2, v4
+; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_ssubsat_v2i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_max_i32_e32 v4, -1, v0
+; GFX10-NEXT:    v_max_i32_e32 v5, -1, v1
+; GFX10-NEXT:    s_brev_b32 s4, -2
+; GFX10-NEXT:    v_min_i32_e32 v6, -1, v0
+; GFX10-NEXT:    v_min_i32_e32 v7, -1, v1
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v4, s4, v4
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v5, s4, v5
+; GFX10-NEXT:    s_mov_b32 s4, 0x80000000
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v6, s4, v6
+; GFX10-NEXT:    v_max_i32_e32 v11, v4, v2
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v7, s4, v7
+; GFX10-NEXT:    v_max_i32_e32 v10, v5, v3
+; GFX10-NEXT:    v_min_i32_e32 v2, v11, v6
+; GFX10-NEXT:    v_min_i32_e32 v3, v10, v7
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v2
+; GFX10-NEXT:    v_sub_nc_u32_e32 v1, v1, v3
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> %lhs, <2 x i32> %rhs)
+  ret <2 x i32> %result
+}
+
+define amdgpu_ps <2 x i32> @s_ssubsat_v2i32(<2 x i32> inreg %lhs, <2 x i32> inreg %rhs) {
+; GFX6-LABEL: s_ssubsat_v2i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    s_cselect_b32 s6, s0, -1
+; GFX6-NEXT:    s_sub_i32 s6, s6, s4
+; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_cselect_b32 s7, s0, -1
+; GFX6-NEXT:    s_sub_i32 s7, s7, s5
+; GFX6-NEXT:    s_cmp_gt_i32 s6, s2
+; GFX6-NEXT:    s_cselect_b32 s2, s6, s2
+; GFX6-NEXT:    s_cmp_lt_i32 s2, s7
+; GFX6-NEXT:    s_cselect_b32 s2, s2, s7
+; GFX6-NEXT:    s_sub_i32 s0, s0, s2
+; GFX6-NEXT:    s_cmp_gt_i32 s1, -1
+; GFX6-NEXT:    s_cselect_b32 s2, s1, -1
+; GFX6-NEXT:    s_sub_i32 s2, s2, s4
+; GFX6-NEXT:    s_cmp_lt_i32 s1, -1
+; GFX6-NEXT:    s_cselect_b32 s4, s1, -1
+; GFX6-NEXT:    s_sub_i32 s4, s4, s5
+; GFX6-NEXT:    s_cmp_gt_i32 s2, s3
+; GFX6-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX6-NEXT:    s_cmp_lt_i32 s2, s4
+; GFX6-NEXT:    s_cselect_b32 s2, s2, s4
+; GFX6-NEXT:    s_sub_i32 s1, s1, s2
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_ssubsat_v2i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX8-NEXT:    s_brev_b32 s4, -2
+; GFX8-NEXT:    s_cselect_b32 s6, s0, -1
+; GFX8-NEXT:    s_sub_i32 s6, s6, s4
+; GFX8-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX8-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX8-NEXT:    s_cselect_b32 s7, s0, -1
+; GFX8-NEXT:    s_sub_i32 s7, s7, s5
+; GFX8-NEXT:    s_cmp_gt_i32 s6, s2
+; GFX8-NEXT:    s_cselect_b32 s2, s6, s2
+; GFX8-NEXT:    s_cmp_lt_i32 s2, s7
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s7
+; GFX8-NEXT:    s_sub_i32 s0, s0, s2
+; GFX8-NEXT:    s_cmp_gt_i32 s1, -1
+; GFX8-NEXT:    s_cselect_b32 s2, s1, -1
+; GFX8-NEXT:    s_sub_i32 s2, s2, s4
+; GFX8-NEXT:    s_cmp_lt_i32 s1, -1
+; GFX8-NEXT:    s_cselect_b32 s4, s1, -1
+; GFX8-NEXT:    s_sub_i32 s4, s4, s5
+; GFX8-NEXT:    s_cmp_gt_i32 s2, s3
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX8-NEXT:    s_cmp_lt_i32 s2, s4
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s4
+; GFX8-NEXT:    s_sub_i32 s1, s1, s2
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_ssubsat_v2i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX9-NEXT:    s_brev_b32 s4, -2
+; GFX9-NEXT:    s_cselect_b32 s6, s0, -1
+; GFX9-NEXT:    s_sub_i32 s6, s6, s4
+; GFX9-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX9-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX9-NEXT:    s_cselect_b32 s7, s0, -1
+; GFX9-NEXT:    s_sub_i32 s7, s7, s5
+; GFX9-NEXT:    s_cmp_gt_i32 s6, s2
+; GFX9-NEXT:    s_cselect_b32 s2, s6, s2
+; GFX9-NEXT:    s_cmp_lt_i32 s2, s7
+; GFX9-NEXT:    s_cselect_b32 s2, s2, s7
+; GFX9-NEXT:    s_sub_i32 s0, s0, s2
+; GFX9-NEXT:    s_cmp_gt_i32 s1, -1
+; GFX9-NEXT:    s_cselect_b32 s2, s1, -1
+; GFX9-NEXT:    s_sub_i32 s2, s2, s4
+; GFX9-NEXT:    s_cmp_lt_i32 s1, -1
+; GFX9-NEXT:    s_cselect_b32 s4, s1, -1
+; GFX9-NEXT:    s_sub_i32 s4, s4, s5
+; GFX9-NEXT:    s_cmp_gt_i32 s2, s3
+; GFX9-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX9-NEXT:    s_cmp_lt_i32 s2, s4
+; GFX9-NEXT:    s_cselect_b32 s2, s2, s4
+; GFX9-NEXT:    s_sub_i32 s1, s1, s2
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_ssubsat_v2i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX10-NEXT:    s_brev_b32 s4, -2
+; GFX10-NEXT:    s_cselect_b32 s5, s0, -1
+; GFX10-NEXT:    s_mov_b32 s6, 0x80000000
+; GFX10-NEXT:    s_sub_i32 s5, s5, s4
+; GFX10-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s7, s0, -1
+; GFX10-NEXT:    s_sub_i32 s7, s7, s6
+; GFX10-NEXT:    s_cmp_gt_i32 s5, s2
+; GFX10-NEXT:    s_cselect_b32 s2, s5, s2
+; GFX10-NEXT:    s_cmp_lt_i32 s2, s7
+; GFX10-NEXT:    s_cselect_b32 s2, s2, s7
+; GFX10-NEXT:    s_sub_i32 s0, s0, s2
+; GFX10-NEXT:    s_cmp_gt_i32 s1, -1
+; GFX10-NEXT:    s_cselect_b32 s2, s1, -1
+; GFX10-NEXT:    s_sub_i32 s2, s2, s4
+; GFX10-NEXT:    s_cmp_lt_i32 s1, -1
+; GFX10-NEXT:    s_cselect_b32 s4, s1, -1
+; GFX10-NEXT:    s_sub_i32 s4, s4, s6
+; GFX10-NEXT:    s_cmp_gt_i32 s2, s3
+; GFX10-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX10-NEXT:    s_cmp_lt_i32 s2, s4
+; GFX10-NEXT:    s_cselect_b32 s2, s2, s4
+; GFX10-NEXT:    s_sub_i32 s1, s1, s2
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> %lhs, <2 x i32> %rhs)
+  ret <2 x i32> %result
+}
+
+define <3 x i32> @v_ssubsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
+; GFX6-LABEL: v_ssubsat_v3i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    v_max_i32_e32 v6, -1, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v6, vcc, s4, v6
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v7, -1, v0
+; GFX6-NEXT:    v_max_i32_e32 v3, v6, v3
+; GFX6-NEXT:    v_subrev_i32_e32 v7, vcc, s5, v7
+; GFX6-NEXT:    v_min_i32_e32 v3, v3, v7
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; GFX6-NEXT:    v_max_i32_e32 v3, -1, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v3, vcc, s4, v3
+; GFX6-NEXT:    v_min_i32_e32 v6, -1, v1
+; GFX6-NEXT:    v_max_i32_e32 v3, v3, v4
+; GFX6-NEXT:    v_subrev_i32_e32 v6, vcc, s5, v6
+; GFX6-NEXT:    v_min_i32_e32 v3, v3, v6
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v3
+; GFX6-NEXT:    v_max_i32_e32 v3, -1, v2
+; GFX6-NEXT:    v_subrev_i32_e32 v3, vcc, s4, v3
+; GFX6-NEXT:    v_min_i32_e32 v4, -1, v2
+; GFX6-NEXT:    v_subrev_i32_e32 v4, vcc, s5, v4
+; GFX6-NEXT:    v_max_i32_e32 v3, v3, v5
+; GFX6-NEXT:    v_min_i32_e32 v3, v3, v4
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v3
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_v3i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_brev_b32 s4, -2
+; GFX8-NEXT:    v_max_i32_e32 v6, -1, v0
+; GFX8-NEXT:    v_subrev_u32_e32 v6, vcc, s4, v6
+; GFX8-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX8-NEXT:    v_min_i32_e32 v7, -1, v0
+; GFX8-NEXT:    v_max_i32_e32 v3, v6, v3
+; GFX8-NEXT:    v_subrev_u32_e32 v7, vcc, s5, v7
+; GFX8-NEXT:    v_min_i32_e32 v3, v3, v7
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, v0, v3
+; GFX8-NEXT:    v_max_i32_e32 v3, -1, v1
+; GFX8-NEXT:    v_subrev_u32_e32 v3, vcc, s4, v3
+; GFX8-NEXT:    v_min_i32_e32 v6, -1, v1
+; GFX8-NEXT:    v_max_i32_e32 v3, v3, v4
+; GFX8-NEXT:    v_subrev_u32_e32 v6, vcc, s5, v6
+; GFX8-NEXT:    v_min_i32_e32 v3, v3, v6
+; GFX8-NEXT:    v_sub_u32_e32 v1, vcc, v1, v3
+; GFX8-NEXT:    v_max_i32_e32 v3, -1, v2
+; GFX8-NEXT:    v_subrev_u32_e32 v3, vcc, s4, v3
+; GFX8-NEXT:    v_min_i32_e32 v4, -1, v2
+; GFX8-NEXT:    v_subrev_u32_e32 v4, vcc, s5, v4
+; GFX8-NEXT:    v_max_i32_e32 v3, v3, v5
+; GFX8-NEXT:    v_min_i32_e32 v3, v3, v4
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, v2, v3
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_v3i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_brev_b32 s4, -2
+; GFX9-NEXT:    v_max_i32_e32 v6, -1, v0
+; GFX9-NEXT:    v_subrev_u32_e32 v6, s4, v6
+; GFX9-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX9-NEXT:    v_min_i32_e32 v7, -1, v0
+; GFX9-NEXT:    v_max_i32_e32 v3, v6, v3
+; GFX9-NEXT:    v_subrev_u32_e32 v7, s5, v7
+; GFX9-NEXT:    v_min_i32_e32 v3, v3, v7
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v3
+; GFX9-NEXT:    v_max_i32_e32 v3, -1, v1
+; GFX9-NEXT:    v_subrev_u32_e32 v3, s4, v3
+; GFX9-NEXT:    v_min_i32_e32 v6, -1, v1
+; GFX9-NEXT:    v_max_i32_e32 v3, v3, v4
+; GFX9-NEXT:    v_subrev_u32_e32 v6, s5, v6
+; GFX9-NEXT:    v_min_i32_e32 v3, v3, v6
+; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v3
+; GFX9-NEXT:    v_max_i32_e32 v3, -1, v2
+; GFX9-NEXT:    v_subrev_u32_e32 v3, s4, v3
+; GFX9-NEXT:    v_min_i32_e32 v4, -1, v2
+; GFX9-NEXT:    v_subrev_u32_e32 v4, s5, v4
+; GFX9-NEXT:    v_max_i32_e32 v3, v3, v5
+; GFX9-NEXT:    v_min_i32_e32 v3, v3, v4
+; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v3
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_ssubsat_v3i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_max_i32_e32 v6, -1, v0
+; GFX10-NEXT:    v_max_i32_e32 v8, -1, v1
+; GFX10-NEXT:    v_max_i32_e32 v9, -1, v2
+; GFX10-NEXT:    s_brev_b32 s4, -2
+; GFX10-NEXT:    v_min_i32_e32 v7, -1, v0
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v6, s4, v6
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v15, s4, v8
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v19, s4, v9
+; GFX10-NEXT:    v_min_i32_e32 v10, -1, v1
+; GFX10-NEXT:    v_min_i32_e32 v11, -1, v2
+; GFX10-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX10-NEXT:    v_max_i32_e32 v14, v6, v3
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v7, s5, v7
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v6, s5, v10
+; GFX10-NEXT:    v_max_i32_e32 v4, v15, v4
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v8, s5, v11
+; GFX10-NEXT:    v_max_i32_e32 v5, v19, v5
+; GFX10-NEXT:    v_min_i32_e32 v3, v14, v7
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_min_i32_e32 v4, v4, v6
+; GFX10-NEXT:    v_min_i32_e32 v5, v5, v8
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v3
+; GFX10-NEXT:    v_sub_nc_u32_e32 v1, v1, v4
+; GFX10-NEXT:    v_sub_nc_u32_e32 v2, v2, v5
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <3 x i32> @llvm.ssub.sat.v3i32(<3 x i32> %lhs, <3 x i32> %rhs)
+  ret <3 x i32> %result
+}
+
+define amdgpu_ps <3 x i32> @s_ssubsat_v3i32(<3 x i32> inreg %lhs, <3 x i32> inreg %rhs) {
+; GFX6-LABEL: s_ssubsat_v3i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX6-NEXT:    s_brev_b32 s6, -2
+; GFX6-NEXT:    s_cselect_b32 s8, s0, -1
+; GFX6-NEXT:    s_sub_i32 s8, s8, s6
+; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX6-NEXT:    s_mov_b32 s7, 0x80000000
+; GFX6-NEXT:    s_cselect_b32 s9, s0, -1
+; GFX6-NEXT:    s_sub_i32 s9, s9, s7
+; GFX6-NEXT:    s_cmp_gt_i32 s8, s3
+; GFX6-NEXT:    s_cselect_b32 s3, s8, s3
+; GFX6-NEXT:    s_cmp_lt_i32 s3, s9
+; GFX6-NEXT:    s_cselect_b32 s3, s3, s9
+; GFX6-NEXT:    s_sub_i32 s0, s0, s3
+; GFX6-NEXT:    s_cmp_gt_i32 s1, -1
+; GFX6-NEXT:    s_cselect_b32 s3, s1, -1
+; GFX6-NEXT:    s_sub_i32 s3, s3, s6
+; GFX6-NEXT:    s_cmp_lt_i32 s1, -1
+; GFX6-NEXT:    s_cselect_b32 s8, s1, -1
+; GFX6-NEXT:    s_sub_i32 s8, s8, s7
+; GFX6-NEXT:    s_cmp_gt_i32 s3, s4
+; GFX6-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX6-NEXT:    s_cmp_lt_i32 s3, s8
+; GFX6-NEXT:    s_cselect_b32 s3, s3, s8
+; GFX6-NEXT:    s_sub_i32 s1, s1, s3
+; GFX6-NEXT:    s_cmp_gt_i32 s2, -1
+; GFX6-NEXT:    s_cselect_b32 s3, s2, -1
+; GFX6-NEXT:    s_sub_i32 s3, s3, s6
+; GFX6-NEXT:    s_cmp_lt_i32 s2, -1
+; GFX6-NEXT:    s_cselect_b32 s4, s2, -1
+; GFX6-NEXT:    s_sub_i32 s4, s4, s7
+; GFX6-NEXT:    s_cmp_gt_i32 s3, s5
+; GFX6-NEXT:    s_cselect_b32 s3, s3, s5
+; GFX6-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX6-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX6-NEXT:    s_sub_i32 s2, s2, s3
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_ssubsat_v3i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX8-NEXT:    s_brev_b32 s6, -2
+; GFX8-NEXT:    s_cselect_b32 s8, s0, -1
+; GFX8-NEXT:    s_sub_i32 s8, s8, s6
+; GFX8-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX8-NEXT:    s_mov_b32 s7, 0x80000000
+; GFX8-NEXT:    s_cselect_b32 s9, s0, -1
+; GFX8-NEXT:    s_sub_i32 s9, s9, s7
+; GFX8-NEXT:    s_cmp_gt_i32 s8, s3
+; GFX8-NEXT:    s_cselect_b32 s3, s8, s3
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s9
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s9
+; GFX8-NEXT:    s_sub_i32 s0, s0, s3
+; GFX8-NEXT:    s_cmp_gt_i32 s1, -1
+; GFX8-NEXT:    s_cselect_b32 s3, s1, -1
+; GFX8-NEXT:    s_sub_i32 s3, s3, s6
+; GFX8-NEXT:    s_cmp_lt_i32 s1, -1
+; GFX8-NEXT:    s_cselect_b32 s8, s1, -1
+; GFX8-NEXT:    s_sub_i32 s8, s8, s7
+; GFX8-NEXT:    s_cmp_gt_i32 s3, s4
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s8
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s8
+; GFX8-NEXT:    s_sub_i32 s1, s1, s3
+; GFX8-NEXT:    s_cmp_gt_i32 s2, -1
+; GFX8-NEXT:    s_cselect_b32 s3, s2, -1
+; GFX8-NEXT:    s_sub_i32 s3, s3, s6
+; GFX8-NEXT:    s_cmp_lt_i32 s2, -1
+; GFX8-NEXT:    s_cselect_b32 s4, s2, -1
+; GFX8-NEXT:    s_sub_i32 s4, s4, s7
+; GFX8-NEXT:    s_cmp_gt_i32 s3, s5
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX8-NEXT:    s_sub_i32 s2, s2, s3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_ssubsat_v3i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX9-NEXT:    s_brev_b32 s6, -2
+; GFX9-NEXT:    s_cselect_b32 s8, s0, -1
+; GFX9-NEXT:    s_sub_i32 s8, s8, s6
+; GFX9-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX9-NEXT:    s_mov_b32 s7, 0x80000000
+; GFX9-NEXT:    s_cselect_b32 s9, s0, -1
+; GFX9-NEXT:    s_sub_i32 s9, s9, s7
+; GFX9-NEXT:    s_cmp_gt_i32 s8, s3
+; GFX9-NEXT:    s_cselect_b32 s3, s8, s3
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s9
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s9
+; GFX9-NEXT:    s_sub_i32 s0, s0, s3
+; GFX9-NEXT:    s_cmp_gt_i32 s1, -1
+; GFX9-NEXT:    s_cselect_b32 s3, s1, -1
+; GFX9-NEXT:    s_sub_i32 s3, s3, s6
+; GFX9-NEXT:    s_cmp_lt_i32 s1, -1
+; GFX9-NEXT:    s_cselect_b32 s8, s1, -1
+; GFX9-NEXT:    s_sub_i32 s8, s8, s7
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s4
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s8
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s8
+; GFX9-NEXT:    s_sub_i32 s1, s1, s3
+; GFX9-NEXT:    s_cmp_gt_i32 s2, -1
+; GFX9-NEXT:    s_cselect_b32 s3, s2, -1
+; GFX9-NEXT:    s_sub_i32 s3, s3, s6
+; GFX9-NEXT:    s_cmp_lt_i32 s2, -1
+; GFX9-NEXT:    s_cselect_b32 s4, s2, -1
+; GFX9-NEXT:    s_sub_i32 s4, s4, s7
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s5
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s5
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX9-NEXT:    s_sub_i32 s2, s2, s3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_ssubsat_v3i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX10-NEXT:    s_brev_b32 s6, -2
+; GFX10-NEXT:    s_cselect_b32 s7, s0, -1
+; GFX10-NEXT:    s_mov_b32 s8, 0x80000000
+; GFX10-NEXT:    s_sub_i32 s7, s7, s6
+; GFX10-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s9, s0, -1
+; GFX10-NEXT:    s_sub_i32 s9, s9, s8
+; GFX10-NEXT:    s_cmp_gt_i32 s7, s3
+; GFX10-NEXT:    s_cselect_b32 s3, s7, s3
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s9
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s9
+; GFX10-NEXT:    s_sub_i32 s0, s0, s3
+; GFX10-NEXT:    s_cmp_gt_i32 s1, -1
+; GFX10-NEXT:    s_cselect_b32 s3, s1, -1
+; GFX10-NEXT:    s_sub_i32 s3, s3, s6
+; GFX10-NEXT:    s_cmp_lt_i32 s1, -1
+; GFX10-NEXT:    s_cselect_b32 s7, s1, -1
+; GFX10-NEXT:    s_sub_i32 s7, s7, s8
+; GFX10-NEXT:    s_cmp_gt_i32 s3, s4
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s7
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s7
+; GFX10-NEXT:    s_sub_i32 s1, s1, s3
+; GFX10-NEXT:    s_cmp_gt_i32 s2, -1
+; GFX10-NEXT:    s_cselect_b32 s3, s2, -1
+; GFX10-NEXT:    s_sub_i32 s3, s3, s6
+; GFX10-NEXT:    s_cmp_lt_i32 s2, -1
+; GFX10-NEXT:    s_cselect_b32 s4, s2, -1
+; GFX10-NEXT:    s_sub_i32 s4, s4, s8
+; GFX10-NEXT:    s_cmp_gt_i32 s3, s5
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s5
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX10-NEXT:    s_sub_i32 s2, s2, s3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <3 x i32> @llvm.ssub.sat.v3i32(<3 x i32> %lhs, <3 x i32> %rhs)
+  ret <3 x i32> %result
+}
+
+define <4 x i32> @v_ssubsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
+; GFX6-LABEL: v_ssubsat_v4i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    v_max_i32_e32 v8, -1, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v8, vcc, s4, v8
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v9, -1, v0
+; GFX6-NEXT:    v_max_i32_e32 v4, v8, v4
+; GFX6-NEXT:    v_subrev_i32_e32 v9, vcc, s5, v9
+; GFX6-NEXT:    v_min_i32_e32 v4, v4, v9
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; GFX6-NEXT:    v_max_i32_e32 v4, -1, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v4, vcc, s4, v4
+; GFX6-NEXT:    v_min_i32_e32 v8, -1, v1
+; GFX6-NEXT:    v_max_i32_e32 v4, v4, v5
+; GFX6-NEXT:    v_subrev_i32_e32 v8, vcc, s5, v8
+; GFX6-NEXT:    v_min_i32_e32 v4, v4, v8
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v4
+; GFX6-NEXT:    v_max_i32_e32 v4, -1, v2
+; GFX6-NEXT:    v_subrev_i32_e32 v4, vcc, s4, v4
+; GFX6-NEXT:    v_min_i32_e32 v5, -1, v2
+; GFX6-NEXT:    v_subrev_i32_e32 v5, vcc, s5, v5
+; GFX6-NEXT:    v_max_i32_e32 v4, v4, v6
+; GFX6-NEXT:    v_min_i32_e32 v4, v4, v5
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v4
+; GFX6-NEXT:    v_max_i32_e32 v4, -1, v3
+; GFX6-NEXT:    v_subrev_i32_e32 v4, vcc, 0x7fffffff, v4
+; GFX6-NEXT:    v_min_i32_e32 v5, -1, v3
+; GFX6-NEXT:    v_subrev_i32_e32 v5, vcc, 0x80000000, v5
+; GFX6-NEXT:    v_max_i32_e32 v4, v4, v7
+; GFX6-NEXT:    v_min_i32_e32 v4, v4, v5
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, v3, v4
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_v4i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_brev_b32 s4, -2
+; GFX8-NEXT:    v_max_i32_e32 v8, -1, v0
+; GFX8-NEXT:    v_subrev_u32_e32 v8, vcc, s4, v8
+; GFX8-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX8-NEXT:    v_min_i32_e32 v9, -1, v0
+; GFX8-NEXT:    v_max_i32_e32 v4, v8, v4
+; GFX8-NEXT:    v_subrev_u32_e32 v9, vcc, s5, v9
+; GFX8-NEXT:    v_min_i32_e32 v4, v4, v9
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, v0, v4
+; GFX8-NEXT:    v_max_i32_e32 v4, -1, v1
+; GFX8-NEXT:    v_subrev_u32_e32 v4, vcc, s4, v4
+; GFX8-NEXT:    v_min_i32_e32 v8, -1, v1
+; GFX8-NEXT:    v_max_i32_e32 v4, v4, v5
+; GFX8-NEXT:    v_subrev_u32_e32 v8, vcc, s5, v8
+; GFX8-NEXT:    v_min_i32_e32 v4, v4, v8
+; GFX8-NEXT:    v_sub_u32_e32 v1, vcc, v1, v4
+; GFX8-NEXT:    v_max_i32_e32 v4, -1, v2
+; GFX8-NEXT:    v_subrev_u32_e32 v4, vcc, s4, v4
+; GFX8-NEXT:    v_min_i32_e32 v5, -1, v2
+; GFX8-NEXT:    v_subrev_u32_e32 v5, vcc, s5, v5
+; GFX8-NEXT:    v_max_i32_e32 v4, v4, v6
+; GFX8-NEXT:    v_min_i32_e32 v4, v4, v5
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, v2, v4
+; GFX8-NEXT:    v_max_i32_e32 v4, -1, v3
+; GFX8-NEXT:    v_subrev_u32_e32 v4, vcc, 0x7fffffff, v4
+; GFX8-NEXT:    v_min_i32_e32 v5, -1, v3
+; GFX8-NEXT:    v_subrev_u32_e32 v5, vcc, 0x80000000, v5
+; GFX8-NEXT:    v_max_i32_e32 v4, v4, v7
+; GFX8-NEXT:    v_min_i32_e32 v4, v4, v5
+; GFX8-NEXT:    v_sub_u32_e32 v3, vcc, v3, v4
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_v4i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_brev_b32 s4, -2
+; GFX9-NEXT:    v_max_i32_e32 v8, -1, v0
+; GFX9-NEXT:    v_subrev_u32_e32 v8, s4, v8
+; GFX9-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX9-NEXT:    v_min_i32_e32 v9, -1, v0
+; GFX9-NEXT:    v_max_i32_e32 v4, v8, v4
+; GFX9-NEXT:    v_subrev_u32_e32 v9, s5, v9
+; GFX9-NEXT:    v_min_i32_e32 v4, v4, v9
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v4
+; GFX9-NEXT:    v_max_i32_e32 v4, -1, v1
+; GFX9-NEXT:    v_subrev_u32_e32 v4, s4, v4
+; GFX9-NEXT:    v_min_i32_e32 v8, -1, v1
+; GFX9-NEXT:    v_max_i32_e32 v4, v4, v5
+; GFX9-NEXT:    v_subrev_u32_e32 v8, s5, v8
+; GFX9-NEXT:    v_min_i32_e32 v4, v4, v8
+; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v4
+; GFX9-NEXT:    v_max_i32_e32 v4, -1, v2
+; GFX9-NEXT:    v_subrev_u32_e32 v4, s4, v4
+; GFX9-NEXT:    v_min_i32_e32 v5, -1, v2
+; GFX9-NEXT:    v_subrev_u32_e32 v5, s5, v5
+; GFX9-NEXT:    v_max_i32_e32 v4, v4, v6
+; GFX9-NEXT:    v_min_i32_e32 v4, v4, v5
+; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v4
+; GFX9-NEXT:    v_max_i32_e32 v4, -1, v3
+; GFX9-NEXT:    v_subrev_u32_e32 v4, 0x7fffffff, v4
+; GFX9-NEXT:    v_min_i32_e32 v5, -1, v3
+; GFX9-NEXT:    v_subrev_u32_e32 v5, 0x80000000, v5
+; GFX9-NEXT:    v_max_i32_e32 v4, v4, v7
+; GFX9-NEXT:    v_min_i32_e32 v4, v4, v5
+; GFX9-NEXT:    v_sub_u32_e32 v3, v3, v4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_ssubsat_v4i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_max_i32_e32 v8, -1, v0
+; GFX10-NEXT:    s_brev_b32 s4, -2
+; GFX10-NEXT:    v_max_i32_e32 v10, -1, v1
+; GFX10-NEXT:    v_max_i32_e32 v12, -1, v3
+; GFX10-NEXT:    v_min_i32_e32 v9, -1, v0
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v15, s4, v8
+; GFX10-NEXT:    v_max_i32_e32 v8, -1, v2
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v10, s4, v10
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v12, 0x7fffffff, v12
+; GFX10-NEXT:    v_min_i32_e32 v11, -1, v1
+; GFX10-NEXT:    v_min_i32_e32 v13, -1, v2
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v8, s4, v8
+; GFX10-NEXT:    v_min_i32_e32 v14, -1, v3
+; GFX10-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX10-NEXT:    v_max_i32_e32 v4, v15, v4
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v9, s5, v9
+; GFX10-NEXT:    v_max_i32_e32 v5, v10, v5
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v11, s5, v11
+; GFX10-NEXT:    v_max_i32_e32 v15, v8, v6
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v10, s5, v13
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v8, 0x80000000, v14
+; GFX10-NEXT:    v_max_i32_e32 v7, v12, v7
+; GFX10-NEXT:    v_min_i32_e32 v19, v4, v9
+; GFX10-NEXT:    v_min_i32_e32 v11, v5, v11
+; GFX10-NEXT:    v_min_i32_e32 v15, v15, v10
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_min_i32_e32 v6, v7, v8
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v19
+; GFX10-NEXT:    v_sub_nc_u32_e32 v1, v1, v11
+; GFX10-NEXT:    v_sub_nc_u32_e32 v2, v2, v15
+; GFX10-NEXT:    v_sub_nc_u32_e32 v3, v3, v6
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %lhs, <4 x i32> %rhs)
+  ret <4 x i32> %result
+}
+
+define amdgpu_ps <4 x i32> @s_ssubsat_v4i32(<4 x i32> inreg %lhs, <4 x i32> inreg %rhs) {
+; GFX6-LABEL: s_ssubsat_v4i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX6-NEXT:    s_brev_b32 s8, -2
+; GFX6-NEXT:    s_cselect_b32 s10, s0, -1
+; GFX6-NEXT:    s_sub_i32 s10, s10, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX6-NEXT:    s_mov_b32 s9, 0x80000000
+; GFX6-NEXT:    s_cselect_b32 s11, s0, -1
+; GFX6-NEXT:    s_sub_i32 s11, s11, s9
+; GFX6-NEXT:    s_cmp_gt_i32 s10, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s10, s4
+; GFX6-NEXT:    s_cmp_lt_i32 s4, s11
+; GFX6-NEXT:    s_cselect_b32 s4, s4, s11
+; GFX6-NEXT:    s_sub_i32 s0, s0, s4
+; GFX6-NEXT:    s_cmp_gt_i32 s1, -1
+; GFX6-NEXT:    s_cselect_b32 s4, s1, -1
+; GFX6-NEXT:    s_sub_i32 s4, s4, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s1, -1
+; GFX6-NEXT:    s_cselect_b32 s10, s1, -1
+; GFX6-NEXT:    s_sub_i32 s10, s10, s9
+; GFX6-NEXT:    s_cmp_gt_i32 s4, s5
+; GFX6-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX6-NEXT:    s_cmp_lt_i32 s4, s10
+; GFX6-NEXT:    s_cselect_b32 s4, s4, s10
+; GFX6-NEXT:    s_sub_i32 s1, s1, s4
+; GFX6-NEXT:    s_cmp_gt_i32 s2, -1
+; GFX6-NEXT:    s_cselect_b32 s4, s2, -1
+; GFX6-NEXT:    s_sub_i32 s4, s4, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s2, -1
+; GFX6-NEXT:    s_cselect_b32 s5, s2, -1
+; GFX6-NEXT:    s_sub_i32 s5, s5, s9
+; GFX6-NEXT:    s_cmp_gt_i32 s4, s6
+; GFX6-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX6-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX6-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX6-NEXT:    s_sub_i32 s2, s2, s4
+; GFX6-NEXT:    s_cmp_gt_i32 s3, -1
+; GFX6-NEXT:    s_cselect_b32 s4, s3, -1
+; GFX6-NEXT:    s_sub_i32 s4, s4, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s3, -1
+; GFX6-NEXT:    s_cselect_b32 s5, s3, -1
+; GFX6-NEXT:    s_sub_i32 s5, s5, s9
+; GFX6-NEXT:    s_cmp_gt_i32 s4, s7
+; GFX6-NEXT:    s_cselect_b32 s4, s4, s7
+; GFX6-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX6-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX6-NEXT:    s_sub_i32 s3, s3, s4
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_ssubsat_v4i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX8-NEXT:    s_brev_b32 s8, -2
+; GFX8-NEXT:    s_cselect_b32 s10, s0, -1
+; GFX8-NEXT:    s_sub_i32 s10, s10, s8
+; GFX8-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX8-NEXT:    s_mov_b32 s9, 0x80000000
+; GFX8-NEXT:    s_cselect_b32 s11, s0, -1
+; GFX8-NEXT:    s_sub_i32 s11, s11, s9
+; GFX8-NEXT:    s_cmp_gt_i32 s10, s4
+; GFX8-NEXT:    s_cselect_b32 s4, s10, s4
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s11
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s11
+; GFX8-NEXT:    s_sub_i32 s0, s0, s4
+; GFX8-NEXT:    s_cmp_gt_i32 s1, -1
+; GFX8-NEXT:    s_cselect_b32 s4, s1, -1
+; GFX8-NEXT:    s_sub_i32 s4, s4, s8
+; GFX8-NEXT:    s_cmp_lt_i32 s1, -1
+; GFX8-NEXT:    s_cselect_b32 s10, s1, -1
+; GFX8-NEXT:    s_sub_i32 s10, s10, s9
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s5
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s10
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s10
+; GFX8-NEXT:    s_sub_i32 s1, s1, s4
+; GFX8-NEXT:    s_cmp_gt_i32 s2, -1
+; GFX8-NEXT:    s_cselect_b32 s4, s2, -1
+; GFX8-NEXT:    s_sub_i32 s4, s4, s8
+; GFX8-NEXT:    s_cmp_lt_i32 s2, -1
+; GFX8-NEXT:    s_cselect_b32 s5, s2, -1
+; GFX8-NEXT:    s_sub_i32 s5, s5, s9
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s6
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX8-NEXT:    s_sub_i32 s2, s2, s4
+; GFX8-NEXT:    s_cmp_gt_i32 s3, -1
+; GFX8-NEXT:    s_cselect_b32 s4, s3, -1
+; GFX8-NEXT:    s_sub_i32 s4, s4, s8
+; GFX8-NEXT:    s_cmp_lt_i32 s3, -1
+; GFX8-NEXT:    s_cselect_b32 s5, s3, -1
+; GFX8-NEXT:    s_sub_i32 s5, s5, s9
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s7
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s7
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX8-NEXT:    s_sub_i32 s3, s3, s4
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_ssubsat_v4i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX9-NEXT:    s_brev_b32 s8, -2
+; GFX9-NEXT:    s_cselect_b32 s10, s0, -1
+; GFX9-NEXT:    s_sub_i32 s10, s10, s8
+; GFX9-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX9-NEXT:    s_mov_b32 s9, 0x80000000
+; GFX9-NEXT:    s_cselect_b32 s11, s0, -1
+; GFX9-NEXT:    s_sub_i32 s11, s11, s9
+; GFX9-NEXT:    s_cmp_gt_i32 s10, s4
+; GFX9-NEXT:    s_cselect_b32 s4, s10, s4
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s11
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s11
+; GFX9-NEXT:    s_sub_i32 s0, s0, s4
+; GFX9-NEXT:    s_cmp_gt_i32 s1, -1
+; GFX9-NEXT:    s_cselect_b32 s4, s1, -1
+; GFX9-NEXT:    s_sub_i32 s4, s4, s8
+; GFX9-NEXT:    s_cmp_lt_i32 s1, -1
+; GFX9-NEXT:    s_cselect_b32 s10, s1, -1
+; GFX9-NEXT:    s_sub_i32 s10, s10, s9
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s5
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s10
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s10
+; GFX9-NEXT:    s_sub_i32 s1, s1, s4
+; GFX9-NEXT:    s_cmp_gt_i32 s2, -1
+; GFX9-NEXT:    s_cselect_b32 s4, s2, -1
+; GFX9-NEXT:    s_sub_i32 s4, s4, s8
+; GFX9-NEXT:    s_cmp_lt_i32 s2, -1
+; GFX9-NEXT:    s_cselect_b32 s5, s2, -1
+; GFX9-NEXT:    s_sub_i32 s5, s5, s9
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s6
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX9-NEXT:    s_sub_i32 s2, s2, s4
+; GFX9-NEXT:    s_cmp_gt_i32 s3, -1
+; GFX9-NEXT:    s_cselect_b32 s4, s3, -1
+; GFX9-NEXT:    s_sub_i32 s4, s4, s8
+; GFX9-NEXT:    s_cmp_lt_i32 s3, -1
+; GFX9-NEXT:    s_cselect_b32 s5, s3, -1
+; GFX9-NEXT:    s_sub_i32 s5, s5, s9
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s7
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s7
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX9-NEXT:    s_sub_i32 s3, s3, s4
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_ssubsat_v4i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX10-NEXT:    s_brev_b32 s8, -2
+; GFX10-NEXT:    s_cselect_b32 s9, s0, -1
+; GFX10-NEXT:    s_mov_b32 s10, 0x80000000
+; GFX10-NEXT:    s_sub_i32 s9, s9, s8
+; GFX10-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s11, s0, -1
+; GFX10-NEXT:    s_sub_i32 s11, s11, s10
+; GFX10-NEXT:    s_cmp_gt_i32 s9, s4
+; GFX10-NEXT:    s_cselect_b32 s4, s9, s4
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s11
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s11
+; GFX10-NEXT:    s_sub_i32 s0, s0, s4
+; GFX10-NEXT:    s_cmp_gt_i32 s1, -1
+; GFX10-NEXT:    s_cselect_b32 s4, s1, -1
+; GFX10-NEXT:    s_sub_i32 s4, s4, s8
+; GFX10-NEXT:    s_cmp_lt_i32 s1, -1
+; GFX10-NEXT:    s_cselect_b32 s9, s1, -1
+; GFX10-NEXT:    s_sub_i32 s9, s9, s10
+; GFX10-NEXT:    s_cmp_gt_i32 s4, s5
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s9
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s9
+; GFX10-NEXT:    s_sub_i32 s1, s1, s4
+; GFX10-NEXT:    s_cmp_gt_i32 s2, -1
+; GFX10-NEXT:    s_cselect_b32 s4, s2, -1
+; GFX10-NEXT:    s_sub_i32 s4, s4, s8
+; GFX10-NEXT:    s_cmp_lt_i32 s2, -1
+; GFX10-NEXT:    s_cselect_b32 s5, s2, -1
+; GFX10-NEXT:    s_sub_i32 s5, s5, s10
+; GFX10-NEXT:    s_cmp_gt_i32 s4, s6
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX10-NEXT:    s_sub_i32 s2, s2, s4
+; GFX10-NEXT:    s_cmp_gt_i32 s3, -1
+; GFX10-NEXT:    s_cselect_b32 s4, s3, -1
+; GFX10-NEXT:    s_sub_i32 s4, s4, s8
+; GFX10-NEXT:    s_cmp_lt_i32 s3, -1
+; GFX10-NEXT:    s_cselect_b32 s5, s3, -1
+; GFX10-NEXT:    s_sub_i32 s5, s5, s10
+; GFX10-NEXT:    s_cmp_gt_i32 s4, s7
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s7
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX10-NEXT:    s_sub_i32 s3, s3, s4
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %lhs, <4 x i32> %rhs)
+  ret <4 x i32> %result
+}
+
+define <5 x i32> @v_ssubsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
+; GFX6-LABEL: v_ssubsat_v5i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    v_max_i32_e32 v10, -1, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v10, vcc, s4, v10
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v12, -1, v0
+; GFX6-NEXT:    v_max_i32_e32 v5, v10, v5
+; GFX6-NEXT:    v_subrev_i32_e32 v12, vcc, s5, v12
+; GFX6-NEXT:    v_min_i32_e32 v5, v5, v12
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v5
+; GFX6-NEXT:    v_max_i32_e32 v5, -1, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v5, vcc, s4, v5
+; GFX6-NEXT:    v_min_i32_e32 v10, -1, v1
+; GFX6-NEXT:    v_max_i32_e32 v5, v5, v6
+; GFX6-NEXT:    v_subrev_i32_e32 v10, vcc, s5, v10
+; GFX6-NEXT:    v_min_i32_e32 v5, v5, v10
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v5
+; GFX6-NEXT:    v_max_i32_e32 v5, -1, v2
+; GFX6-NEXT:    v_subrev_i32_e32 v5, vcc, s4, v5
+; GFX6-NEXT:    v_min_i32_e32 v6, -1, v2
+; GFX6-NEXT:    v_subrev_i32_e32 v6, vcc, s5, v6
+; GFX6-NEXT:    v_max_i32_e32 v5, v5, v7
+; GFX6-NEXT:    v_min_i32_e32 v5, v5, v6
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v5
+; GFX6-NEXT:    v_bfrev_b32_e32 v11, -2
+; GFX6-NEXT:    v_max_i32_e32 v5, -1, v3
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, v5, v11
+; GFX6-NEXT:    v_mov_b32_e32 v13, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v6, -1, v3
+; GFX6-NEXT:    v_sub_i32_e32 v6, vcc, v6, v13
+; GFX6-NEXT:    v_max_i32_e32 v5, v5, v8
+; GFX6-NEXT:    v_min_i32_e32 v5, v5, v6
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, v3, v5
+; GFX6-NEXT:    v_max_i32_e32 v5, -1, v4
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, v5, v11
+; GFX6-NEXT:    v_min_i32_e32 v6, -1, v4
+; GFX6-NEXT:    v_sub_i32_e32 v6, vcc, v6, v13
+; GFX6-NEXT:    v_max_i32_e32 v5, v5, v9
+; GFX6-NEXT:    v_min_i32_e32 v5, v5, v6
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, v4, v5
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_v5i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_brev_b32 s4, -2
+; GFX8-NEXT:    v_max_i32_e32 v10, -1, v0
+; GFX8-NEXT:    v_subrev_u32_e32 v10, vcc, s4, v10
+; GFX8-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX8-NEXT:    v_min_i32_e32 v12, -1, v0
+; GFX8-NEXT:    v_max_i32_e32 v5, v10, v5
+; GFX8-NEXT:    v_subrev_u32_e32 v12, vcc, s5, v12
+; GFX8-NEXT:    v_min_i32_e32 v5, v5, v12
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, v0, v5
+; GFX8-NEXT:    v_max_i32_e32 v5, -1, v1
+; GFX8-NEXT:    v_subrev_u32_e32 v5, vcc, s4, v5
+; GFX8-NEXT:    v_min_i32_e32 v10, -1, v1
+; GFX8-NEXT:    v_max_i32_e32 v5, v5, v6
+; GFX8-NEXT:    v_subrev_u32_e32 v10, vcc, s5, v10
+; GFX8-NEXT:    v_min_i32_e32 v5, v5, v10
+; GFX8-NEXT:    v_sub_u32_e32 v1, vcc, v1, v5
+; GFX8-NEXT:    v_max_i32_e32 v5, -1, v2
+; GFX8-NEXT:    v_subrev_u32_e32 v5, vcc, s4, v5
+; GFX8-NEXT:    v_min_i32_e32 v6, -1, v2
+; GFX8-NEXT:    v_subrev_u32_e32 v6, vcc, s5, v6
+; GFX8-NEXT:    v_max_i32_e32 v5, v5, v7
+; GFX8-NEXT:    v_min_i32_e32 v5, v5, v6
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, v2, v5
+; GFX8-NEXT:    v_bfrev_b32_e32 v11, -2
+; GFX8-NEXT:    v_max_i32_e32 v5, -1, v3
+; GFX8-NEXT:    v_sub_u32_e32 v5, vcc, v5, v11
+; GFX8-NEXT:    v_mov_b32_e32 v13, 0x80000000
+; GFX8-NEXT:    v_min_i32_e32 v6, -1, v3
+; GFX8-NEXT:    v_sub_u32_e32 v6, vcc, v6, v13
+; GFX8-NEXT:    v_max_i32_e32 v5, v5, v8
+; GFX8-NEXT:    v_min_i32_e32 v5, v5, v6
+; GFX8-NEXT:    v_sub_u32_e32 v3, vcc, v3, v5
+; GFX8-NEXT:    v_max_i32_e32 v5, -1, v4
+; GFX8-NEXT:    v_sub_u32_e32 v5, vcc, v5, v11
+; GFX8-NEXT:    v_min_i32_e32 v6, -1, v4
+; GFX8-NEXT:    v_sub_u32_e32 v6, vcc, v6, v13
+; GFX8-NEXT:    v_max_i32_e32 v5, v5, v9
+; GFX8-NEXT:    v_min_i32_e32 v5, v5, v6
+; GFX8-NEXT:    v_sub_u32_e32 v4, vcc, v4, v5
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_v5i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_brev_b32 s4, -2
+; GFX9-NEXT:    v_max_i32_e32 v10, -1, v0
+; GFX9-NEXT:    v_subrev_u32_e32 v10, s4, v10
+; GFX9-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX9-NEXT:    v_min_i32_e32 v12, -1, v0
+; GFX9-NEXT:    v_max_i32_e32 v5, v10, v5
+; GFX9-NEXT:    v_subrev_u32_e32 v12, s5, v12
+; GFX9-NEXT:    v_min_i32_e32 v5, v5, v12
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v5
+; GFX9-NEXT:    v_max_i32_e32 v5, -1, v1
+; GFX9-NEXT:    v_subrev_u32_e32 v5, s4, v5
+; GFX9-NEXT:    v_min_i32_e32 v10, -1, v1
+; GFX9-NEXT:    v_max_i32_e32 v5, v5, v6
+; GFX9-NEXT:    v_subrev_u32_e32 v10, s5, v10
+; GFX9-NEXT:    v_min_i32_e32 v5, v5, v10
+; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v5
+; GFX9-NEXT:    v_max_i32_e32 v5, -1, v2
+; GFX9-NEXT:    v_subrev_u32_e32 v5, s4, v5
+; GFX9-NEXT:    v_min_i32_e32 v6, -1, v2
+; GFX9-NEXT:    v_subrev_u32_e32 v6, s5, v6
+; GFX9-NEXT:    v_max_i32_e32 v5, v5, v7
+; GFX9-NEXT:    v_min_i32_e32 v5, v5, v6
+; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v5
+; GFX9-NEXT:    v_bfrev_b32_e32 v11, -2
+; GFX9-NEXT:    v_max_i32_e32 v5, -1, v3
+; GFX9-NEXT:    v_sub_u32_e32 v5, v5, v11
+; GFX9-NEXT:    v_mov_b32_e32 v13, 0x80000000
+; GFX9-NEXT:    v_min_i32_e32 v6, -1, v3
+; GFX9-NEXT:    v_sub_u32_e32 v6, v6, v13
+; GFX9-NEXT:    v_max_i32_e32 v5, v5, v8
+; GFX9-NEXT:    v_min_i32_e32 v5, v5, v6
+; GFX9-NEXT:    v_sub_u32_e32 v3, v3, v5
+; GFX9-NEXT:    v_max_i32_e32 v5, -1, v4
+; GFX9-NEXT:    v_sub_u32_e32 v5, v5, v11
+; GFX9-NEXT:    v_min_i32_e32 v6, -1, v4
+; GFX9-NEXT:    v_sub_u32_e32 v6, v6, v13
+; GFX9-NEXT:    v_max_i32_e32 v5, v5, v9
+; GFX9-NEXT:    v_min_i32_e32 v5, v5, v6
+; GFX9-NEXT:    v_sub_u32_e32 v4, v4, v5
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_ssubsat_v5i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_max_i32_e32 v10, -1, v0
+; GFX10-NEXT:    v_max_i32_e32 v13, -1, v1
+; GFX10-NEXT:    s_brev_b32 s4, -2
+; GFX10-NEXT:    v_bfrev_b32_e32 v11, -2
+; GFX10-NEXT:    v_max_i32_e32 v17, -1, v4
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v10, s4, v10
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v13, s4, v13
+; GFX10-NEXT:    v_min_i32_e32 v12, -1, v0
+; GFX10-NEXT:    v_mov_b32_e32 v14, 0x80000000
+; GFX10-NEXT:    v_min_i32_e32 v15, -1, v1
+; GFX10-NEXT:    v_max_i32_e32 v5, v10, v5
+; GFX10-NEXT:    v_max_i32_e32 v10, -1, v2
+; GFX10-NEXT:    v_max_i32_e32 v6, v13, v6
+; GFX10-NEXT:    v_max_i32_e32 v13, -1, v3
+; GFX10-NEXT:    v_min_i32_e32 v16, -1, v2
+; GFX10-NEXT:    v_min_i32_e32 v23, -1, v3
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v10, s4, v10
+; GFX10-NEXT:    v_min_i32_e32 v19, -1, v4
+; GFX10-NEXT:    v_sub_nc_u32_e32 v13, v13, v11
+; GFX10-NEXT:    v_sub_nc_u32_e32 v11, v17, v11
+; GFX10-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX10-NEXT:    v_max_i32_e32 v7, v10, v7
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v12, s5, v12
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v15, s5, v15
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v16, s5, v16
+; GFX10-NEXT:    v_max_i32_e32 v8, v13, v8
+; GFX10-NEXT:    v_sub_nc_u32_e32 v10, v23, v14
+; GFX10-NEXT:    v_sub_nc_u32_e32 v13, v19, v14
+; GFX10-NEXT:    v_max_i32_e32 v11, v11, v9
+; GFX10-NEXT:    v_min_i32_e32 v5, v5, v12
+; GFX10-NEXT:    v_min_i32_e32 v6, v6, v15
+; GFX10-NEXT:    v_min_i32_e32 v7, v7, v16
+; GFX10-NEXT:    v_min_i32_e32 v8, v8, v10
+; GFX10-NEXT:    v_min_i32_e32 v9, v11, v13
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v5
+; GFX10-NEXT:    v_sub_nc_u32_e32 v1, v1, v6
+; GFX10-NEXT:    v_sub_nc_u32_e32 v2, v2, v7
+; GFX10-NEXT:    v_sub_nc_u32_e32 v3, v3, v8
+; GFX10-NEXT:    v_sub_nc_u32_e32 v4, v4, v9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <5 x i32> @llvm.ssub.sat.v5i32(<5 x i32> %lhs, <5 x i32> %rhs)
+  ret <5 x i32> %result
+}
+
+define amdgpu_ps <5 x i32> @s_ssubsat_v5i32(<5 x i32> inreg %lhs, <5 x i32> inreg %rhs) {
+; GFX6-LABEL: s_ssubsat_v5i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX6-NEXT:    s_brev_b32 s10, -2
+; GFX6-NEXT:    s_cselect_b32 s12, s0, -1
+; GFX6-NEXT:    s_sub_i32 s12, s12, s10
+; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX6-NEXT:    s_mov_b32 s11, 0x80000000
+; GFX6-NEXT:    s_cselect_b32 s13, s0, -1
+; GFX6-NEXT:    s_sub_i32 s13, s13, s11
+; GFX6-NEXT:    s_cmp_gt_i32 s12, s5
+; GFX6-NEXT:    s_cselect_b32 s5, s12, s5
+; GFX6-NEXT:    s_cmp_lt_i32 s5, s13
+; GFX6-NEXT:    s_cselect_b32 s5, s5, s13
+; GFX6-NEXT:    s_sub_i32 s0, s0, s5
+; GFX6-NEXT:    s_cmp_gt_i32 s1, -1
+; GFX6-NEXT:    s_cselect_b32 s5, s1, -1
+; GFX6-NEXT:    s_sub_i32 s5, s5, s10
+; GFX6-NEXT:    s_cmp_lt_i32 s1, -1
+; GFX6-NEXT:    s_cselect_b32 s12, s1, -1
+; GFX6-NEXT:    s_sub_i32 s12, s12, s11
+; GFX6-NEXT:    s_cmp_gt_i32 s5, s6
+; GFX6-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX6-NEXT:    s_cmp_lt_i32 s5, s12
+; GFX6-NEXT:    s_cselect_b32 s5, s5, s12
+; GFX6-NEXT:    s_sub_i32 s1, s1, s5
+; GFX6-NEXT:    s_cmp_gt_i32 s2, -1
+; GFX6-NEXT:    s_cselect_b32 s5, s2, -1
+; GFX6-NEXT:    s_sub_i32 s5, s5, s10
+; GFX6-NEXT:    s_cmp_lt_i32 s2, -1
+; GFX6-NEXT:    s_cselect_b32 s6, s2, -1
+; GFX6-NEXT:    s_sub_i32 s6, s6, s11
+; GFX6-NEXT:    s_cmp_gt_i32 s5, s7
+; GFX6-NEXT:    s_cselect_b32 s5, s5, s7
+; GFX6-NEXT:    s_cmp_lt_i32 s5, s6
+; GFX6-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX6-NEXT:    s_sub_i32 s2, s2, s5
+; GFX6-NEXT:    s_cmp_gt_i32 s3, -1
+; GFX6-NEXT:    s_cselect_b32 s5, s3, -1
+; GFX6-NEXT:    s_sub_i32 s5, s5, s10
+; GFX6-NEXT:    s_cmp_lt_i32 s3, -1
+; GFX6-NEXT:    s_cselect_b32 s6, s3, -1
+; GFX6-NEXT:    s_sub_i32 s6, s6, s11
+; GFX6-NEXT:    s_cmp_gt_i32 s5, s8
+; GFX6-NEXT:    s_cselect_b32 s5, s5, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s5, s6
+; GFX6-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX6-NEXT:    s_sub_i32 s3, s3, s5
+; GFX6-NEXT:    s_cmp_gt_i32 s4, -1
+; GFX6-NEXT:    s_cselect_b32 s5, s4, -1
+; GFX6-NEXT:    s_sub_i32 s5, s5, s10
+; GFX6-NEXT:    s_cmp_lt_i32 s4, -1
+; GFX6-NEXT:    s_cselect_b32 s6, s4, -1
+; GFX6-NEXT:    s_sub_i32 s6, s6, s11
+; GFX6-NEXT:    s_cmp_gt_i32 s5, s9
+; GFX6-NEXT:    s_cselect_b32 s5, s5, s9
+; GFX6-NEXT:    s_cmp_lt_i32 s5, s6
+; GFX6-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX6-NEXT:    s_sub_i32 s4, s4, s5
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_ssubsat_v5i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX8-NEXT:    s_brev_b32 s10, -2
+; GFX8-NEXT:    s_cselect_b32 s12, s0, -1
+; GFX8-NEXT:    s_sub_i32 s12, s12, s10
+; GFX8-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX8-NEXT:    s_mov_b32 s11, 0x80000000
+; GFX8-NEXT:    s_cselect_b32 s13, s0, -1
+; GFX8-NEXT:    s_sub_i32 s13, s13, s11
+; GFX8-NEXT:    s_cmp_gt_i32 s12, s5
+; GFX8-NEXT:    s_cselect_b32 s5, s12, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s5, s13
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s13
+; GFX8-NEXT:    s_sub_i32 s0, s0, s5
+; GFX8-NEXT:    s_cmp_gt_i32 s1, -1
+; GFX8-NEXT:    s_cselect_b32 s5, s1, -1
+; GFX8-NEXT:    s_sub_i32 s5, s5, s10
+; GFX8-NEXT:    s_cmp_lt_i32 s1, -1
+; GFX8-NEXT:    s_cselect_b32 s12, s1, -1
+; GFX8-NEXT:    s_sub_i32 s12, s12, s11
+; GFX8-NEXT:    s_cmp_gt_i32 s5, s6
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX8-NEXT:    s_cmp_lt_i32 s5, s12
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s12
+; GFX8-NEXT:    s_sub_i32 s1, s1, s5
+; GFX8-NEXT:    s_cmp_gt_i32 s2, -1
+; GFX8-NEXT:    s_cselect_b32 s5, s2, -1
+; GFX8-NEXT:    s_sub_i32 s5, s5, s10
+; GFX8-NEXT:    s_cmp_lt_i32 s2, -1
+; GFX8-NEXT:    s_cselect_b32 s6, s2, -1
+; GFX8-NEXT:    s_sub_i32 s6, s6, s11
+; GFX8-NEXT:    s_cmp_gt_i32 s5, s7
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s7
+; GFX8-NEXT:    s_cmp_lt_i32 s5, s6
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX8-NEXT:    s_sub_i32 s2, s2, s5
+; GFX8-NEXT:    s_cmp_gt_i32 s3, -1
+; GFX8-NEXT:    s_cselect_b32 s5, s3, -1
+; GFX8-NEXT:    s_sub_i32 s5, s5, s10
+; GFX8-NEXT:    s_cmp_lt_i32 s3, -1
+; GFX8-NEXT:    s_cselect_b32 s6, s3, -1
+; GFX8-NEXT:    s_sub_i32 s6, s6, s11
+; GFX8-NEXT:    s_cmp_gt_i32 s5, s8
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s8
+; GFX8-NEXT:    s_cmp_lt_i32 s5, s6
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX8-NEXT:    s_sub_i32 s3, s3, s5
+; GFX8-NEXT:    s_cmp_gt_i32 s4, -1
+; GFX8-NEXT:    s_cselect_b32 s5, s4, -1
+; GFX8-NEXT:    s_sub_i32 s5, s5, s10
+; GFX8-NEXT:    s_cmp_lt_i32 s4, -1
+; GFX8-NEXT:    s_cselect_b32 s6, s4, -1
+; GFX8-NEXT:    s_sub_i32 s6, s6, s11
+; GFX8-NEXT:    s_cmp_gt_i32 s5, s9
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s9
+; GFX8-NEXT:    s_cmp_lt_i32 s5, s6
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX8-NEXT:    s_sub_i32 s4, s4, s5
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_ssubsat_v5i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX9-NEXT:    s_brev_b32 s10, -2
+; GFX9-NEXT:    s_cselect_b32 s12, s0, -1
+; GFX9-NEXT:    s_sub_i32 s12, s12, s10
+; GFX9-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX9-NEXT:    s_mov_b32 s11, 0x80000000
+; GFX9-NEXT:    s_cselect_b32 s13, s0, -1
+; GFX9-NEXT:    s_sub_i32 s13, s13, s11
+; GFX9-NEXT:    s_cmp_gt_i32 s12, s5
+; GFX9-NEXT:    s_cselect_b32 s5, s12, s5
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s13
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s13
+; GFX9-NEXT:    s_sub_i32 s0, s0, s5
+; GFX9-NEXT:    s_cmp_gt_i32 s1, -1
+; GFX9-NEXT:    s_cselect_b32 s5, s1, -1
+; GFX9-NEXT:    s_sub_i32 s5, s5, s10
+; GFX9-NEXT:    s_cmp_lt_i32 s1, -1
+; GFX9-NEXT:    s_cselect_b32 s12, s1, -1
+; GFX9-NEXT:    s_sub_i32 s12, s12, s11
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s6
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s12
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s12
+; GFX9-NEXT:    s_sub_i32 s1, s1, s5
+; GFX9-NEXT:    s_cmp_gt_i32 s2, -1
+; GFX9-NEXT:    s_cselect_b32 s5, s2, -1
+; GFX9-NEXT:    s_sub_i32 s5, s5, s10
+; GFX9-NEXT:    s_cmp_lt_i32 s2, -1
+; GFX9-NEXT:    s_cselect_b32 s6, s2, -1
+; GFX9-NEXT:    s_sub_i32 s6, s6, s11
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s7
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s7
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s6
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX9-NEXT:    s_sub_i32 s2, s2, s5
+; GFX9-NEXT:    s_cmp_gt_i32 s3, -1
+; GFX9-NEXT:    s_cselect_b32 s5, s3, -1
+; GFX9-NEXT:    s_sub_i32 s5, s5, s10
+; GFX9-NEXT:    s_cmp_lt_i32 s3, -1
+; GFX9-NEXT:    s_cselect_b32 s6, s3, -1
+; GFX9-NEXT:    s_sub_i32 s6, s6, s11
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s8
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s8
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s6
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX9-NEXT:    s_sub_i32 s3, s3, s5
+; GFX9-NEXT:    s_cmp_gt_i32 s4, -1
+; GFX9-NEXT:    s_cselect_b32 s5, s4, -1
+; GFX9-NEXT:    s_sub_i32 s5, s5, s10
+; GFX9-NEXT:    s_cmp_lt_i32 s4, -1
+; GFX9-NEXT:    s_cselect_b32 s6, s4, -1
+; GFX9-NEXT:    s_sub_i32 s6, s6, s11
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s9
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s9
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s6
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX9-NEXT:    s_sub_i32 s4, s4, s5
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_ssubsat_v5i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX10-NEXT:    s_brev_b32 s10, -2
+; GFX10-NEXT:    s_cselect_b32 s11, s0, -1
+; GFX10-NEXT:    s_mov_b32 s12, 0x80000000
+; GFX10-NEXT:    s_sub_i32 s11, s11, s10
+; GFX10-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s13, s0, -1
+; GFX10-NEXT:    s_sub_i32 s13, s13, s12
+; GFX10-NEXT:    s_cmp_gt_i32 s11, s5
+; GFX10-NEXT:    s_cselect_b32 s5, s11, s5
+; GFX10-NEXT:    s_cmp_lt_i32 s5, s13
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s13
+; GFX10-NEXT:    s_sub_i32 s0, s0, s5
+; GFX10-NEXT:    s_cmp_gt_i32 s1, -1
+; GFX10-NEXT:    s_cselect_b32 s5, s1, -1
+; GFX10-NEXT:    s_sub_i32 s5, s5, s10
+; GFX10-NEXT:    s_cmp_lt_i32 s1, -1
+; GFX10-NEXT:    s_cselect_b32 s11, s1, -1
+; GFX10-NEXT:    s_sub_i32 s11, s11, s12
+; GFX10-NEXT:    s_cmp_gt_i32 s5, s6
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX10-NEXT:    s_cmp_lt_i32 s5, s11
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s11
+; GFX10-NEXT:    s_sub_i32 s1, s1, s5
+; GFX10-NEXT:    s_cmp_gt_i32 s2, -1
+; GFX10-NEXT:    s_cselect_b32 s5, s2, -1
+; GFX10-NEXT:    s_sub_i32 s5, s5, s10
+; GFX10-NEXT:    s_cmp_lt_i32 s2, -1
+; GFX10-NEXT:    s_cselect_b32 s6, s2, -1
+; GFX10-NEXT:    s_sub_i32 s6, s6, s12
+; GFX10-NEXT:    s_cmp_gt_i32 s5, s7
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s7
+; GFX10-NEXT:    s_cmp_lt_i32 s5, s6
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX10-NEXT:    s_sub_i32 s2, s2, s5
+; GFX10-NEXT:    s_cmp_gt_i32 s3, -1
+; GFX10-NEXT:    s_cselect_b32 s5, s3, -1
+; GFX10-NEXT:    s_sub_i32 s5, s5, s10
+; GFX10-NEXT:    s_cmp_lt_i32 s3, -1
+; GFX10-NEXT:    s_cselect_b32 s6, s3, -1
+; GFX10-NEXT:    s_sub_i32 s6, s6, s12
+; GFX10-NEXT:    s_cmp_gt_i32 s5, s8
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s8
+; GFX10-NEXT:    s_cmp_lt_i32 s5, s6
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX10-NEXT:    s_sub_i32 s3, s3, s5
+; GFX10-NEXT:    s_cmp_gt_i32 s4, -1
+; GFX10-NEXT:    s_cselect_b32 s5, s4, -1
+; GFX10-NEXT:    s_sub_i32 s5, s5, s10
+; GFX10-NEXT:    s_cmp_lt_i32 s4, -1
+; GFX10-NEXT:    s_cselect_b32 s6, s4, -1
+; GFX10-NEXT:    s_sub_i32 s6, s6, s12
+; GFX10-NEXT:    s_cmp_gt_i32 s5, s9
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s9
+; GFX10-NEXT:    s_cmp_lt_i32 s5, s6
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX10-NEXT:    s_sub_i32 s4, s4, s5
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <5 x i32> @llvm.ssub.sat.v5i32(<5 x i32> %lhs, <5 x i32> %rhs)
+  ret <5 x i32> %result
+}
+
+define <16 x i32> @v_ssubsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
+; GFX6-LABEL: v_ssubsat_v16i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    v_max_i32_e32 v32, -1, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v32, vcc, s4, v32
+; GFX6-NEXT:    v_max_i32_e32 v16, v32, v16
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v32, -1, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v32, vcc, s5, v32
+; GFX6-NEXT:    v_min_i32_e32 v16, v16, v32
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v16
+; GFX6-NEXT:    v_max_i32_e32 v16, -1, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v16, vcc, s4, v16
+; GFX6-NEXT:    v_max_i32_e32 v16, v16, v17
+; GFX6-NEXT:    v_min_i32_e32 v17, -1, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v17, vcc, s5, v17
+; GFX6-NEXT:    v_min_i32_e32 v16, v16, v17
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v16
+; GFX6-NEXT:    v_max_i32_e32 v16, -1, v2
+; GFX6-NEXT:    v_subrev_i32_e32 v16, vcc, s4, v16
+; GFX6-NEXT:    v_min_i32_e32 v17, -1, v2
+; GFX6-NEXT:    v_max_i32_e32 v16, v16, v18
+; GFX6-NEXT:    v_subrev_i32_e32 v17, vcc, s5, v17
+; GFX6-NEXT:    v_min_i32_e32 v16, v16, v17
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v16
+; GFX6-NEXT:    v_bfrev_b32_e32 v16, -2
+; GFX6-NEXT:    v_max_i32_e32 v17, -1, v3
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v17, v16
+; GFX6-NEXT:    v_max_i32_e32 v17, v17, v19
+; GFX6-NEXT:    v_mov_b32_e32 v18, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v19, -1, v3
+; GFX6-NEXT:    v_sub_i32_e32 v19, vcc, v19, v18
+; GFX6-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, v3, v17
+; GFX6-NEXT:    v_max_i32_e32 v17, -1, v4
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v17, v16
+; GFX6-NEXT:    v_min_i32_e32 v19, -1, v4
+; GFX6-NEXT:    v_max_i32_e32 v17, v17, v20
+; GFX6-NEXT:    v_sub_i32_e32 v19, vcc, v19, v18
+; GFX6-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, v4, v17
+; GFX6-NEXT:    v_max_i32_e32 v17, -1, v5
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v17, v16
+; GFX6-NEXT:    v_min_i32_e32 v19, -1, v5
+; GFX6-NEXT:    v_max_i32_e32 v17, v17, v21
+; GFX6-NEXT:    v_sub_i32_e32 v19, vcc, v19, v18
+; GFX6-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, v5, v17
+; GFX6-NEXT:    v_max_i32_e32 v17, -1, v6
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v17, v16
+; GFX6-NEXT:    v_min_i32_e32 v19, -1, v6
+; GFX6-NEXT:    v_max_i32_e32 v17, v17, v22
+; GFX6-NEXT:    v_sub_i32_e32 v19, vcc, v19, v18
+; GFX6-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX6-NEXT:    v_sub_i32_e32 v6, vcc, v6, v17
+; GFX6-NEXT:    v_max_i32_e32 v17, -1, v7
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v17, v16
+; GFX6-NEXT:    v_min_i32_e32 v19, -1, v7
+; GFX6-NEXT:    v_max_i32_e32 v17, v17, v23
+; GFX6-NEXT:    v_sub_i32_e32 v19, vcc, v19, v18
+; GFX6-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX6-NEXT:    v_sub_i32_e32 v7, vcc, v7, v17
+; GFX6-NEXT:    v_max_i32_e32 v17, -1, v8
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v17, v16
+; GFX6-NEXT:    v_min_i32_e32 v19, -1, v8
+; GFX6-NEXT:    v_max_i32_e32 v17, v17, v24
+; GFX6-NEXT:    v_sub_i32_e32 v19, vcc, v19, v18
+; GFX6-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX6-NEXT:    v_sub_i32_e32 v8, vcc, v8, v17
+; GFX6-NEXT:    v_max_i32_e32 v17, -1, v9
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v17, v16
+; GFX6-NEXT:    v_min_i32_e32 v19, -1, v9
+; GFX6-NEXT:    v_max_i32_e32 v17, v17, v25
+; GFX6-NEXT:    v_sub_i32_e32 v19, vcc, v19, v18
+; GFX6-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX6-NEXT:    v_sub_i32_e32 v9, vcc, v9, v17
+; GFX6-NEXT:    v_max_i32_e32 v17, -1, v10
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v17, v16
+; GFX6-NEXT:    v_min_i32_e32 v19, -1, v10
+; GFX6-NEXT:    v_max_i32_e32 v17, v17, v26
+; GFX6-NEXT:    v_sub_i32_e32 v19, vcc, v19, v18
+; GFX6-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX6-NEXT:    v_sub_i32_e32 v10, vcc, v10, v17
+; GFX6-NEXT:    v_max_i32_e32 v17, -1, v11
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v17, v16
+; GFX6-NEXT:    v_min_i32_e32 v19, -1, v11
+; GFX6-NEXT:    v_max_i32_e32 v17, v17, v27
+; GFX6-NEXT:    v_sub_i32_e32 v19, vcc, v19, v18
+; GFX6-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX6-NEXT:    v_sub_i32_e32 v11, vcc, v11, v17
+; GFX6-NEXT:    v_max_i32_e32 v17, -1, v12
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v17, v16
+; GFX6-NEXT:    v_min_i32_e32 v19, -1, v12
+; GFX6-NEXT:    v_max_i32_e32 v17, v17, v28
+; GFX6-NEXT:    v_sub_i32_e32 v19, vcc, v19, v18
+; GFX6-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX6-NEXT:    v_sub_i32_e32 v12, vcc, v12, v17
+; GFX6-NEXT:    v_max_i32_e32 v17, -1, v13
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v17, v16
+; GFX6-NEXT:    v_min_i32_e32 v19, -1, v13
+; GFX6-NEXT:    v_max_i32_e32 v17, v17, v29
+; GFX6-NEXT:    v_sub_i32_e32 v19, vcc, v19, v18
+; GFX6-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX6-NEXT:    v_sub_i32_e32 v13, vcc, v13, v17
+; GFX6-NEXT:    v_max_i32_e32 v17, -1, v14
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v17, v16
+; GFX6-NEXT:    v_min_i32_e32 v19, -1, v14
+; GFX6-NEXT:    v_max_i32_e32 v17, v17, v30
+; GFX6-NEXT:    v_sub_i32_e32 v19, vcc, v19, v18
+; GFX6-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX6-NEXT:    v_sub_i32_e32 v14, vcc, v14, v17
+; GFX6-NEXT:    v_max_i32_e32 v17, -1, v15
+; GFX6-NEXT:    v_sub_i32_e32 v16, vcc, v17, v16
+; GFX6-NEXT:    v_min_i32_e32 v17, -1, v15
+; GFX6-NEXT:    v_sub_i32_e32 v17, vcc, v17, v18
+; GFX6-NEXT:    v_max_i32_e32 v16, v16, v31
+; GFX6-NEXT:    v_min_i32_e32 v16, v16, v17
+; GFX6-NEXT:    v_sub_i32_e32 v15, vcc, v15, v16
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_v16i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_brev_b32 s4, -2
+; GFX8-NEXT:    v_max_i32_e32 v32, -1, v0
+; GFX8-NEXT:    v_subrev_u32_e32 v32, vcc, s4, v32
+; GFX8-NEXT:    v_max_i32_e32 v16, v32, v16
+; GFX8-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX8-NEXT:    v_min_i32_e32 v32, -1, v0
+; GFX8-NEXT:    v_subrev_u32_e32 v32, vcc, s5, v32
+; GFX8-NEXT:    v_min_i32_e32 v16, v16, v32
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, v0, v16
+; GFX8-NEXT:    v_max_i32_e32 v16, -1, v1
+; GFX8-NEXT:    v_subrev_u32_e32 v16, vcc, s4, v16
+; GFX8-NEXT:    v_max_i32_e32 v16, v16, v17
+; GFX8-NEXT:    v_min_i32_e32 v17, -1, v1
+; GFX8-NEXT:    v_subrev_u32_e32 v17, vcc, s5, v17
+; GFX8-NEXT:    v_min_i32_e32 v16, v16, v17
+; GFX8-NEXT:    v_sub_u32_e32 v1, vcc, v1, v16
+; GFX8-NEXT:    v_max_i32_e32 v16, -1, v2
+; GFX8-NEXT:    v_subrev_u32_e32 v16, vcc, s4, v16
+; GFX8-NEXT:    v_min_i32_e32 v17, -1, v2
+; GFX8-NEXT:    v_max_i32_e32 v16, v16, v18
+; GFX8-NEXT:    v_subrev_u32_e32 v17, vcc, s5, v17
+; GFX8-NEXT:    v_min_i32_e32 v16, v16, v17
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, v2, v16
+; GFX8-NEXT:    v_bfrev_b32_e32 v16, -2
+; GFX8-NEXT:    v_max_i32_e32 v17, -1, v3
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v17, v16
+; GFX8-NEXT:    v_max_i32_e32 v17, v17, v19
+; GFX8-NEXT:    v_mov_b32_e32 v18, 0x80000000
+; GFX8-NEXT:    v_min_i32_e32 v19, -1, v3
+; GFX8-NEXT:    v_sub_u32_e32 v19, vcc, v19, v18
+; GFX8-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX8-NEXT:    v_sub_u32_e32 v3, vcc, v3, v17
+; GFX8-NEXT:    v_max_i32_e32 v17, -1, v4
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v17, v16
+; GFX8-NEXT:    v_min_i32_e32 v19, -1, v4
+; GFX8-NEXT:    v_max_i32_e32 v17, v17, v20
+; GFX8-NEXT:    v_sub_u32_e32 v19, vcc, v19, v18
+; GFX8-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX8-NEXT:    v_sub_u32_e32 v4, vcc, v4, v17
+; GFX8-NEXT:    v_max_i32_e32 v17, -1, v5
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v17, v16
+; GFX8-NEXT:    v_min_i32_e32 v19, -1, v5
+; GFX8-NEXT:    v_max_i32_e32 v17, v17, v21
+; GFX8-NEXT:    v_sub_u32_e32 v19, vcc, v19, v18
+; GFX8-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX8-NEXT:    v_sub_u32_e32 v5, vcc, v5, v17
+; GFX8-NEXT:    v_max_i32_e32 v17, -1, v6
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v17, v16
+; GFX8-NEXT:    v_min_i32_e32 v19, -1, v6
+; GFX8-NEXT:    v_max_i32_e32 v17, v17, v22
+; GFX8-NEXT:    v_sub_u32_e32 v19, vcc, v19, v18
+; GFX8-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX8-NEXT:    v_sub_u32_e32 v6, vcc, v6, v17
+; GFX8-NEXT:    v_max_i32_e32 v17, -1, v7
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v17, v16
+; GFX8-NEXT:    v_min_i32_e32 v19, -1, v7
+; GFX8-NEXT:    v_max_i32_e32 v17, v17, v23
+; GFX8-NEXT:    v_sub_u32_e32 v19, vcc, v19, v18
+; GFX8-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX8-NEXT:    v_sub_u32_e32 v7, vcc, v7, v17
+; GFX8-NEXT:    v_max_i32_e32 v17, -1, v8
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v17, v16
+; GFX8-NEXT:    v_min_i32_e32 v19, -1, v8
+; GFX8-NEXT:    v_max_i32_e32 v17, v17, v24
+; GFX8-NEXT:    v_sub_u32_e32 v19, vcc, v19, v18
+; GFX8-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX8-NEXT:    v_sub_u32_e32 v8, vcc, v8, v17
+; GFX8-NEXT:    v_max_i32_e32 v17, -1, v9
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v17, v16
+; GFX8-NEXT:    v_min_i32_e32 v19, -1, v9
+; GFX8-NEXT:    v_max_i32_e32 v17, v17, v25
+; GFX8-NEXT:    v_sub_u32_e32 v19, vcc, v19, v18
+; GFX8-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX8-NEXT:    v_sub_u32_e32 v9, vcc, v9, v17
+; GFX8-NEXT:    v_max_i32_e32 v17, -1, v10
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v17, v16
+; GFX8-NEXT:    v_min_i32_e32 v19, -1, v10
+; GFX8-NEXT:    v_max_i32_e32 v17, v17, v26
+; GFX8-NEXT:    v_sub_u32_e32 v19, vcc, v19, v18
+; GFX8-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX8-NEXT:    v_sub_u32_e32 v10, vcc, v10, v17
+; GFX8-NEXT:    v_max_i32_e32 v17, -1, v11
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v17, v16
+; GFX8-NEXT:    v_min_i32_e32 v19, -1, v11
+; GFX8-NEXT:    v_max_i32_e32 v17, v17, v27
+; GFX8-NEXT:    v_sub_u32_e32 v19, vcc, v19, v18
+; GFX8-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX8-NEXT:    v_sub_u32_e32 v11, vcc, v11, v17
+; GFX8-NEXT:    v_max_i32_e32 v17, -1, v12
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v17, v16
+; GFX8-NEXT:    v_min_i32_e32 v19, -1, v12
+; GFX8-NEXT:    v_max_i32_e32 v17, v17, v28
+; GFX8-NEXT:    v_sub_u32_e32 v19, vcc, v19, v18
+; GFX8-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX8-NEXT:    v_sub_u32_e32 v12, vcc, v12, v17
+; GFX8-NEXT:    v_max_i32_e32 v17, -1, v13
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v17, v16
+; GFX8-NEXT:    v_min_i32_e32 v19, -1, v13
+; GFX8-NEXT:    v_max_i32_e32 v17, v17, v29
+; GFX8-NEXT:    v_sub_u32_e32 v19, vcc, v19, v18
+; GFX8-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX8-NEXT:    v_sub_u32_e32 v13, vcc, v13, v17
+; GFX8-NEXT:    v_max_i32_e32 v17, -1, v14
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v17, v16
+; GFX8-NEXT:    v_min_i32_e32 v19, -1, v14
+; GFX8-NEXT:    v_max_i32_e32 v17, v17, v30
+; GFX8-NEXT:    v_sub_u32_e32 v19, vcc, v19, v18
+; GFX8-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX8-NEXT:    v_sub_u32_e32 v14, vcc, v14, v17
+; GFX8-NEXT:    v_max_i32_e32 v17, -1, v15
+; GFX8-NEXT:    v_sub_u32_e32 v16, vcc, v17, v16
+; GFX8-NEXT:    v_min_i32_e32 v17, -1, v15
+; GFX8-NEXT:    v_sub_u32_e32 v17, vcc, v17, v18
+; GFX8-NEXT:    v_max_i32_e32 v16, v16, v31
+; GFX8-NEXT:    v_min_i32_e32 v16, v16, v17
+; GFX8-NEXT:    v_sub_u32_e32 v15, vcc, v15, v16
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_v16i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_brev_b32 s4, -2
+; GFX9-NEXT:    v_max_i32_e32 v32, -1, v0
+; GFX9-NEXT:    v_subrev_u32_e32 v32, s4, v32
+; GFX9-NEXT:    v_max_i32_e32 v16, v32, v16
+; GFX9-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX9-NEXT:    v_min_i32_e32 v32, -1, v0
+; GFX9-NEXT:    v_subrev_u32_e32 v32, s5, v32
+; GFX9-NEXT:    v_min_i32_e32 v16, v16, v32
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v16
+; GFX9-NEXT:    v_max_i32_e32 v16, -1, v1
+; GFX9-NEXT:    v_subrev_u32_e32 v16, s4, v16
+; GFX9-NEXT:    v_max_i32_e32 v16, v16, v17
+; GFX9-NEXT:    v_min_i32_e32 v17, -1, v1
+; GFX9-NEXT:    v_subrev_u32_e32 v17, s5, v17
+; GFX9-NEXT:    v_min_i32_e32 v16, v16, v17
+; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v16
+; GFX9-NEXT:    v_max_i32_e32 v16, -1, v2
+; GFX9-NEXT:    v_subrev_u32_e32 v16, s4, v16
+; GFX9-NEXT:    v_min_i32_e32 v17, -1, v2
+; GFX9-NEXT:    v_max_i32_e32 v16, v16, v18
+; GFX9-NEXT:    v_subrev_u32_e32 v17, s5, v17
+; GFX9-NEXT:    v_min_i32_e32 v16, v16, v17
+; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v16
+; GFX9-NEXT:    v_bfrev_b32_e32 v16, -2
+; GFX9-NEXT:    v_max_i32_e32 v17, -1, v3
+; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v16
+; GFX9-NEXT:    v_max_i32_e32 v17, v17, v19
+; GFX9-NEXT:    v_mov_b32_e32 v18, 0x80000000
+; GFX9-NEXT:    v_min_i32_e32 v19, -1, v3
+; GFX9-NEXT:    v_sub_u32_e32 v19, v19, v18
+; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX9-NEXT:    v_sub_u32_e32 v3, v3, v17
+; GFX9-NEXT:    v_max_i32_e32 v17, -1, v4
+; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v16
+; GFX9-NEXT:    v_min_i32_e32 v19, -1, v4
+; GFX9-NEXT:    v_max_i32_e32 v17, v17, v20
+; GFX9-NEXT:    v_sub_u32_e32 v19, v19, v18
+; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX9-NEXT:    v_sub_u32_e32 v4, v4, v17
+; GFX9-NEXT:    v_max_i32_e32 v17, -1, v5
+; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v16
+; GFX9-NEXT:    v_min_i32_e32 v19, -1, v5
+; GFX9-NEXT:    v_max_i32_e32 v17, v17, v21
+; GFX9-NEXT:    v_sub_u32_e32 v19, v19, v18
+; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX9-NEXT:    v_sub_u32_e32 v5, v5, v17
+; GFX9-NEXT:    v_max_i32_e32 v17, -1, v6
+; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v16
+; GFX9-NEXT:    v_min_i32_e32 v19, -1, v6
+; GFX9-NEXT:    v_max_i32_e32 v17, v17, v22
+; GFX9-NEXT:    v_sub_u32_e32 v19, v19, v18
+; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX9-NEXT:    v_sub_u32_e32 v6, v6, v17
+; GFX9-NEXT:    v_max_i32_e32 v17, -1, v7
+; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v16
+; GFX9-NEXT:    v_min_i32_e32 v19, -1, v7
+; GFX9-NEXT:    v_max_i32_e32 v17, v17, v23
+; GFX9-NEXT:    v_sub_u32_e32 v19, v19, v18
+; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX9-NEXT:    v_sub_u32_e32 v7, v7, v17
+; GFX9-NEXT:    v_max_i32_e32 v17, -1, v8
+; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v16
+; GFX9-NEXT:    v_min_i32_e32 v19, -1, v8
+; GFX9-NEXT:    v_max_i32_e32 v17, v17, v24
+; GFX9-NEXT:    v_sub_u32_e32 v19, v19, v18
+; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX9-NEXT:    v_sub_u32_e32 v8, v8, v17
+; GFX9-NEXT:    v_max_i32_e32 v17, -1, v9
+; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v16
+; GFX9-NEXT:    v_min_i32_e32 v19, -1, v9
+; GFX9-NEXT:    v_max_i32_e32 v17, v17, v25
+; GFX9-NEXT:    v_sub_u32_e32 v19, v19, v18
+; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX9-NEXT:    v_sub_u32_e32 v9, v9, v17
+; GFX9-NEXT:    v_max_i32_e32 v17, -1, v10
+; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v16
+; GFX9-NEXT:    v_min_i32_e32 v19, -1, v10
+; GFX9-NEXT:    v_max_i32_e32 v17, v17, v26
+; GFX9-NEXT:    v_sub_u32_e32 v19, v19, v18
+; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX9-NEXT:    v_sub_u32_e32 v10, v10, v17
+; GFX9-NEXT:    v_max_i32_e32 v17, -1, v11
+; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v16
+; GFX9-NEXT:    v_min_i32_e32 v19, -1, v11
+; GFX9-NEXT:    v_max_i32_e32 v17, v17, v27
+; GFX9-NEXT:    v_sub_u32_e32 v19, v19, v18
+; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX9-NEXT:    v_sub_u32_e32 v11, v11, v17
+; GFX9-NEXT:    v_max_i32_e32 v17, -1, v12
+; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v16
+; GFX9-NEXT:    v_min_i32_e32 v19, -1, v12
+; GFX9-NEXT:    v_max_i32_e32 v17, v17, v28
+; GFX9-NEXT:    v_sub_u32_e32 v19, v19, v18
+; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX9-NEXT:    v_sub_u32_e32 v12, v12, v17
+; GFX9-NEXT:    v_max_i32_e32 v17, -1, v13
+; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v16
+; GFX9-NEXT:    v_min_i32_e32 v19, -1, v13
+; GFX9-NEXT:    v_max_i32_e32 v17, v17, v29
+; GFX9-NEXT:    v_sub_u32_e32 v19, v19, v18
+; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX9-NEXT:    v_sub_u32_e32 v13, v13, v17
+; GFX9-NEXT:    v_max_i32_e32 v17, -1, v14
+; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v16
+; GFX9-NEXT:    v_min_i32_e32 v19, -1, v14
+; GFX9-NEXT:    v_max_i32_e32 v17, v17, v30
+; GFX9-NEXT:    v_sub_u32_e32 v19, v19, v18
+; GFX9-NEXT:    v_min_i32_e32 v17, v17, v19
+; GFX9-NEXT:    v_sub_u32_e32 v14, v14, v17
+; GFX9-NEXT:    v_max_i32_e32 v17, -1, v15
+; GFX9-NEXT:    v_sub_u32_e32 v16, v17, v16
+; GFX9-NEXT:    v_min_i32_e32 v17, -1, v15
+; GFX9-NEXT:    v_sub_u32_e32 v17, v17, v18
+; GFX9-NEXT:    v_max_i32_e32 v16, v16, v31
+; GFX9-NEXT:    v_min_i32_e32 v16, v16, v17
+; GFX9-NEXT:    v_sub_u32_e32 v15, v15, v16
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_ssubsat_v16i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_max_i32_e32 v32, -1, v0
+; GFX10-NEXT:    s_brev_b32 s4, -2
+; GFX10-NEXT:    v_min_i32_e32 v33, -1, v0
+; GFX10-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX10-NEXT:    v_max_i32_e32 v36, -1, v2
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v35, s4, v32
+; GFX10-NEXT:    v_max_i32_e32 v32, -1, v1
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v33, s5, v33
+; GFX10-NEXT:    v_bfrev_b32_e32 v34, -2
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v36, s4, v36
+; GFX10-NEXT:    v_max_i32_e32 v16, v35, v16
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v32, s4, v32
+; GFX10-NEXT:    v_max_i32_e32 v39, -1, v3
+; GFX10-NEXT:    v_min_i32_e32 v37, -1, v1
+; GFX10-NEXT:    v_max_i32_e32 v18, v36, v18
+; GFX10-NEXT:    v_min_i32_e32 v16, v16, v33
+; GFX10-NEXT:    v_min_i32_e32 v33, -1, v2
+; GFX10-NEXT:    v_max_i32_e32 v38, v32, v17
+; GFX10-NEXT:    v_max_i32_e32 v17, -1, v4
+; GFX10-NEXT:    v_sub_nc_u32_e32 v36, v39, v34
+; GFX10-NEXT:    v_mov_b32_e32 v35, 0x80000000
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v32, s5, v33
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v16
+; GFX10-NEXT:    v_min_i32_e32 v33, -1, v3
+; GFX10-NEXT:    v_sub_nc_u32_e32 v17, v17, v34
+; GFX10-NEXT:    v_subrev_nc_u32_e32 v37, s5, v37
+; GFX10-NEXT:    v_min_i32_e32 v16, v18, v32
+; GFX10-NEXT:    v_max_i32_e32 v19, v36, v19
+; GFX10-NEXT:    v_sub_nc_u32_e32 v18, v33, v35
+; GFX10-NEXT:    v_max_i32_e32 v17, v17, v20
+; GFX10-NEXT:    v_min_i32_e32 v39, v38, v37
+; GFX10-NEXT:    v_sub_nc_u32_e32 v2, v2, v16
+; GFX10-NEXT:    v_min_i32_e32 v16, -1, v4
+; GFX10-NEXT:    v_min_i32_e32 v18, v19, v18
+; GFX10-NEXT:    v_max_i32_e32 v19, -1, v5
+; GFX10-NEXT:    v_max_i32_e32 v32, -1, v6
+; GFX10-NEXT:    v_min_i32_e32 v33, -1, v5
+; GFX10-NEXT:    v_sub_nc_u32_e32 v16, v16, v35
+; GFX10-NEXT:    v_sub_nc_u32_e32 v1, v1, v39
+; GFX10-NEXT:    v_sub_nc_u32_e32 v19, v19, v34
+; GFX10-NEXT:    v_sub_nc_u32_e32 v32, v32, v34
+; GFX10-NEXT:    v_min_i32_e32 v36, -1, v6
+; GFX10-NEXT:    v_min_i32_e32 v39, v17, v16
+; GFX10-NEXT:    v_max_i32_e32 v17, -1, v7
+; GFX10-NEXT:    v_min_i32_e32 v16, -1, v7
+; GFX10-NEXT:    v_max_i32_e32 v19, v19, v21
+; GFX10-NEXT:    v_sub_nc_u32_e32 v20, v33, v35
+; GFX10-NEXT:    v_sub_nc_u32_e32 v4, v4, v39
+; GFX10-NEXT:    v_sub_nc_u32_e32 v17, v17, v34
+; GFX10-NEXT:    v_sub_nc_u32_e32 v21, v36, v35
+; GFX10-NEXT:    v_max_i32_e32 v22, v32, v22
+; GFX10-NEXT:    v_min_i32_e32 v38, v19, v20
+; GFX10-NEXT:    v_max_i32_e32 v20, -1, v9
+; GFX10-NEXT:    v_max_i32_e32 v39, -1, v8
+; GFX10-NEXT:    v_sub_nc_u32_e32 v16, v16, v35
+; GFX10-NEXT:    v_max_i32_e32 v17, v17, v23
+; GFX10-NEXT:    v_min_i32_e32 v19, v22, v21
+; GFX10-NEXT:    v_min_i32_e32 v21, -1, v9
+; GFX10-NEXT:    v_sub_nc_u32_e32 v20, v20, v34
+; GFX10-NEXT:    v_sub_nc_u32_e32 v3, v3, v18
+; GFX10-NEXT:    v_sub_nc_u32_e32 v18, v39, v34
+; GFX10-NEXT:    v_max_i32_e32 v39, -1, v10
+; GFX10-NEXT:    v_min_i32_e32 v16, v17, v16
+; GFX10-NEXT:    v_min_i32_e32 v22, -1, v8
+; GFX10-NEXT:    v_sub_nc_u32_e32 v5, v5, v38
+; GFX10-NEXT:    v_sub_nc_u32_e32 v6, v6, v19
+; GFX10-NEXT:    v_max_i32_e32 v18, v18, v24
+; GFX10-NEXT:    v_max_i32_e32 v20, v20, v25
+; GFX10-NEXT:    v_sub_nc_u32_e32 v19, v22, v35
+; GFX10-NEXT:    v_sub_nc_u32_e32 v21, v21, v35
+; GFX10-NEXT:    v_sub_nc_u32_e32 v7, v7, v16
+; GFX10-NEXT:    v_max_i32_e32 v16, -1, v11
+; GFX10-NEXT:    v_min_i32_e32 v38, -1, v10
+; GFX10-NEXT:    v_sub_nc_u32_e32 v23, v39, v34
+; GFX10-NEXT:    v_min_i32_e32 v17, v18, v19
+; GFX10-NEXT:    v_min_i32_e32 v20, v20, v21
+; GFX10-NEXT:    v_sub_nc_u32_e32 v16, v16, v34
+; GFX10-NEXT:    v_sub_nc_u32_e32 v18, v38, v35
+; GFX10-NEXT:    v_max_i32_e32 v19, v23, v26
+; GFX10-NEXT:    v_sub_nc_u32_e32 v8, v8, v17
+; GFX10-NEXT:    v_sub_nc_u32_e32 v9, v9, v20
+; GFX10-NEXT:    v_max_i32_e32 v20, -1, v13
+; GFX10-NEXT:    v_max_i32_e32 v16, v16, v27
+; GFX10-NEXT:    v_min_i32_e32 v17, v19, v18
+; GFX10-NEXT:    v_max_i32_e32 v19, -1, v12
+; GFX10-NEXT:    v_max_i32_e32 v27, -1, v14
+; GFX10-NEXT:    v_max_i32_e32 v23, -1, v15
+; GFX10-NEXT:    v_min_i32_e32 v18, -1, v11
+; GFX10-NEXT:    v_min_i32_e32 v21, -1, v13
+; GFX10-NEXT:    v_sub_nc_u32_e32 v19, v19, v34
+; GFX10-NEXT:    v_sub_nc_u32_e32 v20, v20, v34
+; GFX10-NEXT:    v_min_i32_e32 v24, -1, v14
+; GFX10-NEXT:    v_min_i32_e32 v25, -1, v15
+; GFX10-NEXT:    v_sub_nc_u32_e32 v26, v23, v34
+; GFX10-NEXT:    v_sub_nc_u32_e32 v10, v10, v17
+; GFX10-NEXT:    v_min_i32_e32 v17, -1, v12
+; GFX10-NEXT:    v_sub_nc_u32_e32 v27, v27, v34
+; GFX10-NEXT:    v_sub_nc_u32_e32 v18, v18, v35
+; GFX10-NEXT:    v_max_i32_e32 v19, v19, v28
+; GFX10-NEXT:    v_sub_nc_u32_e32 v21, v21, v35
+; GFX10-NEXT:    v_sub_nc_u32_e32 v17, v17, v35
+; GFX10-NEXT:    v_max_i32_e32 v20, v20, v29
+; GFX10-NEXT:    v_sub_nc_u32_e32 v24, v24, v35
+; GFX10-NEXT:    v_max_i32_e32 v22, v27, v30
+; GFX10-NEXT:    v_sub_nc_u32_e32 v25, v25, v35
+; GFX10-NEXT:    v_max_i32_e32 v23, v26, v31
+; GFX10-NEXT:    v_min_i32_e32 v16, v16, v18
+; GFX10-NEXT:    v_min_i32_e32 v17, v19, v17
+; GFX10-NEXT:    v_min_i32_e32 v18, v20, v21
+; GFX10-NEXT:    v_min_i32_e32 v19, v22, v24
+; GFX10-NEXT:    v_min_i32_e32 v20, v23, v25
+; GFX10-NEXT:    v_sub_nc_u32_e32 v11, v11, v16
+; GFX10-NEXT:    v_sub_nc_u32_e32 v12, v12, v17
+; GFX10-NEXT:    v_sub_nc_u32_e32 v13, v13, v18
+; GFX10-NEXT:    v_sub_nc_u32_e32 v14, v14, v19
+; GFX10-NEXT:    v_sub_nc_u32_e32 v15, v15, v20
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
+  ret <16 x i32> %result
+}
+
+define amdgpu_ps <16 x i32> @s_ssubsat_v16i32(<16 x i32> inreg %lhs, <16 x i32> inreg %rhs) {
+; GFX6-LABEL: s_ssubsat_v16i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX6-NEXT:    s_brev_b32 s32, -2
+; GFX6-NEXT:    s_cselect_b32 s34, s0, -1
+; GFX6-NEXT:    s_sub_i32 s34, s34, s32
+; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX6-NEXT:    s_mov_b32 s33, 0x80000000
+; GFX6-NEXT:    s_cselect_b32 s35, s0, -1
+; GFX6-NEXT:    s_sub_i32 s35, s35, s33
+; GFX6-NEXT:    s_cmp_gt_i32 s34, s16
+; GFX6-NEXT:    s_cselect_b32 s16, s34, s16
+; GFX6-NEXT:    s_cmp_lt_i32 s16, s35
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s35
+; GFX6-NEXT:    s_sub_i32 s0, s0, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s1, -1
+; GFX6-NEXT:    s_cselect_b32 s16, s1, -1
+; GFX6-NEXT:    s_sub_i32 s16, s16, s32
+; GFX6-NEXT:    s_cmp_lt_i32 s1, -1
+; GFX6-NEXT:    s_cselect_b32 s34, s1, -1
+; GFX6-NEXT:    s_sub_i32 s34, s34, s33
+; GFX6-NEXT:    s_cmp_gt_i32 s16, s17
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX6-NEXT:    s_cmp_lt_i32 s16, s34
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s34
+; GFX6-NEXT:    s_sub_i32 s1, s1, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s2, -1
+; GFX6-NEXT:    s_cselect_b32 s16, s2, -1
+; GFX6-NEXT:    s_sub_i32 s16, s16, s32
+; GFX6-NEXT:    s_cmp_lt_i32 s2, -1
+; GFX6-NEXT:    s_cselect_b32 s17, s2, -1
+; GFX6-NEXT:    s_sub_i32 s17, s17, s33
+; GFX6-NEXT:    s_cmp_gt_i32 s16, s18
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s18
+; GFX6-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX6-NEXT:    s_sub_i32 s2, s2, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s3, -1
+; GFX6-NEXT:    s_cselect_b32 s16, s3, -1
+; GFX6-NEXT:    s_sub_i32 s16, s16, s32
+; GFX6-NEXT:    s_cmp_lt_i32 s3, -1
+; GFX6-NEXT:    s_cselect_b32 s17, s3, -1
+; GFX6-NEXT:    s_sub_i32 s17, s17, s33
+; GFX6-NEXT:    s_cmp_gt_i32 s16, s19
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s19
+; GFX6-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX6-NEXT:    s_sub_i32 s3, s3, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s4, -1
+; GFX6-NEXT:    s_cselect_b32 s16, s4, -1
+; GFX6-NEXT:    s_sub_i32 s16, s16, s32
+; GFX6-NEXT:    s_cmp_lt_i32 s4, -1
+; GFX6-NEXT:    s_cselect_b32 s17, s4, -1
+; GFX6-NEXT:    s_sub_i32 s17, s17, s33
+; GFX6-NEXT:    s_cmp_gt_i32 s16, s20
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s20
+; GFX6-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX6-NEXT:    s_sub_i32 s4, s4, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s5, -1
+; GFX6-NEXT:    s_cselect_b32 s16, s5, -1
+; GFX6-NEXT:    s_sub_i32 s16, s16, s32
+; GFX6-NEXT:    s_cmp_lt_i32 s5, -1
+; GFX6-NEXT:    s_cselect_b32 s17, s5, -1
+; GFX6-NEXT:    s_sub_i32 s17, s17, s33
+; GFX6-NEXT:    s_cmp_gt_i32 s16, s21
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s21
+; GFX6-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX6-NEXT:    s_sub_i32 s5, s5, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s6, -1
+; GFX6-NEXT:    s_cselect_b32 s16, s6, -1
+; GFX6-NEXT:    s_sub_i32 s16, s16, s32
+; GFX6-NEXT:    s_cmp_lt_i32 s6, -1
+; GFX6-NEXT:    s_cselect_b32 s17, s6, -1
+; GFX6-NEXT:    s_sub_i32 s17, s17, s33
+; GFX6-NEXT:    s_cmp_gt_i32 s16, s22
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s22
+; GFX6-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX6-NEXT:    s_sub_i32 s6, s6, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s7, -1
+; GFX6-NEXT:    s_cselect_b32 s16, s7, -1
+; GFX6-NEXT:    s_sub_i32 s16, s16, s32
+; GFX6-NEXT:    s_cmp_lt_i32 s7, -1
+; GFX6-NEXT:    s_cselect_b32 s17, s7, -1
+; GFX6-NEXT:    s_sub_i32 s17, s17, s33
+; GFX6-NEXT:    s_cmp_gt_i32 s16, s23
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s23
+; GFX6-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX6-NEXT:    s_sub_i32 s7, s7, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s8, -1
+; GFX6-NEXT:    s_cselect_b32 s16, s8, -1
+; GFX6-NEXT:    s_sub_i32 s16, s16, s32
+; GFX6-NEXT:    s_cmp_lt_i32 s8, -1
+; GFX6-NEXT:    s_cselect_b32 s17, s8, -1
+; GFX6-NEXT:    s_sub_i32 s17, s17, s33
+; GFX6-NEXT:    s_cmp_gt_i32 s16, s24
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s24
+; GFX6-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX6-NEXT:    s_sub_i32 s8, s8, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s9, -1
+; GFX6-NEXT:    s_cselect_b32 s16, s9, -1
+; GFX6-NEXT:    s_sub_i32 s16, s16, s32
+; GFX6-NEXT:    s_cmp_lt_i32 s9, -1
+; GFX6-NEXT:    s_cselect_b32 s17, s9, -1
+; GFX6-NEXT:    s_sub_i32 s17, s17, s33
+; GFX6-NEXT:    s_cmp_gt_i32 s16, s25
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s25
+; GFX6-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX6-NEXT:    s_sub_i32 s9, s9, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s10, -1
+; GFX6-NEXT:    s_cselect_b32 s16, s10, -1
+; GFX6-NEXT:    s_sub_i32 s16, s16, s32
+; GFX6-NEXT:    s_cmp_lt_i32 s10, -1
+; GFX6-NEXT:    s_cselect_b32 s17, s10, -1
+; GFX6-NEXT:    s_sub_i32 s17, s17, s33
+; GFX6-NEXT:    s_cmp_gt_i32 s16, s26
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s26
+; GFX6-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX6-NEXT:    s_sub_i32 s10, s10, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s11, -1
+; GFX6-NEXT:    s_cselect_b32 s16, s11, -1
+; GFX6-NEXT:    s_sub_i32 s16, s16, s32
+; GFX6-NEXT:    s_cmp_lt_i32 s11, -1
+; GFX6-NEXT:    s_cselect_b32 s17, s11, -1
+; GFX6-NEXT:    s_sub_i32 s17, s17, s33
+; GFX6-NEXT:    s_cmp_gt_i32 s16, s27
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s27
+; GFX6-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX6-NEXT:    s_sub_i32 s11, s11, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s12, -1
+; GFX6-NEXT:    s_cselect_b32 s16, s12, -1
+; GFX6-NEXT:    s_sub_i32 s16, s16, s32
+; GFX6-NEXT:    s_cmp_lt_i32 s12, -1
+; GFX6-NEXT:    s_cselect_b32 s17, s12, -1
+; GFX6-NEXT:    s_sub_i32 s17, s17, s33
+; GFX6-NEXT:    s_cmp_gt_i32 s16, s28
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s28
+; GFX6-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX6-NEXT:    s_sub_i32 s12, s12, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s13, -1
+; GFX6-NEXT:    s_cselect_b32 s16, s13, -1
+; GFX6-NEXT:    s_sub_i32 s16, s16, s32
+; GFX6-NEXT:    s_cmp_lt_i32 s13, -1
+; GFX6-NEXT:    s_cselect_b32 s17, s13, -1
+; GFX6-NEXT:    s_sub_i32 s17, s17, s33
+; GFX6-NEXT:    s_cmp_gt_i32 s16, s29
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s29
+; GFX6-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX6-NEXT:    s_sub_i32 s13, s13, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s14, -1
+; GFX6-NEXT:    s_cselect_b32 s16, s14, -1
+; GFX6-NEXT:    s_sub_i32 s16, s16, s32
+; GFX6-NEXT:    s_cmp_lt_i32 s14, -1
+; GFX6-NEXT:    s_cselect_b32 s17, s14, -1
+; GFX6-NEXT:    s_sub_i32 s17, s17, s33
+; GFX6-NEXT:    s_cmp_gt_i32 s16, s30
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s30
+; GFX6-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX6-NEXT:    s_sub_i32 s14, s14, s16
+; GFX6-NEXT:    s_cmp_gt_i32 s15, -1
+; GFX6-NEXT:    s_cselect_b32 s16, s15, -1
+; GFX6-NEXT:    s_sub_i32 s16, s16, s32
+; GFX6-NEXT:    s_cmp_lt_i32 s15, -1
+; GFX6-NEXT:    s_cselect_b32 s17, s15, -1
+; GFX6-NEXT:    s_sub_i32 s17, s17, s33
+; GFX6-NEXT:    s_cmp_gt_i32 s16, s31
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s31
+; GFX6-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX6-NEXT:    s_sub_i32 s15, s15, s16
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_ssubsat_v16i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX8-NEXT:    s_brev_b32 s32, -2
+; GFX8-NEXT:    s_cselect_b32 s34, s0, -1
+; GFX8-NEXT:    s_sub_i32 s34, s34, s32
+; GFX8-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX8-NEXT:    s_mov_b32 s33, 0x80000000
+; GFX8-NEXT:    s_cselect_b32 s35, s0, -1
+; GFX8-NEXT:    s_sub_i32 s35, s35, s33
+; GFX8-NEXT:    s_cmp_gt_i32 s34, s16
+; GFX8-NEXT:    s_cselect_b32 s16, s34, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s16, s35
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s35
+; GFX8-NEXT:    s_sub_i32 s0, s0, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s1, -1
+; GFX8-NEXT:    s_cselect_b32 s16, s1, -1
+; GFX8-NEXT:    s_sub_i32 s16, s16, s32
+; GFX8-NEXT:    s_cmp_lt_i32 s1, -1
+; GFX8-NEXT:    s_cselect_b32 s34, s1, -1
+; GFX8-NEXT:    s_sub_i32 s34, s34, s33
+; GFX8-NEXT:    s_cmp_gt_i32 s16, s17
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX8-NEXT:    s_cmp_lt_i32 s16, s34
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s34
+; GFX8-NEXT:    s_sub_i32 s1, s1, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s2, -1
+; GFX8-NEXT:    s_cselect_b32 s16, s2, -1
+; GFX8-NEXT:    s_sub_i32 s16, s16, s32
+; GFX8-NEXT:    s_cmp_lt_i32 s2, -1
+; GFX8-NEXT:    s_cselect_b32 s17, s2, -1
+; GFX8-NEXT:    s_sub_i32 s17, s17, s33
+; GFX8-NEXT:    s_cmp_gt_i32 s16, s18
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s18
+; GFX8-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX8-NEXT:    s_sub_i32 s2, s2, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s3, -1
+; GFX8-NEXT:    s_cselect_b32 s16, s3, -1
+; GFX8-NEXT:    s_sub_i32 s16, s16, s32
+; GFX8-NEXT:    s_cmp_lt_i32 s3, -1
+; GFX8-NEXT:    s_cselect_b32 s17, s3, -1
+; GFX8-NEXT:    s_sub_i32 s17, s17, s33
+; GFX8-NEXT:    s_cmp_gt_i32 s16, s19
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s19
+; GFX8-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX8-NEXT:    s_sub_i32 s3, s3, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s4, -1
+; GFX8-NEXT:    s_cselect_b32 s16, s4, -1
+; GFX8-NEXT:    s_sub_i32 s16, s16, s32
+; GFX8-NEXT:    s_cmp_lt_i32 s4, -1
+; GFX8-NEXT:    s_cselect_b32 s17, s4, -1
+; GFX8-NEXT:    s_sub_i32 s17, s17, s33
+; GFX8-NEXT:    s_cmp_gt_i32 s16, s20
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s20
+; GFX8-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX8-NEXT:    s_sub_i32 s4, s4, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s5, -1
+; GFX8-NEXT:    s_cselect_b32 s16, s5, -1
+; GFX8-NEXT:    s_sub_i32 s16, s16, s32
+; GFX8-NEXT:    s_cmp_lt_i32 s5, -1
+; GFX8-NEXT:    s_cselect_b32 s17, s5, -1
+; GFX8-NEXT:    s_sub_i32 s17, s17, s33
+; GFX8-NEXT:    s_cmp_gt_i32 s16, s21
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s21
+; GFX8-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX8-NEXT:    s_sub_i32 s5, s5, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s6, -1
+; GFX8-NEXT:    s_cselect_b32 s16, s6, -1
+; GFX8-NEXT:    s_sub_i32 s16, s16, s32
+; GFX8-NEXT:    s_cmp_lt_i32 s6, -1
+; GFX8-NEXT:    s_cselect_b32 s17, s6, -1
+; GFX8-NEXT:    s_sub_i32 s17, s17, s33
+; GFX8-NEXT:    s_cmp_gt_i32 s16, s22
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s22
+; GFX8-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX8-NEXT:    s_sub_i32 s6, s6, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s7, -1
+; GFX8-NEXT:    s_cselect_b32 s16, s7, -1
+; GFX8-NEXT:    s_sub_i32 s16, s16, s32
+; GFX8-NEXT:    s_cmp_lt_i32 s7, -1
+; GFX8-NEXT:    s_cselect_b32 s17, s7, -1
+; GFX8-NEXT:    s_sub_i32 s17, s17, s33
+; GFX8-NEXT:    s_cmp_gt_i32 s16, s23
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s23
+; GFX8-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX8-NEXT:    s_sub_i32 s7, s7, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s8, -1
+; GFX8-NEXT:    s_cselect_b32 s16, s8, -1
+; GFX8-NEXT:    s_sub_i32 s16, s16, s32
+; GFX8-NEXT:    s_cmp_lt_i32 s8, -1
+; GFX8-NEXT:    s_cselect_b32 s17, s8, -1
+; GFX8-NEXT:    s_sub_i32 s17, s17, s33
+; GFX8-NEXT:    s_cmp_gt_i32 s16, s24
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s24
+; GFX8-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX8-NEXT:    s_sub_i32 s8, s8, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s9, -1
+; GFX8-NEXT:    s_cselect_b32 s16, s9, -1
+; GFX8-NEXT:    s_sub_i32 s16, s16, s32
+; GFX8-NEXT:    s_cmp_lt_i32 s9, -1
+; GFX8-NEXT:    s_cselect_b32 s17, s9, -1
+; GFX8-NEXT:    s_sub_i32 s17, s17, s33
+; GFX8-NEXT:    s_cmp_gt_i32 s16, s25
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s25
+; GFX8-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX8-NEXT:    s_sub_i32 s9, s9, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s10, -1
+; GFX8-NEXT:    s_cselect_b32 s16, s10, -1
+; GFX8-NEXT:    s_sub_i32 s16, s16, s32
+; GFX8-NEXT:    s_cmp_lt_i32 s10, -1
+; GFX8-NEXT:    s_cselect_b32 s17, s10, -1
+; GFX8-NEXT:    s_sub_i32 s17, s17, s33
+; GFX8-NEXT:    s_cmp_gt_i32 s16, s26
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s26
+; GFX8-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX8-NEXT:    s_sub_i32 s10, s10, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s11, -1
+; GFX8-NEXT:    s_cselect_b32 s16, s11, -1
+; GFX8-NEXT:    s_sub_i32 s16, s16, s32
+; GFX8-NEXT:    s_cmp_lt_i32 s11, -1
+; GFX8-NEXT:    s_cselect_b32 s17, s11, -1
+; GFX8-NEXT:    s_sub_i32 s17, s17, s33
+; GFX8-NEXT:    s_cmp_gt_i32 s16, s27
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s27
+; GFX8-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX8-NEXT:    s_sub_i32 s11, s11, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s12, -1
+; GFX8-NEXT:    s_cselect_b32 s16, s12, -1
+; GFX8-NEXT:    s_sub_i32 s16, s16, s32
+; GFX8-NEXT:    s_cmp_lt_i32 s12, -1
+; GFX8-NEXT:    s_cselect_b32 s17, s12, -1
+; GFX8-NEXT:    s_sub_i32 s17, s17, s33
+; GFX8-NEXT:    s_cmp_gt_i32 s16, s28
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s28
+; GFX8-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX8-NEXT:    s_sub_i32 s12, s12, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s13, -1
+; GFX8-NEXT:    s_cselect_b32 s16, s13, -1
+; GFX8-NEXT:    s_sub_i32 s16, s16, s32
+; GFX8-NEXT:    s_cmp_lt_i32 s13, -1
+; GFX8-NEXT:    s_cselect_b32 s17, s13, -1
+; GFX8-NEXT:    s_sub_i32 s17, s17, s33
+; GFX8-NEXT:    s_cmp_gt_i32 s16, s29
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s29
+; GFX8-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX8-NEXT:    s_sub_i32 s13, s13, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s14, -1
+; GFX8-NEXT:    s_cselect_b32 s16, s14, -1
+; GFX8-NEXT:    s_sub_i32 s16, s16, s32
+; GFX8-NEXT:    s_cmp_lt_i32 s14, -1
+; GFX8-NEXT:    s_cselect_b32 s17, s14, -1
+; GFX8-NEXT:    s_sub_i32 s17, s17, s33
+; GFX8-NEXT:    s_cmp_gt_i32 s16, s30
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s30
+; GFX8-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX8-NEXT:    s_sub_i32 s14, s14, s16
+; GFX8-NEXT:    s_cmp_gt_i32 s15, -1
+; GFX8-NEXT:    s_cselect_b32 s16, s15, -1
+; GFX8-NEXT:    s_sub_i32 s16, s16, s32
+; GFX8-NEXT:    s_cmp_lt_i32 s15, -1
+; GFX8-NEXT:    s_cselect_b32 s17, s15, -1
+; GFX8-NEXT:    s_sub_i32 s17, s17, s33
+; GFX8-NEXT:    s_cmp_gt_i32 s16, s31
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s31
+; GFX8-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX8-NEXT:    s_sub_i32 s15, s15, s16
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_ssubsat_v16i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX9-NEXT:    s_brev_b32 s32, -2
+; GFX9-NEXT:    s_cselect_b32 s34, s0, -1
+; GFX9-NEXT:    s_sub_i32 s34, s34, s32
+; GFX9-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX9-NEXT:    s_mov_b32 s33, 0x80000000
+; GFX9-NEXT:    s_cselect_b32 s35, s0, -1
+; GFX9-NEXT:    s_sub_i32 s35, s35, s33
+; GFX9-NEXT:    s_cmp_gt_i32 s34, s16
+; GFX9-NEXT:    s_cselect_b32 s16, s34, s16
+; GFX9-NEXT:    s_cmp_lt_i32 s16, s35
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s35
+; GFX9-NEXT:    s_sub_i32 s0, s0, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s1, -1
+; GFX9-NEXT:    s_cselect_b32 s16, s1, -1
+; GFX9-NEXT:    s_sub_i32 s16, s16, s32
+; GFX9-NEXT:    s_cmp_lt_i32 s1, -1
+; GFX9-NEXT:    s_cselect_b32 s34, s1, -1
+; GFX9-NEXT:    s_sub_i32 s34, s34, s33
+; GFX9-NEXT:    s_cmp_gt_i32 s16, s17
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX9-NEXT:    s_cmp_lt_i32 s16, s34
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s34
+; GFX9-NEXT:    s_sub_i32 s1, s1, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s2, -1
+; GFX9-NEXT:    s_cselect_b32 s16, s2, -1
+; GFX9-NEXT:    s_sub_i32 s16, s16, s32
+; GFX9-NEXT:    s_cmp_lt_i32 s2, -1
+; GFX9-NEXT:    s_cselect_b32 s17, s2, -1
+; GFX9-NEXT:    s_sub_i32 s17, s17, s33
+; GFX9-NEXT:    s_cmp_gt_i32 s16, s18
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s18
+; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX9-NEXT:    s_sub_i32 s2, s2, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s3, -1
+; GFX9-NEXT:    s_cselect_b32 s16, s3, -1
+; GFX9-NEXT:    s_sub_i32 s16, s16, s32
+; GFX9-NEXT:    s_cmp_lt_i32 s3, -1
+; GFX9-NEXT:    s_cselect_b32 s17, s3, -1
+; GFX9-NEXT:    s_sub_i32 s17, s17, s33
+; GFX9-NEXT:    s_cmp_gt_i32 s16, s19
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s19
+; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX9-NEXT:    s_sub_i32 s3, s3, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s4, -1
+; GFX9-NEXT:    s_cselect_b32 s16, s4, -1
+; GFX9-NEXT:    s_sub_i32 s16, s16, s32
+; GFX9-NEXT:    s_cmp_lt_i32 s4, -1
+; GFX9-NEXT:    s_cselect_b32 s17, s4, -1
+; GFX9-NEXT:    s_sub_i32 s17, s17, s33
+; GFX9-NEXT:    s_cmp_gt_i32 s16, s20
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s20
+; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX9-NEXT:    s_sub_i32 s4, s4, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s5, -1
+; GFX9-NEXT:    s_cselect_b32 s16, s5, -1
+; GFX9-NEXT:    s_sub_i32 s16, s16, s32
+; GFX9-NEXT:    s_cmp_lt_i32 s5, -1
+; GFX9-NEXT:    s_cselect_b32 s17, s5, -1
+; GFX9-NEXT:    s_sub_i32 s17, s17, s33
+; GFX9-NEXT:    s_cmp_gt_i32 s16, s21
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s21
+; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX9-NEXT:    s_sub_i32 s5, s5, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s6, -1
+; GFX9-NEXT:    s_cselect_b32 s16, s6, -1
+; GFX9-NEXT:    s_sub_i32 s16, s16, s32
+; GFX9-NEXT:    s_cmp_lt_i32 s6, -1
+; GFX9-NEXT:    s_cselect_b32 s17, s6, -1
+; GFX9-NEXT:    s_sub_i32 s17, s17, s33
+; GFX9-NEXT:    s_cmp_gt_i32 s16, s22
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s22
+; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX9-NEXT:    s_sub_i32 s6, s6, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s7, -1
+; GFX9-NEXT:    s_cselect_b32 s16, s7, -1
+; GFX9-NEXT:    s_sub_i32 s16, s16, s32
+; GFX9-NEXT:    s_cmp_lt_i32 s7, -1
+; GFX9-NEXT:    s_cselect_b32 s17, s7, -1
+; GFX9-NEXT:    s_sub_i32 s17, s17, s33
+; GFX9-NEXT:    s_cmp_gt_i32 s16, s23
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s23
+; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX9-NEXT:    s_sub_i32 s7, s7, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s8, -1
+; GFX9-NEXT:    s_cselect_b32 s16, s8, -1
+; GFX9-NEXT:    s_sub_i32 s16, s16, s32
+; GFX9-NEXT:    s_cmp_lt_i32 s8, -1
+; GFX9-NEXT:    s_cselect_b32 s17, s8, -1
+; GFX9-NEXT:    s_sub_i32 s17, s17, s33
+; GFX9-NEXT:    s_cmp_gt_i32 s16, s24
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s24
+; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX9-NEXT:    s_sub_i32 s8, s8, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s9, -1
+; GFX9-NEXT:    s_cselect_b32 s16, s9, -1
+; GFX9-NEXT:    s_sub_i32 s16, s16, s32
+; GFX9-NEXT:    s_cmp_lt_i32 s9, -1
+; GFX9-NEXT:    s_cselect_b32 s17, s9, -1
+; GFX9-NEXT:    s_sub_i32 s17, s17, s33
+; GFX9-NEXT:    s_cmp_gt_i32 s16, s25
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s25
+; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX9-NEXT:    s_sub_i32 s9, s9, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s10, -1
+; GFX9-NEXT:    s_cselect_b32 s16, s10, -1
+; GFX9-NEXT:    s_sub_i32 s16, s16, s32
+; GFX9-NEXT:    s_cmp_lt_i32 s10, -1
+; GFX9-NEXT:    s_cselect_b32 s17, s10, -1
+; GFX9-NEXT:    s_sub_i32 s17, s17, s33
+; GFX9-NEXT:    s_cmp_gt_i32 s16, s26
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s26
+; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX9-NEXT:    s_sub_i32 s10, s10, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s11, -1
+; GFX9-NEXT:    s_cselect_b32 s16, s11, -1
+; GFX9-NEXT:    s_sub_i32 s16, s16, s32
+; GFX9-NEXT:    s_cmp_lt_i32 s11, -1
+; GFX9-NEXT:    s_cselect_b32 s17, s11, -1
+; GFX9-NEXT:    s_sub_i32 s17, s17, s33
+; GFX9-NEXT:    s_cmp_gt_i32 s16, s27
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s27
+; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX9-NEXT:    s_sub_i32 s11, s11, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s12, -1
+; GFX9-NEXT:    s_cselect_b32 s16, s12, -1
+; GFX9-NEXT:    s_sub_i32 s16, s16, s32
+; GFX9-NEXT:    s_cmp_lt_i32 s12, -1
+; GFX9-NEXT:    s_cselect_b32 s17, s12, -1
+; GFX9-NEXT:    s_sub_i32 s17, s17, s33
+; GFX9-NEXT:    s_cmp_gt_i32 s16, s28
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s28
+; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX9-NEXT:    s_sub_i32 s12, s12, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s13, -1
+; GFX9-NEXT:    s_cselect_b32 s16, s13, -1
+; GFX9-NEXT:    s_sub_i32 s16, s16, s32
+; GFX9-NEXT:    s_cmp_lt_i32 s13, -1
+; GFX9-NEXT:    s_cselect_b32 s17, s13, -1
+; GFX9-NEXT:    s_sub_i32 s17, s17, s33
+; GFX9-NEXT:    s_cmp_gt_i32 s16, s29
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s29
+; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX9-NEXT:    s_sub_i32 s13, s13, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s14, -1
+; GFX9-NEXT:    s_cselect_b32 s16, s14, -1
+; GFX9-NEXT:    s_sub_i32 s16, s16, s32
+; GFX9-NEXT:    s_cmp_lt_i32 s14, -1
+; GFX9-NEXT:    s_cselect_b32 s17, s14, -1
+; GFX9-NEXT:    s_sub_i32 s17, s17, s33
+; GFX9-NEXT:    s_cmp_gt_i32 s16, s30
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s30
+; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX9-NEXT:    s_sub_i32 s14, s14, s16
+; GFX9-NEXT:    s_cmp_gt_i32 s15, -1
+; GFX9-NEXT:    s_cselect_b32 s16, s15, -1
+; GFX9-NEXT:    s_sub_i32 s16, s16, s32
+; GFX9-NEXT:    s_cmp_lt_i32 s15, -1
+; GFX9-NEXT:    s_cselect_b32 s17, s15, -1
+; GFX9-NEXT:    s_sub_i32 s17, s17, s33
+; GFX9-NEXT:    s_cmp_gt_i32 s16, s31
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s31
+; GFX9-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX9-NEXT:    s_sub_i32 s15, s15, s16
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_ssubsat_v16i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX10-NEXT:    s_brev_b32 s46, -2
+; GFX10-NEXT:    s_cselect_b32 s33, s0, -1
+; GFX10-NEXT:    s_mov_b32 s34, 0x80000000
+; GFX10-NEXT:    s_sub_i32 s47, s33, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s35, s0, -1
+; GFX10-NEXT:    s_sub_i32 s35, s35, s34
+; GFX10-NEXT:    s_cmp_gt_i32 s47, s16
+; GFX10-NEXT:    s_cselect_b32 s16, s47, s16
+; GFX10-NEXT:    s_cmp_lt_i32 s16, s35
+; GFX10-NEXT:    s_cselect_b32 s47, s16, s35
+; GFX10-NEXT:    s_sub_i32 s0, s0, s47
+; GFX10-NEXT:    s_cmp_gt_i32 s1, -1
+; GFX10-NEXT:    s_cselect_b32 s16, s1, -1
+; GFX10-NEXT:    s_sub_i32 s16, s16, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s1, -1
+; GFX10-NEXT:    s_cselect_b32 s33, s1, -1
+; GFX10-NEXT:    s_sub_i32 s47, s33, s34
+; GFX10-NEXT:    s_cmp_gt_i32 s16, s17
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX10-NEXT:    s_cmp_lt_i32 s16, s47
+; GFX10-NEXT:    s_cselect_b32 s47, s16, s47
+; GFX10-NEXT:    s_sub_i32 s1, s1, s47
+; GFX10-NEXT:    s_cmp_gt_i32 s2, -1
+; GFX10-NEXT:    s_cselect_b32 s16, s2, -1
+; GFX10-NEXT:    s_sub_i32 s16, s16, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s2, -1
+; GFX10-NEXT:    s_cselect_b32 s17, s2, -1
+; GFX10-NEXT:    s_sub_i32 s17, s17, s34
+; GFX10-NEXT:    s_cmp_gt_i32 s16, s18
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s18
+; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX10-NEXT:    s_sub_i32 s2, s2, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s3, -1
+; GFX10-NEXT:    s_cselect_b32 s16, s3, -1
+; GFX10-NEXT:    s_sub_i32 s16, s16, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s3, -1
+; GFX10-NEXT:    s_cselect_b32 s17, s3, -1
+; GFX10-NEXT:    s_sub_i32 s17, s17, s34
+; GFX10-NEXT:    s_cmp_gt_i32 s16, s19
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s19
+; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX10-NEXT:    s_sub_i32 s3, s3, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s4, -1
+; GFX10-NEXT:    s_cselect_b32 s16, s4, -1
+; GFX10-NEXT:    s_sub_i32 s16, s16, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s4, -1
+; GFX10-NEXT:    s_cselect_b32 s17, s4, -1
+; GFX10-NEXT:    s_sub_i32 s17, s17, s34
+; GFX10-NEXT:    s_cmp_gt_i32 s16, s20
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s20
+; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX10-NEXT:    s_sub_i32 s4, s4, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s5, -1
+; GFX10-NEXT:    s_cselect_b32 s16, s5, -1
+; GFX10-NEXT:    s_sub_i32 s16, s16, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s5, -1
+; GFX10-NEXT:    s_cselect_b32 s17, s5, -1
+; GFX10-NEXT:    s_sub_i32 s17, s17, s34
+; GFX10-NEXT:    s_cmp_gt_i32 s16, s21
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s21
+; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX10-NEXT:    s_sub_i32 s5, s5, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s6, -1
+; GFX10-NEXT:    s_cselect_b32 s16, s6, -1
+; GFX10-NEXT:    s_sub_i32 s16, s16, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s6, -1
+; GFX10-NEXT:    s_cselect_b32 s17, s6, -1
+; GFX10-NEXT:    s_sub_i32 s17, s17, s34
+; GFX10-NEXT:    s_cmp_gt_i32 s16, s22
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s22
+; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX10-NEXT:    s_sub_i32 s6, s6, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s7, -1
+; GFX10-NEXT:    s_cselect_b32 s16, s7, -1
+; GFX10-NEXT:    s_sub_i32 s16, s16, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s7, -1
+; GFX10-NEXT:    s_cselect_b32 s17, s7, -1
+; GFX10-NEXT:    s_sub_i32 s17, s17, s34
+; GFX10-NEXT:    s_cmp_gt_i32 s16, s23
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s23
+; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX10-NEXT:    s_sub_i32 s7, s7, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s8, -1
+; GFX10-NEXT:    s_cselect_b32 s16, s8, -1
+; GFX10-NEXT:    s_sub_i32 s16, s16, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s8, -1
+; GFX10-NEXT:    s_cselect_b32 s17, s8, -1
+; GFX10-NEXT:    s_sub_i32 s17, s17, s34
+; GFX10-NEXT:    s_cmp_gt_i32 s16, s24
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s24
+; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX10-NEXT:    s_sub_i32 s8, s8, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s9, -1
+; GFX10-NEXT:    s_cselect_b32 s16, s9, -1
+; GFX10-NEXT:    s_sub_i32 s16, s16, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s9, -1
+; GFX10-NEXT:    s_cselect_b32 s17, s9, -1
+; GFX10-NEXT:    s_sub_i32 s17, s17, s34
+; GFX10-NEXT:    s_cmp_gt_i32 s16, s25
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s25
+; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX10-NEXT:    s_sub_i32 s9, s9, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s10, -1
+; GFX10-NEXT:    s_cselect_b32 s16, s10, -1
+; GFX10-NEXT:    s_sub_i32 s16, s16, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s10, -1
+; GFX10-NEXT:    s_cselect_b32 s17, s10, -1
+; GFX10-NEXT:    s_sub_i32 s17, s17, s34
+; GFX10-NEXT:    s_cmp_gt_i32 s16, s26
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s26
+; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX10-NEXT:    s_sub_i32 s10, s10, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s11, -1
+; GFX10-NEXT:    s_cselect_b32 s16, s11, -1
+; GFX10-NEXT:    s_sub_i32 s16, s16, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s11, -1
+; GFX10-NEXT:    s_cselect_b32 s17, s11, -1
+; GFX10-NEXT:    s_sub_i32 s17, s17, s34
+; GFX10-NEXT:    s_cmp_gt_i32 s16, s27
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s27
+; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX10-NEXT:    s_sub_i32 s11, s11, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s12, -1
+; GFX10-NEXT:    s_cselect_b32 s16, s12, -1
+; GFX10-NEXT:    s_sub_i32 s16, s16, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s12, -1
+; GFX10-NEXT:    s_cselect_b32 s17, s12, -1
+; GFX10-NEXT:    s_sub_i32 s17, s17, s34
+; GFX10-NEXT:    s_cmp_gt_i32 s16, s28
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s28
+; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX10-NEXT:    s_sub_i32 s12, s12, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s13, -1
+; GFX10-NEXT:    s_cselect_b32 s16, s13, -1
+; GFX10-NEXT:    s_sub_i32 s16, s16, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s13, -1
+; GFX10-NEXT:    s_cselect_b32 s17, s13, -1
+; GFX10-NEXT:    s_sub_i32 s17, s17, s34
+; GFX10-NEXT:    s_cmp_gt_i32 s16, s29
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s29
+; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX10-NEXT:    s_sub_i32 s13, s13, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s14, -1
+; GFX10-NEXT:    s_cselect_b32 s16, s14, -1
+; GFX10-NEXT:    s_sub_i32 s16, s16, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s14, -1
+; GFX10-NEXT:    s_cselect_b32 s17, s14, -1
+; GFX10-NEXT:    s_sub_i32 s17, s17, s34
+; GFX10-NEXT:    s_cmp_gt_i32 s16, s30
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s30
+; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX10-NEXT:    s_sub_i32 s14, s14, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s15, -1
+; GFX10-NEXT:    s_cselect_b32 s16, s15, -1
+; GFX10-NEXT:    s_sub_i32 s16, s16, s46
+; GFX10-NEXT:    s_cmp_lt_i32 s15, -1
+; GFX10-NEXT:    s_cselect_b32 s17, s15, -1
+; GFX10-NEXT:    s_sub_i32 s17, s17, s34
+; GFX10-NEXT:    s_cmp_gt_i32 s16, s31
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s31
+; GFX10-NEXT:    s_cmp_lt_i32 s16, s17
+; GFX10-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX10-NEXT:    s_sub_i32 s15, s15, s16
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
+  ret <16 x i32> %result
+}
+
+define i16 @v_ssubsat_i16(i16 %lhs, i16 %rhs) {
+; GFX6-LABEL: v_ssubsat_i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_max_i32_e32 v2, -1, v0
+; GFX6-NEXT:    v_min_i32_e32 v3, -1, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, 0x7fffffff, v2
+; GFX6-NEXT:    v_subrev_i32_e32 v3, vcc, 0x80000000, v3
+; GFX6-NEXT:    v_max_i32_e32 v1, v2, v1
+; GFX6-NEXT:    v_min_i32_e32 v1, v1, v3
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s4, 0xffff
+; GFX8-NEXT:    v_max_i16_e32 v2, s4, v0
+; GFX8-NEXT:    v_min_i16_e32 v3, s4, v0
+; GFX8-NEXT:    v_subrev_u16_e32 v2, 0x7fff, v2
+; GFX8-NEXT:    v_subrev_u16_e32 v3, 0x8000, v3
+; GFX8-NEXT:    v_max_i16_e32 v1, v2, v1
+; GFX8-NEXT:    v_min_i16_e32 v1, v1, v3
+; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s4, 0xffff
+; GFX9-NEXT:    v_max_i16_e32 v2, s4, v0
+; GFX9-NEXT:    v_min_i16_e32 v3, s4, v0
+; GFX9-NEXT:    v_subrev_u16_e32 v2, 0x7fff, v2
+; GFX9-NEXT:    v_subrev_u16_e32 v3, 0x8000, v3
+; GFX9-NEXT:    v_max_i16_e32 v1, v2, v1
+; GFX9-NEXT:    v_min_i16_e32 v1, v1, v3
+; GFX9-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_ssubsat_i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_mov_b32 s4, 0xffff
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_max_i16_e64 v2, v0, s4
+; GFX10-NEXT:    v_min_i16_e64 v3, v0, s4
+; GFX10-NEXT:    v_sub_nc_u16_e64 v2, v2, 0x7fff
+; GFX10-NEXT:    v_sub_nc_u16_e64 v3, v3, 0x8000
+; GFX10-NEXT:    v_max_i16_e64 v1, v2, v1
+; GFX10-NEXT:    v_min_i16_e64 v1, v1, v3
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i16 @llvm.ssub.sat.i16(i16 %lhs, i16 %rhs)
+  ret i16 %result
+}
+
+define amdgpu_ps i16 @s_ssubsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
+; GFX6-LABEL: s_ssubsat_i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX6-NEXT:    s_cselect_b32 s2, s0, -1
+; GFX6-NEXT:    s_sub_i32 s2, s2, 0x7fffffff
+; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX6-NEXT:    s_cselect_b32 s3, s0, -1
+; GFX6-NEXT:    s_sub_i32 s3, s3, 0x80000000
+; GFX6-NEXT:    s_cmp_gt_i32 s2, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX6-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX6-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX6-NEXT:    s_sub_i32 s0, s0, s1
+; GFX6-NEXT:    s_ashr_i32 s0, s0, 16
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_ssubsat_i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_sext_i32_i16 s2, s0
+; GFX8-NEXT:    s_sext_i32_i16 s3, 0xffff
+; GFX8-NEXT:    s_cmp_gt_i32 s2, s3
+; GFX8-NEXT:    s_cselect_b32 s4, s2, s3
+; GFX8-NEXT:    s_sub_i32 s4, s4, 0x7fff
+; GFX8-NEXT:    s_cmp_lt_i32 s2, s3
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX8-NEXT:    s_sub_i32 s2, s2, 0x8000
+; GFX8-NEXT:    s_sext_i32_i16 s3, s4
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_cmp_gt_i32 s3, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_sext_i32_i16 s2, s2
+; GFX8-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX8-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX8-NEXT:    s_sub_i32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_ssubsat_i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_sext_i32_i16 s2, s0
+; GFX9-NEXT:    s_sext_i32_i16 s3, 0xffff
+; GFX9-NEXT:    s_cmp_gt_i32 s2, s3
+; GFX9-NEXT:    s_cselect_b32 s4, s2, s3
+; GFX9-NEXT:    s_sub_i32 s4, s4, 0x7fff
+; GFX9-NEXT:    s_cmp_lt_i32 s2, s3
+; GFX9-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX9-NEXT:    s_sub_i32 s2, s2, 0x8000
+; GFX9-NEXT:    s_sext_i32_i16 s3, s4
+; GFX9-NEXT:    s_sext_i32_i16 s1, s1
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX9-NEXT:    s_sext_i32_i16 s1, s1
+; GFX9-NEXT:    s_sext_i32_i16 s2, s2
+; GFX9-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX9-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX9-NEXT:    s_sub_i32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_ssubsat_i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_sext_i32_i16 s2, 0xffff
+; GFX10-NEXT:    s_sext_i32_i16 s3, s0
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_cmp_gt_i32 s3, s2
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s4, s3, s2
+; GFX10-NEXT:    s_sub_i32 s4, s4, 0x7fff
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s2
+; GFX10-NEXT:    s_cselect_b32 s2, s3, s2
+; GFX10-NEXT:    s_sext_i32_i16 s3, s4
+; GFX10-NEXT:    s_sub_i32 s2, s2, 0x8000
+; GFX10-NEXT:    s_cmp_gt_i32 s3, s1
+; GFX10-NEXT:    s_sext_i32_i16 s2, s2
+; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX10-NEXT:    s_sext_i32_i16 s1, s1
+; GFX10-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX10-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX10-NEXT:    s_sub_i32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i16 @llvm.ssub.sat.i16(i16 %lhs, i16 %rhs)
+  ret i16 %result
+}
+
+define amdgpu_ps half @ssubsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
+; GFX6-LABEL: ssubsat_i16_sv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX6-NEXT:    s_cselect_b32 s1, s0, -1
+; GFX6-NEXT:    s_sub_i32 s1, s1, 0x7fffffff
+; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX6-NEXT:    s_cselect_b32 s2, s0, -1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    s_sub_i32 s2, s2, 0x80000000
+; GFX6-NEXT:    v_max_i32_e32 v0, s1, v0
+; GFX6-NEXT:    v_min_i32_e32 v0, s2, v0
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: ssubsat_i16_sv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_sext_i32_i16 s1, s0
+; GFX8-NEXT:    s_sext_i32_i16 s2, 0xffff
+; GFX8-NEXT:    s_cmp_gt_i32 s1, s2
+; GFX8-NEXT:    s_cselect_b32 s3, s1, s2
+; GFX8-NEXT:    s_sub_i32 s3, s3, 0x7fff
+; GFX8-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX8-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX8-NEXT:    s_sub_i32 s1, s1, 0x8000
+; GFX8-NEXT:    v_max_i16_e32 v0, s3, v0
+; GFX8-NEXT:    v_min_i16_e32 v0, s1, v0
+; GFX8-NEXT:    v_sub_u16_e32 v0, s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: ssubsat_i16_sv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_sext_i32_i16 s1, s0
+; GFX9-NEXT:    s_sext_i32_i16 s2, 0xffff
+; GFX9-NEXT:    s_cmp_gt_i32 s1, s2
+; GFX9-NEXT:    s_cselect_b32 s3, s1, s2
+; GFX9-NEXT:    s_sub_i32 s3, s3, 0x7fff
+; GFX9-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX9-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX9-NEXT:    s_sub_i32 s1, s1, 0x8000
+; GFX9-NEXT:    v_max_i16_e32 v0, s3, v0
+; GFX9-NEXT:    v_min_i16_e32 v0, s1, v0
+; GFX9-NEXT:    v_sub_u16_e32 v0, s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: ssubsat_i16_sv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_sext_i32_i16 s1, s0
+; GFX10-NEXT:    s_sext_i32_i16 s2, 0xffff
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cmp_gt_i32 s1, s2
+; GFX10-NEXT:    s_cselect_b32 s3, s1, s2
+; GFX10-NEXT:    s_sub_i32 s3, s3, 0x7fff
+; GFX10-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX10-NEXT:    v_max_i16_e64 v0, s3, v0
+; GFX10-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX10-NEXT:    s_sub_i32 s1, s1, 0x8000
+; GFX10-NEXT:    v_min_i16_e64 v0, v0, s1
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i16 @llvm.ssub.sat.i16(i16 %lhs, i16 %rhs)
+  %cast = bitcast i16 %result to half
+  ret half %cast
+}
+
+define amdgpu_ps half @ssubsat_i16_vs(i16 %lhs, i16 inreg %rhs) {
+; GFX6-LABEL: ssubsat_i16_vs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_max_i32_e32 v1, -1, v0
+; GFX6-NEXT:    v_min_i32_e32 v2, -1, v0
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    v_subrev_i32_e32 v1, vcc, 0x7fffffff, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, 0x80000000, v2
+; GFX6-NEXT:    v_max_i32_e32 v1, s0, v1
+; GFX6-NEXT:    v_min_i32_e32 v1, v1, v2
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: ssubsat_i16_vs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s1, 0xffff
+; GFX8-NEXT:    v_max_i16_e32 v1, s1, v0
+; GFX8-NEXT:    v_subrev_u16_e32 v1, 0x7fff, v1
+; GFX8-NEXT:    v_min_i16_e32 v2, s1, v0
+; GFX8-NEXT:    v_subrev_u16_e32 v2, 0x8000, v2
+; GFX8-NEXT:    v_max_i16_e32 v1, s0, v1
+; GFX8-NEXT:    v_min_i16_e32 v1, v1, v2
+; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: ssubsat_i16_vs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s1, 0xffff
+; GFX9-NEXT:    v_max_i16_e32 v1, s1, v0
+; GFX9-NEXT:    v_subrev_u16_e32 v1, 0x7fff, v1
+; GFX9-NEXT:    v_min_i16_e32 v2, s1, v0
+; GFX9-NEXT:    v_subrev_u16_e32 v2, 0x8000, v2
+; GFX9-NEXT:    v_max_i16_e32 v1, s0, v1
+; GFX9-NEXT:    v_min_i16_e32 v1, v1, v2
+; GFX9-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: ssubsat_i16_vs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s1, 0xffff
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_max_i16_e64 v1, v0, s1
+; GFX10-NEXT:    v_min_i16_e64 v2, v0, s1
+; GFX10-NEXT:    v_sub_nc_u16_e64 v1, v1, 0x7fff
+; GFX10-NEXT:    v_sub_nc_u16_e64 v2, v2, 0x8000
+; GFX10-NEXT:    v_max_i16_e64 v1, v1, s0
+; GFX10-NEXT:    v_min_i16_e64 v1, v1, v2
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i16 @llvm.ssub.sat.i16(i16 %lhs, i16 %rhs)
+  %cast = bitcast i16 %result to half
+  ret half %cast
+}
+
+define <2 x i16> @v_ssubsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
+; GFX6-LABEL: v_ssubsat_v2i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    v_max_i32_e32 v4, -1, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_subrev_i32_e32 v4, vcc, s4, v4
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v5, -1, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v5, vcc, s5, v5
+; GFX6-NEXT:    v_max_i32_e32 v2, v4, v2
+; GFX6-NEXT:    v_min_i32_e32 v2, v2, v5
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
+; GFX6-NEXT:    v_max_i32_e32 v3, -1, v1
+; GFX6-NEXT:    v_min_i32_e32 v4, -1, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v3, vcc, s4, v3
+; GFX6-NEXT:    v_subrev_i32_e32 v4, vcc, s5, v4
+; GFX6-NEXT:    v_max_i32_e32 v2, v3, v2
+; GFX6-NEXT:    v_min_i32_e32 v2, v2, v4
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_v2i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s6, 0xffff
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX8-NEXT:    v_max_i16_e32 v3, s6, v0
+; GFX8-NEXT:    v_subrev_u16_e32 v3, s4, v3
+; GFX8-NEXT:    s_mov_b32 s5, 0x8000
+; GFX8-NEXT:    v_min_i16_e32 v4, s6, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX8-NEXT:    v_subrev_u16_e32 v4, s5, v4
+; GFX8-NEXT:    v_max_i16_e32 v3, v3, v1
+; GFX8-NEXT:    v_min_i16_e32 v3, v3, v4
+; GFX8-NEXT:    v_max_i16_e32 v4, s6, v2
+; GFX8-NEXT:    v_min_i16_e32 v5, s6, v2
+; GFX8-NEXT:    v_subrev_u16_e32 v4, s4, v4
+; GFX8-NEXT:    v_subrev_u16_e32 v5, s5, v5
+; GFX8-NEXT:    v_max_i16_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_min_i16_e32 v1, v1, v5
+; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v3
+; GFX8-NEXT:    v_sub_u16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, -1, -1
+; GFX9-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
+; GFX9-NEXT:    v_pk_max_i16 v2, v0, s6
+; GFX9-NEXT:    v_pk_sub_i16 v2, v2, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
+; GFX9-NEXT:    v_pk_min_i16 v3, v0, s6
+; GFX9-NEXT:    v_pk_sub_i16 v3, v3, s5
+; GFX9-NEXT:    v_pk_max_i16 v1, v2, v1
+; GFX9-NEXT:    v_pk_min_i16 v1, v1, v3
+; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_ssubsat_v2i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
+; GFX10-NEXT:    s_movk_i32 s5, 0x7fff
+; GFX10-NEXT:    v_pk_max_i16 v2, v0, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
+; GFX10-NEXT:    v_pk_min_i16 v3, v0, s4
+; GFX10-NEXT:    s_mov_b32 s6, 0xffff8000
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_pk_sub_i16 v2, v2, s5
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s6, s6
+; GFX10-NEXT:    v_pk_sub_i16 v3, v3, s4
+; GFX10-NEXT:    v_pk_max_i16 v1, v2, v1
+; GFX10-NEXT:    v_pk_min_i16 v1, v1, v3
+; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
+  ret <2 x i16> %result
+}
+
+define amdgpu_ps i32 @s_ssubsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs) {
+; GFX6-LABEL: s_ssubsat_v2i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    s_cselect_b32 s6, s0, -1
+; GFX6-NEXT:    s_sub_i32 s6, s6, s4
+; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    s_cselect_b32 s7, s0, -1
+; GFX6-NEXT:    s_sub_i32 s7, s7, s5
+; GFX6-NEXT:    s_cmp_gt_i32 s6, s2
+; GFX6-NEXT:    s_cselect_b32 s2, s6, s2
+; GFX6-NEXT:    s_cmp_lt_i32 s2, s7
+; GFX6-NEXT:    s_cselect_b32 s2, s2, s7
+; GFX6-NEXT:    s_sub_i32 s0, s0, s2
+; GFX6-NEXT:    s_ashr_i32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s2, s3, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s1, -1
+; GFX6-NEXT:    s_cselect_b32 s3, s1, -1
+; GFX6-NEXT:    s_sub_i32 s3, s3, s4
+; GFX6-NEXT:    s_cmp_lt_i32 s1, -1
+; GFX6-NEXT:    s_cselect_b32 s4, s1, -1
+; GFX6-NEXT:    s_sub_i32 s4, s4, s5
+; GFX6-NEXT:    s_cmp_gt_i32 s3, s2
+; GFX6-NEXT:    s_cselect_b32 s2, s3, s2
+; GFX6-NEXT:    s_cmp_lt_i32 s2, s4
+; GFX6-NEXT:    s_cselect_b32 s2, s2, s4
+; GFX6-NEXT:    s_sub_i32 s1, s1, s2
+; GFX6-NEXT:    s_mov_b32 s2, 0xffff
+; GFX6-NEXT:    s_ashr_i32 s1, s1, 16
+; GFX6-NEXT:    s_and_b32 s1, s1, s2
+; GFX6-NEXT:    s_and_b32 s0, s0, s2
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_ssubsat_v2i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX8-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX8-NEXT:    s_sext_i32_i16 s6, s0
+; GFX8-NEXT:    s_sext_i32_i16 s7, 0xffff
+; GFX8-NEXT:    s_cmp_gt_i32 s6, s7
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX8-NEXT:    s_cselect_b32 s8, s6, s7
+; GFX8-NEXT:    s_sub_i32 s8, s8, s4
+; GFX8-NEXT:    s_cmp_lt_i32 s6, s7
+; GFX8-NEXT:    s_mov_b32 s5, 0x8000
+; GFX8-NEXT:    s_cselect_b32 s6, s6, s7
+; GFX8-NEXT:    s_sub_i32 s6, s6, s5
+; GFX8-NEXT:    s_sext_i32_i16 s8, s8
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_cmp_gt_i32 s8, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s8, s1
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_sext_i32_i16 s6, s6
+; GFX8-NEXT:    s_cmp_lt_i32 s1, s6
+; GFX8-NEXT:    s_cselect_b32 s1, s1, s6
+; GFX8-NEXT:    s_sub_i32 s0, s0, s1
+; GFX8-NEXT:    s_sext_i32_i16 s1, s2
+; GFX8-NEXT:    s_cmp_gt_i32 s1, s7
+; GFX8-NEXT:    s_cselect_b32 s6, s1, s7
+; GFX8-NEXT:    s_sub_i32 s4, s6, s4
+; GFX8-NEXT:    s_cmp_lt_i32 s1, s7
+; GFX8-NEXT:    s_cselect_b32 s1, s1, s7
+; GFX8-NEXT:    s_sub_i32 s1, s1, s5
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s3
+; GFX8-NEXT:    s_cselect_b32 s3, s4, s3
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_sext_i32_i16 s1, s1
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX8-NEXT:    s_sub_i32 s1, s2, s1
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_ssubsat_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
+; GFX9-NEXT:    s_sext_i32_i16 s7, s4
+; GFX9-NEXT:    s_sext_i32_i16 s5, s0
+; GFX9-NEXT:    s_ashr_i32 s6, s0, 16
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s7
+; GFX9-NEXT:    s_cselect_b32 s8, s5, s7
+; GFX9-NEXT:    s_cmp_gt_i32 s6, s4
+; GFX9-NEXT:    s_movk_i32 s2, 0x7fff
+; GFX9-NEXT:    s_cselect_b32 s9, s6, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s8, s8, s9
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s2
+; GFX9-NEXT:    s_lshr_b32 s10, s2, 16
+; GFX9-NEXT:    s_lshr_b32 s9, s8, 16
+; GFX9-NEXT:    s_sub_i32 s2, s8, s2
+; GFX9-NEXT:    s_sub_i32 s8, s9, s10
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s7
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s7
+; GFX9-NEXT:    s_cmp_lt_i32 s6, s4
+; GFX9-NEXT:    s_mov_b32 s3, 0xffff8000
+; GFX9-NEXT:    s_cselect_b32 s4, s6, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s5, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s3
+; GFX9-NEXT:    s_lshr_b32 s6, s3, 16
+; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
+; GFX9-NEXT:    s_sub_i32 s3, s4, s3
+; GFX9-NEXT:    s_sub_i32 s4, s5, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
+; GFX9-NEXT:    s_sext_i32_i16 s4, s2
+; GFX9-NEXT:    s_sext_i32_i16 s5, s1
+; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX9-NEXT:    s_ashr_i32 s1, s1, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s5
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX9-NEXT:    s_cmp_gt_i32 s2, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s4, s1
+; GFX9-NEXT:    s_sext_i32_i16 s2, s1
+; GFX9-NEXT:    s_sext_i32_i16 s4, s3
+; GFX9-NEXT:    s_ashr_i32 s1, s1, 16
+; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s2, s4
+; GFX9-NEXT:    s_cselect_b32 s2, s2, s4
+; GFX9-NEXT:    s_cmp_lt_i32 s1, s3
+; GFX9-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s2, s1
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX9-NEXT:    s_sub_i32 s0, s0, s1
+; GFX9-NEXT:    s_sub_i32 s1, s2, s3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_ssubsat_v2i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, -1, -1
+; GFX10-NEXT:    s_sext_i32_i16 s3, s0
+; GFX10-NEXT:    s_sext_i32_i16 s5, s2
+; GFX10-NEXT:    s_ashr_i32 s4, s0, 16
+; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s3, s5
+; GFX10-NEXT:    s_movk_i32 s8, 0x7fff
+; GFX10-NEXT:    s_cselect_b32 s6, s3, s5
+; GFX10-NEXT:    s_cmp_gt_i32 s4, s2
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s7, s4, s2
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s7
+; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s8, s8
+; GFX10-NEXT:    s_lshr_b32 s8, s6, 16
+; GFX10-NEXT:    s_lshr_b32 s9, s7, 16
+; GFX10-NEXT:    s_sub_i32 s6, s6, s7
+; GFX10-NEXT:    s_sub_i32 s7, s8, s9
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s5
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s5
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s2
+; GFX10-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX10-NEXT:    s_cselect_b32 s2, s4, s2
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s5, s5
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s3, s2
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s6, s7
+; GFX10-NEXT:    s_lshr_b32 s5, s2, 16
+; GFX10-NEXT:    s_lshr_b32 s6, s4, 16
+; GFX10-NEXT:    s_sub_i32 s2, s2, s4
+; GFX10-NEXT:    s_sub_i32 s4, s5, s6
+; GFX10-NEXT:    s_sext_i32_i16 s5, s3
+; GFX10-NEXT:    s_sext_i32_i16 s6, s1
+; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX10-NEXT:    s_ashr_i32 s1, s1, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s5, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX10-NEXT:    s_cmp_gt_i32 s3, s1
+; GFX10-NEXT:    s_sext_i32_i16 s4, s2
+; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s5, s1
+; GFX10-NEXT:    s_sext_i32_i16 s3, s1
+; GFX10-NEXT:    s_ashr_i32 s1, s1, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX10-NEXT:    s_cmp_lt_i32 s1, s2
+; GFX10-NEXT:    s_cselect_b32 s1, s1, s2
+; GFX10-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s3, s1
+; GFX10-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX10-NEXT:    s_sub_i32 s0, s0, s1
+; GFX10-NEXT:    s_sub_i32 s1, s2, s3
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
+  %cast = bitcast <2 x i16> %result to i32
+  ret i32 %cast
+}
+
+define amdgpu_ps float @ssubsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
+; GFX6-LABEL: ssubsat_v2i16_sv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX6-NEXT:    s_brev_b32 s2, -2
+; GFX6-NEXT:    s_cselect_b32 s4, s0, -1
+; GFX6-NEXT:    s_sub_i32 s4, s4, s2
+; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    s_mov_b32 s3, 0x80000000
+; GFX6-NEXT:    s_cselect_b32 s5, s0, -1
+; GFX6-NEXT:    s_sub_i32 s5, s5, s3
+; GFX6-NEXT:    v_max_i32_e32 v0, s4, v0
+; GFX6-NEXT:    v_min_i32_e32 v0, s5, v0
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
+; GFX6-NEXT:    s_lshl_b32 s0, s1, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX6-NEXT:    s_cselect_b32 s1, s0, -1
+; GFX6-NEXT:    s_sub_i32 s1, s1, s2
+; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX6-NEXT:    s_cselect_b32 s2, s0, -1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    s_sub_i32 s2, s2, s3
+; GFX6-NEXT:    v_max_i32_e32 v1, s1, v1
+; GFX6-NEXT:    v_min_i32_e32 v1, s2, v1
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, s0, v1
+; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
+; GFX6-NEXT:    s_mov_b32 s0, 0xffff
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, s0, v1
+; GFX6-NEXT:    v_and_b32_e32 v0, s0, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: ssubsat_v2i16_sv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_lshr_b32 s1, s0, 16
+; GFX8-NEXT:    s_sext_i32_i16 s4, s0
+; GFX8-NEXT:    s_sext_i32_i16 s5, 0xffff
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s5
+; GFX8-NEXT:    s_movk_i32 s2, 0x7fff
+; GFX8-NEXT:    s_cselect_b32 s6, s4, s5
+; GFX8-NEXT:    s_sub_i32 s6, s6, s2
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX8-NEXT:    s_mov_b32 s3, 0x8000
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX8-NEXT:    s_sub_i32 s4, s4, s3
+; GFX8-NEXT:    v_max_i16_e32 v1, s6, v0
+; GFX8-NEXT:    v_min_i16_e32 v1, s4, v1
+; GFX8-NEXT:    s_sext_i32_i16 s4, s1
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s5
+; GFX8-NEXT:    s_cselect_b32 s6, s4, s5
+; GFX8-NEXT:    s_sub_i32 s2, s6, s2
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX8-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-NEXT:    v_max_i16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    s_sub_i32 s3, s4, s3
+; GFX8-NEXT:    v_min_i16_e32 v0, s3, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    v_sub_u16_e32 v1, s0, v1
+; GFX8-NEXT:    v_sub_u16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: ssubsat_v2i16_sv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, -1, -1
+; GFX9-NEXT:    s_sext_i32_i16 s6, s3
+; GFX9-NEXT:    s_sext_i32_i16 s4, s0
+; GFX9-NEXT:    s_ashr_i32 s5, s0, 16
+; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s6
+; GFX9-NEXT:    s_cselect_b32 s7, s4, s6
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s3
+; GFX9-NEXT:    s_movk_i32 s1, 0x7fff
+; GFX9-NEXT:    s_cselect_b32 s8, s5, s3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s7, s7, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s1
+; GFX9-NEXT:    s_lshr_b32 s9, s1, 16
+; GFX9-NEXT:    s_lshr_b32 s8, s7, 16
+; GFX9-NEXT:    s_sub_i32 s1, s7, s1
+; GFX9-NEXT:    s_sub_i32 s7, s8, s9
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s6
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s3
+; GFX9-NEXT:    s_mov_b32 s2, 0xffff8000
+; GFX9-NEXT:    s_cselect_b32 s3, s5, s3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s4, s3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s2
+; GFX9-NEXT:    s_lshr_b32 s5, s2, 16
+; GFX9-NEXT:    s_lshr_b32 s4, s3, 16
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s7
+; GFX9-NEXT:    s_sub_i32 s2, s3, s2
+; GFX9-NEXT:    s_sub_i32 s3, s4, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
+; GFX9-NEXT:    v_pk_max_i16 v0, s1, v0
+; GFX9-NEXT:    v_pk_min_i16 v0, v0, s2
+; GFX9-NEXT:    v_pk_sub_i16 v0, s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: ssubsat_v2i16_sv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, -1, -1
+; GFX10-NEXT:    s_sext_i32_i16 s2, s0
+; GFX10-NEXT:    s_sext_i32_i16 s4, s1
+; GFX10-NEXT:    s_ashr_i32 s3, s0, 16
+; GFX10-NEXT:    s_ashr_i32 s1, s1, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s2, s4
+; GFX10-NEXT:    s_movk_i32 s7, 0x7fff
+; GFX10-NEXT:    s_cselect_b32 s5, s2, s4
+; GFX10-NEXT:    s_cmp_gt_i32 s3, s1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s6, s3, s1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s7, s7
+; GFX10-NEXT:    s_lshr_b32 s7, s5, 16
+; GFX10-NEXT:    s_lshr_b32 s8, s6, 16
+; GFX10-NEXT:    s_sub_i32 s5, s5, s6
+; GFX10-NEXT:    s_sub_i32 s6, s7, s8
+; GFX10-NEXT:    s_cmp_lt_i32 s2, s4
+; GFX10-NEXT:    s_cselect_b32 s2, s2, s4
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s1
+; GFX10-NEXT:    s_mov_b32 s4, 0xffff8000
+; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s4, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s2, s1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s5, s6
+; GFX10-NEXT:    s_lshr_b32 s4, s1, 16
+; GFX10-NEXT:    s_lshr_b32 s5, s3, 16
+; GFX10-NEXT:    v_pk_max_i16 v0, s2, v0
+; GFX10-NEXT:    s_sub_i32 s1, s1, s3
+; GFX10-NEXT:    s_sub_i32 s2, s4, s5
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
+; GFX10-NEXT:    v_pk_min_i16 v0, v0, s1
+; GFX10-NEXT:    v_pk_sub_i16 v0, s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
+  %cast = bitcast <2 x i16> %result to float
+  ret float %cast
+}
+
+define amdgpu_ps float @ssubsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
+; GFX6-LABEL: ssubsat_v2i16_vs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    s_brev_b32 s2, -2
+; GFX6-NEXT:    v_max_i32_e32 v2, -1, v0
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, s2, v2
+; GFX6-NEXT:    s_mov_b32 s3, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v3, -1, v0
+; GFX6-NEXT:    v_max_i32_e32 v2, s0, v2
+; GFX6-NEXT:    v_subrev_i32_e32 v3, vcc, s3, v3
+; GFX6-NEXT:    v_min_i32_e32 v2, v2, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT:    v_max_i32_e32 v2, -1, v1
+; GFX6-NEXT:    v_min_i32_e32 v3, -1, v1
+; GFX6-NEXT:    s_lshl_b32 s0, s1, 16
+; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, s2, v2
+; GFX6-NEXT:    v_max_i32_e32 v2, s0, v2
+; GFX6-NEXT:    v_subrev_i32_e32 v3, vcc, s3, v3
+; GFX6-NEXT:    v_min_i32_e32 v2, v2, v3
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
+; GFX6-NEXT:    s_mov_b32 s0, 0xffff
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, s0, v1
+; GFX6-NEXT:    v_and_b32_e32 v0, s0, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: ssubsat_v2i16_vs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s4, 0xffff
+; GFX8-NEXT:    s_movk_i32 s2, 0x7fff
+; GFX8-NEXT:    v_max_i16_e32 v2, s4, v0
+; GFX8-NEXT:    v_subrev_u16_e32 v2, s2, v2
+; GFX8-NEXT:    s_mov_b32 s3, 0x8000
+; GFX8-NEXT:    v_min_i16_e32 v3, s4, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; GFX8-NEXT:    v_subrev_u16_e32 v3, s3, v3
+; GFX8-NEXT:    v_max_i16_e32 v2, s0, v2
+; GFX8-NEXT:    v_min_i16_e32 v2, v2, v3
+; GFX8-NEXT:    v_max_i16_e32 v3, s4, v1
+; GFX8-NEXT:    v_min_i16_e32 v4, s4, v1
+; GFX8-NEXT:    s_lshr_b32 s1, s0, 16
+; GFX8-NEXT:    v_subrev_u16_e32 v3, s2, v3
+; GFX8-NEXT:    v_subrev_u16_e32 v4, s3, v4
+; GFX8-NEXT:    v_max_i16_e32 v3, s1, v3
+; GFX8-NEXT:    v_min_i16_e32 v3, v3, v4
+; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v2
+; GFX8-NEXT:    v_sub_u16_sdwa v1, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: ssubsat_v2i16_vs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_movk_i32 s1, 0x7fff
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, -1, -1
+; GFX9-NEXT:    s_mov_b32 s2, 0xffff8000
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s1
+; GFX9-NEXT:    v_pk_max_i16 v1, v0, s3
+; GFX9-NEXT:    v_pk_sub_i16 v1, v1, s1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s2
+; GFX9-NEXT:    v_pk_min_i16 v2, v0, s3
+; GFX9-NEXT:    v_pk_sub_i16 v2, v2, s2
+; GFX9-NEXT:    v_pk_max_i16 v1, v1, s0
+; GFX9-NEXT:    v_pk_min_i16 v1, v1, v2
+; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: ssubsat_v2i16_vs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, -1, -1
+; GFX10-NEXT:    s_movk_i32 s2, 0x7fff
+; GFX10-NEXT:    v_pk_max_i16 v1, v0, s1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s2
+; GFX10-NEXT:    v_pk_min_i16 v2, v0, s1
+; GFX10-NEXT:    s_mov_b32 s3, 0xffff8000
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_pk_sub_i16 v1, v1, s2
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s3, s3
+; GFX10-NEXT:    v_pk_sub_i16 v2, v2, s1
+; GFX10-NEXT:    v_pk_max_i16 v1, v1, s0
+; GFX10-NEXT:    v_pk_min_i16 v1, v1, v2
+; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
+  %cast = bitcast <2 x i16> %result to float
+  ret float %cast
+}
+
+; FIXME: v3i16 insert/extract
+; define <3 x i16> @v_ssubsat_v3i16(<3 x i16> %lhs, <3 x i16> %rhs) {
+;   %result = call <3 x i16> @llvm.ssub.sat.v3i16(<3 x i16> %lhs, <3 x i16> %rhs)
+;   ret <3 x i16> %result
+; }
+
+; define amdgpu_ps <3 x i16> @s_ssubsat_v3i16(<3 x i16> inreg %lhs, <3 x i16> inreg %rhs) {
+;   %result = call <3 x i16> @llvm.ssub.sat.v3i16(<3 x i16> %lhs, <3 x i16> %rhs)
+;   ret <3 x i16> %result
+; }
+
+define <2 x float> @v_ssubsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
+; GFX6-LABEL: v_ssubsat_v4i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    v_max_i32_e32 v8, -1, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT:    v_subrev_i32_e32 v8, vcc, s4, v8
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v10, -1, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v10, vcc, s5, v10
+; GFX6-NEXT:    v_max_i32_e32 v4, v8, v4
+; GFX6-NEXT:    v_min_i32_e32 v4, v4, v10
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
+; GFX6-NEXT:    v_max_i32_e32 v5, -1, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v5, vcc, s4, v5
+; GFX6-NEXT:    v_min_i32_e32 v8, -1, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v8, vcc, s5, v8
+; GFX6-NEXT:    v_max_i32_e32 v4, v5, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_min_i32_e32 v4, v4, v8
+; GFX6-NEXT:    v_bfrev_b32_e32 v9, -2
+; GFX6-NEXT:    v_max_i32_e32 v5, -1, v2
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v6
+; GFX6-NEXT:    v_min_i32_e32 v6, -1, v2
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, v5, v9
+; GFX6-NEXT:    v_subrev_i32_e32 v6, vcc, s5, v6
+; GFX6-NEXT:    v_max_i32_e32 v4, v5, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_min_i32_e32 v4, v4, v6
+; GFX6-NEXT:    v_max_i32_e32 v5, -1, v3
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v4
+; GFX6-NEXT:    v_mov_b32_e32 v11, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v6, -1, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v7
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, v5, v9
+; GFX6-NEXT:    v_sub_i32_e32 v6, vcc, v6, v11
+; GFX6-NEXT:    v_max_i32_e32 v4, v5, v4
+; GFX6-NEXT:    v_min_i32_e32 v4, v4, v6
+; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, v3, v4
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
+; GFX6-NEXT:    v_ashrrev_i32_e32 v3, 16, v3
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_v4i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s6, 0xffff
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX8-NEXT:    v_max_i16_e32 v6, s6, v0
+; GFX8-NEXT:    v_subrev_u16_e32 v6, s4, v6
+; GFX8-NEXT:    s_mov_b32 s5, 0x8000
+; GFX8-NEXT:    v_min_i16_e32 v7, s6, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
+; GFX8-NEXT:    v_subrev_u16_e32 v7, s5, v7
+; GFX8-NEXT:    v_max_i16_e32 v6, v6, v2
+; GFX8-NEXT:    v_min_i16_e32 v6, v6, v7
+; GFX8-NEXT:    v_max_i16_e32 v7, s6, v4
+; GFX8-NEXT:    v_min_i16_e32 v8, s6, v4
+; GFX8-NEXT:    v_subrev_u16_e32 v7, s4, v7
+; GFX8-NEXT:    v_max_i16_sdwa v2, v7, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_max_i16_e32 v7, s6, v1
+; GFX8-NEXT:    v_subrev_u16_e32 v8, s5, v8
+; GFX8-NEXT:    v_min_i16_e32 v2, v2, v8
+; GFX8-NEXT:    v_subrev_u16_e32 v7, s4, v7
+; GFX8-NEXT:    v_min_i16_e32 v8, s6, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
+; GFX8-NEXT:    v_subrev_u16_e32 v8, s5, v8
+; GFX8-NEXT:    v_max_i16_e32 v7, v7, v3
+; GFX8-NEXT:    v_min_i16_e32 v7, v7, v8
+; GFX8-NEXT:    v_max_i16_e32 v8, s6, v5
+; GFX8-NEXT:    v_min_i16_e32 v9, -1, v5
+; GFX8-NEXT:    v_subrev_u16_e32 v8, s4, v8
+; GFX8-NEXT:    v_subrev_u16_e32 v9, s5, v9
+; GFX8-NEXT:    v_max_i16_sdwa v3, v8, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_min_i16_e32 v3, v3, v9
+; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v6
+; GFX8-NEXT:    v_sub_u16_sdwa v2, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT:    v_sub_u16_e32 v1, v1, v7
+; GFX8-NEXT:    v_sub_u16_sdwa v2, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_v4i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, -1, -1
+; GFX9-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
+; GFX9-NEXT:    v_pk_max_i16 v4, v0, s6
+; GFX9-NEXT:    v_pk_sub_i16 v4, v4, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
+; GFX9-NEXT:    v_pk_min_i16 v5, v0, s6
+; GFX9-NEXT:    v_pk_max_i16 v2, v4, v2
+; GFX9-NEXT:    v_pk_sub_i16 v5, v5, s5
+; GFX9-NEXT:    v_pk_min_i16 v2, v2, v5
+; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v2
+; GFX9-NEXT:    v_pk_max_i16 v2, v1, s6
+; GFX9-NEXT:    v_pk_sub_i16 v2, v2, s4
+; GFX9-NEXT:    v_pk_min_i16 v4, v1, s6
+; GFX9-NEXT:    v_pk_sub_i16 v4, v4, s5
+; GFX9-NEXT:    v_pk_max_i16 v2, v2, v3
+; GFX9-NEXT:    v_pk_min_i16 v2, v2, v4
+; GFX9-NEXT:    v_pk_sub_i16 v1, v1, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_ssubsat_v4i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, -1, -1
+; GFX10-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX10-NEXT:    v_pk_max_i16 v4, v0, s5
+; GFX10-NEXT:    v_pk_max_i16 v5, v1, s5
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
+; GFX10-NEXT:    v_pk_min_i16 v6, v0, s5
+; GFX10-NEXT:    v_pk_min_i16 v7, v1, s5
+; GFX10-NEXT:    v_pk_sub_i16 v4, v4, s4
+; GFX10-NEXT:    v_pk_sub_i16 v5, v5, s4
+; GFX10-NEXT:    s_mov_b32 s6, 0xffff8000
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s6
+; GFX10-NEXT:    v_pk_max_i16 v11, v4, v2
+; GFX10-NEXT:    v_pk_sub_i16 v6, v6, s6
+; GFX10-NEXT:    v_pk_sub_i16 v4, v7, s6
+; GFX10-NEXT:    v_pk_max_i16 v3, v5, v3
+; GFX10-NEXT:    v_pk_min_i16 v2, v11, v6
+; GFX10-NEXT:    v_pk_min_i16 v3, v3, v4
+; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v2
+; GFX10-NEXT:    v_pk_sub_i16 v1, v1, v3
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> %lhs, <4 x i16> %rhs)
+  %cast = bitcast <4 x i16> %result to <2 x float>
+  ret <2 x float> %cast
+}
+
+define amdgpu_ps <2 x i32> @s_ssubsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inreg %rhs) {
+; GFX6-LABEL: s_ssubsat_v4i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX6-NEXT:    s_brev_b32 s8, -2
+; GFX6-NEXT:    s_cselect_b32 s10, s0, -1
+; GFX6-NEXT:    s_sub_i32 s10, s10, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX6-NEXT:    s_mov_b32 s9, 0x80000000
+; GFX6-NEXT:    s_cselect_b32 s11, s0, -1
+; GFX6-NEXT:    s_sub_i32 s11, s11, s9
+; GFX6-NEXT:    s_cmp_gt_i32 s10, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s10, s4
+; GFX6-NEXT:    s_cmp_lt_i32 s4, s11
+; GFX6-NEXT:    s_cselect_b32 s4, s4, s11
+; GFX6-NEXT:    s_sub_i32 s0, s0, s4
+; GFX6-NEXT:    s_ashr_i32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s4, s5, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s1, -1
+; GFX6-NEXT:    s_cselect_b32 s5, s1, -1
+; GFX6-NEXT:    s_sub_i32 s5, s5, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s1, -1
+; GFX6-NEXT:    s_cselect_b32 s10, s1, -1
+; GFX6-NEXT:    s_sub_i32 s10, s10, s9
+; GFX6-NEXT:    s_cmp_gt_i32 s5, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX6-NEXT:    s_cmp_lt_i32 s4, s10
+; GFX6-NEXT:    s_cselect_b32 s4, s4, s10
+; GFX6-NEXT:    s_sub_i32 s1, s1, s4
+; GFX6-NEXT:    s_ashr_i32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_lshl_b32 s4, s6, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s2, -1
+; GFX6-NEXT:    s_cselect_b32 s5, s2, -1
+; GFX6-NEXT:    s_sub_i32 s5, s5, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s2, -1
+; GFX6-NEXT:    s_cselect_b32 s6, s2, -1
+; GFX6-NEXT:    s_sub_i32 s6, s6, s9
+; GFX6-NEXT:    s_cmp_gt_i32 s5, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX6-NEXT:    s_cmp_lt_i32 s4, s6
+; GFX6-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX6-NEXT:    s_sub_i32 s2, s2, s4
+; GFX6-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX6-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX6-NEXT:    s_lshl_b32 s4, s7, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s3, -1
+; GFX6-NEXT:    s_cselect_b32 s5, s3, -1
+; GFX6-NEXT:    s_sub_i32 s5, s5, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s3, -1
+; GFX6-NEXT:    s_cselect_b32 s6, s3, -1
+; GFX6-NEXT:    s_sub_i32 s6, s6, s9
+; GFX6-NEXT:    s_cmp_gt_i32 s5, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX6-NEXT:    s_cmp_lt_i32 s4, s6
+; GFX6-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX6-NEXT:    s_sub_i32 s3, s3, s4
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    s_and_b32 s1, s1, s4
+; GFX6-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX6-NEXT:    s_and_b32 s0, s0, s4
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_and_b32 s1, s2, s4
+; GFX6-NEXT:    s_and_b32 s2, s3, s4
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_or_b32 s1, s1, s2
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_ssubsat_v4i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_lshr_b32 s6, s2, 16
+; GFX8-NEXT:    s_lshr_b32 s7, s3, 16
+; GFX8-NEXT:    s_lshr_b32 s4, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s5, s1, 16
+; GFX8-NEXT:    s_sext_i32_i16 s10, s0
+; GFX8-NEXT:    s_sext_i32_i16 s11, 0xffff
+; GFX8-NEXT:    s_cmp_gt_i32 s10, s11
+; GFX8-NEXT:    s_movk_i32 s8, 0x7fff
+; GFX8-NEXT:    s_cselect_b32 s12, s10, s11
+; GFX8-NEXT:    s_sub_i32 s12, s12, s8
+; GFX8-NEXT:    s_cmp_lt_i32 s10, s11
+; GFX8-NEXT:    s_mov_b32 s9, 0x8000
+; GFX8-NEXT:    s_cselect_b32 s10, s10, s11
+; GFX8-NEXT:    s_sub_i32 s10, s10, s9
+; GFX8-NEXT:    s_sext_i32_i16 s12, s12
+; GFX8-NEXT:    s_sext_i32_i16 s2, s2
+; GFX8-NEXT:    s_cmp_gt_i32 s12, s2
+; GFX8-NEXT:    s_cselect_b32 s2, s12, s2
+; GFX8-NEXT:    s_sext_i32_i16 s2, s2
+; GFX8-NEXT:    s_sext_i32_i16 s10, s10
+; GFX8-NEXT:    s_cmp_lt_i32 s2, s10
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s10
+; GFX8-NEXT:    s_sub_i32 s0, s0, s2
+; GFX8-NEXT:    s_sext_i32_i16 s2, s4
+; GFX8-NEXT:    s_cmp_gt_i32 s2, s11
+; GFX8-NEXT:    s_cselect_b32 s10, s2, s11
+; GFX8-NEXT:    s_sub_i32 s10, s10, s8
+; GFX8-NEXT:    s_cmp_lt_i32 s2, s11
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s11
+; GFX8-NEXT:    s_sub_i32 s2, s2, s9
+; GFX8-NEXT:    s_sext_i32_i16 s10, s10
+; GFX8-NEXT:    s_sext_i32_i16 s6, s6
+; GFX8-NEXT:    s_cmp_gt_i32 s10, s6
+; GFX8-NEXT:    s_cselect_b32 s6, s10, s6
+; GFX8-NEXT:    s_sext_i32_i16 s6, s6
+; GFX8-NEXT:    s_sext_i32_i16 s2, s2
+; GFX8-NEXT:    s_cmp_lt_i32 s6, s2
+; GFX8-NEXT:    s_cselect_b32 s2, s6, s2
+; GFX8-NEXT:    s_sub_i32 s2, s4, s2
+; GFX8-NEXT:    s_sext_i32_i16 s4, s1
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s11
+; GFX8-NEXT:    s_cselect_b32 s6, s4, s11
+; GFX8-NEXT:    s_sub_i32 s6, s6, s8
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s11
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s11
+; GFX8-NEXT:    s_sub_i32 s4, s4, s9
+; GFX8-NEXT:    s_sext_i32_i16 s6, s6
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_cmp_gt_i32 s6, s3
+; GFX8-NEXT:    s_cselect_b32 s3, s6, s3
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX8-NEXT:    s_sub_i32 s1, s1, s3
+; GFX8-NEXT:    s_sext_i32_i16 s3, s5
+; GFX8-NEXT:    s_cmp_gt_i32 s3, s11
+; GFX8-NEXT:    s_cselect_b32 s4, s3, s11
+; GFX8-NEXT:    s_sub_i32 s4, s4, s8
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s11
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s11
+; GFX8-NEXT:    s_sub_i32 s3, s3, s9
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_sext_i32_i16 s6, s7
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s6
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s3
+; GFX8-NEXT:    s_cselect_b32 s3, s4, s3
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_sub_i32 s3, s5, s3
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX8-NEXT:    s_or_b32 s0, s0, s2
+; GFX8-NEXT:    s_bfe_u32 s2, s3, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX8-NEXT:    s_or_b32 s1, s1, s2
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_ssubsat_v4i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, -1, -1
+; GFX9-NEXT:    s_sext_i32_i16 s9, s6
+; GFX9-NEXT:    s_sext_i32_i16 s7, s0
+; GFX9-NEXT:    s_ashr_i32 s8, s0, 16
+; GFX9-NEXT:    s_ashr_i32 s6, s6, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s7, s9
+; GFX9-NEXT:    s_cselect_b32 s10, s7, s9
+; GFX9-NEXT:    s_cmp_gt_i32 s8, s6
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    s_cselect_b32 s11, s8, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s11
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
+; GFX9-NEXT:    s_lshr_b32 s11, s10, 16
+; GFX9-NEXT:    s_lshr_b32 s12, s4, 16
+; GFX9-NEXT:    s_sub_i32 s10, s10, s4
+; GFX9-NEXT:    s_sub_i32 s11, s11, s12
+; GFX9-NEXT:    s_cmp_lt_i32 s7, s9
+; GFX9-NEXT:    s_cselect_b32 s7, s7, s9
+; GFX9-NEXT:    s_cmp_lt_i32 s8, s6
+; GFX9-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX9-NEXT:    s_cselect_b32 s8, s8, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s7, s7, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s11
+; GFX9-NEXT:    s_lshr_b32 s8, s7, 16
+; GFX9-NEXT:    s_lshr_b32 s11, s5, 16
+; GFX9-NEXT:    s_sub_i32 s7, s7, s5
+; GFX9-NEXT:    s_sub_i32 s8, s8, s11
+; GFX9-NEXT:    s_pack_ll_b32_b16 s7, s7, s8
+; GFX9-NEXT:    s_sext_i32_i16 s8, s10
+; GFX9-NEXT:    s_sext_i32_i16 s13, s2
+; GFX9-NEXT:    s_ashr_i32 s10, s10, 16
+; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s8, s13
+; GFX9-NEXT:    s_cselect_b32 s8, s8, s13
+; GFX9-NEXT:    s_cmp_gt_i32 s10, s2
+; GFX9-NEXT:    s_cselect_b32 s2, s10, s2
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s8, s2
+; GFX9-NEXT:    s_sext_i32_i16 s8, s2
+; GFX9-NEXT:    s_sext_i32_i16 s10, s7
+; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX9-NEXT:    s_ashr_i32 s7, s7, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s8, s10
+; GFX9-NEXT:    s_cselect_b32 s8, s8, s10
+; GFX9-NEXT:    s_cmp_lt_i32 s2, s7
+; GFX9-NEXT:    s_cselect_b32 s2, s2, s7
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s8, s2
+; GFX9-NEXT:    s_lshr_b32 s7, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s8, s2, 16
+; GFX9-NEXT:    s_sub_i32 s0, s0, s2
+; GFX9-NEXT:    s_sub_i32 s2, s7, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT:    s_sext_i32_i16 s2, s1
+; GFX9-NEXT:    s_ashr_i32 s7, s1, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s2, s9
+; GFX9-NEXT:    s_cselect_b32 s8, s2, s9
+; GFX9-NEXT:    s_cmp_gt_i32 s7, s6
+; GFX9-NEXT:    s_cselect_b32 s10, s7, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s8, s8, s10
+; GFX9-NEXT:    s_lshr_b32 s10, s8, 16
+; GFX9-NEXT:    s_sub_i32 s4, s8, s4
+; GFX9-NEXT:    s_sub_i32 s8, s10, s12
+; GFX9-NEXT:    s_cmp_lt_i32 s2, s9
+; GFX9-NEXT:    s_cselect_b32 s2, s2, s9
+; GFX9-NEXT:    s_cmp_lt_i32 s7, s6
+; GFX9-NEXT:    s_cselect_b32 s6, s7, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s6
+; GFX9-NEXT:    s_lshr_b32 s6, s2, 16
+; GFX9-NEXT:    s_sub_i32 s2, s2, s5
+; GFX9-NEXT:    s_sub_i32 s5, s6, s11
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s5
+; GFX9-NEXT:    s_sext_i32_i16 s5, s4
+; GFX9-NEXT:    s_sext_i32_i16 s6, s3
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s6
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s3
+; GFX9-NEXT:    s_cselect_b32 s3, s4, s3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s5, s3
+; GFX9-NEXT:    s_sext_i32_i16 s4, s3
+; GFX9-NEXT:    s_sext_i32_i16 s5, s2
+; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX9-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s5
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s2
+; GFX9-NEXT:    s_cselect_b32 s2, s3, s2
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s4, s2
+; GFX9-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX9-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX9-NEXT:    s_sub_i32 s1, s1, s2
+; GFX9-NEXT:    s_sub_i32 s2, s3, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_ssubsat_v4i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
+; GFX10-NEXT:    s_sext_i32_i16 s5, s0
+; GFX10-NEXT:    s_sext_i32_i16 s7, s4
+; GFX10-NEXT:    s_ashr_i32 s6, s0, 16
+; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s5, s7
+; GFX10-NEXT:    s_movk_i32 s10, 0x7fff
+; GFX10-NEXT:    s_cselect_b32 s8, s5, s7
+; GFX10-NEXT:    s_cmp_gt_i32 s6, s4
+; GFX10-NEXT:    s_mov_b32 s12, 0xffff8000
+; GFX10-NEXT:    s_cselect_b32 s9, s6, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s12, s12, s12
+; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s9
+; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s10, s10
+; GFX10-NEXT:    s_lshr_b32 s10, s8, 16
+; GFX10-NEXT:    s_lshr_b32 s11, s9, 16
+; GFX10-NEXT:    s_sub_i32 s8, s8, s9
+; GFX10-NEXT:    s_sub_i32 s10, s10, s11
+; GFX10-NEXT:    s_cmp_lt_i32 s5, s7
+; GFX10-NEXT:    s_sext_i32_i16 s14, s2
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s7
+; GFX10-NEXT:    s_cmp_lt_i32 s6, s4
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s6, s6, s4
+; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s8, s10
+; GFX10-NEXT:    s_lshr_b32 s8, s5, 16
+; GFX10-NEXT:    s_lshr_b32 s10, s12, 16
+; GFX10-NEXT:    s_sext_i32_i16 s13, s6
+; GFX10-NEXT:    s_sub_i32 s5, s5, s12
+; GFX10-NEXT:    s_sub_i32 s8, s8, s10
+; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s13, s14
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s8
+; GFX10-NEXT:    s_cselect_b32 s13, s13, s14
+; GFX10-NEXT:    s_cmp_gt_i32 s6, s2
+; GFX10-NEXT:    s_sext_i32_i16 s8, s5
+; GFX10-NEXT:    s_cselect_b32 s2, s6, s2
+; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s13, s2
+; GFX10-NEXT:    s_sext_i32_i16 s6, s2
+; GFX10-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s6, s8
+; GFX10-NEXT:    s_cselect_b32 s6, s6, s8
+; GFX10-NEXT:    s_cmp_lt_i32 s2, s5
+; GFX10-NEXT:    s_cselect_b32 s2, s2, s5
+; GFX10-NEXT:    s_lshr_b32 s5, s0, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s6, s2
+; GFX10-NEXT:    s_lshr_b32 s6, s2, 16
+; GFX10-NEXT:    s_sub_i32 s0, s0, s2
+; GFX10-NEXT:    s_sub_i32 s2, s5, s6
+; GFX10-NEXT:    s_sext_i32_i16 s5, s1
+; GFX10-NEXT:    s_ashr_i32 s6, s1, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s5, s7
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
+; GFX10-NEXT:    s_cselect_b32 s8, s5, s7
+; GFX10-NEXT:    s_cmp_gt_i32 s6, s4
+; GFX10-NEXT:    s_cselect_b32 s13, s6, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s13
+; GFX10-NEXT:    s_lshr_b32 s13, s8, 16
+; GFX10-NEXT:    s_sub_i32 s8, s8, s9
+; GFX10-NEXT:    s_sub_i32 s9, s13, s11
+; GFX10-NEXT:    s_cmp_lt_i32 s5, s7
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s7
+; GFX10-NEXT:    s_cmp_lt_i32 s6, s4
+; GFX10-NEXT:    s_cselect_b32 s4, s6, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s5, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s8, s9
+; GFX10-NEXT:    s_lshr_b32 s6, s4, 16
+; GFX10-NEXT:    s_sext_i32_i16 s7, s5
+; GFX10-NEXT:    s_sext_i32_i16 s8, s3
+; GFX10-NEXT:    s_sub_i32 s4, s4, s12
+; GFX10-NEXT:    s_sub_i32 s6, s6, s10
+; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
+; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s7, s8
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s6
+; GFX10-NEXT:    s_cselect_b32 s7, s7, s8
+; GFX10-NEXT:    s_cmp_gt_i32 s5, s3
+; GFX10-NEXT:    s_sext_i32_i16 s6, s4
+; GFX10-NEXT:    s_cselect_b32 s3, s5, s3
+; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s7, s3
+; GFX10-NEXT:    s_sext_i32_i16 s5, s3
+; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s5, s6
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s4
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX10-NEXT:    s_lshr_b32 s4, s1, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s5, s3
+; GFX10-NEXT:    s_lshr_b32 s5, s3, 16
+; GFX10-NEXT:    s_sub_i32 s1, s1, s3
+; GFX10-NEXT:    s_sub_i32 s3, s4, s5
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> %lhs, <4 x i16> %rhs)
+  %cast = bitcast <4 x i16> %result to <2 x i32>
+  ret <2 x i32> %cast
+}
+
+; FIXME
+; define <5 x i16> @v_ssubsat_v5i16(<5 x i16> %lhs, <5 x i16> %rhs) {
+;   %result = call <5 x i16> @llvm.ssub.sat.v5i16(<5 x i16> %lhs, <5 x i16> %rhs)
+;   ret <5 x i16> %result
+; }
+
+; define amdgpu_ps <5 x i16> @s_ssubsat_v5i16(<5 x i16> inreg %lhs, <5 x i16> inreg %rhs) {
+;   %result = call <5 x i16> @llvm.ssub.sat.v5i16(<5 x i16> %lhs, <5 x i16> %rhs)
+;   ret <5 x i16> %result
+; }
+
+define <3 x float> @v_ssubsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
+; GFX6-LABEL: v_ssubsat_v6i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    v_max_i32_e32 v12, -1, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GFX6-NEXT:    v_subrev_i32_e32 v12, vcc, s4, v12
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v14, -1, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v14, vcc, s5, v14
+; GFX6-NEXT:    v_max_i32_e32 v6, v12, v6
+; GFX6-NEXT:    v_min_i32_e32 v6, v6, v14
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
+; GFX6-NEXT:    v_max_i32_e32 v7, -1, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v7, vcc, s4, v7
+; GFX6-NEXT:    v_min_i32_e32 v12, -1, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v12, vcc, s5, v12
+; GFX6-NEXT:    v_max_i32_e32 v6, v7, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_min_i32_e32 v6, v6, v12
+; GFX6-NEXT:    v_bfrev_b32_e32 v13, -2
+; GFX6-NEXT:    v_max_i32_e32 v7, -1, v2
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v8
+; GFX6-NEXT:    v_min_i32_e32 v8, -1, v2
+; GFX6-NEXT:    v_sub_i32_e32 v7, vcc, v7, v13
+; GFX6-NEXT:    v_subrev_i32_e32 v8, vcc, s5, v8
+; GFX6-NEXT:    v_max_i32_e32 v6, v7, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_min_i32_e32 v6, v6, v8
+; GFX6-NEXT:    v_max_i32_e32 v7, -1, v3
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v6
+; GFX6-NEXT:    v_mov_b32_e32 v15, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v8, -1, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v9
+; GFX6-NEXT:    v_sub_i32_e32 v7, vcc, v7, v13
+; GFX6-NEXT:    v_sub_i32_e32 v8, vcc, v8, v15
+; GFX6-NEXT:    v_max_i32_e32 v6, v7, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT:    v_min_i32_e32 v6, v6, v8
+; GFX6-NEXT:    v_max_i32_e32 v7, -1, v4
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, v3, v6
+; GFX6-NEXT:    v_min_i32_e32 v8, -1, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v10
+; GFX6-NEXT:    v_sub_i32_e32 v7, vcc, v7, v13
+; GFX6-NEXT:    v_sub_i32_e32 v8, vcc, v8, v15
+; GFX6-NEXT:    v_max_i32_e32 v6, v7, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GFX6-NEXT:    v_min_i32_e32 v6, v6, v8
+; GFX6-NEXT:    v_max_i32_e32 v7, -1, v5
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, v4, v6
+; GFX6-NEXT:    v_min_i32_e32 v8, -1, v5
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v11
+; GFX6-NEXT:    v_sub_i32_e32 v7, vcc, v7, v13
+; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    v_sub_i32_e32 v8, vcc, v8, v15
+; GFX6-NEXT:    v_max_i32_e32 v6, v7, v6
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_min_i32_e32 v6, v6, v8
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, v5, v6
+; GFX6-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
+; GFX6-NEXT:    v_ashrrev_i32_e32 v3, 16, v3
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v3
+; GFX6-NEXT:    v_ashrrev_i32_e32 v5, 16, v5
+; GFX6-NEXT:    v_and_b32_e32 v3, s4, v5
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_ashrrev_i32_e32 v4, 16, v4
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_v6i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s6, 0xffff
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX8-NEXT:    v_max_i16_e32 v10, s6, v0
+; GFX8-NEXT:    v_subrev_u16_e32 v10, s4, v10
+; GFX8-NEXT:    s_mov_b32 s5, 0x8000
+; GFX8-NEXT:    v_min_i16_e32 v12, s6, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
+; GFX8-NEXT:    v_subrev_u16_e32 v12, s5, v12
+; GFX8-NEXT:    v_max_i16_e32 v10, v10, v3
+; GFX8-NEXT:    v_min_i16_e32 v10, v10, v12
+; GFX8-NEXT:    v_max_i16_e32 v12, s6, v6
+; GFX8-NEXT:    v_min_i16_e32 v14, s6, v6
+; GFX8-NEXT:    v_subrev_u16_e32 v12, s4, v12
+; GFX8-NEXT:    v_max_i16_sdwa v3, v12, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_max_i16_e32 v12, s6, v1
+; GFX8-NEXT:    v_subrev_u16_e32 v14, s5, v14
+; GFX8-NEXT:    v_min_i16_e32 v3, v3, v14
+; GFX8-NEXT:    v_subrev_u16_e32 v12, s4, v12
+; GFX8-NEXT:    v_min_i16_e32 v14, s6, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v1
+; GFX8-NEXT:    v_subrev_u16_e32 v14, s5, v14
+; GFX8-NEXT:    v_max_i16_e32 v12, v12, v4
+; GFX8-NEXT:    v_min_i16_e32 v12, v12, v14
+; GFX8-NEXT:    v_max_i16_e32 v14, s6, v7
+; GFX8-NEXT:    v_mov_b32_e32 v9, 0xffff
+; GFX8-NEXT:    v_min_i16_e32 v15, v7, v9
+; GFX8-NEXT:    v_subrev_u16_e32 v14, s4, v14
+; GFX8-NEXT:    v_max_i16_sdwa v4, v14, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_subrev_u16_e32 v15, s5, v15
+; GFX8-NEXT:    v_mov_b32_e32 v11, 0x7fff
+; GFX8-NEXT:    v_max_i16_e32 v14, v2, v9
+; GFX8-NEXT:    v_sub_u16_e32 v14, v14, v11
+; GFX8-NEXT:    v_min_i16_e32 v4, v4, v15
+; GFX8-NEXT:    v_mov_b32_e32 v13, 0x8000
+; GFX8-NEXT:    v_min_i16_e32 v15, v2, v9
+; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
+; GFX8-NEXT:    v_sub_u16_e32 v15, v15, v13
+; GFX8-NEXT:    v_max_i16_e32 v14, v14, v5
+; GFX8-NEXT:    v_min_i16_e32 v14, v14, v15
+; GFX8-NEXT:    v_max_i16_e32 v15, v8, v9
+; GFX8-NEXT:    v_min_i16_e32 v9, v8, v9
+; GFX8-NEXT:    v_sub_u16_e32 v11, v15, v11
+; GFX8-NEXT:    v_sub_u16_e32 v9, v9, v13
+; GFX8-NEXT:    v_max_i16_sdwa v5, v11, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v10
+; GFX8-NEXT:    v_sub_u16_sdwa v3, v6, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX8-NEXT:    v_min_i16_e32 v5, v5, v9
+; GFX8-NEXT:    v_sub_u16_e32 v1, v1, v12
+; GFX8-NEXT:    v_sub_u16_sdwa v3, v7, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v3
+; GFX8-NEXT:    v_sub_u16_e32 v2, v2, v14
+; GFX8-NEXT:    v_sub_u16_sdwa v3, v8, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_v6i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, -1, -1
+; GFX9-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
+; GFX9-NEXT:    v_pk_max_i16 v6, v0, s6
+; GFX9-NEXT:    v_pk_sub_i16 v6, v6, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
+; GFX9-NEXT:    v_pk_min_i16 v7, v0, s6
+; GFX9-NEXT:    v_pk_max_i16 v3, v6, v3
+; GFX9-NEXT:    v_pk_sub_i16 v7, v7, s5
+; GFX9-NEXT:    v_pk_min_i16 v3, v3, v7
+; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v3
+; GFX9-NEXT:    v_pk_max_i16 v3, v1, s6
+; GFX9-NEXT:    v_pk_sub_i16 v3, v3, s4
+; GFX9-NEXT:    v_pk_min_i16 v6, v1, s6
+; GFX9-NEXT:    v_pk_max_i16 v3, v3, v4
+; GFX9-NEXT:    v_pk_sub_i16 v6, v6, s5
+; GFX9-NEXT:    v_pk_min_i16 v3, v3, v6
+; GFX9-NEXT:    v_pk_sub_i16 v1, v1, v3
+; GFX9-NEXT:    v_pk_max_i16 v3, v2, s6
+; GFX9-NEXT:    v_pk_sub_i16 v3, v3, s4
+; GFX9-NEXT:    v_pk_min_i16 v4, v2, s6
+; GFX9-NEXT:    v_pk_sub_i16 v4, v4, s5
+; GFX9-NEXT:    v_pk_max_i16 v3, v3, v5
+; GFX9-NEXT:    v_pk_min_i16 v3, v3, v4
+; GFX9-NEXT:    v_pk_sub_i16 v2, v2, v3
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_ssubsat_v6i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, -1, -1
+; GFX10-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX10-NEXT:    v_pk_max_i16 v6, v0, s5
+; GFX10-NEXT:    v_pk_max_i16 v8, v1, s5
+; GFX10-NEXT:    v_pk_max_i16 v9, v2, s5
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
+; GFX10-NEXT:    v_pk_min_i16 v7, v0, s5
+; GFX10-NEXT:    v_pk_sub_i16 v6, v6, s4
+; GFX10-NEXT:    v_pk_sub_i16 v15, v8, s4
+; GFX10-NEXT:    v_pk_sub_i16 v19, v9, s4
+; GFX10-NEXT:    v_pk_min_i16 v10, v1, s5
+; GFX10-NEXT:    v_pk_min_i16 v11, v2, s5
+; GFX10-NEXT:    s_mov_b32 s6, 0xffff8000
+; GFX10-NEXT:    v_pk_max_i16 v14, v6, v3
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s6
+; GFX10-NEXT:    v_pk_max_i16 v4, v15, v4
+; GFX10-NEXT:    v_pk_sub_i16 v7, v7, s6
+; GFX10-NEXT:    v_pk_sub_i16 v6, v10, s6
+; GFX10-NEXT:    v_pk_sub_i16 v8, v11, s6
+; GFX10-NEXT:    v_pk_max_i16 v5, v19, v5
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_pk_min_i16 v3, v14, v7
+; GFX10-NEXT:    v_pk_min_i16 v4, v4, v6
+; GFX10-NEXT:    v_pk_min_i16 v5, v5, v8
+; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v3
+; GFX10-NEXT:    v_pk_sub_i16 v1, v1, v4
+; GFX10-NEXT:    v_pk_sub_i16 v2, v2, v5
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <6 x i16> @llvm.ssub.sat.v6i16(<6 x i16> %lhs, <6 x i16> %rhs)
+  %cast = bitcast <6 x i16> %result to <3 x float>
+  ret <3 x float> %cast
+}
+
+define amdgpu_ps <3 x i32> @s_ssubsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inreg %rhs) {
+; GFX6-LABEL: s_ssubsat_v6i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s6, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX6-NEXT:    s_brev_b32 s12, -2
+; GFX6-NEXT:    s_cselect_b32 s14, s0, -1
+; GFX6-NEXT:    s_sub_i32 s14, s14, s12
+; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX6-NEXT:    s_mov_b32 s13, 0x80000000
+; GFX6-NEXT:    s_cselect_b32 s15, s0, -1
+; GFX6-NEXT:    s_sub_i32 s15, s15, s13
+; GFX6-NEXT:    s_cmp_gt_i32 s14, s6
+; GFX6-NEXT:    s_cselect_b32 s6, s14, s6
+; GFX6-NEXT:    s_cmp_lt_i32 s6, s15
+; GFX6-NEXT:    s_cselect_b32 s6, s6, s15
+; GFX6-NEXT:    s_sub_i32 s0, s0, s6
+; GFX6-NEXT:    s_ashr_i32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s7, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s1, -1
+; GFX6-NEXT:    s_cselect_b32 s7, s1, -1
+; GFX6-NEXT:    s_sub_i32 s7, s7, s12
+; GFX6-NEXT:    s_cmp_lt_i32 s1, -1
+; GFX6-NEXT:    s_cselect_b32 s14, s1, -1
+; GFX6-NEXT:    s_sub_i32 s14, s14, s13
+; GFX6-NEXT:    s_cmp_gt_i32 s7, s6
+; GFX6-NEXT:    s_cselect_b32 s6, s7, s6
+; GFX6-NEXT:    s_cmp_lt_i32 s6, s14
+; GFX6-NEXT:    s_cselect_b32 s6, s6, s14
+; GFX6-NEXT:    s_sub_i32 s1, s1, s6
+; GFX6-NEXT:    s_ashr_i32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s8, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s2, -1
+; GFX6-NEXT:    s_cselect_b32 s7, s2, -1
+; GFX6-NEXT:    s_sub_i32 s7, s7, s12
+; GFX6-NEXT:    s_cmp_lt_i32 s2, -1
+; GFX6-NEXT:    s_cselect_b32 s8, s2, -1
+; GFX6-NEXT:    s_sub_i32 s8, s8, s13
+; GFX6-NEXT:    s_cmp_gt_i32 s7, s6
+; GFX6-NEXT:    s_cselect_b32 s6, s7, s6
+; GFX6-NEXT:    s_cmp_lt_i32 s6, s8
+; GFX6-NEXT:    s_cselect_b32 s6, s6, s8
+; GFX6-NEXT:    s_sub_i32 s2, s2, s6
+; GFX6-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX6-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s9, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s3, -1
+; GFX6-NEXT:    s_cselect_b32 s7, s3, -1
+; GFX6-NEXT:    s_sub_i32 s7, s7, s12
+; GFX6-NEXT:    s_cmp_lt_i32 s3, -1
+; GFX6-NEXT:    s_cselect_b32 s8, s3, -1
+; GFX6-NEXT:    s_sub_i32 s8, s8, s13
+; GFX6-NEXT:    s_cmp_gt_i32 s7, s6
+; GFX6-NEXT:    s_cselect_b32 s6, s7, s6
+; GFX6-NEXT:    s_cmp_lt_i32 s6, s8
+; GFX6-NEXT:    s_cselect_b32 s6, s6, s8
+; GFX6-NEXT:    s_sub_i32 s3, s3, s6
+; GFX6-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX6-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s10, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s4, -1
+; GFX6-NEXT:    s_cselect_b32 s7, s4, -1
+; GFX6-NEXT:    s_sub_i32 s7, s7, s12
+; GFX6-NEXT:    s_cmp_lt_i32 s4, -1
+; GFX6-NEXT:    s_cselect_b32 s8, s4, -1
+; GFX6-NEXT:    s_sub_i32 s8, s8, s13
+; GFX6-NEXT:    s_cmp_gt_i32 s7, s6
+; GFX6-NEXT:    s_cselect_b32 s6, s7, s6
+; GFX6-NEXT:    s_cmp_lt_i32 s6, s8
+; GFX6-NEXT:    s_cselect_b32 s6, s6, s8
+; GFX6-NEXT:    s_sub_i32 s4, s4, s6
+; GFX6-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX6-NEXT:    s_lshl_b32 s5, s5, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s11, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s5, -1
+; GFX6-NEXT:    s_cselect_b32 s7, s5, -1
+; GFX6-NEXT:    s_sub_i32 s7, s7, s12
+; GFX6-NEXT:    s_cmp_lt_i32 s5, -1
+; GFX6-NEXT:    s_cselect_b32 s8, s5, -1
+; GFX6-NEXT:    s_sub_i32 s8, s8, s13
+; GFX6-NEXT:    s_cmp_gt_i32 s7, s6
+; GFX6-NEXT:    s_cselect_b32 s6, s7, s6
+; GFX6-NEXT:    s_cmp_lt_i32 s6, s8
+; GFX6-NEXT:    s_cselect_b32 s6, s6, s8
+; GFX6-NEXT:    s_sub_i32 s5, s5, s6
+; GFX6-NEXT:    s_mov_b32 s6, 0xffff
+; GFX6-NEXT:    s_and_b32 s1, s1, s6
+; GFX6-NEXT:    s_and_b32 s0, s0, s6
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_and_b32 s1, s2, s6
+; GFX6-NEXT:    s_and_b32 s2, s3, s6
+; GFX6-NEXT:    s_ashr_i32 s5, s5, 16
+; GFX6-NEXT:    s_and_b32 s3, s5, s6
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_or_b32 s1, s1, s2
+; GFX6-NEXT:    s_and_b32 s2, s4, s6
+; GFX6-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX6-NEXT:    s_or_b32 s2, s2, s3
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_ssubsat_v6i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_lshr_b32 s9, s3, 16
+; GFX8-NEXT:    s_lshr_b32 s10, s4, 16
+; GFX8-NEXT:    s_lshr_b32 s11, s5, 16
+; GFX8-NEXT:    s_lshr_b32 s6, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s7, s1, 16
+; GFX8-NEXT:    s_lshr_b32 s8, s2, 16
+; GFX8-NEXT:    s_sext_i32_i16 s14, s0
+; GFX8-NEXT:    s_sext_i32_i16 s15, 0xffff
+; GFX8-NEXT:    s_cmp_gt_i32 s14, s15
+; GFX8-NEXT:    s_movk_i32 s12, 0x7fff
+; GFX8-NEXT:    s_cselect_b32 s16, s14, s15
+; GFX8-NEXT:    s_sub_i32 s16, s16, s12
+; GFX8-NEXT:    s_cmp_lt_i32 s14, s15
+; GFX8-NEXT:    s_mov_b32 s13, 0x8000
+; GFX8-NEXT:    s_cselect_b32 s14, s14, s15
+; GFX8-NEXT:    s_sub_i32 s14, s14, s13
+; GFX8-NEXT:    s_sext_i32_i16 s16, s16
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_cmp_gt_i32 s16, s3
+; GFX8-NEXT:    s_cselect_b32 s3, s16, s3
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_sext_i32_i16 s14, s14
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s14
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s14
+; GFX8-NEXT:    s_sub_i32 s0, s0, s3
+; GFX8-NEXT:    s_sext_i32_i16 s3, s6
+; GFX8-NEXT:    s_cmp_gt_i32 s3, s15
+; GFX8-NEXT:    s_cselect_b32 s14, s3, s15
+; GFX8-NEXT:    s_sub_i32 s14, s14, s12
+; GFX8-NEXT:    s_cmp_lt_i32 s3, s15
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s15
+; GFX8-NEXT:    s_sub_i32 s3, s3, s13
+; GFX8-NEXT:    s_sext_i32_i16 s14, s14
+; GFX8-NEXT:    s_sext_i32_i16 s9, s9
+; GFX8-NEXT:    s_cmp_gt_i32 s14, s9
+; GFX8-NEXT:    s_cselect_b32 s9, s14, s9
+; GFX8-NEXT:    s_sext_i32_i16 s9, s9
+; GFX8-NEXT:    s_sext_i32_i16 s3, s3
+; GFX8-NEXT:    s_cmp_lt_i32 s9, s3
+; GFX8-NEXT:    s_cselect_b32 s3, s9, s3
+; GFX8-NEXT:    s_sub_i32 s3, s6, s3
+; GFX8-NEXT:    s_sext_i32_i16 s6, s1
+; GFX8-NEXT:    s_cmp_gt_i32 s6, s15
+; GFX8-NEXT:    s_cselect_b32 s9, s6, s15
+; GFX8-NEXT:    s_sub_i32 s9, s9, s12
+; GFX8-NEXT:    s_cmp_lt_i32 s6, s15
+; GFX8-NEXT:    s_cselect_b32 s6, s6, s15
+; GFX8-NEXT:    s_sub_i32 s6, s6, s13
+; GFX8-NEXT:    s_sext_i32_i16 s9, s9
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_cmp_gt_i32 s9, s4
+; GFX8-NEXT:    s_cselect_b32 s4, s9, s4
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_sext_i32_i16 s6, s6
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s6
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX8-NEXT:    s_sub_i32 s1, s1, s4
+; GFX8-NEXT:    s_sext_i32_i16 s4, s7
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s15
+; GFX8-NEXT:    s_cselect_b32 s6, s4, s15
+; GFX8-NEXT:    s_sub_i32 s6, s6, s12
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s15
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s15
+; GFX8-NEXT:    s_sub_i32 s4, s4, s13
+; GFX8-NEXT:    s_sext_i32_i16 s6, s6
+; GFX8-NEXT:    s_sext_i32_i16 s9, s10
+; GFX8-NEXT:    s_cmp_gt_i32 s6, s9
+; GFX8-NEXT:    s_cselect_b32 s6, s6, s9
+; GFX8-NEXT:    s_sext_i32_i16 s6, s6
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_cmp_lt_i32 s6, s4
+; GFX8-NEXT:    s_cselect_b32 s4, s6, s4
+; GFX8-NEXT:    s_sub_i32 s4, s7, s4
+; GFX8-NEXT:    s_sext_i32_i16 s6, s2
+; GFX8-NEXT:    s_cmp_gt_i32 s6, s15
+; GFX8-NEXT:    s_cselect_b32 s7, s6, s15
+; GFX8-NEXT:    s_sub_i32 s7, s7, s12
+; GFX8-NEXT:    s_cmp_lt_i32 s6, s15
+; GFX8-NEXT:    s_cselect_b32 s6, s6, s15
+; GFX8-NEXT:    s_sub_i32 s6, s6, s13
+; GFX8-NEXT:    s_sext_i32_i16 s7, s7
+; GFX8-NEXT:    s_sext_i32_i16 s5, s5
+; GFX8-NEXT:    s_cmp_gt_i32 s7, s5
+; GFX8-NEXT:    s_cselect_b32 s5, s7, s5
+; GFX8-NEXT:    s_sext_i32_i16 s5, s5
+; GFX8-NEXT:    s_sext_i32_i16 s6, s6
+; GFX8-NEXT:    s_cmp_lt_i32 s5, s6
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX8-NEXT:    s_sub_i32 s2, s2, s5
+; GFX8-NEXT:    s_sext_i32_i16 s5, s8
+; GFX8-NEXT:    s_cmp_gt_i32 s5, s15
+; GFX8-NEXT:    s_cselect_b32 s6, s5, s15
+; GFX8-NEXT:    s_sub_i32 s6, s6, s12
+; GFX8-NEXT:    s_cmp_lt_i32 s5, s15
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s15
+; GFX8-NEXT:    s_sub_i32 s5, s5, s13
+; GFX8-NEXT:    s_sext_i32_i16 s6, s6
+; GFX8-NEXT:    s_sext_i32_i16 s7, s11
+; GFX8-NEXT:    s_cmp_gt_i32 s6, s7
+; GFX8-NEXT:    s_cselect_b32 s6, s6, s7
+; GFX8-NEXT:    s_sext_i32_i16 s6, s6
+; GFX8-NEXT:    s_sext_i32_i16 s5, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s6, s5
+; GFX8-NEXT:    s_cselect_b32 s5, s6, s5
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX8-NEXT:    s_or_b32 s0, s0, s3
+; GFX8-NEXT:    s_bfe_u32 s3, s4, 0x100000
+; GFX8-NEXT:    s_sub_i32 s5, s8, s5
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX8-NEXT:    s_or_b32 s1, s1, s3
+; GFX8-NEXT:    s_bfe_u32 s3, s5, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX8-NEXT:    s_or_b32 s2, s2, s3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_ssubsat_v6i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_pack_ll_b32_b16 s8, -1, -1
+; GFX9-NEXT:    s_sext_i32_i16 s11, s8
+; GFX9-NEXT:    s_sext_i32_i16 s9, s0
+; GFX9-NEXT:    s_ashr_i32 s10, s0, 16
+; GFX9-NEXT:    s_ashr_i32 s8, s8, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s9, s11
+; GFX9-NEXT:    s_cselect_b32 s12, s9, s11
+; GFX9-NEXT:    s_cmp_gt_i32 s10, s8
+; GFX9-NEXT:    s_movk_i32 s6, 0x7fff
+; GFX9-NEXT:    s_cselect_b32 s13, s10, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s12, s12, s13
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s6
+; GFX9-NEXT:    s_lshr_b32 s13, s12, 16
+; GFX9-NEXT:    s_lshr_b32 s14, s6, 16
+; GFX9-NEXT:    s_sub_i32 s12, s12, s6
+; GFX9-NEXT:    s_sub_i32 s13, s13, s14
+; GFX9-NEXT:    s_cmp_lt_i32 s9, s11
+; GFX9-NEXT:    s_cselect_b32 s9, s9, s11
+; GFX9-NEXT:    s_cmp_lt_i32 s10, s8
+; GFX9-NEXT:    s_mov_b32 s7, 0xffff8000
+; GFX9-NEXT:    s_cselect_b32 s10, s10, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s7, s7, s7
+; GFX9-NEXT:    s_pack_ll_b32_b16 s12, s12, s13
+; GFX9-NEXT:    s_lshr_b32 s10, s9, 16
+; GFX9-NEXT:    s_lshr_b32 s13, s7, 16
+; GFX9-NEXT:    s_sub_i32 s9, s9, s7
+; GFX9-NEXT:    s_sub_i32 s10, s10, s13
+; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
+; GFX9-NEXT:    s_sext_i32_i16 s10, s12
+; GFX9-NEXT:    s_sext_i32_i16 s15, s3
+; GFX9-NEXT:    s_ashr_i32 s12, s12, 16
+; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s10, s15
+; GFX9-NEXT:    s_cselect_b32 s10, s10, s15
+; GFX9-NEXT:    s_cmp_gt_i32 s12, s3
+; GFX9-NEXT:    s_cselect_b32 s3, s12, s3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s10, s3
+; GFX9-NEXT:    s_sext_i32_i16 s10, s3
+; GFX9-NEXT:    s_sext_i32_i16 s12, s9
+; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX9-NEXT:    s_ashr_i32 s9, s9, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s10, s12
+; GFX9-NEXT:    s_cselect_b32 s10, s10, s12
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s9
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s9
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s10, s3
+; GFX9-NEXT:    s_lshr_b32 s9, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s10, s3, 16
+; GFX9-NEXT:    s_sub_i32 s0, s0, s3
+; GFX9-NEXT:    s_sub_i32 s3, s9, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s3
+; GFX9-NEXT:    s_sext_i32_i16 s3, s1
+; GFX9-NEXT:    s_ashr_i32 s9, s1, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s11
+; GFX9-NEXT:    s_cselect_b32 s10, s3, s11
+; GFX9-NEXT:    s_cmp_gt_i32 s9, s8
+; GFX9-NEXT:    s_cselect_b32 s12, s9, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s12
+; GFX9-NEXT:    s_lshr_b32 s12, s10, 16
+; GFX9-NEXT:    s_sub_i32 s10, s10, s6
+; GFX9-NEXT:    s_sub_i32 s12, s12, s14
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s11
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s11
+; GFX9-NEXT:    s_cmp_lt_i32 s9, s8
+; GFX9-NEXT:    s_cselect_b32 s9, s9, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s9
+; GFX9-NEXT:    s_lshr_b32 s9, s3, 16
+; GFX9-NEXT:    s_pack_ll_b32_b16 s10, s10, s12
+; GFX9-NEXT:    s_sub_i32 s3, s3, s7
+; GFX9-NEXT:    s_sub_i32 s9, s9, s13
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s9
+; GFX9-NEXT:    s_sext_i32_i16 s9, s10
+; GFX9-NEXT:    s_sext_i32_i16 s12, s4
+; GFX9-NEXT:    s_ashr_i32 s10, s10, 16
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s9, s12
+; GFX9-NEXT:    s_cselect_b32 s9, s9, s12
+; GFX9-NEXT:    s_cmp_gt_i32 s10, s4
+; GFX9-NEXT:    s_cselect_b32 s4, s10, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s9, s4
+; GFX9-NEXT:    s_sext_i32_i16 s9, s4
+; GFX9-NEXT:    s_sext_i32_i16 s10, s3
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s9, s10
+; GFX9-NEXT:    s_cselect_b32 s9, s9, s10
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s3
+; GFX9-NEXT:    s_cselect_b32 s3, s4, s3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s9, s3
+; GFX9-NEXT:    s_lshr_b32 s4, s1, 16
+; GFX9-NEXT:    s_lshr_b32 s9, s3, 16
+; GFX9-NEXT:    s_sub_i32 s1, s1, s3
+; GFX9-NEXT:    s_sub_i32 s3, s4, s9
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s3
+; GFX9-NEXT:    s_sext_i32_i16 s3, s2
+; GFX9-NEXT:    s_ashr_i32 s4, s2, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s3, s11
+; GFX9-NEXT:    s_cselect_b32 s9, s3, s11
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s8
+; GFX9-NEXT:    s_cselect_b32 s10, s4, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
+; GFX9-NEXT:    s_lshr_b32 s10, s9, 16
+; GFX9-NEXT:    s_sub_i32 s6, s9, s6
+; GFX9-NEXT:    s_sub_i32 s9, s10, s14
+; GFX9-NEXT:    s_cmp_lt_i32 s3, s11
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s11
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s8
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
+; GFX9-NEXT:    s_lshr_b32 s4, s3, 16
+; GFX9-NEXT:    s_sub_i32 s3, s3, s7
+; GFX9-NEXT:    s_sub_i32 s4, s4, s13
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s9
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
+; GFX9-NEXT:    s_sext_i32_i16 s4, s6
+; GFX9-NEXT:    s_sext_i32_i16 s7, s5
+; GFX9-NEXT:    s_ashr_i32 s6, s6, 16
+; GFX9-NEXT:    s_ashr_i32 s5, s5, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s7
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s7
+; GFX9-NEXT:    s_cmp_gt_i32 s6, s5
+; GFX9-NEXT:    s_cselect_b32 s5, s6, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
+; GFX9-NEXT:    s_sext_i32_i16 s5, s4
+; GFX9-NEXT:    s_sext_i32_i16 s6, s3
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX9-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s6
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s3
+; GFX9-NEXT:    s_cselect_b32 s3, s4, s3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s5, s3
+; GFX9-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX9-NEXT:    s_lshr_b32 s5, s3, 16
+; GFX9-NEXT:    s_sub_i32 s2, s2, s3
+; GFX9-NEXT:    s_sub_i32 s3, s4, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_ssubsat_v6i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, -1, -1
+; GFX10-NEXT:    s_sext_i32_i16 s7, s0
+; GFX10-NEXT:    s_sext_i32_i16 s9, s6
+; GFX10-NEXT:    s_ashr_i32 s8, s0, 16
+; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s7, s9
+; GFX10-NEXT:    s_movk_i32 s12, 0x7fff
+; GFX10-NEXT:    s_cselect_b32 s10, s7, s9
+; GFX10-NEXT:    s_cmp_gt_i32 s8, s6
+; GFX10-NEXT:    s_mov_b32 s14, 0xffff8000
+; GFX10-NEXT:    s_cselect_b32 s11, s8, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s14, s14, s14
+; GFX10-NEXT:    s_pack_ll_b32_b16 s10, s10, s11
+; GFX10-NEXT:    s_pack_ll_b32_b16 s11, s12, s12
+; GFX10-NEXT:    s_lshr_b32 s12, s10, 16
+; GFX10-NEXT:    s_lshr_b32 s13, s11, 16
+; GFX10-NEXT:    s_sub_i32 s10, s10, s11
+; GFX10-NEXT:    s_sub_i32 s12, s12, s13
+; GFX10-NEXT:    s_cmp_lt_i32 s7, s9
+; GFX10-NEXT:    s_sext_i32_i16 s16, s3
+; GFX10-NEXT:    s_cselect_b32 s7, s7, s9
+; GFX10-NEXT:    s_cmp_lt_i32 s8, s6
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s8, s8, s6
+; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s7, s8
+; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s10, s12
+; GFX10-NEXT:    s_lshr_b32 s10, s7, 16
+; GFX10-NEXT:    s_lshr_b32 s12, s14, 16
+; GFX10-NEXT:    s_sext_i32_i16 s15, s8
+; GFX10-NEXT:    s_sub_i32 s7, s7, s14
+; GFX10-NEXT:    s_sub_i32 s10, s10, s12
+; GFX10-NEXT:    s_ashr_i32 s8, s8, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s15, s16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s7, s10
+; GFX10-NEXT:    s_cselect_b32 s15, s15, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s8, s3
+; GFX10-NEXT:    s_sext_i32_i16 s10, s7
+; GFX10-NEXT:    s_cselect_b32 s3, s8, s3
+; GFX10-NEXT:    s_ashr_i32 s7, s7, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s15, s3
+; GFX10-NEXT:    s_sext_i32_i16 s16, s4
+; GFX10-NEXT:    s_sext_i32_i16 s8, s3
+; GFX10-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s8, s10
+; GFX10-NEXT:    s_cselect_b32 s8, s8, s10
+; GFX10-NEXT:    s_cmp_lt_i32 s3, s7
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s7
+; GFX10-NEXT:    s_lshr_b32 s7, s0, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s8, s3
+; GFX10-NEXT:    s_lshr_b32 s8, s3, 16
+; GFX10-NEXT:    s_sub_i32 s0, s0, s3
+; GFX10-NEXT:    s_sub_i32 s3, s7, s8
+; GFX10-NEXT:    s_sext_i32_i16 s7, s1
+; GFX10-NEXT:    s_ashr_i32 s8, s1, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s7, s9
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s3
+; GFX10-NEXT:    s_cselect_b32 s10, s7, s9
+; GFX10-NEXT:    s_cmp_gt_i32 s8, s6
+; GFX10-NEXT:    s_cselect_b32 s15, s8, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s10, s10, s15
+; GFX10-NEXT:    s_lshr_b32 s15, s10, 16
+; GFX10-NEXT:    s_sub_i32 s10, s10, s11
+; GFX10-NEXT:    s_sub_i32 s15, s15, s13
+; GFX10-NEXT:    s_cmp_lt_i32 s7, s9
+; GFX10-NEXT:    s_cselect_b32 s7, s7, s9
+; GFX10-NEXT:    s_cmp_lt_i32 s8, s6
+; GFX10-NEXT:    s_cselect_b32 s8, s8, s6
+; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s7, s8
+; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s10, s15
+; GFX10-NEXT:    s_lshr_b32 s10, s7, 16
+; GFX10-NEXT:    s_sext_i32_i16 s15, s8
+; GFX10-NEXT:    s_sub_i32 s7, s7, s14
+; GFX10-NEXT:    s_sub_i32 s10, s10, s12
+; GFX10-NEXT:    s_ashr_i32 s8, s8, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s15, s16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s7, s10
+; GFX10-NEXT:    s_cselect_b32 s15, s15, s16
+; GFX10-NEXT:    s_cmp_gt_i32 s8, s4
+; GFX10-NEXT:    s_sext_i32_i16 s10, s7
+; GFX10-NEXT:    s_cselect_b32 s4, s8, s4
+; GFX10-NEXT:    s_ashr_i32 s7, s7, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s15, s4
+; GFX10-NEXT:    s_sext_i32_i16 s8, s4
+; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s8, s10
+; GFX10-NEXT:    s_cselect_b32 s8, s8, s10
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s7
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s7
+; GFX10-NEXT:    s_lshr_b32 s7, s1, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s8, s4
+; GFX10-NEXT:    s_lshr_b32 s8, s4, 16
+; GFX10-NEXT:    s_sub_i32 s1, s1, s4
+; GFX10-NEXT:    s_sub_i32 s4, s7, s8
+; GFX10-NEXT:    s_sext_i32_i16 s7, s2
+; GFX10-NEXT:    s_ashr_i32 s8, s2, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s7, s9
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
+; GFX10-NEXT:    s_cselect_b32 s10, s7, s9
+; GFX10-NEXT:    s_cmp_gt_i32 s8, s6
+; GFX10-NEXT:    s_cselect_b32 s15, s8, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s10, s10, s15
+; GFX10-NEXT:    s_lshr_b32 s15, s10, 16
+; GFX10-NEXT:    s_sub_i32 s10, s10, s11
+; GFX10-NEXT:    s_sub_i32 s11, s15, s13
+; GFX10-NEXT:    s_cmp_lt_i32 s7, s9
+; GFX10-NEXT:    s_cselect_b32 s7, s7, s9
+; GFX10-NEXT:    s_cmp_lt_i32 s8, s6
+; GFX10-NEXT:    s_cselect_b32 s6, s8, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s7, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s10, s11
+; GFX10-NEXT:    s_lshr_b32 s8, s6, 16
+; GFX10-NEXT:    s_sext_i32_i16 s9, s7
+; GFX10-NEXT:    s_sext_i32_i16 s10, s5
+; GFX10-NEXT:    s_sub_i32 s6, s6, s14
+; GFX10-NEXT:    s_sub_i32 s8, s8, s12
+; GFX10-NEXT:    s_ashr_i32 s7, s7, 16
+; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s9, s10
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s8
+; GFX10-NEXT:    s_cselect_b32 s9, s9, s10
+; GFX10-NEXT:    s_cmp_gt_i32 s7, s5
+; GFX10-NEXT:    s_sext_i32_i16 s8, s6
+; GFX10-NEXT:    s_cselect_b32 s5, s7, s5
+; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s9, s5
+; GFX10-NEXT:    s_sext_i32_i16 s7, s5
+; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s7, s8
+; GFX10-NEXT:    s_cselect_b32 s7, s7, s8
+; GFX10-NEXT:    s_cmp_lt_i32 s5, s6
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s7, s5
+; GFX10-NEXT:    s_lshr_b32 s5, s2, 16
+; GFX10-NEXT:    s_lshr_b32 s6, s3, 16
+; GFX10-NEXT:    s_sub_i32 s2, s2, s3
+; GFX10-NEXT:    s_sub_i32 s3, s5, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <6 x i16> @llvm.ssub.sat.v6i16(<6 x i16> %lhs, <6 x i16> %rhs)
+  %cast = bitcast <6 x i16> %result to <3 x i32>
+  ret <3 x i32> %cast
+}
+
+define <4 x float> @v_ssubsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
+; GFX6-LABEL: v_ssubsat_v8i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    s_brev_b32 s4, -2
+; GFX6-NEXT:    v_max_i32_e32 v16, -1, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GFX6-NEXT:    v_subrev_i32_e32 v16, vcc, s4, v16
+; GFX6-NEXT:    s_mov_b32 s5, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v18, -1, v0
+; GFX6-NEXT:    v_subrev_i32_e32 v18, vcc, s5, v18
+; GFX6-NEXT:    v_max_i32_e32 v8, v16, v8
+; GFX6-NEXT:    v_min_i32_e32 v8, v8, v18
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v9
+; GFX6-NEXT:    v_max_i32_e32 v9, -1, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v9, vcc, s4, v9
+; GFX6-NEXT:    v_min_i32_e32 v16, -1, v1
+; GFX6-NEXT:    v_subrev_i32_e32 v16, vcc, s5, v16
+; GFX6-NEXT:    v_max_i32_e32 v8, v9, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_min_i32_e32 v8, v8, v16
+; GFX6-NEXT:    v_bfrev_b32_e32 v17, -2
+; GFX6-NEXT:    v_max_i32_e32 v9, -1, v2
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v10
+; GFX6-NEXT:    v_min_i32_e32 v10, -1, v2
+; GFX6-NEXT:    v_sub_i32_e32 v9, vcc, v9, v17
+; GFX6-NEXT:    v_subrev_i32_e32 v10, vcc, s5, v10
+; GFX6-NEXT:    v_max_i32_e32 v8, v9, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_min_i32_e32 v8, v8, v10
+; GFX6-NEXT:    v_max_i32_e32 v9, -1, v3
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v8
+; GFX6-NEXT:    v_mov_b32_e32 v19, 0x80000000
+; GFX6-NEXT:    v_min_i32_e32 v10, -1, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v11
+; GFX6-NEXT:    v_sub_i32_e32 v9, vcc, v9, v17
+; GFX6-NEXT:    v_sub_i32_e32 v10, vcc, v10, v19
+; GFX6-NEXT:    v_max_i32_e32 v8, v9, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT:    v_min_i32_e32 v8, v8, v10
+; GFX6-NEXT:    v_max_i32_e32 v9, -1, v4
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, v3, v8
+; GFX6-NEXT:    v_min_i32_e32 v10, -1, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v12
+; GFX6-NEXT:    v_sub_i32_e32 v9, vcc, v9, v17
+; GFX6-NEXT:    v_sub_i32_e32 v10, vcc, v10, v19
+; GFX6-NEXT:    v_max_i32_e32 v8, v9, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GFX6-NEXT:    v_min_i32_e32 v8, v8, v10
+; GFX6-NEXT:    v_max_i32_e32 v9, -1, v5
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, v4, v8
+; GFX6-NEXT:    v_min_i32_e32 v10, -1, v5
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v13
+; GFX6-NEXT:    v_sub_i32_e32 v9, vcc, v9, v17
+; GFX6-NEXT:    v_sub_i32_e32 v10, vcc, v10, v19
+; GFX6-NEXT:    v_max_i32_e32 v8, v9, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GFX6-NEXT:    v_min_i32_e32 v8, v8, v10
+; GFX6-NEXT:    v_max_i32_e32 v9, -1, v6
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, v5, v8
+; GFX6-NEXT:    v_min_i32_e32 v10, -1, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v14
+; GFX6-NEXT:    v_sub_i32_e32 v9, vcc, v9, v17
+; GFX6-NEXT:    v_sub_i32_e32 v10, vcc, v10, v19
+; GFX6-NEXT:    v_max_i32_e32 v8, v9, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GFX6-NEXT:    v_min_i32_e32 v8, v8, v10
+; GFX6-NEXT:    v_max_i32_e32 v9, -1, v7
+; GFX6-NEXT:    v_sub_i32_e32 v6, vcc, v6, v8
+; GFX6-NEXT:    v_min_i32_e32 v10, -1, v7
+; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 16, v1
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v15
+; GFX6-NEXT:    v_sub_i32_e32 v9, vcc, v9, v17
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_sub_i32_e32 v10, vcc, v10, v19
+; GFX6-NEXT:    v_max_i32_e32 v8, v9, v8
+; GFX6-NEXT:    v_min_i32_e32 v8, v8, v10
+; GFX6-NEXT:    v_ashrrev_i32_e32 v2, 16, v2
+; GFX6-NEXT:    v_ashrrev_i32_e32 v3, 16, v3
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_sub_i32_e32 v7, vcc, v7, v8
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v3
+; GFX6-NEXT:    v_ashrrev_i32_e32 v5, 16, v5
+; GFX6-NEXT:    v_and_b32_e32 v3, s4, v5
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_ashrrev_i32_e32 v4, 16, v4
+; GFX6-NEXT:    v_ashrrev_i32_e32 v7, 16, v7
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v4
+; GFX6-NEXT:    v_and_b32_e32 v4, s4, v7
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_ashrrev_i32_e32 v6, 16, v6
+; GFX6-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX6-NEXT:    v_and_b32_e32 v3, s4, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT:    v_or_b32_e32 v3, v3, v4
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_v8i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s6, 0xffff
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX8-NEXT:    v_max_i16_e32 v13, s6, v0
+; GFX8-NEXT:    v_subrev_u16_e32 v13, s4, v13
+; GFX8-NEXT:    s_mov_b32 s5, 0x8000
+; GFX8-NEXT:    v_min_i16_e32 v15, s6, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 16, v0
+; GFX8-NEXT:    v_subrev_u16_e32 v15, s5, v15
+; GFX8-NEXT:    v_max_i16_e32 v13, v13, v4
+; GFX8-NEXT:    v_min_i16_e32 v13, v13, v15
+; GFX8-NEXT:    v_max_i16_e32 v15, s6, v8
+; GFX8-NEXT:    v_min_i16_e32 v17, s6, v8
+; GFX8-NEXT:    v_subrev_u16_e32 v15, s4, v15
+; GFX8-NEXT:    v_max_i16_sdwa v4, v15, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_max_i16_e32 v15, s6, v1
+; GFX8-NEXT:    v_subrev_u16_e32 v17, s5, v17
+; GFX8-NEXT:    v_min_i16_e32 v4, v4, v17
+; GFX8-NEXT:    v_subrev_u16_e32 v15, s4, v15
+; GFX8-NEXT:    v_min_i16_e32 v17, s6, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v9, 16, v1
+; GFX8-NEXT:    v_subrev_u16_e32 v17, s5, v17
+; GFX8-NEXT:    v_max_i16_e32 v15, v15, v5
+; GFX8-NEXT:    v_min_i16_e32 v15, v15, v17
+; GFX8-NEXT:    v_max_i16_e32 v17, s6, v9
+; GFX8-NEXT:    v_mov_b32_e32 v12, 0xffff
+; GFX8-NEXT:    v_min_i16_e32 v18, v9, v12
+; GFX8-NEXT:    v_subrev_u16_e32 v17, s4, v17
+; GFX8-NEXT:    v_max_i16_sdwa v5, v17, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_subrev_u16_e32 v18, s5, v18
+; GFX8-NEXT:    v_mov_b32_e32 v14, 0x7fff
+; GFX8-NEXT:    v_max_i16_e32 v17, v2, v12
+; GFX8-NEXT:    v_sub_u16_e32 v17, v17, v14
+; GFX8-NEXT:    v_min_i16_e32 v5, v5, v18
+; GFX8-NEXT:    v_mov_b32_e32 v16, 0x8000
+; GFX8-NEXT:    v_min_i16_e32 v18, v2, v12
+; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
+; GFX8-NEXT:    v_sub_u16_e32 v18, v18, v16
+; GFX8-NEXT:    v_max_i16_e32 v17, v17, v6
+; GFX8-NEXT:    v_min_i16_e32 v17, v17, v18
+; GFX8-NEXT:    v_max_i16_e32 v18, v10, v12
+; GFX8-NEXT:    v_min_i16_e32 v19, v10, v12
+; GFX8-NEXT:    v_sub_u16_e32 v18, v18, v14
+; GFX8-NEXT:    v_max_i16_sdwa v6, v18, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_max_i16_e32 v18, v3, v12
+; GFX8-NEXT:    v_sub_u16_e32 v19, v19, v16
+; GFX8-NEXT:    v_sub_u16_e32 v18, v18, v14
+; GFX8-NEXT:    v_min_i16_e32 v6, v6, v19
+; GFX8-NEXT:    v_min_i16_e32 v19, v3, v12
+; GFX8-NEXT:    v_lshrrev_b32_e32 v11, 16, v3
+; GFX8-NEXT:    v_sub_u16_e32 v19, v19, v16
+; GFX8-NEXT:    v_max_i16_e32 v18, v18, v7
+; GFX8-NEXT:    v_min_i16_e32 v18, v18, v19
+; GFX8-NEXT:    v_max_i16_e32 v19, v11, v12
+; GFX8-NEXT:    v_min_i16_e32 v12, v11, v12
+; GFX8-NEXT:    v_sub_u16_e32 v14, v19, v14
+; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v13
+; GFX8-NEXT:    v_sub_u16_sdwa v4, v8, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT:    v_sub_u16_e32 v12, v12, v16
+; GFX8-NEXT:    v_max_i16_sdwa v7, v14, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_u16_e32 v1, v1, v15
+; GFX8-NEXT:    v_sub_u16_sdwa v4, v9, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v4
+; GFX8-NEXT:    v_min_i16_e32 v7, v7, v12
+; GFX8-NEXT:    v_sub_u16_e32 v2, v2, v17
+; GFX8-NEXT:    v_sub_u16_sdwa v4, v10, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v2, v2, v4
+; GFX8-NEXT:    v_sub_u16_e32 v3, v3, v18
+; GFX8-NEXT:    v_sub_u16_sdwa v4, v11, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v3, v3, v4
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_v8i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, -1, -1
+; GFX9-NEXT:    s_mov_b32 s5, 0xffff8000
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
+; GFX9-NEXT:    v_pk_max_i16 v8, v0, s6
+; GFX9-NEXT:    v_pk_sub_i16 v8, v8, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
+; GFX9-NEXT:    v_pk_min_i16 v9, v0, s6
+; GFX9-NEXT:    v_pk_max_i16 v4, v8, v4
+; GFX9-NEXT:    v_pk_sub_i16 v9, v9, s5
+; GFX9-NEXT:    v_pk_min_i16 v4, v4, v9
+; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v4
+; GFX9-NEXT:    v_pk_max_i16 v4, v1, s6
+; GFX9-NEXT:    v_pk_sub_i16 v4, v4, s4
+; GFX9-NEXT:    v_pk_min_i16 v8, v1, s6
+; GFX9-NEXT:    v_pk_max_i16 v4, v4, v5
+; GFX9-NEXT:    v_pk_sub_i16 v8, v8, s5
+; GFX9-NEXT:    v_pk_min_i16 v4, v4, v8
+; GFX9-NEXT:    v_pk_sub_i16 v1, v1, v4
+; GFX9-NEXT:    v_pk_max_i16 v4, v2, s6
+; GFX9-NEXT:    v_pk_sub_i16 v4, v4, s4
+; GFX9-NEXT:    v_pk_min_i16 v5, v2, s6
+; GFX9-NEXT:    v_pk_sub_i16 v5, v5, s5
+; GFX9-NEXT:    v_pk_max_i16 v4, v4, v6
+; GFX9-NEXT:    v_pk_min_i16 v4, v4, v5
+; GFX9-NEXT:    v_pk_sub_i16 v2, v2, v4
+; GFX9-NEXT:    v_pk_max_i16 v4, v3, s6
+; GFX9-NEXT:    v_pk_sub_i16 v4, v4, s4
+; GFX9-NEXT:    v_pk_min_i16 v5, v3, s6
+; GFX9-NEXT:    v_pk_sub_i16 v5, v5, s5
+; GFX9-NEXT:    v_pk_max_i16 v4, v4, v7
+; GFX9-NEXT:    v_pk_min_i16 v4, v4, v5
+; GFX9-NEXT:    v_pk_sub_i16 v3, v3, v4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_ssubsat_v8i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
+; GFX10-NEXT:    s_movk_i32 s5, 0x7fff
+; GFX10-NEXT:    v_pk_max_i16 v8, v0, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s5, s5
+; GFX10-NEXT:    v_pk_max_i16 v10, v1, s4
+; GFX10-NEXT:    v_pk_max_i16 v12, v3, s4
+; GFX10-NEXT:    v_pk_min_i16 v9, v0, s4
+; GFX10-NEXT:    v_pk_sub_i16 v15, v8, s5
+; GFX10-NEXT:    v_pk_max_i16 v8, v2, s4
+; GFX10-NEXT:    v_pk_sub_i16 v10, v10, s5
+; GFX10-NEXT:    v_pk_sub_i16 v12, v12, s5
+; GFX10-NEXT:    v_pk_min_i16 v11, v1, s4
+; GFX10-NEXT:    v_pk_min_i16 v13, v2, s4
+; GFX10-NEXT:    v_pk_sub_i16 v8, v8, s5
+; GFX10-NEXT:    v_pk_min_i16 v14, v3, s4
+; GFX10-NEXT:    s_mov_b32 s6, 0xffff8000
+; GFX10-NEXT:    v_pk_max_i16 v4, v15, v4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s6, s6
+; GFX10-NEXT:    v_pk_max_i16 v5, v10, v5
+; GFX10-NEXT:    v_pk_sub_i16 v11, v11, s6
+; GFX10-NEXT:    v_pk_sub_i16 v9, v9, s6
+; GFX10-NEXT:    v_pk_max_i16 v15, v8, v6
+; GFX10-NEXT:    v_pk_sub_i16 v10, v13, s6
+; GFX10-NEXT:    v_pk_sub_i16 v8, v14, s6
+; GFX10-NEXT:    v_pk_max_i16 v7, v12, v7
+; GFX10-NEXT:    v_pk_min_i16 v19, v4, v9
+; GFX10-NEXT:    v_pk_min_i16 v11, v5, v11
+; GFX10-NEXT:    v_pk_min_i16 v15, v15, v10
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_pk_min_i16 v6, v7, v8
+; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v19
+; GFX10-NEXT:    v_pk_sub_i16 v1, v1, v11
+; GFX10-NEXT:    v_pk_sub_i16 v2, v2, v15
+; GFX10-NEXT:    v_pk_sub_i16 v3, v3, v6
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %lhs, <8 x i16> %rhs)
+  %cast = bitcast <8 x i16> %result to <4 x float>
+  ret <4 x float> %cast
+}
+
+define amdgpu_ps <4 x i32> @s_ssubsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inreg %rhs) {
+; GFX6-LABEL: s_ssubsat_v8i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s8, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s0, -1
+; GFX6-NEXT:    s_brev_b32 s16, -2
+; GFX6-NEXT:    s_cselect_b32 s18, s0, -1
+; GFX6-NEXT:    s_sub_i32 s18, s18, s16
+; GFX6-NEXT:    s_cmp_lt_i32 s0, -1
+; GFX6-NEXT:    s_mov_b32 s17, 0x80000000
+; GFX6-NEXT:    s_cselect_b32 s19, s0, -1
+; GFX6-NEXT:    s_sub_i32 s19, s19, s17
+; GFX6-NEXT:    s_cmp_gt_i32 s18, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s18, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s8, s19
+; GFX6-NEXT:    s_cselect_b32 s8, s8, s19
+; GFX6-NEXT:    s_sub_i32 s0, s0, s8
+; GFX6-NEXT:    s_ashr_i32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s9, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s1, -1
+; GFX6-NEXT:    s_cselect_b32 s9, s1, -1
+; GFX6-NEXT:    s_sub_i32 s9, s9, s16
+; GFX6-NEXT:    s_cmp_lt_i32 s1, -1
+; GFX6-NEXT:    s_cselect_b32 s18, s1, -1
+; GFX6-NEXT:    s_sub_i32 s18, s18, s17
+; GFX6-NEXT:    s_cmp_gt_i32 s9, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s9, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s8, s18
+; GFX6-NEXT:    s_cselect_b32 s8, s8, s18
+; GFX6-NEXT:    s_sub_i32 s1, s1, s8
+; GFX6-NEXT:    s_ashr_i32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s10, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s2, -1
+; GFX6-NEXT:    s_cselect_b32 s9, s2, -1
+; GFX6-NEXT:    s_sub_i32 s9, s9, s16
+; GFX6-NEXT:    s_cmp_lt_i32 s2, -1
+; GFX6-NEXT:    s_cselect_b32 s10, s2, -1
+; GFX6-NEXT:    s_sub_i32 s10, s10, s17
+; GFX6-NEXT:    s_cmp_gt_i32 s9, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s9, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s8, s10
+; GFX6-NEXT:    s_cselect_b32 s8, s8, s10
+; GFX6-NEXT:    s_sub_i32 s2, s2, s8
+; GFX6-NEXT:    s_ashr_i32 s2, s2, 16
+; GFX6-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s11, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s3, -1
+; GFX6-NEXT:    s_cselect_b32 s9, s3, -1
+; GFX6-NEXT:    s_sub_i32 s9, s9, s16
+; GFX6-NEXT:    s_cmp_lt_i32 s3, -1
+; GFX6-NEXT:    s_cselect_b32 s10, s3, -1
+; GFX6-NEXT:    s_sub_i32 s10, s10, s17
+; GFX6-NEXT:    s_cmp_gt_i32 s9, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s9, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s8, s10
+; GFX6-NEXT:    s_cselect_b32 s8, s8, s10
+; GFX6-NEXT:    s_sub_i32 s3, s3, s8
+; GFX6-NEXT:    s_ashr_i32 s3, s3, 16
+; GFX6-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s12, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s4, -1
+; GFX6-NEXT:    s_cselect_b32 s9, s4, -1
+; GFX6-NEXT:    s_sub_i32 s9, s9, s16
+; GFX6-NEXT:    s_cmp_lt_i32 s4, -1
+; GFX6-NEXT:    s_cselect_b32 s10, s4, -1
+; GFX6-NEXT:    s_sub_i32 s10, s10, s17
+; GFX6-NEXT:    s_cmp_gt_i32 s9, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s9, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s8, s10
+; GFX6-NEXT:    s_cselect_b32 s8, s8, s10
+; GFX6-NEXT:    s_sub_i32 s4, s4, s8
+; GFX6-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX6-NEXT:    s_lshl_b32 s5, s5, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s13, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s5, -1
+; GFX6-NEXT:    s_cselect_b32 s9, s5, -1
+; GFX6-NEXT:    s_sub_i32 s9, s9, s16
+; GFX6-NEXT:    s_cmp_lt_i32 s5, -1
+; GFX6-NEXT:    s_cselect_b32 s10, s5, -1
+; GFX6-NEXT:    s_sub_i32 s10, s10, s17
+; GFX6-NEXT:    s_cmp_gt_i32 s9, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s9, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s8, s10
+; GFX6-NEXT:    s_cselect_b32 s8, s8, s10
+; GFX6-NEXT:    s_sub_i32 s5, s5, s8
+; GFX6-NEXT:    s_ashr_i32 s5, s5, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s6, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s14, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s6, -1
+; GFX6-NEXT:    s_cselect_b32 s9, s6, -1
+; GFX6-NEXT:    s_sub_i32 s9, s9, s16
+; GFX6-NEXT:    s_cmp_lt_i32 s6, -1
+; GFX6-NEXT:    s_cselect_b32 s10, s6, -1
+; GFX6-NEXT:    s_sub_i32 s10, s10, s17
+; GFX6-NEXT:    s_cmp_gt_i32 s9, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s9, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s8, s10
+; GFX6-NEXT:    s_cselect_b32 s8, s8, s10
+; GFX6-NEXT:    s_sub_i32 s6, s6, s8
+; GFX6-NEXT:    s_ashr_i32 s6, s6, 16
+; GFX6-NEXT:    s_lshl_b32 s7, s7, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s15, 16
+; GFX6-NEXT:    s_cmp_gt_i32 s7, -1
+; GFX6-NEXT:    s_cselect_b32 s9, s7, -1
+; GFX6-NEXT:    s_sub_i32 s9, s9, s16
+; GFX6-NEXT:    s_cmp_lt_i32 s7, -1
+; GFX6-NEXT:    s_cselect_b32 s10, s7, -1
+; GFX6-NEXT:    s_sub_i32 s10, s10, s17
+; GFX6-NEXT:    s_cmp_gt_i32 s9, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s9, s8
+; GFX6-NEXT:    s_cmp_lt_i32 s8, s10
+; GFX6-NEXT:    s_cselect_b32 s8, s8, s10
+; GFX6-NEXT:    s_sub_i32 s7, s7, s8
+; GFX6-NEXT:    s_mov_b32 s8, 0xffff
+; GFX6-NEXT:    s_and_b32 s1, s1, s8
+; GFX6-NEXT:    s_and_b32 s0, s0, s8
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_and_b32 s1, s2, s8
+; GFX6-NEXT:    s_and_b32 s2, s3, s8
+; GFX6-NEXT:    s_and_b32 s3, s5, s8
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_ashr_i32 s7, s7, 16
+; GFX6-NEXT:    s_or_b32 s1, s1, s2
+; GFX6-NEXT:    s_and_b32 s2, s4, s8
+; GFX6-NEXT:    s_and_b32 s4, s7, s8
+; GFX6-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX6-NEXT:    s_or_b32 s2, s2, s3
+; GFX6-NEXT:    s_and_b32 s3, s6, s8
+; GFX6-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX6-NEXT:    s_or_b32 s3, s3, s4
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_ssubsat_v8i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_lshr_b32 s12, s4, 16
+; GFX8-NEXT:    s_lshr_b32 s13, s5, 16
+; GFX8-NEXT:    s_lshr_b32 s14, s6, 16
+; GFX8-NEXT:    s_lshr_b32 s15, s7, 16
+; GFX8-NEXT:    s_lshr_b32 s8, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s9, s1, 16
+; GFX8-NEXT:    s_lshr_b32 s10, s2, 16
+; GFX8-NEXT:    s_lshr_b32 s11, s3, 16
+; GFX8-NEXT:    s_sext_i32_i16 s18, s0
+; GFX8-NEXT:    s_sext_i32_i16 s19, 0xffff
+; GFX8-NEXT:    s_cmp_gt_i32 s18, s19
+; GFX8-NEXT:    s_movk_i32 s16, 0x7fff
+; GFX8-NEXT:    s_cselect_b32 s20, s18, s19
+; GFX8-NEXT:    s_sub_i32 s20, s20, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s18, s19
+; GFX8-NEXT:    s_mov_b32 s17, 0x8000
+; GFX8-NEXT:    s_cselect_b32 s18, s18, s19
+; GFX8-NEXT:    s_sub_i32 s18, s18, s17
+; GFX8-NEXT:    s_sext_i32_i16 s20, s20
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_cmp_gt_i32 s20, s4
+; GFX8-NEXT:    s_cselect_b32 s4, s20, s4
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_sext_i32_i16 s18, s18
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s18
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s18
+; GFX8-NEXT:    s_sub_i32 s0, s0, s4
+; GFX8-NEXT:    s_sext_i32_i16 s4, s8
+; GFX8-NEXT:    s_cmp_gt_i32 s4, s19
+; GFX8-NEXT:    s_cselect_b32 s18, s4, s19
+; GFX8-NEXT:    s_sub_i32 s18, s18, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s4, s19
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s19
+; GFX8-NEXT:    s_sub_i32 s4, s4, s17
+; GFX8-NEXT:    s_sext_i32_i16 s18, s18
+; GFX8-NEXT:    s_sext_i32_i16 s12, s12
+; GFX8-NEXT:    s_cmp_gt_i32 s18, s12
+; GFX8-NEXT:    s_cselect_b32 s12, s18, s12
+; GFX8-NEXT:    s_sext_i32_i16 s12, s12
+; GFX8-NEXT:    s_sext_i32_i16 s4, s4
+; GFX8-NEXT:    s_cmp_lt_i32 s12, s4
+; GFX8-NEXT:    s_cselect_b32 s4, s12, s4
+; GFX8-NEXT:    s_sub_i32 s4, s8, s4
+; GFX8-NEXT:    s_sext_i32_i16 s8, s1
+; GFX8-NEXT:    s_cmp_gt_i32 s8, s19
+; GFX8-NEXT:    s_cselect_b32 s12, s8, s19
+; GFX8-NEXT:    s_sub_i32 s12, s12, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s8, s19
+; GFX8-NEXT:    s_cselect_b32 s8, s8, s19
+; GFX8-NEXT:    s_sub_i32 s8, s8, s17
+; GFX8-NEXT:    s_sext_i32_i16 s12, s12
+; GFX8-NEXT:    s_sext_i32_i16 s5, s5
+; GFX8-NEXT:    s_cmp_gt_i32 s12, s5
+; GFX8-NEXT:    s_cselect_b32 s5, s12, s5
+; GFX8-NEXT:    s_sext_i32_i16 s5, s5
+; GFX8-NEXT:    s_sext_i32_i16 s8, s8
+; GFX8-NEXT:    s_cmp_lt_i32 s5, s8
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s8
+; GFX8-NEXT:    s_sub_i32 s1, s1, s5
+; GFX8-NEXT:    s_sext_i32_i16 s5, s9
+; GFX8-NEXT:    s_cmp_gt_i32 s5, s19
+; GFX8-NEXT:    s_cselect_b32 s8, s5, s19
+; GFX8-NEXT:    s_sub_i32 s8, s8, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s5, s19
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s19
+; GFX8-NEXT:    s_sub_i32 s5, s5, s17
+; GFX8-NEXT:    s_sext_i32_i16 s8, s8
+; GFX8-NEXT:    s_sext_i32_i16 s12, s13
+; GFX8-NEXT:    s_cmp_gt_i32 s8, s12
+; GFX8-NEXT:    s_cselect_b32 s8, s8, s12
+; GFX8-NEXT:    s_sext_i32_i16 s8, s8
+; GFX8-NEXT:    s_sext_i32_i16 s5, s5
+; GFX8-NEXT:    s_cmp_lt_i32 s8, s5
+; GFX8-NEXT:    s_cselect_b32 s5, s8, s5
+; GFX8-NEXT:    s_sub_i32 s5, s9, s5
+; GFX8-NEXT:    s_sext_i32_i16 s8, s2
+; GFX8-NEXT:    s_cmp_gt_i32 s8, s19
+; GFX8-NEXT:    s_cselect_b32 s9, s8, s19
+; GFX8-NEXT:    s_sub_i32 s9, s9, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s8, s19
+; GFX8-NEXT:    s_cselect_b32 s8, s8, s19
+; GFX8-NEXT:    s_sub_i32 s8, s8, s17
+; GFX8-NEXT:    s_sext_i32_i16 s9, s9
+; GFX8-NEXT:    s_sext_i32_i16 s6, s6
+; GFX8-NEXT:    s_cmp_gt_i32 s9, s6
+; GFX8-NEXT:    s_cselect_b32 s6, s9, s6
+; GFX8-NEXT:    s_sext_i32_i16 s6, s6
+; GFX8-NEXT:    s_sext_i32_i16 s8, s8
+; GFX8-NEXT:    s_cmp_lt_i32 s6, s8
+; GFX8-NEXT:    s_cselect_b32 s6, s6, s8
+; GFX8-NEXT:    s_sub_i32 s2, s2, s6
+; GFX8-NEXT:    s_sext_i32_i16 s6, s10
+; GFX8-NEXT:    s_cmp_gt_i32 s6, s19
+; GFX8-NEXT:    s_cselect_b32 s8, s6, s19
+; GFX8-NEXT:    s_sub_i32 s8, s8, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s6, s19
+; GFX8-NEXT:    s_cselect_b32 s6, s6, s19
+; GFX8-NEXT:    s_sub_i32 s6, s6, s17
+; GFX8-NEXT:    s_sext_i32_i16 s8, s8
+; GFX8-NEXT:    s_sext_i32_i16 s9, s14
+; GFX8-NEXT:    s_cmp_gt_i32 s8, s9
+; GFX8-NEXT:    s_cselect_b32 s8, s8, s9
+; GFX8-NEXT:    s_sext_i32_i16 s8, s8
+; GFX8-NEXT:    s_sext_i32_i16 s6, s6
+; GFX8-NEXT:    s_cmp_lt_i32 s8, s6
+; GFX8-NEXT:    s_cselect_b32 s6, s8, s6
+; GFX8-NEXT:    s_sub_i32 s6, s10, s6
+; GFX8-NEXT:    s_sext_i32_i16 s8, s3
+; GFX8-NEXT:    s_cmp_gt_i32 s8, s19
+; GFX8-NEXT:    s_cselect_b32 s9, s8, s19
+; GFX8-NEXT:    s_sub_i32 s9, s9, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s8, s19
+; GFX8-NEXT:    s_cselect_b32 s8, s8, s19
+; GFX8-NEXT:    s_sub_i32 s8, s8, s17
+; GFX8-NEXT:    s_sext_i32_i16 s9, s9
+; GFX8-NEXT:    s_sext_i32_i16 s7, s7
+; GFX8-NEXT:    s_cmp_gt_i32 s9, s7
+; GFX8-NEXT:    s_cselect_b32 s7, s9, s7
+; GFX8-NEXT:    s_sext_i32_i16 s7, s7
+; GFX8-NEXT:    s_sext_i32_i16 s8, s8
+; GFX8-NEXT:    s_cmp_lt_i32 s7, s8
+; GFX8-NEXT:    s_cselect_b32 s7, s7, s8
+; GFX8-NEXT:    s_sub_i32 s3, s3, s7
+; GFX8-NEXT:    s_sext_i32_i16 s7, s11
+; GFX8-NEXT:    s_cmp_gt_i32 s7, s19
+; GFX8-NEXT:    s_cselect_b32 s8, s7, s19
+; GFX8-NEXT:    s_sub_i32 s8, s8, s16
+; GFX8-NEXT:    s_cmp_lt_i32 s7, s19
+; GFX8-NEXT:    s_cselect_b32 s7, s7, s19
+; GFX8-NEXT:    s_sub_i32 s7, s7, s17
+; GFX8-NEXT:    s_sext_i32_i16 s8, s8
+; GFX8-NEXT:    s_sext_i32_i16 s9, s15
+; GFX8-NEXT:    s_cmp_gt_i32 s8, s9
+; GFX8-NEXT:    s_cselect_b32 s8, s8, s9
+; GFX8-NEXT:    s_sext_i32_i16 s8, s8
+; GFX8-NEXT:    s_sext_i32_i16 s7, s7
+; GFX8-NEXT:    s_cmp_lt_i32 s8, s7
+; GFX8-NEXT:    s_cselect_b32 s7, s8, s7
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX8-NEXT:    s_or_b32 s0, s0, s4
+; GFX8-NEXT:    s_bfe_u32 s4, s5, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX8-NEXT:    s_or_b32 s1, s1, s4
+; GFX8-NEXT:    s_bfe_u32 s4, s6, 0x100000
+; GFX8-NEXT:    s_sub_i32 s7, s11, s7
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX8-NEXT:    s_or_b32 s2, s2, s4
+; GFX8-NEXT:    s_bfe_u32 s4, s7, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX8-NEXT:    s_or_b32 s3, s3, s4
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_ssubsat_v8i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_pack_ll_b32_b16 s10, -1, -1
+; GFX9-NEXT:    s_sext_i32_i16 s13, s10
+; GFX9-NEXT:    s_sext_i32_i16 s11, s0
+; GFX9-NEXT:    s_ashr_i32 s12, s0, 16
+; GFX9-NEXT:    s_ashr_i32 s10, s10, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s11, s13
+; GFX9-NEXT:    s_cselect_b32 s14, s11, s13
+; GFX9-NEXT:    s_cmp_gt_i32 s12, s10
+; GFX9-NEXT:    s_movk_i32 s8, 0x7fff
+; GFX9-NEXT:    s_cselect_b32 s15, s12, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s14, s14, s15
+; GFX9-NEXT:    s_pack_ll_b32_b16 s8, s8, s8
+; GFX9-NEXT:    s_lshr_b32 s15, s14, 16
+; GFX9-NEXT:    s_lshr_b32 s16, s8, 16
+; GFX9-NEXT:    s_sub_i32 s14, s14, s8
+; GFX9-NEXT:    s_sub_i32 s15, s15, s16
+; GFX9-NEXT:    s_cmp_lt_i32 s11, s13
+; GFX9-NEXT:    s_cselect_b32 s11, s11, s13
+; GFX9-NEXT:    s_cmp_lt_i32 s12, s10
+; GFX9-NEXT:    s_mov_b32 s9, 0xffff8000
+; GFX9-NEXT:    s_cselect_b32 s12, s12, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s12
+; GFX9-NEXT:    s_pack_ll_b32_b16 s9, s9, s9
+; GFX9-NEXT:    s_pack_ll_b32_b16 s14, s14, s15
+; GFX9-NEXT:    s_lshr_b32 s12, s11, 16
+; GFX9-NEXT:    s_lshr_b32 s15, s9, 16
+; GFX9-NEXT:    s_sub_i32 s11, s11, s9
+; GFX9-NEXT:    s_sub_i32 s12, s12, s15
+; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s12
+; GFX9-NEXT:    s_sext_i32_i16 s12, s14
+; GFX9-NEXT:    s_sext_i32_i16 s17, s4
+; GFX9-NEXT:    s_ashr_i32 s14, s14, 16
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s12, s17
+; GFX9-NEXT:    s_cselect_b32 s12, s12, s17
+; GFX9-NEXT:    s_cmp_gt_i32 s14, s4
+; GFX9-NEXT:    s_cselect_b32 s4, s14, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s12, s4
+; GFX9-NEXT:    s_sext_i32_i16 s12, s4
+; GFX9-NEXT:    s_sext_i32_i16 s14, s11
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX9-NEXT:    s_ashr_i32 s11, s11, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s12, s14
+; GFX9-NEXT:    s_cselect_b32 s12, s12, s14
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s11
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s11
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s12, s4
+; GFX9-NEXT:    s_lshr_b32 s11, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s12, s4, 16
+; GFX9-NEXT:    s_sub_i32 s0, s0, s4
+; GFX9-NEXT:    s_sub_i32 s4, s11, s12
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s4
+; GFX9-NEXT:    s_sext_i32_i16 s4, s1
+; GFX9-NEXT:    s_ashr_i32 s11, s1, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s13
+; GFX9-NEXT:    s_cselect_b32 s12, s4, s13
+; GFX9-NEXT:    s_cmp_gt_i32 s11, s10
+; GFX9-NEXT:    s_cselect_b32 s14, s11, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s12, s12, s14
+; GFX9-NEXT:    s_lshr_b32 s14, s12, 16
+; GFX9-NEXT:    s_sub_i32 s12, s12, s8
+; GFX9-NEXT:    s_sub_i32 s14, s14, s16
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s13
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s13
+; GFX9-NEXT:    s_cmp_lt_i32 s11, s10
+; GFX9-NEXT:    s_cselect_b32 s11, s11, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s11
+; GFX9-NEXT:    s_lshr_b32 s11, s4, 16
+; GFX9-NEXT:    s_pack_ll_b32_b16 s12, s12, s14
+; GFX9-NEXT:    s_sub_i32 s4, s4, s9
+; GFX9-NEXT:    s_sub_i32 s11, s11, s15
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s11
+; GFX9-NEXT:    s_sext_i32_i16 s11, s12
+; GFX9-NEXT:    s_sext_i32_i16 s14, s5
+; GFX9-NEXT:    s_ashr_i32 s12, s12, 16
+; GFX9-NEXT:    s_ashr_i32 s5, s5, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s11, s14
+; GFX9-NEXT:    s_cselect_b32 s11, s11, s14
+; GFX9-NEXT:    s_cmp_gt_i32 s12, s5
+; GFX9-NEXT:    s_cselect_b32 s5, s12, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s11, s5
+; GFX9-NEXT:    s_sext_i32_i16 s11, s5
+; GFX9-NEXT:    s_sext_i32_i16 s12, s4
+; GFX9-NEXT:    s_ashr_i32 s5, s5, 16
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s11, s12
+; GFX9-NEXT:    s_cselect_b32 s11, s11, s12
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s4
+; GFX9-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s11, s4
+; GFX9-NEXT:    s_lshr_b32 s5, s1, 16
+; GFX9-NEXT:    s_lshr_b32 s11, s4, 16
+; GFX9-NEXT:    s_sub_i32 s1, s1, s4
+; GFX9-NEXT:    s_sub_i32 s4, s5, s11
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
+; GFX9-NEXT:    s_sext_i32_i16 s4, s2
+; GFX9-NEXT:    s_ashr_i32 s5, s2, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s13
+; GFX9-NEXT:    s_cselect_b32 s11, s4, s13
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s10
+; GFX9-NEXT:    s_cselect_b32 s12, s5, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s12
+; GFX9-NEXT:    s_lshr_b32 s12, s11, 16
+; GFX9-NEXT:    s_sub_i32 s11, s11, s8
+; GFX9-NEXT:    s_sub_i32 s12, s12, s16
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s13
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s13
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s10
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
+; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
+; GFX9-NEXT:    s_pack_ll_b32_b16 s11, s11, s12
+; GFX9-NEXT:    s_sub_i32 s4, s4, s9
+; GFX9-NEXT:    s_sub_i32 s5, s5, s15
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
+; GFX9-NEXT:    s_sext_i32_i16 s5, s11
+; GFX9-NEXT:    s_sext_i32_i16 s12, s6
+; GFX9-NEXT:    s_ashr_i32 s11, s11, 16
+; GFX9-NEXT:    s_ashr_i32 s6, s6, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s12
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s12
+; GFX9-NEXT:    s_cmp_gt_i32 s11, s6
+; GFX9-NEXT:    s_cselect_b32 s6, s11, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
+; GFX9-NEXT:    s_sext_i32_i16 s6, s5
+; GFX9-NEXT:    s_sext_i32_i16 s11, s4
+; GFX9-NEXT:    s_ashr_i32 s5, s5, 16
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s6, s11
+; GFX9-NEXT:    s_cselect_b32 s6, s6, s11
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s4
+; GFX9-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s6, s4
+; GFX9-NEXT:    s_lshr_b32 s5, s2, 16
+; GFX9-NEXT:    s_lshr_b32 s6, s4, 16
+; GFX9-NEXT:    s_sub_i32 s2, s2, s4
+; GFX9-NEXT:    s_sub_i32 s4, s5, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
+; GFX9-NEXT:    s_sext_i32_i16 s4, s3
+; GFX9-NEXT:    s_ashr_i32 s5, s3, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s4, s13
+; GFX9-NEXT:    s_cselect_b32 s6, s4, s13
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s10
+; GFX9-NEXT:    s_cselect_b32 s11, s5, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s11
+; GFX9-NEXT:    s_lshr_b32 s11, s6, 16
+; GFX9-NEXT:    s_sub_i32 s6, s6, s8
+; GFX9-NEXT:    s_sub_i32 s8, s11, s16
+; GFX9-NEXT:    s_cmp_lt_i32 s4, s13
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s13
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s10
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
+; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, s6, s8
+; GFX9-NEXT:    s_sub_i32 s4, s4, s9
+; GFX9-NEXT:    s_sub_i32 s5, s5, s15
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
+; GFX9-NEXT:    s_sext_i32_i16 s5, s6
+; GFX9-NEXT:    s_sext_i32_i16 s8, s7
+; GFX9-NEXT:    s_ashr_i32 s6, s6, 16
+; GFX9-NEXT:    s_ashr_i32 s7, s7, 16
+; GFX9-NEXT:    s_cmp_gt_i32 s5, s8
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s8
+; GFX9-NEXT:    s_cmp_gt_i32 s6, s7
+; GFX9-NEXT:    s_cselect_b32 s6, s6, s7
+; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
+; GFX9-NEXT:    s_sext_i32_i16 s6, s5
+; GFX9-NEXT:    s_sext_i32_i16 s7, s4
+; GFX9-NEXT:    s_ashr_i32 s5, s5, 16
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX9-NEXT:    s_cmp_lt_i32 s6, s7
+; GFX9-NEXT:    s_cselect_b32 s6, s6, s7
+; GFX9-NEXT:    s_cmp_lt_i32 s5, s4
+; GFX9-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s6, s4
+; GFX9-NEXT:    s_lshr_b32 s5, s3, 16
+; GFX9-NEXT:    s_lshr_b32 s6, s4, 16
+; GFX9-NEXT:    s_sub_i32 s3, s3, s4
+; GFX9-NEXT:    s_sub_i32 s4, s5, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_ssubsat_v8i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_pack_ll_b32_b16 s8, -1, -1
+; GFX10-NEXT:    s_sext_i32_i16 s9, s0
+; GFX10-NEXT:    s_sext_i32_i16 s11, s8
+; GFX10-NEXT:    s_ashr_i32 s10, s0, 16
+; GFX10-NEXT:    s_ashr_i32 s8, s8, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s9, s11
+; GFX10-NEXT:    s_movk_i32 s14, 0x7fff
+; GFX10-NEXT:    s_cselect_b32 s12, s9, s11
+; GFX10-NEXT:    s_cmp_gt_i32 s10, s8
+; GFX10-NEXT:    s_mov_b32 s16, 0xffff8000
+; GFX10-NEXT:    s_cselect_b32 s13, s10, s8
+; GFX10-NEXT:    s_pack_ll_b32_b16 s16, s16, s16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s12, s12, s13
+; GFX10-NEXT:    s_pack_ll_b32_b16 s13, s14, s14
+; GFX10-NEXT:    s_lshr_b32 s14, s12, 16
+; GFX10-NEXT:    s_lshr_b32 s15, s13, 16
+; GFX10-NEXT:    s_sub_i32 s12, s12, s13
+; GFX10-NEXT:    s_sub_i32 s14, s14, s15
+; GFX10-NEXT:    s_cmp_lt_i32 s9, s11
+; GFX10-NEXT:    s_sext_i32_i16 s18, s4
+; GFX10-NEXT:    s_cselect_b32 s9, s9, s11
+; GFX10-NEXT:    s_cmp_lt_i32 s10, s8
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s10, s10, s8
+; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
+; GFX10-NEXT:    s_pack_ll_b32_b16 s10, s12, s14
+; GFX10-NEXT:    s_lshr_b32 s12, s9, 16
+; GFX10-NEXT:    s_lshr_b32 s14, s16, 16
+; GFX10-NEXT:    s_sext_i32_i16 s17, s10
+; GFX10-NEXT:    s_sub_i32 s9, s9, s16
+; GFX10-NEXT:    s_sub_i32 s12, s12, s14
+; GFX10-NEXT:    s_ashr_i32 s10, s10, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s17, s18
+; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s12
+; GFX10-NEXT:    s_cselect_b32 s17, s17, s18
+; GFX10-NEXT:    s_cmp_gt_i32 s10, s4
+; GFX10-NEXT:    s_sext_i32_i16 s12, s9
+; GFX10-NEXT:    s_cselect_b32 s4, s10, s4
+; GFX10-NEXT:    s_ashr_i32 s9, s9, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s17, s4
+; GFX10-NEXT:    s_sext_i32_i16 s18, s5
+; GFX10-NEXT:    s_sext_i32_i16 s10, s4
+; GFX10-NEXT:    s_ashr_i32 s4, s4, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s10, s12
+; GFX10-NEXT:    s_cselect_b32 s10, s10, s12
+; GFX10-NEXT:    s_cmp_lt_i32 s4, s9
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s9
+; GFX10-NEXT:    s_lshr_b32 s9, s0, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s10, s4
+; GFX10-NEXT:    s_lshr_b32 s10, s4, 16
+; GFX10-NEXT:    s_sub_i32 s0, s0, s4
+; GFX10-NEXT:    s_sub_i32 s4, s9, s10
+; GFX10-NEXT:    s_sext_i32_i16 s9, s1
+; GFX10-NEXT:    s_ashr_i32 s10, s1, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s9, s11
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s4
+; GFX10-NEXT:    s_cselect_b32 s12, s9, s11
+; GFX10-NEXT:    s_cmp_gt_i32 s10, s8
+; GFX10-NEXT:    s_cselect_b32 s17, s10, s8
+; GFX10-NEXT:    s_pack_ll_b32_b16 s12, s12, s17
+; GFX10-NEXT:    s_lshr_b32 s17, s12, 16
+; GFX10-NEXT:    s_sub_i32 s12, s12, s13
+; GFX10-NEXT:    s_sub_i32 s17, s17, s15
+; GFX10-NEXT:    s_cmp_lt_i32 s9, s11
+; GFX10-NEXT:    s_cselect_b32 s9, s9, s11
+; GFX10-NEXT:    s_cmp_lt_i32 s10, s8
+; GFX10-NEXT:    s_cselect_b32 s10, s10, s8
+; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
+; GFX10-NEXT:    s_pack_ll_b32_b16 s10, s12, s17
+; GFX10-NEXT:    s_lshr_b32 s12, s9, 16
+; GFX10-NEXT:    s_sext_i32_i16 s17, s10
+; GFX10-NEXT:    s_sub_i32 s9, s9, s16
+; GFX10-NEXT:    s_sub_i32 s12, s12, s14
+; GFX10-NEXT:    s_ashr_i32 s10, s10, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s17, s18
+; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s12
+; GFX10-NEXT:    s_cselect_b32 s17, s17, s18
+; GFX10-NEXT:    s_cmp_gt_i32 s10, s5
+; GFX10-NEXT:    s_sext_i32_i16 s12, s9
+; GFX10-NEXT:    s_cselect_b32 s5, s10, s5
+; GFX10-NEXT:    s_ashr_i32 s9, s9, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s17, s5
+; GFX10-NEXT:    s_sext_i32_i16 s18, s6
+; GFX10-NEXT:    s_sext_i32_i16 s10, s5
+; GFX10-NEXT:    s_ashr_i32 s5, s5, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s10, s12
+; GFX10-NEXT:    s_cselect_b32 s10, s10, s12
+; GFX10-NEXT:    s_cmp_lt_i32 s5, s9
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s9
+; GFX10-NEXT:    s_lshr_b32 s9, s1, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s10, s5
+; GFX10-NEXT:    s_lshr_b32 s10, s5, 16
+; GFX10-NEXT:    s_sub_i32 s1, s1, s5
+; GFX10-NEXT:    s_sub_i32 s5, s9, s10
+; GFX10-NEXT:    s_sext_i32_i16 s9, s2
+; GFX10-NEXT:    s_ashr_i32 s10, s2, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s9, s11
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s5
+; GFX10-NEXT:    s_cselect_b32 s12, s9, s11
+; GFX10-NEXT:    s_cmp_gt_i32 s10, s8
+; GFX10-NEXT:    s_cselect_b32 s17, s10, s8
+; GFX10-NEXT:    s_pack_ll_b32_b16 s12, s12, s17
+; GFX10-NEXT:    s_lshr_b32 s17, s12, 16
+; GFX10-NEXT:    s_sub_i32 s12, s12, s13
+; GFX10-NEXT:    s_sub_i32 s17, s17, s15
+; GFX10-NEXT:    s_cmp_lt_i32 s9, s11
+; GFX10-NEXT:    s_cselect_b32 s9, s9, s11
+; GFX10-NEXT:    s_cmp_lt_i32 s10, s8
+; GFX10-NEXT:    s_cselect_b32 s10, s10, s8
+; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s10
+; GFX10-NEXT:    s_pack_ll_b32_b16 s10, s12, s17
+; GFX10-NEXT:    s_lshr_b32 s12, s9, 16
+; GFX10-NEXT:    s_sext_i32_i16 s17, s10
+; GFX10-NEXT:    s_sub_i32 s9, s9, s16
+; GFX10-NEXT:    s_sub_i32 s12, s12, s14
+; GFX10-NEXT:    s_ashr_i32 s10, s10, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s17, s18
+; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s9, s12
+; GFX10-NEXT:    s_cselect_b32 s17, s17, s18
+; GFX10-NEXT:    s_cmp_gt_i32 s10, s6
+; GFX10-NEXT:    s_sext_i32_i16 s12, s9
+; GFX10-NEXT:    s_cselect_b32 s6, s10, s6
+; GFX10-NEXT:    s_ashr_i32 s9, s9, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s17, s6
+; GFX10-NEXT:    s_sext_i32_i16 s10, s6
+; GFX10-NEXT:    s_ashr_i32 s6, s6, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s10, s12
+; GFX10-NEXT:    s_cselect_b32 s10, s10, s12
+; GFX10-NEXT:    s_cmp_lt_i32 s6, s9
+; GFX10-NEXT:    s_cselect_b32 s6, s6, s9
+; GFX10-NEXT:    s_lshr_b32 s9, s2, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s10, s6
+; GFX10-NEXT:    s_lshr_b32 s10, s6, 16
+; GFX10-NEXT:    s_sub_i32 s2, s2, s6
+; GFX10-NEXT:    s_sub_i32 s6, s9, s10
+; GFX10-NEXT:    s_sext_i32_i16 s9, s3
+; GFX10-NEXT:    s_ashr_i32 s10, s3, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s9, s11
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s6
+; GFX10-NEXT:    s_cselect_b32 s12, s9, s11
+; GFX10-NEXT:    s_cmp_gt_i32 s10, s8
+; GFX10-NEXT:    s_cselect_b32 s17, s10, s8
+; GFX10-NEXT:    s_pack_ll_b32_b16 s12, s12, s17
+; GFX10-NEXT:    s_lshr_b32 s17, s12, 16
+; GFX10-NEXT:    s_sub_i32 s12, s12, s13
+; GFX10-NEXT:    s_sub_i32 s13, s17, s15
+; GFX10-NEXT:    s_cmp_lt_i32 s9, s11
+; GFX10-NEXT:    s_cselect_b32 s9, s9, s11
+; GFX10-NEXT:    s_cmp_lt_i32 s10, s8
+; GFX10-NEXT:    s_cselect_b32 s8, s10, s8
+; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s9, s8
+; GFX10-NEXT:    s_pack_ll_b32_b16 s9, s12, s13
+; GFX10-NEXT:    s_lshr_b32 s10, s8, 16
+; GFX10-NEXT:    s_sext_i32_i16 s11, s9
+; GFX10-NEXT:    s_sext_i32_i16 s12, s7
+; GFX10-NEXT:    s_sub_i32 s8, s8, s16
+; GFX10-NEXT:    s_sub_i32 s10, s10, s14
+; GFX10-NEXT:    s_ashr_i32 s9, s9, 16
+; GFX10-NEXT:    s_ashr_i32 s7, s7, 16
+; GFX10-NEXT:    s_cmp_gt_i32 s11, s12
+; GFX10-NEXT:    s_pack_ll_b32_b16 s8, s8, s10
+; GFX10-NEXT:    s_cselect_b32 s11, s11, s12
+; GFX10-NEXT:    s_cmp_gt_i32 s9, s7
+; GFX10-NEXT:    s_sext_i32_i16 s10, s8
+; GFX10-NEXT:    s_cselect_b32 s7, s9, s7
+; GFX10-NEXT:    s_ashr_i32 s8, s8, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s7, s11, s7
+; GFX10-NEXT:    s_sext_i32_i16 s9, s7
+; GFX10-NEXT:    s_ashr_i32 s7, s7, 16
+; GFX10-NEXT:    s_cmp_lt_i32 s9, s10
+; GFX10-NEXT:    s_cselect_b32 s9, s9, s10
+; GFX10-NEXT:    s_cmp_lt_i32 s7, s8
+; GFX10-NEXT:    s_cselect_b32 s4, s7, s8
+; GFX10-NEXT:    s_lshr_b32 s5, s3, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s9, s4
+; GFX10-NEXT:    s_lshr_b32 s7, s4, 16
+; GFX10-NEXT:    s_sub_i32 s3, s3, s4
+; GFX10-NEXT:    s_sub_i32 s4, s5, s7
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %lhs, <8 x i16> %rhs)
+  %cast = bitcast <8 x i16> %result to <4 x i32>
+  ret <4 x i32> %cast
+}
+
+; FIXME: i48 broken because i48 add broken
+; define i48 @v_ssubsat_i48(i48 %lhs, i48 %rhs) {
+;   %result = call i48 @llvm.ssub.sat.i48(i48 %lhs, i48 %rhs)
+;   ret i48 %result
+; }
+
+; define amdgpu_ps i48 @s_ssubsat_i48(i48 inreg %lhs, i48 inreg %rhs) {
+;   %result = call i48 @llvm.ssub.sat.i48(i48 %lhs, i48 %rhs)
+;   ret i48 %result
+; }
+
+; define amdgpu_ps <2 x float> @ssubsat_i48_sv(i48 inreg %lhs, i48 %rhs) {
+;   %result = call i48 @llvm.ssub.sat.i48(i48 %lhs, i48 %rhs)
+;   %ext.result = zext i48 %result to i64
+;   %cast = bitcast i64 %ext.result to <2 x float>
+;   ret <2 x float> %cast
+; }
+
+; define amdgpu_ps <2 x float> @ssubsat_i48_vs(i48 %lhs, i48 inreg %rhs) {
+;   %result = call i48 @llvm.ssub.sat.i48(i48 %lhs, i48 %rhs)
+;   %ext.result = zext i48 %result to i64
+;   %cast = bitcast i64 %ext.result to <2 x float>
+;   ret <2 x float> %cast
+; }
+
+define i64 @v_ssubsat_i64(i64 %lhs, i64 %rhs) {
+; GFX6-LABEL: v_ssubsat_i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, v0, v2
+; GFX6-NEXT:    v_subb_u32_e32 v5, vcc, v1, v3, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, v[4:5], v[0:1]
+; GFX6-NEXT:    v_cmp_lt_i64_e64 s[4:5], 0, v[2:3]
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 31, v5
+; GFX6-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX6-NEXT:    v_add_i32_e64 v2, s[6:7], 0, v0
+; GFX6-NEXT:    v_addc_u32_e64 v1, s[6:7], v0, v1, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v4, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_u32_e32 v4, vcc, v0, v2
+; GFX8-NEXT:    v_subb_u32_e32 v5, vcc, v1, v3, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, v[4:5], v[0:1]
+; GFX8-NEXT:    v_cmp_lt_i64_e64 s[4:5], 0, v[2:3]
+; GFX8-NEXT:    v_ashrrev_i32_e32 v0, 31, v5
+; GFX8-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX8-NEXT:    v_add_u32_e64 v2, s[6:7], 0, v0
+; GFX8-NEXT:    v_addc_u32_e64 v1, s[6:7], v0, v1, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_co_u32_e32 v4, vcc, v0, v2
+; GFX9-NEXT:    v_subb_co_u32_e32 v5, vcc, v1, v3, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, v[4:5], v[0:1]
+; GFX9-NEXT:    v_cmp_lt_i64_e64 s[4:5], 0, v[2:3]
+; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 31, v5
+; GFX9-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX9-NEXT:    v_add_co_u32_e64 v2, s[6:7], 0, v0
+; GFX9-NEXT:    v_addc_co_u32_e64 v1, s[6:7], v0, v1, s[6:7]
+; GFX9-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_ssubsat_i64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_sub_co_u32_e64 v10, vcc_lo, v0, v2
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_sub_co_ci_u32_e32 v11, vcc_lo, v1, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_i64_e32 vcc_lo, 0, v[2:3]
+; GFX10-NEXT:    v_ashrrev_i32_e32 v6, 31, v11
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s4, v[10:11], v[0:1]
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s5, v6, 0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s5, 0x80000000, v6, s5
+; GFX10-NEXT:    s_xor_b32 vcc_lo, vcc_lo, s4
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v11, v1, vcc_lo
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i64 @llvm.ssub.sat.i64(i64 %lhs, i64 %rhs)
+  ret i64 %result
+}
+
+define amdgpu_ps i64 @s_ssubsat_i64(i64 inreg %lhs, i64 inreg %rhs) {
+; GFX6-LABEL: s_ssubsat_i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_sub_u32 s4, s0, s2
+; GFX6-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX6-NEXT:    s_and_b32 s5, s5, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s5, 0
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_subb_u32 s5, s1, s3
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
+; GFX6-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[2:3], 0
+; GFX6-NEXT:    s_ashr_i32 s2, s5, 31
+; GFX6-NEXT:    s_xor_b64 vcc, s[0:1], vcc
+; GFX6-NEXT:    s_add_u32 s0, s2, 0
+; GFX6-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX6-NEXT:    s_and_b32 s1, s1, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX6-NEXT:    s_addc_u32 s1, s2, 0x80000000
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    v_mov_b32_e32 v2, s1
+; GFX6-NEXT:    v_mov_b32_e32 v3, s5
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_ssubsat_i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_sub_u32 s4, s0, s2
+; GFX8-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX8-NEXT:    s_and_b32 s5, s5, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    s_subb_u32 s5, s1, s3
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
+; GFX8-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[2:3], 0
+; GFX8-NEXT:    s_ashr_i32 s2, s5, 31
+; GFX8-NEXT:    s_xor_b64 vcc, s[0:1], vcc
+; GFX8-NEXT:    s_add_u32 s0, s2, 0
+; GFX8-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX8-NEXT:    s_and_b32 s1, s1, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX8-NEXT:    s_addc_u32 s1, s2, 0x80000000
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    v_mov_b32_e32 v3, s5
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_ssubsat_i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_sub_u32 s4, s0, s2
+; GFX9-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX9-NEXT:    s_and_b32 s5, s5, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s5, 0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    s_subb_u32 s5, s1, s3
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[0:1]
+; GFX9-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[2:3], 0
+; GFX9-NEXT:    s_ashr_i32 s2, s5, 31
+; GFX9-NEXT:    s_xor_b64 vcc, s[0:1], vcc
+; GFX9-NEXT:    s_add_u32 s0, s2, 0
+; GFX9-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX9-NEXT:    s_and_b32 s1, s1, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX9-NEXT:    s_addc_u32 s1, s2, 0x80000000
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v2, vcc
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_ssubsat_i64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_sub_u32 s4, s0, s2
+; GFX10-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX10-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-NEXT:    s_and_b32 s5, s5, 1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cmp_lg_u32 s5, 0
+; GFX10-NEXT:    s_subb_u32 s5, s1, s3
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s0, s[4:5], s[0:1]
+; GFX10-NEXT:    v_cmp_gt_i64_e64 s1, s[2:3], 0
+; GFX10-NEXT:    s_ashr_i32 s2, s5, 31
+; GFX10-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-NEXT:    s_xor_b32 s3, s1, s0
+; GFX10-NEXT:    s_add_u32 s0, s2, 0
+; GFX10-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, s0, s3
+; GFX10-NEXT:    s_and_b32 s1, s1, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX10-NEXT:    s_addc_u32 s1, s2, 0x80000000
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s1, s3
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i64 @llvm.ssub.sat.i64(i64 %lhs, i64 %rhs)
+  ret i64 %result
+}
+
+define amdgpu_ps <2 x float> @ssubsat_i64_sv(i64 inreg %lhs, i64 %rhs) {
+; GFX6-LABEL: ssubsat_i64_sv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v3, s1
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, s0, v0
+; GFX6-NEXT:    v_subb_u32_e32 v3, vcc, v3, v1, vcc
+; GFX6-NEXT:    v_cmp_gt_i64_e32 vcc, s[0:1], v[2:3]
+; GFX6-NEXT:    v_cmp_lt_i64_e64 s[0:1], 0, v[0:1]
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 31, v3
+; GFX6-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX6-NEXT:    v_add_i32_e64 v4, s[2:3], 0, v0
+; GFX6-NEXT:    v_addc_u32_e64 v1, s[2:3], v0, v1, s[2:3]
+; GFX6-NEXT:    s_xor_b64 vcc, s[0:1], vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: ssubsat_i64_sv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, s0, v0
+; GFX8-NEXT:    v_subb_u32_e32 v3, vcc, v3, v1, vcc
+; GFX8-NEXT:    v_cmp_gt_i64_e32 vcc, s[0:1], v[2:3]
+; GFX8-NEXT:    v_cmp_lt_i64_e64 s[0:1], 0, v[0:1]
+; GFX8-NEXT:    v_ashrrev_i32_e32 v0, 31, v3
+; GFX8-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX8-NEXT:    v_add_u32_e64 v4, s[2:3], 0, v0
+; GFX8-NEXT:    v_addc_u32_e64 v1, s[2:3], v0, v1, s[2:3]
+; GFX8-NEXT:    s_xor_b64 vcc, s[0:1], vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: ssubsat_i64_sv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_sub_co_u32_e32 v2, vcc, s0, v0
+; GFX9-NEXT:    v_subb_co_u32_e32 v3, vcc, v3, v1, vcc
+; GFX9-NEXT:    v_cmp_gt_i64_e32 vcc, s[0:1], v[2:3]
+; GFX9-NEXT:    v_cmp_lt_i64_e64 s[0:1], 0, v[0:1]
+; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 31, v3
+; GFX9-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX9-NEXT:    v_add_co_u32_e64 v4, s[2:3], 0, v0
+; GFX9-NEXT:    v_addc_co_u32_e64 v1, s[2:3], v0, v1, s[2:3]
+; GFX9-NEXT:    s_xor_b64 vcc, s[0:1], vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: ssubsat_i64_sv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_sub_co_u32_e64 v2, vcc_lo, s0, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_sub_co_ci_u32_e32 v3, vcc_lo, s1, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_i64_e32 vcc_lo, 0, v[0:1]
+; GFX10-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
+; GFX10-NEXT:    v_cmp_gt_i64_e64 s0, s[0:1], v[2:3]
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s1, v4, 0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s1, 0x80000000, v4, s1
+; GFX10-NEXT:    s_xor_b32 vcc_lo, vcc_lo, s0
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i64 @llvm.ssub.sat.i64(i64 %lhs, i64 %rhs)
+  %cast = bitcast i64 %result to <2 x float>
+  ret <2 x float> %cast
+}
+
+define amdgpu_ps <2 x float> @ssubsat_i64_vs(i64 %lhs, i64 inreg %rhs) {
+; GFX6-LABEL: ssubsat_i64_vs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v3, s1
+; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, s0, v0
+; GFX6-NEXT:    v_subb_u32_e32 v3, vcc, v1, v3, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, v[2:3], v[0:1]
+; GFX6-NEXT:    v_cmp_gt_i64_e64 s[2:3], s[0:1], 0
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 31, v3
+; GFX6-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX6-NEXT:    v_add_i32_e64 v4, s[0:1], 0, v0
+; GFX6-NEXT:    v_addc_u32_e64 v1, s[0:1], v0, v1, s[0:1]
+; GFX6-NEXT:    s_xor_b64 vcc, s[2:3], vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: ssubsat_i64_vs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_subrev_u32_e32 v2, vcc, s0, v0
+; GFX8-NEXT:    v_subb_u32_e32 v3, vcc, v1, v3, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, v[2:3], v[0:1]
+; GFX8-NEXT:    v_cmp_gt_i64_e64 s[2:3], s[0:1], 0
+; GFX8-NEXT:    v_ashrrev_i32_e32 v0, 31, v3
+; GFX8-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX8-NEXT:    v_add_u32_e64 v4, s[0:1], 0, v0
+; GFX8-NEXT:    v_addc_u32_e64 v1, s[0:1], v0, v1, s[0:1]
+; GFX8-NEXT:    s_xor_b64 vcc, s[2:3], vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: ssubsat_i64_vs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_subrev_co_u32_e32 v2, vcc, s0, v0
+; GFX9-NEXT:    v_subb_co_u32_e32 v3, vcc, v1, v3, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, v[2:3], v[0:1]
+; GFX9-NEXT:    v_cmp_gt_i64_e64 s[2:3], s[0:1], 0
+; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 31, v3
+; GFX9-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX9-NEXT:    v_add_co_u32_e64 v4, s[0:1], 0, v0
+; GFX9-NEXT:    v_addc_co_u32_e64 v1, s[0:1], v0, v1, s[0:1]
+; GFX9-NEXT:    s_xor_b64 vcc, s[2:3], vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: ssubsat_i64_vs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_sub_co_u32_e64 v2, vcc_lo, v0, s0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_subrev_co_ci_u32_e32 v3, vcc_lo, s1, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_i64_e64 s1, s[0:1], 0
+; GFX10-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
+; GFX10-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, v4, 0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, 0x80000000, v4, s0
+; GFX10-NEXT:    s_xor_b32 vcc_lo, s1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc_lo
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i64 @llvm.ssub.sat.i64(i64 %lhs, i64 %rhs)
+  %cast = bitcast i64 %result to <2 x float>
+  ret <2 x float> %cast
+}
+
+define <2 x i64> @v_ssubsat_v2i64(<2 x i64> %lhs, <2 x i64> %rhs) {
+; GFX6-LABEL: v_ssubsat_v2i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_sub_i32_e32 v8, vcc, v0, v4
+; GFX6-NEXT:    v_subb_u32_e32 v9, vcc, v1, v5, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, v[8:9], v[0:1]
+; GFX6-NEXT:    v_cmp_lt_i64_e64 s[4:5], 0, v[4:5]
+; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 31, v9
+; GFX6-NEXT:    s_brev_b32 s8, 1
+; GFX6-NEXT:    v_mov_b32_e32 v1, s8
+; GFX6-NEXT:    v_add_i32_e64 v4, s[6:7], 0, v0
+; GFX6-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX6-NEXT:    v_addc_u32_e64 v1, s[6:7], v0, v1, s[6:7]
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v8, v4, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, v2, v6
+; GFX6-NEXT:    v_subb_u32_e32 v5, vcc, v3, v7, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, v[4:5], v[2:3]
+; GFX6-NEXT:    v_cmp_lt_i64_e64 s[4:5], 0, v[6:7]
+; GFX6-NEXT:    v_ashrrev_i32_e32 v2, 31, v5
+; GFX6-NEXT:    v_mov_b32_e32 v3, s8
+; GFX6-NEXT:    v_add_i32_e64 v6, s[6:7], 0, v2
+; GFX6-NEXT:    v_addc_u32_e64 v3, s[6:7], v2, v3, s[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v4, v6, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_v2i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_u32_e32 v8, vcc, v0, v4
+; GFX8-NEXT:    v_subb_u32_e32 v9, vcc, v1, v5, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, v[8:9], v[0:1]
+; GFX8-NEXT:    v_cmp_lt_i64_e64 s[4:5], 0, v[4:5]
+; GFX8-NEXT:    v_ashrrev_i32_e32 v0, 31, v9
+; GFX8-NEXT:    s_brev_b32 s8, 1
+; GFX8-NEXT:    v_mov_b32_e32 v1, s8
+; GFX8-NEXT:    v_add_u32_e64 v4, s[6:7], 0, v0
+; GFX8-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX8-NEXT:    v_addc_u32_e64 v1, s[6:7], v0, v1, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v8, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
+; GFX8-NEXT:    v_sub_u32_e32 v4, vcc, v2, v6
+; GFX8-NEXT:    v_subb_u32_e32 v5, vcc, v3, v7, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, v[4:5], v[2:3]
+; GFX8-NEXT:    v_cmp_lt_i64_e64 s[4:5], 0, v[6:7]
+; GFX8-NEXT:    v_ashrrev_i32_e32 v2, 31, v5
+; GFX8-NEXT:    v_mov_b32_e32 v3, s8
+; GFX8-NEXT:    v_add_u32_e64 v6, s[6:7], 0, v2
+; GFX8-NEXT:    v_addc_u32_e64 v3, s[6:7], v2, v3, s[6:7]
+; GFX8-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v4, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_v2i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_co_u32_e32 v8, vcc, v0, v4
+; GFX9-NEXT:    v_subb_co_u32_e32 v9, vcc, v1, v5, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, v[8:9], v[0:1]
+; GFX9-NEXT:    v_cmp_lt_i64_e64 s[4:5], 0, v[4:5]
+; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 31, v9
+; GFX9-NEXT:    s_brev_b32 s8, 1
+; GFX9-NEXT:    v_mov_b32_e32 v1, s8
+; GFX9-NEXT:    v_add_co_u32_e64 v4, s[6:7], 0, v0
+; GFX9-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX9-NEXT:    v_addc_co_u32_e64 v1, s[6:7], v0, v1, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v8, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
+; GFX9-NEXT:    v_sub_co_u32_e32 v4, vcc, v2, v6
+; GFX9-NEXT:    v_subb_co_u32_e32 v5, vcc, v3, v7, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, v[4:5], v[2:3]
+; GFX9-NEXT:    v_cmp_lt_i64_e64 s[4:5], 0, v[6:7]
+; GFX9-NEXT:    v_ashrrev_i32_e32 v2, 31, v5
+; GFX9-NEXT:    v_mov_b32_e32 v3, s8
+; GFX9-NEXT:    v_add_co_u32_e64 v6, s[6:7], 0, v2
+; GFX9-NEXT:    v_addc_co_u32_e64 v3, s[6:7], v2, v3, s[6:7]
+; GFX9-NEXT:    s_xor_b64 vcc, s[4:5], vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_ssubsat_v2i64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_mov_b32_e32 v9, v0
+; GFX10-NEXT:    v_mov_b32_e32 v10, v1
+; GFX10-NEXT:    v_mov_b32_e32 v13, v2
+; GFX10-NEXT:    v_mov_b32_e32 v14, v3
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s4, 0, v[4:5]
+; GFX10-NEXT:    v_sub_co_u32_e64 v19, vcc_lo, v9, v4
+; GFX10-NEXT:    s_brev_b32 s8, 1
+; GFX10-NEXT:    v_sub_co_ci_u32_e32 v20, vcc_lo, v10, v5, vcc_lo
+; GFX10-NEXT:    v_sub_co_u32_e64 v23, vcc_lo, v13, v6
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s6, 0, v[6:7]
+; GFX10-NEXT:    v_sub_co_ci_u32_e32 v24, vcc_lo, v14, v7, vcc_lo
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_ashrrev_i32_e32 v0, 31, v20
+; GFX10-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[19:20], v[9:10]
+; GFX10-NEXT:    v_ashrrev_i32_e32 v1, 31, v24
+; GFX10-NEXT:    v_add_co_u32_e64 v4, s5, v0, 0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v5, s5, s8, v0, s5
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s5, v[23:24], v[13:14]
+; GFX10-NEXT:    v_add_co_u32_e64 v2, s7, v1, 0
+; GFX10-NEXT:    s_xor_b32 vcc_lo, s4, vcc_lo
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, s7, s8, v1, s7
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v19, v4, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v20, v5, vcc_lo
+; GFX10-NEXT:    s_xor_b32 vcc_lo, s6, s5
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v23, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v24, v3, vcc_lo
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> %lhs, <2 x i64> %rhs)
+  ret <2 x i64> %result
+}
+
+define amdgpu_ps <2 x i64> @s_ssubsat_v2i64(<2 x i64> inreg %lhs, <2 x i64> inreg %rhs) {
+; GFX6-LABEL: s_ssubsat_v2i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_sub_u32 s8, s0, s4
+; GFX6-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX6-NEXT:    s_and_b32 s9, s9, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_subb_u32 s9, s1, s5
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[4:5], 0
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, s[8:9], v[0:1]
+; GFX6-NEXT:    s_ashr_i32 s4, s9, 31
+; GFX6-NEXT:    s_xor_b64 vcc, s[0:1], vcc
+; GFX6-NEXT:    s_add_u32 s0, s4, 0
+; GFX6-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX6-NEXT:    s_and_b32 s1, s1, 1
+; GFX6-NEXT:    s_brev_b32 s5, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX6-NEXT:    s_addc_u32 s1, s4, s5
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_sub_u32 s0, s2, s6
+; GFX6-NEXT:    v_mov_b32_e32 v2, s1
+; GFX6-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX6-NEXT:    s_and_b32 s1, s1, 1
+; GFX6-NEXT:    v_mov_b32_e32 v0, s8
+; GFX6-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, v0, v1, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    s_subb_u32 s1, s3, s7
+; GFX6-NEXT:    v_mov_b32_e32 v3, s9
+; GFX6-NEXT:    v_mov_b32_e32 v1, s3
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, s[0:1], v[0:1]
+; GFX6-NEXT:    v_cmp_gt_i64_e64 s[2:3], s[6:7], 0
+; GFX6-NEXT:    s_ashr_i32 s4, s1, 31
+; GFX6-NEXT:    s_xor_b64 vcc, s[2:3], vcc
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_add_u32 s0, s4, 0
+; GFX6-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX6-NEXT:    s_and_b32 s2, s2, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX6-NEXT:    s_addc_u32 s3, s4, s5
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    v_mov_b32_e32 v5, s1
+; GFX6-NEXT:    v_mov_b32_e32 v3, s3
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX6-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX6-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s3, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_ssubsat_v2i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_sub_u32 s8, s0, s4
+; GFX8-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX8-NEXT:    s_and_b32 s9, s9, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    s_subb_u32 s9, s1, s5
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[4:5], 0
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[8:9], v[0:1]
+; GFX8-NEXT:    s_ashr_i32 s4, s9, 31
+; GFX8-NEXT:    s_xor_b64 vcc, s[0:1], vcc
+; GFX8-NEXT:    s_add_u32 s0, s4, 0
+; GFX8-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX8-NEXT:    s_and_b32 s1, s1, 1
+; GFX8-NEXT:    s_brev_b32 s5, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX8-NEXT:    s_addc_u32 s1, s4, s5
+; GFX8-NEXT:    v_mov_b32_e32 v1, s0
+; GFX8-NEXT:    s_sub_u32 s0, s2, s6
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX8-NEXT:    s_and_b32 s1, s1, 1
+; GFX8-NEXT:    v_mov_b32_e32 v0, s8
+; GFX8-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v0, v1, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    s_subb_u32 s1, s3, s7
+; GFX8-NEXT:    v_mov_b32_e32 v3, s9
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[0:1], v[0:1]
+; GFX8-NEXT:    v_cmp_gt_i64_e64 s[2:3], s[6:7], 0
+; GFX8-NEXT:    s_ashr_i32 s4, s1, 31
+; GFX8-NEXT:    s_xor_b64 vcc, s[2:3], vcc
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    s_add_u32 s0, s4, 0
+; GFX8-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX8-NEXT:    s_and_b32 s2, s2, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX8-NEXT:    s_addc_u32 s3, s4, s5
+; GFX8-NEXT:    v_mov_b32_e32 v1, s0
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v3, s3
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_ssubsat_v2i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_sub_u32 s8, s0, s4
+; GFX9-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX9-NEXT:    s_and_b32 s9, s9, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    s_subb_u32 s9, s1, s5
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[4:5], 0
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, s[8:9], v[0:1]
+; GFX9-NEXT:    s_ashr_i32 s4, s9, 31
+; GFX9-NEXT:    s_xor_b64 vcc, s[0:1], vcc
+; GFX9-NEXT:    s_add_u32 s0, s4, 0
+; GFX9-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX9-NEXT:    s_and_b32 s1, s1, 1
+; GFX9-NEXT:    s_brev_b32 s5, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX9-NEXT:    s_addc_u32 s1, s4, s5
+; GFX9-NEXT:    v_mov_b32_e32 v1, s0
+; GFX9-NEXT:    s_sub_u32 s0, s2, s6
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX9-NEXT:    s_and_b32 s1, s1, 1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s8
+; GFX9-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v0, v1, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    s_subb_u32 s1, s3, s7
+; GFX9-NEXT:    v_mov_b32_e32 v3, s9
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, s[0:1], v[0:1]
+; GFX9-NEXT:    v_cmp_gt_i64_e64 s[2:3], s[6:7], 0
+; GFX9-NEXT:    s_ashr_i32 s4, s1, 31
+; GFX9-NEXT:    s_xor_b64 vcc, s[2:3], vcc
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    s_add_u32 s0, s4, 0
+; GFX9-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX9-NEXT:    s_and_b32 s2, s2, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX9-NEXT:    s_addc_u32 s3, s4, s5
+; GFX9-NEXT:    v_mov_b32_e32 v1, s0
+; GFX9-NEXT:    v_mov_b32_e32 v5, s1
+; GFX9-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_ssubsat_v2i64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_sub_u32 s8, s0, s4
+; GFX10-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX10-NEXT:    v_cmp_gt_i64_e64 s4, s[4:5], 0
+; GFX10-NEXT:    s_and_b32 s9, s9, 1
+; GFX10-NEXT:    v_mov_b32_e32 v0, s8
+; GFX10-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX10-NEXT:    s_brev_b32 s10, 1
+; GFX10-NEXT:    s_subb_u32 s9, s1, s5
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s0, s[8:9], s[0:1]
+; GFX10-NEXT:    s_ashr_i32 s1, s9, 31
+; GFX10-NEXT:    v_mov_b32_e32 v1, s9
+; GFX10-NEXT:    s_xor_b32 s8, s4, s0
+; GFX10-NEXT:    s_add_u32 s0, s1, 0
+; GFX10-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, s0, s8
+; GFX10-NEXT:    s_and_b32 s4, s4, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX10-NEXT:    s_addc_u32 s1, s1, s10
+; GFX10-NEXT:    s_sub_u32 s4, s2, s6
+; GFX10-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s1, s8
+; GFX10-NEXT:    s_and_b32 s5, s5, 1
+; GFX10-NEXT:    v_mov_b32_e32 v2, s4
+; GFX10-NEXT:    s_cmp_lg_u32 s5, 0
+; GFX10-NEXT:    s_subb_u32 s5, s3, s7
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s2, s[4:5], s[2:3]
+; GFX10-NEXT:    v_cmp_gt_i64_e64 s3, s[6:7], 0
+; GFX10-NEXT:    s_ashr_i32 s1, s5, 31
+; GFX10-NEXT:    v_mov_b32_e32 v3, s5
+; GFX10-NEXT:    s_xor_b32 s2, s3, s2
+; GFX10-NEXT:    s_add_u32 s0, s1, 0
+; GFX10-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s0, s2
+; GFX10-NEXT:    s_and_b32 s3, s3, 1
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    s_cmp_lg_u32 s3, 0
+; GFX10-NEXT:    s_addc_u32 s1, s1, s10
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, s1, s2
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> %lhs, <2 x i64> %rhs)
+  ret <2 x i64> %result
+}
+
+define amdgpu_ps i128 @s_ssubsat_i128(i128 inreg %lhs, i128 inreg %rhs) {
+; GFX6-LABEL: s_ssubsat_i128:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_sub_u32 s8, s0, s4
+; GFX6-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX6-NEXT:    s_and_b32 s9, s9, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX6-NEXT:    s_subb_u32 s9, s1, s5
+; GFX6-NEXT:    s_cselect_b32 s10, 1, 0
+; GFX6-NEXT:    s_and_b32 s10, s10, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX6-NEXT:    s_subb_u32 s10, s2, s6
+; GFX6-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX6-NEXT:    v_mov_b32_e32 v3, s1
+; GFX6-NEXT:    s_and_b32 s11, s11, 1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[8:9], v[2:3]
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    s_subb_u32 s11, s3, s7
+; GFX6-NEXT:    v_mov_b32_e32 v1, s3
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, s[10:11], v[0:1]
+; GFX6-NEXT:    v_cmp_gt_u64_e64 s[0:1], s[4:5], 0
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, s[10:11], v[0:1]
+; GFX6-NEXT:    s_movk_i32 s2, 0x7f
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; GFX6-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[6:7], 0
+; GFX6-NEXT:    v_cmp_eq_u64_e64 vcc, s[6:7], 0
+; GFX6-NEXT:    s_sub_i32 s6, s2, 64
+; GFX6-NEXT:    s_sub_i32 s4, 64, s2
+; GFX6-NEXT:    s_cmp_lt_u32 s2, 64
+; GFX6-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s2, 0
+; GFX6-NEXT:    s_cselect_b32 s13, 1, 0
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX6-NEXT:    s_ashr_i64 s[0:1], s[10:11], s2
+; GFX6-NEXT:    s_lshr_b64 s[2:3], s[8:9], s2
+; GFX6-NEXT:    s_lshl_b64 s[4:5], s[10:11], s4
+; GFX6-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX6-NEXT:    s_ashr_i32 s4, s11, 31
+; GFX6-NEXT:    s_ashr_i64 s[6:7], s[10:11], s6
+; GFX6-NEXT:    s_and_b32 s12, s12, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX6-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[6:7]
+; GFX6-NEXT:    s_and_b32 s6, s13, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s6, 0
+; GFX6-NEXT:    s_cselect_b64 s[2:3], s[8:9], s[2:3]
+; GFX6-NEXT:    s_mov_b32 s5, s4
+; GFX6-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[4:5]
+; GFX6-NEXT:    s_add_u32 s2, s2, 0
+; GFX6-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX6-NEXT:    s_and_b32 s4, s4, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX6-NEXT:    s_addc_u32 s3, s3, 0
+; GFX6-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX6-NEXT:    s_and_b32 s4, s4, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX6-NEXT:    s_addc_u32 s0, s0, 0
+; GFX6-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX6-NEXT:    v_xor_b32_e32 v0, v1, v0
+; GFX6-NEXT:    s_and_b32 s4, s4, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX6-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX6-NEXT:    s_addc_u32 s1, s1, 0x80000000
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v2, s3
+; GFX6-NEXT:    v_mov_b32_e32 v3, s8
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v4, s9
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v3, v1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v4, v2, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    v_mov_b32_e32 v3, s1
+; GFX6-NEXT:    v_mov_b32_e32 v4, s10
+; GFX6-NEXT:    v_mov_b32_e32 v5, s11
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX6-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX6-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_ssubsat_i128:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_sub_u32 s8, s0, s4
+; GFX8-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX8-NEXT:    s_and_b32 s9, s9, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX8-NEXT:    s_subb_u32 s9, s1, s5
+; GFX8-NEXT:    s_cselect_b32 s10, 1, 0
+; GFX8-NEXT:    s_and_b32 s10, s10, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX8-NEXT:    s_subb_u32 s10, s2, s6
+; GFX8-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX8-NEXT:    s_and_b32 s11, s11, 1
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:    s_subb_u32 s11, s3, s7
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[8:9], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    s_cmp_eq_u64 s[10:11], s[2:3]
+; GFX8-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[10:11], v[0:1]
+; GFX8-NEXT:    s_and_b32 s0, 1, s2
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    v_cmp_gt_u64_e64 s[0:1], s[4:5], 0
+; GFX8-NEXT:    s_cmp_eq_u64 s[6:7], 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; GFX8-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[6:7], 0
+; GFX8-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX8-NEXT:    s_and_b32 s0, 1, s2
+; GFX8-NEXT:    s_movk_i32 s2, 0x7f
+; GFX8-NEXT:    s_sub_i32 s6, s2, 64
+; GFX8-NEXT:    s_sub_i32 s4, 64, s2
+; GFX8-NEXT:    s_cmp_lt_u32 s2, 64
+; GFX8-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s2, 0
+; GFX8-NEXT:    s_cselect_b32 s13, 1, 0
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    s_ashr_i64 s[0:1], s[10:11], s2
+; GFX8-NEXT:    s_lshr_b64 s[2:3], s[8:9], s2
+; GFX8-NEXT:    s_lshl_b64 s[4:5], s[10:11], s4
+; GFX8-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX8-NEXT:    s_ashr_i32 s4, s11, 31
+; GFX8-NEXT:    s_ashr_i64 s[6:7], s[10:11], s6
+; GFX8-NEXT:    s_and_b32 s12, s12, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX8-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[6:7]
+; GFX8-NEXT:    s_and_b32 s6, s13, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s6, 0
+; GFX8-NEXT:    s_cselect_b64 s[2:3], s[8:9], s[2:3]
+; GFX8-NEXT:    s_mov_b32 s5, s4
+; GFX8-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[4:5]
+; GFX8-NEXT:    s_add_u32 s2, s2, 0
+; GFX8-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX8-NEXT:    s_and_b32 s4, s4, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX8-NEXT:    s_addc_u32 s3, s3, 0
+; GFX8-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX8-NEXT:    s_and_b32 s4, s4, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX8-NEXT:    s_addc_u32 s0, s0, 0
+; GFX8-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX8-NEXT:    v_xor_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_and_b32 s4, s4, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX8-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX8-NEXT:    s_addc_u32 s1, s1, 0x80000000
+; GFX8-NEXT:    v_mov_b32_e32 v1, s2
+; GFX8-NEXT:    v_mov_b32_e32 v2, s3
+; GFX8-NEXT:    v_mov_b32_e32 v3, s8
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s9
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v4, v2, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s10
+; GFX8-NEXT:    v_mov_b32_e32 v5, s11
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_ssubsat_i128:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_sub_u32 s8, s0, s4
+; GFX9-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX9-NEXT:    s_and_b32 s9, s9, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX9-NEXT:    s_subb_u32 s9, s1, s5
+; GFX9-NEXT:    s_cselect_b32 s10, 1, 0
+; GFX9-NEXT:    s_and_b32 s10, s10, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX9-NEXT:    s_subb_u32 s10, s2, s6
+; GFX9-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX9-NEXT:    s_and_b32 s11, s11, 1
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s0
+; GFX9-NEXT:    s_subb_u32 s11, s3, s7
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[8:9], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    s_cmp_eq_u64 s[10:11], s[2:3]
+; GFX9-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, s[10:11], v[0:1]
+; GFX9-NEXT:    s_and_b32 s0, 1, s2
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    v_cmp_gt_u64_e64 s[0:1], s[4:5], 0
+; GFX9-NEXT:    s_cmp_eq_u64 s[6:7], 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; GFX9-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[6:7], 0
+; GFX9-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX9-NEXT:    s_and_b32 s0, 1, s2
+; GFX9-NEXT:    s_movk_i32 s2, 0x7f
+; GFX9-NEXT:    s_sub_i32 s6, s2, 64
+; GFX9-NEXT:    s_sub_i32 s4, 64, s2
+; GFX9-NEXT:    s_cmp_lt_u32 s2, 64
+; GFX9-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s2, 0
+; GFX9-NEXT:    s_cselect_b32 s13, 1, 0
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    s_ashr_i64 s[0:1], s[10:11], s2
+; GFX9-NEXT:    s_lshr_b64 s[2:3], s[8:9], s2
+; GFX9-NEXT:    s_lshl_b64 s[4:5], s[10:11], s4
+; GFX9-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX9-NEXT:    s_ashr_i32 s4, s11, 31
+; GFX9-NEXT:    s_ashr_i64 s[6:7], s[10:11], s6
+; GFX9-NEXT:    s_and_b32 s12, s12, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX9-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[6:7]
+; GFX9-NEXT:    s_and_b32 s6, s13, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s6, 0
+; GFX9-NEXT:    s_cselect_b64 s[2:3], s[8:9], s[2:3]
+; GFX9-NEXT:    s_mov_b32 s5, s4
+; GFX9-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[4:5]
+; GFX9-NEXT:    s_add_u32 s2, s2, 0
+; GFX9-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX9-NEXT:    s_and_b32 s4, s4, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX9-NEXT:    s_addc_u32 s3, s3, 0
+; GFX9-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX9-NEXT:    s_and_b32 s4, s4, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX9-NEXT:    s_addc_u32 s0, s0, 0
+; GFX9-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    v_xor_b32_e32 v0, v1, v0
+; GFX9-NEXT:    s_and_b32 s4, s4, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    s_addc_u32 s1, s1, 0x80000000
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-NEXT:    v_mov_b32_e32 v3, s8
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    v_mov_b32_e32 v4, s9
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v2, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v2, s0
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_mov_b32_e32 v4, s10
+; GFX9-NEXT:    v_mov_b32_e32 v5, s11
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_ssubsat_i128:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_sub_u32 s8, s0, s4
+; GFX10-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX10-NEXT:    s_movk_i32 s12, 0x7f
+; GFX10-NEXT:    s_and_b32 s9, s9, 1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX10-NEXT:    s_subb_u32 s9, s1, s5
+; GFX10-NEXT:    s_cselect_b32 s10, 1, 0
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s0, s[8:9], s[0:1]
+; GFX10-NEXT:    s_and_b32 s10, s10, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX10-NEXT:    s_subb_u32 s10, s2, s6
+; GFX10-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX10-NEXT:    s_and_b32 s11, s11, 1
+; GFX10-NEXT:    v_mov_b32_e32 v3, s10
+; GFX10-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX10-NEXT:    s_subb_u32 s11, s3, s7
+; GFX10-NEXT:    s_cmp_eq_u64 s[10:11], s[2:3]
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s1, s[10:11], s[2:3]
+; GFX10-NEXT:    s_cselect_b32 s0, 1, 0
+; GFX10-NEXT:    v_mov_b32_e32 v4, s11
+; GFX10-NEXT:    s_and_b32 s0, 1, s0
+; GFX10-NEXT:    s_cmp_eq_u64 s[6:7], 0
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s0
+; GFX10-NEXT:    v_cmp_gt_u64_e64 s0, s[4:5], 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s1
+; GFX10-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX10-NEXT:    s_sub_i32 s13, s12, 64
+; GFX10-NEXT:    s_and_b32 s14, 1, s1
+; GFX10-NEXT:    s_sub_i32 s2, 64, s12
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX10-NEXT:    v_cmp_gt_i64_e64 s0, s[6:7], 0
+; GFX10-NEXT:    s_cmp_lt_u32 s12, 64
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s14
+; GFX10-NEXT:    s_cselect_b32 s15, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s12, 0
+; GFX10-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s0
+; GFX10-NEXT:    s_lshr_b64 s[0:1], s[8:9], s12
+; GFX10-NEXT:    s_lshl_b64 s[2:3], s[10:11], s2
+; GFX10-NEXT:    s_ashr_i64 s[4:5], s[10:11], s12
+; GFX10-NEXT:    s_and_b32 s12, s15, 1
+; GFX10-NEXT:    s_or_b64 s[0:1], s[0:1], s[2:3]
+; GFX10-NEXT:    s_ashr_i32 s2, s11, 31
+; GFX10-NEXT:    s_ashr_i64 s[6:7], s[10:11], s13
+; GFX10-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX10-NEXT:    s_mov_b32 s3, s2
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[6:7]
+; GFX10-NEXT:    s_and_b32 s6, s16, 1
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc_lo
+; GFX10-NEXT:    s_cmp_lg_u32 s6, 0
+; GFX10-NEXT:    v_mov_b32_e32 v2, s9
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[8:9], s[0:1]
+; GFX10-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX10-NEXT:    v_xor_b32_e32 v0, v1, v0
+; GFX10-NEXT:    s_cselect_b64 s[2:3], s[4:5], s[2:3]
+; GFX10-NEXT:    s_add_u32 s0, s0, 0
+; GFX10-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s8
+; GFX10-NEXT:    s_and_b32 s4, s4, 1
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX10-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX10-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX10-NEXT:    s_and_b32 s4, s4, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX10-NEXT:    s_addc_u32 s2, s2, 0
+; GFX10-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v1, s0, vcc_lo
+; GFX10-NEXT:    s_and_b32 s4, s4, 1
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v2, s1, vcc_lo
+; GFX10-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v3, s2, vcc_lo
+; GFX10-NEXT:    s_addc_u32 s3, s3, 0x80000000
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v4, s3, vcc_lo
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i128 @llvm.ssub.sat.i128(i128 %lhs, i128 %rhs)
+  ret i128 %result
+}
+
+define amdgpu_ps <4 x float> @ssubsat_i128_sv(i128 inreg %lhs, i128 %rhs) {
+; GFX6-LABEL: ssubsat_i128_sv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v5, s1
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, s0, v0
+; GFX6-NEXT:    v_subb_u32_e32 v5, vcc, v5, v1, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v6, s2
+; GFX6-NEXT:    v_mov_b32_e32 v7, s3
+; GFX6-NEXT:    v_subb_u32_e32 v6, vcc, v6, v2, vcc
+; GFX6-NEXT:    v_subb_u32_e32 v7, vcc, v7, v3, vcc
+; GFX6-NEXT:    v_cmp_gt_u64_e32 vcc, s[0:1], v[4:5]
+; GFX6-NEXT:    s_movk_i32 s0, 0x7f
+; GFX6-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_gt_i64_e32 vcc, s[2:3], v[6:7]
+; GFX6-NEXT:    s_sub_i32 s1, s0, 64
+; GFX6-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, s[2:3], v[6:7]
+; GFX6-NEXT:    s_sub_i32 s2, 64, s0
+; GFX6-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, 0, v[0:1]
+; GFX6-NEXT:    s_cmp_lt_u32 s0, 64
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, 0, v[2:3]
+; GFX6-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX6-NEXT:    v_lshl_b64 v[2:3], v[6:7], s2
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX6-NEXT:    v_xor_b32_e32 v10, v0, v8
+; GFX6-NEXT:    v_lshr_b64 v[0:1], v[4:5], s0
+; GFX6-NEXT:    s_cmp_eq_u32 s0, 0
+; GFX6-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX6-NEXT:    v_ashr_i64 v[8:9], v[6:7], s0
+; GFX6-NEXT:    s_and_b32 s0, 1, s3
+; GFX6-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX6-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX6-NEXT:    v_ashr_i64 v[0:1], v[6:7], s1
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX6-NEXT:    s_and_b32 s0, 1, s4
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX6-NEXT:    s_and_b32 s0, 1, s3
+; GFX6-NEXT:    v_ashrrev_i32_e32 v11, 31, v7
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v11, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v11, v9, vcc
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, 0, v0
+; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX6-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v3, v8, vcc
+; GFX6-NEXT:    v_and_b32_e32 v8, 1, v10
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: ssubsat_i128_sv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_sub_u32_e32 v4, vcc, s0, v0
+; GFX8-NEXT:    v_subb_u32_e32 v5, vcc, v5, v1, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v6, s2
+; GFX8-NEXT:    v_mov_b32_e32 v7, s3
+; GFX8-NEXT:    v_subb_u32_e32 v6, vcc, v6, v2, vcc
+; GFX8-NEXT:    v_subb_u32_e32 v7, vcc, v7, v3, vcc
+; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[0:1], v[4:5]
+; GFX8-NEXT:    s_movk_i32 s0, 0x7f
+; GFX8-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_gt_i64_e32 vcc, s[2:3], v[6:7]
+; GFX8-NEXT:    s_sub_i32 s1, s0, 64
+; GFX8-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, s[2:3], v[6:7]
+; GFX8-NEXT:    s_sub_i32 s2, 64, s0
+; GFX8-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, 0, v[0:1]
+; GFX8-NEXT:    s_cmp_lt_u32 s0, 64
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, 0, v[2:3]
+; GFX8-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX8-NEXT:    v_lshlrev_b64 v[2:3], s2, v[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT:    v_xor_b32_e32 v10, v0, v8
+; GFX8-NEXT:    v_lshrrev_b64 v[0:1], s0, v[4:5]
+; GFX8-NEXT:    s_cmp_eq_u32 s0, 0
+; GFX8-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX8-NEXT:    v_ashrrev_i64 v[8:9], s0, v[6:7]
+; GFX8-NEXT:    s_and_b32 s0, 1, s3
+; GFX8-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX8-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX8-NEXT:    v_ashrrev_i64 v[0:1], s1, v[6:7]
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    s_and_b32 s0, 1, s4
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    s_and_b32 s0, 1, s3
+; GFX8-NEXT:    v_ashrrev_i32_e32 v11, 31, v7
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v11, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v11, v9, vcc
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 0, v0
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX8-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, v3, v8, vcc
+; GFX8-NEXT:    v_and_b32_e32 v8, 1, v10
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: ssubsat_i128_sv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v5, s1
+; GFX9-NEXT:    v_sub_co_u32_e32 v4, vcc, s0, v0
+; GFX9-NEXT:    v_subb_co_u32_e32 v5, vcc, v5, v1, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v6, s2
+; GFX9-NEXT:    v_mov_b32_e32 v7, s3
+; GFX9-NEXT:    v_subb_co_u32_e32 v6, vcc, v6, v2, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v7, vcc, v7, v3, vcc
+; GFX9-NEXT:    v_cmp_gt_u64_e32 vcc, s[0:1], v[4:5]
+; GFX9-NEXT:    s_movk_i32 s0, 0x7f
+; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_gt_i64_e32 vcc, s[2:3], v[6:7]
+; GFX9-NEXT:    s_sub_i32 s1, s0, 64
+; GFX9-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, s[2:3], v[6:7]
+; GFX9-NEXT:    s_sub_i32 s2, 64, s0
+; GFX9-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, 0, v[0:1]
+; GFX9-NEXT:    s_cmp_lt_u32 s0, 64
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, 0, v[2:3]
+; GFX9-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX9-NEXT:    v_lshlrev_b64 v[2:3], s2, v[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX9-NEXT:    v_xor_b32_e32 v10, v0, v8
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], s0, v[4:5]
+; GFX9-NEXT:    s_cmp_eq_u32 s0, 0
+; GFX9-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX9-NEXT:    v_ashrrev_i64 v[8:9], s0, v[6:7]
+; GFX9-NEXT:    s_and_b32 s0, 1, s3
+; GFX9-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX9-NEXT:    v_ashrrev_i64 v[0:1], s1, v[6:7]
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    s_and_b32 s0, 1, s4
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    s_and_b32 s0, 1, s3
+; GFX9-NEXT:    v_ashrrev_i32_e32 v11, 31, v7
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v11, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v11, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v8, vcc
+; GFX9-NEXT:    v_and_b32_e32 v8, 1, v10
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: ssubsat_i128_sv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_sub_co_u32_e64 v4, vcc_lo, s0, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_sub_co_ci_u32_e32 v5, vcc_lo, s1, v1, vcc_lo
+; GFX10-NEXT:    v_sub_co_ci_u32_e32 v6, vcc_lo, s2, v2, vcc_lo
+; GFX10-NEXT:    v_sub_co_ci_u32_e32 v7, vcc_lo, s3, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[0:1], v[4:5]
+; GFX10-NEXT:    s_movk_i32 s0, 0x7f
+; GFX10-NEXT:    s_sub_i32 s1, 64, s0
+; GFX10-NEXT:    v_lshrrev_b64 v[15:16], s0, v[4:5]
+; GFX10-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_i64_e32 vcc_lo, s[2:3], v[6:7]
+; GFX10-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[2:3], v[6:7]
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v9, v8, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, 0, v[0:1]
+; GFX10-NEXT:    v_lshlrev_b64 v[8:9], s1, v[6:7]
+; GFX10-NEXT:    s_sub_i32 s1, s0, 64
+; GFX10-NEXT:    s_cmp_lt_u32 s0, 64
+; GFX10-NEXT:    v_ashrrev_i64 v[0:1], s0, v[6:7]
+; GFX10-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_i64_e32 vcc_lo, 0, v[2:3]
+; GFX10-NEXT:    v_or_b32_e32 v8, v15, v8
+; GFX10-NEXT:    v_or_b32_e32 v9, v16, v9
+; GFX10-NEXT:    v_ashrrev_i32_e32 v15, 31, v7
+; GFX10-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[2:3]
+; GFX10-NEXT:    v_ashrrev_i64 v[2:3], s1, v[6:7]
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v12, v11, vcc_lo
+; GFX10-NEXT:    s_cselect_b32 vcc_lo, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s0, 0
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v9, vcc_lo
+; GFX10-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc_lo
+; GFX10-NEXT:    s_and_b32 s0, 1, s1
+; GFX10-NEXT:    s_and_b32 s1, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s0
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s0, 0, s1
+; GFX10-NEXT:    v_xor_b32_e32 v9, v11, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v15, v0, s0
+; GFX10-NEXT:    v_and_b32_e32 v8, 1, v9
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v15, v1, s0
+; GFX10-NEXT:    v_add_co_u32_e64 v2, vcc_lo, v2, 0
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s0, 0, v8
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, 0, v0, vcc_lo
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, 0x80000000, v1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v4, v2, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v5, v3, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v6, v8, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i128 @llvm.ssub.sat.i128(i128 %lhs, i128 %rhs)
+  %cast = bitcast i128 %result to <4 x float>
+  ret <4 x float> %cast
+}
+
+define amdgpu_ps <4 x float> @ssubsat_i128_vs(i128 %lhs, i128 inreg %rhs) {
+; GFX6-LABEL: ssubsat_i128_vs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v5, s1
+; GFX6-NEXT:    v_subrev_i32_e32 v4, vcc, s0, v0
+; GFX6-NEXT:    v_subb_u32_e32 v5, vcc, v1, v5, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v6, s2
+; GFX6-NEXT:    v_mov_b32_e32 v7, s3
+; GFX6-NEXT:    v_subb_u32_e32 v6, vcc, v2, v6, vcc
+; GFX6-NEXT:    v_subb_u32_e32 v7, vcc, v3, v7, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, v[4:5], v[0:1]
+; GFX6-NEXT:    v_cmp_gt_u64_e64 s[0:1], s[0:1], 0
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, v[6:7], v[2:3]
+; GFX6-NEXT:    v_ashrrev_i32_e32 v11, 31, v7
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; GFX6-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[2:3], 0
+; GFX6-NEXT:    v_cmp_eq_u64_e64 vcc, s[2:3], 0
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX6-NEXT:    s_movk_i32 s0, 0x7f
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX6-NEXT:    s_sub_i32 s2, 64, s0
+; GFX6-NEXT:    s_sub_i32 s1, s0, 64
+; GFX6-NEXT:    v_xor_b32_e32 v10, v1, v0
+; GFX6-NEXT:    s_cmp_lt_u32 s0, 64
+; GFX6-NEXT:    v_lshr_b64 v[0:1], v[4:5], s0
+; GFX6-NEXT:    v_lshl_b64 v[2:3], v[6:7], s2
+; GFX6-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s0, 0
+; GFX6-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX6-NEXT:    v_ashr_i64 v[8:9], v[6:7], s0
+; GFX6-NEXT:    s_and_b32 s0, 1, s3
+; GFX6-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX6-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX6-NEXT:    v_ashr_i64 v[0:1], v[6:7], s1
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX6-NEXT:    s_and_b32 s0, 1, s4
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX6-NEXT:    s_and_b32 s0, 1, s3
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v11, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v11, v9, vcc
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, 0, v0
+; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX6-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v3, v8, vcc
+; GFX6-NEXT:    v_and_b32_e32 v8, 1, v10
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: ssubsat_i128_vs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_subrev_u32_e32 v4, vcc, s0, v0
+; GFX8-NEXT:    v_subb_u32_e32 v5, vcc, v1, v5, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v6, s2
+; GFX8-NEXT:    v_mov_b32_e32 v7, s3
+; GFX8-NEXT:    v_subb_u32_e32 v6, vcc, v2, v6, vcc
+; GFX8-NEXT:    v_subb_u32_e32 v7, vcc, v3, v7, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[4:5], v[0:1]
+; GFX8-NEXT:    v_cmp_gt_u64_e64 s[0:1], s[0:1], 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, v[6:7], v[2:3]
+; GFX8-NEXT:    s_cmp_eq_u64 s[2:3], 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX8-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; GFX8-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[2:3], 0
+; GFX8-NEXT:    v_ashrrev_i32_e32 v11, 31, v7
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX8-NEXT:    s_and_b32 s0, 1, s4
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    s_movk_i32 s0, 0x7f
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX8-NEXT:    s_sub_i32 s2, 64, s0
+; GFX8-NEXT:    s_sub_i32 s1, s0, 64
+; GFX8-NEXT:    v_xor_b32_e32 v10, v1, v0
+; GFX8-NEXT:    s_cmp_lt_u32 s0, 64
+; GFX8-NEXT:    v_lshrrev_b64 v[0:1], s0, v[4:5]
+; GFX8-NEXT:    v_lshlrev_b64 v[2:3], s2, v[6:7]
+; GFX8-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s0, 0
+; GFX8-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX8-NEXT:    v_ashrrev_i64 v[8:9], s0, v[6:7]
+; GFX8-NEXT:    s_and_b32 s0, 1, s3
+; GFX8-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX8-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX8-NEXT:    v_ashrrev_i64 v[0:1], s1, v[6:7]
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    s_and_b32 s0, 1, s4
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    s_and_b32 s0, 1, s3
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v11, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v11, v9, vcc
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 0, v0
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX8-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, v3, v8, vcc
+; GFX8-NEXT:    v_and_b32_e32 v8, 1, v10
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: ssubsat_i128_vs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v5, s1
+; GFX9-NEXT:    v_subrev_co_u32_e32 v4, vcc, s0, v0
+; GFX9-NEXT:    v_subb_co_u32_e32 v5, vcc, v1, v5, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v6, s2
+; GFX9-NEXT:    v_mov_b32_e32 v7, s3
+; GFX9-NEXT:    v_subb_co_u32_e32 v6, vcc, v2, v6, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v7, vcc, v3, v7, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[4:5], v[0:1]
+; GFX9-NEXT:    v_cmp_gt_u64_e64 s[0:1], s[0:1], 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, v[6:7], v[2:3]
+; GFX9-NEXT:    s_cmp_eq_u64 s[2:3], 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX9-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; GFX9-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[2:3], 0
+; GFX9-NEXT:    v_ashrrev_i32_e32 v11, 31, v7
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX9-NEXT:    s_and_b32 s0, 1, s4
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    s_movk_i32 s0, 0x7f
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    s_sub_i32 s2, 64, s0
+; GFX9-NEXT:    s_sub_i32 s1, s0, 64
+; GFX9-NEXT:    v_xor_b32_e32 v10, v1, v0
+; GFX9-NEXT:    s_cmp_lt_u32 s0, 64
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], s0, v[4:5]
+; GFX9-NEXT:    v_lshlrev_b64 v[2:3], s2, v[6:7]
+; GFX9-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s0, 0
+; GFX9-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX9-NEXT:    v_ashrrev_i64 v[8:9], s0, v[6:7]
+; GFX9-NEXT:    s_and_b32 s0, 1, s3
+; GFX9-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX9-NEXT:    v_ashrrev_i64 v[0:1], s1, v[6:7]
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    s_and_b32 s0, 1, s4
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    s_and_b32 s0, 1, s3
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v11, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v11, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v8, vcc
+; GFX9-NEXT:    v_and_b32_e32 v8, 1, v10
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: ssubsat_i128_vs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_mov_b32_e32 v5, v0
+; GFX10-NEXT:    v_mov_b32_e32 v6, v1
+; GFX10-NEXT:    v_mov_b32_e32 v9, v2
+; GFX10-NEXT:    v_mov_b32_e32 v10, v3
+; GFX10-NEXT:    s_cmp_eq_u64 s[2:3], 0
+; GFX10-NEXT:    v_sub_co_u32_e64 v15, vcc_lo, v5, s0
+; GFX10-NEXT:    v_cmp_gt_u64_e64 s0, s[0:1], 0
+; GFX10-NEXT:    v_subrev_co_ci_u32_e32 v16, vcc_lo, s1, v6, vcc_lo
+; GFX10-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX10-NEXT:    v_subrev_co_ci_u32_e32 v19, vcc_lo, s2, v9, vcc_lo
+; GFX10-NEXT:    s_and_b32 s1, 1, s4
+; GFX10-NEXT:    v_subrev_co_ci_u32_e32 v20, vcc_lo, s3, v10, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[15:16], v[5:6]
+; GFX10-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s0
+; GFX10-NEXT:    v_cmp_gt_i64_e64 s0, s[2:3], 0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_ashrrev_i32_e32 v7, 31, v20
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[19:20], v[9:10]
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[19:20], v[9:10]
+; GFX10-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s0
+; GFX10-NEXT:    s_movk_i32 s0, 0x7f
+; GFX10-NEXT:    s_sub_i32 s2, 64, s0
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v1, v0, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s1
+; GFX10-NEXT:    v_lshrrev_b64 v[0:1], s0, v[15:16]
+; GFX10-NEXT:    v_lshlrev_b64 v[2:3], s2, v[19:20]
+; GFX10-NEXT:    s_sub_i32 s1, s0, 64
+; GFX10-NEXT:    s_cmp_lt_u32 s0, 64
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v9, v8, vcc_lo
+; GFX10-NEXT:    s_cselect_b32 vcc_lo, 1, 0
+; GFX10-NEXT:    v_ashrrev_i64 v[8:9], s1, v[19:20]
+; GFX10-NEXT:    s_cmp_eq_u32 s0, 0
+; GFX10-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX10-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX10-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX10-NEXT:    v_ashrrev_i64 v[0:1], s0, v[19:20]
+; GFX10-NEXT:    s_and_b32 s0, 1, s1
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc_lo
+; GFX10-NEXT:    s_and_b32 s1, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s0
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s0, 0, s1
+; GFX10-NEXT:    v_xor_b32_e32 v9, v11, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v15, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v16, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v7, v0, s0
+; GFX10-NEXT:    v_and_b32_e32 v8, 1, v9
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v7, v1, s0
+; GFX10-NEXT:    v_add_co_u32_e64 v2, vcc_lo, v2, 0
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s0, 0, v8
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, 0, v0, vcc_lo
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, 0x80000000, v1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v15, v2, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v16, v3, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v19, v8, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v20, v9, s0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i128 @llvm.ssub.sat.i128(i128 %lhs, i128 %rhs)
+  %cast = bitcast i128 %result to <4 x float>
+  ret <4 x float> %cast
+}
+
+define <2 x i128> @v_ssubsat_v2i128(<2 x i128> %lhs, <2 x i128> %rhs) {
+; GFX6-LABEL: v_ssubsat_v2i128:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_sub_i32_e32 v16, vcc, v0, v8
+; GFX6-NEXT:    v_subb_u32_e32 v17, vcc, v1, v9, vcc
+; GFX6-NEXT:    v_subb_u32_e32 v18, vcc, v2, v10, vcc
+; GFX6-NEXT:    v_subb_u32_e32 v19, vcc, v3, v11, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, v[16:17], v[0:1]
+; GFX6-NEXT:    s_movk_i32 s6, 0x7f
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, v[18:19], v[2:3]
+; GFX6-NEXT:    s_sub_i32 s7, s6, 64
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, v[18:19], v[2:3]
+; GFX6-NEXT:    s_sub_i32 s8, 64, s6
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, 0, v[8:9]
+; GFX6-NEXT:    s_cmp_lt_u32 s6, 64
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, 0, v[10:11]
+; GFX6-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[10:11]
+; GFX6-NEXT:    s_cmp_eq_u32 s6, 0
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX6-NEXT:    v_xor_b32_e32 v10, v1, v0
+; GFX6-NEXT:    v_lshr_b64 v[0:1], v[16:17], s6
+; GFX6-NEXT:    v_lshl_b64 v[2:3], v[18:19], s8
+; GFX6-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX6-NEXT:    s_and_b32 s4, 1, s4
+; GFX6-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX6-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX6-NEXT:    v_ashr_i64 v[0:1], v[18:19], s7
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX6-NEXT:    s_and_b32 s4, 1, s5
+; GFX6-NEXT:    v_ashr_i64 v[8:9], v[18:19], s6
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, s4
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX6-NEXT:    v_ashrrev_i32_e32 v11, 31, v19
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v0, v16, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v1, v17, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v11, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v11, v9, vcc
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, 0, v0
+; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX6-NEXT:    s_brev_b32 s4, 1
+; GFX6-NEXT:    v_mov_b32_e32 v8, s4
+; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v3, v8, vcc
+; GFX6-NEXT:    v_and_b32_e32 v8, 1, v10
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v18, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v19, v3, vcc
+; GFX6-NEXT:    v_sub_i32_e32 v8, vcc, v4, v12
+; GFX6-NEXT:    v_subb_u32_e32 v9, vcc, v5, v13, vcc
+; GFX6-NEXT:    v_subb_u32_e32 v10, vcc, v6, v14, vcc
+; GFX6-NEXT:    v_subb_u32_e32 v11, vcc, v7, v15, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
+; GFX6-NEXT:    s_cmp_lt_u32 s6, 64
+; GFX6-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, v[10:11], v[6:7]
+; GFX6-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX6-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, v[10:11], v[6:7]
+; GFX6-NEXT:    s_cmp_eq_u32 s6, 0
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, 0, v[12:13]
+; GFX6-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX6-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, 0, v[14:15]
+; GFX6-NEXT:    v_ashr_i64 v[12:13], v[10:11], s6
+; GFX6-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[14:15]
+; GFX6-NEXT:    v_ashrrev_i32_e32 v15, 31, v11
+; GFX6-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
+; GFX6-NEXT:    v_xor_b32_e32 v14, v5, v4
+; GFX6-NEXT:    v_lshr_b64 v[4:5], v[8:9], s6
+; GFX6-NEXT:    v_lshl_b64 v[6:7], v[10:11], s8
+; GFX6-NEXT:    s_and_b32 s6, 1, s5
+; GFX6-NEXT:    v_or_b32_e32 v6, v4, v6
+; GFX6-NEXT:    v_or_b32_e32 v7, v5, v7
+; GFX6-NEXT:    v_ashr_i64 v[4:5], v[10:11], s7
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s6
+; GFX6-NEXT:    s_and_b32 s6, 1, s9
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s6
+; GFX6-NEXT:    s_and_b32 s5, 1, s5
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, v4, v8, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v5, v5, v9, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s5
+; GFX6-NEXT:    v_cndmask_b32_e32 v6, v15, v12, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v7, v15, v13, vcc
+; GFX6-NEXT:    v_add_i32_e32 v4, vcc, 0, v4
+; GFX6-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v12, s4
+; GFX6-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v7, vcc, v7, v12, vcc
+; GFX6-NEXT:    v_and_b32_e32 v12, 1, v14
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v5, v9, v5, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v6, v10, v6, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v7, v11, v7, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_ssubsat_v2i128:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_u32_e32 v16, vcc, v0, v8
+; GFX8-NEXT:    v_subb_u32_e32 v17, vcc, v1, v9, vcc
+; GFX8-NEXT:    v_subb_u32_e32 v18, vcc, v2, v10, vcc
+; GFX8-NEXT:    v_subb_u32_e32 v19, vcc, v3, v11, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[16:17], v[0:1]
+; GFX8-NEXT:    s_movk_i32 s6, 0x7f
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, v[18:19], v[2:3]
+; GFX8-NEXT:    s_sub_i32 s7, s6, 64
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[18:19], v[2:3]
+; GFX8-NEXT:    s_sub_i32 s8, 64, s6
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, 0, v[8:9]
+; GFX8-NEXT:    s_cmp_lt_u32 s6, 64
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, 0, v[10:11]
+; GFX8-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[10:11]
+; GFX8-NEXT:    s_cmp_eq_u32 s6, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX8-NEXT:    v_xor_b32_e32 v10, v1, v0
+; GFX8-NEXT:    v_lshrrev_b64 v[0:1], s6, v[16:17]
+; GFX8-NEXT:    v_lshlrev_b64 v[2:3], s8, v[18:19]
+; GFX8-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX8-NEXT:    s_and_b32 s4, 1, s4
+; GFX8-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX8-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX8-NEXT:    v_ashrrev_i64 v[0:1], s7, v[18:19]
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    s_and_b32 s4, 1, s5
+; GFX8-NEXT:    v_ashrrev_i64 v[8:9], s6, v[18:19]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, s4
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX8-NEXT:    v_ashrrev_i32_e32 v11, 31, v19
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, v16, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, v17, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v11, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v11, v9, vcc
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 0, v0
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT:    s_brev_b32 s4, 1
+; GFX8-NEXT:    v_mov_b32_e32 v8, s4
+; GFX8-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, v3, v8, vcc
+; GFX8-NEXT:    v_and_b32_e32 v8, 1, v10
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v18, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v19, v3, vcc
+; GFX8-NEXT:    v_sub_u32_e32 v8, vcc, v4, v12
+; GFX8-NEXT:    v_subb_u32_e32 v9, vcc, v5, v13, vcc
+; GFX8-NEXT:    v_subb_u32_e32 v10, vcc, v6, v14, vcc
+; GFX8-NEXT:    v_subb_u32_e32 v11, vcc, v7, v15, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
+; GFX8-NEXT:    s_cmp_lt_u32 s6, 64
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, v[10:11], v[6:7]
+; GFX8-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[10:11], v[6:7]
+; GFX8-NEXT:    s_cmp_eq_u32 s6, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, 0, v[12:13]
+; GFX8-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, 0, v[14:15]
+; GFX8-NEXT:    v_ashrrev_i64 v[12:13], s6, v[10:11]
+; GFX8-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[14:15]
+; GFX8-NEXT:    v_ashrrev_i32_e32 v15, 31, v11
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
+; GFX8-NEXT:    v_xor_b32_e32 v14, v5, v4
+; GFX8-NEXT:    v_lshrrev_b64 v[4:5], s6, v[8:9]
+; GFX8-NEXT:    v_lshlrev_b64 v[6:7], s8, v[10:11]
+; GFX8-NEXT:    s_and_b32 s6, 1, s5
+; GFX8-NEXT:    v_or_b32_e32 v6, v4, v6
+; GFX8-NEXT:    v_or_b32_e32 v7, v5, v7
+; GFX8-NEXT:    v_ashrrev_i64 v[4:5], s7, v[10:11]
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s6
+; GFX8-NEXT:    s_and_b32 s6, 1, s9
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s6
+; GFX8-NEXT:    s_and_b32 s5, 1, s5
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v8, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v9, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s5
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v15, v12, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v15, v13, vcc
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 0, v4
+; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v12, s4
+; GFX8-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v7, vcc, v7, v12, vcc
+; GFX8-NEXT:    v_and_b32_e32 v12, 1, v14
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v9, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v10, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v11, v7, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_ssubsat_v2i128:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_co_u32_e32 v16, vcc, v0, v8
+; GFX9-NEXT:    v_subb_co_u32_e32 v17, vcc, v1, v9, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v18, vcc, v2, v10, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v19, vcc, v3, v11, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[16:17], v[0:1]
+; GFX9-NEXT:    s_movk_i32 s6, 0x7f
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, v[18:19], v[2:3]
+; GFX9-NEXT:    s_sub_i32 s7, s6, 64
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[18:19], v[2:3]
+; GFX9-NEXT:    s_sub_i32 s8, 64, s6
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, 0, v[8:9]
+; GFX9-NEXT:    s_cmp_lt_u32 s6, 64
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, 0, v[10:11]
+; GFX9-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[10:11]
+; GFX9-NEXT:    s_cmp_eq_u32 s6, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    v_xor_b32_e32 v10, v1, v0
+; GFX9-NEXT:    v_lshrrev_b64 v[0:1], s6, v[16:17]
+; GFX9-NEXT:    v_lshlrev_b64 v[2:3], s8, v[18:19]
+; GFX9-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX9-NEXT:    s_and_b32 s4, 1, s4
+; GFX9-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX9-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX9-NEXT:    v_ashrrev_i64 v[0:1], s7, v[18:19]
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    s_and_b32 s4, 1, s5
+; GFX9-NEXT:    v_ashrrev_i64 v[8:9], s6, v[18:19]
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, s4
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX9-NEXT:    v_ashrrev_i32_e32 v11, 31, v19
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, v16, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v17, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v11, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v11, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    s_brev_b32 s4, 1
+; GFX9-NEXT:    v_mov_b32_e32 v8, s4
+; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v8, vcc
+; GFX9-NEXT:    v_and_b32_e32 v8, 1, v10
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v17, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v18, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v19, v3, vcc
+; GFX9-NEXT:    v_sub_co_u32_e32 v8, vcc, v4, v12
+; GFX9-NEXT:    v_subb_co_u32_e32 v9, vcc, v5, v13, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v10, vcc, v6, v14, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v11, vcc, v7, v15, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[4:5]
+; GFX9-NEXT:    s_cmp_lt_u32 s6, 64
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, v[10:11], v[6:7]
+; GFX9-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[10:11], v[6:7]
+; GFX9-NEXT:    s_cmp_eq_u32 s6, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, 0, v[12:13]
+; GFX9-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, 0, v[14:15]
+; GFX9-NEXT:    v_ashrrev_i64 v[12:13], s6, v[10:11]
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[14:15]
+; GFX9-NEXT:    v_ashrrev_i32_e32 v15, 31, v11
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
+; GFX9-NEXT:    v_xor_b32_e32 v14, v5, v4
+; GFX9-NEXT:    v_lshrrev_b64 v[4:5], s6, v[8:9]
+; GFX9-NEXT:    v_lshlrev_b64 v[6:7], s8, v[10:11]
+; GFX9-NEXT:    s_and_b32 s6, 1, s5
+; GFX9-NEXT:    v_or_b32_e32 v6, v4, v6
+; GFX9-NEXT:    v_or_b32_e32 v7, v5, v7
+; GFX9-NEXT:    v_ashrrev_i64 v[4:5], s7, v[10:11]
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s6
+; GFX9-NEXT:    s_and_b32 s6, 1, s9
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s6
+; GFX9-NEXT:    s_and_b32 s5, 1, s5
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v8, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v9, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s5
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, v15, v12, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v7, v15, v13, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v12, s4
+; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, 0, v6, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, v7, v12, vcc
+; GFX9-NEXT:    v_and_b32_e32 v12, 1, v14
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v9, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, v10, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v7, v11, v7, vcc
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_ssubsat_v2i128:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_mov_b32_e32 v22, v0
+; GFX10-NEXT:    v_mov_b32_e32 v23, v1
+; GFX10-NEXT:    v_mov_b32_e32 v20, v2
+; GFX10-NEXT:    v_mov_b32_e32 v21, v3
+; GFX10-NEXT:    s_movk_i32 s5, 0x7f
+; GFX10-NEXT:    v_sub_co_u32_e64 v16, vcc_lo, v22, v8
+; GFX10-NEXT:    s_sub_i32 s6, 64, s5
+; GFX10-NEXT:    v_sub_co_ci_u32_e32 v17, vcc_lo, v23, v9, vcc_lo
+; GFX10-NEXT:    s_sub_i32 s7, s5, 64
+; GFX10-NEXT:    v_sub_co_ci_u32_e32 v18, vcc_lo, v20, v10, vcc_lo
+; GFX10-NEXT:    s_cmp_lt_u32 s5, 64
+; GFX10-NEXT:    v_sub_co_ci_u32_e32 v19, vcc_lo, v21, v11, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[16:17], v[22:23]
+; GFX10-NEXT:    v_mov_b32_e32 v26, v4
+; GFX10-NEXT:    v_mov_b32_e32 v27, v5
+; GFX10-NEXT:    v_mov_b32_e32 v24, v6
+; GFX10-NEXT:    v_lshlrev_b64 v[2:3], s6, v[18:19]
+; GFX10-NEXT:    v_mov_b32_e32 v25, v7
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[18:19], v[20:21]
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[18:19], v[20:21]
+; GFX10-NEXT:    v_cndmask_b32_e32 v20, v1, v0, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, 0, v[8:9]
+; GFX10-NEXT:    v_lshrrev_b64 v[0:1], s5, v[16:17]
+; GFX10-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_i64_e32 vcc_lo, 0, v[10:11]
+; GFX10-NEXT:    v_or_b32_e32 v2, v0, v2
+; GFX10-NEXT:    v_or_b32_e32 v3, v1, v3
+; GFX10-NEXT:    v_ashrrev_i64 v[0:1], s5, v[18:19]
+; GFX10-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[10:11]
+; GFX10-NEXT:    v_ashrrev_i32_e32 v11, 31, v19
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v9, v8, vcc_lo
+; GFX10-NEXT:    s_cselect_b32 vcc_lo, 1, 0
+; GFX10-NEXT:    v_ashrrev_i64 v[8:9], s7, v[18:19]
+; GFX10-NEXT:    s_cmp_eq_u32 s5, 0
+; GFX10-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX10-NEXT:    s_and_b32 s8, 1, vcc_lo
+; GFX10-NEXT:    s_and_b32 s4, 1, s4
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s4
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s4, 0, s8
+; GFX10-NEXT:    v_xor_b32_e32 v9, v10, v20
+; GFX10-NEXT:    s_brev_b32 s8, 1
+; GFX10-NEXT:    s_cmp_lt_u32 s5, 64
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v16, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v17, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v11, v0, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v11, v1, s4
+; GFX10-NEXT:    v_and_b32_e32 v8, 1, v9
+; GFX10-NEXT:    v_add_co_u32_e64 v2, vcc_lo, v2, 0
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v20, vcc_lo, 0, v0, vcc_lo
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v21, vcc_lo, s8, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v8
+; GFX10-NEXT:    v_sub_co_u32_e64 v8, s4, v26, v12
+; GFX10-NEXT:    v_sub_co_ci_u32_e64 v9, s4, v27, v13, s4
+; GFX10-NEXT:    v_sub_co_ci_u32_e64 v10, s4, v24, v14, s4
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v16, v2, vcc_lo
+; GFX10-NEXT:    v_sub_co_ci_u32_e64 v11, s4, v25, v15, s4
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s4, v[8:9], v[26:27]
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v17, v3, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v18, v20, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b64 v[3:4], s5, v[8:9]
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, 0, 1, s4
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s4, v[10:11], v[24:25]
+; GFX10-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s4
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s4, 0, v[12:13]
+; GFX10-NEXT:    v_lshlrev_b64 v[12:13], s6, v[10:11]
+; GFX10-NEXT:    v_cndmask_b32_e64 v17, 0, 1, s4
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s4, 0, v[14:15]
+; GFX10-NEXT:    v_or_b32_e32 v12, v3, v12
+; GFX10-NEXT:    v_or_b32_e32 v13, v4, v13
+; GFX10-NEXT:    v_ashrrev_i64 v[3:4], s5, v[10:11]
+; GFX10-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s4
+; GFX10-NEXT:    v_cmp_eq_u64_e64 s4, v[10:11], v[24:25]
+; GFX10-NEXT:    v_cndmask_b32_e64 v7, v16, v5, s4
+; GFX10-NEXT:    v_cmp_eq_u64_e64 s4, 0, v[14:15]
+; GFX10-NEXT:    v_ashrrev_i64 v[5:6], s7, v[10:11]
+; GFX10-NEXT:    v_cndmask_b32_e64 v14, v18, v17, s4
+; GFX10-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s5, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, v12, s4
+; GFX10-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, v13, s4
+; GFX10-NEXT:    s_and_b32 s5, 1, s6
+; GFX10-NEXT:    s_and_b32 s6, 1, s4
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s4, 0, s5
+; GFX10-NEXT:    v_xor_b32_e32 v7, v14, v7
+; GFX10-NEXT:    v_ashrrev_i32_e32 v18, 31, v11
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s5, 0, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, v8, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, v9, s4
+; GFX10-NEXT:    v_and_b32_e32 v7, 1, v7
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v18, v3, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, v18, v4, s5
+; GFX10-NEXT:    v_add_co_u32_e64 v5, s4, v5, 0
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v6, s4, 0, v6, s4
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s5, 0, v7
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v7, s4, 0, v3, s4
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v19, v21, vcc_lo
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v12, s4, s8, v4, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, v8, v5, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v9, v6, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v10, v7, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v7, v11, v12, s5
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i128> @llvm.ssub.sat.v2i128(<2 x i128> %lhs, <2 x i128> %rhs)
+  ret <2 x i128> %result
+}
+
+define amdgpu_ps <2 x i128> @s_ssubsat_v2i128(<2 x i128> inreg %lhs, <2 x i128> inreg %rhs) {
+; GFX6-LABEL: s_ssubsat_v2i128:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_sub_u32 s16, s0, s8
+; GFX6-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX6-NEXT:    s_and_b32 s17, s17, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX6-NEXT:    s_subb_u32 s17, s1, s9
+; GFX6-NEXT:    s_cselect_b32 s18, 1, 0
+; GFX6-NEXT:    s_and_b32 s18, s18, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s18, 0
+; GFX6-NEXT:    s_subb_u32 s18, s2, s10
+; GFX6-NEXT:    s_cselect_b32 s19, 1, 0
+; GFX6-NEXT:    v_mov_b32_e32 v3, s1
+; GFX6-NEXT:    s_and_b32 s19, s19, 1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[16:17], v[2:3]
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    s_subb_u32 s19, s3, s11
+; GFX6-NEXT:    s_movk_i32 s20, 0x7f
+; GFX6-NEXT:    v_mov_b32_e32 v1, s3
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, s[18:19], v[0:1]
+; GFX6-NEXT:    v_cmp_gt_u64_e64 s[0:1], s[8:9], 0
+; GFX6-NEXT:    s_sub_i32 s21, s20, 64
+; GFX6-NEXT:    s_sub_i32 s22, 64, s20
+; GFX6-NEXT:    s_cmp_lt_u32 s20, 64
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, s[18:19], v[0:1]
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; GFX6-NEXT:    s_cselect_b32 s23, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s20, 0
+; GFX6-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[10:11], 0
+; GFX6-NEXT:    s_cselect_b32 s24, 1, 0
+; GFX6-NEXT:    s_lshr_b64 s[2:3], s[16:17], s20
+; GFX6-NEXT:    s_lshl_b64 s[8:9], s[18:19], s22
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX6-NEXT:    v_cmp_eq_u64_e64 vcc, s[10:11], 0
+; GFX6-NEXT:    s_or_b64 s[2:3], s[2:3], s[8:9]
+; GFX6-NEXT:    s_ashr_i32 s8, s19, 31
+; GFX6-NEXT:    s_ashr_i64 s[0:1], s[18:19], s20
+; GFX6-NEXT:    s_ashr_i64 s[10:11], s[18:19], s21
+; GFX6-NEXT:    s_and_b32 s23, s23, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s23, 0
+; GFX6-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[10:11]
+; GFX6-NEXT:    s_and_b32 s10, s24, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX6-NEXT:    s_cselect_b64 s[2:3], s[16:17], s[2:3]
+; GFX6-NEXT:    s_cmp_lg_u32 s23, 0
+; GFX6-NEXT:    s_mov_b32 s9, s8
+; GFX6-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[8:9]
+; GFX6-NEXT:    s_add_u32 s2, s2, 0
+; GFX6-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX6-NEXT:    s_and_b32 s8, s8, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX6-NEXT:    s_addc_u32 s3, s3, 0
+; GFX6-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX6-NEXT:    s_and_b32 s8, s8, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX6-NEXT:    s_addc_u32 s0, s0, 0
+; GFX6-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX6-NEXT:    s_and_b32 s8, s8, 1
+; GFX6-NEXT:    v_xor_b32_e32 v0, v1, v0
+; GFX6-NEXT:    s_brev_b32 s23, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX6-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX6-NEXT:    s_addc_u32 s1, s1, s23
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v3, s16
+; GFX6-NEXT:    s_sub_u32 s0, s4, s12
+; GFX6-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX6-NEXT:    s_and_b32 s1, s1, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX6-NEXT:    s_subb_u32 s1, s5, s13
+; GFX6-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX6-NEXT:    s_and_b32 s2, s2, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX6-NEXT:    v_mov_b32_e32 v2, s3
+; GFX6-NEXT:    v_mov_b32_e32 v4, s17
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v2, s18
+; GFX6-NEXT:    v_mov_b32_e32 v3, s19
+; GFX6-NEXT:    s_subb_u32 s2, s6, s14
+; GFX6-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX6-NEXT:    v_cndmask_b32_e32 v6, v2, v0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v7, v3, v1, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v2, s4
+; GFX6-NEXT:    s_and_b32 s3, s3, 1
+; GFX6-NEXT:    v_mov_b32_e32 v3, s5
+; GFX6-NEXT:    s_cmp_lg_u32 s3, 0
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX6-NEXT:    v_mov_b32_e32 v0, s6
+; GFX6-NEXT:    s_subb_u32 s3, s7, s15
+; GFX6-NEXT:    v_mov_b32_e32 v1, s7
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
+; GFX6-NEXT:    v_cmp_gt_u64_e64 s[4:5], s[12:13], 0
+; GFX6-NEXT:    s_cmp_lt_u32 s20, 64
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, s[2:3], v[0:1]
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[4:5]
+; GFX6-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX6-NEXT:    s_cmp_eq_u32 s20, 0
+; GFX6-NEXT:    v_cmp_gt_i64_e64 s[4:5], s[14:15], 0
+; GFX6-NEXT:    s_cselect_b32 s13, 1, 0
+; GFX6-NEXT:    s_lshr_b64 s[6:7], s[0:1], s20
+; GFX6-NEXT:    s_lshl_b64 s[8:9], s[2:3], s22
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[4:5]
+; GFX6-NEXT:    s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX6-NEXT:    s_ashr_i32 s8, s3, 31
+; GFX6-NEXT:    s_ashr_i64 s[4:5], s[2:3], s20
+; GFX6-NEXT:    s_ashr_i64 s[10:11], s[2:3], s21
+; GFX6-NEXT:    s_and_b32 s12, s12, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX6-NEXT:    s_cselect_b64 s[6:7], s[6:7], s[10:11]
+; GFX6-NEXT:    s_and_b32 s10, s13, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX6-NEXT:    s_cselect_b64 s[6:7], s[0:1], s[6:7]
+; GFX6-NEXT:    s_mov_b32 s9, s8
+; GFX6-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX6-NEXT:    s_cselect_b64 s[4:5], s[4:5], s[8:9]
+; GFX6-NEXT:    s_add_u32 s6, s6, 0
+; GFX6-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX6-NEXT:    s_and_b32 s8, s8, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX6-NEXT:    s_addc_u32 s7, s7, 0
+; GFX6-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX6-NEXT:    s_and_b32 s8, s8, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX6-NEXT:    v_cmp_eq_u64_e64 vcc, s[14:15], 0
+; GFX6-NEXT:    s_addc_u32 s4, s4, 0
+; GFX6-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX6-NEXT:    v_xor_b32_e32 v0, v1, v0
+; GFX6-NEXT:    s_and_b32 s8, s8, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX6-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX6-NEXT:    v_mov_b32_e32 v3, s0
+; GFX6-NEXT:    v_mov_b32_e32 v8, s1
+; GFX6-NEXT:    s_addc_u32 s5, s5, s23
+; GFX6-NEXT:    v_mov_b32_e32 v1, s6
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v2, s7
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v3, v1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v8, v2, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v2, s4
+; GFX6-NEXT:    v_mov_b32_e32 v8, s2
+; GFX6-NEXT:    v_mov_b32_e32 v3, s5
+; GFX6-NEXT:    v_mov_b32_e32 v9, s3
+; GFX6-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v5
+; GFX6-NEXT:    v_readfirstlane_b32 s1, v4
+; GFX6-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX6-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX6-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX6-NEXT:    v_readfirstlane_b32 s6, v2
+; GFX6-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_ssubsat_v2i128:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_sub_u32 s16, s0, s8
+; GFX8-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX8-NEXT:    s_and_b32 s17, s17, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX8-NEXT:    s_subb_u32 s17, s1, s9
+; GFX8-NEXT:    s_cselect_b32 s18, 1, 0
+; GFX8-NEXT:    s_and_b32 s18, s18, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s18, 0
+; GFX8-NEXT:    s_subb_u32 s18, s2, s10
+; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
+; GFX8-NEXT:    s_and_b32 s19, s19, 1
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:    s_subb_u32 s19, s3, s11
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[16:17], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    s_cmp_eq_u64 s[18:19], s[2:3]
+; GFX8-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[18:19], v[0:1]
+; GFX8-NEXT:    s_and_b32 s0, 1, s2
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    v_cmp_gt_u64_e64 s[0:1], s[8:9], 0
+; GFX8-NEXT:    s_cmp_eq_u64 s[10:11], 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; GFX8-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[10:11], 0
+; GFX8-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX8-NEXT:    s_movk_i32 s20, 0x7f
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX8-NEXT:    s_and_b32 s0, 1, s2
+; GFX8-NEXT:    s_sub_i32 s21, s20, 64
+; GFX8-NEXT:    s_sub_i32 s22, 64, s20
+; GFX8-NEXT:    s_cmp_lt_u32 s20, 64
+; GFX8-NEXT:    s_cselect_b32 s23, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s20, 0
+; GFX8-NEXT:    s_cselect_b32 s24, 1, 0
+; GFX8-NEXT:    s_lshr_b64 s[2:3], s[16:17], s20
+; GFX8-NEXT:    s_lshl_b64 s[8:9], s[18:19], s22
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    s_or_b64 s[2:3], s[2:3], s[8:9]
+; GFX8-NEXT:    s_ashr_i32 s8, s19, 31
+; GFX8-NEXT:    s_ashr_i64 s[0:1], s[18:19], s20
+; GFX8-NEXT:    s_ashr_i64 s[10:11], s[18:19], s21
+; GFX8-NEXT:    s_and_b32 s23, s23, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s23, 0
+; GFX8-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[10:11]
+; GFX8-NEXT:    s_and_b32 s10, s24, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX8-NEXT:    s_cselect_b64 s[2:3], s[16:17], s[2:3]
+; GFX8-NEXT:    s_cmp_lg_u32 s23, 0
+; GFX8-NEXT:    s_mov_b32 s9, s8
+; GFX8-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[8:9]
+; GFX8-NEXT:    s_add_u32 s2, s2, 0
+; GFX8-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX8-NEXT:    s_and_b32 s8, s8, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX8-NEXT:    s_addc_u32 s3, s3, 0
+; GFX8-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX8-NEXT:    s_and_b32 s8, s8, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX8-NEXT:    s_addc_u32 s0, s0, 0
+; GFX8-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX8-NEXT:    s_and_b32 s8, s8, 1
+; GFX8-NEXT:    v_xor_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_brev_b32 s23, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX8-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX8-NEXT:    s_addc_u32 s1, s1, s23
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s16
+; GFX8-NEXT:    s_sub_u32 s0, s4, s12
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX8-NEXT:    s_and_b32 s1, s1, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX8-NEXT:    s_subb_u32 s1, s5, s13
+; GFX8-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX8-NEXT:    s_and_b32 s2, s2, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s3
+; GFX8-NEXT:    v_mov_b32_e32 v4, s17
+; GFX8-NEXT:    s_subb_u32 s2, s6, s14
+; GFX8-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v2, s18
+; GFX8-NEXT:    v_mov_b32_e32 v3, s19
+; GFX8-NEXT:    s_and_b32 s3, s3, 1
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v2, v0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v3, v1, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    s_cmp_lg_u32 s3, 0
+; GFX8-NEXT:    v_mov_b32_e32 v3, s5
+; GFX8-NEXT:    s_subb_u32 s3, s7, s15
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s6
+; GFX8-NEXT:    v_mov_b32_e32 v1, s7
+; GFX8-NEXT:    s_cmp_eq_u64 s[2:3], s[6:7]
+; GFX8-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
+; GFX8-NEXT:    s_and_b32 s4, 1, s6
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    v_cmp_gt_u64_e64 s[4:5], s[12:13], 0
+; GFX8-NEXT:    s_cmp_eq_u64 s[14:15], 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[4:5]
+; GFX8-NEXT:    v_cmp_gt_i64_e64 s[4:5], s[14:15], 0
+; GFX8-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[4:5]
+; GFX8-NEXT:    s_and_b32 s4, 1, s6
+; GFX8-NEXT:    s_cmp_lt_u32 s20, 64
+; GFX8-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX8-NEXT:    s_cmp_eq_u32 s20, 0
+; GFX8-NEXT:    s_cselect_b32 s13, 1, 0
+; GFX8-NEXT:    s_lshr_b64 s[6:7], s[0:1], s20
+; GFX8-NEXT:    s_lshl_b64 s[8:9], s[2:3], s22
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX8-NEXT:    s_ashr_i32 s8, s3, 31
+; GFX8-NEXT:    s_ashr_i64 s[4:5], s[2:3], s20
+; GFX8-NEXT:    s_ashr_i64 s[10:11], s[2:3], s21
+; GFX8-NEXT:    s_and_b32 s12, s12, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX8-NEXT:    s_cselect_b64 s[6:7], s[6:7], s[10:11]
+; GFX8-NEXT:    s_and_b32 s10, s13, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX8-NEXT:    s_cselect_b64 s[6:7], s[0:1], s[6:7]
+; GFX8-NEXT:    s_mov_b32 s9, s8
+; GFX8-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX8-NEXT:    s_cselect_b64 s[4:5], s[4:5], s[8:9]
+; GFX8-NEXT:    s_add_u32 s6, s6, 0
+; GFX8-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX8-NEXT:    s_and_b32 s8, s8, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX8-NEXT:    s_addc_u32 s7, s7, 0
+; GFX8-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX8-NEXT:    s_and_b32 s8, s8, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX8-NEXT:    s_addc_u32 s4, s4, 0
+; GFX8-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX8-NEXT:    v_xor_b32_e32 v0, v1, v0
+; GFX8-NEXT:    s_and_b32 s8, s8, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX8-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v3, s0
+; GFX8-NEXT:    v_mov_b32_e32 v8, s1
+; GFX8-NEXT:    s_addc_u32 s5, s5, s23
+; GFX8-NEXT:    v_mov_b32_e32 v1, s6
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s7
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v8, v2, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_mov_b32_e32 v8, s2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s5
+; GFX8-NEXT:    v_mov_b32_e32 v9, s3
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v5
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v4
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX8-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s6, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_ssubsat_v2i128:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_sub_u32 s16, s0, s8
+; GFX9-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX9-NEXT:    s_and_b32 s17, s17, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX9-NEXT:    s_subb_u32 s17, s1, s9
+; GFX9-NEXT:    s_cselect_b32 s18, 1, 0
+; GFX9-NEXT:    s_and_b32 s18, s18, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s18, 0
+; GFX9-NEXT:    s_subb_u32 s18, s2, s10
+; GFX9-NEXT:    s_cselect_b32 s19, 1, 0
+; GFX9-NEXT:    s_and_b32 s19, s19, 1
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s0
+; GFX9-NEXT:    s_subb_u32 s19, s3, s11
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[16:17], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    s_cmp_eq_u64 s[18:19], s[2:3]
+; GFX9-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, s[18:19], v[0:1]
+; GFX9-NEXT:    s_and_b32 s0, 1, s2
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    v_cmp_gt_u64_e64 s[0:1], s[8:9], 0
+; GFX9-NEXT:    s_cmp_eq_u64 s[10:11], 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; GFX9-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[10:11], 0
+; GFX9-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX9-NEXT:    s_movk_i32 s20, 0x7f
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX9-NEXT:    s_and_b32 s0, 1, s2
+; GFX9-NEXT:    s_sub_i32 s21, s20, 64
+; GFX9-NEXT:    s_sub_i32 s22, 64, s20
+; GFX9-NEXT:    s_cmp_lt_u32 s20, 64
+; GFX9-NEXT:    s_cselect_b32 s23, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s20, 0
+; GFX9-NEXT:    s_cselect_b32 s24, 1, 0
+; GFX9-NEXT:    s_lshr_b64 s[2:3], s[16:17], s20
+; GFX9-NEXT:    s_lshl_b64 s[8:9], s[18:19], s22
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    s_or_b64 s[2:3], s[2:3], s[8:9]
+; GFX9-NEXT:    s_ashr_i32 s8, s19, 31
+; GFX9-NEXT:    s_ashr_i64 s[0:1], s[18:19], s20
+; GFX9-NEXT:    s_ashr_i64 s[10:11], s[18:19], s21
+; GFX9-NEXT:    s_and_b32 s23, s23, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s23, 0
+; GFX9-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[10:11]
+; GFX9-NEXT:    s_and_b32 s10, s24, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX9-NEXT:    s_cselect_b64 s[2:3], s[16:17], s[2:3]
+; GFX9-NEXT:    s_cmp_lg_u32 s23, 0
+; GFX9-NEXT:    s_mov_b32 s9, s8
+; GFX9-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[8:9]
+; GFX9-NEXT:    s_add_u32 s2, s2, 0
+; GFX9-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX9-NEXT:    s_and_b32 s8, s8, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX9-NEXT:    s_addc_u32 s3, s3, 0
+; GFX9-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX9-NEXT:    s_and_b32 s8, s8, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX9-NEXT:    s_addc_u32 s0, s0, 0
+; GFX9-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    s_and_b32 s8, s8, 1
+; GFX9-NEXT:    v_xor_b32_e32 v0, v1, v0
+; GFX9-NEXT:    s_brev_b32 s23, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    s_addc_u32 s1, s1, s23
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    v_mov_b32_e32 v3, s16
+; GFX9-NEXT:    s_sub_u32 s0, s4, s12
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v3, v1, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX9-NEXT:    s_and_b32 s1, s1, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX9-NEXT:    s_subb_u32 s1, s5, s13
+; GFX9-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX9-NEXT:    s_and_b32 s2, s2, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-NEXT:    v_mov_b32_e32 v4, s17
+; GFX9-NEXT:    s_subb_u32 s2, s6, s14
+; GFX9-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v2, s18
+; GFX9-NEXT:    v_mov_b32_e32 v3, s19
+; GFX9-NEXT:    s_and_b32 s3, s3, 1
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, v2, v0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v7, v3, v1, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    s_cmp_lg_u32 s3, 0
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
+; GFX9-NEXT:    s_subb_u32 s3, s7, s15
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v0, s6
+; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    s_cmp_eq_u64 s[2:3], s[6:7]
+; GFX9-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
+; GFX9-NEXT:    s_and_b32 s4, 1, s6
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    v_cmp_gt_u64_e64 s[4:5], s[12:13], 0
+; GFX9-NEXT:    s_cmp_eq_u64 s[14:15], 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[4:5]
+; GFX9-NEXT:    v_cmp_gt_i64_e64 s[4:5], s[14:15], 0
+; GFX9-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[4:5]
+; GFX9-NEXT:    s_and_b32 s4, 1, s6
+; GFX9-NEXT:    s_cmp_lt_u32 s20, 64
+; GFX9-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX9-NEXT:    s_cmp_eq_u32 s20, 0
+; GFX9-NEXT:    s_cselect_b32 s13, 1, 0
+; GFX9-NEXT:    s_lshr_b64 s[6:7], s[0:1], s20
+; GFX9-NEXT:    s_lshl_b64 s[8:9], s[2:3], s22
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    s_or_b64 s[6:7], s[6:7], s[8:9]
+; GFX9-NEXT:    s_ashr_i32 s8, s3, 31
+; GFX9-NEXT:    s_ashr_i64 s[4:5], s[2:3], s20
+; GFX9-NEXT:    s_ashr_i64 s[10:11], s[2:3], s21
+; GFX9-NEXT:    s_and_b32 s12, s12, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX9-NEXT:    s_cselect_b64 s[6:7], s[6:7], s[10:11]
+; GFX9-NEXT:    s_and_b32 s10, s13, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX9-NEXT:    s_cselect_b64 s[6:7], s[0:1], s[6:7]
+; GFX9-NEXT:    s_mov_b32 s9, s8
+; GFX9-NEXT:    s_cmp_lg_u32 s12, 0
+; GFX9-NEXT:    s_cselect_b64 s[4:5], s[4:5], s[8:9]
+; GFX9-NEXT:    s_add_u32 s6, s6, 0
+; GFX9-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX9-NEXT:    s_and_b32 s8, s8, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX9-NEXT:    s_addc_u32 s7, s7, 0
+; GFX9-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX9-NEXT:    s_and_b32 s8, s8, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX9-NEXT:    s_addc_u32 s4, s4, 0
+; GFX9-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-NEXT:    v_xor_b32_e32 v0, v1, v0
+; GFX9-NEXT:    s_and_b32 s8, s8, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v3, s0
+; GFX9-NEXT:    v_mov_b32_e32 v8, s1
+; GFX9-NEXT:    s_addc_u32 s5, s5, s23
+; GFX9-NEXT:    v_mov_b32_e32 v1, s6
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s7
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v3, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v8, v2, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v8, s2
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
+; GFX9-NEXT:    v_mov_b32_e32 v9, s3
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v5
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v4
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX9-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s6, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_ssubsat_v2i128:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_sub_u32 s28, s0, s8
+; GFX10-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX10-NEXT:    s_mov_b32 s46, s0
+; GFX10-NEXT:    s_and_b32 s17, s17, 1
+; GFX10-NEXT:    s_mov_b32 s47, s1
+; GFX10-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_subb_u32 s29, s1, s9
+; GFX10-NEXT:    s_cselect_b32 s18, 1, 0
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s0, s[28:29], s[46:47]
+; GFX10-NEXT:    s_and_b32 s18, s18, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s18, 0
+; GFX10-NEXT:    s_subb_u32 s30, s2, s10
+; GFX10-NEXT:    s_cselect_b32 s19, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX10-NEXT:    s_and_b32 s19, s19, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX10-NEXT:    s_subb_u32 s31, s3, s11
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s0, s[30:31], s[2:3]
+; GFX10-NEXT:    s_cmp_eq_u64 s[30:31], s[2:3]
+; GFX10-NEXT:    v_cmp_gt_u64_e64 s2, s[8:9], 0
+; GFX10-NEXT:    s_cselect_b32 s20, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX10-NEXT:    s_and_b32 s0, 1, s20
+; GFX10-NEXT:    s_cmp_eq_u64 s[10:11], 0
+; GFX10-NEXT:    s_movk_i32 s20, 0x7f
+; GFX10-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s2
+; GFX10-NEXT:    v_cmp_gt_i64_e64 s2, s[10:11], 0
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s0
+; GFX10-NEXT:    s_and_b32 s1, 1, s1
+; GFX10-NEXT:    s_sub_i32 s21, s20, 64
+; GFX10-NEXT:    s_sub_i32 s22, 64, s20
+; GFX10-NEXT:    s_cmp_lt_u32 s20, 64
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX10-NEXT:    s_cselect_b32 s10, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s20, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, 0, 1, s2
+; GFX10-NEXT:    s_cselect_b32 s23, 1, 0
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s1
+; GFX10-NEXT:    s_lshr_b64 s[0:1], s[28:29], s20
+; GFX10-NEXT:    s_lshl_b64 s[2:3], s[30:31], s22
+; GFX10-NEXT:    s_and_b32 s24, s10, 1
+; GFX10-NEXT:    s_or_b64 s[0:1], s[0:1], s[2:3]
+; GFX10-NEXT:    s_ashr_i32 s2, s31, 31
+; GFX10-NEXT:    s_ashr_i64 s[8:9], s[30:31], s20
+; GFX10-NEXT:    s_ashr_i64 s[10:11], s[30:31], s21
+; GFX10-NEXT:    s_cmp_lg_u32 s24, 0
+; GFX10-NEXT:    s_mov_b32 s3, s2
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[0:1], s[10:11]
+; GFX10-NEXT:    s_and_b32 s10, s23, 1
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v3, v2, vcc_lo
+; GFX10-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX10-NEXT:    v_mov_b32_e32 v2, s29
+; GFX10-NEXT:    s_cselect_b64 s[0:1], s[28:29], s[0:1]
+; GFX10-NEXT:    s_cmp_lg_u32 s24, 0
+; GFX10-NEXT:    v_xor_b32_e32 v0, v1, v0
+; GFX10-NEXT:    s_cselect_b64 s[2:3], s[8:9], s[2:3]
+; GFX10-NEXT:    s_add_u32 s0, s0, 0
+; GFX10-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s28
+; GFX10-NEXT:    s_and_b32 s8, s8, 1
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX10-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX10-NEXT:    s_brev_b32 s23, 1
+; GFX10-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX10-NEXT:    s_and_b32 s8, s8, 1
+; GFX10-NEXT:    v_mov_b32_e32 v3, s31
+; GFX10-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX10-NEXT:    s_addc_u32 s2, s2, 0
+; GFX10-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v1, s0, vcc_lo
+; GFX10-NEXT:    s_and_b32 s8, s8, 1
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v2, s1, vcc_lo
+; GFX10-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX10-NEXT:    v_mov_b32_e32 v2, s30
+; GFX10-NEXT:    s_addc_u32 s3, s3, s23
+; GFX10-NEXT:    s_sub_u32 s0, s4, s12
+; GFX10-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, s3, vcc_lo
+; GFX10-NEXT:    s_and_b32 s1, s1, 1
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s2, vcc_lo
+; GFX10-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX10-NEXT:    s_subb_u32 s1, s5, s13
+; GFX10-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s3, s[0:1], s[4:5]
+; GFX10-NEXT:    s_and_b32 s8, s8, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX10-NEXT:    s_subb_u32 s8, s6, s14
+; GFX10-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s3
+; GFX10-NEXT:    s_and_b32 s9, s9, 1
+; GFX10-NEXT:    v_mov_b32_e32 v7, s8
+; GFX10-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX10-NEXT:    s_subb_u32 s9, s7, s15
+; GFX10-NEXT:    s_cmp_eq_u64 s[8:9], s[6:7]
+; GFX10-NEXT:    v_cmp_lt_i64_e64 s3, s[8:9], s[6:7]
+; GFX10-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX10-NEXT:    v_mov_b32_e32 v8, s9
+; GFX10-NEXT:    s_and_b32 s2, 1, s2
+; GFX10-NEXT:    s_cmp_eq_u64 s[14:15], 0
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s2
+; GFX10-NEXT:    v_cmp_gt_u64_e64 s2, s[12:13], 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, 0, 1, s3
+; GFX10-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX10-NEXT:    s_and_b32 s16, 1, s3
+; GFX10-NEXT:    s_cmp_lt_u32 s20, 64
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, 0, 1, s2
+; GFX10-NEXT:    v_cmp_gt_i64_e64 s2, s[14:15], 0
+; GFX10-NEXT:    s_cselect_b32 s10, 1, 0
+; GFX10-NEXT:    s_cmp_eq_u32 s20, 0
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s16
+; GFX10-NEXT:    s_cselect_b32 s12, 1, 0
+; GFX10-NEXT:    s_lshl_b64 s[4:5], s[8:9], s22
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s2
+; GFX10-NEXT:    s_lshr_b64 s[2:3], s[0:1], s20
+; GFX10-NEXT:    s_and_b32 s13, s10, 1
+; GFX10-NEXT:    s_or_b64 s[2:3], s[2:3], s[4:5]
+; GFX10-NEXT:    s_ashr_i32 s4, s9, 31
+; GFX10-NEXT:    s_ashr_i64 s[6:7], s[8:9], s20
+; GFX10-NEXT:    s_ashr_i64 s[10:11], s[8:9], s21
+; GFX10-NEXT:    s_cmp_lg_u32 s13, 0
+; GFX10-NEXT:    s_mov_b32 s5, s4
+; GFX10-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[10:11]
+; GFX10-NEXT:    s_and_b32 s10, s12, 1
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc_lo
+; GFX10-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX10-NEXT:    v_mov_b32_e32 v6, s1
+; GFX10-NEXT:    s_cselect_b64 s[2:3], s[0:1], s[2:3]
+; GFX10-NEXT:    s_cmp_lg_u32 s13, 0
+; GFX10-NEXT:    v_xor_b32_e32 v4, v5, v4
+; GFX10-NEXT:    s_cselect_b64 s[4:5], s[6:7], s[4:5]
+; GFX10-NEXT:    s_add_u32 s2, s2, 0
+; GFX10-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX10-NEXT:    v_mov_b32_e32 v5, s0
+; GFX10-NEXT:    s_and_b32 s6, s6, 1
+; GFX10-NEXT:    v_and_b32_e32 v4, 1, v4
+; GFX10-NEXT:    s_cmp_lg_u32 s6, 0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    s_addc_u32 s3, s3, 0
+; GFX10-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v4
+; GFX10-NEXT:    s_and_b32 s6, s6, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s6, 0
+; GFX10-NEXT:    s_addc_u32 s4, s4, 0
+; GFX10-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, v5, s2, vcc_lo
+; GFX10-NEXT:    s_and_b32 s6, s6, 1
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v6, s3, vcc_lo
+; GFX10-NEXT:    s_cmp_lg_u32 s6, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v7, s4, vcc_lo
+; GFX10-NEXT:    s_addc_u32 s1, s5, s23
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_cndmask_b32_e64 v7, v8, s1, vcc_lo
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX10-NEXT:    v_readfirstlane_b32 s4, v4
+; GFX10-NEXT:    v_readfirstlane_b32 s5, v5
+; GFX10-NEXT:    v_readfirstlane_b32 s6, v6
+; GFX10-NEXT:    v_readfirstlane_b32 s7, v7
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i128> @llvm.ssub.sat.v2i128(<2 x i128> %lhs, <2 x i128> %rhs)
+  ret <2 x i128> %result
+}
+
+declare i7 @llvm.ssub.sat.i7(i7, i7) #0
+declare i8 @llvm.ssub.sat.i8(i8, i8) #0
+declare <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8>, <2 x i8>) #0
+declare <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8>, <4 x i8>) #0
+
+declare i16 @llvm.ssub.sat.i16(i16, i16) #0
+declare <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16>, <2 x i16>) #0
+declare <3 x i16> @llvm.ssub.sat.v3i16(<3 x i16>, <3 x i16>) #0
+declare <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16>, <4 x i16>) #0
+declare <5 x i16> @llvm.ssub.sat.v5i16(<5 x i16>, <5 x i16>) #0
+declare <6 x i16> @llvm.ssub.sat.v6i16(<6 x i16>, <6 x i16>) #0
+declare <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16>, <8 x i16>) #0
+
+declare i24 @llvm.ssub.sat.i24(i24, i24) #0
+
+declare i32 @llvm.ssub.sat.i32(i32, i32) #0
+declare <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32>, <2 x i32>) #0
+declare <3 x i32> @llvm.ssub.sat.v3i32(<3 x i32>, <3 x i32>) #0
+declare <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32>, <4 x i32>) #0
+declare <5 x i32> @llvm.ssub.sat.v5i32(<5 x i32>, <5 x i32>) #0
+declare <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32>, <16 x i32>) #0
+
+declare i48 @llvm.ssub.sat.i48(i48, i48) #0
+
+declare i64 @llvm.ssub.sat.i64(i64, i64) #0
+declare <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64>, <2 x i64>) #0
+
+declare i128 @llvm.ssub.sat.i128(i128, i128) #0
+declare <2 x i128> @llvm.ssub.sat.v2i128(<2 x i128>, <2 x i128>) #0
+
+attributes #0 = { nounwind readnone speculatable willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
new file mode 100644
index 0000000000000..413bc71038470
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
@@ -0,0 +1,4979 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=tahiti -o - %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=fiji -o - %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -o - %s | FileCheck -check-prefix=GFX10 %s
+
+define i7 @v_uaddsat_i7(i7 %lhs, i7 %rhs) {
+; GFX6-LABEL: v_uaddsat_i7:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 25, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 25, v1
+; GFX6-NEXT:    v_xor_b32_e32 v2, -1, v0
+; GFX6-NEXT:    v_min_u32_e32 v1, v2, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 25, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_i7:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 9, v0
+; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 9, v1
+; GFX8-NEXT:    v_xor_b32_e32 v2, 0xffff, v0
+; GFX8-NEXT:    v_min_u16_e32 v1, v2, v1
+; GFX8-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_lshrrev_b16_e32 v0, 9, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_i7:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 9, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 9, v1
+; GFX9-NEXT:    v_xor_b32_e32 v2, 0xffff, v0
+; GFX9-NEXT:    v_min_u16_e32 v1, v2, v1
+; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 9, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_uaddsat_i7:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 9, v0
+; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 9, v1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_xor_b32_e32 v2, 0xffff, v0
+; GFX10-NEXT:    v_min_u16_e64 v1, v2, v1
+; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    v_lshrrev_b16_e64 v0, 9, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i7 @llvm.uadd.sat.i7(i7 %lhs, i7 %rhs)
+  ret i7 %result
+}
+
+define amdgpu_ps i7 @s_uaddsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
+; GFX6-LABEL: s_uaddsat_i7:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 25
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 25
+; GFX6-NEXT:    s_not_b32 s2, s0
+; GFX6-NEXT:    s_cmp_lt_u32 s2, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX6-NEXT:    s_add_i32 s0, s0, s1
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 25
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_uaddsat_i7:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_bfe_u32 s2, 9, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX8-NEXT:    s_xor_b32 s3, s0, 0xffff
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s3, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX8-NEXT:    s_add_i32 s0, s0, s1
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_uaddsat_i7:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_bfe_u32 s2, 9, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX9-NEXT:    s_xor_b32 s3, s0, 0xffff
+; GFX9-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_cmp_lt_u32 s3, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX9-NEXT:    s_add_i32 s0, s0, s1
+; GFX9-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_uaddsat_i7:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_bfe_u32 s2, 9, 0x100000
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX10-NEXT:    s_xor_b32 s3, s0, 0xffff
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX10-NEXT:    s_cmp_lt_u32 s3, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX10-NEXT:    s_add_i32 s0, s0, s1
+; GFX10-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i7 @llvm.uadd.sat.i7(i7 %lhs, i7 %rhs)
+  ret i7 %result
+}
+
+define i8 @v_uaddsat_i8(i8 %lhs, i8 %rhs) {
+; GFX6-LABEL: v_uaddsat_i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
+; GFX6-NEXT:    v_xor_b32_e32 v2, -1, v0
+; GFX6-NEXT:    v_min_u32_e32 v1, v2, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX8-NEXT:    v_xor_b32_e32 v2, 0xffff, v0
+; GFX8-NEXT:    v_min_u16_e32 v1, v2, v1
+; GFX8-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-NEXT:    v_xor_b32_e32 v2, 0xffff, v0
+; GFX9-NEXT:    v_min_u16_e32 v1, v2, v1
+; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_uaddsat_i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 8, v0
+; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 8, v1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_xor_b32_e32 v2, 0xffff, v0
+; GFX10-NEXT:    v_min_u16_e64 v1, v2, v1
+; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    v_lshrrev_b16_e64 v0, 8, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i8 @llvm.uadd.sat.i8(i8 %lhs, i8 %rhs)
+  ret i8 %result
+}
+
+define amdgpu_ps i8 @s_uaddsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
+; GFX6-LABEL: s_uaddsat_i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 24
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX6-NEXT:    s_not_b32 s2, s0
+; GFX6-NEXT:    s_cmp_lt_u32 s2, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX6-NEXT:    s_add_i32 s0, s0, s1
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 24
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_uaddsat_i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_bfe_u32 s2, 8, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX8-NEXT:    s_xor_b32 s3, s0, 0xffff
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s3, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX8-NEXT:    s_add_i32 s0, s0, s1
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_uaddsat_i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_bfe_u32 s2, 8, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX9-NEXT:    s_xor_b32 s3, s0, 0xffff
+; GFX9-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_cmp_lt_u32 s3, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX9-NEXT:    s_add_i32 s0, s0, s1
+; GFX9-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_uaddsat_i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_bfe_u32 s2, 8, 0x100000
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX10-NEXT:    s_xor_b32 s3, s0, 0xffff
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX10-NEXT:    s_cmp_lt_u32 s3, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX10-NEXT:    s_add_i32 s0, s0, s1
+; GFX10-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i8 @llvm.uadd.sat.i8(i8 %lhs, i8 %rhs)
+  ret i8 %result
+}
+
+define i16 @v_uaddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
+; GFX6-LABEL: v_uaddsat_v2i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 8, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 8, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
+; GFX6-NEXT:    v_xor_b32_e32 v4, -1, v0
+; GFX6-NEXT:    v_min_u32_e32 v1, v4, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 24, v3
+; GFX6-NEXT:    v_xor_b32_e32 v3, -1, v1
+; GFX6-NEXT:    v_min_u32_e32 v2, v3, v2
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 24, v1
+; GFX6-NEXT:    v_mov_b32_e32 v2, 0xff
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, v1, v2
+; GFX6-NEXT:    v_and_b32_e32 v0, v0, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_v2i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v2, 8
+; GFX8-NEXT:    v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX8-NEXT:    s_mov_b32 s4, 0xffff
+; GFX8-NEXT:    v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX8-NEXT:    v_xor_b32_e32 v4, s4, v0
+; GFX8-NEXT:    v_min_u16_e32 v1, v4, v1
+; GFX8-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_xor_b32_e32 v1, s4, v3
+; GFX8-NEXT:    v_min_u16_e32 v1, v1, v2
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0xff
+; GFX8-NEXT:    v_add_u16_e32 v1, v3, v1
+; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v1, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_v2i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    s_mov_b32 s4, 0xffff
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-NEXT:    v_xor_b32_e32 v4, s4, v0
+; GFX9-NEXT:    v_min_u16_e32 v1, v4, v1
+; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_xor_b32_e32 v1, s4, v2
+; GFX9-NEXT:    v_min_u16_e32 v1, v1, v3
+; GFX9-NEXT:    s_movk_i32 s4, 0xff
+; GFX9-NEXT:    v_add_u16_e32 v1, v2, v1
+; GFX9-NEXT:    v_and_b32_sdwa v0, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v1, v1, s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_uaddsat_v2i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_mov_b32 s4, 8
+; GFX10-NEXT:    v_lshlrev_b16_e64 v2, 8, v0
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v0, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    s_mov_b32 s5, 0xffff
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 8, v1
+; GFX10-NEXT:    v_xor_b32_e32 v4, s5, v2
+; GFX10-NEXT:    v_xor_b32_e32 v5, s5, v0
+; GFX10-NEXT:    s_movk_i32 s4, 0xff
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_min_u16_e64 v1, v4, v1
+; GFX10-NEXT:    v_min_u16_e64 v3, v5, v3
+; GFX10-NEXT:    v_add_nc_u16_e64 v1, v2, v1
+; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v3
+; GFX10-NEXT:    v_and_b32_sdwa v1, v1, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v0, v0, s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %lhs = bitcast i16 %lhs.arg to <2 x i8>
+  %rhs = bitcast i16 %rhs.arg to <2 x i8>
+  %result = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %lhs, <2 x i8> %rhs)
+  %cast.result = bitcast <2 x i8> %result to i16
+  ret i16 %cast.result
+}
+
+define amdgpu_ps i16 @s_uaddsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
+; GFX6-LABEL: s_uaddsat_v2i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 24
+; GFX6-NEXT:    s_lshr_b32 s3, s1, 8
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX6-NEXT:    s_not_b32 s4, s0
+; GFX6-NEXT:    s_cmp_lt_u32 s4, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s4, s1
+; GFX6-NEXT:    s_add_i32 s0, s0, s1
+; GFX6-NEXT:    s_lshl_b32 s1, s2, 24
+; GFX6-NEXT:    s_lshl_b32 s2, s3, 24
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 24
+; GFX6-NEXT:    s_not_b32 s3, s1
+; GFX6-NEXT:    s_cmp_lt_u32 s3, s2
+; GFX6-NEXT:    s_cselect_b32 s2, s3, s2
+; GFX6-NEXT:    s_add_i32 s1, s1, s2
+; GFX6-NEXT:    s_movk_i32 s2, 0xff
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 24
+; GFX6-NEXT:    s_and_b32 s1, s1, s2
+; GFX6-NEXT:    s_and_b32 s0, s0, s2
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_uaddsat_v2i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_bfe_u32 s4, 8, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s4
+; GFX8-NEXT:    s_mov_b32 s5, 0xffff
+; GFX8-NEXT:    s_xor_b32 s6, s0, s5
+; GFX8-NEXT:    s_lshr_b32 s3, s1, 8
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s4
+; GFX8-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s6, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s6, s1
+; GFX8-NEXT:    s_add_i32 s0, s0, s1
+; GFX8-NEXT:    s_lshl_b32 s1, s2, s4
+; GFX8-NEXT:    s_lshl_b32 s2, s3, s4
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_xor_b32 s3, s1, s5
+; GFX8-NEXT:    s_lshr_b32 s0, s0, s4
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s3, s2
+; GFX8-NEXT:    s_cselect_b32 s2, s3, s2
+; GFX8-NEXT:    s_add_i32 s1, s1, s2
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_movk_i32 s2, 0xff
+; GFX8-NEXT:    s_lshr_b32 s1, s1, s4
+; GFX8-NEXT:    s_and_b32 s1, s1, s2
+; GFX8-NEXT:    s_and_b32 s0, s0, s2
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s4
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_uaddsat_v2i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_bfe_u32 s4, 8, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s4
+; GFX9-NEXT:    s_mov_b32 s5, 0xffff
+; GFX9-NEXT:    s_xor_b32 s6, s0, s5
+; GFX9-NEXT:    s_lshr_b32 s3, s1, 8
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s4
+; GFX9-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_cmp_lt_u32 s6, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s6, s1
+; GFX9-NEXT:    s_add_i32 s0, s0, s1
+; GFX9-NEXT:    s_lshl_b32 s1, s2, s4
+; GFX9-NEXT:    s_lshl_b32 s2, s3, s4
+; GFX9-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX9-NEXT:    s_xor_b32 s3, s1, s5
+; GFX9-NEXT:    s_lshr_b32 s0, s0, s4
+; GFX9-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX9-NEXT:    s_cmp_lt_u32 s3, s2
+; GFX9-NEXT:    s_cselect_b32 s2, s3, s2
+; GFX9-NEXT:    s_add_i32 s1, s1, s2
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_movk_i32 s2, 0xff
+; GFX9-NEXT:    s_lshr_b32 s1, s1, s4
+; GFX9-NEXT:    s_and_b32 s1, s1, s2
+; GFX9-NEXT:    s_and_b32 s0, s0, s2
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s4
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_uaddsat_v2i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_bfe_u32 s2, 8, 0x100000
+; GFX10-NEXT:    s_mov_b32 s4, 0xffff
+; GFX10-NEXT:    s_lshl_b32 s3, s0, s2
+; GFX10-NEXT:    s_lshl_b32 s6, s1, s2
+; GFX10-NEXT:    s_xor_b32 s5, s3, s4
+; GFX10-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s0, s0, 8
+; GFX10-NEXT:    s_lshr_b32 s1, s1, 8
+; GFX10-NEXT:    s_cmp_lt_u32 s5, s6
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX10-NEXT:    s_add_i32 s3, s3, s5
+; GFX10-NEXT:    s_xor_b32 s4, s0, s4
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX10-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s3, s3, s2
+; GFX10-NEXT:    s_cmp_lt_u32 s4, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s4, s1
+; GFX10-NEXT:    s_add_i32 s0, s0, s1
+; GFX10-NEXT:    s_movk_i32 s1, 0xff
+; GFX10-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX10-NEXT:    s_and_b32 s0, s0, s1
+; GFX10-NEXT:    s_and_b32 s1, s3, s1
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX10-NEXT:    s_or_b32 s0, s1, s0
+; GFX10-NEXT:    ; return to shader part epilog
+  %lhs = bitcast i16 %lhs.arg to <2 x i8>
+  %rhs = bitcast i16 %rhs.arg to <2 x i8>
+  %result = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %lhs, <2 x i8> %rhs)
+  %cast.result = bitcast <2 x i8> %result to i16
+  ret i16 %cast.result
+}
+
+define i32 @v_uaddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
+; GFX6-LABEL: v_uaddsat_v4i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 8, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 24, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v5, 8, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
+; GFX6-NEXT:    v_xor_b32_e32 v8, -1, v0
+; GFX6-NEXT:    v_min_u32_e32 v1, v8, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 24, v5
+; GFX6-NEXT:    v_xor_b32_e32 v5, -1, v1
+; GFX6-NEXT:    v_min_u32_e32 v2, v5, v2
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 24, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 24, v6
+; GFX6-NEXT:    v_xor_b32_e32 v5, -1, v2
+; GFX6-NEXT:    v_min_u32_e32 v3, v5, v3
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 24, v4
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 24, v1
+; GFX6-NEXT:    s_movk_i32 s4, 0xff
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 24, v7
+; GFX6-NEXT:    v_xor_b32_e32 v5, -1, v3
+; GFX6-NEXT:    v_min_u32_e32 v4, v5, v4
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 24, v2
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 24, v3
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_v4i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v2, 8
+; GFX8-NEXT:    v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 24, v0
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX8-NEXT:    s_mov_b32 s4, 0xffff
+; GFX8-NEXT:    v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
+; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX8-NEXT:    v_xor_b32_e32 v8, s4, v0
+; GFX8-NEXT:    v_min_u16_e32 v1, v8, v1
+; GFX8-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_xor_b32_e32 v1, s4, v3
+; GFX8-NEXT:    v_min_u16_e32 v1, v1, v2
+; GFX8-NEXT:    v_lshlrev_b16_e32 v2, 8, v4
+; GFX8-NEXT:    v_add_u16_e32 v1, v3, v1
+; GFX8-NEXT:    v_lshlrev_b16_e32 v3, 8, v6
+; GFX8-NEXT:    v_xor_b32_e32 v4, s4, v2
+; GFX8-NEXT:    v_min_u16_e32 v3, v4, v3
+; GFX8-NEXT:    v_add_u16_e32 v2, v2, v3
+; GFX8-NEXT:    v_lshlrev_b16_e32 v3, 8, v5
+; GFX8-NEXT:    v_lshlrev_b16_e32 v4, 8, v7
+; GFX8-NEXT:    v_xor_b32_e32 v5, s4, v3
+; GFX8-NEXT:    v_min_u16_e32 v4, v5, v4
+; GFX8-NEXT:    v_add_u16_e32 v3, v3, v4
+; GFX8-NEXT:    v_mov_b32_e32 v4, 0xff
+; GFX8-NEXT:    v_and_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_and_b32_sdwa v1, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_and_b32_sdwa v1, v3, v4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_v4i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_lshrrev_b32_sdwa v5, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 24, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    s_mov_b32 s4, 0xffff
+; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-NEXT:    v_xor_b32_e32 v8, s4, v0
+; GFX9-NEXT:    v_min_u16_e32 v1, v8, v1
+; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_xor_b32_e32 v1, s4, v2
+; GFX9-NEXT:    v_min_u16_e32 v1, v1, v5
+; GFX9-NEXT:    v_add_u16_e32 v1, v2, v1
+; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v3
+; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v6
+; GFX9-NEXT:    v_xor_b32_e32 v5, s4, v2
+; GFX9-NEXT:    v_min_u16_e32 v3, v5, v3
+; GFX9-NEXT:    v_add_u16_e32 v2, v2, v3
+; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v4
+; GFX9-NEXT:    v_xor_b32_e32 v5, s4, v3
+; GFX9-NEXT:    v_lshlrev_b16_e32 v4, 8, v7
+; GFX9-NEXT:    s_movk_i32 s4, 0xff
+; GFX9-NEXT:    v_and_b32_sdwa v1, v1, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_min_u16_e32 v4, v5, v4
+; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX9-NEXT:    v_add_u16_e32 v3, v3, v4
+; GFX9-NEXT:    v_and_or_b32 v0, v0, s4, v1
+; GFX9-NEXT:    v_and_b32_sdwa v1, v2, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v2, v3, s4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_uaddsat_v4i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_mov_b32 s4, 8
+; GFX10-NEXT:    s_mov_b32 s7, 0xffff
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshlrev_b16_e64 v4, 8, v0
+; GFX10-NEXT:    s_mov_b32 s5, 16
+; GFX10-NEXT:    s_mov_b32 s6, 24
+; GFX10-NEXT:    v_xor_b32_e32 v5, s7, v2
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v6, s5, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v0, s6, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshlrev_b16_e64 v7, 8, v1
+; GFX10-NEXT:    v_xor_b32_e32 v8, s7, v4
+; GFX10-NEXT:    v_min_u16_e64 v3, v5, v3
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v5, s5, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_xor_b32_e32 v11, s7, v6
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v1, s6, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_min_u16_e64 v7, v8, v7
+; GFX10-NEXT:    v_add_nc_u16_e64 v2, v2, v3
+; GFX10-NEXT:    v_xor_b32_e32 v3, s7, v0
+; GFX10-NEXT:    s_movk_i32 s4, 0xff
+; GFX10-NEXT:    v_min_u16_e64 v5, v11, v5
+; GFX10-NEXT:    v_add_nc_u16_e64 v4, v4, v7
+; GFX10-NEXT:    v_and_b32_sdwa v2, v2, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_min_u16_e64 v1, v3, v1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_nc_u16_e64 v3, v6, v5
+; GFX10-NEXT:    v_lshrrev_b16_e64 v4, 8, v4
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    v_and_b32_sdwa v1, v3, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_or_b32 v2, v4, s4, v2
+; GFX10-NEXT:    v_and_b32_sdwa v0, v0, s4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_or3_b32 v0, v2, v1, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %lhs = bitcast i32 %lhs.arg to <4 x i8>
+  %rhs = bitcast i32 %rhs.arg to <4 x i8>
+  %result = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> %lhs, <4 x i8> %rhs)
+  %cast.result = bitcast <4 x i8> %result to i32
+  ret i32 %cast.result
+}
+
+define amdgpu_ps i32 @s_uaddsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
+; GFX6-LABEL: s_uaddsat_v4i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX6-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX6-NEXT:    s_lshr_b32 s4, s0, 24
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 24
+; GFX6-NEXT:    s_lshr_b32 s5, s1, 8
+; GFX6-NEXT:    s_lshr_b32 s6, s1, 16
+; GFX6-NEXT:    s_lshr_b32 s7, s1, 24
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX6-NEXT:    s_not_b32 s8, s0
+; GFX6-NEXT:    s_cmp_lt_u32 s8, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s8, s1
+; GFX6-NEXT:    s_add_i32 s0, s0, s1
+; GFX6-NEXT:    s_lshl_b32 s1, s2, 24
+; GFX6-NEXT:    s_lshl_b32 s2, s5, 24
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 24
+; GFX6-NEXT:    s_not_b32 s5, s1
+; GFX6-NEXT:    s_cmp_lt_u32 s5, s2
+; GFX6-NEXT:    s_cselect_b32 s2, s5, s2
+; GFX6-NEXT:    s_add_i32 s1, s1, s2
+; GFX6-NEXT:    s_lshl_b32 s2, s3, 24
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 24
+; GFX6-NEXT:    s_lshl_b32 s3, s6, 24
+; GFX6-NEXT:    s_not_b32 s5, s2
+; GFX6-NEXT:    s_cmp_lt_u32 s5, s3
+; GFX6-NEXT:    s_cselect_b32 s3, s5, s3
+; GFX6-NEXT:    s_add_i32 s2, s2, s3
+; GFX6-NEXT:    s_lshl_b32 s3, s4, 24
+; GFX6-NEXT:    s_lshr_b32 s2, s2, 24
+; GFX6-NEXT:    s_lshl_b32 s4, s7, 24
+; GFX6-NEXT:    s_not_b32 s5, s3
+; GFX6-NEXT:    s_cmp_lt_u32 s5, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX6-NEXT:    s_add_i32 s3, s3, s4
+; GFX6-NEXT:    s_movk_i32 s4, 0xff
+; GFX6-NEXT:    s_and_b32 s1, s1, s4
+; GFX6-NEXT:    s_and_b32 s0, s0, s4
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_and_b32 s1, s2, s4
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_lshr_b32 s3, s3, 24
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_and_b32 s1, s3, s4
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_uaddsat_v4i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_bfe_u32 s8, 8, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX8-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s4, s0, 24
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s8
+; GFX8-NEXT:    s_mov_b32 s9, 0xffff
+; GFX8-NEXT:    s_xor_b32 s10, s0, s9
+; GFX8-NEXT:    s_lshr_b32 s5, s1, 8
+; GFX8-NEXT:    s_lshr_b32 s6, s1, 16
+; GFX8-NEXT:    s_lshr_b32 s7, s1, 24
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s8
+; GFX8-NEXT:    s_bfe_u32 s10, s10, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s10, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s10, s1
+; GFX8-NEXT:    s_add_i32 s0, s0, s1
+; GFX8-NEXT:    s_lshl_b32 s1, s2, s8
+; GFX8-NEXT:    s_lshl_b32 s2, s5, s8
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_xor_b32 s5, s1, s9
+; GFX8-NEXT:    s_lshr_b32 s0, s0, s8
+; GFX8-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s5, s2
+; GFX8-NEXT:    s_cselect_b32 s2, s5, s2
+; GFX8-NEXT:    s_add_i32 s1, s1, s2
+; GFX8-NEXT:    s_lshl_b32 s2, s3, s8
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s3, s6, s8
+; GFX8-NEXT:    s_xor_b32 s5, s2, s9
+; GFX8-NEXT:    s_lshr_b32 s1, s1, s8
+; GFX8-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s5, s3
+; GFX8-NEXT:    s_cselect_b32 s3, s5, s3
+; GFX8-NEXT:    s_add_i32 s2, s2, s3
+; GFX8-NEXT:    s_lshl_b32 s3, s4, s8
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s4, s7, s8
+; GFX8-NEXT:    s_xor_b32 s5, s3, s9
+; GFX8-NEXT:    s_lshr_b32 s2, s2, s8
+; GFX8-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s5, s4
+; GFX8-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX8-NEXT:    s_add_i32 s3, s3, s4
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_and_b32 s1, s1, s4
+; GFX8-NEXT:    s_and_b32 s0, s0, s4
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    s_and_b32 s1, s2, s4
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX8-NEXT:    s_lshr_b32 s3, s3, s8
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    s_and_b32 s1, s3, s4
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_uaddsat_v4i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_bfe_u32 s8, 8, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX9-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s4, s0, 24
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s8
+; GFX9-NEXT:    s_mov_b32 s9, 0xffff
+; GFX9-NEXT:    s_xor_b32 s10, s0, s9
+; GFX9-NEXT:    s_lshr_b32 s5, s1, 8
+; GFX9-NEXT:    s_lshr_b32 s6, s1, 16
+; GFX9-NEXT:    s_lshr_b32 s7, s1, 24
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s8
+; GFX9-NEXT:    s_bfe_u32 s10, s10, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_cmp_lt_u32 s10, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s10, s1
+; GFX9-NEXT:    s_add_i32 s0, s0, s1
+; GFX9-NEXT:    s_lshl_b32 s1, s2, s8
+; GFX9-NEXT:    s_lshl_b32 s2, s5, s8
+; GFX9-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX9-NEXT:    s_xor_b32 s5, s1, s9
+; GFX9-NEXT:    s_lshr_b32 s0, s0, s8
+; GFX9-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX9-NEXT:    s_cmp_lt_u32 s5, s2
+; GFX9-NEXT:    s_cselect_b32 s2, s5, s2
+; GFX9-NEXT:    s_add_i32 s1, s1, s2
+; GFX9-NEXT:    s_lshl_b32 s2, s3, s8
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s3, s6, s8
+; GFX9-NEXT:    s_xor_b32 s5, s2, s9
+; GFX9-NEXT:    s_lshr_b32 s1, s1, s8
+; GFX9-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX9-NEXT:    s_cmp_lt_u32 s5, s3
+; GFX9-NEXT:    s_cselect_b32 s3, s5, s3
+; GFX9-NEXT:    s_add_i32 s2, s2, s3
+; GFX9-NEXT:    s_lshl_b32 s3, s4, s8
+; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s4, s7, s8
+; GFX9-NEXT:    s_xor_b32 s5, s3, s9
+; GFX9-NEXT:    s_lshr_b32 s2, s2, s8
+; GFX9-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX9-NEXT:    s_cmp_lt_u32 s5, s4
+; GFX9-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX9-NEXT:    s_add_i32 s3, s3, s4
+; GFX9-NEXT:    s_movk_i32 s4, 0xff
+; GFX9-NEXT:    s_and_b32 s1, s1, s4
+; GFX9-NEXT:    s_and_b32 s0, s0, s4
+; GFX9-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX9-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    s_and_b32 s1, s2, s4
+; GFX9-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX9-NEXT:    s_lshr_b32 s3, s3, s8
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    s_and_b32 s1, s3, s4
+; GFX9-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_uaddsat_v4i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_bfe_u32 s5, 8, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX10-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX10-NEXT:    s_lshr_b32 s4, s0, 24
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s5
+; GFX10-NEXT:    s_mov_b32 s7, 0xffff
+; GFX10-NEXT:    s_lshl_b32 s10, s1, s5
+; GFX10-NEXT:    s_xor_b32 s9, s0, s7
+; GFX10-NEXT:    s_bfe_u32 s10, s10, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s9, s9, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s6, s1, 8
+; GFX10-NEXT:    s_lshr_b32 s8, s1, 16
+; GFX10-NEXT:    s_lshr_b32 s1, s1, 24
+; GFX10-NEXT:    s_cmp_lt_u32 s9, s10
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s9, s9, s10
+; GFX10-NEXT:    s_lshl_b32 s2, s2, s5
+; GFX10-NEXT:    s_add_i32 s0, s0, s9
+; GFX10-NEXT:    s_xor_b32 s9, s2, s7
+; GFX10-NEXT:    s_lshl_b32 s6, s6, s5
+; GFX10-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s9, s9, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s0, s0, s5
+; GFX10-NEXT:    s_cmp_lt_u32 s9, s6
+; GFX10-NEXT:    s_cselect_b32 s6, s9, s6
+; GFX10-NEXT:    s_lshl_b32 s3, s3, s5
+; GFX10-NEXT:    s_add_i32 s2, s2, s6
+; GFX10-NEXT:    s_xor_b32 s6, s3, s7
+; GFX10-NEXT:    s_lshl_b32 s8, s8, s5
+; GFX10-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s8, s8, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s2, s2, s5
+; GFX10-NEXT:    s_cmp_lt_u32 s6, s8
+; GFX10-NEXT:    s_cselect_b32 s6, s6, s8
+; GFX10-NEXT:    s_lshl_b32 s4, s4, s5
+; GFX10-NEXT:    s_add_i32 s3, s3, s6
+; GFX10-NEXT:    s_xor_b32 s6, s4, s7
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s5
+; GFX10-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s3, s3, s5
+; GFX10-NEXT:    s_cmp_lt_u32 s6, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s6, s1
+; GFX10-NEXT:    s_add_i32 s4, s4, s1
+; GFX10-NEXT:    s_bfe_u32 s1, s4, 0x100000
+; GFX10-NEXT:    s_movk_i32 s4, 0xff
+; GFX10-NEXT:    s_lshr_b32 s1, s1, s5
+; GFX10-NEXT:    s_and_b32 s2, s2, s4
+; GFX10-NEXT:    s_and_b32 s0, s0, s4
+; GFX10-NEXT:    s_lshl_b32 s2, s2, 8
+; GFX10-NEXT:    s_and_b32 s3, s3, s4
+; GFX10-NEXT:    s_and_b32 s1, s1, s4
+; GFX10-NEXT:    s_or_b32 s0, s0, s2
+; GFX10-NEXT:    s_lshl_b32 s2, s3, 16
+; GFX10-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX10-NEXT:    s_or_b32 s0, s0, s2
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %lhs = bitcast i32 %lhs.arg to <4 x i8>
+  %rhs = bitcast i32 %rhs.arg to <4 x i8>
+  %result = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> %lhs, <4 x i8> %rhs)
+  %cast.result = bitcast <4 x i8> %result to i32
+  ret i32 %cast.result
+}
+
+define i24 @v_uaddsat_i24(i24 %lhs, i24 %rhs) {
+; GFX6-LABEL: v_uaddsat_i24:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX6-NEXT:    v_xor_b32_e32 v2, -1, v0
+; GFX6-NEXT:    v_min_u32_e32 v1, v2, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_i24:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v0
+; GFX8-NEXT:    v_min_u32_e32 v1, v2, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_i24:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v0
+; GFX9-NEXT:    v_min_u32_e32 v1, v2, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_uaddsat_i24:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_xor_b32_e32 v2, -1, v0
+; GFX10-NEXT:    v_min_u32_e32 v1, v2, v1
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i24 @llvm.uadd.sat.i24(i24 %lhs, i24 %rhs)
+  ret i24 %result
+}
+
+define amdgpu_ps i24 @s_uaddsat_i24(i24 inreg %lhs, i24 inreg %rhs) {
+; GFX6-LABEL: s_uaddsat_i24:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 8
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX6-NEXT:    s_not_b32 s2, s0
+; GFX6-NEXT:    s_cmp_lt_u32 s2, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX6-NEXT:    s_add_i32 s0, s0, s1
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 8
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_uaddsat_i24:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_lshl_b32 s0, s0, 8
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX8-NEXT:    s_not_b32 s2, s0
+; GFX8-NEXT:    s_cmp_lt_u32 s2, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX8-NEXT:    s_add_i32 s0, s0, s1
+; GFX8-NEXT:    s_lshr_b32 s0, s0, 8
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_uaddsat_i24:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 8
+; GFX9-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX9-NEXT:    s_not_b32 s2, s0
+; GFX9-NEXT:    s_cmp_lt_u32 s2, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX9-NEXT:    s_add_i32 s0, s0, s1
+; GFX9-NEXT:    s_lshr_b32 s0, s0, 8
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_uaddsat_i24:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_lshl_b32 s0, s0, 8
+; GFX10-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX10-NEXT:    s_not_b32 s2, s0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cmp_lt_u32 s2, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX10-NEXT:    s_add_i32 s0, s0, s1
+; GFX10-NEXT:    s_lshr_b32 s0, s0, 8
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i24 @llvm.uadd.sat.i24(i24 %lhs, i24 %rhs)
+  ret i24 %result
+}
+
+define i32 @v_uaddsat_i32(i32 %lhs, i32 %rhs) {
+; GFX6-LABEL: v_uaddsat_i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_xor_b32_e32 v2, -1, v0
+; GFX6-NEXT:    v_min_u32_e32 v1, v2, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v0
+; GFX8-NEXT:    v_min_u32_e32 v1, v2, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v0
+; GFX9-NEXT:    v_min_u32_e32 v1, v2, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_uaddsat_i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_xor_b32_e32 v2, -1, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_min_u32_e32 v1, v2, v1
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i32 @llvm.uadd.sat.i32(i32 %lhs, i32 %rhs)
+  ret i32 %result
+}
+
+define amdgpu_ps i32 @s_uaddsat_i32(i32 inreg %lhs, i32 inreg %rhs) {
+; GFX6-LABEL: s_uaddsat_i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_not_b32 s2, s0
+; GFX6-NEXT:    s_cmp_lt_u32 s2, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX6-NEXT:    s_add_i32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_uaddsat_i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_not_b32 s2, s0
+; GFX8-NEXT:    s_cmp_lt_u32 s2, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX8-NEXT:    s_add_i32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_uaddsat_i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_not_b32 s2, s0
+; GFX9-NEXT:    s_cmp_lt_u32 s2, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX9-NEXT:    s_add_i32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_uaddsat_i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_not_b32 s2, s0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cmp_lt_u32 s2, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX10-NEXT:    s_add_i32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i32 @llvm.uadd.sat.i32(i32 %lhs, i32 %rhs)
+  ret i32 %result
+}
+
+define amdgpu_ps float @uaddsat_i32_sv(i32 inreg %lhs, i32 %rhs) {
+; GFX6-LABEL: uaddsat_i32_sv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_not_b32 s1, s0
+; GFX6-NEXT:    v_min_u32_e32 v0, s1, v0
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: uaddsat_i32_sv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_not_b32 s1, s0
+; GFX8-NEXT:    v_min_u32_e32 v0, s1, v0
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: uaddsat_i32_sv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_not_b32 s1, s0
+; GFX9-NEXT:    v_min_u32_e32 v0, s1, v0
+; GFX9-NEXT:    v_add_u32_e32 v0, s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: uaddsat_i32_sv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_not_b32 s1, s0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_min_u32_e32 v0, s1, v0
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i32 @llvm.uadd.sat.i32(i32 %lhs, i32 %rhs)
+  %cast = bitcast i32 %result to float
+  ret float %cast
+}
+
+define amdgpu_ps float @uaddsat_i32_vs(i32 %lhs, i32 inreg %rhs) {
+; GFX6-LABEL: uaddsat_i32_vs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_xor_b32_e32 v1, -1, v0
+; GFX6-NEXT:    v_min_u32_e32 v1, s0, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: uaddsat_i32_vs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_xor_b32_e32 v1, -1, v0
+; GFX8-NEXT:    v_min_u32_e32 v1, s0, v1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: uaddsat_i32_vs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_xor_b32_e32 v1, -1, v0
+; GFX9-NEXT:    v_min_u32_e32 v1, s0, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: uaddsat_i32_vs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_xor_b32_e32 v1, -1, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_min_u32_e32 v1, s0, v1
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i32 @llvm.uadd.sat.i32(i32 %lhs, i32 %rhs)
+  %cast = bitcast i32 %result to float
+  ret float %cast
+}
+
+define <2 x i32> @v_uaddsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
+; GFX6-LABEL: v_uaddsat_v2i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_xor_b32_e32 v4, -1, v0
+; GFX6-NEXT:    v_min_u32_e32 v2, v4, v2
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT:    v_xor_b32_e32 v2, -1, v1
+; GFX6-NEXT:    v_min_u32_e32 v2, v2, v3
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_v2i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v4, -1, v0
+; GFX8-NEXT:    v_min_u32_e32 v2, v4, v2
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT:    v_xor_b32_e32 v2, -1, v1
+; GFX8-NEXT:    v_min_u32_e32 v2, v2, v3
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v2
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_v2i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v0
+; GFX9-NEXT:    v_min_u32_e32 v2, v4, v2
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v2
+; GFX9-NEXT:    v_xor_b32_e32 v2, -1, v1
+; GFX9-NEXT:    v_min_u32_e32 v2, v2, v3
+; GFX9-NEXT:    v_add_u32_e32 v1, v1, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_uaddsat_v2i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_xor_b32_e32 v4, -1, v0
+; GFX10-NEXT:    v_xor_b32_e32 v5, -1, v1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_min_u32_e32 v2, v4, v2
+; GFX10-NEXT:    v_min_u32_e32 v3, v5, v3
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v2
+; GFX10-NEXT:    v_add_nc_u32_e32 v1, v1, v3
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> %lhs, <2 x i32> %rhs)
+  ret <2 x i32> %result
+}
+
+define amdgpu_ps <2 x i32> @s_uaddsat_v2i32(<2 x i32> inreg %lhs, <2 x i32> inreg %rhs) {
+; GFX6-LABEL: s_uaddsat_v2i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_not_b32 s4, s0
+; GFX6-NEXT:    s_cmp_lt_u32 s4, s2
+; GFX6-NEXT:    s_cselect_b32 s2, s4, s2
+; GFX6-NEXT:    s_add_i32 s0, s0, s2
+; GFX6-NEXT:    s_not_b32 s2, s1
+; GFX6-NEXT:    s_cmp_lt_u32 s2, s3
+; GFX6-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX6-NEXT:    s_add_i32 s1, s1, s2
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_uaddsat_v2i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_not_b32 s4, s0
+; GFX8-NEXT:    s_cmp_lt_u32 s4, s2
+; GFX8-NEXT:    s_cselect_b32 s2, s4, s2
+; GFX8-NEXT:    s_add_i32 s0, s0, s2
+; GFX8-NEXT:    s_not_b32 s2, s1
+; GFX8-NEXT:    s_cmp_lt_u32 s2, s3
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX8-NEXT:    s_add_i32 s1, s1, s2
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_uaddsat_v2i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_not_b32 s4, s0
+; GFX9-NEXT:    s_cmp_lt_u32 s4, s2
+; GFX9-NEXT:    s_cselect_b32 s2, s4, s2
+; GFX9-NEXT:    s_add_i32 s0, s0, s2
+; GFX9-NEXT:    s_not_b32 s2, s1
+; GFX9-NEXT:    s_cmp_lt_u32 s2, s3
+; GFX9-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX9-NEXT:    s_add_i32 s1, s1, s2
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_uaddsat_v2i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_not_b32 s4, s0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cmp_lt_u32 s4, s2
+; GFX10-NEXT:    s_cselect_b32 s2, s4, s2
+; GFX10-NEXT:    s_not_b32 s4, s1
+; GFX10-NEXT:    s_add_i32 s0, s0, s2
+; GFX10-NEXT:    s_cmp_lt_u32 s4, s3
+; GFX10-NEXT:    s_cselect_b32 s2, s4, s3
+; GFX10-NEXT:    s_add_i32 s1, s1, s2
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> %lhs, <2 x i32> %rhs)
+  ret <2 x i32> %result
+}
+
+define <3 x i32> @v_uaddsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
+; GFX6-LABEL: v_uaddsat_v3i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_xor_b32_e32 v6, -1, v0
+; GFX6-NEXT:    v_min_u32_e32 v3, v6, v3
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
+; GFX6-NEXT:    v_xor_b32_e32 v3, -1, v1
+; GFX6-NEXT:    v_min_u32_e32 v3, v3, v4
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GFX6-NEXT:    v_xor_b32_e32 v3, -1, v2
+; GFX6-NEXT:    v_min_u32_e32 v3, v3, v5
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_v3i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v6, -1, v0
+; GFX8-NEXT:    v_min_u32_e32 v3, v6, v3
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v3
+; GFX8-NEXT:    v_xor_b32_e32 v3, -1, v1
+; GFX8-NEXT:    v_min_u32_e32 v3, v3, v4
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v3
+; GFX8-NEXT:    v_xor_b32_e32 v3, -1, v2
+; GFX8-NEXT:    v_min_u32_e32 v3, v3, v5
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v3
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_v3i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v6, -1, v0
+; GFX9-NEXT:    v_min_u32_e32 v3, v6, v3
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v3
+; GFX9-NEXT:    v_xor_b32_e32 v3, -1, v1
+; GFX9-NEXT:    v_min_u32_e32 v3, v3, v4
+; GFX9-NEXT:    v_add_u32_e32 v1, v1, v3
+; GFX9-NEXT:    v_xor_b32_e32 v3, -1, v2
+; GFX9-NEXT:    v_min_u32_e32 v3, v3, v5
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v3
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_uaddsat_v3i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_xor_b32_e32 v6, -1, v0
+; GFX10-NEXT:    v_xor_b32_e32 v7, -1, v1
+; GFX10-NEXT:    v_xor_b32_e32 v8, -1, v2
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_min_u32_e32 v3, v6, v3
+; GFX10-NEXT:    v_min_u32_e32 v4, v7, v4
+; GFX10-NEXT:    v_min_u32_e32 v5, v8, v5
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v3
+; GFX10-NEXT:    v_add_nc_u32_e32 v1, v1, v4
+; GFX10-NEXT:    v_add_nc_u32_e32 v2, v2, v5
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <3 x i32> @llvm.uadd.sat.v3i32(<3 x i32> %lhs, <3 x i32> %rhs)
+  ret <3 x i32> %result
+}
+
+define amdgpu_ps <3 x i32> @s_uaddsat_v3i32(<3 x i32> inreg %lhs, <3 x i32> inreg %rhs) {
+; GFX6-LABEL: s_uaddsat_v3i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_not_b32 s6, s0
+; GFX6-NEXT:    s_cmp_lt_u32 s6, s3
+; GFX6-NEXT:    s_cselect_b32 s3, s6, s3
+; GFX6-NEXT:    s_add_i32 s0, s0, s3
+; GFX6-NEXT:    s_not_b32 s3, s1
+; GFX6-NEXT:    s_cmp_lt_u32 s3, s4
+; GFX6-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX6-NEXT:    s_add_i32 s1, s1, s3
+; GFX6-NEXT:    s_not_b32 s3, s2
+; GFX6-NEXT:    s_cmp_lt_u32 s3, s5
+; GFX6-NEXT:    s_cselect_b32 s3, s3, s5
+; GFX6-NEXT:    s_add_i32 s2, s2, s3
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_uaddsat_v3i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_not_b32 s6, s0
+; GFX8-NEXT:    s_cmp_lt_u32 s6, s3
+; GFX8-NEXT:    s_cselect_b32 s3, s6, s3
+; GFX8-NEXT:    s_add_i32 s0, s0, s3
+; GFX8-NEXT:    s_not_b32 s3, s1
+; GFX8-NEXT:    s_cmp_lt_u32 s3, s4
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX8-NEXT:    s_add_i32 s1, s1, s3
+; GFX8-NEXT:    s_not_b32 s3, s2
+; GFX8-NEXT:    s_cmp_lt_u32 s3, s5
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s5
+; GFX8-NEXT:    s_add_i32 s2, s2, s3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_uaddsat_v3i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_not_b32 s6, s0
+; GFX9-NEXT:    s_cmp_lt_u32 s6, s3
+; GFX9-NEXT:    s_cselect_b32 s3, s6, s3
+; GFX9-NEXT:    s_add_i32 s0, s0, s3
+; GFX9-NEXT:    s_not_b32 s3, s1
+; GFX9-NEXT:    s_cmp_lt_u32 s3, s4
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX9-NEXT:    s_add_i32 s1, s1, s3
+; GFX9-NEXT:    s_not_b32 s3, s2
+; GFX9-NEXT:    s_cmp_lt_u32 s3, s5
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s5
+; GFX9-NEXT:    s_add_i32 s2, s2, s3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_uaddsat_v3i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_not_b32 s6, s0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cmp_lt_u32 s6, s3
+; GFX10-NEXT:    s_cselect_b32 s3, s6, s3
+; GFX10-NEXT:    s_not_b32 s6, s1
+; GFX10-NEXT:    s_add_i32 s0, s0, s3
+; GFX10-NEXT:    s_cmp_lt_u32 s6, s4
+; GFX10-NEXT:    s_cselect_b32 s3, s6, s4
+; GFX10-NEXT:    s_not_b32 s4, s2
+; GFX10-NEXT:    s_add_i32 s1, s1, s3
+; GFX10-NEXT:    s_cmp_lt_u32 s4, s5
+; GFX10-NEXT:    s_cselect_b32 s3, s4, s5
+; GFX10-NEXT:    s_add_i32 s2, s2, s3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <3 x i32> @llvm.uadd.sat.v3i32(<3 x i32> %lhs, <3 x i32> %rhs)
+  ret <3 x i32> %result
+}
+
+define <4 x i32> @v_uaddsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
+; GFX6-LABEL: v_uaddsat_v4i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_xor_b32_e32 v8, -1, v0
+; GFX6-NEXT:    v_min_u32_e32 v4, v8, v4
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; GFX6-NEXT:    v_xor_b32_e32 v4, -1, v1
+; GFX6-NEXT:    v_min_u32_e32 v4, v4, v5
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
+; GFX6-NEXT:    v_xor_b32_e32 v4, -1, v2
+; GFX6-NEXT:    v_min_u32_e32 v4, v4, v6
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; GFX6-NEXT:    v_xor_b32_e32 v4, -1, v3
+; GFX6-NEXT:    v_min_u32_e32 v4, v4, v7
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_v4i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v8, -1, v0
+; GFX8-NEXT:    v_min_u32_e32 v4, v8, v4
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v4
+; GFX8-NEXT:    v_xor_b32_e32 v4, -1, v1
+; GFX8-NEXT:    v_min_u32_e32 v4, v4, v5
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v4
+; GFX8-NEXT:    v_xor_b32_e32 v4, -1, v2
+; GFX8-NEXT:    v_min_u32_e32 v4, v4, v6
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v4
+; GFX8-NEXT:    v_xor_b32_e32 v4, -1, v3
+; GFX8-NEXT:    v_min_u32_e32 v4, v4, v7
+; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v4
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_v4i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v8, -1, v0
+; GFX9-NEXT:    v_min_u32_e32 v4, v8, v4
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v4
+; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v1
+; GFX9-NEXT:    v_min_u32_e32 v4, v4, v5
+; GFX9-NEXT:    v_add_u32_e32 v1, v1, v4
+; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v2
+; GFX9-NEXT:    v_min_u32_e32 v4, v4, v6
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v4
+; GFX9-NEXT:    v_xor_b32_e32 v4, -1, v3
+; GFX9-NEXT:    v_min_u32_e32 v4, v4, v7
+; GFX9-NEXT:    v_add_u32_e32 v3, v3, v4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_uaddsat_v4i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_xor_b32_e32 v15, -1, v0
+; GFX10-NEXT:    v_xor_b32_e32 v19, -1, v1
+; GFX10-NEXT:    v_xor_b32_e32 v23, -1, v2
+; GFX10-NEXT:    v_xor_b32_e32 v10, -1, v3
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_min_u32_e32 v11, v15, v4
+; GFX10-NEXT:    v_min_u32_e32 v15, v19, v5
+; GFX10-NEXT:    v_min_u32_e32 v19, v23, v6
+; GFX10-NEXT:    v_min_u32_e32 v6, v10, v7
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v11
+; GFX10-NEXT:    v_add_nc_u32_e32 v1, v1, v15
+; GFX10-NEXT:    v_add_nc_u32_e32 v2, v2, v19
+; GFX10-NEXT:    v_add_nc_u32_e32 v3, v3, v6
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %lhs, <4 x i32> %rhs)
+  ret <4 x i32> %result
+}
+
+define amdgpu_ps <4 x i32> @s_uaddsat_v4i32(<4 x i32> inreg %lhs, <4 x i32> inreg %rhs) {
+; GFX6-LABEL: s_uaddsat_v4i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_not_b32 s8, s0
+; GFX6-NEXT:    s_cmp_lt_u32 s8, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s8, s4
+; GFX6-NEXT:    s_add_i32 s0, s0, s4
+; GFX6-NEXT:    s_not_b32 s4, s1
+; GFX6-NEXT:    s_cmp_lt_u32 s4, s5
+; GFX6-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX6-NEXT:    s_add_i32 s1, s1, s4
+; GFX6-NEXT:    s_not_b32 s4, s2
+; GFX6-NEXT:    s_cmp_lt_u32 s4, s6
+; GFX6-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX6-NEXT:    s_add_i32 s2, s2, s4
+; GFX6-NEXT:    s_not_b32 s4, s3
+; GFX6-NEXT:    s_cmp_lt_u32 s4, s7
+; GFX6-NEXT:    s_cselect_b32 s4, s4, s7
+; GFX6-NEXT:    s_add_i32 s3, s3, s4
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_uaddsat_v4i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_not_b32 s8, s0
+; GFX8-NEXT:    s_cmp_lt_u32 s8, s4
+; GFX8-NEXT:    s_cselect_b32 s4, s8, s4
+; GFX8-NEXT:    s_add_i32 s0, s0, s4
+; GFX8-NEXT:    s_not_b32 s4, s1
+; GFX8-NEXT:    s_cmp_lt_u32 s4, s5
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX8-NEXT:    s_add_i32 s1, s1, s4
+; GFX8-NEXT:    s_not_b32 s4, s2
+; GFX8-NEXT:    s_cmp_lt_u32 s4, s6
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX8-NEXT:    s_add_i32 s2, s2, s4
+; GFX8-NEXT:    s_not_b32 s4, s3
+; GFX8-NEXT:    s_cmp_lt_u32 s4, s7
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s7
+; GFX8-NEXT:    s_add_i32 s3, s3, s4
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_uaddsat_v4i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_not_b32 s8, s0
+; GFX9-NEXT:    s_cmp_lt_u32 s8, s4
+; GFX9-NEXT:    s_cselect_b32 s4, s8, s4
+; GFX9-NEXT:    s_add_i32 s0, s0, s4
+; GFX9-NEXT:    s_not_b32 s4, s1
+; GFX9-NEXT:    s_cmp_lt_u32 s4, s5
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX9-NEXT:    s_add_i32 s1, s1, s4
+; GFX9-NEXT:    s_not_b32 s4, s2
+; GFX9-NEXT:    s_cmp_lt_u32 s4, s6
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX9-NEXT:    s_add_i32 s2, s2, s4
+; GFX9-NEXT:    s_not_b32 s4, s3
+; GFX9-NEXT:    s_cmp_lt_u32 s4, s7
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s7
+; GFX9-NEXT:    s_add_i32 s3, s3, s4
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_uaddsat_v4i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_not_b32 s8, s0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cmp_lt_u32 s8, s4
+; GFX10-NEXT:    s_cselect_b32 s4, s8, s4
+; GFX10-NEXT:    s_not_b32 s8, s1
+; GFX10-NEXT:    s_add_i32 s0, s0, s4
+; GFX10-NEXT:    s_cmp_lt_u32 s8, s5
+; GFX10-NEXT:    s_cselect_b32 s4, s8, s5
+; GFX10-NEXT:    s_not_b32 s5, s2
+; GFX10-NEXT:    s_add_i32 s1, s1, s4
+; GFX10-NEXT:    s_cmp_lt_u32 s5, s6
+; GFX10-NEXT:    s_cselect_b32 s4, s5, s6
+; GFX10-NEXT:    s_not_b32 s5, s3
+; GFX10-NEXT:    s_add_i32 s2, s2, s4
+; GFX10-NEXT:    s_cmp_lt_u32 s5, s7
+; GFX10-NEXT:    s_cselect_b32 s4, s5, s7
+; GFX10-NEXT:    s_add_i32 s3, s3, s4
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %lhs, <4 x i32> %rhs)
+  ret <4 x i32> %result
+}
+
+define <5 x i32> @v_uaddsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
+; GFX6-LABEL: v_uaddsat_v5i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_xor_b32_e32 v10, -1, v0
+; GFX6-NEXT:    v_min_u32_e32 v5, v10, v5
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v5
+; GFX6-NEXT:    v_xor_b32_e32 v5, -1, v1
+; GFX6-NEXT:    v_min_u32_e32 v5, v5, v6
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v5
+; GFX6-NEXT:    v_xor_b32_e32 v5, -1, v2
+; GFX6-NEXT:    v_min_u32_e32 v5, v5, v7
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GFX6-NEXT:    v_xor_b32_e32 v5, -1, v3
+; GFX6-NEXT:    v_min_u32_e32 v5, v5, v8
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GFX6-NEXT:    v_xor_b32_e32 v5, -1, v4
+; GFX6-NEXT:    v_min_u32_e32 v5, v5, v9
+; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_v5i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v10, -1, v0
+; GFX8-NEXT:    v_min_u32_e32 v5, v10, v5
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v5
+; GFX8-NEXT:    v_xor_b32_e32 v5, -1, v1
+; GFX8-NEXT:    v_min_u32_e32 v5, v5, v6
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v5
+; GFX8-NEXT:    v_xor_b32_e32 v5, -1, v2
+; GFX8-NEXT:    v_min_u32_e32 v5, v5, v7
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v5
+; GFX8-NEXT:    v_xor_b32_e32 v5, -1, v3
+; GFX8-NEXT:    v_min_u32_e32 v5, v5, v8
+; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v5
+; GFX8-NEXT:    v_xor_b32_e32 v5, -1, v4
+; GFX8-NEXT:    v_min_u32_e32 v5, v5, v9
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v4, v5
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_v5i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v10, -1, v0
+; GFX9-NEXT:    v_min_u32_e32 v5, v10, v5
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v5
+; GFX9-NEXT:    v_xor_b32_e32 v5, -1, v1
+; GFX9-NEXT:    v_min_u32_e32 v5, v5, v6
+; GFX9-NEXT:    v_add_u32_e32 v1, v1, v5
+; GFX9-NEXT:    v_xor_b32_e32 v5, -1, v2
+; GFX9-NEXT:    v_min_u32_e32 v5, v5, v7
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v5
+; GFX9-NEXT:    v_xor_b32_e32 v5, -1, v3
+; GFX9-NEXT:    v_min_u32_e32 v5, v5, v8
+; GFX9-NEXT:    v_add_u32_e32 v3, v3, v5
+; GFX9-NEXT:    v_xor_b32_e32 v5, -1, v4
+; GFX9-NEXT:    v_min_u32_e32 v5, v5, v9
+; GFX9-NEXT:    v_add_u32_e32 v4, v4, v5
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_uaddsat_v5i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_xor_b32_e32 v10, -1, v0
+; GFX10-NEXT:    v_xor_b32_e32 v11, -1, v1
+; GFX10-NEXT:    v_xor_b32_e32 v12, -1, v2
+; GFX10-NEXT:    v_xor_b32_e32 v13, -1, v3
+; GFX10-NEXT:    v_xor_b32_e32 v14, -1, v4
+; GFX10-NEXT:    v_min_u32_e32 v5, v10, v5
+; GFX10-NEXT:    v_min_u32_e32 v6, v11, v6
+; GFX10-NEXT:    v_min_u32_e32 v7, v12, v7
+; GFX10-NEXT:    v_min_u32_e32 v8, v13, v8
+; GFX10-NEXT:    v_min_u32_e32 v9, v14, v9
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v5
+; GFX10-NEXT:    v_add_nc_u32_e32 v1, v1, v6
+; GFX10-NEXT:    v_add_nc_u32_e32 v2, v2, v7
+; GFX10-NEXT:    v_add_nc_u32_e32 v3, v3, v8
+; GFX10-NEXT:    v_add_nc_u32_e32 v4, v4, v9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <5 x i32> @llvm.uadd.sat.v5i32(<5 x i32> %lhs, <5 x i32> %rhs)
+  ret <5 x i32> %result
+}
+
+define amdgpu_ps <5 x i32> @s_uaddsat_v5i32(<5 x i32> inreg %lhs, <5 x i32> inreg %rhs) {
+; GFX6-LABEL: s_uaddsat_v5i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_not_b32 s10, s0
+; GFX6-NEXT:    s_cmp_lt_u32 s10, s5
+; GFX6-NEXT:    s_cselect_b32 s5, s10, s5
+; GFX6-NEXT:    s_add_i32 s0, s0, s5
+; GFX6-NEXT:    s_not_b32 s5, s1
+; GFX6-NEXT:    s_cmp_lt_u32 s5, s6
+; GFX6-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX6-NEXT:    s_add_i32 s1, s1, s5
+; GFX6-NEXT:    s_not_b32 s5, s2
+; GFX6-NEXT:    s_cmp_lt_u32 s5, s7
+; GFX6-NEXT:    s_cselect_b32 s5, s5, s7
+; GFX6-NEXT:    s_add_i32 s2, s2, s5
+; GFX6-NEXT:    s_not_b32 s5, s3
+; GFX6-NEXT:    s_cmp_lt_u32 s5, s8
+; GFX6-NEXT:    s_cselect_b32 s5, s5, s8
+; GFX6-NEXT:    s_add_i32 s3, s3, s5
+; GFX6-NEXT:    s_not_b32 s5, s4
+; GFX6-NEXT:    s_cmp_lt_u32 s5, s9
+; GFX6-NEXT:    s_cselect_b32 s5, s5, s9
+; GFX6-NEXT:    s_add_i32 s4, s4, s5
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_uaddsat_v5i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_not_b32 s10, s0
+; GFX8-NEXT:    s_cmp_lt_u32 s10, s5
+; GFX8-NEXT:    s_cselect_b32 s5, s10, s5
+; GFX8-NEXT:    s_add_i32 s0, s0, s5
+; GFX8-NEXT:    s_not_b32 s5, s1
+; GFX8-NEXT:    s_cmp_lt_u32 s5, s6
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX8-NEXT:    s_add_i32 s1, s1, s5
+; GFX8-NEXT:    s_not_b32 s5, s2
+; GFX8-NEXT:    s_cmp_lt_u32 s5, s7
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s7
+; GFX8-NEXT:    s_add_i32 s2, s2, s5
+; GFX8-NEXT:    s_not_b32 s5, s3
+; GFX8-NEXT:    s_cmp_lt_u32 s5, s8
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s8
+; GFX8-NEXT:    s_add_i32 s3, s3, s5
+; GFX8-NEXT:    s_not_b32 s5, s4
+; GFX8-NEXT:    s_cmp_lt_u32 s5, s9
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s9
+; GFX8-NEXT:    s_add_i32 s4, s4, s5
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_uaddsat_v5i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_not_b32 s10, s0
+; GFX9-NEXT:    s_cmp_lt_u32 s10, s5
+; GFX9-NEXT:    s_cselect_b32 s5, s10, s5
+; GFX9-NEXT:    s_add_i32 s0, s0, s5
+; GFX9-NEXT:    s_not_b32 s5, s1
+; GFX9-NEXT:    s_cmp_lt_u32 s5, s6
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX9-NEXT:    s_add_i32 s1, s1, s5
+; GFX9-NEXT:    s_not_b32 s5, s2
+; GFX9-NEXT:    s_cmp_lt_u32 s5, s7
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s7
+; GFX9-NEXT:    s_add_i32 s2, s2, s5
+; GFX9-NEXT:    s_not_b32 s5, s3
+; GFX9-NEXT:    s_cmp_lt_u32 s5, s8
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s8
+; GFX9-NEXT:    s_add_i32 s3, s3, s5
+; GFX9-NEXT:    s_not_b32 s5, s4
+; GFX9-NEXT:    s_cmp_lt_u32 s5, s9
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s9
+; GFX9-NEXT:    s_add_i32 s4, s4, s5
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_uaddsat_v5i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_not_b32 s10, s0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cmp_lt_u32 s10, s5
+; GFX10-NEXT:    s_cselect_b32 s5, s10, s5
+; GFX10-NEXT:    s_not_b32 s10, s1
+; GFX10-NEXT:    s_add_i32 s0, s0, s5
+; GFX10-NEXT:    s_cmp_lt_u32 s10, s6
+; GFX10-NEXT:    s_cselect_b32 s5, s10, s6
+; GFX10-NEXT:    s_not_b32 s6, s2
+; GFX10-NEXT:    s_add_i32 s1, s1, s5
+; GFX10-NEXT:    s_cmp_lt_u32 s6, s7
+; GFX10-NEXT:    s_cselect_b32 s5, s6, s7
+; GFX10-NEXT:    s_not_b32 s6, s3
+; GFX10-NEXT:    s_add_i32 s2, s2, s5
+; GFX10-NEXT:    s_cmp_lt_u32 s6, s8
+; GFX10-NEXT:    s_cselect_b32 s5, s6, s8
+; GFX10-NEXT:    s_not_b32 s6, s4
+; GFX10-NEXT:    s_add_i32 s3, s3, s5
+; GFX10-NEXT:    s_cmp_lt_u32 s6, s9
+; GFX10-NEXT:    s_cselect_b32 s5, s6, s9
+; GFX10-NEXT:    s_add_i32 s4, s4, s5
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <5 x i32> @llvm.uadd.sat.v5i32(<5 x i32> %lhs, <5 x i32> %rhs)
+  ret <5 x i32> %result
+}
+
+define <16 x i32> @v_uaddsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
+; GFX6-LABEL: v_uaddsat_v16i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_xor_b32_e32 v32, -1, v0
+; GFX6-NEXT:    v_min_u32_e32 v16, v32, v16
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v16
+; GFX6-NEXT:    v_xor_b32_e32 v16, -1, v1
+; GFX6-NEXT:    v_min_u32_e32 v16, v16, v17
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v16
+; GFX6-NEXT:    v_xor_b32_e32 v16, -1, v2
+; GFX6-NEXT:    v_min_u32_e32 v16, v16, v18
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v16
+; GFX6-NEXT:    v_xor_b32_e32 v16, -1, v3
+; GFX6-NEXT:    v_min_u32_e32 v16, v16, v19
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v16
+; GFX6-NEXT:    v_xor_b32_e32 v16, -1, v4
+; GFX6-NEXT:    v_min_u32_e32 v16, v16, v20
+; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v4, v16
+; GFX6-NEXT:    v_xor_b32_e32 v16, -1, v5
+; GFX6-NEXT:    v_min_u32_e32 v16, v16, v21
+; GFX6-NEXT:    v_add_i32_e32 v5, vcc, v5, v16
+; GFX6-NEXT:    v_xor_b32_e32 v16, -1, v6
+; GFX6-NEXT:    v_min_u32_e32 v16, v16, v22
+; GFX6-NEXT:    v_add_i32_e32 v6, vcc, v6, v16
+; GFX6-NEXT:    v_xor_b32_e32 v16, -1, v7
+; GFX6-NEXT:    v_min_u32_e32 v16, v16, v23
+; GFX6-NEXT:    v_add_i32_e32 v7, vcc, v7, v16
+; GFX6-NEXT:    v_xor_b32_e32 v16, -1, v8
+; GFX6-NEXT:    v_min_u32_e32 v16, v16, v24
+; GFX6-NEXT:    v_add_i32_e32 v8, vcc, v8, v16
+; GFX6-NEXT:    v_xor_b32_e32 v16, -1, v9
+; GFX6-NEXT:    v_min_u32_e32 v16, v16, v25
+; GFX6-NEXT:    v_add_i32_e32 v9, vcc, v9, v16
+; GFX6-NEXT:    v_xor_b32_e32 v16, -1, v10
+; GFX6-NEXT:    v_min_u32_e32 v16, v16, v26
+; GFX6-NEXT:    v_add_i32_e32 v10, vcc, v10, v16
+; GFX6-NEXT:    v_xor_b32_e32 v16, -1, v11
+; GFX6-NEXT:    v_min_u32_e32 v16, v16, v27
+; GFX6-NEXT:    v_add_i32_e32 v11, vcc, v11, v16
+; GFX6-NEXT:    v_xor_b32_e32 v16, -1, v12
+; GFX6-NEXT:    v_min_u32_e32 v16, v16, v28
+; GFX6-NEXT:    v_add_i32_e32 v12, vcc, v12, v16
+; GFX6-NEXT:    v_xor_b32_e32 v16, -1, v13
+; GFX6-NEXT:    v_min_u32_e32 v16, v16, v29
+; GFX6-NEXT:    v_add_i32_e32 v13, vcc, v13, v16
+; GFX6-NEXT:    v_xor_b32_e32 v16, -1, v14
+; GFX6-NEXT:    v_min_u32_e32 v16, v16, v30
+; GFX6-NEXT:    v_add_i32_e32 v14, vcc, v14, v16
+; GFX6-NEXT:    v_xor_b32_e32 v16, -1, v15
+; GFX6-NEXT:    v_min_u32_e32 v16, v16, v31
+; GFX6-NEXT:    v_add_i32_e32 v15, vcc, v15, v16
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_v16i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v32, -1, v0
+; GFX8-NEXT:    v_min_u32_e32 v16, v32, v16
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v16
+; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v1
+; GFX8-NEXT:    v_min_u32_e32 v16, v16, v17
+; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v16
+; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v2
+; GFX8-NEXT:    v_min_u32_e32 v16, v16, v18
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v16
+; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v3
+; GFX8-NEXT:    v_min_u32_e32 v16, v16, v19
+; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v16
+; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v4
+; GFX8-NEXT:    v_min_u32_e32 v16, v16, v20
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v4, v16
+; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v5
+; GFX8-NEXT:    v_min_u32_e32 v16, v16, v21
+; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v16
+; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v6
+; GFX8-NEXT:    v_min_u32_e32 v16, v16, v22
+; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v6, v16
+; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v7
+; GFX8-NEXT:    v_min_u32_e32 v16, v16, v23
+; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v16
+; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v8
+; GFX8-NEXT:    v_min_u32_e32 v16, v16, v24
+; GFX8-NEXT:    v_add_u32_e32 v8, vcc, v8, v16
+; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v9
+; GFX8-NEXT:    v_min_u32_e32 v16, v16, v25
+; GFX8-NEXT:    v_add_u32_e32 v9, vcc, v9, v16
+; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v10
+; GFX8-NEXT:    v_min_u32_e32 v16, v16, v26
+; GFX8-NEXT:    v_add_u32_e32 v10, vcc, v10, v16
+; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v11
+; GFX8-NEXT:    v_min_u32_e32 v16, v16, v27
+; GFX8-NEXT:    v_add_u32_e32 v11, vcc, v11, v16
+; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v12
+; GFX8-NEXT:    v_min_u32_e32 v16, v16, v28
+; GFX8-NEXT:    v_add_u32_e32 v12, vcc, v12, v16
+; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v13
+; GFX8-NEXT:    v_min_u32_e32 v16, v16, v29
+; GFX8-NEXT:    v_add_u32_e32 v13, vcc, v13, v16
+; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v14
+; GFX8-NEXT:    v_min_u32_e32 v16, v16, v30
+; GFX8-NEXT:    v_add_u32_e32 v14, vcc, v14, v16
+; GFX8-NEXT:    v_xor_b32_e32 v16, -1, v15
+; GFX8-NEXT:    v_min_u32_e32 v16, v16, v31
+; GFX8-NEXT:    v_add_u32_e32 v15, vcc, v15, v16
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_v16i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v32, -1, v0
+; GFX9-NEXT:    v_min_u32_e32 v16, v32, v16
+; GFX9-NEXT:    v_add_u32_e32 v0, v0, v16
+; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v1
+; GFX9-NEXT:    v_min_u32_e32 v16, v16, v17
+; GFX9-NEXT:    v_add_u32_e32 v1, v1, v16
+; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v2
+; GFX9-NEXT:    v_min_u32_e32 v16, v16, v18
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v16
+; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v3
+; GFX9-NEXT:    v_min_u32_e32 v16, v16, v19
+; GFX9-NEXT:    v_add_u32_e32 v3, v3, v16
+; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v4
+; GFX9-NEXT:    v_min_u32_e32 v16, v16, v20
+; GFX9-NEXT:    v_add_u32_e32 v4, v4, v16
+; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v5
+; GFX9-NEXT:    v_min_u32_e32 v16, v16, v21
+; GFX9-NEXT:    v_add_u32_e32 v5, v5, v16
+; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v6
+; GFX9-NEXT:    v_min_u32_e32 v16, v16, v22
+; GFX9-NEXT:    v_add_u32_e32 v6, v6, v16
+; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v7
+; GFX9-NEXT:    v_min_u32_e32 v16, v16, v23
+; GFX9-NEXT:    v_add_u32_e32 v7, v7, v16
+; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v8
+; GFX9-NEXT:    v_min_u32_e32 v16, v16, v24
+; GFX9-NEXT:    v_add_u32_e32 v8, v8, v16
+; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v9
+; GFX9-NEXT:    v_min_u32_e32 v16, v16, v25
+; GFX9-NEXT:    v_add_u32_e32 v9, v9, v16
+; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v10
+; GFX9-NEXT:    v_min_u32_e32 v16, v16, v26
+; GFX9-NEXT:    v_add_u32_e32 v10, v10, v16
+; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v11
+; GFX9-NEXT:    v_min_u32_e32 v16, v16, v27
+; GFX9-NEXT:    v_add_u32_e32 v11, v11, v16
+; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v12
+; GFX9-NEXT:    v_min_u32_e32 v16, v16, v28
+; GFX9-NEXT:    v_add_u32_e32 v12, v12, v16
+; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v13
+; GFX9-NEXT:    v_min_u32_e32 v16, v16, v29
+; GFX9-NEXT:    v_add_u32_e32 v13, v13, v16
+; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v14
+; GFX9-NEXT:    v_min_u32_e32 v16, v16, v30
+; GFX9-NEXT:    v_add_u32_e32 v14, v14, v16
+; GFX9-NEXT:    v_xor_b32_e32 v16, -1, v15
+; GFX9-NEXT:    v_min_u32_e32 v16, v16, v31
+; GFX9-NEXT:    v_add_u32_e32 v15, v15, v16
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_uaddsat_v16i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_xor_b32_e32 v35, -1, v0
+; GFX10-NEXT:    v_xor_b32_e32 v32, -1, v2
+; GFX10-NEXT:    v_xor_b32_e32 v33, -1, v3
+; GFX10-NEXT:    v_xor_b32_e32 v34, -1, v4
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_min_u32_e32 v35, v35, v16
+; GFX10-NEXT:    v_xor_b32_e32 v16, -1, v1
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v35
+; GFX10-NEXT:    v_xor_b32_e32 v35, -1, v5
+; GFX10-NEXT:    v_min_u32_e32 v16, v16, v17
+; GFX10-NEXT:    v_min_u32_e32 v17, v32, v18
+; GFX10-NEXT:    v_min_u32_e32 v18, v33, v19
+; GFX10-NEXT:    v_min_u32_e32 v19, v34, v20
+; GFX10-NEXT:    v_min_u32_e32 v20, v35, v21
+; GFX10-NEXT:    v_add_nc_u32_e32 v1, v1, v16
+; GFX10-NEXT:    v_xor_b32_e32 v16, -1, v6
+; GFX10-NEXT:    v_add_nc_u32_e32 v2, v2, v17
+; GFX10-NEXT:    v_xor_b32_e32 v17, -1, v7
+; GFX10-NEXT:    v_add_nc_u32_e32 v3, v3, v18
+; GFX10-NEXT:    v_xor_b32_e32 v18, -1, v8
+; GFX10-NEXT:    v_add_nc_u32_e32 v4, v4, v19
+; GFX10-NEXT:    v_xor_b32_e32 v19, -1, v9
+; GFX10-NEXT:    v_add_nc_u32_e32 v5, v5, v20
+; GFX10-NEXT:    v_xor_b32_e32 v20, -1, v10
+; GFX10-NEXT:    v_min_u32_e32 v16, v16, v22
+; GFX10-NEXT:    v_min_u32_e32 v17, v17, v23
+; GFX10-NEXT:    v_min_u32_e32 v18, v18, v24
+; GFX10-NEXT:    v_min_u32_e32 v19, v19, v25
+; GFX10-NEXT:    v_min_u32_e32 v20, v20, v26
+; GFX10-NEXT:    v_add_nc_u32_e32 v6, v6, v16
+; GFX10-NEXT:    v_xor_b32_e32 v16, -1, v11
+; GFX10-NEXT:    v_add_nc_u32_e32 v7, v7, v17
+; GFX10-NEXT:    v_xor_b32_e32 v17, -1, v12
+; GFX10-NEXT:    v_add_nc_u32_e32 v8, v8, v18
+; GFX10-NEXT:    v_xor_b32_e32 v18, -1, v13
+; GFX10-NEXT:    v_add_nc_u32_e32 v9, v9, v19
+; GFX10-NEXT:    v_xor_b32_e32 v19, -1, v14
+; GFX10-NEXT:    v_add_nc_u32_e32 v10, v10, v20
+; GFX10-NEXT:    v_xor_b32_e32 v20, -1, v15
+; GFX10-NEXT:    v_min_u32_e32 v16, v16, v27
+; GFX10-NEXT:    v_min_u32_e32 v17, v17, v28
+; GFX10-NEXT:    v_min_u32_e32 v18, v18, v29
+; GFX10-NEXT:    v_min_u32_e32 v19, v19, v30
+; GFX10-NEXT:    v_min_u32_e32 v20, v20, v31
+; GFX10-NEXT:    v_add_nc_u32_e32 v11, v11, v16
+; GFX10-NEXT:    v_add_nc_u32_e32 v12, v12, v17
+; GFX10-NEXT:    v_add_nc_u32_e32 v13, v13, v18
+; GFX10-NEXT:    v_add_nc_u32_e32 v14, v14, v19
+; GFX10-NEXT:    v_add_nc_u32_e32 v15, v15, v20
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
+  ret <16 x i32> %result
+}
+
+define amdgpu_ps <16 x i32> @s_uaddsat_v16i32(<16 x i32> inreg %lhs, <16 x i32> inreg %rhs) {
+; GFX6-LABEL: s_uaddsat_v16i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_not_b32 s32, s0
+; GFX6-NEXT:    s_cmp_lt_u32 s32, s16
+; GFX6-NEXT:    s_cselect_b32 s16, s32, s16
+; GFX6-NEXT:    s_add_i32 s0, s0, s16
+; GFX6-NEXT:    s_not_b32 s16, s1
+; GFX6-NEXT:    s_cmp_lt_u32 s16, s17
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX6-NEXT:    s_add_i32 s1, s1, s16
+; GFX6-NEXT:    s_not_b32 s16, s2
+; GFX6-NEXT:    s_cmp_lt_u32 s16, s18
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s18
+; GFX6-NEXT:    s_add_i32 s2, s2, s16
+; GFX6-NEXT:    s_not_b32 s16, s3
+; GFX6-NEXT:    s_cmp_lt_u32 s16, s19
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s19
+; GFX6-NEXT:    s_add_i32 s3, s3, s16
+; GFX6-NEXT:    s_not_b32 s16, s4
+; GFX6-NEXT:    s_cmp_lt_u32 s16, s20
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s20
+; GFX6-NEXT:    s_add_i32 s4, s4, s16
+; GFX6-NEXT:    s_not_b32 s16, s5
+; GFX6-NEXT:    s_cmp_lt_u32 s16, s21
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s21
+; GFX6-NEXT:    s_add_i32 s5, s5, s16
+; GFX6-NEXT:    s_not_b32 s16, s6
+; GFX6-NEXT:    s_cmp_lt_u32 s16, s22
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s22
+; GFX6-NEXT:    s_add_i32 s6, s6, s16
+; GFX6-NEXT:    s_not_b32 s16, s7
+; GFX6-NEXT:    s_cmp_lt_u32 s16, s23
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s23
+; GFX6-NEXT:    s_add_i32 s7, s7, s16
+; GFX6-NEXT:    s_not_b32 s16, s8
+; GFX6-NEXT:    s_cmp_lt_u32 s16, s24
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s24
+; GFX6-NEXT:    s_add_i32 s8, s8, s16
+; GFX6-NEXT:    s_not_b32 s16, s9
+; GFX6-NEXT:    s_cmp_lt_u32 s16, s25
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s25
+; GFX6-NEXT:    s_add_i32 s9, s9, s16
+; GFX6-NEXT:    s_not_b32 s16, s10
+; GFX6-NEXT:    s_cmp_lt_u32 s16, s26
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s26
+; GFX6-NEXT:    s_add_i32 s10, s10, s16
+; GFX6-NEXT:    s_not_b32 s16, s11
+; GFX6-NEXT:    s_cmp_lt_u32 s16, s27
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s27
+; GFX6-NEXT:    s_add_i32 s11, s11, s16
+; GFX6-NEXT:    s_not_b32 s16, s12
+; GFX6-NEXT:    s_cmp_lt_u32 s16, s28
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s28
+; GFX6-NEXT:    s_add_i32 s12, s12, s16
+; GFX6-NEXT:    s_not_b32 s16, s13
+; GFX6-NEXT:    s_cmp_lt_u32 s16, s29
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s29
+; GFX6-NEXT:    s_add_i32 s13, s13, s16
+; GFX6-NEXT:    s_not_b32 s16, s14
+; GFX6-NEXT:    s_cmp_lt_u32 s16, s30
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s30
+; GFX6-NEXT:    s_add_i32 s14, s14, s16
+; GFX6-NEXT:    s_not_b32 s16, s15
+; GFX6-NEXT:    s_cmp_lt_u32 s16, s31
+; GFX6-NEXT:    s_cselect_b32 s16, s16, s31
+; GFX6-NEXT:    s_add_i32 s15, s15, s16
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_uaddsat_v16i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_not_b32 s32, s0
+; GFX8-NEXT:    s_cmp_lt_u32 s32, s16
+; GFX8-NEXT:    s_cselect_b32 s16, s32, s16
+; GFX8-NEXT:    s_add_i32 s0, s0, s16
+; GFX8-NEXT:    s_not_b32 s16, s1
+; GFX8-NEXT:    s_cmp_lt_u32 s16, s17
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX8-NEXT:    s_add_i32 s1, s1, s16
+; GFX8-NEXT:    s_not_b32 s16, s2
+; GFX8-NEXT:    s_cmp_lt_u32 s16, s18
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s18
+; GFX8-NEXT:    s_add_i32 s2, s2, s16
+; GFX8-NEXT:    s_not_b32 s16, s3
+; GFX8-NEXT:    s_cmp_lt_u32 s16, s19
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s19
+; GFX8-NEXT:    s_add_i32 s3, s3, s16
+; GFX8-NEXT:    s_not_b32 s16, s4
+; GFX8-NEXT:    s_cmp_lt_u32 s16, s20
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s20
+; GFX8-NEXT:    s_add_i32 s4, s4, s16
+; GFX8-NEXT:    s_not_b32 s16, s5
+; GFX8-NEXT:    s_cmp_lt_u32 s16, s21
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s21
+; GFX8-NEXT:    s_add_i32 s5, s5, s16
+; GFX8-NEXT:    s_not_b32 s16, s6
+; GFX8-NEXT:    s_cmp_lt_u32 s16, s22
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s22
+; GFX8-NEXT:    s_add_i32 s6, s6, s16
+; GFX8-NEXT:    s_not_b32 s16, s7
+; GFX8-NEXT:    s_cmp_lt_u32 s16, s23
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s23
+; GFX8-NEXT:    s_add_i32 s7, s7, s16
+; GFX8-NEXT:    s_not_b32 s16, s8
+; GFX8-NEXT:    s_cmp_lt_u32 s16, s24
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s24
+; GFX8-NEXT:    s_add_i32 s8, s8, s16
+; GFX8-NEXT:    s_not_b32 s16, s9
+; GFX8-NEXT:    s_cmp_lt_u32 s16, s25
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s25
+; GFX8-NEXT:    s_add_i32 s9, s9, s16
+; GFX8-NEXT:    s_not_b32 s16, s10
+; GFX8-NEXT:    s_cmp_lt_u32 s16, s26
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s26
+; GFX8-NEXT:    s_add_i32 s10, s10, s16
+; GFX8-NEXT:    s_not_b32 s16, s11
+; GFX8-NEXT:    s_cmp_lt_u32 s16, s27
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s27
+; GFX8-NEXT:    s_add_i32 s11, s11, s16
+; GFX8-NEXT:    s_not_b32 s16, s12
+; GFX8-NEXT:    s_cmp_lt_u32 s16, s28
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s28
+; GFX8-NEXT:    s_add_i32 s12, s12, s16
+; GFX8-NEXT:    s_not_b32 s16, s13
+; GFX8-NEXT:    s_cmp_lt_u32 s16, s29
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s29
+; GFX8-NEXT:    s_add_i32 s13, s13, s16
+; GFX8-NEXT:    s_not_b32 s16, s14
+; GFX8-NEXT:    s_cmp_lt_u32 s16, s30
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s30
+; GFX8-NEXT:    s_add_i32 s14, s14, s16
+; GFX8-NEXT:    s_not_b32 s16, s15
+; GFX8-NEXT:    s_cmp_lt_u32 s16, s31
+; GFX8-NEXT:    s_cselect_b32 s16, s16, s31
+; GFX8-NEXT:    s_add_i32 s15, s15, s16
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_uaddsat_v16i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_not_b32 s32, s0
+; GFX9-NEXT:    s_cmp_lt_u32 s32, s16
+; GFX9-NEXT:    s_cselect_b32 s16, s32, s16
+; GFX9-NEXT:    s_add_i32 s0, s0, s16
+; GFX9-NEXT:    s_not_b32 s16, s1
+; GFX9-NEXT:    s_cmp_lt_u32 s16, s17
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s17
+; GFX9-NEXT:    s_add_i32 s1, s1, s16
+; GFX9-NEXT:    s_not_b32 s16, s2
+; GFX9-NEXT:    s_cmp_lt_u32 s16, s18
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s18
+; GFX9-NEXT:    s_add_i32 s2, s2, s16
+; GFX9-NEXT:    s_not_b32 s16, s3
+; GFX9-NEXT:    s_cmp_lt_u32 s16, s19
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s19
+; GFX9-NEXT:    s_add_i32 s3, s3, s16
+; GFX9-NEXT:    s_not_b32 s16, s4
+; GFX9-NEXT:    s_cmp_lt_u32 s16, s20
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s20
+; GFX9-NEXT:    s_add_i32 s4, s4, s16
+; GFX9-NEXT:    s_not_b32 s16, s5
+; GFX9-NEXT:    s_cmp_lt_u32 s16, s21
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s21
+; GFX9-NEXT:    s_add_i32 s5, s5, s16
+; GFX9-NEXT:    s_not_b32 s16, s6
+; GFX9-NEXT:    s_cmp_lt_u32 s16, s22
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s22
+; GFX9-NEXT:    s_add_i32 s6, s6, s16
+; GFX9-NEXT:    s_not_b32 s16, s7
+; GFX9-NEXT:    s_cmp_lt_u32 s16, s23
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s23
+; GFX9-NEXT:    s_add_i32 s7, s7, s16
+; GFX9-NEXT:    s_not_b32 s16, s8
+; GFX9-NEXT:    s_cmp_lt_u32 s16, s24
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s24
+; GFX9-NEXT:    s_add_i32 s8, s8, s16
+; GFX9-NEXT:    s_not_b32 s16, s9
+; GFX9-NEXT:    s_cmp_lt_u32 s16, s25
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s25
+; GFX9-NEXT:    s_add_i32 s9, s9, s16
+; GFX9-NEXT:    s_not_b32 s16, s10
+; GFX9-NEXT:    s_cmp_lt_u32 s16, s26
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s26
+; GFX9-NEXT:    s_add_i32 s10, s10, s16
+; GFX9-NEXT:    s_not_b32 s16, s11
+; GFX9-NEXT:    s_cmp_lt_u32 s16, s27
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s27
+; GFX9-NEXT:    s_add_i32 s11, s11, s16
+; GFX9-NEXT:    s_not_b32 s16, s12
+; GFX9-NEXT:    s_cmp_lt_u32 s16, s28
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s28
+; GFX9-NEXT:    s_add_i32 s12, s12, s16
+; GFX9-NEXT:    s_not_b32 s16, s13
+; GFX9-NEXT:    s_cmp_lt_u32 s16, s29
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s29
+; GFX9-NEXT:    s_add_i32 s13, s13, s16
+; GFX9-NEXT:    s_not_b32 s16, s14
+; GFX9-NEXT:    s_cmp_lt_u32 s16, s30
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s30
+; GFX9-NEXT:    s_add_i32 s14, s14, s16
+; GFX9-NEXT:    s_not_b32 s16, s15
+; GFX9-NEXT:    s_cmp_lt_u32 s16, s31
+; GFX9-NEXT:    s_cselect_b32 s16, s16, s31
+; GFX9-NEXT:    s_add_i32 s15, s15, s16
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_uaddsat_v16i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_not_b32 s46, s0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cmp_lt_u32 s46, s16
+; GFX10-NEXT:    s_cselect_b32 s46, s46, s16
+; GFX10-NEXT:    s_not_b32 s47, s1
+; GFX10-NEXT:    s_add_i32 s0, s0, s46
+; GFX10-NEXT:    s_cmp_lt_u32 s47, s17
+; GFX10-NEXT:    s_cselect_b32 s46, s47, s17
+; GFX10-NEXT:    s_not_b32 s17, s2
+; GFX10-NEXT:    s_add_i32 s1, s1, s46
+; GFX10-NEXT:    s_cmp_lt_u32 s17, s18
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s18
+; GFX10-NEXT:    s_not_b32 s17, s3
+; GFX10-NEXT:    s_add_i32 s2, s2, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s17, s19
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s19
+; GFX10-NEXT:    s_not_b32 s17, s4
+; GFX10-NEXT:    s_add_i32 s3, s3, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s17, s20
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s20
+; GFX10-NEXT:    s_not_b32 s17, s5
+; GFX10-NEXT:    s_add_i32 s4, s4, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s17, s21
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s21
+; GFX10-NEXT:    s_not_b32 s17, s6
+; GFX10-NEXT:    s_add_i32 s5, s5, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s17, s22
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s22
+; GFX10-NEXT:    s_not_b32 s17, s7
+; GFX10-NEXT:    s_add_i32 s6, s6, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s17, s23
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s23
+; GFX10-NEXT:    s_not_b32 s17, s8
+; GFX10-NEXT:    s_add_i32 s7, s7, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s17, s24
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s24
+; GFX10-NEXT:    s_not_b32 s17, s9
+; GFX10-NEXT:    s_add_i32 s8, s8, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s17, s25
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s25
+; GFX10-NEXT:    s_not_b32 s17, s10
+; GFX10-NEXT:    s_add_i32 s9, s9, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s17, s26
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s26
+; GFX10-NEXT:    s_not_b32 s17, s11
+; GFX10-NEXT:    s_add_i32 s10, s10, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s17, s27
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s27
+; GFX10-NEXT:    s_not_b32 s17, s12
+; GFX10-NEXT:    s_add_i32 s11, s11, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s17, s28
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s28
+; GFX10-NEXT:    s_not_b32 s17, s13
+; GFX10-NEXT:    s_add_i32 s12, s12, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s17, s29
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s29
+; GFX10-NEXT:    s_not_b32 s17, s14
+; GFX10-NEXT:    s_add_i32 s13, s13, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s17, s30
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s30
+; GFX10-NEXT:    s_not_b32 s17, s15
+; GFX10-NEXT:    s_add_i32 s14, s14, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s17, s31
+; GFX10-NEXT:    s_cselect_b32 s16, s17, s31
+; GFX10-NEXT:    s_add_i32 s15, s15, s16
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
+  ret <16 x i32> %result
+}
+
+define i16 @v_uaddsat_i16(i16 %lhs, i16 %rhs) {
+; GFX6-LABEL: v_uaddsat_i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_xor_b32_e32 v2, -1, v0
+; GFX6-NEXT:    v_min_u32_e32 v1, v2, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v2, 0xffff, v0
+; GFX8-NEXT:    v_min_u16_e32 v1, v2, v1
+; GFX8-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v2, 0xffff, v0
+; GFX9-NEXT:    v_min_u16_e32 v1, v2, v1
+; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_uaddsat_i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_xor_b32_e32 v2, 0xffff, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_min_u16_e64 v1, v2, v1
+; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i16 @llvm.uadd.sat.i16(i16 %lhs, i16 %rhs)
+  ret i16 %result
+}
+
+define amdgpu_ps i16 @s_uaddsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
+; GFX6-LABEL: s_uaddsat_i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_not_b32 s2, s0
+; GFX6-NEXT:    s_cmp_lt_u32 s2, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX6-NEXT:    s_add_i32 s0, s0, s1
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_uaddsat_i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_xor_b32 s2, s0, 0xffff
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s2, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX8-NEXT:    s_add_i32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_uaddsat_i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_xor_b32 s2, s0, 0xffff
+; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_cmp_lt_u32 s2, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX9-NEXT:    s_add_i32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_uaddsat_i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_xor_b32 s2, s0, 0xffff
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cmp_lt_u32 s2, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX10-NEXT:    s_add_i32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i16 @llvm.uadd.sat.i16(i16 %lhs, i16 %rhs)
+  ret i16 %result
+}
+
+define amdgpu_ps half @uaddsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
+; GFX6-LABEL: uaddsat_i16_sv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    s_not_b32 s1, s0
+; GFX6-NEXT:    v_min_u32_e32 v0, s1, v0
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: uaddsat_i16_sv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_xor_b32 s1, s0, 0xffff
+; GFX8-NEXT:    v_min_u16_e32 v0, s1, v0
+; GFX8-NEXT:    v_add_u16_e32 v0, s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: uaddsat_i16_sv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_xor_b32 s1, s0, 0xffff
+; GFX9-NEXT:    v_min_u16_e32 v0, s1, v0
+; GFX9-NEXT:    v_add_u16_e32 v0, s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: uaddsat_i16_sv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_xor_b32 s1, s0, 0xffff
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_min_u16_e64 v0, s1, v0
+; GFX10-NEXT:    v_add_nc_u16_e64 v0, s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i16 @llvm.uadd.sat.i16(i16 %lhs, i16 %rhs)
+  %cast = bitcast i16 %result to half
+  ret half %cast
+}
+
+define amdgpu_ps half @uaddsat_i16_vs(i16 %lhs, i16 inreg %rhs) {
+; GFX6-LABEL: uaddsat_i16_vs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    v_xor_b32_e32 v1, -1, v0
+; GFX6-NEXT:    v_min_u32_e32 v1, s0, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: uaddsat_i16_vs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_xor_b32_e32 v1, 0xffff, v0
+; GFX8-NEXT:    v_min_u16_e32 v1, s0, v1
+; GFX8-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: uaddsat_i16_vs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_xor_b32_e32 v1, 0xffff, v0
+; GFX9-NEXT:    v_min_u16_e32 v1, s0, v1
+; GFX9-NEXT:    v_add_u16_e32 v0, v0, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: uaddsat_i16_vs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_xor_b32_e32 v1, 0xffff, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_min_u16_e64 v1, v1, s0
+; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i16 @llvm.uadd.sat.i16(i16 %lhs, i16 %rhs)
+  %cast = bitcast i16 %result to half
+  ret half %cast
+}
+
+define <2 x i16> @v_uaddsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
+; GFX6-LABEL: v_uaddsat_v2i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_xor_b32_e32 v4, -1, v0
+; GFX6-NEXT:    v_min_u32_e32 v2, v4, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
+; GFX6-NEXT:    v_xor_b32_e32 v3, -1, v1
+; GFX6-NEXT:    v_min_u32_e32 v2, v3, v2
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_v2i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s4, 0xffff
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX8-NEXT:    v_xor_b32_e32 v3, s4, v0
+; GFX8-NEXT:    v_xor_b32_e32 v4, s4, v2
+; GFX8-NEXT:    v_min_u16_e32 v3, v3, v1
+; GFX8-NEXT:    v_min_u16_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_add_u16_e32 v0, v0, v3
+; GFX8-NEXT:    v_add_u16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
+; GFX9-NEXT:    v_xor_b32_e32 v2, s4, v0
+; GFX9-NEXT:    v_pk_min_u16 v1, v2, v1
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_uaddsat_v2i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_xor_b32_e32 v2, s4, v0
+; GFX10-NEXT:    v_pk_min_u16 v1, v2, v1
+; GFX10-NEXT:    v_pk_add_u16 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
+  ret <2 x i16> %result
+}
+
+define amdgpu_ps i32 @s_uaddsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs) {
+; GFX6-LABEL: s_uaddsat_v2i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_not_b32 s4, s0
+; GFX6-NEXT:    s_cmp_lt_u32 s4, s2
+; GFX6-NEXT:    s_cselect_b32 s2, s4, s2
+; GFX6-NEXT:    s_add_i32 s0, s0, s2
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s2, s3, 16
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX6-NEXT:    s_not_b32 s3, s1
+; GFX6-NEXT:    s_cmp_lt_u32 s3, s2
+; GFX6-NEXT:    s_cselect_b32 s2, s3, s2
+; GFX6-NEXT:    s_add_i32 s1, s1, s2
+; GFX6-NEXT:    s_mov_b32 s2, 0xffff
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 16
+; GFX6-NEXT:    s_and_b32 s1, s1, s2
+; GFX6-NEXT:    s_and_b32 s0, s0, s2
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_uaddsat_v2i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s4, 0xffff
+; GFX8-NEXT:    s_xor_b32 s5, s0, s4
+; GFX8-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX8-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX8-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s5, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s5, s1
+; GFX8-NEXT:    s_add_i32 s0, s0, s1
+; GFX8-NEXT:    s_xor_b32 s1, s2, s4
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s1, s3
+; GFX8-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX8-NEXT:    s_add_i32 s2, s2, s1
+; GFX8-NEXT:    s_bfe_u32 s1, s2, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_uaddsat_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, -1, -1
+; GFX9-NEXT:    s_xor_b32 s2, s0, s2
+; GFX9-NEXT:    s_mov_b32 s4, 0xffff
+; GFX9-NEXT:    s_lshr_b32 s3, s2, 16
+; GFX9-NEXT:    s_lshr_b32 s5, s1, 16
+; GFX9-NEXT:    s_and_b32 s2, s2, s4
+; GFX9-NEXT:    s_and_b32 s1, s1, s4
+; GFX9-NEXT:    s_cmp_lt_u32 s2, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX9-NEXT:    s_cmp_lt_u32 s3, s5
+; GFX9-NEXT:    s_cselect_b32 s2, s3, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX9-NEXT:    s_add_i32 s0, s0, s1
+; GFX9-NEXT:    s_add_i32 s2, s2, s3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_uaddsat_v2i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, -1, -1
+; GFX10-NEXT:    s_mov_b32 s3, 0xffff
+; GFX10-NEXT:    s_xor_b32 s2, s0, s2
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX10-NEXT:    s_and_b32 s2, s2, s3
+; GFX10-NEXT:    s_and_b32 s3, s1, s3
+; GFX10-NEXT:    s_lshr_b32 s1, s1, 16
+; GFX10-NEXT:    s_cmp_lt_u32 s2, s3
+; GFX10-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX10-NEXT:    s_cmp_lt_u32 s4, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s4, s1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s2, s1
+; GFX10-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX10-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX10-NEXT:    s_add_i32 s0, s0, s1
+; GFX10-NEXT:    s_add_i32 s2, s2, s3
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
+  %cast = bitcast <2 x i16> %result to i32
+  ret i32 %cast
+}
+
+define amdgpu_ps float @uaddsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
+; GFX6-LABEL: uaddsat_v2i16_sv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    s_not_b32 s2, s0
+; GFX6-NEXT:    v_min_u32_e32 v0, s2, v0
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; GFX6-NEXT:    s_lshl_b32 s0, s1, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    s_not_b32 s1, s0
+; GFX6-NEXT:    v_min_u32_e32 v1, s1, v1
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, s0, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    s_mov_b32 s0, 0xffff
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, s0, v1
+; GFX6-NEXT:    v_and_b32_e32 v0, s0, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: uaddsat_v2i16_sv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s2, 0xffff
+; GFX8-NEXT:    s_lshr_b32 s1, s0, 16
+; GFX8-NEXT:    s_xor_b32 s3, s0, s2
+; GFX8-NEXT:    s_xor_b32 s2, s1, s2
+; GFX8-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-NEXT:    v_min_u16_e32 v1, s3, v0
+; GFX8-NEXT:    v_min_u16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    v_add_u16_e32 v1, s0, v1
+; GFX8-NEXT:    v_add_u16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: uaddsat_v2i16_sv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, -1, -1
+; GFX9-NEXT:    s_xor_b32 s1, s0, s1
+; GFX9-NEXT:    v_pk_min_u16 v0, s1, v0
+; GFX9-NEXT:    v_pk_add_u16 v0, s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: uaddsat_v2i16_sv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, -1, -1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_xor_b32 s1, s0, s1
+; GFX10-NEXT:    v_pk_min_u16 v0, s1, v0
+; GFX10-NEXT:    v_pk_add_u16 v0, s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
+  %cast = bitcast <2 x i16> %result to float
+  ret float %cast
+}
+
+define amdgpu_ps float @uaddsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
+; GFX6-LABEL: uaddsat_v2i16_vs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    v_xor_b32_e32 v2, -1, v0
+; GFX6-NEXT:    v_min_u32_e32 v2, s0, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT:    s_lshl_b32 s0, s1, 16
+; GFX6-NEXT:    v_xor_b32_e32 v2, -1, v1
+; GFX6-NEXT:    v_min_u32_e32 v2, s0, v2
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    s_mov_b32 s0, 0xffff
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, s0, v1
+; GFX6-NEXT:    v_and_b32_e32 v0, s0, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: uaddsat_v2i16_vs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s2, 0xffff
+; GFX8-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; GFX8-NEXT:    v_xor_b32_e32 v2, s2, v0
+; GFX8-NEXT:    s_lshr_b32 s1, s0, 16
+; GFX8-NEXT:    v_xor_b32_e32 v3, s2, v1
+; GFX8-NEXT:    v_min_u16_e32 v2, s0, v2
+; GFX8-NEXT:    v_min_u16_e32 v3, s1, v3
+; GFX8-NEXT:    v_add_u16_e32 v0, v0, v2
+; GFX8-NEXT:    v_add_u16_sdwa v1, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: uaddsat_v2i16_vs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, -1, -1
+; GFX9-NEXT:    v_xor_b32_e32 v1, s1, v0
+; GFX9-NEXT:    v_pk_min_u16 v1, v1, s0
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: uaddsat_v2i16_vs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, -1, -1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_xor_b32_e32 v1, s1, v0
+; GFX10-NEXT:    v_pk_min_u16 v1, v1, s0
+; GFX10-NEXT:    v_pk_add_u16 v0, v0, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
+  %cast = bitcast <2 x i16> %result to float
+  ret float %cast
+}
+
+; FIXME: v3i16 insert/extract
+; define <3 x i16> @v_uaddsat_v3i16(<3 x i16> %lhs, <3 x i16> %rhs) {
+;   %result = call <3 x i16> @llvm.uadd.sat.v3i16(<3 x i16> %lhs, <3 x i16> %rhs)
+;   ret <3 x i16> %result
+; }
+
+; define amdgpu_ps <3 x i16> @s_uaddsat_v3i16(<3 x i16> inreg %lhs, <3 x i16> inreg %rhs) {
+;   %result = call <3 x i16> @llvm.uadd.sat.v3i16(<3 x i16> %lhs, <3 x i16> %rhs)
+;   ret <3 x i16> %result
+; }
+
+define <2 x float> @v_uaddsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
+; GFX6-LABEL: v_uaddsat_v4i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT:    v_xor_b32_e32 v8, -1, v0
+; GFX6-NEXT:    v_min_u32_e32 v4, v8, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
+; GFX6-NEXT:    v_xor_b32_e32 v5, -1, v1
+; GFX6-NEXT:    v_min_u32_e32 v4, v5, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v6
+; GFX6-NEXT:    v_xor_b32_e32 v5, -1, v2
+; GFX6-NEXT:    v_min_u32_e32 v4, v5, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v7
+; GFX6-NEXT:    v_xor_b32_e32 v5, -1, v3
+; GFX6-NEXT:    v_min_u32_e32 v4, v5, v4
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_v4i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s4, 0xffff
+; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
+; GFX8-NEXT:    v_xor_b32_e32 v6, s4, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
+; GFX8-NEXT:    v_xor_b32_e32 v7, s4, v4
+; GFX8-NEXT:    v_min_u16_e32 v6, v6, v2
+; GFX8-NEXT:    v_min_u16_sdwa v2, v7, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_xor_b32_e32 v7, s4, v1
+; GFX8-NEXT:    v_xor_b32_e32 v8, s4, v5
+; GFX8-NEXT:    v_min_u16_e32 v7, v7, v3
+; GFX8-NEXT:    v_min_u16_sdwa v3, v8, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_add_u16_e32 v0, v0, v6
+; GFX8-NEXT:    v_add_u16_sdwa v2, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT:    v_add_u16_e32 v1, v1, v7
+; GFX8-NEXT:    v_add_u16_sdwa v2, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_v4i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
+; GFX9-NEXT:    v_xor_b32_e32 v4, s4, v0
+; GFX9-NEXT:    v_pk_min_u16 v2, v4, v2
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, v2
+; GFX9-NEXT:    v_xor_b32_e32 v2, s4, v1
+; GFX9-NEXT:    v_pk_min_u16 v2, v2, v3
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_uaddsat_v4i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_xor_b32_e32 v4, s4, v0
+; GFX10-NEXT:    v_xor_b32_e32 v5, s4, v1
+; GFX10-NEXT:    v_pk_min_u16 v2, v4, v2
+; GFX10-NEXT:    v_pk_min_u16 v3, v5, v3
+; GFX10-NEXT:    v_pk_add_u16 v0, v0, v2
+; GFX10-NEXT:    v_pk_add_u16 v1, v1, v3
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> %lhs, <4 x i16> %rhs)
+  %cast = bitcast <4 x i16> %result to <2 x float>
+  ret <2 x float> %cast
+}
+
+define amdgpu_ps <2 x i32> @s_uaddsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inreg %rhs) {
+; GFX6-LABEL: s_uaddsat_v4i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX6-NEXT:    s_not_b32 s8, s0
+; GFX6-NEXT:    s_cmp_lt_u32 s8, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s8, s4
+; GFX6-NEXT:    s_add_i32 s0, s0, s4
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s4, s5, 16
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX6-NEXT:    s_not_b32 s5, s1
+; GFX6-NEXT:    s_cmp_lt_u32 s5, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX6-NEXT:    s_add_i32 s1, s1, s4
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s4, s6, 16
+; GFX6-NEXT:    s_not_b32 s5, s2
+; GFX6-NEXT:    s_cmp_lt_u32 s5, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX6-NEXT:    s_add_i32 s2, s2, s4
+; GFX6-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX6-NEXT:    s_lshr_b32 s2, s2, 16
+; GFX6-NEXT:    s_lshl_b32 s4, s7, 16
+; GFX6-NEXT:    s_not_b32 s5, s3
+; GFX6-NEXT:    s_cmp_lt_u32 s5, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX6-NEXT:    s_add_i32 s3, s3, s4
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    s_and_b32 s1, s1, s4
+; GFX6-NEXT:    s_lshr_b32 s3, s3, 16
+; GFX6-NEXT:    s_and_b32 s0, s0, s4
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_and_b32 s1, s2, s4
+; GFX6-NEXT:    s_and_b32 s2, s3, s4
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_or_b32 s1, s1, s2
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_uaddsat_v4i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s8, 0xffff
+; GFX8-NEXT:    s_xor_b32 s9, s0, s8
+; GFX8-NEXT:    s_lshr_b32 s6, s2, 16
+; GFX8-NEXT:    s_lshr_b32 s4, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s5, s1, 16
+; GFX8-NEXT:    s_lshr_b32 s7, s3, 16
+; GFX8-NEXT:    s_bfe_u32 s9, s9, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s9, s2
+; GFX8-NEXT:    s_cselect_b32 s2, s9, s2
+; GFX8-NEXT:    s_add_i32 s0, s0, s2
+; GFX8-NEXT:    s_xor_b32 s2, s4, s8
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s2, s6
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s6
+; GFX8-NEXT:    s_add_i32 s4, s4, s2
+; GFX8-NEXT:    s_xor_b32 s2, s1, s8
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s2, s3
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX8-NEXT:    s_add_i32 s1, s1, s2
+; GFX8-NEXT:    s_xor_b32 s2, s5, s8
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s3, s7, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s2, s3
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX8-NEXT:    s_add_i32 s5, s5, s2
+; GFX8-NEXT:    s_bfe_u32 s2, s4, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX8-NEXT:    s_or_b32 s0, s0, s2
+; GFX8-NEXT:    s_bfe_u32 s2, s5, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX8-NEXT:    s_or_b32 s1, s1, s2
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_uaddsat_v4i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
+; GFX9-NEXT:    s_xor_b32 s5, s0, s4
+; GFX9-NEXT:    s_mov_b32 s7, 0xffff
+; GFX9-NEXT:    s_lshr_b32 s6, s5, 16
+; GFX9-NEXT:    s_lshr_b32 s8, s2, 16
+; GFX9-NEXT:    s_and_b32 s5, s5, s7
+; GFX9-NEXT:    s_and_b32 s2, s2, s7
+; GFX9-NEXT:    s_cmp_lt_u32 s5, s2
+; GFX9-NEXT:    s_cselect_b32 s2, s5, s2
+; GFX9-NEXT:    s_cmp_lt_u32 s6, s8
+; GFX9-NEXT:    s_cselect_b32 s5, s6, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s5
+; GFX9-NEXT:    s_lshr_b32 s5, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s6, s2, 16
+; GFX9-NEXT:    s_add_i32 s0, s0, s2
+; GFX9-NEXT:    s_xor_b32 s2, s1, s4
+; GFX9-NEXT:    s_add_i32 s5, s5, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s5
+; GFX9-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX9-NEXT:    s_lshr_b32 s5, s3, 16
+; GFX9-NEXT:    s_and_b32 s2, s2, s7
+; GFX9-NEXT:    s_and_b32 s3, s3, s7
+; GFX9-NEXT:    s_cmp_lt_u32 s2, s3
+; GFX9-NEXT:    s_cselect_b32 s2, s2, s3
+; GFX9-NEXT:    s_cmp_lt_u32 s4, s5
+; GFX9-NEXT:    s_cselect_b32 s3, s4, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
+; GFX9-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX9-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX9-NEXT:    s_add_i32 s1, s1, s2
+; GFX9-NEXT:    s_add_i32 s3, s3, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_uaddsat_v4i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
+; GFX10-NEXT:    s_mov_b32 s6, 0xffff
+; GFX10-NEXT:    s_xor_b32 s5, s0, s4
+; GFX10-NEXT:    s_and_b32 s8, s2, s6
+; GFX10-NEXT:    s_lshr_b32 s7, s5, 16
+; GFX10-NEXT:    s_and_b32 s5, s5, s6
+; GFX10-NEXT:    s_lshr_b32 s2, s2, 16
+; GFX10-NEXT:    s_cmp_lt_u32 s5, s8
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s5, s5, s8
+; GFX10-NEXT:    s_cmp_lt_u32 s7, s2
+; GFX10-NEXT:    s_cselect_b32 s2, s7, s2
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s5, s2
+; GFX10-NEXT:    s_lshr_b32 s5, s0, 16
+; GFX10-NEXT:    s_lshr_b32 s7, s2, 16
+; GFX10-NEXT:    s_add_i32 s0, s0, s2
+; GFX10-NEXT:    s_xor_b32 s2, s1, s4
+; GFX10-NEXT:    s_add_i32 s5, s5, s7
+; GFX10-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX10-NEXT:    s_and_b32 s2, s2, s6
+; GFX10-NEXT:    s_and_b32 s6, s3, s6
+; GFX10-NEXT:    s_lshr_b32 s3, s3, 16
+; GFX10-NEXT:    s_cmp_lt_u32 s2, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s5
+; GFX10-NEXT:    s_cselect_b32 s2, s2, s6
+; GFX10-NEXT:    s_cmp_lt_u32 s4, s3
+; GFX10-NEXT:    s_cselect_b32 s3, s4, s3
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
+; GFX10-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX10-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX10-NEXT:    s_add_i32 s1, s1, s2
+; GFX10-NEXT:    s_add_i32 s3, s3, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> %lhs, <4 x i16> %rhs)
+  %cast = bitcast <4 x i16> %result to <2 x i32>
+  ret <2 x i32> %cast
+}
+
+; FIXME
+; define <5 x i16> @v_uaddsat_v5i16(<5 x i16> %lhs, <5 x i16> %rhs) {
+;   %result = call <5 x i16> @llvm.uadd.sat.v5i16(<5 x i16> %lhs, <5 x i16> %rhs)
+;   ret <5 x i16> %result
+; }
+
+; define amdgpu_ps <5 x i16> @s_uaddsat_v5i16(<5 x i16> inreg %lhs, <5 x i16> inreg %rhs) {
+;   %result = call <5 x i16> @llvm.uadd.sat.v5i16(<5 x i16> %lhs, <5 x i16> %rhs)
+;   ret <5 x i16> %result
+; }
+
+define <3 x float> @v_uaddsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
+; GFX6-LABEL: v_uaddsat_v6i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GFX6-NEXT:    v_xor_b32_e32 v12, -1, v0
+; GFX6-NEXT:    v_min_u32_e32 v6, v12, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
+; GFX6-NEXT:    v_xor_b32_e32 v7, -1, v1
+; GFX6-NEXT:    v_min_u32_e32 v6, v7, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v8
+; GFX6-NEXT:    v_xor_b32_e32 v7, -1, v2
+; GFX6-NEXT:    v_min_u32_e32 v6, v7, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v9
+; GFX6-NEXT:    v_xor_b32_e32 v7, -1, v3
+; GFX6-NEXT:    v_min_u32_e32 v6, v7, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v10
+; GFX6-NEXT:    v_xor_b32_e32 v7, -1, v4
+; GFX6-NEXT:    v_min_u32_e32 v6, v7, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v11
+; GFX6-NEXT:    v_xor_b32_e32 v7, -1, v5
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_min_u32_e32 v6, v7, v6
+; GFX6-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v3
+; GFX6-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
+; GFX6-NEXT:    v_and_b32_e32 v3, s4, v5
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_v6i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s4, 0xffff
+; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
+; GFX8-NEXT:    v_xor_b32_e32 v9, s4, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v1
+; GFX8-NEXT:    v_xor_b32_e32 v10, s4, v6
+; GFX8-NEXT:    v_min_u16_e32 v9, v9, v3
+; GFX8-NEXT:    v_min_u16_sdwa v3, v10, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_xor_b32_e32 v10, s4, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
+; GFX8-NEXT:    v_xor_b32_e32 v11, s4, v7
+; GFX8-NEXT:    v_min_u16_e32 v10, v10, v4
+; GFX8-NEXT:    v_min_u16_sdwa v4, v11, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_xor_b32_e32 v11, s4, v2
+; GFX8-NEXT:    v_xor_b32_e32 v12, s4, v8
+; GFX8-NEXT:    v_add_u16_e32 v0, v0, v9
+; GFX8-NEXT:    v_add_u16_sdwa v3, v6, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_min_u16_e32 v11, v11, v5
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX8-NEXT:    v_min_u16_sdwa v5, v12, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_add_u16_e32 v1, v1, v10
+; GFX8-NEXT:    v_add_u16_sdwa v3, v7, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v3
+; GFX8-NEXT:    v_add_u16_e32 v2, v2, v11
+; GFX8-NEXT:    v_add_u16_sdwa v3, v8, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_v6i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
+; GFX9-NEXT:    v_xor_b32_e32 v6, s4, v0
+; GFX9-NEXT:    v_pk_min_u16 v3, v6, v3
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, v3
+; GFX9-NEXT:    v_xor_b32_e32 v3, s4, v1
+; GFX9-NEXT:    v_pk_min_u16 v3, v3, v4
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, v3
+; GFX9-NEXT:    v_xor_b32_e32 v3, s4, v2
+; GFX9-NEXT:    v_pk_min_u16 v3, v3, v5
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, v3
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_uaddsat_v6i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_xor_b32_e32 v6, s4, v0
+; GFX10-NEXT:    v_xor_b32_e32 v7, s4, v1
+; GFX10-NEXT:    v_xor_b32_e32 v8, s4, v2
+; GFX10-NEXT:    v_pk_min_u16 v3, v6, v3
+; GFX10-NEXT:    v_pk_min_u16 v4, v7, v4
+; GFX10-NEXT:    v_pk_min_u16 v5, v8, v5
+; GFX10-NEXT:    v_pk_add_u16 v0, v0, v3
+; GFX10-NEXT:    v_pk_add_u16 v1, v1, v4
+; GFX10-NEXT:    v_pk_add_u16 v2, v2, v5
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <6 x i16> @llvm.uadd.sat.v6i16(<6 x i16> %lhs, <6 x i16> %rhs)
+  %cast = bitcast <6 x i16> %result to <3 x float>
+  ret <3 x float> %cast
+}
+
+define amdgpu_ps <3 x i32> @s_uaddsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inreg %rhs) {
+; GFX6-LABEL: s_uaddsat_v6i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s6, 16
+; GFX6-NEXT:    s_not_b32 s12, s0
+; GFX6-NEXT:    s_cmp_lt_u32 s12, s6
+; GFX6-NEXT:    s_cselect_b32 s6, s12, s6
+; GFX6-NEXT:    s_add_i32 s0, s0, s6
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s7, 16
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX6-NEXT:    s_not_b32 s7, s1
+; GFX6-NEXT:    s_cmp_lt_u32 s7, s6
+; GFX6-NEXT:    s_cselect_b32 s6, s7, s6
+; GFX6-NEXT:    s_add_i32 s1, s1, s6
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s8, 16
+; GFX6-NEXT:    s_not_b32 s7, s2
+; GFX6-NEXT:    s_cmp_lt_u32 s7, s6
+; GFX6-NEXT:    s_cselect_b32 s6, s7, s6
+; GFX6-NEXT:    s_add_i32 s2, s2, s6
+; GFX6-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX6-NEXT:    s_lshr_b32 s2, s2, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s9, 16
+; GFX6-NEXT:    s_not_b32 s7, s3
+; GFX6-NEXT:    s_cmp_lt_u32 s7, s6
+; GFX6-NEXT:    s_cselect_b32 s6, s7, s6
+; GFX6-NEXT:    s_add_i32 s3, s3, s6
+; GFX6-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX6-NEXT:    s_lshr_b32 s3, s3, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s10, 16
+; GFX6-NEXT:    s_not_b32 s7, s4
+; GFX6-NEXT:    s_cmp_lt_u32 s7, s6
+; GFX6-NEXT:    s_cselect_b32 s6, s7, s6
+; GFX6-NEXT:    s_add_i32 s4, s4, s6
+; GFX6-NEXT:    s_lshl_b32 s5, s5, 16
+; GFX6-NEXT:    s_lshr_b32 s4, s4, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s11, 16
+; GFX6-NEXT:    s_not_b32 s7, s5
+; GFX6-NEXT:    s_cmp_lt_u32 s7, s6
+; GFX6-NEXT:    s_cselect_b32 s6, s7, s6
+; GFX6-NEXT:    s_add_i32 s5, s5, s6
+; GFX6-NEXT:    s_mov_b32 s6, 0xffff
+; GFX6-NEXT:    s_and_b32 s1, s1, s6
+; GFX6-NEXT:    s_and_b32 s0, s0, s6
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_and_b32 s1, s2, s6
+; GFX6-NEXT:    s_and_b32 s2, s3, s6
+; GFX6-NEXT:    s_lshr_b32 s5, s5, 16
+; GFX6-NEXT:    s_and_b32 s3, s5, s6
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_or_b32 s1, s1, s2
+; GFX6-NEXT:    s_and_b32 s2, s4, s6
+; GFX6-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX6-NEXT:    s_or_b32 s2, s2, s3
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_uaddsat_v6i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s12, 0xffff
+; GFX8-NEXT:    s_xor_b32 s13, s0, s12
+; GFX8-NEXT:    s_lshr_b32 s9, s3, 16
+; GFX8-NEXT:    s_lshr_b32 s6, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s7, s1, 16
+; GFX8-NEXT:    s_lshr_b32 s8, s2, 16
+; GFX8-NEXT:    s_lshr_b32 s10, s4, 16
+; GFX8-NEXT:    s_lshr_b32 s11, s5, 16
+; GFX8-NEXT:    s_bfe_u32 s13, s13, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s13, s3
+; GFX8-NEXT:    s_cselect_b32 s3, s13, s3
+; GFX8-NEXT:    s_add_i32 s0, s0, s3
+; GFX8-NEXT:    s_xor_b32 s3, s6, s12
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s9, s9, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s3, s9
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s9
+; GFX8-NEXT:    s_add_i32 s6, s6, s3
+; GFX8-NEXT:    s_xor_b32 s3, s1, s12
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s3, s4
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX8-NEXT:    s_add_i32 s1, s1, s3
+; GFX8-NEXT:    s_xor_b32 s3, s7, s12
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s4, s10, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s3, s4
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX8-NEXT:    s_add_i32 s7, s7, s3
+; GFX8-NEXT:    s_xor_b32 s3, s2, s12
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s4, s5, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s3, s4
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX8-NEXT:    s_add_i32 s2, s2, s3
+; GFX8-NEXT:    s_xor_b32 s3, s8, s12
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s4, s11, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s3, s4
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX8-NEXT:    s_add_i32 s8, s8, s3
+; GFX8-NEXT:    s_bfe_u32 s3, s6, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX8-NEXT:    s_or_b32 s0, s0, s3
+; GFX8-NEXT:    s_bfe_u32 s3, s7, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX8-NEXT:    s_or_b32 s1, s1, s3
+; GFX8-NEXT:    s_bfe_u32 s3, s8, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX8-NEXT:    s_or_b32 s2, s2, s3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_uaddsat_v6i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_pack_ll_b32_b16 s6, -1, -1
+; GFX9-NEXT:    s_xor_b32 s7, s0, s6
+; GFX9-NEXT:    s_mov_b32 s9, 0xffff
+; GFX9-NEXT:    s_lshr_b32 s8, s7, 16
+; GFX9-NEXT:    s_lshr_b32 s10, s3, 16
+; GFX9-NEXT:    s_and_b32 s7, s7, s9
+; GFX9-NEXT:    s_and_b32 s3, s3, s9
+; GFX9-NEXT:    s_cmp_lt_u32 s7, s3
+; GFX9-NEXT:    s_cselect_b32 s3, s7, s3
+; GFX9-NEXT:    s_cmp_lt_u32 s8, s10
+; GFX9-NEXT:    s_cselect_b32 s7, s8, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s7
+; GFX9-NEXT:    s_lshr_b32 s7, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s8, s3, 16
+; GFX9-NEXT:    s_add_i32 s0, s0, s3
+; GFX9-NEXT:    s_add_i32 s7, s7, s8
+; GFX9-NEXT:    s_xor_b32 s3, s1, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s7
+; GFX9-NEXT:    s_lshr_b32 s7, s3, 16
+; GFX9-NEXT:    s_lshr_b32 s8, s4, 16
+; GFX9-NEXT:    s_and_b32 s3, s3, s9
+; GFX9-NEXT:    s_and_b32 s4, s4, s9
+; GFX9-NEXT:    s_cmp_lt_u32 s3, s4
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX9-NEXT:    s_cmp_lt_u32 s7, s8
+; GFX9-NEXT:    s_cselect_b32 s4, s7, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
+; GFX9-NEXT:    s_lshr_b32 s4, s1, 16
+; GFX9-NEXT:    s_lshr_b32 s7, s3, 16
+; GFX9-NEXT:    s_add_i32 s1, s1, s3
+; GFX9-NEXT:    s_add_i32 s4, s4, s7
+; GFX9-NEXT:    s_xor_b32 s3, s2, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
+; GFX9-NEXT:    s_lshr_b32 s4, s3, 16
+; GFX9-NEXT:    s_lshr_b32 s6, s5, 16
+; GFX9-NEXT:    s_and_b32 s3, s3, s9
+; GFX9-NEXT:    s_and_b32 s5, s5, s9
+; GFX9-NEXT:    s_cmp_lt_u32 s3, s5
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s5
+; GFX9-NEXT:    s_cmp_lt_u32 s4, s6
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
+; GFX9-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX9-NEXT:    s_lshr_b32 s5, s3, 16
+; GFX9-NEXT:    s_add_i32 s2, s2, s3
+; GFX9-NEXT:    s_add_i32 s4, s4, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_uaddsat_v6i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, -1, -1
+; GFX10-NEXT:    s_mov_b32 s8, 0xffff
+; GFX10-NEXT:    s_xor_b32 s7, s0, s6
+; GFX10-NEXT:    s_and_b32 s10, s3, s8
+; GFX10-NEXT:    s_lshr_b32 s9, s7, 16
+; GFX10-NEXT:    s_and_b32 s7, s7, s8
+; GFX10-NEXT:    s_lshr_b32 s3, s3, 16
+; GFX10-NEXT:    s_cmp_lt_u32 s7, s10
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s7, s7, s10
+; GFX10-NEXT:    s_cmp_lt_u32 s9, s3
+; GFX10-NEXT:    s_cselect_b32 s3, s9, s3
+; GFX10-NEXT:    s_and_b32 s10, s4, s8
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s7, s3
+; GFX10-NEXT:    s_lshr_b32 s7, s0, 16
+; GFX10-NEXT:    s_lshr_b32 s9, s3, 16
+; GFX10-NEXT:    s_add_i32 s0, s0, s3
+; GFX10-NEXT:    s_xor_b32 s3, s1, s6
+; GFX10-NEXT:    s_add_i32 s7, s7, s9
+; GFX10-NEXT:    s_lshr_b32 s9, s3, 16
+; GFX10-NEXT:    s_and_b32 s3, s3, s8
+; GFX10-NEXT:    s_lshr_b32 s4, s4, 16
+; GFX10-NEXT:    s_cmp_lt_u32 s3, s10
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s7
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s10
+; GFX10-NEXT:    s_cmp_lt_u32 s9, s4
+; GFX10-NEXT:    s_cselect_b32 s4, s9, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
+; GFX10-NEXT:    s_lshr_b32 s4, s1, 16
+; GFX10-NEXT:    s_lshr_b32 s9, s3, 16
+; GFX10-NEXT:    s_add_i32 s1, s1, s3
+; GFX10-NEXT:    s_xor_b32 s3, s2, s6
+; GFX10-NEXT:    s_add_i32 s4, s4, s9
+; GFX10-NEXT:    s_lshr_b32 s6, s3, 16
+; GFX10-NEXT:    s_and_b32 s3, s3, s8
+; GFX10-NEXT:    s_and_b32 s8, s5, s8
+; GFX10-NEXT:    s_lshr_b32 s5, s5, 16
+; GFX10-NEXT:    s_cmp_lt_u32 s3, s8
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
+; GFX10-NEXT:    s_cselect_b32 s3, s3, s8
+; GFX10-NEXT:    s_cmp_lt_u32 s6, s5
+; GFX10-NEXT:    s_cselect_b32 s5, s6, s5
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s5
+; GFX10-NEXT:    s_lshr_b32 s5, s2, 16
+; GFX10-NEXT:    s_lshr_b32 s6, s3, 16
+; GFX10-NEXT:    s_add_i32 s2, s2, s3
+; GFX10-NEXT:    s_add_i32 s5, s5, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s5
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <6 x i16> @llvm.uadd.sat.v6i16(<6 x i16> %lhs, <6 x i16> %rhs)
+  %cast = bitcast <6 x i16> %result to <3 x i32>
+  ret <3 x i32> %cast
+}
+
+define <4 x float> @v_uaddsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
+; GFX6-LABEL: v_uaddsat_v8i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GFX6-NEXT:    v_xor_b32_e32 v16, -1, v0
+; GFX6-NEXT:    v_min_u32_e32 v8, v16, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v9
+; GFX6-NEXT:    v_xor_b32_e32 v9, -1, v1
+; GFX6-NEXT:    v_min_u32_e32 v8, v9, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v10
+; GFX6-NEXT:    v_xor_b32_e32 v9, -1, v2
+; GFX6-NEXT:    v_min_u32_e32 v8, v9, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v11
+; GFX6-NEXT:    v_xor_b32_e32 v9, -1, v3
+; GFX6-NEXT:    v_min_u32_e32 v8, v9, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v12
+; GFX6-NEXT:    v_xor_b32_e32 v9, -1, v4
+; GFX6-NEXT:    v_min_u32_e32 v8, v9, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v13
+; GFX6-NEXT:    v_xor_b32_e32 v9, -1, v5
+; GFX6-NEXT:    v_min_u32_e32 v8, v9, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GFX6-NEXT:    v_add_i32_e32 v5, vcc, v5, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v14
+; GFX6-NEXT:    v_xor_b32_e32 v9, -1, v6
+; GFX6-NEXT:    v_min_u32_e32 v8, v9, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v15
+; GFX6-NEXT:    v_xor_b32_e32 v9, -1, v7
+; GFX6-NEXT:    v_min_u32_e32 v8, v9, v8
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v3
+; GFX6-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
+; GFX6-NEXT:    v_and_b32_e32 v3, s4, v5
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
+; GFX6-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v4
+; GFX6-NEXT:    v_and_b32_e32 v4, s4, v7
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_lshrrev_b32_e32 v6, 16, v6
+; GFX6-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX6-NEXT:    v_and_b32_e32 v3, s4, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT:    v_or_b32_e32 v3, v3, v4
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_v8i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s4, 0xffff
+; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 16, v0
+; GFX8-NEXT:    v_xor_b32_e32 v12, s4, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v9, 16, v1
+; GFX8-NEXT:    v_xor_b32_e32 v13, s4, v8
+; GFX8-NEXT:    v_min_u16_e32 v12, v12, v4
+; GFX8-NEXT:    v_min_u16_sdwa v4, v13, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_xor_b32_e32 v13, s4, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
+; GFX8-NEXT:    v_xor_b32_e32 v14, s4, v9
+; GFX8-NEXT:    v_min_u16_e32 v13, v13, v5
+; GFX8-NEXT:    v_lshrrev_b32_e32 v11, 16, v3
+; GFX8-NEXT:    v_min_u16_sdwa v5, v14, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_xor_b32_e32 v14, s4, v2
+; GFX8-NEXT:    v_xor_b32_e32 v15, s4, v10
+; GFX8-NEXT:    v_add_u16_e32 v0, v0, v12
+; GFX8-NEXT:    v_add_u16_sdwa v4, v8, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_min_u16_e32 v14, v14, v6
+; GFX8-NEXT:    v_min_u16_sdwa v6, v15, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_xor_b32_e32 v15, s4, v3
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT:    v_xor_b32_e32 v16, s4, v11
+; GFX8-NEXT:    v_add_u16_e32 v1, v1, v13
+; GFX8-NEXT:    v_add_u16_sdwa v4, v9, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_min_u16_e32 v15, v15, v7
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v4
+; GFX8-NEXT:    v_min_u16_sdwa v7, v16, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_add_u16_e32 v2, v2, v14
+; GFX8-NEXT:    v_add_u16_sdwa v4, v10, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v2, v2, v4
+; GFX8-NEXT:    v_add_u16_e32 v3, v3, v15
+; GFX8-NEXT:    v_add_u16_sdwa v4, v11, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v3, v3, v4
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_v8i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
+; GFX9-NEXT:    v_xor_b32_e32 v8, s4, v0
+; GFX9-NEXT:    v_pk_min_u16 v4, v8, v4
+; GFX9-NEXT:    v_pk_add_u16 v0, v0, v4
+; GFX9-NEXT:    v_xor_b32_e32 v4, s4, v1
+; GFX9-NEXT:    v_pk_min_u16 v4, v4, v5
+; GFX9-NEXT:    v_pk_add_u16 v1, v1, v4
+; GFX9-NEXT:    v_xor_b32_e32 v4, s4, v2
+; GFX9-NEXT:    v_pk_min_u16 v4, v4, v6
+; GFX9-NEXT:    v_pk_add_u16 v2, v2, v4
+; GFX9-NEXT:    v_xor_b32_e32 v4, s4, v3
+; GFX9-NEXT:    v_pk_min_u16 v4, v4, v7
+; GFX9-NEXT:    v_pk_add_u16 v3, v3, v4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_uaddsat_v8i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, -1, -1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_xor_b32_e32 v15, s4, v0
+; GFX10-NEXT:    v_xor_b32_e32 v19, s4, v1
+; GFX10-NEXT:    v_xor_b32_e32 v23, s4, v2
+; GFX10-NEXT:    v_xor_b32_e32 v10, s4, v3
+; GFX10-NEXT:    v_pk_min_u16 v11, v15, v4
+; GFX10-NEXT:    v_pk_min_u16 v15, v19, v5
+; GFX10-NEXT:    v_pk_min_u16 v19, v23, v6
+; GFX10-NEXT:    v_pk_min_u16 v6, v10, v7
+; GFX10-NEXT:    v_pk_add_u16 v0, v0, v11
+; GFX10-NEXT:    v_pk_add_u16 v1, v1, v15
+; GFX10-NEXT:    v_pk_add_u16 v2, v2, v19
+; GFX10-NEXT:    v_pk_add_u16 v3, v3, v6
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %lhs, <8 x i16> %rhs)
+  %cast = bitcast <8 x i16> %result to <4 x float>
+  ret <4 x float> %cast
+}
+
+define amdgpu_ps <4 x i32> @s_uaddsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inreg %rhs) {
+; GFX6-LABEL: s_uaddsat_v8i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s8, 16
+; GFX6-NEXT:    s_not_b32 s16, s0
+; GFX6-NEXT:    s_cmp_lt_u32 s16, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s16, s8
+; GFX6-NEXT:    s_add_i32 s0, s0, s8
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s9, 16
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX6-NEXT:    s_not_b32 s9, s1
+; GFX6-NEXT:    s_cmp_lt_u32 s9, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s9, s8
+; GFX6-NEXT:    s_add_i32 s1, s1, s8
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s10, 16
+; GFX6-NEXT:    s_not_b32 s9, s2
+; GFX6-NEXT:    s_cmp_lt_u32 s9, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s9, s8
+; GFX6-NEXT:    s_add_i32 s2, s2, s8
+; GFX6-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX6-NEXT:    s_lshr_b32 s2, s2, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s11, 16
+; GFX6-NEXT:    s_not_b32 s9, s3
+; GFX6-NEXT:    s_cmp_lt_u32 s9, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s9, s8
+; GFX6-NEXT:    s_add_i32 s3, s3, s8
+; GFX6-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX6-NEXT:    s_lshr_b32 s3, s3, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s12, 16
+; GFX6-NEXT:    s_not_b32 s9, s4
+; GFX6-NEXT:    s_cmp_lt_u32 s9, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s9, s8
+; GFX6-NEXT:    s_add_i32 s4, s4, s8
+; GFX6-NEXT:    s_lshl_b32 s5, s5, 16
+; GFX6-NEXT:    s_lshr_b32 s4, s4, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s13, 16
+; GFX6-NEXT:    s_not_b32 s9, s5
+; GFX6-NEXT:    s_cmp_lt_u32 s9, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s9, s8
+; GFX6-NEXT:    s_add_i32 s5, s5, s8
+; GFX6-NEXT:    s_lshl_b32 s6, s6, 16
+; GFX6-NEXT:    s_lshr_b32 s5, s5, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s14, 16
+; GFX6-NEXT:    s_not_b32 s9, s6
+; GFX6-NEXT:    s_cmp_lt_u32 s9, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s9, s8
+; GFX6-NEXT:    s_add_i32 s6, s6, s8
+; GFX6-NEXT:    s_lshl_b32 s7, s7, 16
+; GFX6-NEXT:    s_lshr_b32 s6, s6, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s15, 16
+; GFX6-NEXT:    s_not_b32 s9, s7
+; GFX6-NEXT:    s_cmp_lt_u32 s9, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s9, s8
+; GFX6-NEXT:    s_add_i32 s7, s7, s8
+; GFX6-NEXT:    s_mov_b32 s8, 0xffff
+; GFX6-NEXT:    s_and_b32 s1, s1, s8
+; GFX6-NEXT:    s_and_b32 s0, s0, s8
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_and_b32 s1, s2, s8
+; GFX6-NEXT:    s_and_b32 s2, s3, s8
+; GFX6-NEXT:    s_and_b32 s3, s5, s8
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_lshr_b32 s7, s7, 16
+; GFX6-NEXT:    s_or_b32 s1, s1, s2
+; GFX6-NEXT:    s_and_b32 s2, s4, s8
+; GFX6-NEXT:    s_and_b32 s4, s7, s8
+; GFX6-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX6-NEXT:    s_or_b32 s2, s2, s3
+; GFX6-NEXT:    s_and_b32 s3, s6, s8
+; GFX6-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX6-NEXT:    s_or_b32 s3, s3, s4
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_uaddsat_v8i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s16, 0xffff
+; GFX8-NEXT:    s_xor_b32 s17, s0, s16
+; GFX8-NEXT:    s_lshr_b32 s12, s4, 16
+; GFX8-NEXT:    s_lshr_b32 s8, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s9, s1, 16
+; GFX8-NEXT:    s_lshr_b32 s10, s2, 16
+; GFX8-NEXT:    s_lshr_b32 s11, s3, 16
+; GFX8-NEXT:    s_lshr_b32 s13, s5, 16
+; GFX8-NEXT:    s_lshr_b32 s14, s6, 16
+; GFX8-NEXT:    s_lshr_b32 s15, s7, 16
+; GFX8-NEXT:    s_bfe_u32 s17, s17, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s17, s4
+; GFX8-NEXT:    s_cselect_b32 s4, s17, s4
+; GFX8-NEXT:    s_add_i32 s0, s0, s4
+; GFX8-NEXT:    s_xor_b32 s4, s8, s16
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s12, s12, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s4, s12
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s12
+; GFX8-NEXT:    s_add_i32 s8, s8, s4
+; GFX8-NEXT:    s_xor_b32 s4, s1, s16
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s4, s5
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX8-NEXT:    s_add_i32 s1, s1, s4
+; GFX8-NEXT:    s_xor_b32 s4, s9, s16
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s5, s13, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s4, s5
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX8-NEXT:    s_add_i32 s9, s9, s4
+; GFX8-NEXT:    s_xor_b32 s4, s2, s16
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s5, s6, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s4, s5
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX8-NEXT:    s_add_i32 s2, s2, s4
+; GFX8-NEXT:    s_xor_b32 s4, s10, s16
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s5, s14, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s4, s5
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX8-NEXT:    s_add_i32 s10, s10, s4
+; GFX8-NEXT:    s_xor_b32 s4, s3, s16
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s5, s7, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s4, s5
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX8-NEXT:    s_add_i32 s3, s3, s4
+; GFX8-NEXT:    s_xor_b32 s4, s11, s16
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s5, s15, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s4, s5
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX8-NEXT:    s_add_i32 s11, s11, s4
+; GFX8-NEXT:    s_bfe_u32 s4, s8, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX8-NEXT:    s_or_b32 s0, s0, s4
+; GFX8-NEXT:    s_bfe_u32 s4, s9, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX8-NEXT:    s_or_b32 s1, s1, s4
+; GFX8-NEXT:    s_bfe_u32 s4, s10, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX8-NEXT:    s_or_b32 s2, s2, s4
+; GFX8-NEXT:    s_bfe_u32 s4, s11, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX8-NEXT:    s_or_b32 s3, s3, s4
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_uaddsat_v8i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_pack_ll_b32_b16 s8, -1, -1
+; GFX9-NEXT:    s_xor_b32 s9, s0, s8
+; GFX9-NEXT:    s_mov_b32 s11, 0xffff
+; GFX9-NEXT:    s_lshr_b32 s10, s9, 16
+; GFX9-NEXT:    s_lshr_b32 s12, s4, 16
+; GFX9-NEXT:    s_and_b32 s9, s9, s11
+; GFX9-NEXT:    s_and_b32 s4, s4, s11
+; GFX9-NEXT:    s_cmp_lt_u32 s9, s4
+; GFX9-NEXT:    s_cselect_b32 s4, s9, s4
+; GFX9-NEXT:    s_cmp_lt_u32 s10, s12
+; GFX9-NEXT:    s_cselect_b32 s9, s10, s12
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s9
+; GFX9-NEXT:    s_lshr_b32 s9, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s10, s4, 16
+; GFX9-NEXT:    s_add_i32 s0, s0, s4
+; GFX9-NEXT:    s_add_i32 s9, s9, s10
+; GFX9-NEXT:    s_xor_b32 s4, s1, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s9
+; GFX9-NEXT:    s_lshr_b32 s9, s4, 16
+; GFX9-NEXT:    s_lshr_b32 s10, s5, 16
+; GFX9-NEXT:    s_and_b32 s4, s4, s11
+; GFX9-NEXT:    s_and_b32 s5, s5, s11
+; GFX9-NEXT:    s_cmp_lt_u32 s4, s5
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX9-NEXT:    s_cmp_lt_u32 s9, s10
+; GFX9-NEXT:    s_cselect_b32 s5, s9, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
+; GFX9-NEXT:    s_lshr_b32 s5, s1, 16
+; GFX9-NEXT:    s_lshr_b32 s9, s4, 16
+; GFX9-NEXT:    s_add_i32 s1, s1, s4
+; GFX9-NEXT:    s_add_i32 s5, s5, s9
+; GFX9-NEXT:    s_xor_b32 s4, s2, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s5
+; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
+; GFX9-NEXT:    s_lshr_b32 s9, s6, 16
+; GFX9-NEXT:    s_and_b32 s4, s4, s11
+; GFX9-NEXT:    s_and_b32 s6, s6, s11
+; GFX9-NEXT:    s_cmp_lt_u32 s4, s6
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX9-NEXT:    s_cmp_lt_u32 s5, s9
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s9
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
+; GFX9-NEXT:    s_lshr_b32 s5, s2, 16
+; GFX9-NEXT:    s_lshr_b32 s6, s4, 16
+; GFX9-NEXT:    s_add_i32 s2, s2, s4
+; GFX9-NEXT:    s_add_i32 s5, s5, s6
+; GFX9-NEXT:    s_xor_b32 s4, s3, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s5
+; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
+; GFX9-NEXT:    s_lshr_b32 s6, s7, 16
+; GFX9-NEXT:    s_and_b32 s4, s4, s11
+; GFX9-NEXT:    s_and_b32 s7, s7, s11
+; GFX9-NEXT:    s_cmp_lt_u32 s4, s7
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s7
+; GFX9-NEXT:    s_cmp_lt_u32 s5, s6
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
+; GFX9-NEXT:    s_lshr_b32 s5, s3, 16
+; GFX9-NEXT:    s_lshr_b32 s6, s4, 16
+; GFX9-NEXT:    s_add_i32 s3, s3, s4
+; GFX9-NEXT:    s_add_i32 s5, s5, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s5
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_uaddsat_v8i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_pack_ll_b32_b16 s8, -1, -1
+; GFX10-NEXT:    s_mov_b32 s10, 0xffff
+; GFX10-NEXT:    s_xor_b32 s9, s0, s8
+; GFX10-NEXT:    s_and_b32 s12, s4, s10
+; GFX10-NEXT:    s_lshr_b32 s11, s9, 16
+; GFX10-NEXT:    s_and_b32 s9, s9, s10
+; GFX10-NEXT:    s_lshr_b32 s4, s4, 16
+; GFX10-NEXT:    s_cmp_lt_u32 s9, s12
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s9, s9, s12
+; GFX10-NEXT:    s_cmp_lt_u32 s11, s4
+; GFX10-NEXT:    s_cselect_b32 s4, s11, s4
+; GFX10-NEXT:    s_and_b32 s12, s5, s10
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s9, s4
+; GFX10-NEXT:    s_lshr_b32 s9, s0, 16
+; GFX10-NEXT:    s_lshr_b32 s11, s4, 16
+; GFX10-NEXT:    s_add_i32 s0, s0, s4
+; GFX10-NEXT:    s_xor_b32 s4, s1, s8
+; GFX10-NEXT:    s_add_i32 s9, s9, s11
+; GFX10-NEXT:    s_lshr_b32 s11, s4, 16
+; GFX10-NEXT:    s_and_b32 s4, s4, s10
+; GFX10-NEXT:    s_lshr_b32 s5, s5, 16
+; GFX10-NEXT:    s_cmp_lt_u32 s4, s12
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s9
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s12
+; GFX10-NEXT:    s_cmp_lt_u32 s11, s5
+; GFX10-NEXT:    s_cselect_b32 s5, s11, s5
+; GFX10-NEXT:    s_and_b32 s12, s6, s10
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
+; GFX10-NEXT:    s_lshr_b32 s5, s1, 16
+; GFX10-NEXT:    s_lshr_b32 s11, s4, 16
+; GFX10-NEXT:    s_add_i32 s1, s1, s4
+; GFX10-NEXT:    s_xor_b32 s4, s2, s8
+; GFX10-NEXT:    s_add_i32 s5, s5, s11
+; GFX10-NEXT:    s_lshr_b32 s11, s4, 16
+; GFX10-NEXT:    s_and_b32 s4, s4, s10
+; GFX10-NEXT:    s_lshr_b32 s6, s6, 16
+; GFX10-NEXT:    s_cmp_lt_u32 s4, s12
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s5
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s12
+; GFX10-NEXT:    s_cmp_lt_u32 s11, s6
+; GFX10-NEXT:    s_cselect_b32 s6, s11, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s6
+; GFX10-NEXT:    s_lshr_b32 s6, s2, 16
+; GFX10-NEXT:    s_lshr_b32 s11, s4, 16
+; GFX10-NEXT:    s_add_i32 s2, s2, s4
+; GFX10-NEXT:    s_xor_b32 s4, s3, s8
+; GFX10-NEXT:    s_add_i32 s6, s6, s11
+; GFX10-NEXT:    s_lshr_b32 s8, s4, 16
+; GFX10-NEXT:    s_and_b32 s4, s4, s10
+; GFX10-NEXT:    s_and_b32 s10, s7, s10
+; GFX10-NEXT:    s_lshr_b32 s7, s7, 16
+; GFX10-NEXT:    s_cmp_lt_u32 s4, s10
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s6
+; GFX10-NEXT:    s_cselect_b32 s4, s4, s10
+; GFX10-NEXT:    s_cmp_lt_u32 s8, s7
+; GFX10-NEXT:    s_cselect_b32 s7, s8, s7
+; GFX10-NEXT:    s_lshr_b32 s5, s3, 16
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s7
+; GFX10-NEXT:    s_lshr_b32 s7, s4, 16
+; GFX10-NEXT:    s_add_i32 s3, s3, s4
+; GFX10-NEXT:    s_add_i32 s5, s5, s7
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s5
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %lhs, <8 x i16> %rhs)
+  %cast = bitcast <8 x i16> %result to <4 x i32>
+  ret <4 x i32> %cast
+}
+
+; FIXME: i48 broken because i48 add broken
+; define i48 @v_uaddsat_i48(i48 %lhs, i48 %rhs) {
+;   %result = call i48 @llvm.uadd.sat.i48(i48 %lhs, i48 %rhs)
+;   ret i48 %result
+; }
+
+; define amdgpu_ps i48 @s_uaddsat_i48(i48 inreg %lhs, i48 inreg %rhs) {
+;   %result = call i48 @llvm.uadd.sat.i48(i48 %lhs, i48 %rhs)
+;   ret i48 %result
+; }
+
+; define amdgpu_ps <2 x float> @uaddsat_i48_sv(i48 inreg %lhs, i48 %rhs) {
+;   %result = call i48 @llvm.uadd.sat.i48(i48 %lhs, i48 %rhs)
+;   %ext.result = zext i48 %result to i64
+;   %cast = bitcast i64 %ext.result to <2 x float>
+;   ret <2 x float> %cast
+; }
+
+; define amdgpu_ps <2 x float> @uaddsat_i48_vs(i48 %lhs, i48 inreg %rhs) {
+;   %result = call i48 @llvm.uadd.sat.i48(i48 %lhs, i48 %rhs)
+;   %ext.result = zext i48 %result to i64
+;   %cast = bitcast i64 %ext.result to <2 x float>
+;   ret <2 x float> %cast
+; }
+
+define i64 @v_uaddsat_i64(i64 %lhs, i64 %rhs) {
+; GFX6-LABEL: v_uaddsat_i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v0, -1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v1, -1, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, -1, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, -1, vcc
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_uaddsat_i64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, v0, v2
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, -1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, -1, vcc_lo
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i64 @llvm.uadd.sat.i64(i64 %lhs, i64 %rhs)
+  ret i64 %result
+}
+
+define amdgpu_ps i64 @s_uaddsat_i64(i64 inreg %lhs, i64 inreg %rhs) {
+; GFX6-LABEL: s_uaddsat_i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_add_u32 s0, s0, s2
+; GFX6-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX6-NEXT:    s_and_b32 s4, s4, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    s_addc_u32 s1, s1, s3
+; GFX6-NEXT:    v_mov_b32_e32 v1, s3
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    v_mov_b32_e32 v3, s1
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v2, -1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v3, -1, vcc
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_uaddsat_i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_add_u32 s0, s0, s2
+; GFX8-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX8-NEXT:    s_and_b32 s4, s4, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    s_addc_u32 s1, s1, s3
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v2, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v3, -1, vcc
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_uaddsat_i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_add_u32 s0, s0, s2
+; GFX9-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX9-NEXT:    s_and_b32 s4, s4, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    s_addc_u32 s1, s1, s3
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v2, s0
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v2, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v3, -1, vcc
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_uaddsat_i64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s0, s0, s2
+; GFX10-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_and_b32 s4, s4, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX10-NEXT:    s_addc_u32 s1, s1, s3
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s2, s[0:1], s[2:3]
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, s0, -1, s2
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, s1, -1, s2
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i64 @llvm.uadd.sat.i64(i64 %lhs, i64 %rhs)
+  ret i64 %result
+}
+
+define amdgpu_ps <2 x float> @uaddsat_i64_sv(i64 inreg %lhs, i64 %rhs) {
+; GFX6-LABEL: uaddsat_i64_sv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v3, s1
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, s0, v0
+; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v3, v1, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, v[2:3], v[0:1]
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v2, -1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v3, -1, vcc
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: uaddsat_i64_sv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s0, v0
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, v3, v1, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[2:3], v[0:1]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v2, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v3, -1, vcc
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: uaddsat_i64_sv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v1, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[2:3], v[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v2, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v3, -1, vcc
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: uaddsat_i64_sv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_add_co_u32_e64 v2, vcc_lo, s0, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, s1, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v2, -1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v3, -1, vcc_lo
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i64 @llvm.uadd.sat.i64(i64 %lhs, i64 %rhs)
+  %cast = bitcast i64 %result to <2 x float>
+  ret <2 x float> %cast
+}
+
+define amdgpu_ps <2 x float> @uaddsat_i64_vs(i64 %lhs, i64 inreg %rhs) {
+; GFX6-LABEL: uaddsat_i64_vs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v2, s1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v2, vcc
+; GFX6-NEXT:    v_cmp_gt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v0, -1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v1, -1, vcc
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: uaddsat_i64_vs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, v1, v2, vcc
+; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, -1, vcc
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: uaddsat_i64_vs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
+; GFX9-NEXT:    v_cmp_gt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, -1, vcc
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: uaddsat_i64_vs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, v0, s0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[0:1], v[0:1]
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, -1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, -1, vcc_lo
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i64 @llvm.uadd.sat.i64(i64 %lhs, i64 %rhs)
+  %cast = bitcast i64 %result to <2 x float>
+  ret <2 x float> %cast
+}
+
+define <2 x i64> @v_uaddsat_v2i64(<2 x i64> %lhs, <2 x i64> %rhs) {
+; GFX6-LABEL: v_uaddsat_v2i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v5, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, v[0:1], v[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v0, -1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v1, -1, vcc
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v6
+; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v3, v7, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, v[2:3], v[6:7]
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, v2, -1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, v3, -1, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_v2i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v4
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, v1, v5, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[0:1], v[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, -1, vcc
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v6
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, v3, v7, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[2:3], v[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, -1, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_v2i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v5, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[0:1], v[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, -1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v7, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[2:3], v[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, -1, vcc
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_uaddsat_v2i64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_mov_b32_e32 v10, v4
+; GFX10-NEXT:    v_mov_b32_e32 v11, v5
+; GFX10-NEXT:    v_mov_b32_e32 v8, v6
+; GFX10-NEXT:    v_mov_b32_e32 v9, v7
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, v0, v10
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v1, v11, vcc_lo
+; GFX10-NEXT:    v_add_co_u32_e64 v2, vcc_lo, v2, v8
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, v3, v9, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[10:11]
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s4, v[2:3], v[8:9]
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, -1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, -1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, -1, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, -1, s4
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> %lhs, <2 x i64> %rhs)
+  ret <2 x i64> %result
+}
+
+define amdgpu_ps <2 x i64> @s_uaddsat_v2i64(<2 x i64> inreg %lhs, <2 x i64> inreg %rhs) {
+; GFX6-LABEL: s_uaddsat_v2i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_add_u32 s0, s0, s4
+; GFX6-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX6-NEXT:    s_and_b32 s8, s8, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    s_addc_u32 s1, s1, s5
+; GFX6-NEXT:    v_mov_b32_e32 v1, s5
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_add_u32 s0, s2, s6
+; GFX6-NEXT:    v_mov_b32_e32 v3, s1
+; GFX6-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX6-NEXT:    s_and_b32 s1, s1, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX6-NEXT:    v_mov_b32_e32 v0, s6
+; GFX6-NEXT:    s_addc_u32 s1, s3, s7
+; GFX6-NEXT:    v_mov_b32_e32 v1, s7
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, v2, -1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, v3, -1, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX6-NEXT:    v_mov_b32_e32 v4, s0
+; GFX6-NEXT:    v_mov_b32_e32 v5, s1
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v4, -1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v5, -1, vcc
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v2
+; GFX6-NEXT:    v_readfirstlane_b32 s1, v3
+; GFX6-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s3, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_uaddsat_v2i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_add_u32 s0, s0, s4
+; GFX8-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX8-NEXT:    s_and_b32 s8, s8, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    s_addc_u32 s1, s1, s5
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:    s_add_u32 s0, s2, s6
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX8-NEXT:    s_and_b32 s1, s1, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s6
+; GFX8-NEXT:    s_addc_u32 s1, s3, s7
+; GFX8-NEXT:    v_mov_b32_e32 v1, s7
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, -1, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v4, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v5, -1, vcc
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v3
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_uaddsat_v2i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_add_u32 s0, s0, s4
+; GFX9-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX9-NEXT:    s_and_b32 s8, s8, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    s_addc_u32 s1, s1, s5
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v2, s0
+; GFX9-NEXT:    s_add_u32 s0, s2, s6
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX9-NEXT:    s_and_b32 s1, s1, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s6
+; GFX9-NEXT:    s_addc_u32 s1, s3, s7
+; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, -1, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v4, s0
+; GFX9-NEXT:    v_mov_b32_e32 v5, s1
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v5, -1, vcc
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v3
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_uaddsat_v2i64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s0, s0, s4
+; GFX10-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_and_b32 s8, s8, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX10-NEXT:    s_addc_u32 s1, s1, s5
+; GFX10-NEXT:    s_add_u32 s2, s2, s6
+; GFX10-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s4, s[0:1], s[4:5]
+; GFX10-NEXT:    s_and_b32 s8, s8, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX10-NEXT:    s_addc_u32 s3, s3, s7
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, s0, -1, s4
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s5, s[2:3], s[6:7]
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, s1, -1, s4
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, s2, -1, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, s3, -1, s5
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> %lhs, <2 x i64> %rhs)
+  ret <2 x i64> %result
+}
+
+define amdgpu_ps i128 @s_uaddsat_i128(i128 inreg %lhs, i128 inreg %rhs) {
+; GFX6-LABEL: s_uaddsat_i128:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_add_u32 s0, s0, s4
+; GFX6-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX6-NEXT:    s_and_b32 s8, s8, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX6-NEXT:    s_addc_u32 s1, s1, s5
+; GFX6-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX6-NEXT:    s_and_b32 s8, s8, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX6-NEXT:    s_addc_u32 s2, s2, s6
+; GFX6-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX6-NEXT:    v_mov_b32_e32 v2, s4
+; GFX6-NEXT:    s_and_b32 s8, s8, 1
+; GFX6-NEXT:    v_mov_b32_e32 v3, s5
+; GFX6-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX6-NEXT:    v_mov_b32_e32 v0, s6
+; GFX6-NEXT:    s_addc_u32 s3, s3, s7
+; GFX6-NEXT:    v_mov_b32_e32 v1, s7
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, s[2:3], v[0:1]
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX6-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v2, s1
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v1, -1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v2, -1, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v2, s2
+; GFX6-NEXT:    v_mov_b32_e32 v3, s3
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, v2, -1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, v3, -1, vcc
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX6-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX6-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_uaddsat_i128:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_add_u32 s0, s0, s4
+; GFX8-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX8-NEXT:    s_and_b32 s8, s8, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX8-NEXT:    s_addc_u32 s1, s1, s5
+; GFX8-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX8-NEXT:    s_and_b32 s8, s8, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX8-NEXT:    s_addc_u32 s2, s2, s6
+; GFX8-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX8-NEXT:    s_and_b32 s8, s8, 1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX8-NEXT:    v_mov_b32_e32 v3, s5
+; GFX8-NEXT:    s_addc_u32 s3, s3, s7
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s6
+; GFX8-NEXT:    v_mov_b32_e32 v1, s7
+; GFX8-NEXT:    s_cmp_eq_u64 s[2:3], s[6:7]
+; GFX8-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
+; GFX8-NEXT:    s_and_b32 s4, 1, s6
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s0
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v1, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v2, -1, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s3
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, -1, vcc
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_uaddsat_i128:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_add_u32 s0, s0, s4
+; GFX9-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX9-NEXT:    s_and_b32 s8, s8, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX9-NEXT:    s_addc_u32 s1, s1, s5
+; GFX9-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX9-NEXT:    s_and_b32 s8, s8, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX9-NEXT:    s_addc_u32 s2, s2, s6
+; GFX9-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX9-NEXT:    s_and_b32 s8, s8, 1
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
+; GFX9-NEXT:    s_addc_u32 s3, s3, s7
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v0, s6
+; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    s_cmp_eq_u64 s[2:3], s[6:7]
+; GFX9-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
+; GFX9-NEXT:    s_and_b32 s4, 1, s6
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s0
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v1, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v2, -1, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v2, s2
+; GFX9-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, -1, vcc
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_uaddsat_i128:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s0, s0, s4
+; GFX10-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_and_b32 s8, s8, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX10-NEXT:    s_addc_u32 s1, s1, s5
+; GFX10-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s4, s[0:1], s[4:5]
+; GFX10-NEXT:    s_and_b32 s8, s8, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX10-NEXT:    s_addc_u32 s2, s2, s6
+; GFX10-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s4
+; GFX10-NEXT:    s_and_b32 s8, s8, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX10-NEXT:    s_addc_u32 s3, s3, s7
+; GFX10-NEXT:    s_cmp_eq_u64 s[2:3], s[6:7]
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s5, s[2:3], s[6:7]
+; GFX10-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX10-NEXT:    s_and_b32 s4, 1, s4
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s5
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, s0, -1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, s1, -1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, s2, -1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, s3, -1, vcc_lo
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i128 @llvm.uadd.sat.i128(i128 %lhs, i128 %rhs)
+  ret i128 %result
+}
+
+define amdgpu_ps <4 x float> @uaddsat_i128_sv(i128 inreg %lhs, i128 %rhs) {
+; GFX6-LABEL: uaddsat_i128_sv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v5, s1
+; GFX6-NEXT:    v_add_i32_e32 v4, vcc, s0, v0
+; GFX6-NEXT:    v_addc_u32_e32 v5, vcc, v5, v1, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v6, s2
+; GFX6-NEXT:    v_mov_b32_e32 v7, s3
+; GFX6-NEXT:    v_addc_u32_e32 v6, vcc, v6, v2, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v7, vcc, v7, v3, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, v[4:5], v[0:1]
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX6-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v4, -1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v5, -1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, v6, -1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, v7, -1, vcc
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: uaddsat_i128_sv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, s0, v0
+; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, v5, v1, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v6, s2
+; GFX8-NEXT:    v_mov_b32_e32 v7, s3
+; GFX8-NEXT:    v_addc_u32_e32 v6, vcc, v6, v2, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v7, vcc, v7, v3, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[4:5], v[0:1]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v4, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v5, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v6, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v7, -1, vcc
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: uaddsat_i128_sv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v5, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, s0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v5, v1, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v6, s2
+; GFX9-NEXT:    v_mov_b32_e32 v7, s3
+; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, v6, v2, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, v7, v3, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[4:5], v[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v5, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v6, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v7, -1, vcc
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: uaddsat_i128_sv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_add_co_u32_e64 v10, vcc_lo, s0, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v11, vcc_lo, s1, v1, vcc_lo
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, s2, v2, vcc_lo
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, s3, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[10:11], v[0:1]
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[4:5], v[2:3]
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[2:3]
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v10, -1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v11, -1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v4, -1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v5, -1, vcc_lo
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i128 @llvm.uadd.sat.i128(i128 %lhs, i128 %rhs)
+  %cast = bitcast i128 %result to <4 x float>
+  ret <4 x float> %cast
+}
+
+define amdgpu_ps <4 x float> @uaddsat_i128_vs(i128 %lhs, i128 inreg %rhs) {
+; GFX6-LABEL: uaddsat_i128_vs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v4, s1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v4, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v4, s2
+; GFX6-NEXT:    v_mov_b32_e32 v5, s3
+; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, v2, v4, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v3, v5, vcc
+; GFX6-NEXT:    v_cmp_gt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX6-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX6-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, s[2:3], v[2:3]
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
+; GFX6-NEXT:    v_and_b32_e32 v4, 1, v4
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v0, -1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v1, -1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, v2, -1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, v3, -1, vcc
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: uaddsat_i128_vs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v4, s1
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, v1, v4, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v4, s2
+; GFX8-NEXT:    v_mov_b32_e32 v5, s3
+; GFX8-NEXT:    v_addc_u32_e32 v2, vcc, v2, v4, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, v3, v5, vcc
+; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
+; GFX8-NEXT:    v_and_b32_e32 v4, 1, v4
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, -1, vcc
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: uaddsat_i128_vs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v4, s1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v4, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v4, s2
+; GFX9-NEXT:    v_mov_b32_e32 v5, s3
+; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, v2, v4, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v5, vcc
+; GFX9-NEXT:    v_cmp_gt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, s[2:3], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
+; GFX9-NEXT:    v_and_b32_e32 v4, 1, v4
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, -1, vcc
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: uaddsat_i128_vs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, v0, s0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, s2, v2, vcc_lo
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, s3, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[0:1], v[0:1]
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[2:3], v[2:3]
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[2:3], v[2:3]
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc_lo
+; GFX10-NEXT:    v_and_b32_e32 v4, 1, v4
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v4
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, -1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, -1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, -1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, -1, vcc_lo
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i128 @llvm.uadd.sat.i128(i128 %lhs, i128 %rhs)
+  %cast = bitcast i128 %result to <4 x float>
+  ret <4 x float> %cast
+}
+
+define <2 x i128> @v_uaddsat_v2i128(<2 x i128> %lhs, <2 x i128> %rhs) {
+; GFX6-LABEL: v_uaddsat_v2i128:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v8
+; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v9, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, v2, v10, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v3, v11, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, v[0:1], v[8:9]
+; GFX6-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, v[2:3], v[10:11]
+; GFX6-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[10:11]
+; GFX6-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
+; GFX6-NEXT:    v_and_b32_e32 v8, 1, v8
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v0, -1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v1, -1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, v2, -1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, v3, -1, vcc
+; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v4, v12
+; GFX6-NEXT:    v_addc_u32_e32 v5, vcc, v5, v13, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v6, vcc, v6, v14, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v7, vcc, v7, v15, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, v[4:5], v[12:13]
+; GFX6-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[14:15]
+; GFX6-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[14:15]
+; GFX6-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
+; GFX6-NEXT:    v_and_b32_e32 v8, 1, v8
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GFX6-NEXT:    v_cndmask_b32_e64 v4, v4, -1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v5, v5, -1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v6, v6, -1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v7, v7, -1, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_uaddsat_v2i128:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v0, v8
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, v1, v9, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v2, vcc, v2, v10, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, v3, v11, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[2:3], v[10:11]
+; GFX8-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[10:11]
+; GFX8-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
+; GFX8-NEXT:    v_and_b32_e32 v8, 1, v8
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v0, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v1, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, -1, vcc
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v4, v12
+; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, v5, v13, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v6, vcc, v6, v14, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v7, vcc, v7, v15, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[4:5], v[12:13]
+; GFX8-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[14:15]
+; GFX8-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[14:15]
+; GFX8-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
+; GFX8-NEXT:    v_and_b32_e32 v8, 1, v8
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, v4, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, v5, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v6, v6, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v7, v7, -1, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uaddsat_v2i128:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v9, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, v2, v10, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v11, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[0:1], v[8:9]
+; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[2:3], v[10:11]
+; GFX9-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[10:11]
+; GFX9-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
+; GFX9-NEXT:    v_and_b32_e32 v8, 1, v8
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, -1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v4, v12
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v5, v13, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, v6, v14, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, v7, v15, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[4:5], v[12:13]
+; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[14:15]
+; GFX9-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[14:15]
+; GFX9-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc
+; GFX9-NEXT:    v_and_b32_e32 v8, 1, v8
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v5, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, v6, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v7, -1, vcc
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_uaddsat_v2i128:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_mov_b32_e32 v18, v8
+; GFX10-NEXT:    v_mov_b32_e32 v19, v9
+; GFX10-NEXT:    v_mov_b32_e32 v16, v10
+; GFX10-NEXT:    v_mov_b32_e32 v17, v11
+; GFX10-NEXT:    v_mov_b32_e32 v10, v12
+; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, v0, v18
+; GFX10-NEXT:    v_mov_b32_e32 v11, v13
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v1, v19, vcc_lo
+; GFX10-NEXT:    v_mov_b32_e32 v20, v14
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, v2, v16, vcc_lo
+; GFX10-NEXT:    v_mov_b32_e32 v21, v15
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, v3, v17, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[18:19]
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc_lo
+; GFX10-NEXT:    v_add_co_u32_e64 v4, vcc_lo, v4, v10
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, v5, v11, vcc_lo
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, v6, v20, vcc_lo
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v7, v21, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[2:3], v[16:17]
+; GFX10-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[4:5], v[10:11]
+; GFX10-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[6:7], v[20:21]
+; GFX10-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[16:17]
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v9, v8, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[20:21]
+; GFX10-NEXT:    v_and_b32_e32 v8, 1, v8
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v13, v12, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v8
+; GFX10-NEXT:    v_and_b32_e32 v9, 1, v9
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, -1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, -1, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s4, 0, v9
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, -1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, -1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, v4, -1, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, -1, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, -1, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v7, v7, -1, s4
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i128> @llvm.uadd.sat.v2i128(<2 x i128> %lhs, <2 x i128> %rhs)
+  ret <2 x i128> %result
+}
+
+define amdgpu_ps <2 x i128> @s_uaddsat_v2i128(<2 x i128> inreg %lhs, <2 x i128> inreg %rhs) {
+; GFX6-LABEL: s_uaddsat_v2i128:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_add_u32 s0, s0, s8
+; GFX6-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX6-NEXT:    s_and_b32 s16, s16, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX6-NEXT:    s_addc_u32 s1, s1, s9
+; GFX6-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX6-NEXT:    s_and_b32 s16, s16, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX6-NEXT:    s_addc_u32 s2, s2, s10
+; GFX6-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX6-NEXT:    v_mov_b32_e32 v2, s8
+; GFX6-NEXT:    s_and_b32 s16, s16, 1
+; GFX6-NEXT:    v_mov_b32_e32 v3, s9
+; GFX6-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX6-NEXT:    v_mov_b32_e32 v0, s10
+; GFX6-NEXT:    s_addc_u32 s3, s3, s11
+; GFX6-NEXT:    v_mov_b32_e32 v1, s11
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, s[2:3], v[0:1]
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_add_u32 s0, s4, s12
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v2, s1
+; GFX6-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX6-NEXT:    s_and_b32 s1, s1, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX6-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX6-NEXT:    s_addc_u32 s1, s5, s13
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX6-NEXT:    s_and_b32 s2, s2, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX6-NEXT:    s_addc_u32 s2, s6, s14
+; GFX6-NEXT:    v_cndmask_b32_e64 v4, v1, -1, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v1, s3
+; GFX6-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX6-NEXT:    v_cndmask_b32_e64 v5, v2, -1, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v2, s12
+; GFX6-NEXT:    s_and_b32 s3, s3, 1
+; GFX6-NEXT:    v_mov_b32_e32 v3, s13
+; GFX6-NEXT:    s_cmp_lg_u32 s3, 0
+; GFX6-NEXT:    v_cndmask_b32_e64 v6, v0, -1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v7, v1, -1, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX6-NEXT:    v_mov_b32_e32 v0, s14
+; GFX6-NEXT:    s_addc_u32 s3, s7, s15
+; GFX6-NEXT:    v_mov_b32_e32 v1, s15
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, s[2:3], v[0:1]
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX6-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v2, s1
+; GFX6-NEXT:    v_mov_b32_e32 v3, s3
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v1, -1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v2, -1, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v2, s2
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, v2, -1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, v3, -1, vcc
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX6-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX6-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX6-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX6-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX6-NEXT:    v_readfirstlane_b32 s6, v2
+; GFX6-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_uaddsat_v2i128:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_add_u32 s0, s0, s8
+; GFX8-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX8-NEXT:    s_and_b32 s16, s16, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX8-NEXT:    s_addc_u32 s1, s1, s9
+; GFX8-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX8-NEXT:    s_and_b32 s16, s16, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX8-NEXT:    s_addc_u32 s2, s2, s10
+; GFX8-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX8-NEXT:    s_and_b32 s16, s16, 1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s8
+; GFX8-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX8-NEXT:    v_mov_b32_e32 v3, s9
+; GFX8-NEXT:    s_addc_u32 s3, s3, s11
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s10
+; GFX8-NEXT:    v_mov_b32_e32 v1, s11
+; GFX8-NEXT:    s_cmp_eq_u64 s[2:3], s[10:11]
+; GFX8-NEXT:    s_cselect_b32 s10, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
+; GFX8-NEXT:    s_and_b32 s8, 1, s10
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s8
+; GFX8-NEXT:    v_mov_b32_e32 v1, s0
+; GFX8-NEXT:    s_add_u32 s0, s4, s12
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX8-NEXT:    s_and_b32 s1, s1, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX8-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX8-NEXT:    s_addc_u32 s1, s5, s13
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX8-NEXT:    s_and_b32 s2, s2, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX8-NEXT:    s_addc_u32 s2, s6, s14
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, v1, -1, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX8-NEXT:    s_and_b32 s3, s3, 1
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, v2, -1, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v2, s12
+; GFX8-NEXT:    s_cmp_lg_u32 s3, 0
+; GFX8-NEXT:    v_mov_b32_e32 v3, s13
+; GFX8-NEXT:    s_addc_u32 s3, s7, s15
+; GFX8-NEXT:    v_cndmask_b32_e64 v6, v0, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v7, v1, -1, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s14
+; GFX8-NEXT:    s_cmp_eq_u64 s[2:3], s[14:15]
+; GFX8-NEXT:    v_mov_b32_e32 v1, s15
+; GFX8-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
+; GFX8-NEXT:    s_and_b32 s4, 1, s4
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s0
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    v_mov_b32_e32 v3, s3
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v1, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v2, -1, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, -1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, -1, vcc
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX8-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s6, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_uaddsat_v2i128:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_add_u32 s0, s0, s8
+; GFX9-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX9-NEXT:    s_and_b32 s16, s16, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX9-NEXT:    s_addc_u32 s1, s1, s9
+; GFX9-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX9-NEXT:    s_and_b32 s16, s16, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX9-NEXT:    s_addc_u32 s2, s2, s10
+; GFX9-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX9-NEXT:    s_and_b32 s16, s16, 1
+; GFX9-NEXT:    v_mov_b32_e32 v2, s8
+; GFX9-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX9-NEXT:    v_mov_b32_e32 v3, s9
+; GFX9-NEXT:    s_addc_u32 s3, s3, s11
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v0, s10
+; GFX9-NEXT:    v_mov_b32_e32 v1, s11
+; GFX9-NEXT:    s_cmp_eq_u64 s[2:3], s[10:11]
+; GFX9-NEXT:    s_cselect_b32 s10, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
+; GFX9-NEXT:    s_and_b32 s8, 1, s10
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s8
+; GFX9-NEXT:    v_mov_b32_e32 v1, s0
+; GFX9-NEXT:    s_add_u32 s0, s4, s12
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX9-NEXT:    s_and_b32 s1, s1, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    s_addc_u32 s1, s5, s13
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX9-NEXT:    s_and_b32 s2, s2, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX9-NEXT:    s_addc_u32 s2, s6, s14
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v1, -1, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX9-NEXT:    s_and_b32 s3, s3, 1
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v2, -1, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v2, s12
+; GFX9-NEXT:    s_cmp_lg_u32 s3, 0
+; GFX9-NEXT:    v_mov_b32_e32 v3, s13
+; GFX9-NEXT:    s_addc_u32 s3, s7, s15
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, v0, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v1, -1, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v0, s14
+; GFX9-NEXT:    s_cmp_eq_u64 s[2:3], s[14:15]
+; GFX9-NEXT:    v_mov_b32_e32 v1, s15
+; GFX9-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
+; GFX9-NEXT:    s_and_b32 s4, 1, s4
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s0
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v1, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v2, -1, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v2, s2
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, -1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, -1, vcc
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX9-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s6, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_uaddsat_v2i128:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_add_u32 s0, s0, s8
+; GFX10-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_and_b32 s16, s16, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX10-NEXT:    s_addc_u32 s1, s1, s9
+; GFX10-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s8, s[0:1], s[8:9]
+; GFX10-NEXT:    s_and_b32 s16, s16, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX10-NEXT:    s_addc_u32 s2, s2, s10
+; GFX10-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s8
+; GFX10-NEXT:    s_and_b32 s16, s16, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s16, 0
+; GFX10-NEXT:    s_addc_u32 s3, s3, s11
+; GFX10-NEXT:    s_cmp_eq_u64 s[2:3], s[10:11]
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s10, s[2:3], s[10:11]
+; GFX10-NEXT:    s_cselect_b32 s16, 1, 0
+; GFX10-NEXT:    s_and_b32 s8, 1, s16
+; GFX10-NEXT:    s_add_u32 s4, s4, s12
+; GFX10-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s8
+; GFX10-NEXT:    s_and_b32 s9, s9, 1
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s10
+; GFX10-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX10-NEXT:    s_addc_u32 s5, s5, s13
+; GFX10-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX10-NEXT:    s_and_b32 s9, s9, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s9, s[4:5], s[12:13]
+; GFX10-NEXT:    s_addc_u32 s6, s6, s14
+; GFX10-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX10-NEXT:    s_and_b32 s8, s8, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s9
+; GFX10-NEXT:    s_addc_u32 s7, s7, s15
+; GFX10-NEXT:    s_cmp_eq_u64 s[6:7], s[14:15]
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s9, s[6:7], s[14:15]
+; GFX10-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX10-NEXT:    s_and_b32 s8, 1, s8
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s8
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s9
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v1
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, s0, -1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, s1, -1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, s2, -1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, s3, -1, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v3
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v4
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, s4, -1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, s5, -1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, s6, -1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, s7, -1, vcc_lo
+; GFX10-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s6, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i128> @llvm.uadd.sat.v2i128(<2 x i128> %lhs, <2 x i128> %rhs)
+  ret <2 x i128> %result
+}
+
+declare i7 @llvm.uadd.sat.i7(i7, i7) #0
+declare i8 @llvm.uadd.sat.i8(i8, i8) #0
+declare <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8>, <2 x i8>) #0
+declare <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8>, <4 x i8>) #0
+
+declare i16 @llvm.uadd.sat.i16(i16, i16) #0
+declare <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16>, <2 x i16>) #0
+declare <3 x i16> @llvm.uadd.sat.v3i16(<3 x i16>, <3 x i16>) #0
+declare <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16>, <4 x i16>) #0
+declare <5 x i16> @llvm.uadd.sat.v5i16(<5 x i16>, <5 x i16>) #0
+declare <6 x i16> @llvm.uadd.sat.v6i16(<6 x i16>, <6 x i16>) #0
+declare <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16>, <8 x i16>) #0
+
+declare i24 @llvm.uadd.sat.i24(i24, i24) #0
+
+declare i32 @llvm.uadd.sat.i32(i32, i32) #0
+declare <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32>, <2 x i32>) #0
+declare <3 x i32> @llvm.uadd.sat.v3i32(<3 x i32>, <3 x i32>) #0
+declare <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32>, <4 x i32>) #0
+declare <5 x i32> @llvm.uadd.sat.v5i32(<5 x i32>, <5 x i32>) #0
+declare <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32>, <16 x i32>) #0
+
+declare i48 @llvm.uadd.sat.i48(i48, i48) #0
+
+declare i64 @llvm.uadd.sat.i64(i64, i64) #0
+declare <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64>, <2 x i64>) #0
+
+declare i128 @llvm.uadd.sat.i128(i128, i128) #0
+declare <2 x i128> @llvm.uadd.sat.v2i128(<2 x i128>, <2 x i128>) #0
+
+attributes #0 = { nounwind readnone speculatable willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
new file mode 100644
index 0000000000000..b111fd31851cf
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
@@ -0,0 +1,4440 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=tahiti -o - %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=fiji -o - %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -o - %s | FileCheck -check-prefix=GFX10 %s
+
+define i7 @v_usubsat_i7(i7 %lhs, i7 %rhs) {
+; GFX6-LABEL: v_usubsat_i7:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 25, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 25, v1
+; GFX6-NEXT:    v_min_u32_e32 v1, v0, v1
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 25, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_i7:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 9, v0
+; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 9, v1
+; GFX8-NEXT:    v_min_u16_e32 v1, v0, v1
+; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_lshrrev_b16_e32 v0, 9, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_i7:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 9, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 9, v1
+; GFX9-NEXT:    v_min_u16_e32 v1, v0, v1
+; GFX9-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 9, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_usubsat_i7:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 9, v0
+; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 9, v1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_min_u16_e64 v1, v0, v1
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    v_lshrrev_b16_e64 v0, 9, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i7 @llvm.usub.sat.i7(i7 %lhs, i7 %rhs)
+  ret i7 %result
+}
+
+define amdgpu_ps i7 @s_usubsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
+; GFX6-LABEL: s_usubsat_i7:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 25
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 25
+; GFX6-NEXT:    s_cmp_lt_u32 s0, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s0, s1
+; GFX6-NEXT:    s_sub_i32 s0, s0, s1
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 25
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_usubsat_i7:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_bfe_u32 s2, 9, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX8-NEXT:    s_bfe_u32 s3, s0, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s3, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX8-NEXT:    s_sub_i32 s0, s0, s1
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_usubsat_i7:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_bfe_u32 s2, 9, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX9-NEXT:    s_bfe_u32 s3, s0, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_cmp_lt_u32 s3, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX9-NEXT:    s_sub_i32 s0, s0, s1
+; GFX9-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_usubsat_i7:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_bfe_u32 s2, 9, 0x100000
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s3, s0, 0x100000
+; GFX10-NEXT:    s_cmp_lt_u32 s3, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX10-NEXT:    s_sub_i32 s0, s0, s1
+; GFX10-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i7 @llvm.usub.sat.i7(i7 %lhs, i7 %rhs)
+  ret i7 %result
+}
+
+define i8 @v_usubsat_i8(i8 %lhs, i8 %rhs) {
+; GFX6-LABEL: v_usubsat_i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
+; GFX6-NEXT:    v_min_u32_e32 v1, v0, v1
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX8-NEXT:    v_min_u16_e32 v1, v0, v1
+; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-NEXT:    v_min_u16_e32 v1, v0, v1
+; GFX9-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_usubsat_i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 8, v0
+; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 8, v1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_min_u16_e64 v1, v0, v1
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    v_lshrrev_b16_e64 v0, 8, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i8 @llvm.usub.sat.i8(i8 %lhs, i8 %rhs)
+  ret i8 %result
+}
+
+define amdgpu_ps i8 @s_usubsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
+; GFX6-LABEL: s_usubsat_i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 24
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX6-NEXT:    s_cmp_lt_u32 s0, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s0, s1
+; GFX6-NEXT:    s_sub_i32 s0, s0, s1
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 24
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_usubsat_i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_bfe_u32 s2, 8, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX8-NEXT:    s_bfe_u32 s3, s0, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s3, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX8-NEXT:    s_sub_i32 s0, s0, s1
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_usubsat_i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_bfe_u32 s2, 8, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX9-NEXT:    s_bfe_u32 s3, s0, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_cmp_lt_u32 s3, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX9-NEXT:    s_sub_i32 s0, s0, s1
+; GFX9-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_usubsat_i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_bfe_u32 s2, 8, 0x100000
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s3, s0, 0x100000
+; GFX10-NEXT:    s_cmp_lt_u32 s3, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX10-NEXT:    s_sub_i32 s0, s0, s1
+; GFX10-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i8 @llvm.usub.sat.i8(i8 %lhs, i8 %rhs)
+  ret i8 %result
+}
+
+define i16 @v_usubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
+; GFX6-LABEL: v_usubsat_v2i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 8, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 8, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
+; GFX6-NEXT:    v_min_u32_e32 v1, v0, v1
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 24, v3
+; GFX6-NEXT:    v_min_u32_e32 v2, v1, v2
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 24, v1
+; GFX6-NEXT:    v_mov_b32_e32 v2, 0xff
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, v1, v2
+; GFX6-NEXT:    v_and_b32_e32 v0, v0, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_v2i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v2, 8
+; GFX8-NEXT:    v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX8-NEXT:    v_min_u16_e32 v1, v0, v1
+; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_min_u16_e32 v1, v3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0xff
+; GFX8-NEXT:    v_sub_u16_e32 v1, v3, v1
+; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v1, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_v2i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-NEXT:    v_min_u16_e32 v1, v0, v1
+; GFX9-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_min_u16_e32 v1, v2, v3
+; GFX9-NEXT:    s_movk_i32 s4, 0xff
+; GFX9-NEXT:    v_sub_u16_e32 v1, v2, v1
+; GFX9-NEXT:    v_and_b32_sdwa v0, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v1, v1, s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_usubsat_v2i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_mov_b32 s4, 8
+; GFX10-NEXT:    v_lshlrev_b16_e64 v3, 8, v1
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v1, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 8, v0
+; GFX10-NEXT:    s_movk_i32 s4, 0xff
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_min_u16_e64 v1, v2, v1
+; GFX10-NEXT:    v_min_u16_e64 v3, v0, v3
+; GFX10-NEXT:    v_sub_nc_u16_e64 v1, v2, v1
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v3
+; GFX10-NEXT:    v_and_b32_sdwa v1, v1, s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v0, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %lhs = bitcast i16 %lhs.arg to <2 x i8>
+  %rhs = bitcast i16 %rhs.arg to <2 x i8>
+  %result = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %lhs, <2 x i8> %rhs)
+  %cast.result = bitcast <2 x i8> %result to i16
+  ret i16 %cast.result
+}
+
+define amdgpu_ps i16 @s_usubsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
+; GFX6-LABEL: s_usubsat_v2i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX6-NEXT:    s_lshr_b32 s3, s1, 8
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 24
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX6-NEXT:    s_cmp_lt_u32 s0, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s0, s1
+; GFX6-NEXT:    s_sub_i32 s0, s0, s1
+; GFX6-NEXT:    s_lshl_b32 s1, s2, 24
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 24
+; GFX6-NEXT:    s_lshl_b32 s2, s3, 24
+; GFX6-NEXT:    s_cmp_lt_u32 s1, s2
+; GFX6-NEXT:    s_cselect_b32 s2, s1, s2
+; GFX6-NEXT:    s_sub_i32 s1, s1, s2
+; GFX6-NEXT:    s_movk_i32 s2, 0xff
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 24
+; GFX6-NEXT:    s_and_b32 s1, s1, s2
+; GFX6-NEXT:    s_and_b32 s0, s0, s2
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_usubsat_v2i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_bfe_u32 s4, 8, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX8-NEXT:    s_lshr_b32 s3, s1, 8
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s4
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s4
+; GFX8-NEXT:    s_bfe_u32 s5, s0, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s5, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s5, s1
+; GFX8-NEXT:    s_sub_i32 s0, s0, s1
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s1, s2, s4
+; GFX8-NEXT:    s_lshl_b32 s2, s3, s4
+; GFX8-NEXT:    s_lshr_b32 s0, s0, s4
+; GFX8-NEXT:    s_bfe_u32 s3, s1, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s3, s2
+; GFX8-NEXT:    s_cselect_b32 s2, s3, s2
+; GFX8-NEXT:    s_sub_i32 s1, s1, s2
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_movk_i32 s2, 0xff
+; GFX8-NEXT:    s_lshr_b32 s1, s1, s4
+; GFX8-NEXT:    s_and_b32 s1, s1, s2
+; GFX8-NEXT:    s_and_b32 s0, s0, s2
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s4
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_usubsat_v2i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_bfe_u32 s4, 8, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX9-NEXT:    s_lshr_b32 s3, s1, 8
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s4
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s4
+; GFX9-NEXT:    s_bfe_u32 s5, s0, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_cmp_lt_u32 s5, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s5, s1
+; GFX9-NEXT:    s_sub_i32 s0, s0, s1
+; GFX9-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s1, s2, s4
+; GFX9-NEXT:    s_lshl_b32 s2, s3, s4
+; GFX9-NEXT:    s_lshr_b32 s0, s0, s4
+; GFX9-NEXT:    s_bfe_u32 s3, s1, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX9-NEXT:    s_cmp_lt_u32 s3, s2
+; GFX9-NEXT:    s_cselect_b32 s2, s3, s2
+; GFX9-NEXT:    s_sub_i32 s1, s1, s2
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_movk_i32 s2, 0xff
+; GFX9-NEXT:    s_lshr_b32 s1, s1, s4
+; GFX9-NEXT:    s_and_b32 s1, s1, s2
+; GFX9-NEXT:    s_and_b32 s0, s0, s2
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s4
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_usubsat_v2i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_bfe_u32 s2, 8, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s3, s0, 8
+; GFX10-NEXT:    s_lshl_b32 s4, s1, s2
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
+; GFX10-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s5, s0, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s1, s1, 8
+; GFX10-NEXT:    s_cmp_lt_u32 s5, s4
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX10-NEXT:    s_sub_i32 s0, s0, s4
+; GFX10-NEXT:    s_lshl_b32 s3, s3, s2
+; GFX10-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s4, s3, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s0, s0, s2
+; GFX10-NEXT:    s_cmp_lt_u32 s4, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s4, s1
+; GFX10-NEXT:    s_sub_i32 s1, s3, s1
+; GFX10-NEXT:    s_movk_i32 s3, 0xff
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_and_b32 s0, s0, s3
+; GFX10-NEXT:    s_lshr_b32 s1, s1, s2
+; GFX10-NEXT:    s_and_b32 s1, s1, s3
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %lhs = bitcast i16 %lhs.arg to <2 x i8>
+  %rhs = bitcast i16 %rhs.arg to <2 x i8>
+  %result = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %lhs, <2 x i8> %rhs)
+  %cast.result = bitcast <2 x i8> %result to i16
+  ret i16 %cast.result
+}
+
+define i32 @v_usubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
+; GFX6-LABEL: v_usubsat_v4i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 8, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 24, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v5, 8, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 24, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
+; GFX6-NEXT:    v_min_u32_e32 v1, v0, v1
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 24, v5
+; GFX6-NEXT:    v_min_u32_e32 v2, v1, v2
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 24, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 24, v6
+; GFX6-NEXT:    v_min_u32_e32 v3, v2, v3
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 24, v1
+; GFX6-NEXT:    s_movk_i32 s4, 0xff
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v3
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 24, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 24, v7
+; GFX6-NEXT:    v_min_u32_e32 v4, v3, v4
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 24, v2
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, v3, v4
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 24, v3
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 24, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_v4i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v2, 8
+; GFX8-NEXT:    v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 24, v0
+; GFX8-NEXT:    v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
+; GFX8-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX8-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX8-NEXT:    v_min_u16_e32 v1, v0, v1
+; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX8-NEXT:    v_min_u16_e32 v1, v3, v2
+; GFX8-NEXT:    v_sub_u16_e32 v1, v3, v1
+; GFX8-NEXT:    v_lshlrev_b16_e32 v2, 8, v4
+; GFX8-NEXT:    v_lshlrev_b16_e32 v3, 8, v6
+; GFX8-NEXT:    v_min_u16_e32 v3, v2, v3
+; GFX8-NEXT:    v_sub_u16_e32 v2, v2, v3
+; GFX8-NEXT:    v_lshlrev_b16_e32 v3, 8, v5
+; GFX8-NEXT:    v_lshlrev_b16_e32 v4, 8, v7
+; GFX8-NEXT:    v_min_u16_e32 v4, v3, v4
+; GFX8-NEXT:    v_sub_u16_e32 v3, v3, v4
+; GFX8-NEXT:    v_mov_b32_e32 v4, 0xff
+; GFX8-NEXT:    v_and_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_and_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_and_b32_sdwa v1, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    v_and_b32_sdwa v1, v3, v4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_v4i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s4, 8
+; GFX9-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_lshrrev_b32_sdwa v5, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 24, v0
+; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-NEXT:    v_min_u16_e32 v1, v0, v1
+; GFX9-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_min_u16_e32 v1, v2, v5
+; GFX9-NEXT:    v_sub_u16_e32 v1, v2, v1
+; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v3
+; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v6
+; GFX9-NEXT:    v_min_u16_e32 v3, v2, v3
+; GFX9-NEXT:    v_sub_u16_e32 v2, v2, v3
+; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v4
+; GFX9-NEXT:    v_lshlrev_b16_e32 v4, 8, v7
+; GFX9-NEXT:    s_movk_i32 s4, 0xff
+; GFX9-NEXT:    v_min_u16_e32 v4, v3, v4
+; GFX9-NEXT:    v_and_b32_sdwa v1, v1, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 8, v0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX9-NEXT:    v_sub_u16_e32 v3, v3, v4
+; GFX9-NEXT:    v_and_or_b32 v0, v0, s4, v1
+; GFX9-NEXT:    v_and_b32_sdwa v1, v2, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v2, v3, s4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_usubsat_v4i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_mov_b32 s4, 8
+; GFX10-NEXT:    v_lshlrev_b16_e64 v4, 8, v0
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v2, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshlrev_b16_e64 v5, 8, v1
+; GFX10-NEXT:    s_mov_b32 s4, 16
+; GFX10-NEXT:    s_mov_b32 s5, 24
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v6, s4, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_min_u16_e64 v3, v2, v3
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v7, s4, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_min_u16_e64 v5, v4, v5
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v0, s5, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v1, s5, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_sub_nc_u16_e64 v2, v2, v3
+; GFX10-NEXT:    s_movk_i32 s4, 0xff
+; GFX10-NEXT:    v_min_u16_e64 v3, v6, v7
+; GFX10-NEXT:    v_sub_nc_u16_e64 v4, v4, v5
+; GFX10-NEXT:    v_min_u16_e64 v1, v0, v1
+; GFX10-NEXT:    v_and_b32_sdwa v2, v2, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_sub_nc_u16_e64 v3, v6, v3
+; GFX10-NEXT:    v_lshrrev_b16_e64 v4, 8, v4
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX10-NEXT:    v_and_b32_sdwa v1, v3, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v0, v0, s4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_or_b32 v2, v4, s4, v2
+; GFX10-NEXT:    v_or3_b32 v0, v2, v1, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %lhs = bitcast i32 %lhs.arg to <4 x i8>
+  %rhs = bitcast i32 %rhs.arg to <4 x i8>
+  %result = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> %lhs, <4 x i8> %rhs)
+  %cast.result = bitcast <4 x i8> %result to i32
+  ret i32 %cast.result
+}
+
+define amdgpu_ps i32 @s_usubsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
+; GFX6-LABEL: s_usubsat_v4i8:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX6-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX6-NEXT:    s_lshr_b32 s4, s0, 24
+; GFX6-NEXT:    s_lshr_b32 s5, s1, 8
+; GFX6-NEXT:    s_lshr_b32 s6, s1, 16
+; GFX6-NEXT:    s_lshr_b32 s7, s1, 24
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 24
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX6-NEXT:    s_cmp_lt_u32 s0, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s0, s1
+; GFX6-NEXT:    s_sub_i32 s0, s0, s1
+; GFX6-NEXT:    s_lshl_b32 s1, s2, 24
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 24
+; GFX6-NEXT:    s_lshl_b32 s2, s5, 24
+; GFX6-NEXT:    s_cmp_lt_u32 s1, s2
+; GFX6-NEXT:    s_cselect_b32 s2, s1, s2
+; GFX6-NEXT:    s_sub_i32 s1, s1, s2
+; GFX6-NEXT:    s_lshl_b32 s2, s3, 24
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 24
+; GFX6-NEXT:    s_lshl_b32 s3, s6, 24
+; GFX6-NEXT:    s_cmp_lt_u32 s2, s3
+; GFX6-NEXT:    s_cselect_b32 s3, s2, s3
+; GFX6-NEXT:    s_sub_i32 s2, s2, s3
+; GFX6-NEXT:    s_lshl_b32 s3, s4, 24
+; GFX6-NEXT:    s_lshr_b32 s2, s2, 24
+; GFX6-NEXT:    s_lshl_b32 s4, s7, 24
+; GFX6-NEXT:    s_cmp_lt_u32 s3, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s3, s4
+; GFX6-NEXT:    s_sub_i32 s3, s3, s4
+; GFX6-NEXT:    s_movk_i32 s4, 0xff
+; GFX6-NEXT:    s_and_b32 s1, s1, s4
+; GFX6-NEXT:    s_and_b32 s0, s0, s4
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_and_b32 s1, s2, s4
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_lshr_b32 s3, s3, 24
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_and_b32 s1, s3, s4
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_usubsat_v4i8:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_bfe_u32 s8, 8, 0x100000
+; GFX8-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX8-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s4, s0, 24
+; GFX8-NEXT:    s_lshl_b32 s0, s0, s8
+; GFX8-NEXT:    s_lshr_b32 s5, s1, 8
+; GFX8-NEXT:    s_lshr_b32 s6, s1, 16
+; GFX8-NEXT:    s_lshr_b32 s7, s1, 24
+; GFX8-NEXT:    s_lshl_b32 s1, s1, s8
+; GFX8-NEXT:    s_bfe_u32 s9, s0, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s9, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s9, s1
+; GFX8-NEXT:    s_sub_i32 s0, s0, s1
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s1, s2, s8
+; GFX8-NEXT:    s_lshl_b32 s2, s5, s8
+; GFX8-NEXT:    s_lshr_b32 s0, s0, s8
+; GFX8-NEXT:    s_bfe_u32 s5, s1, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s5, s2
+; GFX8-NEXT:    s_cselect_b32 s2, s5, s2
+; GFX8-NEXT:    s_sub_i32 s1, s1, s2
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s2, s3, s8
+; GFX8-NEXT:    s_lshl_b32 s3, s6, s8
+; GFX8-NEXT:    s_lshr_b32 s1, s1, s8
+; GFX8-NEXT:    s_bfe_u32 s5, s2, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s5, s3
+; GFX8-NEXT:    s_cselect_b32 s3, s5, s3
+; GFX8-NEXT:    s_sub_i32 s2, s2, s3
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s3, s4, s8
+; GFX8-NEXT:    s_lshl_b32 s4, s7, s8
+; GFX8-NEXT:    s_lshr_b32 s2, s2, s8
+; GFX8-NEXT:    s_bfe_u32 s5, s3, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s5, s4
+; GFX8-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX8-NEXT:    s_sub_i32 s3, s3, s4
+; GFX8-NEXT:    s_movk_i32 s4, 0xff
+; GFX8-NEXT:    s_and_b32 s1, s1, s4
+; GFX8-NEXT:    s_and_b32 s0, s0, s4
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    s_and_b32 s1, s2, s4
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX8-NEXT:    s_lshr_b32 s3, s3, s8
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    s_and_b32 s1, s3, s4
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_usubsat_v4i8:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_bfe_u32 s8, 8, 0x100000
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX9-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX9-NEXT:    s_lshr_b32 s4, s0, 24
+; GFX9-NEXT:    s_lshl_b32 s0, s0, s8
+; GFX9-NEXT:    s_lshr_b32 s5, s1, 8
+; GFX9-NEXT:    s_lshr_b32 s6, s1, 16
+; GFX9-NEXT:    s_lshr_b32 s7, s1, 24
+; GFX9-NEXT:    s_lshl_b32 s1, s1, s8
+; GFX9-NEXT:    s_bfe_u32 s9, s0, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_cmp_lt_u32 s9, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s9, s1
+; GFX9-NEXT:    s_sub_i32 s0, s0, s1
+; GFX9-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s1, s2, s8
+; GFX9-NEXT:    s_lshl_b32 s2, s5, s8
+; GFX9-NEXT:    s_lshr_b32 s0, s0, s8
+; GFX9-NEXT:    s_bfe_u32 s5, s1, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX9-NEXT:    s_cmp_lt_u32 s5, s2
+; GFX9-NEXT:    s_cselect_b32 s2, s5, s2
+; GFX9-NEXT:    s_sub_i32 s1, s1, s2
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s2, s3, s8
+; GFX9-NEXT:    s_lshl_b32 s3, s6, s8
+; GFX9-NEXT:    s_lshr_b32 s1, s1, s8
+; GFX9-NEXT:    s_bfe_u32 s5, s2, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX9-NEXT:    s_cmp_lt_u32 s5, s3
+; GFX9-NEXT:    s_cselect_b32 s3, s5, s3
+; GFX9-NEXT:    s_sub_i32 s2, s2, s3
+; GFX9-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX9-NEXT:    s_lshl_b32 s3, s4, s8
+; GFX9-NEXT:    s_lshl_b32 s4, s7, s8
+; GFX9-NEXT:    s_lshr_b32 s2, s2, s8
+; GFX9-NEXT:    s_bfe_u32 s5, s3, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX9-NEXT:    s_cmp_lt_u32 s5, s4
+; GFX9-NEXT:    s_cselect_b32 s4, s5, s4
+; GFX9-NEXT:    s_sub_i32 s3, s3, s4
+; GFX9-NEXT:    s_movk_i32 s4, 0xff
+; GFX9-NEXT:    s_and_b32 s1, s1, s4
+; GFX9-NEXT:    s_and_b32 s0, s0, s4
+; GFX9-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX9-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    s_and_b32 s1, s2, s4
+; GFX9-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX9-NEXT:    s_lshr_b32 s3, s3, s8
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    s_and_b32 s1, s3, s4
+; GFX9-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX9-NEXT:    s_or_b32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_usubsat_v4i8:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_bfe_u32 s6, 8, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s2, s0, 8
+; GFX10-NEXT:    s_lshl_b32 s8, s1, s6
+; GFX10-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX10-NEXT:    s_lshr_b32 s4, s0, 24
+; GFX10-NEXT:    s_lshl_b32 s0, s0, s6
+; GFX10-NEXT:    s_bfe_u32 s8, s8, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s9, s0, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s5, s1, 8
+; GFX10-NEXT:    s_lshr_b32 s7, s1, 16
+; GFX10-NEXT:    s_lshr_b32 s1, s1, 24
+; GFX10-NEXT:    s_cmp_lt_u32 s9, s8
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s8, s9, s8
+; GFX10-NEXT:    s_lshl_b32 s5, s5, s6
+; GFX10-NEXT:    s_sub_i32 s0, s0, s8
+; GFX10-NEXT:    s_lshl_b32 s2, s2, s6
+; GFX10-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s8, s2, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s0, s0, s6
+; GFX10-NEXT:    s_cmp_lt_u32 s8, s5
+; GFX10-NEXT:    s_cselect_b32 s5, s8, s5
+; GFX10-NEXT:    s_lshl_b32 s3, s3, s6
+; GFX10-NEXT:    s_sub_i32 s2, s2, s5
+; GFX10-NEXT:    s_lshl_b32 s5, s7, s6
+; GFX10-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s7, s3, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s2, s2, s6
+; GFX10-NEXT:    s_cmp_lt_u32 s7, s5
+; GFX10-NEXT:    s_cselect_b32 s5, s7, s5
+; GFX10-NEXT:    s_lshl_b32 s1, s1, s6
+; GFX10-NEXT:    s_sub_i32 s3, s3, s5
+; GFX10-NEXT:    s_lshl_b32 s4, s4, s6
+; GFX10-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s5, s4, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_lshr_b32 s3, s3, s6
+; GFX10-NEXT:    s_cmp_lt_u32 s5, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s5, s1
+; GFX10-NEXT:    s_sub_i32 s1, s4, s1
+; GFX10-NEXT:    s_movk_i32 s4, 0xff
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    s_and_b32 s2, s2, s4
+; GFX10-NEXT:    s_lshr_b32 s1, s1, s6
+; GFX10-NEXT:    s_and_b32 s0, s0, s4
+; GFX10-NEXT:    s_lshl_b32 s2, s2, 8
+; GFX10-NEXT:    s_and_b32 s3, s3, s4
+; GFX10-NEXT:    s_and_b32 s1, s1, s4
+; GFX10-NEXT:    s_or_b32 s0, s0, s2
+; GFX10-NEXT:    s_lshl_b32 s2, s3, 16
+; GFX10-NEXT:    s_lshl_b32 s1, s1, 24
+; GFX10-NEXT:    s_or_b32 s0, s0, s2
+; GFX10-NEXT:    s_or_b32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %lhs = bitcast i32 %lhs.arg to <4 x i8>
+  %rhs = bitcast i32 %rhs.arg to <4 x i8>
+  %result = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> %lhs, <4 x i8> %rhs)
+  %cast.result = bitcast <4 x i8> %result to i32
+  ret i32 %cast.result
+}
+
+define i24 @v_usubsat_i24(i24 %lhs, i24 %rhs) {
+; GFX6-LABEL: v_usubsat_i24:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX6-NEXT:    v_min_u32_e32 v1, v0, v1
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_i24:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX8-NEXT:    v_min_u32_e32 v1, v0, v1
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_i24:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX9-NEXT:    v_min_u32_e32 v1, v0, v1
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v1
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_usubsat_i24:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_min_u32_e32 v1, v0, v1
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i24 @llvm.usub.sat.i24(i24 %lhs, i24 %rhs)
+  ret i24 %result
+}
+
+define amdgpu_ps i24 @s_usubsat_i24(i24 inreg %lhs, i24 inreg %rhs) {
+; GFX6-LABEL: s_usubsat_i24:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 8
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX6-NEXT:    s_cmp_lt_u32 s0, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s0, s1
+; GFX6-NEXT:    s_sub_i32 s0, s0, s1
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 8
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_usubsat_i24:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_lshl_b32 s0, s0, 8
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX8-NEXT:    s_cmp_lt_u32 s0, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s0, s1
+; GFX8-NEXT:    s_sub_i32 s0, s0, s1
+; GFX8-NEXT:    s_lshr_b32 s0, s0, 8
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_usubsat_i24:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_lshl_b32 s0, s0, 8
+; GFX9-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX9-NEXT:    s_cmp_lt_u32 s0, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s0, s1
+; GFX9-NEXT:    s_sub_i32 s0, s0, s1
+; GFX9-NEXT:    s_lshr_b32 s0, s0, 8
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_usubsat_i24:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_lshl_b32 s0, s0, 8
+; GFX10-NEXT:    s_lshl_b32 s1, s1, 8
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cmp_lt_u32 s0, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s0, s1
+; GFX10-NEXT:    s_sub_i32 s0, s0, s1
+; GFX10-NEXT:    s_lshr_b32 s0, s0, 8
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i24 @llvm.usub.sat.i24(i24 %lhs, i24 %rhs)
+  ret i24 %result
+}
+
+define i32 @v_usubsat_i32(i32 %lhs, i32 %rhs) {
+; GFX6-LABEL: v_usubsat_i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_min_u32_e32 v1, v0, v1
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_min_u32_e32 v1, v0, v1
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_min_u32_e32 v1, v0, v1
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_usubsat_i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_min_u32_e32 v1, v0, v1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i32 @llvm.usub.sat.i32(i32 %lhs, i32 %rhs)
+  ret i32 %result
+}
+
+define amdgpu_ps i32 @s_usubsat_i32(i32 inreg %lhs, i32 inreg %rhs) {
+; GFX6-LABEL: s_usubsat_i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_cmp_lt_u32 s0, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s0, s1
+; GFX6-NEXT:    s_sub_i32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_usubsat_i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_cmp_lt_u32 s0, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s0, s1
+; GFX8-NEXT:    s_sub_i32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_usubsat_i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_cmp_lt_u32 s0, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s0, s1
+; GFX9-NEXT:    s_sub_i32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_usubsat_i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_cmp_lt_u32 s0, s1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s1, s0, s1
+; GFX10-NEXT:    s_sub_i32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i32 @llvm.usub.sat.i32(i32 %lhs, i32 %rhs)
+  ret i32 %result
+}
+
+define amdgpu_ps float @usubsat_i32_sv(i32 inreg %lhs, i32 %rhs) {
+; GFX6-LABEL: usubsat_i32_sv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_min_u32_e32 v0, s0, v0
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: usubsat_i32_sv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_min_u32_e32 v0, s0, v0
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: usubsat_i32_sv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_min_u32_e32 v0, s0, v0
+; GFX9-NEXT:    v_sub_u32_e32 v0, s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: usubsat_i32_sv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_min_u32_e32 v0, s0, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i32 @llvm.usub.sat.i32(i32 %lhs, i32 %rhs)
+  %cast = bitcast i32 %result to float
+  ret float %cast
+}
+
+define amdgpu_ps float @usubsat_i32_vs(i32 %lhs, i32 inreg %rhs) {
+; GFX6-LABEL: usubsat_i32_vs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_min_u32_e32 v1, s0, v0
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: usubsat_i32_vs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_min_u32_e32 v1, s0, v0
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: usubsat_i32_vs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_min_u32_e32 v1, s0, v0
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: usubsat_i32_vs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_min_u32_e32 v1, s0, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i32 @llvm.usub.sat.i32(i32 %lhs, i32 %rhs)
+  %cast = bitcast i32 %result to float
+  ret float %cast
+}
+
+define <2 x i32> @v_usubsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
+; GFX6-LABEL: v_usubsat_v2i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_min_u32_e32 v2, v0, v2
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT:    v_min_u32_e32 v2, v1, v3
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_v2i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_min_u32_e32 v2, v0, v2
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, v0, v2
+; GFX8-NEXT:    v_min_u32_e32 v2, v1, v3
+; GFX8-NEXT:    v_sub_u32_e32 v1, vcc, v1, v2
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_v2i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_min_u32_e32 v2, v0, v2
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v2
+; GFX9-NEXT:    v_min_u32_e32 v2, v1, v3
+; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_usubsat_v2i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_min_u32_e32 v2, v0, v2
+; GFX10-NEXT:    v_min_u32_e32 v3, v1, v3
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v2
+; GFX10-NEXT:    v_sub_nc_u32_e32 v1, v1, v3
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> %lhs, <2 x i32> %rhs)
+  ret <2 x i32> %result
+}
+
+define amdgpu_ps <2 x i32> @s_usubsat_v2i32(<2 x i32> inreg %lhs, <2 x i32> inreg %rhs) {
+; GFX6-LABEL: s_usubsat_v2i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_cmp_lt_u32 s0, s2
+; GFX6-NEXT:    s_cselect_b32 s2, s0, s2
+; GFX6-NEXT:    s_sub_i32 s0, s0, s2
+; GFX6-NEXT:    s_cmp_lt_u32 s1, s3
+; GFX6-NEXT:    s_cselect_b32 s2, s1, s3
+; GFX6-NEXT:    s_sub_i32 s1, s1, s2
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_usubsat_v2i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_cmp_lt_u32 s0, s2
+; GFX8-NEXT:    s_cselect_b32 s2, s0, s2
+; GFX8-NEXT:    s_sub_i32 s0, s0, s2
+; GFX8-NEXT:    s_cmp_lt_u32 s1, s3
+; GFX8-NEXT:    s_cselect_b32 s2, s1, s3
+; GFX8-NEXT:    s_sub_i32 s1, s1, s2
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_usubsat_v2i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_cmp_lt_u32 s0, s2
+; GFX9-NEXT:    s_cselect_b32 s2, s0, s2
+; GFX9-NEXT:    s_sub_i32 s0, s0, s2
+; GFX9-NEXT:    s_cmp_lt_u32 s1, s3
+; GFX9-NEXT:    s_cselect_b32 s2, s1, s3
+; GFX9-NEXT:    s_sub_i32 s1, s1, s2
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_usubsat_v2i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_cmp_lt_u32 s0, s2
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s2, s0, s2
+; GFX10-NEXT:    s_sub_i32 s0, s0, s2
+; GFX10-NEXT:    s_cmp_lt_u32 s1, s3
+; GFX10-NEXT:    s_cselect_b32 s2, s1, s3
+; GFX10-NEXT:    s_sub_i32 s1, s1, s2
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> %lhs, <2 x i32> %rhs)
+  ret <2 x i32> %result
+}
+
+define <3 x i32> @v_usubsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
+; GFX6-LABEL: v_usubsat_v3i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_min_u32_e32 v3, v0, v3
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; GFX6-NEXT:    v_min_u32_e32 v3, v1, v4
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v3
+; GFX6-NEXT:    v_min_u32_e32 v3, v2, v5
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v3
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_v3i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_min_u32_e32 v3, v0, v3
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, v0, v3
+; GFX8-NEXT:    v_min_u32_e32 v3, v1, v4
+; GFX8-NEXT:    v_sub_u32_e32 v1, vcc, v1, v3
+; GFX8-NEXT:    v_min_u32_e32 v3, v2, v5
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, v2, v3
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_v3i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_min_u32_e32 v3, v0, v3
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v3
+; GFX9-NEXT:    v_min_u32_e32 v3, v1, v4
+; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v3
+; GFX9-NEXT:    v_min_u32_e32 v3, v2, v5
+; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v3
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_usubsat_v3i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_min_u32_e32 v3, v0, v3
+; GFX10-NEXT:    v_min_u32_e32 v4, v1, v4
+; GFX10-NEXT:    v_min_u32_e32 v5, v2, v5
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v3
+; GFX10-NEXT:    v_sub_nc_u32_e32 v1, v1, v4
+; GFX10-NEXT:    v_sub_nc_u32_e32 v2, v2, v5
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <3 x i32> @llvm.usub.sat.v3i32(<3 x i32> %lhs, <3 x i32> %rhs)
+  ret <3 x i32> %result
+}
+
+define amdgpu_ps <3 x i32> @s_usubsat_v3i32(<3 x i32> inreg %lhs, <3 x i32> inreg %rhs) {
+; GFX6-LABEL: s_usubsat_v3i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_cmp_lt_u32 s0, s3
+; GFX6-NEXT:    s_cselect_b32 s3, s0, s3
+; GFX6-NEXT:    s_sub_i32 s0, s0, s3
+; GFX6-NEXT:    s_cmp_lt_u32 s1, s4
+; GFX6-NEXT:    s_cselect_b32 s3, s1, s4
+; GFX6-NEXT:    s_sub_i32 s1, s1, s3
+; GFX6-NEXT:    s_cmp_lt_u32 s2, s5
+; GFX6-NEXT:    s_cselect_b32 s3, s2, s5
+; GFX6-NEXT:    s_sub_i32 s2, s2, s3
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_usubsat_v3i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_cmp_lt_u32 s0, s3
+; GFX8-NEXT:    s_cselect_b32 s3, s0, s3
+; GFX8-NEXT:    s_sub_i32 s0, s0, s3
+; GFX8-NEXT:    s_cmp_lt_u32 s1, s4
+; GFX8-NEXT:    s_cselect_b32 s3, s1, s4
+; GFX8-NEXT:    s_sub_i32 s1, s1, s3
+; GFX8-NEXT:    s_cmp_lt_u32 s2, s5
+; GFX8-NEXT:    s_cselect_b32 s3, s2, s5
+; GFX8-NEXT:    s_sub_i32 s2, s2, s3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_usubsat_v3i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_cmp_lt_u32 s0, s3
+; GFX9-NEXT:    s_cselect_b32 s3, s0, s3
+; GFX9-NEXT:    s_sub_i32 s0, s0, s3
+; GFX9-NEXT:    s_cmp_lt_u32 s1, s4
+; GFX9-NEXT:    s_cselect_b32 s3, s1, s4
+; GFX9-NEXT:    s_sub_i32 s1, s1, s3
+; GFX9-NEXT:    s_cmp_lt_u32 s2, s5
+; GFX9-NEXT:    s_cselect_b32 s3, s2, s5
+; GFX9-NEXT:    s_sub_i32 s2, s2, s3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_usubsat_v3i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_cmp_lt_u32 s0, s3
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s3, s0, s3
+; GFX10-NEXT:    s_sub_i32 s0, s0, s3
+; GFX10-NEXT:    s_cmp_lt_u32 s1, s4
+; GFX10-NEXT:    s_cselect_b32 s3, s1, s4
+; GFX10-NEXT:    s_sub_i32 s1, s1, s3
+; GFX10-NEXT:    s_cmp_lt_u32 s2, s5
+; GFX10-NEXT:    s_cselect_b32 s3, s2, s5
+; GFX10-NEXT:    s_sub_i32 s2, s2, s3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <3 x i32> @llvm.usub.sat.v3i32(<3 x i32> %lhs, <3 x i32> %rhs)
+  ret <3 x i32> %result
+}
+
+define <4 x i32> @v_usubsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
+; GFX6-LABEL: v_usubsat_v4i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_min_u32_e32 v4, v0, v4
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; GFX6-NEXT:    v_min_u32_e32 v4, v1, v5
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v4
+; GFX6-NEXT:    v_min_u32_e32 v4, v2, v6
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v4
+; GFX6-NEXT:    v_min_u32_e32 v4, v3, v7
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, v3, v4
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_v4i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_min_u32_e32 v4, v0, v4
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, v0, v4
+; GFX8-NEXT:    v_min_u32_e32 v4, v1, v5
+; GFX8-NEXT:    v_sub_u32_e32 v1, vcc, v1, v4
+; GFX8-NEXT:    v_min_u32_e32 v4, v2, v6
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, v2, v4
+; GFX8-NEXT:    v_min_u32_e32 v4, v3, v7
+; GFX8-NEXT:    v_sub_u32_e32 v3, vcc, v3, v4
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_v4i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_min_u32_e32 v4, v0, v4
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v4
+; GFX9-NEXT:    v_min_u32_e32 v4, v1, v5
+; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v4
+; GFX9-NEXT:    v_min_u32_e32 v4, v2, v6
+; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v4
+; GFX9-NEXT:    v_min_u32_e32 v4, v3, v7
+; GFX9-NEXT:    v_sub_u32_e32 v3, v3, v4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_usubsat_v4i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_min_u32_e32 v19, v2, v6
+; GFX10-NEXT:    v_min_u32_e32 v11, v0, v4
+; GFX10-NEXT:    v_min_u32_e32 v15, v1, v5
+; GFX10-NEXT:    v_min_u32_e32 v6, v3, v7
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_sub_nc_u32_e32 v2, v2, v19
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v11
+; GFX10-NEXT:    v_sub_nc_u32_e32 v1, v1, v15
+; GFX10-NEXT:    v_sub_nc_u32_e32 v3, v3, v6
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %lhs, <4 x i32> %rhs)
+  ret <4 x i32> %result
+}
+
+define amdgpu_ps <4 x i32> @s_usubsat_v4i32(<4 x i32> inreg %lhs, <4 x i32> inreg %rhs) {
+; GFX6-LABEL: s_usubsat_v4i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_cmp_lt_u32 s0, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s0, s4
+; GFX6-NEXT:    s_sub_i32 s0, s0, s4
+; GFX6-NEXT:    s_cmp_lt_u32 s1, s5
+; GFX6-NEXT:    s_cselect_b32 s4, s1, s5
+; GFX6-NEXT:    s_sub_i32 s1, s1, s4
+; GFX6-NEXT:    s_cmp_lt_u32 s2, s6
+; GFX6-NEXT:    s_cselect_b32 s4, s2, s6
+; GFX6-NEXT:    s_sub_i32 s2, s2, s4
+; GFX6-NEXT:    s_cmp_lt_u32 s3, s7
+; GFX6-NEXT:    s_cselect_b32 s4, s3, s7
+; GFX6-NEXT:    s_sub_i32 s3, s3, s4
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_usubsat_v4i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_cmp_lt_u32 s0, s4
+; GFX8-NEXT:    s_cselect_b32 s4, s0, s4
+; GFX8-NEXT:    s_sub_i32 s0, s0, s4
+; GFX8-NEXT:    s_cmp_lt_u32 s1, s5
+; GFX8-NEXT:    s_cselect_b32 s4, s1, s5
+; GFX8-NEXT:    s_sub_i32 s1, s1, s4
+; GFX8-NEXT:    s_cmp_lt_u32 s2, s6
+; GFX8-NEXT:    s_cselect_b32 s4, s2, s6
+; GFX8-NEXT:    s_sub_i32 s2, s2, s4
+; GFX8-NEXT:    s_cmp_lt_u32 s3, s7
+; GFX8-NEXT:    s_cselect_b32 s4, s3, s7
+; GFX8-NEXT:    s_sub_i32 s3, s3, s4
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_usubsat_v4i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_cmp_lt_u32 s0, s4
+; GFX9-NEXT:    s_cselect_b32 s4, s0, s4
+; GFX9-NEXT:    s_sub_i32 s0, s0, s4
+; GFX9-NEXT:    s_cmp_lt_u32 s1, s5
+; GFX9-NEXT:    s_cselect_b32 s4, s1, s5
+; GFX9-NEXT:    s_sub_i32 s1, s1, s4
+; GFX9-NEXT:    s_cmp_lt_u32 s2, s6
+; GFX9-NEXT:    s_cselect_b32 s4, s2, s6
+; GFX9-NEXT:    s_sub_i32 s2, s2, s4
+; GFX9-NEXT:    s_cmp_lt_u32 s3, s7
+; GFX9-NEXT:    s_cselect_b32 s4, s3, s7
+; GFX9-NEXT:    s_sub_i32 s3, s3, s4
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_usubsat_v4i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_cmp_lt_u32 s0, s4
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s4, s0, s4
+; GFX10-NEXT:    s_sub_i32 s0, s0, s4
+; GFX10-NEXT:    s_cmp_lt_u32 s1, s5
+; GFX10-NEXT:    s_cselect_b32 s4, s1, s5
+; GFX10-NEXT:    s_sub_i32 s1, s1, s4
+; GFX10-NEXT:    s_cmp_lt_u32 s2, s6
+; GFX10-NEXT:    s_cselect_b32 s4, s2, s6
+; GFX10-NEXT:    s_sub_i32 s2, s2, s4
+; GFX10-NEXT:    s_cmp_lt_u32 s3, s7
+; GFX10-NEXT:    s_cselect_b32 s4, s3, s7
+; GFX10-NEXT:    s_sub_i32 s3, s3, s4
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %lhs, <4 x i32> %rhs)
+  ret <4 x i32> %result
+}
+
+define <5 x i32> @v_usubsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
+; GFX6-LABEL: v_usubsat_v5i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_min_u32_e32 v5, v0, v5
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v5
+; GFX6-NEXT:    v_min_u32_e32 v5, v1, v6
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v5
+; GFX6-NEXT:    v_min_u32_e32 v5, v2, v7
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v5
+; GFX6-NEXT:    v_min_u32_e32 v5, v3, v8
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, v3, v5
+; GFX6-NEXT:    v_min_u32_e32 v5, v4, v9
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, v4, v5
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_v5i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_min_u32_e32 v5, v0, v5
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, v0, v5
+; GFX8-NEXT:    v_min_u32_e32 v5, v1, v6
+; GFX8-NEXT:    v_sub_u32_e32 v1, vcc, v1, v5
+; GFX8-NEXT:    v_min_u32_e32 v5, v2, v7
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, v2, v5
+; GFX8-NEXT:    v_min_u32_e32 v5, v3, v8
+; GFX8-NEXT:    v_sub_u32_e32 v3, vcc, v3, v5
+; GFX8-NEXT:    v_min_u32_e32 v5, v4, v9
+; GFX8-NEXT:    v_sub_u32_e32 v4, vcc, v4, v5
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_v5i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_min_u32_e32 v5, v0, v5
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v5
+; GFX9-NEXT:    v_min_u32_e32 v5, v1, v6
+; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v5
+; GFX9-NEXT:    v_min_u32_e32 v5, v2, v7
+; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v5
+; GFX9-NEXT:    v_min_u32_e32 v5, v3, v8
+; GFX9-NEXT:    v_sub_u32_e32 v3, v3, v5
+; GFX9-NEXT:    v_min_u32_e32 v5, v4, v9
+; GFX9-NEXT:    v_sub_u32_e32 v4, v4, v5
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_usubsat_v5i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_min_u32_e32 v5, v0, v5
+; GFX10-NEXT:    v_min_u32_e32 v6, v1, v6
+; GFX10-NEXT:    v_min_u32_e32 v7, v2, v7
+; GFX10-NEXT:    v_min_u32_e32 v8, v3, v8
+; GFX10-NEXT:    v_min_u32_e32 v9, v4, v9
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v5
+; GFX10-NEXT:    v_sub_nc_u32_e32 v1, v1, v6
+; GFX10-NEXT:    v_sub_nc_u32_e32 v2, v2, v7
+; GFX10-NEXT:    v_sub_nc_u32_e32 v3, v3, v8
+; GFX10-NEXT:    v_sub_nc_u32_e32 v4, v4, v9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <5 x i32> @llvm.usub.sat.v5i32(<5 x i32> %lhs, <5 x i32> %rhs)
+  ret <5 x i32> %result
+}
+
+define amdgpu_ps <5 x i32> @s_usubsat_v5i32(<5 x i32> inreg %lhs, <5 x i32> inreg %rhs) {
+; GFX6-LABEL: s_usubsat_v5i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_cmp_lt_u32 s0, s5
+; GFX6-NEXT:    s_cselect_b32 s5, s0, s5
+; GFX6-NEXT:    s_sub_i32 s0, s0, s5
+; GFX6-NEXT:    s_cmp_lt_u32 s1, s6
+; GFX6-NEXT:    s_cselect_b32 s5, s1, s6
+; GFX6-NEXT:    s_sub_i32 s1, s1, s5
+; GFX6-NEXT:    s_cmp_lt_u32 s2, s7
+; GFX6-NEXT:    s_cselect_b32 s5, s2, s7
+; GFX6-NEXT:    s_sub_i32 s2, s2, s5
+; GFX6-NEXT:    s_cmp_lt_u32 s3, s8
+; GFX6-NEXT:    s_cselect_b32 s5, s3, s8
+; GFX6-NEXT:    s_sub_i32 s3, s3, s5
+; GFX6-NEXT:    s_cmp_lt_u32 s4, s9
+; GFX6-NEXT:    s_cselect_b32 s5, s4, s9
+; GFX6-NEXT:    s_sub_i32 s4, s4, s5
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_usubsat_v5i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_cmp_lt_u32 s0, s5
+; GFX8-NEXT:    s_cselect_b32 s5, s0, s5
+; GFX8-NEXT:    s_sub_i32 s0, s0, s5
+; GFX8-NEXT:    s_cmp_lt_u32 s1, s6
+; GFX8-NEXT:    s_cselect_b32 s5, s1, s6
+; GFX8-NEXT:    s_sub_i32 s1, s1, s5
+; GFX8-NEXT:    s_cmp_lt_u32 s2, s7
+; GFX8-NEXT:    s_cselect_b32 s5, s2, s7
+; GFX8-NEXT:    s_sub_i32 s2, s2, s5
+; GFX8-NEXT:    s_cmp_lt_u32 s3, s8
+; GFX8-NEXT:    s_cselect_b32 s5, s3, s8
+; GFX8-NEXT:    s_sub_i32 s3, s3, s5
+; GFX8-NEXT:    s_cmp_lt_u32 s4, s9
+; GFX8-NEXT:    s_cselect_b32 s5, s4, s9
+; GFX8-NEXT:    s_sub_i32 s4, s4, s5
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_usubsat_v5i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_cmp_lt_u32 s0, s5
+; GFX9-NEXT:    s_cselect_b32 s5, s0, s5
+; GFX9-NEXT:    s_sub_i32 s0, s0, s5
+; GFX9-NEXT:    s_cmp_lt_u32 s1, s6
+; GFX9-NEXT:    s_cselect_b32 s5, s1, s6
+; GFX9-NEXT:    s_sub_i32 s1, s1, s5
+; GFX9-NEXT:    s_cmp_lt_u32 s2, s7
+; GFX9-NEXT:    s_cselect_b32 s5, s2, s7
+; GFX9-NEXT:    s_sub_i32 s2, s2, s5
+; GFX9-NEXT:    s_cmp_lt_u32 s3, s8
+; GFX9-NEXT:    s_cselect_b32 s5, s3, s8
+; GFX9-NEXT:    s_sub_i32 s3, s3, s5
+; GFX9-NEXT:    s_cmp_lt_u32 s4, s9
+; GFX9-NEXT:    s_cselect_b32 s5, s4, s9
+; GFX9-NEXT:    s_sub_i32 s4, s4, s5
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_usubsat_v5i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_cmp_lt_u32 s0, s5
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s5, s0, s5
+; GFX10-NEXT:    s_sub_i32 s0, s0, s5
+; GFX10-NEXT:    s_cmp_lt_u32 s1, s6
+; GFX10-NEXT:    s_cselect_b32 s5, s1, s6
+; GFX10-NEXT:    s_sub_i32 s1, s1, s5
+; GFX10-NEXT:    s_cmp_lt_u32 s2, s7
+; GFX10-NEXT:    s_cselect_b32 s5, s2, s7
+; GFX10-NEXT:    s_sub_i32 s2, s2, s5
+; GFX10-NEXT:    s_cmp_lt_u32 s3, s8
+; GFX10-NEXT:    s_cselect_b32 s5, s3, s8
+; GFX10-NEXT:    s_sub_i32 s3, s3, s5
+; GFX10-NEXT:    s_cmp_lt_u32 s4, s9
+; GFX10-NEXT:    s_cselect_b32 s5, s4, s9
+; GFX10-NEXT:    s_sub_i32 s4, s4, s5
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <5 x i32> @llvm.usub.sat.v5i32(<5 x i32> %lhs, <5 x i32> %rhs)
+  ret <5 x i32> %result
+}
+
+define <16 x i32> @v_usubsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
+; GFX6-LABEL: v_usubsat_v16i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_min_u32_e32 v16, v0, v16
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v16
+; GFX6-NEXT:    v_min_u32_e32 v16, v1, v17
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v16
+; GFX6-NEXT:    v_min_u32_e32 v16, v2, v18
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v16
+; GFX6-NEXT:    v_min_u32_e32 v16, v3, v19
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, v3, v16
+; GFX6-NEXT:    v_min_u32_e32 v16, v4, v20
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, v4, v16
+; GFX6-NEXT:    v_min_u32_e32 v16, v5, v21
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, v5, v16
+; GFX6-NEXT:    v_min_u32_e32 v16, v6, v22
+; GFX6-NEXT:    v_sub_i32_e32 v6, vcc, v6, v16
+; GFX6-NEXT:    v_min_u32_e32 v16, v7, v23
+; GFX6-NEXT:    v_sub_i32_e32 v7, vcc, v7, v16
+; GFX6-NEXT:    v_min_u32_e32 v16, v8, v24
+; GFX6-NEXT:    v_sub_i32_e32 v8, vcc, v8, v16
+; GFX6-NEXT:    v_min_u32_e32 v16, v9, v25
+; GFX6-NEXT:    v_sub_i32_e32 v9, vcc, v9, v16
+; GFX6-NEXT:    v_min_u32_e32 v16, v10, v26
+; GFX6-NEXT:    v_sub_i32_e32 v10, vcc, v10, v16
+; GFX6-NEXT:    v_min_u32_e32 v16, v11, v27
+; GFX6-NEXT:    v_sub_i32_e32 v11, vcc, v11, v16
+; GFX6-NEXT:    v_min_u32_e32 v16, v12, v28
+; GFX6-NEXT:    v_sub_i32_e32 v12, vcc, v12, v16
+; GFX6-NEXT:    v_min_u32_e32 v16, v13, v29
+; GFX6-NEXT:    v_sub_i32_e32 v13, vcc, v13, v16
+; GFX6-NEXT:    v_min_u32_e32 v16, v14, v30
+; GFX6-NEXT:    v_sub_i32_e32 v14, vcc, v14, v16
+; GFX6-NEXT:    v_min_u32_e32 v16, v15, v31
+; GFX6-NEXT:    v_sub_i32_e32 v15, vcc, v15, v16
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_v16i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_min_u32_e32 v16, v0, v16
+; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, v0, v16
+; GFX8-NEXT:    v_min_u32_e32 v16, v1, v17
+; GFX8-NEXT:    v_sub_u32_e32 v1, vcc, v1, v16
+; GFX8-NEXT:    v_min_u32_e32 v16, v2, v18
+; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, v2, v16
+; GFX8-NEXT:    v_min_u32_e32 v16, v3, v19
+; GFX8-NEXT:    v_sub_u32_e32 v3, vcc, v3, v16
+; GFX8-NEXT:    v_min_u32_e32 v16, v4, v20
+; GFX8-NEXT:    v_sub_u32_e32 v4, vcc, v4, v16
+; GFX8-NEXT:    v_min_u32_e32 v16, v5, v21
+; GFX8-NEXT:    v_sub_u32_e32 v5, vcc, v5, v16
+; GFX8-NEXT:    v_min_u32_e32 v16, v6, v22
+; GFX8-NEXT:    v_sub_u32_e32 v6, vcc, v6, v16
+; GFX8-NEXT:    v_min_u32_e32 v16, v7, v23
+; GFX8-NEXT:    v_sub_u32_e32 v7, vcc, v7, v16
+; GFX8-NEXT:    v_min_u32_e32 v16, v8, v24
+; GFX8-NEXT:    v_sub_u32_e32 v8, vcc, v8, v16
+; GFX8-NEXT:    v_min_u32_e32 v16, v9, v25
+; GFX8-NEXT:    v_sub_u32_e32 v9, vcc, v9, v16
+; GFX8-NEXT:    v_min_u32_e32 v16, v10, v26
+; GFX8-NEXT:    v_sub_u32_e32 v10, vcc, v10, v16
+; GFX8-NEXT:    v_min_u32_e32 v16, v11, v27
+; GFX8-NEXT:    v_sub_u32_e32 v11, vcc, v11, v16
+; GFX8-NEXT:    v_min_u32_e32 v16, v12, v28
+; GFX8-NEXT:    v_sub_u32_e32 v12, vcc, v12, v16
+; GFX8-NEXT:    v_min_u32_e32 v16, v13, v29
+; GFX8-NEXT:    v_sub_u32_e32 v13, vcc, v13, v16
+; GFX8-NEXT:    v_min_u32_e32 v16, v14, v30
+; GFX8-NEXT:    v_sub_u32_e32 v14, vcc, v14, v16
+; GFX8-NEXT:    v_min_u32_e32 v16, v15, v31
+; GFX8-NEXT:    v_sub_u32_e32 v15, vcc, v15, v16
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_v16i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_min_u32_e32 v16, v0, v16
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v16
+; GFX9-NEXT:    v_min_u32_e32 v16, v1, v17
+; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v16
+; GFX9-NEXT:    v_min_u32_e32 v16, v2, v18
+; GFX9-NEXT:    v_sub_u32_e32 v2, v2, v16
+; GFX9-NEXT:    v_min_u32_e32 v16, v3, v19
+; GFX9-NEXT:    v_sub_u32_e32 v3, v3, v16
+; GFX9-NEXT:    v_min_u32_e32 v16, v4, v20
+; GFX9-NEXT:    v_sub_u32_e32 v4, v4, v16
+; GFX9-NEXT:    v_min_u32_e32 v16, v5, v21
+; GFX9-NEXT:    v_sub_u32_e32 v5, v5, v16
+; GFX9-NEXT:    v_min_u32_e32 v16, v6, v22
+; GFX9-NEXT:    v_sub_u32_e32 v6, v6, v16
+; GFX9-NEXT:    v_min_u32_e32 v16, v7, v23
+; GFX9-NEXT:    v_sub_u32_e32 v7, v7, v16
+; GFX9-NEXT:    v_min_u32_e32 v16, v8, v24
+; GFX9-NEXT:    v_sub_u32_e32 v8, v8, v16
+; GFX9-NEXT:    v_min_u32_e32 v16, v9, v25
+; GFX9-NEXT:    v_sub_u32_e32 v9, v9, v16
+; GFX9-NEXT:    v_min_u32_e32 v16, v10, v26
+; GFX9-NEXT:    v_sub_u32_e32 v10, v10, v16
+; GFX9-NEXT:    v_min_u32_e32 v16, v11, v27
+; GFX9-NEXT:    v_sub_u32_e32 v11, v11, v16
+; GFX9-NEXT:    v_min_u32_e32 v16, v12, v28
+; GFX9-NEXT:    v_sub_u32_e32 v12, v12, v16
+; GFX9-NEXT:    v_min_u32_e32 v16, v13, v29
+; GFX9-NEXT:    v_sub_u32_e32 v13, v13, v16
+; GFX9-NEXT:    v_min_u32_e32 v16, v14, v30
+; GFX9-NEXT:    v_sub_u32_e32 v14, v14, v16
+; GFX9-NEXT:    v_min_u32_e32 v16, v15, v31
+; GFX9-NEXT:    v_sub_u32_e32 v15, v15, v16
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_usubsat_v16i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_min_u32_e32 v35, v0, v16
+; GFX10-NEXT:    v_min_u32_e32 v16, v1, v17
+; GFX10-NEXT:    v_min_u32_e32 v17, v2, v18
+; GFX10-NEXT:    v_min_u32_e32 v18, v3, v19
+; GFX10-NEXT:    v_min_u32_e32 v19, v4, v20
+; GFX10-NEXT:    v_min_u32_e32 v20, v5, v21
+; GFX10-NEXT:    v_sub_nc_u32_e32 v1, v1, v16
+; GFX10-NEXT:    v_min_u32_e32 v16, v6, v22
+; GFX10-NEXT:    v_sub_nc_u32_e32 v2, v2, v17
+; GFX10-NEXT:    v_min_u32_e32 v17, v7, v23
+; GFX10-NEXT:    v_sub_nc_u32_e32 v3, v3, v18
+; GFX10-NEXT:    v_min_u32_e32 v18, v8, v24
+; GFX10-NEXT:    v_sub_nc_u32_e32 v4, v4, v19
+; GFX10-NEXT:    v_min_u32_e32 v19, v9, v25
+; GFX10-NEXT:    v_sub_nc_u32_e32 v5, v5, v20
+; GFX10-NEXT:    v_min_u32_e32 v20, v10, v26
+; GFX10-NEXT:    v_sub_nc_u32_e32 v6, v6, v16
+; GFX10-NEXT:    v_min_u32_e32 v16, v11, v27
+; GFX10-NEXT:    v_sub_nc_u32_e32 v7, v7, v17
+; GFX10-NEXT:    v_min_u32_e32 v17, v12, v28
+; GFX10-NEXT:    v_sub_nc_u32_e32 v8, v8, v18
+; GFX10-NEXT:    v_min_u32_e32 v18, v13, v29
+; GFX10-NEXT:    v_sub_nc_u32_e32 v9, v9, v19
+; GFX10-NEXT:    v_min_u32_e32 v19, v14, v30
+; GFX10-NEXT:    v_sub_nc_u32_e32 v10, v10, v20
+; GFX10-NEXT:    v_min_u32_e32 v20, v15, v31
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v35
+; GFX10-NEXT:    v_sub_nc_u32_e32 v11, v11, v16
+; GFX10-NEXT:    v_sub_nc_u32_e32 v12, v12, v17
+; GFX10-NEXT:    v_sub_nc_u32_e32 v13, v13, v18
+; GFX10-NEXT:    v_sub_nc_u32_e32 v14, v14, v19
+; GFX10-NEXT:    v_sub_nc_u32_e32 v15, v15, v20
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
+  ret <16 x i32> %result
+}
+
+define amdgpu_ps <16 x i32> @s_usubsat_v16i32(<16 x i32> inreg %lhs, <16 x i32> inreg %rhs) {
+; GFX6-LABEL: s_usubsat_v16i32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_cmp_lt_u32 s0, s16
+; GFX6-NEXT:    s_cselect_b32 s16, s0, s16
+; GFX6-NEXT:    s_sub_i32 s0, s0, s16
+; GFX6-NEXT:    s_cmp_lt_u32 s1, s17
+; GFX6-NEXT:    s_cselect_b32 s16, s1, s17
+; GFX6-NEXT:    s_sub_i32 s1, s1, s16
+; GFX6-NEXT:    s_cmp_lt_u32 s2, s18
+; GFX6-NEXT:    s_cselect_b32 s16, s2, s18
+; GFX6-NEXT:    s_sub_i32 s2, s2, s16
+; GFX6-NEXT:    s_cmp_lt_u32 s3, s19
+; GFX6-NEXT:    s_cselect_b32 s16, s3, s19
+; GFX6-NEXT:    s_sub_i32 s3, s3, s16
+; GFX6-NEXT:    s_cmp_lt_u32 s4, s20
+; GFX6-NEXT:    s_cselect_b32 s16, s4, s20
+; GFX6-NEXT:    s_sub_i32 s4, s4, s16
+; GFX6-NEXT:    s_cmp_lt_u32 s5, s21
+; GFX6-NEXT:    s_cselect_b32 s16, s5, s21
+; GFX6-NEXT:    s_sub_i32 s5, s5, s16
+; GFX6-NEXT:    s_cmp_lt_u32 s6, s22
+; GFX6-NEXT:    s_cselect_b32 s16, s6, s22
+; GFX6-NEXT:    s_sub_i32 s6, s6, s16
+; GFX6-NEXT:    s_cmp_lt_u32 s7, s23
+; GFX6-NEXT:    s_cselect_b32 s16, s7, s23
+; GFX6-NEXT:    s_sub_i32 s7, s7, s16
+; GFX6-NEXT:    s_cmp_lt_u32 s8, s24
+; GFX6-NEXT:    s_cselect_b32 s16, s8, s24
+; GFX6-NEXT:    s_sub_i32 s8, s8, s16
+; GFX6-NEXT:    s_cmp_lt_u32 s9, s25
+; GFX6-NEXT:    s_cselect_b32 s16, s9, s25
+; GFX6-NEXT:    s_sub_i32 s9, s9, s16
+; GFX6-NEXT:    s_cmp_lt_u32 s10, s26
+; GFX6-NEXT:    s_cselect_b32 s16, s10, s26
+; GFX6-NEXT:    s_sub_i32 s10, s10, s16
+; GFX6-NEXT:    s_cmp_lt_u32 s11, s27
+; GFX6-NEXT:    s_cselect_b32 s16, s11, s27
+; GFX6-NEXT:    s_sub_i32 s11, s11, s16
+; GFX6-NEXT:    s_cmp_lt_u32 s12, s28
+; GFX6-NEXT:    s_cselect_b32 s16, s12, s28
+; GFX6-NEXT:    s_sub_i32 s12, s12, s16
+; GFX6-NEXT:    s_cmp_lt_u32 s13, s29
+; GFX6-NEXT:    s_cselect_b32 s16, s13, s29
+; GFX6-NEXT:    s_sub_i32 s13, s13, s16
+; GFX6-NEXT:    s_cmp_lt_u32 s14, s30
+; GFX6-NEXT:    s_cselect_b32 s16, s14, s30
+; GFX6-NEXT:    s_sub_i32 s14, s14, s16
+; GFX6-NEXT:    s_cmp_lt_u32 s15, s31
+; GFX6-NEXT:    s_cselect_b32 s16, s15, s31
+; GFX6-NEXT:    s_sub_i32 s15, s15, s16
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_usubsat_v16i32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_cmp_lt_u32 s0, s16
+; GFX8-NEXT:    s_cselect_b32 s16, s0, s16
+; GFX8-NEXT:    s_sub_i32 s0, s0, s16
+; GFX8-NEXT:    s_cmp_lt_u32 s1, s17
+; GFX8-NEXT:    s_cselect_b32 s16, s1, s17
+; GFX8-NEXT:    s_sub_i32 s1, s1, s16
+; GFX8-NEXT:    s_cmp_lt_u32 s2, s18
+; GFX8-NEXT:    s_cselect_b32 s16, s2, s18
+; GFX8-NEXT:    s_sub_i32 s2, s2, s16
+; GFX8-NEXT:    s_cmp_lt_u32 s3, s19
+; GFX8-NEXT:    s_cselect_b32 s16, s3, s19
+; GFX8-NEXT:    s_sub_i32 s3, s3, s16
+; GFX8-NEXT:    s_cmp_lt_u32 s4, s20
+; GFX8-NEXT:    s_cselect_b32 s16, s4, s20
+; GFX8-NEXT:    s_sub_i32 s4, s4, s16
+; GFX8-NEXT:    s_cmp_lt_u32 s5, s21
+; GFX8-NEXT:    s_cselect_b32 s16, s5, s21
+; GFX8-NEXT:    s_sub_i32 s5, s5, s16
+; GFX8-NEXT:    s_cmp_lt_u32 s6, s22
+; GFX8-NEXT:    s_cselect_b32 s16, s6, s22
+; GFX8-NEXT:    s_sub_i32 s6, s6, s16
+; GFX8-NEXT:    s_cmp_lt_u32 s7, s23
+; GFX8-NEXT:    s_cselect_b32 s16, s7, s23
+; GFX8-NEXT:    s_sub_i32 s7, s7, s16
+; GFX8-NEXT:    s_cmp_lt_u32 s8, s24
+; GFX8-NEXT:    s_cselect_b32 s16, s8, s24
+; GFX8-NEXT:    s_sub_i32 s8, s8, s16
+; GFX8-NEXT:    s_cmp_lt_u32 s9, s25
+; GFX8-NEXT:    s_cselect_b32 s16, s9, s25
+; GFX8-NEXT:    s_sub_i32 s9, s9, s16
+; GFX8-NEXT:    s_cmp_lt_u32 s10, s26
+; GFX8-NEXT:    s_cselect_b32 s16, s10, s26
+; GFX8-NEXT:    s_sub_i32 s10, s10, s16
+; GFX8-NEXT:    s_cmp_lt_u32 s11, s27
+; GFX8-NEXT:    s_cselect_b32 s16, s11, s27
+; GFX8-NEXT:    s_sub_i32 s11, s11, s16
+; GFX8-NEXT:    s_cmp_lt_u32 s12, s28
+; GFX8-NEXT:    s_cselect_b32 s16, s12, s28
+; GFX8-NEXT:    s_sub_i32 s12, s12, s16
+; GFX8-NEXT:    s_cmp_lt_u32 s13, s29
+; GFX8-NEXT:    s_cselect_b32 s16, s13, s29
+; GFX8-NEXT:    s_sub_i32 s13, s13, s16
+; GFX8-NEXT:    s_cmp_lt_u32 s14, s30
+; GFX8-NEXT:    s_cselect_b32 s16, s14, s30
+; GFX8-NEXT:    s_sub_i32 s14, s14, s16
+; GFX8-NEXT:    s_cmp_lt_u32 s15, s31
+; GFX8-NEXT:    s_cselect_b32 s16, s15, s31
+; GFX8-NEXT:    s_sub_i32 s15, s15, s16
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_usubsat_v16i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_cmp_lt_u32 s0, s16
+; GFX9-NEXT:    s_cselect_b32 s16, s0, s16
+; GFX9-NEXT:    s_sub_i32 s0, s0, s16
+; GFX9-NEXT:    s_cmp_lt_u32 s1, s17
+; GFX9-NEXT:    s_cselect_b32 s16, s1, s17
+; GFX9-NEXT:    s_sub_i32 s1, s1, s16
+; GFX9-NEXT:    s_cmp_lt_u32 s2, s18
+; GFX9-NEXT:    s_cselect_b32 s16, s2, s18
+; GFX9-NEXT:    s_sub_i32 s2, s2, s16
+; GFX9-NEXT:    s_cmp_lt_u32 s3, s19
+; GFX9-NEXT:    s_cselect_b32 s16, s3, s19
+; GFX9-NEXT:    s_sub_i32 s3, s3, s16
+; GFX9-NEXT:    s_cmp_lt_u32 s4, s20
+; GFX9-NEXT:    s_cselect_b32 s16, s4, s20
+; GFX9-NEXT:    s_sub_i32 s4, s4, s16
+; GFX9-NEXT:    s_cmp_lt_u32 s5, s21
+; GFX9-NEXT:    s_cselect_b32 s16, s5, s21
+; GFX9-NEXT:    s_sub_i32 s5, s5, s16
+; GFX9-NEXT:    s_cmp_lt_u32 s6, s22
+; GFX9-NEXT:    s_cselect_b32 s16, s6, s22
+; GFX9-NEXT:    s_sub_i32 s6, s6, s16
+; GFX9-NEXT:    s_cmp_lt_u32 s7, s23
+; GFX9-NEXT:    s_cselect_b32 s16, s7, s23
+; GFX9-NEXT:    s_sub_i32 s7, s7, s16
+; GFX9-NEXT:    s_cmp_lt_u32 s8, s24
+; GFX9-NEXT:    s_cselect_b32 s16, s8, s24
+; GFX9-NEXT:    s_sub_i32 s8, s8, s16
+; GFX9-NEXT:    s_cmp_lt_u32 s9, s25
+; GFX9-NEXT:    s_cselect_b32 s16, s9, s25
+; GFX9-NEXT:    s_sub_i32 s9, s9, s16
+; GFX9-NEXT:    s_cmp_lt_u32 s10, s26
+; GFX9-NEXT:    s_cselect_b32 s16, s10, s26
+; GFX9-NEXT:    s_sub_i32 s10, s10, s16
+; GFX9-NEXT:    s_cmp_lt_u32 s11, s27
+; GFX9-NEXT:    s_cselect_b32 s16, s11, s27
+; GFX9-NEXT:    s_sub_i32 s11, s11, s16
+; GFX9-NEXT:    s_cmp_lt_u32 s12, s28
+; GFX9-NEXT:    s_cselect_b32 s16, s12, s28
+; GFX9-NEXT:    s_sub_i32 s12, s12, s16
+; GFX9-NEXT:    s_cmp_lt_u32 s13, s29
+; GFX9-NEXT:    s_cselect_b32 s16, s13, s29
+; GFX9-NEXT:    s_sub_i32 s13, s13, s16
+; GFX9-NEXT:    s_cmp_lt_u32 s14, s30
+; GFX9-NEXT:    s_cselect_b32 s16, s14, s30
+; GFX9-NEXT:    s_sub_i32 s14, s14, s16
+; GFX9-NEXT:    s_cmp_lt_u32 s15, s31
+; GFX9-NEXT:    s_cselect_b32 s16, s15, s31
+; GFX9-NEXT:    s_sub_i32 s15, s15, s16
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_usubsat_v16i32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_cmp_lt_u32 s0, s16
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s46, s0, s16
+; GFX10-NEXT:    s_sub_i32 s0, s0, s46
+; GFX10-NEXT:    s_cmp_lt_u32 s1, s17
+; GFX10-NEXT:    s_cselect_b32 s46, s1, s17
+; GFX10-NEXT:    s_sub_i32 s1, s1, s46
+; GFX10-NEXT:    s_cmp_lt_u32 s2, s18
+; GFX10-NEXT:    s_cselect_b32 s16, s2, s18
+; GFX10-NEXT:    s_sub_i32 s2, s2, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s3, s19
+; GFX10-NEXT:    s_cselect_b32 s16, s3, s19
+; GFX10-NEXT:    s_sub_i32 s3, s3, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s4, s20
+; GFX10-NEXT:    s_cselect_b32 s16, s4, s20
+; GFX10-NEXT:    s_sub_i32 s4, s4, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s5, s21
+; GFX10-NEXT:    s_cselect_b32 s16, s5, s21
+; GFX10-NEXT:    s_sub_i32 s5, s5, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s6, s22
+; GFX10-NEXT:    s_cselect_b32 s16, s6, s22
+; GFX10-NEXT:    s_sub_i32 s6, s6, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s7, s23
+; GFX10-NEXT:    s_cselect_b32 s16, s7, s23
+; GFX10-NEXT:    s_sub_i32 s7, s7, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s8, s24
+; GFX10-NEXT:    s_cselect_b32 s16, s8, s24
+; GFX10-NEXT:    s_sub_i32 s8, s8, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s9, s25
+; GFX10-NEXT:    s_cselect_b32 s16, s9, s25
+; GFX10-NEXT:    s_sub_i32 s9, s9, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s10, s26
+; GFX10-NEXT:    s_cselect_b32 s16, s10, s26
+; GFX10-NEXT:    s_sub_i32 s10, s10, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s11, s27
+; GFX10-NEXT:    s_cselect_b32 s16, s11, s27
+; GFX10-NEXT:    s_sub_i32 s11, s11, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s12, s28
+; GFX10-NEXT:    s_cselect_b32 s16, s12, s28
+; GFX10-NEXT:    s_sub_i32 s12, s12, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s13, s29
+; GFX10-NEXT:    s_cselect_b32 s16, s13, s29
+; GFX10-NEXT:    s_sub_i32 s13, s13, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s14, s30
+; GFX10-NEXT:    s_cselect_b32 s16, s14, s30
+; GFX10-NEXT:    s_sub_i32 s14, s14, s16
+; GFX10-NEXT:    s_cmp_lt_u32 s15, s31
+; GFX10-NEXT:    s_cselect_b32 s16, s15, s31
+; GFX10-NEXT:    s_sub_i32 s15, s15, s16
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
+  ret <16 x i32> %result
+}
+
+define i16 @v_usubsat_i16(i16 %lhs, i16 %rhs) {
+; GFX6-LABEL: v_usubsat_i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_min_u32_e32 v1, v0, v1
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_min_u16_e32 v1, v0, v1
+; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_min_u16_e32 v1, v0, v1
+; GFX9-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_usubsat_i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_min_u16_e64 v1, v0, v1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i16 @llvm.usub.sat.i16(i16 %lhs, i16 %rhs)
+  ret i16 %result
+}
+
+define amdgpu_ps i16 @s_usubsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
+; GFX6-LABEL: s_usubsat_i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_cmp_lt_u32 s0, s1
+; GFX6-NEXT:    s_cselect_b32 s1, s0, s1
+; GFX6-NEXT:    s_sub_i32 s0, s0, s1
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_usubsat_i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_bfe_u32 s2, s0, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s2, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX8-NEXT:    s_sub_i32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_usubsat_i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_bfe_u32 s2, s0, 0x100000
+; GFX9-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX9-NEXT:    s_cmp_lt_u32 s2, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX9-NEXT:    s_sub_i32 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_usubsat_i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_bfe_u32 s2, s0, 0x100000
+; GFX10-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cmp_lt_u32 s2, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s2, s1
+; GFX10-NEXT:    s_sub_i32 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i16 @llvm.usub.sat.i16(i16 %lhs, i16 %rhs)
+  ret i16 %result
+}
+
+define amdgpu_ps half @usubsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
+; GFX6-LABEL: usubsat_i16_sv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_min_u32_e32 v0, s0, v0
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: usubsat_i16_sv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_min_u16_e32 v0, s0, v0
+; GFX8-NEXT:    v_sub_u16_e32 v0, s0, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: usubsat_i16_sv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_min_u16_e32 v0, s0, v0
+; GFX9-NEXT:    v_sub_u16_e32 v0, s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: usubsat_i16_sv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_min_u16_e64 v0, s0, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i16 @llvm.usub.sat.i16(i16 %lhs, i16 %rhs)
+  %cast = bitcast i16 %result to half
+  ret half %cast
+}
+
+define amdgpu_ps half @usubsat_i16_vs(i16 %lhs, i16 inreg %rhs) {
+; GFX6-LABEL: usubsat_i16_vs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    v_min_u32_e32 v1, s0, v0
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: usubsat_i16_vs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_min_u16_e32 v1, s0, v0
+; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: usubsat_i16_vs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_min_u16_e32 v1, s0, v0
+; GFX9-NEXT:    v_sub_u16_e32 v0, v0, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: usubsat_i16_vs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_min_u16_e64 v1, v0, s0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i16 @llvm.usub.sat.i16(i16 %lhs, i16 %rhs)
+  %cast = bitcast i16 %result to half
+  ret half %cast
+}
+
+define <2 x i16> @v_usubsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
+; GFX6-LABEL: v_usubsat_v2i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_min_u32_e32 v2, v0, v2
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
+; GFX6-NEXT:    v_min_u32_e32 v2, v1, v2
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_v2i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
+; GFX8-NEXT:    v_min_u16_e32 v3, v0, v1
+; GFX8-NEXT:    v_min_u16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v3
+; GFX8-NEXT:    v_sub_u16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_min_u16 v1, v0, v1
+; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_usubsat_v2i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_pk_min_u16 v1, v0, v1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v1
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
+  ret <2 x i16> %result
+}
+
+define amdgpu_ps i32 @s_usubsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs) {
+; GFX6-LABEL: s_usubsat_v2i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_cmp_lt_u32 s0, s2
+; GFX6-NEXT:    s_cselect_b32 s2, s0, s2
+; GFX6-NEXT:    s_sub_i32 s0, s0, s2
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s2, s3, 16
+; GFX6-NEXT:    s_cmp_lt_u32 s1, s2
+; GFX6-NEXT:    s_cselect_b32 s2, s1, s2
+; GFX6-NEXT:    s_sub_i32 s1, s1, s2
+; GFX6-NEXT:    s_mov_b32 s2, 0xffff
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 16
+; GFX6-NEXT:    s_and_b32 s1, s1, s2
+; GFX6-NEXT:    s_and_b32 s0, s0, s2
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_usubsat_v2i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX8-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX8-NEXT:    s_bfe_u32 s4, s0, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s4, s1
+; GFX8-NEXT:    s_cselect_b32 s1, s4, s1
+; GFX8-NEXT:    s_sub_i32 s0, s0, s1
+; GFX8-NEXT:    s_bfe_u32 s1, s2, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s1, s3
+; GFX8-NEXT:    s_cselect_b32 s1, s1, s3
+; GFX8-NEXT:    s_sub_i32 s1, s2, s1
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX8-NEXT:    s_or_b32 s0, s0, s1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_usubsat_v2i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s3, 0xffff
+; GFX9-NEXT:    s_lshr_b32 s5, s1, 16
+; GFX9-NEXT:    s_lshr_b32 s2, s0, 16
+; GFX9-NEXT:    s_and_b32 s4, s0, s3
+; GFX9-NEXT:    s_and_b32 s1, s1, s3
+; GFX9-NEXT:    s_cmp_lt_u32 s4, s1
+; GFX9-NEXT:    s_cselect_b32 s1, s4, s1
+; GFX9-NEXT:    s_cmp_lt_u32 s2, s5
+; GFX9-NEXT:    s_cselect_b32 s3, s2, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s3
+; GFX9-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX9-NEXT:    s_sub_i32 s0, s0, s1
+; GFX9-NEXT:    s_sub_i32 s1, s2, s3
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_usubsat_v2i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s2, 0xffff
+; GFX10-NEXT:    s_lshr_b32 s3, s0, 16
+; GFX10-NEXT:    s_and_b32 s4, s0, s2
+; GFX10-NEXT:    s_and_b32 s2, s1, s2
+; GFX10-NEXT:    s_lshr_b32 s1, s1, 16
+; GFX10-NEXT:    s_cmp_lt_u32 s4, s2
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s2, s4, s2
+; GFX10-NEXT:    s_cmp_lt_u32 s3, s1
+; GFX10-NEXT:    s_cselect_b32 s1, s3, s1
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s2, s1
+; GFX10-NEXT:    s_lshr_b32 s2, s1, 16
+; GFX10-NEXT:    s_sub_i32 s0, s0, s1
+; GFX10-NEXT:    s_sub_i32 s1, s3, s2
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
+  %cast = bitcast <2 x i16> %result to i32
+  ret i32 %cast
+}
+
+define amdgpu_ps float @usubsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
+; GFX6-LABEL: usubsat_v2i16_sv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_min_u32_e32 v0, s0, v0
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
+; GFX6-NEXT:    s_lshl_b32 s0, s1, 16
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_min_u32_e32 v1, s0, v1
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, s0, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    s_mov_b32 s0, 0xffff
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, s0, v1
+; GFX6-NEXT:    v_and_b32_e32 v0, s0, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: usubsat_v2i16_sv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_lshr_b32 s1, s0, 16
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    v_min_u16_e32 v1, s0, v0
+; GFX8-NEXT:    v_min_u16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_u16_e32 v1, s0, v1
+; GFX8-NEXT:    v_sub_u16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: usubsat_v2i16_sv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_pk_min_u16 v0, s0, v0
+; GFX9-NEXT:    v_pk_sub_i16 v0, s0, v0
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: usubsat_v2i16_sv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_pk_min_u16 v0, s0, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_pk_sub_i16 v0, s0, v0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
+  %cast = bitcast <2 x i16> %result to float
+  ret float %cast
+}
+
+define amdgpu_ps float @usubsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
+; GFX6-LABEL: usubsat_v2i16_vs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    v_min_u32_e32 v2, s0, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    s_lshl_b32 s0, s1, 16
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT:    v_min_u32_e32 v2, s0, v1
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    s_mov_b32 s0, 0xffff
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, s0, v1
+; GFX6-NEXT:    v_and_b32_e32 v0, s0, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: usubsat_v2i16_vs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; GFX8-NEXT:    s_lshr_b32 s1, s0, 16
+; GFX8-NEXT:    v_min_u16_e32 v2, s0, v0
+; GFX8-NEXT:    v_min_u16_e32 v3, s1, v1
+; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v2
+; GFX8-NEXT:    v_sub_u16_sdwa v1, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: usubsat_v2i16_vs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_pk_min_u16 v1, v0, s0
+; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: usubsat_v2i16_vs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_pk_min_u16 v1, v0, s0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
+  %cast = bitcast <2 x i16> %result to float
+  ret float %cast
+}
+
+; FIXME: v3i16 insert/extract
+; define <3 x i16> @v_usubsat_v3i16(<3 x i16> %lhs, <3 x i16> %rhs) {
+;   %result = call <3 x i16> @llvm.usub.sat.v3i16(<3 x i16> %lhs, <3 x i16> %rhs)
+;   ret <3 x i16> %result
+; }
+
+; define amdgpu_ps <3 x i16> @s_usubsat_v3i16(<3 x i16> inreg %lhs, <3 x i16> inreg %rhs) {
+;   %result = call <3 x i16> @llvm.usub.sat.v3i16(<3 x i16> %lhs, <3 x i16> %rhs)
+;   ret <3 x i16> %result
+; }
+
+define <2 x float> @v_usubsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
+; GFX6-LABEL: v_usubsat_v4i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT:    v_min_u32_e32 v4, v0, v4
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v5
+; GFX6-NEXT:    v_min_u32_e32 v4, v1, v4
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v6
+; GFX6-NEXT:    v_min_u32_e32 v4, v2, v4
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v7
+; GFX6-NEXT:    v_min_u32_e32 v4, v3, v4
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, v3, v4
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_v4i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
+; GFX8-NEXT:    v_min_u16_e32 v6, v0, v2
+; GFX8-NEXT:    v_min_u16_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
+; GFX8-NEXT:    v_min_u16_e32 v7, v1, v3
+; GFX8-NEXT:    v_min_u16_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v6
+; GFX8-NEXT:    v_sub_u16_sdwa v2, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT:    v_sub_u16_e32 v1, v1, v7
+; GFX8-NEXT:    v_sub_u16_sdwa v2, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_v4i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_min_u16 v2, v0, v2
+; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v2
+; GFX9-NEXT:    v_pk_min_u16 v2, v1, v3
+; GFX9-NEXT:    v_pk_sub_i16 v1, v1, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_usubsat_v4i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_pk_min_u16 v2, v0, v2
+; GFX10-NEXT:    v_pk_min_u16 v3, v1, v3
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v2
+; GFX10-NEXT:    v_pk_sub_i16 v1, v1, v3
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <4 x i16> @llvm.usub.sat.v4i16(<4 x i16> %lhs, <4 x i16> %rhs)
+  %cast = bitcast <4 x i16> %result to <2 x float>
+  ret <2 x float> %cast
+}
+
+define amdgpu_ps <2 x i32> @s_usubsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inreg %rhs) {
+; GFX6-LABEL: s_usubsat_v4i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX6-NEXT:    s_cmp_lt_u32 s0, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s0, s4
+; GFX6-NEXT:    s_sub_i32 s0, s0, s4
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s4, s5, 16
+; GFX6-NEXT:    s_cmp_lt_u32 s1, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s1, s4
+; GFX6-NEXT:    s_sub_i32 s1, s1, s4
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_lshl_b32 s4, s6, 16
+; GFX6-NEXT:    s_cmp_lt_u32 s2, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s2, s4
+; GFX6-NEXT:    s_sub_i32 s2, s2, s4
+; GFX6-NEXT:    s_lshr_b32 s2, s2, 16
+; GFX6-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX6-NEXT:    s_lshl_b32 s4, s7, 16
+; GFX6-NEXT:    s_cmp_lt_u32 s3, s4
+; GFX6-NEXT:    s_cselect_b32 s4, s3, s4
+; GFX6-NEXT:    s_sub_i32 s3, s3, s4
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    s_and_b32 s1, s1, s4
+; GFX6-NEXT:    s_lshr_b32 s3, s3, 16
+; GFX6-NEXT:    s_and_b32 s0, s0, s4
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_and_b32 s1, s2, s4
+; GFX6-NEXT:    s_and_b32 s2, s3, s4
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_or_b32 s1, s1, s2
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_usubsat_v4i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_lshr_b32 s6, s2, 16
+; GFX8-NEXT:    s_lshr_b32 s4, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s5, s1, 16
+; GFX8-NEXT:    s_lshr_b32 s7, s3, 16
+; GFX8-NEXT:    s_bfe_u32 s8, s0, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s8, s2
+; GFX8-NEXT:    s_cselect_b32 s2, s8, s2
+; GFX8-NEXT:    s_sub_i32 s0, s0, s2
+; GFX8-NEXT:    s_bfe_u32 s2, s4, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s2, s6
+; GFX8-NEXT:    s_cselect_b32 s2, s2, s6
+; GFX8-NEXT:    s_sub_i32 s2, s4, s2
+; GFX8-NEXT:    s_bfe_u32 s4, s1, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s4, s3
+; GFX8-NEXT:    s_cselect_b32 s3, s4, s3
+; GFX8-NEXT:    s_sub_i32 s1, s1, s3
+; GFX8-NEXT:    s_bfe_u32 s3, s5, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s4, s7, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s3, s4
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s4
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_sub_i32 s3, s5, s3
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX8-NEXT:    s_or_b32 s0, s0, s2
+; GFX8-NEXT:    s_bfe_u32 s2, s3, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX8-NEXT:    s_or_b32 s1, s1, s2
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_usubsat_v4i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s5, 0xffff
+; GFX9-NEXT:    s_lshr_b32 s7, s2, 16
+; GFX9-NEXT:    s_lshr_b32 s4, s0, 16
+; GFX9-NEXT:    s_and_b32 s6, s0, s5
+; GFX9-NEXT:    s_and_b32 s2, s2, s5
+; GFX9-NEXT:    s_cmp_lt_u32 s6, s2
+; GFX9-NEXT:    s_cselect_b32 s2, s6, s2
+; GFX9-NEXT:    s_cmp_lt_u32 s4, s7
+; GFX9-NEXT:    s_cselect_b32 s6, s4, s7
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s6
+; GFX9-NEXT:    s_lshr_b32 s6, s2, 16
+; GFX9-NEXT:    s_sub_i32 s0, s0, s2
+; GFX9-NEXT:    s_sub_i32 s2, s4, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT:    s_lshr_b32 s6, s3, 16
+; GFX9-NEXT:    s_lshr_b32 s2, s1, 16
+; GFX9-NEXT:    s_and_b32 s4, s1, s5
+; GFX9-NEXT:    s_and_b32 s3, s3, s5
+; GFX9-NEXT:    s_cmp_lt_u32 s4, s3
+; GFX9-NEXT:    s_cselect_b32 s3, s4, s3
+; GFX9-NEXT:    s_cmp_lt_u32 s2, s6
+; GFX9-NEXT:    s_cselect_b32 s4, s2, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
+; GFX9-NEXT:    s_lshr_b32 s4, s3, 16
+; GFX9-NEXT:    s_sub_i32 s1, s1, s3
+; GFX9-NEXT:    s_sub_i32 s2, s2, s4
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_usubsat_v4i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s4, 0xffff
+; GFX10-NEXT:    s_lshr_b32 s5, s0, 16
+; GFX10-NEXT:    s_and_b32 s7, s2, s4
+; GFX10-NEXT:    s_and_b32 s6, s0, s4
+; GFX10-NEXT:    s_lshr_b32 s2, s2, 16
+; GFX10-NEXT:    s_cmp_lt_u32 s6, s7
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s6, s6, s7
+; GFX10-NEXT:    s_cmp_lt_u32 s5, s2
+; GFX10-NEXT:    s_cselect_b32 s2, s5, s2
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s6, s2
+; GFX10-NEXT:    s_lshr_b32 s6, s2, 16
+; GFX10-NEXT:    s_sub_i32 s0, s0, s2
+; GFX10-NEXT:    s_sub_i32 s2, s5, s6
+; GFX10-NEXT:    s_and_b32 s6, s1, s4
+; GFX10-NEXT:    s_and_b32 s4, s3, s4
+; GFX10-NEXT:    s_lshr_b32 s5, s1, 16
+; GFX10-NEXT:    s_lshr_b32 s3, s3, 16
+; GFX10-NEXT:    s_cmp_lt_u32 s6, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s2
+; GFX10-NEXT:    s_cselect_b32 s4, s6, s4
+; GFX10-NEXT:    s_cmp_lt_u32 s5, s3
+; GFX10-NEXT:    s_cselect_b32 s3, s5, s3
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s4, s3
+; GFX10-NEXT:    s_lshr_b32 s4, s3, 16
+; GFX10-NEXT:    s_sub_i32 s1, s1, s3
+; GFX10-NEXT:    s_sub_i32 s3, s5, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <4 x i16> @llvm.usub.sat.v4i16(<4 x i16> %lhs, <4 x i16> %rhs)
+  %cast = bitcast <4 x i16> %result to <2 x i32>
+  ret <2 x i32> %cast
+}
+
+; FIXME
+; define <5 x i16> @v_usubsat_v5i16(<5 x i16> %lhs, <5 x i16> %rhs) {
+;   %result = call <5 x i16> @llvm.usub.sat.v5i16(<5 x i16> %lhs, <5 x i16> %rhs)
+;   ret <5 x i16> %result
+; }
+
+; define amdgpu_ps <5 x i16> @s_usubsat_v5i16(<5 x i16> inreg %lhs, <5 x i16> inreg %rhs) {
+;   %result = call <5 x i16> @llvm.usub.sat.v5i16(<5 x i16> %lhs, <5 x i16> %rhs)
+;   ret <5 x i16> %result
+; }
+
+define <3 x float> @v_usubsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
+; GFX6-LABEL: v_usubsat_v6i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GFX6-NEXT:    v_min_u32_e32 v6, v0, v6
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v7
+; GFX6-NEXT:    v_min_u32_e32 v6, v1, v6
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v8
+; GFX6-NEXT:    v_min_u32_e32 v6, v2, v6
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v9
+; GFX6-NEXT:    v_min_u32_e32 v6, v3, v6
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, v3, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v10
+; GFX6-NEXT:    v_min_u32_e32 v6, v4, v6
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, v4, v6
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v11
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_min_u32_e32 v6, v5, v6
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, v5, v6
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v3
+; GFX6-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
+; GFX6-NEXT:    v_and_b32_e32 v3, s4, v5
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_v6i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
+; GFX8-NEXT:    v_min_u16_e32 v9, v0, v3
+; GFX8-NEXT:    v_min_u16_sdwa v3, v6, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 16, v1
+; GFX8-NEXT:    v_min_u16_e32 v10, v1, v4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
+; GFX8-NEXT:    v_min_u16_sdwa v4, v7, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v9
+; GFX8-NEXT:    v_sub_u16_sdwa v3, v6, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_min_u16_e32 v11, v2, v5
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX8-NEXT:    v_min_u16_sdwa v5, v8, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_u16_e32 v1, v1, v10
+; GFX8-NEXT:    v_sub_u16_sdwa v3, v7, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v3
+; GFX8-NEXT:    v_sub_u16_e32 v2, v2, v11
+; GFX8-NEXT:    v_sub_u16_sdwa v3, v8, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_v6i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_min_u16 v3, v0, v3
+; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v3
+; GFX9-NEXT:    v_pk_min_u16 v3, v1, v4
+; GFX9-NEXT:    v_pk_sub_i16 v1, v1, v3
+; GFX9-NEXT:    v_pk_min_u16 v3, v2, v5
+; GFX9-NEXT:    v_pk_sub_i16 v2, v2, v3
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_usubsat_v6i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_pk_min_u16 v3, v0, v3
+; GFX10-NEXT:    v_pk_min_u16 v4, v1, v4
+; GFX10-NEXT:    v_pk_min_u16 v5, v2, v5
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v3
+; GFX10-NEXT:    v_pk_sub_i16 v1, v1, v4
+; GFX10-NEXT:    v_pk_sub_i16 v2, v2, v5
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <6 x i16> @llvm.usub.sat.v6i16(<6 x i16> %lhs, <6 x i16> %rhs)
+  %cast = bitcast <6 x i16> %result to <3 x float>
+  ret <3 x float> %cast
+}
+
+define amdgpu_ps <3 x i32> @s_usubsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inreg %rhs) {
+; GFX6-LABEL: s_usubsat_v6i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s6, 16
+; GFX6-NEXT:    s_cmp_lt_u32 s0, s6
+; GFX6-NEXT:    s_cselect_b32 s6, s0, s6
+; GFX6-NEXT:    s_sub_i32 s0, s0, s6
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s7, 16
+; GFX6-NEXT:    s_cmp_lt_u32 s1, s6
+; GFX6-NEXT:    s_cselect_b32 s6, s1, s6
+; GFX6-NEXT:    s_sub_i32 s1, s1, s6
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s8, 16
+; GFX6-NEXT:    s_cmp_lt_u32 s2, s6
+; GFX6-NEXT:    s_cselect_b32 s6, s2, s6
+; GFX6-NEXT:    s_sub_i32 s2, s2, s6
+; GFX6-NEXT:    s_lshr_b32 s2, s2, 16
+; GFX6-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s9, 16
+; GFX6-NEXT:    s_cmp_lt_u32 s3, s6
+; GFX6-NEXT:    s_cselect_b32 s6, s3, s6
+; GFX6-NEXT:    s_sub_i32 s3, s3, s6
+; GFX6-NEXT:    s_lshr_b32 s3, s3, 16
+; GFX6-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s10, 16
+; GFX6-NEXT:    s_cmp_lt_u32 s4, s6
+; GFX6-NEXT:    s_cselect_b32 s6, s4, s6
+; GFX6-NEXT:    s_sub_i32 s4, s4, s6
+; GFX6-NEXT:    s_lshr_b32 s4, s4, 16
+; GFX6-NEXT:    s_lshl_b32 s5, s5, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s11, 16
+; GFX6-NEXT:    s_cmp_lt_u32 s5, s6
+; GFX6-NEXT:    s_cselect_b32 s6, s5, s6
+; GFX6-NEXT:    s_sub_i32 s5, s5, s6
+; GFX6-NEXT:    s_mov_b32 s6, 0xffff
+; GFX6-NEXT:    s_and_b32 s1, s1, s6
+; GFX6-NEXT:    s_and_b32 s0, s0, s6
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_and_b32 s1, s2, s6
+; GFX6-NEXT:    s_and_b32 s2, s3, s6
+; GFX6-NEXT:    s_lshr_b32 s5, s5, 16
+; GFX6-NEXT:    s_and_b32 s3, s5, s6
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_or_b32 s1, s1, s2
+; GFX6-NEXT:    s_and_b32 s2, s4, s6
+; GFX6-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX6-NEXT:    s_or_b32 s2, s2, s3
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_usubsat_v6i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_lshr_b32 s9, s3, 16
+; GFX8-NEXT:    s_lshr_b32 s6, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s7, s1, 16
+; GFX8-NEXT:    s_lshr_b32 s8, s2, 16
+; GFX8-NEXT:    s_lshr_b32 s10, s4, 16
+; GFX8-NEXT:    s_lshr_b32 s11, s5, 16
+; GFX8-NEXT:    s_bfe_u32 s12, s0, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s12, s3
+; GFX8-NEXT:    s_cselect_b32 s3, s12, s3
+; GFX8-NEXT:    s_sub_i32 s0, s0, s3
+; GFX8-NEXT:    s_bfe_u32 s3, s6, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s9, s9, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s3, s9
+; GFX8-NEXT:    s_cselect_b32 s3, s3, s9
+; GFX8-NEXT:    s_sub_i32 s3, s6, s3
+; GFX8-NEXT:    s_bfe_u32 s6, s1, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s6, s4
+; GFX8-NEXT:    s_cselect_b32 s4, s6, s4
+; GFX8-NEXT:    s_sub_i32 s1, s1, s4
+; GFX8-NEXT:    s_bfe_u32 s4, s7, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s6, s10, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s4, s6
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s6
+; GFX8-NEXT:    s_sub_i32 s4, s7, s4
+; GFX8-NEXT:    s_bfe_u32 s6, s2, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s6, s5
+; GFX8-NEXT:    s_cselect_b32 s5, s6, s5
+; GFX8-NEXT:    s_sub_i32 s2, s2, s5
+; GFX8-NEXT:    s_bfe_u32 s5, s8, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s6, s11, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s5, s6
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX8-NEXT:    s_or_b32 s0, s0, s3
+; GFX8-NEXT:    s_bfe_u32 s3, s4, 0x100000
+; GFX8-NEXT:    s_sub_i32 s5, s8, s5
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX8-NEXT:    s_or_b32 s1, s1, s3
+; GFX8-NEXT:    s_bfe_u32 s3, s5, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX8-NEXT:    s_or_b32 s2, s2, s3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_usubsat_v6i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s7, 0xffff
+; GFX9-NEXT:    s_lshr_b32 s9, s3, 16
+; GFX9-NEXT:    s_lshr_b32 s6, s0, 16
+; GFX9-NEXT:    s_and_b32 s8, s0, s7
+; GFX9-NEXT:    s_and_b32 s3, s3, s7
+; GFX9-NEXT:    s_cmp_lt_u32 s8, s3
+; GFX9-NEXT:    s_cselect_b32 s3, s8, s3
+; GFX9-NEXT:    s_cmp_lt_u32 s6, s9
+; GFX9-NEXT:    s_cselect_b32 s8, s6, s9
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s8
+; GFX9-NEXT:    s_lshr_b32 s8, s3, 16
+; GFX9-NEXT:    s_sub_i32 s0, s0, s3
+; GFX9-NEXT:    s_sub_i32 s3, s6, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s3
+; GFX9-NEXT:    s_lshr_b32 s8, s4, 16
+; GFX9-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX9-NEXT:    s_and_b32 s6, s1, s7
+; GFX9-NEXT:    s_and_b32 s4, s4, s7
+; GFX9-NEXT:    s_cmp_lt_u32 s6, s4
+; GFX9-NEXT:    s_cselect_b32 s4, s6, s4
+; GFX9-NEXT:    s_cmp_lt_u32 s3, s8
+; GFX9-NEXT:    s_cselect_b32 s6, s3, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s6
+; GFX9-NEXT:    s_lshr_b32 s6, s4, 16
+; GFX9-NEXT:    s_sub_i32 s1, s1, s4
+; GFX9-NEXT:    s_sub_i32 s3, s3, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s3
+; GFX9-NEXT:    s_lshr_b32 s6, s5, 16
+; GFX9-NEXT:    s_lshr_b32 s3, s2, 16
+; GFX9-NEXT:    s_and_b32 s4, s2, s7
+; GFX9-NEXT:    s_and_b32 s5, s5, s7
+; GFX9-NEXT:    s_cmp_lt_u32 s4, s5
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s5
+; GFX9-NEXT:    s_cmp_lt_u32 s3, s6
+; GFX9-NEXT:    s_cselect_b32 s5, s3, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s5
+; GFX9-NEXT:    s_lshr_b32 s5, s4, 16
+; GFX9-NEXT:    s_sub_i32 s2, s2, s4
+; GFX9-NEXT:    s_sub_i32 s3, s3, s5
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_usubsat_v6i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s6, 0xffff
+; GFX10-NEXT:    s_lshr_b32 s7, s0, 16
+; GFX10-NEXT:    s_and_b32 s9, s3, s6
+; GFX10-NEXT:    s_and_b32 s8, s0, s6
+; GFX10-NEXT:    s_lshr_b32 s3, s3, 16
+; GFX10-NEXT:    s_cmp_lt_u32 s8, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s8, s8, s9
+; GFX10-NEXT:    s_cmp_lt_u32 s7, s3
+; GFX10-NEXT:    s_cselect_b32 s3, s7, s3
+; GFX10-NEXT:    s_and_b32 s9, s4, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s8, s3
+; GFX10-NEXT:    s_lshr_b32 s4, s4, 16
+; GFX10-NEXT:    s_lshr_b32 s8, s3, 16
+; GFX10-NEXT:    s_sub_i32 s0, s0, s3
+; GFX10-NEXT:    s_sub_i32 s3, s7, s8
+; GFX10-NEXT:    s_and_b32 s8, s1, s6
+; GFX10-NEXT:    s_lshr_b32 s7, s1, 16
+; GFX10-NEXT:    s_cmp_lt_u32 s8, s9
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s3
+; GFX10-NEXT:    s_cselect_b32 s8, s8, s9
+; GFX10-NEXT:    s_cmp_lt_u32 s7, s4
+; GFX10-NEXT:    s_cselect_b32 s4, s7, s4
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s8, s4
+; GFX10-NEXT:    s_lshr_b32 s8, s4, 16
+; GFX10-NEXT:    s_sub_i32 s1, s1, s4
+; GFX10-NEXT:    s_sub_i32 s4, s7, s8
+; GFX10-NEXT:    s_and_b32 s8, s2, s6
+; GFX10-NEXT:    s_and_b32 s6, s5, s6
+; GFX10-NEXT:    s_lshr_b32 s7, s2, 16
+; GFX10-NEXT:    s_lshr_b32 s5, s5, 16
+; GFX10-NEXT:    s_cmp_lt_u32 s8, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
+; GFX10-NEXT:    s_cselect_b32 s6, s8, s6
+; GFX10-NEXT:    s_cmp_lt_u32 s7, s5
+; GFX10-NEXT:    s_cselect_b32 s5, s7, s5
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s6, s5
+; GFX10-NEXT:    s_lshr_b32 s3, s5, 16
+; GFX10-NEXT:    s_sub_i32 s2, s2, s5
+; GFX10-NEXT:    s_sub_i32 s3, s7, s3
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <6 x i16> @llvm.usub.sat.v6i16(<6 x i16> %lhs, <6 x i16> %rhs)
+  %cast = bitcast <6 x i16> %result to <3 x i32>
+  ret <3 x i32> %cast
+}
+
+define <4 x float> @v_usubsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
+; GFX6-LABEL: v_usubsat_v8i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
+; GFX6-NEXT:    v_min_u32_e32 v8, v0, v8
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v9
+; GFX6-NEXT:    v_min_u32_e32 v8, v1, v8
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v10
+; GFX6-NEXT:    v_min_u32_e32 v8, v2, v8
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v11
+; GFX6-NEXT:    v_min_u32_e32 v8, v3, v8
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, v3, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v12
+; GFX6-NEXT:    v_min_u32_e32 v8, v4, v8
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, v4, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v13
+; GFX6-NEXT:    v_min_u32_e32 v8, v5, v8
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, v5, v8
+; GFX6-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v14
+; GFX6-NEXT:    v_min_u32_e32 v8, v6, v8
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    s_mov_b32 s4, 0xffff
+; GFX6-NEXT:    v_sub_i32_e32 v6, vcc, v6, v8
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v1
+; GFX6-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GFX6-NEXT:    v_lshlrev_b32_e32 v8, 16, v15
+; GFX6-NEXT:    v_min_u32_e32 v8, v7, v8
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT:    v_sub_i32_e32 v7, vcc, v7, v8
+; GFX6-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT:    v_and_b32_e32 v1, s4, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v3
+; GFX6-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
+; GFX6-NEXT:    v_and_b32_e32 v3, s4, v5
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
+; GFX6-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
+; GFX6-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT:    v_and_b32_e32 v2, s4, v4
+; GFX6-NEXT:    v_and_b32_e32 v4, s4, v7
+; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT:    v_lshrrev_b32_e32 v6, 16, v6
+; GFX6-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX6-NEXT:    v_and_b32_e32 v3, s4, v6
+; GFX6-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT:    v_or_b32_e32 v3, v3, v4
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_v8i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 16, v0
+; GFX8-NEXT:    v_min_u16_e32 v12, v0, v4
+; GFX8-NEXT:    v_min_u16_sdwa v4, v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_lshrrev_b32_e32 v9, 16, v1
+; GFX8-NEXT:    v_min_u16_e32 v13, v1, v5
+; GFX8-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
+; GFX8-NEXT:    v_min_u16_sdwa v5, v9, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_u16_e32 v0, v0, v12
+; GFX8-NEXT:    v_sub_u16_sdwa v4, v8, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_min_u16_e32 v14, v2, v6
+; GFX8-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v11, 16, v3
+; GFX8-NEXT:    v_min_u16_sdwa v6, v10, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_u16_e32 v1, v1, v13
+; GFX8-NEXT:    v_sub_u16_sdwa v4, v9, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_min_u16_e32 v15, v3, v7
+; GFX8-NEXT:    v_or_b32_e32 v1, v1, v4
+; GFX8-NEXT:    v_min_u16_sdwa v7, v11, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_u16_e32 v2, v2, v14
+; GFX8-NEXT:    v_sub_u16_sdwa v4, v10, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v2, v2, v4
+; GFX8-NEXT:    v_sub_u16_e32 v3, v3, v15
+; GFX8-NEXT:    v_sub_u16_sdwa v4, v11, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_e32 v3, v3, v4
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_v8i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_pk_min_u16 v4, v0, v4
+; GFX9-NEXT:    v_pk_sub_i16 v0, v0, v4
+; GFX9-NEXT:    v_pk_min_u16 v4, v1, v5
+; GFX9-NEXT:    v_pk_sub_i16 v1, v1, v4
+; GFX9-NEXT:    v_pk_min_u16 v4, v2, v6
+; GFX9-NEXT:    v_pk_sub_i16 v2, v2, v4
+; GFX9-NEXT:    v_pk_min_u16 v4, v3, v7
+; GFX9-NEXT:    v_pk_sub_i16 v3, v3, v4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_usubsat_v8i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_pk_min_u16 v19, v2, v6
+; GFX10-NEXT:    v_pk_min_u16 v11, v0, v4
+; GFX10-NEXT:    v_pk_min_u16 v15, v1, v5
+; GFX10-NEXT:    v_pk_min_u16 v6, v3, v7
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_pk_sub_i16 v2, v2, v19
+; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v11
+; GFX10-NEXT:    v_pk_sub_i16 v1, v1, v15
+; GFX10-NEXT:    v_pk_sub_i16 v3, v3, v6
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %lhs, <8 x i16> %rhs)
+  %cast = bitcast <8 x i16> %result to <4 x float>
+  ret <4 x float> %cast
+}
+
+define amdgpu_ps <4 x i32> @s_usubsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inreg %rhs) {
+; GFX6-LABEL: s_usubsat_v8i16:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_lshl_b32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s8, 16
+; GFX6-NEXT:    s_cmp_lt_u32 s0, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s0, s8
+; GFX6-NEXT:    s_sub_i32 s0, s0, s8
+; GFX6-NEXT:    s_lshr_b32 s0, s0, 16
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s9, 16
+; GFX6-NEXT:    s_cmp_lt_u32 s1, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s1, s8
+; GFX6-NEXT:    s_sub_i32 s1, s1, s8
+; GFX6-NEXT:    s_lshr_b32 s1, s1, 16
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s10, 16
+; GFX6-NEXT:    s_cmp_lt_u32 s2, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s2, s8
+; GFX6-NEXT:    s_sub_i32 s2, s2, s8
+; GFX6-NEXT:    s_lshr_b32 s2, s2, 16
+; GFX6-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s11, 16
+; GFX6-NEXT:    s_cmp_lt_u32 s3, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s3, s8
+; GFX6-NEXT:    s_sub_i32 s3, s3, s8
+; GFX6-NEXT:    s_lshr_b32 s3, s3, 16
+; GFX6-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s12, 16
+; GFX6-NEXT:    s_cmp_lt_u32 s4, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s4, s8
+; GFX6-NEXT:    s_sub_i32 s4, s4, s8
+; GFX6-NEXT:    s_lshr_b32 s4, s4, 16
+; GFX6-NEXT:    s_lshl_b32 s5, s5, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s13, 16
+; GFX6-NEXT:    s_cmp_lt_u32 s5, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s5, s8
+; GFX6-NEXT:    s_sub_i32 s5, s5, s8
+; GFX6-NEXT:    s_lshr_b32 s5, s5, 16
+; GFX6-NEXT:    s_lshl_b32 s6, s6, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s14, 16
+; GFX6-NEXT:    s_cmp_lt_u32 s6, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s6, s8
+; GFX6-NEXT:    s_sub_i32 s6, s6, s8
+; GFX6-NEXT:    s_lshr_b32 s6, s6, 16
+; GFX6-NEXT:    s_lshl_b32 s7, s7, 16
+; GFX6-NEXT:    s_lshl_b32 s8, s15, 16
+; GFX6-NEXT:    s_cmp_lt_u32 s7, s8
+; GFX6-NEXT:    s_cselect_b32 s8, s7, s8
+; GFX6-NEXT:    s_sub_i32 s7, s7, s8
+; GFX6-NEXT:    s_mov_b32 s8, 0xffff
+; GFX6-NEXT:    s_and_b32 s1, s1, s8
+; GFX6-NEXT:    s_and_b32 s0, s0, s8
+; GFX6-NEXT:    s_lshl_b32 s1, s1, 16
+; GFX6-NEXT:    s_or_b32 s0, s0, s1
+; GFX6-NEXT:    s_and_b32 s1, s2, s8
+; GFX6-NEXT:    s_and_b32 s2, s3, s8
+; GFX6-NEXT:    s_and_b32 s3, s5, s8
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_lshr_b32 s7, s7, 16
+; GFX6-NEXT:    s_or_b32 s1, s1, s2
+; GFX6-NEXT:    s_and_b32 s2, s4, s8
+; GFX6-NEXT:    s_and_b32 s4, s7, s8
+; GFX6-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX6-NEXT:    s_or_b32 s2, s2, s3
+; GFX6-NEXT:    s_and_b32 s3, s6, s8
+; GFX6-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX6-NEXT:    s_or_b32 s3, s3, s4
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_usubsat_v8i16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_lshr_b32 s12, s4, 16
+; GFX8-NEXT:    s_lshr_b32 s8, s0, 16
+; GFX8-NEXT:    s_lshr_b32 s9, s1, 16
+; GFX8-NEXT:    s_lshr_b32 s10, s2, 16
+; GFX8-NEXT:    s_lshr_b32 s11, s3, 16
+; GFX8-NEXT:    s_lshr_b32 s13, s5, 16
+; GFX8-NEXT:    s_lshr_b32 s14, s6, 16
+; GFX8-NEXT:    s_lshr_b32 s15, s7, 16
+; GFX8-NEXT:    s_bfe_u32 s16, s0, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s16, s4
+; GFX8-NEXT:    s_cselect_b32 s4, s16, s4
+; GFX8-NEXT:    s_sub_i32 s0, s0, s4
+; GFX8-NEXT:    s_bfe_u32 s4, s8, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s12, s12, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s4, s12
+; GFX8-NEXT:    s_cselect_b32 s4, s4, s12
+; GFX8-NEXT:    s_sub_i32 s4, s8, s4
+; GFX8-NEXT:    s_bfe_u32 s8, s1, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s5, s5, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s8, s5
+; GFX8-NEXT:    s_cselect_b32 s5, s8, s5
+; GFX8-NEXT:    s_sub_i32 s1, s1, s5
+; GFX8-NEXT:    s_bfe_u32 s5, s9, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s8, s13, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s5, s8
+; GFX8-NEXT:    s_cselect_b32 s5, s5, s8
+; GFX8-NEXT:    s_sub_i32 s5, s9, s5
+; GFX8-NEXT:    s_bfe_u32 s8, s2, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s6, s6, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s8, s6
+; GFX8-NEXT:    s_cselect_b32 s6, s8, s6
+; GFX8-NEXT:    s_sub_i32 s2, s2, s6
+; GFX8-NEXT:    s_bfe_u32 s6, s10, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s8, s14, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s6, s8
+; GFX8-NEXT:    s_cselect_b32 s6, s6, s8
+; GFX8-NEXT:    s_sub_i32 s6, s10, s6
+; GFX8-NEXT:    s_bfe_u32 s8, s3, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s7, s7, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s8, s7
+; GFX8-NEXT:    s_cselect_b32 s7, s8, s7
+; GFX8-NEXT:    s_sub_i32 s3, s3, s7
+; GFX8-NEXT:    s_bfe_u32 s7, s11, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s8, s15, 0x100000
+; GFX8-NEXT:    s_cmp_lt_u32 s7, s8
+; GFX8-NEXT:    s_cselect_b32 s7, s7, s8
+; GFX8-NEXT:    s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX8-NEXT:    s_or_b32 s0, s0, s4
+; GFX8-NEXT:    s_bfe_u32 s4, s5, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX8-NEXT:    s_or_b32 s1, s1, s4
+; GFX8-NEXT:    s_bfe_u32 s4, s6, 0x100000
+; GFX8-NEXT:    s_sub_i32 s7, s11, s7
+; GFX8-NEXT:    s_bfe_u32 s2, s2, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX8-NEXT:    s_or_b32 s2, s2, s4
+; GFX8-NEXT:    s_bfe_u32 s4, s7, 0x100000
+; GFX8-NEXT:    s_bfe_u32 s3, s3, 0x100000
+; GFX8-NEXT:    s_lshl_b32 s4, s4, 16
+; GFX8-NEXT:    s_or_b32 s3, s3, s4
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_usubsat_v8i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s9, 0xffff
+; GFX9-NEXT:    s_lshr_b32 s11, s4, 16
+; GFX9-NEXT:    s_lshr_b32 s8, s0, 16
+; GFX9-NEXT:    s_and_b32 s10, s0, s9
+; GFX9-NEXT:    s_and_b32 s4, s4, s9
+; GFX9-NEXT:    s_cmp_lt_u32 s10, s4
+; GFX9-NEXT:    s_cselect_b32 s4, s10, s4
+; GFX9-NEXT:    s_cmp_lt_u32 s8, s11
+; GFX9-NEXT:    s_cselect_b32 s10, s8, s11
+; GFX9-NEXT:    s_pack_ll_b32_b16 s4, s4, s10
+; GFX9-NEXT:    s_lshr_b32 s10, s4, 16
+; GFX9-NEXT:    s_sub_i32 s0, s0, s4
+; GFX9-NEXT:    s_sub_i32 s4, s8, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s4
+; GFX9-NEXT:    s_lshr_b32 s10, s5, 16
+; GFX9-NEXT:    s_lshr_b32 s4, s1, 16
+; GFX9-NEXT:    s_and_b32 s8, s1, s9
+; GFX9-NEXT:    s_and_b32 s5, s5, s9
+; GFX9-NEXT:    s_cmp_lt_u32 s8, s5
+; GFX9-NEXT:    s_cselect_b32 s5, s8, s5
+; GFX9-NEXT:    s_cmp_lt_u32 s4, s10
+; GFX9-NEXT:    s_cselect_b32 s8, s4, s10
+; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s8
+; GFX9-NEXT:    s_lshr_b32 s8, s5, 16
+; GFX9-NEXT:    s_sub_i32 s1, s1, s5
+; GFX9-NEXT:    s_sub_i32 s4, s4, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s1, s4
+; GFX9-NEXT:    s_lshr_b32 s8, s6, 16
+; GFX9-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX9-NEXT:    s_and_b32 s5, s2, s9
+; GFX9-NEXT:    s_and_b32 s6, s6, s9
+; GFX9-NEXT:    s_cmp_lt_u32 s5, s6
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s6
+; GFX9-NEXT:    s_cmp_lt_u32 s4, s8
+; GFX9-NEXT:    s_cselect_b32 s6, s4, s8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
+; GFX9-NEXT:    s_lshr_b32 s6, s5, 16
+; GFX9-NEXT:    s_sub_i32 s2, s2, s5
+; GFX9-NEXT:    s_sub_i32 s4, s4, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s2, s2, s4
+; GFX9-NEXT:    s_lshr_b32 s6, s7, 16
+; GFX9-NEXT:    s_lshr_b32 s4, s3, 16
+; GFX9-NEXT:    s_and_b32 s5, s3, s9
+; GFX9-NEXT:    s_and_b32 s7, s7, s9
+; GFX9-NEXT:    s_cmp_lt_u32 s5, s7
+; GFX9-NEXT:    s_cselect_b32 s5, s5, s7
+; GFX9-NEXT:    s_cmp_lt_u32 s4, s6
+; GFX9-NEXT:    s_cselect_b32 s6, s4, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
+; GFX9-NEXT:    s_lshr_b32 s6, s5, 16
+; GFX9-NEXT:    s_sub_i32 s3, s3, s5
+; GFX9-NEXT:    s_sub_i32 s4, s4, s6
+; GFX9-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_usubsat_v8i16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s8, 0xffff
+; GFX10-NEXT:    s_lshr_b32 s9, s0, 16
+; GFX10-NEXT:    s_and_b32 s11, s4, s8
+; GFX10-NEXT:    s_and_b32 s10, s0, s8
+; GFX10-NEXT:    s_lshr_b32 s4, s4, 16
+; GFX10-NEXT:    s_cmp_lt_u32 s10, s11
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cselect_b32 s10, s10, s11
+; GFX10-NEXT:    s_cmp_lt_u32 s9, s4
+; GFX10-NEXT:    s_cselect_b32 s4, s9, s4
+; GFX10-NEXT:    s_and_b32 s11, s5, s8
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s10, s4
+; GFX10-NEXT:    s_lshr_b32 s5, s5, 16
+; GFX10-NEXT:    s_lshr_b32 s10, s4, 16
+; GFX10-NEXT:    s_sub_i32 s0, s0, s4
+; GFX10-NEXT:    s_sub_i32 s4, s9, s10
+; GFX10-NEXT:    s_and_b32 s10, s1, s8
+; GFX10-NEXT:    s_lshr_b32 s9, s1, 16
+; GFX10-NEXT:    s_cmp_lt_u32 s10, s11
+; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s4
+; GFX10-NEXT:    s_cselect_b32 s10, s10, s11
+; GFX10-NEXT:    s_cmp_lt_u32 s9, s5
+; GFX10-NEXT:    s_cselect_b32 s5, s9, s5
+; GFX10-NEXT:    s_and_b32 s11, s6, s8
+; GFX10-NEXT:    s_pack_ll_b32_b16 s5, s10, s5
+; GFX10-NEXT:    s_lshr_b32 s6, s6, 16
+; GFX10-NEXT:    s_lshr_b32 s10, s5, 16
+; GFX10-NEXT:    s_sub_i32 s1, s1, s5
+; GFX10-NEXT:    s_sub_i32 s5, s9, s10
+; GFX10-NEXT:    s_and_b32 s10, s2, s8
+; GFX10-NEXT:    s_lshr_b32 s9, s2, 16
+; GFX10-NEXT:    s_cmp_lt_u32 s10, s11
+; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s5
+; GFX10-NEXT:    s_cselect_b32 s10, s10, s11
+; GFX10-NEXT:    s_cmp_lt_u32 s9, s6
+; GFX10-NEXT:    s_cselect_b32 s6, s9, s6
+; GFX10-NEXT:    s_pack_ll_b32_b16 s6, s10, s6
+; GFX10-NEXT:    s_lshr_b32 s10, s6, 16
+; GFX10-NEXT:    s_sub_i32 s2, s2, s6
+; GFX10-NEXT:    s_sub_i32 s6, s9, s10
+; GFX10-NEXT:    s_and_b32 s10, s3, s8
+; GFX10-NEXT:    s_and_b32 s8, s7, s8
+; GFX10-NEXT:    s_lshr_b32 s9, s3, 16
+; GFX10-NEXT:    s_lshr_b32 s7, s7, 16
+; GFX10-NEXT:    s_cmp_lt_u32 s10, s8
+; GFX10-NEXT:    s_pack_ll_b32_b16 s2, s2, s6
+; GFX10-NEXT:    s_cselect_b32 s8, s10, s8
+; GFX10-NEXT:    s_cmp_lt_u32 s9, s7
+; GFX10-NEXT:    s_cselect_b32 s7, s9, s7
+; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s8, s7
+; GFX10-NEXT:    s_lshr_b32 s5, s4, 16
+; GFX10-NEXT:    s_sub_i32 s3, s3, s4
+; GFX10-NEXT:    s_sub_i32 s4, s9, s5
+; GFX10-NEXT:    s_pack_ll_b32_b16 s3, s3, s4
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %lhs, <8 x i16> %rhs)
+  %cast = bitcast <8 x i16> %result to <4 x i32>
+  ret <4 x i32> %cast
+}
+
+; FIXME: i48 broken because i48 add broken
+; define i48 @v_usubsat_i48(i48 %lhs, i48 %rhs) {
+;   %result = call i48 @llvm.usub.sat.i48(i48 %lhs, i48 %rhs)
+;   ret i48 %result
+; }
+
+; define amdgpu_ps i48 @s_usubsat_i48(i48 inreg %lhs, i48 inreg %rhs) {
+;   %result = call i48 @llvm.usub.sat.i48(i48 %lhs, i48 %rhs)
+;   ret i48 %result
+; }
+
+; define amdgpu_ps <2 x float> @usubsat_i48_sv(i48 inreg %lhs, i48 %rhs) {
+;   %result = call i48 @llvm.usub.sat.i48(i48 %lhs, i48 %rhs)
+;   %ext.result = zext i48 %result to i64
+;   %cast = bitcast i64 %ext.result to <2 x float>
+;   ret <2 x float> %cast
+; }
+
+; define amdgpu_ps <2 x float> @usubsat_i48_vs(i48 %lhs, i48 inreg %rhs) {
+;   %result = call i48 @llvm.usub.sat.i48(i48 %lhs, i48 %rhs)
+;   %ext.result = zext i48 %result to i64
+;   %cast = bitcast i64 %ext.result to <2 x float>
+;   ret <2 x float> %cast
+; }
+
+define i64 @v_usubsat_i64(i64 %lhs, i64 %rhs) {
+; GFX6-LABEL: v_usubsat_i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, v0, v2
+; GFX6-NEXT:    v_subb_u32_e32 v5, vcc, v1, v3, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v5, 0, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_u32_e32 v4, vcc, v0, v2
+; GFX8-NEXT:    v_subb_u32_e32 v5, vcc, v1, v3, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v5, 0, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_co_u32_e32 v4, vcc, v0, v2
+; GFX9-NEXT:    v_subb_co_u32_e32 v5, vcc, v1, v3, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v5, 0, vcc
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_usubsat_i64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_sub_co_u32_e64 v4, vcc_lo, v0, v2
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_sub_co_ci_u32_e32 v5, vcc_lo, v1, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v5, 0, vcc_lo
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call i64 @llvm.usub.sat.i64(i64 %lhs, i64 %rhs)
+  ret i64 %result
+}
+
+define amdgpu_ps i64 @s_usubsat_i64(i64 inreg %lhs, i64 inreg %rhs) {
+; GFX6-LABEL: s_usubsat_i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_sub_u32 s4, s0, s2
+; GFX6-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX6-NEXT:    s_and_b32 s5, s5, 1
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    s_cmp_lg_u32 s5, 0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s3
+; GFX6-NEXT:    s_subb_u32 s5, s1, s3
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX6-NEXT:    v_mov_b32_e32 v2, s4
+; GFX6-NEXT:    v_mov_b32_e32 v3, s5
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v2, 0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v3, 0, vcc
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_usubsat_i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_sub_u32 s4, s0, s2
+; GFX8-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX8-NEXT:    s_and_b32 s5, s5, 1
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    s_cmp_lg_u32 s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    s_subb_u32 s5, s1, s3
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    v_mov_b32_e32 v3, s5
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v2, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v3, 0, vcc
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_usubsat_i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_sub_u32 s4, s0, s2
+; GFX9-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX9-NEXT:    s_and_b32 s5, s5, 1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    s_cmp_lg_u32 s5, 0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    s_subb_u32 s5, s1, s3
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v2, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v3, 0, vcc
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_usubsat_i64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_sub_u32 s4, s0, s2
+; GFX10-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s0, s[0:1], s[2:3]
+; GFX10-NEXT:    s_and_b32 s5, s5, 1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cmp_lg_u32 s5, 0
+; GFX10-NEXT:    s_subb_u32 s1, s1, s3
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, s4, 0, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, s1, 0, s0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i64 @llvm.usub.sat.i64(i64 %lhs, i64 %rhs)
+  ret i64 %result
+}
+
+define amdgpu_ps <2 x float> @usubsat_i64_sv(i64 inreg %lhs, i64 %rhs) {
+; GFX6-LABEL: usubsat_i64_sv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v2, s1
+; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, s0, v0
+; GFX6-NEXT:    v_subb_u32_e32 v2, vcc, v2, v1, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v3, 0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v2, 0, vcc
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: usubsat_i64_sv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    v_sub_u32_e32 v3, vcc, s0, v0
+; GFX8-NEXT:    v_subb_u32_e32 v2, vcc, v2, v1, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v3, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v2, 0, vcc
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: usubsat_i64_sv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_sub_co_u32_e32 v3, vcc, s0, v0
+; GFX9-NEXT:    v_subb_co_u32_e32 v2, vcc, v2, v1, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v3, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v2, 0, vcc
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: usubsat_i64_sv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_sub_co_u32_e64 v2, vcc_lo, s0, v0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_sub_co_ci_u32_e32 v3, vcc_lo, s1, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[0:1], v[0:1]
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v2, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v3, 0, vcc_lo
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i64 @llvm.usub.sat.i64(i64 %lhs, i64 %rhs)
+  %cast = bitcast i64 %result to <2 x float>
+  ret <2 x float> %cast
+}
+
+define amdgpu_ps <2 x float> @usubsat_i64_vs(i64 %lhs, i64 inreg %rhs) {
+; GFX6-LABEL: usubsat_i64_vs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v2, s1
+; GFX6-NEXT:    v_subrev_i32_e32 v3, vcc, s0, v0
+; GFX6-NEXT:    v_subb_u32_e32 v2, vcc, v1, v2, vcc
+; GFX6-NEXT:    v_cmp_gt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v3, 0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v2, 0, vcc
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: usubsat_i64_vs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    v_subrev_u32_e32 v3, vcc, s0, v0
+; GFX8-NEXT:    v_subb_u32_e32 v2, vcc, v1, v2, vcc
+; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v3, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v2, 0, vcc
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: usubsat_i64_vs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_subrev_co_u32_e32 v3, vcc, s0, v0
+; GFX9-NEXT:    v_subb_co_u32_e32 v2, vcc, v1, v2, vcc
+; GFX9-NEXT:    v_cmp_gt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v3, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v2, 0, vcc
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: usubsat_i64_vs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_sub_co_u32_e64 v2, vcc_lo, v0, s0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_subrev_co_ci_u32_e32 v3, vcc_lo, s1, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[0:1], v[0:1]
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v2, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v3, 0, vcc_lo
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i64 @llvm.usub.sat.i64(i64 %lhs, i64 %rhs)
+  %cast = bitcast i64 %result to <2 x float>
+  ret <2 x float> %cast
+}
+
+define <2 x i64> @v_usubsat_v2i64(<2 x i64> %lhs, <2 x i64> %rhs) {
+; GFX6-LABEL: v_usubsat_v2i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_sub_i32_e32 v8, vcc, v0, v4
+; GFX6-NEXT:    v_subb_u32_e32 v9, vcc, v1, v5, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, v[0:1], v[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v9, 0, vcc
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, v2, v6
+; GFX6-NEXT:    v_subb_u32_e32 v5, vcc, v3, v7, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, v[2:3], v[6:7]
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, v4, 0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, v5, 0, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_v2i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_u32_e32 v8, vcc, v0, v4
+; GFX8-NEXT:    v_subb_u32_e32 v9, vcc, v1, v5, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[0:1], v[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v9, 0, vcc
+; GFX8-NEXT:    v_sub_u32_e32 v4, vcc, v2, v6
+; GFX8-NEXT:    v_subb_u32_e32 v5, vcc, v3, v7, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[2:3], v[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v4, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v5, 0, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_v2i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_co_u32_e32 v8, vcc, v0, v4
+; GFX9-NEXT:    v_subb_co_u32_e32 v9, vcc, v1, v5, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[0:1], v[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v9, 0, vcc
+; GFX9-NEXT:    v_sub_co_u32_e32 v4, vcc, v2, v6
+; GFX9-NEXT:    v_subb_co_u32_e32 v5, vcc, v3, v7, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[2:3], v[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v4, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v5, 0, vcc
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_usubsat_v2i64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_mov_b32_e32 v10, v0
+; GFX10-NEXT:    v_mov_b32_e32 v11, v1
+; GFX10-NEXT:    v_mov_b32_e32 v0, v2
+; GFX10-NEXT:    v_mov_b32_e32 v1, v3
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_sub_co_u32_e64 v8, vcc_lo, v10, v4
+; GFX10-NEXT:    v_sub_co_ci_u32_e32 v9, vcc_lo, v11, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[10:11], v[4:5]
+; GFX10-NEXT:    v_sub_co_u32_e64 v4, s4, v0, v6
+; GFX10-NEXT:    v_sub_co_ci_u32_e64 v5, s4, v1, v7, s4
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s4, v[0:1], v[6:7]
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v8, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v9, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v4, 0, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v5, 0, s4
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> %lhs, <2 x i64> %rhs)
+  ret <2 x i64> %result
+}
+
+define amdgpu_ps <2 x i64> @s_usubsat_v2i64(<2 x i64> inreg %lhs, <2 x i64> inreg %rhs) {
+; GFX6-LABEL: s_usubsat_v2i64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_sub_u32 s8, s0, s4
+; GFX6-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX6-NEXT:    s_and_b32 s9, s9, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    s_subb_u32 s9, s1, s5
+; GFX6-NEXT:    v_mov_b32_e32 v1, s5
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX6-NEXT:    s_sub_u32 s0, s2, s6
+; GFX6-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX6-NEXT:    s_and_b32 s1, s1, 1
+; GFX6-NEXT:    v_mov_b32_e32 v0, s6
+; GFX6-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX6-NEXT:    v_mov_b32_e32 v2, s8
+; GFX6-NEXT:    v_mov_b32_e32 v3, s9
+; GFX6-NEXT:    v_mov_b32_e32 v1, s7
+; GFX6-NEXT:    s_subb_u32 s1, s3, s7
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, v2, 0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, v3, 0, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
+; GFX6-NEXT:    v_mov_b32_e32 v4, s0
+; GFX6-NEXT:    v_mov_b32_e32 v5, s1
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v5, 0, vcc
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v2
+; GFX6-NEXT:    v_readfirstlane_b32 s1, v3
+; GFX6-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s3, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_usubsat_v2i64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_sub_u32 s8, s0, s4
+; GFX8-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX8-NEXT:    s_and_b32 s9, s9, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    s_subb_u32 s9, s1, s5
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX8-NEXT:    s_sub_u32 s0, s2, s6
+; GFX8-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX8-NEXT:    s_and_b32 s1, s1, 1
+; GFX8-NEXT:    v_mov_b32_e32 v0, s6
+; GFX8-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s8
+; GFX8-NEXT:    v_mov_b32_e32 v3, s9
+; GFX8-NEXT:    v_mov_b32_e32 v1, s7
+; GFX8-NEXT:    s_subb_u32 s1, s3, s7
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, 0, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v5, 0, vcc
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v3
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_usubsat_v2i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_sub_u32 s8, s0, s4
+; GFX9-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX9-NEXT:    s_and_b32 s9, s9, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    s_subb_u32 s9, s1, s5
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX9-NEXT:    s_sub_u32 s0, s2, s6
+; GFX9-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX9-NEXT:    s_and_b32 s1, s1, 1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s6
+; GFX9-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s8
+; GFX9-NEXT:    v_mov_b32_e32 v3, s9
+; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    s_subb_u32 s1, s3, s7
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, 0, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v4, s0
+; GFX9-NEXT:    v_mov_b32_e32 v5, s1
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v5, 0, vcc
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v3
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_usubsat_v2i64:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_sub_u32 s8, s0, s4
+; GFX10-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_and_b32 s9, s9, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX10-NEXT:    s_subb_u32 s9, s1, s5
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s1, s[0:1], s[4:5]
+; GFX10-NEXT:    s_sub_u32 s0, s2, s6
+; GFX10-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s2, s[2:3], s[6:7]
+; GFX10-NEXT:    s_and_b32 s4, s4, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, s8, 0, s1
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, s9, 0, s1
+; GFX10-NEXT:    s_subb_u32 s1, s3, s7
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, s0, 0, s2
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, s1, 0, s2
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> %lhs, <2 x i64> %rhs)
+  ret <2 x i64> %result
+}
+
+define amdgpu_ps i128 @s_usubsat_i128(i128 inreg %lhs, i128 inreg %rhs) {
+; GFX6-LABEL: s_usubsat_i128:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_sub_u32 s8, s0, s4
+; GFX6-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX6-NEXT:    s_and_b32 s9, s9, 1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s4
+; GFX6-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX6-NEXT:    v_mov_b32_e32 v3, s5
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX6-NEXT:    s_subb_u32 s9, s1, s5
+; GFX6-NEXT:    v_mov_b32_e32 v0, s6
+; GFX6-NEXT:    s_cselect_b32 s10, 1, 0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s7
+; GFX6-NEXT:    s_and_b32 s10, s10, 1
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
+; GFX6-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX6-NEXT:    s_subb_u32 s10, s2, s6
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, s[2:3], v[0:1]
+; GFX6-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX6-NEXT:    s_and_b32 s11, s11, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX6-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX6-NEXT:    s_subb_u32 s11, s3, s7
+; GFX6-NEXT:    v_mov_b32_e32 v1, s8
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v2, s9
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v1, 0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v2, 0, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v2, s10
+; GFX6-NEXT:    v_mov_b32_e32 v3, s11
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, v2, 0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, v3, 0, vcc
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX6-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX6-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_usubsat_i128:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_sub_u32 s8, s0, s4
+; GFX8-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX8-NEXT:    s_and_b32 s9, s9, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX8-NEXT:    s_subb_u32 s9, s1, s5
+; GFX8-NEXT:    s_cselect_b32 s10, 1, 0
+; GFX8-NEXT:    s_and_b32 s10, s10, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX8-NEXT:    s_subb_u32 s10, s2, s6
+; GFX8-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX8-NEXT:    s_and_b32 s11, s11, 1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX8-NEXT:    v_mov_b32_e32 v3, s5
+; GFX8-NEXT:    s_subb_u32 s11, s3, s7
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s6
+; GFX8-NEXT:    v_mov_b32_e32 v1, s7
+; GFX8-NEXT:    s_cmp_eq_u64 s[2:3], s[6:7]
+; GFX8-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
+; GFX8-NEXT:    s_and_b32 s0, 1, s6
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s8
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s9
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v1, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v2, 0, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v2, s10
+; GFX8-NEXT:    v_mov_b32_e32 v3, s11
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, 0, vcc
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_usubsat_i128:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_sub_u32 s8, s0, s4
+; GFX9-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX9-NEXT:    s_and_b32 s9, s9, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX9-NEXT:    s_subb_u32 s9, s1, s5
+; GFX9-NEXT:    s_cselect_b32 s10, 1, 0
+; GFX9-NEXT:    s_and_b32 s10, s10, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX9-NEXT:    s_subb_u32 s10, s2, s6
+; GFX9-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX9-NEXT:    s_and_b32 s11, s11, 1
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
+; GFX9-NEXT:    s_subb_u32 s11, s3, s7
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v0, s6
+; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    s_cmp_eq_u64 s[2:3], s[6:7]
+; GFX9-NEXT:    s_cselect_b32 s6, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
+; GFX9-NEXT:    s_and_b32 s0, 1, s6
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s8
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s9
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v1, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v2, 0, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v2, s10
+; GFX9-NEXT:    v_mov_b32_e32 v3, s11
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, 0, vcc
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_usubsat_i128:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_sub_u32 s8, s0, s4
+; GFX10-NEXT:    s_cselect_b32 s9, 1, 0
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s0, s[0:1], s[4:5]
+; GFX10-NEXT:    s_and_b32 s9, s9, 1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cmp_lg_u32 s9, 0
+; GFX10-NEXT:    s_subb_u32 s9, s1, s5
+; GFX10-NEXT:    s_cselect_b32 s10, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX10-NEXT:    s_and_b32 s10, s10, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s10, 0
+; GFX10-NEXT:    s_subb_u32 s14, s2, s6
+; GFX10-NEXT:    s_cselect_b32 s11, 1, 0
+; GFX10-NEXT:    s_and_b32 s11, s11, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s11, 0
+; GFX10-NEXT:    s_subb_u32 s1, s3, s7
+; GFX10-NEXT:    s_cmp_eq_u64 s[2:3], s[6:7]
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s2, s[2:3], s[6:7]
+; GFX10-NEXT:    s_cselect_b32 s0, 1, 0
+; GFX10-NEXT:    s_and_b32 s0, 1, s0
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, s8, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, s14, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, s1, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, s9, 0, vcc_lo
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i128 @llvm.usub.sat.i128(i128 %lhs, i128 %rhs)
+  ret i128 %result
+}
+
+define amdgpu_ps <4 x float> @usubsat_i128_sv(i128 inreg %lhs, i128 %rhs) {
+; GFX6-LABEL: usubsat_i128_sv:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v4, s1
+; GFX6-NEXT:    v_sub_i32_e32 v5, vcc, s0, v0
+; GFX6-NEXT:    v_subb_u32_e32 v4, vcc, v4, v1, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v6, s2
+; GFX6-NEXT:    v_mov_b32_e32 v7, s3
+; GFX6-NEXT:    v_subb_u32_e32 v6, vcc, v6, v2, vcc
+; GFX6-NEXT:    v_subb_u32_e32 v7, vcc, v7, v3, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, s[2:3], v[2:3]
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX6-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v5, 0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v4, 0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, v6, 0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, v7, 0, vcc
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: usubsat_i128_sv:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v4, s1
+; GFX8-NEXT:    v_sub_u32_e32 v5, vcc, s0, v0
+; GFX8-NEXT:    v_subb_u32_e32 v4, vcc, v4, v1, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v6, s2
+; GFX8-NEXT:    v_mov_b32_e32 v7, s3
+; GFX8-NEXT:    v_subb_u32_e32 v6, vcc, v6, v2, vcc
+; GFX8-NEXT:    v_subb_u32_e32 v7, vcc, v7, v3, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v5, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v4, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v6, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v7, 0, vcc
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: usubsat_i128_sv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v4, s1
+; GFX9-NEXT:    v_sub_co_u32_e32 v5, vcc, s0, v0
+; GFX9-NEXT:    v_subb_co_u32_e32 v4, vcc, v4, v1, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v6, s2
+; GFX9-NEXT:    v_mov_b32_e32 v7, s3
+; GFX9-NEXT:    v_subb_co_u32_e32 v6, vcc, v6, v2, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v7, vcc, v7, v3, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, s[2:3], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v5, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v4, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v6, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v7, 0, vcc
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: usubsat_i128_sv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[0:1], v[0:1]
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[2:3], v[2:3]
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[2:3], v[2:3]
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc_lo
+; GFX10-NEXT:    v_sub_co_u32_e64 v0, vcc_lo, s0, v0
+; GFX10-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
+; GFX10-NEXT:    v_and_b32_e32 v4, 1, v4
+; GFX10-NEXT:    v_sub_co_ci_u32_e32 v2, vcc_lo, s2, v2, vcc_lo
+; GFX10-NEXT:    v_sub_co_ci_u32_e32 v3, vcc_lo, s3, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s0, 0, v4
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, 0, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, 0, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, 0, s0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i128 @llvm.usub.sat.i128(i128 %lhs, i128 %rhs)
+  %cast = bitcast i128 %result to <4 x float>
+  ret <4 x float> %cast
+}
+
+define amdgpu_ps <4 x float> @usubsat_i128_vs(i128 %lhs, i128 inreg %rhs) {
+; GFX6-LABEL: usubsat_i128_vs:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    v_mov_b32_e32 v4, s1
+; GFX6-NEXT:    v_subrev_i32_e32 v5, vcc, s0, v0
+; GFX6-NEXT:    v_subb_u32_e32 v4, vcc, v1, v4, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v6, s2
+; GFX6-NEXT:    v_mov_b32_e32 v7, s3
+; GFX6-NEXT:    v_subb_u32_e32 v6, vcc, v2, v6, vcc
+; GFX6-NEXT:    v_subb_u32_e32 v7, vcc, v3, v7, vcc
+; GFX6-NEXT:    v_cmp_gt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, s[2:3], v[2:3]
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX6-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v5, 0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v4, 0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, v6, 0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, v7, 0, vcc
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: usubsat_i128_vs:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    v_mov_b32_e32 v4, s1
+; GFX8-NEXT:    v_subrev_u32_e32 v5, vcc, s0, v0
+; GFX8-NEXT:    v_subb_u32_e32 v4, vcc, v1, v4, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v6, s2
+; GFX8-NEXT:    v_mov_b32_e32 v7, s3
+; GFX8-NEXT:    v_subb_u32_e32 v6, vcc, v2, v6, vcc
+; GFX8-NEXT:    v_subb_u32_e32 v7, vcc, v3, v7, vcc
+; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v5, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v4, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v6, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v7, 0, vcc
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: usubsat_i128_vs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v4, s1
+; GFX9-NEXT:    v_subrev_co_u32_e32 v5, vcc, s0, v0
+; GFX9-NEXT:    v_subb_co_u32_e32 v4, vcc, v1, v4, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v6, s2
+; GFX9-NEXT:    v_mov_b32_e32 v7, s3
+; GFX9-NEXT:    v_subb_co_u32_e32 v6, vcc, v2, v6, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v7, vcc, v3, v7, vcc
+; GFX9-NEXT:    v_cmp_gt_u64_e32 vcc, s[0:1], v[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, s[2:3], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v5, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v4, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v6, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v7, 0, vcc
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: usubsat_i128_vs:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[0:1], v[0:1]
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[2:3], v[2:3]
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[2:3], v[2:3]
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc_lo
+; GFX10-NEXT:    v_sub_co_u32_e64 v0, vcc_lo, v0, s0
+; GFX10-NEXT:    v_subrev_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
+; GFX10-NEXT:    v_and_b32_e32 v4, 1, v4
+; GFX10-NEXT:    v_subrev_co_ci_u32_e32 v2, vcc_lo, s2, v2, vcc_lo
+; GFX10-NEXT:    v_subrev_co_ci_u32_e32 v3, vcc_lo, s3, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s0, 0, v4
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, 0, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, 0, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, 0, s0
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call i128 @llvm.usub.sat.i128(i128 %lhs, i128 %rhs)
+  %cast = bitcast i128 %result to <4 x float>
+  ret <4 x float> %cast
+}
+
+define <2 x i128> @v_usubsat_v2i128(<2 x i128> %lhs, <2 x i128> %rhs) {
+; GFX6-LABEL: v_usubsat_v2i128:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_sub_i32_e32 v16, vcc, v0, v8
+; GFX6-NEXT:    v_subb_u32_e32 v17, vcc, v1, v9, vcc
+; GFX6-NEXT:    v_subb_u32_e32 v18, vcc, v2, v10, vcc
+; GFX6-NEXT:    v_subb_u32_e32 v19, vcc, v3, v11, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, v[0:1], v[8:9]
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, v[2:3], v[10:11]
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[10:11]
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX6-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v17, 0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, v18, 0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, v19, 0, vcc
+; GFX6-NEXT:    v_sub_i32_e32 v8, vcc, v4, v12
+; GFX6-NEXT:    v_subb_u32_e32 v9, vcc, v5, v13, vcc
+; GFX6-NEXT:    v_subb_u32_e32 v10, vcc, v6, v14, vcc
+; GFX6-NEXT:    v_subb_u32_e32 v11, vcc, v7, v15, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, v[4:5], v[12:13]
+; GFX6-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[14:15]
+; GFX6-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[14:15]
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
+; GFX6-NEXT:    v_and_b32_e32 v4, 1, v4
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; GFX6-NEXT:    v_cndmask_b32_e64 v4, v8, 0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v5, v9, 0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v6, v10, 0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v7, v11, 0, vcc
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_usubsat_v2i128:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_u32_e32 v16, vcc, v0, v8
+; GFX8-NEXT:    v_subb_u32_e32 v17, vcc, v1, v9, vcc
+; GFX8-NEXT:    v_subb_u32_e32 v18, vcc, v2, v10, vcc
+; GFX8-NEXT:    v_subb_u32_e32 v19, vcc, v3, v11, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[2:3], v[10:11]
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[10:11]
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v17, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v18, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v19, 0, vcc
+; GFX8-NEXT:    v_sub_u32_e32 v8, vcc, v4, v12
+; GFX8-NEXT:    v_subb_u32_e32 v9, vcc, v5, v13, vcc
+; GFX8-NEXT:    v_subb_u32_e32 v10, vcc, v6, v14, vcc
+; GFX8-NEXT:    v_subb_u32_e32 v11, vcc, v7, v15, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[4:5], v[12:13]
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[14:15]
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[14:15]
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
+; GFX8-NEXT:    v_and_b32_e32 v4, 1, v4
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, v8, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, v9, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v6, v10, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v7, v11, 0, vcc
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_usubsat_v2i128:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_co_u32_e32 v16, vcc, v0, v8
+; GFX9-NEXT:    v_subb_co_u32_e32 v17, vcc, v1, v9, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v18, vcc, v2, v10, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v19, vcc, v3, v11, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[0:1], v[8:9]
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[2:3], v[10:11]
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[10:11]
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v16, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v17, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v18, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v19, 0, vcc
+; GFX9-NEXT:    v_sub_co_u32_e32 v8, vcc, v4, v12
+; GFX9-NEXT:    v_subb_co_u32_e32 v9, vcc, v5, v13, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v10, vcc, v6, v14, vcc
+; GFX9-NEXT:    v_subb_co_u32_e32 v11, vcc, v7, v15, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[4:5], v[12:13]
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[14:15]
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[14:15]
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
+; GFX9-NEXT:    v_and_b32_e32 v4, 1, v4
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v8, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v9, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, v10, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v11, 0, vcc
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_usubsat_v2i128:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_mov_b32_e32 v22, v0
+; GFX10-NEXT:    v_mov_b32_e32 v23, v1
+; GFX10-NEXT:    v_mov_b32_e32 v20, v2
+; GFX10-NEXT:    v_mov_b32_e32 v21, v3
+; GFX10-NEXT:    v_mov_b32_e32 v26, v4
+; GFX10-NEXT:    v_mov_b32_e32 v27, v5
+; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[22:23], v[8:9]
+; GFX10-NEXT:    v_mov_b32_e32 v24, v6
+; GFX10-NEXT:    v_mov_b32_e32 v25, v7
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[20:21], v[10:11]
+; GFX10-NEXT:    v_cmp_eq_u64_e64 s5, v[24:25], v[14:15]
+; GFX10-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[20:21], v[10:11]
+; GFX10-NEXT:    v_cndmask_b32_e32 v16, v17, v16, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[26:27], v[12:13]
+; GFX10-NEXT:    v_and_b32_e32 v16, 1, v16
+; GFX10-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc_lo
+; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[24:25], v[14:15]
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s4, 0, v16
+; GFX10-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc_lo
+; GFX10-NEXT:    v_sub_co_u32_e64 v0, vcc_lo, v22, v8
+; GFX10-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, v23, v9, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v8, v18, v17, s5
+; GFX10-NEXT:    v_sub_co_ci_u32_e32 v2, vcc_lo, v20, v10, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s4
+; GFX10-NEXT:    v_sub_co_ci_u32_e32 v3, vcc_lo, v21, v11, vcc_lo
+; GFX10-NEXT:    v_sub_co_u32_e64 v4, vcc_lo, v26, v12
+; GFX10-NEXT:    v_and_b32_e32 v8, 1, v8
+; GFX10-NEXT:    v_sub_co_ci_u32_e32 v5, vcc_lo, v27, v13, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, 0, s4
+; GFX10-NEXT:    v_sub_co_ci_u32_e32 v6, vcc_lo, v24, v14, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, 0, s4
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s5, 0, v8
+; GFX10-NEXT:    v_sub_co_ci_u32_e32 v7, vcc_lo, v25, v15, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, 0, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, v4, 0, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, 0, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, 0, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v7, v7, 0, s5
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %result = call <2 x i128> @llvm.usub.sat.v2i128(<2 x i128> %lhs, <2 x i128> %rhs)
+  ret <2 x i128> %result
+}
+
+define amdgpu_ps <2 x i128> @s_usubsat_v2i128(<2 x i128> inreg %lhs, <2 x i128> inreg %rhs) {
+; GFX6-LABEL: s_usubsat_v2i128:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_sub_u32 s16, s0, s8
+; GFX6-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX6-NEXT:    s_and_b32 s17, s17, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX6-NEXT:    s_subb_u32 s17, s1, s9
+; GFX6-NEXT:    v_mov_b32_e32 v2, s8
+; GFX6-NEXT:    s_cselect_b32 s18, 1, 0
+; GFX6-NEXT:    v_mov_b32_e32 v3, s9
+; GFX6-NEXT:    s_and_b32 s18, s18, 1
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX6-NEXT:    v_mov_b32_e32 v0, s10
+; GFX6-NEXT:    s_cmp_lg_u32 s18, 0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s11
+; GFX6-NEXT:    s_subb_u32 s18, s2, s10
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
+; GFX6-NEXT:    s_cselect_b32 s19, 1, 0
+; GFX6-NEXT:    s_and_b32 s19, s19, 1
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, s[2:3], v[0:1]
+; GFX6-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX6-NEXT:    s_subb_u32 s19, s3, s11
+; GFX6-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX6-NEXT:    s_sub_u32 s0, s4, s12
+; GFX6-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v2, s17
+; GFX6-NEXT:    s_and_b32 s1, s1, 1
+; GFX6-NEXT:    v_mov_b32_e32 v1, s16
+; GFX6-NEXT:    v_cndmask_b32_e64 v5, v2, 0, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v2, s12
+; GFX6-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX6-NEXT:    v_cndmask_b32_e64 v4, v1, 0, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v0, s18
+; GFX6-NEXT:    v_mov_b32_e32 v1, s19
+; GFX6-NEXT:    v_mov_b32_e32 v3, s13
+; GFX6-NEXT:    s_subb_u32 s1, s5, s13
+; GFX6-NEXT:    v_cndmask_b32_e64 v6, v0, 0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v7, v1, 0, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
+; GFX6-NEXT:    v_mov_b32_e32 v0, s14
+; GFX6-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s15
+; GFX6-NEXT:    s_and_b32 s2, s2, 1
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
+; GFX6-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX6-NEXT:    s_subb_u32 s2, s6, s14
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
+; GFX6-NEXT:    v_cmp_eq_u64_e32 vcc, s[6:7], v[0:1]
+; GFX6-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
+; GFX6-NEXT:    s_and_b32 s3, s3, 1
+; GFX6-NEXT:    s_cmp_lg_u32 s3, 0
+; GFX6-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX6-NEXT:    s_subb_u32 s3, s7, s15
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v2, s1
+; GFX6-NEXT:    v_mov_b32_e32 v3, s3
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v1, 0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v2, 0, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v2, s2
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, v2, 0, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, v3, 0, vcc
+; GFX6-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX6-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX6-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX6-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX6-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX6-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX6-NEXT:    v_readfirstlane_b32 s6, v2
+; GFX6-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_usubsat_v2i128:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_sub_u32 s16, s0, s8
+; GFX8-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX8-NEXT:    s_and_b32 s17, s17, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX8-NEXT:    s_subb_u32 s17, s1, s9
+; GFX8-NEXT:    s_cselect_b32 s18, 1, 0
+; GFX8-NEXT:    s_and_b32 s18, s18, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s18, 0
+; GFX8-NEXT:    s_subb_u32 s18, s2, s10
+; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
+; GFX8-NEXT:    s_and_b32 s19, s19, 1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s8
+; GFX8-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX8-NEXT:    v_mov_b32_e32 v3, s9
+; GFX8-NEXT:    s_subb_u32 s19, s3, s11
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s10
+; GFX8-NEXT:    v_mov_b32_e32 v1, s11
+; GFX8-NEXT:    s_cmp_eq_u64 s[2:3], s[10:11]
+; GFX8-NEXT:    s_cselect_b32 s10, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
+; GFX8-NEXT:    s_and_b32 s0, 1, s10
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX8-NEXT:    s_sub_u32 s0, s4, s12
+; GFX8-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX8-NEXT:    s_and_b32 s1, s1, 1
+; GFX8-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX8-NEXT:    s_subb_u32 s1, s5, s13
+; GFX8-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX8-NEXT:    s_and_b32 s2, s2, 1
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX8-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX8-NEXT:    s_subb_u32 s2, s6, s14
+; GFX8-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s17
+; GFX8-NEXT:    s_and_b32 s3, s3, 1
+; GFX8-NEXT:    v_mov_b32_e32 v1, s16
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, v2, 0, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v2, s12
+; GFX8-NEXT:    s_cmp_lg_u32 s3, 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, v1, 0, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v0, s18
+; GFX8-NEXT:    v_mov_b32_e32 v1, s19
+; GFX8-NEXT:    v_mov_b32_e32 v3, s13
+; GFX8-NEXT:    s_subb_u32 s3, s7, s15
+; GFX8-NEXT:    v_cndmask_b32_e64 v6, v0, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v7, v1, 0, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s14
+; GFX8-NEXT:    s_cmp_eq_u64 s[6:7], s[14:15]
+; GFX8-NEXT:    v_mov_b32_e32 v1, s15
+; GFX8-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
+; GFX8-NEXT:    s_and_b32 s4, 1, s8
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX8-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s0
+; GFX8-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    v_mov_b32_e32 v3, s3
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v1, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v2, 0, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, 0, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, 0, vcc
+; GFX8-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX8-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX8-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX8-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX8-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX8-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX8-NEXT:    v_readfirstlane_b32 s6, v2
+; GFX8-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: s_usubsat_v2i128:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_sub_u32 s16, s0, s8
+; GFX9-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX9-NEXT:    s_and_b32 s17, s17, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX9-NEXT:    s_subb_u32 s17, s1, s9
+; GFX9-NEXT:    s_cselect_b32 s18, 1, 0
+; GFX9-NEXT:    s_and_b32 s18, s18, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s18, 0
+; GFX9-NEXT:    s_subb_u32 s18, s2, s10
+; GFX9-NEXT:    s_cselect_b32 s19, 1, 0
+; GFX9-NEXT:    s_and_b32 s19, s19, 1
+; GFX9-NEXT:    v_mov_b32_e32 v2, s8
+; GFX9-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX9-NEXT:    v_mov_b32_e32 v3, s9
+; GFX9-NEXT:    s_subb_u32 s19, s3, s11
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v0, s10
+; GFX9-NEXT:    v_mov_b32_e32 v1, s11
+; GFX9-NEXT:    s_cmp_eq_u64 s[2:3], s[10:11]
+; GFX9-NEXT:    s_cselect_b32 s10, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
+; GFX9-NEXT:    s_and_b32 s0, 1, s10
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s0
+; GFX9-NEXT:    s_sub_u32 s0, s4, s12
+; GFX9-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX9-NEXT:    s_and_b32 s1, s1, 1
+; GFX9-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX9-NEXT:    s_subb_u32 s1, s5, s13
+; GFX9-NEXT:    s_cselect_b32 s2, 1, 0
+; GFX9-NEXT:    s_and_b32 s2, s2, 1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    s_subb_u32 s2, s6, s14
+; GFX9-NEXT:    s_cselect_b32 s3, 1, 0
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s17
+; GFX9-NEXT:    s_and_b32 s3, s3, 1
+; GFX9-NEXT:    v_mov_b32_e32 v1, s16
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v2, 0, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v2, s12
+; GFX9-NEXT:    s_cmp_lg_u32 s3, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v1, 0, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v0, s18
+; GFX9-NEXT:    v_mov_b32_e32 v1, s19
+; GFX9-NEXT:    v_mov_b32_e32 v3, s13
+; GFX9-NEXT:    s_subb_u32 s3, s7, s15
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, v0, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v1, 0, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v0, s14
+; GFX9-NEXT:    s_cmp_eq_u64 s[6:7], s[14:15]
+; GFX9-NEXT:    v_mov_b32_e32 v1, s15
+; GFX9-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
+; GFX9-NEXT:    s_and_b32 s4, 1, s8
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 vcc, 0, s4
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s0
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v1, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v2, 0, vcc
+; GFX9-NEXT:    v_mov_b32_e32 v2, s2
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, 0, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, 0, vcc
+; GFX9-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX9-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX9-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX9-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX9-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX9-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s6, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: s_usubsat_v2i128:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_sub_u32 s16, s0, s8
+; GFX10-NEXT:    s_cselect_b32 s17, 1, 0
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s0, s[0:1], s[8:9]
+; GFX10-NEXT:    s_and_b32 s17, s17, 1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_cmp_lg_u32 s17, 0
+; GFX10-NEXT:    s_subb_u32 s17, s1, s9
+; GFX10-NEXT:    s_cselect_b32 s18, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX10-NEXT:    s_and_b32 s18, s18, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s18, 0
+; GFX10-NEXT:    s_subb_u32 s18, s2, s10
+; GFX10-NEXT:    s_cselect_b32 s19, 1, 0
+; GFX10-NEXT:    s_and_b32 s19, s19, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s19, 0
+; GFX10-NEXT:    s_subb_u32 s19, s3, s11
+; GFX10-NEXT:    s_cmp_eq_u64 s[2:3], s[10:11]
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s2, s[2:3], s[10:11]
+; GFX10-NEXT:    s_cselect_b32 s20, 1, 0
+; GFX10-NEXT:    s_and_b32 s0, 1, s20
+; GFX10-NEXT:    s_sub_u32 s8, s4, s12
+; GFX10-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s0
+; GFX10-NEXT:    s_and_b32 s1, s1, 1
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX10-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX10-NEXT:    s_subb_u32 s3, s5, s13
+; GFX10-NEXT:    s_cselect_b32 s1, 1, 0
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX10-NEXT:    s_and_b32 s1, s1, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s1, s[4:5], s[12:13]
+; GFX10-NEXT:    s_subb_u32 s30, s6, s14
+; GFX10-NEXT:    s_cselect_b32 s0, 1, 0
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX10-NEXT:    s_and_b32 s0, s0, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s0, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s1
+; GFX10-NEXT:    s_subb_u32 s9, s7, s15
+; GFX10-NEXT:    s_cmp_eq_u64 s[6:7], s[14:15]
+; GFX10-NEXT:    v_cmp_lt_u64_e64 s1, s[6:7], s[14:15]
+; GFX10-NEXT:    s_cselect_b32 s0, 1, 0
+; GFX10-NEXT:    s_and_b32 s0, 1, s0
+; GFX10-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s1
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v1
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, s16, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, s17, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, s18, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, s19, 0, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v3
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, s3, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, s8, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, s30, 0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, s9, 0, vcc_lo
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v4
+; GFX10-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s4, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s6, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s7, v3
+; GFX10-NEXT:    ; return to shader part epilog
+  %result = call <2 x i128> @llvm.usub.sat.v2i128(<2 x i128> %lhs, <2 x i128> %rhs)
+  ret <2 x i128> %result
+}
+
+declare i7 @llvm.usub.sat.i7(i7, i7) #0
+declare i8 @llvm.usub.sat.i8(i8, i8) #0
+declare <2 x i8> @llvm.usub.sat.v2i8(<2 x i8>, <2 x i8>) #0
+declare <4 x i8> @llvm.usub.sat.v4i8(<4 x i8>, <4 x i8>) #0
+
+declare i16 @llvm.usub.sat.i16(i16, i16) #0
+declare <2 x i16> @llvm.usub.sat.v2i16(<2 x i16>, <2 x i16>) #0
+declare <3 x i16> @llvm.usub.sat.v3i16(<3 x i16>, <3 x i16>) #0
+declare <4 x i16> @llvm.usub.sat.v4i16(<4 x i16>, <4 x i16>) #0
+declare <5 x i16> @llvm.usub.sat.v5i16(<5 x i16>, <5 x i16>) #0
+declare <6 x i16> @llvm.usub.sat.v6i16(<6 x i16>, <6 x i16>) #0
+declare <8 x i16> @llvm.usub.sat.v8i16(<8 x i16>, <8 x i16>) #0
+
+declare i24 @llvm.usub.sat.i24(i24, i24) #0
+
+declare i32 @llvm.usub.sat.i32(i32, i32) #0
+declare <2 x i32> @llvm.usub.sat.v2i32(<2 x i32>, <2 x i32>) #0
+declare <3 x i32> @llvm.usub.sat.v3i32(<3 x i32>, <3 x i32>) #0
+declare <4 x i32> @llvm.usub.sat.v4i32(<4 x i32>, <4 x i32>) #0
+declare <5 x i32> @llvm.usub.sat.v5i32(<5 x i32>, <5 x i32>) #0
+declare <16 x i32> @llvm.usub.sat.v16i32(<16 x i32>, <16 x i32>) #0
+
+declare i48 @llvm.usub.sat.i48(i48, i48) #0
+
+declare i64 @llvm.usub.sat.i64(i64, i64) #0
+declare <2 x i64> @llvm.usub.sat.v2i64(<2 x i64>, <2 x i64>) #0
+
+declare i128 @llvm.usub.sat.i128(i128, i128) #0
+declare <2 x i128> @llvm.usub.sat.v2i128(<2 x i128>, <2 x i128>) #0
+
+attributes #0 = { nounwind readnone speculatable willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/add.i16.ll b/llvm/test/CodeGen/AMDGPU/add.i16.ll
index 98848295a73b2..619cfb81e1e0d 100644
--- a/llvm/test/CodeGen/AMDGPU/add.i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/add.i16.ll
@@ -36,7 +36,7 @@ define amdgpu_kernel void @v_test_add_i16_constant(i16 addrspace(1)* %out, i16 a
 ; FIXME: Need to handle non-uniform case for function below (load without gep).
 ; GCN-LABEL: {{^}}v_test_add_i16_neg_constant:
 ; VI: flat_load_ushort [[A:v[0-9]+]]
-; VI: v_add_u16_e32 [[ADD:v[0-9]+]], 0xfffffcb3, [[A]]
+; VI: v_add_u16_e32 [[ADD:v[0-9]+]], 0xfcb3, [[A]]
 ; VI-NEXT: buffer_store_short [[ADD]]
 define amdgpu_kernel void @v_test_add_i16_neg_constant(i16 addrspace(1)* %out, i16 addrspace(1)* %in0) #1 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/add.v2i16.ll b/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
index 09f9abc23a6f7..5e85f88ed4302 100644
--- a/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
@@ -89,7 +89,7 @@ define amdgpu_kernel void @v_test_add_v2i16_constant(<2 x i16> addrspace(1)* %ou
 ; GFX9: s_mov_b32 [[CONST:s[0-9]+]], 0xfc21fcb3{{$}}
 ; GFX9: v_pk_add_u16 v{{[0-9]+}}, v{{[0-9]+}}, [[CONST]]
 
-; VI-DAG: v_add_u16_e32 v{{[0-9]+}}, 0xfffffcb3, v{{[0-9]+}}
+; VI-DAG: v_add_u16_e32 v{{[0-9]+}}, 0xfcb3, v{{[0-9]+}}
 ; VI-DAG: v_mov_b32_e32 v[[SCONST:[0-9]+]], 0xfffffc21
 ; VI-DAG: v_add_u16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v[[SCONST]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 define amdgpu_kernel void @v_test_add_v2i16_neg_constant(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in0) #1 {
diff --git a/llvm/test/CodeGen/AMDGPU/amdpal-elf.ll b/llvm/test/CodeGen/AMDGPU/amdpal-elf.ll
index 412815fb9c825..dfff47a926ff8 100644
--- a/llvm/test/CodeGen/AMDGPU/amdpal-elf.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdpal-elf.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -mtriple=amdgcn--amdpal -mcpu=kaveri -filetype=obj -mattr=-code-object-v3 | llvm-readobj -symbols -s -sd | FileCheck --check-prefix=ELF %s
-; RUN: llc < %s -mtriple=amdgcn--amdpal -mcpu=kaveri -mattr=-code-object-v3 | llvm-mc -filetype=obj -triple amdgcn--amdpal -mcpu=kaveri -mattr=-code-object-v3 | llvm-readobj -symbols -s -sd | FileCheck %s --check-prefix=ELF
+; RUN: llc < %s -mtriple=amdgcn--amdpal -mcpu=kaveri -filetype=obj -mattr=-code-object-v3 | llvm-readobj -symbols -s -sd - | FileCheck --check-prefix=ELF %s
+; RUN: llc < %s -mtriple=amdgcn--amdpal -mcpu=kaveri -mattr=-code-object-v3 | llvm-mc -filetype=obj -triple amdgcn--amdpal -mcpu=kaveri -mattr=-code-object-v3 | llvm-readobj -symbols -s -sd - | FileCheck %s --check-prefix=ELF
 ; RUN: llc < %s -mtriple=amdgcn--amdpal -mcpu=gfx1010 -mattr=+WavefrontSize32,-WavefrontSize64,-code-object-v3 | FileCheck --check-prefix=GFX10-W32 %s
 ; RUN: llc < %s -mtriple=amdgcn--amdpal -mcpu=gfx1010 -mattr=-WavefrontSize32,+WavefrontSize64,-code-object-v3 | FileCheck --check-prefix=GFX10-W64 %s
 
diff --git a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll
index f372fcb42667c..859fbc1e651a7 100644
--- a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll
+++ b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 < %s | FileCheck --check-prefix=CHECK %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -mattr=-code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck --check-prefix=HSAMD %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -mattr=-code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=HSAMD %s
 
 ; CHECK-LABEL: {{^}}min_64_max_64:
 ; CHECK: SGPRBlocks: 0
diff --git a/llvm/test/CodeGen/AMDGPU/code-object-v3.ll b/llvm/test/CodeGen/AMDGPU/code-object-v3.ll
index 2dc41a9659625..dafedf1dcd869 100644
--- a/llvm/test/CodeGen/AMDGPU/code-object-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/code-object-v3.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -mattr=+code-object-v3 < %s | FileCheck --check-prefixes=ALL-ASM,OSABI-AMDHSA-ASM %s
-; RUN: llc -filetype=obj -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -mattr=+code-object-v3 < %s | llvm-readelf --notes -relocations -sections -symbols | FileCheck --check-prefixes=ALL-ELF,OSABI-AMDHSA-ELF %s
+; RUN: llc -filetype=obj -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -mattr=+code-object-v3 < %s | llvm-readelf --notes -relocations -sections -symbols - | FileCheck --check-prefixes=ALL-ELF,OSABI-AMDHSA-ELF %s
 
 ; ALL-ASM-LABEL: {{^}}fadd:
 
diff --git a/llvm/test/CodeGen/AMDGPU/elf-notes.ll b/llvm/test/CodeGen/AMDGPU/elf-notes.ll
index a9b77d9713304..3a73b91249d51 100644
--- a/llvm/test/CodeGen/AMDGPU/elf-notes.ll
+++ b/llvm/test/CodeGen/AMDGPU/elf-notes.ll
@@ -1,12 +1,12 @@
 ; RUN: llc -mtriple=amdgcn-amd-unknown -mcpu=gfx802 -mattr=-code-object-v3 < %s | FileCheck --check-prefix=GCN --check-prefix=OSABI-UNK --check-prefix=GFX802 %s
 ; RUN: llc -mtriple=amdgcn-amd-unknown -mcpu=iceland -mattr=-code-object-v3 < %s | FileCheck --check-prefix=GCN --check-prefix=OSABI-UNK --check-prefix=GFX802 %s
-; RUN: llc -mtriple=amdgcn-amd-unknown -mcpu=gfx802 -filetype=obj -mattr=-code-object-v3 < %s | llvm-readelf --notes  | FileCheck --check-prefix=GCN --check-prefix=OSABI-UNK-ELF --check-prefix=GFX802 %s
+; RUN: llc -mtriple=amdgcn-amd-unknown -mcpu=gfx802 -filetype=obj -mattr=-code-object-v3 < %s | llvm-readelf --notes  - | FileCheck --check-prefix=GCN --check-prefix=OSABI-UNK-ELF --check-prefix=GFX802 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx802 -mattr=-code-object-v3 < %s | FileCheck --check-prefix=GCN --check-prefix=OSABI-HSA --check-prefix=GFX802 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=iceland -mattr=-code-object-v3 < %s | FileCheck --check-prefix=GCN --check-prefix=OSABI-HSA --check-prefix=GFX802 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx802 -filetype=obj -mattr=-code-object-v3 < %s | llvm-readelf --notes  | FileCheck --check-prefix=GCN --check-prefix=OSABI-HSA-ELF --check-prefix=GFX802 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx802 -filetype=obj -mattr=-code-object-v3 < %s | llvm-readelf --notes  - | FileCheck --check-prefix=GCN --check-prefix=OSABI-HSA-ELF --check-prefix=GFX802 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx802 -mattr=-code-object-v3 < %s | FileCheck --check-prefix=GCN --check-prefix=OSABI-PAL --check-prefix=GFX802 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=iceland -mattr=-code-object-v3 < %s | FileCheck --check-prefix=GCN --check-prefix=OSABI-PAL --check-prefix=GFX802 %s
-; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx802 -filetype=obj -mattr=-code-object-v3 < %s | llvm-readelf --notes  | FileCheck --check-prefix=GCN --check-prefix=OSABI-PAL-ELF --check-prefix=GFX802 %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx802 -filetype=obj -mattr=-code-object-v3 < %s | llvm-readelf --notes  - | FileCheck --check-prefix=GCN --check-prefix=OSABI-PAL-ELF --check-prefix=GFX802 %s
 ; RUN: llc -march=r600 < %s | FileCheck --check-prefix=R600 %s
 
 ; OSABI-UNK-NOT: .hsa_code_object_version
diff --git a/llvm/test/CodeGen/AMDGPU/fptosi.f16.ll b/llvm/test/CodeGen/AMDGPU/fptosi.f16.ll
index 57d8c62ec7ce6..7fa4f3c32ba8f 100644
--- a/llvm/test/CodeGen/AMDGPU/fptosi.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptosi.f16.ll
@@ -37,8 +37,11 @@ entry:
 ; test checks code generated for 'i64 = fp_to_sint f32'.
 
 ; GCN-LABEL: {{^}}fptosi_f16_to_i64
-; GCN: buffer_load_ushort
-; GCN: v_cvt_f32_f16_e32
+; GCN: buffer_load_ushort v[[A_F16:[0-9]+]]
+; GCN: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]]
+; GCN: v_cvt_i32_f32_e32 v[[R_I64_Low:[0-9]+]], v[[A_F32]]
+; GCN: v_ashrrev_i32_e32 v[[R_I64_High:[0-9]+]], 31, v[[R_I64_Low]]
+; GCN: buffer_store_dwordx2 v{{\[}}[[R_I64_Low]]{{\:}}[[R_I64_High]]{{\]}}
 ; GCN: s_endpgm
 define amdgpu_kernel void @fptosi_f16_to_i64(
     i64 addrspace(1)* %r,
@@ -104,10 +107,21 @@ entry:
 ; test checks code generated for 'i64 = fp_to_sint f32'.
 
 ; GCN-LABEL: {{^}}fptosi_v2f16_to_v2i64
-; GCN: buffer_load_dword
-; GCN: v_cvt_f32_f16_e32
-; SI: v_cvt_f32_f16_e32
-; VI: v_cvt_f32_f16_sdwa
+; GCN: buffer_load_dword v[[A_F16_0:[0-9]+]]
+; SI: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_F16_0]]
+; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_F16_0]]
+; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]]
+; SI: v_cvt_i32_f32_e32 v[[R_I64_0_Low:[0-9]+]], v[[A_F32_0]]
+; SI: v_ashrrev_i32_e32 v[[R_I64_0_High:[0-9]+]], 31, v[[R_I64_0_Low]]
+; SI: v_cvt_i32_f32_e32 v[[R_I64_1_Low:[0-9]+]], v[[A_F32_1]]
+; SI: v_ashrrev_i32_e32 v[[R_I64_1_High:[0-9]+]], 31, v[[R_I64_1_Low]]
+; VI: v_cvt_f32_f16_sdwa v[[A_F32_1:[0-9]+]], v[[A_F16_0]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_F16_0]]
+; VI: v_cvt_i32_f32_e32 v[[R_I64_1_Low:[0-9]+]], v[[A_F32_1]]
+; VI: v_cvt_i32_f32_e32 v[[R_I64_0_Low:[0-9]+]], v[[A_F32_0]]
+; VI: v_ashrrev_i32_e32 v[[R_I64_1_High:[0-9]+]], 31, v[[R_I64_1_Low]]
+; VI: v_ashrrev_i32_e32 v[[R_I64_0_High:[0-9]+]], 31, v[[R_I64_0_Low]]
+; GCN: buffer_store_dwordx4 v{{\[}}[[R_I64_0_Low]]{{\:}}[[R_I64_1_High]]{{\]}}
 ; GCN: s_endpgm
 define amdgpu_kernel void @fptosi_v2f16_to_v2i64(
     <2 x i64> addrspace(1)* %r,
diff --git a/llvm/test/CodeGen/AMDGPU/fptoui.f16.ll b/llvm/test/CodeGen/AMDGPU/fptoui.f16.ll
index e2c8aacfb2ac7..11c946b308b09 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoui.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoui.f16.ll
@@ -38,8 +38,11 @@ entry:
 ; test checks code generated for 'i64 = fp_to_uint f32'.
 
 ; GCN-LABEL: {{^}}fptoui_f16_to_i64
-; GCN: buffer_load_ushort
-; GCN: v_cvt_f32_f16_e32
+; GCN: buffer_load_ushort v[[A_F16:[0-9]+]]
+; GCN: v_mov_b32_e32 v[[R_I64_High:[0-9]+]], 0
+; GCN: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]]
+; GCN: v_cvt_u32_f32_e32 v[[R_I64_Low:[0-9]+]], v[[A_F32]]
+; GCN: buffer_store_dwordx2 v{{\[}}[[R_I64_Low]]{{\:}}[[R_I64_High]]{{\]}}
 ; GCN: s_endpgm
 define amdgpu_kernel void @fptoui_f16_to_i64(
     i64 addrspace(1)* %r,
@@ -104,10 +107,19 @@ entry:
 ; test checks code generated for 'i64 = fp_to_uint f32'.
 
 ; GCN-LABEL: {{^}}fptoui_v2f16_to_v2i64
-; GCN: buffer_load_dword
-; GCN: v_cvt_f32_f16_e32
-; SI: v_cvt_f32_f16_e32
-; VI: v_cvt_f32_f16_sdwa
+; GCN: buffer_load_dword v[[A_F16_0:[0-9]+]]
+; GCN: v_mov_b32_e32 v[[R_I64_1_High:[0-9]+]], 0
+; SI: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_F16_0]]
+; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_F16_0]]
+; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]]
+; SI: v_cvt_u32_f32_e32 v[[R_I64_0_Low:[0-9]+]], v[[A_F32_0]]
+; SI: v_cvt_u32_f32_e32 v[[R_I64_1_Low:[0-9]+]], v[[A_F32_1]]
+; VI: v_cvt_f32_f16_sdwa v[[A_F32_1:[0-9]+]], v[[A_F16_0]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_F16_0]]
+; VI: v_cvt_u32_f32_e32 v[[R_I64_1_Low:[0-9]+]], v[[A_F32_1]]
+; VI: v_cvt_u32_f32_e32 v[[R_I64_0_Low:[0-9]+]], v[[A_F32_0]]
+; GCN: v_mov_b32_e32 v[[R_I64_0_High:[0-9]+]], 0
+; GCN: buffer_store_dwordx4 v{{\[}}[[R_I64_0_Low]]{{\:}}[[R_I64_1_High]]{{\]}}
 ; GCN: s_endpgm
 define amdgpu_kernel void @fptoui_v2f16_to_v2i64(
     <2 x i64> addrspace(1)* %r,
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-func-align.ll b/llvm/test/CodeGen/AMDGPU/hsa-func-align.ll
index 005540092c93c..db0f60e924e3e 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-func-align.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-func-align.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri < %s | FileCheck -check-prefix=HSA %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -filetype=obj < %s | llvm-readobj --symbols -S --sd | FileCheck -check-prefix=ELF %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -filetype=obj < %s | llvm-readobj --symbols -S --sd - | FileCheck -check-prefix=ELF %s
 
 ; ELF: Section {
 ; ELF: Name: .text
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-func.ll b/llvm/test/CodeGen/AMDGPU/hsa-func.ll
index b04ad1ce96371..4409384b0cb51 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-func.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-func.ll
@@ -2,8 +2,8 @@
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mattr=-code-object-v3 -mcpu=kaveri | FileCheck --check-prefix=HSA-CI %s
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mattr=-code-object-v3 -mcpu=carrizo  | FileCheck --check-prefix=HSA %s
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mattr=-code-object-v3 -mcpu=carrizo | FileCheck --check-prefix=HSA-VI %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mattr=-code-object-v3 -mcpu=kaveri -filetype=obj | llvm-readobj --symbols -S --sd | FileCheck --check-prefix=ELF %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mattr=-code-object-v3 -mcpu=kaveri | llvm-mc -filetype=obj -triple amdgcn--amdhsa -mattr=-code-object-v3 -mcpu=kaveri | llvm-readobj --symbols -S --sd | FileCheck %s --check-prefix=ELF
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mattr=-code-object-v3 -mcpu=kaveri -filetype=obj | llvm-readobj --symbols -S --sd - | FileCheck --check-prefix=ELF %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mattr=-code-object-v3 -mcpu=kaveri | llvm-mc -filetype=obj -triple amdgcn--amdhsa -mattr=-code-object-v3 -mcpu=kaveri | llvm-readobj --symbols -S --sd - | FileCheck %s --check-prefix=ELF
 
 ; The SHT_NOTE section contains the output from the .hsa_code_object_*
 ; directives.
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-deduce-ro-arg-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-deduce-ro-arg-v3.ll
index 4afa9f35499f1..90714fa8bdd92 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-deduce-ro-arg-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-deduce-ro-arg-v3.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mattr=+code-object-v3 -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck %s
+; RUN: llc -mattr=+code-object-v3 -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck %s
 
 ; CHECK:        - .args:
 ; CHECK-NEXT:       - .access:         read_only
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-deduce-ro-arg.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-deduce-ro-arg.ll
index 6e22f85f453f1..c4374452b4947 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-deduce-ro-arg.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-deduce-ro-arg.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck %s
 
 ; CHECK:      - Name:            test_ro_arg
 ; CHECK-NEXT:   SymbolName:      'test_ro_arg@kd'
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-enqueue-kernel-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-enqueue-kernel-v3.ll
index 5f77fef31bfa8..5768fa1f03085 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-enqueue-kernel-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-enqueue-kernel-v3.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=+code-object-v3 -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=+code-object-v3 -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=+code-object-v3 -mcpu=gfx900 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
 
 ; CHECK:              ---
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-enqueue-kernel.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-enqueue-kernel.ll
index 086c810c9ded3..4ea88155c8ac8 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-enqueue-kernel.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-enqueue-kernel.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=-code-object-v3 -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=-code-object-v3 -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=-code-object-v3 -mcpu=gfx900 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
 
 ; CHECK: ---
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full-v3.ll
index ef441b9c3cc26..2c392c7c31b6c 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full-v3.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -mattr=+code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck --check-prefix=CHECK --check-prefix=GFX700 --check-prefix=NOTES %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx802 -mattr=+code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck --check-prefix=CHECK --check-prefix=GFX802 --check-prefix=NOTES %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -mattr=+code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX700 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx802 -mattr=+code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX802 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -mattr=+code-object-v3 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx802 -mattr=+code-object-v3 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+code-object-v3 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
@@ -53,6 +53,80 @@ define amdgpu_kernel void @test_char(i8 %a) #0
   ret void
 }
 
+; CHECK:        - .args:
+; CHECK-NEXT:       - .name:           a
+; CHECK-NEXT:         .offset:         0
+; CHECK-NEXT:         .size:           1
+; CHECK-NEXT:         .type_name:      char
+; CHECK-NEXT:         .value_kind:     by_value
+; CHECK-NEXT:       - .offset:         8
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
+; CHECK-NEXT:       - .offset:         16
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
+; CHECK-NEXT:       - .offset:         24
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
+; CHECK-NEXT:       - .address_space:  global
+; CHECK-NEXT:         .offset:         32
+; CHECK-NEXT:         .size:           8
+; CHECK-NOT:          .value_kind:     hidden_default_queue
+; CHECK-NOT:          .value_kind:     hidden_completion_action
+; CHECK-NOT:          .value_kind:     hidden_hostcall_buffer
+; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
+; CHECK:              .value_kind:     hidden_multigrid_sync_arg
+; CHECK:          .language:       OpenCL C
+; CHECK-NEXT:     .language_version:
+; CHECK-NEXT:       - 2
+; CHECK-NEXT:       - 0
+; CHECK:          .name:           test_char_byref_constant
+; CHECK:          .symbol:         test_char_byref_constant.kd
+define amdgpu_kernel void @test_char_byref_constant(i8 addrspace(4)* byref(i8) %a) #0
+    !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !9
+    !kernel_arg_base_type !9 !kernel_arg_type_qual !4 {
+  ret void
+}
+
+; CHECK:        - .args:
+; CHECK-NEXT:       - .offset:         0
+; CHECK-NEXT:         .size:           1
+; CHECK-NEXT:         .type_name:      char
+; CHECK-NEXT:         .value_kind:     by_value
+; CHECK-NEXT:       - .name:           a
+; CHECK-NEXT:         .offset:         512
+; CHECK-NEXT:         .size:           1
+; CHECK-NEXT:         .type_name:      char
+; CHECK-NEXT:         .value_kind:     by_value
+; CHECK-NEXT:       - .offset:         520
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
+; CHECK-NEXT:       - .offset:         528
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
+; CHECK-NEXT:       - .offset:         536
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
+; CHECK-NEXT:       - .address_space:  global
+; CHECK-NEXT:         .offset:         544
+; CHECK-NEXT:         .size:           8
+; CHECK-NOT:          .value_kind:     hidden_default_queue
+; CHECK-NOT:          .value_kind:     hidden_completion_action
+; CHECK-NOT:          .value_kind:     hidden_hostcall_buffer
+; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
+; CHECK:              .value_kind:     hidden_multigrid_sync_arg
+; CHECK:          .language:       OpenCL C
+; CHECK-NEXT:     .language_version:
+; CHECK-NEXT:       - 2
+; CHECK-NEXT:       - 0
+; CHECK:          .name:           test_char_byref_constant_align512
+; CHECK:          .symbol:         test_char_byref_constant_align512.kd
+define amdgpu_kernel void @test_char_byref_constant_align512(i8, i8 addrspace(4)* byref(i8) align(512) %a) #0
+    !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !111
+    !kernel_arg_base_type !9 !kernel_arg_type_qual !4 {
+  ret void
+}
+
 ; CHECK:        - .args:
 ; CHECK-NEXT:       - .name:           a
 ; CHECK-NEXT:         .offset:         0
@@ -529,6 +603,49 @@ define amdgpu_kernel void @test_struct(%struct.A %a) #0
   ret void
 }
 
+; CHECK:        - .args:
+; CHECK-NEXT:         .name:           a
+; CHECK-NEXT:         .offset:         0
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .type_name:      struct A
+; CHECK-NEXT:         .value_kind:     by_value
+; CHECK-NEXT:       - .offset:         8
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
+; CHECK-NEXT:       - .offset:         16
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
+; CHECK-NEXT:       - .offset:         24
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
+; CHECK-NEXT:       - .address_space:  global
+; CHECK-NEXT:         .offset:         32
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
+; CHECK-NEXT:       - .address_space:  global
+; CHECK-NEXT:         .offset:         40
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_none
+; CHECK-NEXT:       - .address_space:  global
+; CHECK-NEXT:         .offset:         48
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_none
+; CHECK-NEXT:       - .address_space:  global
+; CHECK-NEXT:         .offset:         56
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
+; CHECK:          .language:       OpenCL C
+; CHECK-NEXT:     .language_version:
+; CHECK-NEXT:       - 2
+; CHECK-NEXT:       - 0
+; CHECK:          .name:           test_struct_byref_constant
+; CHECK:          .symbol:         test_struct_byref_constant.kd
+define amdgpu_kernel void @test_struct_byref_constant(%struct.A addrspace(4)* byref(%struct.A) %a) #0
+    !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !20
+    !kernel_arg_base_type !20 !kernel_arg_type_qual !4 {
+  ret void
+}
+
 ; CHECK:        - .args:
 ; CHECK-NEXT:         .name:           a
 ; CHECK-NEXT:         .offset:         0
@@ -572,6 +689,49 @@ define amdgpu_kernel void @test_array([32 x i8] %a) #0
   ret void
 }
 
+; CHECK:        - .args:
+; CHECK-NEXT:         .name:           a
+; CHECK-NEXT:         .offset:         0
+; CHECK-NEXT:         .size:           32
+; CHECK-NEXT:         .type_name:      struct A
+; CHECK-NEXT:         .value_kind:     by_value
+; CHECK-NEXT:       - .offset:         32
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_global_offset_x
+; CHECK-NEXT:       - .offset:         40
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_global_offset_y
+; CHECK-NEXT:       - .offset:         48
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_global_offset_z
+; CHECK-NEXT:       - .address_space:  global
+; CHECK-NEXT:         .offset:         56
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_printf_buffer
+; CHECK-NEXT:       - .address_space:  global
+; CHECK-NEXT:         .offset:         64
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_none
+; CHECK-NEXT:       - .address_space:  global
+; CHECK-NEXT:         .offset:         72
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_none
+; CHECK-NEXT:       - .address_space:  global
+; CHECK-NEXT:         .offset:         80
+; CHECK-NEXT:         .size:           8
+; CHECK-NEXT:         .value_kind:     hidden_multigrid_sync_arg
+; CHECK:          .language:       OpenCL C
+; CHECK-NEXT:     .language_version:
+; CHECK-NEXT:       - 2
+; CHECK-NEXT:       - 0
+; CHECK:          .name:           test_array_byref_constant
+; CHECK:          .symbol:         test_array_byref_constant.kd
+define amdgpu_kernel void @test_array_byref_constant([32 x i8] addrspace(4)* byref([32 x i8]) %a) #0
+    !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !20
+    !kernel_arg_base_type !20 !kernel_arg_type_qual !4 {
+  ret void
+}
+
 ; CHECK:        - .args:
 ; CHECK-NEXT:       - .name:           a
 ; CHECK-NEXT:         .offset:         0
@@ -1792,5 +1952,6 @@ attributes #2 = { "amdgpu-implicitarg-num-bytes"="56" "calls-enqueue-kernel" }
 !100 = !{!"1:1:4:%d\5Cn"}
 !101 = !{!"2:1:8:%g\5Cn"}
 !110 = !{!"__block_literal"}
+!111 = !{!"char", !"char"}
 
 ; PARSER: AMDGPU HSA Metadata Parser Test: PASS
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full.ll
index 8969c7b45bbbd..5c5960f8b1fe7 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -mattr=-code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck --check-prefix=CHECK --check-prefix=GFX700 --check-prefix=NOTES %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx802 -mattr=-code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck --check-prefix=CHECK --check-prefix=GFX802 --check-prefix=NOTES %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=-code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -mattr=-code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX700 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx802 -mattr=-code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX802 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=-code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -mattr=-code-object-v3 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx802 -mattr=-code-object-v3 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=-code-object-v3 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
@@ -58,6 +58,80 @@ define amdgpu_kernel void @test_char(i8 %a) #0
   ret void
 }
 
+; CHECK:      - Name:            test_char_byref_constant
+; CHECK-NEXT:   SymbolName:      'test_char_byref_constant@kd'
+; CHECK-NEXT:   Language:        OpenCL C
+; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
+; CHECK-NEXT:   Args:
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      char
+; CHECK-NEXT:       Size:          1
+; CHECK-NEXT:       Align:         1
+; CHECK-NEXT:       ValueKind:     ByValue
+; CHECK-NEXT:       AccQual:       Default
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NOT:        ValueKind:     HiddenHostcallBuffer
+; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
+; CHECK-NEXT:       AddrSpaceQual: Global
+; CHECK-NOT:        ValueKind:     HiddenDefaultQueue
+; CHECK-NOT:        ValueKind:     HiddenCompletionAction
+; CHECK:            ValueKind:     HiddenMultiGridSyncArg
+; CHECK-NEXT:       AddrSpaceQual: Global
+define amdgpu_kernel void @test_char_byref_constant(i8 addrspace(4)* byref(i8) %a) #0
+    !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !9
+    !kernel_arg_base_type !9 !kernel_arg_type_qual !4 {
+  ret void
+}
+
+; CHECK:      - Name:            test_char_byref_constant_align512
+; CHECK-NEXT:   SymbolName:      'test_char_byref_constant_align512@kd'
+; CHECK-NEXT:   Language:        OpenCL C
+; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
+; CHECK-NEXT:   Args:
+; CHECK-NEXT:       TypeName:      char
+; CHECK-NEXT:       Size:          1
+; CHECK-NEXT:       Align:         1
+; CHECK-NEXT:       ValueKind:     ByValue
+; CHECK-NEXT:       AccQual:       Default
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      char
+; CHECK-NEXT:       Size:          1
+; CHECK-NEXT:       Align:         512
+; CHECK-NEXT:       ValueKind:     ByValue
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NOT:        ValueKind:     HiddenHostcallBuffer
+; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
+; CHECK-NEXT:       AddrSpaceQual: Global
+; CHECK-NOT:        ValueKind:     HiddenDefaultQueue
+; CHECK-NOT:        ValueKind:     HiddenCompletionAction
+; CHECK:            ValueKind:     HiddenMultiGridSyncArg
+; CHECK-NEXT:       AddrSpaceQual: Global
+define amdgpu_kernel void @test_char_byref_constant_align512(i8, i8 addrspace(4)* byref(i8) align 512 %a) #0
+    !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !111
+    !kernel_arg_base_type !9 !kernel_arg_type_qual !4 {
+  ret void
+}
+
 ; CHECK:      - Name:            test_ushort2
 ; CHECK-NEXT:   SymbolName:      'test_ushort2@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
@@ -523,6 +597,48 @@ define amdgpu_kernel void @test_struct(%struct.A %a) #0
   ret void
 }
 
+; CHECK:      - Name:            test_struct_byref_constant
+; CHECK-NEXT:   SymbolName:      'test_struct_byref_constant@kd'
+; CHECK-NEXT:   Language:        OpenCL C
+; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
+; CHECK-NEXT:   Args:
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      struct A
+; CHECK-NEXT:       Size:          8
+; CHECK-NEXT:       Align:         4
+; CHECK-NEXT:       ValueKind:     ByValue
+; CHECK-NEXT:       AccQual:       Default
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
+; CHECK-NEXT:       AddrSpaceQual: Global
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenNone
+; CHECK-NEXT:       AddrSpaceQual: Global
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenNone
+; CHECK-NEXT:       AddrSpaceQual: Global
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
+; CHECK-NEXT:       AddrSpaceQual: Global
+define amdgpu_kernel void @test_struct_byref_constant(%struct.A addrspace(4)* byref(%struct.A) %a) #0
+    !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !20
+    !kernel_arg_base_type !20 !kernel_arg_type_qual !4 {
+  ret void
+}
+
 ; CHECK:      - Name:            test_array
 ; CHECK-NEXT:   SymbolName:      'test_array@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
@@ -565,6 +681,48 @@ define amdgpu_kernel void @test_array([8 x i8] %a) #0
   ret void
 }
 
+; CHECK:      - Name:            test_array_byref_constant
+; CHECK-NEXT:   SymbolName:      'test_array_byref_constant@kd'
+; CHECK-NEXT:   Language:        OpenCL C
+; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
+; CHECK-NEXT:   Args:
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      struct A
+; CHECK-NEXT:       Size:          8
+; CHECK-NEXT:       Align:         1
+; CHECK-NEXT:       ValueKind:     ByValue
+; CHECK-NEXT:       AccQual:       Default
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
+; CHECK-NEXT:       AddrSpaceQual: Global
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenNone
+; CHECK-NEXT:       AddrSpaceQual: Global
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenNone
+; CHECK-NEXT:       AddrSpaceQual: Global
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenMultiGridSyncArg
+; CHECK-NEXT:       AddrSpaceQual: Global
+define amdgpu_kernel void @test_array_byref_constant([8 x i8] addrspace(4)* byref([8 x i8]) %a) #0
+    !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !20
+    !kernel_arg_base_type !20 !kernel_arg_type_qual !4 {
+  ret void
+}
+
 ; CHECK:      - Name:            test_i128
 ; CHECK-NEXT:   SymbolName:      'test_i128@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
@@ -1766,5 +1924,5 @@ attributes #2 = { "amdgpu-implicitarg-num-bytes"="56" "calls-enqueue-kernel" }
 !100 = !{!"1:1:4:%d\5Cn"}
 !101 = !{!"2:1:8:%g\5Cn"}
 !110 = !{!"__block_literal"}
-
+!111 = !{!"char", !"char"}
 ; PARSER: AMDGPU HSA Metadata Parser Test: PASS
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args-v3.ll
index 67e2bfce50356..235530c5cfd33 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args-v3.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -mattr=+code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck --check-prefix=CHECK --check-prefix=GFX700 --check-prefix=NOTES %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -mattr=+code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck --check-prefix=CHECK --check-prefix=GFX803 --check-prefix=NOTES %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -mattr=+code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX700 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -mattr=+code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX803 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
 
 ; CHECK:              ---
 ; CHECK:      amdhsa.kernels:
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args.ll
index 91e382dcce5fb..5322c60d68d64 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hidden-args.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -mattr=-code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck --check-prefix=CHECK --check-prefix=GFX700 --check-prefix=NOTES %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -mattr=-code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck --check-prefix=CHECK --check-prefix=GFX803 --check-prefix=NOTES %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=-code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -mattr=-code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX700 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -mattr=-code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX803 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=-code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
 
 ; CHECK: ---
 ; CHECK:  Version: [ 1, 0 ]
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent-v3.ll
index c52e09158eeb8..1a81d8f90bb4f 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent-v3.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=+code-object-v3 -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=+code-object-v3 -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=+code-object-v3 -mcpu=gfx900 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
 
 ; CHECK:              ---
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent.ll
index fb5874acbe7d7..4f843b28851a3 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-absent.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=-code-object-v3 -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=-code-object-v3 -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=-code-object-v3 -mcpu=gfx900 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
 
 ; CHECK: ---
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present-v3.ll
index 8fd62c89a6033..f4cff1a5b20dc 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present-v3.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=+code-object-v3 -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=+code-object-v3 -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=+code-object-v3 -mcpu=gfx900 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
 
 ; CHECK:              ---
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present.ll
index e8db4d2a866eb..277055c8cb9a6 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-hostcall-present.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=-code-object-v3 -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=-code-object-v3 -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=-code-object-v3 -mcpu=gfx900 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
 
 ; CHECK: ---
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-images-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-images-v3.ll
index 5ee17a8c4855a..17f419dbdf562 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-images-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-images-v3.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mattr=+code-object-v3 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck --check-prefix=CHECK --check-prefix=GFX700 --check-prefix=NOTES %s
-; RUN: llc -mattr=+code-object-v3 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx802 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck --check-prefix=CHECK --check-prefix=GFX802 --check-prefix=NOTES %s
-; RUN: llc -mattr=+code-object-v3 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
+; RUN: llc -mattr=+code-object-v3 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX700 --check-prefix=NOTES %s
+; RUN: llc -mattr=+code-object-v3 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx802 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX802 --check-prefix=NOTES %s
+; RUN: llc -mattr=+code-object-v3 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
 
 %opencl.image1d_t = type opaque
 %opencl.image1d_array_t = type opaque
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-images.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-images.ll
index 79e0cfcffd812..d843a0190013d 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-images.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-images.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -mattr=-code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck --check-prefix=CHECK --check-prefix=GFX700 --check-prefix=NOTES %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx802 -mattr=-code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck --check-prefix=CHECK --check-prefix=GFX802 --check-prefix=NOTES %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=-code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -mattr=-code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX700 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx802 -mattr=-code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX802 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=-code-object-v3 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
 
 %opencl.image1d_t = type opaque
 %opencl.image1d_array_t = type opaque
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-1-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-1-v3.ll
index 1c6103275c92c..95f8cde2c8b0d 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-1-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-1-v3.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mattr=+code-object-v3 -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck %s
+; RUN: llc -mattr=+code-object-v3 -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck %s
 
 ; Make sure llc does not crash for invalid opencl version metadata.
 
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-1.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-1.ll
index 6ce365efbc60f..1fbb097ef65fa 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-1.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-1.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck %s
 
 ; Make sure llc does not crash for invalid opencl version metadata.
 
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-2-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-2-v3.ll
index 5f55ffb55c54d..d9c1bebf1e29d 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-2-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-2-v3.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mattr=+code-object-v3 -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck %s
+; RUN: llc -mattr=+code-object-v3 -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck %s
 
 ; Make sure llc does not crash for invalid opencl version metadata.
 
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-2.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-2.ll
index 74a5413aa610e..89f6afb246e2e 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-2.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-2.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck %s
 
 ; Make sure llc does not crash for invalid opencl version metadata.
 
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-3-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-3-v3.ll
index 31c77f0a031e9..0ab0df27c5cb9 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-3-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-3-v3.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mattr=+code-object-v3 -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck %s
+; RUN: llc -mattr=+code-object-v3 -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck %s
 
 ; Make sure llc does not crash for invalid opencl version metadata.
 
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-3.ll
index ca6a3d5d32643..01e826c7f2a19 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-3.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck %s
 
 ; Make sure llc does not crash for invalid opencl version metadata.
 
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props-v3.ll
index 858bd54123679..d24af8482fdef 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props-v3.ll
@@ -1,7 +1,7 @@
-; RUN: llc -mattr=+code-object-v3 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -enable-misched=0 -filetype=obj -o - < %s | llvm-readobj -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX700 --check-prefix=WAVE64 --check-prefix=NOTES %s
-; RUN: llc -mattr=+code-object-v3 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -enable-misched=0 -filetype=obj -o - < %s | llvm-readobj -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX803 --check-prefix=WAVE64 --check-prefix=NOTES %s
-; RUN: llc -mattr=+code-object-v3 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -enable-misched=0 -filetype=obj -o - < %s | llvm-readobj -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=WAVE64 --check-prefix=NOTES %s
-; RUN: llc -mattr=+code-object-v3 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -enable-misched=0 -filetype=obj -o - < %s | llvm-readobj -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX1010 --check-prefix=WAVE32 --check-prefix=NOTES %s
+; RUN: llc -mattr=+code-object-v3 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -enable-misched=0 -filetype=obj -o - < %s | llvm-readobj -elf-output-style=GNU -notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX700 --check-prefix=WAVE64 --check-prefix=NOTES %s
+; RUN: llc -mattr=+code-object-v3 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -enable-misched=0 -filetype=obj -o - < %s | llvm-readobj -elf-output-style=GNU -notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX803 --check-prefix=WAVE64 --check-prefix=NOTES %s
+; RUN: llc -mattr=+code-object-v3 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -enable-misched=0 -filetype=obj -o - < %s | llvm-readobj -elf-output-style=GNU -notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=WAVE64 --check-prefix=NOTES %s
+; RUN: llc -mattr=+code-object-v3 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -enable-misched=0 -filetype=obj -o - < %s | llvm-readobj -elf-output-style=GNU -notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX1010 --check-prefix=WAVE32 --check-prefix=NOTES %s
 
 @var = addrspace(1) global float 0.0
 
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props.ll
index 12d8973333f0e..86306127b33a2 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -mattr=-code-object-v3 -enable-misched=0 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck --check-prefix=CHECK --check-prefix=GFX700 --check-prefix=NOTES %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -mattr=-code-object-v3 -enable-misched=0 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck --check-prefix=CHECK --check-prefix=GFX803 --check-prefix=NOTES %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=-code-object-v3 -enable-misched=0 -filetype=obj -o - < %s | llvm-readelf --notes | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -mattr=-code-object-v3 -enable-misched=0 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX700 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -mattr=-code-object-v3 -enable-misched=0 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX803 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=-code-object-v3 -enable-misched=0 -filetype=obj -o - < %s | llvm-readelf --notes - | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
 
 @var = addrspace(1) global float 0.0
 
diff --git a/llvm/test/CodeGen/AMDGPU/hsa.ll b/llvm/test/CodeGen/AMDGPU/hsa.ll
index 3462398b58511..f23ce2389984a 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa.ll
@@ -2,8 +2,8 @@
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=kaveri -mattr=-code-object-v3,-flat-for-global | FileCheck --check-prefix=HSA-CI %s
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=carrizo -mattr=-code-object-v3 | FileCheck --check-prefix=HSA %s
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=carrizo -mattr=-code-object-v3,-flat-for-global | FileCheck --check-prefix=HSA-VI %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=kaveri -filetype=obj -mattr=-code-object-v3 | llvm-readobj -symbols -s -sd | FileCheck --check-prefix=ELF %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=kaveri -mattr=-code-object-v3 | llvm-mc -filetype=obj -triple amdgcn--amdhsa -mcpu=kaveri -mattr=-code-object-v3 | llvm-readobj -symbols -s -sd | FileCheck %s --check-prefix=ELF
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=kaveri -filetype=obj -mattr=-code-object-v3 | llvm-readobj -symbols -s -sd - | FileCheck --check-prefix=ELF %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=kaveri -mattr=-code-object-v3 | llvm-mc -filetype=obj -triple amdgcn--amdhsa -mcpu=kaveri -mattr=-code-object-v3 | llvm-readobj -symbols -s -sd - | FileCheck %s --check-prefix=ELF
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx1010 -mattr=+WavefrontSize32,-WavefrontSize64,-code-object-v3 | FileCheck --check-prefix=HSA --check-prefix=GFX10 --check-prefix=GFX10-W32 %s
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx1010 -mattr=-WavefrontSize32,+WavefrontSize64,-code-object-v3 | FileCheck --check-prefix=HSA --check-prefix=GFX10 --check-prefix=GFX10-W64 %s
 
diff --git a/llvm/test/CodeGen/AMDGPU/huge-number-operand-folds.mir b/llvm/test/CodeGen/AMDGPU/huge-number-operand-folds.mir
new file mode 100644
index 0000000000000..1a8feddeea82b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/huge-number-operand-folds.mir
@@ -0,0 +1,22 @@
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -verify-machineinstrs -run-pass=si-fold-operands  %s -o - | FileCheck -check-prefix=GCN %s
+
+# We were storing fold candidate uses in an unsigned char, which this exceeds.
+# The use operand overflows and the expected register operand hits the immediate 0.
+# We never have more than a handful of non-implicit operands, so don't try to fold into
+# implicit operands to avoid this problem.
+
+---
+name: op_idx_overflows_uchar
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr12_sgpr13_sgpr14_sgpr15
+
+    ; GCN-LABEL: name: op_idx_overflows_uchar
+    ; GCN: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
+    %0:sreg_32 = S_MOV_B32 0
+    %1:vgpr_32 = COPY %0
+    S_ENDPGM 0, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1, implicit %1
+...
diff --git a/llvm/test/CodeGen/AMDGPU/imm16.ll b/llvm/test/CodeGen/AMDGPU/imm16.ll
index 534b04e58d157..8aebe29ae6584 100644
--- a/llvm/test/CodeGen/AMDGPU/imm16.ll
+++ b/llvm/test/CodeGen/AMDGPU/imm16.ll
@@ -1438,7 +1438,7 @@ define void @mul_inline_imm_neg_0.5_i16(i16 addrspace(1)* %out, i16 %x) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x8c,0xbf]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
-; GFX10-NEXT:    v_mul_lo_u16_e64 v2, 0xffffb800, v2 ; encoding: [0x02,0x00,0x05,0xd7,0xff,0x04,0x02,0x00,0x00,0xb8,0xff,0xff]
+; GFX10-NEXT:    v_mul_lo_u16_e64 v2, 0xb800, v2 ; encoding: [0x02,0x00,0x05,0xd7,0xff,0x04,0x02,0x00,0x00,0xb8,0xff,0xff]
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    global_store_short v[0:1], v2, off ; encoding: [0x00,0x80,0x68,0xdc,0x00,0x02,0x7d,0x00]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
@@ -1447,7 +1447,7 @@ define void @mul_inline_imm_neg_0.5_i16(i16 addrspace(1)* %out, i16 %x) {
 ; VI-LABEL: mul_inline_imm_neg_0.5_i16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x8c,0xbf]
-; VI-NEXT:    v_mul_lo_u16_e32 v2, 0xffffb800, v2 ; encoding: [0xff,0x04,0x04,0x52,0x00,0xb8,0xff,0xff]
+; VI-NEXT:    v_mul_lo_u16_e32 v2, 0xb800, v2 ; encoding: [0xff,0x04,0x04,0x52,0x00,0xb8,0xff,0xff]
 ; VI-NEXT:    flat_store_short v[0:1], v2 ; encoding: [0x00,0x00,0x68,0xdc,0x00,0x02,0x00,0x00]
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0) ; encoding: [0x70,0x00,0x8c,0xbf]
 ; VI-NEXT:    s_setpc_b64 s[30:31] ; encoding: [0x1e,0x1d,0x80,0xbe]
@@ -1510,7 +1510,7 @@ define void @mul_inline_imm_neg_1.0_i16(i16 addrspace(1)* %out, i16 %x) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x8c,0xbf]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
-; GFX10-NEXT:    v_mul_lo_u16_e64 v2, 0xffffbc00, v2 ; encoding: [0x02,0x00,0x05,0xd7,0xff,0x04,0x02,0x00,0x00,0xbc,0xff,0xff]
+; GFX10-NEXT:    v_mul_lo_u16_e64 v2, 0xbc00, v2 ; encoding: [0x02,0x00,0x05,0xd7,0xff,0x04,0x02,0x00,0x00,0xbc,0xff,0xff]
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    global_store_short v[0:1], v2, off ; encoding: [0x00,0x80,0x68,0xdc,0x00,0x02,0x7d,0x00]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
@@ -1519,7 +1519,7 @@ define void @mul_inline_imm_neg_1.0_i16(i16 addrspace(1)* %out, i16 %x) {
 ; VI-LABEL: mul_inline_imm_neg_1.0_i16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x8c,0xbf]
-; VI-NEXT:    v_mul_lo_u16_e32 v2, 0xffffbc00, v2 ; encoding: [0xff,0x04,0x04,0x52,0x00,0xbc,0xff,0xff]
+; VI-NEXT:    v_mul_lo_u16_e32 v2, 0xbc00, v2 ; encoding: [0xff,0x04,0x04,0x52,0x00,0xbc,0xff,0xff]
 ; VI-NEXT:    flat_store_short v[0:1], v2 ; encoding: [0x00,0x00,0x68,0xdc,0x00,0x02,0x00,0x00]
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0) ; encoding: [0x70,0x00,0x8c,0xbf]
 ; VI-NEXT:    s_setpc_b64 s[30:31] ; encoding: [0x1e,0x1d,0x80,0xbe]
@@ -1583,7 +1583,7 @@ define void @shl_inline_imm_neg_2.0_i16(i16 addrspace(1)* %out, i16 %x) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x8c,0xbf]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
-; GFX10-NEXT:    v_lshlrev_b16_e64 v2, v2, 0xffffc000 ; encoding: [0x02,0x00,0x14,0xd7,0x02,0xff,0x01,0x00,0x00,0xc0,0xff,0xff]
+; GFX10-NEXT:    v_lshlrev_b16_e64 v2, v2, 0xc000 ; encoding: [0x02,0x00,0x14,0xd7,0x02,0xff,0x01,0x00,0x00,0xc0,0xff,0xff]
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    global_store_short v[0:1], v2, off ; encoding: [0x00,0x80,0x68,0xdc,0x00,0x02,0x7d,0x00]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
@@ -1656,7 +1656,7 @@ define void @mul_inline_imm_neg_4.0_i16(i16 addrspace(1)* %out, i16 %x) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x8c,0xbf]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
-; GFX10-NEXT:    v_mul_lo_u16_e64 v2, 0xffffc400, v2 ; encoding: [0x02,0x00,0x05,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0xff,0xff]
+; GFX10-NEXT:    v_mul_lo_u16_e64 v2, 0xc400, v2 ; encoding: [0x02,0x00,0x05,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0xff,0xff]
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    global_store_short v[0:1], v2, off ; encoding: [0x00,0x80,0x68,0xdc,0x00,0x02,0x7d,0x00]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
@@ -1665,7 +1665,7 @@ define void @mul_inline_imm_neg_4.0_i16(i16 addrspace(1)* %out, i16 %x) {
 ; VI-LABEL: mul_inline_imm_neg_4.0_i16:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x8c,0xbf]
-; VI-NEXT:    v_mul_lo_u16_e32 v2, 0xffffc400, v2 ; encoding: [0xff,0x04,0x04,0x52,0x00,0xc4,0xff,0xff]
+; VI-NEXT:    v_mul_lo_u16_e32 v2, 0xc400, v2 ; encoding: [0xff,0x04,0x04,0x52,0x00,0xc4,0xff,0xff]
 ; VI-NEXT:    flat_store_short v[0:1], v2 ; encoding: [0x00,0x00,0x68,0xdc,0x00,0x02,0x00,0x00]
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0) ; encoding: [0x70,0x00,0x8c,0xbf]
 ; VI-NEXT:    s_setpc_b64 s[30:31] ; encoding: [0x1e,0x1d,0x80,0xbe]
diff --git a/llvm/test/CodeGen/AMDGPU/kernel-args.ll b/llvm/test/CodeGen/AMDGPU/kernel-args.ll
index 49c2bf08ba3e1..72f8c9cfc55ff 100644
--- a/llvm/test/CodeGen/AMDGPU/kernel-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/kernel-args.ll
@@ -920,3 +920,25 @@ define amdgpu_kernel void @small_array_round_down_offset(i8, [1 x i8] %arg) {
   store volatile i8 %val, i8 addrspace(1)* undef
   ret void
 }
+
+; GCN-LABEL: {{^}}byref_align_constant_i32_arg:
+; HSA-GFX9: kernarg_segment_byte_size = 264
+; HSA-GFX9-DAG: s_load_dwordx2 {{s\[[0-9]+:[0-9]+\]}}, s[4:5], 0x100{{$}}
+define amdgpu_kernel void @byref_align_constant_i32_arg(i32 addrspace(1)* nocapture %out, i32 addrspace(4)* byref(i32) align(256) %in.byref, i32 %after.offset) {
+  %in = load i32, i32 addrspace(4)* %in.byref
+  store volatile i32 %in, i32 addrspace(1)* %out, align 4
+  store volatile i32 %after.offset, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}byref_natural_align_constant_v16i32_arg:
+; HSA-GFX9: kernarg_segment_byte_size = 132
+; HSA-GFX9-DAG: s_load_dword s{{[0-9]+}}, s[4:5], 0x80
+; HSA-GFX9-DAG: s_load_dwordx16 s{{\[[0-9]+:[0-9]+\]}}, s[4:5], 0x40{{$}}
+define amdgpu_kernel void @byref_natural_align_constant_v16i32_arg(i32 addrspace(1)* nocapture %out, i8, <16 x i32> addrspace(4)* byref(<16 x i32>) %in.byref, i32 %after.offset) {
+  %in = load <16 x i32>, <16 x i32> addrspace(4)* %in.byref
+  %cast.out = bitcast i32 addrspace(1)* %out to <16 x i32> addrspace(1)*
+  store volatile <16 x i32> %in, <16 x i32> addrspace(1)* %cast.out, align 4
+  store volatile i32 %after.offset, i32 addrspace(1)* %out, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/kernel-argument-dag-lowering.ll b/llvm/test/CodeGen/AMDGPU/kernel-argument-dag-lowering.ll
index 8a33e92557cb1..7250ef262bf16 100644
--- a/llvm/test/CodeGen/AMDGPU/kernel-argument-dag-lowering.ll
+++ b/llvm/test/CodeGen/AMDGPU/kernel-argument-dag-lowering.ll
@@ -152,3 +152,137 @@ entry:
   store <3 x i15> %in, <3 x i15> addrspace(1)* %out, align 4
   ret void
 }
+
+; Byref pointers should only be treated as offsets from kernarg
+; GCN-LABEL: {{^}}byref_constant_i8_arg:
+; GCN: kernarg_segment_byte_size = 12
+; GCN: v_mov_b32_e32 v[[VPTR_LO:[0-9]+]], s4
+; GCN: v_mov_b32_e32 v[[VPTR_HI:[0-9]+]], s5
+; GCN: global_load_ubyte v{{[0-9]+}}, v{{\[}}[[VPTR_LO]]:[[VPTR_HI]]{{\]}}, off offset:8
+define amdgpu_kernel void @byref_constant_i8_arg(i32 addrspace(1)* nocapture %out, i8 addrspace(4)* byref(i8) %in.byref) {
+  %in = load i8, i8 addrspace(4)* %in.byref
+  %ext = zext i8 %in to i32
+  store i32 %ext, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}byref_constant_i16_arg:
+; GCN: kernarg_segment_byte_size = 12
+; GCN: v_mov_b32_e32 v[[VPTR_LO:[0-9]+]], s4
+; GCN: v_mov_b32_e32 v[[VPTR_HI:[0-9]+]], s5
+; GCN: global_load_ushort v{{[0-9]+}}, v{{\[}}[[VPTR_LO]]:[[VPTR_HI]]{{\]}}, off offset:8
+define amdgpu_kernel void @byref_constant_i16_arg(i32 addrspace(1)* nocapture %out, i16 addrspace(4)* byref(i16) %in.byref) {
+  %in = load i16, i16 addrspace(4)* %in.byref
+  %ext = zext i16 %in to i32
+  store i32 %ext, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}byref_constant_i32_arg:
+; GCN: kernarg_segment_byte_size = 16
+; GCN: s_load_dword [[IN:s[0-9]+]], s[4:5], 0x8{{$}}
+; GCN: s_load_dword [[OFFSET:s[0-9]+]], s[4:5], 0xc{{$}}
+define amdgpu_kernel void @byref_constant_i32_arg(i32 addrspace(1)* nocapture %out, i32 addrspace(4)* byref(i32) %in.byref, i32 %after.offset) {
+  %in = load i32, i32 addrspace(4)* %in.byref
+  store volatile i32 %in, i32 addrspace(1)* %out, align 4
+  store volatile i32 %after.offset, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}byref_constant_v4i32_arg:
+; GCN: kernarg_segment_byte_size = 36
+; GCN: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s[4:5], 0x10{{$}}
+; GCN: s_load_dword s{{[0-9]+}}, s[4:5], 0x20{{$}}
+define amdgpu_kernel void @byref_constant_v4i32_arg(<4 x i32> addrspace(1)* nocapture %out, <4 x i32> addrspace(4)* byref(<4 x i32>) %in.byref, i32 %after.offset) {
+  %in = load <4 x i32>, <4 x i32> addrspace(4)* %in.byref
+  store volatile <4 x i32> %in, <4 x i32> addrspace(1)* %out, align 4
+  %out.cast = bitcast <4 x i32> addrspace(1)* %out to i32 addrspace(1)*
+  store volatile i32 %after.offset, i32 addrspace(1)* %out.cast, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}byref_align_constant_i32_arg:
+; GCN: kernarg_segment_byte_size = 264
+; GCN-DAG: s_load_dword [[IN:s[0-9]+]], s[4:5], 0x100{{$}}
+; GCN-DAG: s_load_dword [[AFTER_OFFSET:s[0-9]+]], s[4:5], 0x104{{$}}
+; GCN-DAG: v_mov_b32_e32 [[V_IN:v[0-9]+]], [[IN]]
+; GCN-DAG: v_mov_b32_e32 [[V_AFTER_OFFSET:v[0-9]+]], [[AFTER_OFFSET]]
+; GCN: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[V_IN]]
+; GCN: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[V_AFTER_OFFSET]]
+define amdgpu_kernel void @byref_align_constant_i32_arg(i32 addrspace(1)* nocapture %out, i32 addrspace(4)* byref(i32) align(256) %in.byref, i32 %after.offset) {
+  %in = load i32, i32 addrspace(4)* %in.byref
+  store volatile i32 %in, i32 addrspace(1)* %out, align 4
+  store volatile i32 %after.offset, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}byref_natural_align_constant_v16i32_arg:
+; GCN: kernarg_segment_byte_size = 132
+; GCN-DAG: s_load_dword s{{[0-9]+}}, s[4:5], 0x80
+; GCN-DAG: s_load_dwordx16 s{{\[[0-9]+:[0-9]+\]}}, s[4:5], 0x40{{$}}
+define amdgpu_kernel void @byref_natural_align_constant_v16i32_arg(i32 addrspace(1)* nocapture %out, i8, <16 x i32> addrspace(4)* byref(<16 x i32>) align(64) %in.byref, i32 %after.offset) {
+  %in = load <16 x i32>, <16 x i32> addrspace(4)* %in.byref
+  %cast.out = bitcast i32 addrspace(1)* %out to <16 x i32> addrspace(1)*
+  store volatile <16 x i32> %in, <16 x i32> addrspace(1)* %cast.out, align 4
+  store volatile i32 %after.offset, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; Also accept byref kernel arguments with other global address spaces.
+; GCN-LABEL: {{^}}byref_global_i32_arg:
+; GCN: kernarg_segment_byte_size = 12
+; GCN: s_load_dword [[IN:s[0-9]+]], s[4:5], 0x8{{$}}
+define amdgpu_kernel void @byref_global_i32_arg(i32 addrspace(1)* nocapture %out, i32 addrspace(1)* byref(i32) %in.byref) {
+  %in = load i32, i32 addrspace(1)* %in.byref
+  store i32 %in, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}byref_flat_i32_arg:
+; GCN: flat_load_dword [[IN:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}} offset:8{{$}}
+define amdgpu_kernel void @byref_flat_i32_arg(i32 addrspace(1)* nocapture %out, i32* byref(i32) %in.byref) {
+  %in = load i32, i32* %in.byref
+  store i32 %in, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}byref_constant_32bit_i32_arg:
+; GCN: s_add_i32 s[[PTR_LO:[0-9]+]], s4, 8
+; GCN: s_mov_b32 s[[PTR_HI:[0-9]+]], 0{{$}}
+; GCN: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0x0{{$}}
+define amdgpu_kernel void @byref_constant_32bit_i32_arg(i32 addrspace(1)* nocapture %out, i32 addrspace(6)* byref(i32) %in.byref) {
+  %in = load i32, i32 addrspace(6)* %in.byref
+  store i32 %in, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; define amdgpu_kernel void @byref_unknown_as_i32_arg(i32 addrspace(1)* nocapture %out, i32 addrspace(999)* byref %in.byref) {
+;   %in = load i32, i32 addrspace(999)* %in.byref
+;   store i32 %in, i32 addrspace(1)* %out, align 4
+;   ret void
+; }
+
+; GCN-LABEL: {{^}}multi_byref_constant_i32_arg:
+; GCN: kernarg_segment_byte_size = 20
+; GCN: s_load_dword {{s[0-9]+}}, s[4:5], 0x8
+; GCN: s_load_dword {{s[0-9]+}}, s[4:5], 0xc
+; GCN: s_load_dword {{s[0-9]+}}, s[4:5], 0x10
+define amdgpu_kernel void @multi_byref_constant_i32_arg(i32 addrspace(1)* nocapture %out, i32 addrspace(4)* byref(i32) %in0.byref, i32 addrspace(4)* byref(i32) %in1.byref, i32 %after.offset) {
+  %in0 = load i32, i32 addrspace(4)* %in0.byref
+  %in1 = load i32, i32 addrspace(4)* %in1.byref
+  store volatile i32 %in0, i32 addrspace(1)* %out, align 4
+  store volatile i32 %in1, i32 addrspace(1)* %out, align 4
+  store volatile i32 %after.offset, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}byref_constant_i32_arg_offset0:
+; GCN: kernarg_segment_byte_size = 4
+; GCN-NOT: s4
+; GCN-NOT: s5
+; GCN: s_load_dword {{s[0-9]+}}, s[4:5], 0x0{{$}}
+define amdgpu_kernel void @byref_constant_i32_arg_offset0(i32 addrspace(4)* byref(i32) %in.byref) {
+  %in = load i32, i32 addrspace(4)* %in.byref
+  store i32 %in, i32 addrspace(1)* undef, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/lds-relocs.ll b/llvm/test/CodeGen/AMDGPU/lds-relocs.ll
index cf88c235e0371..3e458378a7101 100644
--- a/llvm/test/CodeGen/AMDGPU/lds-relocs.ll
+++ b/llvm/test/CodeGen/AMDGPU/lds-relocs.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefixes=GCN %s
-; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -filetype=obj < %s | llvm-readobj -r -t | FileCheck -check-prefixes=ELF %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -filetype=obj < %s | llvm-readobj -r -t - | FileCheck -check-prefixes=ELF %s
 
 @lds.external = external unnamed_addr addrspace(3) global [0 x i32]
 @lds.defined = unnamed_addr addrspace(3) global [8 x i32] undef, align 8
diff --git a/llvm/test/CodeGen/AMDGPU/lower-kernargs.ll b/llvm/test/CodeGen/AMDGPU/lower-kernargs.ll
index 9daaeee89cea5..4d03f827ac2f1 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-kernargs.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-kernargs.ll
@@ -1486,6 +1486,432 @@ define amdgpu_kernel void @dyn_alloca_kernarg_i32(i32 %n) {
   ret void
 }
 
+; Byref pointers should only be treated as offsets from kernarg
+define amdgpu_kernel void @byref_constant_i8_arg(i32 addrspace(1)* nocapture %out, i8 addrspace(4)* byref(i8) %in.byref) {
+; HSA-LABEL: @byref_constant_i8_arg(
+; HSA-NEXT:    [[BYREF_CONSTANT_I8_ARG_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(12) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
+; HSA-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_CONSTANT_I8_ARG_KERNARG_SEGMENT]], i64 0
+; HSA-NEXT:    [[OUT_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[OUT_KERNARG_OFFSET]] to i32 addrspace(1)* addrspace(4)*
+; HSA-NEXT:    [[OUT_LOAD:%.*]] = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(4)* [[OUT_KERNARG_OFFSET_CAST]], align 16, !invariant.load !0
+; HSA-NEXT:    [[IN_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_CONSTANT_I8_ARG_KERNARG_SEGMENT]], i64 8
+; HSA-NEXT:    [[IN:%.*]] = load i8, i8 addrspace(4)* [[IN_BYREF_BYVAL_KERNARG_OFFSET]], align 1
+; HSA-NEXT:    [[EXT:%.*]] = zext i8 [[IN]] to i32
+; HSA-NEXT:    store i32 [[EXT]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; HSA-NEXT:    ret void
+;
+; MESA-LABEL: @byref_constant_i8_arg(
+; MESA-NEXT:    [[BYREF_CONSTANT_I8_ARG_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(48) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
+; MESA-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_CONSTANT_I8_ARG_KERNARG_SEGMENT]], i64 36
+; MESA-NEXT:    [[OUT_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[OUT_KERNARG_OFFSET]] to i32 addrspace(1)* addrspace(4)*
+; MESA-NEXT:    [[OUT_LOAD:%.*]] = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(4)* [[OUT_KERNARG_OFFSET_CAST]], align 4, !invariant.load !0
+; MESA-NEXT:    [[IN_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_CONSTANT_I8_ARG_KERNARG_SEGMENT]], i64 44
+; MESA-NEXT:    [[IN:%.*]] = load i8, i8 addrspace(4)* [[IN_BYREF_BYVAL_KERNARG_OFFSET]], align 1
+; MESA-NEXT:    [[EXT:%.*]] = zext i8 [[IN]] to i32
+; MESA-NEXT:    store i32 [[EXT]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; MESA-NEXT:    ret void
+;
+  %in = load i8, i8 addrspace(4)* %in.byref
+  %ext = zext i8 %in to i32
+  store i32 %ext, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @byref_constant_i16_arg(i32 addrspace(1)* nocapture %out, i16 addrspace(4)* byref(i16) %in.byref) {
+; HSA-LABEL: @byref_constant_i16_arg(
+; HSA-NEXT:    [[BYREF_CONSTANT_I16_ARG_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(12) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
+; HSA-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_CONSTANT_I16_ARG_KERNARG_SEGMENT]], i64 0
+; HSA-NEXT:    [[OUT_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[OUT_KERNARG_OFFSET]] to i32 addrspace(1)* addrspace(4)*
+; HSA-NEXT:    [[OUT_LOAD:%.*]] = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(4)* [[OUT_KERNARG_OFFSET_CAST]], align 16, !invariant.load !0
+; HSA-NEXT:    [[IN_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_CONSTANT_I16_ARG_KERNARG_SEGMENT]], i64 8
+; HSA-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(4)* [[IN_BYREF_BYVAL_KERNARG_OFFSET]] to i16 addrspace(4)*
+; HSA-NEXT:    [[IN:%.*]] = load i16, i16 addrspace(4)* [[TMP1]], align 2
+; HSA-NEXT:    [[EXT:%.*]] = zext i16 [[IN]] to i32
+; HSA-NEXT:    store i32 [[EXT]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; HSA-NEXT:    ret void
+;
+; MESA-LABEL: @byref_constant_i16_arg(
+; MESA-NEXT:    [[BYREF_CONSTANT_I16_ARG_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(48) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
+; MESA-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_CONSTANT_I16_ARG_KERNARG_SEGMENT]], i64 36
+; MESA-NEXT:    [[OUT_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[OUT_KERNARG_OFFSET]] to i32 addrspace(1)* addrspace(4)*
+; MESA-NEXT:    [[OUT_LOAD:%.*]] = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(4)* [[OUT_KERNARG_OFFSET_CAST]], align 4, !invariant.load !0
+; MESA-NEXT:    [[IN_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_CONSTANT_I16_ARG_KERNARG_SEGMENT]], i64 44
+; MESA-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(4)* [[IN_BYREF_BYVAL_KERNARG_OFFSET]] to i16 addrspace(4)*
+; MESA-NEXT:    [[IN:%.*]] = load i16, i16 addrspace(4)* [[TMP1]], align 2
+; MESA-NEXT:    [[EXT:%.*]] = zext i16 [[IN]] to i32
+; MESA-NEXT:    store i32 [[EXT]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; MESA-NEXT:    ret void
+;
+  %in = load i16, i16 addrspace(4)* %in.byref
+  %ext = zext i16 %in to i32
+  store i32 %ext, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @byref_constant_i32_arg(i32 addrspace(1)* nocapture %out, i32 addrspace(4)* byref(i32) %in.byref, i32 %after.offset) {
+; HSA-LABEL: @byref_constant_i32_arg(
+; HSA-NEXT:    [[BYREF_CONSTANT_I32_ARG_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(16) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
+; HSA-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_CONSTANT_I32_ARG_KERNARG_SEGMENT]], i64 0
+; HSA-NEXT:    [[OUT_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[OUT_KERNARG_OFFSET]] to i32 addrspace(1)* addrspace(4)*
+; HSA-NEXT:    [[OUT_LOAD:%.*]] = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(4)* [[OUT_KERNARG_OFFSET_CAST]], align 16, !invariant.load !0
+; HSA-NEXT:    [[IN_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_CONSTANT_I32_ARG_KERNARG_SEGMENT]], i64 8
+; HSA-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(4)* [[IN_BYREF_BYVAL_KERNARG_OFFSET]] to i32 addrspace(4)*
+; HSA-NEXT:    [[AFTER_OFFSET_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_CONSTANT_I32_ARG_KERNARG_SEGMENT]], i64 12
+; HSA-NEXT:    [[AFTER_OFFSET_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[AFTER_OFFSET_KERNARG_OFFSET]] to i32 addrspace(4)*
+; HSA-NEXT:    [[AFTER_OFFSET_LOAD:%.*]] = load i32, i32 addrspace(4)* [[AFTER_OFFSET_KERNARG_OFFSET_CAST]], align 4, !invariant.load !0
+; HSA-NEXT:    [[IN:%.*]] = load i32, i32 addrspace(4)* [[TMP1]], align 4
+; HSA-NEXT:    store volatile i32 [[IN]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; HSA-NEXT:    store volatile i32 [[AFTER_OFFSET_LOAD]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; HSA-NEXT:    ret void
+;
+; MESA-LABEL: @byref_constant_i32_arg(
+; MESA-NEXT:    [[BYREF_CONSTANT_I32_ARG_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(52) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
+; MESA-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_CONSTANT_I32_ARG_KERNARG_SEGMENT]], i64 36
+; MESA-NEXT:    [[OUT_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[OUT_KERNARG_OFFSET]] to i32 addrspace(1)* addrspace(4)*
+; MESA-NEXT:    [[OUT_LOAD:%.*]] = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(4)* [[OUT_KERNARG_OFFSET_CAST]], align 4, !invariant.load !0
+; MESA-NEXT:    [[IN_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_CONSTANT_I32_ARG_KERNARG_SEGMENT]], i64 44
+; MESA-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(4)* [[IN_BYREF_BYVAL_KERNARG_OFFSET]] to i32 addrspace(4)*
+; MESA-NEXT:    [[AFTER_OFFSET_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_CONSTANT_I32_ARG_KERNARG_SEGMENT]], i64 48
+; MESA-NEXT:    [[AFTER_OFFSET_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[AFTER_OFFSET_KERNARG_OFFSET]] to i32 addrspace(4)*
+; MESA-NEXT:    [[AFTER_OFFSET_LOAD:%.*]] = load i32, i32 addrspace(4)* [[AFTER_OFFSET_KERNARG_OFFSET_CAST]], align 16, !invariant.load !0
+; MESA-NEXT:    [[IN:%.*]] = load i32, i32 addrspace(4)* [[TMP1]], align 4
+; MESA-NEXT:    store volatile i32 [[IN]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; MESA-NEXT:    store volatile i32 [[AFTER_OFFSET_LOAD]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; MESA-NEXT:    ret void
+;
+  %in = load i32, i32 addrspace(4)* %in.byref
+  store volatile i32 %in, i32 addrspace(1)* %out, align 4
+  store volatile i32 %after.offset, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @byref_constant_v4i32_arg(<4 x i32> addrspace(1)* nocapture %out, <4 x i32> addrspace(4)* byref(<4 x i32>) %in.byref, i32 %after.offset) {
+; HSA-LABEL: @byref_constant_v4i32_arg(
+; HSA-NEXT:    [[BYREF_CONSTANT_V4I32_ARG_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(36) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
+; HSA-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_CONSTANT_V4I32_ARG_KERNARG_SEGMENT]], i64 0
+; HSA-NEXT:    [[OUT_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[OUT_KERNARG_OFFSET]] to <4 x i32> addrspace(1)* addrspace(4)*
+; HSA-NEXT:    [[OUT_LOAD:%.*]] = load <4 x i32> addrspace(1)*, <4 x i32> addrspace(1)* addrspace(4)* [[OUT_KERNARG_OFFSET_CAST]], align 16, !invariant.load !0
+; HSA-NEXT:    [[IN_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_CONSTANT_V4I32_ARG_KERNARG_SEGMENT]], i64 16
+; HSA-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(4)* [[IN_BYREF_BYVAL_KERNARG_OFFSET]] to <4 x i32> addrspace(4)*
+; HSA-NEXT:    [[AFTER_OFFSET_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_CONSTANT_V4I32_ARG_KERNARG_SEGMENT]], i64 32
+; HSA-NEXT:    [[AFTER_OFFSET_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[AFTER_OFFSET_KERNARG_OFFSET]] to i32 addrspace(4)*
+; HSA-NEXT:    [[AFTER_OFFSET_LOAD:%.*]] = load i32, i32 addrspace(4)* [[AFTER_OFFSET_KERNARG_OFFSET_CAST]], align 16, !invariant.load !0
+; HSA-NEXT:    [[IN:%.*]] = load <4 x i32>, <4 x i32> addrspace(4)* [[TMP1]], align 16
+; HSA-NEXT:    store volatile <4 x i32> [[IN]], <4 x i32> addrspace(1)* [[OUT_LOAD]], align 4
+; HSA-NEXT:    [[OUT_CAST:%.*]] = bitcast <4 x i32> addrspace(1)* [[OUT_LOAD]] to i32 addrspace(1)*
+; HSA-NEXT:    store volatile i32 [[AFTER_OFFSET_LOAD]], i32 addrspace(1)* [[OUT_CAST]], align 4
+; HSA-NEXT:    ret void
+;
+; MESA-LABEL: @byref_constant_v4i32_arg(
+; MESA-NEXT:    [[BYREF_CONSTANT_V4I32_ARG_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(72) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
+; MESA-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_CONSTANT_V4I32_ARG_KERNARG_SEGMENT]], i64 36
+; MESA-NEXT:    [[OUT_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[OUT_KERNARG_OFFSET]] to <4 x i32> addrspace(1)* addrspace(4)*
+; MESA-NEXT:    [[OUT_LOAD:%.*]] = load <4 x i32> addrspace(1)*, <4 x i32> addrspace(1)* addrspace(4)* [[OUT_KERNARG_OFFSET_CAST]], align 4, !invariant.load !0
+; MESA-NEXT:    [[IN_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_CONSTANT_V4I32_ARG_KERNARG_SEGMENT]], i64 52
+; MESA-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(4)* [[IN_BYREF_BYVAL_KERNARG_OFFSET]] to <4 x i32> addrspace(4)*
+; MESA-NEXT:    [[AFTER_OFFSET_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_CONSTANT_V4I32_ARG_KERNARG_SEGMENT]], i64 68
+; MESA-NEXT:    [[AFTER_OFFSET_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[AFTER_OFFSET_KERNARG_OFFSET]] to i32 addrspace(4)*
+; MESA-NEXT:    [[AFTER_OFFSET_LOAD:%.*]] = load i32, i32 addrspace(4)* [[AFTER_OFFSET_KERNARG_OFFSET_CAST]], align 4, !invariant.load !0
+; MESA-NEXT:    [[IN:%.*]] = load <4 x i32>, <4 x i32> addrspace(4)* [[TMP1]], align 16
+; MESA-NEXT:    store volatile <4 x i32> [[IN]], <4 x i32> addrspace(1)* [[OUT_LOAD]], align 4
+; MESA-NEXT:    [[OUT_CAST:%.*]] = bitcast <4 x i32> addrspace(1)* [[OUT_LOAD]] to i32 addrspace(1)*
+; MESA-NEXT:    store volatile i32 [[AFTER_OFFSET_LOAD]], i32 addrspace(1)* [[OUT_CAST]], align 4
+; MESA-NEXT:    ret void
+;
+  %in = load <4 x i32>, <4 x i32> addrspace(4)* %in.byref
+  store volatile <4 x i32> %in, <4 x i32> addrspace(1)* %out, align 4
+  %out.cast = bitcast <4 x i32> addrspace(1)* %out to i32 addrspace(1)*
+  store volatile i32 %after.offset, i32 addrspace(1)* %out.cast, align 4
+  ret void
+}
+
+define amdgpu_kernel void @byref_align_constant_i32_arg(i32 addrspace(1)* nocapture %out, i32 addrspace(4)* byref(i32) align(256) %in.byref, i32 %after.offset) {
+; HSA-LABEL: @byref_align_constant_i32_arg(
+; HSA-NEXT:    [[BYREF_ALIGN_CONSTANT_I32_ARG_KERNARG_SEGMENT:%.*]] = call nonnull align 256 dereferenceable(264) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
+; HSA-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_ALIGN_CONSTANT_I32_ARG_KERNARG_SEGMENT]], i64 0
+; HSA-NEXT:    [[OUT_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[OUT_KERNARG_OFFSET]] to i32 addrspace(1)* addrspace(4)*
+; HSA-NEXT:    [[OUT_LOAD:%.*]] = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(4)* [[OUT_KERNARG_OFFSET_CAST]], align 16, !invariant.load !0
+; HSA-NEXT:    [[IN_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_ALIGN_CONSTANT_I32_ARG_KERNARG_SEGMENT]], i64 256
+; HSA-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(4)* [[IN_BYREF_BYVAL_KERNARG_OFFSET]] to i32 addrspace(4)*
+; HSA-NEXT:    [[AFTER_OFFSET_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_ALIGN_CONSTANT_I32_ARG_KERNARG_SEGMENT]], i64 260
+; HSA-NEXT:    [[AFTER_OFFSET_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[AFTER_OFFSET_KERNARG_OFFSET]] to i32 addrspace(4)*
+; HSA-NEXT:    [[AFTER_OFFSET_LOAD:%.*]] = load i32, i32 addrspace(4)* [[AFTER_OFFSET_KERNARG_OFFSET_CAST]], align 4, !invariant.load !0
+; HSA-NEXT:    [[IN:%.*]] = load i32, i32 addrspace(4)* [[TMP1]], align 4
+; HSA-NEXT:    store volatile i32 [[IN]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; HSA-NEXT:    store volatile i32 [[AFTER_OFFSET_LOAD]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; HSA-NEXT:    ret void
+;
+; MESA-LABEL: @byref_align_constant_i32_arg(
+; MESA-NEXT:    [[BYREF_ALIGN_CONSTANT_I32_ARG_KERNARG_SEGMENT:%.*]] = call nonnull align 256 dereferenceable(300) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
+; MESA-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_ALIGN_CONSTANT_I32_ARG_KERNARG_SEGMENT]], i64 36
+; MESA-NEXT:    [[OUT_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[OUT_KERNARG_OFFSET]] to i32 addrspace(1)* addrspace(4)*
+; MESA-NEXT:    [[OUT_LOAD:%.*]] = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(4)* [[OUT_KERNARG_OFFSET_CAST]], align 4, !invariant.load !0
+; MESA-NEXT:    [[IN_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_ALIGN_CONSTANT_I32_ARG_KERNARG_SEGMENT]], i64 292
+; MESA-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(4)* [[IN_BYREF_BYVAL_KERNARG_OFFSET]] to i32 addrspace(4)*
+; MESA-NEXT:    [[AFTER_OFFSET_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_ALIGN_CONSTANT_I32_ARG_KERNARG_SEGMENT]], i64 296
+; MESA-NEXT:    [[AFTER_OFFSET_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[AFTER_OFFSET_KERNARG_OFFSET]] to i32 addrspace(4)*
+; MESA-NEXT:    [[AFTER_OFFSET_LOAD:%.*]] = load i32, i32 addrspace(4)* [[AFTER_OFFSET_KERNARG_OFFSET_CAST]], align 8, !invariant.load !0
+; MESA-NEXT:    [[IN:%.*]] = load i32, i32 addrspace(4)* [[TMP1]], align 4
+; MESA-NEXT:    store volatile i32 [[IN]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; MESA-NEXT:    store volatile i32 [[AFTER_OFFSET_LOAD]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; MESA-NEXT:    ret void
+;
+  %in = load i32, i32 addrspace(4)* %in.byref
+  store volatile i32 %in, i32 addrspace(1)* %out, align 4
+  store volatile i32 %after.offset, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @byref_natural_align_constant_v16i32_arg(i32 addrspace(1)* nocapture %out, i8, <16 x i32> addrspace(4)* byref(<16 x i32>) %in.byref, i32 %after.offset) {
+; HSA-LABEL: @byref_natural_align_constant_v16i32_arg(
+; HSA-NEXT:    [[BYREF_NATURAL_ALIGN_CONSTANT_V16I32_ARG_KERNARG_SEGMENT:%.*]] = call nonnull align 64 dereferenceable(132) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
+; HSA-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_NATURAL_ALIGN_CONSTANT_V16I32_ARG_KERNARG_SEGMENT]], i64 0
+; HSA-NEXT:    [[OUT_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[OUT_KERNARG_OFFSET]] to i32 addrspace(1)* addrspace(4)*
+; HSA-NEXT:    [[OUT_LOAD:%.*]] = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(4)* [[OUT_KERNARG_OFFSET_CAST]], align 16, !invariant.load !0
+; HSA-NEXT:    [[IN_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_NATURAL_ALIGN_CONSTANT_V16I32_ARG_KERNARG_SEGMENT]], i64 64
+; HSA-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(4)* [[IN_BYREF_BYVAL_KERNARG_OFFSET]] to <16 x i32> addrspace(4)*
+; HSA-NEXT:    [[AFTER_OFFSET_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_NATURAL_ALIGN_CONSTANT_V16I32_ARG_KERNARG_SEGMENT]], i64 128
+; HSA-NEXT:    [[AFTER_OFFSET_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[AFTER_OFFSET_KERNARG_OFFSET]] to i32 addrspace(4)*
+; HSA-NEXT:    [[AFTER_OFFSET_LOAD:%.*]] = load i32, i32 addrspace(4)* [[AFTER_OFFSET_KERNARG_OFFSET_CAST]], align 16, !invariant.load !0
+; HSA-NEXT:    [[IN:%.*]] = load <16 x i32>, <16 x i32> addrspace(4)* [[TMP2]], align 64
+; HSA-NEXT:    [[CAST_OUT:%.*]] = bitcast i32 addrspace(1)* [[OUT_LOAD]] to <16 x i32> addrspace(1)*
+; HSA-NEXT:    store volatile <16 x i32> [[IN]], <16 x i32> addrspace(1)* [[CAST_OUT]], align 4
+; HSA-NEXT:    store volatile i32 [[AFTER_OFFSET_LOAD]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; HSA-NEXT:    ret void
+;
+; MESA-LABEL: @byref_natural_align_constant_v16i32_arg(
+; MESA-NEXT:    [[BYREF_NATURAL_ALIGN_CONSTANT_V16I32_ARG_KERNARG_SEGMENT:%.*]] = call nonnull align 64 dereferenceable(168) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
+; MESA-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_NATURAL_ALIGN_CONSTANT_V16I32_ARG_KERNARG_SEGMENT]], i64 36
+; MESA-NEXT:    [[OUT_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[OUT_KERNARG_OFFSET]] to i32 addrspace(1)* addrspace(4)*
+; MESA-NEXT:    [[OUT_LOAD:%.*]] = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(4)* [[OUT_KERNARG_OFFSET_CAST]], align 4, !invariant.load !0
+; MESA-NEXT:    [[IN_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_NATURAL_ALIGN_CONSTANT_V16I32_ARG_KERNARG_SEGMENT]], i64 100
+; MESA-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(4)* [[IN_BYREF_BYVAL_KERNARG_OFFSET]] to <16 x i32> addrspace(4)*
+; MESA-NEXT:    [[AFTER_OFFSET_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_NATURAL_ALIGN_CONSTANT_V16I32_ARG_KERNARG_SEGMENT]], i64 164
+; MESA-NEXT:    [[AFTER_OFFSET_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[AFTER_OFFSET_KERNARG_OFFSET]] to i32 addrspace(4)*
+; MESA-NEXT:    [[AFTER_OFFSET_LOAD:%.*]] = load i32, i32 addrspace(4)* [[AFTER_OFFSET_KERNARG_OFFSET_CAST]], align 4, !invariant.load !0
+; MESA-NEXT:    [[IN:%.*]] = load <16 x i32>, <16 x i32> addrspace(4)* [[TMP2]], align 64
+; MESA-NEXT:    [[CAST_OUT:%.*]] = bitcast i32 addrspace(1)* [[OUT_LOAD]] to <16 x i32> addrspace(1)*
+; MESA-NEXT:    store volatile <16 x i32> [[IN]], <16 x i32> addrspace(1)* [[CAST_OUT]], align 4
+; MESA-NEXT:    store volatile i32 [[AFTER_OFFSET_LOAD]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; MESA-NEXT:    ret void
+;
+  %in = load <16 x i32>, <16 x i32> addrspace(4)* %in.byref
+  %cast.out = bitcast i32 addrspace(1)* %out to <16 x i32> addrspace(1)*
+  store volatile <16 x i32> %in, <16 x i32> addrspace(1)* %cast.out, align 4
+  store volatile i32 %after.offset, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; Also accept byref kernel arguments with other global address spaces.
+define amdgpu_kernel void @byref_global_i32_arg(i32 addrspace(1)* nocapture %out, i32 addrspace(1)* byref(i32) %in.byref) {
+; HSA-LABEL: @byref_global_i32_arg(
+; HSA-NEXT:    [[BYREF_GLOBAL_I32_ARG_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(12) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
+; HSA-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_GLOBAL_I32_ARG_KERNARG_SEGMENT]], i64 0
+; HSA-NEXT:    [[OUT_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[OUT_KERNARG_OFFSET]] to i32 addrspace(1)* addrspace(4)*
+; HSA-NEXT:    [[OUT_LOAD:%.*]] = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(4)* [[OUT_KERNARG_OFFSET_CAST]], align 16, !invariant.load !0
+; HSA-NEXT:    [[IN_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_GLOBAL_I32_ARG_KERNARG_SEGMENT]], i64 8
+; HSA-NEXT:    [[TMP1:%.*]] = addrspacecast i8 addrspace(4)* [[IN_BYREF_BYVAL_KERNARG_OFFSET]] to i32 addrspace(1)*
+; HSA-NEXT:    [[IN:%.*]] = load i32, i32 addrspace(1)* [[TMP1]], align 4
+; HSA-NEXT:    store i32 [[IN]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; HSA-NEXT:    ret void
+;
+; MESA-LABEL: @byref_global_i32_arg(
+; MESA-NEXT:    [[BYREF_GLOBAL_I32_ARG_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(48) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
+; MESA-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_GLOBAL_I32_ARG_KERNARG_SEGMENT]], i64 36
+; MESA-NEXT:    [[OUT_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[OUT_KERNARG_OFFSET]] to i32 addrspace(1)* addrspace(4)*
+; MESA-NEXT:    [[OUT_LOAD:%.*]] = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(4)* [[OUT_KERNARG_OFFSET_CAST]], align 4, !invariant.load !0
+; MESA-NEXT:    [[IN_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_GLOBAL_I32_ARG_KERNARG_SEGMENT]], i64 44
+; MESA-NEXT:    [[TMP1:%.*]] = addrspacecast i8 addrspace(4)* [[IN_BYREF_BYVAL_KERNARG_OFFSET]] to i32 addrspace(1)*
+; MESA-NEXT:    [[IN:%.*]] = load i32, i32 addrspace(1)* [[TMP1]], align 4
+; MESA-NEXT:    store i32 [[IN]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; MESA-NEXT:    ret void
+;
+  %in = load i32, i32 addrspace(1)* %in.byref
+  store i32 %in, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @byref_flat_i32_arg(i32 addrspace(1)* nocapture %out, i32* byref(i32) %in.byref) {
+; HSA-LABEL: @byref_flat_i32_arg(
+; HSA-NEXT:    [[BYREF_FLAT_I32_ARG_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(12) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
+; HSA-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_FLAT_I32_ARG_KERNARG_SEGMENT]], i64 0
+; HSA-NEXT:    [[OUT_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[OUT_KERNARG_OFFSET]] to i32 addrspace(1)* addrspace(4)*
+; HSA-NEXT:    [[OUT_LOAD:%.*]] = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(4)* [[OUT_KERNARG_OFFSET_CAST]], align 16, !invariant.load !0
+; HSA-NEXT:    [[IN_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_FLAT_I32_ARG_KERNARG_SEGMENT]], i64 8
+; HSA-NEXT:    [[TMP1:%.*]] = addrspacecast i8 addrspace(4)* [[IN_BYREF_BYVAL_KERNARG_OFFSET]] to i32*
+; HSA-NEXT:    [[IN:%.*]] = load i32, i32* [[TMP1]], align 4
+; HSA-NEXT:    store i32 [[IN]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; HSA-NEXT:    ret void
+;
+; MESA-LABEL: @byref_flat_i32_arg(
+; MESA-NEXT:    [[BYREF_FLAT_I32_ARG_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(48) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
+; MESA-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_FLAT_I32_ARG_KERNARG_SEGMENT]], i64 36
+; MESA-NEXT:    [[OUT_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[OUT_KERNARG_OFFSET]] to i32 addrspace(1)* addrspace(4)*
+; MESA-NEXT:    [[OUT_LOAD:%.*]] = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(4)* [[OUT_KERNARG_OFFSET_CAST]], align 4, !invariant.load !0
+; MESA-NEXT:    [[IN_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_FLAT_I32_ARG_KERNARG_SEGMENT]], i64 44
+; MESA-NEXT:    [[TMP1:%.*]] = addrspacecast i8 addrspace(4)* [[IN_BYREF_BYVAL_KERNARG_OFFSET]] to i32*
+; MESA-NEXT:    [[IN:%.*]] = load i32, i32* [[TMP1]], align 4
+; MESA-NEXT:    store i32 [[IN]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; MESA-NEXT:    ret void
+;
+  %in = load i32, i32* %in.byref
+  store i32 %in, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @byref_constant_32bit_i32_arg(i32 addrspace(1)* nocapture %out, i32 addrspace(6)* byref(i32) %in.byref) {
+; HSA-LABEL: @byref_constant_32bit_i32_arg(
+; HSA-NEXT:    [[BYREF_CONSTANT_32BIT_I32_ARG_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(12) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
+; HSA-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_CONSTANT_32BIT_I32_ARG_KERNARG_SEGMENT]], i64 0
+; HSA-NEXT:    [[OUT_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[OUT_KERNARG_OFFSET]] to i32 addrspace(1)* addrspace(4)*
+; HSA-NEXT:    [[OUT_LOAD:%.*]] = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(4)* [[OUT_KERNARG_OFFSET_CAST]], align 16, !invariant.load !0
+; HSA-NEXT:    [[IN_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_CONSTANT_32BIT_I32_ARG_KERNARG_SEGMENT]], i64 8
+; HSA-NEXT:    [[TMP1:%.*]] = addrspacecast i8 addrspace(4)* [[IN_BYREF_BYVAL_KERNARG_OFFSET]] to i32 addrspace(6)*
+; HSA-NEXT:    [[IN:%.*]] = load i32, i32 addrspace(6)* [[TMP1]], align 4
+; HSA-NEXT:    store i32 [[IN]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; HSA-NEXT:    ret void
+;
+; MESA-LABEL: @byref_constant_32bit_i32_arg(
+; MESA-NEXT:    [[BYREF_CONSTANT_32BIT_I32_ARG_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(48) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
+; MESA-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_CONSTANT_32BIT_I32_ARG_KERNARG_SEGMENT]], i64 36
+; MESA-NEXT:    [[OUT_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[OUT_KERNARG_OFFSET]] to i32 addrspace(1)* addrspace(4)*
+; MESA-NEXT:    [[OUT_LOAD:%.*]] = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(4)* [[OUT_KERNARG_OFFSET_CAST]], align 4, !invariant.load !0
+; MESA-NEXT:    [[IN_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_CONSTANT_32BIT_I32_ARG_KERNARG_SEGMENT]], i64 44
+; MESA-NEXT:    [[TMP1:%.*]] = addrspacecast i8 addrspace(4)* [[IN_BYREF_BYVAL_KERNARG_OFFSET]] to i32 addrspace(6)*
+; MESA-NEXT:    [[IN:%.*]] = load i32, i32 addrspace(6)* [[TMP1]], align 4
+; MESA-NEXT:    store i32 [[IN]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; MESA-NEXT:    ret void
+;
+  %in = load i32, i32 addrspace(6)* %in.byref
+  store i32 %in, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @byref_unknown_as_i32_arg(i32 addrspace(1)* nocapture %out, i32 addrspace(999)* byref(i32) %in.byref) {
+; HSA-LABEL: @byref_unknown_as_i32_arg(
+; HSA-NEXT:    [[BYREF_UNKNOWN_AS_I32_ARG_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(12) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
+; HSA-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_UNKNOWN_AS_I32_ARG_KERNARG_SEGMENT]], i64 0
+; HSA-NEXT:    [[OUT_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[OUT_KERNARG_OFFSET]] to i32 addrspace(1)* addrspace(4)*
+; HSA-NEXT:    [[OUT_LOAD:%.*]] = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(4)* [[OUT_KERNARG_OFFSET_CAST]], align 16, !invariant.load !0
+; HSA-NEXT:    [[IN_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_UNKNOWN_AS_I32_ARG_KERNARG_SEGMENT]], i64 8
+; HSA-NEXT:    [[TMP1:%.*]] = addrspacecast i8 addrspace(4)* [[IN_BYREF_BYVAL_KERNARG_OFFSET]] to i32 addrspace(999)*
+; HSA-NEXT:    [[IN:%.*]] = load i32, i32 addrspace(999)* [[TMP1]], align 4
+; HSA-NEXT:    store i32 [[IN]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; HSA-NEXT:    ret void
+;
+; MESA-LABEL: @byref_unknown_as_i32_arg(
+; MESA-NEXT:    [[BYREF_UNKNOWN_AS_I32_ARG_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(48) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
+; MESA-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_UNKNOWN_AS_I32_ARG_KERNARG_SEGMENT]], i64 36
+; MESA-NEXT:    [[OUT_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[OUT_KERNARG_OFFSET]] to i32 addrspace(1)* addrspace(4)*
+; MESA-NEXT:    [[OUT_LOAD:%.*]] = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(4)* [[OUT_KERNARG_OFFSET_CAST]], align 4, !invariant.load !0
+; MESA-NEXT:    [[IN_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_UNKNOWN_AS_I32_ARG_KERNARG_SEGMENT]], i64 44
+; MESA-NEXT:    [[TMP1:%.*]] = addrspacecast i8 addrspace(4)* [[IN_BYREF_BYVAL_KERNARG_OFFSET]] to i32 addrspace(999)*
+; MESA-NEXT:    [[IN:%.*]] = load i32, i32 addrspace(999)* [[TMP1]], align 4
+; MESA-NEXT:    store i32 [[IN]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; MESA-NEXT:    ret void
+;
+  %in = load i32, i32 addrspace(999)* %in.byref
+  store i32 %in, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; Invalid, but should not crash.
+define amdgpu_kernel void @byref_local_i32_arg(i32 addrspace(1)* nocapture %out, i32 addrspace(3)* byref(i32) %in.byref) {
+; HSA-LABEL: @byref_local_i32_arg(
+; HSA-NEXT:    [[BYREF_LOCAL_I32_ARG_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(12) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
+; HSA-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_LOCAL_I32_ARG_KERNARG_SEGMENT]], i64 0
+; HSA-NEXT:    [[OUT_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[OUT_KERNARG_OFFSET]] to i32 addrspace(1)* addrspace(4)*
+; HSA-NEXT:    [[OUT_LOAD:%.*]] = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(4)* [[OUT_KERNARG_OFFSET_CAST]], align 16, !invariant.load !0
+; HSA-NEXT:    [[IN_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_LOCAL_I32_ARG_KERNARG_SEGMENT]], i64 8
+; HSA-NEXT:    [[TMP1:%.*]] = addrspacecast i8 addrspace(4)* [[IN_BYREF_BYVAL_KERNARG_OFFSET]] to i32 addrspace(3)*
+; HSA-NEXT:    [[IN:%.*]] = load i32, i32 addrspace(3)* [[TMP1]], align 4
+; HSA-NEXT:    store i32 [[IN]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; HSA-NEXT:    ret void
+;
+; MESA-LABEL: @byref_local_i32_arg(
+; MESA-NEXT:    [[BYREF_LOCAL_I32_ARG_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(48) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
+; MESA-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_LOCAL_I32_ARG_KERNARG_SEGMENT]], i64 36
+; MESA-NEXT:    [[OUT_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[OUT_KERNARG_OFFSET]] to i32 addrspace(1)* addrspace(4)*
+; MESA-NEXT:    [[OUT_LOAD:%.*]] = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(4)* [[OUT_KERNARG_OFFSET_CAST]], align 4, !invariant.load !0
+; MESA-NEXT:    [[IN_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_LOCAL_I32_ARG_KERNARG_SEGMENT]], i64 44
+; MESA-NEXT:    [[TMP1:%.*]] = addrspacecast i8 addrspace(4)* [[IN_BYREF_BYVAL_KERNARG_OFFSET]] to i32 addrspace(3)*
+; MESA-NEXT:    [[IN:%.*]] = load i32, i32 addrspace(3)* [[TMP1]], align 4
+; MESA-NEXT:    store i32 [[IN]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; MESA-NEXT:    ret void
+;
+  %in = load i32, i32 addrspace(3)* %in.byref
+  store i32 %in, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @multi_byref_constant_i32_arg(i32 addrspace(1)* nocapture %out, i32 addrspace(4)* byref(i32) %in0.byref, i32 addrspace(4)* byref(i32) %in1.byref, i32 %after.offset) {
+; HSA-LABEL: @multi_byref_constant_i32_arg(
+; HSA-NEXT:    [[MULTI_BYREF_CONSTANT_I32_ARG_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(20) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
+; HSA-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[MULTI_BYREF_CONSTANT_I32_ARG_KERNARG_SEGMENT]], i64 0
+; HSA-NEXT:    [[OUT_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[OUT_KERNARG_OFFSET]] to i32 addrspace(1)* addrspace(4)*
+; HSA-NEXT:    [[OUT_LOAD:%.*]] = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(4)* [[OUT_KERNARG_OFFSET_CAST]], align 16, !invariant.load !0
+; HSA-NEXT:    [[IN0_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[MULTI_BYREF_CONSTANT_I32_ARG_KERNARG_SEGMENT]], i64 8
+; HSA-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(4)* [[IN0_BYREF_BYVAL_KERNARG_OFFSET]] to i32 addrspace(4)*
+; HSA-NEXT:    [[IN1_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[MULTI_BYREF_CONSTANT_I32_ARG_KERNARG_SEGMENT]], i64 12
+; HSA-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(4)* [[IN1_BYREF_BYVAL_KERNARG_OFFSET]] to i32 addrspace(4)*
+; HSA-NEXT:    [[AFTER_OFFSET_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[MULTI_BYREF_CONSTANT_I32_ARG_KERNARG_SEGMENT]], i64 16
+; HSA-NEXT:    [[AFTER_OFFSET_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[AFTER_OFFSET_KERNARG_OFFSET]] to i32 addrspace(4)*
+; HSA-NEXT:    [[AFTER_OFFSET_LOAD:%.*]] = load i32, i32 addrspace(4)* [[AFTER_OFFSET_KERNARG_OFFSET_CAST]], align 16, !invariant.load !0
+; HSA-NEXT:    [[IN0:%.*]] = load i32, i32 addrspace(4)* [[TMP1]], align 4
+; HSA-NEXT:    [[IN1:%.*]] = load i32, i32 addrspace(4)* [[TMP2]], align 4
+; HSA-NEXT:    store volatile i32 [[IN0]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; HSA-NEXT:    store volatile i32 [[IN1]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; HSA-NEXT:    store volatile i32 [[AFTER_OFFSET_LOAD]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; HSA-NEXT:    ret void
+;
+; MESA-LABEL: @multi_byref_constant_i32_arg(
+; MESA-NEXT:    [[MULTI_BYREF_CONSTANT_I32_ARG_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(56) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
+; MESA-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[MULTI_BYREF_CONSTANT_I32_ARG_KERNARG_SEGMENT]], i64 36
+; MESA-NEXT:    [[OUT_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[OUT_KERNARG_OFFSET]] to i32 addrspace(1)* addrspace(4)*
+; MESA-NEXT:    [[OUT_LOAD:%.*]] = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(4)* [[OUT_KERNARG_OFFSET_CAST]], align 4, !invariant.load !0
+; MESA-NEXT:    [[IN0_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[MULTI_BYREF_CONSTANT_I32_ARG_KERNARG_SEGMENT]], i64 44
+; MESA-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(4)* [[IN0_BYREF_BYVAL_KERNARG_OFFSET]] to i32 addrspace(4)*
+; MESA-NEXT:    [[IN1_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[MULTI_BYREF_CONSTANT_I32_ARG_KERNARG_SEGMENT]], i64 48
+; MESA-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(4)* [[IN1_BYREF_BYVAL_KERNARG_OFFSET]] to i32 addrspace(4)*
+; MESA-NEXT:    [[AFTER_OFFSET_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[MULTI_BYREF_CONSTANT_I32_ARG_KERNARG_SEGMENT]], i64 52
+; MESA-NEXT:    [[AFTER_OFFSET_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[AFTER_OFFSET_KERNARG_OFFSET]] to i32 addrspace(4)*
+; MESA-NEXT:    [[AFTER_OFFSET_LOAD:%.*]] = load i32, i32 addrspace(4)* [[AFTER_OFFSET_KERNARG_OFFSET_CAST]], align 4, !invariant.load !0
+; MESA-NEXT:    [[IN0:%.*]] = load i32, i32 addrspace(4)* [[TMP1]], align 4
+; MESA-NEXT:    [[IN1:%.*]] = load i32, i32 addrspace(4)* [[TMP2]], align 4
+; MESA-NEXT:    store volatile i32 [[IN0]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; MESA-NEXT:    store volatile i32 [[IN1]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; MESA-NEXT:    store volatile i32 [[AFTER_OFFSET_LOAD]], i32 addrspace(1)* [[OUT_LOAD]], align 4
+; MESA-NEXT:    ret void
+;
+  %in0 = load i32, i32 addrspace(4)* %in0.byref
+  %in1 = load i32, i32 addrspace(4)* %in1.byref
+  store volatile i32 %in0, i32 addrspace(1)* %out, align 4
+  store volatile i32 %in1, i32 addrspace(1)* %out, align 4
+  store volatile i32 %after.offset, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @byref_constant_i32_arg_offset0(i32 addrspace(4)* byref(i32) %in.byref) {
+; HSA-LABEL: @byref_constant_i32_arg_offset0(
+; HSA-NEXT:    [[BYREF_CONSTANT_I32_ARG_OFFSET0_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(4) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
+; HSA-NEXT:    [[IN_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_CONSTANT_I32_ARG_OFFSET0_KERNARG_SEGMENT]], i64 0
+; HSA-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(4)* [[IN_BYREF_BYVAL_KERNARG_OFFSET]] to i32 addrspace(4)*
+; HSA-NEXT:    [[IN:%.*]] = load i32, i32 addrspace(4)* [[TMP1]], align 4
+; HSA-NEXT:    store i32 [[IN]], i32 addrspace(1)* undef, align 4
+; HSA-NEXT:    ret void
+;
+; MESA-LABEL: @byref_constant_i32_arg_offset0(
+; MESA-NEXT:    [[BYREF_CONSTANT_I32_ARG_OFFSET0_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(40) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
+; MESA-NEXT:    [[IN_BYREF_BYVAL_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[BYREF_CONSTANT_I32_ARG_OFFSET0_KERNARG_SEGMENT]], i64 36
+; MESA-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(4)* [[IN_BYREF_BYVAL_KERNARG_OFFSET]] to i32 addrspace(4)*
+; MESA-NEXT:    [[IN:%.*]] = load i32, i32 addrspace(4)* [[TMP1]], align 4
+; MESA-NEXT:    store i32 [[IN]], i32 addrspace(1)* undef, align 4
+; MESA-NEXT:    ret void
+;
+  %in = load i32, i32 addrspace(4)* %in.byref
+  store i32 %in, i32 addrspace(1)* undef, align 4
+  ret void
+}
+
 attributes #0 = { nounwind "target-cpu"="kaveri" }
 attributes #1 = { nounwind "target-cpu"="kaveri" "amdgpu-implicitarg-num-bytes"="40" }
 attributes #2 = { nounwind "target-cpu"="tahiti" }
diff --git a/llvm/test/CodeGen/AMDGPU/no-initializer-constant-addrspace.ll b/llvm/test/CodeGen/AMDGPU/no-initializer-constant-addrspace.ll
index 2312112b7bc7b..feb582e28ff05 100644
--- a/llvm/test/CodeGen/AMDGPU/no-initializer-constant-addrspace.ll
+++ b/llvm/test/CodeGen/AMDGPU/no-initializer-constant-addrspace.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mtriple=amdgcn- -mcpu=tahiti -filetype=obj < %s | llvm-readobj -r --symbols | FileCheck %s -check-prefix=GCN
-; RUN: llc -mtriple=amdgcn- -mcpu=tonga -filetype=obj < %s | llvm-readobj -r --symbols | FileCheck %s -check-prefix=GCN
-; RUN: llc -mtriple=r600- -mcpu=cypress -filetype=obj < %s | llvm-readobj -r --symbols | FileCheck %s -check-prefix=EG
+; RUN: llc -mtriple=amdgcn- -mcpu=tahiti -filetype=obj < %s | llvm-readobj -r --symbols - | FileCheck %s -check-prefix=GCN
+; RUN: llc -mtriple=amdgcn- -mcpu=tonga -filetype=obj < %s | llvm-readobj -r --symbols - | FileCheck %s -check-prefix=GCN
+; RUN: llc -mtriple=r600- -mcpu=cypress -filetype=obj < %s | llvm-readobj -r --symbols - | FileCheck %s -check-prefix=EG
 
 ; GCN: R_AMDGPU_GOTPCREL32_LO extern_const_addrspace 0x0
 ; EG: R_AMDGPU_ABS32 extern_const_addrspace
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-constantexpr-use.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-constantexpr-use.ll
new file mode 100644
index 0000000000000..a6511910603e1
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-constantexpr-use.ll
@@ -0,0 +1,35 @@
+; RUN: opt -S -disable-promote-alloca-to-vector -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-promote-alloca < %s | FileCheck -check-prefix=IR %s
+; RUN: llc -disable-promote-alloca-to-vector -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefix=ASM %s
+
+target datalayout = "A5"
+
+@all_lds = internal unnamed_addr addrspace(3) global [16384 x i32] undef, align 4
+
+; This function cannot promote to using LDS because of the size of the
+; constant expression use in the function, which was previously not
+; detected.
+; IR-LABEL: @constant_expression_uses_lds(
+; IR: alloca
+
+; ASM-LABEL: constant_expression_uses_lds:
+; ASM: .group_segment_fixed_size: 65536
+define amdgpu_kernel void @constant_expression_uses_lds(i32 addrspace(1)* nocapture %out, i32 %idx) #0 {
+entry:
+  %stack = alloca [4 x i32], align 4, addrspace(5)
+  %gep0 = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 0
+  %gep1 = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 1
+  %gep2 = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 2
+  %gep3 = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 3
+  store i32 9, i32 addrspace(5)* %gep0
+  store i32 10, i32 addrspace(5)* %gep1
+  store i32 99, i32 addrspace(5)* %gep2
+  store i32 43, i32 addrspace(5)* %gep3
+  %arrayidx = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %stack, i32 0, i32 %idx
+  %load = load i32, i32 addrspace(5)* %arrayidx, align 4
+  store i32 %load, i32 addrspace(1)* %out
+
+  store volatile i32 ptrtoint ([16384 x i32] addrspace(3)* @all_lds to i32), i32 addrspace(1)* undef
+  ret void
+}
+
+attributes #0 = { "amdgpu-waves-per-eu"="1,5" }
diff --git a/llvm/test/CodeGen/AMDGPU/r600-constant-array-fixup.ll b/llvm/test/CodeGen/AMDGPU/r600-constant-array-fixup.ll
index 7166a88b481dd..54460a77c4d0b 100644
--- a/llvm/test/CodeGen/AMDGPU/r600-constant-array-fixup.ll
+++ b/llvm/test/CodeGen/AMDGPU/r600-constant-array-fixup.ll
@@ -1,4 +1,4 @@
-; RUN: llc -filetype=obj -mtriple=r600-mesa-mesa3d -mcpu=cypress -verify-machineinstrs < %s | llvm-readobj -r --symbols | FileCheck %s
+; RUN: llc -filetype=obj -mtriple=r600-mesa-mesa3d -mcpu=cypress -verify-machineinstrs < %s | llvm-readobj -r --symbols - | FileCheck %s
 
 @arr = internal unnamed_addr addrspace(4) constant [4 x i32] [i32 4, i32 5, i32 6, i32 7], align 4
 
diff --git a/llvm/test/CodeGen/AMDGPU/scratch-simple.ll b/llvm/test/CodeGen/AMDGPU/scratch-simple.ll
index 2c852f066c4fd..5fe1329ecf61b 100644
--- a/llvm/test/CodeGen/AMDGPU/scratch-simple.ll
+++ b/llvm/test/CodeGen/AMDGPU/scratch-simple.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=verde -amdgpu-use-divergent-register-indexing -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,SI,SIVI %s
 ; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=gfx803 -mattr=-flat-for-global -amdgpu-use-divergent-register-indexing -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,VI,SIVI %s
 ; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-use-divergent-register-indexing -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX9,GFX9_10 %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=gfx900 -filetype=obj -amdgpu-use-divergent-register-indexing < %s | llvm-readobj -r | FileCheck --check-prefix=RELS %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=gfx900 -filetype=obj -amdgpu-use-divergent-register-indexing < %s | llvm-readobj -r - | FileCheck --check-prefix=RELS %s
 ; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=gfx1010 -mattr=-flat-for-global -amdgpu-use-divergent-register-indexing -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX10_W32,GFX9_10 %s
 ; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=gfx1010 -mattr=-flat-for-global,+wavefrontsize64 -amdgpu-use-divergent-register-indexing -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX10_W64,GFX9_10 %s
 
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
new file mode 100644
index 0000000000000..88a2e1d772bfa
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
@@ -0,0 +1,196 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji < %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; FIXME: promotion not handled without f16 insts
+
+define half @v_constained_fsub_f16_fpexcept_strict(half %x, half %y) #0 {
+; GCN-LABEL: v_constained_fsub_f16_fpexcept_strict:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_sub_f16_e32 v0, v0, v1
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %val = call half @llvm.experimental.constrained.fsub.f16(half %x, half %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  ret half %val
+}
+
+define half @v_constained_fsub_f16_fpexcept_ignore(half %x, half %y) #0 {
+; GCN-LABEL: v_constained_fsub_f16_fpexcept_ignore:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_sub_f16_e32 v0, v0, v1
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %val = call half @llvm.experimental.constrained.fsub.f16(half %x, half %y, metadata !"round.tonearest", metadata !"fpexcept.ignore")
+  ret half %val
+}
+
+define half @v_constained_fsub_f16_fpexcept_maytrap(half %x, half %y) #0 {
+; GCN-LABEL: v_constained_fsub_f16_fpexcept_maytrap:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_sub_f16_e32 v0, v0, v1
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %val = call half @llvm.experimental.constrained.fsub.f16(half %x, half %y, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  ret half %val
+}
+
+define <2 x half> @v_constained_fsub_v2f16_fpexcept_strict(<2 x half> %x, <2 x half> %y) #0 {
+; GFX9-LABEL: v_constained_fsub_v2f16_fpexcept_strict:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_f16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_sub_f16_e32 v0, v0, v1
+; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT:    v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_constained_fsub_v2f16_fpexcept_strict:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_f16_e32 v0, v0, v1
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+  %val = call <2 x half> @llvm.experimental.constrained.fsub.v2f16(<2 x half> %x, <2 x half> %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  ret <2 x half> %val
+}
+
+define <2 x half> @v_constained_fsub_v2f16_fpexcept_ignore(<2 x half> %x, <2 x half> %y) #0 {
+; GFX9-LABEL: v_constained_fsub_v2f16_fpexcept_ignore:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_f16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_sub_f16_e32 v0, v0, v1
+; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT:    v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_constained_fsub_v2f16_fpexcept_ignore:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_f16_e32 v0, v0, v1
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+  %val = call <2 x half> @llvm.experimental.constrained.fsub.v2f16(<2 x half> %x, <2 x half> %y, metadata !"round.tonearest", metadata !"fpexcept.ignore")
+  ret <2 x half> %val
+}
+
+define <2 x half> @v_constained_fsub_v2f16_fpexcept_maytrap(<2 x half> %x, <2 x half> %y) #0 {
+; GFX9-LABEL: v_constained_fsub_v2f16_fpexcept_maytrap:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_f16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_sub_f16_e32 v0, v0, v1
+; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT:    v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_constained_fsub_v2f16_fpexcept_maytrap:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_f16_e32 v0, v0, v1
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+  %val = call <2 x half> @llvm.experimental.constrained.fsub.v2f16(<2 x half> %x, <2 x half> %y, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  ret <2 x half> %val
+}
+
+define <3 x half> @v_constained_fsub_v3f16_fpexcept_strict(<3 x half> %x, <3 x half> %y) #0 {
+; GFX9-LABEL: v_constained_fsub_v3f16_fpexcept_strict:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_f16_sdwa v4, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_sub_f16_e32 v0, v0, v2
+; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT:    v_lshl_or_b32 v0, v4, 16, v0
+; GFX9-NEXT:    v_sub_f16_e32 v1, v1, v3
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_constained_fsub_v3f16_fpexcept_strict:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_f16_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_f16_e32 v0, v0, v2
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_sub_f16_e32 v1, v1, v3
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+  %val = call <3 x half> @llvm.experimental.constrained.fsub.v3f16(<3 x half> %x, <3 x half> %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  ret <3 x half> %val
+}
+
+; FIXME: Scalarized
+define <4 x half> @v_constained_fsub_v4f16_fpexcept_strict(<4 x half> %x, <4 x half> %y) #0 {
+; GFX9-LABEL: v_constained_fsub_v4f16_fpexcept_strict:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_f16_sdwa v4, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_sub_f16_sdwa v5, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_sub_f16_e32 v0, v0, v2
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX9-NEXT:    v_sub_f16_e32 v1, v1, v3
+; GFX9-NEXT:    v_and_b32_e32 v0, v2, v0
+; GFX9-NEXT:    v_and_b32_e32 v1, v2, v1
+; GFX9-NEXT:    v_lshl_or_b32 v0, v5, 16, v0
+; GFX9-NEXT:    v_lshl_or_b32 v1, v4, 16, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_constained_fsub_v4f16_fpexcept_strict:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_f16_sdwa v4, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_f16_e32 v1, v1, v3
+; GFX8-NEXT:    v_sub_f16_sdwa v5, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT:    v_sub_f16_e32 v0, v0, v2
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+  %val = call <4 x half> @llvm.experimental.constrained.fsub.v4f16(<4 x half> %x, <4 x half> %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  ret <4 x half> %val
+}
+
+define amdgpu_ps half @s_constained_fsub_f16_fpexcept_strict(half inreg %x, half inreg %y) #0 {
+; GCN-LABEL: s_constained_fsub_f16_fpexcept_strict:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_mov_b32_e32 v0, s3
+; GCN-NEXT:    v_sub_f16_e32 v0, s2, v0
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call half @llvm.experimental.constrained.fsub.f16(half %x, half %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  ret half %val
+}
+
+define amdgpu_ps <2 x half> @s_constained_fsub_v2f16_fpexcept_strict(<2 x half> inreg %x, <2 x half> inreg %y) #0 {
+; GFX9-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-NEXT:    s_lshr_b32 s0, s3, 16
+; GFX9-NEXT:    v_sub_f16_e32 v1, s2, v1
+; GFX9-NEXT:    s_lshr_b32 s1, s2, 16
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_sub_f16_e32 v0, s1, v0
+; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT:    v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_lshr_b32 s0, s3, 16
+; GFX8-NEXT:    s_lshr_b32 s1, s2, 16
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_sub_f16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    v_sub_f16_e32 v1, s2, v1
+; GFX8-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    ; return to shader part epilog
+  %val = call <2 x half> @llvm.experimental.constrained.fsub.v2f16(<2 x half> %x, <2 x half> %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  ret <2 x half> %val
+}
+
+declare half @llvm.experimental.constrained.fsub.f16(half, half, metadata, metadata) #1
+declare <2 x half> @llvm.experimental.constrained.fsub.v2f16(<2 x half>, <2 x half>, metadata, metadata) #1
+declare <3 x half> @llvm.experimental.constrained.fsub.v3f16(<3 x half>, <3 x half>, metadata, metadata) #1
+declare <4 x half> @llvm.experimental.constrained.fsub.v4f16(<4 x half>, <4 x half>, metadata, metadata) #1
+
+attributes #0 = { strictfp }
+attributes #1 = { inaccessiblememonly nounwind willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fsub.f32.ll b/llvm/test/CodeGen/AMDGPU/strict_fsub.f32.ll
new file mode 100644
index 0000000000000..2364a9cac57ed
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/strict_fsub.f32.ll
@@ -0,0 +1,129 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 < %s | FileCheck -check-prefix=GCN %s
+
+define float @v_constained_fsub_f32_fpexcept_strict(float %x, float %y) #0 {
+; GCN-LABEL: v_constained_fsub_f32_fpexcept_strict:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_sub_f32_e32 v0, v0, v1
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %val = call float @llvm.experimental.constrained.fsub.f32(float %x, float %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  ret float %val
+}
+
+define float @v_constained_fsub_f32_fpexcept_ignore(float %x, float %y) #0 {
+; GCN-LABEL: v_constained_fsub_f32_fpexcept_ignore:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_sub_f32_e32 v0, v0, v1
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %val = call float @llvm.experimental.constrained.fsub.f32(float %x, float %y, metadata !"round.tonearest", metadata !"fpexcept.ignore")
+  ret float %val
+}
+
+define float @v_constained_fsub_f32_fpexcept_maytrap(float %x, float %y) #0 {
+; GCN-LABEL: v_constained_fsub_f32_fpexcept_maytrap:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_sub_f32_e32 v0, v0, v1
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %val = call float @llvm.experimental.constrained.fsub.f32(float %x, float %y, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  ret float %val
+}
+
+define <2 x float> @v_constained_fsub_v2f32_fpexcept_strict(<2 x float> %x, <2 x float> %y) #0 {
+; GCN-LABEL: v_constained_fsub_v2f32_fpexcept_strict:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_sub_f32_e32 v0, v0, v2
+; GCN-NEXT:    v_sub_f32_e32 v1, v1, v3
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %val = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> %x, <2 x float> %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  ret <2 x float> %val
+}
+
+define <2 x float> @v_constained_fsub_v2f32_fpexcept_ignore(<2 x float> %x, <2 x float> %y) #0 {
+; GCN-LABEL: v_constained_fsub_v2f32_fpexcept_ignore:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_sub_f32_e32 v0, v0, v2
+; GCN-NEXT:    v_sub_f32_e32 v1, v1, v3
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %val = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> %x, <2 x float> %y, metadata !"round.tonearest", metadata !"fpexcept.ignore")
+  ret <2 x float> %val
+}
+
+define <2 x float> @v_constained_fsub_v2f32_fpexcept_maytrap(<2 x float> %x, <2 x float> %y) #0 {
+; GCN-LABEL: v_constained_fsub_v2f32_fpexcept_maytrap:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_sub_f32_e32 v0, v0, v2
+; GCN-NEXT:    v_sub_f32_e32 v1, v1, v3
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %val = call <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float> %x, <2 x float> %y, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  ret <2 x float> %val
+}
+
+define <3 x float> @v_constained_fsub_v3f32_fpexcept_strict(<3 x float> %x, <3 x float> %y) #0 {
+; GCN-LABEL: v_constained_fsub_v3f32_fpexcept_strict:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_sub_f32_e32 v0, v0, v3
+; GCN-NEXT:    v_sub_f32_e32 v1, v1, v4
+; GCN-NEXT:    v_sub_f32_e32 v2, v2, v5
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %val = call <3 x float> @llvm.experimental.constrained.fsub.v3f32(<3 x float> %x, <3 x float> %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  ret <3 x float> %val
+}
+
+define amdgpu_ps float @s_constained_fsub_f32_fpexcept_strict(float inreg %x, float inreg %y) #0 {
+; GCN-LABEL: s_constained_fsub_f32_fpexcept_strict:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_mov_b32_e32 v0, s3
+; GCN-NEXT:    v_sub_f32_e32 v0, s2, v0
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call float @llvm.experimental.constrained.fsub.f32(float %x, float %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  ret float %val
+}
+
+define float @v_constained_fsub_f32_fpexcept_strict_fabs_lhs(float %x, float %y) #0 {
+; GCN-LABEL: v_constained_fsub_f32_fpexcept_strict_fabs_lhs:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_sub_f32_e64 v0, |v0|, v1
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fabs.x = call float @llvm.fabs.f32(float %x)
+  %val = call float @llvm.experimental.constrained.fsub.f32(float %fabs.x, float %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  ret float %val
+}
+
+define float @v_constained_fsub_f32_fpexcept_strict_fabs_rhs(float %x, float %y) #0 {
+; GCN-LABEL: v_constained_fsub_f32_fpexcept_strict_fabs_rhs:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_sub_f32_e64 v0, v0, |v1|
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fabs.y = call float @llvm.fabs.f32(float %y)
+  %val = call float @llvm.experimental.constrained.fsub.f32(float %x, float %fabs.y, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  ret float %val
+}
+
+define float @v_constained_fsub_f32_fpexcept_strict_fneg_fabs_lhs(float %x, float %y) #0 {
+; GCN-LABEL: v_constained_fsub_f32_fpexcept_strict_fneg_fabs_lhs:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_sub_f32_e64 v0, -|v0|, v1
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fabs.x = call float @llvm.fabs.f32(float %x)
+  %neg.fabs.x = fneg float %fabs.x
+  %val = call float @llvm.experimental.constrained.fsub.f32(float %neg.fabs.x, float %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  ret float %val
+}
+
+declare float @llvm.fabs.f32(float) #1
+declare float @llvm.experimental.constrained.fsub.f32(float, float, metadata, metadata) #1
+declare <2 x float> @llvm.experimental.constrained.fsub.v2f32(<2 x float>, <2 x float>, metadata, metadata) #1
+declare <3 x float> @llvm.experimental.constrained.fsub.v3f32(<3 x float>, <3 x float>, metadata, metadata) #1
+
+attributes #0 = { strictfp }
+attributes #1 = { inaccessiblememonly nounwind willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fsub.f64.ll b/llvm/test/CodeGen/AMDGPU/strict_fsub.f64.ll
new file mode 100644
index 0000000000000..a185397a5d4e4
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/strict_fsub.f64.ll
@@ -0,0 +1,96 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 < %s | FileCheck -check-prefix=GCN %s
+
+define double @v_constained_fsub_f64_fpexcept_strict(double %x, double %y) #0 {
+; GCN-LABEL: v_constained_fsub_f64_fpexcept_strict:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], -v[2:3]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %val = call double @llvm.experimental.constrained.fsub.f64(double %x, double %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  ret double %val
+}
+
+define double @v_constained_fsub_f64_fpexcept_ignore(double %x, double %y) #0 {
+; GCN-LABEL: v_constained_fsub_f64_fpexcept_ignore:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], -v[2:3]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %val = call double @llvm.experimental.constrained.fsub.f64(double %x, double %y, metadata !"round.tonearest", metadata !"fpexcept.ignore")
+  ret double %val
+}
+
+define double @v_constained_fsub_f64_fpexcept_maytrap(double %x, double %y) #0 {
+; GCN-LABEL: v_constained_fsub_f64_fpexcept_maytrap:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], -v[2:3]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %val = call double @llvm.experimental.constrained.fsub.f64(double %x, double %y, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  ret double %val
+}
+
+define <2 x double> @v_constained_fsub_v2f64_fpexcept_strict(<2 x double> %x, <2 x double> %y) #0 {
+; GCN-LABEL: v_constained_fsub_v2f64_fpexcept_strict:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], -v[4:5]
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], -v[6:7]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %val = call <2 x double> @llvm.experimental.constrained.fsub.v2f64(<2 x double> %x, <2 x double> %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  ret <2 x double> %val
+}
+
+define <2 x double> @v_constained_fsub_v2f64_fpexcept_ignore(<2 x double> %x, <2 x double> %y) #0 {
+; GCN-LABEL: v_constained_fsub_v2f64_fpexcept_ignore:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], -v[4:5]
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], -v[6:7]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %val = call <2 x double> @llvm.experimental.constrained.fsub.v2f64(<2 x double> %x, <2 x double> %y, metadata !"round.tonearest", metadata !"fpexcept.ignore")
+  ret <2 x double> %val
+}
+
+define <2 x double> @v_constained_fsub_v2f64_fpexcept_maytrap(<2 x double> %x, <2 x double> %y) #0 {
+; GCN-LABEL: v_constained_fsub_v2f64_fpexcept_maytrap:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], -v[4:5]
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], -v[6:7]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %val = call <2 x double> @llvm.experimental.constrained.fsub.v2f64(<2 x double> %x, <2 x double> %y, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  ret <2 x double> %val
+}
+
+define <3 x double> @v_constained_fsub_v3f64_fpexcept_strict(<3 x double> %x, <3 x double> %y) #0 {
+; GCN-LABEL: v_constained_fsub_v3f64_fpexcept_strict:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_add_f64 v[0:1], v[0:1], -v[6:7]
+; GCN-NEXT:    v_add_f64 v[2:3], v[2:3], -v[8:9]
+; GCN-NEXT:    v_add_f64 v[4:5], v[4:5], -v[10:11]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %val = call <3 x double> @llvm.experimental.constrained.fsub.v3f64(<3 x double> %x, <3 x double> %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  ret <3 x double> %val
+}
+
+define amdgpu_ps <2 x float> @s_constained_fsub_f64_fpexcept_strict(double inreg %x, double inreg %y) #0 {
+; GCN-LABEL: s_constained_fsub_f64_fpexcept_strict:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    v_mov_b32_e32 v1, s5
+; GCN-NEXT:    v_add_f64 v[0:1], s[2:3], -v[0:1]
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call double @llvm.experimental.constrained.fsub.f64(double %x, double %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  %cast = bitcast double %val to <2 x float>
+  ret <2 x float> %cast
+}
+
+declare double @llvm.experimental.constrained.fsub.f64(double, double, metadata, metadata) #1
+declare <2 x double> @llvm.experimental.constrained.fsub.v2f64(<2 x double>, <2 x double>, metadata, metadata) #1
+declare <3 x double> @llvm.experimental.constrained.fsub.v3f64(<3 x double>, <3 x double>, metadata, metadata) #1
+
+attributes #0 = { strictfp }
+attributes #1 = { inaccessiblememonly nounwind willreturn }
diff --git a/llvm/test/CodeGen/AMDGPU/sub.i16.ll b/llvm/test/CodeGen/AMDGPU/sub.i16.ll
index 5f98b7b1fc9c3..7fe003ff54209 100644
--- a/llvm/test/CodeGen/AMDGPU/sub.i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/sub.i16.ll
@@ -22,7 +22,7 @@ define amdgpu_kernel void @v_test_sub_i16(i16 addrspace(1)* %out, i16 addrspace(
 ; FIXME: Need to handle non-uniform case for function below (load without gep).
 ; GCN-LABEL: {{^}}v_test_sub_i16_constant:
 ; VI: flat_load_ushort [[A:v[0-9]+]]
-; VI: v_add_u16_e32 [[ADD:v[0-9]+]], 0xffffff85, [[A]]
+; VI: v_add_u16_e32 [[ADD:v[0-9]+]], 0xff85, [[A]]
 ; VI-NEXT: buffer_store_short [[ADD]]
 define amdgpu_kernel void @v_test_sub_i16_constant(i16 addrspace(1)* %out, i16 addrspace(1)* %in0) #1 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll b/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
index 43536131582d2..30b681fcaa393 100644
--- a/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
@@ -194,7 +194,7 @@ define amdgpu_kernel void @v_test_sub_v2i16_constant(<2 x i16> addrspace(1)* %ou
 ; VI-NEXT:    s_mov_b32 s3, 0xf000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_add_u16_e32 v2, 0xffffff85, v0
+; VI-NEXT:    v_add_u16_e32 v2, 0xff85, v0
 ; VI-NEXT:    v_add_u16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; VI-NEXT:    v_or_b32_e32 v0, v2, v0
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
diff --git a/llvm/test/CodeGen/ARM/2010-12-08-tpsoft.ll b/llvm/test/CodeGen/ARM/2010-12-08-tpsoft.ll
index 14883f4f1d5eb..2df9fb76b9747 100644
--- a/llvm/test/CodeGen/ARM/2010-12-08-tpsoft.ll
+++ b/llvm/test/CodeGen/ARM/2010-12-08-tpsoft.ll
@@ -1,7 +1,7 @@
 ; RUN: llc  %s -mtriple=armv7-linux-gnueabi -o - | \
 ; RUN:    FileCheck  -check-prefix=ELFASM %s 
 ; RUN: llc  %s -mtriple=armv7-linux-gnueabi -filetype=obj -o - | \
-; RUN:    llvm-readobj -S --sd | FileCheck  -check-prefix=ELFOBJ %s
+; RUN:    llvm-readobj -S --sd - | FileCheck  -check-prefix=ELFOBJ %s
 
 ;; Make sure that bl __aeabi_read_tp is materialized and fixed up correctly
 ;; in the obj case. 
diff --git a/llvm/test/CodeGen/ARM/2010-12-15-elf-lcomm.ll b/llvm/test/CodeGen/ARM/2010-12-15-elf-lcomm.ll
index bf53ab24dab5a..ccebe2a003cdf 100644
--- a/llvm/test/CodeGen/ARM/2010-12-15-elf-lcomm.ll
+++ b/llvm/test/CodeGen/ARM/2010-12-15-elf-lcomm.ll
@@ -1,5 +1,5 @@
 ; RUN: llc  %s -mtriple=armv7-linux-gnueabi -filetype=obj -o - | \
-; RUN:    llvm-readobj -S --symbols | FileCheck  -check-prefix=OBJ %s
+; RUN:    llvm-readobj -S --symbols - | FileCheck  -check-prefix=OBJ %s
 ; RUN: llc  %s -mtriple=armv7-linux-gnueabi -o - | \
 ; RUN:    FileCheck  -check-prefix=ASM %s
 
diff --git a/llvm/test/CodeGen/ARM/MachO-subtypes.ll b/llvm/test/CodeGen/ARM/MachO-subtypes.ll
index bd002622e303a..f5cc20a4d0703 100644
--- a/llvm/test/CodeGen/ARM/MachO-subtypes.ll
+++ b/llvm/test/CodeGen/ARM/MachO-subtypes.ll
@@ -1,57 +1,57 @@
 ; Check that MachO ARM CPU Subtypes are respected
 
 ; RUN: llc -mtriple=armv4t-apple-darwin -filetype=obj -o - < %s \
-; RUN: | llvm-readobj --file-headers | FileCheck %s --check-prefix=CHECK-V4T
+; RUN: | llvm-readobj --file-headers - | FileCheck %s --check-prefix=CHECK-V4T
 
 ; RUN: llc -mtriple=armv5-apple-darwin -filetype=obj -o - < %s \
-; RUN: | llvm-readobj --file-headers | FileCheck %s --check-prefix=CHECK-V5
+; RUN: | llvm-readobj --file-headers - | FileCheck %s --check-prefix=CHECK-V5
 ; RUN: llc -mtriple=armv5e-apple-darwin -filetype=obj -o - < %s \
-; RUN: | llvm-readobj --file-headers | FileCheck %s --check-prefix=CHECK-V5
+; RUN: | llvm-readobj --file-headers - | FileCheck %s --check-prefix=CHECK-V5
 ; RUN: llc -mtriple=armv5t-apple-darwin -filetype=obj -o - < %s \
-; RUN: | llvm-readobj --file-headers | FileCheck %s --check-prefix=CHECK-V5
+; RUN: | llvm-readobj --file-headers - | FileCheck %s --check-prefix=CHECK-V5
 ; RUN: llc -mtriple=armv5te-apple-darwin -filetype=obj -o - < %s \
-; RUN: | llvm-readobj --file-headers | FileCheck %s --check-prefix=CHECK-V5
+; RUN: | llvm-readobj --file-headers - | FileCheck %s --check-prefix=CHECK-V5
 ; RUN: llc -mtriple=armv5tej-apple-darwin -filetype=obj -o - < %s \
-; RUN: | llvm-readobj --file-headers | FileCheck %s --check-prefix=CHECK-V5
+; RUN: | llvm-readobj --file-headers - | FileCheck %s --check-prefix=CHECK-V5
 
 ; RUN: llc -mtriple=armv6-apple-darwin -filetype=obj -o - < %s \
-; RUN: | llvm-readobj --file-headers | FileCheck %s --check-prefix=CHECK-V6
+; RUN: | llvm-readobj --file-headers - | FileCheck %s --check-prefix=CHECK-V6
 ; RUN: llc -mtriple=armv6k-apple-darwin -filetype=obj -o - < %s \
-; RUN: | llvm-readobj --file-headers | FileCheck %s --check-prefix=CHECK-V6
+; RUN: | llvm-readobj --file-headers - | FileCheck %s --check-prefix=CHECK-V6
 ; RUN: llc -mtriple=thumbv6-apple-darwin -filetype=obj -o - < %s \
-; RUN: | llvm-readobj --file-headers | FileCheck %s --check-prefix=CHECK-V6
+; RUN: | llvm-readobj --file-headers - | FileCheck %s --check-prefix=CHECK-V6
 ; RUN: llc -mtriple=thumbv6k-apple-darwin -filetype=obj -o - < %s \
-; RUN: | llvm-readobj --file-headers | FileCheck %s --check-prefix=CHECK-V6
+; RUN: | llvm-readobj --file-headers - | FileCheck %s --check-prefix=CHECK-V6
 
 ; RUN: llc -mtriple=armv6m-apple-darwin -filetype=obj -o - < %s \
-; RUN: | llvm-readobj --file-headers | FileCheck %s --check-prefix=CHECK-V6M
+; RUN: | llvm-readobj --file-headers - | FileCheck %s --check-prefix=CHECK-V6M
 ; RUN: llc -mtriple=thumbv6m-apple-darwin -filetype=obj -o - < %s \
-; RUN: | llvm-readobj --file-headers | FileCheck %s --check-prefix=CHECK-V6M
+; RUN: | llvm-readobj --file-headers - | FileCheck %s --check-prefix=CHECK-V6M
 
 ; RUN: llc -mtriple=armv7-apple-darwin -filetype=obj -o - < %s \
-; RUN: | llvm-readobj --file-headers | FileCheck %s --check-prefix=CHECK-V7
+; RUN: | llvm-readobj --file-headers - | FileCheck %s --check-prefix=CHECK-V7
 ; RUN: llc -mtriple=thumbv7-apple-darwin -filetype=obj -o - < %s \
-; RUN: | llvm-readobj --file-headers | FileCheck %s --check-prefix=CHECK-V7
+; RUN: | llvm-readobj --file-headers - | FileCheck %s --check-prefix=CHECK-V7
 
 ; RUN: llc -mtriple=thumbv7em-apple-darwin -mcpu=cortex-m4 -filetype=obj -o - < %s \
-; RUN: | llvm-readobj --file-headers | FileCheck %s --check-prefix=CHECK-V7EM
+; RUN: | llvm-readobj --file-headers - | FileCheck %s --check-prefix=CHECK-V7EM
 ; RUN: llc -mtriple=thumbv7em-apple-darwin -mcpu=cortex-m7 -filetype=obj -o - < %s \
-; RUN: | llvm-readobj --file-headers | FileCheck %s --check-prefix=CHECK-V7EM
+; RUN: | llvm-readobj --file-headers - | FileCheck %s --check-prefix=CHECK-V7EM
 
 ; RUN: llc -mtriple=armv7k-apple-darwin -filetype=obj -o - < %s \
-; RUN: | llvm-readobj --file-headers | FileCheck %s --check-prefix=CHECK-V7K
+; RUN: | llvm-readobj --file-headers - | FileCheck %s --check-prefix=CHECK-V7K
 ; RUN: llc -mtriple=thumbv7k-apple-darwin -filetype=obj -o - < %s \
-; RUN: | llvm-readobj --file-headers | FileCheck %s --check-prefix=CHECK-V7K
+; RUN: | llvm-readobj --file-headers - | FileCheck %s --check-prefix=CHECK-V7K
 
 ; RUN: llc -mtriple=thumbv7m-apple-darwin -mcpu=sc300 -filetype=obj -o - < %s \
-; RUN: | llvm-readobj --file-headers | FileCheck %s --check-prefix=CHECK-V7M
+; RUN: | llvm-readobj --file-headers - | FileCheck %s --check-prefix=CHECK-V7M
 ; RUN: llc -mtriple=thumbv7m-apple-darwin -mcpu=cortex-m3 -filetype=obj -o - < %s \
-; RUN: | llvm-readobj --file-headers | FileCheck %s --check-prefix=CHECK-V7M
+; RUN: | llvm-readobj --file-headers - | FileCheck %s --check-prefix=CHECK-V7M
 
 ; RUN: llc -mtriple=armv7s-apple-darwin -filetype=obj -o - < %s \
-; RUN: | llvm-readobj --file-headers | FileCheck %s --check-prefix=CHECK-V7S
+; RUN: | llvm-readobj --file-headers - | FileCheck %s --check-prefix=CHECK-V7S
 ; RUN: llc -mtriple=thumbv7s-apple-darwin -filetype=obj -o - < %s \
-; RUN: | llvm-readobj --file-headers | FileCheck %s --check-prefix=CHECK-V7S
+; RUN: | llvm-readobj --file-headers - | FileCheck %s --check-prefix=CHECK-V7S
 
 define void @_test() {
   ret void
diff --git a/llvm/test/CodeGen/ARM/build-attributes-encoding.s b/llvm/test/CodeGen/ARM/build-attributes-encoding.s
index 356185079d625..87e28f0adf813 100644
--- a/llvm/test/CodeGen/ARM/build-attributes-encoding.s
+++ b/llvm/test/CodeGen/ARM/build-attributes-encoding.s
@@ -1,7 +1,7 @@
 // This tests that ARM attributes are properly encoded.
 
 // RUN: llvm-mc < %s -triple=arm-linux-gnueabi -filetype=obj -o - \
-// RUN:   | llvm-readobj -S --sd | FileCheck %s
+// RUN:   | llvm-readobj -S --sd - | FileCheck %s
 
 // Tag_CPU_name (=5)
 .cpu cortex-a8
diff --git a/llvm/test/CodeGen/ARM/const-load-align-thumb.mir b/llvm/test/CodeGen/ARM/const-load-align-thumb.mir
new file mode 100644
index 0000000000000..daa8a91033094
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/const-load-align-thumb.mir
@@ -0,0 +1,59 @@
+# RUN: llc -mtriple=arm-eabi -run-pass=arm-cp-islands %s -o - | FileCheck %s
+--- |
+  target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "thumbv8.2a-arm-none-eabi"
+
+  define hidden i32 @main() {
+  entry:
+    %P5 = alloca half, align 2
+    store half 0xH3FE0, half* %P5, align 2
+    %0 = load half, half* %P5, align 2
+    call void @z_bar(half %0)
+    ret i32 0
+  }
+
+  declare dso_local void @z_bar(half)
+
+...
+---
+name:            main
+alignment:       2
+tracksRegLiveness: true
+frameInfo:
+  stackSize:       16
+  maxAlignment:    4
+  adjustsStack:    true
+  hasCalls:        true
+  maxCallFrameSize: 0
+  localFrameSize:  2
+stack:
+  - { id: 0, name: P5, offset: -10, size: 2, alignment: 2, local-offset: -2 }
+  - { id: 1, type: spill-slot, offset: -4, size: 4, alignment: 4, callee-saved-register: '$lr',
+      callee-saved-restored: false }
+  - { id: 2, type: spill-slot, offset: -8, size: 4, alignment: 4, callee-saved-register: '$r7' }
+constants:
+  - id:              0
+    value:           half 0xH3FE0
+    alignment:       2
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    liveins: $r7, $lr
+
+    frame-setup tPUSH 14 /* CC::al */, $noreg, killed $r7, killed $lr, implicit-def $sp, implicit $sp
+    frame-setup CFI_INSTRUCTION def_cfa_offset 8
+    frame-setup CFI_INSTRUCTION offset $lr, -4
+    frame-setup CFI_INSTRUCTION offset $r7, -8
+    $sp = frame-setup tSUBspi $sp, 2, 14 /* CC::al */, $noreg
+    frame-setup CFI_INSTRUCTION def_cfa_offset 16
+    renamable $s0 = VLDRH %const.0, 0, 14, $noreg :: (load 2 from constant-pool)
+    VSTRH killed renamable $s0, $sp, 3, 14, $noreg :: (store 2 into %ir.P5)
+    renamable $r0 = t2LDRHi12 $sp, 6, 14 /* CC::al */, $noreg :: (dereferenceable load 2 from %ir.P5)
+    tBL 14 /* CC::al */, $noreg, @z_bar, csr_aapcs, implicit-def dead $lr, implicit $sp, implicit killed $r0, implicit-def $sp
+    renamable $r0, dead $cpsr = tMOVi8 0, 14 /* CC::al */, $noreg
+    $sp = frame-destroy tADDspi $sp, 2, 14 /* CC::al */, $noreg
+    frame-destroy tPOP_RET 14 /* CC::al */, $noreg, def $r7, def $pc, implicit killed $r0
+
+  ; CHECK: name:            main
+  ; CHECK-NEXT: alignment:       4
+...
diff --git a/llvm/test/CodeGen/ARM/ifcvt-branch-weight-bug.ll b/llvm/test/CodeGen/ARM/ifcvt-branch-weight-bug.ll
index 539c80bb1f09b..2f4143547a01a 100644
--- a/llvm/test/CodeGen/ARM/ifcvt-branch-weight-bug.ll
+++ b/llvm/test/CodeGen/ARM/ifcvt-branch-weight-bug.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=armv4t--linux-androideabi -print-machineinstrs=if-converter -o /dev/null 2>&1 | FileCheck %s
+; RUN: llc < %s -mtriple=armv4t--linux-androideabi -stop-after=if-converter | FileCheck %s
 ; Fix a bug triggered in IfConverterTriangle when CvtBB has multiple
 ; predecessors.
 ; PR18752
diff --git a/llvm/test/CodeGen/ARM/ifcvt-branch-weight.ll b/llvm/test/CodeGen/ARM/ifcvt-branch-weight.ll
index f4733b615303c..bf985516ceb2d 100644
--- a/llvm/test/CodeGen/ARM/ifcvt-branch-weight.ll
+++ b/llvm/test/CodeGen/ARM/ifcvt-branch-weight.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=thumbv8 -print-machineinstrs=if-converter -arm-atomic-cfg-tidy=0 -o /dev/null 2>&1 | FileCheck %s
+; RUN: llc < %s -mtriple=thumbv8 -stop-after=if-converter -arm-atomic-cfg-tidy=0 | FileCheck %s
 
 %struct.S = type { i8* (i8*)*, [1 x i8] }
 define internal zeroext i8 @bar(%struct.S* %x, %struct.S* nocapture %y) nounwind readonly {
diff --git a/llvm/test/CodeGen/ARM/ifcvt-iter-indbr.ll b/llvm/test/CodeGen/ARM/ifcvt-iter-indbr.ll
index 79b9900fa9c91..a39a8e8da00cc 100644
--- a/llvm/test/CodeGen/ARM/ifcvt-iter-indbr.ll
+++ b/llvm/test/CodeGen/ARM/ifcvt-iter-indbr.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s -mtriple thumbv7s-apple-darwin  -asm-verbose=false | FileCheck %s
-; RUN: llc < %s -mtriple thumbv7s-apple-darwin  -asm-verbose=false -print-machineinstrs=if-converter 2>&1 | FileCheck --check-prefix=CHECK-PROB %s
+; RUN: llc < %s -mtriple thumbv7s-apple-darwin  -asm-verbose=false -stop-after=if-converter | FileCheck --check-prefix=CHECK-PROB %s
 
 declare i32 @foo(i32)
 declare i8* @bar(i32, i8*, i8*)
diff --git a/llvm/test/CodeGen/ARM/tail-merge-branch-weight.ll b/llvm/test/CodeGen/ARM/tail-merge-branch-weight.ll
index 05121c5727fd4..33444078c616f 100644
--- a/llvm/test/CodeGen/ARM/tail-merge-branch-weight.ll
+++ b/llvm/test/CodeGen/ARM/tail-merge-branch-weight.ll
@@ -1,5 +1,4 @@
-; RUN: llc -mtriple=arm-apple-ios -print-machineinstrs=branch-folder \
-; RUN: %s -o /dev/null 2>&1 | FileCheck %s
+; RUN: llc < %s -mtriple=arm-apple-ios -stop-after=branch-folder | FileCheck %s
 
 ; Branch probability of tailed-merged block:
 ;
@@ -8,11 +7,9 @@
 ; p(L0_L1 -> L3) = p(entry -> L0) * p(L0 -> L3) + p(entry -> L1) * p(L1 -> L3)
 ;                = 0.2 * 0.4 + 0.8 * 0.7 = 0.64
 
-; CHECK: # Machine code for function test0:
 ; CHECK: successors: %bb.{{[0-9]+}}(0x1999999a), %bb.{{[0-9]+}}(0x66666666)
 ; CHECK: bb.{{[0-9]+}}{{[0-9a-zA-Z.]*}}:
 ; CHECK: bb.{{[0-9]+}}{{[0-9a-zA-Z.]*}}:
-; CHECK: # End machine code for function test0.
 
 define i32 @test0(i32 %n, i32 %m, i32* nocapture %a, i32* nocapture %b) {
 entry:
diff --git a/llvm/test/CodeGen/ARM/taildup-branch-weight.ll b/llvm/test/CodeGen/ARM/taildup-branch-weight.ll
index 72a2611f23cf6..bfc4d837ea856 100644
--- a/llvm/test/CodeGen/ARM/taildup-branch-weight.ll
+++ b/llvm/test/CodeGen/ARM/taildup-branch-weight.ll
@@ -1,8 +1,7 @@
-; RUN: llc -mtriple=arm-eabi -print-machineinstrs=tailduplication -tail-dup-size=100 \
-; RUN:      -enable-tail-merge=false -disable-cgp %s -o /dev/null 2>&1 \
-; RUN:	| FileCheck %s
+; RUN: llc < %s -mtriple=arm-eabi -stop-after=tailduplication -tail-dup-size=100 \
+; RUN:      -enable-tail-merge=false -disable-cgp | FileCheck %s
 
-; CHECK: Machine code for function test0:
+; CHECK: name:            test0
 ; CHECK: successors: %bb.1(0x04000000), %bb.2(0x7c000000)
 
 define void @test0(i32 %a, i32 %b, i32* %c, i32* %d) {
@@ -29,7 +28,7 @@ B4:
 
 !0 = !{!"branch_weights", i32 4, i32 124}
 
-; CHECK: Machine code for function test1:
+; CHECK: name:            test1
 ; CHECK: successors: %bb.2(0x7c000000), %bb.1(0x04000000)
 
 @g0 = common global i32 0, align 4
diff --git a/llvm/test/CodeGen/BPF/BTF/binary-format.ll b/llvm/test/CodeGen/BPF/BTF/binary-format.ll
index ec4363cce1801..d5b4b4786c16c 100644
--- a/llvm/test/CodeGen/BPF/BTF/binary-format.ll
+++ b/llvm/test/CodeGen/BPF/BTF/binary-format.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=bpfel -filetype=obj -o - %s | llvm-readelf -x ".BTF" -x ".BTF.ext" | FileCheck -check-prefixes=CHECK,CHECK-EL %s
-; RUN: llc -march=bpfeb -filetype=obj -o - %s | llvm-readelf -x ".BTF" -x ".BTF.ext" | FileCheck -check-prefixes=CHECK,CHECK-EB %s
+; RUN: llc -march=bpfel -filetype=obj -o - %s | llvm-readelf -x ".BTF" -x ".BTF.ext" - | FileCheck -check-prefixes=CHECK,CHECK-EL %s
+; RUN: llc -march=bpfeb -filetype=obj -o - %s | llvm-readelf -x ".BTF" -x ".BTF.ext" - | FileCheck -check-prefixes=CHECK,CHECK-EB %s
 
 ; Source code:
 ;   int f(int a) { return a; }
diff --git a/llvm/test/CodeGen/BPF/CORE/no-elf-ama-symbol.ll b/llvm/test/CodeGen/BPF/CORE/no-elf-ama-symbol.ll
index 8851c502b6f0b..3ac50f120a11a 100644
--- a/llvm/test/CodeGen/BPF/CORE/no-elf-ama-symbol.ll
+++ b/llvm/test/CodeGen/BPF/CORE/no-elf-ama-symbol.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=bpfel -filetype=obj -o - %s | llvm-readelf -s | FileCheck -check-prefixes=CHECK %s
-; RUN: llc -march=bpfeb -filetype=obj -o - %s | llvm-readelf -s | FileCheck -check-prefixes=CHECK %s
-; RUN: llc -march=bpfel -filetype=obj -addrsig -o - %s | llvm-readelf -s | FileCheck -check-prefixes=CHECK %s
-; RUN: llc -march=bpfeb -filetype=obj -addrsig -o - %s | llvm-readelf -s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -filetype=obj -o - %s | llvm-readelf -s - | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=obj -o - %s | llvm-readelf -s - | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -filetype=obj -addrsig -o - %s | llvm-readelf -s - | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=obj -addrsig -o - %s | llvm-readelf -s - | FileCheck -check-prefixes=CHECK %s
 ;
 ; Source Code:
 ;   struct tt { int a; } __attribute__((preserve_access_index));
diff --git a/llvm/test/CodeGen/BPF/elf-symbol-information.ll b/llvm/test/CodeGen/BPF/elf-symbol-information.ll
index 37af7eddea510..5e400dab32e01 100644
--- a/llvm/test/CodeGen/BPF/elf-symbol-information.ll
+++ b/llvm/test/CodeGen/BPF/elf-symbol-information.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=bpfel -filetype=obj -o - %s | llvm-readelf --symbols | FileCheck %s
+; RUN: llc -march=bpfel -filetype=obj -o - %s | llvm-readelf --symbols - | FileCheck %s
 
 ; CHECK: 0 FILE    LOCAL  DEFAULT  ABS elf-symbol-information.ll
 ; CHECK: 8 FUNC    GLOBAL DEFAULT    2 test_func
diff --git a/llvm/test/CodeGen/Generic/print-machineinstrs.ll b/llvm/test/CodeGen/Generic/print-machineinstrs.ll
deleted file mode 100644
index b33e0929edc8c..0000000000000
--- a/llvm/test/CodeGen/Generic/print-machineinstrs.ll
+++ /dev/null
@@ -1,27 +0,0 @@
-; RUN: llc < %s -O3 -debug-pass=Structure -print-machineinstrs=branch-folder -verify-machineinstrs -o /dev/null 2>&1 \
-; RUN:   | FileCheck %s -check-prefix=PRINT-BRANCH-FOLD
-; RUN: llc < %s -O3 -debug-pass=Structure -print-machineinstrs -verify-machineinstrs -o /dev/null 2>&1 \
-; RUN:   | FileCheck %s -check-prefix=PRINT
-; RUN: llc < %s -O3 -debug-pass=Structure -print-machineinstrs= -verify-machineinstrs -o /dev/null 2>&1 \
-; RUN:   | FileCheck %s -check-prefix=PRINT
-
-; Note: -verify-machineinstrs is used in order to make this test compatible with EXPENSIVE_CHECKS.
-
-define i64 @foo(i64 %a, i64 %b) nounwind {
-; PRINT-BRANCH-FOLD: -branch-folder -machineverifier -machineinstr-printer
-; PRINT-BRANCH-FOLD: Control Flow Optimizer
-; PRINT-BRANCH-FOLD-NEXT: Verify generated machine code
-; PRINT-BRANCH-FOLD-NEXT: MachineFunction Printer
-; PRINT-BRANCH-FOLD: Machine code for function foo:
-
-; PRINT: -branch-folder -machineinstr-printer
-; PRINT: Control Flow Optimizer
-; PRINT-NEXT: MachineFunction Printer
-; PRINT-NEXT: Verify generated machine code
-; PRINT: Machine code for function foo:
-
-  %c = add i64 %a, %b
-  %d = trunc i64 %c to i32
-  %e = zext i32 %d to i64
-  ret i64 %e
-}
diff --git a/llvm/test/CodeGen/Hexagon/ifcvt-edge-weight.ll b/llvm/test/CodeGen/Hexagon/ifcvt-edge-weight.ll
index 21b2617cb98d1..3fa095e857a7c 100644
--- a/llvm/test/CodeGen/Hexagon/ifcvt-edge-weight.ll
+++ b/llvm/test/CodeGen/Hexagon/ifcvt-edge-weight.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -mcpu=hexagonv5 -hexagon-eif=0 -print-machineinstrs=if-converter %s -o /dev/null 2>&1 | FileCheck %s
+; RUN: llc -march=hexagon -mcpu=hexagonv5 -hexagon-eif=0 -stop-after=if-converter < %s | FileCheck %s
 ; Check that the edge weights are updated correctly after if-conversion.
 
 ; CHECK: bb.3.if{{[0-9a-zA-Z.]*}}:
diff --git a/llvm/test/CodeGen/Hexagon/simple_addend.ll b/llvm/test/CodeGen/Hexagon/simple_addend.ll
index e0308cdc5cf72..4a6f0323bcbc7 100644
--- a/llvm/test/CodeGen/Hexagon/simple_addend.ll
+++ b/llvm/test/CodeGen/Hexagon/simple_addend.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -filetype=obj -o - < %s | llvm-readobj -r | FileCheck %s
+; RUN: llc -march=hexagon -filetype=obj -o - < %s | llvm-readobj -r - | FileCheck %s
 
 declare void @bar(i32);
 
diff --git a/llvm/test/CodeGen/MIR/X86/diexpr-win32.mir b/llvm/test/CodeGen/MIR/X86/diexpr-win32.mir
index f12e4922f34ec..8f88926376f71 100644
--- a/llvm/test/CodeGen/MIR/X86/diexpr-win32.mir
+++ b/llvm/test/CodeGen/MIR/X86/diexpr-win32.mir
@@ -1,4 +1,4 @@
-# RUN: llc -start-after=prologepilog -filetype=obj -O0 %s -o - | llvm-readobj --codeview | FileCheck %s
+# RUN: llc -start-after=prologepilog -filetype=obj -O0 %s -o - | llvm-readobj --codeview - | FileCheck %s
 
 # Offsets are now CFA, or VFRAME, relative. Both the NRVO sret pointer and the
 # string* parameter are on the stack, NRVO at offset 4 (after RA), and Str at
diff --git a/llvm/test/CodeGen/Mips/cpus.ll b/llvm/test/CodeGen/Mips/cpus.ll
index 182d13b1c7e30..5f9740c2ce866 100644
--- a/llvm/test/CodeGen/Mips/cpus.ll
+++ b/llvm/test/CodeGen/Mips/cpus.ll
@@ -1,57 +1,57 @@
 ; Check that the CPU names work.
 
 ; RUN: llc -mtriple=mips -mcpu=generic -filetype=obj < %s \
-; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=GENERIC
+; RUN:   | llvm-readelf -A - | FileCheck %s --check-prefix=GENERIC
 ; GENERIC: ISA: MIPS32
 
 ; RUN: llc -mtriple=mips -mcpu=mips2 -filetype=obj < %s \
-; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=MIPS2
+; RUN:   | llvm-readelf -A - | FileCheck %s --check-prefix=MIPS2
 ; MIPS2: ISA: MIPS2
 ; RUN: llc -mtriple=mips64 -mcpu=mips3 -filetype=obj < %s \
-; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=MIPS3
+; RUN:   | llvm-readelf -A - | FileCheck %s --check-prefix=MIPS3
 ; MIPS3: ISA: MIPS3
 ; RUN: llc -mtriple=mips64 -mcpu=mips4 -filetype=obj < %s \
-; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=MIPS4
+; RUN:   | llvm-readelf -A - | FileCheck %s --check-prefix=MIPS4
 ; MIPS4: ISA: MIPS4
 
 ; RUN: llc -mtriple=mips -mcpu=mips32 -filetype=obj < %s \
-; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=MIPS32
+; RUN:   | llvm-readelf -A - | FileCheck %s --check-prefix=MIPS32
 ; MIPS32: ISA: MIPS32
 ; RUN: llc -mtriple=mips -mcpu=mips32r2 -filetype=obj < %s \
-; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=MIPS32R2
+; RUN:   | llvm-readelf -A - | FileCheck %s --check-prefix=MIPS32R2
 ; MIPS32R2: ISA: MIPS32r2
 ; RUN: llc -mtriple=mips -mcpu=mips32r3 -filetype=obj < %s \
-; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=MIPS32R3
+; RUN:   | llvm-readelf -A - | FileCheck %s --check-prefix=MIPS32R3
 ; MIPS32R3: ISA: MIPS32r3
 ; RUN: llc -mtriple=mips -mcpu=mips32r5 -filetype=obj < %s \
-; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=MIPS32R5
+; RUN:   | llvm-readelf -A - | FileCheck %s --check-prefix=MIPS32R5
 ; MIPS32R5: ISA: MIPS32r5
 ; RUN: llc -mtriple=mips -mcpu=mips32r6 -filetype=obj < %s \
-; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=MIPS32R6
+; RUN:   | llvm-readelf -A - | FileCheck %s --check-prefix=MIPS32R6
 ; MIPS32R6: ISA: MIPS32r6
 
 ; RUN: llc -mtriple=mips64 -mcpu=mips64 -filetype=obj < %s \
-; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=MIPS64
+; RUN:   | llvm-readelf -A - | FileCheck %s --check-prefix=MIPS64
 ; MIPS64: ISA: MIPS64
 ; RUN: llc -mtriple=mips64 -mcpu=mips64r2 -filetype=obj < %s \
-; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=MIPS64R2
+; RUN:   | llvm-readelf -A - | FileCheck %s --check-prefix=MIPS64R2
 ; MIPS64R2: ISA: MIPS64r2
 ; RUN: llc -mtriple=mips64 -mcpu=mips64r3 -filetype=obj < %s \
-; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=MIPS64R3
+; RUN:   | llvm-readelf -A - | FileCheck %s --check-prefix=MIPS64R3
 ; MIPS64R3: ISA: MIPS64r3
 ; RUN: llc -mtriple=mips64 -mcpu=mips64r5 -filetype=obj < %s \
-; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=MIPS64R5
+; RUN:   | llvm-readelf -A - | FileCheck %s --check-prefix=MIPS64R5
 ; MIPS64R5: ISA: MIPS64r5
 ; RUN: llc -mtriple=mips64 -mcpu=mips64r6 -filetype=obj < %s \
-; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=MIPS64R6
+; RUN:   | llvm-readelf -A - | FileCheck %s --check-prefix=MIPS64R6
 ; MIPS64R6: ISA: MIPS64r6
 
 ; RUN: llc -mtriple=mips64 -mcpu=octeon -filetype=obj < %s \
-; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=OCTEON
+; RUN:   | llvm-readelf -A - | FileCheck %s --check-prefix=OCTEON
 ; OCTEON: ISA: MIPS64r2
 ; OCTEON: ISA Extension: Cavium Networks Octeon
 ; RUN: llc -mtriple=mips64 -mcpu=octeon+ -filetype=obj < %s \
-; RUN:   | llvm-readelf -A | FileCheck %s --check-prefix=OCTEONP
+; RUN:   | llvm-readelf -A - | FileCheck %s --check-prefix=OCTEONP
 ; OCTEONP: ISA: MIPS64r2
 ; OCTEONP: ISA Extension: Cavium Networks OcteonP
 
diff --git a/llvm/test/CodeGen/Mips/micromips-ase-function-attribute.ll b/llvm/test/CodeGen/Mips/micromips-ase-function-attribute.ll
index cd78166d372a6..edb94496b3e92 100644
--- a/llvm/test/CodeGen/Mips/micromips-ase-function-attribute.ll
+++ b/llvm/test/CodeGen/Mips/micromips-ase-function-attribute.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -mtriple=mips-unknown-linux -filetype=obj %s -o - | \
-; RUN:   llvm-readobj -A | \
+; RUN:   llvm-readobj -A - | \
 ; RUN:   FileCheck --check-prefix=ASE-MICROMIPS %s
 
 define void @_Z3foov() #0 {
diff --git a/llvm/test/CodeGen/Mips/mips-shf-gprel.s b/llvm/test/CodeGen/Mips/mips-shf-gprel.s
index 0228a3fa24b59..51a513a77859f 100644
--- a/llvm/test/CodeGen/Mips/mips-shf-gprel.s
+++ b/llvm/test/CodeGen/Mips/mips-shf-gprel.s
@@ -2,7 +2,7 @@
 # and proper section types.
 
 # RUN: llvm-mc -filetype=obj -triple=mips-unknown-linux %s -o - \
-# RUN:   | llvm-readobj -S | FileCheck %s
+# RUN:   | llvm-readobj -S - | FileCheck %s
 
   .sdata
   .word 0
diff --git a/llvm/test/CodeGen/PowerPC/aix-alias-unsupported.ll b/llvm/test/CodeGen/PowerPC/aix-alias-unsupported.ll
new file mode 100644
index 0000000000000..342f7d419f337
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-alias-unsupported.ll
@@ -0,0 +1,6 @@
+; RUN: not --crash llc < %s -mtriple powerpc-ibm-aix-xcoff 2>&1 | FileCheck %s
+; RUN: not --crash llc < %s -mtriple powerpc64-ibm-aix-xcoff 2>&1 | FileCheck %s
+; CHECK: ERROR: alias without a base object is not yet supported on AIX
+
+@bar = global i32 42
+@test = alias i32, inttoptr(i32 42 to i32*)
diff --git a/llvm/test/CodeGen/PowerPC/aix-alias.ll b/llvm/test/CodeGen/PowerPC/aix-alias.ll
index 2183c2e8c5579..1fdd7c90f5ada 100644
--- a/llvm/test/CodeGen/PowerPC/aix-alias.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-alias.ll
@@ -1,10 +1,116 @@
-; RUN: not --crash llc < %s -mtriple powerpc-ibm-aix-xcoff 2>&1 | FileCheck %s
-; RUN: not --crash llc < %s -mtriple powerpc64-ibm-aix-xcoff 2>&1 | FileCheck %s
+; TODO: Add object generation test when visibility for object generation
+;       is implemnted.
 
-; Check that, while generation of aliases on AIX remains unimplemented, llc dies
-; with an appropriate message instead of generating incorrect output when an
-; alias is encountered.
+; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr4 \
+; RUN:     -mattr=-altivec < %s | \
+; RUN:   FileCheck --check-prefix=ASM %s
+; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff -mcpu=pwr4 \
+; RUN:     -mattr=-altivec < %s | \
+; RUN:   FileCheck --check-prefix=ASM %s
 
-define i32 @a() { ret i32 0 }
-; CHECK: ERROR: module has aliases
-@b = internal alias i32 (), i32 ()* @a
+@var = global i32 42
+@var1 = alias i32, i32* @var
+@var2 = alias i32, i32* @var1
+@var_l = linkonce_odr alias i32, i32* @var
+@var_i = internal alias i32, i32* @var
+@var_h = hidden alias i32, i32* @var
+@var_p = protected alias i32, i32* @var
+
+@array = global [2 x i32] [i32 1, i32 2], align 4
+@x = global i32* bitcast (i8* getelementptr (i8, i8* bitcast ([2 x i32]* @array to i8*), i64 4) to i32*), align 4
+@bitcast_alias = alias i32*, i32** @x
+
+define i32 @fun() {
+  ret i32 0
+}
+
+%FunTy = type i32()
+@fun_weak = weak alias %FunTy, %FunTy* @fun
+@fun_hidden = hidden alias %FunTy, %FunTy* @fun
+@fun_ptr = global i32()* @fun_weak
+
+define i32 @test() {
+entry:
+   %tmp = load i32, i32* @var1
+   %tmp1 = load i32, i32* @var2
+   %tmp0 = load i32, i32* @var_i
+   %tmp2 = call i32 @fun()
+   %tmp3 = add i32 %tmp, %tmp2
+   %tmp4 = call i32 @fun_weak()
+   %tmp5 = add i32 %tmp3, %tmp4
+   %tmp6 = add i32 %tmp1, %tmp5
+   %tmp7 = add i32 %tmp6, %tmp0
+   %fun_ptr1 = alloca i32 ()*
+   store i32 ()* @fun_weak, i32 ()** %fun_ptr1
+   %callee.knr.cast = bitcast i32 ()** %fun_ptr1 to i32 ()*
+   %tmp8 = call i32 %callee.knr.cast()
+   %tmp9 = call i32 @fun_hidden()
+   %tmp10 = add i32 %tmp7, %tmp8
+   %tmp11 = add i32 %tmp10, %tmp9
+   ret i32 %tmp11
+}
+
+; ASM:         .globl  fun[DS]
+; ASM-NEXT:    .globl  .fun
+; ASM-NEXT:    .align  4
+; ASM-NEXT:    .csect fun[DS]
+; ASM-NEXT:  fun_weak:                               # @fun
+; ASM-NEXT:  fun_hidden:
+; ASM:         .csect .text[PR],2
+; ASM-NEXT:  .fun:
+; ASM-NEXT:  .fun_weak:
+; ASM-NEXT:  .fun_hidden:
+; ASM-NEXT:  # %bb.0:
+; ASM-NEXT:    li 3, 0
+; ASM-NEXT:    blr
+; ASM-NEXT:                                          # -- End function
+; ASM:       .csect .text[PR],2
+; ASM-NEXT:  .test:
+; ASM-NEXT:  # %bb.0:                                # %entry
+; ASM:         bl .fun
+; ASM-NEXT:    nop
+; ASM:         bl .fun_weak
+; ASM-NEXT:    nop
+; ASM:         bl .fun_hidden
+; ASM:                                               # -- End function
+; ASM-NEXT:    .csect .data[RW]
+; ASM-NEXT:    .globl  var
+; ASM:       var:
+; ASM-NEXT:  var1:
+; ASM-NEXT:  var2:
+; ASM-NEXT:  var_l:
+; ASM-NEXT:  var_i:
+; ASM-NEXT:  var_h:
+; ASM-NEXT:  var_p:
+; ASM-NEXT:    .vbyte  4, 42
+; ASM-NEXT:   .globl array
+; ASM:       array:
+; ASM-NEXT:    .vbyte 4, 1 # 0x1
+; ASM-NEXT:    .vbyte 4, 2 # 0x2
+; ASM-NEXT:    .globl x
+; ASM:       x:
+; ASM-NEXT:  bitcast_alias:
+; ASM-NEXT:    .vbyte  {{[0-9]+}}, array+4
+; ASM-NEXT:    .globl  fun_ptr
+; ASM:       fun_ptr:
+; ASM-NEXT:    .vbyte  {{[0-9]+}}, fun_weak
+; ASM-NEXT:    .globl  var1
+; ASM-NEXT:    .globl  var2
+; ASM-NEXT:    .weak var_l
+; ASM-NEXT:    .lglobl var_i
+; ASM-NEXT:    .globl  var_h,hidden
+; ASM-NEXT:    .globl  var_p,protected
+; ASM-NEXT:    .globl bitcast_alias
+; ASM-NEXT:    .weak fun_weak
+; ASM-NEXT:    .weak .fun_weak
+; ASM-NEXT:    .globl  fun_hidden,hidden
+; ASM-NEXT:    .globl  .fun_hidden,hidden
+; ASM-NEXT:    .toc
+; ASM-NEXT:  L..C0:
+; ASM-NEXT:    .tc var1[TC],var1
+; ASM-NEXT:  L..C1:
+; ASM-NEXT:    .tc var2[TC],var2
+; ASM-NEXT:  L..C2:
+; ASM-NEXT:    .tc var_i[TC],var_i
+; ASM-NEXT:  L..C3:
+; ASM-NEXT:    .tc fun_weak[TC],fun_weak
diff --git a/llvm/test/CodeGen/PowerPC/aix-cc-byval-mem.ll b/llvm/test/CodeGen/PowerPC/aix-cc-byval-mem.ll
index 53f18cd39ae0a..82927fdda86ae 100644
--- a/llvm/test/CodeGen/PowerPC/aix-cc-byval-mem.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-cc-byval-mem.ll
@@ -88,7 +88,7 @@ entry:
 ; 32BIT-DAG:      $r3 = COPY %0
 ; 32BIT-DAG:      $r4 = COPY %1
 ; 32BIT-DAG:      $r5 = COPY %2
-; 32BIT-NEXT:     BL_NOP <mcsymbol .memcpy>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $r3, implicit $r4, implicit $r5, implicit $r2, implicit-def $r1, implicit-def $r3
+; 32BIT-NEXT:     BL_NOP &.memcpy, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $r3, implicit $r4, implicit $r5, implicit $r2, implicit-def $r1, implicit-def $r3
 ; 32BIT-NEXT:     ADJCALLSTACKUP 56, 0, implicit-def dead $r1, implicit $r1
 ; 32BIT:          ADJCALLSTACKDOWN 312, 0, implicit-def dead $r1, implicit $r1
 ; 32BIT-DAG:      $r3 = COPY %{{[0-9]+}}
@@ -120,7 +120,7 @@ entry:
 ; 64BIT-DAG:      $x3 = COPY %0
 ; 64BIT-DAG:      $x4 = COPY %1
 ; 64BIT-DAG:      $x5 = COPY %2
-; 64BIT-NEXT:     BL8_NOP <mcsymbol .memcpy>, csr_ppc64, implicit-def dead $lr8, implicit $rm, implicit $x3, implicit $x4, implicit $x5, implicit $x2, implicit-def $r1, implicit-def $x3
+; 64BIT-NEXT:     BL8_NOP &.memcpy, csr_ppc64, implicit-def dead $lr8, implicit $rm, implicit $x3, implicit $x4, implicit $x5, implicit $x2, implicit-def $r1, implicit-def $x3
 ; 64BIT-NEXT:     ADJCALLSTACKUP 112, 0, implicit-def dead $r1, implicit $r1
 ; 64BIT:          ADJCALLSTACKDOWN 368, 0, implicit-def dead $r1, implicit $r1
 ; 64BIT-DAG:      $x3 = COPY %{{[0-9]+}}
@@ -187,7 +187,7 @@ entry:
 ; 32BIT-DAG:      $r3 = COPY %2
 ; 32BIT-DAG:      $r4 = COPY %1
 ; 32BIT-DAG:      $r5 = COPY %3
-; 32BIT-NEXT:     BL_NOP <mcsymbol .memcpy>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $r3, implicit $r4, implicit $r5, implicit $r2, implicit-def $r1, implicit-def $r3
+; 32BIT-NEXT:     BL_NOP &.memcpy, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $r3, implicit $r4, implicit $r5, implicit $r2, implicit-def $r1, implicit-def $r3
 ; 32BIT-NEXT:     ADJCALLSTACKUP 56, 0, implicit-def dead $r1, implicit $r1
 ; 32BIT:          ADJCALLSTACKDOWN 92, 0, implicit-def dead $r1, implicit $r1
 ; 32BIT-DAG:      $r3 = COPY %{{[0-9]+}}
@@ -305,7 +305,7 @@ entry:
 ; 32BIT-DAG:      $r3 = COPY %3
 ; 32BIT-DAG:      $r4 = COPY %4
 ; 32BIT-DAG:      $r5 = COPY %5
-; 32BIT-NEXT:     BL_NOP <mcsymbol .memcpy>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $r3, implicit $r4, implicit $r5, implicit $r2, implicit-def $r1, implicit-def $r3
+; 32BIT-NEXT:     BL_NOP &.memcpy, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $r3, implicit $r4, implicit $r5, implicit $r2, implicit-def $r1, implicit-def $r3
 ; 32BIT-NEXT:     ADJCALLSTACKUP 56, 0, implicit-def dead $r1, implicit $r1
 ; 32BIT:          ADJCALLSTACKDOWN 316, 0, implicit-def dead $r1, implicit $r1
 ; 32BIT-DAG:      $r3 = COPY %{{[0-9]+}}
@@ -349,7 +349,7 @@ entry:
 ; 64BIT-DAG:      $x3 = COPY %2
 ; 64BIT-DAG:      $x4 = COPY %1
 ; 64BIT-DAG:      $x5 = COPY %3
-; 64BIT-NEXT:     BL8_NOP <mcsymbol .memcpy>, csr_ppc64, implicit-def dead $lr8, implicit $rm, implicit $x3, implicit $x4, implicit $x5, implicit $x2, implicit-def $r1, implicit-def $x3
+; 64BIT-NEXT:     BL8_NOP &.memcpy, csr_ppc64, implicit-def dead $lr8, implicit $rm, implicit $x3, implicit $x4, implicit $x5, implicit $x2, implicit-def $r1, implicit-def $x3
 ; 64BIT-NEXT:     ADJCALLSTACKUP 112, 0, implicit-def dead $r1, implicit $r1
 ; 64BIT:          ADJCALLSTACKDOWN 344, 0, implicit-def dead $r1, implicit $r1
 ; 64BIT-DAG:      $x3 = COPY %{{[0-9]+}}
diff --git a/llvm/test/CodeGen/PowerPC/aix-external-sym-sdnode-lowering.ll b/llvm/test/CodeGen/PowerPC/aix-external-sym-sdnode-lowering.ll
index a4957a514e092..690ff99d05304 100644
--- a/llvm/test/CodeGen/PowerPC/aix-external-sym-sdnode-lowering.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-external-sym-sdnode-lowering.ll
@@ -14,5 +14,5 @@ entry:
 
 declare double @llvm.ceil.f64(double)
 
-; 32BIT: BL_NOP <mcsymbol .ceil>
-; 64BIT: BL8_NOP <mcsymbol .ceil>
+; 32BIT: BL_NOP &.ceil
+; 64BIT: BL8_NOP &.ceil
diff --git a/llvm/test/CodeGen/PowerPC/aix-llvm-intrinsic.ll b/llvm/test/CodeGen/PowerPC/aix-llvm-intrinsic.ll
new file mode 100644
index 0000000000000..cc0baf5d0436c
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-llvm-intrinsic.ll
@@ -0,0 +1,70 @@
+; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr4 -mattr=-altivec < %s | \
+; RUN:   FileCheck %s
+
+; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff -mcpu=pwr4 -mattr=-altivec < %s | \
+; RUN:   FileCheck %s
+
+; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr4 \
+; RUN:     -mattr=-altivec -filetype=obj -o %t.o < %s
+; RUN: llvm-readobj --symbols %t.o | FileCheck --check-prefix=CHECKSYM %s
+; RUN: llvm-objdump -r -d --symbol-description %t.o | FileCheck --check-prefix=CHECKRELOC %s
+
+; RUN: not --crash llc -verify-machineinstrs -mcpu=pwr4 -mtriple powerpc64-ibm-aix-xcoff \
+; RUN:                 -mattr=-altivec -filetype=obj -o %t.o 2>&1 < %s | \
+; RUN:   FileCheck --check-prefix=XCOFF64 %s
+; XCOFF64: LLVM ERROR: 64-bit XCOFF object files are not supported yet.
+
+%struct.S = type { i32, i32 }
+
+@s = external global %struct.S, align 4
+
+define void @bar() {
+entry:
+  %0 = load i32, i32* getelementptr inbounds (%struct.S, %struct.S* @s, i32 0, i32 1), align 4
+  %1 = trunc i32 %0 to i8
+  %2 = load i32, i32* getelementptr inbounds (%struct.S, %struct.S* @s, i32 0, i32 1), align 4
+  call void @llvm.memset.p0i8.i32(i8* align 4 bitcast (%struct.S* @s to i8*), i8 %1, i32 %2, i1 false)
+  ret void
+}
+
+declare void @llvm.memset.p0i8.i32(i8* nocapture writeonly, i8, i32, i1 immarg)
+
+; CHECK-LABEL: .bar:
+; CHECK-NEXT: # %bb.0:                                # %entry
+; CHECK-NEXT:         mflr 0
+
+; CHECK:              bl .memset
+
+; CHECK:              .extern .memset
+
+; CHECKSYM:      Symbol {
+; CHECKSYM-NEXT:     Index: 0
+; CHECKSYM-NEXT:     Name: .memset
+; CHECKSYM-NEXT:     Value (RelocatableAddress): 0x0
+; CHECKSYM-NEXT:     Section: N_UNDEF
+; CHECKSYM-NEXT:     Type: 0x0
+; CHECKSYM-NEXT:     StorageClass: C_EXT (0x2)
+; CHECKSYM-NEXT:     NumberOfAuxEntries: 1
+; CHECKSYM-NEXT:     CSECT Auxiliary Entry {
+; CHECKSYM-NEXT:       Index: 1
+; CHECKSYM-NEXT:       SectionLen: 0
+; CHECKSYM-NEXT:       ParameterHashIndex: 0x0
+; CHECKSYM-NEXT:       TypeChkSectNum: 0x0
+; CHECKSYM-NEXT:       SymbolAlignmentLog2: 0
+; CHECKSYM-NEXT:       SymbolType: XTY_ER (0x0)
+; CHECKSYM-NEXT:       StorageMappingClass: XMC_PR (0x0)
+; CHECKSYM-NEXT:       StabInfoIndex: 0x0
+; CHECKSYM-NEXT:       StabSectNum: 0x0
+; CHECKSYM-NEXT:     }
+; CHECKSYM-NEXT:   }
+
+; CHECKRELOC:      00000000 (idx: 6) .bar:
+; CHECKRELOC-NEXT:        0: 7c 08 02 a6                        mflr 0
+; CHECKRELOC-NEXT:        4: 90 01 00 08                        stw 0, 8(1)
+; CHECKRELOC-NEXT:        8: 94 21 ff c0                        stwu 1, -64(1)
+; CHECKRELOC-NEXT:        c: 80 62 00 00                        lwz 3, 0(2)
+; CHECKRELOC-NEXT:                      0000000e:  R_TOC        (idx: 12) s[TC]
+; CHECKRELOC-NEXT:       10: 80 83 00 04                        lwz 4, 4(3)
+; CHECKRELOC-NEXT:       14: 7c 85 23 78                        mr 5, 4
+; CHECKRELOC-NEXT:       18: 4b ff ff e9                        bl 0x0
+; CHECKRELOC-NEXT:                      00000018:  R_RBR        (idx: 0) .memset[PR]
diff --git a/llvm/test/CodeGen/PowerPC/aix-user-defined-memcpy.ll b/llvm/test/CodeGen/PowerPC/aix-user-defined-memcpy.ll
index 0e6fbcb6a3efa..c24108895c2ac 100644
--- a/llvm/test/CodeGen/PowerPC/aix-user-defined-memcpy.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-user-defined-memcpy.ll
@@ -8,6 +8,10 @@
 
 ; RUN: llvm-objdump -D %t.o | FileCheck --check-prefix=32-DIS %s
 
+; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff \
+; RUN:     -mcpu=pwr4 -mattr=-altivec < %s | \
+; RUN:   FileCheck %s
+
 ; RUN: not --crash llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff \
 ; RUN: -mcpu=pwr4 -mattr=-altivec -filetype=obj < %s 2>&1 | FileCheck \
 ; RUN: --check-prefix=64-CHECK %s
@@ -35,6 +39,8 @@ declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture writeonly, i8* nocapture r
 ; 2. There is no relocation associated with the call, since callee is defined.
 ; 3. Branch instruction in raw data is branching back to the right callee location.
 
+; CHECK-NOT: .extern .memcpy
+
 ; 32-SYM:      Symbol {{[{][[:space:]] *}}Index: [[#Index:]]{{[[:space:]] *}}Name: .memcpy 
 ; 32-SYM-NEXT:    Value (RelocatableAddress): 0x0
 ; 32-SYM-NEXT:    Section: .text
diff --git a/llvm/test/CodeGen/PowerPC/builtins-ppc-p10permute.ll b/llvm/test/CodeGen/PowerPC/builtins-ppc-p10permute.ll
index 3e4a509dc943b..73a174297cacc 100644
--- a/llvm/test/CodeGen/PowerPC/builtins-ppc-p10permute.ll
+++ b/llvm/test/CodeGen/PowerPC/builtins-ppc-p10permute.ll
@@ -170,67 +170,67 @@ entry:
 }
 declare <2 x i64> @llvm.ppc.altivec.vinsdrx(<2 x i64>, i64, i64)
 
-define <16 x i8> @testVINSBVLX(<16 x i8> %a, i64 %b, <16 x i8> %c) {
+define <16 x i8> @testVINSBVLX(<16 x i8> %a, i32 %b, <16 x i8> %c) {
 ; CHECK-LABEL: testVINSBVLX:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vinsbvlx v2, r5, v3
 ; CHECK-NEXT:    blr
 entry:
-  %0 = tail call <16 x i8> @llvm.ppc.altivec.vinsbvlx(<16 x i8> %a, i64 %b, <16 x i8> %c)
+  %0 = tail call <16 x i8> @llvm.ppc.altivec.vinsbvlx(<16 x i8> %a, i32 %b, <16 x i8> %c)
   ret <16 x i8> %0
 }
-declare <16 x i8> @llvm.ppc.altivec.vinsbvlx(<16 x i8>, i64, <16 x i8>)
+declare <16 x i8> @llvm.ppc.altivec.vinsbvlx(<16 x i8>, i32, <16 x i8>)
 
-define <16 x i8> @testVINSBVRX(<16 x i8> %a, i64 %b, <16 x i8> %c) {
+define <16 x i8> @testVINSBVRX(<16 x i8> %a, i32 %b, <16 x i8> %c) {
 ; CHECK-LABEL: testVINSBVRX:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vinsbvrx v2, r5, v3
 ; CHECK-NEXT:    blr
 entry:
-  %0 = tail call <16 x i8> @llvm.ppc.altivec.vinsbvrx(<16 x i8> %a, i64 %b, <16 x i8> %c)
+  %0 = tail call <16 x i8> @llvm.ppc.altivec.vinsbvrx(<16 x i8> %a, i32 %b, <16 x i8> %c)
   ret <16 x i8> %0
 }
-declare <16 x i8> @llvm.ppc.altivec.vinsbvrx(<16 x i8>, i64, <16 x i8>)
+declare <16 x i8> @llvm.ppc.altivec.vinsbvrx(<16 x i8>, i32, <16 x i8>)
 
-define <8 x i16> @testVINSHVLX(<8 x i16> %a, i64 %b, <8 x i16> %c) {
+define <8 x i16> @testVINSHVLX(<8 x i16> %a, i32 %b, <8 x i16> %c) {
 ; CHECK-LABEL: testVINSHVLX:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vinshvlx v2, r5, v3
 ; CHECK-NEXT:    blr
 entry:
-  %0 = tail call <8 x i16> @llvm.ppc.altivec.vinshvlx(<8 x i16> %a, i64 %b, <8 x i16> %c)
+  %0 = tail call <8 x i16> @llvm.ppc.altivec.vinshvlx(<8 x i16> %a, i32 %b, <8 x i16> %c)
   ret <8 x i16> %0
 }
-declare <8 x i16> @llvm.ppc.altivec.vinshvlx(<8 x i16>, i64, <8 x i16>)
+declare <8 x i16> @llvm.ppc.altivec.vinshvlx(<8 x i16>, i32, <8 x i16>)
 
-define <8 x i16> @testVINSHVRX(<8 x i16> %a, i64 %b, <8 x i16> %c) {
+define <8 x i16> @testVINSHVRX(<8 x i16> %a, i32 %b, <8 x i16> %c) {
 entry:
-  %0 = tail call <8 x i16> @llvm.ppc.altivec.vinshvrx(<8 x i16> %a, i64 %b, <8 x i16> %c)
+  %0 = tail call <8 x i16> @llvm.ppc.altivec.vinshvrx(<8 x i16> %a, i32 %b, <8 x i16> %c)
   ret <8 x i16> %0
 }
-declare <8 x i16> @llvm.ppc.altivec.vinshvrx(<8 x i16>, i64, <8 x i16>)
+declare <8 x i16> @llvm.ppc.altivec.vinshvrx(<8 x i16>, i32, <8 x i16>)
 
-define <4 x i32> @testVINSWVLX(<4 x i32> %a, i64 %b, <4 x i32> %c) {
+define <4 x i32> @testVINSWVLX(<4 x i32> %a, i32 %b, <4 x i32> %c) {
 ; CHECK-LABEL: testVINSWVLX:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vinswvlx v2, r5, v3
 ; CHECK-NEXT:    blr
 entry:
-  %0 = tail call <4 x i32> @llvm.ppc.altivec.vinswvlx(<4 x i32> %a, i64 %b, <4 x i32> %c)
+  %0 = tail call <4 x i32> @llvm.ppc.altivec.vinswvlx(<4 x i32> %a, i32 %b, <4 x i32> %c)
   ret <4 x i32> %0
 }
-declare <4 x i32> @llvm.ppc.altivec.vinswvlx(<4 x i32>, i64, <4 x i32>)
+declare <4 x i32> @llvm.ppc.altivec.vinswvlx(<4 x i32>, i32, <4 x i32>)
 
-define <4 x i32> @testVINSWVRX(<4 x i32> %a, i64 %b, <4 x i32> %c) {
+define <4 x i32> @testVINSWVRX(<4 x i32> %a, i32 %b, <4 x i32> %c) {
 ; CHECK-LABEL: testVINSWVRX:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vinswvrx v2, r5, v3
 ; CHECK-NEXT:    blr
 entry:
-  %0 = tail call <4 x i32> @llvm.ppc.altivec.vinswvrx(<4 x i32> %a, i64 %b, <4 x i32> %c)
+  %0 = tail call <4 x i32> @llvm.ppc.altivec.vinswvrx(<4 x i32> %a, i32 %b, <4 x i32> %c)
   ret <4 x i32> %0
 }
-declare <4 x i32> @llvm.ppc.altivec.vinswvrx(<4 x i32>, i64, <4 x i32>)
+declare <4 x i32> @llvm.ppc.altivec.vinswvrx(<4 x i32>, i32, <4 x i32>)
 
 define <4 x i32> @testVINSW(<4 x i32> %a, i32 %b) {
 ; CHECK-LABEL: testVINSW:
diff --git a/llvm/test/CodeGen/PowerPC/builtins-ppc-p10vsx.ll b/llvm/test/CodeGen/PowerPC/builtins-ppc-p10vsx.ll
index d4e71d18c6ebb..2ac1b2b7514bc 100644
--- a/llvm/test/CodeGen/PowerPC/builtins-ppc-p10vsx.ll
+++ b/llvm/test/CodeGen/PowerPC/builtins-ppc-p10vsx.ll
@@ -2,11 +2,14 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
 ; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
 ; RUN:   FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O0 \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
+; RUN:   FileCheck %s
 
 ; These test cases aims to test the builtins for the Power10 VSX vector
 ; instructions introduced in ISA 3.1.
 
-declare i32 @llvm.ppc.vsx.xvtlsbb(<16 x i8>, i1)
+declare i32 @llvm.ppc.vsx.xvtlsbb(<16 x i8>, i32)
 
 define signext i32 @test_vec_test_lsbb_all_ones(<16 x i8> %vuca) {
 ; CHECK-LABEL: test_vec_test_lsbb_all_ones:
@@ -17,7 +20,7 @@ define signext i32 @test_vec_test_lsbb_all_ones(<16 x i8> %vuca) {
 ; CHECK-NEXT:    extsw r3, r3
 ; CHECK-NEXT:    blr
 entry:
-  %0 = tail call i32 @llvm.ppc.vsx.xvtlsbb(<16 x i8> %vuca, i1 1)
+  %0 = tail call i32 @llvm.ppc.vsx.xvtlsbb(<16 x i8> %vuca, i32 1)
   ret i32 %0
 }
 
@@ -30,6 +33,6 @@ define signext i32 @test_vec_test_lsbb_all_zeros(<16 x i8> %vuca) {
 ; CHECK-NEXT:    extsw r3, r3
 ; CHECK-NEXT:    blr
 entry:
-  %0 = tail call i32 @llvm.ppc.vsx.xvtlsbb(<16 x i8> %vuca, i1 0)
+  %0 = tail call i32 @llvm.ppc.vsx.xvtlsbb(<16 x i8> %vuca, i32 0)
   ret i32 %0
 }
diff --git a/llvm/test/CodeGen/PowerPC/constant-pool.ll b/llvm/test/CodeGen/PowerPC/constant-pool.ll
index 4355cfa6ba211..af2d823884968 100644
--- a/llvm/test/CodeGen/PowerPC/constant-pool.ll
+++ b/llvm/test/CodeGen/PowerPC/constant-pool.ll
@@ -1,5 +1,7 @@
-; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
-; RUN:   -mcpu=future -ppc-asm-full-reg-names < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-- \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -target-abi=elfv2 -mtriple=powerpc64-- \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names < %s | FileCheck %s
 
  define float @FloatConstantPool() {
 ; CHECK-LABEL: FloatConstantPool:
diff --git a/llvm/test/CodeGen/PowerPC/fixup-kill-dead-flag-crash.mir b/llvm/test/CodeGen/PowerPC/fixup-kill-dead-flag-crash.mir
new file mode 100644
index 0000000000000..be2671fa9b5d0
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/fixup-kill-dead-flag-crash.mir
@@ -0,0 +1,21 @@
+# RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -verify-machineinstrs -start-before ppc-mi-peepholes \
+# RUN:   -stop-after ppc-mi-peepholes %s -o - | FileCheck %s
+
+---
+name: test
+#CHECK : name : test
+tracksRegLiveness: true
+body: |
+  bb.0.entry:
+    liveins: $x3
+    %0:g8rc = COPY $x3
+    %1:gprc = COPY %0.sub_32:g8rc
+    %2:g8rc = LI8 63
+
+  bb.1:
+    %3:gprc = COPY %2.sub_32:g8rc
+    ; CHECK: %4:gprc = LI 0
+    %4:gprc = XORI killed %3:gprc, 63
+    STW killed %4:gprc, %4:gprc, 100
+    BLR8 implicit $lr8, implicit $rm
+...
diff --git a/llvm/test/CodeGen/PowerPC/global-address-non-got-indirect-access.ll b/llvm/test/CodeGen/PowerPC/global-address-non-got-indirect-access.ll
index 5c49760a9f45a..af88fa02c2669 100644
--- a/llvm/test/CodeGen/PowerPC/global-address-non-got-indirect-access.ll
+++ b/llvm/test/CodeGen/PowerPC/global-address-non-got-indirect-access.ll
@@ -1,6 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
-; RUN:   -mcpu=future -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s \
+; RUN:   | FileCheck %s
+; RUN: llc -verify-machineinstrs -target-abi=elfv2 -mtriple=powerpc64-- \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s \
 ; RUN:   | FileCheck %s
 
  @_ZL13StaticBoolVar = internal unnamed_addr global i8 0, align 1
diff --git a/llvm/test/CodeGen/PowerPC/ifcvt.mir b/llvm/test/CodeGen/PowerPC/ifcvt.mir
index b54c972baa17f..3572ce83fc0eb 100644
--- a/llvm/test/CodeGen/PowerPC/ifcvt.mir
+++ b/llvm/test/CodeGen/PowerPC/ifcvt.mir
@@ -49,5 +49,5 @@ body:             |
 
 
   ; CHECK-LABEL: name: testBDZLR
-  ; CHECK: BDZLR implicit $lr, implicit $rm
+  ; CHECK: BDZLR implicit $lr, implicit $rm, implicit $ctr, implicit-def $ctr
 ...
diff --git a/llvm/test/CodeGen/PowerPC/mcm-obj-2.ll b/llvm/test/CodeGen/PowerPC/mcm-obj-2.ll
index 0884d34a88a01..ce305800de83f 100644
--- a/llvm/test/CodeGen/PowerPC/mcm-obj-2.ll
+++ b/llvm/test/CodeGen/PowerPC/mcm-obj-2.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -relocation-model=static -verify-machineinstrs -O1 -mcpu=pwr7 -code-model=medium -filetype=obj %s -o - | \
-; RUN: llvm-readobj -r | FileCheck %s
+; RUN: llvm-readobj -r - | FileCheck %s
 
 ; FIXME: When asm-parse is available, could make this an assembly test.
 
diff --git a/llvm/test/CodeGen/PowerPC/mcm-obj.ll b/llvm/test/CodeGen/PowerPC/mcm-obj.ll
index fa899b5b30167..452aa83fd1d4f 100644
--- a/llvm/test/CodeGen/PowerPC/mcm-obj.ll
+++ b/llvm/test/CodeGen/PowerPC/mcm-obj.ll
@@ -1,13 +1,13 @@
 ; RUN: llc -relocation-model=static -verify-machineinstrs -O0 -mcpu=pwr7 -code-model=medium -filetype=obj -fast-isel=false %s -o - | \
-; RUN: llvm-readobj -r | FileCheck -check-prefix=MEDIUM %s
+; RUN: llvm-readobj -r - | FileCheck -check-prefix=MEDIUM %s
 ; RUN: llc -relocation-model=static -verify-machineinstrs -O0 -mcpu=pwr7 -code-model=large -filetype=obj -fast-isel=false %s -o - | \
-; RUN: llvm-readobj -r | FileCheck -check-prefix=LARGE %s
+; RUN: llvm-readobj -r - | FileCheck -check-prefix=LARGE %s
 
 ; Run jump table test separately since jump tables aren't generated at -O0.
 ; RUN: llc -relocation-model=static -verify-machineinstrs -mcpu=pwr7 -code-model=medium -filetype=obj -fast-isel=false %s -o - | \
-; RUN: llvm-readobj -r | FileCheck -check-prefix=MEDIUM-JT %s
+; RUN: llvm-readobj -r - | FileCheck -check-prefix=MEDIUM-JT %s
 ; RUN: llc -relocation-model=static -verify-machineinstrs -mcpu=pwr7 -code-model=large -filetype=obj -fast-isel=false %s -o - | \
-; RUN: llvm-readobj -r | FileCheck -check-prefix=LARGE-JT %s
+; RUN: llvm-readobj -r - | FileCheck -check-prefix=LARGE-JT %s
 
 ; FIXME: When asm-parse is available, could make this an assembly test.
 
diff --git a/llvm/test/CodeGen/PowerPC/p10-vector-divide.ll b/llvm/test/CodeGen/PowerPC/p10-vector-divide.ll
new file mode 100644
index 0000000000000..4ecc3a17fedbe
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/p10-vector-divide.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
+; RUN:   FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
+; RUN:   FileCheck %s
+
+; This test case aims to test the vector divide instructions on Power10.
+; This includes the low order and extended versions of vector divide,
+; that operate on signed and unsigned words and doublewords.
+
+define <2 x i64> @test_vdivud(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vdivud:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vdivud v2, v2, v3
+; CHECK-NEXT:    blr
+entry:
+  %div = udiv <2 x i64> %a, %b
+  ret <2 x i64> %div
+}
+
+define <2 x i64> @test_vdivsd(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vdivsd:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vdivsd v2, v2, v3
+; CHECK-NEXT:    blr
+entry:
+  %div = sdiv <2 x i64> %a, %b
+  ret <2 x i64> %div
+}
+
+define <4 x i32> @test_vdivuw(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vdivuw:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vdivuw v2, v2, v3
+; CHECK-NEXT:    blr
+entry:
+  %div = udiv <4 x i32> %a, %b
+  ret <4 x i32> %div
+}
+
+define <4 x i32> @test_vdivsw(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vdivsw:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vdivsw v2, v2, v3
+; CHECK-NEXT:    blr
+entry:
+  %div = sdiv <4 x i32> %a, %b
+  ret <4 x i32> %div
+}
diff --git a/llvm/test/CodeGen/PowerPC/p10-vector-modulo.ll b/llvm/test/CodeGen/PowerPC/p10-vector-modulo.ll
new file mode 100644
index 0000000000000..e4ef0380ae8bd
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/p10-vector-modulo.ll
@@ -0,0 +1,107 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
+; RUN:   FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
+; RUN:   FileCheck %s
+
+; This test case aims to test the vector modulo instructions on Power10.
+; The vector modulo instructions operate on signed and unsigned words
+; and doublewords.
+
+define <2 x i64> @test_vmodud(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vmodud:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmodud v2, v2, v3
+; CHECK-NEXT:    blr
+entry:
+  %rem = urem <2 x i64> %a, %b
+  ret <2 x i64> %rem
+}
+
+define <2 x i64> @test_vmodsd(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vmodsd:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmodsd v2, v2, v3
+; CHECK-NEXT:    blr
+entry:
+  %rem = srem <2 x i64> %a, %b
+  ret <2 x i64> %rem
+}
+
+define <4 x i32> @test_vmoduw(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vmoduw:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmoduw v2, v2, v3
+; CHECK-NEXT:    blr
+entry:
+  %rem = urem <4 x i32> %a, %b
+  ret <4 x i32> %rem
+}
+
+define <4 x i32> @test_vmodsw(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vmodsw:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmodsw v2, v2, v3
+; CHECK-NEXT:    blr
+entry:
+  %rem = srem <4 x i32> %a, %b
+  ret <4 x i32> %rem
+}
+
+define <2 x i64> @test_vmodud_with_div(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vmodud_with_div:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmodud v4, v2, v3
+; CHECK-NEXT:    vdivud v2, v2, v3
+; CHECK-NEXT:    vaddudm v2, v4, v2
+; CHECK-NEXT:    blr
+entry:
+  %rem = urem <2 x i64> %a, %b
+  %div = udiv <2 x i64> %a, %b
+  %add = add <2 x i64> %rem, %div
+  ret <2 x i64> %add
+}
+
+define <2 x i64> @test_vmodsd_with_div(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vmodsd_with_div:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmodsd v4, v2, v3
+; CHECK-NEXT:    vdivsd v2, v2, v3
+; CHECK-NEXT:    vaddudm v2, v4, v2
+; CHECK-NEXT:    blr
+entry:
+  %rem = srem <2 x i64> %a, %b
+  %div = sdiv <2 x i64> %a, %b
+  %add = add <2 x i64> %rem, %div
+  ret <2 x i64> %add
+}
+
+define <4 x i32> @test_vmoduw_with_div(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vmoduw_with_div:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmoduw v4, v2, v3
+; CHECK-NEXT:    vdivuw v2, v2, v3
+; CHECK-NEXT:    vadduwm v2, v4, v2
+; CHECK-NEXT:    blr
+entry:
+  %rem = urem <4 x i32> %a, %b
+  %div = udiv <4 x i32> %a, %b
+  %add = add <4 x i32> %rem, %div
+  ret <4 x i32> %add
+}
+
+define <4 x i32> @test_vmodsw_div(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: test_vmodsw_div:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmodsw v4, v2, v3
+; CHECK-NEXT:    vdivsw v2, v2, v3
+; CHECK-NEXT:    vadduwm v2, v4, v2
+; CHECK-NEXT:    blr
+entry:
+  %rem = srem <4 x i32> %a, %b
+  %div = sdiv <4 x i32> %a, %b
+  %add = add <4 x i32> %rem, %div
+  ret <4 x i32> %add
+}
diff --git a/llvm/test/CodeGen/PowerPC/p10-vector-multiply.ll b/llvm/test/CodeGen/PowerPC/p10-vector-multiply.ll
new file mode 100644
index 0000000000000..e8f77574f66cc
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/p10-vector-multiply.ll
@@ -0,0 +1,19 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
+; RUN:   FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
+; RUN:   FileCheck %s
+
+; This test case aims to test the vector multiply instructions on Power10.
+
+define <2 x i64> @test_vmulld(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_vmulld:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmulld v2, v3, v2
+; CHECK-NEXT:    blr
+entry:
+  %mul = mul <2 x i64> %b, %a
+  ret <2 x i64> %mul
+}
diff --git a/llvm/test/CodeGen/PowerPC/pcrel-block-address.ll b/llvm/test/CodeGen/PowerPC/pcrel-block-address.ll
index b7219da940276..76eea940aeef1 100644
--- a/llvm/test/CodeGen/PowerPC/pcrel-block-address.ll
+++ b/llvm/test/CodeGen/PowerPC/pcrel-block-address.ll
@@ -1,5 +1,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
-; RUN:   -mcpu=future -ppc-asm-full-reg-names < %s | FileCheck %s
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -target-abi=elfv2 -mtriple=powerpc64-- \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names < %s | FileCheck %s
 define dso_local void @blockaddress() {
 ; CHECK-LABEL: blockaddress:
 ; CHECK:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/PowerPC/pcrel-call-linkage-leaf.ll b/llvm/test/CodeGen/PowerPC/pcrel-call-linkage-leaf.ll
index 498963371c9c1..9141fdc735a0e 100644
--- a/llvm/test/CodeGen/PowerPC/pcrel-call-linkage-leaf.ll
+++ b/llvm/test/CodeGen/PowerPC/pcrel-call-linkage-leaf.ll
@@ -1,5 +1,8 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
-; RUN:   -mcpu=future -ppc-asm-full-reg-names < %s \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names < %s \
+; RUN:   | FileCheck %s --check-prefixes=CHECK-S,CHECK-ALL
+; RUN: llc -verify-machineinstrs -target-abi=elfv2 -mtriple=powerpc64-- \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names < %s \
 ; RUN:   | FileCheck %s --check-prefixes=CHECK-S,CHECK-ALL
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
 ; RUN:   -mcpu=pwr9 -ppc-asm-full-reg-names < %s \
diff --git a/llvm/test/CodeGen/PowerPC/pcrel-call-linkage-simple.ll b/llvm/test/CodeGen/PowerPC/pcrel-call-linkage-simple.ll
index 649aecf8f7202..013ae7345956b 100644
--- a/llvm/test/CodeGen/PowerPC/pcrel-call-linkage-simple.ll
+++ b/llvm/test/CodeGen/PowerPC/pcrel-call-linkage-simple.ll
@@ -1,8 +1,14 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
-; RUN:   -mcpu=future -ppc-asm-full-reg-names < %s \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names < %s \
 ; RUN:   | FileCheck %s --check-prefix=CHECK-S
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
-; RUN:   -mcpu=future -ppc-asm-full-reg-names --filetype=obj < %s | \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names --filetype=obj < %s | \
+; RUN:   llvm-objdump -dr - | FileCheck %s --check-prefix=CHECK-O
+; RUN: llc -verify-machineinstrs -target-abi=elfv2 -mtriple=powerpc64-- \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names < %s \
+; RUN:   | FileCheck %s --check-prefix=CHECK-S
+; RUN: llc -verify-machineinstrs -target-abi=elfv2 -mtriple=powerpc64-- \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names --filetype=obj < %s | \
 ; RUN:   llvm-objdump -dr - | FileCheck %s --check-prefix=CHECK-O
 
 
diff --git a/llvm/test/CodeGen/PowerPC/pcrel-call-linkage-with-calls.ll b/llvm/test/CodeGen/PowerPC/pcrel-call-linkage-with-calls.ll
index bb6b5052ee588..0a4f2f38c816b 100644
--- a/llvm/test/CodeGen/PowerPC/pcrel-call-linkage-with-calls.ll
+++ b/llvm/test/CodeGen/PowerPC/pcrel-call-linkage-with-calls.ll
@@ -1,5 +1,8 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
-; RUN:   -mcpu=future -ppc-asm-full-reg-names < %s \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names < %s \
+; RUN:   | FileCheck %s --check-prefixes=CHECK-S,CHECK-ALL
+; RUN: llc -verify-machineinstrs -target-abi=elfv2 -mtriple=powerpc64-- \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names < %s \
 ; RUN:   | FileCheck %s --check-prefixes=CHECK-S,CHECK-ALL
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
 ; RUN:   -mcpu=pwr9 -ppc-asm-full-reg-names < %s \
@@ -64,6 +67,8 @@ define dso_local signext i32 @DirectCallLocal2(i32 signext %a, i32 signext %b) l
 ; CHECK-S-NEXT:    extsw r3, r3
 ; CHECK-S-NEXT:    bl localCall@notoc
 ; CHECK-S-NEXT:    pld r4, externGlobalVar@got@pcrel(0), 1
+; CHECK-S-NEXT: .Lpcrel:
+; CHECK-S-NEXT:    .reloc .Lpcrel-8,R_PPC64_PCREL_OPT,.-(.Lpcrel-8)
 ; CHECK-S-NEXT:    lwz r4, 0(r4)
 ; CHECK-S-NEXT:    mullw r3, r4, r3
 ; CHECK-S-NEXT:    extsw r3, r3
@@ -147,6 +152,8 @@ define dso_local signext i32 @DirectCallExtern2(i32 signext %a, i32 signext %b)
 ; CHECK-S-NEXT:    extsw r3, r3
 ; CHECK-S-NEXT:    bl externCall@notoc
 ; CHECK-S-NEXT:    pld r4, externGlobalVar@got@pcrel(0), 1
+; CHECK-S-NEXT:  .Lpcrel0:
+; CHECK-S-NEXT:    .reloc .Lpcrel0-8,R_PPC64_PCREL_OPT,.-(.Lpcrel0-8)
 ; CHECK-S-NEXT:    lwz r4, 0(r4)
 ; CHECK-S-NEXT:    mullw r3, r4, r3
 ; CHECK-S-NEXT:    extsw r3, r3
@@ -209,6 +216,8 @@ define dso_local signext i32 @TailCallLocal2(i32 signext %a) local_unnamed_addr
 ; CHECK-S:         .localentry     TailCallLocal2
 ; CHECK-S:       # %bb.0: # %entry
 ; CHECK-S:         pld r4, externGlobalVar@got@pcrel(0), 1
+; CHECK-S-NEXT:  .Lpcrel1:
+; CHECK-S-NEXT:    .reloc .Lpcrel1-8,R_PPC64_PCREL_OPT,.-(.Lpcrel1-8)
 ; CHECK-S-NEXT:    lwz r4, 0(r4)
 ; CHECK-S-NEXT:    add r3, r4, r3
 ; CHECK-S-NEXT:    extsw r3, r3
@@ -251,6 +260,8 @@ define dso_local signext i32 @TailCallExtern2(i32 signext %a) local_unnamed_addr
 ; CHECK-S:         .localentry     TailCallExtern2
 ; CHECK-S:       # %bb.0: # %entry
 ; CHECK-S:         pld r4, externGlobalVar@got@pcrel(0), 1
+; CHECK-S-NEXT:  .Lpcrel2:
+; CHECK-S-NEXT:    .reloc .Lpcrel2-8,R_PPC64_PCREL_OPT,.-(.Lpcrel2-8)
 ; CHECK-S-NEXT:    lwz r4, 0(r4)
 ; CHECK-S-NEXT:    add r3, r4, r3
 ; CHECK-S-NEXT:    extsw r3, r3
@@ -316,6 +327,8 @@ define dso_local signext i32 @IndirectCall2(i32 signext %a, i32 signext %b) loca
 ; CHECK-S-NEXT:    mtctr r12
 ; CHECK-S-NEXT:    bctrl
 ; CHECK-S-NEXT:    pld r4, externGlobalVar@got@pcrel(0), 1
+; CHECK-S-NEXT:  .Lpcrel3:
+; CHECK-S-NEXT:    .reloc .Lpcrel3-8,R_PPC64_PCREL_OPT,.-(.Lpcrel3-8)
 ; CHECK-S-NEXT:    lwz r4, 0(r4)
 ; CHECK-S-NEXT:    mullw r3, r4, r3
 ; CHECK-S-NEXT:    extsw r3, r3
diff --git a/llvm/test/CodeGen/PowerPC/pcrel-got-indirect.ll b/llvm/test/CodeGen/PowerPC/pcrel-got-indirect.ll
index c838308f3aa65..4d61b66d3bb77 100644
--- a/llvm/test/CodeGen/PowerPC/pcrel-got-indirect.ll
+++ b/llvm/test/CodeGen/PowerPC/pcrel-got-indirect.ll
@@ -1,7 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
-; RUN:   -mcpu=future -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s \
-; RUN:   | FileCheck %s
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s \
+; RUN:   | FileCheck %s --check-prefix=LE
+; RUN: llc -verify-machineinstrs -target-abi=elfv2 -mtriple=powerpc64-- \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s \
+; RUN:   | FileCheck %s --check-prefix=BE
+
 
 %struct.Struct = type { i8, i16, i32 }
 
@@ -16,11 +20,21 @@
 @ptrfunc = external local_unnamed_addr global void (...)*, align 8
 
 define dso_local signext i32 @ReadGlobalVarChar() local_unnamed_addr  {
-; CHECK-LABEL: ReadGlobalVarChar:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pld r3, valChar@got@pcrel(0), 1
-; CHECK-NEXT:    lbz r3, 0(r3)
-; CHECK-NEXT:    blr
+; LE-LABEL: ReadGlobalVarChar:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    pld r3, valChar@got@pcrel(0), 1
+; LE-NEXT:  .Lpcrel:
+; LE-NEXT:    .reloc .Lpcrel-8,R_PPC64_PCREL_OPT,.-(.Lpcrel-8)
+; LE-NEXT:    lbz r3, 0(r3)
+; LE-NEXT:    blr
+;
+; BE-LABEL: ReadGlobalVarChar:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    pld r3, valChar@got@pcrel(0), 1
+; BE-NEXT:  .Lpcrel:
+; BE-NEXT:    .reloc .Lpcrel-8,R_PPC64_PCREL_OPT,.-(.Lpcrel-8)
+; BE-NEXT:    lbz r3, 0(r3)
+; BE-NEXT:    blr
 entry:
   %0 = load i8, i8* @valChar, align 1
   %conv = zext i8 %0 to i32
@@ -28,23 +42,40 @@ entry:
 }
 
 define dso_local void @WriteGlobalVarChar() local_unnamed_addr  {
-; CHECK-LABEL: WriteGlobalVarChar:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pld r3, valChar@got@pcrel(0), 1
-; CHECK-NEXT:    li r4, 3
-; CHECK-NEXT:    stb r4, 0(r3)
-; CHECK-NEXT:    blr
+; LE-LABEL: WriteGlobalVarChar:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    pld r3, valChar@got@pcrel(0), 1
+; LE-NEXT:    li r4, 3
+; LE-NEXT:    stb r4, 0(r3)
+; LE-NEXT:    blr
+;
+; BE-LABEL: WriteGlobalVarChar:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    pld r3, valChar@got@pcrel(0), 1
+; BE-NEXT:    li r4, 3
+; BE-NEXT:    stb r4, 0(r3)
+; BE-NEXT:    blr
 entry:
   store i8 3, i8* @valChar, align 1
   ret void
 }
 
 define dso_local signext i32 @ReadGlobalVarShort() local_unnamed_addr  {
-; CHECK-LABEL: ReadGlobalVarShort:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pld r3, valShort@got@pcrel(0), 1
-; CHECK-NEXT:    lha r3, 0(r3)
-; CHECK-NEXT:    blr
+; LE-LABEL: ReadGlobalVarShort:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    pld r3, valShort@got@pcrel(0), 1
+; LE-NEXT:  .Lpcrel0:
+; LE-NEXT:    .reloc .Lpcrel0-8,R_PPC64_PCREL_OPT,.-(.Lpcrel0-8)
+; LE-NEXT:    lha r3, 0(r3)
+; LE-NEXT:    blr
+;
+; BE-LABEL: ReadGlobalVarShort:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    pld r3, valShort@got@pcrel(0), 1
+; BE-NEXT:  .Lpcrel0:
+; BE-NEXT:    .reloc .Lpcrel0-8,R_PPC64_PCREL_OPT,.-(.Lpcrel0-8)
+; BE-NEXT:    lha r3, 0(r3)
+; BE-NEXT:    blr
 entry:
   %0 = load i16, i16* @valShort, align 2
   %conv = sext i16 %0 to i32
@@ -52,69 +83,120 @@ entry:
 }
 
 define dso_local void @WriteGlobalVarShort() local_unnamed_addr  {
-; CHECK-LABEL: WriteGlobalVarShort:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pld r3, valShort@got@pcrel(0), 1
-; CHECK-NEXT:    li r4, 3
-; CHECK-NEXT:    sth r4, 0(r3)
-; CHECK-NEXT:    blr
+; LE-LABEL: WriteGlobalVarShort:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    pld r3, valShort@got@pcrel(0), 1
+; LE-NEXT:    li r4, 3
+; LE-NEXT:    sth r4, 0(r3)
+; LE-NEXT:    blr
+;
+; BE-LABEL: WriteGlobalVarShort:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    pld r3, valShort@got@pcrel(0), 1
+; BE-NEXT:    li r4, 3
+; BE-NEXT:    sth r4, 0(r3)
+; BE-NEXT:    blr
 entry:
   store i16 3, i16* @valShort, align 2
   ret void
 }
 
 define dso_local signext i32 @ReadGlobalVarInt() local_unnamed_addr  {
-; CHECK-LABEL: ReadGlobalVarInt:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pld r3, valInt@got@pcrel(0), 1
-; CHECK-NEXT:    lwa r3, 0(r3)
-; CHECK-NEXT:    blr
+; LE-LABEL: ReadGlobalVarInt:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    pld r3, valInt@got@pcrel(0), 1
+; LE-NEXT:  .Lpcrel1:
+; LE-NEXT:    .reloc .Lpcrel1-8,R_PPC64_PCREL_OPT,.-(.Lpcrel1-8)
+; LE-NEXT:    lwa r3, 0(r3)
+; LE-NEXT:    blr
+;
+; BE-LABEL: ReadGlobalVarInt:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    pld r3, valInt@got@pcrel(0), 1
+; BE-NEXT:  .Lpcrel1:
+; BE-NEXT:    .reloc .Lpcrel1-8,R_PPC64_PCREL_OPT,.-(.Lpcrel1-8)
+; BE-NEXT:    lwa r3, 0(r3)
+; BE-NEXT:    blr
 entry:
   %0 = load i32, i32* @valInt, align 4
   ret i32 %0
 }
 
 define dso_local void @WriteGlobalVarInt() local_unnamed_addr  {
-; CHECK-LABEL: WriteGlobalVarInt:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pld r3, valInt@got@pcrel(0), 1
-; CHECK-NEXT:    li r4, 33
-; CHECK-NEXT:    stw r4, 0(r3)
-; CHECK-NEXT:    blr
+; LE-LABEL: WriteGlobalVarInt:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    pld r3, valInt@got@pcrel(0), 1
+; LE-NEXT:    li r4, 33
+; LE-NEXT:    stw r4, 0(r3)
+; LE-NEXT:    blr
+;
+; BE-LABEL: WriteGlobalVarInt:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    pld r3, valInt@got@pcrel(0), 1
+; BE-NEXT:    li r4, 33
+; BE-NEXT:    stw r4, 0(r3)
+; BE-NEXT:    blr
 entry:
   store i32 33, i32* @valInt, align 4
   ret void
 }
 
 define dso_local signext i32 @ReadGlobalVarUnsigned() local_unnamed_addr  {
-; CHECK-LABEL: ReadGlobalVarUnsigned:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pld r3, valUnsigned@got@pcrel(0), 1
-; CHECK-NEXT:    lwa r3, 0(r3)
-; CHECK-NEXT:    blr
+; LE-LABEL: ReadGlobalVarUnsigned:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    pld r3, valUnsigned@got@pcrel(0), 1
+; LE-NEXT:  .Lpcrel2:
+; LE-NEXT:    .reloc .Lpcrel2-8,R_PPC64_PCREL_OPT,.-(.Lpcrel2-8)
+; LE-NEXT:    lwa r3, 0(r3)
+; LE-NEXT:    blr
+;
+; BE-LABEL: ReadGlobalVarUnsigned:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    pld r3, valUnsigned@got@pcrel(0), 1
+; BE-NEXT:  .Lpcrel2:
+; BE-NEXT:    .reloc .Lpcrel2-8,R_PPC64_PCREL_OPT,.-(.Lpcrel2-8)
+; BE-NEXT:    lwa r3, 0(r3)
+; BE-NEXT:    blr
 entry:
   %0 = load i32, i32* @valUnsigned, align 4
   ret i32 %0
 }
 
 define dso_local void @WriteGlobalVarUnsigned() local_unnamed_addr  {
-; CHECK-LABEL: WriteGlobalVarUnsigned:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pld r3, valUnsigned@got@pcrel(0), 1
-; CHECK-NEXT:    li r4, 33
-; CHECK-NEXT:    stw r4, 0(r3)
-; CHECK-NEXT:    blr
+; LE-LABEL: WriteGlobalVarUnsigned:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    pld r3, valUnsigned@got@pcrel(0), 1
+; LE-NEXT:    li r4, 33
+; LE-NEXT:    stw r4, 0(r3)
+; LE-NEXT:    blr
+;
+; BE-LABEL: WriteGlobalVarUnsigned:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    pld r3, valUnsigned@got@pcrel(0), 1
+; BE-NEXT:    li r4, 33
+; BE-NEXT:    stw r4, 0(r3)
+; BE-NEXT:    blr
 entry:
   store i32 33, i32* @valUnsigned, align 4
   ret void
 }
 
 define dso_local signext i32 @ReadGlobalVarLong() local_unnamed_addr  {
-; CHECK-LABEL: ReadGlobalVarLong:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pld r3, valLong@got@pcrel(0), 1
-; CHECK-NEXT:    lwa r3, 0(r3)
-; CHECK-NEXT:    blr
+; LE-LABEL: ReadGlobalVarLong:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    pld r3, valLong@got@pcrel(0), 1
+; LE-NEXT:  .Lpcrel3:
+; LE-NEXT:    .reloc .Lpcrel3-8,R_PPC64_PCREL_OPT,.-(.Lpcrel3-8)
+; LE-NEXT:    lwa r3, 0(r3)
+; LE-NEXT:    blr
+;
+; BE-LABEL: ReadGlobalVarLong:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    pld r3, valLong@got@pcrel(0), 1
+; BE-NEXT:  .Lpcrel3:
+; BE-NEXT:    .reloc .Lpcrel3-8,R_PPC64_PCREL_OPT,.-(.Lpcrel3-8)
+; BE-NEXT:    lwa r3, 4(r3)
+; BE-NEXT:    blr
 entry:
   %0 = load i64, i64* @valLong, align 8
   %conv = trunc i64 %0 to i32
@@ -122,36 +204,65 @@ entry:
 }
 
 define dso_local void @WriteGlobalVarLong() local_unnamed_addr  {
-; CHECK-LABEL: WriteGlobalVarLong:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pld r3, valLong@got@pcrel(0), 1
-; CHECK-NEXT:    li r4, 3333
-; CHECK-NEXT:    std r4, 0(r3)
-; CHECK-NEXT:    blr
+; LE-LABEL: WriteGlobalVarLong:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    pld r3, valLong@got@pcrel(0), 1
+; LE-NEXT:    li r4, 3333
+; LE-NEXT:    std r4, 0(r3)
+; LE-NEXT:    blr
+;
+; BE-LABEL: WriteGlobalVarLong:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    pld r3, valLong@got@pcrel(0), 1
+; BE-NEXT:    li r4, 3333
+; BE-NEXT:    std r4, 0(r3)
+; BE-NEXT:    blr
 entry:
   store i64 3333, i64* @valLong, align 8
   ret void
 }
 
 define dso_local i32* @ReadGlobalPtr() local_unnamed_addr  {
-; CHECK-LABEL: ReadGlobalPtr:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pld r3, ptr@got@pcrel(0), 1
-; CHECK-NEXT:    ld r3, 0(r3)
-; CHECK-NEXT:    blr
+; LE-LABEL: ReadGlobalPtr:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    pld r3, ptr@got@pcrel(0), 1
+; LE-NEXT:  .Lpcrel4:
+; LE-NEXT:    .reloc .Lpcrel4-8,R_PPC64_PCREL_OPT,.-(.Lpcrel4-8)
+; LE-NEXT:    ld r3, 0(r3)
+; LE-NEXT:    blr
+;
+; BE-LABEL: ReadGlobalPtr:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    pld r3, ptr@got@pcrel(0), 1
+; BE-NEXT:  .Lpcrel4:
+; BE-NEXT:    .reloc .Lpcrel4-8,R_PPC64_PCREL_OPT,.-(.Lpcrel4-8)
+; BE-NEXT:    ld r3, 0(r3)
+; BE-NEXT:    blr
 entry:
   %0 = load i32*, i32** @ptr, align 8
   ret i32* %0
 }
 
 define dso_local void @WriteGlobalPtr() local_unnamed_addr  {
-; CHECK-LABEL: WriteGlobalPtr:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pld r3, ptr@got@pcrel(0), 1
-; CHECK-NEXT:    li r4, 3
-; CHECK-NEXT:    ld r3, 0(r3)
-; CHECK-NEXT:    stw r4, 0(r3)
-; CHECK-NEXT:    blr
+; LE-LABEL: WriteGlobalPtr:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    pld r3, ptr@got@pcrel(0), 1
+; LE-NEXT:  .Lpcrel5:
+; LE-NEXT:    li r4, 3
+; LE-NEXT:    .reloc .Lpcrel5-8,R_PPC64_PCREL_OPT,.-(.Lpcrel5-8)
+; LE-NEXT:    ld r3, 0(r3)
+; LE-NEXT:    stw r4, 0(r3)
+; LE-NEXT:    blr
+;
+; BE-LABEL: WriteGlobalPtr:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    pld r3, ptr@got@pcrel(0), 1
+; BE-NEXT:  .Lpcrel5:
+; BE-NEXT:    li r4, 3
+; BE-NEXT:    .reloc .Lpcrel5-8,R_PPC64_PCREL_OPT,.-(.Lpcrel5-8)
+; BE-NEXT:    ld r3, 0(r3)
+; BE-NEXT:    stw r4, 0(r3)
+; BE-NEXT:    blr
 entry:
   %0 = load i32*, i32** @ptr, align 8
   store i32 3, i32* %0, align 4
@@ -159,69 +270,121 @@ entry:
 }
 
 define dso_local nonnull i32* @GlobalVarAddr() local_unnamed_addr  {
-; CHECK-LABEL: GlobalVarAddr:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pld r3, valInt@got@pcrel(0), 1
-; CHECK-NEXT:    blr
+; LE-LABEL: GlobalVarAddr:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    pld r3, valInt@got@pcrel(0), 1
+; LE-NEXT:    blr
+;
+; BE-LABEL: GlobalVarAddr:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    pld r3, valInt@got@pcrel(0), 1
+; BE-NEXT:    blr
 entry:
   ret i32* @valInt
 }
 
 define dso_local signext i32 @ReadGlobalArray() local_unnamed_addr  {
-; CHECK-LABEL: ReadGlobalArray:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pld r3, array@got@pcrel(0), 1
-; CHECK-NEXT:    lwa r3, 12(r3)
-; CHECK-NEXT:    blr
+; LE-LABEL: ReadGlobalArray:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    pld r3, array@got@pcrel(0), 1
+; LE-NEXT:  .Lpcrel6:
+; LE-NEXT:    .reloc .Lpcrel6-8,R_PPC64_PCREL_OPT,.-(.Lpcrel6-8)
+; LE-NEXT:    lwa r3, 12(r3)
+; LE-NEXT:    blr
+;
+; BE-LABEL: ReadGlobalArray:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    pld r3, array@got@pcrel(0), 1
+; BE-NEXT:  .Lpcrel6:
+; BE-NEXT:    .reloc .Lpcrel6-8,R_PPC64_PCREL_OPT,.-(.Lpcrel6-8)
+; BE-NEXT:    lwa r3, 12(r3)
+; BE-NEXT:    blr
 entry:
   %0 = load i32, i32* getelementptr inbounds ([10 x i32], [10 x i32]* @array, i64 0, i64 3), align 4
   ret i32 %0
 }
 
 define dso_local void @WriteGlobalArray() local_unnamed_addr  {
-; CHECK-LABEL: WriteGlobalArray:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pld r3, array@got@pcrel(0), 1
-; CHECK-NEXT:    li r4, 5
-; CHECK-NEXT:    stw r4, 12(r3)
-; CHECK-NEXT:    blr
+; LE-LABEL: WriteGlobalArray:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    pld r3, array@got@pcrel(0), 1
+; LE-NEXT:    li r4, 5
+; LE-NEXT:    stw r4, 12(r3)
+; LE-NEXT:    blr
+;
+; BE-LABEL: WriteGlobalArray:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    pld r3, array@got@pcrel(0), 1
+; BE-NEXT:    li r4, 5
+; BE-NEXT:    stw r4, 12(r3)
+; BE-NEXT:    blr
 entry:
   store i32 5, i32* getelementptr inbounds ([10 x i32], [10 x i32]* @array, i64 0, i64 3), align 4
   ret void
 }
 
 define dso_local signext i32 @ReadGlobalStruct() local_unnamed_addr  {
-; CHECK-LABEL: ReadGlobalStruct:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pld r3, structure@got@pcrel(0), 1
-; CHECK-NEXT:    lwa r3, 4(r3)
-; CHECK-NEXT:    blr
+; LE-LABEL: ReadGlobalStruct:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    pld r3, structure@got@pcrel(0), 1
+; LE-NEXT:  .Lpcrel7:
+; LE-NEXT:    .reloc .Lpcrel7-8,R_PPC64_PCREL_OPT,.-(.Lpcrel7-8)
+; LE-NEXT:    lwa r3, 4(r3)
+; LE-NEXT:    blr
+;
+; BE-LABEL: ReadGlobalStruct:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    pld r3, structure@got@pcrel(0), 1
+; BE-NEXT:  .Lpcrel7:
+; BE-NEXT:    .reloc .Lpcrel7-8,R_PPC64_PCREL_OPT,.-(.Lpcrel7-8)
+; BE-NEXT:    lwa r3, 4(r3)
+; BE-NEXT:    blr
 entry:
   %0 = load i32, i32* getelementptr inbounds (%struct.Struct, %struct.Struct* @structure, i64 0, i32 2), align 4
   ret i32 %0
 }
 
 define dso_local void @WriteGlobalStruct() local_unnamed_addr  {
-; CHECK-LABEL: WriteGlobalStruct:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pld r3, structure@got@pcrel(0), 1
-; CHECK-NEXT:    li r4, 3
-; CHECK-NEXT:    stw r4, 4(r3)
-; CHECK-NEXT:    blr
+; LE-LABEL: WriteGlobalStruct:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    pld r3, structure@got@pcrel(0), 1
+; LE-NEXT:    li r4, 3
+; LE-NEXT:    stw r4, 4(r3)
+; LE-NEXT:    blr
+;
+; BE-LABEL: WriteGlobalStruct:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    pld r3, structure@got@pcrel(0), 1
+; BE-NEXT:    li r4, 3
+; BE-NEXT:    stw r4, 4(r3)
+; BE-NEXT:    blr
 entry:
   store i32 3, i32* getelementptr inbounds (%struct.Struct, %struct.Struct* @structure, i64 0, i32 2), align 4
   ret void
 }
 
 define dso_local void @ReadFuncPtr() local_unnamed_addr  {
-; CHECK-LABEL: ReadFuncPtr:
-; CHECK:         .localentry ReadFuncPtr, 1
-; CHECK-NEXT:  # %bb.0: # %entry
-; CHECK-NEXT:    pld r3, ptrfunc@got@pcrel(0), 1
-; CHECK-NEXT:    ld r12, 0(r3)
-; CHECK-NEXT:    mtctr r12
-; CHECK-NEXT:    bctr
-; CHECK-NEXT:    #TC_RETURNr8 ctr 0
+; LE-LABEL: ReadFuncPtr:
+; LE:         .localentry ReadFuncPtr, 1
+; LE-NEXT:  # %bb.0: # %entry
+; LE-NEXT:    pld r3, ptrfunc@got@pcrel(0), 1
+; LE-NEXT:  .Lpcrel8:
+; LE-NEXT:    .reloc .Lpcrel8-8,R_PPC64_PCREL_OPT,.-(.Lpcrel8-8)
+; LE-NEXT:    ld r12, 0(r3)
+; LE-NEXT:    mtctr r12
+; LE-NEXT:    bctr
+; LE-NEXT:    #TC_RETURNr8 ctr 0
+;
+; BE-LABEL: ReadFuncPtr:
+; BE:         .localentry ReadFuncPtr, 1
+; BE-NEXT:  # %bb.0: # %entry
+; BE-NEXT:    pld r3, ptrfunc@got@pcrel(0), 1
+; BE-NEXT:  .Lpcrel8:
+; BE-NEXT:    .reloc .Lpcrel8-8,R_PPC64_PCREL_OPT,.-(.Lpcrel8-8)
+; BE-NEXT:    ld r12, 0(r3)
+; BE-NEXT:    mtctr r12
+; BE-NEXT:    bctr
+; BE-NEXT:    #TC_RETURNr8 ctr 0
 entry:
   %0 = load void ()*, void ()** bitcast (void (...)** @ptrfunc to void ()**), align 8
   tail call void %0()
@@ -229,12 +392,19 @@ entry:
 }
 
 define dso_local void @WriteFuncPtr() local_unnamed_addr  {
-; CHECK-LABEL: WriteFuncPtr:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pld r3, ptrfunc@got@pcrel(0), 1
-; CHECK-NEXT:    pld r4, function@got@pcrel(0), 1
-; CHECK-NEXT:    std r4, 0(r3)
-; CHECK-NEXT:    blr
+; LE-LABEL: WriteFuncPtr:
+; LE:       # %bb.0: # %entry
+; LE-NEXT:    pld r3, ptrfunc@got@pcrel(0), 1
+; LE-NEXT:    pld r4, function@got@pcrel(0), 1
+; LE-NEXT:    std r4, 0(r3)
+; LE-NEXT:    blr
+;
+; BE-LABEL: WriteFuncPtr:
+; BE:       # %bb.0: # %entry
+; BE-NEXT:    pld r3, ptrfunc@got@pcrel(0), 1
+; BE-NEXT:    pld r4, function@got@pcrel(0), 1
+; BE-NEXT:    std r4, 0(r3)
+; BE-NEXT:    blr
 entry:
   store void (...)* @function, void (...)** @ptrfunc, align 8
   ret void
diff --git a/llvm/test/CodeGen/PowerPC/pcrel-indirect-call.ll b/llvm/test/CodeGen/PowerPC/pcrel-indirect-call.ll
index d7df6f10a6be1..88f25db58b49e 100644
--- a/llvm/test/CodeGen/PowerPC/pcrel-indirect-call.ll
+++ b/llvm/test/CodeGen/PowerPC/pcrel-indirect-call.ll
@@ -1,5 +1,8 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
-; RUN:   -mcpu=future -ppc-asm-full-reg-names < %s | FileCheck %s
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -target-abi=elfv2 -mtriple=powerpc64-- \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names < %s | FileCheck %s
+
 
 ; The test checks the behavior of PC Relative indirect calls. When using
 ; PC Relative, TOC save and restore are no longer required. Function pointer
diff --git a/llvm/test/CodeGen/PowerPC/pcrel-jump-table.ll b/llvm/test/CodeGen/PowerPC/pcrel-jump-table.ll
index a7d3ca7e6d738..3a742588d23b5 100644
--- a/llvm/test/CodeGen/PowerPC/pcrel-jump-table.ll
+++ b/llvm/test/CodeGen/PowerPC/pcrel-jump-table.ll
@@ -1,10 +1,18 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
-; RUN:   -mcpu=future -ppc-asm-full-reg-names < %s | FileCheck %s \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names < %s | FileCheck %s \
 ; RUN:   --check-prefix=CHECK-R
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
-; RUN:   -mcpu=future -ppc-use-absolute-jumptables \
+; RUN:   -mcpu=pwr10 -ppc-use-absolute-jumptables \
 ; RUN:   -ppc-asm-full-reg-names < %s | FileCheck %s \
-; RUN:   --check-prefix=CHECK-A
+; RUN:   --check-prefix=CHECK-A-LE
+; RUN: llc -verify-machineinstrs -target-abi=elfv2 -mtriple=powerpc64-- \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names < %s | FileCheck %s \
+; RUN:   --check-prefix=CHECK-R
+; RUN: llc -verify-machineinstrs -target-abi=elfv2 -mtriple=powerpc64-- \
+; RUN:   -mcpu=pwr10 -ppc-use-absolute-jumptables \
+; RUN:   -ppc-asm-full-reg-names < %s | FileCheck %s \
+; RUN:   --check-prefix=CHECK-A-BE
+
 
 ; This test checks for getting relative and absolute jump table base address
 ; using PC Relative addressing.
@@ -18,13 +26,21 @@ define dso_local signext i32 @jumptable(i32 signext %param) {
 ; CHECK-R-NEXT:    add r4, r4, r5
 ; CHECK-R-NEXT:    mtctr r4
 ; CHECK-R-NEXT:    bctr
-; CHECK-A-LABEL: jumptable:
-; CHECK-A:       # %bb.1: # %entry
-; CHECK-A-NEXT:    rldic r4, r4
-; CHECK-A-NEXT:    paddi r5, 0, .LJTI0_0@PCREL, 1
-; CHECK-A-NEXT:    ldx r4, r4, r5
-; CHECK-A-NEXT:    mtctr r4
-; CHECK-A-NEXT:    bctr
+; CHECK-A-LE-LABEL: jumptable:
+; CHECK-A-LE:       # %bb.1: # %entry
+; CHECK-A-LE-NEXT:    rldic r4, r4
+; CHECK-A-LE-NEXT:    paddi r5, 0, .LJTI0_0@PCREL, 1
+; CHECK-A-LE-NEXT:    ldx r4, r4, r5
+; CHECK-A-LE-NEXT:    mtctr r4
+; CHECK-A-LE-NEXT:    bctr
+; CHECK-A-BE-LABEL: jumptable:
+; CHECK-A-BE:       # %bb.1: # %entry
+; CHECK-A-BE-NEXT:    rldic r4, r4
+; CHECK-A-BE-NEXT:    paddi r5, 0, .LJTI0_0@PCREL, 1
+; CHECK-A-BE-NEXT:    lwax r4, r4, r5
+; CHECK-A-BE-NEXT:    mtctr r4
+; CHECK-A-BE-NEXT:    bctr
+
 
 entry:
   switch i32 %param, label %sw.default [
diff --git a/llvm/test/CodeGen/PowerPC/pcrel-linkeropt.ll b/llvm/test/CodeGen/PowerPC/pcrel-linkeropt.ll
new file mode 100644
index 0000000000000..e878e74399115
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/pcrel-linkeropt.ll
@@ -0,0 +1,395 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:   -mcpu=future -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
+; RUN:   < %s | FileCheck %s
+
+; On future CPU with PC Relative addressing enabled, it is possible for the
+; linker to optimize GOT indirect accesses. In order for the linker to do this
+; the compiler needs to add a hint using the R_PPC64_PCREL_OPT relocation.
+; This test checks that the compiler adds the R_PPC64_PCREL_OPT relocation
+; correctly.
+
+@input8 = external local_unnamed_addr global i8, align 1
+@output8 = external local_unnamed_addr global i8, align 1
+@input16 = external local_unnamed_addr global i16, align 2
+@output16 = external local_unnamed_addr global i16, align 2
+@input32 = external global i32, align 4
+@output32 = external local_unnamed_addr global i32, align 4
+@input64 = external local_unnamed_addr global i64, align 8
+@output64 = external local_unnamed_addr global i64, align 8
+@input128 = external local_unnamed_addr global i128, align 16
+@output128 = external local_unnamed_addr global i128, align 16
+@inputf32 = external local_unnamed_addr global float, align 4
+@outputf32 = external local_unnamed_addr global float, align 4
+@inputf64 = external local_unnamed_addr global double, align 8
+@outputf64 = external local_unnamed_addr global double, align 8
+@inputVi32 = external local_unnamed_addr global <4 x i32>, align 16
+@outputVi32 = external local_unnamed_addr global <4 x i32>, align 16
+@inputVi64 = external local_unnamed_addr global <2 x i64>, align 16
+@outputVi64 = external local_unnamed_addr global <2 x i64>, align 16
+@ArrayIn = external global [10 x i32], align 4
+@ArrayOut = external local_unnamed_addr global [10 x i32], align 4
+@IntPtrIn = external local_unnamed_addr global i32*, align 8
+@IntPtrOut = external local_unnamed_addr global i32*, align 8
+@FuncPtrIn = external local_unnamed_addr global void (...)*, align 8
+@FuncPtrOut = external local_unnamed_addr global void (...)*, align 8
+
+define dso_local void @ReadWrite8() local_unnamed_addr #0 {
+; CHECK-LABEL: ReadWrite8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pld r3, input8@got@pcrel(0), 1
+; CHECK-NEXT:  .Lpcrel:
+; CHECK-NEXT:    pld r4, output8@got@pcrel(0), 1
+; CHECK-NEXT:    .reloc .Lpcrel-8,R_PPC64_PCREL_OPT,.-(.Lpcrel-8)
+; CHECK-NEXT:    lbz r3, 0(r3)
+; In this test the stb r3, 0(r4) cannot be optimized because it
+; uses the register r3 and that register is defined by lbz r3, 0(r3)
+; which is defined between the pld and the stb.
+; CHECK-NEXT:    stb r3, 0(r4)
+; CHECK-NEXT:    blr
+entry:
+  %0 = load i8, i8* @input8, align 1
+  store i8 %0, i8* @output8, align 1
+  ret void
+}
+
+define dso_local void @ReadWrite16() local_unnamed_addr #0 {
+; CHECK-LABEL: ReadWrite16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pld r3, input16@got@pcrel(0), 1
+; CHECK-NEXT:  .Lpcrel0:
+; CHECK-NEXT:    pld r4, output16@got@pcrel(0), 1
+; CHECK-NEXT:    .reloc .Lpcrel0-8,R_PPC64_PCREL_OPT,.-(.Lpcrel0-8)
+; CHECK-NEXT:    lhz r3, 0(r3)
+; In this test the sth r3, 0(r4) cannot be optimized because it
+; uses the register r3 and that register is defined by lhz r3, 0(r3)
+; which is defined between the pld and the sth.
+; CHECK-NEXT:    sth r3, 0(r4)
+; CHECK-NEXT:    blr
+entry:
+  %0 = load i16, i16* @input16, align 2
+  store i16 %0, i16* @output16, align 2
+  ret void
+}
+
+define dso_local void @ReadWrite32() local_unnamed_addr #0 {
+; CHECK-LABEL: ReadWrite32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pld r3, input32@got@pcrel(0), 1
+; CHECK-NEXT:  .Lpcrel1:
+; CHECK-NEXT:    pld r4, output32@got@pcrel(0), 1
+; CHECK-NEXT:    .reloc .Lpcrel1-8,R_PPC64_PCREL_OPT,.-(.Lpcrel1-8)
+; CHECK-NEXT:    lwz r3, 0(r3)
+; CHECK-NEXT:    stw r3, 0(r4)
+; CHECK-NEXT:    blr
+entry:
+  %0 = load i32, i32* @input32, align 4
+  store i32 %0, i32* @output32, align 4
+  ret void
+}
+
+define dso_local void @ReadWrite64() local_unnamed_addr #0 {
+; CHECK-LABEL: ReadWrite64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pld r3, input64@got@pcrel(0), 1
+; CHECK-NEXT:  .Lpcrel2:
+; CHECK-NEXT:    pld r4, output64@got@pcrel(0), 1
+; CHECK-NEXT:    .reloc .Lpcrel2-8,R_PPC64_PCREL_OPT,.-(.Lpcrel2-8)
+; CHECK-NEXT:    ld r3, 0(r3)
+; CHECK-NEXT:    std r3, 0(r4)
+; CHECK-NEXT:    blr
+entry:
+  %0 = load i64, i64* @input64, align 8
+  store i64 %0, i64* @output64, align 8
+  ret void
+}
+
+; FIXME: we should always convert X-Form instructions that use
+; PPC::ZERO[8] to the corresponding D-Form so we can perform this opt.
+define dso_local void @ReadWrite128() local_unnamed_addr #0 {
+; CHECK-LABEL: ReadWrite128:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pld r3, input128@got@pcrel(0), 1
+; CHECK-NEXT:    lxvx vs0, 0, r3
+; CHECK-NEXT:    pld r3, output128@got@pcrel(0), 1
+; CHECK-NEXT:    stxvx vs0, 0, r3
+; CHECK-NEXT:    blr
+entry:
+  %0 = load i128, i128* @input128, align 16
+  store i128 %0, i128* @output128, align 16
+  ret void
+}
+
+define dso_local void @ReadWritef32() local_unnamed_addr #0 {
+; CHECK-LABEL: ReadWritef32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pld r3, inputf32@got@pcrel(0), 1
+; CHECK-NEXT:  .Lpcrel3:
+; CHECK-NEXT:    xxspltidp vs1, 1078103900
+; CHECK-NEXT:    .reloc .Lpcrel3-8,R_PPC64_PCREL_OPT,.-(.Lpcrel3-8)
+; CHECK-NEXT:    lfs f0, 0(r3)
+; CHECK-NEXT:    pld r3, outputf32@got@pcrel(0), 1
+; CHECK-NEXT:    xsaddsp f0, f0, f1
+; CHECK-NEXT:    stfs f0, 0(r3)
+; CHECK-NEXT:    blr
+entry:
+  %0 = load float, float* @inputf32, align 4
+  %add = fadd float %0, 0x400851EB80000000
+  store float %add, float* @outputf32, align 4
+  ret void
+}
+
+define dso_local void @ReadWritef64() local_unnamed_addr #0 {
+; CHECK-LABEL: ReadWritef64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pld r3, inputf64@got@pcrel(0), 1
+; CHECK-NEXT:  .Lpcrel4:
+; CHECK-NEXT:    plfd f1, .LCPI6_0@PCREL(0), 1
+; CHECK-NEXT:    .reloc .Lpcrel4-8,R_PPC64_PCREL_OPT,.-(.Lpcrel4-8)
+; CHECK-NEXT:    lfd f0, 0(r3)
+; CHECK-NEXT:    pld r3, outputf64@got@pcrel(0), 1
+; CHECK-NEXT:    xsadddp f0, f0, f1
+; CHECK-NEXT:    stfd f0, 0(r3)
+; CHECK-NEXT:    blr
+entry:
+  %0 = load double, double* @inputf64, align 8
+  %add = fadd double %0, 6.800000e+00
+  store double %add, double* @outputf64, align 8
+  ret void
+}
+
+; FIXME: we should always convert X-Form instructions that use
+; PPC::ZERO[8] to the corresponding D-Form so we can perform this opt.
+define dso_local void @ReadWriteVi32() local_unnamed_addr #0 {
+; CHECK-LABEL: ReadWriteVi32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pld r3, inputVi32@got@pcrel(0), 1
+; CHECK-NEXT:    li r4, 45
+; CHECK-NEXT:    mtfprwz f1, r4
+; CHECK-NEXT:    lxvx vs0, 0, r3
+; CHECK-NEXT:    pld r3, outputVi32@got@pcrel(0), 1
+; CHECK-NEXT:    xxinsertw vs0, vs1, 8
+; CHECK-NEXT:    stxvx vs0, 0, r3
+; CHECK-NEXT:    blr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* @inputVi32, align 16
+  %vecins = insertelement <4 x i32> %0, i32 45, i32 1
+  store <4 x i32> %vecins, <4 x i32>* @outputVi32, align 16
+  ret void
+}
+
+define dso_local void @ReadWriteVi64() local_unnamed_addr #0 {
+; CHECK-LABEL: ReadWriteVi64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pld r3, inputVi64@got@pcrel(0), 1
+; CHECK-NEXT:    lxvx vs0, 0, r3
+; CHECK-NEXT:    pld r3, outputVi64@got@pcrel(0), 1
+; CHECK-NEXT:    stxvx vs0, 0, r3
+; CHECK-NEXT:    blr
+entry:
+  %0 = load <2 x i64>, <2 x i64>* @inputVi64, align 16
+  store <2 x i64> %0, <2 x i64>* @outputVi64, align 16
+  ret void
+}
+
+define dso_local void @ReadWriteArray() local_unnamed_addr #0 {
+; CHECK-LABEL: ReadWriteArray:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pld r3, ArrayIn@got@pcrel(0), 1
+; CHECK-NEXT:  .Lpcrel5:
+; CHECK-NEXT:    pld r4, ArrayOut@got@pcrel(0), 1
+; CHECK-NEXT:    .reloc .Lpcrel5-8,R_PPC64_PCREL_OPT,.-(.Lpcrel5-8)
+; CHECK-NEXT:    lwz r3, 28(r3)
+; CHECK-NEXT:    addi r3, r3, 42
+; CHECK-NEXT:    stw r3, 8(r4)
+; CHECK-NEXT:    blr
+entry:
+  %0 = load i32, i32* getelementptr inbounds ([10 x i32], [10 x i32]* @ArrayIn, i64 0, i64 7), align 4
+  %add = add nsw i32 %0, 42
+  store i32 %add, i32* getelementptr inbounds ([10 x i32], [10 x i32]* @ArrayOut, i64 0, i64 2), align 4
+  ret void
+}
+
+define dso_local void @ReadWriteSameArray() local_unnamed_addr #0 {
+; CHECK-LABEL: ReadWriteSameArray:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pld r3, ArrayIn@got@pcrel(0), 1
+; CHECK-NEXT:    lwz r4, 12(r3)
+; CHECK-NEXT:    addi r4, r4, 8
+; CHECK-NEXT:    stw r4, 24(r3)
+; CHECK-NEXT:    blr
+entry:
+  %0 = load i32, i32* getelementptr inbounds ([10 x i32], [10 x i32]* @ArrayIn, i64 0, i64 3), align 4
+  %add = add nsw i32 %0, 8
+  store i32 %add, i32* getelementptr inbounds ([10 x i32], [10 x i32]* @ArrayIn, i64 0, i64 6), align 4
+  ret void
+}
+
+define dso_local void @ReadWriteIntPtr() local_unnamed_addr #0 {
+; CHECK-LABEL: ReadWriteIntPtr:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pld r3, IntPtrIn@got@pcrel(0), 1
+; CHECK-NEXT:  .Lpcrel6:
+; CHECK-NEXT:    pld r4, IntPtrOut@got@pcrel(0), 1
+; CHECK-NEXT:  .Lpcrel7:
+; CHECK-NEXT:    .reloc .Lpcrel6-8,R_PPC64_PCREL_OPT,.-(.Lpcrel6-8)
+; CHECK-NEXT:    ld r3, 0(r3)
+; CHECK-NEXT:    .reloc .Lpcrel7-8,R_PPC64_PCREL_OPT,.-(.Lpcrel7-8)
+; CHECK-NEXT:    ld r4, 0(r4)
+; CHECK-NEXT:    lwz r5, 216(r3)
+; CHECK-NEXT:    lwz r3, 48(r3)
+; CHECK-NEXT:    add r3, r3, r5
+; CHECK-NEXT:    stw r3, 136(r4)
+; CHECK-NEXT:    blr
+entry:
+  %0 = load i32*, i32** @IntPtrIn, align 8
+  %arrayidx = getelementptr inbounds i32, i32* %0, i64 54
+  %1 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %0, i64 12
+  %2 = load i32, i32* %arrayidx1, align 4
+  %add = add nsw i32 %2, %1
+  %3 = load i32*, i32** @IntPtrOut, align 8
+  %arrayidx2 = getelementptr inbounds i32, i32* %3, i64 34
+  store i32 %add, i32* %arrayidx2, align 4
+  ret void
+}
+
+define dso_local void @ReadWriteFuncPtr() local_unnamed_addr #0 {
+; CHECK-LABEL: ReadWriteFuncPtr:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pld r3, FuncPtrIn@got@pcrel(0), 1
+; CHECK-NEXT:  .Lpcrel8:
+; CHECK-NEXT:    pld r4, FuncPtrOut@got@pcrel(0), 1
+; CHECK-NEXT:    .reloc .Lpcrel8-8,R_PPC64_PCREL_OPT,.-(.Lpcrel8-8)
+; CHECK-NEXT:    ld r3, 0(r3)
+; CHECK-NEXT:    std r3, 0(r4)
+; CHECK-NEXT:    blr
+entry:
+  %0 = load i64, i64* bitcast (void (...)** @FuncPtrIn to i64*), align 8
+  store i64 %0, i64* bitcast (void (...)** @FuncPtrOut to i64*), align 8
+  ret void
+}
+
+define dso_local void @FuncPtrCopy() local_unnamed_addr #0 {
+; CHECK-LABEL: FuncPtrCopy:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pld r3, FuncPtrOut@got@pcrel(0), 1
+; CHECK-NEXT:    pld r4, Callee@got@pcrel(0), 1
+; CHECK-NEXT:    std r4, 0(r3)
+; CHECK-NEXT:    blr
+entry:
+  store void (...)* @Callee, void (...)** @FuncPtrOut, align 8
+  ret void
+}
+
+declare void @Callee(...)
+
+define dso_local void @FuncPtrCall() local_unnamed_addr #0 {
+; CHECK-LABEL: FuncPtrCall:
+; CHECK:         .localentry FuncPtrCall, 1
+; CHECK-NEXT:  # %bb.0: # %entry
+; CHECK-NEXT:    pld r3, FuncPtrIn@got@pcrel(0), 1
+; CHECK-NEXT:  .Lpcrel9:
+; CHECK-NEXT:    .reloc .Lpcrel9-8,R_PPC64_PCREL_OPT,.-(.Lpcrel9-8)
+; CHECK-NEXT:    ld r12, 0(r3)
+; CHECK-NEXT:    mtctr r12
+; CHECK-NEXT:    bctr
+; CHECK-NEXT:    #TC_RETURNr8 ctr 0
+entry:
+  %0 = load void ()*, void ()** bitcast (void (...)** @FuncPtrIn to void ()**), align 8
+  tail call void %0()
+  ret void
+}
+
+define dso_local signext i32 @ReadVecElement() local_unnamed_addr #0 {
+; CHECK-LABEL: ReadVecElement:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pld r3, inputVi32@got@pcrel(0), 1
+; CHECK-NEXT:  .Lpcrel10:
+; CHECK-NEXT:    .reloc .Lpcrel10-8,R_PPC64_PCREL_OPT,.-(.Lpcrel10-8)
+; CHECK-NEXT:    lwa r3, 4(r3)
+; CHECK-NEXT:    blr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* @inputVi32, align 16
+  %vecext = extractelement <4 x i32> %0, i32 1
+  ret i32 %vecext
+}
+
+define dso_local signext i32 @VecMultiUse() local_unnamed_addr #0 {
+; CHECK-LABEL: VecMultiUse:
+; CHECK:         .localentry VecMultiUse, 1
+; CHECK-NEXT:  # %bb.0: # %entry
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    std r29, -24(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r0, 16(r1)
+; CHECK-NEXT:    stdu r1, -64(r1)
+; CHECK-NEXT:    pld r30, inputVi32@got@pcrel(0), 1
+; CHECK-NEXT:    lwz r29, 4(r30)
+; CHECK-NEXT:    bl Callee@notoc
+; CHECK-NEXT:    lwz r3, 8(r30)
+; CHECK-NEXT:    add r29, r3, r29
+; CHECK-NEXT:    bl Callee@notoc
+; CHECK-NEXT:    lwz r3, 0(r30)
+; CHECK-NEXT:    add r3, r29, r3
+; CHECK-NEXT:    extsw r3, r3
+; CHECK-NEXT:    addi r1, r1, 64
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r29, -24(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* @inputVi32, align 16
+  tail call void bitcast (void (...)* @Callee to void ()*)()
+  %1 = load <4 x i32>, <4 x i32>* @inputVi32, align 16
+  %2 = extractelement <4 x i32> %1, i32 2
+  %3 = extractelement <4 x i32> %0, i64 1
+  %4 = add nsw i32 %2, %3
+  tail call void bitcast (void (...)* @Callee to void ()*)()
+  %5 = load <4 x i32>, <4 x i32>* @inputVi32, align 16
+  %vecext2 = extractelement <4 x i32> %5, i32 0
+  %add3 = add nsw i32 %4, %vecext2
+  ret i32 %add3
+}
+
+define dso_local signext i32 @UseAddr(i32 signext %a) local_unnamed_addr #0 {
+; CHECK-LABEL: UseAddr:
+; CHECK:         .localentry UseAddr, 1
+; CHECK-NEXT:  # %bb.0: # %entry
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r0, 16(r1)
+; CHECK-NEXT:    stdu r1, -48(r1)
+; CHECK-NEXT:    pld r4, ArrayIn@got@pcrel(0), 1
+; CHECK-NEXT:    lwz r5, 16(r4)
+; CHECK-NEXT:    add r30, r5, r3
+; CHECK-NEXT:    mr r3, r4
+; CHECK-NEXT:    bl getAddr@notoc
+; CHECK-NEXT:    add r3, r30, r3
+; CHECK-NEXT:    extsw r3, r3
+; CHECK-NEXT:    addi r1, r1, 48
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+entry:
+  %0 = load i32, i32* getelementptr inbounds ([10 x i32], [10 x i32]* @ArrayIn, i64 0, i64 4), align 4
+  %add = add nsw i32 %0, %a
+  %call = tail call signext i32 @getAddr(i32* getelementptr inbounds ([10 x i32], [10 x i32]* @ArrayIn, i64 0, i64 0))
+  %add1 = add nsw i32 %add, %call
+  ret i32 %add1
+}
+
+declare signext i32 @getAddr(i32*) local_unnamed_addr
+
+define dso_local nonnull i32* @AddrTaken32() local_unnamed_addr #0 {
+; CHECK-LABEL: AddrTaken32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pld r3, input32@got@pcrel(0), 1
+; CHECK-NEXT:    blr
+entry:
+  ret i32* @input32
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/PowerPC/pcrel-relocation-plus-offset.ll b/llvm/test/CodeGen/PowerPC/pcrel-relocation-plus-offset.ll
index 373b339d45e13..44d3f7a50e9bf 100644
--- a/llvm/test/CodeGen/PowerPC/pcrel-relocation-plus-offset.ll
+++ b/llvm/test/CodeGen/PowerPC/pcrel-relocation-plus-offset.ll
@@ -1,10 +1,18 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
-; RUN:   -mcpu=future -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
 ; RUN:   FileCheck %s --check-prefix=CHECK-S
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
-; RUN:   -mcpu=future -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
 ; RUN:   --filetype=obj < %s | \
-; RUN:   llvm-objdump --mcpu=future -dr - | FileCheck %s --check-prefix=CHECK-O
+; RUN:   llvm-objdump --mcpu=pwr10 -dr - | FileCheck %s --check-prefix=CHECK-O
+; RUN: llc -verify-machineinstrs -target-abi=elfv2 -mtriple=powerpc64-- \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
+; RUN:   FileCheck %s --check-prefix=CHECK-S
+; RUN: llc -verify-machineinstrs -target-abi=elfv2 -mtriple=powerpc64-- \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
+; RUN:   --filetype=obj < %s | \
+; RUN:   llvm-objdump --mcpu=pwr10 -dr - | FileCheck %s --check-prefix=CHECK-O
+
 
 
 @array1 = external local_unnamed_addr global [10 x i32], align 4
@@ -16,9 +24,9 @@ define dso_local signext i32 @getElementLocal7() local_unnamed_addr {
 ; CHECK-S-NEXT:    plwa r3, array2@PCREL+28(0), 1
 ; CHECK-S-NEXT:    blr
 ; CHECK-O-LABEL: <getElementLocal7>:
-; CHECK-O:         00 00 10 04 00 00 60 a4       plwa 3, 0(0), 1
-; CHECK-O-NEXT:    0000000000000000:  R_PPC64_PCREL34      array2+0x1c
-; CHECK-O-NEXT:    20 00 80 4e                   blr
+; CHECK-O:         plwa 3, 0(0), 1
+; CHECK-O-NEXT:      R_PPC64_PCREL34      array2+0x1c
+; CHECK-O-NEXT:    blr
 entry:
   %0 = load i32, i32* getelementptr inbounds ([10 x i32], [10 x i32]* @array2, i64 0, i64 7), align 4
   ret i32 %0
@@ -30,9 +38,9 @@ define dso_local signext i32 @getElementLocalNegative() local_unnamed_addr {
 ; CHECK-S-NEXT:    plwa r3, array2@PCREL-8(0), 1
 ; CHECK-S-NEXT:    blr
 ; CHECK-O-LABEL: <getElementLocalNegative>:
-; CHECK-O:         00 00 10 04 00 00 60 a4       plwa 3, 0(0), 1
-; CHECK-O-NEXT:    0000000000000020:  R_PPC64_PCREL34      array2-0x8
-; CHECK-O-NEXT:    20 00 80 4e                   blr
+; CHECK-O:         plwa 3, 0(0), 1
+; CHECK-O-NEXT:      R_PPC64_PCREL34      array2-0x8
+; CHECK-O-NEXT:    blr
 entry:
   %0 = load i32, i32* getelementptr inbounds ([10 x i32], [10 x i32]* @array2, i64 0, i64 -2), align 4
   ret i32 %0
@@ -42,13 +50,16 @@ define dso_local signext i32 @getElementExtern4() local_unnamed_addr {
 ; CHECK-S-LABEL: getElementExtern4:
 ; CHECK-S:       # %bb.0: # %entry
 ; CHECK-S-NEXT:    pld r3, array1@got@pcrel(0), 1
+; CHECK-S-NEXT:  .Lpcrel:
+; CHECK-S-NEXT:    .reloc .Lpcrel-8,R_PPC64_PCREL_OPT,.-(.Lpcrel-8)
 ; CHECK-S-NEXT:    lwa r3, 16(r3)
 ; CHECK-S-NEXT:    blr
 ; CHECK-O-LABEL: <getElementExtern4>:
-; CHECK-O:         00 00 10 04 00 00 60 e4       pld 3, 0(0), 1
-; CHECK-O-NEXT:    0000000000000040:  R_PPC64_GOT_PCREL34  array1
-; CHECK-O-NEXT:    12 00 63 e8                   lwa 3, 16(3)
-; CHECK-O-NEXT:    20 00 80 4e                   blr
+; CHECK-O:         pld 3, 0(0), 1
+; CHECK-O-NEXT:      R_PPC64_GOT_PCREL34  array1
+; CHECK-O-NEXT:      R_PPC64_PCREL_OPT *ABS*+0x8
+; CHECK-O:         lwa 3, 16(3)
+; CHECK-O-NEXT:    blr
 entry:
   %0 = load i32, i32* getelementptr inbounds ([10 x i32], [10 x i32]* @array1, i64 0, i64 4), align 4
   ret i32 %0
@@ -58,13 +69,16 @@ define dso_local signext i32 @getElementExternNegative() local_unnamed_addr {
 ; CHECK-S-LABEL: getElementExternNegative:
 ; CHECK-S:       # %bb.0: # %entry
 ; CHECK-S-NEXT:    pld r3, array1@got@pcrel(0), 1
+; CHECK-S-NEXT:  .Lpcrel0:
+; CHECK-S-NEXT:    .reloc .Lpcrel0-8,R_PPC64_PCREL_OPT,.-(.Lpcrel0-8)
 ; CHECK-S-NEXT:    lwa r3, -4(r3)
 ; CHECK-S-NEXT:    blr
 ; CHECK-O-LABEL: <getElementExternNegative>:
-; CHECK-O:         00 00 10 04 00 00 60 e4       pld 3, 0(0), 1
-; CHECK-O-NEXT:    0000000000000060:  R_PPC64_GOT_PCREL34  array1
-; CHECK-O-NEXT:    fe ff 63 e8                   lwa 3, -4(3)
-; CHECK-O-NEXT:    20 00 80 4e                   blr
+; CHECK-O:         pld 3, 0(0), 1
+; CHECK-O-NEXT:      R_PPC64_GOT_PCREL34  array1
+; CHECK-O-NEXT:      R_PPC64_PCREL_OPT *ABS*+0x8
+; CHECK-O:         lwa 3, -4(3)
+; CHECK-O-NEXT:    blr
 entry:
   %0 = load i32, i32* getelementptr inbounds ([10 x i32], [10 x i32]* @array1, i64 0, i64 -1), align 4
   ret i32 %0
diff --git a/llvm/test/CodeGen/PowerPC/pcrel-tail-calls.ll b/llvm/test/CodeGen/PowerPC/pcrel-tail-calls.ll
index 78b01601f1faa..56e49780c5f0f 100644
--- a/llvm/test/CodeGen/PowerPC/pcrel-tail-calls.ll
+++ b/llvm/test/CodeGen/PowerPC/pcrel-tail-calls.ll
@@ -1,7 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
-; RUN:   -mcpu=future -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
 ; RUN:   FileCheck %s
+; RUN: llc -verify-machineinstrs -target-abi=elfv2 -mtriple=powerpc64-- \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
+; RUN:   FileCheck %s
+
 
 ; The tests check the behaviour of PC Relative tail calls. When using
 ; PC Relative we are able to do more tail calling than we have done in
@@ -47,6 +51,8 @@ define dso_local void @TailCallExtrnFuncPtr() local_unnamed_addr {
 ; CHECK:         .localentry TailCallExtrnFuncPtr, 1
 ; CHECK-NEXT:  # %bb.0: # %entry
 ; CHECK-NEXT:    pld r3, Func@got@pcrel(0), 1
+; CHECK-NEXT:  .Lpcrel:
+; CHECK-NEXT:    .reloc .Lpcrel-8,R_PPC64_PCREL_OPT,.-(.Lpcrel-8)
 ; CHECK-NEXT:    ld r12, 0(r3)
 ; CHECK-NEXT:    mtctr r12
 ; CHECK-NEXT:    bctr
diff --git a/llvm/test/CodeGen/PowerPC/pcrel.ll b/llvm/test/CodeGen/PowerPC/pcrel.ll
index 539c6a909637b..55783180cfaca 100644
--- a/llvm/test/CodeGen/PowerPC/pcrel.ll
+++ b/llvm/test/CodeGen/PowerPC/pcrel.ll
@@ -1,10 +1,10 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
-; RUN:   -mcpu=future -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
 ; RUN:   FileCheck %s --check-prefix=CHECK-S
-; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
-; RUN:   -mcpu=future -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
+; RUN: llc -verify-machineinstrs -target-abi=elfv2 -mtriple=powerpc64-- \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
 ; RUN:   --filetype=obj < %s | \
-; RUN:   llvm-objdump --mcpu=future -dr - | FileCheck %s --check-prefix=CHECK-O
+; RUN:   llvm-objdump --mcpu=pwr10 -dr - | FileCheck %s --check-prefix=CHECK-O
 
 ; Constant Pool Index.
 ; CHECK-S-LABEL: ConstPool
@@ -41,12 +41,15 @@ define dso_local signext i32 @ReadGlobalVarInt() local_unnamed_addr  {
 ; CHECK-S-LABEL: ReadGlobalVarInt
 ; CHECK-S:       # %bb.0: # %entry
 ; CHECK-S-NEXT:    pld r3, valIntGlob@got@pcrel(0), 1
+; CHECK-S-NEXT: .Lpcrel:
+; CHECK-S-NEXT:    .reloc .Lpcrel-8,R_PPC64_PCREL_OPT,.-(.Lpcrel-8)
 ; CHECK-S-NEXT:    lwa r3, 0(r3)
 ; CHECK-S-NEXT:    blr
 
 ; CHECK-O-LABEL: ReadGlobalVarInt
 ; CHECK-O:         pld 3, 0(0), 1
 ; CHECK-O-NEXT:    R_PPC64_GOT_PCREL34 valIntGlob
+; CHECK-O-NEXT:    R_PPC64_PCREL_OPT *ABS*+0x8
 ; CHECK-O-NEXT:    lwa 3, 0(3)
 ; CHECK-O-NEXT:    blr
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/pr15359.ll b/llvm/test/CodeGen/PowerPC/pr15359.ll
index 23f9e211e93d8..273663e4f5aa1 100644
--- a/llvm/test/CodeGen/PowerPC/pr15359.ll
+++ b/llvm/test/CodeGen/PowerPC/pr15359.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -verify-machineinstrs -O0 -mcpu=pwr7 -filetype=obj %s -o - | \
-; RUN: llvm-readobj --symbols | FileCheck %s
+; RUN: llvm-readobj --symbols - | FileCheck %s
 
 target datalayout = "E-p:64:64:64-S0-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f16:16:16-f32:32:32-f64:64:64-f128:128:128-v64:64:64-v128:128:128-a0:0:64-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
diff --git a/llvm/test/CodeGen/PowerPC/pr22711.ll b/llvm/test/CodeGen/PowerPC/pr22711.ll
index fec38ac8952a5..044486553a6de 100644
--- a/llvm/test/CodeGen/PowerPC/pr22711.ll
+++ b/llvm/test/CodeGen/PowerPC/pr22711.ll
@@ -1,6 +1,6 @@
 ; Verify that the .toc section is aligned on an 8-byte boundary.
 
-; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 -filetype=obj -o - | llvm-readobj --sections | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 -filetype=obj -o - | llvm-readobj --sections - | FileCheck %s
 
 define void @test(i32* %a) {
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/pr46759.ll b/llvm/test/CodeGen/PowerPC/pr46759.ll
index 2c0af89500998..d1d68a5db7e31 100644
--- a/llvm/test/CodeGen/PowerPC/pr46759.ll
+++ b/llvm/test/CodeGen/PowerPC/pr46759.ll
@@ -12,7 +12,7 @@ define void @foo(i32 %vla_size) #0 {
 ; CHECK-LE-NEXT:    mr r12, r1
 ; CHECK-LE-NEXT:    .cfi_def_cfa r12, 0
 ; CHECK-LE-NEXT:    clrldi r0, r12, 53
-; CHECK-LE-NEXT:    stdux r12, r1, r0
+; CHECK-LE-NEXT:    subc r1, r1, r0
 ; CHECK-LE-NEXT:    stdu r12, -2048(r1)
 ; CHECK-LE-NEXT:    stdu r12, -4096(r1)
 ; CHECK-LE-NEXT:    .cfi_def_cfa_register r1
@@ -20,26 +20,27 @@ define void @foo(i32 %vla_size) #0 {
 ; CHECK-LE-NEXT:    .cfi_offset r31, -8
 ; CHECK-LE-NEXT:    .cfi_offset r30, -16
 ; CHECK-LE-NEXT:    clrldi r3, r3, 32
-; CHECK-LE-NEXT:    li r6, -4096
-; CHECK-LE-NEXT:    ld r4, 0(r1)
+; CHECK-LE-NEXT:    li r5, -2048
 ; CHECK-LE-NEXT:    mr r31, r1
 ; CHECK-LE-NEXT:    addi r3, r3, 15
 ; CHECK-LE-NEXT:    rldicl r3, r3, 60, 4
 ; CHECK-LE-NEXT:    rldicl r3, r3, 4, 31
-; CHECK-LE-NEXT:    neg r5, r3
-; CHECK-LE-NEXT:    li r3, -2048
-; CHECK-LE-NEXT:    divd r7, r5, r6
-; CHECK-LE-NEXT:    and r3, r5, r3
-; CHECK-LE-NEXT:    add r3, r1, r3
-; CHECK-LE-NEXT:    mulld r6, r7, r6
-; CHECK-LE-NEXT:    sub r5, r5, r6
-; CHECK-LE-NEXT:    stdux r4, r1, r5
-; CHECK-LE-NEXT:    cmpd r1, r3
+; CHECK-LE-NEXT:    neg r4, r3
+; CHECK-LE-NEXT:    ld r3, 0(r1)
+; CHECK-LE-NEXT:    and r5, r4, r5
+; CHECK-LE-NEXT:    mr r4, r5
+; CHECK-LE-NEXT:    li r5, -4096
+; CHECK-LE-NEXT:    divd r6, r4, r5
+; CHECK-LE-NEXT:    mulld r5, r6, r5
+; CHECK-LE-NEXT:    sub r5, r4, r5
+; CHECK-LE-NEXT:    add r4, r1, r4
+; CHECK-LE-NEXT:    stdux r3, r1, r5
+; CHECK-LE-NEXT:    cmpd r1, r4
 ; CHECK-LE-NEXT:    beq cr0, .LBB0_2
 ; CHECK-LE-NEXT:  .LBB0_1: # %entry
 ; CHECK-LE-NEXT:    #
-; CHECK-LE-NEXT:    stdu r4, -4096(r1)
-; CHECK-LE-NEXT:    cmpd r1, r3
+; CHECK-LE-NEXT:    stdu r3, -4096(r1)
+; CHECK-LE-NEXT:    cmpd r1, r4
 ; CHECK-LE-NEXT:    bne cr0, .LBB0_1
 ; CHECK-LE-NEXT:  .LBB0_2: # %entry
 ; CHECK-LE-NEXT:    addi r3, r1, 2048
diff --git a/llvm/test/CodeGen/PowerPC/stack-clash-dynamic-alloca.ll b/llvm/test/CodeGen/PowerPC/stack-clash-dynamic-alloca.ll
index b475a2f7fbf1c..eef02e77c2b16 100644
--- a/llvm/test/CodeGen/PowerPC/stack-clash-dynamic-alloca.ll
+++ b/llvm/test/CodeGen/PowerPC/stack-clash-dynamic-alloca.ll
@@ -18,23 +18,23 @@ define i32 @foo(i32 %n) local_unnamed_addr #0 "stack-probe-size"="32768" nounwin
 ; CHECK-LE-NEXT:    std r31, -8(r1)
 ; CHECK-LE-NEXT:    stdu r1, -48(r1)
 ; CHECK-LE-NEXT:    rldic r3, r3, 2, 30
-; CHECK-LE-NEXT:    li r6, -32768
+; CHECK-LE-NEXT:    li r5, -32768
 ; CHECK-LE-NEXT:    mr r31, r1
 ; CHECK-LE-NEXT:    addi r3, r3, 15
-; CHECK-LE-NEXT:    addi r4, r31, 48
 ; CHECK-LE-NEXT:    rldicl r3, r3, 60, 4
 ; CHECK-LE-NEXT:    rldicl r3, r3, 4, 29
-; CHECK-LE-NEXT:    neg r5, r3
-; CHECK-LE-NEXT:    divd r7, r5, r6
-; CHECK-LE-NEXT:    add r3, r1, r5
-; CHECK-LE-NEXT:    mulld r6, r7, r6
-; CHECK-LE-NEXT:    sub r5, r5, r6
-; CHECK-LE-NEXT:    stdux r4, r1, r5
-; CHECK-LE-NEXT:    cmpd r1, r3
+; CHECK-LE-NEXT:    neg r4, r3
+; CHECK-LE-NEXT:    addi r3, r31, 48
+; CHECK-LE-NEXT:    divd r6, r4, r5
+; CHECK-LE-NEXT:    mulld r5, r6, r5
+; CHECK-LE-NEXT:    sub r5, r4, r5
+; CHECK-LE-NEXT:    add r4, r1, r4
+; CHECK-LE-NEXT:    stdux r3, r1, r5
+; CHECK-LE-NEXT:    cmpd r1, r4
 ; CHECK-LE-NEXT:    beq cr0, .LBB0_2
 ; CHECK-LE-NEXT:  .LBB0_1:
-; CHECK-LE-NEXT:    stdu r4, -32768(r1)
-; CHECK-LE-NEXT:    cmpd r1, r3
+; CHECK-LE-NEXT:    stdu r3, -32768(r1)
+; CHECK-LE-NEXT:    cmpd r1, r4
 ; CHECK-LE-NEXT:    bne cr0, .LBB0_1
 ; CHECK-LE-NEXT:  .LBB0_2:
 ; CHECK-LE-NEXT:    li r4, 1
@@ -53,20 +53,20 @@ define i32 @foo(i32 %n) local_unnamed_addr #0 "stack-probe-size"="32768" nounwin
 ; CHECK-P9-LE-NEXT:    addi r3, r3, 15
 ; CHECK-P9-LE-NEXT:    li r6, -32768
 ; CHECK-P9-LE-NEXT:    mr r31, r1
-; CHECK-P9-LE-NEXT:    addi r4, r31, 48
 ; CHECK-P9-LE-NEXT:    rldicl r3, r3, 60, 4
 ; CHECK-P9-LE-NEXT:    rldicl r3, r3, 4, 29
 ; CHECK-P9-LE-NEXT:    neg r5, r3
+; CHECK-P9-LE-NEXT:    addi r3, r31, 48
 ; CHECK-P9-LE-NEXT:    divd r7, r5, r6
-; CHECK-P9-LE-NEXT:    add r3, r1, r5
+; CHECK-P9-LE-NEXT:    add r4, r1, r5
 ; CHECK-P9-LE-NEXT:    mulld r6, r7, r6
 ; CHECK-P9-LE-NEXT:    sub r5, r5, r6
-; CHECK-P9-LE-NEXT:    stdux r4, r1, r5
-; CHECK-P9-LE-NEXT:    cmpd r1, r3
+; CHECK-P9-LE-NEXT:    stdux r3, r1, r5
+; CHECK-P9-LE-NEXT:    cmpd r1, r4
 ; CHECK-P9-LE-NEXT:    beq cr0, .LBB0_2
 ; CHECK-P9-LE-NEXT:  .LBB0_1:
-; CHECK-P9-LE-NEXT:    stdu r4, -32768(r1)
-; CHECK-P9-LE-NEXT:    cmpd r1, r3
+; CHECK-P9-LE-NEXT:    stdu r3, -32768(r1)
+; CHECK-P9-LE-NEXT:    cmpd r1, r4
 ; CHECK-P9-LE-NEXT:    bne cr0, .LBB0_1
 ; CHECK-P9-LE-NEXT:  .LBB0_2:
 ; CHECK-P9-LE-NEXT:    li r4, 1
@@ -82,23 +82,23 @@ define i32 @foo(i32 %n) local_unnamed_addr #0 "stack-probe-size"="32768" nounwin
 ; CHECK-BE-NEXT:    std r31, -8(r1)
 ; CHECK-BE-NEXT:    stdu r1, -64(r1)
 ; CHECK-BE-NEXT:    rldic r3, r3, 2, 30
-; CHECK-BE-NEXT:    li r6, -32768
+; CHECK-BE-NEXT:    li r5, -32768
 ; CHECK-BE-NEXT:    addi r3, r3, 15
 ; CHECK-BE-NEXT:    rldicl r3, r3, 60, 4
 ; CHECK-BE-NEXT:    mr r31, r1
 ; CHECK-BE-NEXT:    rldicl r3, r3, 4, 29
-; CHECK-BE-NEXT:    addi r4, r31, 64
-; CHECK-BE-NEXT:    neg r5, r3
-; CHECK-BE-NEXT:    divd r7, r5, r6
-; CHECK-BE-NEXT:    add r3, r1, r5
-; CHECK-BE-NEXT:    mulld r6, r7, r6
-; CHECK-BE-NEXT:    sub r5, r5, r6
-; CHECK-BE-NEXT:    stdux r4, r1, r5
-; CHECK-BE-NEXT:    cmpd r1, r3
+; CHECK-BE-NEXT:    neg r4, r3
+; CHECK-BE-NEXT:    divd r6, r4, r5
+; CHECK-BE-NEXT:    addi r3, r31, 64
+; CHECK-BE-NEXT:    mulld r5, r6, r5
+; CHECK-BE-NEXT:    sub r5, r4, r5
+; CHECK-BE-NEXT:    add r4, r1, r4
+; CHECK-BE-NEXT:    stdux r3, r1, r5
+; CHECK-BE-NEXT:    cmpd r1, r4
 ; CHECK-BE-NEXT:    beq cr0, .LBB0_2
 ; CHECK-BE-NEXT:  .LBB0_1:
-; CHECK-BE-NEXT:    stdu r4, -32768(r1)
-; CHECK-BE-NEXT:    cmpd r1, r3
+; CHECK-BE-NEXT:    stdu r3, -32768(r1)
+; CHECK-BE-NEXT:    cmpd r1, r4
 ; CHECK-BE-NEXT:    bne cr0, .LBB0_1
 ; CHECK-BE-NEXT:  .LBB0_2:
 ; CHECK-BE-NEXT:    li r4, 1
@@ -115,21 +115,21 @@ define i32 @foo(i32 %n) local_unnamed_addr #0 "stack-probe-size"="32768" nounwin
 ; CHECK-32-NEXT:    slwi r3, r3, 2
 ; CHECK-32-NEXT:    addi r3, r3, 15
 ; CHECK-32-NEXT:    rlwinm r3, r3, 0, 0, 27
-; CHECK-32-NEXT:    neg r5, r3
-; CHECK-32-NEXT:    li r6, -32768
-; CHECK-32-NEXT:    divw r7, r5, r6
+; CHECK-32-NEXT:    neg r4, r3
+; CHECK-32-NEXT:    li r5, -32768
+; CHECK-32-NEXT:    divw r6, r4, r5
 ; CHECK-32-NEXT:    stw r31, 28(r1)
 ; CHECK-32-NEXT:    mr r31, r1
-; CHECK-32-NEXT:    addi r4, r31, 32
-; CHECK-32-NEXT:    add r3, r1, r5
-; CHECK-32-NEXT:    mullw r6, r7, r6
-; CHECK-32-NEXT:    sub r5, r5, r6
-; CHECK-32-NEXT:    stwux r4, r1, r5
-; CHECK-32-NEXT:    cmpw r1, r3
+; CHECK-32-NEXT:    addi r3, r31, 32
+; CHECK-32-NEXT:    mullw r5, r6, r5
+; CHECK-32-NEXT:    sub r5, r4, r5
+; CHECK-32-NEXT:    add r4, r1, r4
+; CHECK-32-NEXT:    stwux r3, r1, r5
+; CHECK-32-NEXT:    cmpw r1, r4
 ; CHECK-32-NEXT:    beq cr0, .LBB0_2
 ; CHECK-32-NEXT:  .LBB0_1:
-; CHECK-32-NEXT:    stwu r4, -32768(r1)
-; CHECK-32-NEXT:    cmpw r1, r3
+; CHECK-32-NEXT:    stwu r3, -32768(r1)
+; CHECK-32-NEXT:    cmpw r1, r4
 ; CHECK-32-NEXT:    bne cr0, .LBB0_1
 ; CHECK-32-NEXT:  .LBB0_2:
 ; CHECK-32-NEXT:    li r4, 1
@@ -154,23 +154,23 @@ define i32 @bar(i32 %n) local_unnamed_addr #0 nounwind {
 ; CHECK-LE-NEXT:    std r31, -8(r1)
 ; CHECK-LE-NEXT:    stdu r1, -48(r1)
 ; CHECK-LE-NEXT:    rldic r4, r3, 2, 30
-; CHECK-LE-NEXT:    li r7, -4096
+; CHECK-LE-NEXT:    li r6, -4096
 ; CHECK-LE-NEXT:    mr r31, r1
 ; CHECK-LE-NEXT:    addi r4, r4, 15
-; CHECK-LE-NEXT:    addi r5, r31, 48
 ; CHECK-LE-NEXT:    rldicl r4, r4, 60, 4
 ; CHECK-LE-NEXT:    rldicl r4, r4, 4, 29
-; CHECK-LE-NEXT:    neg r6, r4
-; CHECK-LE-NEXT:    divd r8, r6, r7
-; CHECK-LE-NEXT:    add r4, r1, r6
-; CHECK-LE-NEXT:    mulld r7, r8, r7
-; CHECK-LE-NEXT:    sub r6, r6, r7
-; CHECK-LE-NEXT:    stdux r5, r1, r6
-; CHECK-LE-NEXT:    cmpd r1, r4
+; CHECK-LE-NEXT:    neg r5, r4
+; CHECK-LE-NEXT:    addi r4, r31, 48
+; CHECK-LE-NEXT:    divd r7, r5, r6
+; CHECK-LE-NEXT:    mulld r6, r7, r6
+; CHECK-LE-NEXT:    sub r6, r5, r6
+; CHECK-LE-NEXT:    add r5, r1, r5
+; CHECK-LE-NEXT:    stdux r4, r1, r6
+; CHECK-LE-NEXT:    cmpd r1, r5
 ; CHECK-LE-NEXT:    beq cr0, .LBB1_2
 ; CHECK-LE-NEXT:  .LBB1_1:
-; CHECK-LE-NEXT:    stdu r5, -4096(r1)
-; CHECK-LE-NEXT:    cmpd r1, r4
+; CHECK-LE-NEXT:    stdu r4, -4096(r1)
+; CHECK-LE-NEXT:    cmpd r1, r5
 ; CHECK-LE-NEXT:    bne cr0, .LBB1_1
 ; CHECK-LE-NEXT:  .LBB1_2:
 ; CHECK-LE-NEXT:    extsw r3, r3
@@ -192,20 +192,20 @@ define i32 @bar(i32 %n) local_unnamed_addr #0 nounwind {
 ; CHECK-P9-LE-NEXT:    addi r4, r4, 15
 ; CHECK-P9-LE-NEXT:    li r7, -4096
 ; CHECK-P9-LE-NEXT:    mr r31, r1
-; CHECK-P9-LE-NEXT:    addi r5, r31, 48
 ; CHECK-P9-LE-NEXT:    rldicl r4, r4, 60, 4
 ; CHECK-P9-LE-NEXT:    rldicl r4, r4, 4, 29
 ; CHECK-P9-LE-NEXT:    neg r6, r4
+; CHECK-P9-LE-NEXT:    addi r4, r31, 48
 ; CHECK-P9-LE-NEXT:    divd r8, r6, r7
-; CHECK-P9-LE-NEXT:    add r4, r1, r6
+; CHECK-P9-LE-NEXT:    add r5, r1, r6
 ; CHECK-P9-LE-NEXT:    mulld r7, r8, r7
 ; CHECK-P9-LE-NEXT:    sub r6, r6, r7
-; CHECK-P9-LE-NEXT:    stdux r5, r1, r6
-; CHECK-P9-LE-NEXT:    cmpd r1, r4
+; CHECK-P9-LE-NEXT:    stdux r4, r1, r6
+; CHECK-P9-LE-NEXT:    cmpd r1, r5
 ; CHECK-P9-LE-NEXT:    beq cr0, .LBB1_2
 ; CHECK-P9-LE-NEXT:  .LBB1_1:
-; CHECK-P9-LE-NEXT:    stdu r5, -4096(r1)
-; CHECK-P9-LE-NEXT:    cmpd r1, r4
+; CHECK-P9-LE-NEXT:    stdu r4, -4096(r1)
+; CHECK-P9-LE-NEXT:    cmpd r1, r5
 ; CHECK-P9-LE-NEXT:    bne cr0, .LBB1_1
 ; CHECK-P9-LE-NEXT:  .LBB1_2:
 ; CHECK-P9-LE-NEXT:    extswsli r3, r3, 2
@@ -223,23 +223,23 @@ define i32 @bar(i32 %n) local_unnamed_addr #0 nounwind {
 ; CHECK-BE-NEXT:    std r31, -8(r1)
 ; CHECK-BE-NEXT:    stdu r1, -64(r1)
 ; CHECK-BE-NEXT:    rldic r4, r3, 2, 30
-; CHECK-BE-NEXT:    li r7, -4096
+; CHECK-BE-NEXT:    li r6, -4096
 ; CHECK-BE-NEXT:    addi r4, r4, 15
 ; CHECK-BE-NEXT:    rldicl r4, r4, 60, 4
 ; CHECK-BE-NEXT:    mr r31, r1
 ; CHECK-BE-NEXT:    rldicl r4, r4, 4, 29
-; CHECK-BE-NEXT:    addi r5, r31, 64
-; CHECK-BE-NEXT:    neg r6, r4
-; CHECK-BE-NEXT:    divd r8, r6, r7
-; CHECK-BE-NEXT:    add r4, r1, r6
-; CHECK-BE-NEXT:    mulld r7, r8, r7
-; CHECK-BE-NEXT:    sub r6, r6, r7
-; CHECK-BE-NEXT:    stdux r5, r1, r6
-; CHECK-BE-NEXT:    cmpd r1, r4
+; CHECK-BE-NEXT:    neg r5, r4
+; CHECK-BE-NEXT:    divd r7, r5, r6
+; CHECK-BE-NEXT:    addi r4, r31, 64
+; CHECK-BE-NEXT:    mulld r6, r7, r6
+; CHECK-BE-NEXT:    sub r6, r5, r6
+; CHECK-BE-NEXT:    add r5, r1, r5
+; CHECK-BE-NEXT:    stdux r4, r1, r6
+; CHECK-BE-NEXT:    cmpd r1, r5
 ; CHECK-BE-NEXT:    beq cr0, .LBB1_2
 ; CHECK-BE-NEXT:  .LBB1_1:
-; CHECK-BE-NEXT:    stdu r5, -4096(r1)
-; CHECK-BE-NEXT:    cmpd r1, r4
+; CHECK-BE-NEXT:    stdu r4, -4096(r1)
+; CHECK-BE-NEXT:    cmpd r1, r5
 ; CHECK-BE-NEXT:    bne cr0, .LBB1_1
 ; CHECK-BE-NEXT:  .LBB1_2:
 ; CHECK-BE-NEXT:    extsw r3, r3
@@ -259,21 +259,21 @@ define i32 @bar(i32 %n) local_unnamed_addr #0 nounwind {
 ; CHECK-32-NEXT:    slwi r3, r3, 2
 ; CHECK-32-NEXT:    addi r4, r3, 15
 ; CHECK-32-NEXT:    rlwinm r4, r4, 0, 0, 27
-; CHECK-32-NEXT:    neg r6, r4
-; CHECK-32-NEXT:    li r7, -4096
-; CHECK-32-NEXT:    divw r8, r6, r7
+; CHECK-32-NEXT:    neg r5, r4
+; CHECK-32-NEXT:    li r6, -4096
+; CHECK-32-NEXT:    divw r7, r5, r6
 ; CHECK-32-NEXT:    stw r31, 28(r1)
 ; CHECK-32-NEXT:    mr r31, r1
-; CHECK-32-NEXT:    addi r5, r31, 32
-; CHECK-32-NEXT:    add r4, r1, r6
-; CHECK-32-NEXT:    mullw r7, r8, r7
-; CHECK-32-NEXT:    sub r6, r6, r7
-; CHECK-32-NEXT:    stwux r5, r1, r6
-; CHECK-32-NEXT:    cmpw r1, r4
+; CHECK-32-NEXT:    addi r4, r31, 32
+; CHECK-32-NEXT:    mullw r6, r7, r6
+; CHECK-32-NEXT:    sub r6, r5, r6
+; CHECK-32-NEXT:    add r5, r1, r5
+; CHECK-32-NEXT:    stwux r4, r1, r6
+; CHECK-32-NEXT:    cmpw r1, r5
 ; CHECK-32-NEXT:    beq cr0, .LBB1_2
 ; CHECK-32-NEXT:  .LBB1_1:
-; CHECK-32-NEXT:    stwu r5, -4096(r1)
-; CHECK-32-NEXT:    cmpw r1, r4
+; CHECK-32-NEXT:    stwu r4, -4096(r1)
+; CHECK-32-NEXT:    cmpw r1, r5
 ; CHECK-32-NEXT:    bne cr0, .LBB1_1
 ; CHECK-32-NEXT:  .LBB1_2:
 ; CHECK-32-NEXT:    addi r4, r1, 16
@@ -300,24 +300,24 @@ define i32 @f(i32 %n) local_unnamed_addr #0 "stack-probe-size"="65536" nounwind
 ; CHECK-LE-NEXT:    std r31, -8(r1)
 ; CHECK-LE-NEXT:    stdu r1, -48(r1)
 ; CHECK-LE-NEXT:    rldic r3, r3, 2, 30
-; CHECK-LE-NEXT:    lis r5, -1
+; CHECK-LE-NEXT:    lis r4, -1
 ; CHECK-LE-NEXT:    mr r31, r1
 ; CHECK-LE-NEXT:    addi r3, r3, 15
-; CHECK-LE-NEXT:    ori r5, r5, 0
-; CHECK-LE-NEXT:    addi r4, r31, 48
+; CHECK-LE-NEXT:    ori r4, r4, 0
 ; CHECK-LE-NEXT:    rldicl r3, r3, 60, 4
 ; CHECK-LE-NEXT:    rldicl r3, r3, 4, 29
-; CHECK-LE-NEXT:    neg r6, r3
-; CHECK-LE-NEXT:    divd r7, r6, r5
-; CHECK-LE-NEXT:    add r3, r1, r6
-; CHECK-LE-NEXT:    mulld r7, r7, r5
-; CHECK-LE-NEXT:    sub r6, r6, r7
-; CHECK-LE-NEXT:    stdux r4, r1, r6
-; CHECK-LE-NEXT:    cmpd r1, r3
+; CHECK-LE-NEXT:    neg r5, r3
+; CHECK-LE-NEXT:    addi r3, r31, 48
+; CHECK-LE-NEXT:    divd r6, r5, r4
+; CHECK-LE-NEXT:    mulld r6, r6, r4
+; CHECK-LE-NEXT:    sub r6, r5, r6
+; CHECK-LE-NEXT:    add r5, r1, r5
+; CHECK-LE-NEXT:    stdux r3, r1, r6
+; CHECK-LE-NEXT:    cmpd r1, r5
 ; CHECK-LE-NEXT:    beq cr0, .LBB2_2
 ; CHECK-LE-NEXT:  .LBB2_1:
-; CHECK-LE-NEXT:    stdux r4, r1, r5
-; CHECK-LE-NEXT:    cmpd r1, r3
+; CHECK-LE-NEXT:    stdux r3, r1, r4
+; CHECK-LE-NEXT:    cmpd r1, r5
 ; CHECK-LE-NEXT:    bne cr0, .LBB2_1
 ; CHECK-LE-NEXT:  .LBB2_2:
 ; CHECK-LE-NEXT:    li r4, 1
@@ -337,20 +337,20 @@ define i32 @f(i32 %n) local_unnamed_addr #0 "stack-probe-size"="65536" nounwind
 ; CHECK-P9-LE-NEXT:    lis r5, -1
 ; CHECK-P9-LE-NEXT:    ori r5, r5, 0
 ; CHECK-P9-LE-NEXT:    mr r31, r1
-; CHECK-P9-LE-NEXT:    addi r4, r31, 48
 ; CHECK-P9-LE-NEXT:    rldicl r3, r3, 60, 4
 ; CHECK-P9-LE-NEXT:    rldicl r3, r3, 4, 29
 ; CHECK-P9-LE-NEXT:    neg r6, r3
+; CHECK-P9-LE-NEXT:    addi r3, r31, 48
 ; CHECK-P9-LE-NEXT:    divd r7, r6, r5
-; CHECK-P9-LE-NEXT:    add r3, r1, r6
+; CHECK-P9-LE-NEXT:    add r4, r1, r6
 ; CHECK-P9-LE-NEXT:    mulld r7, r7, r5
 ; CHECK-P9-LE-NEXT:    sub r6, r6, r7
-; CHECK-P9-LE-NEXT:    stdux r4, r1, r6
-; CHECK-P9-LE-NEXT:    cmpd r1, r3
+; CHECK-P9-LE-NEXT:    stdux r3, r1, r6
+; CHECK-P9-LE-NEXT:    cmpd r1, r4
 ; CHECK-P9-LE-NEXT:    beq cr0, .LBB2_2
 ; CHECK-P9-LE-NEXT:  .LBB2_1:
-; CHECK-P9-LE-NEXT:    stdux r4, r1, r5
-; CHECK-P9-LE-NEXT:    cmpd r1, r3
+; CHECK-P9-LE-NEXT:    stdux r3, r1, r5
+; CHECK-P9-LE-NEXT:    cmpd r1, r4
 ; CHECK-P9-LE-NEXT:    bne cr0, .LBB2_1
 ; CHECK-P9-LE-NEXT:  .LBB2_2:
 ; CHECK-P9-LE-NEXT:    li r4, 1
@@ -366,24 +366,24 @@ define i32 @f(i32 %n) local_unnamed_addr #0 "stack-probe-size"="65536" nounwind
 ; CHECK-BE-NEXT:    std r31, -8(r1)
 ; CHECK-BE-NEXT:    stdu r1, -64(r1)
 ; CHECK-BE-NEXT:    rldic r3, r3, 2, 30
-; CHECK-BE-NEXT:    lis r5, -1
+; CHECK-BE-NEXT:    lis r4, -1
 ; CHECK-BE-NEXT:    addi r3, r3, 15
 ; CHECK-BE-NEXT:    rldicl r3, r3, 60, 4
-; CHECK-BE-NEXT:    ori r5, r5, 0
+; CHECK-BE-NEXT:    ori r4, r4, 0
 ; CHECK-BE-NEXT:    rldicl r3, r3, 4, 29
 ; CHECK-BE-NEXT:    mr r31, r1
-; CHECK-BE-NEXT:    neg r6, r3
-; CHECK-BE-NEXT:    divd r7, r6, r5
-; CHECK-BE-NEXT:    addi r4, r31, 64
-; CHECK-BE-NEXT:    mulld r7, r7, r5
-; CHECK-BE-NEXT:    add r3, r1, r6
-; CHECK-BE-NEXT:    sub r6, r6, r7
-; CHECK-BE-NEXT:    stdux r4, r1, r6
-; CHECK-BE-NEXT:    cmpd r1, r3
+; CHECK-BE-NEXT:    neg r5, r3
+; CHECK-BE-NEXT:    divd r6, r5, r4
+; CHECK-BE-NEXT:    addi r3, r31, 64
+; CHECK-BE-NEXT:    mulld r6, r6, r4
+; CHECK-BE-NEXT:    sub r6, r5, r6
+; CHECK-BE-NEXT:    add r5, r1, r5
+; CHECK-BE-NEXT:    stdux r3, r1, r6
+; CHECK-BE-NEXT:    cmpd r1, r5
 ; CHECK-BE-NEXT:    beq cr0, .LBB2_2
 ; CHECK-BE-NEXT:  .LBB2_1:
-; CHECK-BE-NEXT:    stdux r4, r1, r5
-; CHECK-BE-NEXT:    cmpd r1, r3
+; CHECK-BE-NEXT:    stdux r3, r1, r4
+; CHECK-BE-NEXT:    cmpd r1, r5
 ; CHECK-BE-NEXT:    bne cr0, .LBB2_1
 ; CHECK-BE-NEXT:  .LBB2_2:
 ; CHECK-BE-NEXT:    li r4, 1
@@ -400,22 +400,22 @@ define i32 @f(i32 %n) local_unnamed_addr #0 "stack-probe-size"="65536" nounwind
 ; CHECK-32-NEXT:    slwi r3, r3, 2
 ; CHECK-32-NEXT:    addi r3, r3, 15
 ; CHECK-32-NEXT:    rlwinm r3, r3, 0, 0, 27
-; CHECK-32-NEXT:    lis r5, -1
-; CHECK-32-NEXT:    neg r6, r3
-; CHECK-32-NEXT:    ori r5, r5, 0
-; CHECK-32-NEXT:    divw r7, r6, r5
+; CHECK-32-NEXT:    lis r4, -1
+; CHECK-32-NEXT:    neg r5, r3
+; CHECK-32-NEXT:    ori r4, r4, 0
+; CHECK-32-NEXT:    divw r6, r5, r4
 ; CHECK-32-NEXT:    stw r31, 28(r1)
 ; CHECK-32-NEXT:    mr r31, r1
-; CHECK-32-NEXT:    addi r4, r31, 32
-; CHECK-32-NEXT:    add r3, r1, r6
-; CHECK-32-NEXT:    mullw r7, r7, r5
-; CHECK-32-NEXT:    sub r6, r6, r7
-; CHECK-32-NEXT:    stwux r4, r1, r6
-; CHECK-32-NEXT:    cmpw r1, r3
+; CHECK-32-NEXT:    addi r3, r31, 32
+; CHECK-32-NEXT:    mullw r6, r6, r4
+; CHECK-32-NEXT:    sub r6, r5, r6
+; CHECK-32-NEXT:    add r5, r1, r5
+; CHECK-32-NEXT:    stwux r3, r1, r6
+; CHECK-32-NEXT:    cmpw r1, r5
 ; CHECK-32-NEXT:    beq cr0, .LBB2_2
 ; CHECK-32-NEXT:  .LBB2_1:
-; CHECK-32-NEXT:    stwux r4, r1, r5
-; CHECK-32-NEXT:    cmpw r1, r3
+; CHECK-32-NEXT:    stwux r3, r1, r4
+; CHECK-32-NEXT:    cmpw r1, r5
 ; CHECK-32-NEXT:    bne cr0, .LBB2_1
 ; CHECK-32-NEXT:  .LBB2_2:
 ; CHECK-32-NEXT:    li r4, 1
diff --git a/llvm/test/CodeGen/RISCV/fixups-diff.ll b/llvm/test/CodeGen/RISCV/fixups-diff.ll
index 08dca52a34561..8b2273ddaa388 100644
--- a/llvm/test/CodeGen/RISCV/fixups-diff.ll
+++ b/llvm/test/CodeGen/RISCV/fixups-diff.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -filetype=obj -mtriple=riscv32 -mattr=+relax %s -o - \
-; RUN:     | llvm-readobj -r | FileCheck -check-prefix=RELAX %s
+; RUN:     | llvm-readobj -r - | FileCheck -check-prefix=RELAX %s
 ; RUN: llc -filetype=obj -mtriple=riscv32 -mattr=-relax %s -o - \
-; RUN:     | llvm-readobj -r | FileCheck -check-prefix=NORELAX %s
+; RUN:     | llvm-readobj -r - | FileCheck -check-prefix=NORELAX %s
 
 ; Check that a difference between two symbols in the same fragment
 ; causes relocations to be emitted if and only if relaxation is enabled.
diff --git a/llvm/test/CodeGen/RISCV/fixups-relax-diff.ll b/llvm/test/CodeGen/RISCV/fixups-relax-diff.ll
index 0fa7998dae04e..2ac335b74d65c 100644
--- a/llvm/test/CodeGen/RISCV/fixups-relax-diff.ll
+++ b/llvm/test/CodeGen/RISCV/fixups-relax-diff.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -filetype=obj -mtriple=riscv32 -mattr=+relax %s -o - \
-; RUN:     | llvm-readobj -r | FileCheck -check-prefix=RELAX %s
+; RUN:     | llvm-readobj -r - | FileCheck -check-prefix=RELAX %s
 ; RUN: llc -filetype=obj -mtriple=riscv32 -mattr=-relax %s -o - \
-; RUN:     | llvm-readobj -r | FileCheck -check-prefix=NORELAX %s
+; RUN:     | llvm-readobj -r - | FileCheck -check-prefix=NORELAX %s
 
 ; This test checks that a diff inserted via inline assembly only causes
 ; relocations when relaxation is enabled. This isn't an assembly test
diff --git a/llvm/test/CodeGen/SPARC/obj-relocs.ll b/llvm/test/CodeGen/SPARC/obj-relocs.ll
index 0e7e04032be07..edeb39ec47edd 100644
--- a/llvm/test/CodeGen/SPARC/obj-relocs.ll
+++ b/llvm/test/CodeGen/SPARC/obj-relocs.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -march=sparcv9 -filetype=obj --relocation-model=static | llvm-readobj -r | FileCheck %s --check-prefix=CHECK-ABS
-; RUN: llc < %s -march=sparcv9 -filetype=obj --relocation-model=pic    | llvm-readobj -r | FileCheck %s --check-prefix=CHECK-PIC
+; RUN: llc < %s -march=sparcv9 -filetype=obj --relocation-model=static | llvm-readobj -r - | FileCheck %s --check-prefix=CHECK-ABS
+; RUN: llc < %s -march=sparcv9 -filetype=obj --relocation-model=pic    | llvm-readobj -r - | FileCheck %s --check-prefix=CHECK-PIC
 
 ;CHECK-ABS:      Relocations [
 ;CHECK-ABS:         0x{{[0-9,A-F]+}} R_SPARC_H44 AGlobalVar 0x0
diff --git a/llvm/test/CodeGen/SPARC/tls.ll b/llvm/test/CodeGen/SPARC/tls.ll
index 843769df8d936..da676ca84d40a 100644
--- a/llvm/test/CodeGen/SPARC/tls.ll
+++ b/llvm/test/CodeGen/SPARC/tls.ll
@@ -3,10 +3,10 @@
 ; RUN: llc <%s -mtriple=sparc-unknown-linux   -relocation-model=pic    | FileCheck %s --check-prefix=pic
 ; RUN: llc <%s -mtriple=sparcv9-unknown-linux -relocation-model=pic    | FileCheck %s --check-prefix=pic
 
-; RUN: llc <%s -mtriple=sparc-unknown-linux   -relocation-model=static -filetype=obj | llvm-readobj -r --symbols | FileCheck %s --check-prefix=v8abs-obj
-; RUN: llc <%s -mtriple=sparcv9-unknown-linux -relocation-model=static -filetype=obj | llvm-readobj -r --symbols | FileCheck %s --check-prefix=v9abs-obj
-; RUN: llc <%s -mtriple=sparc-unknown-linux   -relocation-model=pic    -filetype=obj | llvm-readobj -r --symbols | FileCheck %s --check-prefix=pic-obj
-; RUN: llc <%s -mtriple=sparcv9-unknown-linux -relocation-model=pic    -filetype=obj | llvm-readobj -r --symbols | FileCheck %s --check-prefix=pic-obj
+; RUN: llc <%s -mtriple=sparc-unknown-linux   -relocation-model=static -filetype=obj | llvm-readobj -r --symbols - | FileCheck %s --check-prefix=v8abs-obj
+; RUN: llc <%s -mtriple=sparcv9-unknown-linux -relocation-model=static -filetype=obj | llvm-readobj -r --symbols - | FileCheck %s --check-prefix=v9abs-obj
+; RUN: llc <%s -mtriple=sparc-unknown-linux   -relocation-model=pic    -filetype=obj | llvm-readobj -r --symbols - | FileCheck %s --check-prefix=pic-obj
+; RUN: llc <%s -mtriple=sparcv9-unknown-linux -relocation-model=pic    -filetype=obj | llvm-readobj -r --symbols - | FileCheck %s --check-prefix=pic-obj
 
 @local_symbol = internal thread_local global i32 0
 @extern_symbol = external thread_local global i32
diff --git a/llvm/test/CodeGen/SystemZ/no-vx.ll b/llvm/test/CodeGen/SystemZ/no-vx.ll
new file mode 100644
index 0000000000000..a590739e547ea
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/no-vx.ll
@@ -0,0 +1,33 @@
+; RUN: llc < %s -mtriple=s390x-linux-gnu -o - | FileCheck %s
+
+; This test verifies that passing the "-vector" feature disables *any*
+; use of vector instructions, even if +vector-enhancements-1 if given.
+
+; Function Attrs: noinline nounwind optnone
+define dso_local signext i32 @foo(fp128* %0, fp128* %1) #0 {
+; CHECK-LABEL: @foo
+; CHECK-NOT: vl
+; CHECK-NOT: vst
+; CHECK: br %r14
+entry:
+  %arg1.addr = alloca fp128, align 8
+  %arg2.addr = alloca fp128, align 8
+  %indirect-arg-temp = alloca fp128, align 8
+  %indirect-arg-temp1 = alloca fp128, align 8
+  %arg1 = load fp128, fp128* %0, align 8
+  %arg2 = load fp128, fp128* %1, align 8
+  store fp128 %arg1, fp128* %arg1.addr, align 8
+  store fp128 %arg2, fp128* %arg2.addr, align 8
+  %2 = load fp128, fp128* %arg1.addr, align 8
+  %3 = load fp128, fp128* %arg2.addr, align 8
+  store fp128 %2, fp128* %indirect-arg-temp, align 8
+  store fp128 %3, fp128* %indirect-arg-temp1, align 8
+  %call = call signext i32 @bar(i32 signext 2, fp128* %indirect-arg-temp, fp128* %indirect-arg-temp1)
+  ret i32 %call
+}
+
+declare dso_local signext i32 @bar(i32 signext, fp128*, fp128*) #1
+
+attributes #0 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="z14" "target-features"="+transactional-execution,+vector-enhancements-1,-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="z14" "target-features"="+transactional-execution,+vector-enhancements-1,-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/cond-vector-reduce-mve-codegen.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/cond-vector-reduce-mve-codegen.ll
index e98276e258abd..eff56a041ee27 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/cond-vector-reduce-mve-codegen.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/cond-vector-reduce-mve-codegen.ll
@@ -99,8 +99,9 @@ define dso_local i32 @vpsel_mul_reduce_add_2(i32* noalias nocapture readonly %a,
 ; CHECK-LABEL: vpsel_mul_reduce_add_2:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    push {r4, r5, r7, lr}
-; CHECK-NEXT:    sub sp, #4
-; CHECK-NEXT:    ldr.w r12, [sp, #20]
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    ldr.w r12, [sp, #40]
 ; CHECK-NEXT:    cmp.w r12, #0
 ; CHECK-NEXT:    beq .LBB1_4
 ; CHECK-NEXT:  @ %bb.1: @ %vector.ph
@@ -116,19 +117,17 @@ define dso_local i32 @vpsel_mul_reduce_add_2(i32* noalias nocapture readonly %a,
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vctp.32 r12
 ; CHECK-NEXT:    and r5, r4, #15
-; CHECK-NEXT:    vstr p0, [sp] @ 4-byte Spill
+; CHECK-NEXT:    vstr p0, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:    vmov q0, q1
-; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vldrwt.u32 q1, [r3], #16
-; CHECK-NEXT:    vldrwt.u32 q2, [r2], #16
-; CHECK-NEXT:    vdup.32 q3, r5
-; CHECK-NEXT:    vsub.i32 q1, q2, q1
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vldrwt.u32 q2, [r1], #16
-; CHECK-NEXT:    vcmp.i32 eq, q3, zr
+; CHECK-NEXT:    vpsttt
+; CHECK-NEXT:    vldrwt.u32 q1, [r1], #16
+; CHECK-NEXT:    vldrwt.u32 q2, [r3], #16
+; CHECK-NEXT:    vldrwt.u32 q3, [r2], #16
+; CHECK-NEXT:    vdup.32 q4, r5
+; CHECK-NEXT:    vpt.i32 eq, q4, zr
+; CHECK-NEXT:    vsubt.i32 q1, q3, q2
 ; CHECK-NEXT:    adds r4, #4
-; CHECK-NEXT:    vpsel q1, q1, q2
-; CHECK-NEXT:    vldr p0, [sp] @ 4-byte Reload
+; CHECK-NEXT:    vldr p0, [sp, #4] @ 4-byte Reload
 ; CHECK-NEXT:    vpst
 ; CHECK-NEXT:    vldrwt.u32 q2, [r0], #16
 ; CHECK-NEXT:    vmul.i32 q1, q1, q2
@@ -138,11 +137,12 @@ define dso_local i32 @vpsel_mul_reduce_add_2(i32* noalias nocapture readonly %a,
 ; CHECK-NEXT:  @ %bb.3: @ %middle.block
 ; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    vaddv.u32 r0, q0
-; CHECK-NEXT:    add sp, #4
-; CHECK-NEXT:    pop {r4, r5, r7, pc}
+; CHECK-NEXT:    b .LBB1_5
 ; CHECK-NEXT:  .LBB1_4:
 ; CHECK-NEXT:    movs r0, #0
-; CHECK-NEXT:    add sp, #4
+; CHECK-NEXT:  .LBB1_5: @ %for.cond.cleanup
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {r4, r5, r7, pc}
                                          i32* noalias nocapture readonly %c, i32* noalias nocapture readonly %d, i32 %N) {
 entry:
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/predicates.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/predicates.ll
index 23220bb552f87..7a2a5559cc8f8 100644
--- a/llvm/test/CodeGen/Thumb2/mve-intrinsics/predicates.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/predicates.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: opt -instcombine %s | llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - | FileCheck %s
+; RUN: opt -instcombine -mtriple=thumbv8.1m.main %s | llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - | FileCheck %s
 
 declare <16 x i1> @llvm.arm.mve.vctp8(i32)
 declare <8 x i1> @llvm.arm.mve.vctp16(i32)
diff --git a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vadc-multiple.ll b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vadc-multiple.ll
index 2bc8c5f589c36..118861ccbb138 100644
--- a/llvm/test/CodeGen/Thumb2/mve-intrinsics/vadc-multiple.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-intrinsics/vadc-multiple.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: opt -instcombine -S %s | FileCheck --check-prefix=IR %s
-; RUN: opt -instcombine    %s | llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -O3 -o - | FileCheck --check-prefix=ASM %s
+; RUN: opt -instcombine -mtriple=thumbv8.1m.main -S %s | FileCheck --check-prefix=IR %s
+; RUN: opt -instcombine -mtriple=thumbv8.1m.main    %s | llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -O3 -o - | FileCheck --check-prefix=ASM %s
 
 %struct.foo = type { [2 x <4 x i32>] }
 
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-selectop.ll b/llvm/test/CodeGen/Thumb2/mve-pred-selectop.ll
new file mode 100644
index 0000000000000..51a25b261746b
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-selectop.ll
@@ -0,0 +1,1202 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK
+
+define arm_aapcs_vfpcc <4 x i32> @add_v4i32(<4 x i32> %z, <4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: add_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vaddt.i32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %z, zeroinitializer
+  %a = add <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %z
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @add_v8i16(<8 x i16> %z, <8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: add_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vaddt.i16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %z, zeroinitializer
+  %a = add <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %z
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @add_v16i8(<16 x i8> %z, <16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: add_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vaddt.i8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %z, zeroinitializer
+  %a = add <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %z
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sub_v4i32(<4 x i32> %z, <4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: sub_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vsubt.i32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %z, zeroinitializer
+  %a = sub <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %z
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sub_v8i16(<8 x i16> %z, <8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: sub_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vsubt.i16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %z, zeroinitializer
+  %a = sub <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %z
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @sub_v16i8(<16 x i8> %z, <16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: sub_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vsubt.i8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %z, zeroinitializer
+  %a = sub <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %z
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @mul_v4i32(<4 x i32> %z, <4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: mul_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vmult.i32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %z, zeroinitializer
+  %a = mul <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %z
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @mul_v8i16(<8 x i16> %z, <8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: mul_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vmult.i16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %z, zeroinitializer
+  %a = mul <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %z
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @mul_v16i8(<16 x i8> %z, <16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: mul_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vmult.i8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %z, zeroinitializer
+  %a = mul <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %z
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @and_v4i32(<4 x i32> %z, <4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: and_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vandt q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %z, zeroinitializer
+  %a = and <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %z
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @and_v8i16(<8 x i16> %z, <8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: and_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vandt q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %z, zeroinitializer
+  %a = and <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %z
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @and_v16i8(<16 x i8> %z, <16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: and_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vandt q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %z, zeroinitializer
+  %a = and <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %z
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @or_v4i32(<4 x i32> %z, <4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: or_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vorrt q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %z, zeroinitializer
+  %a = or <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %z
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @or_v8i16(<8 x i16> %z, <8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: or_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vorrt q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %z, zeroinitializer
+  %a = or <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %z
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @or_v16i8(<16 x i8> %z, <16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: or_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vorrt q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %z, zeroinitializer
+  %a = or <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %z
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @xor_v4i32(<4 x i32> %z, <4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: xor_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    veort q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %z, zeroinitializer
+  %a = xor <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %z
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @xor_v8i16(<8 x i16> %z, <8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: xor_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    veort q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %z, zeroinitializer
+  %a = xor <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %z
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @xor_v16i8(<16 x i8> %z, <16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: xor_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    veort q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %z, zeroinitializer
+  %a = xor <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %z
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @andnot_v4i32(<4 x i32> %z, <4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: andnot_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vbict q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %z, zeroinitializer
+  %y1 = xor <4 x i32> %y, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %a = and <4 x i32> %x, %y1
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %z
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @andnot_v8i16(<8 x i16> %z, <8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: andnot_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vbict q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %z, zeroinitializer
+  %y1 = xor <8 x i16> %y, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+  %a = and <8 x i16> %x, %y1
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %z
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @andnot_v16i8(<16 x i8> %z, <16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: andnot_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vbict q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %z, zeroinitializer
+  %y1 = xor <16 x i8> %y, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %a = and <16 x i8> %x, %y1
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %z
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ornot_v4i32(<4 x i32> %z, <4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: ornot_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vornt q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %z, zeroinitializer
+  %y1 = xor <4 x i32> %y, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %a = or <4 x i32> %x, %y1
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %z
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ornot_v8i16(<8 x i16> %z, <8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: ornot_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vornt q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %z, zeroinitializer
+  %y1 = xor <8 x i16> %y, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+  %a = or <8 x i16> %x, %y1
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %z
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @ornot_v16i8(<16 x i8> %z, <16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: ornot_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vornt q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %z, zeroinitializer
+  %y1 = xor <16 x i8> %y, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %a = or <16 x i8> %x, %y1
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %z
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fadd_v4f32(<4 x float> %z, <4 x float> %x, <4 x float> %y) {
+; CHECK-LABEL: fadd_v4f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.f32 eq, q0, zr
+; CHECK-NEXT:    vaddt.f32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = fcmp oeq <4 x float> %z, zeroinitializer
+  %a = fadd <4 x float> %x, %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %z
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fadd_v8f16(<8 x half> %z, <8 x half> %x, <8 x half> %y) {
+; CHECK-LABEL: fadd_v8f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.f16 eq, q0, zr
+; CHECK-NEXT:    vaddt.f16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = fcmp oeq <8 x half> %z, zeroinitializer
+  %a = fadd <8 x half> %x, %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %z
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fsub_v4f32(<4 x float> %z, <4 x float> %x, <4 x float> %y) {
+; CHECK-LABEL: fsub_v4f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.f32 eq, q0, zr
+; CHECK-NEXT:    vsubt.f32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = fcmp oeq <4 x float> %z, zeroinitializer
+  %a = fsub <4 x float> %x, %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %z
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fsub_v8f16(<8 x half> %z, <8 x half> %x, <8 x half> %y) {
+; CHECK-LABEL: fsub_v8f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.f16 eq, q0, zr
+; CHECK-NEXT:    vsubt.f16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = fcmp oeq <8 x half> %z, zeroinitializer
+  %a = fsub <8 x half> %x, %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %z
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fmul_v4f32(<4 x float> %z, <4 x float> %x, <4 x float> %y) {
+; CHECK-LABEL: fmul_v4f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.f32 eq, q0, zr
+; CHECK-NEXT:    vmult.f32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = fcmp oeq <4 x float> %z, zeroinitializer
+  %a = fmul <4 x float> %x, %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %z
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fmul_v8f16(<8 x half> %z, <8 x half> %x, <8 x half> %y) {
+; CHECK-LABEL: fmul_v8f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.f16 eq, q0, zr
+; CHECK-NEXT:    vmult.f16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = fcmp oeq <8 x half> %z, zeroinitializer
+  %a = fmul <8 x half> %x, %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %z
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @icmp_slt_v4i32(<4 x i32> %z, <4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: icmp_slt_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vmint.s32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %z, zeroinitializer
+  %a1 = icmp slt <4 x i32> %x, %y
+  %a = select <4 x i1> %a1, <4 x i32> %x, <4 x i32> %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %z
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @icmp_slt_v8i16(<8 x i16> %z, <8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: icmp_slt_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vmint.s16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %z, zeroinitializer
+  %a1 = icmp slt <8 x i16> %x, %y
+  %a = select <8 x i1> %a1, <8 x i16> %x, <8 x i16> %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %z
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @icmp_slt_v16i8(<16 x i8> %z, <16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: icmp_slt_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vmint.s8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %z, zeroinitializer
+  %a1 = icmp slt <16 x i8> %x, %y
+  %a = select <16 x i1> %a1, <16 x i8> %x, <16 x i8> %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %z
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @icmp_sgt_v4i32(<4 x i32> %z, <4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: icmp_sgt_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vmaxt.s32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %z, zeroinitializer
+  %a1 = icmp sgt <4 x i32> %x, %y
+  %a = select <4 x i1> %a1, <4 x i32> %x, <4 x i32> %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %z
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @icmp_sgt_v8i16(<8 x i16> %z, <8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: icmp_sgt_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vmaxt.s16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %z, zeroinitializer
+  %a1 = icmp sgt <8 x i16> %x, %y
+  %a = select <8 x i1> %a1, <8 x i16> %x, <8 x i16> %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %z
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @icmp_sgt_v16i8(<16 x i8> %z, <16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: icmp_sgt_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vmaxt.s8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %z, zeroinitializer
+  %a1 = icmp sgt <16 x i8> %x, %y
+  %a = select <16 x i1> %a1, <16 x i8> %x, <16 x i8> %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %z
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @icmp_ult_v4i32(<4 x i32> %z, <4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: icmp_ult_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vmint.u32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %z, zeroinitializer
+  %a1 = icmp ult <4 x i32> %x, %y
+  %a = select <4 x i1> %a1, <4 x i32> %x, <4 x i32> %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %z
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @icmp_ult_v8i16(<8 x i16> %z, <8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: icmp_ult_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vmint.u16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %z, zeroinitializer
+  %a1 = icmp ult <8 x i16> %x, %y
+  %a = select <8 x i1> %a1, <8 x i16> %x, <8 x i16> %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %z
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @icmp_ult_v16i8(<16 x i8> %z, <16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: icmp_ult_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vmint.u8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %z, zeroinitializer
+  %a1 = icmp ult <16 x i8> %x, %y
+  %a = select <16 x i1> %a1, <16 x i8> %x, <16 x i8> %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %z
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @icmp_ugt_v4i32(<4 x i32> %z, <4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: icmp_ugt_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vmaxt.u32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %z, zeroinitializer
+  %a1 = icmp ugt <4 x i32> %x, %y
+  %a = select <4 x i1> %a1, <4 x i32> %x, <4 x i32> %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %z
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @icmp_ugt_v8i16(<8 x i16> %z, <8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: icmp_ugt_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vmaxt.u16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %z, zeroinitializer
+  %a1 = icmp ugt <8 x i16> %x, %y
+  %a = select <8 x i1> %a1, <8 x i16> %x, <8 x i16> %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %z
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @icmp_ugt_v16i8(<16 x i8> %z, <16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: icmp_ugt_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vmaxt.u8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %z, zeroinitializer
+  %a1 = icmp ugt <16 x i8> %x, %y
+  %a = select <16 x i1> %a1, <16 x i8> %x, <16 x i8> %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %z
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fcmp_fast_olt_v4f32(<4 x float> %z, <4 x float> %x, <4 x float> %y) {
+; CHECK-LABEL: fcmp_fast_olt_v4f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.f32 eq, q0, zr
+; CHECK-NEXT:    vminnmt.f32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = fcmp oeq <4 x float> %z, zeroinitializer
+  %a1 = fcmp fast olt <4 x float> %x, %y
+  %a = select <4 x i1> %a1, <4 x float> %x, <4 x float> %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %z
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fcmp_fast_olt_v8f16(<8 x half> %z, <8 x half> %x, <8 x half> %y) {
+; CHECK-LABEL: fcmp_fast_olt_v8f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.f16 eq, q0, zr
+; CHECK-NEXT:    vminnmt.f16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = fcmp oeq <8 x half> %z, zeroinitializer
+  %a1 = fcmp fast olt <8 x half> %x, %y
+  %a = select <8 x i1> %a1, <8 x half> %x, <8 x half> %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %z
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fcmp_fast_ogt_v4f32(<4 x float> %z, <4 x float> %x, <4 x float> %y) {
+; CHECK-LABEL: fcmp_fast_ogt_v4f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.f32 eq, q0, zr
+; CHECK-NEXT:    vmaxnmt.f32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = fcmp oeq <4 x float> %z, zeroinitializer
+  %a1 = fcmp fast ogt <4 x float> %x, %y
+  %a = select <4 x i1> %a1, <4 x float> %x, <4 x float> %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %z
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fcmp_fast_ogt_v8f16(<8 x half> %z, <8 x half> %x, <8 x half> %y) {
+; CHECK-LABEL: fcmp_fast_ogt_v8f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.f16 eq, q0, zr
+; CHECK-NEXT:    vmaxnmt.f16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = fcmp oeq <8 x half> %z, zeroinitializer
+  %a1 = fcmp fast ogt <8 x half> %x, %y
+  %a = select <8 x i1> %a1, <8 x half> %x, <8 x half> %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %z
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sadd_sat_v4i32(<4 x i32> %z, <4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: sadd_sat_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vqaddt.s32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %z, zeroinitializer
+  %a = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %z
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sadd_sat_v8i16(<8 x i16> %z, <8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: sadd_sat_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vqaddt.s16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %z, zeroinitializer
+  %a = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %z
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @sadd_sat_v16i8(<16 x i8> %z, <16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: sadd_sat_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vqaddt.s8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %z, zeroinitializer
+  %a = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %z
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @uadd_sat_v4i32(<4 x i32> %z, <4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: uadd_sat_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vqaddt.u32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %z, zeroinitializer
+  %a = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %z
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @uadd_sat_v8i16(<8 x i16> %z, <8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: uadd_sat_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vqaddt.u16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %z, zeroinitializer
+  %a = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %z
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @uadd_sat_v16i8(<16 x i8> %z, <16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: uadd_sat_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vqaddt.u8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %z, zeroinitializer
+  %a = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %z
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ssub_sat_v4i32(<4 x i32> %z, <4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: ssub_sat_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vqsubt.s32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %z, zeroinitializer
+  %a = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %z
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ssub_sat_v8i16(<8 x i16> %z, <8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: ssub_sat_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vqsubt.s16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %z, zeroinitializer
+  %a = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %z
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @ssub_sat_v16i8(<16 x i8> %z, <16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: ssub_sat_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vqsubt.s8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %z, zeroinitializer
+  %a = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %z
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @usub_sat_v4i32(<4 x i32> %z, <4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: usub_sat_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vqsubt.u32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %z, zeroinitializer
+  %a = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %z
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @usub_sat_v8i16(<8 x i16> %z, <8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: usub_sat_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vqsubt.u16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %z, zeroinitializer
+  %a = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %z
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @usub_sat_v16i8(<16 x i8> %z, <16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: usub_sat_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vqsubt.u8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %z, zeroinitializer
+  %a = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %z
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @addqr_v4i32(<4 x i32> %z, <4 x i32> %x, i32 %y) {
+; CHECK-LABEL: addqr_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vaddt.i32 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %z, zeroinitializer
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = add <4 x i32> %x, %ys
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %z
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @addqr_v8i16(<8 x i16> %z, <8 x i16> %x, i16 %y) {
+; CHECK-LABEL: addqr_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vaddt.i16 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %z, zeroinitializer
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = add <8 x i16> %x, %ys
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %z
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @addqr_v16i8(<16 x i8> %z, <16 x i8> %x, i8 %y) {
+; CHECK-LABEL: addqr_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vaddt.i8 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %z, zeroinitializer
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = add <16 x i8> %x, %ys
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %z
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @subqr_v4i32(<4 x i32> %z, <4 x i32> %x, i32 %y) {
+; CHECK-LABEL: subqr_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vsubt.i32 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %z, zeroinitializer
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = sub <4 x i32> %x, %ys
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %z
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @subqr_v8i16(<8 x i16> %z, <8 x i16> %x, i16 %y) {
+; CHECK-LABEL: subqr_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vsubt.i16 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %z, zeroinitializer
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = sub <8 x i16> %x, %ys
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %z
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @subqr_v16i8(<16 x i8> %z, <16 x i8> %x, i8 %y) {
+; CHECK-LABEL: subqr_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vsubt.i8 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %z, zeroinitializer
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = sub <16 x i8> %x, %ys
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %z
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @mulqr_v4i32(<4 x i32> %z, <4 x i32> %x, i32 %y) {
+; CHECK-LABEL: mulqr_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vmult.i32 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %z, zeroinitializer
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = mul <4 x i32> %x, %ys
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %z
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @mulqr_v8i16(<8 x i16> %z, <8 x i16> %x, i16 %y) {
+; CHECK-LABEL: mulqr_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vmult.i16 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %z, zeroinitializer
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = mul <8 x i16> %x, %ys
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %z
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @mulqr_v16i8(<16 x i8> %z, <16 x i8> %x, i8 %y) {
+; CHECK-LABEL: mulqr_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vmult.i8 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %z, zeroinitializer
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = mul <16 x i8> %x, %ys
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %z
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @faddqr_v4f32(<4 x float> %z, <4 x float> %x, float %y) {
+; CHECK-LABEL: faddqr_v4f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vpt.f32 eq, q0, zr
+; CHECK-NEXT:    vaddt.f32 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = fcmp oeq <4 x float> %z, zeroinitializer
+  %i = insertelement <4 x float> undef, float %y, i32 0
+  %ys = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %a = fadd <4 x float> %x, %ys
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %z
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @faddqr_v8f16(<8 x half> %z, <8 x half> %x, half %y) {
+; CHECK-LABEL: faddqr_v8f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 r0, s8
+; CHECK-NEXT:    vpt.f16 eq, q0, zr
+; CHECK-NEXT:    vaddt.f16 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = fcmp oeq <8 x half> %z, zeroinitializer
+  %i = insertelement <8 x half> undef, half %y, i32 0
+  %ys = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %a = fadd <8 x half> %x, %ys
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %z
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fsubqr_v4f32(<4 x float> %z, <4 x float> %x, float %y) {
+; CHECK-LABEL: fsubqr_v4f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vpt.f32 eq, q0, zr
+; CHECK-NEXT:    vsubt.f32 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = fcmp oeq <4 x float> %z, zeroinitializer
+  %i = insertelement <4 x float> undef, float %y, i32 0
+  %ys = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %a = fsub <4 x float> %x, %ys
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %z
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fsubqr_v8f16(<8 x half> %z, <8 x half> %x, half %y) {
+; CHECK-LABEL: fsubqr_v8f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 r0, s8
+; CHECK-NEXT:    vpt.f16 eq, q0, zr
+; CHECK-NEXT:    vsubt.f16 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = fcmp oeq <8 x half> %z, zeroinitializer
+  %i = insertelement <8 x half> undef, half %y, i32 0
+  %ys = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %a = fsub <8 x half> %x, %ys
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %z
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fmulqr_v4f32(<4 x float> %z, <4 x float> %x, float %y) {
+; CHECK-LABEL: fmulqr_v4f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vpt.f32 eq, q0, zr
+; CHECK-NEXT:    vmult.f32 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = fcmp oeq <4 x float> %z, zeroinitializer
+  %i = insertelement <4 x float> undef, float %y, i32 0
+  %ys = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %a = fmul <4 x float> %x, %ys
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %z
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fmulqr_v8f16(<8 x half> %z, <8 x half> %x, half %y) {
+; CHECK-LABEL: fmulqr_v8f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 r0, s8
+; CHECK-NEXT:    vpt.f16 eq, q0, zr
+; CHECK-NEXT:    vmult.f16 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = fcmp oeq <8 x half> %z, zeroinitializer
+  %i = insertelement <8 x half> undef, half %y, i32 0
+  %ys = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %a = fmul <8 x half> %x, %ys
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %z
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sadd_satqr_v4i32(<4 x i32> %z, <4 x i32> %x, i32 %y) {
+; CHECK-LABEL: sadd_satqr_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vqaddt.s32 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %z, zeroinitializer
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %z
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sadd_satqr_v8i16(<8 x i16> %z, <8 x i16> %x, i16 %y) {
+; CHECK-LABEL: sadd_satqr_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vqaddt.s16 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %z, zeroinitializer
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %z
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @sadd_satqr_v16i8(<16 x i8> %z, <16 x i8> %x, i8 %y) {
+; CHECK-LABEL: sadd_satqr_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vqaddt.s8 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %z, zeroinitializer
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %z
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @uadd_satqr_v4i32(<4 x i32> %z, <4 x i32> %x, i32 %y) {
+; CHECK-LABEL: uadd_satqr_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vqaddt.u32 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %z, zeroinitializer
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %z
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @uadd_satqr_v8i16(<8 x i16> %z, <8 x i16> %x, i16 %y) {
+; CHECK-LABEL: uadd_satqr_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vqaddt.u16 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %z, zeroinitializer
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %z
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @uadd_satqr_v16i8(<16 x i8> %z, <16 x i8> %x, i8 %y) {
+; CHECK-LABEL: uadd_satqr_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vqaddt.u8 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %z, zeroinitializer
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %z
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ssub_satqr_v4i32(<4 x i32> %z, <4 x i32> %x, i32 %y) {
+; CHECK-LABEL: ssub_satqr_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vqsubt.s32 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %z, zeroinitializer
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %z
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ssub_satqr_v8i16(<8 x i16> %z, <8 x i16> %x, i16 %y) {
+; CHECK-LABEL: ssub_satqr_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vqsubt.s16 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %z, zeroinitializer
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %z
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @ssub_satqr_v16i8(<16 x i8> %z, <16 x i8> %x, i8 %y) {
+; CHECK-LABEL: ssub_satqr_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vqsubt.s8 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %z, zeroinitializer
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %z
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @usub_satqr_v4i32(<4 x i32> %z, <4 x i32> %x, i32 %y) {
+; CHECK-LABEL: usub_satqr_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vqsubt.u32 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %z, zeroinitializer
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %z
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @usub_satqr_v8i16(<8 x i16> %z, <8 x i16> %x, i16 %y) {
+; CHECK-LABEL: usub_satqr_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vqsubt.u16 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %z, zeroinitializer
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %z
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @usub_satqr_v16i8(<16 x i8> %z, <16 x i8> %x, i8 %y) {
+; CHECK-LABEL: usub_satqr_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vqsubt.u8 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %z, zeroinitializer
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %z
+  ret <16 x i8> %b
+}
+
+declare <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %src1, <16 x i8> %src2)
+declare <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %src1, <8 x i16> %src2)
+declare <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %src1, <4 x i32> %src2)
+declare <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %src1, <16 x i8> %src2)
+declare <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %src1, <8 x i16> %src2)
+declare <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %src1, <4 x i32> %src2)
+declare <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %src1, <16 x i8> %src2)
+declare <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %src1, <8 x i16> %src2)
+declare <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %src1, <4 x i32> %src2)
+declare <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %src1, <16 x i8> %src2)
+declare <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %src1, <8 x i16> %src2)
+declare <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %src1, <4 x i32> %src2)
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-selectop2.ll b/llvm/test/CodeGen/Thumb2/mve-pred-selectop2.ll
new file mode 100644
index 0000000000000..62271a250ac2e
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-selectop2.ll
@@ -0,0 +1,2675 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK
+
+define arm_aapcs_vfpcc <4 x i32> @add_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: add_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = add <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @add_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: add_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = add <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @add_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: add_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = add <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sub_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: sub_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.i32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = sub <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sub_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: sub_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.i16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = sub <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @sub_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: sub_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.i8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = sub <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @mul_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: mul_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = mul <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @mul_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: mul_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = mul <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @mul_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: mul_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = mul <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @and_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: and_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vandt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = and <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @and_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: and_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vandt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = and <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @and_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: and_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vandt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = and <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @or_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: or_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vorrt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = or <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @or_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: or_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vorrt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = or <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @or_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: or_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vorrt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = or <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @xor_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: xor_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    veort q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = xor <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @xor_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: xor_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    veort q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = xor <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @xor_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: xor_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    veort q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = xor <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @andnot_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: andnot_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vbict q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %y1 = xor <4 x i32> %y, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %a = and <4 x i32> %x, %y1
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @andnot_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: andnot_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vbict q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %y1 = xor <8 x i16> %y, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+  %a = and <8 x i16> %x, %y1
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @andnot_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: andnot_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vbict q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %y1 = xor <16 x i8> %y, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %a = and <16 x i8> %x, %y1
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ornot_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: ornot_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vornt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %y1 = xor <4 x i32> %y, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %a = or <4 x i32> %x, %y1
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ornot_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: ornot_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vornt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %y1 = xor <8 x i16> %y, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+  %a = or <8 x i16> %x, %y1
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @ornot_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: ornot_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vornt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %y1 = xor <16 x i8> %y, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %a = or <16 x i8> %x, %y1
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fadd_v4f32_x(<4 x float> %x, <4 x float> %y, i32 %n) {
+; CHECK-LABEL: fadd_v4f32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.f32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = fadd <4 x float> %x, %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %x
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fadd_v8f16_x(<8 x half> %x, <8 x half> %y, i32 %n) {
+; CHECK-LABEL: fadd_v8f16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.f16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = fadd <8 x half> %x, %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %x
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fsub_v4f32_x(<4 x float> %x, <4 x float> %y, i32 %n) {
+; CHECK-LABEL: fsub_v4f32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.f32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = fsub <4 x float> %x, %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %x
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fsub_v8f16_x(<8 x half> %x, <8 x half> %y, i32 %n) {
+; CHECK-LABEL: fsub_v8f16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.f16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = fsub <8 x half> %x, %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %x
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fmul_v4f32_x(<4 x float> %x, <4 x float> %y, i32 %n) {
+; CHECK-LABEL: fmul_v4f32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.f32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = fmul <4 x float> %x, %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %x
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fmul_v8f16_x(<8 x half> %x, <8 x half> %y, i32 %n) {
+; CHECK-LABEL: fmul_v8f16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.f16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = fmul <8 x half> %x, %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %x
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @icmp_slt_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: icmp_slt_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a1 = icmp slt <4 x i32> %x, %y
+  %a = select <4 x i1> %a1, <4 x i32> %x, <4 x i32> %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @icmp_slt_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: icmp_slt_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a1 = icmp slt <8 x i16> %x, %y
+  %a = select <8 x i1> %a1, <8 x i16> %x, <8 x i16> %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @icmp_slt_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: icmp_slt_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a1 = icmp slt <16 x i8> %x, %y
+  %a = select <16 x i1> %a1, <16 x i8> %x, <16 x i8> %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @icmp_sgt_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: icmp_sgt_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a1 = icmp sgt <4 x i32> %x, %y
+  %a = select <4 x i1> %a1, <4 x i32> %x, <4 x i32> %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @icmp_sgt_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: icmp_sgt_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a1 = icmp sgt <8 x i16> %x, %y
+  %a = select <8 x i1> %a1, <8 x i16> %x, <8 x i16> %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @icmp_sgt_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: icmp_sgt_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a1 = icmp sgt <16 x i8> %x, %y
+  %a = select <16 x i1> %a1, <16 x i8> %x, <16 x i8> %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @icmp_ult_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: icmp_ult_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a1 = icmp ult <4 x i32> %x, %y
+  %a = select <4 x i1> %a1, <4 x i32> %x, <4 x i32> %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @icmp_ult_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: icmp_ult_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a1 = icmp ult <8 x i16> %x, %y
+  %a = select <8 x i1> %a1, <8 x i16> %x, <8 x i16> %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @icmp_ult_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: icmp_ult_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a1 = icmp ult <16 x i8> %x, %y
+  %a = select <16 x i1> %a1, <16 x i8> %x, <16 x i8> %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @icmp_ugt_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: icmp_ugt_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a1 = icmp ugt <4 x i32> %x, %y
+  %a = select <4 x i1> %a1, <4 x i32> %x, <4 x i32> %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @icmp_ugt_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: icmp_ugt_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a1 = icmp ugt <8 x i16> %x, %y
+  %a = select <8 x i1> %a1, <8 x i16> %x, <8 x i16> %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @icmp_ugt_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: icmp_ugt_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a1 = icmp ugt <16 x i8> %x, %y
+  %a = select <16 x i1> %a1, <16 x i8> %x, <16 x i8> %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fcmp_fast_olt_v4f32_x(<4 x float> %x, <4 x float> %y, i32 %n) {
+; CHECK-LABEL: fcmp_fast_olt_v4f32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vminnmt.f32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a1 = fcmp fast olt <4 x float> %x, %y
+  %a = select <4 x i1> %a1, <4 x float> %x, <4 x float> %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %x
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fcmp_fast_olt_v8f16_x(<8 x half> %x, <8 x half> %y, i32 %n) {
+; CHECK-LABEL: fcmp_fast_olt_v8f16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vminnmt.f16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a1 = fcmp fast olt <8 x half> %x, %y
+  %a = select <8 x i1> %a1, <8 x half> %x, <8 x half> %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %x
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fcmp_fast_ogt_v4f32_x(<4 x float> %x, <4 x float> %y, i32 %n) {
+; CHECK-LABEL: fcmp_fast_ogt_v4f32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxnmt.f32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a1 = fcmp fast ogt <4 x float> %x, %y
+  %a = select <4 x i1> %a1, <4 x float> %x, <4 x float> %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %x
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fcmp_fast_ogt_v8f16_x(<8 x half> %x, <8 x half> %y, i32 %n) {
+; CHECK-LABEL: fcmp_fast_ogt_v8f16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxnmt.f16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a1 = fcmp fast ogt <8 x half> %x, %y
+  %a = select <8 x i1> %a1, <8 x half> %x, <8 x half> %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %x
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sadd_sat_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: sadd_sat_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sadd_sat_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: sadd_sat_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @sadd_sat_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: sadd_sat_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @uadd_sat_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: uadd_sat_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @uadd_sat_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: uadd_sat_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @uadd_sat_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: uadd_sat_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ssub_sat_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: ssub_sat_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ssub_sat_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: ssub_sat_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @ssub_sat_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: ssub_sat_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @usub_sat_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: usub_sat_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @usub_sat_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: usub_sat_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @usub_sat_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: usub_sat_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @addqr_v4i32_x(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: addqr_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i32 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = add <4 x i32> %x, %ys
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @addqr_v8i16_x(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: addqr_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i16 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = add <8 x i16> %x, %ys
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @addqr_v16i8_x(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: addqr_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i8 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = add <16 x i8> %x, %ys
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @subqr_v4i32_x(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: subqr_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.i32 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = sub <4 x i32> %x, %ys
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @subqr_v8i16_x(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: subqr_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.i16 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = sub <8 x i16> %x, %ys
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @subqr_v16i8_x(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: subqr_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.i8 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = sub <16 x i8> %x, %ys
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @mulqr_v4i32_x(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: mulqr_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i32 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = mul <4 x i32> %x, %ys
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @mulqr_v8i16_x(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: mulqr_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i16 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = mul <8 x i16> %x, %ys
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @mulqr_v16i8_x(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: mulqr_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i8 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = mul <16 x i8> %x, %ys
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @faddqr_v4f32_x(<4 x float> %x, float %y, i32 %n) {
+; CHECK-LABEL: faddqr_v4f32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.f32 q0, q0, r1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x float> undef, float %y, i32 0
+  %ys = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %a = fadd <4 x float> %x, %ys
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %x
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @faddqr_v8f16_x(<8 x half> %x, half %y, i32 %n) {
+; CHECK-LABEL: faddqr_v8f16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 r1, s4
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.f16 q0, q0, r1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x half> undef, half %y, i32 0
+  %ys = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %a = fadd <8 x half> %x, %ys
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %x
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fsubqr_v4f32_x(<4 x float> %x, float %y, i32 %n) {
+; CHECK-LABEL: fsubqr_v4f32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.f32 q0, q0, r1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x float> undef, float %y, i32 0
+  %ys = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %a = fsub <4 x float> %x, %ys
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %x
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fsubqr_v8f16_x(<8 x half> %x, half %y, i32 %n) {
+; CHECK-LABEL: fsubqr_v8f16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 r1, s4
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.f16 q0, q0, r1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x half> undef, half %y, i32 0
+  %ys = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %a = fsub <8 x half> %x, %ys
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %x
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fmulqr_v4f32_x(<4 x float> %x, float %y, i32 %n) {
+; CHECK-LABEL: fmulqr_v4f32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.f32 q0, q0, r1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x float> undef, float %y, i32 0
+  %ys = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %a = fmul <4 x float> %x, %ys
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %x
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fmulqr_v8f16_x(<8 x half> %x, half %y, i32 %n) {
+; CHECK-LABEL: fmulqr_v8f16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 r1, s4
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.f16 q0, q0, r1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x half> undef, half %y, i32 0
+  %ys = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %a = fmul <8 x half> %x, %ys
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %x
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sadd_satqr_v4i32_x(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: sadd_satqr_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.s32 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sadd_satqr_v8i16_x(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: sadd_satqr_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.s16 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @sadd_satqr_v16i8_x(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: sadd_satqr_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.s8 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @uadd_satqr_v4i32_x(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: uadd_satqr_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.u32 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @uadd_satqr_v8i16_x(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: uadd_satqr_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.u16 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @uadd_satqr_v16i8_x(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: uadd_satqr_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.u8 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ssub_satqr_v4i32_x(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: ssub_satqr_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.s32 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ssub_satqr_v8i16_x(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: ssub_satqr_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.s16 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @ssub_satqr_v16i8_x(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: ssub_satqr_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.s8 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @usub_satqr_v4i32_x(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: usub_satqr_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.u32 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @usub_satqr_v8i16_x(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: usub_satqr_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.u16 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @usub_satqr_v16i8_x(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: usub_satqr_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.u8 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @add_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: add_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i32 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = add <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @add_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: add_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i16 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = add <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @add_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: add_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i8 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = add <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sub_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: sub_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.i32 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = sub <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sub_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: sub_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.i16 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = sub <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @sub_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: sub_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.i8 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = sub <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @mul_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: mul_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i32 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = mul <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @mul_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: mul_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i16 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = mul <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @mul_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: mul_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i8 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = mul <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @and_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: and_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vandt q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = and <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @and_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: and_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vandt q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = and <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @and_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: and_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vandt q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = and <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @or_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: or_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vorrt q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = or <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @or_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: or_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vorrt q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = or <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @or_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: or_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vorrt q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = or <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @xor_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: xor_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    veort q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = xor <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @xor_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: xor_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    veort q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = xor <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @xor_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: xor_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    veort q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = xor <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @andnot_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: andnot_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vbict q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %y1 = xor <4 x i32> %y, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %a = and <4 x i32> %x, %y1
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @andnot_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: andnot_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vbict q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %y1 = xor <8 x i16> %y, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+  %a = and <8 x i16> %x, %y1
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @andnot_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: andnot_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vbict q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %y1 = xor <16 x i8> %y, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %a = and <16 x i8> %x, %y1
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ornot_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: ornot_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vornt q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %y1 = xor <4 x i32> %y, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %a = or <4 x i32> %x, %y1
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ornot_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: ornot_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vornt q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %y1 = xor <8 x i16> %y, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+  %a = or <8 x i16> %x, %y1
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @ornot_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: ornot_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vornt q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %y1 = xor <16 x i8> %y, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %a = or <16 x i8> %x, %y1
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fadd_v4f32_y(<4 x float> %x, <4 x float> %y, i32 %n) {
+; CHECK-LABEL: fadd_v4f32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.f32 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = fadd <4 x float> %x, %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %y
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fadd_v8f16_y(<8 x half> %x, <8 x half> %y, i32 %n) {
+; CHECK-LABEL: fadd_v8f16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.f16 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = fadd <8 x half> %x, %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %y
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fsub_v4f32_y(<4 x float> %x, <4 x float> %y, i32 %n) {
+; CHECK-LABEL: fsub_v4f32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.f32 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = fsub <4 x float> %x, %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %y
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fsub_v8f16_y(<8 x half> %x, <8 x half> %y, i32 %n) {
+; CHECK-LABEL: fsub_v8f16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.f16 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = fsub <8 x half> %x, %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %y
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fmul_v4f32_y(<4 x float> %x, <4 x float> %y, i32 %n) {
+; CHECK-LABEL: fmul_v4f32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.f32 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = fmul <4 x float> %x, %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %y
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fmul_v8f16_y(<8 x half> %x, <8 x half> %y, i32 %n) {
+; CHECK-LABEL: fmul_v8f16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.f16 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = fmul <8 x half> %x, %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %y
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @icmp_slt_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: icmp_slt_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.s32 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a1 = icmp slt <4 x i32> %x, %y
+  %a = select <4 x i1> %a1, <4 x i32> %x, <4 x i32> %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @icmp_slt_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: icmp_slt_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.s16 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a1 = icmp slt <8 x i16> %x, %y
+  %a = select <8 x i1> %a1, <8 x i16> %x, <8 x i16> %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @icmp_slt_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: icmp_slt_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.s8 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a1 = icmp slt <16 x i8> %x, %y
+  %a = select <16 x i1> %a1, <16 x i8> %x, <16 x i8> %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @icmp_sgt_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: icmp_sgt_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.s32 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a1 = icmp sgt <4 x i32> %x, %y
+  %a = select <4 x i1> %a1, <4 x i32> %x, <4 x i32> %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @icmp_sgt_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: icmp_sgt_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.s16 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a1 = icmp sgt <8 x i16> %x, %y
+  %a = select <8 x i1> %a1, <8 x i16> %x, <8 x i16> %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @icmp_sgt_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: icmp_sgt_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.s8 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a1 = icmp sgt <16 x i8> %x, %y
+  %a = select <16 x i1> %a1, <16 x i8> %x, <16 x i8> %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @icmp_ult_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: icmp_ult_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.u32 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a1 = icmp ult <4 x i32> %x, %y
+  %a = select <4 x i1> %a1, <4 x i32> %x, <4 x i32> %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @icmp_ult_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: icmp_ult_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.u16 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a1 = icmp ult <8 x i16> %x, %y
+  %a = select <8 x i1> %a1, <8 x i16> %x, <8 x i16> %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @icmp_ult_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: icmp_ult_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.u8 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a1 = icmp ult <16 x i8> %x, %y
+  %a = select <16 x i1> %a1, <16 x i8> %x, <16 x i8> %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @icmp_ugt_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: icmp_ugt_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.u32 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a1 = icmp ugt <4 x i32> %x, %y
+  %a = select <4 x i1> %a1, <4 x i32> %x, <4 x i32> %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @icmp_ugt_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: icmp_ugt_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.u16 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a1 = icmp ugt <8 x i16> %x, %y
+  %a = select <8 x i1> %a1, <8 x i16> %x, <8 x i16> %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @icmp_ugt_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: icmp_ugt_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.u8 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a1 = icmp ugt <16 x i8> %x, %y
+  %a = select <16 x i1> %a1, <16 x i8> %x, <16 x i8> %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fcmp_fast_olt_v4f32_y(<4 x float> %x, <4 x float> %y, i32 %n) {
+; CHECK-LABEL: fcmp_fast_olt_v4f32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vminnmt.f32 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a1 = fcmp fast olt <4 x float> %x, %y
+  %a = select <4 x i1> %a1, <4 x float> %x, <4 x float> %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %y
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fcmp_fast_olt_v8f16_y(<8 x half> %x, <8 x half> %y, i32 %n) {
+; CHECK-LABEL: fcmp_fast_olt_v8f16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vminnmt.f16 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a1 = fcmp fast olt <8 x half> %x, %y
+  %a = select <8 x i1> %a1, <8 x half> %x, <8 x half> %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %y
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fcmp_fast_ogt_v4f32_y(<4 x float> %x, <4 x float> %y, i32 %n) {
+; CHECK-LABEL: fcmp_fast_ogt_v4f32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxnmt.f32 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a1 = fcmp fast ogt <4 x float> %x, %y
+  %a = select <4 x i1> %a1, <4 x float> %x, <4 x float> %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %y
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fcmp_fast_ogt_v8f16_y(<8 x half> %x, <8 x half> %y, i32 %n) {
+; CHECK-LABEL: fcmp_fast_ogt_v8f16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxnmt.f16 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a1 = fcmp fast ogt <8 x half> %x, %y
+  %a = select <8 x i1> %a1, <8 x half> %x, <8 x half> %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %y
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sadd_sat_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: sadd_sat_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.s32 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sadd_sat_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: sadd_sat_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.s16 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @sadd_sat_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: sadd_sat_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.s8 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @uadd_sat_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: uadd_sat_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.u32 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @uadd_sat_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: uadd_sat_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.u16 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @uadd_sat_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: uadd_sat_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.u8 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ssub_sat_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: ssub_sat_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.s32 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ssub_sat_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: ssub_sat_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.s16 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @ssub_sat_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: ssub_sat_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.s8 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @usub_sat_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: usub_sat_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.u32 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @usub_sat_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: usub_sat_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.u16 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @usub_sat_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: usub_sat_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.u8 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @addqr_v4i32_y(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: addqr_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i32 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = add <4 x i32> %x, %ys
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %ys
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @addqr_v8i16_y(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: addqr_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.16 q1, r0
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i16 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = add <8 x i16> %x, %ys
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %ys
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @addqr_v16i8_y(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: addqr_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.8 q1, r0
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i8 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = add <16 x i8> %x, %ys
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %ys
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @subqr_v4i32_y(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: subqr_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.i32 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = sub <4 x i32> %x, %ys
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %ys
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @subqr_v8i16_y(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: subqr_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.16 q1, r0
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.i16 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = sub <8 x i16> %x, %ys
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %ys
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @subqr_v16i8_y(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: subqr_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.8 q1, r0
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.i8 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = sub <16 x i8> %x, %ys
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %ys
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @mulqr_v4i32_y(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: mulqr_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i32 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = mul <4 x i32> %x, %ys
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %ys
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @mulqr_v8i16_y(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: mulqr_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.16 q1, r0
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i16 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = mul <8 x i16> %x, %ys
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %ys
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @mulqr_v16i8_y(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: mulqr_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.8 q1, r0
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i8 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = mul <16 x i8> %x, %ys
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %ys
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @faddqr_v4f32_y(<4 x float> %x, float %y, i32 %n) {
+; CHECK-LABEL: faddqr_v4f32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vdup.32 q1, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.f32 q1, q0, r1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x float> undef, float %y, i32 0
+  %ys = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %a = fadd <4 x float> %x, %ys
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %ys
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @faddqr_v8f16_y(<8 x half> %x, half %y, i32 %n) {
+; CHECK-LABEL: faddqr_v8f16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 r1, s4
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vdup.16 q1, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.f16 q1, q0, r1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x half> undef, half %y, i32 0
+  %ys = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %a = fadd <8 x half> %x, %ys
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %ys
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fsubqr_v4f32_y(<4 x float> %x, float %y, i32 %n) {
+; CHECK-LABEL: fsubqr_v4f32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vdup.32 q1, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.f32 q1, q0, r1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x float> undef, float %y, i32 0
+  %ys = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %a = fsub <4 x float> %x, %ys
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %ys
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fsubqr_v8f16_y(<8 x half> %x, half %y, i32 %n) {
+; CHECK-LABEL: fsubqr_v8f16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 r1, s4
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vdup.16 q1, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.f16 q1, q0, r1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x half> undef, half %y, i32 0
+  %ys = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %a = fsub <8 x half> %x, %ys
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %ys
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fmulqr_v4f32_y(<4 x float> %x, float %y, i32 %n) {
+; CHECK-LABEL: fmulqr_v4f32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vdup.32 q1, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.f32 q1, q0, r1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x float> undef, float %y, i32 0
+  %ys = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %a = fmul <4 x float> %x, %ys
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %ys
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fmulqr_v8f16_y(<8 x half> %x, half %y, i32 %n) {
+; CHECK-LABEL: fmulqr_v8f16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 r1, s4
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vdup.16 q1, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.f16 q1, q0, r1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x half> undef, half %y, i32 0
+  %ys = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %a = fmul <8 x half> %x, %ys
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %ys
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sadd_satqr_v4i32_y(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: sadd_satqr_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.s32 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %ys
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sadd_satqr_v8i16_y(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: sadd_satqr_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.16 q1, r0
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.s16 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %ys
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @sadd_satqr_v16i8_y(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: sadd_satqr_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.8 q1, r0
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.s8 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %ys
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @uadd_satqr_v4i32_y(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: uadd_satqr_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.u32 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %ys
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @uadd_satqr_v8i16_y(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: uadd_satqr_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.16 q1, r0
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.u16 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %ys
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @uadd_satqr_v16i8_y(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: uadd_satqr_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.8 q1, r0
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.u8 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %ys
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ssub_satqr_v4i32_y(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: ssub_satqr_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.s32 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %ys
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ssub_satqr_v8i16_y(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: ssub_satqr_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.16 q1, r0
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.s16 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %ys
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @ssub_satqr_v16i8_y(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: ssub_satqr_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.8 q1, r0
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.s8 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %ys
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @usub_satqr_v4i32_y(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: usub_satqr_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.u32 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %ys
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @usub_satqr_v8i16_y(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: usub_satqr_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.16 q1, r0
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.u16 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %ys
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @usub_satqr_v16i8_y(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: usub_satqr_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.8 q1, r0
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.u8 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %ys
+  ret <16 x i8> %b
+}
+
+declare <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32>, <4 x i32>)
+declare <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32>, <4 x i32>)
+declare <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32>, <4 x i32>)
+declare <16 x i8> @llvm.usub.sat.v16i8(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.usub.sat.v8i16(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.usub.sat.v4i32(<4 x i32>, <4 x i32>)
+
+declare <16 x i1> @llvm.arm.mve.vctp8(i32)
+declare <8 x i1> @llvm.arm.mve.vctp16(i32)
+declare <4 x i1> @llvm.arm.mve.vctp32(i32)
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-selectop3.ll b/llvm/test/CodeGen/Thumb2/mve-pred-selectop3.ll
new file mode 100644
index 0000000000000..df92d30da6af4
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-selectop3.ll
@@ -0,0 +1,2681 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK
+
+define arm_aapcs_vfpcc <4 x i32> @add_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: add_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = select <4 x i1> %c, <4 x i32> %y, <4 x i32> zeroinitializer
+  %b = add <4 x i32> %a, %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @add_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: add_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = select <8 x i1> %c, <8 x i16> %y, <8 x i16> zeroinitializer
+  %b = add <8 x i16> %a, %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @add_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: add_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = select <16 x i1> %c, <16 x i8> %y, <16 x i8> zeroinitializer
+  %b = add <16 x i8> %a, %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sub_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: sub_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.i32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = select <4 x i1> %c, <4 x i32> %y, <4 x i32> zeroinitializer
+  %b = sub <4 x i32> %x, %a
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sub_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: sub_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.i16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = select <8 x i1> %c, <8 x i16> %y, <8 x i16> zeroinitializer
+  %b = sub <8 x i16> %x, %a
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @sub_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: sub_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.i8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = select <16 x i1> %c, <16 x i8> %y, <16 x i8> zeroinitializer
+  %b = sub <16 x i8> %x, %a
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @mul_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: mul_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = select <4 x i1> %c, <4 x i32> %y, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %b = mul <4 x i32> %a, %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @mul_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: mul_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = select <8 x i1> %c, <8 x i16> %y, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %b = mul <8 x i16> %a, %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @mul_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: mul_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = select <16 x i1> %c, <16 x i8> %y, <16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+  %b = mul <16 x i8> %a, %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @and_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: and_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vandt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = select <4 x i1> %c, <4 x i32> %y, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
+  %b = and <4 x i32> %a, %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @and_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: and_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vandt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = select <8 x i1> %c, <8 x i16> %y, <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+  %b = and <8 x i16> %a, %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @and_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: and_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vandt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = select <16 x i1> %c, <16 x i8> %y, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %b = and <16 x i8> %a, %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @or_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: or_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vorrt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = select <4 x i1> %c, <4 x i32> %y, <4 x i32> zeroinitializer
+  %b = or <4 x i32> %a, %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @or_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: or_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vorrt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = select <8 x i1> %c, <8 x i16> %y, <8 x i16> zeroinitializer
+  %b = or <8 x i16> %a, %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @or_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: or_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vorrt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = select <16 x i1> %c, <16 x i8> %y, <16 x i8> zeroinitializer
+  %b = or <16 x i8> %a, %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @xor_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: xor_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    veort q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = select <4 x i1> %c, <4 x i32> %y, <4 x i32> zeroinitializer
+  %b = xor <4 x i32> %a, %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @xor_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: xor_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    veort q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = select <8 x i1> %c, <8 x i16> %y, <8 x i16> zeroinitializer
+  %b = xor <8 x i16> %a, %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @xor_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: xor_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    veort q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = select <16 x i1> %c, <16 x i8> %y, <16 x i8> zeroinitializer
+  %b = xor <16 x i8> %a, %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @andnot_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: andnot_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmvn q1, q1
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vandt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %y1 = xor <4 x i32> %y, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %a = select <4 x i1> %c, <4 x i32> %y1, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
+  %b = and <4 x i32> %a, %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @andnot_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: andnot_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmvn q1, q1
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vandt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %y1 = xor <8 x i16> %y, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+  %a = select <8 x i1> %c, <8 x i16> %y1, <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+  %b = and <8 x i16> %a, %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @andnot_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: andnot_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmvn q1, q1
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vandt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %y1 = xor <16 x i8> %y, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %a = select <16 x i1> %c, <16 x i8> %y1, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %b = and <16 x i8> %a, %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ornot_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: ornot_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmvn q1, q1
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vorrt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %y1 = xor <4 x i32> %y, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %a = select <4 x i1> %c, <4 x i32> %y1, <4 x i32> zeroinitializer
+  %b = or <4 x i32> %a, %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ornot_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: ornot_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmvn q1, q1
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vorrt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %y1 = xor <8 x i16> %y, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+  %a = select <8 x i1> %c, <8 x i16> %y1, <8 x i16> zeroinitializer
+  %b = or <8 x i16> %a, %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @ornot_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: ornot_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmvn q1, q1
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vorrt q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %y1 = xor <16 x i8> %y, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %a = select <16 x i1> %c, <16 x i8> %y1, <16 x i8> zeroinitializer
+  %b = or <16 x i8> %a, %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fadd_v4f32_x(<4 x float> %x, <4 x float> %y, i32 %n) {
+; CHECK-LABEL: fadd_v4f32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.f32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = fadd <4 x float> %x, %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %x
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fadd_v8f16_x(<8 x half> %x, <8 x half> %y, i32 %n) {
+; CHECK-LABEL: fadd_v8f16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.f16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = fadd <8 x half> %x, %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %x
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fsub_v4f32_x(<4 x float> %x, <4 x float> %y, i32 %n) {
+; CHECK-LABEL: fsub_v4f32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.f32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = fsub <4 x float> %x, %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %x
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fsub_v8f16_x(<8 x half> %x, <8 x half> %y, i32 %n) {
+; CHECK-LABEL: fsub_v8f16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.f16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = fsub <8 x half> %x, %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %x
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fmul_v4f32_x(<4 x float> %x, <4 x float> %y, i32 %n) {
+; CHECK-LABEL: fmul_v4f32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.f32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = fmul <4 x float> %x, %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %x
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fmul_v8f16_x(<8 x half> %x, <8 x half> %y, i32 %n) {
+; CHECK-LABEL: fmul_v8f16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.f16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = fmul <8 x half> %x, %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %x
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @icmp_slt_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: icmp_slt_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a1 = icmp slt <4 x i32> %x, %y
+  %a = select <4 x i1> %a1, <4 x i32> %x, <4 x i32> %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @icmp_slt_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: icmp_slt_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a1 = icmp slt <8 x i16> %x, %y
+  %a = select <8 x i1> %a1, <8 x i16> %x, <8 x i16> %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @icmp_slt_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: icmp_slt_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a1 = icmp slt <16 x i8> %x, %y
+  %a = select <16 x i1> %a1, <16 x i8> %x, <16 x i8> %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @icmp_sgt_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: icmp_sgt_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a1 = icmp sgt <4 x i32> %x, %y
+  %a = select <4 x i1> %a1, <4 x i32> %x, <4 x i32> %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @icmp_sgt_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: icmp_sgt_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a1 = icmp sgt <8 x i16> %x, %y
+  %a = select <8 x i1> %a1, <8 x i16> %x, <8 x i16> %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @icmp_sgt_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: icmp_sgt_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a1 = icmp sgt <16 x i8> %x, %y
+  %a = select <16 x i1> %a1, <16 x i8> %x, <16 x i8> %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @icmp_ult_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: icmp_ult_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a1 = icmp ult <4 x i32> %x, %y
+  %a = select <4 x i1> %a1, <4 x i32> %x, <4 x i32> %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @icmp_ult_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: icmp_ult_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a1 = icmp ult <8 x i16> %x, %y
+  %a = select <8 x i1> %a1, <8 x i16> %x, <8 x i16> %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @icmp_ult_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: icmp_ult_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a1 = icmp ult <16 x i8> %x, %y
+  %a = select <16 x i1> %a1, <16 x i8> %x, <16 x i8> %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @icmp_ugt_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: icmp_ugt_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a1 = icmp ugt <4 x i32> %x, %y
+  %a = select <4 x i1> %a1, <4 x i32> %x, <4 x i32> %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @icmp_ugt_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: icmp_ugt_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a1 = icmp ugt <8 x i16> %x, %y
+  %a = select <8 x i1> %a1, <8 x i16> %x, <8 x i16> %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @icmp_ugt_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: icmp_ugt_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a1 = icmp ugt <16 x i8> %x, %y
+  %a = select <16 x i1> %a1, <16 x i8> %x, <16 x i8> %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fcmp_fast_olt_v4f32_x(<4 x float> %x, <4 x float> %y, i32 %n) {
+; CHECK-LABEL: fcmp_fast_olt_v4f32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vminnmt.f32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a1 = fcmp fast olt <4 x float> %x, %y
+  %a = select <4 x i1> %a1, <4 x float> %x, <4 x float> %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %x
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fcmp_fast_olt_v8f16_x(<8 x half> %x, <8 x half> %y, i32 %n) {
+; CHECK-LABEL: fcmp_fast_olt_v8f16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vminnmt.f16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a1 = fcmp fast olt <8 x half> %x, %y
+  %a = select <8 x i1> %a1, <8 x half> %x, <8 x half> %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %x
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fcmp_fast_ogt_v4f32_x(<4 x float> %x, <4 x float> %y, i32 %n) {
+; CHECK-LABEL: fcmp_fast_ogt_v4f32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxnmt.f32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a1 = fcmp fast ogt <4 x float> %x, %y
+  %a = select <4 x i1> %a1, <4 x float> %x, <4 x float> %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %x
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fcmp_fast_ogt_v8f16_x(<8 x half> %x, <8 x half> %y, i32 %n) {
+; CHECK-LABEL: fcmp_fast_ogt_v8f16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxnmt.f16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a1 = fcmp fast ogt <8 x half> %x, %y
+  %a = select <8 x i1> %a1, <8 x half> %x, <8 x half> %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %x
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sadd_sat_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: sadd_sat_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sadd_sat_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: sadd_sat_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @sadd_sat_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: sadd_sat_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @uadd_sat_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: uadd_sat_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @uadd_sat_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: uadd_sat_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @uadd_sat_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: uadd_sat_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ssub_sat_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: ssub_sat_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ssub_sat_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: ssub_sat_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @ssub_sat_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: ssub_sat_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @usub_sat_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: usub_sat_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @usub_sat_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: usub_sat_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @usub_sat_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: usub_sat_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @addqr_v4i32_x(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: addqr_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i32 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = select <4 x i1> %c, <4 x i32> %ys, <4 x i32> zeroinitializer
+  %b = add <4 x i32> %a, %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @addqr_v8i16_x(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: addqr_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i16 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = select <8 x i1> %c, <8 x i16> %ys, <8 x i16> zeroinitializer
+  %b = add <8 x i16> %a, %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @addqr_v16i8_x(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: addqr_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i8 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = select <16 x i1> %c, <16 x i8> %ys, <16 x i8> zeroinitializer
+  %b = add <16 x i8> %a, %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @subqr_v4i32_x(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: subqr_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.i32 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = select <4 x i1> %c, <4 x i32> %ys, <4 x i32> zeroinitializer
+  %b = sub <4 x i32> %x, %a
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @subqr_v8i16_x(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: subqr_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.i16 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = select <8 x i1> %c, <8 x i16> %ys, <8 x i16> zeroinitializer
+  %b = sub <8 x i16> %x, %a
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @subqr_v16i8_x(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: subqr_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.i8 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = select <16 x i1> %c, <16 x i8> %ys, <16 x i8> zeroinitializer
+  %b = sub <16 x i8> %x, %a
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @mulqr_v4i32_x(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: mulqr_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i32 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = select <4 x i1> %c, <4 x i32> %ys, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %b = mul <4 x i32> %a, %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @mulqr_v8i16_x(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: mulqr_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i16 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = select <8 x i1> %c, <8 x i16> %ys, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %b = mul <8 x i16> %a, %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @mulqr_v16i8_x(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: mulqr_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i8 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = select <16 x i1> %c, <16 x i8> %ys, <16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+  %b = mul <16 x i8> %a, %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @faddqr_v4f32_x(<4 x float> %x, float %y, i32 %n) {
+; CHECK-LABEL: faddqr_v4f32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.f32 q0, q0, r1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x float> undef, float %y, i32 0
+  %ys = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %a = fadd <4 x float> %ys, %x
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %x
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @faddqr_v8f16_x(<8 x half> %x, half %y, i32 %n) {
+; CHECK-LABEL: faddqr_v8f16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 r1, s4
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.f16 q0, q0, r1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x half> undef, half %y, i32 0
+  %ys = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %a = fadd <8 x half> %ys, %x
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %x
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fsubqr_v4f32_x(<4 x float> %x, float %y, i32 %n) {
+; CHECK-LABEL: fsubqr_v4f32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.f32 q0, q0, r1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x float> undef, float %y, i32 0
+  %ys = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %a = fsub <4 x float> %x, %ys
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %x
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fsubqr_v8f16_x(<8 x half> %x, half %y, i32 %n) {
+; CHECK-LABEL: fsubqr_v8f16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 r1, s4
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.f16 q0, q0, r1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x half> undef, half %y, i32 0
+  %ys = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %a = fsub <8 x half> %x, %ys
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %x
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fmulqr_v4f32_x(<4 x float> %x, float %y, i32 %n) {
+; CHECK-LABEL: fmulqr_v4f32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.f32 q0, q0, r1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x float> undef, float %y, i32 0
+  %ys = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %a = fmul <4 x float> %ys, %x
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %x
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fmulqr_v8f16_x(<8 x half> %x, half %y, i32 %n) {
+; CHECK-LABEL: fmulqr_v8f16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 r1, s4
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.f16 q0, q0, r1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x half> undef, half %y, i32 0
+  %ys = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %a = fmul <8 x half> %ys, %x
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %x
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sadd_satqr_v4i32_x(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: sadd_satqr_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.s32 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sadd_satqr_v8i16_x(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: sadd_satqr_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.s16 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @sadd_satqr_v16i8_x(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: sadd_satqr_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.s8 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @uadd_satqr_v4i32_x(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: uadd_satqr_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.u32 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @uadd_satqr_v8i16_x(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: uadd_satqr_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.u16 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @uadd_satqr_v16i8_x(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: uadd_satqr_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.u8 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ssub_satqr_v4i32_x(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: ssub_satqr_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.s32 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ssub_satqr_v8i16_x(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: ssub_satqr_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.s16 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @ssub_satqr_v16i8_x(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: ssub_satqr_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.s8 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @usub_satqr_v4i32_x(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: usub_satqr_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.u32 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @usub_satqr_v8i16_x(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: usub_satqr_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.u16 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @usub_satqr_v16i8_x(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: usub_satqr_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.u8 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @add_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: add_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i32 q1, q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = select <4 x i1> %c, <4 x i32> %x, <4 x i32> zeroinitializer
+  %b = add <4 x i32> %a, %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @add_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: add_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i16 q1, q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = select <8 x i1> %c, <8 x i16> %x, <8 x i16> zeroinitializer
+  %b = add <8 x i16> %a, %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @add_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: add_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i8 q1, q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = select <16 x i1> %c, <16 x i8> %x, <16 x i8> zeroinitializer
+  %b = add <16 x i8> %a, %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sub_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: sub_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.i32 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = sub <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sub_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: sub_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.i16 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = sub <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @sub_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: sub_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.i8 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = sub <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @mul_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: mul_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i32 q1, q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = select <4 x i1> %c, <4 x i32> %x, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %b = mul <4 x i32> %a, %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @mul_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: mul_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i16 q1, q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = select <8 x i1> %c, <8 x i16> %x, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %b = mul <8 x i16> %a, %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @mul_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: mul_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i8 q1, q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = select <16 x i1> %c, <16 x i8> %x, <16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+  %b = mul <16 x i8> %a, %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @and_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: and_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vandt q1, q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = select <4 x i1> %c, <4 x i32> %x, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
+  %b = and <4 x i32> %a, %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @and_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: and_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vandt q1, q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = select <8 x i1> %c, <8 x i16> %x, <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+  %b = and <8 x i16> %a, %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @and_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: and_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vandt q1, q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = select <16 x i1> %c, <16 x i8> %x, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %b = and <16 x i8> %a, %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @or_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: or_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vorrt q1, q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = select <4 x i1> %c, <4 x i32> %x, <4 x i32> zeroinitializer
+  %b = or <4 x i32> %a, %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @or_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: or_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vorrt q1, q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = select <8 x i1> %c, <8 x i16> %x, <8 x i16> zeroinitializer
+  %b = or <8 x i16> %a, %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @or_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: or_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vorrt q1, q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = select <16 x i1> %c, <16 x i8> %x, <16 x i8> zeroinitializer
+  %b = or <16 x i8> %a, %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @xor_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: xor_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    veort q1, q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = select <4 x i1> %c, <4 x i32> %x, <4 x i32> zeroinitializer
+  %b = xor <4 x i32> %a, %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @xor_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: xor_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    veort q1, q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = select <8 x i1> %c, <8 x i16> %x, <8 x i16> zeroinitializer
+  %b = xor <8 x i16> %a, %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @xor_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: xor_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    veort q1, q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = select <16 x i1> %c, <16 x i8> %x, <16 x i8> zeroinitializer
+  %b = xor <16 x i8> %a, %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @andnot_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: andnot_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vbict q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %y1 = xor <4 x i32> %y, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %a = and <4 x i32> %y1, %x
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @andnot_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: andnot_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vbict q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %y1 = xor <8 x i16> %y, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+  %a = and <8 x i16> %y1, %x
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @andnot_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: andnot_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vbict q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %y1 = xor <16 x i8> %y, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %a = and <16 x i8> %y1, %x
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ornot_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: ornot_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vornt q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %y1 = xor <4 x i32> %y, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %a = or <4 x i32> %y1, %x
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ornot_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: ornot_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vornt q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %y1 = xor <8 x i16> %y, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+  %a = or <8 x i16> %y1, %x
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @ornot_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: ornot_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vornt q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %y1 = xor <16 x i8> %y, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %a = or <16 x i8> %y1, %x
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fadd_v4f32_y(<4 x float> %x, <4 x float> %y, i32 %n) {
+; CHECK-LABEL: fadd_v4f32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.f32 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = fadd <4 x float> %x, %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %y
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fadd_v8f16_y(<8 x half> %x, <8 x half> %y, i32 %n) {
+; CHECK-LABEL: fadd_v8f16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.f16 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = fadd <8 x half> %x, %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %y
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fsub_v4f32_y(<4 x float> %x, <4 x float> %y, i32 %n) {
+; CHECK-LABEL: fsub_v4f32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.f32 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = fsub <4 x float> %x, %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %y
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fsub_v8f16_y(<8 x half> %x, <8 x half> %y, i32 %n) {
+; CHECK-LABEL: fsub_v8f16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.f16 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = fsub <8 x half> %x, %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %y
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fmul_v4f32_y(<4 x float> %x, <4 x float> %y, i32 %n) {
+; CHECK-LABEL: fmul_v4f32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.f32 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = fmul <4 x float> %x, %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %y
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fmul_v8f16_y(<8 x half> %x, <8 x half> %y, i32 %n) {
+; CHECK-LABEL: fmul_v8f16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.f16 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = fmul <8 x half> %x, %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %y
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @icmp_slt_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: icmp_slt_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.s32 gt, q1, q0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a1 = icmp slt <4 x i32> %x, %y
+  %0 = and <4 x i1> %c, %a1
+  %b = select <4 x i1> %0, <4 x i32> %x, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @icmp_slt_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: icmp_slt_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.s16 gt, q1, q0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a1 = icmp slt <8 x i16> %x, %y
+  %0 = and <8 x i1> %c, %a1
+  %b = select <8 x i1> %0, <8 x i16> %x, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @icmp_slt_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: icmp_slt_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.s8 gt, q1, q0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a1 = icmp slt <16 x i8> %x, %y
+  %0 = and <16 x i1> %c, %a1
+  %b = select <16 x i1> %0, <16 x i8> %x, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @icmp_sgt_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: icmp_sgt_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.s32 gt, q0, q1
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a1 = icmp sgt <4 x i32> %x, %y
+  %0 = and <4 x i1> %c, %a1
+  %b = select <4 x i1> %0, <4 x i32> %x, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @icmp_sgt_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: icmp_sgt_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.s16 gt, q0, q1
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a1 = icmp sgt <8 x i16> %x, %y
+  %0 = and <8 x i1> %c, %a1
+  %b = select <8 x i1> %0, <8 x i16> %x, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @icmp_sgt_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: icmp_sgt_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.s8 gt, q0, q1
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a1 = icmp sgt <16 x i8> %x, %y
+  %0 = and <16 x i1> %c, %a1
+  %b = select <16 x i1> %0, <16 x i8> %x, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @icmp_ult_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: icmp_ult_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.u32 hi, q1, q0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a1 = icmp ult <4 x i32> %x, %y
+  %0 = and <4 x i1> %c, %a1
+  %b = select <4 x i1> %0, <4 x i32> %x, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @icmp_ult_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: icmp_ult_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.u16 hi, q1, q0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a1 = icmp ult <8 x i16> %x, %y
+  %0 = and <8 x i1> %c, %a1
+  %b = select <8 x i1> %0, <8 x i16> %x, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @icmp_ult_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: icmp_ult_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.u8 hi, q1, q0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a1 = icmp ult <16 x i8> %x, %y
+  %0 = and <16 x i1> %c, %a1
+  %b = select <16 x i1> %0, <16 x i8> %x, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @icmp_ugt_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: icmp_ugt_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.u32 hi, q0, q1
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a1 = icmp ugt <4 x i32> %x, %y
+  %0 = and <4 x i1> %c, %a1
+  %b = select <4 x i1> %0, <4 x i32> %x, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @icmp_ugt_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: icmp_ugt_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.u16 hi, q0, q1
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a1 = icmp ugt <8 x i16> %x, %y
+  %0 = and <8 x i1> %c, %a1
+  %b = select <8 x i1> %0, <8 x i16> %x, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @icmp_ugt_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: icmp_ugt_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.u8 hi, q0, q1
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a1 = icmp ugt <16 x i8> %x, %y
+  %0 = and <16 x i1> %c, %a1
+  %b = select <16 x i1> %0, <16 x i8> %x, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fcmp_fast_olt_v4f32_y(<4 x float> %x, <4 x float> %y, i32 %n) {
+; CHECK-LABEL: fcmp_fast_olt_v4f32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.f32 gt, q1, q0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a1 = fcmp fast olt <4 x float> %x, %y
+  %0 = and <4 x i1> %c, %a1
+  %b = select <4 x i1> %0, <4 x float> %x, <4 x float> %y
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fcmp_fast_olt_v8f16_y(<8 x half> %x, <8 x half> %y, i32 %n) {
+; CHECK-LABEL: fcmp_fast_olt_v8f16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.f16 gt, q1, q0
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a1 = fcmp fast olt <8 x half> %x, %y
+  %0 = and <8 x i1> %c, %a1
+  %b = select <8 x i1> %0, <8 x half> %x, <8 x half> %y
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fcmp_fast_ogt_v4f32_y(<4 x float> %x, <4 x float> %y, i32 %n) {
+; CHECK-LABEL: fcmp_fast_ogt_v4f32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.f32 gt, q0, q1
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a1 = fcmp fast ogt <4 x float> %x, %y
+  %0 = and <4 x i1> %c, %a1
+  %b = select <4 x i1> %0, <4 x float> %x, <4 x float> %y
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fcmp_fast_ogt_v8f16_y(<8 x half> %x, <8 x half> %y, i32 %n) {
+; CHECK-LABEL: fcmp_fast_ogt_v8f16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.f16 gt, q0, q1
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a1 = fcmp fast ogt <8 x half> %x, %y
+  %0 = and <8 x i1> %c, %a1
+  %b = select <8 x i1> %0, <8 x half> %x, <8 x half> %y
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sadd_sat_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: sadd_sat_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.s32 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sadd_sat_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: sadd_sat_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.s16 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @sadd_sat_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: sadd_sat_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.s8 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @uadd_sat_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: uadd_sat_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.u32 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @uadd_sat_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: uadd_sat_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.u16 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @uadd_sat_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: uadd_sat_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.u8 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ssub_sat_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: ssub_sat_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.s32 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ssub_sat_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: ssub_sat_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.s16 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @ssub_sat_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: ssub_sat_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.s8 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @usub_sat_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: usub_sat_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.u32 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @usub_sat_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: usub_sat_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.u16 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @usub_sat_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: usub_sat_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.u8 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @addqr_v4i32_y(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: addqr_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i32 q1, q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = select <4 x i1> %c, <4 x i32> %x, <4 x i32> zeroinitializer
+  %b = add <4 x i32> %ys, %a
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @addqr_v8i16_y(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: addqr_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.16 q1, r0
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i16 q1, q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = select <8 x i1> %c, <8 x i16> %x, <8 x i16> zeroinitializer
+  %b = add <8 x i16> %ys, %a
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @addqr_v16i8_y(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: addqr_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.8 q1, r0
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.i8 q1, q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = select <16 x i1> %c, <16 x i8> %x, <16 x i8> zeroinitializer
+  %b = add <16 x i8> %ys, %a
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @subqr_v4i32_y(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: subqr_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.i32 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = sub <4 x i32> %x, %ys
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %ys
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @subqr_v8i16_y(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: subqr_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.16 q1, r0
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.i16 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = sub <8 x i16> %x, %ys
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %ys
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @subqr_v16i8_y(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: subqr_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.8 q1, r0
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.i8 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = sub <16 x i8> %x, %ys
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %ys
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @mulqr_v4i32_y(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: mulqr_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i32 q1, q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = select <4 x i1> %c, <4 x i32> %x, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %b = mul <4 x i32> %ys, %a
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @mulqr_v8i16_y(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: mulqr_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.16 q1, r0
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i16 q1, q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = select <8 x i1> %c, <8 x i16> %x, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %b = mul <8 x i16> %ys, %a
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @mulqr_v16i8_y(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: mulqr_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.8 q1, r0
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i8 q1, q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = select <16 x i1> %c, <16 x i8> %x, <16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+  %b = mul <16 x i8> %ys, %a
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @faddqr_v4f32_y(<4 x float> %x, float %y, i32 %n) {
+; CHECK-LABEL: faddqr_v4f32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vdup.32 q1, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.f32 q1, q0, r1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x float> undef, float %y, i32 0
+  %ys = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %a = fadd <4 x float> %ys, %x
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %ys
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @faddqr_v8f16_y(<8 x half> %x, half %y, i32 %n) {
+; CHECK-LABEL: faddqr_v8f16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 r1, s4
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vdup.16 q1, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddt.f16 q1, q0, r1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x half> undef, half %y, i32 0
+  %ys = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %a = fadd <8 x half> %ys, %x
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %ys
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fsubqr_v4f32_y(<4 x float> %x, float %y, i32 %n) {
+; CHECK-LABEL: fsubqr_v4f32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vdup.32 q1, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.f32 q1, q0, r1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x float> undef, float %y, i32 0
+  %ys = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %a = fsub <4 x float> %x, %ys
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %ys
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fsubqr_v8f16_y(<8 x half> %x, half %y, i32 %n) {
+; CHECK-LABEL: fsubqr_v8f16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 r1, s4
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vdup.16 q1, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vsubt.f16 q1, q0, r1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x half> undef, half %y, i32 0
+  %ys = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %a = fsub <8 x half> %x, %ys
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %ys
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fmulqr_v4f32_y(<4 x float> %x, float %y, i32 %n) {
+; CHECK-LABEL: fmulqr_v4f32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vdup.32 q1, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.f32 q1, q0, r1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x float> undef, float %y, i32 0
+  %ys = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
+  %a = fmul <4 x float> %ys, %x
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %ys
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fmulqr_v8f16_y(<8 x half> %x, half %y, i32 %n) {
+; CHECK-LABEL: fmulqr_v8f16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f16 r1, s4
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vdup.16 q1, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.f16 q1, q0, r1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x half> undef, half %y, i32 0
+  %ys = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
+  %a = fmul <8 x half> %ys, %x
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %ys
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sadd_satqr_v4i32_y(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: sadd_satqr_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.s32 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %ys
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sadd_satqr_v8i16_y(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: sadd_satqr_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.16 q1, r0
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.s16 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %ys
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @sadd_satqr_v16i8_y(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: sadd_satqr_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.8 q1, r0
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.s8 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %ys
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @uadd_satqr_v4i32_y(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: uadd_satqr_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.u32 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %ys
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @uadd_satqr_v8i16_y(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: uadd_satqr_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.16 q1, r0
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.u16 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %ys
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @uadd_satqr_v16i8_y(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: uadd_satqr_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.8 q1, r0
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqaddt.u8 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %ys
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ssub_satqr_v4i32_y(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: ssub_satqr_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.s32 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %ys
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ssub_satqr_v8i16_y(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: ssub_satqr_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.16 q1, r0
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.s16 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %ys
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @ssub_satqr_v16i8_y(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: ssub_satqr_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.8 q1, r0
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.s8 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %ys
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @usub_satqr_v4i32_y(<4 x i32> %x, i32 %y, i32 %n) {
+; CHECK-LABEL: usub_satqr_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.u32 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %a = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %ys
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @usub_satqr_v8i16_y(<8 x i16> %x, i16 %y, i32 %n) {
+; CHECK-LABEL: usub_satqr_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.16 q1, r0
+; CHECK-NEXT:    vctp.16 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.u16 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
+  %a = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %ys
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @usub_satqr_v16i8_y(<16 x i8> %x, i8 %y, i32 %n) {
+; CHECK-LABEL: usub_satqr_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdup.8 q1, r0
+; CHECK-NEXT:    vctp.8 r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vqsubt.u8 q1, q0, r0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
+  %a = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %ys
+  ret <16 x i8> %b
+}
+
+declare <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32>, <4 x i32>)
+declare <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32>, <4 x i32>)
+declare <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32>, <4 x i32>)
+declare <16 x i8> @llvm.usub.sat.v16i8(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.usub.sat.v8i16(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.usub.sat.v4i32(<4 x i32>, <4 x i32>)
+
+declare <16 x i1> @llvm.arm.mve.vctp8(i32)
+declare <8 x i1> @llvm.arm.mve.vctp16(i32)
+declare <4 x i1> @llvm.arm.mve.vctp32(i32)
diff --git a/llvm/test/CodeGen/Thumb2/mve-vcvt.ll b/llvm/test/CodeGen/Thumb2/mve-vcvt.ll
index ddc36f3be583f..831ca04993335 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vcvt.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vcvt.ll
@@ -45,8 +45,8 @@ define arm_aapcs_vfpcc <4 x i32> @foo_int32_float(<4 x float> %src) {
 ; CHECK-MVE:       @ %bb.0: @ %entry
 ; CHECK-MVE-NEXT:    vcvt.s32.f32 s4, s0
 ; CHECK-MVE-NEXT:    vcvt.s32.f32 s6, s1
-; CHECK-MVE-NEXT:    vcvt.s32.f32 s8, s3
 ; CHECK-MVE-NEXT:    vcvt.s32.f32 s10, s2
+; CHECK-MVE-NEXT:    vcvt.s32.f32 s8, s3
 ; CHECK-MVE-NEXT:    vmov r0, s4
 ; CHECK-MVE-NEXT:    vmov.32 q0[0], r0
 ; CHECK-MVE-NEXT:    vmov r0, s6
@@ -71,8 +71,8 @@ define arm_aapcs_vfpcc <4 x i32> @foo_uint32_float(<4 x float> %src) {
 ; CHECK-MVE:       @ %bb.0: @ %entry
 ; CHECK-MVE-NEXT:    vcvt.u32.f32 s4, s0
 ; CHECK-MVE-NEXT:    vcvt.u32.f32 s6, s1
-; CHECK-MVE-NEXT:    vcvt.u32.f32 s8, s3
 ; CHECK-MVE-NEXT:    vcvt.u32.f32 s10, s2
+; CHECK-MVE-NEXT:    vcvt.u32.f32 s8, s3
 ; CHECK-MVE-NEXT:    vmov r0, s4
 ; CHECK-MVE-NEXT:    vmov.32 q0[0], r0
 ; CHECK-MVE-NEXT:    vmov r0, s6
diff --git a/llvm/test/CodeGen/Thumb2/mve-vecreduce-addpred.ll b/llvm/test/CodeGen/Thumb2/mve-vecreduce-addpred.ll
new file mode 100644
index 0000000000000..eb47d1dc235bf
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vecreduce-addpred.ll
@@ -0,0 +1,3127 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK
+
+define arm_aapcs_vfpcc i32 @add_v4i32_v4i32(<4 x i32> %x, <4 x i32> %b) {
+; CHECK-LABEL: add_v4i32_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q1, zr
+; CHECK-NEXT:    vaddvt.u32 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %b, zeroinitializer
+  %s = select <4 x i1> %c, <4 x i32> %x, <4 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %s)
+  ret i32 %z
+}
+
+define arm_aapcs_vfpcc i64 @add_v4i32_v4i64_zext(<4 x i32> %x, <4 x i32> %b) {
+; CHECK-LABEL: add_v4i32_v4i64_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q1, zr
+; CHECK-NEXT:    vaddlvt.u32 r0, r1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %b, zeroinitializer
+  %xx = zext <4 x i32> %x to <4 x i64>
+  %s = select <4 x i1> %c, <4 x i64> %xx, <4 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v4i64(<4 x i64> %s)
+  ret i64 %z
+}
+
+define arm_aapcs_vfpcc i64 @add_v4i32_v4i64_sext(<4 x i32> %x, <4 x i32> %b) {
+; CHECK-LABEL: add_v4i32_v4i64_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q1, zr
+; CHECK-NEXT:    vaddlvt.s32 r0, r1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %b, zeroinitializer
+  %xx = sext <4 x i32> %x to <4 x i64>
+  %s = select <4 x i1> %c, <4 x i64> %xx, <4 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v4i64(<4 x i64> %s)
+  ret i64 %z
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i32_v2i64_zext(<2 x i32> %x, <2 x i32> %b) {
+; CHECK-LABEL: add_v2i32_v2i64_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.i64 q2, #0xffffffff
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <2 x i32> %b, zeroinitializer
+  %xx = zext <2 x i32> %x to <2 x i64>
+  %s = select <2 x i1> %c, <2 x i64> %xx, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  ret i64 %z
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i32_v2i64_sext(<2 x i32> %x, <2 x i32> %b) {
+; CHECK-LABEL: add_v2i32_v2i64_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    asrs r0, r0, #31
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    asrs r0, r0, #31
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q0[2], r0
+; CHECK-NEXT:    vmov.32 q0[3], r0
+; CHECK-NEXT:    vand q0, q2, q0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <2 x i32> %b, zeroinitializer
+  %xx = sext <2 x i32> %x to <2 x i64>
+  %s = select <2 x i1> %c, <2 x i64> %xx, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  ret i64 %z
+}
+
+define arm_aapcs_vfpcc i32 @add_v8i16_v8i32_zext(<8 x i16> %x, <8 x i16> %b) {
+; CHECK-LABEL: add_v8i16_v8i32_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q1, zr
+; CHECK-NEXT:    vaddvt.u16 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %b, zeroinitializer
+  %xx = zext <8 x i16> %x to <8 x i32>
+  %s = select <8 x i1> %c, <8 x i32> %xx, <8 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> %s)
+  ret i32 %z
+}
+
+define arm_aapcs_vfpcc i32 @add_v8i16_v8i32_sext(<8 x i16> %x, <8 x i16> %b) {
+; CHECK-LABEL: add_v8i16_v8i32_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q1, zr
+; CHECK-NEXT:    vaddvt.s16 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %b, zeroinitializer
+  %xx = sext <8 x i16> %x to <8 x i32>
+  %s = select <8 x i1> %c, <8 x i32> %xx, <8 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> %s)
+  ret i32 %z
+}
+
+define arm_aapcs_vfpcc i32 @add_v4i16_v4i32_zext(<4 x i16> %x, <4 x i16> %b) {
+; CHECK-LABEL: add_v4i16_v4i32_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.u16 q0, q0
+; CHECK-NEXT:    vmovlb.u16 q1, q1
+; CHECK-NEXT:    vpt.i32 eq, q1, zr
+; CHECK-NEXT:    vaddvt.u32 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i16> %b, zeroinitializer
+  %xx = zext <4 x i16> %x to <4 x i32>
+  %s = select <4 x i1> %c, <4 x i32> %xx, <4 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %s)
+  ret i32 %z
+}
+
+define arm_aapcs_vfpcc i32 @add_v4i16_v4i32_sext(<4 x i16> %x, <4 x i16> %b) {
+; CHECK-LABEL: add_v4i16_v4i32_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    vmovlb.u16 q1, q1
+; CHECK-NEXT:    vpt.i32 eq, q1, zr
+; CHECK-NEXT:    vaddvt.u32 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i16> %b, zeroinitializer
+  %xx = sext <4 x i16> %x to <4 x i32>
+  %s = select <4 x i1> %c, <4 x i32> %xx, <4 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %s)
+  ret i32 %z
+}
+
+define arm_aapcs_vfpcc zeroext i16 @add_v8i16_v8i16(<8 x i16> %x, <8 x i16> %b) {
+; CHECK-LABEL: add_v8i16_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q1, zr
+; CHECK-NEXT:    vaddvt.u16 r0, q0
+; CHECK-NEXT:    uxth r0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %b, zeroinitializer
+  %s = select <8 x i1> %c, <8 x i16> %x, <8 x i16> zeroinitializer
+  %z = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> %s)
+  ret i16 %z
+}
+
+define arm_aapcs_vfpcc i64 @add_v8i16_v8i64_zext(<8 x i16> %x, <8 x i16> %b) {
+; CHECK-LABEL: add_v8i16_v8i64_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vmov.i8 q2, #0x0
+; CHECK-NEXT:    vmov.i8 q3, #0xff
+; CHECK-NEXT:    vcmp.i16 eq, q1, zr
+; CHECK-NEXT:    vpsel q2, q3, q2
+; CHECK-NEXT:    vmov.u16 r0, q2[0]
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[1]
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[2]
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[3]
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vcmp.i32 ne, q1, zr
+; CHECK-NEXT:    vmov.i64 q1, #0xffff
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    and r1, r0, #1
+; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    vmov.32 q3[0], r1
+; CHECK-NEXT:    vmov.32 q3[1], r1
+; CHECK-NEXT:    ubfx r1, r0, #4, #1
+; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    vmov.32 q3[2], r1
+; CHECK-NEXT:    vmov.32 q3[3], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[0]
+; CHECK-NEXT:    vmov.32 q4[0], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[1]
+; CHECK-NEXT:    vmov.32 q4[2], r1
+; CHECK-NEXT:    vand q4, q4, q1
+; CHECK-NEXT:    vand q3, q4, q3
+; CHECK-NEXT:    vmov r1, s15
+; CHECK-NEXT:    vmov r2, s13
+; CHECK-NEXT:    vmov r3, s12
+; CHECK-NEXT:    orrs r1, r2
+; CHECK-NEXT:    vmov r2, s14
+; CHECK-NEXT:    add r2, r3
+; CHECK-NEXT:    ubfx r3, r0, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r0, r0, #12, #1
+; CHECK-NEXT:    vmov.32 q3[0], r3
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vmov.32 q3[1], r3
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[2]
+; CHECK-NEXT:    vmov.32 q4[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[3]
+; CHECK-NEXT:    vmov.32 q4[2], r0
+; CHECK-NEXT:    vand q4, q4, q1
+; CHECK-NEXT:    vand q3, q4, q3
+; CHECK-NEXT:    vmov r3, s12
+; CHECK-NEXT:    vmov r0, s13
+; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    vmov r3, s15
+; CHECK-NEXT:    adcs r1, r0
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    adds.w r12, r2, r0
+; CHECK-NEXT:    vmov.u16 r2, q2[4]
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[5]
+; CHECK-NEXT:    vmov.32 q3[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[6]
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[7]
+; CHECK-NEXT:    vmov.32 q3[3], r2
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    vcmp.i32 ne, q3, zr
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r3, r2, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q2[0], r3
+; CHECK-NEXT:    vmov.32 q2[1], r3
+; CHECK-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q2[2], r3
+; CHECK-NEXT:    vmov.32 q2[3], r3
+; CHECK-NEXT:    vmov.u16 r3, q0[4]
+; CHECK-NEXT:    vmov.32 q3[0], r3
+; CHECK-NEXT:    vmov.u16 r3, q0[5]
+; CHECK-NEXT:    vmov.32 q3[2], r3
+; CHECK-NEXT:    vand q3, q3, q1
+; CHECK-NEXT:    vand q2, q3, q2
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov r3, s9
+; CHECK-NEXT:    adds.w r12, r12, r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    vmov r3, s11
+; CHECK-NEXT:    adds.w r0, r0, r12
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    vmov.32 q2[0], r3
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov.32 q2[1], r3
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vmov.u16 r2, q0[6]
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q0[7]
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    vand q0, q3, q1
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %b, zeroinitializer
+  %xx = zext <8 x i16> %x to <8 x i64>
+  %s = select <8 x i1> %c, <8 x i64> %xx, <8 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v8i64(<8 x i64> %s)
+  ret i64 %z
+}
+
+define arm_aapcs_vfpcc i64 @add_v8i16_v8i64_sext(<8 x i16> %x, <8 x i16> %b) {
+; CHECK-LABEL: add_v8i16_v8i64_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i8 q2, #0x0
+; CHECK-NEXT:    vmov.i8 q3, #0xff
+; CHECK-NEXT:    vcmp.i16 eq, q1, zr
+; CHECK-NEXT:    vpsel q1, q3, q2
+; CHECK-NEXT:    vmov.u16 r0, q1[0]
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[1]
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[2]
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[3]
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vcmp.i32 ne, q2, zr
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    and r1, r0, #1
+; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    vmov.32 q2[0], r1
+; CHECK-NEXT:    vmov.32 q2[1], r1
+; CHECK-NEXT:    ubfx r1, r0, #4, #1
+; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    vmov.32 q2[2], r1
+; CHECK-NEXT:    vmov.32 q2[3], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[0]
+; CHECK-NEXT:    sxth r1, r1
+; CHECK-NEXT:    vmov.32 q3[0], r1
+; CHECK-NEXT:    asrs r1, r1, #31
+; CHECK-NEXT:    vmov.32 q3[1], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[1]
+; CHECK-NEXT:    sxth r1, r1
+; CHECK-NEXT:    vmov.32 q3[2], r1
+; CHECK-NEXT:    asrs r1, r1, #31
+; CHECK-NEXT:    vmov.32 q3[3], r1
+; CHECK-NEXT:    vand q2, q3, q2
+; CHECK-NEXT:    vmov r1, s10
+; CHECK-NEXT:    vmov r2, s8
+; CHECK-NEXT:    vmov r12, s11
+; CHECK-NEXT:    vmov r3, s9
+; CHECK-NEXT:    adds r1, r1, r2
+; CHECK-NEXT:    adc.w r2, r3, r12
+; CHECK-NEXT:    ubfx r3, r0, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r0, r0, #12, #1
+; CHECK-NEXT:    vmov.32 q2[0], r3
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vmov.32 q2[1], r3
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[2]
+; CHECK-NEXT:    sxth r0, r0
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    asrs r0, r0, #31
+; CHECK-NEXT:    vmov.32 q3[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[3]
+; CHECK-NEXT:    sxth r0, r0
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    asrs r0, r0, #31
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vand q2, q3, q2
+; CHECK-NEXT:    vmov r3, s8
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    adds r1, r1, r3
+; CHECK-NEXT:    vmov r3, s11
+; CHECK-NEXT:    adcs r2, r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    adds.w r12, r1, r0
+; CHECK-NEXT:    adc.w r1, r2, r3
+; CHECK-NEXT:    vmov.u16 r2, q1[4]
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[5]
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[6]
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[7]
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vcmp.i32 ne, q2, zr
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r3, r2, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q1[0], r3
+; CHECK-NEXT:    vmov.32 q1[1], r3
+; CHECK-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q1[2], r3
+; CHECK-NEXT:    vmov.32 q1[3], r3
+; CHECK-NEXT:    vmov.u16 r3, q0[4]
+; CHECK-NEXT:    sxth r3, r3
+; CHECK-NEXT:    vmov.32 q2[0], r3
+; CHECK-NEXT:    asrs r3, r3, #31
+; CHECK-NEXT:    vmov.32 q2[1], r3
+; CHECK-NEXT:    vmov.u16 r3, q0[5]
+; CHECK-NEXT:    sxth r3, r3
+; CHECK-NEXT:    vmov.32 q2[2], r3
+; CHECK-NEXT:    asrs r3, r3, #31
+; CHECK-NEXT:    vmov.32 q2[3], r3
+; CHECK-NEXT:    vand q1, q2, q1
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r3, s5
+; CHECK-NEXT:    adds.w r12, r12, r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    vmov r3, s7
+; CHECK-NEXT:    adds.w r0, r0, r12
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    vmov.32 q1[0], r3
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov.32 q1[1], r3
+; CHECK-NEXT:    vmov.32 q1[2], r2
+; CHECK-NEXT:    vmov.32 q1[3], r2
+; CHECK-NEXT:    vmov.u16 r2, q0[6]
+; CHECK-NEXT:    sxth r2, r2
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q0[7]
+; CHECK-NEXT:    sxth r2, r2
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vand q0, q2, q1
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %b, zeroinitializer
+  %xx = sext <8 x i16> %x to <8 x i64>
+  %s = select <8 x i1> %c, <8 x i64> %xx, <8 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v8i64(<8 x i64> %s)
+  ret i64 %z
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_zext(<2 x i16> %x, <2 x i16> %b) {
+; CHECK-LABEL: add_v2i16_v2i64_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i64 q2, #0xffff
+; CHECK-NEXT:    vand q1, q1, q2
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov.32 q3[1], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vand q0, q0, q3
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    add r0, r1
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    orrs r1, r2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <2 x i16> %b, zeroinitializer
+  %xx = zext <2 x i16> %x to <2 x i64>
+  %s = select <2 x i1> %c, <2 x i64> %xx, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  ret i64 %z
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_sext(<2 x i16> %x, <2 x i16> %b) {
+; CHECK-LABEL: add_v2i16_v2i64_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i32 q2, #0xffff
+; CHECK-NEXT:    vand q2, q1, q2
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    sxth r0, r0
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    asrs r0, r0, #31
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    sxth r0, r0
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    asrs r0, r0, #31
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vand q0, q2, q1
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <2 x i16> %b, zeroinitializer
+  %xx = sext <2 x i16> %x to <2 x i64>
+  %s = select <2 x i1> %c, <2 x i64> %xx, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  ret i64 %z
+}
+
+define arm_aapcs_vfpcc i32 @add_v16i8_v16i32_zext(<16 x i8> %x, <16 x i8> %b) {
+; CHECK-LABEL: add_v16i8_v16i32_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q1, zr
+; CHECK-NEXT:    vaddvt.u8 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %xx = zext <16 x i8> %x to <16 x i32>
+  %s = select <16 x i1> %c, <16 x i32> %xx, <16 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v16i32(<16 x i32> %s)
+  ret i32 %z
+}
+
+define arm_aapcs_vfpcc i32 @add_v16i8_v16i32_sext(<16 x i8> %x, <16 x i8> %b) {
+; CHECK-LABEL: add_v16i8_v16i32_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q1, zr
+; CHECK-NEXT:    vaddvt.s8 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %xx = sext <16 x i8> %x to <16 x i32>
+  %s = select <16 x i1> %c, <16 x i32> %xx, <16 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v16i32(<16 x i32> %s)
+  ret i32 %z
+}
+
+define arm_aapcs_vfpcc i32 @add_v4i8_v4i32_zext(<4 x i8> %x, <4 x i8> %b) {
+; CHECK-LABEL: add_v4i8_v4i32_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i32 q2, #0xff
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vand q1, q1, q2
+; CHECK-NEXT:    vpt.i32 eq, q1, zr
+; CHECK-NEXT:    vaddvt.u32 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i8> %b, zeroinitializer
+  %xx = zext <4 x i8> %x to <4 x i32>
+  %s = select <4 x i1> %c, <4 x i32> %xx, <4 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %s)
+  ret i32 %z
+}
+
+define arm_aapcs_vfpcc i32 @add_v4i8_v4i32_sext(<4 x i8> %x, <4 x i8> %b) {
+; CHECK-LABEL: add_v4i8_v4i32_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    vmov.i32 q2, #0xff
+; CHECK-NEXT:    vand q1, q1, q2
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    vpt.i32 eq, q1, zr
+; CHECK-NEXT:    vaddvt.u32 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i8> %b, zeroinitializer
+  %xx = sext <4 x i8> %x to <4 x i32>
+  %s = select <4 x i1> %c, <4 x i32> %xx, <4 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %s)
+  ret i32 %z
+}
+
+define arm_aapcs_vfpcc zeroext i16 @add_v16i8_v16i16_zext(<16 x i8> %x, <16 x i8> %b) {
+; CHECK-LABEL: add_v16i8_v16i16_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i8 eq, q1, zr
+; CHECK-NEXT:    vmov.i8 q1, #0x0
+; CHECK-NEXT:    vmov.i8 q2, #0xff
+; CHECK-NEXT:    vmov.i32 q3, #0x0
+; CHECK-NEXT:    vpsel q1, q2, q1
+; CHECK-NEXT:    vmov.u8 r0, q1[0]
+; CHECK-NEXT:    vmov.16 q2[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[1]
+; CHECK-NEXT:    vmov.16 q2[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[2]
+; CHECK-NEXT:    vmov.16 q2[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[3]
+; CHECK-NEXT:    vmov.16 q2[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[4]
+; CHECK-NEXT:    vmov.16 q2[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[5]
+; CHECK-NEXT:    vmov.16 q2[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[6]
+; CHECK-NEXT:    vmov.16 q2[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[7]
+; CHECK-NEXT:    vmov.16 q2[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[0]
+; CHECK-NEXT:    vcmp.i16 ne, q2, zr
+; CHECK-NEXT:    vmov.16 q2[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[1]
+; CHECK-NEXT:    vmov.16 q2[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[2]
+; CHECK-NEXT:    vmov.16 q2[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[3]
+; CHECK-NEXT:    vmov.16 q2[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[4]
+; CHECK-NEXT:    vmov.16 q2[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[5]
+; CHECK-NEXT:    vmov.16 q2[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[6]
+; CHECK-NEXT:    vmov.16 q2[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[7]
+; CHECK-NEXT:    vmov.16 q2[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[8]
+; CHECK-NEXT:    vmovlb.u8 q2, q2
+; CHECK-NEXT:    vpsel q2, q2, q3
+; CHECK-NEXT:    vmov.16 q3[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[9]
+; CHECK-NEXT:    vmov.16 q3[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[10]
+; CHECK-NEXT:    vmov.16 q3[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[11]
+; CHECK-NEXT:    vmov.16 q3[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[12]
+; CHECK-NEXT:    vmov.16 q3[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[13]
+; CHECK-NEXT:    vmov.16 q3[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[14]
+; CHECK-NEXT:    vmov.16 q3[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[15]
+; CHECK-NEXT:    vmov.16 q3[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[8]
+; CHECK-NEXT:    vmov.16 q1[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[9]
+; CHECK-NEXT:    vmov.16 q1[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[10]
+; CHECK-NEXT:    vmov.16 q1[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[11]
+; CHECK-NEXT:    vmov.16 q1[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[12]
+; CHECK-NEXT:    vmov.16 q1[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[13]
+; CHECK-NEXT:    vmov.16 q1[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[14]
+; CHECK-NEXT:    vmov.16 q1[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[15]
+; CHECK-NEXT:    vmov.16 q1[7], r0
+; CHECK-NEXT:    vmovlb.u8 q0, q1
+; CHECK-NEXT:    vpt.i16 ne, q3, zr
+; CHECK-NEXT:    vaddt.i16 q2, q2, q0
+; CHECK-NEXT:    vaddv.u16 r0, q2
+; CHECK-NEXT:    uxth r0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %xx = zext <16 x i8> %x to <16 x i16>
+  %s = select <16 x i1> %c, <16 x i16> %xx, <16 x i16> zeroinitializer
+  %z = call i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16> %s)
+  ret i16 %z
+}
+
+define arm_aapcs_vfpcc signext i16 @add_v16i8_v16i16_sext(<16 x i8> %x, <16 x i8> %b) {
+; CHECK-LABEL: add_v16i8_v16i16_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i8 eq, q1, zr
+; CHECK-NEXT:    vmov.i8 q1, #0x0
+; CHECK-NEXT:    vmov.i8 q2, #0xff
+; CHECK-NEXT:    vmov.i32 q3, #0x0
+; CHECK-NEXT:    vpsel q1, q2, q1
+; CHECK-NEXT:    vmov.u8 r0, q1[0]
+; CHECK-NEXT:    vmov.16 q2[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[1]
+; CHECK-NEXT:    vmov.16 q2[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[2]
+; CHECK-NEXT:    vmov.16 q2[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[3]
+; CHECK-NEXT:    vmov.16 q2[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[4]
+; CHECK-NEXT:    vmov.16 q2[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[5]
+; CHECK-NEXT:    vmov.16 q2[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[6]
+; CHECK-NEXT:    vmov.16 q2[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[7]
+; CHECK-NEXT:    vmov.16 q2[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[0]
+; CHECK-NEXT:    vcmp.i16 ne, q2, zr
+; CHECK-NEXT:    vmov.16 q2[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[1]
+; CHECK-NEXT:    vmov.16 q2[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[2]
+; CHECK-NEXT:    vmov.16 q2[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[3]
+; CHECK-NEXT:    vmov.16 q2[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[4]
+; CHECK-NEXT:    vmov.16 q2[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[5]
+; CHECK-NEXT:    vmov.16 q2[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[6]
+; CHECK-NEXT:    vmov.16 q2[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[7]
+; CHECK-NEXT:    vmov.16 q2[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[8]
+; CHECK-NEXT:    vmovlb.s8 q2, q2
+; CHECK-NEXT:    vpsel q2, q2, q3
+; CHECK-NEXT:    vmov.16 q3[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[9]
+; CHECK-NEXT:    vmov.16 q3[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[10]
+; CHECK-NEXT:    vmov.16 q3[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[11]
+; CHECK-NEXT:    vmov.16 q3[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[12]
+; CHECK-NEXT:    vmov.16 q3[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[13]
+; CHECK-NEXT:    vmov.16 q3[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[14]
+; CHECK-NEXT:    vmov.16 q3[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[15]
+; CHECK-NEXT:    vmov.16 q3[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[8]
+; CHECK-NEXT:    vmov.16 q1[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[9]
+; CHECK-NEXT:    vmov.16 q1[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[10]
+; CHECK-NEXT:    vmov.16 q1[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[11]
+; CHECK-NEXT:    vmov.16 q1[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[12]
+; CHECK-NEXT:    vmov.16 q1[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[13]
+; CHECK-NEXT:    vmov.16 q1[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[14]
+; CHECK-NEXT:    vmov.16 q1[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[15]
+; CHECK-NEXT:    vmov.16 q1[7], r0
+; CHECK-NEXT:    vmovlb.s8 q0, q1
+; CHECK-NEXT:    vpt.i16 ne, q3, zr
+; CHECK-NEXT:    vaddt.i16 q2, q2, q0
+; CHECK-NEXT:    vaddv.u16 r0, q2
+; CHECK-NEXT:    sxth r0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %xx = sext <16 x i8> %x to <16 x i16>
+  %s = select <16 x i1> %c, <16 x i16> %xx, <16 x i16> zeroinitializer
+  %z = call i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16> %s)
+  ret i16 %z
+}
+
+define arm_aapcs_vfpcc zeroext i16 @add_v8i8_v8i16_zext(<8 x i8> %x, <8 x i8> %b) {
+; CHECK-LABEL: add_v8i8_v8i16_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.u8 q0, q0
+; CHECK-NEXT:    vmovlb.u8 q1, q1
+; CHECK-NEXT:    vpt.i16 eq, q1, zr
+; CHECK-NEXT:    vaddvt.u16 r0, q0
+; CHECK-NEXT:    uxth r0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i8> %b, zeroinitializer
+  %xx = zext <8 x i8> %x to <8 x i16>
+  %s = select <8 x i1> %c, <8 x i16> %xx, <8 x i16> zeroinitializer
+  %z = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> %s)
+  ret i16 %z
+}
+
+define arm_aapcs_vfpcc signext i16 @add_v8i8_v8i16_sext(<8 x i8> %x, <8 x i8> %b) {
+; CHECK-LABEL: add_v8i8_v8i16_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    vmovlb.u8 q1, q1
+; CHECK-NEXT:    vpt.i16 eq, q1, zr
+; CHECK-NEXT:    vaddvt.u16 r0, q0
+; CHECK-NEXT:    sxth r0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i8> %b, zeroinitializer
+  %xx = sext <8 x i8> %x to <8 x i16>
+  %s = select <8 x i1> %c, <8 x i16> %xx, <8 x i16> zeroinitializer
+  %z = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> %s)
+  ret i16 %z
+}
+
+define arm_aapcs_vfpcc zeroext i8 @add_v16i8_v16i8(<16 x i8> %x, <16 x i8> %b) {
+; CHECK-LABEL: add_v16i8_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q1, zr
+; CHECK-NEXT:    vaddvt.u8 r0, q0
+; CHECK-NEXT:    uxtb r0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %s = select <16 x i1> %c, <16 x i8> %x, <16 x i8> zeroinitializer
+  %z = call i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8> %s)
+  ret i8 %z
+}
+
+define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_zext(<16 x i8> %x, <16 x i8> %b) {
+; CHECK-LABEL: add_v16i8_v16i64_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vcmp.i8 eq, q1, zr
+; CHECK-NEXT:    vmov.i8 q2, #0x0
+; CHECK-NEXT:    vmov.i8 q3, #0xff
+; CHECK-NEXT:    vpsel q4, q3, q2
+; CHECK-NEXT:    vmov.u8 r0, q4[0]
+; CHECK-NEXT:    vmov.16 q1[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q4[1]
+; CHECK-NEXT:    vmov.16 q1[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q4[2]
+; CHECK-NEXT:    vmov.16 q1[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q4[3]
+; CHECK-NEXT:    vmov.16 q1[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q4[4]
+; CHECK-NEXT:    vmov.16 q1[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q4[5]
+; CHECK-NEXT:    vmov.16 q1[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q4[6]
+; CHECK-NEXT:    vmov.16 q1[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q4[7]
+; CHECK-NEXT:    vmov.16 q1[7], r0
+; CHECK-NEXT:    vcmp.i16 ne, q1, zr
+; CHECK-NEXT:    vpsel q5, q3, q2
+; CHECK-NEXT:    vmov.u16 r0, q5[0]
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q5[1]
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q5[2]
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q5[3]
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vcmp.i32 ne, q1, zr
+; CHECK-NEXT:    vmov.i64 q1, #0xff
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    and r1, r0, #1
+; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    vmov.32 q6[0], r1
+; CHECK-NEXT:    vmov.32 q6[1], r1
+; CHECK-NEXT:    ubfx r1, r0, #4, #1
+; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    vmov.32 q6[2], r1
+; CHECK-NEXT:    vmov.32 q6[3], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[0]
+; CHECK-NEXT:    vmov.32 q7[0], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[1]
+; CHECK-NEXT:    vmov.32 q7[2], r1
+; CHECK-NEXT:    vand q7, q7, q1
+; CHECK-NEXT:    vand q6, q7, q6
+; CHECK-NEXT:    vmov r1, s27
+; CHECK-NEXT:    vmov r2, s25
+; CHECK-NEXT:    vmov r3, s24
+; CHECK-NEXT:    orrs r1, r2
+; CHECK-NEXT:    vmov r2, s26
+; CHECK-NEXT:    add r2, r3
+; CHECK-NEXT:    ubfx r3, r0, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r0, r0, #12, #1
+; CHECK-NEXT:    vmov.32 q6[0], r3
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vmov.32 q6[1], r3
+; CHECK-NEXT:    vmov.32 q6[2], r0
+; CHECK-NEXT:    vmov.32 q6[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[2]
+; CHECK-NEXT:    vmov.32 q7[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[3]
+; CHECK-NEXT:    vmov.32 q7[2], r0
+; CHECK-NEXT:    vand q7, q7, q1
+; CHECK-NEXT:    vand q6, q7, q6
+; CHECK-NEXT:    vmov r3, s24
+; CHECK-NEXT:    vmov r0, s25
+; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    vmov r3, s27
+; CHECK-NEXT:    adcs r1, r0
+; CHECK-NEXT:    vmov r0, s26
+; CHECK-NEXT:    adds.w r12, r2, r0
+; CHECK-NEXT:    vmov.u16 r2, q5[4]
+; CHECK-NEXT:    vmov.32 q6[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q5[5]
+; CHECK-NEXT:    vmov.32 q6[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q5[6]
+; CHECK-NEXT:    vmov.32 q6[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q5[7]
+; CHECK-NEXT:    vmov.32 q6[3], r2
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    vcmp.i32 ne, q6, zr
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r3, r2, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q5[0], r3
+; CHECK-NEXT:    vmov.32 q5[1], r3
+; CHECK-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q5[2], r3
+; CHECK-NEXT:    vmov.32 q5[3], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[4]
+; CHECK-NEXT:    vmov.32 q6[0], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[5]
+; CHECK-NEXT:    vmov.32 q6[2], r3
+; CHECK-NEXT:    vand q6, q6, q1
+; CHECK-NEXT:    vand q5, q6, q5
+; CHECK-NEXT:    vmov r0, s20
+; CHECK-NEXT:    vmov r3, s21
+; CHECK-NEXT:    adds.w r12, r12, r0
+; CHECK-NEXT:    vmov r0, s22
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    vmov r3, s23
+; CHECK-NEXT:    adds.w r0, r0, r12
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    vmov.32 q5[0], r3
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov.32 q5[1], r3
+; CHECK-NEXT:    vmov.32 q5[2], r2
+; CHECK-NEXT:    vmov.32 q5[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[6]
+; CHECK-NEXT:    vmov.32 q6[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[7]
+; CHECK-NEXT:    vmov.32 q6[2], r2
+; CHECK-NEXT:    vand q6, q6, q1
+; CHECK-NEXT:    vand q5, q6, q5
+; CHECK-NEXT:    vmov r3, s20
+; CHECK-NEXT:    vmov r2, s21
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    vmov r3, s22
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    vmov r2, s23
+; CHECK-NEXT:    adds.w r12, r0, r3
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    vmov.u8 r2, q4[8]
+; CHECK-NEXT:    vmov.16 q5[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[9]
+; CHECK-NEXT:    vmov.16 q5[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[10]
+; CHECK-NEXT:    vmov.16 q5[2], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[11]
+; CHECK-NEXT:    vmov.16 q5[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[12]
+; CHECK-NEXT:    vmov.16 q5[4], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[13]
+; CHECK-NEXT:    vmov.16 q5[5], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[14]
+; CHECK-NEXT:    vmov.16 q5[6], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[15]
+; CHECK-NEXT:    vmov.16 q5[7], r2
+; CHECK-NEXT:    vcmp.i16 ne, q5, zr
+; CHECK-NEXT:    vpsel q2, q3, q2
+; CHECK-NEXT:    vmov.u16 r2, q2[0]
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[1]
+; CHECK-NEXT:    vmov.32 q3[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[2]
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[3]
+; CHECK-NEXT:    vmov.32 q3[3], r2
+; CHECK-NEXT:    vcmp.i32 ne, q3, zr
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r3, r2, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q3[0], r3
+; CHECK-NEXT:    vmov.32 q3[1], r3
+; CHECK-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q3[2], r3
+; CHECK-NEXT:    vmov.32 q3[3], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[8]
+; CHECK-NEXT:    vmov.32 q4[0], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[9]
+; CHECK-NEXT:    vmov.32 q4[2], r3
+; CHECK-NEXT:    vand q4, q4, q1
+; CHECK-NEXT:    vand q3, q4, q3
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmov r3, s13
+; CHECK-NEXT:    adds.w r12, r12, r0
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    vmov r3, s15
+; CHECK-NEXT:    adds.w r0, r0, r12
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    vmov.32 q3[0], r3
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov.32 q3[1], r3
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    vmov.32 q3[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[10]
+; CHECK-NEXT:    vmov.32 q4[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[11]
+; CHECK-NEXT:    vmov.32 q4[2], r2
+; CHECK-NEXT:    vand q4, q4, q1
+; CHECK-NEXT:    vand q3, q4, q3
+; CHECK-NEXT:    vmov r3, s12
+; CHECK-NEXT:    vmov r2, s13
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    vmov r3, s14
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    vmov r2, s15
+; CHECK-NEXT:    adds.w r12, r0, r3
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    vmov.u16 r2, q2[4]
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[5]
+; CHECK-NEXT:    vmov.32 q3[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[6]
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[7]
+; CHECK-NEXT:    vmov.32 q3[3], r2
+; CHECK-NEXT:    vcmp.i32 ne, q3, zr
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r3, r2, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q2[0], r3
+; CHECK-NEXT:    vmov.32 q2[1], r3
+; CHECK-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q2[2], r3
+; CHECK-NEXT:    vmov.32 q2[3], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[12]
+; CHECK-NEXT:    vmov.32 q3[0], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[13]
+; CHECK-NEXT:    vmov.32 q3[2], r3
+; CHECK-NEXT:    vand q3, q3, q1
+; CHECK-NEXT:    vand q2, q3, q2
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov r3, s9
+; CHECK-NEXT:    adds.w r12, r12, r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    vmov r3, s11
+; CHECK-NEXT:    adds.w r0, r0, r12
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    vmov.32 q2[0], r3
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov.32 q2[1], r3
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[14]
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[15]
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    vand q0, q3, q1
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %xx = zext <16 x i8> %x to <16 x i64>
+  %s = select <16 x i1> %c, <16 x i64> %xx, <16 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v16i64(<16 x i64> %s)
+  ret i64 %z
+}
+
+define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_sext(<16 x i8> %x, <16 x i8> %b) {
+; CHECK-LABEL: add_v16i8_v16i64_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vcmp.i8 eq, q1, zr
+; CHECK-NEXT:    vmov.i8 q1, #0x0
+; CHECK-NEXT:    vmov.i8 q2, #0xff
+; CHECK-NEXT:    vpsel q3, q2, q1
+; CHECK-NEXT:    vmov.u8 r0, q3[0]
+; CHECK-NEXT:    vmov.16 q4[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[1]
+; CHECK-NEXT:    vmov.16 q4[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[2]
+; CHECK-NEXT:    vmov.16 q4[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[3]
+; CHECK-NEXT:    vmov.16 q4[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[4]
+; CHECK-NEXT:    vmov.16 q4[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[5]
+; CHECK-NEXT:    vmov.16 q4[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[6]
+; CHECK-NEXT:    vmov.16 q4[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[7]
+; CHECK-NEXT:    vmov.16 q4[7], r0
+; CHECK-NEXT:    vcmp.i16 ne, q4, zr
+; CHECK-NEXT:    vpsel q4, q2, q1
+; CHECK-NEXT:    vmov.u16 r0, q4[0]
+; CHECK-NEXT:    vmov.32 q5[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q4[1]
+; CHECK-NEXT:    vmov.32 q5[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q4[2]
+; CHECK-NEXT:    vmov.32 q5[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q4[3]
+; CHECK-NEXT:    vmov.32 q5[3], r0
+; CHECK-NEXT:    vcmp.i32 ne, q5, zr
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    and r1, r0, #1
+; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    vmov.32 q5[0], r1
+; CHECK-NEXT:    vmov.32 q5[1], r1
+; CHECK-NEXT:    ubfx r1, r0, #4, #1
+; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    vmov.32 q5[2], r1
+; CHECK-NEXT:    vmov.32 q5[3], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[0]
+; CHECK-NEXT:    sxtb r1, r1
+; CHECK-NEXT:    vmov.32 q6[0], r1
+; CHECK-NEXT:    asrs r1, r1, #31
+; CHECK-NEXT:    vmov.32 q6[1], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[1]
+; CHECK-NEXT:    sxtb r1, r1
+; CHECK-NEXT:    vmov.32 q6[2], r1
+; CHECK-NEXT:    asrs r1, r1, #31
+; CHECK-NEXT:    vmov.32 q6[3], r1
+; CHECK-NEXT:    vand q5, q6, q5
+; CHECK-NEXT:    vmov r1, s22
+; CHECK-NEXT:    vmov r2, s20
+; CHECK-NEXT:    vmov r12, s23
+; CHECK-NEXT:    vmov r3, s21
+; CHECK-NEXT:    adds r1, r1, r2
+; CHECK-NEXT:    adc.w r2, r3, r12
+; CHECK-NEXT:    ubfx r3, r0, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r0, r0, #12, #1
+; CHECK-NEXT:    vmov.32 q5[0], r3
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vmov.32 q5[1], r3
+; CHECK-NEXT:    vmov.32 q5[2], r0
+; CHECK-NEXT:    vmov.32 q5[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[2]
+; CHECK-NEXT:    sxtb r0, r0
+; CHECK-NEXT:    vmov.32 q6[0], r0
+; CHECK-NEXT:    asrs r0, r0, #31
+; CHECK-NEXT:    vmov.32 q6[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[3]
+; CHECK-NEXT:    sxtb r0, r0
+; CHECK-NEXT:    vmov.32 q6[2], r0
+; CHECK-NEXT:    asrs r0, r0, #31
+; CHECK-NEXT:    vmov.32 q6[3], r0
+; CHECK-NEXT:    vand q5, q6, q5
+; CHECK-NEXT:    vmov r3, s20
+; CHECK-NEXT:    vmov r0, s21
+; CHECK-NEXT:    adds r1, r1, r3
+; CHECK-NEXT:    vmov r3, s23
+; CHECK-NEXT:    adcs r2, r0
+; CHECK-NEXT:    vmov r0, s22
+; CHECK-NEXT:    adds.w r12, r1, r0
+; CHECK-NEXT:    adc.w r1, r2, r3
+; CHECK-NEXT:    vmov.u16 r2, q4[4]
+; CHECK-NEXT:    vmov.32 q5[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q4[5]
+; CHECK-NEXT:    vmov.32 q5[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q4[6]
+; CHECK-NEXT:    vmov.32 q5[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q4[7]
+; CHECK-NEXT:    vmov.32 q5[3], r2
+; CHECK-NEXT:    vcmp.i32 ne, q5, zr
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r3, r2, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q4[0], r3
+; CHECK-NEXT:    vmov.32 q4[1], r3
+; CHECK-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q4[2], r3
+; CHECK-NEXT:    vmov.32 q4[3], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[4]
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    vmov.32 q5[0], r3
+; CHECK-NEXT:    asrs r3, r3, #31
+; CHECK-NEXT:    vmov.32 q5[1], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[5]
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    vmov.32 q5[2], r3
+; CHECK-NEXT:    asrs r3, r3, #31
+; CHECK-NEXT:    vmov.32 q5[3], r3
+; CHECK-NEXT:    vand q4, q5, q4
+; CHECK-NEXT:    vmov r0, s16
+; CHECK-NEXT:    vmov r3, s17
+; CHECK-NEXT:    adds.w r12, r12, r0
+; CHECK-NEXT:    vmov r0, s18
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    vmov r3, s19
+; CHECK-NEXT:    adds.w r0, r0, r12
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    vmov.32 q4[0], r3
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov.32 q4[1], r3
+; CHECK-NEXT:    vmov.32 q4[2], r2
+; CHECK-NEXT:    vmov.32 q4[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[6]
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    vmov.32 q5[0], r2
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vmov.32 q5[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[7]
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    vmov.32 q5[2], r2
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vmov.32 q5[3], r2
+; CHECK-NEXT:    vand q4, q5, q4
+; CHECK-NEXT:    vmov r3, s16
+; CHECK-NEXT:    vmov r2, s17
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    vmov r3, s18
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    vmov r2, s19
+; CHECK-NEXT:    adds.w r12, r0, r3
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    vmov.u8 r2, q3[8]
+; CHECK-NEXT:    vmov.16 q4[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q3[9]
+; CHECK-NEXT:    vmov.16 q4[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q3[10]
+; CHECK-NEXT:    vmov.16 q4[2], r2
+; CHECK-NEXT:    vmov.u8 r2, q3[11]
+; CHECK-NEXT:    vmov.16 q4[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q3[12]
+; CHECK-NEXT:    vmov.16 q4[4], r2
+; CHECK-NEXT:    vmov.u8 r2, q3[13]
+; CHECK-NEXT:    vmov.16 q4[5], r2
+; CHECK-NEXT:    vmov.u8 r2, q3[14]
+; CHECK-NEXT:    vmov.16 q4[6], r2
+; CHECK-NEXT:    vmov.u8 r2, q3[15]
+; CHECK-NEXT:    vmov.16 q4[7], r2
+; CHECK-NEXT:    vcmp.i16 ne, q4, zr
+; CHECK-NEXT:    vpsel q1, q2, q1
+; CHECK-NEXT:    vmov.u16 r2, q1[0]
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[1]
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[2]
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[3]
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vcmp.i32 ne, q2, zr
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r3, r2, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q2[0], r3
+; CHECK-NEXT:    vmov.32 q2[1], r3
+; CHECK-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q2[2], r3
+; CHECK-NEXT:    vmov.32 q2[3], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[8]
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    vmov.32 q3[0], r3
+; CHECK-NEXT:    asrs r3, r3, #31
+; CHECK-NEXT:    vmov.32 q3[1], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[9]
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    vmov.32 q3[2], r3
+; CHECK-NEXT:    asrs r3, r3, #31
+; CHECK-NEXT:    vmov.32 q3[3], r3
+; CHECK-NEXT:    vand q2, q3, q2
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov r3, s9
+; CHECK-NEXT:    adds.w r12, r12, r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    vmov r3, s11
+; CHECK-NEXT:    adds.w r0, r0, r12
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    vmov.32 q2[0], r3
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov.32 q2[1], r3
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[10]
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vmov.32 q3[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[11]
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vmov.32 q3[3], r2
+; CHECK-NEXT:    vand q2, q3, q2
+; CHECK-NEXT:    vmov r3, s8
+; CHECK-NEXT:    vmov r2, s9
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    vmov r3, s10
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    vmov r2, s11
+; CHECK-NEXT:    adds.w r12, r0, r3
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    vmov.u16 r2, q1[4]
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[5]
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[6]
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[7]
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vcmp.i32 ne, q2, zr
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r3, r2, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q1[0], r3
+; CHECK-NEXT:    vmov.32 q1[1], r3
+; CHECK-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q1[2], r3
+; CHECK-NEXT:    vmov.32 q1[3], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[12]
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    vmov.32 q2[0], r3
+; CHECK-NEXT:    asrs r3, r3, #31
+; CHECK-NEXT:    vmov.32 q2[1], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[13]
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    vmov.32 q2[2], r3
+; CHECK-NEXT:    asrs r3, r3, #31
+; CHECK-NEXT:    vmov.32 q2[3], r3
+; CHECK-NEXT:    vand q1, q2, q1
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r3, s5
+; CHECK-NEXT:    adds.w r12, r12, r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    vmov r3, s7
+; CHECK-NEXT:    adds.w r0, r0, r12
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    vmov.32 q1[0], r3
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov.32 q1[1], r3
+; CHECK-NEXT:    vmov.32 q1[2], r2
+; CHECK-NEXT:    vmov.32 q1[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[14]
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[15]
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vand q0, q2, q1
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %xx = sext <16 x i8> %x to <16 x i64>
+  %s = select <16 x i1> %c, <16 x i64> %xx, <16 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v16i64(<16 x i64> %s)
+  ret i64 %z
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_zext(<2 x i8> %x, <2 x i8> %b) {
+; CHECK-LABEL: add_v2i8_v2i64_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i64 q2, #0xff
+; CHECK-NEXT:    vand q1, q1, q2
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov.32 q3[1], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vand q0, q0, q3
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    add r0, r1
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    orrs r1, r2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <2 x i8> %b, zeroinitializer
+  %xx = zext <2 x i8> %x to <2 x i64>
+  %s = select <2 x i1> %c, <2 x i64> %xx, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  ret i64 %z
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_sext(<2 x i8> %x, <2 x i8> %b) {
+; CHECK-LABEL: add_v2i8_v2i64_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i32 q2, #0xff
+; CHECK-NEXT:    vand q2, q1, q2
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    sxtb r0, r0
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    asrs r0, r0, #31
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    sxtb r0, r0
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    asrs r0, r0, #31
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vand q0, q2, q1
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <2 x i8> %b, zeroinitializer
+  %xx = sext <2 x i8> %x to <2 x i64>
+  %s = select <2 x i1> %c, <2 x i64> %xx, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  ret i64 %z
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i64_v2i64(<2 x i64> %x, <2 x i64> %b) {
+; CHECK-LABEL: add_v2i64_v2i64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov r1, s4
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s6
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <2 x i64> %b, zeroinitializer
+  %s = select <2 x i1> %c, <2 x i64> %x, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  ret i64 %z
+}
+
+define arm_aapcs_vfpcc i32 @add_v4i32_v4i32_acc(<4 x i32> %x, <4 x i32> %b, i32 %a) {
+; CHECK-LABEL: add_v4i32_v4i32_acc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q1, zr
+; CHECK-NEXT:    vaddvat.u32 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %b, zeroinitializer
+  %s = select <4 x i1> %c, <4 x i32> %x, <4 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %s)
+  %r = add i32 %z, %a
+  ret i32 %r
+}
+
+define arm_aapcs_vfpcc i64 @add_v4i32_v4i64_acc_zext(<4 x i32> %x, <4 x i32> %b, i64 %a) {
+; CHECK-LABEL: add_v4i32_v4i64_acc_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q1, zr
+; CHECK-NEXT:    vaddlvat.u32 r0, r1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %b, zeroinitializer
+  %xx = zext <4 x i32> %x to <4 x i64>
+  %s = select <4 x i1> %c, <4 x i64> %xx, <4 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v4i64(<4 x i64> %s)
+  %r = add i64 %z, %a
+  ret i64 %r
+}
+
+define arm_aapcs_vfpcc i64 @add_v4i32_v4i64_acc_sext(<4 x i32> %x, <4 x i32> %b, i64 %a) {
+; CHECK-LABEL: add_v4i32_v4i64_acc_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q1, zr
+; CHECK-NEXT:    vaddlvat.s32 r0, r1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %b, zeroinitializer
+  %xx = sext <4 x i32> %x to <4 x i64>
+  %s = select <4 x i1> %c, <4 x i64> %xx, <4 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v4i64(<4 x i64> %s)
+  %r = add i64 %z, %a
+  ret i64 %r
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i32_v2i64_acc_zext(<2 x i32> %x, <2 x i32> %b, i64 %a) {
+; CHECK-LABEL: add_v2i32_v2i64_acc_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    vmov.i64 q2, #0xffffffff
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r12, s3
+; CHECK-NEXT:    vmov lr, s1
+; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    adc.w r3, lr, r12
+; CHECK-NEXT:    adds r0, r0, r2
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %c = icmp eq <2 x i32> %b, zeroinitializer
+  %xx = zext <2 x i32> %x to <2 x i64>
+  %s = select <2 x i1> %c, <2 x i64> %xx, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  %r = add i64 %z, %a
+  ret i64 %r
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i32_v2i64_acc_sext(<2 x i32> %x, <2 x i32> %b, i64 %a) {
+; CHECK-LABEL: add_v2i32_v2i64_acc_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q0[2], r2
+; CHECK-NEXT:    vmov.32 q0[3], r2
+; CHECK-NEXT:    vand q0, q2, q0
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r12, s3
+; CHECK-NEXT:    vmov lr, s1
+; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    adc.w r3, lr, r12
+; CHECK-NEXT:    adds r0, r0, r2
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %c = icmp eq <2 x i32> %b, zeroinitializer
+  %xx = sext <2 x i32> %x to <2 x i64>
+  %s = select <2 x i1> %c, <2 x i64> %xx, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  %r = add i64 %z, %a
+  ret i64 %r
+}
+
+define arm_aapcs_vfpcc i32 @add_v8i16_v8i32_acc_zext(<8 x i16> %x, <8 x i16> %b, i32 %a) {
+; CHECK-LABEL: add_v8i16_v8i32_acc_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q1, zr
+; CHECK-NEXT:    vaddvat.u16 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %b, zeroinitializer
+  %xx = zext <8 x i16> %x to <8 x i32>
+  %s = select <8 x i1> %c, <8 x i32> %xx, <8 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> %s)
+  %r = add i32 %z, %a
+  ret i32 %r
+}
+
+define arm_aapcs_vfpcc i32 @add_v8i16_v8i32_acc_sext(<8 x i16> %x, <8 x i16> %b, i32 %a) {
+; CHECK-LABEL: add_v8i16_v8i32_acc_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q1, zr
+; CHECK-NEXT:    vaddvat.s16 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %b, zeroinitializer
+  %xx = sext <8 x i16> %x to <8 x i32>
+  %s = select <8 x i1> %c, <8 x i32> %xx, <8 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> %s)
+  %r = add i32 %z, %a
+  ret i32 %r
+}
+
+define arm_aapcs_vfpcc i32 @add_v4i16_v4i32_acc_zext(<4 x i16> %x, <4 x i16> %b, i32 %a) {
+; CHECK-LABEL: add_v4i16_v4i32_acc_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.u16 q0, q0
+; CHECK-NEXT:    vmovlb.u16 q1, q1
+; CHECK-NEXT:    vpt.i32 eq, q1, zr
+; CHECK-NEXT:    vaddvat.u32 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i16> %b, zeroinitializer
+  %xx = zext <4 x i16> %x to <4 x i32>
+  %s = select <4 x i1> %c, <4 x i32> %xx, <4 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %s)
+  %r = add i32 %z, %a
+  ret i32 %r
+}
+
+define arm_aapcs_vfpcc i32 @add_v4i16_v4i32_acc_sext(<4 x i16> %x, <4 x i16> %b, i32 %a) {
+; CHECK-LABEL: add_v4i16_v4i32_acc_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    vmovlb.u16 q1, q1
+; CHECK-NEXT:    vpt.i32 eq, q1, zr
+; CHECK-NEXT:    vaddvat.u32 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i16> %b, zeroinitializer
+  %xx = sext <4 x i16> %x to <4 x i32>
+  %s = select <4 x i1> %c, <4 x i32> %xx, <4 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %s)
+  %r = add i32 %z, %a
+  ret i32 %r
+}
+
+define arm_aapcs_vfpcc zeroext i16 @add_v8i16_v8i16_acc(<8 x i16> %x, <8 x i16> %b, i16 %a) {
+; CHECK-LABEL: add_v8i16_v8i16_acc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q1, zr
+; CHECK-NEXT:    vaddvat.u16 r0, q0
+; CHECK-NEXT:    uxth r0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %b, zeroinitializer
+  %s = select <8 x i1> %c, <8 x i16> %x, <8 x i16> zeroinitializer
+  %z = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> %s)
+  %r = add i16 %z, %a
+  ret i16 %r
+}
+
+define arm_aapcs_vfpcc i64 @add_v8i16_v8i64_acc_zext(<8 x i16> %x, <8 x i16> %b, i64 %a) {
+; CHECK-LABEL: add_v8i16_v8i64_acc_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vmov.i8 q2, #0x0
+; CHECK-NEXT:    vmov.i8 q3, #0xff
+; CHECK-NEXT:    vcmp.i16 eq, q1, zr
+; CHECK-NEXT:    vpsel q2, q3, q2
+; CHECK-NEXT:    vmov.u16 r2, q2[0]
+; CHECK-NEXT:    vmov.32 q1[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[1]
+; CHECK-NEXT:    vmov.32 q1[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[2]
+; CHECK-NEXT:    vmov.32 q1[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[3]
+; CHECK-NEXT:    vmov.32 q1[3], r2
+; CHECK-NEXT:    vcmp.i32 ne, q1, zr
+; CHECK-NEXT:    vmov.i64 q1, #0xffff
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r3, r2, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q3[0], r3
+; CHECK-NEXT:    vmov.32 q3[1], r3
+; CHECK-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q3[2], r3
+; CHECK-NEXT:    vmov.32 q3[3], r3
+; CHECK-NEXT:    vmov.u16 r3, q0[0]
+; CHECK-NEXT:    vmov.32 q4[0], r3
+; CHECK-NEXT:    vmov.u16 r3, q0[1]
+; CHECK-NEXT:    vmov.32 q4[2], r3
+; CHECK-NEXT:    vand q4, q4, q1
+; CHECK-NEXT:    vand q3, q4, q3
+; CHECK-NEXT:    vmov r12, s15
+; CHECK-NEXT:    vmov r3, s13
+; CHECK-NEXT:    vmov lr, s14
+; CHECK-NEXT:    orr.w r12, r12, r3
+; CHECK-NEXT:    vmov r3, s12
+; CHECK-NEXT:    add lr, r3
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    vmov.32 q3[0], r3
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov.32 q3[1], r3
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    vmov.32 q3[3], r2
+; CHECK-NEXT:    vmov.u16 r2, q0[2]
+; CHECK-NEXT:    vmov.32 q4[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q0[3]
+; CHECK-NEXT:    vmov.32 q4[2], r2
+; CHECK-NEXT:    vand q4, q4, q1
+; CHECK-NEXT:    vand q3, q4, q3
+; CHECK-NEXT:    vmov r3, s12
+; CHECK-NEXT:    vmov r2, s13
+; CHECK-NEXT:    adds.w r4, lr, r3
+; CHECK-NEXT:    vmov r3, s14
+; CHECK-NEXT:    adc.w lr, r12, r2
+; CHECK-NEXT:    vmov r2, s15
+; CHECK-NEXT:    adds.w r12, r4, r3
+; CHECK-NEXT:    adc.w lr, lr, r2
+; CHECK-NEXT:    vmov.u16 r2, q2[4]
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[5]
+; CHECK-NEXT:    vmov.32 q3[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[6]
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[7]
+; CHECK-NEXT:    vmov.32 q3[3], r2
+; CHECK-NEXT:    vcmp.i32 ne, q3, zr
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r4, r2, #1
+; CHECK-NEXT:    rsbs r4, r4, #0
+; CHECK-NEXT:    vmov.32 q2[0], r4
+; CHECK-NEXT:    vmov.32 q2[1], r4
+; CHECK-NEXT:    ubfx r4, r2, #4, #1
+; CHECK-NEXT:    rsbs r4, r4, #0
+; CHECK-NEXT:    vmov.32 q2[2], r4
+; CHECK-NEXT:    vmov.32 q2[3], r4
+; CHECK-NEXT:    vmov.u16 r4, q0[4]
+; CHECK-NEXT:    vmov.32 q3[0], r4
+; CHECK-NEXT:    vmov.u16 r4, q0[5]
+; CHECK-NEXT:    vmov.32 q3[2], r4
+; CHECK-NEXT:    vand q3, q3, q1
+; CHECK-NEXT:    vand q2, q3, q2
+; CHECK-NEXT:    vmov r3, s8
+; CHECK-NEXT:    vmov r4, s9
+; CHECK-NEXT:    adds.w r12, r12, r3
+; CHECK-NEXT:    vmov r3, s11
+; CHECK-NEXT:    adc.w lr, lr, r4
+; CHECK-NEXT:    vmov r4, s10
+; CHECK-NEXT:    adds.w r4, r4, r12
+; CHECK-NEXT:    adc.w r12, lr, r3
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    vmov.32 q2[0], r3
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov.32 q2[1], r3
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vmov.u16 r2, q0[6]
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q0[7]
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    vand q0, q3, q1
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    adds r3, r3, r4
+; CHECK-NEXT:    vmov r4, s3
+; CHECK-NEXT:    adc.w r12, r12, r2
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    adc.w r3, r12, r4
+; CHECK-NEXT:    adds r0, r0, r2
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    pop {r4, pc}
+entry:
+  %c = icmp eq <8 x i16> %b, zeroinitializer
+  %xx = zext <8 x i16> %x to <8 x i64>
+  %s = select <8 x i1> %c, <8 x i64> %xx, <8 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v8i64(<8 x i64> %s)
+  %r = add i64 %z, %a
+  ret i64 %r
+}
+
+define arm_aapcs_vfpcc i64 @add_v8i16_v8i64_acc_sext(<8 x i16> %x, <8 x i16> %b, i64 %a) {
+; CHECK-LABEL: add_v8i16_v8i64_acc_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    vmov.i8 q2, #0x0
+; CHECK-NEXT:    vmov.i8 q3, #0xff
+; CHECK-NEXT:    vcmp.i16 eq, q1, zr
+; CHECK-NEXT:    vpsel q1, q3, q2
+; CHECK-NEXT:    vmov.u16 r2, q1[0]
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[1]
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[2]
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[3]
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vcmp.i32 ne, q2, zr
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r3, r2, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q2[0], r3
+; CHECK-NEXT:    vmov.32 q2[1], r3
+; CHECK-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q2[2], r3
+; CHECK-NEXT:    vmov.32 q2[3], r3
+; CHECK-NEXT:    vmov.u16 r3, q0[0]
+; CHECK-NEXT:    sxth r3, r3
+; CHECK-NEXT:    vmov.32 q3[0], r3
+; CHECK-NEXT:    asrs r3, r3, #31
+; CHECK-NEXT:    vmov.32 q3[1], r3
+; CHECK-NEXT:    vmov.u16 r3, q0[1]
+; CHECK-NEXT:    sxth r3, r3
+; CHECK-NEXT:    vmov.32 q3[2], r3
+; CHECK-NEXT:    asrs r3, r3, #31
+; CHECK-NEXT:    vmov.32 q3[3], r3
+; CHECK-NEXT:    vand q2, q3, q2
+; CHECK-NEXT:    vmov r3, s10
+; CHECK-NEXT:    vmov r4, s8
+; CHECK-NEXT:    vmov r12, s11
+; CHECK-NEXT:    vmov r5, s9
+; CHECK-NEXT:    adds.w lr, r4, r3
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    rsb.w r3, r3, #0
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    vmov.32 q2[0], r3
+; CHECK-NEXT:    rsb.w r2, r2, #0
+; CHECK-NEXT:    vmov.32 q2[1], r3
+; CHECK-NEXT:    adc.w r12, r12, r5
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vmov.u16 r2, q0[2]
+; CHECK-NEXT:    sxth r2, r2
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vmov.32 q3[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q0[3]
+; CHECK-NEXT:    sxth r2, r2
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vmov.32 q3[3], r2
+; CHECK-NEXT:    vand q2, q3, q2
+; CHECK-NEXT:    vmov r3, s8
+; CHECK-NEXT:    vmov r2, s9
+; CHECK-NEXT:    vmov r4, s10
+; CHECK-NEXT:    vmov r5, s11
+; CHECK-NEXT:    adds.w r3, r3, lr
+; CHECK-NEXT:    adc.w r2, r2, r12
+; CHECK-NEXT:    adds.w r12, r3, r4
+; CHECK-NEXT:    adc.w r3, r2, r5
+; CHECK-NEXT:    vmov.u16 r2, q1[4]
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[5]
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[6]
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[7]
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vcmp.i32 ne, q2, zr
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r5, r2, #1
+; CHECK-NEXT:    rsbs r5, r5, #0
+; CHECK-NEXT:    vmov.32 q1[0], r5
+; CHECK-NEXT:    vmov.32 q1[1], r5
+; CHECK-NEXT:    ubfx r5, r2, #4, #1
+; CHECK-NEXT:    rsbs r5, r5, #0
+; CHECK-NEXT:    vmov.32 q1[2], r5
+; CHECK-NEXT:    vmov.32 q1[3], r5
+; CHECK-NEXT:    vmov.u16 r5, q0[4]
+; CHECK-NEXT:    sxth r5, r5
+; CHECK-NEXT:    vmov.32 q2[0], r5
+; CHECK-NEXT:    asrs r5, r5, #31
+; CHECK-NEXT:    vmov.32 q2[1], r5
+; CHECK-NEXT:    vmov.u16 r5, q0[5]
+; CHECK-NEXT:    sxth r5, r5
+; CHECK-NEXT:    vmov.32 q2[2], r5
+; CHECK-NEXT:    asrs r5, r5, #31
+; CHECK-NEXT:    vmov.32 q2[3], r5
+; CHECK-NEXT:    vand q1, q2, q1
+; CHECK-NEXT:    vmov r4, s4
+; CHECK-NEXT:    vmov r5, s5
+; CHECK-NEXT:    adds.w r12, r12, r4
+; CHECK-NEXT:    vmov r4, s6
+; CHECK-NEXT:    adcs r5, r3
+; CHECK-NEXT:    vmov r3, s7
+; CHECK-NEXT:    adds.w r4, r4, r12
+; CHECK-NEXT:    adc.w r12, r5, r3
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    vmov.32 q1[0], r3
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov.32 q1[1], r3
+; CHECK-NEXT:    vmov.32 q1[2], r2
+; CHECK-NEXT:    vmov.32 q1[3], r2
+; CHECK-NEXT:    vmov.u16 r2, q0[6]
+; CHECK-NEXT:    sxth r2, r2
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q0[7]
+; CHECK-NEXT:    sxth r2, r2
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vand q0, q2, q1
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vmov r5, s3
+; CHECK-NEXT:    adds r3, r3, r4
+; CHECK-NEXT:    vmov r4, s2
+; CHECK-NEXT:    adc.w r2, r2, r12
+; CHECK-NEXT:    adds r3, r3, r4
+; CHECK-NEXT:    adcs r2, r5
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+entry:
+  %c = icmp eq <8 x i16> %b, zeroinitializer
+  %xx = sext <8 x i16> %x to <8 x i64>
+  %s = select <8 x i1> %c, <8 x i64> %xx, <8 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v8i64(<8 x i64> %s)
+  %r = add i64 %z, %a
+  ret i64 %r
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_acc_zext(<2 x i16> %x, <2 x i16> %b, i64 %a) {
+; CHECK-LABEL: add_v2i16_v2i64_acc_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i64 q2, #0xffff
+; CHECK-NEXT:    vand q1, q1, q2
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    vmov.32 q3[1], r2
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    vmov.32 q3[3], r2
+; CHECK-NEXT:    vand q0, q0, q3
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r3, s1
+; CHECK-NEXT:    orr.w r12, r3, r2
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    add r2, r3
+; CHECK-NEXT:    adds r0, r0, r2
+; CHECK-NEXT:    adc.w r1, r1, r12
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <2 x i16> %b, zeroinitializer
+  %xx = zext <2 x i16> %x to <2 x i64>
+  %s = select <2 x i1> %c, <2 x i64> %xx, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  %r = add i64 %z, %a
+  ret i64 %r
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_acc_sext(<2 x i16> %x, <2 x i16> %b, i64 %a) {
+; CHECK-LABEL: add_v2i16_v2i64_acc_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vmov.i32 q2, #0xffff
+; CHECK-NEXT:    vand q2, q1, q2
+; CHECK-NEXT:    vmov r2, s8
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q1[0], r2
+; CHECK-NEXT:    vmov.32 q1[1], r2
+; CHECK-NEXT:    vmov r2, s10
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q1[2], r2
+; CHECK-NEXT:    vmov.32 q1[3], r2
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    sxth r2, r2
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    sxth r2, r2
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vand q0, q2, q1
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r12, s3
+; CHECK-NEXT:    vmov lr, s1
+; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    adc.w r3, lr, r12
+; CHECK-NEXT:    adds r0, r0, r2
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %c = icmp eq <2 x i16> %b, zeroinitializer
+  %xx = sext <2 x i16> %x to <2 x i64>
+  %s = select <2 x i1> %c, <2 x i64> %xx, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  %r = add i64 %z, %a
+  ret i64 %r
+}
+
+define arm_aapcs_vfpcc i32 @add_v16i8_v16i32_acc_zext(<16 x i8> %x, <16 x i8> %b, i32 %a) {
+; CHECK-LABEL: add_v16i8_v16i32_acc_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q1, zr
+; CHECK-NEXT:    vaddvat.u8 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %xx = zext <16 x i8> %x to <16 x i32>
+  %s = select <16 x i1> %c, <16 x i32> %xx, <16 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v16i32(<16 x i32> %s)
+  %r = add i32 %z, %a
+  ret i32 %r
+}
+
+define arm_aapcs_vfpcc i32 @add_v16i8_v16i32_acc_sext(<16 x i8> %x, <16 x i8> %b, i32 %a) {
+; CHECK-LABEL: add_v16i8_v16i32_acc_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q1, zr
+; CHECK-NEXT:    vaddvat.s8 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %xx = sext <16 x i8> %x to <16 x i32>
+  %s = select <16 x i1> %c, <16 x i32> %xx, <16 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v16i32(<16 x i32> %s)
+  %r = add i32 %z, %a
+  ret i32 %r
+}
+
+define arm_aapcs_vfpcc i32 @add_v4i8_v4i32_acc_zext(<4 x i8> %x, <4 x i8> %b, i32 %a) {
+; CHECK-LABEL: add_v4i8_v4i32_acc_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i32 q2, #0xff
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vand q1, q1, q2
+; CHECK-NEXT:    vpt.i32 eq, q1, zr
+; CHECK-NEXT:    vaddvat.u32 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i8> %b, zeroinitializer
+  %xx = zext <4 x i8> %x to <4 x i32>
+  %s = select <4 x i1> %c, <4 x i32> %xx, <4 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %s)
+  %r = add i32 %z, %a
+  ret i32 %r
+}
+
+define arm_aapcs_vfpcc i32 @add_v4i8_v4i32_acc_sext(<4 x i8> %x, <4 x i8> %b, i32 %a) {
+; CHECK-LABEL: add_v4i8_v4i32_acc_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    vmov.i32 q2, #0xff
+; CHECK-NEXT:    vand q1, q1, q2
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    vpt.i32 eq, q1, zr
+; CHECK-NEXT:    vaddvat.u32 r0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i8> %b, zeroinitializer
+  %xx = sext <4 x i8> %x to <4 x i32>
+  %s = select <4 x i1> %c, <4 x i32> %xx, <4 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %s)
+  %r = add i32 %z, %a
+  ret i32 %r
+}
+
+define arm_aapcs_vfpcc zeroext i16 @add_v16i8_v16i16_acc_zext(<16 x i8> %x, <16 x i8> %b, i16 %a) {
+; CHECK-LABEL: add_v16i8_v16i16_acc_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i8 eq, q1, zr
+; CHECK-NEXT:    vmov.i8 q1, #0x0
+; CHECK-NEXT:    vmov.i8 q2, #0xff
+; CHECK-NEXT:    vmov.i32 q3, #0x0
+; CHECK-NEXT:    vpsel q1, q2, q1
+; CHECK-NEXT:    vmov.u8 r1, q1[0]
+; CHECK-NEXT:    vmov.16 q2[0], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[1]
+; CHECK-NEXT:    vmov.16 q2[1], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[2]
+; CHECK-NEXT:    vmov.16 q2[2], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[3]
+; CHECK-NEXT:    vmov.16 q2[3], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[4]
+; CHECK-NEXT:    vmov.16 q2[4], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[5]
+; CHECK-NEXT:    vmov.16 q2[5], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[6]
+; CHECK-NEXT:    vmov.16 q2[6], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[7]
+; CHECK-NEXT:    vmov.16 q2[7], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[0]
+; CHECK-NEXT:    vcmp.i16 ne, q2, zr
+; CHECK-NEXT:    vmov.16 q2[0], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[1]
+; CHECK-NEXT:    vmov.16 q2[1], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[2]
+; CHECK-NEXT:    vmov.16 q2[2], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[3]
+; CHECK-NEXT:    vmov.16 q2[3], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[4]
+; CHECK-NEXT:    vmov.16 q2[4], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[5]
+; CHECK-NEXT:    vmov.16 q2[5], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[6]
+; CHECK-NEXT:    vmov.16 q2[6], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[7]
+; CHECK-NEXT:    vmov.16 q2[7], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[8]
+; CHECK-NEXT:    vmovlb.u8 q2, q2
+; CHECK-NEXT:    vpsel q2, q2, q3
+; CHECK-NEXT:    vmov.16 q3[0], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[9]
+; CHECK-NEXT:    vmov.16 q3[1], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[10]
+; CHECK-NEXT:    vmov.16 q3[2], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[11]
+; CHECK-NEXT:    vmov.16 q3[3], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[12]
+; CHECK-NEXT:    vmov.16 q3[4], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[13]
+; CHECK-NEXT:    vmov.16 q3[5], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[14]
+; CHECK-NEXT:    vmov.16 q3[6], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[15]
+; CHECK-NEXT:    vmov.16 q3[7], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[8]
+; CHECK-NEXT:    vmov.16 q1[0], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[9]
+; CHECK-NEXT:    vmov.16 q1[1], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[10]
+; CHECK-NEXT:    vmov.16 q1[2], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[11]
+; CHECK-NEXT:    vmov.16 q1[3], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[12]
+; CHECK-NEXT:    vmov.16 q1[4], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[13]
+; CHECK-NEXT:    vmov.16 q1[5], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[14]
+; CHECK-NEXT:    vmov.16 q1[6], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[15]
+; CHECK-NEXT:    vmov.16 q1[7], r1
+; CHECK-NEXT:    vmovlb.u8 q0, q1
+; CHECK-NEXT:    vpt.i16 ne, q3, zr
+; CHECK-NEXT:    vaddt.i16 q2, q2, q0
+; CHECK-NEXT:    vaddva.u16 r0, q2
+; CHECK-NEXT:    uxth r0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %xx = zext <16 x i8> %x to <16 x i16>
+  %s = select <16 x i1> %c, <16 x i16> %xx, <16 x i16> zeroinitializer
+  %z = call i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16> %s)
+  %r = add i16 %z, %a
+  ret i16 %r
+}
+
+define arm_aapcs_vfpcc signext i16 @add_v16i8_v16i16_acc_sext(<16 x i8> %x, <16 x i8> %b, i16 %a) {
+; CHECK-LABEL: add_v16i8_v16i16_acc_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vcmp.i8 eq, q1, zr
+; CHECK-NEXT:    vmov.i8 q1, #0x0
+; CHECK-NEXT:    vmov.i8 q2, #0xff
+; CHECK-NEXT:    vmov.i32 q3, #0x0
+; CHECK-NEXT:    vpsel q1, q2, q1
+; CHECK-NEXT:    vmov.u8 r1, q1[0]
+; CHECK-NEXT:    vmov.16 q2[0], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[1]
+; CHECK-NEXT:    vmov.16 q2[1], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[2]
+; CHECK-NEXT:    vmov.16 q2[2], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[3]
+; CHECK-NEXT:    vmov.16 q2[3], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[4]
+; CHECK-NEXT:    vmov.16 q2[4], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[5]
+; CHECK-NEXT:    vmov.16 q2[5], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[6]
+; CHECK-NEXT:    vmov.16 q2[6], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[7]
+; CHECK-NEXT:    vmov.16 q2[7], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[0]
+; CHECK-NEXT:    vcmp.i16 ne, q2, zr
+; CHECK-NEXT:    vmov.16 q2[0], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[1]
+; CHECK-NEXT:    vmov.16 q2[1], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[2]
+; CHECK-NEXT:    vmov.16 q2[2], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[3]
+; CHECK-NEXT:    vmov.16 q2[3], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[4]
+; CHECK-NEXT:    vmov.16 q2[4], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[5]
+; CHECK-NEXT:    vmov.16 q2[5], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[6]
+; CHECK-NEXT:    vmov.16 q2[6], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[7]
+; CHECK-NEXT:    vmov.16 q2[7], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[8]
+; CHECK-NEXT:    vmovlb.s8 q2, q2
+; CHECK-NEXT:    vpsel q2, q2, q3
+; CHECK-NEXT:    vmov.16 q3[0], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[9]
+; CHECK-NEXT:    vmov.16 q3[1], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[10]
+; CHECK-NEXT:    vmov.16 q3[2], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[11]
+; CHECK-NEXT:    vmov.16 q3[3], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[12]
+; CHECK-NEXT:    vmov.16 q3[4], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[13]
+; CHECK-NEXT:    vmov.16 q3[5], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[14]
+; CHECK-NEXT:    vmov.16 q3[6], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[15]
+; CHECK-NEXT:    vmov.16 q3[7], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[8]
+; CHECK-NEXT:    vmov.16 q1[0], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[9]
+; CHECK-NEXT:    vmov.16 q1[1], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[10]
+; CHECK-NEXT:    vmov.16 q1[2], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[11]
+; CHECK-NEXT:    vmov.16 q1[3], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[12]
+; CHECK-NEXT:    vmov.16 q1[4], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[13]
+; CHECK-NEXT:    vmov.16 q1[5], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[14]
+; CHECK-NEXT:    vmov.16 q1[6], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[15]
+; CHECK-NEXT:    vmov.16 q1[7], r1
+; CHECK-NEXT:    vmovlb.s8 q0, q1
+; CHECK-NEXT:    vpt.i16 ne, q3, zr
+; CHECK-NEXT:    vaddt.i16 q2, q2, q0
+; CHECK-NEXT:    vaddva.u16 r0, q2
+; CHECK-NEXT:    sxth r0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %xx = sext <16 x i8> %x to <16 x i16>
+  %s = select <16 x i1> %c, <16 x i16> %xx, <16 x i16> zeroinitializer
+  %z = call i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16> %s)
+  %r = add i16 %z, %a
+  ret i16 %r
+}
+
+define arm_aapcs_vfpcc zeroext i16 @add_v8i8_v8i16_acc_zext(<8 x i8> %x, <8 x i8> %b, i16 %a) {
+; CHECK-LABEL: add_v8i8_v8i16_acc_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.u8 q0, q0
+; CHECK-NEXT:    vmovlb.u8 q1, q1
+; CHECK-NEXT:    vpt.i16 eq, q1, zr
+; CHECK-NEXT:    vaddvat.u16 r0, q0
+; CHECK-NEXT:    uxth r0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i8> %b, zeroinitializer
+  %xx = zext <8 x i8> %x to <8 x i16>
+  %s = select <8 x i1> %c, <8 x i16> %xx, <8 x i16> zeroinitializer
+  %z = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> %s)
+  %r = add i16 %z, %a
+  ret i16 %r
+}
+
+define arm_aapcs_vfpcc signext i16 @add_v8i8_v8i16_acc_sext(<8 x i8> %x, <8 x i8> %b, i16 %a) {
+; CHECK-LABEL: add_v8i8_v8i16_acc_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    vmovlb.u8 q1, q1
+; CHECK-NEXT:    vpt.i16 eq, q1, zr
+; CHECK-NEXT:    vaddvat.u16 r0, q0
+; CHECK-NEXT:    sxth r0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i8> %b, zeroinitializer
+  %xx = sext <8 x i8> %x to <8 x i16>
+  %s = select <8 x i1> %c, <8 x i16> %xx, <8 x i16> zeroinitializer
+  %z = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> %s)
+  %r = add i16 %z, %a
+  ret i16 %r
+}
+
+define arm_aapcs_vfpcc zeroext i8 @add_v16i8_v16i8_acc(<16 x i8> %x, <16 x i8> %b, i8 %a) {
+; CHECK-LABEL: add_v16i8_v16i8_acc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q1, zr
+; CHECK-NEXT:    vaddvat.u8 r0, q0
+; CHECK-NEXT:    uxtb r0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %s = select <16 x i1> %c, <16 x i8> %x, <16 x i8> zeroinitializer
+  %z = call i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8> %s)
+  %r = add i8 %z, %a
+  ret i8 %r
+}
+
+define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_acc_zext(<16 x i8> %x, <16 x i8> %b, i64 %a) {
+; CHECK-LABEL: add_v16i8_v16i64_acc_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vcmp.i8 eq, q1, zr
+; CHECK-NEXT:    vmov.i8 q2, #0x0
+; CHECK-NEXT:    vmov.i8 q3, #0xff
+; CHECK-NEXT:    vpsel q4, q3, q2
+; CHECK-NEXT:    vmov.u8 r2, q4[0]
+; CHECK-NEXT:    vmov.16 q1[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[1]
+; CHECK-NEXT:    vmov.16 q1[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[2]
+; CHECK-NEXT:    vmov.16 q1[2], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[3]
+; CHECK-NEXT:    vmov.16 q1[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[4]
+; CHECK-NEXT:    vmov.16 q1[4], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[5]
+; CHECK-NEXT:    vmov.16 q1[5], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[6]
+; CHECK-NEXT:    vmov.16 q1[6], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[7]
+; CHECK-NEXT:    vmov.16 q1[7], r2
+; CHECK-NEXT:    vcmp.i16 ne, q1, zr
+; CHECK-NEXT:    vpsel q5, q3, q2
+; CHECK-NEXT:    vmov.u16 r2, q5[0]
+; CHECK-NEXT:    vmov.32 q1[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q5[1]
+; CHECK-NEXT:    vmov.32 q1[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q5[2]
+; CHECK-NEXT:    vmov.32 q1[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q5[3]
+; CHECK-NEXT:    vmov.32 q1[3], r2
+; CHECK-NEXT:    vcmp.i32 ne, q1, zr
+; CHECK-NEXT:    vmov.i64 q1, #0xff
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r3, r2, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q6[0], r3
+; CHECK-NEXT:    vmov.32 q6[1], r3
+; CHECK-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q6[2], r3
+; CHECK-NEXT:    vmov.32 q6[3], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[0]
+; CHECK-NEXT:    vmov.32 q7[0], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[1]
+; CHECK-NEXT:    vmov.32 q7[2], r3
+; CHECK-NEXT:    vand q7, q7, q1
+; CHECK-NEXT:    vand q6, q7, q6
+; CHECK-NEXT:    vmov r12, s27
+; CHECK-NEXT:    vmov r3, s25
+; CHECK-NEXT:    vmov lr, s26
+; CHECK-NEXT:    orr.w r12, r12, r3
+; CHECK-NEXT:    vmov r3, s24
+; CHECK-NEXT:    add lr, r3
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    vmov.32 q6[0], r3
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov.32 q6[1], r3
+; CHECK-NEXT:    vmov.32 q6[2], r2
+; CHECK-NEXT:    vmov.32 q6[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[2]
+; CHECK-NEXT:    vmov.32 q7[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[3]
+; CHECK-NEXT:    vmov.32 q7[2], r2
+; CHECK-NEXT:    vand q7, q7, q1
+; CHECK-NEXT:    vand q6, q7, q6
+; CHECK-NEXT:    vmov r3, s24
+; CHECK-NEXT:    vmov r2, s25
+; CHECK-NEXT:    adds.w r4, lr, r3
+; CHECK-NEXT:    vmov r3, s26
+; CHECK-NEXT:    adc.w lr, r12, r2
+; CHECK-NEXT:    vmov r2, s27
+; CHECK-NEXT:    adds.w r12, r4, r3
+; CHECK-NEXT:    adc.w lr, lr, r2
+; CHECK-NEXT:    vmov.u16 r2, q5[4]
+; CHECK-NEXT:    vmov.32 q6[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q5[5]
+; CHECK-NEXT:    vmov.32 q6[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q5[6]
+; CHECK-NEXT:    vmov.32 q6[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q5[7]
+; CHECK-NEXT:    vmov.32 q6[3], r2
+; CHECK-NEXT:    vcmp.i32 ne, q6, zr
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r4, r2, #1
+; CHECK-NEXT:    rsbs r4, r4, #0
+; CHECK-NEXT:    vmov.32 q5[0], r4
+; CHECK-NEXT:    vmov.32 q5[1], r4
+; CHECK-NEXT:    ubfx r4, r2, #4, #1
+; CHECK-NEXT:    rsbs r4, r4, #0
+; CHECK-NEXT:    vmov.32 q5[2], r4
+; CHECK-NEXT:    vmov.32 q5[3], r4
+; CHECK-NEXT:    vmov.u8 r4, q0[4]
+; CHECK-NEXT:    vmov.32 q6[0], r4
+; CHECK-NEXT:    vmov.u8 r4, q0[5]
+; CHECK-NEXT:    vmov.32 q6[2], r4
+; CHECK-NEXT:    vand q6, q6, q1
+; CHECK-NEXT:    vand q5, q6, q5
+; CHECK-NEXT:    vmov r3, s20
+; CHECK-NEXT:    vmov r4, s21
+; CHECK-NEXT:    adds.w r12, r12, r3
+; CHECK-NEXT:    vmov r3, s23
+; CHECK-NEXT:    adc.w lr, lr, r4
+; CHECK-NEXT:    vmov r4, s22
+; CHECK-NEXT:    adds.w r4, r4, r12
+; CHECK-NEXT:    adc.w r12, lr, r3
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    vmov.32 q5[0], r3
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov.32 q5[1], r3
+; CHECK-NEXT:    vmov.32 q5[2], r2
+; CHECK-NEXT:    vmov.32 q5[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[6]
+; CHECK-NEXT:    vmov.32 q6[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[7]
+; CHECK-NEXT:    vmov.32 q6[2], r2
+; CHECK-NEXT:    vand q6, q6, q1
+; CHECK-NEXT:    vand q5, q6, q5
+; CHECK-NEXT:    vmov r3, s20
+; CHECK-NEXT:    vmov r2, s21
+; CHECK-NEXT:    adds r3, r3, r4
+; CHECK-NEXT:    vmov r4, s23
+; CHECK-NEXT:    adc.w lr, r12, r2
+; CHECK-NEXT:    vmov r2, s22
+; CHECK-NEXT:    adds.w r12, r3, r2
+; CHECK-NEXT:    vmov.u8 r2, q4[8]
+; CHECK-NEXT:    vmov.16 q5[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[9]
+; CHECK-NEXT:    vmov.16 q5[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[10]
+; CHECK-NEXT:    vmov.16 q5[2], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[11]
+; CHECK-NEXT:    vmov.16 q5[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[12]
+; CHECK-NEXT:    vmov.16 q5[4], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[13]
+; CHECK-NEXT:    vmov.16 q5[5], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[14]
+; CHECK-NEXT:    vmov.16 q5[6], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[15]
+; CHECK-NEXT:    vmov.16 q5[7], r2
+; CHECK-NEXT:    adc.w lr, lr, r4
+; CHECK-NEXT:    vcmp.i16 ne, q5, zr
+; CHECK-NEXT:    vpsel q2, q3, q2
+; CHECK-NEXT:    vmov.u16 r2, q2[0]
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[1]
+; CHECK-NEXT:    vmov.32 q3[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[2]
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[3]
+; CHECK-NEXT:    vmov.32 q3[3], r2
+; CHECK-NEXT:    vcmp.i32 ne, q3, zr
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r4, r2, #1
+; CHECK-NEXT:    rsbs r4, r4, #0
+; CHECK-NEXT:    vmov.32 q3[0], r4
+; CHECK-NEXT:    vmov.32 q3[1], r4
+; CHECK-NEXT:    ubfx r4, r2, #4, #1
+; CHECK-NEXT:    rsbs r4, r4, #0
+; CHECK-NEXT:    vmov.32 q3[2], r4
+; CHECK-NEXT:    vmov.32 q3[3], r4
+; CHECK-NEXT:    vmov.u8 r4, q0[8]
+; CHECK-NEXT:    vmov.32 q4[0], r4
+; CHECK-NEXT:    vmov.u8 r4, q0[9]
+; CHECK-NEXT:    vmov.32 q4[2], r4
+; CHECK-NEXT:    vand q4, q4, q1
+; CHECK-NEXT:    vand q3, q4, q3
+; CHECK-NEXT:    vmov r3, s12
+; CHECK-NEXT:    vmov r4, s13
+; CHECK-NEXT:    adds.w r12, r12, r3
+; CHECK-NEXT:    vmov r3, s15
+; CHECK-NEXT:    adc.w lr, lr, r4
+; CHECK-NEXT:    vmov r4, s14
+; CHECK-NEXT:    adds.w r4, r4, r12
+; CHECK-NEXT:    adc.w r12, lr, r3
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    vmov.32 q3[0], r3
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov.32 q3[1], r3
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    vmov.32 q3[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[10]
+; CHECK-NEXT:    vmov.32 q4[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[11]
+; CHECK-NEXT:    vmov.32 q4[2], r2
+; CHECK-NEXT:    vand q4, q4, q1
+; CHECK-NEXT:    vand q3, q4, q3
+; CHECK-NEXT:    vmov r3, s12
+; CHECK-NEXT:    vmov r2, s13
+; CHECK-NEXT:    adds r3, r3, r4
+; CHECK-NEXT:    vmov r4, s15
+; CHECK-NEXT:    adc.w lr, r12, r2
+; CHECK-NEXT:    vmov r2, s14
+; CHECK-NEXT:    adds.w r12, r3, r2
+; CHECK-NEXT:    vmov.u16 r2, q2[4]
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[5]
+; CHECK-NEXT:    vmov.32 q3[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[6]
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[7]
+; CHECK-NEXT:    vmov.32 q3[3], r2
+; CHECK-NEXT:    adc.w lr, lr, r4
+; CHECK-NEXT:    vcmp.i32 ne, q3, zr
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r4, r2, #1
+; CHECK-NEXT:    rsbs r4, r4, #0
+; CHECK-NEXT:    vmov.32 q2[0], r4
+; CHECK-NEXT:    vmov.32 q2[1], r4
+; CHECK-NEXT:    ubfx r4, r2, #4, #1
+; CHECK-NEXT:    rsbs r4, r4, #0
+; CHECK-NEXT:    vmov.32 q2[2], r4
+; CHECK-NEXT:    vmov.32 q2[3], r4
+; CHECK-NEXT:    vmov.u8 r4, q0[12]
+; CHECK-NEXT:    vmov.32 q3[0], r4
+; CHECK-NEXT:    vmov.u8 r4, q0[13]
+; CHECK-NEXT:    vmov.32 q3[2], r4
+; CHECK-NEXT:    vand q3, q3, q1
+; CHECK-NEXT:    vand q2, q3, q2
+; CHECK-NEXT:    vmov r3, s8
+; CHECK-NEXT:    vmov r4, s9
+; CHECK-NEXT:    adds.w r12, r12, r3
+; CHECK-NEXT:    vmov r3, s11
+; CHECK-NEXT:    adc.w lr, lr, r4
+; CHECK-NEXT:    vmov r4, s10
+; CHECK-NEXT:    adds.w r4, r4, r12
+; CHECK-NEXT:    adc.w r12, lr, r3
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    vmov.32 q2[0], r3
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov.32 q2[1], r3
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[14]
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[15]
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    vand q0, q3, q1
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    adds r3, r3, r4
+; CHECK-NEXT:    vmov r4, s3
+; CHECK-NEXT:    adc.w r12, r12, r2
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    adc.w r3, r12, r4
+; CHECK-NEXT:    adds r0, r0, r2
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    pop {r4, pc}
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %xx = zext <16 x i8> %x to <16 x i64>
+  %s = select <16 x i1> %c, <16 x i64> %xx, <16 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v16i64(<16 x i64> %s)
+  %r = add i64 %z, %a
+  ret i64 %r
+}
+
+define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_acc_sext(<16 x i8> %x, <16 x i8> %b, i64 %a) {
+; CHECK-LABEL: add_v16i8_v16i64_acc_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vcmp.i8 eq, q1, zr
+; CHECK-NEXT:    vmov.i8 q1, #0x0
+; CHECK-NEXT:    vmov.i8 q2, #0xff
+; CHECK-NEXT:    vpsel q3, q2, q1
+; CHECK-NEXT:    vmov.u8 r2, q3[0]
+; CHECK-NEXT:    vmov.16 q4[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q3[1]
+; CHECK-NEXT:    vmov.16 q4[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q3[2]
+; CHECK-NEXT:    vmov.16 q4[2], r2
+; CHECK-NEXT:    vmov.u8 r2, q3[3]
+; CHECK-NEXT:    vmov.16 q4[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q3[4]
+; CHECK-NEXT:    vmov.16 q4[4], r2
+; CHECK-NEXT:    vmov.u8 r2, q3[5]
+; CHECK-NEXT:    vmov.16 q4[5], r2
+; CHECK-NEXT:    vmov.u8 r2, q3[6]
+; CHECK-NEXT:    vmov.16 q4[6], r2
+; CHECK-NEXT:    vmov.u8 r2, q3[7]
+; CHECK-NEXT:    vmov.16 q4[7], r2
+; CHECK-NEXT:    vcmp.i16 ne, q4, zr
+; CHECK-NEXT:    vpsel q4, q2, q1
+; CHECK-NEXT:    vmov.u16 r2, q4[0]
+; CHECK-NEXT:    vmov.32 q5[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q4[1]
+; CHECK-NEXT:    vmov.32 q5[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q4[2]
+; CHECK-NEXT:    vmov.32 q5[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q4[3]
+; CHECK-NEXT:    vmov.32 q5[3], r2
+; CHECK-NEXT:    vcmp.i32 ne, q5, zr
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r3, r2, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q5[0], r3
+; CHECK-NEXT:    vmov.32 q5[1], r3
+; CHECK-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q5[2], r3
+; CHECK-NEXT:    vmov.32 q5[3], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[0]
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    vmov.32 q6[0], r3
+; CHECK-NEXT:    asrs r3, r3, #31
+; CHECK-NEXT:    vmov.32 q6[1], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[1]
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    vmov.32 q6[2], r3
+; CHECK-NEXT:    asrs r3, r3, #31
+; CHECK-NEXT:    vmov.32 q6[3], r3
+; CHECK-NEXT:    vand q5, q6, q5
+; CHECK-NEXT:    vmov r3, s22
+; CHECK-NEXT:    vmov r4, s20
+; CHECK-NEXT:    vmov r12, s23
+; CHECK-NEXT:    vmov r5, s21
+; CHECK-NEXT:    adds.w lr, r4, r3
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    rsb.w r3, r3, #0
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    vmov.32 q5[0], r3
+; CHECK-NEXT:    rsb.w r2, r2, #0
+; CHECK-NEXT:    vmov.32 q5[1], r3
+; CHECK-NEXT:    adc.w r12, r12, r5
+; CHECK-NEXT:    vmov.32 q5[2], r2
+; CHECK-NEXT:    vmov.32 q5[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[2]
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    vmov.32 q6[0], r2
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vmov.32 q6[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[3]
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    vmov.32 q6[2], r2
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vmov.32 q6[3], r2
+; CHECK-NEXT:    vand q5, q6, q5
+; CHECK-NEXT:    vmov r3, s20
+; CHECK-NEXT:    vmov r2, s21
+; CHECK-NEXT:    vmov r4, s22
+; CHECK-NEXT:    vmov r5, s23
+; CHECK-NEXT:    adds.w r3, r3, lr
+; CHECK-NEXT:    adc.w r2, r2, r12
+; CHECK-NEXT:    adds.w r12, r3, r4
+; CHECK-NEXT:    adc.w r3, r2, r5
+; CHECK-NEXT:    vmov.u16 r2, q4[4]
+; CHECK-NEXT:    vmov.32 q5[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q4[5]
+; CHECK-NEXT:    vmov.32 q5[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q4[6]
+; CHECK-NEXT:    vmov.32 q5[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q4[7]
+; CHECK-NEXT:    vmov.32 q5[3], r2
+; CHECK-NEXT:    vcmp.i32 ne, q5, zr
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r5, r2, #1
+; CHECK-NEXT:    rsbs r5, r5, #0
+; CHECK-NEXT:    vmov.32 q4[0], r5
+; CHECK-NEXT:    vmov.32 q4[1], r5
+; CHECK-NEXT:    ubfx r5, r2, #4, #1
+; CHECK-NEXT:    rsbs r5, r5, #0
+; CHECK-NEXT:    vmov.32 q4[2], r5
+; CHECK-NEXT:    vmov.32 q4[3], r5
+; CHECK-NEXT:    vmov.u8 r5, q0[4]
+; CHECK-NEXT:    sxtb r5, r5
+; CHECK-NEXT:    vmov.32 q5[0], r5
+; CHECK-NEXT:    asrs r5, r5, #31
+; CHECK-NEXT:    vmov.32 q5[1], r5
+; CHECK-NEXT:    vmov.u8 r5, q0[5]
+; CHECK-NEXT:    sxtb r5, r5
+; CHECK-NEXT:    vmov.32 q5[2], r5
+; CHECK-NEXT:    asrs r5, r5, #31
+; CHECK-NEXT:    vmov.32 q5[3], r5
+; CHECK-NEXT:    vand q4, q5, q4
+; CHECK-NEXT:    vmov r4, s16
+; CHECK-NEXT:    vmov r5, s17
+; CHECK-NEXT:    adds.w r12, r12, r4
+; CHECK-NEXT:    vmov r4, s18
+; CHECK-NEXT:    adcs r5, r3
+; CHECK-NEXT:    vmov r3, s19
+; CHECK-NEXT:    adds.w r4, r4, r12
+; CHECK-NEXT:    adc.w r12, r5, r3
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    vmov.32 q4[0], r3
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov.32 q4[1], r3
+; CHECK-NEXT:    vmov.32 q4[2], r2
+; CHECK-NEXT:    vmov.32 q4[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[6]
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    vmov.32 q5[0], r2
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vmov.32 q5[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[7]
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    vmov.32 q5[2], r2
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vmov.32 q5[3], r2
+; CHECK-NEXT:    vand q4, q5, q4
+; CHECK-NEXT:    vmov r3, s16
+; CHECK-NEXT:    vmov r2, s17
+; CHECK-NEXT:    vmov r5, s19
+; CHECK-NEXT:    adds r3, r3, r4
+; CHECK-NEXT:    vmov r4, s18
+; CHECK-NEXT:    adc.w r2, r2, r12
+; CHECK-NEXT:    adds.w r12, r3, r4
+; CHECK-NEXT:    adc.w r3, r2, r5
+; CHECK-NEXT:    vmov.u8 r2, q3[8]
+; CHECK-NEXT:    vmov.16 q4[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q3[9]
+; CHECK-NEXT:    vmov.16 q4[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q3[10]
+; CHECK-NEXT:    vmov.16 q4[2], r2
+; CHECK-NEXT:    vmov.u8 r2, q3[11]
+; CHECK-NEXT:    vmov.16 q4[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q3[12]
+; CHECK-NEXT:    vmov.16 q4[4], r2
+; CHECK-NEXT:    vmov.u8 r2, q3[13]
+; CHECK-NEXT:    vmov.16 q4[5], r2
+; CHECK-NEXT:    vmov.u8 r2, q3[14]
+; CHECK-NEXT:    vmov.16 q4[6], r2
+; CHECK-NEXT:    vmov.u8 r2, q3[15]
+; CHECK-NEXT:    vmov.16 q4[7], r2
+; CHECK-NEXT:    vcmp.i16 ne, q4, zr
+; CHECK-NEXT:    vpsel q1, q2, q1
+; CHECK-NEXT:    vmov.u16 r2, q1[0]
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[1]
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[2]
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[3]
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vcmp.i32 ne, q2, zr
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r5, r2, #1
+; CHECK-NEXT:    rsbs r5, r5, #0
+; CHECK-NEXT:    vmov.32 q2[0], r5
+; CHECK-NEXT:    vmov.32 q2[1], r5
+; CHECK-NEXT:    ubfx r5, r2, #4, #1
+; CHECK-NEXT:    rsbs r5, r5, #0
+; CHECK-NEXT:    vmov.32 q2[2], r5
+; CHECK-NEXT:    vmov.32 q2[3], r5
+; CHECK-NEXT:    vmov.u8 r5, q0[8]
+; CHECK-NEXT:    sxtb r5, r5
+; CHECK-NEXT:    vmov.32 q3[0], r5
+; CHECK-NEXT:    asrs r5, r5, #31
+; CHECK-NEXT:    vmov.32 q3[1], r5
+; CHECK-NEXT:    vmov.u8 r5, q0[9]
+; CHECK-NEXT:    sxtb r5, r5
+; CHECK-NEXT:    vmov.32 q3[2], r5
+; CHECK-NEXT:    asrs r5, r5, #31
+; CHECK-NEXT:    vmov.32 q3[3], r5
+; CHECK-NEXT:    vand q2, q3, q2
+; CHECK-NEXT:    vmov r4, s8
+; CHECK-NEXT:    vmov r5, s9
+; CHECK-NEXT:    adds.w r12, r12, r4
+; CHECK-NEXT:    vmov r4, s10
+; CHECK-NEXT:    adcs r5, r3
+; CHECK-NEXT:    vmov r3, s11
+; CHECK-NEXT:    adds.w r4, r4, r12
+; CHECK-NEXT:    adc.w r12, r5, r3
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    vmov.32 q2[0], r3
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov.32 q2[1], r3
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[10]
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vmov.32 q3[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[11]
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vmov.32 q3[3], r2
+; CHECK-NEXT:    vand q2, q3, q2
+; CHECK-NEXT:    vmov r3, s8
+; CHECK-NEXT:    vmov r2, s9
+; CHECK-NEXT:    vmov r5, s11
+; CHECK-NEXT:    adds r3, r3, r4
+; CHECK-NEXT:    vmov r4, s10
+; CHECK-NEXT:    adc.w r2, r2, r12
+; CHECK-NEXT:    adds.w r12, r3, r4
+; CHECK-NEXT:    adc.w r3, r2, r5
+; CHECK-NEXT:    vmov.u16 r2, q1[4]
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[5]
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[6]
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[7]
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vcmp.i32 ne, q2, zr
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r5, r2, #1
+; CHECK-NEXT:    rsbs r5, r5, #0
+; CHECK-NEXT:    vmov.32 q1[0], r5
+; CHECK-NEXT:    vmov.32 q1[1], r5
+; CHECK-NEXT:    ubfx r5, r2, #4, #1
+; CHECK-NEXT:    rsbs r5, r5, #0
+; CHECK-NEXT:    vmov.32 q1[2], r5
+; CHECK-NEXT:    vmov.32 q1[3], r5
+; CHECK-NEXT:    vmov.u8 r5, q0[12]
+; CHECK-NEXT:    sxtb r5, r5
+; CHECK-NEXT:    vmov.32 q2[0], r5
+; CHECK-NEXT:    asrs r5, r5, #31
+; CHECK-NEXT:    vmov.32 q2[1], r5
+; CHECK-NEXT:    vmov.u8 r5, q0[13]
+; CHECK-NEXT:    sxtb r5, r5
+; CHECK-NEXT:    vmov.32 q2[2], r5
+; CHECK-NEXT:    asrs r5, r5, #31
+; CHECK-NEXT:    vmov.32 q2[3], r5
+; CHECK-NEXT:    vand q1, q2, q1
+; CHECK-NEXT:    vmov r4, s4
+; CHECK-NEXT:    vmov r5, s5
+; CHECK-NEXT:    adds.w r12, r12, r4
+; CHECK-NEXT:    vmov r4, s6
+; CHECK-NEXT:    adcs r5, r3
+; CHECK-NEXT:    vmov r3, s7
+; CHECK-NEXT:    adds.w r4, r4, r12
+; CHECK-NEXT:    adc.w r12, r5, r3
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    vmov.32 q1[0], r3
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov.32 q1[1], r3
+; CHECK-NEXT:    vmov.32 q1[2], r2
+; CHECK-NEXT:    vmov.32 q1[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[14]
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[15]
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vand q0, q2, q1
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vmov r5, s3
+; CHECK-NEXT:    adds r3, r3, r4
+; CHECK-NEXT:    vmov r4, s2
+; CHECK-NEXT:    adc.w r2, r2, r12
+; CHECK-NEXT:    adds r3, r3, r4
+; CHECK-NEXT:    adcs r2, r5
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %xx = sext <16 x i8> %x to <16 x i64>
+  %s = select <16 x i1> %c, <16 x i64> %xx, <16 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v16i64(<16 x i64> %s)
+  %r = add i64 %z, %a
+  ret i64 %r
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_acc_zext(<2 x i8> %x, <2 x i8> %b, i64 %a) {
+; CHECK-LABEL: add_v2i8_v2i64_acc_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i64 q2, #0xff
+; CHECK-NEXT:    vand q1, q1, q2
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    vmov.32 q3[1], r2
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    vmov.32 q3[3], r2
+; CHECK-NEXT:    vand q0, q0, q3
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    vmov r3, s1
+; CHECK-NEXT:    orr.w r12, r3, r2
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    add r2, r3
+; CHECK-NEXT:    adds r0, r0, r2
+; CHECK-NEXT:    adc.w r1, r1, r12
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <2 x i8> %b, zeroinitializer
+  %xx = zext <2 x i8> %x to <2 x i64>
+  %s = select <2 x i1> %c, <2 x i64> %xx, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  %r = add i64 %z, %a
+  ret i64 %r
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_acc_sext(<2 x i8> %x, <2 x i8> %b, i64 %a) {
+; CHECK-LABEL: add_v2i8_v2i64_acc_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vmov.i32 q2, #0xff
+; CHECK-NEXT:    vand q2, q1, q2
+; CHECK-NEXT:    vmov r2, s8
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q1[0], r2
+; CHECK-NEXT:    vmov.32 q1[1], r2
+; CHECK-NEXT:    vmov r2, s10
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q1[2], r2
+; CHECK-NEXT:    vmov.32 q1[3], r2
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vand q0, q2, q1
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r12, s3
+; CHECK-NEXT:    vmov lr, s1
+; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    adc.w r3, lr, r12
+; CHECK-NEXT:    adds r0, r0, r2
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %c = icmp eq <2 x i8> %b, zeroinitializer
+  %xx = sext <2 x i8> %x to <2 x i64>
+  %s = select <2 x i1> %c, <2 x i64> %xx, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  %r = add i64 %z, %a
+  ret i64 %r
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i64_v2i64_acc(<2 x i64> %x, <2 x i64> %b, i64 %a) {
+; CHECK-LABEL: add_v2i64_v2i64_acc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vmov r2, s5
+; CHECK-NEXT:    vmov r3, s4
+; CHECK-NEXT:    orrs r2, r3
+; CHECK-NEXT:    vmov r3, s6
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov r2, s7
+; CHECK-NEXT:    orrs r2, r3
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r12, s3
+; CHECK-NEXT:    vmov lr, s1
+; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    adc.w r3, lr, r12
+; CHECK-NEXT:    adds r0, r0, r2
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %c = icmp eq <2 x i64> %b, zeroinitializer
+  %s = select <2 x i1> %c, <2 x i64> %x, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  %r = add i64 %z, %a
+  ret i64 %r
+}
+
+declare i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16>)
+declare i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16>)
+declare i32 @llvm.experimental.vector.reduce.add.v16i32(<16 x i32>)
+declare i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32>)
+declare i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32>)
+declare i64 @llvm.experimental.vector.reduce.add.v16i64(<16 x i64>)
+declare i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64>)
+declare i64 @llvm.experimental.vector.reduce.add.v4i64(<4 x i64>)
+declare i64 @llvm.experimental.vector.reduce.add.v8i64(<8 x i64>)
+declare i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-vecreduce-mlapred.ll b/llvm/test/CodeGen/Thumb2/mve-vecreduce-mlapred.ll
new file mode 100644
index 0000000000000..6510affb482c4
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vecreduce-mlapred.ll
@@ -0,0 +1,3343 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK
+
+define arm_aapcs_vfpcc i32 @add_v4i32_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %b) {
+; CHECK-LABEL: add_v4i32_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q2, zr
+; CHECK-NEXT:    vmlavt.u32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %b, zeroinitializer
+  %m = mul <4 x i32> %x, %y
+  %s = select <4 x i1> %c, <4 x i32> %m, <4 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %s)
+  ret i32 %z
+}
+
+define arm_aapcs_vfpcc i64 @add_v4i32_v4i64_zext(<4 x i32> %x, <4 x i32> %y, <4 x i32> %b) {
+; CHECK-LABEL: add_v4i32_v4i64_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q2, zr
+; CHECK-NEXT:    vmlalvt.u32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %b, zeroinitializer
+  %xx = zext <4 x i32> %x to <4 x i64>
+  %yy = zext <4 x i32> %y to <4 x i64>
+  %m = mul <4 x i64> %xx, %yy
+  %s = select <4 x i1> %c, <4 x i64> %m, <4 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v4i64(<4 x i64> %s)
+  ret i64 %z
+}
+
+define arm_aapcs_vfpcc i64 @add_v4i32_v4i64_sext(<4 x i32> %x, <4 x i32> %y, <4 x i32> %b) {
+; CHECK-LABEL: add_v4i32_v4i64_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q2, zr
+; CHECK-NEXT:    vmlalvt.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %b, zeroinitializer
+  %xx = sext <4 x i32> %x to <4 x i64>
+  %yy = sext <4 x i32> %y to <4 x i64>
+  %m = mul <4 x i64> %xx, %yy
+  %s = select <4 x i1> %c, <4 x i64> %m, <4 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v4i64(<4 x i64> %s)
+  ret i64 %z
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i32_v2i64_zext(<2 x i32> %x, <2 x i32> %y, <2 x i32> %b) {
+; CHECK-LABEL: add_v2i32_v2i64_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmullb.u32 q3, q0, q1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q0[2], r0
+; CHECK-NEXT:    vmov.32 q0[3], r0
+; CHECK-NEXT:    vand q0, q3, q0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <2 x i32> %b, zeroinitializer
+  %xx = zext <2 x i32> %x to <2 x i64>
+  %yy = zext <2 x i32> %y to <2 x i64>
+  %m = mul <2 x i64> %xx, %yy
+  %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  ret i64 %z
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i32_v2i64_sext(<2 x i32> %x, <2 x i32> %y, <2 x i32> %b) {
+; CHECK-LABEL: add_v2i32_v2i64_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmullb.s32 q3, q0, q1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q0[2], r0
+; CHECK-NEXT:    vmov.32 q0[3], r0
+; CHECK-NEXT:    vand q0, q3, q0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <2 x i32> %b, zeroinitializer
+  %xx = sext <2 x i32> %x to <2 x i64>
+  %yy = sext <2 x i32> %y to <2 x i64>
+  %m = mul <2 x i64> %xx, %yy
+  %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  ret i64 %z
+}
+
+define arm_aapcs_vfpcc i32 @add_v8i16_v8i32_zext(<8 x i16> %x, <8 x i16> %y, <8 x i16> %b) {
+; CHECK-LABEL: add_v8i16_v8i32_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q2, zr
+; CHECK-NEXT:    vmlavt.u16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %b, zeroinitializer
+  %xx = zext <8 x i16> %x to <8 x i32>
+  %yy = zext <8 x i16> %y to <8 x i32>
+  %m = mul <8 x i32> %xx, %yy
+  %s = select <8 x i1> %c, <8 x i32> %m, <8 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> %s)
+  ret i32 %z
+}
+
+define arm_aapcs_vfpcc i32 @add_v8i16_v8i32_sext(<8 x i16> %x, <8 x i16> %y, <8 x i16> %b) {
+; CHECK-LABEL: add_v8i16_v8i32_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q2, zr
+; CHECK-NEXT:    vmlavt.s16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %b, zeroinitializer
+  %xx = sext <8 x i16> %x to <8 x i32>
+  %yy = sext <8 x i16> %y to <8 x i32>
+  %m = mul <8 x i32> %xx, %yy
+  %s = select <8 x i1> %c, <8 x i32> %m, <8 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> %s)
+  ret i32 %z
+}
+
+define arm_aapcs_vfpcc i32 @add_v4i16_v4i32_zext(<4 x i16> %x, <4 x i16> %y, <4 x i16> %b) {
+; CHECK-LABEL: add_v4i16_v4i32_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.u16 q1, q1
+; CHECK-NEXT:    vmovlb.u16 q0, q0
+; CHECK-NEXT:    vmovlb.u16 q2, q2
+; CHECK-NEXT:    vpt.i32 eq, q2, zr
+; CHECK-NEXT:    vmlavt.u32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i16> %b, zeroinitializer
+  %xx = zext <4 x i16> %x to <4 x i32>
+  %yy = zext <4 x i16> %y to <4 x i32>
+  %m = mul <4 x i32> %xx, %yy
+  %s = select <4 x i1> %c, <4 x i32> %m, <4 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %s)
+  ret i32 %z
+}
+
+define arm_aapcs_vfpcc i32 @add_v4i16_v4i32_sext(<4 x i16> %x, <4 x i16> %y, <4 x i16> %b) {
+; CHECK-LABEL: add_v4i16_v4i32_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.s16 q1, q1
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    vmovlb.u16 q2, q2
+; CHECK-NEXT:    vpt.i32 eq, q2, zr
+; CHECK-NEXT:    vmlavt.u32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i16> %b, zeroinitializer
+  %xx = sext <4 x i16> %x to <4 x i32>
+  %yy = sext <4 x i16> %y to <4 x i32>
+  %m = mul <4 x i32> %xx, %yy
+  %s = select <4 x i1> %c, <4 x i32> %m, <4 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %s)
+  ret i32 %z
+}
+
+define arm_aapcs_vfpcc zeroext i16 @add_v8i16_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %b) {
+; CHECK-LABEL: add_v8i16_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q2, zr
+; CHECK-NEXT:    vmlavt.u16 r0, q0, q1
+; CHECK-NEXT:    uxth r0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %b, zeroinitializer
+  %m = mul <8 x i16> %x, %y
+  %s = select <8 x i1> %c, <8 x i16> %m, <8 x i16> zeroinitializer
+  %z = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> %s)
+  ret i16 %z
+}
+
+define arm_aapcs_vfpcc i64 @add_v8i16_v8i64_zext(<8 x i16> %x, <8 x i16> %y, <8 x i16> %b) {
+; CHECK-LABEL: add_v8i16_v8i64_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q2, zr
+; CHECK-NEXT:    vmlalvt.u16 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %b, zeroinitializer
+  %xx = zext <8 x i16> %x to <8 x i64>
+  %yy = zext <8 x i16> %y to <8 x i64>
+  %m = mul <8 x i64> %xx, %yy
+  %s = select <8 x i1> %c, <8 x i64> %m, <8 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v8i64(<8 x i64> %s)
+  ret i64 %z
+}
+
+define arm_aapcs_vfpcc i64 @add_v8i16_v8i64_sext(<8 x i16> %x, <8 x i16> %y, <8 x i16> %b) {
+; CHECK-LABEL: add_v8i16_v8i64_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q2, zr
+; CHECK-NEXT:    vmlalvt.s16 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %b, zeroinitializer
+  %xx = sext <8 x i16> %x to <8 x i64>
+  %yy = sext <8 x i16> %y to <8 x i64>
+  %m = mul <8 x i64> %xx, %yy
+  %s = select <8 x i1> %c, <8 x i64> %m, <8 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v8i64(<8 x i64> %s)
+  ret i64 %z
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_zext(<2 x i16> %x, <2 x i16> %y, <2 x i16> %b) {
+; CHECK-LABEL: add_v2i16_v2i64_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vmov.i64 q3, #0xffff
+; CHECK-NEXT:    vand q1, q1, q3
+; CHECK-NEXT:    vand q4, q0, q3
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r1, s16
+; CHECK-NEXT:    umull r0, r1, r1, r0
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vmov.32 q0[1], r1
+; CHECK-NEXT:    vmov r1, s18
+; CHECK-NEXT:    vand q1, q2, q3
+; CHECK-NEXT:    umull r0, r1, r1, r0
+; CHECK-NEXT:    vmov.32 q0[2], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <2 x i16> %b, zeroinitializer
+  %xx = zext <2 x i16> %x to <2 x i64>
+  %yy = zext <2 x i16> %y to <2 x i64>
+  %m = mul <2 x i64> %xx, %yy
+  %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  ret i64 %z
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_sext(<2 x i16> %x, <2 x i16> %y, <2 x i16> %b) {
+; CHECK-LABEL: add_v2i16_v2i64_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i32 q3, #0xffff
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vand q3, q2, q3
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    sxth r1, r1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    sxth r0, r0
+; CHECK-NEXT:    smull r0, r1, r1, r0
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vmov.32 q3[1], r1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    sxth r0, r0
+; CHECK-NEXT:    sxth r1, r1
+; CHECK-NEXT:    smull r0, r1, r1, r0
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r1
+; CHECK-NEXT:    vand q0, q3, q2
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <2 x i16> %b, zeroinitializer
+  %xx = sext <2 x i16> %x to <2 x i64>
+  %yy = sext <2 x i16> %y to <2 x i64>
+  %m = mul <2 x i64> %xx, %yy
+  %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  ret i64 %z
+}
+
+define arm_aapcs_vfpcc i32 @add_v16i8_v16i32_zext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b) {
+; CHECK-LABEL: add_v16i8_v16i32_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q2, zr
+; CHECK-NEXT:    vmlavt.u8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %xx = zext <16 x i8> %x to <16 x i32>
+  %yy = zext <16 x i8> %y to <16 x i32>
+  %m = mul <16 x i32> %xx, %yy
+  %s = select <16 x i1> %c, <16 x i32> %m, <16 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v16i32(<16 x i32> %s)
+  ret i32 %z
+}
+
+define arm_aapcs_vfpcc i32 @add_v16i8_v16i32_sext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b) {
+; CHECK-LABEL: add_v16i8_v16i32_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q2, zr
+; CHECK-NEXT:    vmlavt.s8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %xx = sext <16 x i8> %x to <16 x i32>
+  %yy = sext <16 x i8> %y to <16 x i32>
+  %m = mul <16 x i32> %xx, %yy
+  %s = select <16 x i1> %c, <16 x i32> %m, <16 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v16i32(<16 x i32> %s)
+  ret i32 %z
+}
+
+define arm_aapcs_vfpcc i32 @add_v4i8_v4i32_zext(<4 x i8> %x, <4 x i8> %y, <4 x i8> %b) {
+; CHECK-LABEL: add_v4i8_v4i32_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i32 q3, #0xff
+; CHECK-NEXT:    vand q1, q1, q3
+; CHECK-NEXT:    vand q0, q0, q3
+; CHECK-NEXT:    vand q2, q2, q3
+; CHECK-NEXT:    vpt.i32 eq, q2, zr
+; CHECK-NEXT:    vmlavt.u32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i8> %b, zeroinitializer
+  %xx = zext <4 x i8> %x to <4 x i32>
+  %yy = zext <4 x i8> %y to <4 x i32>
+  %m = mul <4 x i32> %xx, %yy
+  %s = select <4 x i1> %c, <4 x i32> %m, <4 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %s)
+  ret i32 %z
+}
+
+define arm_aapcs_vfpcc i32 @add_v4i8_v4i32_sext(<4 x i8> %x, <4 x i8> %y, <4 x i8> %b) {
+; CHECK-LABEL: add_v4i8_v4i32_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.s8 q1, q1
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    vmov.i32 q3, #0xff
+; CHECK-NEXT:    vmovlb.s16 q1, q1
+; CHECK-NEXT:    vand q2, q2, q3
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    vpt.i32 eq, q2, zr
+; CHECK-NEXT:    vmlavt.u32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i8> %b, zeroinitializer
+  %xx = sext <4 x i8> %x to <4 x i32>
+  %yy = sext <4 x i8> %y to <4 x i32>
+  %m = mul <4 x i32> %xx, %yy
+  %s = select <4 x i1> %c, <4 x i32> %m, <4 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %s)
+  ret i32 %z
+}
+
+define arm_aapcs_vfpcc zeroext i16 @add_v16i8_v16i16_zext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b) {
+; CHECK-LABEL: add_v16i8_v16i16_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vcmp.i8 eq, q2, zr
+; CHECK-NEXT:    vmov.i8 q2, #0x0
+; CHECK-NEXT:    vmov.i8 q3, #0xff
+; CHECK-NEXT:    vpsel q2, q3, q2
+; CHECK-NEXT:    vmov.u8 r0, q2[0]
+; CHECK-NEXT:    vmov.16 q3[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[1]
+; CHECK-NEXT:    vmov.16 q3[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[2]
+; CHECK-NEXT:    vmov.16 q3[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[3]
+; CHECK-NEXT:    vmov.16 q3[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[4]
+; CHECK-NEXT:    vmov.16 q3[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[5]
+; CHECK-NEXT:    vmov.16 q3[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[6]
+; CHECK-NEXT:    vmov.16 q3[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[7]
+; CHECK-NEXT:    vmov.16 q3[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[0]
+; CHECK-NEXT:    vcmp.i16 ne, q3, zr
+; CHECK-NEXT:    vmov.16 q3[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[1]
+; CHECK-NEXT:    vmov.16 q3[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[2]
+; CHECK-NEXT:    vmov.16 q3[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[3]
+; CHECK-NEXT:    vmov.16 q3[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[4]
+; CHECK-NEXT:    vmov.16 q3[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[5]
+; CHECK-NEXT:    vmov.16 q3[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[6]
+; CHECK-NEXT:    vmov.16 q3[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[7]
+; CHECK-NEXT:    vmov.16 q3[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[0]
+; CHECK-NEXT:    vmovlb.u8 q4, q3
+; CHECK-NEXT:    vmov.16 q3[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[1]
+; CHECK-NEXT:    vmov.16 q3[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[2]
+; CHECK-NEXT:    vmov.16 q3[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[3]
+; CHECK-NEXT:    vmov.16 q3[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[4]
+; CHECK-NEXT:    vmov.16 q3[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[5]
+; CHECK-NEXT:    vmov.16 q3[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[6]
+; CHECK-NEXT:    vmov.16 q3[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[7]
+; CHECK-NEXT:    vmov.16 q3[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[8]
+; CHECK-NEXT:    vmovlb.u8 q5, q3
+; CHECK-NEXT:    vmov.i32 q3, #0x0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i16 q3, q5, q4
+; CHECK-NEXT:    vmov.16 q4[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[9]
+; CHECK-NEXT:    vmov.16 q4[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[10]
+; CHECK-NEXT:    vmov.16 q4[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[11]
+; CHECK-NEXT:    vmov.16 q4[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[12]
+; CHECK-NEXT:    vmov.16 q4[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[13]
+; CHECK-NEXT:    vmov.16 q4[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[14]
+; CHECK-NEXT:    vmov.16 q4[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[15]
+; CHECK-NEXT:    vmov.16 q4[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[8]
+; CHECK-NEXT:    vmov.16 q2[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[9]
+; CHECK-NEXT:    vmov.16 q2[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[10]
+; CHECK-NEXT:    vmov.16 q2[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[11]
+; CHECK-NEXT:    vmov.16 q2[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[12]
+; CHECK-NEXT:    vmov.16 q2[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[13]
+; CHECK-NEXT:    vmov.16 q2[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[14]
+; CHECK-NEXT:    vmov.16 q2[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[15]
+; CHECK-NEXT:    vmov.16 q2[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[8]
+; CHECK-NEXT:    vmov.16 q1[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[9]
+; CHECK-NEXT:    vmov.16 q1[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[10]
+; CHECK-NEXT:    vmov.16 q1[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[11]
+; CHECK-NEXT:    vmov.16 q1[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[12]
+; CHECK-NEXT:    vmov.16 q1[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[13]
+; CHECK-NEXT:    vmov.16 q1[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[14]
+; CHECK-NEXT:    vmov.16 q1[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[15]
+; CHECK-NEXT:    vmov.16 q1[7], r0
+; CHECK-NEXT:    vmullb.u8 q0, q1, q2
+; CHECK-NEXT:    vpt.i16 ne, q4, zr
+; CHECK-NEXT:    vaddt.i16 q3, q3, q0
+; CHECK-NEXT:    vaddv.u16 r0, q3
+; CHECK-NEXT:    uxth r0, r0
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %xx = zext <16 x i8> %x to <16 x i16>
+  %yy = zext <16 x i8> %y to <16 x i16>
+  %m = mul <16 x i16> %xx, %yy
+  %s = select <16 x i1> %c, <16 x i16> %m, <16 x i16> zeroinitializer
+  %z = call i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16> %s)
+  ret i16 %z
+}
+
+define arm_aapcs_vfpcc signext i16 @add_v16i8_v16i16_sext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b) {
+; CHECK-LABEL: add_v16i8_v16i16_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vcmp.i8 eq, q2, zr
+; CHECK-NEXT:    vmov.i8 q2, #0x0
+; CHECK-NEXT:    vmov.i8 q3, #0xff
+; CHECK-NEXT:    vpsel q2, q3, q2
+; CHECK-NEXT:    vmov.u8 r0, q2[0]
+; CHECK-NEXT:    vmov.16 q3[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[1]
+; CHECK-NEXT:    vmov.16 q3[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[2]
+; CHECK-NEXT:    vmov.16 q3[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[3]
+; CHECK-NEXT:    vmov.16 q3[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[4]
+; CHECK-NEXT:    vmov.16 q3[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[5]
+; CHECK-NEXT:    vmov.16 q3[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[6]
+; CHECK-NEXT:    vmov.16 q3[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[7]
+; CHECK-NEXT:    vmov.16 q3[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[0]
+; CHECK-NEXT:    vcmp.i16 ne, q3, zr
+; CHECK-NEXT:    vmov.16 q3[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[1]
+; CHECK-NEXT:    vmov.16 q3[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[2]
+; CHECK-NEXT:    vmov.16 q3[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[3]
+; CHECK-NEXT:    vmov.16 q3[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[4]
+; CHECK-NEXT:    vmov.16 q3[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[5]
+; CHECK-NEXT:    vmov.16 q3[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[6]
+; CHECK-NEXT:    vmov.16 q3[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[7]
+; CHECK-NEXT:    vmov.16 q3[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[0]
+; CHECK-NEXT:    vmovlb.s8 q4, q3
+; CHECK-NEXT:    vmov.16 q3[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[1]
+; CHECK-NEXT:    vmov.16 q3[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[2]
+; CHECK-NEXT:    vmov.16 q3[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[3]
+; CHECK-NEXT:    vmov.16 q3[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[4]
+; CHECK-NEXT:    vmov.16 q3[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[5]
+; CHECK-NEXT:    vmov.16 q3[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[6]
+; CHECK-NEXT:    vmov.16 q3[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[7]
+; CHECK-NEXT:    vmov.16 q3[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[8]
+; CHECK-NEXT:    vmovlb.s8 q5, q3
+; CHECK-NEXT:    vmov.i32 q3, #0x0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i16 q3, q5, q4
+; CHECK-NEXT:    vmov.16 q4[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[9]
+; CHECK-NEXT:    vmov.16 q4[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[10]
+; CHECK-NEXT:    vmov.16 q4[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[11]
+; CHECK-NEXT:    vmov.16 q4[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[12]
+; CHECK-NEXT:    vmov.16 q4[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[13]
+; CHECK-NEXT:    vmov.16 q4[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[14]
+; CHECK-NEXT:    vmov.16 q4[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q2[15]
+; CHECK-NEXT:    vmov.16 q4[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[8]
+; CHECK-NEXT:    vmov.16 q2[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[9]
+; CHECK-NEXT:    vmov.16 q2[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[10]
+; CHECK-NEXT:    vmov.16 q2[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[11]
+; CHECK-NEXT:    vmov.16 q2[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[12]
+; CHECK-NEXT:    vmov.16 q2[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[13]
+; CHECK-NEXT:    vmov.16 q2[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[14]
+; CHECK-NEXT:    vmov.16 q2[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[15]
+; CHECK-NEXT:    vmov.16 q2[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[8]
+; CHECK-NEXT:    vmov.16 q1[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[9]
+; CHECK-NEXT:    vmov.16 q1[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[10]
+; CHECK-NEXT:    vmov.16 q1[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[11]
+; CHECK-NEXT:    vmov.16 q1[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[12]
+; CHECK-NEXT:    vmov.16 q1[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[13]
+; CHECK-NEXT:    vmov.16 q1[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[14]
+; CHECK-NEXT:    vmov.16 q1[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[15]
+; CHECK-NEXT:    vmov.16 q1[7], r0
+; CHECK-NEXT:    vmullb.s8 q0, q1, q2
+; CHECK-NEXT:    vpt.i16 ne, q4, zr
+; CHECK-NEXT:    vaddt.i16 q3, q3, q0
+; CHECK-NEXT:    vaddv.u16 r0, q3
+; CHECK-NEXT:    sxth r0, r0
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %xx = sext <16 x i8> %x to <16 x i16>
+  %yy = sext <16 x i8> %y to <16 x i16>
+  %m = mul <16 x i16> %xx, %yy
+  %s = select <16 x i1> %c, <16 x i16> %m, <16 x i16> zeroinitializer
+  %z = call i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16> %s)
+  ret i16 %z
+}
+
+define arm_aapcs_vfpcc zeroext i16 @add_v8i8_v8i16_zext(<8 x i8> %x, <8 x i8> %y, <8 x i8> %b) {
+; CHECK-LABEL: add_v8i8_v8i16_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.u8 q1, q1
+; CHECK-NEXT:    vmovlb.u8 q0, q0
+; CHECK-NEXT:    vmovlb.u8 q2, q2
+; CHECK-NEXT:    vpt.i16 eq, q2, zr
+; CHECK-NEXT:    vmlavt.u16 r0, q0, q1
+; CHECK-NEXT:    uxth r0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i8> %b, zeroinitializer
+  %xx = zext <8 x i8> %x to <8 x i16>
+  %yy = zext <8 x i8> %y to <8 x i16>
+  %m = mul <8 x i16> %xx, %yy
+  %s = select <8 x i1> %c, <8 x i16> %m, <8 x i16> zeroinitializer
+  %z = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> %s)
+  ret i16 %z
+}
+
+define arm_aapcs_vfpcc signext i16 @add_v8i8_v8i16_sext(<8 x i8> %x, <8 x i8> %y, <8 x i8> %b) {
+; CHECK-LABEL: add_v8i8_v8i16_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.s8 q1, q1
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    vmovlb.u8 q2, q2
+; CHECK-NEXT:    vpt.i16 eq, q2, zr
+; CHECK-NEXT:    vmlavt.u16 r0, q0, q1
+; CHECK-NEXT:    sxth r0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i8> %b, zeroinitializer
+  %xx = sext <8 x i8> %x to <8 x i16>
+  %yy = sext <8 x i8> %y to <8 x i16>
+  %m = mul <8 x i16> %xx, %yy
+  %s = select <8 x i1> %c, <8 x i16> %m, <8 x i16> zeroinitializer
+  %z = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> %s)
+  ret i16 %z
+}
+
+define arm_aapcs_vfpcc zeroext i8 @add_v16i8_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b) {
+; CHECK-LABEL: add_v16i8_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q2, zr
+; CHECK-NEXT:    vmlavt.u8 r0, q0, q1
+; CHECK-NEXT:    uxtb r0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %m = mul <16 x i8> %x, %y
+  %s = select <16 x i1> %c, <16 x i8> %m, <16 x i8> zeroinitializer
+  %z = call i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8> %s)
+  ret i8 %z
+}
+
+define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_zext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b) {
+; CHECK-LABEL: add_v16i8_v16i64_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #88
+; CHECK-NEXT:    sub sp, #88
+; CHECK-NEXT:    vmov q3, q1
+; CHECK-NEXT:    vstrw.32 q0, [sp, #64] @ 16-byte Spill
+; CHECK-NEXT:    vmov.i8 q0, #0x0
+; CHECK-NEXT:    vmov.i8 q1, #0xff
+; CHECK-NEXT:    vcmp.i8 eq, q2, zr
+; CHECK-NEXT:    vstrw.32 q1, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vpsel q5, q1, q0
+; CHECK-NEXT:    vstrw.32 q0, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    vmov.u8 r0, q5[0]
+; CHECK-NEXT:    vmov.i64 q4, #0xff
+; CHECK-NEXT:    vmov.16 q2[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[1]
+; CHECK-NEXT:    vmov.16 q2[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[2]
+; CHECK-NEXT:    vmov.16 q2[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[3]
+; CHECK-NEXT:    vmov.16 q2[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[4]
+; CHECK-NEXT:    vmov.16 q2[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[5]
+; CHECK-NEXT:    vmov.16 q2[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[6]
+; CHECK-NEXT:    vmov.16 q2[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q5[7]
+; CHECK-NEXT:    vmov.16 q2[7], r0
+; CHECK-NEXT:    vcmp.i16 ne, q2, zr
+; CHECK-NEXT:    vpsel q6, q1, q0
+; CHECK-NEXT:    vmov.u16 r0, q6[0]
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q6[1]
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q6[2]
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q6[3]
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vcmp.i32 ne, q2, zr
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #64] @ 16-byte Reload
+; CHECK-NEXT:    vmov.u8 r2, q1[0]
+; CHECK-NEXT:    and r1, r0, #1
+; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    vmov.32 q7[0], r1
+; CHECK-NEXT:    vmov.32 q7[1], r1
+; CHECK-NEXT:    ubfx r1, r0, #4, #1
+; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    vmov.32 q7[2], r1
+; CHECK-NEXT:    vmov.32 q7[3], r1
+; CHECK-NEXT:    vmov.u8 r1, q3[0]
+; CHECK-NEXT:    vmov.32 q0[0], r1
+; CHECK-NEXT:    vmov.u8 r1, q3[1]
+; CHECK-NEXT:    vmov.32 q0[2], r1
+; CHECK-NEXT:    vand q2, q0, q4
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q1[1]
+; CHECK-NEXT:    vmov r1, s8
+; CHECK-NEXT:    vmov.32 q0[2], r2
+; CHECK-NEXT:    vand q1, q0, q4
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    umull r1, r2, r2, r1
+; CHECK-NEXT:    vmov.32 q0[0], r1
+; CHECK-NEXT:    vmov r1, s10
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    vldrw.u32 q2, [sp, #64] @ 16-byte Reload
+; CHECK-NEXT:    umull r1, r2, r2, r1
+; CHECK-NEXT:    vmov.32 q0[2], r1
+; CHECK-NEXT:    vmov.32 q0[3], r2
+; CHECK-NEXT:    vand q0, q0, q7
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r12, s3
+; CHECK-NEXT:    vmov r3, s1
+; CHECK-NEXT:    adds r1, r1, r2
+; CHECK-NEXT:    adc.w r2, r3, r12
+; CHECK-NEXT:    ubfx r3, r0, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r0, r0, #12, #1
+; CHECK-NEXT:    vmov.32 q7[0], r3
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vmov.32 q7[1], r3
+; CHECK-NEXT:    vmov.u8 r3, q2[2]
+; CHECK-NEXT:    vmov.32 q7[2], r0
+; CHECK-NEXT:    vmov.32 q1[0], r3
+; CHECK-NEXT:    vmov.32 q7[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[2]
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[3]
+; CHECK-NEXT:    vmov.u8 r3, q2[3]
+; CHECK-NEXT:    vmov.32 q0[2], r0
+; CHECK-NEXT:    vmov.32 q1[2], r3
+; CHECK-NEXT:    vand q0, q0, q4
+; CHECK-NEXT:    vand q1, q1, q4
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov r3, s4
+; CHECK-NEXT:    umull r0, r3, r3, r0
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov.32 q2[1], r3
+; CHECK-NEXT:    vmov r3, s6
+; CHECK-NEXT:    umull r0, r3, r3, r0
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.32 q2[3], r3
+; CHECK-NEXT:    vand q0, q2, q7
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    adds r1, r1, r3
+; CHECK-NEXT:    vmov r3, s3
+; CHECK-NEXT:    adcs r2, r0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    adds.w r12, r1, r0
+; CHECK-NEXT:    adc.w r1, r2, r3
+; CHECK-NEXT:    vmov.u16 r2, q6[4]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q6[5]
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q6[6]
+; CHECK-NEXT:    vmov.32 q0[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q6[7]
+; CHECK-NEXT:    vmov.32 q0[3], r2
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    vmrs lr, p0
+; CHECK-NEXT:    vstrw.32 q3, [sp, #48] @ 16-byte Spill
+; CHECK-NEXT:    vstrw.32 q4, [sp] @ 16-byte Spill
+; CHECK-NEXT:    and r3, lr, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q6[0], r3
+; CHECK-NEXT:    vmov.32 q6[1], r3
+; CHECK-NEXT:    ubfx r3, lr, #4, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q6[2], r3
+; CHECK-NEXT:    vmov.32 q6[3], r3
+; CHECK-NEXT:    vmov.u8 r3, q3[4]
+; CHECK-NEXT:    vmov.32 q0[0], r3
+; CHECK-NEXT:    vmov.u8 r3, q3[5]
+; CHECK-NEXT:    vldrw.u32 q3, [sp, #64] @ 16-byte Reload
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vand q0, q0, q4
+; CHECK-NEXT:    vmov.u8 r0, q3[4]
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q3[5]
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov q7, q3
+; CHECK-NEXT:    vand q1, q1, q4
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    umull r0, r3, r0, r3
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov.32 q2[1], r3
+; CHECK-NEXT:    vmov r3, s6
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #48] @ 16-byte Reload
+; CHECK-NEXT:    umull r0, r3, r3, r0
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.32 q2[3], r3
+; CHECK-NEXT:    vand q0, q2, q6
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    adds.w r3, r3, r12
+; CHECK-NEXT:    adcs r1, r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    adds r3, r3, r2
+; CHECK-NEXT:    vmov.u8 r2, q3[6]
+; CHECK-NEXT:    adcs r1, r0
+; CHECK-NEXT:    ubfx r0, lr, #8, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vmov.32 q6[0], r0
+; CHECK-NEXT:    vmov.32 q6[1], r0
+; CHECK-NEXT:    ubfx r0, lr, #12, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vmov.32 q6[2], r0
+; CHECK-NEXT:    vmov.32 q6[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[6]
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[7]
+; CHECK-NEXT:    vmov.32 q1[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q3[7]
+; CHECK-NEXT:    vmov.32 q0[2], r0
+; CHECK-NEXT:    vmov.32 q1[2], r2
+; CHECK-NEXT:    vand q0, q0, q4
+; CHECK-NEXT:    vand q1, q1, q4
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    umull r0, r2, r2, r0
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    umull r0, r2, r2, r0
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vand q0, q2, q6
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    vmov r3, s3
+; CHECK-NEXT:    adcs r1, r0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT:    adds.w r12, r2, r0
+; CHECK-NEXT:    vmov.u8 r2, q5[8]
+; CHECK-NEXT:    vmov.16 q6[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q5[9]
+; CHECK-NEXT:    vmov.16 q6[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q5[10]
+; CHECK-NEXT:    vmov.16 q6[2], r2
+; CHECK-NEXT:    vmov.u8 r2, q5[11]
+; CHECK-NEXT:    vmov.16 q6[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q5[12]
+; CHECK-NEXT:    vmov.16 q6[4], r2
+; CHECK-NEXT:    vmov.u8 r2, q5[13]
+; CHECK-NEXT:    vmov.16 q6[5], r2
+; CHECK-NEXT:    vmov.u8 r2, q5[14]
+; CHECK-NEXT:    vmov.16 q6[6], r2
+; CHECK-NEXT:    vmov.u8 r2, q5[15]
+; CHECK-NEXT:    vmov.16 q6[7], r2
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    vcmp.i16 ne, q6, zr
+; CHECK-NEXT:    vmov.u8 r0, q7[8]
+; CHECK-NEXT:    vpsel q3, q1, q0
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.u16 r2, q3[0]
+; CHECK-NEXT:    vmov.u8 r0, q7[9]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q3[1]
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q3[2]
+; CHECK-NEXT:    vmov.32 q0[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q3[3]
+; CHECK-NEXT:    vmov.32 q0[3], r2
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    vmrs lr, p0
+; CHECK-NEXT:    vldrw.u32 q6, [sp, #48] @ 16-byte Reload
+; CHECK-NEXT:    vldrw.u32 q5, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vand q1, q1, q5
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    and r3, lr, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q4[0], r3
+; CHECK-NEXT:    vmov.32 q4[1], r3
+; CHECK-NEXT:    ubfx r3, lr, #4, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q4[2], r3
+; CHECK-NEXT:    vmov.32 q4[3], r3
+; CHECK-NEXT:    vmov.u8 r3, q6[8]
+; CHECK-NEXT:    vmov.32 q0[0], r3
+; CHECK-NEXT:    vmov.u8 r3, q6[9]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vand q0, q0, q5
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    umull r0, r3, r0, r3
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov.32 q2[1], r3
+; CHECK-NEXT:    vmov r3, s6
+; CHECK-NEXT:    umull r0, r3, r3, r0
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.32 q2[3], r3
+; CHECK-NEXT:    vand q0, q2, q4
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    adds.w r3, r3, r12
+; CHECK-NEXT:    adcs r1, r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    adds r3, r3, r2
+; CHECK-NEXT:    vmov.u8 r2, q7[10]
+; CHECK-NEXT:    vmov.32 q1[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q7[11]
+; CHECK-NEXT:    vmov.32 q1[2], r2
+; CHECK-NEXT:    vand q1, q1, q5
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    adcs r1, r0
+; CHECK-NEXT:    ubfx r0, lr, #8, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vmov.32 q4[0], r0
+; CHECK-NEXT:    vmov.32 q4[1], r0
+; CHECK-NEXT:    ubfx r0, lr, #12, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vmov.32 q4[2], r0
+; CHECK-NEXT:    vmov.32 q4[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q6[10]
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q6[11]
+; CHECK-NEXT:    vmov.32 q0[2], r0
+; CHECK-NEXT:    vand q0, q0, q5
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    umull r0, r2, r2, r0
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    umull r0, r2, r2, r0
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vand q0, q2, q4
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    vmov r3, s3
+; CHECK-NEXT:    adcs r1, r0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    adds.w r12, r2, r0
+; CHECK-NEXT:    vmov.u16 r2, q3[4]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q3[5]
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q3[6]
+; CHECK-NEXT:    vmov.32 q0[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q3[7]
+; CHECK-NEXT:    vmov.32 q0[3], r2
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    vmov.u8 r0, q7[12]
+; CHECK-NEXT:    vmrs lr, p0
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q7[13]
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vand q1, q1, q5
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    and r3, lr, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q3[0], r3
+; CHECK-NEXT:    vmov.32 q3[1], r3
+; CHECK-NEXT:    ubfx r3, lr, #4, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q3[2], r3
+; CHECK-NEXT:    vmov.32 q3[3], r3
+; CHECK-NEXT:    vmov.u8 r3, q6[12]
+; CHECK-NEXT:    vmov.32 q0[0], r3
+; CHECK-NEXT:    vmov.u8 r3, q6[13]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vand q0, q0, q5
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    umull r0, r3, r0, r3
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov.32 q2[1], r3
+; CHECK-NEXT:    vmov r3, s6
+; CHECK-NEXT:    umull r0, r3, r3, r0
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.32 q2[3], r3
+; CHECK-NEXT:    vand q0, q2, q3
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    adds.w r3, r3, r12
+; CHECK-NEXT:    adcs r1, r0
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    adds r3, r3, r2
+; CHECK-NEXT:    vmov.u8 r2, q7[14]
+; CHECK-NEXT:    vmov.32 q1[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q7[15]
+; CHECK-NEXT:    vmov.32 q1[2], r2
+; CHECK-NEXT:    vand q1, q1, q5
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    adcs r1, r0
+; CHECK-NEXT:    ubfx r0, lr, #8, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov.32 q3[1], r0
+; CHECK-NEXT:    ubfx r0, lr, #12, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q6[14]
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q6[15]
+; CHECK-NEXT:    vmov.32 q0[2], r0
+; CHECK-NEXT:    vand q0, q0, q5
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    umull r0, r2, r2, r0
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    umull r0, r2, r2, r0
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vand q0, q2, q3
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    vmov r3, s3
+; CHECK-NEXT:    adcs r1, r0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    adds r0, r0, r2
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    add sp, #88
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %xx = zext <16 x i8> %x to <16 x i64>
+  %yy = zext <16 x i8> %y to <16 x i64>
+  %m = mul <16 x i64> %xx, %yy
+  %s = select <16 x i1> %c, <16 x i64> %m, <16 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v16i64(<16 x i64> %s)
+  ret i64 %z
+}
+
+define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_sext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b) {
+; CHECK-LABEL: add_v16i8_v16i64_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vcmp.i8 eq, q2, zr
+; CHECK-NEXT:    vmov.i8 q2, #0x0
+; CHECK-NEXT:    vmov.i8 q3, #0xff
+; CHECK-NEXT:    vmov.u8 r2, q0[0]
+; CHECK-NEXT:    vpsel q4, q3, q2
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    vmov.u8 r0, q4[0]
+; CHECK-NEXT:    vmov.16 q5[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q4[1]
+; CHECK-NEXT:    vmov.16 q5[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q4[2]
+; CHECK-NEXT:    vmov.16 q5[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q4[3]
+; CHECK-NEXT:    vmov.16 q5[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q4[4]
+; CHECK-NEXT:    vmov.16 q5[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q4[5]
+; CHECK-NEXT:    vmov.16 q5[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q4[6]
+; CHECK-NEXT:    vmov.16 q5[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q4[7]
+; CHECK-NEXT:    vmov.16 q5[7], r0
+; CHECK-NEXT:    vcmp.i16 ne, q5, zr
+; CHECK-NEXT:    vpsel q5, q3, q2
+; CHECK-NEXT:    vmov.u16 r0, q5[0]
+; CHECK-NEXT:    vmov.32 q6[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q5[1]
+; CHECK-NEXT:    vmov.32 q6[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q5[2]
+; CHECK-NEXT:    vmov.32 q6[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q5[3]
+; CHECK-NEXT:    vmov.32 q6[3], r0
+; CHECK-NEXT:    vcmp.i32 ne, q6, zr
+; CHECK-NEXT:    vmrs r0, p0
+; CHECK-NEXT:    and r1, r0, #1
+; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    vmov.32 q6[0], r1
+; CHECK-NEXT:    vmov.32 q6[1], r1
+; CHECK-NEXT:    ubfx r1, r0, #4, #1
+; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    vmov.32 q6[2], r1
+; CHECK-NEXT:    vmov.32 q6[3], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[0]
+; CHECK-NEXT:    sxtb r1, r1
+; CHECK-NEXT:    smull r1, r2, r2, r1
+; CHECK-NEXT:    vmov.32 q7[0], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[1]
+; CHECK-NEXT:    vmov.32 q7[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[1]
+; CHECK-NEXT:    sxtb r1, r1
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    smull r1, r2, r2, r1
+; CHECK-NEXT:    vmov.32 q7[2], r1
+; CHECK-NEXT:    vmov.32 q7[3], r2
+; CHECK-NEXT:    vand q6, q7, q6
+; CHECK-NEXT:    vmov r1, s26
+; CHECK-NEXT:    vmov r2, s24
+; CHECK-NEXT:    vmov r12, s27
+; CHECK-NEXT:    vmov r3, s25
+; CHECK-NEXT:    adds r1, r1, r2
+; CHECK-NEXT:    adc.w r2, r3, r12
+; CHECK-NEXT:    ubfx r3, r0, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r0, r0, #12, #1
+; CHECK-NEXT:    vmov.32 q6[0], r3
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vmov.32 q6[1], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[2]
+; CHECK-NEXT:    vmov.32 q6[2], r0
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    vmov.32 q6[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[2]
+; CHECK-NEXT:    sxtb r0, r0
+; CHECK-NEXT:    smull r0, r3, r3, r0
+; CHECK-NEXT:    vmov.32 q7[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[3]
+; CHECK-NEXT:    vmov.32 q7[1], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[3]
+; CHECK-NEXT:    sxtb r0, r0
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    smull r0, r3, r3, r0
+; CHECK-NEXT:    vmov.32 q7[2], r0
+; CHECK-NEXT:    vmov.32 q7[3], r3
+; CHECK-NEXT:    vand q6, q7, q6
+; CHECK-NEXT:    vmov r3, s24
+; CHECK-NEXT:    vmov r0, s25
+; CHECK-NEXT:    adds r1, r1, r3
+; CHECK-NEXT:    vmov r3, s27
+; CHECK-NEXT:    adcs r2, r0
+; CHECK-NEXT:    vmov r0, s26
+; CHECK-NEXT:    adds.w r12, r1, r0
+; CHECK-NEXT:    vmov.u8 r0, q0[4]
+; CHECK-NEXT:    adc.w r1, r2, r3
+; CHECK-NEXT:    vmov.u16 r2, q5[4]
+; CHECK-NEXT:    vmov.32 q6[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q5[5]
+; CHECK-NEXT:    vmov.32 q6[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q5[6]
+; CHECK-NEXT:    vmov.32 q6[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q5[7]
+; CHECK-NEXT:    vmov.32 q6[3], r2
+; CHECK-NEXT:    sxtb r0, r0
+; CHECK-NEXT:    vcmp.i32 ne, q6, zr
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r3, r2, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q5[0], r3
+; CHECK-NEXT:    vmov.32 q5[1], r3
+; CHECK-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q5[2], r3
+; CHECK-NEXT:    vmov.32 q5[3], r3
+; CHECK-NEXT:    vmov.u8 r3, q1[4]
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    smull r0, r3, r0, r3
+; CHECK-NEXT:    vmov.32 q6[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[5]
+; CHECK-NEXT:    vmov.32 q6[1], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[5]
+; CHECK-NEXT:    sxtb r0, r0
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    smull r0, r3, r3, r0
+; CHECK-NEXT:    vmov.32 q6[2], r0
+; CHECK-NEXT:    vmov.32 q6[3], r3
+; CHECK-NEXT:    vand q5, q6, q5
+; CHECK-NEXT:    vmov r3, s20
+; CHECK-NEXT:    vmov r0, s21
+; CHECK-NEXT:    adds.w r3, r3, r12
+; CHECK-NEXT:    adc.w r12, r1, r0
+; CHECK-NEXT:    vmov r1, s22
+; CHECK-NEXT:    vmov r0, s23
+; CHECK-NEXT:    adds r3, r3, r1
+; CHECK-NEXT:    adc.w r1, r12, r0
+; CHECK-NEXT:    ubfx r0, r2, #8, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vmov.32 q5[0], r0
+; CHECK-NEXT:    vmov.32 q5[1], r0
+; CHECK-NEXT:    ubfx r0, r2, #12, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vmov.u8 r2, q0[6]
+; CHECK-NEXT:    vmov.32 q5[2], r0
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    vmov.32 q5[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[6]
+; CHECK-NEXT:    sxtb r0, r0
+; CHECK-NEXT:    smull r0, r2, r2, r0
+; CHECK-NEXT:    vmov.32 q6[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[7]
+; CHECK-NEXT:    vmov.32 q6[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[7]
+; CHECK-NEXT:    sxtb r0, r0
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    smull r0, r2, r2, r0
+; CHECK-NEXT:    vmov.32 q6[2], r0
+; CHECK-NEXT:    vmov.32 q6[3], r2
+; CHECK-NEXT:    vand q5, q6, q5
+; CHECK-NEXT:    vmov r2, s20
+; CHECK-NEXT:    vmov r0, s21
+; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    vmov r3, s23
+; CHECK-NEXT:    adcs r1, r0
+; CHECK-NEXT:    vmov r0, s22
+; CHECK-NEXT:    adds.w r12, r2, r0
+; CHECK-NEXT:    vmov.u8 r2, q4[8]
+; CHECK-NEXT:    vmov.16 q5[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[9]
+; CHECK-NEXT:    vmov.16 q5[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[10]
+; CHECK-NEXT:    vmov.16 q5[2], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[11]
+; CHECK-NEXT:    vmov.16 q5[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[12]
+; CHECK-NEXT:    vmov.16 q5[4], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[13]
+; CHECK-NEXT:    vmov.16 q5[5], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[14]
+; CHECK-NEXT:    vmov.16 q5[6], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[15]
+; CHECK-NEXT:    vmov.16 q5[7], r2
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    vcmp.i16 ne, q5, zr
+; CHECK-NEXT:    vmov.u8 r0, q0[8]
+; CHECK-NEXT:    vpsel q2, q3, q2
+; CHECK-NEXT:    sxtb r0, r0
+; CHECK-NEXT:    vmov.u16 r2, q2[0]
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[1]
+; CHECK-NEXT:    vmov.32 q3[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[2]
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[3]
+; CHECK-NEXT:    vmov.32 q3[3], r2
+; CHECK-NEXT:    vcmp.i32 ne, q3, zr
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r3, r2, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q3[0], r3
+; CHECK-NEXT:    vmov.32 q3[1], r3
+; CHECK-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q3[2], r3
+; CHECK-NEXT:    vmov.32 q3[3], r3
+; CHECK-NEXT:    vmov.u8 r3, q1[8]
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    smull r0, r3, r0, r3
+; CHECK-NEXT:    vmov.32 q4[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[9]
+; CHECK-NEXT:    vmov.32 q4[1], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[9]
+; CHECK-NEXT:    sxtb r0, r0
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    smull r0, r3, r3, r0
+; CHECK-NEXT:    vmov.32 q4[2], r0
+; CHECK-NEXT:    vmov.32 q4[3], r3
+; CHECK-NEXT:    vand q3, q4, q3
+; CHECK-NEXT:    vmov r3, s12
+; CHECK-NEXT:    vmov r0, s13
+; CHECK-NEXT:    adds.w r3, r3, r12
+; CHECK-NEXT:    adc.w r12, r1, r0
+; CHECK-NEXT:    vmov r1, s14
+; CHECK-NEXT:    vmov r0, s15
+; CHECK-NEXT:    adds r3, r3, r1
+; CHECK-NEXT:    adc.w r1, r12, r0
+; CHECK-NEXT:    ubfx r0, r2, #8, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov.32 q3[1], r0
+; CHECK-NEXT:    ubfx r0, r2, #12, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vmov.u8 r2, q0[10]
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[10]
+; CHECK-NEXT:    sxtb r0, r0
+; CHECK-NEXT:    smull r0, r2, r2, r0
+; CHECK-NEXT:    vmov.32 q4[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[11]
+; CHECK-NEXT:    vmov.32 q4[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[11]
+; CHECK-NEXT:    sxtb r0, r0
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    smull r0, r2, r2, r0
+; CHECK-NEXT:    vmov.32 q4[2], r0
+; CHECK-NEXT:    vmov.32 q4[3], r2
+; CHECK-NEXT:    vand q3, q4, q3
+; CHECK-NEXT:    vmov r2, s12
+; CHECK-NEXT:    vmov r0, s13
+; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    vmov r3, s15
+; CHECK-NEXT:    adcs r1, r0
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    adds.w r12, r2, r0
+; CHECK-NEXT:    vmov.u16 r2, q2[4]
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[5]
+; CHECK-NEXT:    vmov.32 q3[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[6]
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[7]
+; CHECK-NEXT:    vmov.32 q3[3], r2
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    vcmp.i32 ne, q3, zr
+; CHECK-NEXT:    vmov.u8 r0, q0[12]
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    sxtb r0, r0
+; CHECK-NEXT:    and r3, r2, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q2[0], r3
+; CHECK-NEXT:    vmov.32 q2[1], r3
+; CHECK-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q2[2], r3
+; CHECK-NEXT:    vmov.32 q2[3], r3
+; CHECK-NEXT:    vmov.u8 r3, q1[12]
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    smull r0, r3, r0, r3
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[13]
+; CHECK-NEXT:    vmov.32 q3[1], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[13]
+; CHECK-NEXT:    sxtb r0, r0
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    smull r0, r3, r3, r0
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r3
+; CHECK-NEXT:    vand q2, q3, q2
+; CHECK-NEXT:    vmov r3, s8
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    adds.w r3, r3, r12
+; CHECK-NEXT:    adc.w r12, r1, r0
+; CHECK-NEXT:    vmov r1, s10
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    adds r3, r3, r1
+; CHECK-NEXT:    adc.w r1, r12, r0
+; CHECK-NEXT:    ubfx r0, r2, #8, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    ubfx r0, r2, #12, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
+; CHECK-NEXT:    vmov.u8 r2, q0[14]
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[14]
+; CHECK-NEXT:    sxtb r0, r0
+; CHECK-NEXT:    smull r0, r2, r2, r0
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q1[15]
+; CHECK-NEXT:    vmov.32 q3[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q0[15]
+; CHECK-NEXT:    sxtb r0, r0
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    smull r0, r2, r2, r0
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r2
+; CHECK-NEXT:    vand q0, q3, q2
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    vmov r3, s3
+; CHECK-NEXT:    adcs r1, r0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    adds r0, r0, r2
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %xx = sext <16 x i8> %x to <16 x i64>
+  %yy = sext <16 x i8> %y to <16 x i64>
+  %m = mul <16 x i64> %xx, %yy
+  %s = select <16 x i1> %c, <16 x i64> %m, <16 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v16i64(<16 x i64> %s)
+  ret i64 %z
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_zext(<2 x i8> %x, <2 x i8> %y, <2 x i8> %b) {
+; CHECK-LABEL: add_v2i8_v2i64_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vmov.i64 q3, #0xff
+; CHECK-NEXT:    vand q1, q1, q3
+; CHECK-NEXT:    vand q4, q0, q3
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r1, s16
+; CHECK-NEXT:    umull r0, r1, r1, r0
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vmov.32 q0[1], r1
+; CHECK-NEXT:    vmov r1, s18
+; CHECK-NEXT:    vand q1, q2, q3
+; CHECK-NEXT:    umull r0, r1, r1, r0
+; CHECK-NEXT:    vmov.32 q0[2], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <2 x i8> %b, zeroinitializer
+  %xx = zext <2 x i8> %x to <2 x i64>
+  %yy = zext <2 x i8> %y to <2 x i64>
+  %m = mul <2 x i64> %xx, %yy
+  %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  ret i64 %z
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_sext(<2 x i8> %x, <2 x i8> %y, <2 x i8> %b) {
+; CHECK-LABEL: add_v2i8_v2i64_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i32 q3, #0xff
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vand q3, q2, q3
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    sxtb r1, r1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    sxtb r0, r0
+; CHECK-NEXT:    smull r0, r1, r1, r0
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vmov.32 q3[1], r1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    sxtb r0, r0
+; CHECK-NEXT:    sxtb r1, r1
+; CHECK-NEXT:    smull r0, r1, r1, r0
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r1
+; CHECK-NEXT:    vand q0, q3, q2
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <2 x i8> %b, zeroinitializer
+  %xx = sext <2 x i8> %x to <2 x i64>
+  %yy = sext <2 x i8> %y to <2 x i64>
+  %m = mul <2 x i64> %xx, %yy
+  %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  ret i64 %z
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i64_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %b) {
+; CHECK-LABEL: add_v2i64_v2i64:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    vmov r2, s5
+; CHECK-NEXT:    umull r12, r3, r1, r0
+; CHECK-NEXT:    mla r1, r1, r2, r3
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vmov.32 q3[0], r12
+; CHECK-NEXT:    mla r0, r2, r0, r1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov.32 q3[1], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    umull r2, r3, r1, r0
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    vmov r2, s7
+; CHECK-NEXT:    mla r1, r1, r2, r3
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    mla r0, r2, r0, r1
+; CHECK-NEXT:    vmov r1, s8
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r1, s10
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    cset r0, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csetm r0, ne
+; CHECK-NEXT:    vmov.32 q0[2], r0
+; CHECK-NEXT:    vmov.32 q0[3], r0
+; CHECK-NEXT:    vand q0, q3, q0
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <2 x i64> %b, zeroinitializer
+  %m = mul <2 x i64> %x, %y
+  %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  ret i64 %z
+}
+
+define arm_aapcs_vfpcc i32 @add_v4i32_v4i32_acc(<4 x i32> %x, <4 x i32> %y, <4 x i32> %b, i32 %a) {
+; CHECK-LABEL: add_v4i32_v4i32_acc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q2, zr
+; CHECK-NEXT:    vmlavat.u32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %b, zeroinitializer
+  %m = mul <4 x i32> %x, %y
+  %s = select <4 x i1> %c, <4 x i32> %m, <4 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %s)
+  %r = add i32 %z, %a
+  ret i32 %r
+}
+
+define arm_aapcs_vfpcc i64 @add_v4i32_v4i64_acc_zext(<4 x i32> %x, <4 x i32> %y, <4 x i32> %b, i64 %a) {
+; CHECK-LABEL: add_v4i32_v4i64_acc_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q2, zr
+; CHECK-NEXT:    vmlalvat.u32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %b, zeroinitializer
+  %xx = zext <4 x i32> %x to <4 x i64>
+  %yy = zext <4 x i32> %y to <4 x i64>
+  %m = mul <4 x i64> %xx, %yy
+  %s = select <4 x i1> %c, <4 x i64> %m, <4 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v4i64(<4 x i64> %s)
+  %r = add i64 %z, %a
+  ret i64 %r
+}
+
+define arm_aapcs_vfpcc i64 @add_v4i32_v4i64_acc_sext(<4 x i32> %x, <4 x i32> %y, <4 x i32> %b, i64 %a) {
+; CHECK-LABEL: add_v4i32_v4i64_acc_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q2, zr
+; CHECK-NEXT:    vmlalvat.s32 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %b, zeroinitializer
+  %xx = sext <4 x i32> %x to <4 x i64>
+  %yy = sext <4 x i32> %y to <4 x i64>
+  %m = mul <4 x i64> %xx, %yy
+  %s = select <4 x i1> %c, <4 x i64> %m, <4 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v4i64(<4 x i64> %s)
+  %r = add i64 %z, %a
+  ret i64 %r
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i32_v2i64_acc_zext(<2 x i32> %x, <2 x i32> %y, <2 x i32> %b, i64 %a) {
+; CHECK-LABEL: add_v2i32_v2i64_acc_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vmov r2, s8
+; CHECK-NEXT:    vmullb.u32 q3, q0, q1
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov r2, s10
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q0[2], r2
+; CHECK-NEXT:    vmov.32 q0[3], r2
+; CHECK-NEXT:    vand q0, q3, q0
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r12, s3
+; CHECK-NEXT:    vmov lr, s1
+; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    adc.w r3, lr, r12
+; CHECK-NEXT:    adds r0, r0, r2
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %c = icmp eq <2 x i32> %b, zeroinitializer
+  %xx = zext <2 x i32> %x to <2 x i64>
+  %yy = zext <2 x i32> %y to <2 x i64>
+  %m = mul <2 x i64> %xx, %yy
+  %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  %r = add i64 %z, %a
+  ret i64 %r
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i32_v2i64_acc_sext(<2 x i32> %x, <2 x i32> %y, <2 x i32> %b, i64 %a) {
+; CHECK-LABEL: add_v2i32_v2i64_acc_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vmov r2, s8
+; CHECK-NEXT:    vmullb.s32 q3, q0, q1
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov r2, s10
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q0[2], r2
+; CHECK-NEXT:    vmov.32 q0[3], r2
+; CHECK-NEXT:    vand q0, q3, q0
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r12, s3
+; CHECK-NEXT:    vmov lr, s1
+; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    adc.w r3, lr, r12
+; CHECK-NEXT:    adds r0, r0, r2
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %c = icmp eq <2 x i32> %b, zeroinitializer
+  %xx = sext <2 x i32> %x to <2 x i64>
+  %yy = sext <2 x i32> %y to <2 x i64>
+  %m = mul <2 x i64> %xx, %yy
+  %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  %r = add i64 %z, %a
+  ret i64 %r
+}
+
+define arm_aapcs_vfpcc i32 @add_v8i16_v8i32_acc_zext(<8 x i16> %x, <8 x i16> %y, <8 x i16> %b, i32 %a) {
+; CHECK-LABEL: add_v8i16_v8i32_acc_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q2, zr
+; CHECK-NEXT:    vmlavat.u16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %b, zeroinitializer
+  %xx = zext <8 x i16> %x to <8 x i32>
+  %yy = zext <8 x i16> %y to <8 x i32>
+  %m = mul <8 x i32> %xx, %yy
+  %s = select <8 x i1> %c, <8 x i32> %m, <8 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> %s)
+  %r = add i32 %z, %a
+  ret i32 %r
+}
+
+define arm_aapcs_vfpcc i32 @add_v8i16_v8i32_acc_sext(<8 x i16> %x, <8 x i16> %y, <8 x i16> %b, i32 %a) {
+; CHECK-LABEL: add_v8i16_v8i32_acc_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q2, zr
+; CHECK-NEXT:    vmlavat.s16 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %b, zeroinitializer
+  %xx = sext <8 x i16> %x to <8 x i32>
+  %yy = sext <8 x i16> %y to <8 x i32>
+  %m = mul <8 x i32> %xx, %yy
+  %s = select <8 x i1> %c, <8 x i32> %m, <8 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> %s)
+  %r = add i32 %z, %a
+  ret i32 %r
+}
+
+define arm_aapcs_vfpcc i32 @add_v4i16_v4i32_acc_zext(<4 x i16> %x, <4 x i16> %y, <4 x i16> %b, i32 %a) {
+; CHECK-LABEL: add_v4i16_v4i32_acc_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.u16 q1, q1
+; CHECK-NEXT:    vmovlb.u16 q0, q0
+; CHECK-NEXT:    vmovlb.u16 q2, q2
+; CHECK-NEXT:    vpt.i32 eq, q2, zr
+; CHECK-NEXT:    vmlavat.u32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i16> %b, zeroinitializer
+  %xx = zext <4 x i16> %x to <4 x i32>
+  %yy = zext <4 x i16> %y to <4 x i32>
+  %m = mul <4 x i32> %xx, %yy
+  %s = select <4 x i1> %c, <4 x i32> %m, <4 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %s)
+  %r = add i32 %z, %a
+  ret i32 %r
+}
+
+define arm_aapcs_vfpcc i32 @add_v4i16_v4i32_acc_sext(<4 x i16> %x, <4 x i16> %y, <4 x i16> %b, i32 %a) {
+; CHECK-LABEL: add_v4i16_v4i32_acc_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.s16 q1, q1
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    vmovlb.u16 q2, q2
+; CHECK-NEXT:    vpt.i32 eq, q2, zr
+; CHECK-NEXT:    vmlavat.u32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i16> %b, zeroinitializer
+  %xx = sext <4 x i16> %x to <4 x i32>
+  %yy = sext <4 x i16> %y to <4 x i32>
+  %m = mul <4 x i32> %xx, %yy
+  %s = select <4 x i1> %c, <4 x i32> %m, <4 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %s)
+  %r = add i32 %z, %a
+  ret i32 %r
+}
+
+define arm_aapcs_vfpcc zeroext i16 @add_v8i16_v8i16_acc(<8 x i16> %x, <8 x i16> %y, <8 x i16> %b, i16 %a) {
+; CHECK-LABEL: add_v8i16_v8i16_acc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q2, zr
+; CHECK-NEXT:    vmlavat.u16 r0, q0, q1
+; CHECK-NEXT:    uxth r0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %b, zeroinitializer
+  %m = mul <8 x i16> %x, %y
+  %s = select <8 x i1> %c, <8 x i16> %m, <8 x i16> zeroinitializer
+  %z = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> %s)
+  %r = add i16 %z, %a
+  ret i16 %r
+}
+
+define arm_aapcs_vfpcc i64 @add_v8i16_v8i64_acc_zext(<8 x i16> %x, <8 x i16> %y, <8 x i16> %b, i64 %a) {
+; CHECK-LABEL: add_v8i16_v8i64_acc_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q2, zr
+; CHECK-NEXT:    vmlalvat.u16 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %b, zeroinitializer
+  %xx = zext <8 x i16> %x to <8 x i64>
+  %yy = zext <8 x i16> %y to <8 x i64>
+  %m = mul <8 x i64> %xx, %yy
+  %s = select <8 x i1> %c, <8 x i64> %m, <8 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v8i64(<8 x i64> %s)
+  %r = add i64 %z, %a
+  ret i64 %r
+}
+
+define arm_aapcs_vfpcc i64 @add_v8i16_v8i64_acc_sext(<8 x i16> %x, <8 x i16> %y, <8 x i16> %b, i64 %a) {
+; CHECK-LABEL: add_v8i16_v8i64_acc_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q2, zr
+; CHECK-NEXT:    vmlalvat.s16 r0, r1, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %b, zeroinitializer
+  %xx = sext <8 x i16> %x to <8 x i64>
+  %yy = sext <8 x i16> %y to <8 x i64>
+  %m = mul <8 x i64> %xx, %yy
+  %s = select <8 x i1> %c, <8 x i64> %m, <8 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v8i64(<8 x i64> %s)
+  %r = add i64 %z, %a
+  ret i64 %r
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_acc_zext(<2 x i16> %x, <2 x i16> %y, <2 x i16> %b, i64 %a) {
+; CHECK-LABEL: add_v2i16_v2i64_acc_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vmov.i64 q3, #0xffff
+; CHECK-NEXT:    vand q1, q1, q3
+; CHECK-NEXT:    vand q4, q0, q3
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    vmov r3, s16
+; CHECK-NEXT:    umull r2, r3, r3, r2
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    vmov.32 q0[1], r3
+; CHECK-NEXT:    vmov r3, s18
+; CHECK-NEXT:    vand q1, q2, q3
+; CHECK-NEXT:    umull r2, r3, r3, r2
+; CHECK-NEXT:    vmov.32 q0[2], r2
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    vmov.32 q0[3], r3
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r12, s3
+; CHECK-NEXT:    vmov lr, s1
+; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    adc.w r3, lr, r12
+; CHECK-NEXT:    adds r0, r0, r2
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %c = icmp eq <2 x i16> %b, zeroinitializer
+  %xx = zext <2 x i16> %x to <2 x i64>
+  %yy = zext <2 x i16> %y to <2 x i64>
+  %m = mul <2 x i64> %xx, %yy
+  %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  %r = add i64 %z, %a
+  ret i64 %r
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_acc_sext(<2 x i16> %x, <2 x i16> %y, <2 x i16> %b, i64 %a) {
+; CHECK-LABEL: add_v2i16_v2i64_acc_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vmov.i32 q3, #0xffff
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vand q3, q2, q3
+; CHECK-NEXT:    vmov r2, s12
+; CHECK-NEXT:    sxth r3, r3
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov r2, s14
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    sxth r2, r2
+; CHECK-NEXT:    smull r2, r3, r3, r2
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    vmov.32 q3[1], r3
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    sxth r2, r2
+; CHECK-NEXT:    sxth r3, r3
+; CHECK-NEXT:    smull r2, r3, r3, r2
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    vmov.32 q3[3], r3
+; CHECK-NEXT:    vand q0, q3, q2
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r12, s3
+; CHECK-NEXT:    vmov lr, s1
+; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    adc.w r3, lr, r12
+; CHECK-NEXT:    adds r0, r0, r2
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %c = icmp eq <2 x i16> %b, zeroinitializer
+  %xx = sext <2 x i16> %x to <2 x i64>
+  %yy = sext <2 x i16> %y to <2 x i64>
+  %m = mul <2 x i64> %xx, %yy
+  %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  %r = add i64 %z, %a
+  ret i64 %r
+}
+
+define arm_aapcs_vfpcc i32 @add_v16i8_v16i32_acc_zext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b, i32 %a) {
+; CHECK-LABEL: add_v16i8_v16i32_acc_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q2, zr
+; CHECK-NEXT:    vmlavat.u8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %xx = zext <16 x i8> %x to <16 x i32>
+  %yy = zext <16 x i8> %y to <16 x i32>
+  %m = mul <16 x i32> %xx, %yy
+  %s = select <16 x i1> %c, <16 x i32> %m, <16 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v16i32(<16 x i32> %s)
+  %r = add i32 %z, %a
+  ret i32 %r
+}
+
+define arm_aapcs_vfpcc i32 @add_v16i8_v16i32_acc_sext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b, i32 %a) {
+; CHECK-LABEL: add_v16i8_v16i32_acc_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q2, zr
+; CHECK-NEXT:    vmlavat.s8 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %xx = sext <16 x i8> %x to <16 x i32>
+  %yy = sext <16 x i8> %y to <16 x i32>
+  %m = mul <16 x i32> %xx, %yy
+  %s = select <16 x i1> %c, <16 x i32> %m, <16 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v16i32(<16 x i32> %s)
+  %r = add i32 %z, %a
+  ret i32 %r
+}
+
+define arm_aapcs_vfpcc i32 @add_v4i8_v4i32_acc_zext(<4 x i8> %x, <4 x i8> %y, <4 x i8> %b, i32 %a) {
+; CHECK-LABEL: add_v4i8_v4i32_acc_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i32 q3, #0xff
+; CHECK-NEXT:    vand q1, q1, q3
+; CHECK-NEXT:    vand q0, q0, q3
+; CHECK-NEXT:    vand q2, q2, q3
+; CHECK-NEXT:    vpt.i32 eq, q2, zr
+; CHECK-NEXT:    vmlavat.u32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i8> %b, zeroinitializer
+  %xx = zext <4 x i8> %x to <4 x i32>
+  %yy = zext <4 x i8> %y to <4 x i32>
+  %m = mul <4 x i32> %xx, %yy
+  %s = select <4 x i1> %c, <4 x i32> %m, <4 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %s)
+  %r = add i32 %z, %a
+  ret i32 %r
+}
+
+define arm_aapcs_vfpcc i32 @add_v4i8_v4i32_acc_sext(<4 x i8> %x, <4 x i8> %y, <4 x i8> %b, i32 %a) {
+; CHECK-LABEL: add_v4i8_v4i32_acc_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.s8 q1, q1
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    vmov.i32 q3, #0xff
+; CHECK-NEXT:    vmovlb.s16 q1, q1
+; CHECK-NEXT:    vand q2, q2, q3
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    vpt.i32 eq, q2, zr
+; CHECK-NEXT:    vmlavat.u32 r0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i8> %b, zeroinitializer
+  %xx = sext <4 x i8> %x to <4 x i32>
+  %yy = sext <4 x i8> %y to <4 x i32>
+  %m = mul <4 x i32> %xx, %yy
+  %s = select <4 x i1> %c, <4 x i32> %m, <4 x i32> zeroinitializer
+  %z = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %s)
+  %r = add i32 %z, %a
+  ret i32 %r
+}
+
+define arm_aapcs_vfpcc zeroext i16 @add_v16i8_v16i16_acc_zext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b, i16 %a) {
+; CHECK-LABEL: add_v16i8_v16i16_acc_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vcmp.i8 eq, q2, zr
+; CHECK-NEXT:    vmov.i8 q2, #0x0
+; CHECK-NEXT:    vmov.i8 q3, #0xff
+; CHECK-NEXT:    vpsel q2, q3, q2
+; CHECK-NEXT:    vmov.u8 r1, q2[0]
+; CHECK-NEXT:    vmov.16 q3[0], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[1]
+; CHECK-NEXT:    vmov.16 q3[1], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[2]
+; CHECK-NEXT:    vmov.16 q3[2], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[3]
+; CHECK-NEXT:    vmov.16 q3[3], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[4]
+; CHECK-NEXT:    vmov.16 q3[4], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[5]
+; CHECK-NEXT:    vmov.16 q3[5], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[6]
+; CHECK-NEXT:    vmov.16 q3[6], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[7]
+; CHECK-NEXT:    vmov.16 q3[7], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[0]
+; CHECK-NEXT:    vcmp.i16 ne, q3, zr
+; CHECK-NEXT:    vmov.16 q3[0], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[1]
+; CHECK-NEXT:    vmov.16 q3[1], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[2]
+; CHECK-NEXT:    vmov.16 q3[2], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[3]
+; CHECK-NEXT:    vmov.16 q3[3], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[4]
+; CHECK-NEXT:    vmov.16 q3[4], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[5]
+; CHECK-NEXT:    vmov.16 q3[5], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[6]
+; CHECK-NEXT:    vmov.16 q3[6], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[7]
+; CHECK-NEXT:    vmov.16 q3[7], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[0]
+; CHECK-NEXT:    vmovlb.u8 q4, q3
+; CHECK-NEXT:    vmov.16 q3[0], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[1]
+; CHECK-NEXT:    vmov.16 q3[1], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[2]
+; CHECK-NEXT:    vmov.16 q3[2], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[3]
+; CHECK-NEXT:    vmov.16 q3[3], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[4]
+; CHECK-NEXT:    vmov.16 q3[4], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[5]
+; CHECK-NEXT:    vmov.16 q3[5], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[6]
+; CHECK-NEXT:    vmov.16 q3[6], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[7]
+; CHECK-NEXT:    vmov.16 q3[7], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[8]
+; CHECK-NEXT:    vmovlb.u8 q5, q3
+; CHECK-NEXT:    vmov.i32 q3, #0x0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i16 q3, q5, q4
+; CHECK-NEXT:    vmov.16 q4[0], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[9]
+; CHECK-NEXT:    vmov.16 q4[1], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[10]
+; CHECK-NEXT:    vmov.16 q4[2], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[11]
+; CHECK-NEXT:    vmov.16 q4[3], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[12]
+; CHECK-NEXT:    vmov.16 q4[4], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[13]
+; CHECK-NEXT:    vmov.16 q4[5], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[14]
+; CHECK-NEXT:    vmov.16 q4[6], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[15]
+; CHECK-NEXT:    vmov.16 q4[7], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[8]
+; CHECK-NEXT:    vmov.16 q2[0], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[9]
+; CHECK-NEXT:    vmov.16 q2[1], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[10]
+; CHECK-NEXT:    vmov.16 q2[2], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[11]
+; CHECK-NEXT:    vmov.16 q2[3], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[12]
+; CHECK-NEXT:    vmov.16 q2[4], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[13]
+; CHECK-NEXT:    vmov.16 q2[5], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[14]
+; CHECK-NEXT:    vmov.16 q2[6], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[15]
+; CHECK-NEXT:    vmov.16 q2[7], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[8]
+; CHECK-NEXT:    vmov.16 q1[0], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[9]
+; CHECK-NEXT:    vmov.16 q1[1], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[10]
+; CHECK-NEXT:    vmov.16 q1[2], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[11]
+; CHECK-NEXT:    vmov.16 q1[3], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[12]
+; CHECK-NEXT:    vmov.16 q1[4], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[13]
+; CHECK-NEXT:    vmov.16 q1[5], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[14]
+; CHECK-NEXT:    vmov.16 q1[6], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[15]
+; CHECK-NEXT:    vmov.16 q1[7], r1
+; CHECK-NEXT:    vmullb.u8 q0, q1, q2
+; CHECK-NEXT:    vpt.i16 ne, q4, zr
+; CHECK-NEXT:    vaddt.i16 q3, q3, q0
+; CHECK-NEXT:    vaddva.u16 r0, q3
+; CHECK-NEXT:    uxth r0, r0
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %xx = zext <16 x i8> %x to <16 x i16>
+  %yy = zext <16 x i8> %y to <16 x i16>
+  %m = mul <16 x i16> %xx, %yy
+  %s = select <16 x i1> %c, <16 x i16> %m, <16 x i16> zeroinitializer
+  %z = call i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16> %s)
+  %r = add i16 %z, %a
+  ret i16 %r
+}
+
+define arm_aapcs_vfpcc signext i16 @add_v16i8_v16i16_acc_sext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b, i16 %a) {
+; CHECK-LABEL: add_v16i8_v16i16_acc_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vcmp.i8 eq, q2, zr
+; CHECK-NEXT:    vmov.i8 q2, #0x0
+; CHECK-NEXT:    vmov.i8 q3, #0xff
+; CHECK-NEXT:    vpsel q2, q3, q2
+; CHECK-NEXT:    vmov.u8 r1, q2[0]
+; CHECK-NEXT:    vmov.16 q3[0], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[1]
+; CHECK-NEXT:    vmov.16 q3[1], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[2]
+; CHECK-NEXT:    vmov.16 q3[2], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[3]
+; CHECK-NEXT:    vmov.16 q3[3], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[4]
+; CHECK-NEXT:    vmov.16 q3[4], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[5]
+; CHECK-NEXT:    vmov.16 q3[5], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[6]
+; CHECK-NEXT:    vmov.16 q3[6], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[7]
+; CHECK-NEXT:    vmov.16 q3[7], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[0]
+; CHECK-NEXT:    vcmp.i16 ne, q3, zr
+; CHECK-NEXT:    vmov.16 q3[0], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[1]
+; CHECK-NEXT:    vmov.16 q3[1], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[2]
+; CHECK-NEXT:    vmov.16 q3[2], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[3]
+; CHECK-NEXT:    vmov.16 q3[3], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[4]
+; CHECK-NEXT:    vmov.16 q3[4], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[5]
+; CHECK-NEXT:    vmov.16 q3[5], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[6]
+; CHECK-NEXT:    vmov.16 q3[6], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[7]
+; CHECK-NEXT:    vmov.16 q3[7], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[0]
+; CHECK-NEXT:    vmovlb.s8 q4, q3
+; CHECK-NEXT:    vmov.16 q3[0], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[1]
+; CHECK-NEXT:    vmov.16 q3[1], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[2]
+; CHECK-NEXT:    vmov.16 q3[2], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[3]
+; CHECK-NEXT:    vmov.16 q3[3], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[4]
+; CHECK-NEXT:    vmov.16 q3[4], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[5]
+; CHECK-NEXT:    vmov.16 q3[5], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[6]
+; CHECK-NEXT:    vmov.16 q3[6], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[7]
+; CHECK-NEXT:    vmov.16 q3[7], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[8]
+; CHECK-NEXT:    vmovlb.s8 q5, q3
+; CHECK-NEXT:    vmov.i32 q3, #0x0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmult.i16 q3, q5, q4
+; CHECK-NEXT:    vmov.16 q4[0], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[9]
+; CHECK-NEXT:    vmov.16 q4[1], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[10]
+; CHECK-NEXT:    vmov.16 q4[2], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[11]
+; CHECK-NEXT:    vmov.16 q4[3], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[12]
+; CHECK-NEXT:    vmov.16 q4[4], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[13]
+; CHECK-NEXT:    vmov.16 q4[5], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[14]
+; CHECK-NEXT:    vmov.16 q4[6], r1
+; CHECK-NEXT:    vmov.u8 r1, q2[15]
+; CHECK-NEXT:    vmov.16 q4[7], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[8]
+; CHECK-NEXT:    vmov.16 q2[0], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[9]
+; CHECK-NEXT:    vmov.16 q2[1], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[10]
+; CHECK-NEXT:    vmov.16 q2[2], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[11]
+; CHECK-NEXT:    vmov.16 q2[3], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[12]
+; CHECK-NEXT:    vmov.16 q2[4], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[13]
+; CHECK-NEXT:    vmov.16 q2[5], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[14]
+; CHECK-NEXT:    vmov.16 q2[6], r1
+; CHECK-NEXT:    vmov.u8 r1, q1[15]
+; CHECK-NEXT:    vmov.16 q2[7], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[8]
+; CHECK-NEXT:    vmov.16 q1[0], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[9]
+; CHECK-NEXT:    vmov.16 q1[1], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[10]
+; CHECK-NEXT:    vmov.16 q1[2], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[11]
+; CHECK-NEXT:    vmov.16 q1[3], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[12]
+; CHECK-NEXT:    vmov.16 q1[4], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[13]
+; CHECK-NEXT:    vmov.16 q1[5], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[14]
+; CHECK-NEXT:    vmov.16 q1[6], r1
+; CHECK-NEXT:    vmov.u8 r1, q0[15]
+; CHECK-NEXT:    vmov.16 q1[7], r1
+; CHECK-NEXT:    vmullb.s8 q0, q1, q2
+; CHECK-NEXT:    vpt.i16 ne, q4, zr
+; CHECK-NEXT:    vaddt.i16 q3, q3, q0
+; CHECK-NEXT:    vaddva.u16 r0, q3
+; CHECK-NEXT:    sxth r0, r0
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %xx = sext <16 x i8> %x to <16 x i16>
+  %yy = sext <16 x i8> %y to <16 x i16>
+  %m = mul <16 x i16> %xx, %yy
+  %s = select <16 x i1> %c, <16 x i16> %m, <16 x i16> zeroinitializer
+  %z = call i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16> %s)
+  %r = add i16 %z, %a
+  ret i16 %r
+}
+
+define arm_aapcs_vfpcc zeroext i16 @add_v8i8_v8i16_acc_zext(<8 x i8> %x, <8 x i8> %y, <8 x i8> %b, i16 %a) {
+; CHECK-LABEL: add_v8i8_v8i16_acc_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.u8 q1, q1
+; CHECK-NEXT:    vmovlb.u8 q0, q0
+; CHECK-NEXT:    vmovlb.u8 q2, q2
+; CHECK-NEXT:    vpt.i16 eq, q2, zr
+; CHECK-NEXT:    vmlavat.u16 r0, q0, q1
+; CHECK-NEXT:    uxth r0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i8> %b, zeroinitializer
+  %xx = zext <8 x i8> %x to <8 x i16>
+  %yy = zext <8 x i8> %y to <8 x i16>
+  %m = mul <8 x i16> %xx, %yy
+  %s = select <8 x i1> %c, <8 x i16> %m, <8 x i16> zeroinitializer
+  %z = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> %s)
+  %r = add i16 %z, %a
+  ret i16 %r
+}
+
+define arm_aapcs_vfpcc signext i16 @add_v8i8_v8i16_acc_sext(<8 x i8> %x, <8 x i8> %y, <8 x i8> %b, i16 %a) {
+; CHECK-LABEL: add_v8i8_v8i16_acc_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.s8 q1, q1
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    vmovlb.u8 q2, q2
+; CHECK-NEXT:    vpt.i16 eq, q2, zr
+; CHECK-NEXT:    vmlavat.u16 r0, q0, q1
+; CHECK-NEXT:    sxth r0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i8> %b, zeroinitializer
+  %xx = sext <8 x i8> %x to <8 x i16>
+  %yy = sext <8 x i8> %y to <8 x i16>
+  %m = mul <8 x i16> %xx, %yy
+  %s = select <8 x i1> %c, <8 x i16> %m, <8 x i16> zeroinitializer
+  %z = call i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16> %s)
+  %r = add i16 %z, %a
+  ret i16 %r
+}
+
+define arm_aapcs_vfpcc zeroext i8 @add_v16i8_v16i8_acc(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b, i8 %a) {
+; CHECK-LABEL: add_v16i8_v16i8_acc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q2, zr
+; CHECK-NEXT:    vmlavat.u8 r0, q0, q1
+; CHECK-NEXT:    uxtb r0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %m = mul <16 x i8> %x, %y
+  %s = select <16 x i1> %c, <16 x i8> %m, <16 x i8> zeroinitializer
+  %z = call i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8> %s)
+  %r = add i8 %z, %a
+  ret i8 %r
+}
+
+define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_acc_zext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b, i64 %a) {
+; CHECK-LABEL: add_v16i8_v16i64_acc_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #80
+; CHECK-NEXT:    sub sp, #80
+; CHECK-NEXT:    vmov q3, q1
+; CHECK-NEXT:    vstrw.32 q0, [sp, #64] @ 16-byte Spill
+; CHECK-NEXT:    vmov.i8 q0, #0x0
+; CHECK-NEXT:    vmov.i8 q1, #0xff
+; CHECK-NEXT:    vcmp.i8 eq, q2, zr
+; CHECK-NEXT:    vstrw.32 q1, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vpsel q5, q1, q0
+; CHECK-NEXT:    vstrw.32 q0, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    vmov.u8 r2, q5[0]
+; CHECK-NEXT:    vmov.i64 q4, #0xff
+; CHECK-NEXT:    vmov.16 q2[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q5[1]
+; CHECK-NEXT:    vmov.16 q2[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q5[2]
+; CHECK-NEXT:    vmov.16 q2[2], r2
+; CHECK-NEXT:    vmov.u8 r2, q5[3]
+; CHECK-NEXT:    vmov.16 q2[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q5[4]
+; CHECK-NEXT:    vmov.16 q2[4], r2
+; CHECK-NEXT:    vmov.u8 r2, q5[5]
+; CHECK-NEXT:    vmov.16 q2[5], r2
+; CHECK-NEXT:    vmov.u8 r2, q5[6]
+; CHECK-NEXT:    vmov.16 q2[6], r2
+; CHECK-NEXT:    vmov.u8 r2, q5[7]
+; CHECK-NEXT:    vmov.16 q2[7], r2
+; CHECK-NEXT:    vcmp.i16 ne, q2, zr
+; CHECK-NEXT:    vpsel q6, q1, q0
+; CHECK-NEXT:    vmov.u16 r2, q6[0]
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q6[1]
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q6[2]
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q6[3]
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vcmp.i32 ne, q2, zr
+; CHECK-NEXT:    vmrs lr, p0
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #64] @ 16-byte Reload
+; CHECK-NEXT:    and r3, lr, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q7[0], r3
+; CHECK-NEXT:    vmov.32 q7[1], r3
+; CHECK-NEXT:    ubfx r3, lr, #4, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q7[2], r3
+; CHECK-NEXT:    vmov.32 q7[3], r3
+; CHECK-NEXT:    vmov.u8 r3, q3[0]
+; CHECK-NEXT:    vmov.32 q0[0], r3
+; CHECK-NEXT:    vmov.u8 r3, q3[1]
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.u8 r3, q1[0]
+; CHECK-NEXT:    vand q2, q0, q4
+; CHECK-NEXT:    vmov.32 q0[0], r3
+; CHECK-NEXT:    vmov.u8 r3, q1[1]
+; CHECK-NEXT:    vmov r12, s8
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vand q1, q0, q4
+; CHECK-NEXT:    vmov r3, s4
+; CHECK-NEXT:    umull r3, r2, r3, r12
+; CHECK-NEXT:    vmov.32 q0[0], r3
+; CHECK-NEXT:    vmov r3, s6
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov r2, s10
+; CHECK-NEXT:    vldrw.u32 q2, [sp, #64] @ 16-byte Reload
+; CHECK-NEXT:    umull r2, r3, r3, r2
+; CHECK-NEXT:    vmov.32 q0[2], r2
+; CHECK-NEXT:    vmov.32 q0[3], r3
+; CHECK-NEXT:    vand q0, q0, q7
+; CHECK-NEXT:    vmov r4, s2
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r12, s3
+; CHECK-NEXT:    vmov r3, s1
+; CHECK-NEXT:    adds r4, r4, r2
+; CHECK-NEXT:    ubfx r2, lr, #8, #1
+; CHECK-NEXT:    rsb.w r2, r2, #0
+; CHECK-NEXT:    vmov.32 q7[0], r2
+; CHECK-NEXT:    adc.w r12, r12, r3
+; CHECK-NEXT:    vmov.32 q7[1], r2
+; CHECK-NEXT:    ubfx r2, lr, #12, #1
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov.u8 r3, q2[2]
+; CHECK-NEXT:    vmov.32 q7[2], r2
+; CHECK-NEXT:    vmov.32 q1[0], r3
+; CHECK-NEXT:    vmov.32 q7[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q3[2]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q3[3]
+; CHECK-NEXT:    vmov.u8 r3, q2[3]
+; CHECK-NEXT:    vmov.32 q0[2], r2
+; CHECK-NEXT:    vmov.32 q1[2], r3
+; CHECK-NEXT:    vand q0, q0, q4
+; CHECK-NEXT:    vand q1, q1, q4
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r3, s4
+; CHECK-NEXT:    umull r2, r3, r3, r2
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov.32 q2[1], r3
+; CHECK-NEXT:    vmov r3, s6
+; CHECK-NEXT:    umull r2, r3, r3, r2
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    vmov.32 q2[3], r3
+; CHECK-NEXT:    vand q0, q2, q7
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    adds r3, r3, r4
+; CHECK-NEXT:    vmov r4, s3
+; CHECK-NEXT:    adc.w lr, r12, r2
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    adds.w r12, r3, r2
+; CHECK-NEXT:    vmov.u16 r2, q6[4]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q6[5]
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q6[6]
+; CHECK-NEXT:    vmov.32 q0[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q6[7]
+; CHECK-NEXT:    vmov.32 q0[3], r2
+; CHECK-NEXT:    adc.w lr, lr, r4
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    vmrs r6, p0
+; CHECK-NEXT:    vstrw.32 q3, [sp, #48] @ 16-byte Spill
+; CHECK-NEXT:    vstrw.32 q4, [sp] @ 16-byte Spill
+; CHECK-NEXT:    and r4, r6, #1
+; CHECK-NEXT:    rsbs r4, r4, #0
+; CHECK-NEXT:    vmov.32 q6[0], r4
+; CHECK-NEXT:    vmov.32 q6[1], r4
+; CHECK-NEXT:    ubfx r4, r6, #4, #1
+; CHECK-NEXT:    rsbs r4, r4, #0
+; CHECK-NEXT:    vmov.32 q6[2], r4
+; CHECK-NEXT:    vmov.32 q6[3], r4
+; CHECK-NEXT:    vmov.u8 r4, q3[4]
+; CHECK-NEXT:    vmov.32 q0[0], r4
+; CHECK-NEXT:    vmov.u8 r4, q3[5]
+; CHECK-NEXT:    vldrw.u32 q3, [sp, #64] @ 16-byte Reload
+; CHECK-NEXT:    vmov.32 q0[2], r4
+; CHECK-NEXT:    vand q0, q0, q4
+; CHECK-NEXT:    vmov.u8 r3, q3[4]
+; CHECK-NEXT:    vmov r4, s0
+; CHECK-NEXT:    vmov.32 q1[0], r3
+; CHECK-NEXT:    vmov.u8 r3, q3[5]
+; CHECK-NEXT:    vmov.32 q1[2], r3
+; CHECK-NEXT:    vmov q7, q3
+; CHECK-NEXT:    vand q1, q1, q4
+; CHECK-NEXT:    vmov r3, s4
+; CHECK-NEXT:    umull r3, r4, r3, r4
+; CHECK-NEXT:    vmov.32 q2[0], r3
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov.32 q2[1], r4
+; CHECK-NEXT:    vmov r4, s6
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #48] @ 16-byte Reload
+; CHECK-NEXT:    umull r3, r4, r4, r3
+; CHECK-NEXT:    vmov.32 q2[2], r3
+; CHECK-NEXT:    vmov.32 q2[3], r4
+; CHECK-NEXT:    vand q0, q2, q6
+; CHECK-NEXT:    vmov r4, s0
+; CHECK-NEXT:    vmov r3, s1
+; CHECK-NEXT:    vmov r5, s2
+; CHECK-NEXT:    vmov r2, s3
+; CHECK-NEXT:    adds.w r4, r4, r12
+; CHECK-NEXT:    adc.w r3, r3, lr
+; CHECK-NEXT:    adds r4, r4, r5
+; CHECK-NEXT:    adc.w r12, r3, r2
+; CHECK-NEXT:    ubfx r2, r6, #8, #1
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov.u8 r3, q3[6]
+; CHECK-NEXT:    vmov.32 q6[0], r2
+; CHECK-NEXT:    vmov.32 q6[1], r2
+; CHECK-NEXT:    ubfx r2, r6, #12, #1
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov.32 q6[2], r2
+; CHECK-NEXT:    vmov.32 q6[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q1[6]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q1[7]
+; CHECK-NEXT:    vmov.32 q1[0], r3
+; CHECK-NEXT:    vmov.u8 r3, q3[7]
+; CHECK-NEXT:    vmov.32 q0[2], r2
+; CHECK-NEXT:    vmov.32 q1[2], r3
+; CHECK-NEXT:    vand q0, q0, q4
+; CHECK-NEXT:    vand q1, q1, q4
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r3, s4
+; CHECK-NEXT:    umull r2, r3, r3, r2
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov.32 q2[1], r3
+; CHECK-NEXT:    vmov r3, s6
+; CHECK-NEXT:    vldrw.u32 q1, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    umull r2, r3, r3, r2
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    vmov.32 q2[3], r3
+; CHECK-NEXT:    vand q0, q2, q6
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vmov r5, s2
+; CHECK-NEXT:    vmov r6, s3
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT:    adds r3, r3, r4
+; CHECK-NEXT:    adc.w r2, r2, r12
+; CHECK-NEXT:    adds.w r12, r3, r5
+; CHECK-NEXT:    vmov.u8 r5, q7[8]
+; CHECK-NEXT:    adc.w r3, r2, r6
+; CHECK-NEXT:    vmov.u8 r2, q5[8]
+; CHECK-NEXT:    vmov.16 q6[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q5[9]
+; CHECK-NEXT:    vmov.16 q6[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q5[10]
+; CHECK-NEXT:    vmov.16 q6[2], r2
+; CHECK-NEXT:    vmov.u8 r2, q5[11]
+; CHECK-NEXT:    vmov.16 q6[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q5[12]
+; CHECK-NEXT:    vmov.16 q6[4], r2
+; CHECK-NEXT:    vmov.u8 r2, q5[13]
+; CHECK-NEXT:    vmov.16 q6[5], r2
+; CHECK-NEXT:    vmov.u8 r2, q5[14]
+; CHECK-NEXT:    vmov.16 q6[6], r2
+; CHECK-NEXT:    vmov.u8 r2, q5[15]
+; CHECK-NEXT:    vmov.16 q6[7], r2
+; CHECK-NEXT:    vcmp.i16 ne, q6, zr
+; CHECK-NEXT:    vpsel q3, q1, q0
+; CHECK-NEXT:    vmov.32 q1[0], r5
+; CHECK-NEXT:    vmov.u16 r2, q3[0]
+; CHECK-NEXT:    vmov.u8 r5, q7[9]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q3[1]
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q3[2]
+; CHECK-NEXT:    vmov.32 q0[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q3[3]
+; CHECK-NEXT:    vmov.32 q0[3], r2
+; CHECK-NEXT:    vmov.32 q1[2], r5
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    vldrw.u32 q6, [sp, #48] @ 16-byte Reload
+; CHECK-NEXT:    vldrw.u32 q5, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vand q1, q1, q5
+; CHECK-NEXT:    vmov r5, s4
+; CHECK-NEXT:    and r6, r2, #1
+; CHECK-NEXT:    rsbs r6, r6, #0
+; CHECK-NEXT:    vmov.32 q4[0], r6
+; CHECK-NEXT:    vmov.32 q4[1], r6
+; CHECK-NEXT:    ubfx r6, r2, #4, #1
+; CHECK-NEXT:    rsbs r6, r6, #0
+; CHECK-NEXT:    vmov.32 q4[2], r6
+; CHECK-NEXT:    vmov.32 q4[3], r6
+; CHECK-NEXT:    vmov.u8 r6, q6[8]
+; CHECK-NEXT:    vmov.32 q0[0], r6
+; CHECK-NEXT:    vmov.u8 r6, q6[9]
+; CHECK-NEXT:    vmov.32 q0[2], r6
+; CHECK-NEXT:    vand q0, q0, q5
+; CHECK-NEXT:    vmov r6, s0
+; CHECK-NEXT:    umull r6, r5, r5, r6
+; CHECK-NEXT:    vmov.32 q2[0], r6
+; CHECK-NEXT:    vmov r6, s2
+; CHECK-NEXT:    vmov.32 q2[1], r5
+; CHECK-NEXT:    vmov r5, s6
+; CHECK-NEXT:    umull r6, r5, r5, r6
+; CHECK-NEXT:    vmov.32 q2[2], r6
+; CHECK-NEXT:    vmov.32 q2[3], r5
+; CHECK-NEXT:    vand q0, q2, q4
+; CHECK-NEXT:    vmov r5, s0
+; CHECK-NEXT:    vmov r6, s1
+; CHECK-NEXT:    vmov r4, s3
+; CHECK-NEXT:    adds.w r5, r5, r12
+; CHECK-NEXT:    adcs r6, r3
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    adds r3, r3, r5
+; CHECK-NEXT:    adc.w r12, r6, r4
+; CHECK-NEXT:    ubfx r6, r2, #8, #1
+; CHECK-NEXT:    rsbs r6, r6, #0
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    vmov.32 q4[0], r6
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov.32 q4[1], r6
+; CHECK-NEXT:    vmov.u8 r6, q7[10]
+; CHECK-NEXT:    vmov.32 q4[2], r2
+; CHECK-NEXT:    vmov.32 q1[0], r6
+; CHECK-NEXT:    vmov.32 q4[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q6[10]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q6[11]
+; CHECK-NEXT:    vmov.u8 r6, q7[11]
+; CHECK-NEXT:    vmov.32 q0[2], r2
+; CHECK-NEXT:    vmov.32 q1[2], r6
+; CHECK-NEXT:    vand q0, q0, q5
+; CHECK-NEXT:    vand q1, q1, q5
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r6, s4
+; CHECK-NEXT:    umull r2, r6, r6, r2
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov.32 q2[1], r6
+; CHECK-NEXT:    vmov r6, s6
+; CHECK-NEXT:    umull r2, r6, r6, r2
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    vmov.32 q2[3], r6
+; CHECK-NEXT:    vand q0, q2, q4
+; CHECK-NEXT:    vmov r6, s0
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vmov r5, s2
+; CHECK-NEXT:    adds r3, r3, r6
+; CHECK-NEXT:    vmov r6, s3
+; CHECK-NEXT:    adc.w r2, r2, r12
+; CHECK-NEXT:    adds.w r12, r3, r5
+; CHECK-NEXT:    vmov.u8 r5, q7[12]
+; CHECK-NEXT:    vmov.32 q1[0], r5
+; CHECK-NEXT:    vmov.u8 r5, q7[13]
+; CHECK-NEXT:    vmov.32 q1[2], r5
+; CHECK-NEXT:    vand q1, q1, q5
+; CHECK-NEXT:    vmov r5, s4
+; CHECK-NEXT:    adc.w r3, r2, r6
+; CHECK-NEXT:    vmov.u16 r2, q3[4]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q3[5]
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q3[6]
+; CHECK-NEXT:    vmov.32 q0[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q3[7]
+; CHECK-NEXT:    vmov.32 q0[3], r2
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r6, r2, #1
+; CHECK-NEXT:    rsbs r6, r6, #0
+; CHECK-NEXT:    vmov.32 q3[0], r6
+; CHECK-NEXT:    vmov.32 q3[1], r6
+; CHECK-NEXT:    ubfx r6, r2, #4, #1
+; CHECK-NEXT:    rsbs r6, r6, #0
+; CHECK-NEXT:    vmov.32 q3[2], r6
+; CHECK-NEXT:    vmov.32 q3[3], r6
+; CHECK-NEXT:    vmov.u8 r6, q6[12]
+; CHECK-NEXT:    vmov.32 q0[0], r6
+; CHECK-NEXT:    vmov.u8 r6, q6[13]
+; CHECK-NEXT:    vmov.32 q0[2], r6
+; CHECK-NEXT:    vand q0, q0, q5
+; CHECK-NEXT:    vmov r6, s0
+; CHECK-NEXT:    umull r6, r5, r5, r6
+; CHECK-NEXT:    vmov.32 q2[0], r6
+; CHECK-NEXT:    vmov r6, s2
+; CHECK-NEXT:    vmov.32 q2[1], r5
+; CHECK-NEXT:    vmov r5, s6
+; CHECK-NEXT:    umull r6, r5, r5, r6
+; CHECK-NEXT:    vmov.32 q2[2], r6
+; CHECK-NEXT:    vmov.32 q2[3], r5
+; CHECK-NEXT:    vand q0, q2, q3
+; CHECK-NEXT:    vmov r5, s0
+; CHECK-NEXT:    vmov r6, s1
+; CHECK-NEXT:    vmov r4, s3
+; CHECK-NEXT:    adds.w r5, r5, r12
+; CHECK-NEXT:    adcs r6, r3
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    adds r3, r3, r5
+; CHECK-NEXT:    adc.w r12, r6, r4
+; CHECK-NEXT:    ubfx r6, r2, #8, #1
+; CHECK-NEXT:    rsbs r6, r6, #0
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    vmov.32 q3[0], r6
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov.32 q3[1], r6
+; CHECK-NEXT:    vmov.u8 r6, q7[14]
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    vmov.32 q1[0], r6
+; CHECK-NEXT:    vmov.32 q3[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q6[14]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q6[15]
+; CHECK-NEXT:    vmov.u8 r6, q7[15]
+; CHECK-NEXT:    vmov.32 q0[2], r2
+; CHECK-NEXT:    vmov.32 q1[2], r6
+; CHECK-NEXT:    vand q0, q0, q5
+; CHECK-NEXT:    vand q1, q1, q5
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r6, s4
+; CHECK-NEXT:    umull r2, r6, r6, r2
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov.32 q2[1], r6
+; CHECK-NEXT:    vmov r6, s6
+; CHECK-NEXT:    umull r2, r6, r6, r2
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    vmov.32 q2[3], r6
+; CHECK-NEXT:    vand q0, q2, q3
+; CHECK-NEXT:    vmov r6, s0
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vmov r5, s2
+; CHECK-NEXT:    adds r3, r3, r6
+; CHECK-NEXT:    vmov r6, s3
+; CHECK-NEXT:    adc.w r2, r2, r12
+; CHECK-NEXT:    adds r3, r3, r5
+; CHECK-NEXT:    adcs r2, r6
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    add sp, #80
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %xx = zext <16 x i8> %x to <16 x i64>
+  %yy = zext <16 x i8> %y to <16 x i64>
+  %m = mul <16 x i64> %xx, %yy
+  %s = select <16 x i1> %c, <16 x i64> %m, <16 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v16i64(<16 x i64> %s)
+  %r = add i64 %z, %a
+  ret i64 %r
+}
+
+define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_acc_sext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b, i64 %a) {
+; CHECK-LABEL: add_v16i8_v16i64_acc_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vcmp.i8 eq, q2, zr
+; CHECK-NEXT:    vmov.i8 q2, #0x0
+; CHECK-NEXT:    vmov.i8 q3, #0xff
+; CHECK-NEXT:    vpsel q4, q3, q2
+; CHECK-NEXT:    vmov.u8 r2, q4[0]
+; CHECK-NEXT:    vmov.16 q5[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[1]
+; CHECK-NEXT:    vmov.16 q5[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[2]
+; CHECK-NEXT:    vmov.16 q5[2], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[3]
+; CHECK-NEXT:    vmov.16 q5[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[4]
+; CHECK-NEXT:    vmov.16 q5[4], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[5]
+; CHECK-NEXT:    vmov.16 q5[5], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[6]
+; CHECK-NEXT:    vmov.16 q5[6], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[7]
+; CHECK-NEXT:    vmov.16 q5[7], r2
+; CHECK-NEXT:    vcmp.i16 ne, q5, zr
+; CHECK-NEXT:    vpsel q5, q3, q2
+; CHECK-NEXT:    vmov.u16 r2, q5[0]
+; CHECK-NEXT:    vmov.32 q6[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q5[1]
+; CHECK-NEXT:    vmov.32 q6[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q5[2]
+; CHECK-NEXT:    vmov.32 q6[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q5[3]
+; CHECK-NEXT:    vmov.32 q6[3], r2
+; CHECK-NEXT:    vcmp.i32 ne, q6, zr
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r3, r2, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q6[0], r3
+; CHECK-NEXT:    vmov.32 q6[1], r3
+; CHECK-NEXT:    ubfx r3, r2, #4, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    vmov.32 q6[2], r3
+; CHECK-NEXT:    vmov.32 q6[3], r3
+; CHECK-NEXT:    vmov.u8 r3, q1[0]
+; CHECK-NEXT:    sxtb.w r12, r3
+; CHECK-NEXT:    vmov.u8 r3, q0[0]
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    smull r3, r12, r3, r12
+; CHECK-NEXT:    vmov.32 q7[0], r3
+; CHECK-NEXT:    vmov.u8 r3, q1[1]
+; CHECK-NEXT:    vmov.32 q7[1], r12
+; CHECK-NEXT:    sxtb.w r12, r3
+; CHECK-NEXT:    vmov.u8 r3, q0[1]
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    smull r3, r12, r3, r12
+; CHECK-NEXT:    vmov.32 q7[2], r3
+; CHECK-NEXT:    vmov.32 q7[3], r12
+; CHECK-NEXT:    vand q6, q7, q6
+; CHECK-NEXT:    vmov r3, s26
+; CHECK-NEXT:    vmov r4, s24
+; CHECK-NEXT:    vmov r12, s27
+; CHECK-NEXT:    vmov r5, s25
+; CHECK-NEXT:    adds.w lr, r4, r3
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    rsb.w r3, r3, #0
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    vmov.32 q6[0], r3
+; CHECK-NEXT:    rsb.w r2, r2, #0
+; CHECK-NEXT:    vmov.32 q6[1], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[2]
+; CHECK-NEXT:    vmov.32 q6[2], r2
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    vmov.32 q6[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q1[2]
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    adc.w r12, r12, r5
+; CHECK-NEXT:    smull r2, r3, r3, r2
+; CHECK-NEXT:    vmov.32 q7[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q1[3]
+; CHECK-NEXT:    vmov.32 q7[1], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[3]
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    smull r2, r3, r3, r2
+; CHECK-NEXT:    vmov.32 q7[2], r2
+; CHECK-NEXT:    vmov.32 q7[3], r3
+; CHECK-NEXT:    vand q6, q7, q6
+; CHECK-NEXT:    vmov r3, s24
+; CHECK-NEXT:    vmov r2, s25
+; CHECK-NEXT:    vmov r4, s26
+; CHECK-NEXT:    vmov r5, s27
+; CHECK-NEXT:    adds.w r3, r3, lr
+; CHECK-NEXT:    adc.w r2, r2, r12
+; CHECK-NEXT:    adds.w r12, r3, r4
+; CHECK-NEXT:    vmov.u8 r4, q0[4]
+; CHECK-NEXT:    adc.w r3, r2, r5
+; CHECK-NEXT:    vmov.u16 r2, q5[4]
+; CHECK-NEXT:    vmov.32 q6[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q5[5]
+; CHECK-NEXT:    vmov.32 q6[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q5[6]
+; CHECK-NEXT:    vmov.32 q6[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q5[7]
+; CHECK-NEXT:    vmov.32 q6[3], r2
+; CHECK-NEXT:    sxtb r4, r4
+; CHECK-NEXT:    vcmp.i32 ne, q6, zr
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r5, r2, #1
+; CHECK-NEXT:    rsbs r5, r5, #0
+; CHECK-NEXT:    vmov.32 q5[0], r5
+; CHECK-NEXT:    vmov.32 q5[1], r5
+; CHECK-NEXT:    ubfx r5, r2, #4, #1
+; CHECK-NEXT:    rsbs r5, r5, #0
+; CHECK-NEXT:    vmov.32 q5[2], r5
+; CHECK-NEXT:    vmov.32 q5[3], r5
+; CHECK-NEXT:    vmov.u8 r5, q1[4]
+; CHECK-NEXT:    sxtb r5, r5
+; CHECK-NEXT:    smull r5, r4, r4, r5
+; CHECK-NEXT:    vmov.32 q6[0], r5
+; CHECK-NEXT:    vmov.u8 r5, q1[5]
+; CHECK-NEXT:    vmov.32 q6[1], r4
+; CHECK-NEXT:    vmov.u8 r4, q0[5]
+; CHECK-NEXT:    sxtb r5, r5
+; CHECK-NEXT:    sxtb r4, r4
+; CHECK-NEXT:    smull r5, r4, r4, r5
+; CHECK-NEXT:    vmov.32 q6[2], r5
+; CHECK-NEXT:    vmov.32 q6[3], r4
+; CHECK-NEXT:    vand q5, q6, q5
+; CHECK-NEXT:    vmov r4, s20
+; CHECK-NEXT:    vmov r5, s21
+; CHECK-NEXT:    adds.w r12, r12, r4
+; CHECK-NEXT:    vmov r4, s22
+; CHECK-NEXT:    adcs r5, r3
+; CHECK-NEXT:    vmov r3, s23
+; CHECK-NEXT:    adds.w r4, r4, r12
+; CHECK-NEXT:    adc.w r12, r5, r3
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    vmov.32 q5[0], r3
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov.32 q5[1], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[6]
+; CHECK-NEXT:    vmov.32 q5[2], r2
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    vmov.32 q5[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q1[6]
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    smull r2, r3, r3, r2
+; CHECK-NEXT:    vmov.32 q6[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q1[7]
+; CHECK-NEXT:    vmov.32 q6[1], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[7]
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    smull r2, r3, r3, r2
+; CHECK-NEXT:    vmov.32 q6[2], r2
+; CHECK-NEXT:    vmov.32 q6[3], r3
+; CHECK-NEXT:    vand q5, q6, q5
+; CHECK-NEXT:    vmov r3, s20
+; CHECK-NEXT:    vmov r2, s21
+; CHECK-NEXT:    vmov r5, s23
+; CHECK-NEXT:    adds r3, r3, r4
+; CHECK-NEXT:    vmov r4, s22
+; CHECK-NEXT:    adc.w r2, r2, r12
+; CHECK-NEXT:    adds.w r12, r3, r4
+; CHECK-NEXT:    vmov.u8 r4, q0[8]
+; CHECK-NEXT:    adc.w r3, r2, r5
+; CHECK-NEXT:    vmov.u8 r2, q4[8]
+; CHECK-NEXT:    vmov.16 q5[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[9]
+; CHECK-NEXT:    vmov.16 q5[1], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[10]
+; CHECK-NEXT:    vmov.16 q5[2], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[11]
+; CHECK-NEXT:    vmov.16 q5[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[12]
+; CHECK-NEXT:    vmov.16 q5[4], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[13]
+; CHECK-NEXT:    vmov.16 q5[5], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[14]
+; CHECK-NEXT:    vmov.16 q5[6], r2
+; CHECK-NEXT:    vmov.u8 r2, q4[15]
+; CHECK-NEXT:    vmov.16 q5[7], r2
+; CHECK-NEXT:    sxtb r4, r4
+; CHECK-NEXT:    vcmp.i16 ne, q5, zr
+; CHECK-NEXT:    vpsel q2, q3, q2
+; CHECK-NEXT:    vmov.u16 r2, q2[0]
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[1]
+; CHECK-NEXT:    vmov.32 q3[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[2]
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[3]
+; CHECK-NEXT:    vmov.32 q3[3], r2
+; CHECK-NEXT:    vcmp.i32 ne, q3, zr
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r5, r2, #1
+; CHECK-NEXT:    rsbs r5, r5, #0
+; CHECK-NEXT:    vmov.32 q3[0], r5
+; CHECK-NEXT:    vmov.32 q3[1], r5
+; CHECK-NEXT:    ubfx r5, r2, #4, #1
+; CHECK-NEXT:    rsbs r5, r5, #0
+; CHECK-NEXT:    vmov.32 q3[2], r5
+; CHECK-NEXT:    vmov.32 q3[3], r5
+; CHECK-NEXT:    vmov.u8 r5, q1[8]
+; CHECK-NEXT:    sxtb r5, r5
+; CHECK-NEXT:    smull r5, r4, r4, r5
+; CHECK-NEXT:    vmov.32 q4[0], r5
+; CHECK-NEXT:    vmov.u8 r5, q1[9]
+; CHECK-NEXT:    vmov.32 q4[1], r4
+; CHECK-NEXT:    vmov.u8 r4, q0[9]
+; CHECK-NEXT:    sxtb r5, r5
+; CHECK-NEXT:    sxtb r4, r4
+; CHECK-NEXT:    smull r5, r4, r4, r5
+; CHECK-NEXT:    vmov.32 q4[2], r5
+; CHECK-NEXT:    vmov.32 q4[3], r4
+; CHECK-NEXT:    vand q3, q4, q3
+; CHECK-NEXT:    vmov r4, s12
+; CHECK-NEXT:    vmov r5, s13
+; CHECK-NEXT:    adds.w r12, r12, r4
+; CHECK-NEXT:    vmov r4, s14
+; CHECK-NEXT:    adcs r5, r3
+; CHECK-NEXT:    vmov r3, s15
+; CHECK-NEXT:    adds.w r4, r4, r12
+; CHECK-NEXT:    adc.w r12, r5, r3
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    vmov.32 q3[0], r3
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov.32 q3[1], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[10]
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    vmov.32 q3[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q1[10]
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    smull r2, r3, r3, r2
+; CHECK-NEXT:    vmov.32 q4[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q1[11]
+; CHECK-NEXT:    vmov.32 q4[1], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[11]
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    smull r2, r3, r3, r2
+; CHECK-NEXT:    vmov.32 q4[2], r2
+; CHECK-NEXT:    vmov.32 q4[3], r3
+; CHECK-NEXT:    vand q3, q4, q3
+; CHECK-NEXT:    vmov r3, s12
+; CHECK-NEXT:    vmov r2, s13
+; CHECK-NEXT:    vmov r5, s15
+; CHECK-NEXT:    adds r3, r3, r4
+; CHECK-NEXT:    vmov r4, s14
+; CHECK-NEXT:    adc.w r2, r2, r12
+; CHECK-NEXT:    adds.w r12, r3, r4
+; CHECK-NEXT:    vmov.u8 r4, q0[12]
+; CHECK-NEXT:    adc.w r3, r2, r5
+; CHECK-NEXT:    vmov.u16 r2, q2[4]
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[5]
+; CHECK-NEXT:    vmov.32 q3[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[6]
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q2[7]
+; CHECK-NEXT:    vmov.32 q3[3], r2
+; CHECK-NEXT:    sxtb r4, r4
+; CHECK-NEXT:    vcmp.i32 ne, q3, zr
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r5, r2, #1
+; CHECK-NEXT:    rsbs r5, r5, #0
+; CHECK-NEXT:    vmov.32 q2[0], r5
+; CHECK-NEXT:    vmov.32 q2[1], r5
+; CHECK-NEXT:    ubfx r5, r2, #4, #1
+; CHECK-NEXT:    rsbs r5, r5, #0
+; CHECK-NEXT:    vmov.32 q2[2], r5
+; CHECK-NEXT:    vmov.32 q2[3], r5
+; CHECK-NEXT:    vmov.u8 r5, q1[12]
+; CHECK-NEXT:    sxtb r5, r5
+; CHECK-NEXT:    smull r5, r4, r4, r5
+; CHECK-NEXT:    vmov.32 q3[0], r5
+; CHECK-NEXT:    vmov.u8 r5, q1[13]
+; CHECK-NEXT:    vmov.32 q3[1], r4
+; CHECK-NEXT:    vmov.u8 r4, q0[13]
+; CHECK-NEXT:    sxtb r5, r5
+; CHECK-NEXT:    sxtb r4, r4
+; CHECK-NEXT:    smull r5, r4, r4, r5
+; CHECK-NEXT:    vmov.32 q3[2], r5
+; CHECK-NEXT:    vmov.32 q3[3], r4
+; CHECK-NEXT:    vand q2, q3, q2
+; CHECK-NEXT:    vmov r4, s8
+; CHECK-NEXT:    vmov r5, s9
+; CHECK-NEXT:    adds.w r12, r12, r4
+; CHECK-NEXT:    vmov r4, s10
+; CHECK-NEXT:    adcs r5, r3
+; CHECK-NEXT:    vmov r3, s11
+; CHECK-NEXT:    adds.w r4, r4, r12
+; CHECK-NEXT:    adc.w r12, r5, r3
+; CHECK-NEXT:    ubfx r3, r2, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    vmov.32 q2[0], r3
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    vmov.32 q2[1], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[14]
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vmov.u8 r2, q1[14]
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    smull r2, r3, r3, r2
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    vmov.u8 r2, q1[15]
+; CHECK-NEXT:    vmov.32 q3[1], r3
+; CHECK-NEXT:    vmov.u8 r3, q0[15]
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    smull r2, r3, r3, r2
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    vmov.32 q3[3], r3
+; CHECK-NEXT:    vand q0, q3, q2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r2, s1
+; CHECK-NEXT:    vmov r5, s3
+; CHECK-NEXT:    adds r3, r3, r4
+; CHECK-NEXT:    vmov r4, s2
+; CHECK-NEXT:    adc.w r2, r2, r12
+; CHECK-NEXT:    adds r3, r3, r4
+; CHECK-NEXT:    adcs r2, r5
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    adcs r1, r2
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+entry:
+  %c = icmp eq <16 x i8> %b, zeroinitializer
+  %xx = sext <16 x i8> %x to <16 x i64>
+  %yy = sext <16 x i8> %y to <16 x i64>
+  %m = mul <16 x i64> %xx, %yy
+  %s = select <16 x i1> %c, <16 x i64> %m, <16 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v16i64(<16 x i64> %s)
+  %r = add i64 %z, %a
+  ret i64 %r
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_acc_zext(<2 x i8> %x, <2 x i8> %y, <2 x i8> %b, i64 %a) {
+; CHECK-LABEL: add_v2i8_v2i64_acc_zext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vmov.i64 q3, #0xff
+; CHECK-NEXT:    vand q1, q1, q3
+; CHECK-NEXT:    vand q4, q0, q3
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    vmov r3, s16
+; CHECK-NEXT:    umull r2, r3, r3, r2
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    vmov.32 q0[1], r3
+; CHECK-NEXT:    vmov r3, s18
+; CHECK-NEXT:    vand q1, q2, q3
+; CHECK-NEXT:    umull r2, r3, r3, r2
+; CHECK-NEXT:    vmov.32 q0[2], r2
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    vmov.32 q0[3], r3
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vand q0, q0, q2
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r12, s3
+; CHECK-NEXT:    vmov lr, s1
+; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    adc.w r3, lr, r12
+; CHECK-NEXT:    adds r0, r0, r2
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %c = icmp eq <2 x i8> %b, zeroinitializer
+  %xx = zext <2 x i8> %x to <2 x i64>
+  %yy = zext <2 x i8> %y to <2 x i64>
+  %m = mul <2 x i64> %xx, %yy
+  %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  %r = add i64 %z, %a
+  ret i64 %r
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_acc_sext(<2 x i8> %x, <2 x i8> %y, <2 x i8> %b, i64 %a) {
+; CHECK-LABEL: add_v2i8_v2i64_acc_sext:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vmov.i32 q3, #0xff
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vand q3, q2, q3
+; CHECK-NEXT:    vmov r2, s12
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q2[0], r2
+; CHECK-NEXT:    vmov.32 q2[1], r2
+; CHECK-NEXT:    vmov r2, s14
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q2[2], r2
+; CHECK-NEXT:    vmov.32 q2[3], r2
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    smull r2, r3, r3, r2
+; CHECK-NEXT:    vmov.32 q3[0], r2
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    vmov.32 q3[1], r3
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    sxtb r2, r2
+; CHECK-NEXT:    sxtb r3, r3
+; CHECK-NEXT:    smull r2, r3, r3, r2
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    vmov.32 q3[3], r3
+; CHECK-NEXT:    vand q0, q3, q2
+; CHECK-NEXT:    vmov r2, s2
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r12, s3
+; CHECK-NEXT:    vmov lr, s1
+; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    adc.w r3, lr, r12
+; CHECK-NEXT:    adds r0, r0, r2
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %c = icmp eq <2 x i8> %b, zeroinitializer
+  %xx = sext <2 x i8> %x to <2 x i64>
+  %yy = sext <2 x i8> %y to <2 x i64>
+  %m = mul <2 x i64> %xx, %yy
+  %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  %r = add i64 %z, %a
+  ret i64 %r
+}
+
+define arm_aapcs_vfpcc i64 @add_v2i64_v2i64_acc(<2 x i64> %x, <2 x i64> %y, <2 x i64> %b, i64 %a) {
+; CHECK-LABEL: add_v2i64_v2i64_acc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    vmov r2, s4
+; CHECK-NEXT:    vmov r3, s0
+; CHECK-NEXT:    vmov r4, s5
+; CHECK-NEXT:    umull r12, lr, r3, r2
+; CHECK-NEXT:    mla r3, r3, r4, lr
+; CHECK-NEXT:    vmov r4, s1
+; CHECK-NEXT:    vmov.32 q3[0], r12
+; CHECK-NEXT:    mla r2, r4, r2, r3
+; CHECK-NEXT:    vmov r3, s2
+; CHECK-NEXT:    vmov.32 q3[1], r2
+; CHECK-NEXT:    vmov r2, s6
+; CHECK-NEXT:    umull r4, r12, r3, r2
+; CHECK-NEXT:    vmov.32 q3[2], r4
+; CHECK-NEXT:    vmov r4, s7
+; CHECK-NEXT:    mla r3, r3, r4, r12
+; CHECK-NEXT:    vmov r4, s3
+; CHECK-NEXT:    mla r2, r4, r2, r3
+; CHECK-NEXT:    vmov r3, s8
+; CHECK-NEXT:    vmov.32 q3[3], r2
+; CHECK-NEXT:    vmov r2, s9
+; CHECK-NEXT:    orrs r2, r3
+; CHECK-NEXT:    vmov r3, s10
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov r2, s11
+; CHECK-NEXT:    orrs r2, r3
+; CHECK-NEXT:    cset r2, eq
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csetm r2, ne
+; CHECK-NEXT:    vmov.32 q0[2], r2
+; CHECK-NEXT:    vmov.32 q0[3], r2
+; CHECK-NEXT:    vand q0, q3, q0
+; CHECK-NEXT:    vmov r4, s2
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    vmov r12, s3
+; CHECK-NEXT:    vmov r3, s1
+; CHECK-NEXT:    adds r2, r2, r4
+; CHECK-NEXT:    adc.w r3, r3, r12
+; CHECK-NEXT:    adds r0, r0, r2
+; CHECK-NEXT:    adcs r1, r3
+; CHECK-NEXT:    pop {r4, pc}
+entry:
+  %c = icmp eq <2 x i64> %b, zeroinitializer
+  %m = mul <2 x i64> %x, %y
+  %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer
+  %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %s)
+  %r = add i64 %z, %a
+  ret i64 %r
+}
+
+declare i16 @llvm.experimental.vector.reduce.add.v16i16(<16 x i16>)
+declare i16 @llvm.experimental.vector.reduce.add.v8i16(<8 x i16>)
+declare i32 @llvm.experimental.vector.reduce.add.v16i32(<16 x i32>)
+declare i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32>)
+declare i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32>)
+declare i64 @llvm.experimental.vector.reduce.add.v16i64(<16 x i64>)
+declare i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64>)
+declare i64 @llvm.experimental.vector.reduce.add.v4i64(<4 x i64>)
+declare i64 @llvm.experimental.vector.reduce.add.v8i64(<8 x i64>)
+declare i8 @llvm.experimental.vector.reduce.add.v16i8(<16 x i8>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-from-intrinsics.ll b/llvm/test/CodeGen/Thumb2/mve-vpt-from-intrinsics.ll
index 07e5185b72769..1303c3f3d30b3 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-from-intrinsics.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-from-intrinsics.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: opt -instcombine %s | llc -mtriple=thumbv8.1m.main-none-eabi -mattr=+mve --verify-machineinstrs -o - | FileCheck %s
+; RUN: opt -instcombine -mtriple=thumbv8.1m.main-none-eabi %s | llc -mtriple=thumbv8.1m.main-none-eabi -mattr=+mve --verify-machineinstrs -o - | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
 
@@ -42,4 +42,3 @@ declare i32 @llvm.arm.mve.pred.v2i.v8i1(<8 x i1>)
 declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32)
 declare <8 x i16> @llvm.arm.mve.add.predicated.v8i16.v8i1(<8 x i16>, <8 x i16>, <8 x i1>, <8 x i16>)
 declare <8 x i1> @llvm.arm.mve.vctp16(i32)
-
diff --git a/llvm/test/CodeGen/Thumb2/tpsoft.ll b/llvm/test/CodeGen/Thumb2/tpsoft.ll
index 7e268c718ea87..443cbdb959758 100644
--- a/llvm/test/CodeGen/Thumb2/tpsoft.ll
+++ b/llvm/test/CodeGen/Thumb2/tpsoft.ll
@@ -3,9 +3,9 @@
 ; RUN: llc  %s -mtriple=thumbebv7-linux-gnueabi -o - | \
 ; RUN:    FileCheck  -check-prefix=ELFASM %s
 ; RUN: llc  %s -mtriple=thumbv7-linux-gnueabi -filetype=obj -o - | \
-; RUN:    llvm-readobj -S --sd | FileCheck  -check-prefix=ELFOBJ -check-prefix=ELFOBJ-LE %s
+; RUN:    llvm-readobj -S --sd - | FileCheck  -check-prefix=ELFOBJ -check-prefix=ELFOBJ-LE %s
 ; RUN: llc  %s -mtriple=thumbebv7-linux-gnueabi -filetype=obj -o - | \
-; RUN:    llvm-readobj -S --sd | FileCheck  -check-prefix=ELFOBJ -check-prefix=ELFOBJ-BE %s
+; RUN:    llvm-readobj -S --sd - | FileCheck  -check-prefix=ELFOBJ -check-prefix=ELFOBJ-BE %s
 
 ;; Make sure that bl __aeabi_read_tp is materialized and fixed up correctly
 ;; in the obj case.
diff --git a/llvm/test/CodeGen/WebAssembly/simd-offset.ll b/llvm/test/CodeGen/WebAssembly/simd-offset.ll
index 7ece5b782ab5c..8d39ddc20f244 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-offset.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-offset.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals -mattr=+simd128 | FileCheck %s --check-prefixes CHECK,SIMD128
-; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals | FileCheck %s --check-prefixes CHECK,NO-SIMD128
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -verify-machineinstrs -mattr=+simd128 | FileCheck %s
 
 ; Test SIMD loads and stores
 
@@ -9,34 +9,37 @@ target triple = "wasm32-unknown-unknown"
 ; ==============================================================================
 ; 16 x i8
 ; ==============================================================================
-; CHECK-LABEL: load_v16i8:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v16i8 (i32) -> (v128){{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 0($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <16 x i8> @load_v16i8(<16 x i8>* %p) {
+; CHECK-LABEL: load_v16i8:
+; CHECK:         .functype load_v16i8 (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.load 0
+; CHECK-NEXT:    # fallthrough-return
   %v = load <16 x i8>, <16 x i8>* %p
   ret <16 x i8> %v
 }
 
-; CHECK-LABEL: load_splat_v16i8:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v16i8 (i32) -> (v128){{$}}
-; SIMD128-NEXT: v8x16.load_splat $push[[R:[0-9]+]]=, 0($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <16 x i8> @load_splat_v16i8(i8* %p) {
+; CHECK-LABEL: load_splat_v16i8:
+; CHECK:         .functype load_splat_v16i8 (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v8x16.load_splat 0
+; CHECK-NEXT:    # fallthrough-return
   %e = load i8, i8* %p
   %v1 = insertelement <16 x i8> undef, i8 %e, i32 0
   %v2 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> zeroinitializer
   ret <16 x i8> %v2
 }
 
-; CHECK-LABEL: load_v16i8_with_folded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v16i8_with_folded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <16 x i8> @load_v16i8_with_folded_offset(<16 x i8>* %p) {
+; CHECK-LABEL: load_v16i8_with_folded_offset:
+; CHECK:         .functype load_v16i8_with_folded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.load 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <16 x i8>* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to <16 x i8>*
@@ -44,12 +47,13 @@ define <16 x i8> @load_v16i8_with_folded_offset(<16 x i8>* %p) {
   ret <16 x i8> %v
 }
 
-; CHECK-LABEL: load_splat_v16i8_with_folded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v16i8_with_folded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: v8x16.load_splat $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <16 x i8> @load_splat_v16i8_with_folded_offset(i8* %p) {
+; CHECK-LABEL: load_splat_v16i8_with_folded_offset:
+; CHECK:         .functype load_splat_v16i8_with_folded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v8x16.load_splat 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint i8* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to i8*
@@ -59,23 +63,25 @@ define <16 x i8> @load_splat_v16i8_with_folded_offset(i8* %p) {
   ret <16 x i8> %v2
 }
 
-; CHECK-LABEL: load_v16i8_with_folded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v16i8_with_folded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <16 x i8> @load_v16i8_with_folded_gep_offset(<16 x i8>* %p) {
+; CHECK-LABEL: load_v16i8_with_folded_gep_offset:
+; CHECK:         .functype load_v16i8_with_folded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.load 16
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <16 x i8>, <16 x i8>* %p, i32 1
   %v = load <16 x i8>, <16 x i8>* %s
   ret <16 x i8> %v
 }
 
-; CHECK-LABEL: load_splat_v16i8_with_folded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v16i8_with_folded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: v8x16.load_splat $push[[R:[0-9]+]]=, 1($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <16 x i8> @load_splat_v16i8_with_folded_gep_offset(i8* %p) {
+; CHECK-LABEL: load_splat_v16i8_with_folded_gep_offset:
+; CHECK:         .functype load_splat_v16i8_with_folded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v8x16.load_splat 1
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i8, i8* %p, i32 1
   %e = load i8, i8* %s
   %v1 = insertelement <16 x i8> undef, i8 %e, i32 0
@@ -83,27 +89,29 @@ define <16 x i8> @load_splat_v16i8_with_folded_gep_offset(i8* %p) {
   ret <16 x i8> %v2
 }
 
-; CHECK-LABEL: load_v16i8_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v16i8_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <16 x i8> @load_v16i8_with_unfolded_gep_negative_offset(<16 x i8>* %p) {
+; CHECK-LABEL: load_v16i8_with_unfolded_gep_negative_offset:
+; CHECK:         .functype load_v16i8_with_unfolded_gep_negative_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const -16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v128.load 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <16 x i8>, <16 x i8>* %p, i32 -1
   %v = load <16 x i8>, <16 x i8>* %s
   ret <16 x i8> %v
 }
 
-; CHECK-LABEL: load_splat_v16i8_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v16i8_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -1{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v8x16.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <16 x i8> @load_splat_v16i8_with_unfolded_gep_negative_offset(i8* %p) {
+; CHECK-LABEL: load_splat_v16i8_with_unfolded_gep_negative_offset:
+; CHECK:         .functype load_splat_v16i8_with_unfolded_gep_negative_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const -1
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v8x16.load_splat 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i8, i8* %p, i32 -1
   %e = load i8, i8* %s
   %v1 = insertelement <16 x i8> undef, i8 %e, i32 0
@@ -111,14 +119,15 @@ define <16 x i8> @load_splat_v16i8_with_unfolded_gep_negative_offset(i8* %p) {
   ret <16 x i8> %v2
 }
 
-; CHECK-LABEL: load_v16i8_with_unfolded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v16i8_with_unfolded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <16 x i8> @load_v16i8_with_unfolded_offset(<16 x i8>* %p) {
+; CHECK-LABEL: load_v16i8_with_unfolded_offset:
+; CHECK:         .functype load_v16i8_with_unfolded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v128.load 0
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <16 x i8>* %p to i32
   %r = add nsw i32 %q, 16
   %s = inttoptr i32 %r to <16 x i8>*
@@ -126,14 +135,15 @@ define <16 x i8> @load_v16i8_with_unfolded_offset(<16 x i8>* %p) {
   ret <16 x i8> %v
 }
 
-; CHECK-LABEL: load_splat_v16i8_with_unfolded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v16i8_with_unfolded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v8x16.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <16 x i8> @load_splat_v16i8_with_unfolded_offset(i8* %p) {
+; CHECK-LABEL: load_splat_v16i8_with_unfolded_offset:
+; CHECK:         .functype load_splat_v16i8_with_unfolded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v8x16.load_splat 0
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint i8* %p to i32
   %r = add nsw i32 %q, 16
   %s = inttoptr i32 %r to i8*
@@ -143,27 +153,29 @@ define <16 x i8> @load_splat_v16i8_with_unfolded_offset(i8* %p) {
   ret <16 x i8> %v2
 }
 
-; CHECK-LABEL: load_v16i8_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v16i8_with_unfolded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <16 x i8> @load_v16i8_with_unfolded_gep_offset(<16 x i8>* %p) {
+; CHECK-LABEL: load_v16i8_with_unfolded_gep_offset:
+; CHECK:         .functype load_v16i8_with_unfolded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v128.load 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr <16 x i8>, <16 x i8>* %p, i32 1
   %v = load <16 x i8>, <16 x i8>* %s
   ret <16 x i8> %v
 }
 
-; CHECK-LABEL: load_splat_v16i8_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v16i8_with_unfolded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 1{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v8x16.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <16 x i8> @load_splat_v16i8_with_unfolded_gep_offset(i8* %p) {
+; CHECK-LABEL: load_splat_v16i8_with_unfolded_gep_offset:
+; CHECK:         .functype load_splat_v16i8_with_unfolded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 1
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v8x16.load_splat 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr i8, i8* %p, i32 1
   %e = load i8, i8* %s
   %v1 = insertelement <16 x i8> undef, i8 %e, i32 0
@@ -171,25 +183,25 @@ define <16 x i8> @load_splat_v16i8_with_unfolded_gep_offset(i8* %p) {
   ret <16 x i8> %v2
 }
 
-; CHECK-LABEL: load_v16i8_from_numeric_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v16i8_from_numeric_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <16 x i8> @load_v16i8_from_numeric_address() {
+; CHECK-LABEL: load_v16i8_from_numeric_address:
+; CHECK:         .functype load_v16i8_from_numeric_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v128.load 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to <16 x i8>*
   %v = load <16 x i8>, <16 x i8>* %s
   ret <16 x i8> %v
 }
 
-; CHECK-LABEL: load_splat_v16i8_from_numeric_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v16i8_from_numeric_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v8x16.load_splat $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <16 x i8> @load_splat_v16i8_from_numeric_address() {
+; CHECK-LABEL: load_splat_v16i8_from_numeric_address:
+; CHECK:         .functype load_splat_v16i8_from_numeric_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v8x16.load_splat 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to i8*
   %e = load i8, i8* %s
   %v1 = insertelement <16 x i8> undef, i8 %e, i32 0
@@ -197,46 +209,52 @@ define <16 x i8> @load_splat_v16i8_from_numeric_address() {
   ret <16 x i8> %v2
 }
 
-; CHECK-LABEL: load_v16i8_from_global_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v16i8_from_global_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, gv_v16i8($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 @gv_v16i8 = global <16 x i8> <i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42>
 define <16 x i8> @load_v16i8_from_global_address() {
+; CHECK-LABEL: load_v16i8_from_global_address:
+; CHECK:         .functype load_v16i8_from_global_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v128.load gv_v16i8
+; CHECK-NEXT:    # fallthrough-return
   %v = load <16 x i8>, <16 x i8>* @gv_v16i8
   ret <16 x i8> %v
 }
 
-; CHECK-LABEL: load_splat_v16i8_from_global_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v16i8_from_global_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v8x16.load_splat $push[[R:[0-9]+]]=, gv_i8($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 @gv_i8 = global i8 42
 define <16 x i8> @load_splat_v16i8_from_global_address() {
+; CHECK-LABEL: load_splat_v16i8_from_global_address:
+; CHECK:         .functype load_splat_v16i8_from_global_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v8x16.load_splat gv_i8
+; CHECK-NEXT:    # fallthrough-return
   %e = load i8, i8* @gv_i8
   %v1 = insertelement <16 x i8> undef, i8 %e, i32 0
   %v2 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> zeroinitializer
   ret <16 x i8> %v2
 }
 
-; CHECK-LABEL: store_v16i8:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v16i8 (v128, i32) -> (){{$}}
-; SIMD128-NEXT: v128.store 0($1), $0{{$}}
 define void @store_v16i8(<16 x i8> %v, <16 x i8>* %p) {
+; CHECK-LABEL: store_v16i8:
+; CHECK:         .functype store_v16i8 (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 0
+; CHECK-NEXT:    # fallthrough-return
   store <16 x i8> %v , <16 x i8>* %p
   ret void
 }
 
-; CHECK-LABEL: store_v16i8_with_folded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v16i8_with_folded_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: v128.store 16($1), $0{{$}}
 define void @store_v16i8_with_folded_offset(<16 x i8> %v, <16 x i8>* %p) {
+; CHECK-LABEL: store_v16i8_with_folded_offset:
+; CHECK:         .functype store_v16i8_with_folded_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <16 x i8>* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to <16 x i8>*
@@ -244,69 +262,87 @@ define void @store_v16i8_with_folded_offset(<16 x i8> %v, <16 x i8>* %p) {
   ret void
 }
 
-; CHECK-LABEL: store_v16i8_with_folded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v16i8_with_folded_gep_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: v128.store 16($1), $0{{$}}
 define void @store_v16i8_with_folded_gep_offset(<16 x i8> %v, <16 x i8>* %p) {
+; CHECK-LABEL: store_v16i8_with_folded_gep_offset:
+; CHECK:         .functype store_v16i8_with_folded_gep_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 16
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <16 x i8>, <16 x i8>* %p, i32 1
   store <16 x i8> %v , <16 x i8>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v16i8_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v16i8_with_unfolded_gep_negative_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -16{{$}}
-; SIMD128-NEXT: i32.add $push[[R:[0-9]+]]=, $1, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.store 0($pop[[R]]), $0{{$}}
 define void @store_v16i8_with_unfolded_gep_negative_offset(<16 x i8> %v, <16 x i8>* %p) {
+; CHECK-LABEL: store_v16i8_with_unfolded_gep_negative_offset:
+; CHECK:         .functype store_v16i8_with_unfolded_gep_negative_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const -16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <16 x i8>, <16 x i8>* %p, i32 -1
   store <16 x i8> %v , <16 x i8>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v16i8_with_unfolded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v16i8_with_unfolded_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -16{{$}}
-; SIMD128-NEXT: i32.add $push[[R:[0-9]+]]=, $1, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.store 0($pop[[R]]), $0{{$}}
 define void @store_v16i8_with_unfolded_offset(<16 x i8> %v, <16 x i8>* %p) {
-  %s = getelementptr inbounds <16 x i8>, <16 x i8>* %p, i32 -1
+; CHECK-LABEL: store_v16i8_with_unfolded_offset:
+; CHECK:         .functype store_v16i8_with_unfolded_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 0
+; CHECK-NEXT:    # fallthrough-return
+  %q = ptrtoint <16 x i8>* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to <16 x i8>*
   store <16 x i8> %v , <16 x i8>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v16i8_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v16i8_with_unfolded_gep_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[R:[0-9]+]]=, $1, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.store 0($pop[[R]]), $0{{$}}
 define void @store_v16i8_with_unfolded_gep_offset(<16 x i8> %v, <16 x i8>* %p) {
+; CHECK-LABEL: store_v16i8_with_unfolded_gep_offset:
+; CHECK:         .functype store_v16i8_with_unfolded_gep_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr <16 x i8>, <16 x i8>* %p, i32 1
   store <16 x i8> %v , <16 x i8>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v16i8_to_numeric_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v16i8_to_numeric_address (v128) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[R:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v128.store 32($pop[[R]]), $0{{$}}
 define void @store_v16i8_to_numeric_address(<16 x i8> %v) {
+; CHECK-LABEL: store_v16i8_to_numeric_address:
+; CHECK:         .functype store_v16i8_to_numeric_address (v128) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to <16 x i8>*
   store <16 x i8> %v , <16 x i8>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v16i8_to_global_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v16i8_to_global_address (v128) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[R:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v128.store gv_v16i8($pop[[R]]), $0{{$}}
 define void @store_v16i8_to_global_address(<16 x i8> %v) {
+; CHECK-LABEL: store_v16i8_to_global_address:
+; CHECK:         .functype store_v16i8_to_global_address (v128) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store gv_v16i8
+; CHECK-NEXT:    # fallthrough-return
   store <16 x i8> %v , <16 x i8>* @gv_v16i8
   ret void
 }
@@ -314,66 +350,72 @@ define void @store_v16i8_to_global_address(<16 x i8> %v) {
 ; ==============================================================================
 ; 8 x i16
 ; ==============================================================================
-; CHECK-LABEL: load_v8i16:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v8i16 (i32) -> (v128){{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 0($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @load_v8i16(<8 x i16>* %p) {
+; CHECK-LABEL: load_v8i16:
+; CHECK:         .functype load_v8i16 (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.load 0
+; CHECK-NEXT:    # fallthrough-return
   %v = load <8 x i16>, <8 x i16>* %p
   ret <8 x i16> %v
 }
 
-; CHECK-LABEL: load_splat_v8i16:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v8i16 (i32) -> (v128){{$}}
-; SIMD128-NEXT: v16x8.load_splat $push[[R:[0-9]+]]=, 0($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @load_splat_v8i16(i16* %p) {
+; CHECK-LABEL: load_splat_v8i16:
+; CHECK:         .functype load_splat_v8i16 (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v16x8.load_splat 0
+; CHECK-NEXT:    # fallthrough-return
   %e = load i16, i16* %p
   %v1 = insertelement <8 x i16> undef, i16 %e, i32 0
   %v2 = shufflevector <8 x i16> %v1, <8 x i16> undef, <8 x i32> zeroinitializer
   ret <8 x i16> %v2
 }
 
-; CHECK-LABEL: load_sext_v8i16:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_sext_v8i16 (i32) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.load8x8_s $push[[R:[0-9]+]]=, 0($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @load_sext_v8i16(<8 x i8>* %p) {
+; CHECK-LABEL: load_sext_v8i16:
+; CHECK:         .functype load_sext_v8i16 (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i16x8.load8x8_s 0
+; CHECK-NEXT:    # fallthrough-return
   %v = load <8 x i8>, <8 x i8>* %p
   %v2 = sext <8 x i8> %v to <8 x i16>
   ret <8 x i16> %v2
 }
 
-; CHECK-LABEL: load_zext_v8i16:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_zext_v8i16 (i32) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 0($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @load_zext_v8i16(<8 x i8>* %p) {
+; CHECK-LABEL: load_zext_v8i16:
+; CHECK:         .functype load_zext_v8i16 (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i16x8.load8x8_u 0
+; CHECK-NEXT:    # fallthrough-return
   %v = load <8 x i8>, <8 x i8>* %p
   %v2 = zext <8 x i8> %v to <8 x i16>
   ret <8 x i16> %v2
 }
 
-; CHECK-LABEL: load_ext_v8i16:
-; NO-SIMD128-NOT: load8x8
-; SIMD128-NEXT: .functype load_ext_v8i16 (i32) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 0($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i8> @load_ext_v8i16(<8 x i8>* %p) {
+; CHECK-LABEL: load_ext_v8i16:
+; CHECK:         .functype load_ext_v8i16 (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i16x8.load8x8_u 0
+; CHECK-NEXT:    # fallthrough-return
   %v = load <8 x i8>, <8 x i8>* %p
   ret <8 x i8> %v
 }
 
-; CHECK-LABEL: load_v8i16_with_folded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v8i16_with_folded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @load_v8i16_with_folded_offset(<8 x i16>* %p) {
+; CHECK-LABEL: load_v8i16_with_folded_offset:
+; CHECK:         .functype load_v8i16_with_folded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.load 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <8 x i16>* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to <8 x i16>*
@@ -381,12 +423,13 @@ define <8 x i16> @load_v8i16_with_folded_offset(<8 x i16>* %p) {
   ret <8 x i16> %v
 }
 
-; CHECK-LABEL: load_splat_v8i16_with_folded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v8i16_with_folded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: v16x8.load_splat $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @load_splat_v8i16_with_folded_offset(i16* %p) {
+; CHECK-LABEL: load_splat_v8i16_with_folded_offset:
+; CHECK:         .functype load_splat_v8i16_with_folded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v16x8.load_splat 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint i16* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to i16*
@@ -396,12 +439,13 @@ define <8 x i16> @load_splat_v8i16_with_folded_offset(i16* %p) {
   ret <8 x i16> %v2
 }
 
-; CHECK-LABEL: load_sext_v8i16_with_folded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_sext_v8i16_with_folded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.load8x8_s $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @load_sext_v8i16_with_folded_offset(<8 x i8>* %p) {
+; CHECK-LABEL: load_sext_v8i16_with_folded_offset:
+; CHECK:         .functype load_sext_v8i16_with_folded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i16x8.load8x8_s 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <8 x i8>* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to <8 x i8>*
@@ -410,12 +454,13 @@ define <8 x i16> @load_sext_v8i16_with_folded_offset(<8 x i8>* %p) {
   ret <8 x i16> %v2
 }
 
-; CHECK-LABEL: load_zext_v8i16_with_folded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_zext_v8i16_with_folded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @load_zext_v8i16_with_folded_offset(<8 x i8>* %p) {
+; CHECK-LABEL: load_zext_v8i16_with_folded_offset:
+; CHECK:         .functype load_zext_v8i16_with_folded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i16x8.load8x8_u 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <8 x i8>* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to <8 x i8>*
@@ -424,12 +469,13 @@ define <8 x i16> @load_zext_v8i16_with_folded_offset(<8 x i8>* %p) {
   ret <8 x i16> %v2
 }
 
-; CHECK-LABEL: load_ext_v8i16_with_folded_offset:
-; NO-SIMD128-NOT: load8x8
-; SIMD128-NEXT: .functype load_ext_v8i16_with_folded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i8> @load_ext_v8i16_with_folded_offset(<8 x i8>* %p) {
+; CHECK-LABEL: load_ext_v8i16_with_folded_offset:
+; CHECK:         .functype load_ext_v8i16_with_folded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i16x8.load8x8_u 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <8 x i8>* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to <8 x i8>*
@@ -437,23 +483,25 @@ define <8 x i8> @load_ext_v8i16_with_folded_offset(<8 x i8>* %p) {
   ret <8 x i8> %v
 }
 
-; CHECK-LABEL: load_v8i16_with_folded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v8i16_with_folded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @load_v8i16_with_folded_gep_offset(<8 x i16>* %p) {
+; CHECK-LABEL: load_v8i16_with_folded_gep_offset:
+; CHECK:         .functype load_v8i16_with_folded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.load 16
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <8 x i16>, <8 x i16>* %p, i32 1
   %v = load <8 x i16>, <8 x i16>* %s
   ret <8 x i16> %v
 }
 
-; CHECK-LABEL: load_splat_v8i16_with_folded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v8i16_with_folded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: v16x8.load_splat $push[[R:[0-9]+]]=, 2($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @load_splat_v8i16_with_folded_gep_offset(i16* %p) {
+; CHECK-LABEL: load_splat_v8i16_with_folded_gep_offset:
+; CHECK:         .functype load_splat_v8i16_with_folded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v16x8.load_splat 2
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i16, i16* %p, i32 1
   %e = load i16, i16* %s
   %v1 = insertelement <8 x i16> undef, i16 %e, i32 0
@@ -461,62 +509,67 @@ define <8 x i16> @load_splat_v8i16_with_folded_gep_offset(i16* %p) {
   ret <8 x i16> %v2
 }
 
-; CHECK-LABEL: load_sext_v8i16_with_folded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_sext_v8i16_with_folded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.load8x8_s $push[[R:[0-9]+]]=, 8($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @load_sext_v8i16_with_folded_gep_offset(<8 x i8>* %p) {
+; CHECK-LABEL: load_sext_v8i16_with_folded_gep_offset:
+; CHECK:         .functype load_sext_v8i16_with_folded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i16x8.load8x8_s 8
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <8 x i8>, <8 x i8>* %p, i32 1
   %v = load <8 x i8>, <8 x i8>* %s
   %v2 = sext <8 x i8> %v to <8 x i16>
   ret <8 x i16> %v2
 }
 
-; CHECK-LABEL: load_zext_v8i16_with_folded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_zext_v8i16_with_folded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 8($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @load_zext_v8i16_with_folded_gep_offset(<8 x i8>* %p) {
+; CHECK-LABEL: load_zext_v8i16_with_folded_gep_offset:
+; CHECK:         .functype load_zext_v8i16_with_folded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i16x8.load8x8_u 8
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <8 x i8>, <8 x i8>* %p, i32 1
   %v = load <8 x i8>, <8 x i8>* %s
   %v2 = zext <8 x i8> %v to <8 x i16>
   ret <8 x i16> %v2
 }
 
-; CHECK-LABEL: load_ext_v8i16_with_folded_gep_offset:
-; NO-SIMD128-NOT: load8x8
-; SIMD128-NEXT: .functype load_ext_v8i16_with_folded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 8($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i8> @load_ext_v8i16_with_folded_gep_offset(<8 x i8>* %p) {
+; CHECK-LABEL: load_ext_v8i16_with_folded_gep_offset:
+; CHECK:         .functype load_ext_v8i16_with_folded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i16x8.load8x8_u 8
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <8 x i8>, <8 x i8>* %p, i32 1
   %v = load <8 x i8>, <8 x i8>* %s
   ret <8 x i8> %v
 }
 
-; CHECK-LABEL: load_v8i16_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v8i16_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @load_v8i16_with_unfolded_gep_negative_offset(<8 x i16>* %p) {
+; CHECK-LABEL: load_v8i16_with_unfolded_gep_negative_offset:
+; CHECK:         .functype load_v8i16_with_unfolded_gep_negative_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const -16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v128.load 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <8 x i16>, <8 x i16>* %p, i32 -1
   %v = load <8 x i16>, <8 x i16>* %s
   ret <8 x i16> %v
 }
 
-; CHECK-LABEL: load_splat_v8i16_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v8i16_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -2{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v16x8.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @load_splat_v8i16_with_unfolded_gep_negative_offset(i16* %p) {
+; CHECK-LABEL: load_splat_v8i16_with_unfolded_gep_negative_offset:
+; CHECK:         .functype load_splat_v8i16_with_unfolded_gep_negative_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const -2
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v16x8.load_splat 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i16, i16* %p, i32 -1
   %e = load i16, i16* %s
   %v1 = insertelement <8 x i16> undef, i16 %e, i32 0
@@ -524,55 +577,59 @@ define <8 x i16> @load_splat_v8i16_with_unfolded_gep_negative_offset(i16* %p) {
   ret <8 x i16> %v2
 }
 
-; CHECK-LABEL: load_sext_v8i16_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_sext_v8i16_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i16x8.load8x8_s $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @load_sext_v8i16_with_unfolded_gep_negative_offset(<8 x i8>* %p) {
+; CHECK-LABEL: load_sext_v8i16_with_unfolded_gep_negative_offset:
+; CHECK:         .functype load_sext_v8i16_with_unfolded_gep_negative_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const -8
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i16x8.load8x8_s 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <8 x i8>, <8 x i8>* %p, i32 -1
   %v = load <8 x i8>, <8 x i8>* %s
   %v2 = sext <8 x i8> %v to <8 x i16>
   ret <8 x i16> %v2
 }
 
-; CHECK-LABEL: load_zext_v8i16_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_zext_v8i16_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @load_zext_v8i16_with_unfolded_gep_negative_offset(<8 x i8>* %p) {
+; CHECK-LABEL: load_zext_v8i16_with_unfolded_gep_negative_offset:
+; CHECK:         .functype load_zext_v8i16_with_unfolded_gep_negative_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const -8
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i16x8.load8x8_u 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <8 x i8>, <8 x i8>* %p, i32 -1
   %v = load <8 x i8>, <8 x i8>* %s
   %v2 = zext <8 x i8> %v to <8 x i16>
   ret <8 x i16> %v2
 }
 
-; CHECK-LABEL: load_ext_v8i16_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: load8x8
-; SIMD128-NEXT: .functype load_ext_v8i16_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i8> @load_ext_v8i16_with_unfolded_gep_negative_offset(<8 x i8>* %p) {
+; CHECK-LABEL: load_ext_v8i16_with_unfolded_gep_negative_offset:
+; CHECK:         .functype load_ext_v8i16_with_unfolded_gep_negative_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const -8
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i16x8.load8x8_u 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <8 x i8>, <8 x i8>* %p, i32 -1
   %v = load <8 x i8>, <8 x i8>* %s
   ret <8 x i8> %v
 }
 
-; CHECK-LABEL: load_v8i16_with_unfolded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v8i16_with_unfolded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.load $push[[L0:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[L0]]{{$}}
 define <8 x i16> @load_v8i16_with_unfolded_offset(<8 x i16>* %p) {
+; CHECK-LABEL: load_v8i16_with_unfolded_offset:
+; CHECK:         .functype load_v8i16_with_unfolded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v128.load 0
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <8 x i16>* %p to i32
   %r = add nsw i32 %q, 16
   %s = inttoptr i32 %r to <8 x i16>*
@@ -580,14 +637,15 @@ define <8 x i16> @load_v8i16_with_unfolded_offset(<8 x i16>* %p) {
   ret <8 x i16> %v
 }
 
-; CHECK-LABEL: load_splat_v8i16_with_unfolded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v8i16_with_unfolded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v16x8.load_splat $push[[L0:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[L0]]{{$}}
 define <8 x i16> @load_splat_v8i16_with_unfolded_offset(i16* %p) {
+; CHECK-LABEL: load_splat_v8i16_with_unfolded_offset:
+; CHECK:         .functype load_splat_v8i16_with_unfolded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v16x8.load_splat 0
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint i16* %p to i32
   %r = add nsw i32 %q, 16
   %s = inttoptr i32 %r to i16*
@@ -597,14 +655,15 @@ define <8 x i16> @load_splat_v8i16_with_unfolded_offset(i16* %p) {
   ret <8 x i16> %v2
 }
 
-; CHECK-LABEL: load_sext_v8i16_with_unfolded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_sext_v8i16_with_unfolded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i16x8.load8x8_s $push[[L0:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[L0]]{{$}}
 define <8 x i16> @load_sext_v8i16_with_unfolded_offset(<8 x i8>* %p) {
+; CHECK-LABEL: load_sext_v8i16_with_unfolded_offset:
+; CHECK:         .functype load_sext_v8i16_with_unfolded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i16x8.load8x8_s 0
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <8 x i8>* %p to i32
   %r = add nsw i32 %q, 16
   %s = inttoptr i32 %r to <8 x i8>*
@@ -613,14 +672,15 @@ define <8 x i16> @load_sext_v8i16_with_unfolded_offset(<8 x i8>* %p) {
   ret <8 x i16> %v2
 }
 
-; CHECK-LABEL: load_zext_v8i16_with_unfolded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_zext_v8i16_with_unfolded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i16x8.load8x8_u $push[[L0:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[L0]]{{$}}
 define <8 x i16> @load_zext_v8i16_with_unfolded_offset(<8 x i8>* %p) {
+; CHECK-LABEL: load_zext_v8i16_with_unfolded_offset:
+; CHECK:         .functype load_zext_v8i16_with_unfolded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i16x8.load8x8_u 0
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <8 x i8>* %p to i32
   %r = add nsw i32 %q, 16
   %s = inttoptr i32 %r to <8 x i8>*
@@ -629,14 +689,15 @@ define <8 x i16> @load_zext_v8i16_with_unfolded_offset(<8 x i8>* %p) {
   ret <8 x i16> %v2
 }
 
-; CHECK-LABEL: load_ext_v8i16_with_unfolded_offset:
-; NO-SIMD128-NOT: load8x8
-; SIMD128-NEXT: .functype load_ext_v8i16_with_unfolded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i16x8.load8x8_u $push[[L0:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[L0]]{{$}}
 define <8 x i8> @load_ext_v8i16_with_unfolded_offset(<8 x i8>* %p) {
+; CHECK-LABEL: load_ext_v8i16_with_unfolded_offset:
+; CHECK:         .functype load_ext_v8i16_with_unfolded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i16x8.load8x8_u 0
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <8 x i8>* %p to i32
   %r = add nsw i32 %q, 16
   %s = inttoptr i32 %r to <8 x i8>*
@@ -644,27 +705,29 @@ define <8 x i8> @load_ext_v8i16_with_unfolded_offset(<8 x i8>* %p) {
   ret <8 x i8> %v
 }
 
-; CHECK-LABEL: load_v8i16_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v8i16_with_unfolded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @load_v8i16_with_unfolded_gep_offset(<8 x i16>* %p) {
+; CHECK-LABEL: load_v8i16_with_unfolded_gep_offset:
+; CHECK:         .functype load_v8i16_with_unfolded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v128.load 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr <8 x i16>, <8 x i16>* %p, i32 1
   %v = load <8 x i16>, <8 x i16>* %s
   ret <8 x i16> %v
 }
 
-; CHECK-LABEL: load_splat_v8i16_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v8i16_with_unfolded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 2{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v16x8.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @load_splat_v8i16_with_unfolded_gep_offset(i16* %p) {
+; CHECK-LABEL: load_splat_v8i16_with_unfolded_gep_offset:
+; CHECK:         .functype load_splat_v8i16_with_unfolded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 2
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v16x8.load_splat 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr i16, i16* %p, i32 1
   %e = load i16, i16* %s
   %v1 = insertelement <8 x i16> undef, i16 %e, i32 0
@@ -672,66 +735,69 @@ define <8 x i16> @load_splat_v8i16_with_unfolded_gep_offset(i16* %p) {
   ret <8 x i16> %v2
 }
 
-; CHECK-LABEL: load_sext_v8i16_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_sext_v8i16_with_unfolded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i16x8.load8x8_s $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @load_sext_v8i16_with_unfolded_gep_offset(<8 x i8>* %p) {
+; CHECK-LABEL: load_sext_v8i16_with_unfolded_gep_offset:
+; CHECK:         .functype load_sext_v8i16_with_unfolded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 8
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i16x8.load8x8_s 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr <8 x i8>, <8 x i8>* %p, i32 1
   %v = load <8 x i8>, <8 x i8>* %s
   %v2 = sext <8 x i8> %v to <8 x i16>
   ret <8 x i16> %v2
 }
 
-; CHECK-LABEL: load_zext_v8i16_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_zext_v8i16_with_unfolded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @load_zext_v8i16_with_unfolded_gep_offset(<8 x i8>* %p) {
+; CHECK-LABEL: load_zext_v8i16_with_unfolded_gep_offset:
+; CHECK:         .functype load_zext_v8i16_with_unfolded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 8
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i16x8.load8x8_u 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr <8 x i8>, <8 x i8>* %p, i32 1
   %v = load <8 x i8>, <8 x i8>* %s
   %v2 = zext <8 x i8> %v to <8 x i16>
   ret <8 x i16> %v2
 }
 
-; CHECK-LABEL: load_ext_v8i16_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: load8x8
-; SIMD128-NEXT: .functype load_ext_v8i16_with_unfolded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i8> @load_ext_v8i16_with_unfolded_gep_offset(<8 x i8>* %p) {
+; CHECK-LABEL: load_ext_v8i16_with_unfolded_gep_offset:
+; CHECK:         .functype load_ext_v8i16_with_unfolded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 8
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i16x8.load8x8_u 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr <8 x i8>, <8 x i8>* %p, i32 1
   %v = load <8 x i8>, <8 x i8>* %s
   ret <8 x i8> %v
 }
 
-; CHECK-LABEL: load_v8i16_from_numeric_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v8i16_from_numeric_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @load_v8i16_from_numeric_address() {
+; CHECK-LABEL: load_v8i16_from_numeric_address:
+; CHECK:         .functype load_v8i16_from_numeric_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v128.load 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to <8 x i16>*
   %v = load <8 x i16>, <8 x i16>* %s
   ret <8 x i16> %v
 }
 
-; CHECK-LABEL: load_splat_v8i16_from_numeric_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v8i16_from_numeric_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v16x8.load_splat $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @load_splat_v8i16_from_numeric_address() {
+; CHECK-LABEL: load_splat_v8i16_from_numeric_address:
+; CHECK:         .functype load_splat_v8i16_from_numeric_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v16x8.load_splat 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to i16*
   %e = load i16, i16* %s
   %v1 = insertelement <8 x i16> undef, i16 %e, i32 0
@@ -739,121 +805,127 @@ define <8 x i16> @load_splat_v8i16_from_numeric_address() {
   ret <8 x i16> %v2
 }
 
-; CHECK-LABEL: load_sext_v8i16_from_numeric_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_sext_v8i16_from_numeric_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: i16x8.load8x8_s $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @load_sext_v8i16_from_numeric_address() {
+; CHECK-LABEL: load_sext_v8i16_from_numeric_address:
+; CHECK:         .functype load_sext_v8i16_from_numeric_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    i16x8.load8x8_s 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to <8 x i8>*
   %v = load <8 x i8>, <8 x i8>* %s
   %v2 = sext <8 x i8> %v to <8 x i16>
   ret <8 x i16> %v2
 }
 
-; CHECK-LABEL: load_zext_v8i16_from_numeric_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_zext_v8i16_from_numeric_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @load_zext_v8i16_from_numeric_address() {
+; CHECK-LABEL: load_zext_v8i16_from_numeric_address:
+; CHECK:         .functype load_zext_v8i16_from_numeric_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    i16x8.load8x8_u 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to <8 x i8>*
   %v = load <8 x i8>, <8 x i8>* %s
   %v2 = zext <8 x i8> %v to <8 x i16>
   ret <8 x i16> %v2
 }
 
-; CHECK-LABEL: load_ext_v8i16_from_numeric_address:
-; NO-SIMD128-NOT: load8x8
-; SIMD128-NEXT: .functype load_ext_v8i16_from_numeric_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i8> @load_ext_v8i16_from_numeric_address() {
+; CHECK-LABEL: load_ext_v8i16_from_numeric_address:
+; CHECK:         .functype load_ext_v8i16_from_numeric_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    i16x8.load8x8_u 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to <8 x i8>*
   %v = load <8 x i8>, <8 x i8>* %s
   ret <8 x i8> %v
 }
 
-; CHECK-LABEL: load_v8i16_from_global_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v8i16_from_global_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, gv_v8i16($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 @gv_v8i16 = global <8 x i16> <i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42>
 define <8 x i16> @load_v8i16_from_global_address() {
+; CHECK-LABEL: load_v8i16_from_global_address:
+; CHECK:         .functype load_v8i16_from_global_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v128.load gv_v8i16
+; CHECK-NEXT:    # fallthrough-return
   %v = load <8 x i16>, <8 x i16>* @gv_v8i16
   ret <8 x i16> %v
 }
 
-; CHECK-LABEL: load_splat_v8i16_from_global_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v8i16_from_global_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v16x8.load_splat $push[[R:[0-9]+]]=, gv_i16($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 @gv_i16 = global i16 42
 define <8 x i16> @load_splat_v8i16_from_global_address() {
+; CHECK-LABEL: load_splat_v8i16_from_global_address:
+; CHECK:         .functype load_splat_v8i16_from_global_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v16x8.load_splat gv_i16
+; CHECK-NEXT:    # fallthrough-return
   %e = load i16, i16* @gv_i16
   %v1 = insertelement <8 x i16> undef, i16 %e, i32 0
   %v2 = shufflevector <8 x i16> %v1, <8 x i16> undef, <8 x i32> zeroinitializer
   ret <8 x i16> %v2
 }
 
-; CHECK-LABEL: load_sext_v8i16_from_global_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_sext_v8i16_from_global_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: i16x8.load8x8_s $push[[R:[0-9]+]]=, gv_v8i8($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 @gv_v8i8 = global <8 x i8> <i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42>
 define <8 x i16> @load_sext_v8i16_from_global_address() {
+; CHECK-LABEL: load_sext_v8i16_from_global_address:
+; CHECK:         .functype load_sext_v8i16_from_global_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    i16x8.load8x8_s gv_v8i8
+; CHECK-NEXT:    # fallthrough-return
   %v = load <8 x i8>, <8 x i8>* @gv_v8i8
   %v2 = sext <8 x i8> %v to <8 x i16>
   ret <8 x i16> %v2
 }
 
-; CHECK-LABEL: load_zext_v8i16_from_global_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_zext_v8i16_from_global_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, gv_v8i8($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @load_zext_v8i16_from_global_address() {
+; CHECK-LABEL: load_zext_v8i16_from_global_address:
+; CHECK:         .functype load_zext_v8i16_from_global_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    i16x8.load8x8_u gv_v8i8
+; CHECK-NEXT:    # fallthrough-return
   %v = load <8 x i8>, <8 x i8>* @gv_v8i8
   %v2 = zext <8 x i8> %v to <8 x i16>
   ret <8 x i16> %v2
 }
 
-; CHECK-LABEL: load_ext_v8i16_from_global_address:
-; NO-SIMD128-NOT: load8x8
-; SIMD128-NEXT: .functype load_ext_v8i16_from_global_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, gv_v8i8($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i8> @load_ext_v8i16_from_global_address() {
+; CHECK-LABEL: load_ext_v8i16_from_global_address:
+; CHECK:         .functype load_ext_v8i16_from_global_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    i16x8.load8x8_u gv_v8i8
+; CHECK-NEXT:    # fallthrough-return
   %v = load <8 x i8>, <8 x i8>* @gv_v8i8
   ret <8 x i8> %v
 }
 
 
-; CHECK-LABEL: store_v8i16:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v8i16 (v128, i32) -> (){{$}}
-; SIMD128-NEXT: v128.store 0($1), $0{{$}}
 define void @store_v8i16(<8 x i16> %v, <8 x i16>* %p) {
+; CHECK-LABEL: store_v8i16:
+; CHECK:         .functype store_v8i16 (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 0
+; CHECK-NEXT:    # fallthrough-return
   store <8 x i16> %v , <8 x i16>* %p
   ret void
 }
 
-; CHECK-LABEL: store_v8i16_with_folded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v8i16_with_folded_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: v128.store 16($1), $0{{$}}
 define void @store_v8i16_with_folded_offset(<8 x i16> %v, <8 x i16>* %p) {
+; CHECK-LABEL: store_v8i16_with_folded_offset:
+; CHECK:         .functype store_v8i16_with_folded_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <8 x i16>* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to <8 x i16>*
@@ -861,69 +933,87 @@ define void @store_v8i16_with_folded_offset(<8 x i16> %v, <8 x i16>* %p) {
   ret void
 }
 
-; CHECK-LABEL: store_v8i16_with_folded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v8i16_with_folded_gep_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: v128.store 16($1), $0{{$}}
 define void @store_v8i16_with_folded_gep_offset(<8 x i16> %v, <8 x i16>* %p) {
+; CHECK-LABEL: store_v8i16_with_folded_gep_offset:
+; CHECK:         .functype store_v8i16_with_folded_gep_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 16
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <8 x i16>, <8 x i16>* %p, i32 1
   store <8 x i16> %v , <8 x i16>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v8i16_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v8i16_with_unfolded_gep_negative_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -16{{$}}
-; SIMD128-NEXT: i32.add $push[[R:[0-9]+]]=, $1, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.store 0($pop[[R]]), $0{{$}}
 define void @store_v8i16_with_unfolded_gep_negative_offset(<8 x i16> %v, <8 x i16>* %p) {
+; CHECK-LABEL: store_v8i16_with_unfolded_gep_negative_offset:
+; CHECK:         .functype store_v8i16_with_unfolded_gep_negative_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const -16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <8 x i16>, <8 x i16>* %p, i32 -1
   store <8 x i16> %v , <8 x i16>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v8i16_with_unfolded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v8i16_with_unfolded_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -16{{$}}
-; SIMD128-NEXT: i32.add $push[[R:[0-9]+]]=, $1, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.store 0($pop[[R]]), $0{{$}}
 define void @store_v8i16_with_unfolded_offset(<8 x i16> %v, <8 x i16>* %p) {
-  %s = getelementptr inbounds <8 x i16>, <8 x i16>* %p, i32 -1
+; CHECK-LABEL: store_v8i16_with_unfolded_offset:
+; CHECK:         .functype store_v8i16_with_unfolded_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 0
+; CHECK-NEXT:    # fallthrough-return
+  %q = ptrtoint <8 x i16>* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to <8 x i16>*
   store <8 x i16> %v , <8 x i16>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v8i16_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v8i16_with_unfolded_gep_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[R:[0-9]+]]=, $1, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.store 0($pop[[R]]), $0{{$}}
 define void @store_v8i16_with_unfolded_gep_offset(<8 x i16> %v, <8 x i16>* %p) {
+; CHECK-LABEL: store_v8i16_with_unfolded_gep_offset:
+; CHECK:         .functype store_v8i16_with_unfolded_gep_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr <8 x i16>, <8 x i16>* %p, i32 1
   store <8 x i16> %v , <8 x i16>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v8i16_to_numeric_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v8i16_to_numeric_address (v128) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v128.store 32($pop[[L0]]), $0{{$}}
 define void @store_v8i16_to_numeric_address(<8 x i16> %v) {
+; CHECK-LABEL: store_v8i16_to_numeric_address:
+; CHECK:         .functype store_v8i16_to_numeric_address (v128) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to <8 x i16>*
   store <8 x i16> %v , <8 x i16>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v8i16_to_global_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v8i16_to_global_address (v128) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[R:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v128.store gv_v8i16($pop[[R]]), $0{{$}}
 define void @store_v8i16_to_global_address(<8 x i16> %v) {
+; CHECK-LABEL: store_v8i16_to_global_address:
+; CHECK:         .functype store_v8i16_to_global_address (v128) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store gv_v8i16
+; CHECK-NEXT:    # fallthrough-return
   store <8 x i16> %v , <8 x i16>* @gv_v8i16
   ret void
 }
@@ -931,65 +1021,72 @@ define void @store_v8i16_to_global_address(<8 x i16> %v) {
 ; ==============================================================================
 ; 4 x i32
 ; ==============================================================================
-; CHECK-LABEL: load_v4i32:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v4i32 (i32) -> (v128){{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 0($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_v4i32(<4 x i32>* %p) {
+; CHECK-LABEL: load_v4i32:
+; CHECK:         .functype load_v4i32 (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.load 0
+; CHECK-NEXT:    # fallthrough-return
   %v = load <4 x i32>, <4 x i32>* %p
   ret <4 x i32> %v
 }
 
-; CHECK-LABEL: load_splat_v4i32:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v4i32 (i32) -> (v128){{$}}
-; SIMD128-NEXT: v32x4.load_splat
 define <4 x i32> @load_splat_v4i32(i32* %addr) {
+; CHECK-LABEL: load_splat_v4i32:
+; CHECK:         .functype load_splat_v4i32 (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v32x4.load_splat 0
+; CHECK-NEXT:    # fallthrough-return
   %e = load i32, i32* %addr, align 4
   %v1 = insertelement <4 x i32> undef, i32 %e, i32 0
   %v2 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> zeroinitializer
   ret <4 x i32> %v2
 }
 
-; CHECK-LABEL: load_sext_v4i32:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_sext_v4i32 (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.load16x4_s $push[[R:[0-9]+]]=, 0($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_sext_v4i32(<4 x i16>* %p) {
+; CHECK-LABEL: load_sext_v4i32:
+; CHECK:         .functype load_sext_v4i32 (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32x4.load16x4_s 0
+; CHECK-NEXT:    # fallthrough-return
   %v = load <4 x i16>, <4 x i16>* %p
   %v2 = sext <4 x i16> %v to <4 x i32>
   ret <4 x i32> %v2
 }
 
-; CHECK-LABEL: load_zext_v4i32:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_zext_v4i32 (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 0($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_zext_v4i32(<4 x i16>* %p) {
+; CHECK-LABEL: load_zext_v4i32:
+; CHECK:         .functype load_zext_v4i32 (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32x4.load16x4_u 0
+; CHECK-NEXT:    # fallthrough-return
   %v = load <4 x i16>, <4 x i16>* %p
   %v2 = zext <4 x i16> %v to <4 x i32>
   ret <4 x i32> %v2
 }
 
-; CHECK-LABEL: load_ext_v4i32:
-; NO-SIMD128-NOT: load16x4
-; SIMD128-NEXT: .functype load_ext_v4i32 (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 0($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i16> @load_ext_v4i32(<4 x i16>* %p) {
+; CHECK-LABEL: load_ext_v4i32:
+; CHECK:         .functype load_ext_v4i32 (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32x4.load16x4_u 0
+; CHECK-NEXT:    # fallthrough-return
   %v = load <4 x i16>, <4 x i16>* %p
   ret <4 x i16> %v
 }
 
-; CHECK-LABEL: load_v4i32_with_folded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v4i32_with_folded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_v4i32_with_folded_offset(<4 x i32>* %p) {
+; CHECK-LABEL: load_v4i32_with_folded_offset:
+; CHECK:         .functype load_v4i32_with_folded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.load 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <4 x i32>* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to <4 x i32>*
@@ -997,12 +1094,13 @@ define <4 x i32> @load_v4i32_with_folded_offset(<4 x i32>* %p) {
   ret <4 x i32> %v
 }
 
-; CHECK-LABEL: load_splat_v4i32_with_folded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v4i32_with_folded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_splat_v4i32_with_folded_offset(i32* %p) {
+; CHECK-LABEL: load_splat_v4i32_with_folded_offset:
+; CHECK:         .functype load_splat_v4i32_with_folded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v32x4.load_splat 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint i32* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to i32*
@@ -1012,12 +1110,13 @@ define <4 x i32> @load_splat_v4i32_with_folded_offset(i32* %p) {
   ret <4 x i32> %v2
 }
 
-; CHECK-LABEL: load_sext_v4i32_with_folded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_sext_v4i32_with_folded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.load16x4_s $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_sext_v4i32_with_folded_offset(<4 x i16>* %p) {
+; CHECK-LABEL: load_sext_v4i32_with_folded_offset:
+; CHECK:         .functype load_sext_v4i32_with_folded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32x4.load16x4_s 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <4 x i16>* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to <4 x i16>*
@@ -1026,12 +1125,13 @@ define <4 x i32> @load_sext_v4i32_with_folded_offset(<4 x i16>* %p) {
   ret <4 x i32> %v2
 }
 
-; CHECK-LABEL: load_zext_v4i32_with_folded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_zext_v4i32_with_folded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_zext_v4i32_with_folded_offset(<4 x i16>* %p) {
+; CHECK-LABEL: load_zext_v4i32_with_folded_offset:
+; CHECK:         .functype load_zext_v4i32_with_folded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32x4.load16x4_u 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <4 x i16>* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to <4 x i16>*
@@ -1040,12 +1140,13 @@ define <4 x i32> @load_zext_v4i32_with_folded_offset(<4 x i16>* %p) {
   ret <4 x i32> %v2
 }
 
-; CHECK-LABEL: load_ext_v4i32_with_folded_offset:
-; NO-SIMD128-NOT: load16x4
-; SIMD128-NEXT: .functype load_ext_v4i32_with_folded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i16> @load_ext_v4i32_with_folded_offset(<4 x i16>* %p) {
+; CHECK-LABEL: load_ext_v4i32_with_folded_offset:
+; CHECK:         .functype load_ext_v4i32_with_folded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32x4.load16x4_u 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <4 x i16>* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to <4 x i16>*
@@ -1053,23 +1154,25 @@ define <4 x i16> @load_ext_v4i32_with_folded_offset(<4 x i16>* %p) {
   ret <4 x i16> %v
 }
 
-; CHECK-LABEL: load_v4i32_with_folded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v4i32_with_folded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_v4i32_with_folded_gep_offset(<4 x i32>* %p) {
+; CHECK-LABEL: load_v4i32_with_folded_gep_offset:
+; CHECK:         .functype load_v4i32_with_folded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.load 16
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <4 x i32>, <4 x i32>* %p, i32 1
   %v = load <4 x i32>, <4 x i32>* %s
   ret <4 x i32> %v
 }
 
-; CHECK-LABEL: load_splat_v4i32_with_folded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v4i32_with_folded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 4($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_splat_v4i32_with_folded_gep_offset(i32* %p) {
+; CHECK-LABEL: load_splat_v4i32_with_folded_gep_offset:
+; CHECK:         .functype load_splat_v4i32_with_folded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v32x4.load_splat 4
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i32, i32* %p, i32 1
   %e = load i32, i32* %s
   %v1 = insertelement <4 x i32> undef, i32 %e, i32 0
@@ -1077,62 +1180,67 @@ define <4 x i32> @load_splat_v4i32_with_folded_gep_offset(i32* %p) {
   ret <4 x i32> %v2
 }
 
-; CHECK-LABEL: load_sext_v4i32_with_folded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_sext_v4i32_with_folded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.load16x4_s $push[[R:[0-9]+]]=, 8($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_sext_v4i32_with_folded_gep_offset(<4 x i16>* %p) {
+; CHECK-LABEL: load_sext_v4i32_with_folded_gep_offset:
+; CHECK:         .functype load_sext_v4i32_with_folded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32x4.load16x4_s 8
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <4 x i16>, <4 x i16>* %p, i32 1
   %v = load <4 x i16>, <4 x i16>* %s
   %v2 = sext <4 x i16> %v to <4 x i32>
   ret <4 x i32> %v2
 }
 
-; CHECK-LABEL: load_zext_v4i32_with_folded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_zext_v4i32_with_folded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 8($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_zext_v4i32_with_folded_gep_offset(<4 x i16>* %p) {
+; CHECK-LABEL: load_zext_v4i32_with_folded_gep_offset:
+; CHECK:         .functype load_zext_v4i32_with_folded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32x4.load16x4_u 8
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <4 x i16>, <4 x i16>* %p, i32 1
   %v = load <4 x i16>, <4 x i16>* %s
   %v2 = zext <4 x i16> %v to <4 x i32>
   ret <4 x i32> %v2
 }
 
-; CHECK-LABEL: load_ext_v4i32_with_folded_gep_offset:
-; NO-SIMD128-NOT: load16x4
-; SIMD128-NEXT: .functype load_ext_v4i32_with_folded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 8($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i16> @load_ext_v4i32_with_folded_gep_offset(<4 x i16>* %p) {
+; CHECK-LABEL: load_ext_v4i32_with_folded_gep_offset:
+; CHECK:         .functype load_ext_v4i32_with_folded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32x4.load16x4_u 8
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <4 x i16>, <4 x i16>* %p, i32 1
   %v = load <4 x i16>, <4 x i16>* %s
   ret <4 x i16> %v
 }
 
-; CHECK-LABEL: load_v4i32_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v4i32_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_v4i32_with_unfolded_gep_negative_offset(<4 x i32>* %p) {
+; CHECK-LABEL: load_v4i32_with_unfolded_gep_negative_offset:
+; CHECK:         .functype load_v4i32_with_unfolded_gep_negative_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const -16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v128.load 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <4 x i32>, <4 x i32>* %p, i32 -1
   %v = load <4 x i32>, <4 x i32>* %s
   ret <4 x i32> %v
 }
 
-; CHECK-LABEL: load_splat_v4i32_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v4i32_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -4{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_splat_v4i32_with_unfolded_gep_negative_offset(i32* %p) {
+; CHECK-LABEL: load_splat_v4i32_with_unfolded_gep_negative_offset:
+; CHECK:         .functype load_splat_v4i32_with_unfolded_gep_negative_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const -4
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v32x4.load_splat 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i32, i32* %p, i32 -1
   %e = load i32, i32* %s
   %v1 = insertelement <4 x i32> undef, i32 %e, i32 0
@@ -1140,55 +1248,59 @@ define <4 x i32> @load_splat_v4i32_with_unfolded_gep_negative_offset(i32* %p) {
   ret <4 x i32> %v2
 }
 
-; CHECK-LABEL: load_sext_v4i32_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_sext_v4i32_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i32x4.load16x4_s $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_sext_v4i32_with_unfolded_gep_negative_offset(<4 x i16>* %p) {
+; CHECK-LABEL: load_sext_v4i32_with_unfolded_gep_negative_offset:
+; CHECK:         .functype load_sext_v4i32_with_unfolded_gep_negative_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const -8
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i32x4.load16x4_s 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <4 x i16>, <4 x i16>* %p, i32 -1
   %v = load <4 x i16>, <4 x i16>* %s
   %v2 = sext <4 x i16> %v to <4 x i32>
   ret <4 x i32> %v2
 }
 
-; CHECK-LABEL: load_zext_v4i32_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_zext_v4i32_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_zext_v4i32_with_unfolded_gep_negative_offset(<4 x i16>* %p) {
+; CHECK-LABEL: load_zext_v4i32_with_unfolded_gep_negative_offset:
+; CHECK:         .functype load_zext_v4i32_with_unfolded_gep_negative_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const -8
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i32x4.load16x4_u 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <4 x i16>, <4 x i16>* %p, i32 -1
   %v = load <4 x i16>, <4 x i16>* %s
   %v2 = zext <4 x i16> %v to <4 x i32>
   ret <4 x i32> %v2
 }
 
-; CHECK-LABEL: load_ext_v4i32_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: load16x4
-; SIMD128-NEXT: .functype load_ext_v4i32_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i16> @load_ext_v4i32_with_unfolded_gep_negative_offset(<4 x i16>* %p) {
+; CHECK-LABEL: load_ext_v4i32_with_unfolded_gep_negative_offset:
+; CHECK:         .functype load_ext_v4i32_with_unfolded_gep_negative_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const -8
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i32x4.load16x4_u 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <4 x i16>, <4 x i16>* %p, i32 -1
   %v = load <4 x i16>, <4 x i16>* %s
   ret <4 x i16> %v
 }
 
-; CHECK-LABEL: load_v4i32_with_unfolded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v4i32_with_unfolded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_v4i32_with_unfolded_offset(<4 x i32>* %p) {
+; CHECK-LABEL: load_v4i32_with_unfolded_offset:
+; CHECK:         .functype load_v4i32_with_unfolded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v128.load 0
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <4 x i32>* %p to i32
   %r = add nsw i32 %q, 16
   %s = inttoptr i32 %r to <4 x i32>*
@@ -1196,14 +1308,15 @@ define <4 x i32> @load_v4i32_with_unfolded_offset(<4 x i32>* %p) {
   ret <4 x i32> %v
 }
 
-; CHECK-LABEL: load_splat_v4i32_with_unfolded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v4i32_with_unfolded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_splat_v4i32_with_unfolded_offset(i32* %p) {
+; CHECK-LABEL: load_splat_v4i32_with_unfolded_offset:
+; CHECK:         .functype load_splat_v4i32_with_unfolded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v32x4.load_splat 0
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint i32* %p to i32
   %r = add nsw i32 %q, 16
   %s = inttoptr i32 %r to i32*
@@ -1213,14 +1326,15 @@ define <4 x i32> @load_splat_v4i32_with_unfolded_offset(i32* %p) {
   ret <4 x i32> %v2
 }
 
-; CHECK-LABEL: load_sext_v4i32_with_unfolded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_sext_v4i32_with_unfolded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i32x4.load16x4_s $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_sext_v4i32_with_unfolded_offset(<4 x i16>* %p) {
+; CHECK-LABEL: load_sext_v4i32_with_unfolded_offset:
+; CHECK:         .functype load_sext_v4i32_with_unfolded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i32x4.load16x4_s 0
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <4 x i16>* %p to i32
   %r = add nsw i32 %q, 16
   %s = inttoptr i32 %r to <4 x i16>*
@@ -1229,14 +1343,15 @@ define <4 x i32> @load_sext_v4i32_with_unfolded_offset(<4 x i16>* %p) {
   ret <4 x i32> %v2
 }
 
-; CHECK-LABEL: load_zext_v4i32_with_unfolded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_zext_v4i32_with_unfolded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_zext_v4i32_with_unfolded_offset(<4 x i16>* %p) {
+; CHECK-LABEL: load_zext_v4i32_with_unfolded_offset:
+; CHECK:         .functype load_zext_v4i32_with_unfolded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i32x4.load16x4_u 0
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <4 x i16>* %p to i32
   %r = add nsw i32 %q, 16
   %s = inttoptr i32 %r to <4 x i16>*
@@ -1245,14 +1360,15 @@ define <4 x i32> @load_zext_v4i32_with_unfolded_offset(<4 x i16>* %p) {
   ret <4 x i32> %v2
 }
 
-; CHECK-LABEL: load_ext_v4i32_with_unfolded_offset:
-; NO-SIMD128-NOT: load16x4
-; SIMD128-NEXT: .functype load_ext_v4i32_with_unfolded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i16> @load_ext_v4i32_with_unfolded_offset(<4 x i16>* %p) {
+; CHECK-LABEL: load_ext_v4i32_with_unfolded_offset:
+; CHECK:         .functype load_ext_v4i32_with_unfolded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i32x4.load16x4_u 0
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <4 x i16>* %p to i32
   %r = add nsw i32 %q, 16
   %s = inttoptr i32 %r to <4 x i16>*
@@ -1260,27 +1376,29 @@ define <4 x i16> @load_ext_v4i32_with_unfolded_offset(<4 x i16>* %p) {
   ret <4 x i16> %v
 }
 
-; CHECK-LABEL: load_v4i32_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v4i32_with_unfolded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_v4i32_with_unfolded_gep_offset(<4 x i32>* %p) {
+; CHECK-LABEL: load_v4i32_with_unfolded_gep_offset:
+; CHECK:         .functype load_v4i32_with_unfolded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v128.load 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr <4 x i32>, <4 x i32>* %p, i32 1
   %v = load <4 x i32>, <4 x i32>* %s
   ret <4 x i32> %v
 }
 
-; CHECK-LABEL: load_splat_v4i32_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v4i32_with_unfolded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 4{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_splat_v4i32_with_unfolded_gep_offset(i32* %p) {
+; CHECK-LABEL: load_splat_v4i32_with_unfolded_gep_offset:
+; CHECK:         .functype load_splat_v4i32_with_unfolded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 4
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v32x4.load_splat 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr i32, i32* %p, i32 1
   %e = load i32, i32* %s
   %v1 = insertelement <4 x i32> undef, i32 %e, i32 0
@@ -1288,66 +1406,69 @@ define <4 x i32> @load_splat_v4i32_with_unfolded_gep_offset(i32* %p) {
   ret <4 x i32> %v2
 }
 
-; CHECK-LABEL: load_sext_v4i32_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_sext_v4i32_with_unfolded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i32x4.load16x4_s $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_sext_v4i32_with_unfolded_gep_offset(<4 x i16>* %p) {
+; CHECK-LABEL: load_sext_v4i32_with_unfolded_gep_offset:
+; CHECK:         .functype load_sext_v4i32_with_unfolded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 8
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i32x4.load16x4_s 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr <4 x i16>, <4 x i16>* %p, i32 1
   %v = load <4 x i16>, <4 x i16>* %s
   %v2 = sext <4 x i16> %v to <4 x i32>
   ret <4 x i32> %v2
 }
 
-; CHECK-LABEL: load_zext_v4i32_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_zext_v4i32_with_unfolded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_zext_v4i32_with_unfolded_gep_offset(<4 x i16>* %p) {
+; CHECK-LABEL: load_zext_v4i32_with_unfolded_gep_offset:
+; CHECK:         .functype load_zext_v4i32_with_unfolded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 8
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i32x4.load16x4_u 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr <4 x i16>, <4 x i16>* %p, i32 1
   %v = load <4 x i16>, <4 x i16>* %s
   %v2 = zext <4 x i16> %v to <4 x i32>
   ret <4 x i32> %v2
 }
 
-; CHECK-LABEL: load_ext_v4i32_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: load16x4
-; SIMD128-NEXT: .functype load_ext_v4i32_with_unfolded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i16> @load_ext_v4i32_with_unfolded_gep_offset(<4 x i16>* %p) {
+; CHECK-LABEL: load_ext_v4i32_with_unfolded_gep_offset:
+; CHECK:         .functype load_ext_v4i32_with_unfolded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 8
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i32x4.load16x4_u 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr <4 x i16>, <4 x i16>* %p, i32 1
   %v = load <4 x i16>, <4 x i16>* %s
   ret <4 x i16> %v
 }
 
-; CHECK-LABEL: load_v4i32_from_numeric_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v4i32_from_numeric_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_v4i32_from_numeric_address() {
+; CHECK-LABEL: load_v4i32_from_numeric_address:
+; CHECK:         .functype load_v4i32_from_numeric_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v128.load 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to <4 x i32>*
   %v = load <4 x i32>, <4 x i32>* %s
   ret <4 x i32> %v
 }
 
-; CHECK-LABEL: load_splat_v4i32_from_numeric_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v4i32_from_numeric_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_splat_v4i32_from_numeric_address() {
+; CHECK-LABEL: load_splat_v4i32_from_numeric_address:
+; CHECK:         .functype load_splat_v4i32_from_numeric_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v32x4.load_splat 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to i32*
   %e = load i32, i32* %s
   %v1 = insertelement <4 x i32> undef, i32 %e, i32 0
@@ -1355,120 +1476,126 @@ define <4 x i32> @load_splat_v4i32_from_numeric_address() {
   ret <4 x i32> %v2
 }
 
-; CHECK-LABEL: load_sext_v4i32_from_numeric_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_sext_v4i32_from_numeric_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: i32x4.load16x4_s $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_sext_v4i32_from_numeric_address() {
+; CHECK-LABEL: load_sext_v4i32_from_numeric_address:
+; CHECK:         .functype load_sext_v4i32_from_numeric_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    i32x4.load16x4_s 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to <4 x i16>*
   %v = load <4 x i16>, <4 x i16>* %s
   %v2 = sext <4 x i16> %v to <4 x i32>
   ret <4 x i32> %v2
 }
 
-; CHECK-LABEL: load_zext_v4i32_from_numeric_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_zext_v4i32_from_numeric_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_zext_v4i32_from_numeric_address() {
+; CHECK-LABEL: load_zext_v4i32_from_numeric_address:
+; CHECK:         .functype load_zext_v4i32_from_numeric_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    i32x4.load16x4_u 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to <4 x i16>*
   %v = load <4 x i16>, <4 x i16>* %s
   %v2 = zext <4 x i16> %v to <4 x i32>
   ret <4 x i32> %v2
 }
 
-; CHECK-LABEL: load_ext_v4i32_from_numeric_address:
-; NO-SIMD128-NOT: load16x4
-; SIMD128-NEXT: .functype load_ext_v4i32_from_numeric_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i16> @load_ext_v4i32_from_numeric_address() {
+; CHECK-LABEL: load_ext_v4i32_from_numeric_address:
+; CHECK:         .functype load_ext_v4i32_from_numeric_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    i32x4.load16x4_u 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to <4 x i16>*
   %v = load <4 x i16>, <4 x i16>* %s
   ret <4 x i16> %v
 }
 
-; CHECK-LABEL: load_v4i32_from_global_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v4i32_from_global_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, gv_v4i32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 @gv_v4i32 = global <4 x i32> <i32 42, i32 42, i32 42, i32 42>
 define <4 x i32> @load_v4i32_from_global_address() {
+; CHECK-LABEL: load_v4i32_from_global_address:
+; CHECK:         .functype load_v4i32_from_global_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v128.load gv_v4i32
+; CHECK-NEXT:    # fallthrough-return
   %v = load <4 x i32>, <4 x i32>* @gv_v4i32
   ret <4 x i32> %v
 }
 
-; CHECK-LABEL: load_splat_v4i32_from_global_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v4i32_from_global_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, gv_i32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 @gv_i32 = global i32 42
 define <4 x i32> @load_splat_v4i32_from_global_address() {
+; CHECK-LABEL: load_splat_v4i32_from_global_address:
+; CHECK:         .functype load_splat_v4i32_from_global_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v32x4.load_splat gv_i32
+; CHECK-NEXT:    # fallthrough-return
   %e = load i32, i32* @gv_i32
   %v1 = insertelement <4 x i32> undef, i32 %e, i32 0
   %v2 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> zeroinitializer
   ret <4 x i32> %v2
 }
 
-; CHECK-LABEL: load_sext_v4i32_from_global_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_sext_v4i32_from_global_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: i32x4.load16x4_s $push[[R:[0-9]+]]=, gv_v4i16($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 @gv_v4i16 = global <4 x i16> <i16 42, i16 42, i16 42, i16 42>
 define <4 x i32> @load_sext_v4i32_from_global_address() {
+; CHECK-LABEL: load_sext_v4i32_from_global_address:
+; CHECK:         .functype load_sext_v4i32_from_global_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    i32x4.load16x4_s gv_v4i16
+; CHECK-NEXT:    # fallthrough-return
   %v = load <4 x i16>, <4 x i16>* @gv_v4i16
   %v2 = sext <4 x i16> %v to <4 x i32>
   ret <4 x i32> %v2
 }
 
-; CHECK-LABEL: load_zext_v4i32_from_global_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_zext_v4i32_from_global_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, gv_v4i16($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i32> @load_zext_v4i32_from_global_address() {
+; CHECK-LABEL: load_zext_v4i32_from_global_address:
+; CHECK:         .functype load_zext_v4i32_from_global_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    i32x4.load16x4_u gv_v4i16
+; CHECK-NEXT:    # fallthrough-return
   %v = load <4 x i16>, <4 x i16>* @gv_v4i16
   %v2 = zext <4 x i16> %v to <4 x i32>
   ret <4 x i32> %v2
 }
 
-; CHECK-LABEL: load_ext_v4i32_from_global_address:
-; NO-SIMD128-NOT: load16x4
-; SIMD128-NEXT: .functype load_ext_v4i32_from_global_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, gv_v4i16($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x i16> @load_ext_v4i32_from_global_address() {
+; CHECK-LABEL: load_ext_v4i32_from_global_address:
+; CHECK:         .functype load_ext_v4i32_from_global_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    i32x4.load16x4_u gv_v4i16
+; CHECK-NEXT:    # fallthrough-return
   %v = load <4 x i16>, <4 x i16>* @gv_v4i16
   ret <4 x i16> %v
 }
 
-; CHECK-LABEL: store_v4i32:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v4i32 (v128, i32) -> (){{$}}
-; SIMD128-NEXT: v128.store 0($1), $0{{$}}
 define void @store_v4i32(<4 x i32> %v, <4 x i32>* %p) {
+; CHECK-LABEL: store_v4i32:
+; CHECK:         .functype store_v4i32 (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 0
+; CHECK-NEXT:    # fallthrough-return
   store <4 x i32> %v , <4 x i32>* %p
   ret void
 }
 
-; CHECK-LABEL: store_v4i32_with_folded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v4i32_with_folded_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: v128.store 16($1), $0{{$}}
 define void @store_v4i32_with_folded_offset(<4 x i32> %v, <4 x i32>* %p) {
+; CHECK-LABEL: store_v4i32_with_folded_offset:
+; CHECK:         .functype store_v4i32_with_folded_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <4 x i32>* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to <4 x i32>*
@@ -1476,69 +1603,87 @@ define void @store_v4i32_with_folded_offset(<4 x i32> %v, <4 x i32>* %p) {
   ret void
 }
 
-; CHECK-LABEL: store_v4i32_with_folded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v4i32_with_folded_gep_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: v128.store 16($1), $0{{$}}
 define void @store_v4i32_with_folded_gep_offset(<4 x i32> %v, <4 x i32>* %p) {
+; CHECK-LABEL: store_v4i32_with_folded_gep_offset:
+; CHECK:         .functype store_v4i32_with_folded_gep_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 16
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <4 x i32>, <4 x i32>* %p, i32 1
   store <4 x i32> %v , <4 x i32>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v4i32_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v4i32_with_unfolded_gep_negative_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -16{{$}}
-; SIMD128-NEXT: i32.add $push[[R:[0-9]+]]=, $1, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.store 0($pop[[R]]), $0{{$}}
 define void @store_v4i32_with_unfolded_gep_negative_offset(<4 x i32> %v, <4 x i32>* %p) {
+; CHECK-LABEL: store_v4i32_with_unfolded_gep_negative_offset:
+; CHECK:         .functype store_v4i32_with_unfolded_gep_negative_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const -16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <4 x i32>, <4 x i32>* %p, i32 -1
   store <4 x i32> %v , <4 x i32>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v4i32_with_unfolded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v4i32_with_unfolded_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -16{{$}}
-; SIMD128-NEXT: i32.add $push[[R:[0-9]+]]=, $1, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.store 0($pop[[R]]), $0{{$}}
 define void @store_v4i32_with_unfolded_offset(<4 x i32> %v, <4 x i32>* %p) {
-  %s = getelementptr inbounds <4 x i32>, <4 x i32>* %p, i32 -1
+; CHECK-LABEL: store_v4i32_with_unfolded_offset:
+; CHECK:         .functype store_v4i32_with_unfolded_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 0
+; CHECK-NEXT:    # fallthrough-return
+  %q = ptrtoint <4 x i32>* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to <4 x i32>*
   store <4 x i32> %v , <4 x i32>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v4i32_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v4i32_with_unfolded_gep_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[R:[0-9]+]]=, $1, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.store 0($pop[[R]]), $0{{$}}
 define void @store_v4i32_with_unfolded_gep_offset(<4 x i32> %v, <4 x i32>* %p) {
+; CHECK-LABEL: store_v4i32_with_unfolded_gep_offset:
+; CHECK:         .functype store_v4i32_with_unfolded_gep_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr <4 x i32>, <4 x i32>* %p, i32 1
   store <4 x i32> %v , <4 x i32>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v4i32_to_numeric_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v4i32_to_numeric_address (v128) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v128.store 32($pop[[L0]]), $0{{$}}
 define void @store_v4i32_to_numeric_address(<4 x i32> %v) {
+; CHECK-LABEL: store_v4i32_to_numeric_address:
+; CHECK:         .functype store_v4i32_to_numeric_address (v128) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to <4 x i32>*
   store <4 x i32> %v , <4 x i32>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v4i32_to_global_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v4i32_to_global_address (v128) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[R:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v128.store gv_v4i32($pop[[R]]), $0{{$}}
 define void @store_v4i32_to_global_address(<4 x i32> %v) {
+; CHECK-LABEL: store_v4i32_to_global_address:
+; CHECK:         .functype store_v4i32_to_global_address (v128) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store gv_v4i32
+; CHECK-NEXT:    # fallthrough-return
   store <4 x i32> %v , <4 x i32>* @gv_v4i32
   ret void
 }
@@ -1546,66 +1691,72 @@ define void @store_v4i32_to_global_address(<4 x i32> %v) {
 ; ==============================================================================
 ; 2 x i64
 ; ==============================================================================
-; CHECK-LABEL: load_v2i64:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v2i64 (i32) -> (v128){{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 0($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_v2i64(<2 x i64>* %p) {
+; CHECK-LABEL: load_v2i64:
+; CHECK:         .functype load_v2i64 (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.load 0
+; CHECK-NEXT:    # fallthrough-return
   %v = load <2 x i64>, <2 x i64>* %p
   ret <2 x i64> %v
 }
 
-; CHECK-LABEL: load_splat_v2i64:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v2i64 (i32) -> (v128){{$}}
-; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 0($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_splat_v2i64(i64* %p) {
+; CHECK-LABEL: load_splat_v2i64:
+; CHECK:         .functype load_splat_v2i64 (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v64x2.load_splat 0
+; CHECK-NEXT:    # fallthrough-return
   %e = load i64, i64* %p
   %v1 = insertelement <2 x i64> undef, i64 %e, i32 0
   %v2 = shufflevector <2 x i64> %v1, <2 x i64> undef, <2 x i32> zeroinitializer
   ret <2 x i64> %v2
 }
 
-; CHECK-LABEL: load_sext_v2i64:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_sext_v2i64 (i32) -> (v128){{$}}
-; SIMD128-NEXT: i64x2.load32x2_s $push[[R:[0-9]+]]=, 0($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_sext_v2i64(<2 x i32>* %p) {
+; CHECK-LABEL: load_sext_v2i64:
+; CHECK:         .functype load_sext_v2i64 (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i64x2.load32x2_s 0
+; CHECK-NEXT:    # fallthrough-return
   %v = load <2 x i32>, <2 x i32>* %p
   %v2 = sext <2 x i32> %v to <2 x i64>
   ret <2 x i64> %v2
 }
 
-; CHECK-LABEL: load_zext_v2i64:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_zext_v2i64 (i32) -> (v128){{$}}
-; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 0($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_zext_v2i64(<2 x i32>* %p) {
+; CHECK-LABEL: load_zext_v2i64:
+; CHECK:         .functype load_zext_v2i64 (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i64x2.load32x2_u 0
+; CHECK-NEXT:    # fallthrough-return
   %v = load <2 x i32>, <2 x i32>* %p
   %v2 = zext <2 x i32> %v to <2 x i64>
   ret <2 x i64> %v2
 }
 
-; CHECK-LABEL: load_ext_v2i64:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_ext_v2i64 (i32) -> (v128){{$}}
-; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 0($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i32> @load_ext_v2i64(<2 x i32>* %p) {
+; CHECK-LABEL: load_ext_v2i64:
+; CHECK:         .functype load_ext_v2i64 (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i64x2.load32x2_u 0
+; CHECK-NEXT:    # fallthrough-return
   %v = load <2 x i32>, <2 x i32>* %p
   ret <2 x i32> %v
 }
 
-; CHECK-LABEL: load_v2i64_with_folded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v2i64_with_folded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_v2i64_with_folded_offset(<2 x i64>* %p) {
+; CHECK-LABEL: load_v2i64_with_folded_offset:
+; CHECK:         .functype load_v2i64_with_folded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.load 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <2 x i64>* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to <2 x i64>*
@@ -1613,12 +1764,13 @@ define <2 x i64> @load_v2i64_with_folded_offset(<2 x i64>* %p) {
   ret <2 x i64> %v
 }
 
-; CHECK-LABEL: load_splat_v2i64_with_folded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v2i64_with_folded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_splat_v2i64_with_folded_offset(i64* %p) {
+; CHECK-LABEL: load_splat_v2i64_with_folded_offset:
+; CHECK:         .functype load_splat_v2i64_with_folded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v64x2.load_splat 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint i64* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to i64*
@@ -1628,12 +1780,13 @@ define <2 x i64> @load_splat_v2i64_with_folded_offset(i64* %p) {
   ret <2 x i64> %v2
 }
 
-; CHECK-LABEL: load_sext_v2i64_with_folded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_sext_v2i64_with_folded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i64x2.load32x2_s $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_sext_v2i64_with_folded_offset(<2 x i32>* %p) {
+; CHECK-LABEL: load_sext_v2i64_with_folded_offset:
+; CHECK:         .functype load_sext_v2i64_with_folded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i64x2.load32x2_s 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <2 x i32>* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to <2 x i32>*
@@ -1642,12 +1795,13 @@ define <2 x i64> @load_sext_v2i64_with_folded_offset(<2 x i32>* %p) {
   ret <2 x i64> %v2
 }
 
-; CHECK-LABEL: load_zext_v2i64_with_folded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_zext_v2i64_with_folded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_zext_v2i64_with_folded_offset(<2 x i32>* %p) {
+; CHECK-LABEL: load_zext_v2i64_with_folded_offset:
+; CHECK:         .functype load_zext_v2i64_with_folded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i64x2.load32x2_u 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <2 x i32>* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to <2 x i32>*
@@ -1656,12 +1810,13 @@ define <2 x i64> @load_zext_v2i64_with_folded_offset(<2 x i32>* %p) {
   ret <2 x i64> %v2
 }
 
-; CHECK-LABEL: load_ext_v2i64_with_folded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_ext_v2i64_with_folded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i32> @load_ext_v2i64_with_folded_offset(<2 x i32>* %p) {
+; CHECK-LABEL: load_ext_v2i64_with_folded_offset:
+; CHECK:         .functype load_ext_v2i64_with_folded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i64x2.load32x2_u 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <2 x i32>* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to <2 x i32>*
@@ -1669,23 +1824,25 @@ define <2 x i32> @load_ext_v2i64_with_folded_offset(<2 x i32>* %p) {
   ret <2 x i32> %v
 }
 
-; CHECK-LABEL: load_v2i64_with_folded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v2i64_with_folded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_v2i64_with_folded_gep_offset(<2 x i64>* %p) {
+; CHECK-LABEL: load_v2i64_with_folded_gep_offset:
+; CHECK:         .functype load_v2i64_with_folded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.load 16
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <2 x i64>, <2 x i64>* %p, i32 1
   %v = load <2 x i64>, <2 x i64>* %s
   ret <2 x i64> %v
 }
 
-; CHECK-LABEL: load_splat_v2i64_with_folded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v2i64_with_folded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 8($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_splat_v2i64_with_folded_gep_offset(i64* %p) {
+; CHECK-LABEL: load_splat_v2i64_with_folded_gep_offset:
+; CHECK:         .functype load_splat_v2i64_with_folded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v64x2.load_splat 8
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i64, i64* %p, i32 1
   %e = load i64, i64* %s
   %v1 = insertelement <2 x i64> undef, i64 %e, i32 0
@@ -1693,62 +1850,67 @@ define <2 x i64> @load_splat_v2i64_with_folded_gep_offset(i64* %p) {
   ret <2 x i64> %v2
 }
 
-; CHECK-LABEL: load_sext_v2i64_with_folded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_sext_v2i64_with_folded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i64x2.load32x2_s $push[[R:[0-9]+]]=, 8($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_sext_v2i64_with_folded_gep_offset(<2 x i32>* %p) {
+; CHECK-LABEL: load_sext_v2i64_with_folded_gep_offset:
+; CHECK:         .functype load_sext_v2i64_with_folded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i64x2.load32x2_s 8
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <2 x i32>, <2 x i32>* %p, i32 1
   %v = load <2 x i32>, <2 x i32>* %s
   %v2 = sext <2 x i32> %v to <2 x i64>
   ret <2 x i64> %v2
 }
 
-; CHECK-LABEL: load_zext_v2i64_with_folded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_zext_v2i64_with_folded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 8($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_zext_v2i64_with_folded_gep_offset(<2 x i32>* %p) {
+; CHECK-LABEL: load_zext_v2i64_with_folded_gep_offset:
+; CHECK:         .functype load_zext_v2i64_with_folded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i64x2.load32x2_u 8
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <2 x i32>, <2 x i32>* %p, i32 1
   %v = load <2 x i32>, <2 x i32>* %s
   %v2 = zext <2 x i32> %v to <2 x i64>
   ret <2 x i64> %v2
 }
 
-; CHECK-LABEL: load_ext_v2i64_with_folded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_ext_v2i64_with_folded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 8($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i32> @load_ext_v2i64_with_folded_gep_offset(<2 x i32>* %p) {
+; CHECK-LABEL: load_ext_v2i64_with_folded_gep_offset:
+; CHECK:         .functype load_ext_v2i64_with_folded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i64x2.load32x2_u 8
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <2 x i32>, <2 x i32>* %p, i32 1
   %v = load <2 x i32>, <2 x i32>* %s
   ret <2 x i32> %v
 }
 
-; CHECK-LABEL: load_v2i64_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v2i64_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_v2i64_with_unfolded_gep_negative_offset(<2 x i64>* %p) {
+; CHECK-LABEL: load_v2i64_with_unfolded_gep_negative_offset:
+; CHECK:         .functype load_v2i64_with_unfolded_gep_negative_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const -16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v128.load 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <2 x i64>, <2 x i64>* %p, i32 -1
   %v = load <2 x i64>, <2 x i64>* %s
   ret <2 x i64> %v
 }
 
-; CHECK-LABEL: load_splat_v2i64_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v2i64_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_splat_v2i64_with_unfolded_gep_negative_offset(i64* %p) {
+; CHECK-LABEL: load_splat_v2i64_with_unfolded_gep_negative_offset:
+; CHECK:         .functype load_splat_v2i64_with_unfolded_gep_negative_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const -8
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v64x2.load_splat 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i64, i64* %p, i32 -1
   %e = load i64, i64* %s
   %v1 = insertelement <2 x i64> undef, i64 %e, i32 0
@@ -1756,55 +1918,59 @@ define <2 x i64> @load_splat_v2i64_with_unfolded_gep_negative_offset(i64* %p) {
   ret <2 x i64> %v2
 }
 
-; CHECK-LABEL: load_sext_v2i64_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_sext_v2i64_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i64x2.load32x2_s $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_sext_v2i64_with_unfolded_gep_negative_offset(<2 x i32>* %p) {
+; CHECK-LABEL: load_sext_v2i64_with_unfolded_gep_negative_offset:
+; CHECK:         .functype load_sext_v2i64_with_unfolded_gep_negative_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const -8
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i64x2.load32x2_s 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <2 x i32>, <2 x i32>* %p, i32 -1
   %v = load <2 x i32>, <2 x i32>* %s
   %v2 = sext <2 x i32> %v to <2 x i64>
   ret <2 x i64> %v2
 }
 
-; CHECK-LABEL: load_zext_v2i64_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_zext_v2i64_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_zext_v2i64_with_unfolded_gep_negative_offset(<2 x i32>* %p) {
+; CHECK-LABEL: load_zext_v2i64_with_unfolded_gep_negative_offset:
+; CHECK:         .functype load_zext_v2i64_with_unfolded_gep_negative_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const -8
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i64x2.load32x2_u 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <2 x i32>, <2 x i32>* %p, i32 -1
   %v = load <2 x i32>, <2 x i32>* %s
   %v2 = zext <2 x i32> %v to <2 x i64>
   ret <2 x i64> %v2
 }
 
-; CHECK-LABEL: load_ext_v2i64_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_ext_v2i64_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i32> @load_ext_v2i64_with_unfolded_gep_negative_offset(<2 x i32>* %p) {
+; CHECK-LABEL: load_ext_v2i64_with_unfolded_gep_negative_offset:
+; CHECK:         .functype load_ext_v2i64_with_unfolded_gep_negative_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const -8
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i64x2.load32x2_u 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <2 x i32>, <2 x i32>* %p, i32 -1
   %v = load <2 x i32>, <2 x i32>* %s
   ret <2 x i32> %v
 }
 
-; CHECK-LABEL: load_v2i64_with_unfolded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v2i64_with_unfolded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_v2i64_with_unfolded_offset(<2 x i64>* %p) {
+; CHECK-LABEL: load_v2i64_with_unfolded_offset:
+; CHECK:         .functype load_v2i64_with_unfolded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v128.load 0
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <2 x i64>* %p to i32
   %r = add nsw i32 %q, 16
   %s = inttoptr i32 %r to <2 x i64>*
@@ -1812,14 +1978,15 @@ define <2 x i64> @load_v2i64_with_unfolded_offset(<2 x i64>* %p) {
   ret <2 x i64> %v
 }
 
-; CHECK-LABEL: load_splat_v2i64_with_unfolded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v2i64_with_unfolded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_splat_v2i64_with_unfolded_offset(i64* %p) {
+; CHECK-LABEL: load_splat_v2i64_with_unfolded_offset:
+; CHECK:         .functype load_splat_v2i64_with_unfolded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v64x2.load_splat 0
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint i64* %p to i32
   %r = add nsw i32 %q, 16
   %s = inttoptr i32 %r to i64*
@@ -1829,14 +1996,15 @@ define <2 x i64> @load_splat_v2i64_with_unfolded_offset(i64* %p) {
   ret <2 x i64> %v2
 }
 
-; CHECK-LABEL: load_sext_v2i64_with_unfolded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_sext_v2i64_with_unfolded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i64x2.load32x2_s $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_sext_v2i64_with_unfolded_offset(<2 x i32>* %p) {
+; CHECK-LABEL: load_sext_v2i64_with_unfolded_offset:
+; CHECK:         .functype load_sext_v2i64_with_unfolded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i64x2.load32x2_s 0
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <2 x i32>* %p to i32
   %r = add nsw i32 %q, 16
   %s = inttoptr i32 %r to <2 x i32>*
@@ -1845,14 +2013,15 @@ define <2 x i64> @load_sext_v2i64_with_unfolded_offset(<2 x i32>* %p) {
   ret <2 x i64> %v2
 }
 
-; CHECK-LABEL: load_zext_v2i64_with_unfolded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_zext_v2i64_with_unfolded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_zext_v2i64_with_unfolded_offset(<2 x i32>* %p) {
+; CHECK-LABEL: load_zext_v2i64_with_unfolded_offset:
+; CHECK:         .functype load_zext_v2i64_with_unfolded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i64x2.load32x2_u 0
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <2 x i32>* %p to i32
   %r = add nsw i32 %q, 16
   %s = inttoptr i32 %r to <2 x i32>*
@@ -1861,14 +2030,15 @@ define <2 x i64> @load_zext_v2i64_with_unfolded_offset(<2 x i32>* %p) {
   ret <2 x i64> %v2
 }
 
-; CHECK-LABEL: load_ext_v2i64_with_unfolded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_ext_v2i64_with_unfolded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i32> @load_ext_v2i64_with_unfolded_offset(<2 x i32>* %p) {
+; CHECK-LABEL: load_ext_v2i64_with_unfolded_offset:
+; CHECK:         .functype load_ext_v2i64_with_unfolded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i64x2.load32x2_u 0
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <2 x i32>* %p to i32
   %r = add nsw i32 %q, 16
   %s = inttoptr i32 %r to <2 x i32>*
@@ -1876,27 +2046,29 @@ define <2 x i32> @load_ext_v2i64_with_unfolded_offset(<2 x i32>* %p) {
   ret <2 x i32> %v
 }
 
-; CHECK-LABEL: load_v2i64_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v2i64_with_unfolded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_v2i64_with_unfolded_gep_offset(<2 x i64>* %p) {
+; CHECK-LABEL: load_v2i64_with_unfolded_gep_offset:
+; CHECK:         .functype load_v2i64_with_unfolded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v128.load 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr <2 x i64>, <2 x i64>* %p, i32 1
   %v = load <2 x i64>, <2 x i64>* %s
   ret <2 x i64> %v
 }
 
-; CHECK-LABEL: load_splat_v2i64_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v2i64_with_unfolded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_splat_v2i64_with_unfolded_gep_offset(i64* %p) {
+; CHECK-LABEL: load_splat_v2i64_with_unfolded_gep_offset:
+; CHECK:         .functype load_splat_v2i64_with_unfolded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 8
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v64x2.load_splat 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr i64, i64* %p, i32 1
   %e = load i64, i64* %s
   %v1 = insertelement <2 x i64> undef, i64 %e, i32 0
@@ -1904,66 +2076,69 @@ define <2 x i64> @load_splat_v2i64_with_unfolded_gep_offset(i64* %p) {
   ret <2 x i64> %v2
 }
 
-; CHECK-LABEL: load_sext_v2i64_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_sext_v2i64_with_unfolded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i64x2.load32x2_s $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_sext_v2i64_with_unfolded_gep_offset(<2 x i32>* %p) {
+; CHECK-LABEL: load_sext_v2i64_with_unfolded_gep_offset:
+; CHECK:         .functype load_sext_v2i64_with_unfolded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 8
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i64x2.load32x2_s 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr <2 x i32>, <2 x i32>* %p, i32 1
   %v = load <2 x i32>, <2 x i32>* %s
   %v2 = sext <2 x i32> %v to <2 x i64>
   ret <2 x i64> %v2
 }
 
-; CHECK-LABEL: load_zext_v2i64_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_zext_v2i64_with_unfolded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_zext_v2i64_with_unfolded_gep_offset(<2 x i32>* %p) {
+; CHECK-LABEL: load_zext_v2i64_with_unfolded_gep_offset:
+; CHECK:         .functype load_zext_v2i64_with_unfolded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 8
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i64x2.load32x2_u 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr <2 x i32>, <2 x i32>* %p, i32 1
   %v = load <2 x i32>, <2 x i32>* %s
   %v2 = zext <2 x i32> %v to <2 x i64>
   ret <2 x i64> %v2
 }
 
-; CHECK-LABEL: load_ext_v2i64_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_ext_v2i64_with_unfolded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i32> @load_ext_v2i64_with_unfolded_gep_offset(<2 x i32>* %p) {
+; CHECK-LABEL: load_ext_v2i64_with_unfolded_gep_offset:
+; CHECK:         .functype load_ext_v2i64_with_unfolded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 8
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    i64x2.load32x2_u 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr <2 x i32>, <2 x i32>* %p, i32 1
   %v = load <2 x i32>, <2 x i32>* %s
   ret <2 x i32> %v
 }
 
-; CHECK-LABEL: load_v2i64_from_numeric_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v2i64_from_numeric_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_v2i64_from_numeric_address() {
+; CHECK-LABEL: load_v2i64_from_numeric_address:
+; CHECK:         .functype load_v2i64_from_numeric_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v128.load 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to <2 x i64>*
   %v = load <2 x i64>, <2 x i64>* %s
   ret <2 x i64> %v
 }
 
-; CHECK-LABEL: load_splat_v2i64_from_numeric_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v2i64_from_numeric_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_splat_v2i64_from_numeric_address() {
+; CHECK-LABEL: load_splat_v2i64_from_numeric_address:
+; CHECK:         .functype load_splat_v2i64_from_numeric_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v64x2.load_splat 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to i64*
   %e = load i64, i64* %s
   %v1 = insertelement <2 x i64> undef, i64 %e, i32 0
@@ -1971,120 +2146,126 @@ define <2 x i64> @load_splat_v2i64_from_numeric_address() {
   ret <2 x i64> %v2
 }
 
-; CHECK-LABEL: load_sext_v2i64_from_numeric_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_sext_v2i64_from_numeric_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: i64x2.load32x2_s $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_sext_v2i64_from_numeric_address() {
+; CHECK-LABEL: load_sext_v2i64_from_numeric_address:
+; CHECK:         .functype load_sext_v2i64_from_numeric_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    i64x2.load32x2_s 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to <2 x i32>*
   %v = load <2 x i32>, <2 x i32>* %s
   %v2 = sext <2 x i32> %v to <2 x i64>
   ret <2 x i64> %v2
 }
 
-; CHECK-LABEL: load_zext_v2i64_from_numeric_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_zext_v2i64_from_numeric_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_zext_v2i64_from_numeric_address() {
+; CHECK-LABEL: load_zext_v2i64_from_numeric_address:
+; CHECK:         .functype load_zext_v2i64_from_numeric_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    i64x2.load32x2_u 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to <2 x i32>*
   %v = load <2 x i32>, <2 x i32>* %s
   %v2 = zext <2 x i32> %v to <2 x i64>
   ret <2 x i64> %v2
 }
 
-; CHECK-LABEL: load_ext_v2i64_from_numeric_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_ext_v2i64_from_numeric_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i32> @load_ext_v2i64_from_numeric_address() {
+; CHECK-LABEL: load_ext_v2i64_from_numeric_address:
+; CHECK:         .functype load_ext_v2i64_from_numeric_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    i64x2.load32x2_u 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to <2 x i32>*
   %v = load <2 x i32>, <2 x i32>* %s
   ret <2 x i32> %v
 }
 
-; CHECK-LABEL: load_v2i64_from_global_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v2i64_from_global_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, gv_v2i64($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 @gv_v2i64 = global <2 x i64> <i64 42, i64 42>
 define <2 x i64> @load_v2i64_from_global_address() {
+; CHECK-LABEL: load_v2i64_from_global_address:
+; CHECK:         .functype load_v2i64_from_global_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v128.load gv_v2i64
+; CHECK-NEXT:    # fallthrough-return
   %v = load <2 x i64>, <2 x i64>* @gv_v2i64
   ret <2 x i64> %v
 }
 
-; CHECK-LABEL: load_splat_v2i64_from_global_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v2i64_from_global_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, gv_i64($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 @gv_i64 = global i64 42
 define <2 x i64> @load_splat_v2i64_from_global_address() {
+; CHECK-LABEL: load_splat_v2i64_from_global_address:
+; CHECK:         .functype load_splat_v2i64_from_global_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v64x2.load_splat gv_i64
+; CHECK-NEXT:    # fallthrough-return
   %e = load i64, i64* @gv_i64
   %v1 = insertelement <2 x i64> undef, i64 %e, i32 0
   %v2 = shufflevector <2 x i64> %v1, <2 x i64> undef, <2 x i32> zeroinitializer
   ret <2 x i64> %v2
 }
 
-; CHECK-LABEL: load_sext_v2i64_from_global_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_sext_v2i64_from_global_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: i64x2.load32x2_s $push[[R:[0-9]+]]=, gv_v2i32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 @gv_v2i32 = global <2 x i32> <i32 42, i32 42>
 define <2 x i64> @load_sext_v2i64_from_global_address() {
+; CHECK-LABEL: load_sext_v2i64_from_global_address:
+; CHECK:         .functype load_sext_v2i64_from_global_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    i64x2.load32x2_s gv_v2i32
+; CHECK-NEXT:    # fallthrough-return
   %v = load <2 x i32>, <2 x i32>* @gv_v2i32
   %v2 = sext <2 x i32> %v to <2 x i64>
   ret <2 x i64> %v2
 }
 
-; CHECK-LABEL: load_zext_v2i64_from_global_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_zext_v2i64_from_global_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, gv_v2i32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @load_zext_v2i64_from_global_address() {
+; CHECK-LABEL: load_zext_v2i64_from_global_address:
+; CHECK:         .functype load_zext_v2i64_from_global_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    i64x2.load32x2_u gv_v2i32
+; CHECK-NEXT:    # fallthrough-return
   %v = load <2 x i32>, <2 x i32>* @gv_v2i32
   %v2 = zext <2 x i32> %v to <2 x i64>
   ret <2 x i64> %v2
 }
 
-; CHECK-LABEL: load_ext_v2i64_from_global_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_ext_v2i64_from_global_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, gv_v2i32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i32> @load_ext_v2i64_from_global_address() {
+; CHECK-LABEL: load_ext_v2i64_from_global_address:
+; CHECK:         .functype load_ext_v2i64_from_global_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    i64x2.load32x2_u gv_v2i32
+; CHECK-NEXT:    # fallthrough-return
   %v = load <2 x i32>, <2 x i32>* @gv_v2i32
   ret <2 x i32> %v
 }
 
-; CHECK-LABEL: store_v2i64:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v2i64 (v128, i32) -> (){{$}}
-; SIMD128-NEXT: v128.store 0($1), $0{{$}}
 define void @store_v2i64(<2 x i64> %v, <2 x i64>* %p) {
+; CHECK-LABEL: store_v2i64:
+; CHECK:         .functype store_v2i64 (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 0
+; CHECK-NEXT:    # fallthrough-return
   store <2 x i64> %v , <2 x i64>* %p
   ret void
 }
 
-; CHECK-LABEL: store_v2i64_with_folded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v2i64_with_folded_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: v128.store 16($1), $0{{$}}
 define void @store_v2i64_with_folded_offset(<2 x i64> %v, <2 x i64>* %p) {
+; CHECK-LABEL: store_v2i64_with_folded_offset:
+; CHECK:         .functype store_v2i64_with_folded_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <2 x i64>* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to <2 x i64>*
@@ -2092,69 +2273,87 @@ define void @store_v2i64_with_folded_offset(<2 x i64> %v, <2 x i64>* %p) {
   ret void
 }
 
-; CHECK-LABEL: store_v2i64_with_folded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v2i64_with_folded_gep_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: v128.store 16($1), $0{{$}}
 define void @store_v2i64_with_folded_gep_offset(<2 x i64> %v, <2 x i64>* %p) {
+; CHECK-LABEL: store_v2i64_with_folded_gep_offset:
+; CHECK:         .functype store_v2i64_with_folded_gep_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 16
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <2 x i64>, <2 x i64>* %p, i32 1
   store <2 x i64> %v , <2 x i64>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v2i64_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v2i64_with_unfolded_gep_negative_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -16{{$}}
-; SIMD128-NEXT: i32.add $push[[R:[0-9]+]]=, $1, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.store 0($pop[[R]]), $0{{$}}
 define void @store_v2i64_with_unfolded_gep_negative_offset(<2 x i64> %v, <2 x i64>* %p) {
+; CHECK-LABEL: store_v2i64_with_unfolded_gep_negative_offset:
+; CHECK:         .functype store_v2i64_with_unfolded_gep_negative_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const -16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <2 x i64>, <2 x i64>* %p, i32 -1
   store <2 x i64> %v , <2 x i64>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v2i64_with_unfolded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v2i64_with_unfolded_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -16{{$}}
-; SIMD128-NEXT: i32.add $push[[R:[0-9]+]]=, $1, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.store 0($pop[[R]]), $0{{$}}
 define void @store_v2i64_with_unfolded_offset(<2 x i64> %v, <2 x i64>* %p) {
-  %s = getelementptr inbounds <2 x i64>, <2 x i64>* %p, i32 -1
+; CHECK-LABEL: store_v2i64_with_unfolded_offset:
+; CHECK:         .functype store_v2i64_with_unfolded_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 0
+; CHECK-NEXT:    # fallthrough-return
+  %q = ptrtoint <2 x i64>* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to <2 x i64>*
   store <2 x i64> %v , <2 x i64>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v2i64_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v2i64_with_unfolded_gep_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[R:[0-9]+]]=, $1, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.store 0($pop[[R]]), $0{{$}}
 define void @store_v2i64_with_unfolded_gep_offset(<2 x i64> %v, <2 x i64>* %p) {
+; CHECK-LABEL: store_v2i64_with_unfolded_gep_offset:
+; CHECK:         .functype store_v2i64_with_unfolded_gep_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr <2 x i64>, <2 x i64>* %p, i32 1
   store <2 x i64> %v , <2 x i64>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v2i64_to_numeric_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v2i64_to_numeric_address (v128) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v128.store 32($pop[[L0]]), $0{{$}}
 define void @store_v2i64_to_numeric_address(<2 x i64> %v) {
+; CHECK-LABEL: store_v2i64_to_numeric_address:
+; CHECK:         .functype store_v2i64_to_numeric_address (v128) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to <2 x i64>*
   store <2 x i64> %v , <2 x i64>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v2i64_to_global_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v2i64_to_global_address (v128) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[R:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v128.store gv_v2i64($pop[[R]]), $0{{$}}
 define void @store_v2i64_to_global_address(<2 x i64> %v) {
+; CHECK-LABEL: store_v2i64_to_global_address:
+; CHECK:         .functype store_v2i64_to_global_address (v128) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store gv_v2i64
+; CHECK-NEXT:    # fallthrough-return
   store <2 x i64> %v , <2 x i64>* @gv_v2i64
   ret void
 }
@@ -2162,34 +2361,37 @@ define void @store_v2i64_to_global_address(<2 x i64> %v) {
 ; ==============================================================================
 ; 4 x float
 ; ==============================================================================
-; CHECK-LABEL: load_v4f32:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v4f32 (i32) -> (v128){{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 0($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x float> @load_v4f32(<4 x float>* %p) {
+; CHECK-LABEL: load_v4f32:
+; CHECK:         .functype load_v4f32 (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.load 0
+; CHECK-NEXT:    # fallthrough-return
   %v = load <4 x float>, <4 x float>* %p
   ret <4 x float> %v
 }
 
-; CHECK-LABEL: load_splat_v4f32:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v4f32 (i32) -> (v128){{$}}
-; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 0($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x float> @load_splat_v4f32(float* %p) {
+; CHECK-LABEL: load_splat_v4f32:
+; CHECK:         .functype load_splat_v4f32 (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v32x4.load_splat 0
+; CHECK-NEXT:    # fallthrough-return
   %e = load float, float* %p
   %v1 = insertelement <4 x float> undef, float %e, i32 0
   %v2 = shufflevector <4 x float> %v1, <4 x float> undef, <4 x i32> zeroinitializer
   ret <4 x float> %v2
 }
 
-; CHECK-LABEL: load_v4f32_with_folded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v4f32_with_folded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x float> @load_v4f32_with_folded_offset(<4 x float>* %p) {
+; CHECK-LABEL: load_v4f32_with_folded_offset:
+; CHECK:         .functype load_v4f32_with_folded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.load 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <4 x float>* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to <4 x float>*
@@ -2197,12 +2399,13 @@ define <4 x float> @load_v4f32_with_folded_offset(<4 x float>* %p) {
   ret <4 x float> %v
 }
 
-; CHECK-LABEL: load_splat_v4f32_with_folded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v4f32_with_folded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x float> @load_splat_v4f32_with_folded_offset(float* %p) {
+; CHECK-LABEL: load_splat_v4f32_with_folded_offset:
+; CHECK:         .functype load_splat_v4f32_with_folded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v32x4.load_splat 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint float* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to float*
@@ -2212,23 +2415,25 @@ define <4 x float> @load_splat_v4f32_with_folded_offset(float* %p) {
   ret <4 x float> %v2
 }
 
-; CHECK-LABEL: load_v4f32_with_folded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v4f32_with_folded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x float> @load_v4f32_with_folded_gep_offset(<4 x float>* %p) {
+; CHECK-LABEL: load_v4f32_with_folded_gep_offset:
+; CHECK:         .functype load_v4f32_with_folded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.load 16
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <4 x float>, <4 x float>* %p, i32 1
   %v = load <4 x float>, <4 x float>* %s
   ret <4 x float> %v
 }
 
-; CHECK-LABEL: load_splat_v4f32_with_folded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v4f32_with_folded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 4($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x float> @load_splat_v4f32_with_folded_gep_offset(float* %p) {
+; CHECK-LABEL: load_splat_v4f32_with_folded_gep_offset:
+; CHECK:         .functype load_splat_v4f32_with_folded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v32x4.load_splat 4
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds float, float* %p, i32 1
   %e = load float, float* %s
   %v1 = insertelement <4 x float> undef, float %e, i32 0
@@ -2236,27 +2441,29 @@ define <4 x float> @load_splat_v4f32_with_folded_gep_offset(float* %p) {
   ret <4 x float> %v2
 }
 
-; CHECK-LABEL: load_v4f32_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v4f32_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x float> @load_v4f32_with_unfolded_gep_negative_offset(<4 x float>* %p) {
+; CHECK-LABEL: load_v4f32_with_unfolded_gep_negative_offset:
+; CHECK:         .functype load_v4f32_with_unfolded_gep_negative_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const -16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v128.load 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <4 x float>, <4 x float>* %p, i32 -1
   %v = load <4 x float>, <4 x float>* %s
   ret <4 x float> %v
 }
 
-; CHECK-LABEL: load_splat_v4f32_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v4f32_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -4{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x float> @load_splat_v4f32_with_unfolded_gep_negative_offset(float* %p) {
+; CHECK-LABEL: load_splat_v4f32_with_unfolded_gep_negative_offset:
+; CHECK:         .functype load_splat_v4f32_with_unfolded_gep_negative_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const -4
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v32x4.load_splat 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds float, float* %p, i32 -1
   %e = load float, float* %s
   %v1 = insertelement <4 x float> undef, float %e, i32 0
@@ -2264,14 +2471,15 @@ define <4 x float> @load_splat_v4f32_with_unfolded_gep_negative_offset(float* %p
   ret <4 x float> %v2
 }
 
-; CHECK-LABEL: load_v4f32_with_unfolded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v4f32_with_unfolded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x float> @load_v4f32_with_unfolded_offset(<4 x float>* %p) {
+; CHECK-LABEL: load_v4f32_with_unfolded_offset:
+; CHECK:         .functype load_v4f32_with_unfolded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v128.load 0
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <4 x float>* %p to i32
   %r = add nsw i32 %q, 16
   %s = inttoptr i32 %r to <4 x float>*
@@ -2279,14 +2487,15 @@ define <4 x float> @load_v4f32_with_unfolded_offset(<4 x float>* %p) {
   ret <4 x float> %v
 }
 
-; CHECK-LABEL: load_splat_v4f32_with_unfolded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v4f32_with_unfolded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x float> @load_splat_v4f32_with_unfolded_offset(float* %p) {
+; CHECK-LABEL: load_splat_v4f32_with_unfolded_offset:
+; CHECK:         .functype load_splat_v4f32_with_unfolded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v32x4.load_splat 0
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint float* %p to i32
   %r = add nsw i32 %q, 16
   %s = inttoptr i32 %r to float*
@@ -2296,27 +2505,29 @@ define <4 x float> @load_splat_v4f32_with_unfolded_offset(float* %p) {
   ret <4 x float> %v2
 }
 
-; CHECK-LABEL: load_v4f32_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v4f32_with_unfolded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x float> @load_v4f32_with_unfolded_gep_offset(<4 x float>* %p) {
+; CHECK-LABEL: load_v4f32_with_unfolded_gep_offset:
+; CHECK:         .functype load_v4f32_with_unfolded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v128.load 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr <4 x float>, <4 x float>* %p, i32 1
   %v = load <4 x float>, <4 x float>* %s
   ret <4 x float> %v
 }
 
-; CHECK-LABEL: load_splat_v4f32_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v4f32_with_unfolded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 4{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x float> @load_splat_v4f32_with_unfolded_gep_offset(float* %p) {
+; CHECK-LABEL: load_splat_v4f32_with_unfolded_gep_offset:
+; CHECK:         .functype load_splat_v4f32_with_unfolded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 4
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v32x4.load_splat 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr float, float* %p, i32 1
   %e = load float, float* %s
   %v1 = insertelement <4 x float> undef, float %e, i32 0
@@ -2324,25 +2535,25 @@ define <4 x float> @load_splat_v4f32_with_unfolded_gep_offset(float* %p) {
   ret <4 x float> %v2
 }
 
-; CHECK-LABEL: load_v4f32_from_numeric_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v4f32_from_numeric_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x float> @load_v4f32_from_numeric_address() {
+; CHECK-LABEL: load_v4f32_from_numeric_address:
+; CHECK:         .functype load_v4f32_from_numeric_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v128.load 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to <4 x float>*
   %v = load <4 x float>, <4 x float>* %s
   ret <4 x float> %v
 }
 
-; CHECK-LABEL: load_splat_v4f32_from_numeric_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v4f32_from_numeric_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <4 x float> @load_splat_v4f32_from_numeric_address() {
+; CHECK-LABEL: load_splat_v4f32_from_numeric_address:
+; CHECK:         .functype load_splat_v4f32_from_numeric_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v32x4.load_splat 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to float*
   %e = load float, float* %s
   %v1 = insertelement <4 x float> undef, float %e, i32 0
@@ -2350,46 +2561,52 @@ define <4 x float> @load_splat_v4f32_from_numeric_address() {
   ret <4 x float> %v2
 }
 
-; CHECK-LABEL: load_v4f32_from_global_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v4f32_from_global_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, gv_v4f32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 @gv_v4f32 = global <4 x float> <float 42., float 42., float 42., float 42.>
 define <4 x float> @load_v4f32_from_global_address() {
+; CHECK-LABEL: load_v4f32_from_global_address:
+; CHECK:         .functype load_v4f32_from_global_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v128.load gv_v4f32
+; CHECK-NEXT:    # fallthrough-return
   %v = load <4 x float>, <4 x float>* @gv_v4f32
   ret <4 x float> %v
 }
 
-; CHECK-LABEL: load_splat_v4f32_from_global_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v4f32_from_global_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, gv_f32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 @gv_f32 = global float 42.
 define <4 x float> @load_splat_v4f32_from_global_address() {
+; CHECK-LABEL: load_splat_v4f32_from_global_address:
+; CHECK:         .functype load_splat_v4f32_from_global_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v32x4.load_splat gv_f32
+; CHECK-NEXT:    # fallthrough-return
   %e = load float, float* @gv_f32
   %v1 = insertelement <4 x float> undef, float %e, i32 0
   %v2 = shufflevector <4 x float> %v1, <4 x float> undef, <4 x i32> zeroinitializer
   ret <4 x float> %v2
 }
 
-; CHECK-LABEL: store_v4f32:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v4f32 (v128, i32) -> (){{$}}
-; SIMD128-NEXT: v128.store 0($1), $0{{$}}
 define void @store_v4f32(<4 x float> %v, <4 x float>* %p) {
+; CHECK-LABEL: store_v4f32:
+; CHECK:         .functype store_v4f32 (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 0
+; CHECK-NEXT:    # fallthrough-return
   store <4 x float> %v , <4 x float>* %p
   ret void
 }
 
-; CHECK-LABEL: store_v4f32_with_folded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v4f32_with_folded_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: v128.store 16($1), $0{{$}}
 define void @store_v4f32_with_folded_offset(<4 x float> %v, <4 x float>* %p) {
+; CHECK-LABEL: store_v4f32_with_folded_offset:
+; CHECK:         .functype store_v4f32_with_folded_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <4 x float>* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to <4 x float>*
@@ -2397,69 +2614,87 @@ define void @store_v4f32_with_folded_offset(<4 x float> %v, <4 x float>* %p) {
   ret void
 }
 
-; CHECK-LABEL: store_v4f32_with_folded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v4f32_with_folded_gep_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: v128.store 16($1), $0{{$}}
 define void @store_v4f32_with_folded_gep_offset(<4 x float> %v, <4 x float>* %p) {
+; CHECK-LABEL: store_v4f32_with_folded_gep_offset:
+; CHECK:         .functype store_v4f32_with_folded_gep_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 16
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <4 x float>, <4 x float>* %p, i32 1
   store <4 x float> %v , <4 x float>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v4f32_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v4f32_with_unfolded_gep_negative_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -16{{$}}
-; SIMD128-NEXT: i32.add $push[[R:[0-9]+]]=, $1, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.store 0($pop[[R]]), $0{{$}}
 define void @store_v4f32_with_unfolded_gep_negative_offset(<4 x float> %v, <4 x float>* %p) {
+; CHECK-LABEL: store_v4f32_with_unfolded_gep_negative_offset:
+; CHECK:         .functype store_v4f32_with_unfolded_gep_negative_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const -16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <4 x float>, <4 x float>* %p, i32 -1
   store <4 x float> %v , <4 x float>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v4f32_with_unfolded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v4f32_with_unfolded_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -16{{$}}
-; SIMD128-NEXT: i32.add $push[[R:[0-9]+]]=, $1, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.store 0($pop[[R]]), $0{{$}}
 define void @store_v4f32_with_unfolded_offset(<4 x float> %v, <4 x float>* %p) {
-  %s = getelementptr inbounds <4 x float>, <4 x float>* %p, i32 -1
+; CHECK-LABEL: store_v4f32_with_unfolded_offset:
+; CHECK:         .functype store_v4f32_with_unfolded_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 0
+; CHECK-NEXT:    # fallthrough-return
+  %q = ptrtoint <4 x float>* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to <4 x float>*
   store <4 x float> %v , <4 x float>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v4f32_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v4f32_with_unfolded_gep_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[R:[0-9]+]]=, $1, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.store 0($pop[[R]]), $0{{$}}
 define void @store_v4f32_with_unfolded_gep_offset(<4 x float> %v, <4 x float>* %p) {
+; CHECK-LABEL: store_v4f32_with_unfolded_gep_offset:
+; CHECK:         .functype store_v4f32_with_unfolded_gep_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr <4 x float>, <4 x float>* %p, i32 1
   store <4 x float> %v , <4 x float>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v4f32_to_numeric_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v4f32_to_numeric_address (v128) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v128.store 32($pop[[L0]]), $0{{$}}
 define void @store_v4f32_to_numeric_address(<4 x float> %v) {
+; CHECK-LABEL: store_v4f32_to_numeric_address:
+; CHECK:         .functype store_v4f32_to_numeric_address (v128) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to <4 x float>*
   store <4 x float> %v , <4 x float>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v4f32_to_global_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v4f32_to_global_address (v128) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[R:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v128.store gv_v4f32($pop[[R]]), $0{{$}}
 define void @store_v4f32_to_global_address(<4 x float> %v) {
+; CHECK-LABEL: store_v4f32_to_global_address:
+; CHECK:         .functype store_v4f32_to_global_address (v128) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store gv_v4f32
+; CHECK-NEXT:    # fallthrough-return
   store <4 x float> %v , <4 x float>* @gv_v4f32
   ret void
 }
@@ -2467,34 +2702,37 @@ define void @store_v4f32_to_global_address(<4 x float> %v) {
 ; ==============================================================================
 ; 2 x double
 ; ==============================================================================
-; CHECK-LABEL: load_v2f64:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v2f64 (i32) -> (v128){{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 0($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x double> @load_v2f64(<2 x double>* %p) {
+; CHECK-LABEL: load_v2f64:
+; CHECK:         .functype load_v2f64 (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.load 0
+; CHECK-NEXT:    # fallthrough-return
   %v = load <2 x double>, <2 x double>* %p
   ret <2 x double> %v
 }
 
-; CHECK-LABEL: load_splat_v2f64:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v2f64 (i32) -> (v128){{$}}
-; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 0($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x double> @load_splat_v2f64(double* %p) {
+; CHECK-LABEL: load_splat_v2f64:
+; CHECK:         .functype load_splat_v2f64 (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v64x2.load_splat 0
+; CHECK-NEXT:    # fallthrough-return
   %e = load double, double* %p
   %v1 = insertelement <2 x double> undef, double %e, i32 0
   %v2 = shufflevector <2 x double> %v1, <2 x double> undef, <2 x i32> zeroinitializer
   ret <2 x double> %v2
 }
 
-; CHECK-LABEL: load_v2f64_with_folded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v2f64_with_folded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x double> @load_v2f64_with_folded_offset(<2 x double>* %p) {
+; CHECK-LABEL: load_v2f64_with_folded_offset:
+; CHECK:         .functype load_v2f64_with_folded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.load 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <2 x double>* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to <2 x double>*
@@ -2502,12 +2740,13 @@ define <2 x double> @load_v2f64_with_folded_offset(<2 x double>* %p) {
   ret <2 x double> %v
 }
 
-; CHECK-LABEL: load_splat_v2f64_with_folded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v2f64_with_folded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x double> @load_splat_v2f64_with_folded_offset(double* %p) {
+; CHECK-LABEL: load_splat_v2f64_with_folded_offset:
+; CHECK:         .functype load_splat_v2f64_with_folded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v64x2.load_splat 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint double* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to double*
@@ -2517,23 +2756,25 @@ define <2 x double> @load_splat_v2f64_with_folded_offset(double* %p) {
   ret <2 x double> %v2
 }
 
-; CHECK-LABEL: load_v2f64_with_folded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v2f64_with_folded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 16($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x double> @load_v2f64_with_folded_gep_offset(<2 x double>* %p) {
+; CHECK-LABEL: load_v2f64_with_folded_gep_offset:
+; CHECK:         .functype load_v2f64_with_folded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.load 16
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <2 x double>, <2 x double>* %p, i32 1
   %v = load <2 x double>, <2 x double>* %s
   ret <2 x double> %v
 }
 
-; CHECK-LABEL: load_splat_v2f64_with_folded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v2f64_with_folded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 8($0){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x double> @load_splat_v2f64_with_folded_gep_offset(double* %p) {
+; CHECK-LABEL: load_splat_v2f64_with_folded_gep_offset:
+; CHECK:         .functype load_splat_v2f64_with_folded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v64x2.load_splat 8
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds double, double* %p, i32 1
   %e = load double, double* %s
   %v1 = insertelement <2 x double> undef, double %e, i32 0
@@ -2541,27 +2782,29 @@ define <2 x double> @load_splat_v2f64_with_folded_gep_offset(double* %p) {
   ret <2 x double> %v2
 }
 
-; CHECK-LABEL: load_v2f64_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v2f64_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x double> @load_v2f64_with_unfolded_gep_negative_offset(<2 x double>* %p) {
+; CHECK-LABEL: load_v2f64_with_unfolded_gep_negative_offset:
+; CHECK:         .functype load_v2f64_with_unfolded_gep_negative_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const -16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v128.load 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <2 x double>, <2 x double>* %p, i32 -1
   %v = load <2 x double>, <2 x double>* %s
   ret <2 x double> %v
 }
 
-; CHECK-LABEL: load_splat_v2f64_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v2f64_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x double> @load_splat_v2f64_with_unfolded_gep_negative_offset(double* %p) {
+; CHECK-LABEL: load_splat_v2f64_with_unfolded_gep_negative_offset:
+; CHECK:         .functype load_splat_v2f64_with_unfolded_gep_negative_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const -8
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v64x2.load_splat 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds double, double* %p, i32 -1
   %e = load double, double* %s
   %v1 = insertelement <2 x double> undef, double %e, i32 0
@@ -2569,14 +2812,15 @@ define <2 x double> @load_splat_v2f64_with_unfolded_gep_negative_offset(double*
   ret <2 x double> %v2
 }
 
-; CHECK-LABEL: load_v2f64_with_unfolded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v2f64_with_unfolded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x double> @load_v2f64_with_unfolded_offset(<2 x double>* %p) {
+; CHECK-LABEL: load_v2f64_with_unfolded_offset:
+; CHECK:         .functype load_v2f64_with_unfolded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v128.load 0
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <2 x double>* %p to i32
   %r = add nsw i32 %q, 16
   %s = inttoptr i32 %r to <2 x double>*
@@ -2584,14 +2828,15 @@ define <2 x double> @load_v2f64_with_unfolded_offset(<2 x double>* %p) {
   ret <2 x double> %v
 }
 
-; CHECK-LABEL: load_splat_v2f64_with_unfolded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v2f64_with_unfolded_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x double> @load_splat_v2f64_with_unfolded_offset(double* %p) {
+; CHECK-LABEL: load_splat_v2f64_with_unfolded_offset:
+; CHECK:         .functype load_splat_v2f64_with_unfolded_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v64x2.load_splat 0
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint double* %p to i32
   %r = add nsw i32 %q, 16
   %s = inttoptr i32 %r to double*
@@ -2601,27 +2846,29 @@ define <2 x double> @load_splat_v2f64_with_unfolded_offset(double* %p) {
   ret <2 x double> %v2
 }
 
-; CHECK-LABEL: load_v2f64_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v2f64_with_unfolded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x double> @load_v2f64_with_unfolded_gep_offset(<2 x double>* %p) {
+; CHECK-LABEL: load_v2f64_with_unfolded_gep_offset:
+; CHECK:         .functype load_v2f64_with_unfolded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v128.load 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr <2 x double>, <2 x double>* %p, i32 1
   %v = load <2 x double>, <2 x double>* %s
   ret <2 x double> %v
 }
 
-; CHECK-LABEL: load_splat_v2f64_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v2f64_with_unfolded_gep_offset (i32) -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
-; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x double> @load_splat_v2f64_with_unfolded_gep_offset(double* %p) {
+; CHECK-LABEL: load_splat_v2f64_with_unfolded_gep_offset:
+; CHECK:         .functype load_splat_v2f64_with_unfolded_gep_offset (i32) -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    i32.const 8
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    v64x2.load_splat 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr double, double* %p, i32 1
   %e = load double, double* %s
   %v1 = insertelement <2 x double> undef, double %e, i32 0
@@ -2629,25 +2876,25 @@ define <2 x double> @load_splat_v2f64_with_unfolded_gep_offset(double* %p) {
   ret <2 x double> %v2
 }
 
-; CHECK-LABEL: load_v2f64_from_numeric_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v2f64_from_numeric_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x double> @load_v2f64_from_numeric_address() {
+; CHECK-LABEL: load_v2f64_from_numeric_address:
+; CHECK:         .functype load_v2f64_from_numeric_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v128.load 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to <2 x double>*
   %v = load <2 x double>, <2 x double>* %s
   ret <2 x double> %v
 }
 
-; CHECK-LABEL: load_splat_v2f64_from_numeric_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v2f64_from_numeric_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x double> @load_splat_v2f64_from_numeric_address() {
+; CHECK-LABEL: load_splat_v2f64_from_numeric_address:
+; CHECK:         .functype load_splat_v2f64_from_numeric_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v64x2.load_splat 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to double*
   %e = load double, double* %s
   %v1 = insertelement <2 x double> undef, double %e, i32 0
@@ -2655,46 +2902,52 @@ define <2 x double> @load_splat_v2f64_from_numeric_address() {
   ret <2 x double> %v2
 }
 
-; CHECK-LABEL: load_v2f64_from_global_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_v2f64_from_global_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, gv_v2f64($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 @gv_v2f64 = global <2 x double> <double 42., double 42.>
 define <2 x double> @load_v2f64_from_global_address() {
+; CHECK-LABEL: load_v2f64_from_global_address:
+; CHECK:         .functype load_v2f64_from_global_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v128.load gv_v2f64
+; CHECK-NEXT:    # fallthrough-return
   %v = load <2 x double>, <2 x double>* @gv_v2f64
   ret <2 x double> %v
 }
 
-; CHECK-LABEL: load_splat_v2f64_from_global_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype load_splat_v2f64_from_global_address () -> (v128){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, gv_f64($pop[[L0]]){{$}}
-; SIMD128-NEXT: return $pop[[R]]{{$}}
 @gv_f64 = global double 42.
 define <2 x double> @load_splat_v2f64_from_global_address() {
+; CHECK-LABEL: load_splat_v2f64_from_global_address:
+; CHECK:         .functype load_splat_v2f64_from_global_address () -> (v128)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    v64x2.load_splat gv_f64
+; CHECK-NEXT:    # fallthrough-return
   %e = load double, double* @gv_f64
   %v1 = insertelement <2 x double> undef, double %e, i32 0
   %v2 = shufflevector <2 x double> %v1, <2 x double> undef, <2 x i32> zeroinitializer
   ret <2 x double> %v2
 }
 
-; CHECK-LABEL: store_v2f64:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v2f64 (v128, i32) -> (){{$}}
-; SIMD128-NEXT: v128.store 0($1), $0{{$}}
 define void @store_v2f64(<2 x double> %v, <2 x double>* %p) {
+; CHECK-LABEL: store_v2f64:
+; CHECK:         .functype store_v2f64 (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 0
+; CHECK-NEXT:    # fallthrough-return
   store <2 x double> %v , <2 x double>* %p
   ret void
 }
 
-; CHECK-LABEL: store_v2f64_with_folded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v2f64_with_folded_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: v128.store 16($1), $0{{$}}
 define void @store_v2f64_with_folded_offset(<2 x double> %v, <2 x double>* %p) {
+; CHECK-LABEL: store_v2f64_with_folded_offset:
+; CHECK:         .functype store_v2f64_with_folded_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 16
+; CHECK-NEXT:    # fallthrough-return
   %q = ptrtoint <2 x double>* %p to i32
   %r = add nuw i32 %q, 16
   %s = inttoptr i32 %r to <2 x double>*
@@ -2702,69 +2955,87 @@ define void @store_v2f64_with_folded_offset(<2 x double> %v, <2 x double>* %p) {
   ret void
 }
 
-; CHECK-LABEL: store_v2f64_with_folded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v2f64_with_folded_gep_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: v128.store 16($1), $0{{$}}
 define void @store_v2f64_with_folded_gep_offset(<2 x double> %v, <2 x double>* %p) {
+; CHECK-LABEL: store_v2f64_with_folded_gep_offset:
+; CHECK:         .functype store_v2f64_with_folded_gep_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 16
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <2 x double>, <2 x double>* %p, i32 1
   store <2 x double> %v , <2 x double>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v2f64_with_unfolded_gep_negative_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v2f64_with_unfolded_gep_negative_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -16{{$}}
-; SIMD128-NEXT: i32.add $push[[R:[0-9]+]]=, $1, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.store 0($pop[[R]]), $0{{$}}
 define void @store_v2f64_with_unfolded_gep_negative_offset(<2 x double> %v, <2 x double>* %p) {
+; CHECK-LABEL: store_v2f64_with_unfolded_gep_negative_offset:
+; CHECK:         .functype store_v2f64_with_unfolded_gep_negative_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const -16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr inbounds <2 x double>, <2 x double>* %p, i32 -1
   store <2 x double> %v , <2 x double>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v2f64_with_unfolded_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v2f64_with_unfolded_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -16{{$}}
-; SIMD128-NEXT: i32.add $push[[R:[0-9]+]]=, $1, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.store 0($pop[[R]]), $0{{$}}
 define void @store_v2f64_with_unfolded_offset(<2 x double> %v, <2 x double>* %p) {
-  %s = getelementptr inbounds <2 x double>, <2 x double>* %p, i32 -1
+; CHECK-LABEL: store_v2f64_with_unfolded_offset:
+; CHECK:         .functype store_v2f64_with_unfolded_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 0
+; CHECK-NEXT:    # fallthrough-return
+  %q = ptrtoint <2 x double>* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to <2 x double>*
   store <2 x double> %v , <2 x double>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v2f64_with_unfolded_gep_offset:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v2f64_with_unfolded_gep_offset (v128, i32) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.add $push[[R:[0-9]+]]=, $1, $pop[[L0]]{{$}}
-; SIMD128-NEXT: v128.store 0($pop[[R]]), $0{{$}}
 define void @store_v2f64_with_unfolded_gep_offset(<2 x double> %v, <2 x double>* %p) {
+; CHECK-LABEL: store_v2f64_with_unfolded_gep_offset:
+; CHECK:         .functype store_v2f64_with_unfolded_gep_offset (v128, i32) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    local.get 1
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 0
+; CHECK-NEXT:    # fallthrough-return
   %s = getelementptr <2 x double>, <2 x double>* %p, i32 1
   store <2 x double> %v , <2 x double>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v2f64_to_numeric_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v2f64_to_numeric_address (v128) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v128.store 32($pop[[L0]]), $0{{$}}
 define void @store_v2f64_to_numeric_address(<2 x double> %v) {
+; CHECK-LABEL: store_v2f64_to_numeric_address:
+; CHECK:         .functype store_v2f64_to_numeric_address (v128) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store 32
+; CHECK-NEXT:    # fallthrough-return
   %s = inttoptr i32 32 to <2 x double>*
   store <2 x double> %v , <2 x double>* %s
   ret void
 }
 
-; CHECK-LABEL: store_v2f64_to_global_address:
-; NO-SIMD128-NOT: v128
-; SIMD128-NEXT: .functype store_v2f64_to_global_address (v128) -> (){{$}}
-; SIMD128-NEXT: i32.const $push[[R:[0-9]+]]=, 0{{$}}
-; SIMD128-NEXT: v128.store gv_v2f64($pop[[R]]), $0{{$}}
 define void @store_v2f64_to_global_address(<2 x double> %v) {
+; CHECK-LABEL: store_v2f64_to_global_address:
+; CHECK:         .functype store_v2f64_to_global_address (v128) -> ()
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const 0
+; CHECK-NEXT:    local.get 0
+; CHECK-NEXT:    v128.store gv_v2f64
+; CHECK-NEXT:    # fallthrough-return
   store <2 x double> %v , <2 x double>* @gv_v2f64
   ret void
 }
diff --git a/llvm/test/CodeGen/X86/abs.ll b/llvm/test/CodeGen/X86/abs.ll
new file mode 100644
index 0000000000000..345830676abaa
--- /dev/null
+++ b/llvm/test/CodeGen/X86/abs.ll
@@ -0,0 +1,618 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s --check-prefixes=X64,SSE
+; RUN: llc < %s -mtriple=x86_64-linux -mattr=avx | FileCheck %s --check-prefixes=X64,AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-linux -mattr=avx2 | FileCheck %s --check-prefixes=X64,AVX,AVX2
+; RUN: llc < %s -mtriple=i686 -mattr=cmov | FileCheck %s --check-prefix=X86
+
+; The i1 parameter is not codegen-relevant right now.
+
+declare i8 @llvm.abs.i8(i8, i1)
+declare i16 @llvm.abs.i16(i16, i1)
+declare i24 @llvm.abs.i24(i24, i1)
+declare i32 @llvm.abs.i32(i32, i1)
+declare i64 @llvm.abs.i64(i64, i1)
+declare i128 @llvm.abs.i128(i128, i1)
+
+declare <1 x i32> @llvm.abs.v1i32(<1 x i32>, i1)
+declare <2 x i32> @llvm.abs.v2i32(<2 x i32>, i1)
+declare <3 x i32> @llvm.abs.v3i32(<3 x i32>, i1)
+declare <4 x i32> @llvm.abs.v4i32(<4 x i32>, i1)
+declare <8 x i32> @llvm.abs.v8i32(<8 x i32>, i1)
+
+declare <8 x i16> @llvm.abs.v8i16(<8 x i16>, i1)
+declare <16 x i8> @llvm.abs.v16i8(<16 x i8>, i1)
+
+define i8 @test_i8(i8 %a) nounwind {
+; X64-LABEL: test_i8:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    sarb $7, %cl
+; X64-NEXT:    leal (%rdi,%rcx), %eax
+; X64-NEXT:    xorb %cl, %al
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i8:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    movl %eax, %ecx
+; X86-NEXT:    sarb $7, %cl
+; X86-NEXT:    addb %cl, %al
+; X86-NEXT:    xorb %cl, %al
+; X86-NEXT:    retl
+  %r = call i8 @llvm.abs.i8(i8 %a, i1 false)
+  ret i8 %r
+}
+
+define i16 @test_i16(i16 %a) nounwind {
+; X64-LABEL: test_i16:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    negw %ax
+; X64-NEXT:    cmovlw %di, %ax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i16:
+; X86:       # %bb.0:
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    negw %ax
+; X86-NEXT:    cmovlw %cx, %ax
+; X86-NEXT:    retl
+  %r = call i16 @llvm.abs.i16(i16 %a, i1 false)
+  ret i16 %r
+}
+
+define i24 @test_i24(i24 %a) nounwind {
+; X64-LABEL: test_i24:
+; X64:       # %bb.0:
+; X64-NEXT:    shll $8, %edi
+; X64-NEXT:    sarl $8, %edi
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    negl %eax
+; X64-NEXT:    cmovll %edi, %eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i24:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    shll $8, %ecx
+; X86-NEXT:    sarl $8, %ecx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    negl %eax
+; X86-NEXT:    cmovll %ecx, %eax
+; X86-NEXT:    retl
+  %r = call i24 @llvm.abs.i24(i24 %a, i1 false)
+  ret i24 %r
+}
+
+define i32 @test_i32(i32 %a) nounwind {
+; X64-LABEL: test_i32:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    negl %eax
+; X64-NEXT:    cmovll %edi, %eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    negl %eax
+; X86-NEXT:    cmovll %ecx, %eax
+; X86-NEXT:    retl
+  %r = call i32 @llvm.abs.i32(i32 %a, i1 false)
+  ret i32 %r
+}
+
+define i64 @test_i64(i64 %a) nounwind {
+; X64-LABEL: test_i64:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    negq %rax
+; X64-NEXT:    cmovlq %rdi, %rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i64:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %edx, %ecx
+; X86-NEXT:    sarl $31, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    addl %ecx, %eax
+; X86-NEXT:    adcl %ecx, %edx
+; X86-NEXT:    xorl %ecx, %edx
+; X86-NEXT:    xorl %ecx, %eax
+; X86-NEXT:    retl
+  %r = call i64 @llvm.abs.i64(i64 %a, i1 false)
+  ret i64 %r
+}
+
+define i128 @test_i128(i128 %a) nounwind {
+; X64-LABEL: test_i128:
+; X64:       # %bb.0:
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    negq %rax
+; X64-NEXT:    sbbq %rsi, %rdx
+; X64-NEXT:    testq %rsi, %rsi
+; X64-NEXT:    cmovnsq %rdi, %rax
+; X64-NEXT:    cmovnsq %rsi, %rdx
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i128:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    xorl %esi, %esi
+; X86-NEXT:    negl %edi
+; X86-NEXT:    movl $0, %ebx
+; X86-NEXT:    sbbl %edx, %ebx
+; X86-NEXT:    movl $0, %ebp
+; X86-NEXT:    sbbl %ecx, %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    sbbl %eax, %esi
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    cmovnsl %eax, %esi
+; X86-NEXT:    cmovnsl %ecx, %ebp
+; X86-NEXT:    cmovnsl %edx, %ebx
+; X86-NEXT:    cmovnsl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %edi, (%eax)
+; X86-NEXT:    movl %ebx, 4(%eax)
+; X86-NEXT:    movl %ebp, 8(%eax)
+; X86-NEXT:    movl %esi, 12(%eax)
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+  %r = call i128 @llvm.abs.i128(i128 %a, i1 false)
+  ret i128 %r
+}
+
+define <1 x i32> @test_v1i32(<1 x i32> %a) nounwind {
+; X64-LABEL: test_v1i32:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    negl %eax
+; X64-NEXT:    cmovll %edi, %eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_v1i32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    negl %eax
+; X86-NEXT:    cmovll %ecx, %eax
+; X86-NEXT:    retl
+  %r = call <1 x i32> @llvm.abs.v1i32(<1 x i32> %a, i1 false)
+  ret <1 x i32> %r
+}
+
+define <2 x i32> @test_v2i32(<2 x i32> %a) nounwind {
+; SSE-LABEL: test_v2i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa %xmm0, %xmm1
+; SSE-NEXT:    psrad $31, %xmm1
+; SSE-NEXT:    paddd %xmm1, %xmm0
+; SSE-NEXT:    pxor %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v2i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpabsd %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_v2i32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %edx, %eax
+; X86-NEXT:    negl %eax
+; X86-NEXT:    cmovll %edx, %eax
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    negl %edx
+; X86-NEXT:    cmovll %ecx, %edx
+; X86-NEXT:    retl
+  %r = call <2 x i32> @llvm.abs.v2i32(<2 x i32> %a, i1 false)
+  ret <2 x i32> %r
+}
+
+define <3 x i32> @test_v3i32(<3 x i32> %a) nounwind {
+; SSE-LABEL: test_v3i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa %xmm0, %xmm1
+; SSE-NEXT:    psrad $31, %xmm1
+; SSE-NEXT:    paddd %xmm1, %xmm0
+; SSE-NEXT:    pxor %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v3i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpabsd %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_v3i32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %edx, %eax
+; X86-NEXT:    negl %eax
+; X86-NEXT:    cmovll %edx, %eax
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    negl %edx
+; X86-NEXT:    cmovll %ecx, %edx
+; X86-NEXT:    movl %esi, %ecx
+; X86-NEXT:    negl %ecx
+; X86-NEXT:    cmovll %esi, %ecx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+  %r = call <3 x i32> @llvm.abs.v3i32(<3 x i32> %a, i1 false)
+  ret <3 x i32> %r
+}
+
+define <4 x i32> @test_v4i32(<4 x i32> %a) nounwind {
+; SSE-LABEL: test_v4i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa %xmm0, %xmm1
+; SSE-NEXT:    psrad $31, %xmm1
+; SSE-NEXT:    paddd %xmm1, %xmm0
+; SSE-NEXT:    pxor %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v4i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpabsd %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_v4i32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, %ebx
+; X86-NEXT:    negl %ebx
+; X86-NEXT:    cmovll %edi, %ebx
+; X86-NEXT:    movl %esi, %edi
+; X86-NEXT:    negl %edi
+; X86-NEXT:    cmovll %esi, %edi
+; X86-NEXT:    movl %edx, %esi
+; X86-NEXT:    negl %esi
+; X86-NEXT:    cmovll %edx, %esi
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    negl %edx
+; X86-NEXT:    cmovll %ecx, %edx
+; X86-NEXT:    movl %edx, 12(%eax)
+; X86-NEXT:    movl %esi, 8(%eax)
+; X86-NEXT:    movl %edi, 4(%eax)
+; X86-NEXT:    movl %ebx, (%eax)
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    retl $4
+  %r = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a, i1 false)
+  ret <4 x i32> %r
+}
+
+define <8 x i32> @test_v8i32(<8 x i32> %a) nounwind {
+; SSE-LABEL: test_v8i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa %xmm0, %xmm2
+; SSE-NEXT:    psrad $31, %xmm2
+; SSE-NEXT:    paddd %xmm2, %xmm0
+; SSE-NEXT:    pxor %xmm2, %xmm0
+; SSE-NEXT:    movdqa %xmm1, %xmm2
+; SSE-NEXT:    psrad $31, %xmm2
+; SSE-NEXT:    paddd %xmm2, %xmm1
+; SSE-NEXT:    pxor %xmm2, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: test_v8i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpabsd %xmm0, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpabsd %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test_v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpabsd %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; X86-LABEL: test_v8i32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %edx, %ecx
+; X86-NEXT:    negl %ecx
+; X86-NEXT:    cmovll %edx, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %ebp, %ecx
+; X86-NEXT:    negl %ecx
+; X86-NEXT:    cmovll %ebp, %ecx
+; X86-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; X86-NEXT:    movl %ebx, %ebp
+; X86-NEXT:    negl %ebp
+; X86-NEXT:    cmovll %ebx, %ebp
+; X86-NEXT:    movl %edi, %ebx
+; X86-NEXT:    negl %ebx
+; X86-NEXT:    cmovll %edi, %ebx
+; X86-NEXT:    movl %esi, %edi
+; X86-NEXT:    negl %edi
+; X86-NEXT:    cmovll %esi, %edi
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    negl %esi
+; X86-NEXT:    cmovll %eax, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    negl %eax
+; X86-NEXT:    cmovll %ecx, %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %edx, %ecx
+; X86-NEXT:    negl %ecx
+; X86-NEXT:    cmovll %edx, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %ecx, 28(%edx)
+; X86-NEXT:    movl %eax, 24(%edx)
+; X86-NEXT:    movl %esi, 20(%edx)
+; X86-NEXT:    movl %edi, 16(%edx)
+; X86-NEXT:    movl %ebx, 12(%edx)
+; X86-NEXT:    movl %ebp, 8(%edx)
+; X86-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, 4(%edx)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, (%edx)
+; X86-NEXT:    movl %edx, %eax
+; X86-NEXT:    addl $8, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+  %r = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a, i1 false)
+  ret <8 x i32> %r
+}
+
+define <8 x i16> @test_v8i16(<8 x i16> %a) nounwind {
+; SSE-LABEL: test_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa %xmm0, %xmm1
+; SSE-NEXT:    psraw $15, %xmm1
+; SSE-NEXT:    paddw %xmm1, %xmm0
+; SSE-NEXT:    pxor %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpabsw %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_v8i16:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %edx, %ecx
+; X86-NEXT:    negw %cx
+; X86-NEXT:    cmovlw %dx, %cx
+; X86-NEXT:    movw %cx, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
+; X86-NEXT:    movl %ebp, %ecx
+; X86-NEXT:    negw %cx
+; X86-NEXT:    cmovlw %bp, %cx
+; X86-NEXT:    movw %cx, (%esp) # 2-byte Spill
+; X86-NEXT:    movl %ebx, %ebp
+; X86-NEXT:    negw %bp
+; X86-NEXT:    cmovlw %bx, %bp
+; X86-NEXT:    movl %edi, %ebx
+; X86-NEXT:    negw %bx
+; X86-NEXT:    cmovlw %di, %bx
+; X86-NEXT:    movl %esi, %edi
+; X86-NEXT:    negw %di
+; X86-NEXT:    cmovlw %si, %di
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    negw %si
+; X86-NEXT:    cmovlw %ax, %si
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    negw %ax
+; X86-NEXT:    cmovlw %cx, %ax
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %edx, %ecx
+; X86-NEXT:    negw %cx
+; X86-NEXT:    cmovlw %dx, %cx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movw %cx, 14(%edx)
+; X86-NEXT:    movw %ax, 12(%edx)
+; X86-NEXT:    movw %si, 10(%edx)
+; X86-NEXT:    movw %di, 8(%edx)
+; X86-NEXT:    movw %bx, 6(%edx)
+; X86-NEXT:    movw %bp, 4(%edx)
+; X86-NEXT:    movzwl (%esp), %eax # 2-byte Folded Reload
+; X86-NEXT:    movw %ax, 2(%edx)
+; X86-NEXT:    movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
+; X86-NEXT:    movw %ax, (%edx)
+; X86-NEXT:    movl %edx, %eax
+; X86-NEXT:    addl $4, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+  %r = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a, i1 false)
+  ret <8 x i16> %r
+}
+
+define <16 x i8> @test_v16i8(<16 x i8> %a) nounwind {
+; SSE-LABEL: test_v16i8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    pcmpgtb %xmm0, %xmm1
+; SSE-NEXT:    paddb %xmm1, %xmm0
+; SSE-NEXT:    pxor %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v16i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpabsb %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_v16i8:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %bh
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %bl
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %dh
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %ch
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %ah
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %dl
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    movb %cl, %al
+; X86-NEXT:    sarb $7, %al
+; X86-NEXT:    addb %al, %cl
+; X86-NEXT:    xorb %al, %cl
+; X86-NEXT:    movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; X86-NEXT:    movb %dl, %al
+; X86-NEXT:    sarb $7, %al
+; X86-NEXT:    addb %al, %dl
+; X86-NEXT:    xorb %al, %dl
+; X86-NEXT:    movb %dl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; X86-NEXT:    movb %ah, %al
+; X86-NEXT:    sarb $7, %al
+; X86-NEXT:    addb %al, %ah
+; X86-NEXT:    xorb %al, %ah
+; X86-NEXT:    movb %ah, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; X86-NEXT:    movb %ch, %al
+; X86-NEXT:    sarb $7, %al
+; X86-NEXT:    addb %al, %ch
+; X86-NEXT:    xorb %al, %ch
+; X86-NEXT:    movb %ch, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; X86-NEXT:    movb %dh, %al
+; X86-NEXT:    sarb $7, %al
+; X86-NEXT:    addb %al, %dh
+; X86-NEXT:    xorb %al, %dh
+; X86-NEXT:    movb %dh, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; X86-NEXT:    movl %ebx, %eax
+; X86-NEXT:    sarb $7, %al
+; X86-NEXT:    addb %al, %bl
+; X86-NEXT:    xorb %al, %bl
+; X86-NEXT:    movb %bl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; X86-NEXT:    movb %bh, %al
+; X86-NEXT:    sarb $7, %al
+; X86-NEXT:    addb %al, %bh
+; X86-NEXT:    xorb %al, %bh
+; X86-NEXT:    movb %bh, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    sarb $7, %al
+; X86-NEXT:    addb %al, %cl
+; X86-NEXT:    xorb %al, %cl
+; X86-NEXT:    movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    sarb $7, %al
+; X86-NEXT:    addb %al, %cl
+; X86-NEXT:    xorb %al, %cl
+; X86-NEXT:    movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %bh
+; X86-NEXT:    movb %bh, %al
+; X86-NEXT:    sarb $7, %al
+; X86-NEXT:    addb %al, %bh
+; X86-NEXT:    xorb %al, %bh
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %bl
+; X86-NEXT:    movl %ebx, %eax
+; X86-NEXT:    sarb $7, %al
+; X86-NEXT:    addb %al, %bl
+; X86-NEXT:    xorb %al, %bl
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %dh
+; X86-NEXT:    movb %dh, %al
+; X86-NEXT:    sarb $7, %al
+; X86-NEXT:    addb %al, %dh
+; X86-NEXT:    xorb %al, %dh
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %ch
+; X86-NEXT:    movb %ch, %al
+; X86-NEXT:    sarb $7, %al
+; X86-NEXT:    addb %al, %ch
+; X86-NEXT:    xorb %al, %ch
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %dl
+; X86-NEXT:    movl %edx, %eax
+; X86-NEXT:    sarb $7, %al
+; X86-NEXT:    addb %al, %dl
+; X86-NEXT:    xorb %al, %dl
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    sarb $7, %al
+; X86-NEXT:    addb %al, %cl
+; X86-NEXT:    xorb %al, %cl
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    movb %al, %ah
+; X86-NEXT:    sarb $7, %ah
+; X86-NEXT:    addb %ah, %al
+; X86-NEXT:    xorb %ah, %al
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movb %al, 15(%esi)
+; X86-NEXT:    movb %cl, 14(%esi)
+; X86-NEXT:    movb %dl, 13(%esi)
+; X86-NEXT:    movb %ch, 12(%esi)
+; X86-NEXT:    movb %dh, 11(%esi)
+; X86-NEXT:    movb %bl, 10(%esi)
+; X86-NEXT:    movb %bh, 9(%esi)
+; X86-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; X86-NEXT:    movb %al, 8(%esi)
+; X86-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; X86-NEXT:    movb %al, 7(%esi)
+; X86-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; X86-NEXT:    movb %al, 6(%esi)
+; X86-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; X86-NEXT:    movb %al, 5(%esi)
+; X86-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; X86-NEXT:    movb %al, 4(%esi)
+; X86-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; X86-NEXT:    movb %al, 3(%esi)
+; X86-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; X86-NEXT:    movb %al, 2(%esi)
+; X86-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; X86-NEXT:    movb %al, 1(%esi)
+; X86-NEXT:    movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; X86-NEXT:    movb %al, (%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    retl $4
+  %r = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a, i1 false)
+  ret <16 x i8> %r
+}
diff --git a/llvm/test/CodeGen/X86/byref.ll b/llvm/test/CodeGen/X86/byref.ll
new file mode 100644
index 0000000000000..90d5e9901b67f
--- /dev/null
+++ b/llvm/test/CodeGen/X86/byref.ll
@@ -0,0 +1,20 @@
+; RUN: llc < %s -mtriple=i686-pc-win32 | FileCheck %s
+
+%Foo = type { i32, i32 }
+
+declare x86_stdcallcc void @foo_byref_stdcall_p(%Foo* byref(%Foo))
+declare x86_stdcallcc void @i(i32)
+
+; byref does not imply a stack copy, so this should append 4 bytes,
+; not 8.
+define void @stdcall(%Foo* %value) {
+; CHECK-LABEL: _stdcall:
+; CHECK: pushl 4(%esp)
+; CHECK: calll _foo_byref_stdcall_p@4
+  call x86_stdcallcc void @foo_byref_stdcall_p(%Foo* byref(%Foo) %value)
+; CHECK-NOT: %esp
+; CHECK: pushl
+; CHECK: calll _i@4
+  call x86_stdcallcc void @i(i32 0)
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/combine-sdiv.ll b/llvm/test/CodeGen/X86/combine-sdiv.ll
index a221e42ade42d..1a52ebfc6cd90 100644
--- a/llvm/test/CodeGen/X86/combine-sdiv.ll
+++ b/llvm/test/CodeGen/X86/combine-sdiv.ll
@@ -2676,11 +2676,11 @@ define <8 x i16> @combine_vec_sdiv_nonuniform5(<8 x i16> %x) {
 ; SSE2-NEXT:    pandn %xmm3, %xmm1
 ; SSE2-NEXT:    por %xmm2, %xmm1
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [65535,65535,0,65535,65535,65535,0,65535]
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    psraw $4, %xmm1
-; SSE2-NEXT:    pandn %xmm1, %xmm2
-; SSE2-NEXT:    por %xmm3, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    psraw $4, %xmm3
+; SSE2-NEXT:    pandn %xmm3, %xmm2
+; SSE2-NEXT:    por %xmm1, %xmm2
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [65535,0,65535,65535,65535,65535,0,65535]
 ; SSE2-NEXT:    movdqa %xmm2, %xmm3
 ; SSE2-NEXT:    pand %xmm1, %xmm3
@@ -2787,27 +2787,27 @@ define <8 x i16> @combine_vec_sdiv_nonuniform6(<8 x i16> %x) {
 ; SSE2-NEXT:    pand %xmm1, %xmm2
 ; SSE2-NEXT:    pandn %xmm0, %xmm1
 ; SSE2-NEXT:    por %xmm2, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [0,65535,65535,65535,65535,0,65535,0]
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    psraw $4, %xmm1
-; SSE2-NEXT:    pandn %xmm1, %xmm2
-; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [0,65535,65535,65535,65535,0,65535,65535]
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    pand %xmm1, %xmm3
-; SSE2-NEXT:    psraw $2, %xmm2
-; SSE2-NEXT:    pandn %xmm2, %xmm1
-; SSE2-NEXT:    por %xmm3, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [65535,65535,65535,65535,65535,0,65535,0]
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    psraw $1, %xmm1
-; SSE2-NEXT:    pandn %xmm1, %xmm2
-; SSE2-NEXT:    por %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psraw $6, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [0,65535,65535,65535,65535,0,65535,65535]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [0,65535,65535,65535,65535,0,65535,0]
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, %xmm5
+; SSE2-NEXT:    psraw $12, %xmm5
+; SSE2-NEXT:    pandn %xmm5, %xmm4
+; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    pand %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm2, %xmm3
+; SSE2-NEXT:    por %xmm4, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [65535,65535,65535,65535,65535,0,65535,0]
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    psraw $1, %xmm3
+; SSE2-NEXT:    pandn %xmm3, %xmm1
+; SSE2-NEXT:    por %xmm2, %xmm1
 ; SSE2-NEXT:    psrlw $15, %xmm0
 ; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    paddw %xmm2, %xmm0
+; SSE2-NEXT:    paddw %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: combine_vec_sdiv_nonuniform6:
diff --git a/llvm/test/CodeGen/X86/combine-sra.ll b/llvm/test/CodeGen/X86/combine-sra.ll
index c22d6b7e7ddcd..e2f3f2dc7523f 100644
--- a/llvm/test/CodeGen/X86/combine-sra.ll
+++ b/llvm/test/CodeGen/X86/combine-sra.ll
@@ -196,14 +196,13 @@ define <4 x i32> @combine_vec_ashr_trunc_lshr(<4 x i64> %x) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
 ; SSE-NEXT:    movaps %xmm0, %xmm2
-; SSE-NEXT:    movaps %xmm0, %xmm1
-; SSE-NEXT:    psrad $2, %xmm1
+; SSE-NEXT:    psrad $2, %xmm2
+; SSE-NEXT:    pblendw {{.*#+}} xmm2 = xmm0[0,1,2,3],xmm2[4,5,6,7]
+; SSE-NEXT:    psrad $1, %xmm0
+; SSE-NEXT:    psrad $3, %xmm1
 ; SSE-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1,2,3],xmm1[4,5,6,7]
-; SSE-NEXT:    psrad $3, %xmm0
-; SSE-NEXT:    psrad $1, %xmm2
-; SSE-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm0[4,5,6,7]
-; SSE-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; SSE-NEXT:    movdqa %xmm2, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: combine_vec_ashr_trunc_lshr:
@@ -235,14 +234,13 @@ define <4 x i32> @combine_vec_ashr_trunc_ashr(<4 x i64> %x) {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
 ; SSE-NEXT:    movaps %xmm0, %xmm2
-; SSE-NEXT:    movaps %xmm0, %xmm1
-; SSE-NEXT:    psrad $2, %xmm1
+; SSE-NEXT:    psrad $2, %xmm2
+; SSE-NEXT:    pblendw {{.*#+}} xmm2 = xmm0[0,1,2,3],xmm2[4,5,6,7]
+; SSE-NEXT:    psrad $1, %xmm0
+; SSE-NEXT:    psrad $3, %xmm1
 ; SSE-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1,2,3],xmm1[4,5,6,7]
-; SSE-NEXT:    psrad $3, %xmm0
-; SSE-NEXT:    psrad $1, %xmm2
-; SSE-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm0[4,5,6,7]
-; SSE-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; SSE-NEXT:    movdqa %xmm2, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: combine_vec_ashr_trunc_ashr:
diff --git a/llvm/test/CodeGen/X86/combine-srl.ll b/llvm/test/CodeGen/X86/combine-srl.ll
index fe7783b1076a2..36fbdaf537025 100644
--- a/llvm/test/CodeGen/X86/combine-srl.ll
+++ b/llvm/test/CodeGen/X86/combine-srl.ll
@@ -188,18 +188,17 @@ define <4 x i32> @combine_vec_lshr_trunc_lshr1(<4 x i64> %x) {
 ; SSE-NEXT:    movdqa %xmm0, %xmm2
 ; SSE-NEXT:    psrlq $33, %xmm2
 ; SSE-NEXT:    psrlq $32, %xmm0
-; SSE-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
-; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    pblendw {{.*#+}} xmm2 = xmm0[0,1,2,3],xmm2[4,5,6,7]
+; SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2],xmm1[0,2]
+; SSE-NEXT:    movaps %xmm2, %xmm1
 ; SSE-NEXT:    psrld $19, %xmm1
-; SSE-NEXT:    movaps %xmm0, %xmm2
-; SSE-NEXT:    psrld $17, %xmm2
-; SSE-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5,6,7]
-; SSE-NEXT:    movaps %xmm0, %xmm1
-; SSE-NEXT:    psrld $18, %xmm1
+; SSE-NEXT:    movaps %xmm2, %xmm3
+; SSE-NEXT:    psrld $17, %xmm3
+; SSE-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm1[4,5,6,7]
+; SSE-NEXT:    psrld $18, %xmm2
 ; SSE-NEXT:    psrld $16, %xmm0
-; SSE-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
-; SSE-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; SSE-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
+; SSE-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
 ; SSE-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: combine_vec_lshr_trunc_lshr1:
diff --git a/llvm/test/CodeGen/X86/dup-cost.ll b/llvm/test/CodeGen/X86/dup-cost.ll
new file mode 100644
index 0000000000000..020250b7ad434
--- /dev/null
+++ b/llvm/test/CodeGen/X86/dup-cost.ll
@@ -0,0 +1,105 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
+
+; Cold function, %dup should not be duplicated into predecessors.
+define i32 @cold(i32 %a, i32* %p, i32* %q) !prof !21 {
+; CHECK-LABEL: cold
+; CHECK:       %entry
+; CHECK:       %true1
+; CHECK:       %dup
+; CHECK:       %true2
+; CHECK:       %false1
+; CHECK:       %false2
+entry:
+  %cond1 = icmp sgt i32 %a, 1
+  br i1 %cond1, label %true1, label %false1, !prof !30
+
+true1:
+  %v1 = load i32, i32* %p, align 4
+  %v2 = add i32 %v1, 2
+  br label %dup
+
+false1:
+  %v3 = load i32, i32* %q, align 4
+  %v4 = sub i32 %v3, 3
+  br label %dup
+
+dup:
+  %v5 = phi i32 [%v2, %true1], [%v4, %false1]
+  %cond2 = icmp sgt i32 %v5, 4
+  br i1 %cond2, label %true2, label %false2, !prof !30
+
+true2:
+  %v6 = xor i32 %v5, %a
+  br label %exit
+
+false2:
+  %v7 = and i32 %v5, %a
+  br label %exit
+
+exit:
+  %v8 = phi i32 [%v6, %true2], [%v7, %false2]
+  ret i32 %v8
+}
+
+; Same code as previous function, but with hot profile count.
+; So %dup should be duplicated into predecessors.
+define i32 @hot(i32 %a, i32* %p, i32* %q) !prof !22 {
+; CHECK-LABEL: hot
+; CHECK:       %entry
+; CHECK:       %true1
+; CHECK:       %false2
+; CHECK:       %false1
+; CHECK:       %true2
+entry:
+  %cond1 = icmp sgt i32 %a, 1
+  br i1 %cond1, label %true1, label %false1, !prof !30
+
+true1:
+  %v1 = load i32, i32* %p, align 4
+  %v2 = add i32 %v1, 2
+  br label %dup
+
+false1:
+  %v3 = load i32, i32* %q, align 4
+  %v4 = sub i32 %v3, 3
+  br label %dup
+
+dup:
+  %v5 = phi i32 [%v2, %true1], [%v4, %false1]
+  %cond2 = icmp sgt i32 %v5, 4
+  br i1 %cond2, label %true2, label %false2, !prof !30
+
+true2:
+  %v6 = xor i32 %v5, %a
+  br label %exit
+
+false2:
+  %v7 = and i32 %v5, %a
+  br label %exit
+
+exit:
+  %v8 = phi i32 [%v6, %true2], [%v7, %false2]
+  ret i32 %v8
+}
+
+
+!llvm.module.flags = !{!1}
+!21 = !{!"function_entry_count", i64 10}
+!22 = !{!"function_entry_count", i64 400}
+
+!30 = !{!"branch_weights", i32 1, i32 1}
+
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
+!3 = !{!"ProfileFormat", !"InstrProf"}
+!4 = !{!"TotalCount", i64 10000}
+!5 = !{!"MaxCount", i64 10}
+!6 = !{!"MaxInternalCount", i64 1}
+!7 = !{!"MaxFunctionCount", i64 1000}
+!8 = !{!"NumCounts", i64 3}
+!9 = !{!"NumFunctions", i64 3}
+!10 = !{!"DetailedSummary", !11}
+!11 = !{!12, !13, !14}
+!12 = !{i32 10000, i64 100, i32 1}
+!13 = !{i32 999000, i64 100, i32 1}
+!14 = !{i32 999999, i64 1, i32 2}
diff --git a/llvm/test/CodeGen/X86/file-directive.ll b/llvm/test/CodeGen/X86/file-directive.ll
index 4650432f50dd0..35a07b0b4f407 100644
--- a/llvm/test/CodeGen/X86/file-directive.ll
+++ b/llvm/test/CodeGen/X86/file-directive.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -mtriple=x86_64-linux-gnu -filetype=asm < %s | FileCheck %s --check-prefix=DIRECTIVE
-; RUN: llc -mtriple=x86_64-linux-gnu -filetype=obj < %s | llvm-readobj --symbols | FileCheck %s --check-prefix=STT-FILE
+; RUN: llc -mtriple=x86_64-linux-gnu -filetype=obj < %s | llvm-readobj --symbols - | FileCheck %s --check-prefix=STT-FILE
 
 ; DIRECTIVE: .file "foobar"
 ; STT-FILE: Name: foobar
diff --git a/llvm/test/CodeGen/X86/haddsub-4.ll b/llvm/test/CodeGen/X86/haddsub-4.ll
new file mode 100644
index 0000000000000..5c8e9a7c72f2a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/haddsub-4.ll
@@ -0,0 +1,405 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3           | FileCheck %s --check-prefixes=SSE,SSSE3,SSSE3-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3,fast-hops | FileCheck %s --check-prefixes=SSE,SSSE3,SSSE3-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx             | FileCheck %s --check-prefixes=AVX,AVX-SLOW,AVX1,AVX1-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx,fast-hops   | FileCheck %s --check-prefixes=AVX,AVX-FAST,AVX1,AVX1-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2            | FileCheck %s --check-prefixes=AVX,AVX-SLOW,AVX2,AVX2-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2,fast-hops  | FileCheck %s --check-prefixes=AVX,AVX-FAST,AVX2,AVX2-FAST
+
+define <8 x i16> @hadd_reverse_v8i16(<8 x i16> %a0, <8 x i16> %a1) nounwind {
+; SSE-LABEL: hadd_reverse_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [14,15,10,11,6,7,2,3,14,15,10,11,12,13,14,15]
+; SSE-NEXT:    movdqa %xmm1, %xmm3
+; SSE-NEXT:    pshufb %xmm2, %xmm3
+; SSE-NEXT:    movdqa %xmm0, %xmm4
+; SSE-NEXT:    pshufb %xmm2, %xmm4
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm3[0]
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [12,13,8,9,4,5,0,1,8,9,12,13,12,13,14,15]
+; SSE-NEXT:    pshufb %xmm2, %xmm1
+; SSE-NEXT:    pshufb %xmm2, %xmm0
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE-NEXT:    paddw %xmm4, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: hadd_reverse_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = [14,15,10,11,6,7,2,3,14,15,10,11,12,13,14,15]
+; AVX-NEXT:    vpshufb %xmm2, %xmm1, %xmm3
+; AVX-NEXT:    vpshufb %xmm2, %xmm0, %xmm2
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = [12,13,8,9,4,5,0,1,8,9,12,13,12,13,14,15]
+; AVX-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
+; AVX-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-NEXT:    vpaddw %xmm0, %xmm2, %xmm0
+; AVX-NEXT:    retq
+  %lhs = shufflevector <8 x i16> %a0, <8 x i16> %a1, <8 x i32> <i32 7, i32 5, i32 3, i32 1, i32 15, i32 13, i32 11, i32 9>
+  %rhs = shufflevector <8 x i16> %a0, <8 x i16> %a1, <8 x i32> <i32 6, i32 4, i32 2, i32 0, i32 14, i32 12, i32 10, i32 8>
+  %add = add <8 x i16> %lhs, %rhs
+  ret <8 x i16> %add
+}
+
+define <8 x i16> @hadd_reverse2_v8i16(<8 x i16> %a0, <8 x i16> %a1) nounwind {
+; SSE-LABEL: hadd_reverse2_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [14,15,12,13,10,11,8,9,6,7,4,5,2,3,0,1]
+; SSE-NEXT:    pshufb %xmm2, %xmm0
+; SSE-NEXT:    pshufb %xmm2, %xmm1
+; SSE-NEXT:    phaddw %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: hadd_reverse2_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = [14,15,12,13,10,11,8,9,6,7,4,5,2,3,0,1]
+; AVX-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vphaddw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %shuf0 = shufflevector <8 x i16> %a0, <8 x i16> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  %shuf1 = shufflevector <8 x i16> %a1, <8 x i16> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  %lhs = shufflevector <8 x i16> %shuf0, <8 x i16> %shuf1, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %rhs = shufflevector <8 x i16> %shuf0, <8 x i16> %shuf1, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  %add = add <8 x i16> %lhs, %rhs
+  ret <8 x i16> %add
+}
+
+define <16 x i16> @hadd_reverse_v16i16(<16 x i16> %a0, <16 x i16> %a1) nounwind {
+; SSE-LABEL: hadd_reverse_v16i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa %xmm0, %xmm4
+; SSE-NEXT:    movdqa {{.*#+}} xmm0 = [14,15,10,11,6,7,2,3,14,15,10,11,12,13,14,15]
+; SSE-NEXT:    movdqa %xmm2, %xmm5
+; SSE-NEXT:    pshufb %xmm0, %xmm5
+; SSE-NEXT:    movdqa %xmm4, %xmm6
+; SSE-NEXT:    pshufb %xmm0, %xmm6
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm5[0]
+; SSE-NEXT:    movdqa %xmm3, %xmm5
+; SSE-NEXT:    pshufb %xmm0, %xmm5
+; SSE-NEXT:    movdqa %xmm1, %xmm7
+; SSE-NEXT:    pshufb %xmm0, %xmm7
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm7 = xmm7[0],xmm5[0]
+; SSE-NEXT:    movdqa {{.*#+}} xmm0 = [12,13,8,9,4,5,0,1,8,9,12,13,12,13,14,15]
+; SSE-NEXT:    pshufb %xmm0, %xmm2
+; SSE-NEXT:    pshufb %xmm0, %xmm4
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
+; SSE-NEXT:    paddw %xmm6, %xmm4
+; SSE-NEXT:    pshufb %xmm0, %xmm3
+; SSE-NEXT:    pshufb %xmm0, %xmm1
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE-NEXT:    paddw %xmm7, %xmm1
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    movdqa %xmm4, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: hadd_reverse_v16i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [14,15,10,11,6,7,2,3,14,15,10,11,12,13,14,15]
+; AVX1-NEXT:    vpshufb %xmm3, %xmm2, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
+; AVX1-NEXT:    vpshufb %xmm3, %xmm5, %xmm6
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm6[0],xmm4[0]
+; AVX1-NEXT:    vpshufb %xmm3, %xmm1, %xmm6
+; AVX1-NEXT:    vpshufb %xmm3, %xmm0, %xmm3
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm6[0]
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [12,13,8,9,4,5,0,1,8,9,12,13,12,13,14,15]
+; AVX1-NEXT:    vpshufb %xmm6, %xmm2, %xmm2
+; AVX1-NEXT:    vpshufb %xmm6, %xmm5, %xmm5
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm5[0],xmm2[0]
+; AVX1-NEXT:    vpaddw %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpshufb %xmm6, %xmm1, %xmm1
+; AVX1-NEXT:    vpshufb %xmm6, %xmm0, %xmm0
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX1-NEXT:    vpaddw %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: hadd_reverse_v16i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpshuflw {{.*#+}} ymm2 = ymm1[3,1,2,3,4,5,6,7,11,9,10,11,12,13,14,15]
+; AVX2-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,7,5,6,7,8,9,10,11,15,13,14,15]
+; AVX2-NEXT:    vpshuflw {{.*#+}} ymm3 = ymm0[3,1,2,3,4,5,6,7,11,9,10,11,12,13,14,15]
+; AVX2-NEXT:    vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,7,5,6,7,8,9,10,11,15,13,14,15]
+; AVX2-NEXT:    vshufps {{.*#+}} ymm2 = ymm3[2,0],ymm2[2,0],ymm3[6,4],ymm2[6,4]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[2,3,0,1]
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,12,13,8,9,4,5,0,1,u,u,u,u,u,u,u,u,28,29,24,25,20,21,16,17]
+; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[12,13,8,9,4,5,0,1,u,u,u,u,u,u,u,u,28,29,24,25,20,21,16,17,u,u,u,u,u,u,u,u]
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; AVX2-NEXT:    vpaddw %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    retq
+  %lhs = shufflevector <16 x i16> %a0, <16 x i16> %a1, <16 x i32> <i32 15, i32 13, i32 11, i32 9, i32 31, i32 29, i32 27, i32 25, i32 7, i32 5, i32 3, i32 1, i32 23, i32 21, i32 19, i32 17>
+  %rhs = shufflevector <16 x i16> %a0, <16 x i16> %a1, <16 x i32> <i32 14, i32 12, i32 10, i32 8, i32 30, i32 28, i32 26, i32 24, i32 6, i32 4, i32 2, i32 0, i32 22, i32 20, i32 18, i32 16>
+  %add = add <16 x i16> %lhs, %rhs
+  ret <16 x i16> %add
+}
+
+define <16 x i16> @hadd_reverse2_v16i16(<16 x i16> %a0, <16 x i16> %a1) nounwind {
+; SSE-LABEL: hadd_reverse2_v16i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa %xmm0, %xmm4
+; SSE-NEXT:    movdqa {{.*#+}} xmm0 = [14,15,12,13,10,11,8,9,6,7,4,5,2,3,0,1]
+; SSE-NEXT:    pshufb %xmm0, %xmm4
+; SSE-NEXT:    pshufb %xmm0, %xmm1
+; SSE-NEXT:    pshufb %xmm0, %xmm2
+; SSE-NEXT:    phaddw %xmm2, %xmm4
+; SSE-NEXT:    pshufb %xmm0, %xmm3
+; SSE-NEXT:    phaddw %xmm3, %xmm1
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    movdqa %xmm4, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: hadd_reverse2_v16i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [14,15,12,13,10,11,8,9,6,7,4,5,2,3,0,1]
+; AVX1-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpshufb %xmm3, %xmm4, %xmm4
+; AVX1-NEXT:    vphaddw %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vphaddw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: hadd_reverse2_v16i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [14,15,12,13,10,11,8,9,6,7,4,5,2,3,0,1,30,31,28,29,26,27,24,25,22,23,20,21,18,19,16,17]
+; AVX2-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; AVX2-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,0,1]
+; AVX2-NEXT:    vphaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %shuf0 = shufflevector <16 x i16> %a0, <16 x i16> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  %shuf1 = shufflevector <16 x i16> %a1, <16 x i16> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  %lhs = shufflevector <16 x i16> %shuf0, <16 x i16> %shuf1, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+  %rhs = shufflevector <16 x i16> %shuf0, <16 x i16> %shuf1, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
+  %add = add <16 x i16> %lhs, %rhs
+  ret <16 x i16> %add
+}
+
+define <8 x double> @hadd_reverse_v8f64(<8 x double> %a0, <8 x double> %a1) nounwind {
+; SSE-LABEL: hadd_reverse_v8f64:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movapd %xmm1, %xmm8
+; SSE-NEXT:    movapd %xmm0, %xmm9
+; SSE-NEXT:    haddpd %xmm7, %xmm3
+; SSE-NEXT:    haddpd %xmm6, %xmm2
+; SSE-NEXT:    haddpd %xmm5, %xmm8
+; SSE-NEXT:    haddpd %xmm4, %xmm9
+; SSE-NEXT:    movapd %xmm3, %xmm0
+; SSE-NEXT:    movapd %xmm2, %xmm1
+; SSE-NEXT:    movapd %xmm8, %xmm2
+; SSE-NEXT:    movapd %xmm9, %xmm3
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: hadd_reverse_v8f64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm2[2,3,0,1]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; AVX1-NEXT:    vhaddpd %ymm2, %ymm0, %ymm2
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm3[2,3,0,1]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,0,1]
+; AVX1-NEXT:    vhaddpd %ymm0, %ymm1, %ymm0
+; AVX1-NEXT:    vmovapd %ymm2, %ymm1
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: hadd_reverse_v8f64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm4 = ymm2[0,3,2,1]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm5 = ymm0[0,3,2,1]
+; AVX2-NEXT:    vunpckhpd {{.*#+}} ymm4 = ymm5[1],ymm4[1],ymm5[3],ymm4[3]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm5 = ymm3[0,3,2,1]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm6 = ymm1[0,3,2,1]
+; AVX2-NEXT:    vunpckhpd {{.*#+}} ymm5 = ymm6[1],ymm5[1],ymm6[3],ymm5[3]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[2,1,0,3]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,1,0,3]
+; AVX2-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[2],ymm2[2]
+; AVX2-NEXT:    vaddpd %ymm0, %ymm4, %ymm2
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm3[2,1,0,3]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[2,1,0,3]
+; AVX2-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
+; AVX2-NEXT:    vaddpd %ymm0, %ymm5, %ymm0
+; AVX2-NEXT:    vmovapd %ymm2, %ymm1
+; AVX2-NEXT:    retq
+  %lhs = shufflevector <8 x double> %a0, <8 x double> %a1, <8 x i32> <i32 7, i32 15, i32 5, i32 13, i32 3, i32 11, i32 1, i32 9>
+  %rhs = shufflevector <8 x double> %a0, <8 x double> %a1, <8 x i32> <i32 6, i32 14, i32 4, i32 12, i32 2, i32 10, i32 0, i32 8>
+  %fadd = fadd <8 x double> %lhs, %rhs
+  ret <8 x double> %fadd
+}
+
+define <8 x double> @hadd_reverse2_v8f64(<8 x double> %a0, <8 x double> %a1) nounwind {
+; SSE-LABEL: hadd_reverse2_v8f64:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movapd %xmm1, %xmm8
+; SSE-NEXT:    movapd %xmm0, %xmm9
+; SSE-NEXT:    shufpd {{.*#+}} xmm9 = xmm9[1],xmm0[0]
+; SSE-NEXT:    shufpd {{.*#+}} xmm8 = xmm8[1],xmm1[0]
+; SSE-NEXT:    shufpd {{.*#+}} xmm2 = xmm2[1,0]
+; SSE-NEXT:    shufpd {{.*#+}} xmm3 = xmm3[1,0]
+; SSE-NEXT:    shufpd {{.*#+}} xmm4 = xmm4[1,0]
+; SSE-NEXT:    haddpd %xmm4, %xmm9
+; SSE-NEXT:    shufpd {{.*#+}} xmm5 = xmm5[1,0]
+; SSE-NEXT:    haddpd %xmm5, %xmm8
+; SSE-NEXT:    shufpd {{.*#+}} xmm6 = xmm6[1,0]
+; SSE-NEXT:    haddpd %xmm6, %xmm2
+; SSE-NEXT:    shufpd {{.*#+}} xmm7 = xmm7[1,0]
+; SSE-NEXT:    haddpd %xmm7, %xmm3
+; SSE-NEXT:    movapd %xmm3, %xmm0
+; SSE-NEXT:    movapd %xmm2, %xmm1
+; SSE-NEXT:    movapd %xmm8, %xmm2
+; SSE-NEXT:    movapd %xmm9, %xmm3
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: hadd_reverse2_v8f64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; AVX1-NEXT:    vpermilpd {{.*#+}} ymm1 = ymm1[1,0,3,2]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm1[2,3,0,1]
+; AVX1-NEXT:    vpermilpd {{.*#+}} ymm1 = ymm2[1,0,3,2]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,0,1]
+; AVX1-NEXT:    vhaddpd %ymm1, %ymm0, %ymm1
+; AVX1-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm3[1,0,3,2]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; AVX1-NEXT:    vhaddpd %ymm0, %ymm4, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: hadd_reverse2_v8f64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[3,2,1,0]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm4 = ymm1[3,2,1,0]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm1 = ymm2[3,2,1,0]
+; AVX2-NEXT:    vhaddpd %ymm1, %ymm0, %ymm1
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm3[3,2,1,0]
+; AVX2-NEXT:    vhaddpd %ymm0, %ymm4, %ymm0
+; AVX2-NEXT:    retq
+  %shuf0 = shufflevector <8 x double> %a0, <8 x double> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  %shuf1 = shufflevector <8 x double> %a1, <8 x double> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  %lhs = shufflevector <8 x double> %shuf0, <8 x double> %shuf1, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+  %rhs = shufflevector <8 x double> %shuf0, <8 x double> %shuf1, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
+  %fadd = fadd <8 x double> %lhs, %rhs
+  ret <8 x double> %fadd
+}
+
+define <16 x float> @hadd_reverse_v16f32(<16 x float> %a0, <16 x float> %a1) nounwind {
+; SSE-LABEL: hadd_reverse_v16f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps %xmm5, %xmm8
+; SSE-NEXT:    movaps %xmm1, %xmm5
+; SSE-NEXT:    movaps %xmm8, %xmm1
+; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,1],xmm4[3,1]
+; SSE-NEXT:    movaps %xmm5, %xmm9
+; SSE-NEXT:    shufps {{.*#+}} xmm9 = xmm9[3,1],xmm0[3,1]
+; SSE-NEXT:    movaps %xmm7, %xmm10
+; SSE-NEXT:    shufps {{.*#+}} xmm10 = xmm10[3,1],xmm6[3,1]
+; SSE-NEXT:    movaps %xmm3, %xmm11
+; SSE-NEXT:    shufps {{.*#+}} xmm11 = xmm11[3,1],xmm2[3,1]
+; SSE-NEXT:    shufps {{.*#+}} xmm8 = xmm8[2,0],xmm4[2,0]
+; SSE-NEXT:    addps %xmm1, %xmm8
+; SSE-NEXT:    shufps {{.*#+}} xmm5 = xmm5[2,0],xmm0[2,0]
+; SSE-NEXT:    addps %xmm9, %xmm5
+; SSE-NEXT:    shufps {{.*#+}} xmm7 = xmm7[2,0],xmm6[2,0]
+; SSE-NEXT:    addps %xmm10, %xmm7
+; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[2,0],xmm2[2,0]
+; SSE-NEXT:    addps %xmm11, %xmm3
+; SSE-NEXT:    movaps %xmm3, %xmm0
+; SSE-NEXT:    movaps %xmm7, %xmm1
+; SSE-NEXT:    movaps %xmm5, %xmm2
+; SSE-NEXT:    movaps %xmm8, %xmm3
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: hadd_reverse_v16f32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm0[2,3],ymm2[2,3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    vshufps {{.*#+}} ymm2 = ymm4[3,1],ymm0[3,1],ymm4[7,5],ymm0[7,5]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm1[2,3],ymm3[2,3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm3
+; AVX1-NEXT:    vshufps {{.*#+}} ymm6 = ymm5[3,1],ymm3[3,1],ymm5[7,5],ymm3[7,5]
+; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm4[2,0],ymm0[2,0],ymm4[6,4],ymm0[6,4]
+; AVX1-NEXT:    vaddps %ymm0, %ymm2, %ymm1
+; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm5[2,0],ymm3[2,0],ymm5[6,4],ymm3[6,4]
+; AVX1-NEXT:    vaddps %ymm0, %ymm6, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: hadd_reverse_v16f32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vshufps {{.*#+}} ymm4 = ymm0[3,1],ymm2[3,1],ymm0[7,5],ymm2[7,5]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[2,0,3,1]
+; AVX2-NEXT:    vshufps {{.*#+}} ymm5 = ymm1[3,1],ymm3[3,1],ymm1[7,5],ymm3[7,5]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[2,0,3,1]
+; AVX2-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm2[2,0],ymm0[6,4],ymm2[6,4]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,0,3,1]
+; AVX2-NEXT:    vaddps %ymm0, %ymm4, %ymm2
+; AVX2-NEXT:    vshufps {{.*#+}} ymm0 = ymm1[2,0],ymm3[2,0],ymm1[6,4],ymm3[6,4]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,0,3,1]
+; AVX2-NEXT:    vaddps %ymm0, %ymm5, %ymm0
+; AVX2-NEXT:    vmovaps %ymm2, %ymm1
+; AVX2-NEXT:    retq
+  %lhs = shufflevector <16 x float> %a0, <16 x float> %a1, <16 x i32> <i32 15, i32 13, i32 11, i32 9, i32 31, i32 29, i32 27, i32 25, i32 7, i32 5, i32 3, i32 1, i32 23, i32 21, i32 19, i32 17>
+  %rhs = shufflevector <16 x float> %a0, <16 x float> %a1, <16 x i32> <i32 14, i32 12, i32 10, i32 8, i32 30, i32 28, i32 26, i32 24, i32 6, i32 4, i32 2, i32 0, i32 22, i32 20, i32 18, i32 16>
+  %fadd = fadd <16 x float> %lhs, %rhs
+  ret <16 x float> %fadd
+}
+
+define <16 x float> @hadd_reverse2_v16f32(<16 x float> %a0, <16 x float> %a1) nounwind {
+; SSE-LABEL: hadd_reverse2_v16f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps %xmm1, %xmm8
+; SSE-NEXT:    movaps %xmm0, %xmm9
+; SSE-NEXT:    shufps {{.*#+}} xmm9 = xmm9[3,2],xmm0[1,0]
+; SSE-NEXT:    shufps {{.*#+}} xmm8 = xmm8[3,2],xmm1[1,0]
+; SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,2,1,0]
+; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,2,1,0]
+; SSE-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,2,1,0]
+; SSE-NEXT:    haddps %xmm4, %xmm9
+; SSE-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,2,1,0]
+; SSE-NEXT:    haddps %xmm5, %xmm8
+; SSE-NEXT:    shufps {{.*#+}} xmm6 = xmm6[3,2,1,0]
+; SSE-NEXT:    haddps %xmm6, %xmm2
+; SSE-NEXT:    shufps {{.*#+}} xmm7 = xmm7[3,2,1,0]
+; SSE-NEXT:    haddps %xmm7, %xmm3
+; SSE-NEXT:    movaps %xmm3, %xmm0
+; SSE-NEXT:    movaps %xmm2, %xmm1
+; SSE-NEXT:    movaps %xmm8, %xmm2
+; SSE-NEXT:    movaps %xmm9, %xmm3
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: hadd_reverse2_v16f32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} ymm1 = ymm1[3,2,1,0,7,6,5,4]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm4 = ymm1[2,3,0,1]
+; AVX1-NEXT:    vpermilps {{.*#+}} ymm1 = ymm2[3,2,1,0,7,6,5,4]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,0,1]
+; AVX1-NEXT:    vhaddps %ymm1, %ymm0, %ymm1
+; AVX1-NEXT:    vpermilps {{.*#+}} ymm0 = ymm3[3,2,1,0,7,6,5,4]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; AVX1-NEXT:    vhaddps %ymm0, %ymm4, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: hadd_reverse2_v16f32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; AVX2-NEXT:    vpermilps {{.*#+}} ymm1 = ymm1[3,2,1,0,7,6,5,4]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm4 = ymm1[2,3,0,1]
+; AVX2-NEXT:    vpermilps {{.*#+}} ymm1 = ymm2[3,2,1,0,7,6,5,4]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[2,3,0,1]
+; AVX2-NEXT:    vhaddps %ymm1, %ymm0, %ymm1
+; AVX2-NEXT:    vpermilps {{.*#+}} ymm0 = ymm3[3,2,1,0,7,6,5,4]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; AVX2-NEXT:    vhaddps %ymm0, %ymm4, %ymm0
+; AVX2-NEXT:    retq
+  %shuf0 = shufflevector <16 x float> %a0, <16 x float> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  %shuf1 = shufflevector <16 x float> %a1, <16 x float> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  %lhs = shufflevector <16 x float> %shuf0, <16 x float> %shuf1, <16 x i32> <i32 0, i32 2, i32 16, i32 18, i32 4, i32 6, i32 20, i32 22, i32 8, i32 10, i32 24, i32 26, i32 12, i32 14, i32 28, i32 30>
+  %rhs = shufflevector <16 x float> %shuf0, <16 x float> %shuf1, <16 x i32> <i32 1, i32 3, i32 17, i32 19, i32 5, i32 7, i32 21, i32 23, i32 9, i32 11, i32 25, i32 27, i32 13, i32 15, i32 29, i32 31>
+  %fadd = fadd <16 x float> %lhs, %rhs
+  ret <16 x float> %fadd
+}
diff --git a/llvm/test/CodeGen/X86/llc-print-machineinstrs.mir b/llvm/test/CodeGen/X86/llc-print-machineinstrs.mir
deleted file mode 100644
index c3f8df0313376..0000000000000
--- a/llvm/test/CodeGen/X86/llc-print-machineinstrs.mir
+++ /dev/null
@@ -1,12 +0,0 @@
-# Check that -print-machineinstrs doesn't assert when it's passed an unknown pass name.
-# RUN: llc -mtriple=x86_64-- -start-before=greedy -print-machineinstrs=greedy %s -o /dev/null
-# RUN: not --crash llc -mtriple=x86_64-- -start-before=greedy -print-machineinstrs=unknown %s -o /dev/null 2>&1 | FileCheck %s
-# CHECK: LLVM ERROR: "unknown" pass is not registered.
-
-...
----
-name: fun
-tracksRegLiveness: true
-body: |
-  bb.0:
-    RET 0
diff --git a/llvm/test/CodeGen/X86/pr46820.ll b/llvm/test/CodeGen/X86/pr46820.ll
new file mode 100644
index 0000000000000..76093801f9d0a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr46820.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=avx512f | FileCheck %s
+
+; The alignment of 16 causes type legalization to split this as 3 loads,
+; v16f32, v4f32, and v4f32. This loads 24 elements, but the load is aligned
+; to 16 bytes so this i safe. There was an issue with type legalization building
+; the proper concat_vectors for this because the two v4f32s don't add up to
+; v16f32 and require padding.
+
+define <23 x float> @load23(<23 x float>* %p) {
+; CHECK-LABEL: load23:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    vmovups 64(%rsi), %ymm0
+; CHECK-NEXT:    vmovups (%rsi), %zmm1
+; CHECK-NEXT:    vmovaps 64(%rsi), %xmm2
+; CHECK-NEXT:    vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; CHECK-NEXT:    vmovss %xmm3, 88(%rdi)
+; CHECK-NEXT:    vmovaps %xmm2, 64(%rdi)
+; CHECK-NEXT:    vmovaps %zmm1, (%rdi)
+; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; CHECK-NEXT:    vmovlps %xmm0, 80(%rdi)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %t0 = load <23 x float>, <23 x float>* %p, align 16
+  ret <23 x float> %t0
+}
+
+; Same test as above with minimal alignment just to demonstrate the different
+; codegen.
+define <23 x float> @load23_align_1(<23 x float>* %p) {
+; CHECK-LABEL: load23_align_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    vmovups (%rsi), %zmm0
+; CHECK-NEXT:    vmovups 64(%rsi), %xmm1
+; CHECK-NEXT:    movq 80(%rsi), %rcx
+; CHECK-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; CHECK-NEXT:    vmovss %xmm2, 88(%rdi)
+; CHECK-NEXT:    movq %rcx, 80(%rdi)
+; CHECK-NEXT:    vmovaps %xmm1, 64(%rdi)
+; CHECK-NEXT:    vmovaps %zmm0, (%rdi)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %t0 = load <23 x float>, <23 x float>* %p, align 1
+  ret <23 x float> %t0
+}
diff --git a/llvm/test/CodeGen/X86/push-cfi-obj.ll b/llvm/test/CodeGen/X86/push-cfi-obj.ll
index 4b608ef94d8eb..4611a2c6b8aeb 100644
--- a/llvm/test/CodeGen/X86/push-cfi-obj.ll
+++ b/llvm/test/CodeGen/X86/push-cfi-obj.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -mtriple=i686-pc-linux -filetype=obj | llvm-readobj -S --sr --sd | FileCheck %s -check-prefix=LINUX
-; RUN: llc < %s -mtriple=i686-darwin-macosx10.7 -filetype=obj | llvm-readobj --sections | FileCheck -check-prefix=DARWIN %s
+; RUN: llc < %s -mtriple=i686-pc-linux -filetype=obj | llvm-readobj -S --sr --sd - | FileCheck %s -check-prefix=LINUX
+; RUN: llc < %s -mtriple=i686-darwin-macosx10.7 -filetype=obj | llvm-readobj --sections - | FileCheck -check-prefix=DARWIN %s
 
 ; On darwin, check that we manage to generate the compact unwind section
 ; DARWIN: Name: __compact_unwind
diff --git a/llvm/test/CodeGen/X86/setuge.ll b/llvm/test/CodeGen/X86/setuge.ll
index ea5562dbd5109..3e0d6dedfd58a 100644
--- a/llvm/test/CodeGen/X86/setuge.ll
+++ b/llvm/test/CodeGen/X86/setuge.ll
@@ -1,5 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-- | FileCheck %s
+; Sanity check that we ignore -sahf in 32-bit mode rather than asserting.
+; RUN: llc < %s -mtriple=i686-- -mattr=-sahf | FileCheck %s
 
 declare i1 @llvm.isunordered.f32(float, float)
 
diff --git a/llvm/test/CodeGen/X86/smax.ll b/llvm/test/CodeGen/X86/smax.ll
new file mode 100644
index 0000000000000..31d0822f8090a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/smax.ll
@@ -0,0 +1,662 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s --check-prefixes=X64,SSE
+; RUN: llc < %s -mtriple=x86_64-linux -mattr=avx | FileCheck %s --check-prefixes=X64,AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-linux -mattr=avx2 | FileCheck %s --check-prefixes=X64,AVX,AVX2
+; RUN: llc < %s -mtriple=i686 -mattr=cmov | FileCheck %s --check-prefix=X86
+
+declare i8 @llvm.smax.i8(i8, i8)
+declare i16 @llvm.smax.i16(i16, i16)
+declare i24 @llvm.smax.i24(i24, i24)
+declare i32 @llvm.smax.i32(i32, i32)
+declare i64 @llvm.smax.i64(i64, i64)
+declare i128 @llvm.smax.i128(i128, i128)
+
+declare <1 x i32> @llvm.smax.v1i32(<1 x i32>, <1 x i32>)
+declare <2 x i32> @llvm.smax.v2i32(<2 x i32>, <2 x i32>)
+declare <3 x i32> @llvm.smax.v3i32(<3 x i32>, <3 x i32>)
+declare <4 x i32> @llvm.smax.v4i32(<4 x i32>, <4 x i32>)
+declare <8 x i32> @llvm.smax.v8i32(<8 x i32>, <8 x i32>)
+
+declare <8 x i16> @llvm.smax.v8i16(<8 x i16>, <8 x i16>)
+declare <16 x i8> @llvm.smax.v16i8(<16 x i8>, <16 x i8>)
+
+define i8 @test_i8(i8 %a, i8 %b) nounwind {
+; X64-LABEL: test_i8:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    cmpb %al, %dil
+; X64-NEXT:    cmovgl %edi, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i8:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmpb %al, %cl
+; X86-NEXT:    cmovgl %ecx, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+  %r = call i8 @llvm.smax.i8(i8 %a, i8 %b)
+  ret i8 %r
+}
+
+define i16 @test_i16(i16 %a, i16 %b) nounwind {
+; X64-LABEL: test_i16:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    cmpw %ax, %di
+; X64-NEXT:    cmovgl %edi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i16:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmpw %ax, %cx
+; X86-NEXT:    cmovgl %ecx, %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    retl
+  %r = call i16 @llvm.smax.i16(i16 %a, i16 %b)
+  ret i16 %r
+}
+
+define i24 @test_i24(i24 %a, i24 %b) nounwind {
+; X64-LABEL: test_i24:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    shll $8, %esi
+; X64-NEXT:    sarl $8, %esi
+; X64-NEXT:    shll $8, %eax
+; X64-NEXT:    sarl $8, %eax
+; X64-NEXT:    cmpl %esi, %eax
+; X64-NEXT:    cmovlel %esi, %eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i24:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    shll $8, %ecx
+; X86-NEXT:    sarl $8, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    shll $8, %eax
+; X86-NEXT:    sarl $8, %eax
+; X86-NEXT:    cmpl %ecx, %eax
+; X86-NEXT:    cmovlel %ecx, %eax
+; X86-NEXT:    retl
+  %r = call i24 @llvm.smax.i24(i24 %a, i24 %b)
+  ret i24 %r
+}
+
+define i32 @test_i32(i32 %a, i32 %b) nounwind {
+; X64-LABEL: test_i32:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    cmpl %esi, %edi
+; X64-NEXT:    cmovgl %edi, %eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmpl %eax, %ecx
+; X86-NEXT:    cmovgl %ecx, %eax
+; X86-NEXT:    retl
+  %r = call i32 @llvm.smax.i32(i32 %a, i32 %b)
+  ret i32 %r
+}
+
+define i64 @test_i64(i64 %a, i64 %b) nounwind {
+; X64-LABEL: test_i64:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rsi, %rax
+; X64-NEXT:    cmpq %rsi, %rdi
+; X64-NEXT:    cmovgq %rdi, %rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i64:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    cmpl %eax, %ecx
+; X86-NEXT:    movl %eax, %edi
+; X86-NEXT:    cmoval %ecx, %edi
+; X86-NEXT:    cmpl %edx, %esi
+; X86-NEXT:    cmovgl %ecx, %eax
+; X86-NEXT:    cmovel %edi, %eax
+; X86-NEXT:    cmovgl %esi, %edx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl
+  %r = call i64 @llvm.smax.i64(i64 %a, i64 %b)
+  ret i64 %r
+}
+
+define i128 @test_i128(i128 %a, i128 %b) nounwind {
+; X64-LABEL: test_i128:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdx, %rax
+; X64-NEXT:    cmpq %rdx, %rdi
+; X64-NEXT:    cmovaq %rdi, %rdx
+; X64-NEXT:    cmpq %rcx, %rsi
+; X64-NEXT:    cmovgq %rdi, %rax
+; X64-NEXT:    cmoveq %rdx, %rax
+; X64-NEXT:    cmovgq %rsi, %rcx
+; X64-NEXT:    movq %rcx, %rdx
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i128:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmpl %ebx, %edx
+; X86-NEXT:    movl %ebx, %eax
+; X86-NEXT:    cmoval %edx, %eax
+; X86-NEXT:    cmpl %esi, %ecx
+; X86-NEXT:    movl %ebx, %ebp
+; X86-NEXT:    cmoval %edx, %ebp
+; X86-NEXT:    cmovel %eax, %ebp
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    cmoval %ecx, %eax
+; X86-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %edx, %ecx
+; X86-NEXT:    sbbl %edi, %ecx
+; X86-NEXT:    cmovll {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    cmovll {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl %edi, %ecx
+; X86-NEXT:    xorl %edx, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    xorl %eax, %edi
+; X86-NEXT:    orl %ecx, %edi
+; X86-NEXT:    cmovel %ebp, %ebx
+; X86-NEXT:    cmovel (%esp), %esi # 4-byte Folded Reload
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    cmpl %eax, %edi
+; X86-NEXT:    movl %eax, %ecx
+; X86-NEXT:    cmoval %edi, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    cmpl %edx, %ebp
+; X86-NEXT:    cmovgl %edi, %eax
+; X86-NEXT:    cmovel %ecx, %eax
+; X86-NEXT:    cmovgl %ebp, %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %edx, 12(%ecx)
+; X86-NEXT:    movl %eax, 8(%ecx)
+; X86-NEXT:    movl %esi, 4(%ecx)
+; X86-NEXT:    movl %ebx, (%ecx)
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    addl $4, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+  %r = call i128 @llvm.smax.i128(i128 %a, i128 %b)
+  ret i128 %r
+}
+
+define <1 x i32> @test_v1i32(<1 x i32> %a, <1 x i32> %b) nounwind {
+; X64-LABEL: test_v1i32:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    cmpl %esi, %edi
+; X64-NEXT:    cmovgl %edi, %eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_v1i32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmpl %eax, %ecx
+; X86-NEXT:    cmovgl %ecx, %eax
+; X86-NEXT:    retl
+  %r = call <1 x i32> @llvm.smax.v1i32(<1 x i32> %a, <1 x i32> %b)
+  ret <1 x i32> %r
+}
+
+define <2 x i32> @test_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
+; SSE-LABEL: test_v2i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa %xmm0, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    pandn %xmm1, %xmm2
+; SSE-NEXT:    por %xmm0, %xmm2
+; SSE-NEXT:    movdqa %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v2i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_v2i32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    cmpl %eax, %esi
+; X86-NEXT:    cmovgl %esi, %eax
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    cmovgl %ecx, %edx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+  %r = call <2 x i32> @llvm.smax.v2i32(<2 x i32> %a, <2 x i32> %b)
+  ret <2 x i32> %r
+}
+
+define <3 x i32> @test_v3i32(<3 x i32> %a, <3 x i32> %b) nounwind {
+; SSE-LABEL: test_v3i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa %xmm0, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    pandn %xmm1, %xmm2
+; SSE-NEXT:    por %xmm0, %xmm2
+; SSE-NEXT:    movdqa %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v3i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_v3i32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    cmpl %eax, %ebx
+; X86-NEXT:    cmovgl %ebx, %eax
+; X86-NEXT:    cmpl %edx, %edi
+; X86-NEXT:    cmovgl %edi, %edx
+; X86-NEXT:    cmpl %ecx, %esi
+; X86-NEXT:    cmovgl %esi, %ecx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    retl
+  %r = call <3 x i32> @llvm.smax.v3i32(<3 x i32> %a, <3 x i32> %b)
+  ret <3 x i32> %r
+}
+
+define <4 x i32> @test_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
+; SSE-LABEL: test_v4i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa %xmm0, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    pandn %xmm1, %xmm2
+; SSE-NEXT:    por %xmm0, %xmm2
+; SSE-NEXT:    movdqa %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v4i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_v4i32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %edi, %eax
+; X86-NEXT:    cmovgl %eax, %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %esi, %eax
+; X86-NEXT:    cmovgl %eax, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %edx, %eax
+; X86-NEXT:    cmovgl %eax, %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %ecx, %eax
+; X86-NEXT:    cmovgl %eax, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %ecx, 12(%eax)
+; X86-NEXT:    movl %edx, 8(%eax)
+; X86-NEXT:    movl %esi, 4(%eax)
+; X86-NEXT:    movl %edi, (%eax)
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+  %r = call <4 x i32> @llvm.smax.v4i32(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %r
+}
+
+define <8 x i32> @test_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
+; SSE-LABEL: test_v8i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa %xmm0, %xmm4
+; SSE-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE-NEXT:    pand %xmm4, %xmm0
+; SSE-NEXT:    pandn %xmm2, %xmm4
+; SSE-NEXT:    por %xmm0, %xmm4
+; SSE-NEXT:    movdqa %xmm1, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE-NEXT:    pand %xmm2, %xmm1
+; SSE-NEXT:    pandn %xmm3, %xmm2
+; SSE-NEXT:    por %xmm1, %xmm2
+; SSE-NEXT:    movdqa %xmm4, %xmm0
+; SSE-NEXT:    movdqa %xmm2, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: test_v8i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpmaxsd %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test_v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; X86-LABEL: test_v8i32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %ebp, %eax
+; X86-NEXT:    cmovgl %eax, %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %ebx, %eax
+; X86-NEXT:    cmovgl %eax, %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %edi, %eax
+; X86-NEXT:    cmovgl %eax, %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %esi, %eax
+; X86-NEXT:    cmovgl %eax, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %edx, %eax
+; X86-NEXT:    cmovgl %eax, %edx
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %ecx, %eax
+; X86-NEXT:    cmovgl %eax, %ecx
+; X86-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %ecx, %eax
+; X86-NEXT:    cmovgl %eax, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    cmpl %eax, %edx
+; X86-NEXT:    cmovgl %edx, %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %eax, 28(%edx)
+; X86-NEXT:    movl %ecx, 24(%edx)
+; X86-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, 20(%edx)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, 16(%edx)
+; X86-NEXT:    movl %esi, 12(%edx)
+; X86-NEXT:    movl %edi, 8(%edx)
+; X86-NEXT:    movl %ebx, 4(%edx)
+; X86-NEXT:    movl %ebp, (%edx)
+; X86-NEXT:    movl %edx, %eax
+; X86-NEXT:    addl $8, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+  %r = call <8 x i32> @llvm.smax.v8i32(<8 x i32> %a, <8 x i32> %b)
+  ret <8 x i32> %r
+}
+
+define <8 x i16> @test_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
+; SSE-LABEL: test_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pmaxsw %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_v8i16:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %bp, %ax
+; X86-NEXT:    cmovgl %eax, %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %bx, %ax
+; X86-NEXT:    cmovgl %eax, %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %di, %ax
+; X86-NEXT:    cmovgl %eax, %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %si, %ax
+; X86-NEXT:    cmovgl %eax, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %dx, %ax
+; X86-NEXT:    cmovgl %eax, %edx
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %cx, %ax
+; X86-NEXT:    cmovgl %eax, %ecx
+; X86-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %dx, %ax
+; X86-NEXT:    cmovgl %eax, %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmpw %ax, %cx
+; X86-NEXT:    cmovgl %ecx, %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movw %ax, 14(%ecx)
+; X86-NEXT:    movw %dx, 12(%ecx)
+; X86-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-NEXT:    movw %ax, 10(%ecx)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movw %ax, 8(%ecx)
+; X86-NEXT:    movw %si, 6(%ecx)
+; X86-NEXT:    movw %di, 4(%ecx)
+; X86-NEXT:    movw %bx, 2(%ecx)
+; X86-NEXT:    movw %bp, (%ecx)
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    addl $8, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+  %r = call <8 x i16> @llvm.smax.v8i16(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %r
+}
+
+define <16 x i8> @test_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
+; SSE-LABEL: test_v16i8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa %xmm0, %xmm2
+; SSE-NEXT:    pcmpgtb %xmm1, %xmm2
+; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    pandn %xmm1, %xmm2
+; SSE-NEXT:    por %xmm0, %xmm2
+; SSE-NEXT:    movdqa %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v16i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_v16i8:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $40, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %bl, %al
+; X86-NEXT:    cmovgl %eax, %ebx
+; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %dl, %al
+; X86-NEXT:    cmovgl %eax, %edx
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovgl %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovgl %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovgl %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovgl %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovgl %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovgl %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovgl %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovgl %eax, %ecx
+; X86-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovgl %eax, %ecx
+; X86-NEXT:    movl %ecx, %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovgl %eax, %ecx
+; X86-NEXT:    movl %ecx, %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovgl %eax, %ecx
+; X86-NEXT:    movl %ecx, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %bl, %al
+; X86-NEXT:    cmovgl %eax, %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %dl, %al
+; X86-NEXT:    cmovgl %eax, %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovgl %eax, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movb %cl, 15(%eax)
+; X86-NEXT:    movb %dl, 14(%eax)
+; X86-NEXT:    movb %bl, 13(%eax)
+; X86-NEXT:    movl %esi, %ecx
+; X86-NEXT:    movb %cl, 12(%eax)
+; X86-NEXT:    movl %edi, %ecx
+; X86-NEXT:    movb %cl, 11(%eax)
+; X86-NEXT:    movl %ebp, %ecx
+; X86-NEXT:    movb %cl, 10(%eax)
+; X86-NEXT:    movl (%esp), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 9(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 8(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 7(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 6(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 5(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 4(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 3(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 2(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 1(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, (%eax)
+; X86-NEXT:    addl $40, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+  %r = call <16 x i8> @llvm.smax.v16i8(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %r
+}
diff --git a/llvm/test/CodeGen/X86/smin.ll b/llvm/test/CodeGen/X86/smin.ll
new file mode 100644
index 0000000000000..70391534f544c
--- /dev/null
+++ b/llvm/test/CodeGen/X86/smin.ll
@@ -0,0 +1,656 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s --check-prefixes=X64,SSE
+; RUN: llc < %s -mtriple=x86_64-linux -mattr=avx | FileCheck %s --check-prefixes=X64,AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-linux -mattr=avx2 | FileCheck %s --check-prefixes=X64,AVX,AVX2
+; RUN: llc < %s -mtriple=i686 -mattr=cmov | FileCheck %s --check-prefix=X86
+
+declare i8 @llvm.smin.i8(i8, i8)
+declare i16 @llvm.smin.i16(i16, i16)
+declare i24 @llvm.smin.i24(i24, i24)
+declare i32 @llvm.smin.i32(i32, i32)
+declare i64 @llvm.smin.i64(i64, i64)
+declare i128 @llvm.smin.i128(i128, i128)
+
+declare <1 x i32> @llvm.smin.v1i32(<1 x i32>, <1 x i32>)
+declare <2 x i32> @llvm.smin.v2i32(<2 x i32>, <2 x i32>)
+declare <3 x i32> @llvm.smin.v3i32(<3 x i32>, <3 x i32>)
+declare <4 x i32> @llvm.smin.v4i32(<4 x i32>, <4 x i32>)
+declare <8 x i32> @llvm.smin.v8i32(<8 x i32>, <8 x i32>)
+
+declare <8 x i16> @llvm.smin.v8i16(<8 x i16>, <8 x i16>)
+declare <16 x i8> @llvm.smin.v16i8(<16 x i8>, <16 x i8>)
+
+define i8 @test_i8(i8 %a, i8 %b) nounwind {
+; X64-LABEL: test_i8:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    cmpb %al, %dil
+; X64-NEXT:    cmovll %edi, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i8:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmpb %al, %cl
+; X86-NEXT:    cmovll %ecx, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+  %r = call i8 @llvm.smin.i8(i8 %a, i8 %b)
+  ret i8 %r
+}
+
+define i16 @test_i16(i16 %a, i16 %b) nounwind {
+; X64-LABEL: test_i16:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    cmpw %ax, %di
+; X64-NEXT:    cmovll %edi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i16:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmpw %ax, %cx
+; X86-NEXT:    cmovll %ecx, %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    retl
+  %r = call i16 @llvm.smin.i16(i16 %a, i16 %b)
+  ret i16 %r
+}
+
+define i24 @test_i24(i24 %a, i24 %b) nounwind {
+; X64-LABEL: test_i24:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    shll $8, %esi
+; X64-NEXT:    sarl $8, %esi
+; X64-NEXT:    shll $8, %eax
+; X64-NEXT:    sarl $8, %eax
+; X64-NEXT:    cmpl %esi, %eax
+; X64-NEXT:    cmovgel %esi, %eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i24:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    shll $8, %ecx
+; X86-NEXT:    sarl $8, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    shll $8, %eax
+; X86-NEXT:    sarl $8, %eax
+; X86-NEXT:    cmpl %ecx, %eax
+; X86-NEXT:    cmovgel %ecx, %eax
+; X86-NEXT:    retl
+  %r = call i24 @llvm.smin.i24(i24 %a, i24 %b)
+  ret i24 %r
+}
+
+define i32 @test_i32(i32 %a, i32 %b) nounwind {
+; X64-LABEL: test_i32:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    cmpl %esi, %edi
+; X64-NEXT:    cmovll %edi, %eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmpl %eax, %ecx
+; X86-NEXT:    cmovll %ecx, %eax
+; X86-NEXT:    retl
+  %r = call i32 @llvm.smin.i32(i32 %a, i32 %b)
+  ret i32 %r
+}
+
+define i64 @test_i64(i64 %a, i64 %b) nounwind {
+; X64-LABEL: test_i64:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rsi, %rax
+; X64-NEXT:    cmpq %rsi, %rdi
+; X64-NEXT:    cmovlq %rdi, %rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i64:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    cmpl %eax, %ecx
+; X86-NEXT:    movl %eax, %edi
+; X86-NEXT:    cmovbl %ecx, %edi
+; X86-NEXT:    cmpl %edx, %esi
+; X86-NEXT:    cmovll %ecx, %eax
+; X86-NEXT:    cmovel %edi, %eax
+; X86-NEXT:    cmovll %esi, %edx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl
+  %r = call i64 @llvm.smin.i64(i64 %a, i64 %b)
+  ret i64 %r
+}
+
+define i128 @test_i128(i128 %a, i128 %b) nounwind {
+; X64-LABEL: test_i128:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdx, %rax
+; X64-NEXT:    cmpq %rdx, %rdi
+; X64-NEXT:    cmovbq %rdi, %rdx
+; X64-NEXT:    cmpq %rcx, %rsi
+; X64-NEXT:    cmovlq %rdi, %rax
+; X64-NEXT:    cmoveq %rdx, %rax
+; X64-NEXT:    cmovlq %rsi, %rcx
+; X64-NEXT:    movq %rcx, %rdx
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i128:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    cmpl %ecx, %eax
+; X86-NEXT:    movl %ecx, %ebx
+; X86-NEXT:    cmovbl %eax, %ebx
+; X86-NEXT:    cmpl %esi, %edi
+; X86-NEXT:    movl %ecx, %ebp
+; X86-NEXT:    cmovbl %eax, %ebp
+; X86-NEXT:    cmovel %ebx, %ebp
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    cmovbl %edi, %eax
+; X86-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    cmpl %edx, %edi
+; X86-NEXT:    movl %edx, %eax
+; X86-NEXT:    cmovbl %edi, %eax
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl %ebx, %edi
+; X86-NEXT:    sbbl %eax, %edi
+; X86-NEXT:    cmovll {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    cmovll {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ebx, %edi
+; X86-NEXT:    xorl %eax, %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    xorl %edx, %ebx
+; X86-NEXT:    orl %edi, %ebx
+; X86-NEXT:    cmovel %ebp, %ecx
+; X86-NEXT:    cmovel (%esp), %esi # 4-byte Folded Reload
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    cmpl %eax, %edi
+; X86-NEXT:    cmovll {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    cmovel {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT:    cmovll %edi, %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %eax, 12(%edi)
+; X86-NEXT:    movl %edx, 8(%edi)
+; X86-NEXT:    movl %esi, 4(%edi)
+; X86-NEXT:    movl %ecx, (%edi)
+; X86-NEXT:    movl %edi, %eax
+; X86-NEXT:    addl $8, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+  %r = call i128 @llvm.smin.i128(i128 %a, i128 %b)
+  ret i128 %r
+}
+
+define <1 x i32> @test_v1i32(<1 x i32> %a, <1 x i32> %b) nounwind {
+; X64-LABEL: test_v1i32:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    cmpl %esi, %edi
+; X64-NEXT:    cmovll %edi, %eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_v1i32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmpl %eax, %ecx
+; X86-NEXT:    cmovll %ecx, %eax
+; X86-NEXT:    retl
+  %r = call <1 x i32> @llvm.smin.v1i32(<1 x i32> %a, <1 x i32> %b)
+  ret <1 x i32> %r
+}
+
+define <2 x i32> @test_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
+; SSE-LABEL: test_v2i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa %xmm1, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    pandn %xmm1, %xmm2
+; SSE-NEXT:    por %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v2i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_v2i32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    cmpl %eax, %esi
+; X86-NEXT:    cmovll %esi, %eax
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    cmovll %ecx, %edx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+  %r = call <2 x i32> @llvm.smin.v2i32(<2 x i32> %a, <2 x i32> %b)
+  ret <2 x i32> %r
+}
+
+define <3 x i32> @test_v3i32(<3 x i32> %a, <3 x i32> %b) nounwind {
+; SSE-LABEL: test_v3i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa %xmm1, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    pandn %xmm1, %xmm2
+; SSE-NEXT:    por %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v3i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_v3i32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    cmpl %eax, %ebx
+; X86-NEXT:    cmovll %ebx, %eax
+; X86-NEXT:    cmpl %edx, %edi
+; X86-NEXT:    cmovll %edi, %edx
+; X86-NEXT:    cmpl %ecx, %esi
+; X86-NEXT:    cmovll %esi, %ecx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    retl
+  %r = call <3 x i32> @llvm.smin.v3i32(<3 x i32> %a, <3 x i32> %b)
+  ret <3 x i32> %r
+}
+
+define <4 x i32> @test_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
+; SSE-LABEL: test_v4i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa %xmm1, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    pandn %xmm1, %xmm2
+; SSE-NEXT:    por %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v4i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_v4i32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %edi, %eax
+; X86-NEXT:    cmovll %eax, %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %esi, %eax
+; X86-NEXT:    cmovll %eax, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %edx, %eax
+; X86-NEXT:    cmovll %eax, %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %ecx, %eax
+; X86-NEXT:    cmovll %eax, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %ecx, 12(%eax)
+; X86-NEXT:    movl %edx, 8(%eax)
+; X86-NEXT:    movl %esi, 4(%eax)
+; X86-NEXT:    movl %edi, (%eax)
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+  %r = call <4 x i32> @llvm.smin.v4i32(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %r
+}
+
+define <8 x i32> @test_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
+; SSE-LABEL: test_v8i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa %xmm2, %xmm4
+; SSE-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE-NEXT:    pand %xmm4, %xmm0
+; SSE-NEXT:    pandn %xmm2, %xmm4
+; SSE-NEXT:    por %xmm4, %xmm0
+; SSE-NEXT:    movdqa %xmm3, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE-NEXT:    pand %xmm2, %xmm1
+; SSE-NEXT:    pandn %xmm3, %xmm2
+; SSE-NEXT:    por %xmm2, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: test_v8i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpminsd %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test_v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpminsd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; X86-LABEL: test_v8i32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %ebp, %eax
+; X86-NEXT:    cmovll %eax, %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %ebx, %eax
+; X86-NEXT:    cmovll %eax, %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %edi, %eax
+; X86-NEXT:    cmovll %eax, %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %esi, %eax
+; X86-NEXT:    cmovll %eax, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %edx, %eax
+; X86-NEXT:    cmovll %eax, %edx
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %ecx, %eax
+; X86-NEXT:    cmovll %eax, %ecx
+; X86-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %ecx, %eax
+; X86-NEXT:    cmovll %eax, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    cmpl %eax, %edx
+; X86-NEXT:    cmovll %edx, %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %eax, 28(%edx)
+; X86-NEXT:    movl %ecx, 24(%edx)
+; X86-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, 20(%edx)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, 16(%edx)
+; X86-NEXT:    movl %esi, 12(%edx)
+; X86-NEXT:    movl %edi, 8(%edx)
+; X86-NEXT:    movl %ebx, 4(%edx)
+; X86-NEXT:    movl %ebp, (%edx)
+; X86-NEXT:    movl %edx, %eax
+; X86-NEXT:    addl $8, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+  %r = call <8 x i32> @llvm.smin.v8i32(<8 x i32> %a, <8 x i32> %b)
+  ret <8 x i32> %r
+}
+
+define <8 x i16> @test_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
+; SSE-LABEL: test_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pminsw %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_v8i16:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %bp, %ax
+; X86-NEXT:    cmovll %eax, %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %bx, %ax
+; X86-NEXT:    cmovll %eax, %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %di, %ax
+; X86-NEXT:    cmovll %eax, %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %si, %ax
+; X86-NEXT:    cmovll %eax, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %dx, %ax
+; X86-NEXT:    cmovll %eax, %edx
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %cx, %ax
+; X86-NEXT:    cmovll %eax, %ecx
+; X86-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %dx, %ax
+; X86-NEXT:    cmovll %eax, %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmpw %ax, %cx
+; X86-NEXT:    cmovll %ecx, %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movw %ax, 14(%ecx)
+; X86-NEXT:    movw %dx, 12(%ecx)
+; X86-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-NEXT:    movw %ax, 10(%ecx)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movw %ax, 8(%ecx)
+; X86-NEXT:    movw %si, 6(%ecx)
+; X86-NEXT:    movw %di, 4(%ecx)
+; X86-NEXT:    movw %bx, 2(%ecx)
+; X86-NEXT:    movw %bp, (%ecx)
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    addl $8, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+  %r = call <8 x i16> @llvm.smin.v8i16(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %r
+}
+
+define <16 x i8> @test_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
+; SSE-LABEL: test_v16i8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa %xmm1, %xmm2
+; SSE-NEXT:    pcmpgtb %xmm0, %xmm2
+; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    pandn %xmm1, %xmm2
+; SSE-NEXT:    por %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v16i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_v16i8:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $40, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %bl, %al
+; X86-NEXT:    cmovll %eax, %ebx
+; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %dl, %al
+; X86-NEXT:    cmovll %eax, %edx
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovll %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovll %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovll %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovll %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovll %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovll %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovll %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovll %eax, %ecx
+; X86-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovll %eax, %ecx
+; X86-NEXT:    movl %ecx, %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovll %eax, %ecx
+; X86-NEXT:    movl %ecx, %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovll %eax, %ecx
+; X86-NEXT:    movl %ecx, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %bl, %al
+; X86-NEXT:    cmovll %eax, %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %dl, %al
+; X86-NEXT:    cmovll %eax, %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovll %eax, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movb %cl, 15(%eax)
+; X86-NEXT:    movb %dl, 14(%eax)
+; X86-NEXT:    movb %bl, 13(%eax)
+; X86-NEXT:    movl %esi, %ecx
+; X86-NEXT:    movb %cl, 12(%eax)
+; X86-NEXT:    movl %edi, %ecx
+; X86-NEXT:    movb %cl, 11(%eax)
+; X86-NEXT:    movl %ebp, %ecx
+; X86-NEXT:    movb %cl, 10(%eax)
+; X86-NEXT:    movl (%esp), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 9(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 8(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 7(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 6(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 5(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 4(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 3(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 2(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 1(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, (%eax)
+; X86-NEXT:    addl $40, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+  %r = call <16 x i8> @llvm.smin.v16i8(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %r
+}
diff --git a/llvm/test/CodeGen/X86/stress-scheduledagrrlist.ll b/llvm/test/CodeGen/X86/stress-scheduledagrrlist.ll
new file mode 100644
index 0000000000000..a699134a8c7c5
--- /dev/null
+++ b/llvm/test/CodeGen/X86/stress-scheduledagrrlist.ll
@@ -0,0 +1,12 @@
+; RUN: llc -O0 -mtriple=x86_64-apple-macosx %s -o %t.s
+
+; Stress test for the list scheduler. The store will be expanded to a very
+; large number of stores during isel, stressing ScheduleDAGRRList. It should
+; compiles in a reasonable amount of time. Run with -O0, to disable most other
+; optimizations.
+
+define void @test(i1000000* %ptr) {
+entry:
+  store i1000000 0, i1000000* %ptr, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/umax.ll b/llvm/test/CodeGen/X86/umax.ll
new file mode 100644
index 0000000000000..14a0248e19145
--- /dev/null
+++ b/llvm/test/CodeGen/X86/umax.ll
@@ -0,0 +1,668 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s --check-prefixes=X64,SSE
+; RUN: llc < %s -mtriple=x86_64-linux -mattr=avx | FileCheck %s --check-prefixes=X64,AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-linux -mattr=avx2 | FileCheck %s --check-prefixes=X64,AVX,AVX2
+; RUN: llc < %s -mtriple=i686 -mattr=cmov | FileCheck %s --check-prefix=X86
+
+declare i8 @llvm.umax.i8(i8, i8)
+declare i16 @llvm.umax.i16(i16, i16)
+declare i24 @llvm.umax.i24(i24, i24)
+declare i32 @llvm.umax.i32(i32, i32)
+declare i64 @llvm.umax.i64(i64, i64)
+declare i128 @llvm.umax.i128(i128, i128)
+
+declare <1 x i32> @llvm.umax.v1i32(<1 x i32>, <1 x i32>)
+declare <2 x i32> @llvm.umax.v2i32(<2 x i32>, <2 x i32>)
+declare <3 x i32> @llvm.umax.v3i32(<3 x i32>, <3 x i32>)
+declare <4 x i32> @llvm.umax.v4i32(<4 x i32>, <4 x i32>)
+declare <8 x i32> @llvm.umax.v8i32(<8 x i32>, <8 x i32>)
+
+declare <8 x i16> @llvm.umax.v8i16(<8 x i16>, <8 x i16>)
+declare <16 x i8> @llvm.umax.v16i8(<16 x i8>, <16 x i8>)
+
+define i8 @test_i8(i8 %a, i8 %b) nounwind {
+; X64-LABEL: test_i8:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    cmpb %al, %dil
+; X64-NEXT:    cmoval %edi, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i8:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmpb %al, %cl
+; X86-NEXT:    cmoval %ecx, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+  %r = call i8 @llvm.umax.i8(i8 %a, i8 %b)
+  ret i8 %r
+}
+
+define i16 @test_i16(i16 %a, i16 %b) nounwind {
+; X64-LABEL: test_i16:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    cmpw %ax, %di
+; X64-NEXT:    cmoval %edi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i16:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmpw %ax, %cx
+; X86-NEXT:    cmoval %ecx, %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    retl
+  %r = call i16 @llvm.umax.i16(i16 %a, i16 %b)
+  ret i16 %r
+}
+
+define i24 @test_i24(i24 %a, i24 %b) nounwind {
+; X64-LABEL: test_i24:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    andl $16777215, %esi # imm = 0xFFFFFF
+; X64-NEXT:    andl $16777215, %eax # imm = 0xFFFFFF
+; X64-NEXT:    cmpl %esi, %eax
+; X64-NEXT:    cmovbel %esi, %eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i24:
+; X86:       # %bb.0:
+; X86-NEXT:    movl $16777215, %eax # imm = 0xFFFFFF
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    andl %eax, %ecx
+; X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %ecx, %eax
+; X86-NEXT:    cmovbel %ecx, %eax
+; X86-NEXT:    retl
+  %r = call i24 @llvm.umax.i24(i24 %a, i24 %b)
+  ret i24 %r
+}
+
+define i32 @test_i32(i32 %a, i32 %b) nounwind {
+; X64-LABEL: test_i32:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    cmpl %esi, %edi
+; X64-NEXT:    cmoval %edi, %eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmpl %eax, %ecx
+; X86-NEXT:    cmoval %ecx, %eax
+; X86-NEXT:    retl
+  %r = call i32 @llvm.umax.i32(i32 %a, i32 %b)
+  ret i32 %r
+}
+
+define i64 @test_i64(i64 %a, i64 %b) nounwind {
+; X64-LABEL: test_i64:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rsi, %rax
+; X64-NEXT:    cmpq %rsi, %rdi
+; X64-NEXT:    cmovaq %rdi, %rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i64:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    cmpl %eax, %ecx
+; X86-NEXT:    movl %eax, %edi
+; X86-NEXT:    cmoval %ecx, %edi
+; X86-NEXT:    cmpl %edx, %esi
+; X86-NEXT:    cmoval %ecx, %eax
+; X86-NEXT:    cmovel %edi, %eax
+; X86-NEXT:    cmoval %esi, %edx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl
+  %r = call i64 @llvm.umax.i64(i64 %a, i64 %b)
+  ret i64 %r
+}
+
+define i128 @test_i128(i128 %a, i128 %b) nounwind {
+; X64-LABEL: test_i128:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdx, %rax
+; X64-NEXT:    cmpq %rdx, %rdi
+; X64-NEXT:    cmovaq %rdi, %rdx
+; X64-NEXT:    cmpq %rcx, %rsi
+; X64-NEXT:    cmovaq %rdi, %rax
+; X64-NEXT:    cmoveq %rdx, %rax
+; X64-NEXT:    cmovaq %rsi, %rcx
+; X64-NEXT:    movq %rcx, %rdx
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i128:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmpl %ebx, %edx
+; X86-NEXT:    movl %ebx, %eax
+; X86-NEXT:    cmoval %edx, %eax
+; X86-NEXT:    cmpl %esi, %ecx
+; X86-NEXT:    movl %ebx, %ebp
+; X86-NEXT:    cmoval %edx, %ebp
+; X86-NEXT:    cmovel %eax, %ebp
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    cmoval %ecx, %eax
+; X86-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %edx, %ecx
+; X86-NEXT:    sbbl %edi, %ecx
+; X86-NEXT:    cmovbl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    cmovbl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl %edi, %ecx
+; X86-NEXT:    xorl %edx, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    xorl %eax, %edi
+; X86-NEXT:    orl %ecx, %edi
+; X86-NEXT:    cmovel %ebp, %ebx
+; X86-NEXT:    cmovel (%esp), %esi # 4-byte Folded Reload
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    cmpl %eax, %edi
+; X86-NEXT:    movl %eax, %ecx
+; X86-NEXT:    cmoval %edi, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    cmpl %edx, %ebp
+; X86-NEXT:    cmoval %edi, %eax
+; X86-NEXT:    cmovel %ecx, %eax
+; X86-NEXT:    cmoval %ebp, %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %edx, 12(%ecx)
+; X86-NEXT:    movl %eax, 8(%ecx)
+; X86-NEXT:    movl %esi, 4(%ecx)
+; X86-NEXT:    movl %ebx, (%ecx)
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    addl $4, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+  %r = call i128 @llvm.umax.i128(i128 %a, i128 %b)
+  ret i128 %r
+}
+
+define <1 x i32> @test_v1i32(<1 x i32> %a, <1 x i32> %b) nounwind {
+; X64-LABEL: test_v1i32:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    cmpl %esi, %edi
+; X64-NEXT:    cmoval %edi, %eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_v1i32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmpl %eax, %ecx
+; X86-NEXT:    cmoval %ecx, %eax
+; X86-NEXT:    retl
+  %r = call <1 x i32> @llvm.umax.v1i32(<1 x i32> %a, <1 x i32> %b)
+  ret <1 x i32> %r
+}
+
+define <2 x i32> @test_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
+; SSE-LABEL: test_v2i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSE-NEXT:    movdqa %xmm1, %xmm3
+; SSE-NEXT:    pxor %xmm2, %xmm3
+; SSE-NEXT:    pxor %xmm0, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    pandn %xmm1, %xmm2
+; SSE-NEXT:    por %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v2i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_v2i32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    cmpl %eax, %esi
+; X86-NEXT:    cmoval %esi, %eax
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    cmoval %ecx, %edx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+  %r = call <2 x i32> @llvm.umax.v2i32(<2 x i32> %a, <2 x i32> %b)
+  ret <2 x i32> %r
+}
+
+define <3 x i32> @test_v3i32(<3 x i32> %a, <3 x i32> %b) nounwind {
+; SSE-LABEL: test_v3i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSE-NEXT:    movdqa %xmm1, %xmm3
+; SSE-NEXT:    pxor %xmm2, %xmm3
+; SSE-NEXT:    pxor %xmm0, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    pandn %xmm1, %xmm2
+; SSE-NEXT:    por %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v3i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_v3i32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    cmpl %eax, %ebx
+; X86-NEXT:    cmoval %ebx, %eax
+; X86-NEXT:    cmpl %edx, %edi
+; X86-NEXT:    cmoval %edi, %edx
+; X86-NEXT:    cmpl %ecx, %esi
+; X86-NEXT:    cmoval %esi, %ecx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    retl
+  %r = call <3 x i32> @llvm.umax.v3i32(<3 x i32> %a, <3 x i32> %b)
+  ret <3 x i32> %r
+}
+
+define <4 x i32> @test_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
+; SSE-LABEL: test_v4i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSE-NEXT:    movdqa %xmm1, %xmm3
+; SSE-NEXT:    pxor %xmm2, %xmm3
+; SSE-NEXT:    pxor %xmm0, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    pandn %xmm1, %xmm2
+; SSE-NEXT:    por %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v4i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_v4i32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %edi, %eax
+; X86-NEXT:    cmoval %eax, %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %esi, %eax
+; X86-NEXT:    cmoval %eax, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %edx, %eax
+; X86-NEXT:    cmoval %eax, %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %ecx, %eax
+; X86-NEXT:    cmoval %eax, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %ecx, 12(%eax)
+; X86-NEXT:    movl %edx, 8(%eax)
+; X86-NEXT:    movl %esi, 4(%eax)
+; X86-NEXT:    movl %edi, (%eax)
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+  %r = call <4 x i32> @llvm.umax.v4i32(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %r
+}
+
+define <8 x i32> @test_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
+; SSE-LABEL: test_v8i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm5 = [2147483648,2147483648,2147483648,2147483648]
+; SSE-NEXT:    movdqa %xmm2, %xmm6
+; SSE-NEXT:    pxor %xmm5, %xmm6
+; SSE-NEXT:    movdqa %xmm0, %xmm4
+; SSE-NEXT:    pxor %xmm5, %xmm4
+; SSE-NEXT:    pcmpgtd %xmm6, %xmm4
+; SSE-NEXT:    pand %xmm4, %xmm0
+; SSE-NEXT:    pandn %xmm2, %xmm4
+; SSE-NEXT:    por %xmm0, %xmm4
+; SSE-NEXT:    movdqa %xmm3, %xmm0
+; SSE-NEXT:    pxor %xmm5, %xmm0
+; SSE-NEXT:    pxor %xmm1, %xmm5
+; SSE-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE-NEXT:    pand %xmm5, %xmm1
+; SSE-NEXT:    pandn %xmm3, %xmm5
+; SSE-NEXT:    por %xmm5, %xmm1
+; SSE-NEXT:    movdqa %xmm4, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: test_v8i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpmaxud %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test_v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmaxud %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; X86-LABEL: test_v8i32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %ebp, %eax
+; X86-NEXT:    cmoval %eax, %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %ebx, %eax
+; X86-NEXT:    cmoval %eax, %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %edi, %eax
+; X86-NEXT:    cmoval %eax, %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %esi, %eax
+; X86-NEXT:    cmoval %eax, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %edx, %eax
+; X86-NEXT:    cmoval %eax, %edx
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %ecx, %eax
+; X86-NEXT:    cmoval %eax, %ecx
+; X86-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %ecx, %eax
+; X86-NEXT:    cmoval %eax, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    cmpl %eax, %edx
+; X86-NEXT:    cmoval %edx, %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %eax, 28(%edx)
+; X86-NEXT:    movl %ecx, 24(%edx)
+; X86-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, 20(%edx)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, 16(%edx)
+; X86-NEXT:    movl %esi, 12(%edx)
+; X86-NEXT:    movl %edi, 8(%edx)
+; X86-NEXT:    movl %ebx, 4(%edx)
+; X86-NEXT:    movl %ebp, (%edx)
+; X86-NEXT:    movl %edx, %eax
+; X86-NEXT:    addl $8, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+  %r = call <8 x i32> @llvm.umax.v8i32(<8 x i32> %a, <8 x i32> %b)
+  ret <8 x i32> %r
+}
+
+define <8 x i16> @test_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
+; SSE-LABEL: test_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
+; SSE-NEXT:    pxor %xmm2, %xmm1
+; SSE-NEXT:    pxor %xmm2, %xmm0
+; SSE-NEXT:    pmaxsw %xmm1, %xmm0
+; SSE-NEXT:    pxor %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_v8i16:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %bp, %ax
+; X86-NEXT:    cmoval %eax, %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %bx, %ax
+; X86-NEXT:    cmoval %eax, %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %di, %ax
+; X86-NEXT:    cmoval %eax, %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %si, %ax
+; X86-NEXT:    cmoval %eax, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %dx, %ax
+; X86-NEXT:    cmoval %eax, %edx
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %cx, %ax
+; X86-NEXT:    cmoval %eax, %ecx
+; X86-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %dx, %ax
+; X86-NEXT:    cmoval %eax, %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmpw %ax, %cx
+; X86-NEXT:    cmoval %ecx, %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movw %ax, 14(%ecx)
+; X86-NEXT:    movw %dx, 12(%ecx)
+; X86-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-NEXT:    movw %ax, 10(%ecx)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movw %ax, 8(%ecx)
+; X86-NEXT:    movw %si, 6(%ecx)
+; X86-NEXT:    movw %di, 4(%ecx)
+; X86-NEXT:    movw %bx, 2(%ecx)
+; X86-NEXT:    movw %bp, (%ecx)
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    addl $8, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+  %r = call <8 x i16> @llvm.umax.v8i16(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %r
+}
+
+define <16 x i8> @test_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
+; SSE-LABEL: test_v16i8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pmaxub %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v16i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_v16i8:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $40, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %bl, %al
+; X86-NEXT:    cmoval %eax, %ebx
+; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %dl, %al
+; X86-NEXT:    cmoval %eax, %edx
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmoval %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmoval %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmoval %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmoval %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmoval %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmoval %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmoval %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmoval %eax, %ecx
+; X86-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmoval %eax, %ecx
+; X86-NEXT:    movl %ecx, %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmoval %eax, %ecx
+; X86-NEXT:    movl %ecx, %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmoval %eax, %ecx
+; X86-NEXT:    movl %ecx, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %bl, %al
+; X86-NEXT:    cmoval %eax, %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %dl, %al
+; X86-NEXT:    cmoval %eax, %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmoval %eax, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movb %cl, 15(%eax)
+; X86-NEXT:    movb %dl, 14(%eax)
+; X86-NEXT:    movb %bl, 13(%eax)
+; X86-NEXT:    movl %esi, %ecx
+; X86-NEXT:    movb %cl, 12(%eax)
+; X86-NEXT:    movl %edi, %ecx
+; X86-NEXT:    movb %cl, 11(%eax)
+; X86-NEXT:    movl %ebp, %ecx
+; X86-NEXT:    movb %cl, 10(%eax)
+; X86-NEXT:    movl (%esp), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 9(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 8(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 7(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 6(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 5(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 4(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 3(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 2(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 1(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, (%eax)
+; X86-NEXT:    addl $40, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+  %r = call <16 x i8> @llvm.umax.v16i8(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %r
+}
diff --git a/llvm/test/CodeGen/X86/umin.ll b/llvm/test/CodeGen/X86/umin.ll
new file mode 100644
index 0000000000000..234c4faf6cd2b
--- /dev/null
+++ b/llvm/test/CodeGen/X86/umin.ll
@@ -0,0 +1,667 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s --check-prefixes=X64,SSE
+; RUN: llc < %s -mtriple=x86_64-linux -mattr=avx | FileCheck %s --check-prefixes=X64,AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-linux -mattr=avx2 | FileCheck %s --check-prefixes=X64,AVX,AVX2
+; RUN: llc < %s -mtriple=i686 -mattr=cmov | FileCheck %s --check-prefix=X86
+
+declare i8 @llvm.umin.i8(i8, i8)
+declare i16 @llvm.umin.i16(i16, i16)
+declare i24 @llvm.umin.i24(i24, i24)
+declare i32 @llvm.umin.i32(i32, i32)
+declare i64 @llvm.umin.i64(i64, i64)
+declare i128 @llvm.umin.i128(i128, i128)
+
+declare <1 x i32> @llvm.umin.v1i32(<1 x i32>, <1 x i32>)
+declare <2 x i32> @llvm.umin.v2i32(<2 x i32>, <2 x i32>)
+declare <3 x i32> @llvm.umin.v3i32(<3 x i32>, <3 x i32>)
+declare <4 x i32> @llvm.umin.v4i32(<4 x i32>, <4 x i32>)
+declare <8 x i32> @llvm.umin.v8i32(<8 x i32>, <8 x i32>)
+
+declare <8 x i16> @llvm.umin.v8i16(<8 x i16>, <8 x i16>)
+declare <16 x i8> @llvm.umin.v16i8(<16 x i8>, <16 x i8>)
+
+define i8 @test_i8(i8 %a, i8 %b) nounwind {
+; X64-LABEL: test_i8:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    cmpb %al, %dil
+; X64-NEXT:    cmovbl %edi, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i8:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmpb %al, %cl
+; X86-NEXT:    cmovbl %ecx, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+  %r = call i8 @llvm.umin.i8(i8 %a, i8 %b)
+  ret i8 %r
+}
+
+define i16 @test_i16(i16 %a, i16 %b) nounwind {
+; X64-LABEL: test_i16:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    cmpw %ax, %di
+; X64-NEXT:    cmovbl %edi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i16:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmpw %ax, %cx
+; X86-NEXT:    cmovbl %ecx, %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    retl
+  %r = call i16 @llvm.umin.i16(i16 %a, i16 %b)
+  ret i16 %r
+}
+
+define i24 @test_i24(i24 %a, i24 %b) nounwind {
+; X64-LABEL: test_i24:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    andl $16777215, %esi # imm = 0xFFFFFF
+; X64-NEXT:    andl $16777215, %eax # imm = 0xFFFFFF
+; X64-NEXT:    cmpl %esi, %eax
+; X64-NEXT:    cmovael %esi, %eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i24:
+; X86:       # %bb.0:
+; X86-NEXT:    movl $16777215, %eax # imm = 0xFFFFFF
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    andl %eax, %ecx
+; X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %ecx, %eax
+; X86-NEXT:    cmovael %ecx, %eax
+; X86-NEXT:    retl
+  %r = call i24 @llvm.umin.i24(i24 %a, i24 %b)
+  ret i24 %r
+}
+
+define i32 @test_i32(i32 %a, i32 %b) nounwind {
+; X64-LABEL: test_i32:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    cmpl %esi, %edi
+; X64-NEXT:    cmovbl %edi, %eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmpl %eax, %ecx
+; X86-NEXT:    cmovbl %ecx, %eax
+; X86-NEXT:    retl
+  %r = call i32 @llvm.umin.i32(i32 %a, i32 %b)
+  ret i32 %r
+}
+
+define i64 @test_i64(i64 %a, i64 %b) nounwind {
+; X64-LABEL: test_i64:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rsi, %rax
+; X64-NEXT:    cmpq %rsi, %rdi
+; X64-NEXT:    cmovbq %rdi, %rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i64:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    cmpl %eax, %ecx
+; X86-NEXT:    movl %eax, %edi
+; X86-NEXT:    cmovbl %ecx, %edi
+; X86-NEXT:    cmpl %edx, %esi
+; X86-NEXT:    cmovbl %ecx, %eax
+; X86-NEXT:    cmovel %edi, %eax
+; X86-NEXT:    cmovbl %esi, %edx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl
+  %r = call i64 @llvm.umin.i64(i64 %a, i64 %b)
+  ret i64 %r
+}
+
+define i128 @test_i128(i128 %a, i128 %b) nounwind {
+; X64-LABEL: test_i128:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdx, %rax
+; X64-NEXT:    cmpq %rdx, %rdi
+; X64-NEXT:    cmovbq %rdi, %rdx
+; X64-NEXT:    cmpq %rcx, %rsi
+; X64-NEXT:    cmovbq %rdi, %rax
+; X64-NEXT:    cmoveq %rdx, %rax
+; X64-NEXT:    cmovbq %rsi, %rcx
+; X64-NEXT:    movq %rcx, %rdx
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i128:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    cmpl %ecx, %eax
+; X86-NEXT:    movl %ecx, %ebx
+; X86-NEXT:    cmovbl %eax, %ebx
+; X86-NEXT:    cmpl %esi, %edi
+; X86-NEXT:    movl %ecx, %ebp
+; X86-NEXT:    cmovbl %eax, %ebp
+; X86-NEXT:    cmovel %ebx, %ebp
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    cmovbl %edi, %eax
+; X86-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    cmpl %edx, %edi
+; X86-NEXT:    movl %edx, %eax
+; X86-NEXT:    cmovbl %edi, %eax
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl %ebx, %edi
+; X86-NEXT:    sbbl %eax, %edi
+; X86-NEXT:    cmovbl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    cmovbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ebx, %edi
+; X86-NEXT:    xorl %eax, %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    xorl %edx, %ebx
+; X86-NEXT:    orl %edi, %ebx
+; X86-NEXT:    cmovel %ebp, %ecx
+; X86-NEXT:    cmovel (%esp), %esi # 4-byte Folded Reload
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    cmpl %eax, %edi
+; X86-NEXT:    cmovbl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    cmovel {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT:    cmovbl %edi, %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %eax, 12(%edi)
+; X86-NEXT:    movl %edx, 8(%edi)
+; X86-NEXT:    movl %esi, 4(%edi)
+; X86-NEXT:    movl %ecx, (%edi)
+; X86-NEXT:    movl %edi, %eax
+; X86-NEXT:    addl $8, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+  %r = call i128 @llvm.umin.i128(i128 %a, i128 %b)
+  ret i128 %r
+}
+
+define <1 x i32> @test_v1i32(<1 x i32> %a, <1 x i32> %b) nounwind {
+; X64-LABEL: test_v1i32:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    cmpl %esi, %edi
+; X64-NEXT:    cmovbl %edi, %eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_v1i32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmpl %eax, %ecx
+; X86-NEXT:    cmovbl %ecx, %eax
+; X86-NEXT:    retl
+  %r = call <1 x i32> @llvm.umin.v1i32(<1 x i32> %a, <1 x i32> %b)
+  ret <1 x i32> %r
+}
+
+define <2 x i32> @test_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
+; SSE-LABEL: test_v2i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSE-NEXT:    movdqa %xmm0, %xmm3
+; SSE-NEXT:    pxor %xmm2, %xmm3
+; SSE-NEXT:    pxor %xmm1, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    pandn %xmm1, %xmm2
+; SSE-NEXT:    por %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v2i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpminud %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_v2i32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    cmpl %eax, %esi
+; X86-NEXT:    cmovbl %esi, %eax
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    cmovbl %ecx, %edx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+  %r = call <2 x i32> @llvm.umin.v2i32(<2 x i32> %a, <2 x i32> %b)
+  ret <2 x i32> %r
+}
+
+define <3 x i32> @test_v3i32(<3 x i32> %a, <3 x i32> %b) nounwind {
+; SSE-LABEL: test_v3i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSE-NEXT:    movdqa %xmm0, %xmm3
+; SSE-NEXT:    pxor %xmm2, %xmm3
+; SSE-NEXT:    pxor %xmm1, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    pandn %xmm1, %xmm2
+; SSE-NEXT:    por %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v3i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpminud %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_v3i32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    cmpl %eax, %ebx
+; X86-NEXT:    cmovbl %ebx, %eax
+; X86-NEXT:    cmpl %edx, %edi
+; X86-NEXT:    cmovbl %edi, %edx
+; X86-NEXT:    cmpl %ecx, %esi
+; X86-NEXT:    cmovbl %esi, %ecx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    retl
+  %r = call <3 x i32> @llvm.umin.v3i32(<3 x i32> %a, <3 x i32> %b)
+  ret <3 x i32> %r
+}
+
+define <4 x i32> @test_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
+; SSE-LABEL: test_v4i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSE-NEXT:    movdqa %xmm0, %xmm3
+; SSE-NEXT:    pxor %xmm2, %xmm3
+; SSE-NEXT:    pxor %xmm1, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    pandn %xmm1, %xmm2
+; SSE-NEXT:    por %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v4i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpminud %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_v4i32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %edi, %eax
+; X86-NEXT:    cmovbl %eax, %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %esi, %eax
+; X86-NEXT:    cmovbl %eax, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %edx, %eax
+; X86-NEXT:    cmovbl %eax, %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %ecx, %eax
+; X86-NEXT:    cmovbl %eax, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %ecx, 12(%eax)
+; X86-NEXT:    movl %edx, 8(%eax)
+; X86-NEXT:    movl %esi, 4(%eax)
+; X86-NEXT:    movl %edi, (%eax)
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+  %r = call <4 x i32> @llvm.umin.v4i32(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %r
+}
+
+define <8 x i32> @test_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
+; SSE-LABEL: test_v8i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
+; SSE-NEXT:    movdqa %xmm0, %xmm5
+; SSE-NEXT:    pxor %xmm4, %xmm5
+; SSE-NEXT:    movdqa %xmm2, %xmm6
+; SSE-NEXT:    pxor %xmm4, %xmm6
+; SSE-NEXT:    pcmpgtd %xmm5, %xmm6
+; SSE-NEXT:    pand %xmm6, %xmm0
+; SSE-NEXT:    pandn %xmm2, %xmm6
+; SSE-NEXT:    por %xmm6, %xmm0
+; SSE-NEXT:    movdqa %xmm1, %xmm2
+; SSE-NEXT:    pxor %xmm4, %xmm2
+; SSE-NEXT:    pxor %xmm3, %xmm4
+; SSE-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE-NEXT:    pand %xmm4, %xmm1
+; SSE-NEXT:    pandn %xmm3, %xmm4
+; SSE-NEXT:    por %xmm4, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: test_v8i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpminud %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test_v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpminud %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; X86-LABEL: test_v8i32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %ebp, %eax
+; X86-NEXT:    cmovbl %eax, %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %ebx, %eax
+; X86-NEXT:    cmovbl %eax, %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %edi, %eax
+; X86-NEXT:    cmovbl %eax, %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %esi, %eax
+; X86-NEXT:    cmovbl %eax, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %edx, %eax
+; X86-NEXT:    cmovbl %eax, %edx
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %ecx, %eax
+; X86-NEXT:    cmovbl %eax, %ecx
+; X86-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl %ecx, %eax
+; X86-NEXT:    cmovbl %eax, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    cmpl %eax, %edx
+; X86-NEXT:    cmovbl %edx, %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %eax, 28(%edx)
+; X86-NEXT:    movl %ecx, 24(%edx)
+; X86-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, 20(%edx)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, 16(%edx)
+; X86-NEXT:    movl %esi, 12(%edx)
+; X86-NEXT:    movl %edi, 8(%edx)
+; X86-NEXT:    movl %ebx, 4(%edx)
+; X86-NEXT:    movl %ebp, (%edx)
+; X86-NEXT:    movl %edx, %eax
+; X86-NEXT:    addl $8, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+  %r = call <8 x i32> @llvm.umin.v8i32(<8 x i32> %a, <8 x i32> %b)
+  ret <8 x i32> %r
+}
+
+define <8 x i16> @test_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
+; SSE-LABEL: test_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
+; SSE-NEXT:    pxor %xmm2, %xmm1
+; SSE-NEXT:    pxor %xmm2, %xmm0
+; SSE-NEXT:    pminsw %xmm1, %xmm0
+; SSE-NEXT:    pxor %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpminuw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_v8i16:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %bp, %ax
+; X86-NEXT:    cmovbl %eax, %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %bx, %ax
+; X86-NEXT:    cmovbl %eax, %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %di, %ax
+; X86-NEXT:    cmovbl %eax, %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %si, %ax
+; X86-NEXT:    cmovbl %eax, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %dx, %ax
+; X86-NEXT:    cmovbl %eax, %edx
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %cx, %ax
+; X86-NEXT:    cmovbl %eax, %ecx
+; X86-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpw %dx, %ax
+; X86-NEXT:    cmovbl %eax, %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmpw %ax, %cx
+; X86-NEXT:    cmovbl %ecx, %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movw %ax, 14(%ecx)
+; X86-NEXT:    movw %dx, 12(%ecx)
+; X86-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-NEXT:    movw %ax, 10(%ecx)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movw %ax, 8(%ecx)
+; X86-NEXT:    movw %si, 6(%ecx)
+; X86-NEXT:    movw %di, 4(%ecx)
+; X86-NEXT:    movw %bx, 2(%ecx)
+; X86-NEXT:    movw %bp, (%ecx)
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    addl $8, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+  %r = call <8 x i16> @llvm.umin.v8i16(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %r
+}
+
+define <16 x i8> @test_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
+; SSE-LABEL: test_v16i8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pminub %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test_v16i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_v16i8:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    subl $40, %esp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %bl, %al
+; X86-NEXT:    cmovbl %eax, %ebx
+; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %dl, %al
+; X86-NEXT:    cmovbl %eax, %edx
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovbl %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovbl %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovbl %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovbl %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovbl %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovbl %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovbl %eax, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovbl %eax, %ecx
+; X86-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovbl %eax, %ecx
+; X86-NEXT:    movl %ecx, %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovbl %eax, %ecx
+; X86-NEXT:    movl %ecx, %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovbl %eax, %ecx
+; X86-NEXT:    movl %ecx, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %bl, %al
+; X86-NEXT:    cmovbl %eax, %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %dl, %al
+; X86-NEXT:    cmovbl %eax, %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpb %cl, %al
+; X86-NEXT:    cmovbl %eax, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movb %cl, 15(%eax)
+; X86-NEXT:    movb %dl, 14(%eax)
+; X86-NEXT:    movb %bl, 13(%eax)
+; X86-NEXT:    movl %esi, %ecx
+; X86-NEXT:    movb %cl, 12(%eax)
+; X86-NEXT:    movl %edi, %ecx
+; X86-NEXT:    movb %cl, 11(%eax)
+; X86-NEXT:    movl %ebp, %ecx
+; X86-NEXT:    movb %cl, 10(%eax)
+; X86-NEXT:    movl (%esp), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 9(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 8(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 7(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 6(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 5(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 4(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 3(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 2(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, 1(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movb %cl, (%eax)
+; X86-NEXT:    addl $40, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+  %r = call <16 x i8> @llvm.umin.v16i8(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %r
+}
diff --git a/llvm/test/CodeGen/X86/urem-seteq-vec-nonsplat.ll b/llvm/test/CodeGen/X86/urem-seteq-vec-nonsplat.ll
index b83dcb6a0cd86..9fdb08ba6d4e8 100644
--- a/llvm/test/CodeGen/X86/urem-seteq-vec-nonsplat.ll
+++ b/llvm/test/CodeGen/X86/urem-seteq-vec-nonsplat.ll
@@ -18,23 +18,22 @@ define <4 x i32> @test_urem_odd_even(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[3,3]
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm1
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
-; CHECK-SSE2-NEXT:    psrld $5, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
 ; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
 ; CHECK-SSE2-NEXT:    psrld $2, %xmm3
 ; CHECK-SSE2-NEXT:    psrld $3, %xmm2
 ; CHECK-SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm3[0],xmm2[1]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [5,14,25,100]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    psrld $5, %xmm1
 ; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm1[3,3]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [5,14,25,100]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm4
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
 ; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    psrld $31, %xmm0
@@ -50,15 +49,14 @@ define <4 x i32> @test_urem_odd_even(<4 x i32> %X) nounwind {
 ; CHECK-SSE41-NEXT:    pmuludq %xmm1, %xmm3
 ; CHECK-SSE41-NEXT:    pmuludq %xmm0, %xmm2
 ; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
 ; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE41-NEXT:    psrld $5, %xmm2
-; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm3
-; CHECK-SSE41-NEXT:    psrld $2, %xmm3
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    psrld $2, %xmm2
+; CHECK-SSE41-NEXT:    psrld $5, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm2[0,1,2,3],xmm3[4,5,6,7]
 ; CHECK-SSE41-NEXT:    psrld $3, %xmm1
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
 ; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm1
 ; CHECK-SSE41-NEXT:    psubd %xmm1, %xmm0
 ; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
@@ -75,13 +73,13 @@ define <4 x i32> @test_urem_odd_even(<4 x i32> %X) nounwind {
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; CHECK-AVX1-NEXT:    vpsrld $5, %xmm1, %xmm2
-; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm3
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; CHECK-AVX1-NEXT:    vpsrld $3, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $2, %xmm3, %xmm3
+; CHECK-AVX1-NEXT:    vpsrld $5, %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $3, %xmm2, %xmm2
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
 ; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
@@ -237,7 +235,6 @@ define <4 x i32> @test_urem_even_allones_eq(<4 x i32> %X) nounwind {
 ; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; CHECK-SSE41-NEXT:    pmuludq {{.*}}(%rip), %xmm2
 ; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
 ; CHECK-SSE41-NEXT:    psrld $31, %xmm1
 ; CHECK-SSE41-NEXT:    psrld $2, %xmm2
 ; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5],xmm2[6,7]
@@ -254,12 +251,12 @@ define <4 x i32> @test_urem_even_allones_eq(<4 x i32> %X) nounwind {
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm1[0,1,2,3],xmm0[4,5],xmm1[6,7]
 ; CHECK-AVX1-NEXT:    vpmuludq {{.*}}(%rip), %xmm2, %xmm2
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm2, %xmm3
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; CHECK-AVX1-NEXT:    vpmuludq {{.*}}(%rip), %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm2
 ; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm1
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4,5],xmm1[6,7]
 ; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
@@ -338,7 +335,6 @@ define <4 x i32> @test_urem_even_allones_ne(<4 x i32> %X) nounwind {
 ; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; CHECK-SSE41-NEXT:    pmuludq {{.*}}(%rip), %xmm2
 ; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
 ; CHECK-SSE41-NEXT:    psrld $31, %xmm1
 ; CHECK-SSE41-NEXT:    psrld $2, %xmm2
 ; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5],xmm2[6,7]
@@ -355,12 +351,12 @@ define <4 x i32> @test_urem_even_allones_ne(<4 x i32> %X) nounwind {
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm1[0,1,2,3],xmm0[4,5],xmm1[6,7]
 ; CHECK-AVX1-NEXT:    vpmuludq {{.*}}(%rip), %xmm2, %xmm2
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm2, %xmm3
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; CHECK-AVX1-NEXT:    vpmuludq {{.*}}(%rip), %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm2
 ; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm1
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm3[4,5],xmm1[6,7]
 ; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
@@ -413,23 +409,22 @@ define <4 x i32> @test_urem_odd_even_allones_eq(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[3,3]
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm1
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
-; CHECK-SSE2-NEXT:    psrld $5, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
 ; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
 ; CHECK-SSE2-NEXT:    psrld $2, %xmm3
 ; CHECK-SSE2-NEXT:    psrld $31, %xmm2
 ; CHECK-SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm3[0],xmm2[1]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [5,14,4294967295,100]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    psrld $5, %xmm1
 ; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm1[3,3]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [5,14,4294967295,100]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm4
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
 ; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    psrld $31, %xmm0
@@ -445,15 +440,14 @@ define <4 x i32> @test_urem_odd_even_allones_eq(<4 x i32> %X) nounwind {
 ; CHECK-SSE41-NEXT:    pmuludq %xmm1, %xmm3
 ; CHECK-SSE41-NEXT:    pmuludq %xmm0, %xmm2
 ; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
 ; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE41-NEXT:    psrld $5, %xmm2
-; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm3
-; CHECK-SSE41-NEXT:    psrld $2, %xmm3
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    psrld $2, %xmm2
+; CHECK-SSE41-NEXT:    psrld $5, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm2[0,1,2,3],xmm3[4,5,6,7]
 ; CHECK-SSE41-NEXT:    psrld $31, %xmm1
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
 ; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm1
 ; CHECK-SSE41-NEXT:    psubd %xmm1, %xmm0
 ; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
@@ -470,13 +464,13 @@ define <4 x i32> @test_urem_odd_even_allones_eq(<4 x i32> %X) nounwind {
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; CHECK-AVX1-NEXT:    vpsrld $5, %xmm1, %xmm2
-; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm3
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $2, %xmm3, %xmm3
+; CHECK-AVX1-NEXT:    vpsrld $5, %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm2, %xmm2
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
 ; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
@@ -527,23 +521,22 @@ define <4 x i32> @test_urem_odd_even_allones_ne(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[3,3]
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm1
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
-; CHECK-SSE2-NEXT:    psrld $5, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
 ; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
 ; CHECK-SSE2-NEXT:    psrld $2, %xmm3
 ; CHECK-SSE2-NEXT:    psrld $31, %xmm2
 ; CHECK-SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm3[0],xmm2[1]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [5,14,4294967295,100]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    psrld $5, %xmm1
 ; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm1[3,3]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [5,14,4294967295,100]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm4
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
 ; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    pandn {{.*}}(%rip), %xmm0
@@ -559,15 +552,14 @@ define <4 x i32> @test_urem_odd_even_allones_ne(<4 x i32> %X) nounwind {
 ; CHECK-SSE41-NEXT:    pmuludq %xmm1, %xmm3
 ; CHECK-SSE41-NEXT:    pmuludq %xmm0, %xmm2
 ; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
 ; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE41-NEXT:    psrld $5, %xmm2
-; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm3
-; CHECK-SSE41-NEXT:    psrld $2, %xmm3
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    psrld $2, %xmm2
+; CHECK-SSE41-NEXT:    psrld $5, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm2[0,1,2,3],xmm3[4,5,6,7]
 ; CHECK-SSE41-NEXT:    psrld $31, %xmm1
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
 ; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm1
 ; CHECK-SSE41-NEXT:    psubd %xmm1, %xmm0
 ; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
@@ -584,13 +576,13 @@ define <4 x i32> @test_urem_odd_even_allones_ne(<4 x i32> %X) nounwind {
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; CHECK-AVX1-NEXT:    vpsrld $5, %xmm1, %xmm2
-; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm3
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $2, %xmm3, %xmm3
+; CHECK-AVX1-NEXT:    vpsrld $5, %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm2, %xmm2
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
 ; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
@@ -835,22 +827,21 @@ define <4 x i32> @test_urem_odd_even_poweroftwo(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[3,3]
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm1
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
-; CHECK-SSE2-NEXT:    psrld $5, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
 ; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
 ; CHECK-SSE2-NEXT:    psrld $2, %xmm3
 ; CHECK-SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm3[0],xmm2[1]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [5,14,16,100]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    psrld $5, %xmm1
 ; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm1[3,3]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [5,14,16,100]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm4
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
 ; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    psrld $31, %xmm0
@@ -868,12 +859,11 @@ define <4 x i32> @test_urem_odd_even_poweroftwo(<4 x i32> %X) nounwind {
 ; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
 ; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
 ; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE41-NEXT:    psrld $5, %xmm2
-; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm3
-; CHECK-SSE41-NEXT:    psrld $2, %xmm3
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm1[4,5,6,7]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    psrld $2, %xmm2
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-SSE41-NEXT:    psrld $5, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm2[0,1,2,3],xmm3[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
 ; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm3
 ; CHECK-SSE41-NEXT:    psubd %xmm3, %xmm0
 ; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
@@ -890,12 +880,12 @@ define <4 x i32> @test_urem_odd_even_poweroftwo(<4 x i32> %X) nounwind {
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; CHECK-AVX1-NEXT:    vpsrld $5, %xmm1, %xmm2
-; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $2, %xmm2, %xmm3
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpsrld $5, %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
 ; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
@@ -1105,24 +1095,23 @@ define <4 x i32> @test_urem_odd_even_one(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[3,3]
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm1
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
-; CHECK-SSE2-NEXT:    psrld $5, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
 ; CHECK-SSE2-NEXT:    psrld $2, %xmm2
-; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm1[3,3]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [5,14,1,100]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    movaps %xmm0, %xmm3
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[2,0],xmm2[3,0]
+; CHECK-SSE2-NEXT:    movaps %xmm2, %xmm4
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,1],xmm3[0,2]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [5,14,1,100]
 ; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm4
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
-; CHECK-SSE2-NEXT:    movaps %xmm0, %xmm4
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[2,0],xmm2[3,0]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm4[0,2]
-; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
+; CHECK-SSE2-NEXT:    psrld $5, %xmm1
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
+; CHECK-SSE2-NEXT:    psubd %xmm4, %xmm0
 ; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    psrld $31, %xmm0
@@ -1139,13 +1128,12 @@ define <4 x i32> @test_urem_odd_even_one(<4 x i32> %X) nounwind {
 ; CHECK-SSE41-NEXT:    pmuludq %xmm0, %xmm2
 ; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
 ; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
-; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE41-NEXT:    psrld $5, %xmm2
 ; CHECK-SSE41-NEXT:    psrld $2, %xmm1
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm0[4,5],xmm1[6,7]
-; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm1
-; CHECK-SSE41-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    psrld $5, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm1[0,1,2,3],xmm3[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm0[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm3
+; CHECK-SSE41-NEXT:    psubd %xmm3, %xmm0
 ; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE41-NEXT:    psrld $31, %xmm0
@@ -1160,10 +1148,10 @@ define <4 x i32> @test_urem_odd_even_one(<4 x i32> %X) nounwind {
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; CHECK-AVX1-NEXT:    vpsrld $5, %xmm1, %xmm2
-; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm1
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $2, %xmm2, %xmm2
+; CHECK-AVX1-NEXT:    vpsrld $5, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm0[4,5],xmm1[6,7]
 ; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
@@ -1409,22 +1397,21 @@ define <4 x i32> @test_urem_odd_even_INT_MIN(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[3,3]
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm1
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
-; CHECK-SSE2-NEXT:    psrld $5, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
 ; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
 ; CHECK-SSE2-NEXT:    psrld $2, %xmm3
 ; CHECK-SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm3[0],xmm2[1]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [5,14,2147483648,100]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    psrld $5, %xmm1
 ; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm1[3,3]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [5,14,2147483648,100]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm4
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
 ; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    psrld $31, %xmm0
@@ -1442,12 +1429,11 @@ define <4 x i32> @test_urem_odd_even_INT_MIN(<4 x i32> %X) nounwind {
 ; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
 ; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
 ; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE41-NEXT:    psrld $5, %xmm2
-; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm3
-; CHECK-SSE41-NEXT:    psrld $2, %xmm3
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm1[4,5,6,7]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    psrld $2, %xmm2
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-SSE41-NEXT:    psrld $5, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm2[0,1,2,3],xmm3[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
 ; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm3
 ; CHECK-SSE41-NEXT:    psubd %xmm3, %xmm0
 ; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
@@ -1464,12 +1450,12 @@ define <4 x i32> @test_urem_odd_even_INT_MIN(<4 x i32> %X) nounwind {
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; CHECK-AVX1-NEXT:    vpsrld $5, %xmm1, %xmm2
-; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $2, %xmm2, %xmm3
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpsrld $5, %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
 ; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
@@ -1526,17 +1512,16 @@ define <4 x i32> @test_urem_odd_allones_and_poweroftwo(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
 ; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; CHECK-SSE2-NEXT:    psrld $2, %xmm1
-; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
 ; CHECK-SSE2-NEXT:    psrld $31, %xmm3
 ; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm1[3,3]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [5,4294967295,16,5]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm5
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,2,2,3]
 ; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],xmm2[2,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [5,4294967295,16,5]
+; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm1
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
 ; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
@@ -1554,13 +1539,12 @@ define <4 x i32> @test_urem_odd_allones_and_poweroftwo(<4 x i32> %X) nounwind {
 ; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
 ; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
 ; CHECK-SSE41-NEXT:    psrld $2, %xmm2
-; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
 ; CHECK-SSE41-NEXT:    psrld $31, %xmm3
 ; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5,6,7]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
-; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
-; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm3
+; CHECK-SSE41-NEXT:    psubd %xmm3, %xmm0
 ; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE41-NEXT:    psrld $31, %xmm0
@@ -1575,11 +1559,11 @@ define <4 x i32> @test_urem_odd_allones_and_poweroftwo(<4 x i32> %X) nounwind {
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm1
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm2
-; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm3
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm2, %xmm2
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
 ; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
@@ -1637,18 +1621,17 @@ define <4 x i32> @test_urem_even_allones_and_poweroftwo(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
 ; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; CHECK-SSE2-NEXT:    psrld $2, %xmm2
-; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm3
-; CHECK-SSE2-NEXT:    psrld $31, %xmm3
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm2[3,3]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [14,4294967295,16,14]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm5
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,2,2,3]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm4
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1],xmm2[3,3]
 ; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm1[2,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm2
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [14,4294967295,16,14]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
 ; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    psrld $31, %xmm0
@@ -1668,13 +1651,12 @@ define <4 x i32> @test_urem_even_allones_and_poweroftwo(<4 x i32> %X) nounwind {
 ; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,3],xmm1[4,5],xmm4[6,7]
 ; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
 ; CHECK-SSE41-NEXT:    psrld $2, %xmm2
-; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm3
-; CHECK-SSE41-NEXT:    psrld $31, %xmm3
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5,6,7]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
-; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
-; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-SSE41-NEXT:    psrld $31, %xmm4
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm1[0,1],xmm4[2,3],xmm1[4,5],xmm4[6,7]
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm4
+; CHECK-SSE41-NEXT:    psubd %xmm4, %xmm0
 ; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE41-NEXT:    psrld $31, %xmm0
@@ -1691,11 +1673,11 @@ define <4 x i32> @test_urem_even_allones_and_poweroftwo(<4 x i32> %X) nounwind {
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm2
-; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm3
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm2, %xmm2
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
 ; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
@@ -1740,29 +1722,26 @@ define <4 x i32> @test_urem_odd_even_allones_and_poweroftwo(<4 x i32> %X) nounwi
 ; CHECK-SSE2-LABEL: test_urem_odd_even_allones_and_poweroftwo:
 ; CHECK-SSE2:       # %bb.0:
 ; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [3435973837,2147483649,268435456,1374389535]
-; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
-; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm3
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
-; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
-; CHECK-SSE2-NEXT:    psrld $2, %xmm3
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,3],xmm2[2,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    movdqa %xmm3, %xmm2
 ; CHECK-SSE2-NEXT:    psrld $5, %xmm2
-; CHECK-SSE2-NEXT:    psrld $31, %xmm1
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[3,3]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm3
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm2[3,3]
 ; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [5,4294967295,16,100]
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm4
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[0,2,2,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm3
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm4
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm0, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    psrld $2, %xmm1
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],xmm4[2,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    psrld $31, %xmm0
@@ -1774,19 +1753,18 @@ define <4 x i32> @test_urem_odd_even_allones_and_poweroftwo(<4 x i32> %X) nounwi
 ; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
 ; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; CHECK-SSE41-NEXT:    pmuludq %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    movdqa %xmm3, %xmm2
+; CHECK-SSE41-NEXT:    psrld $5, %xmm2
+; CHECK-SSE41-NEXT:    psrld $31, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm2[4,5,6,7]
 ; CHECK-SSE41-NEXT:    pmuludq %xmm0, %xmm1
 ; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
 ; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm3
-; CHECK-SSE41-NEXT:    psrld $2, %xmm3
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm1[4,5,6,7]
-; CHECK-SSE41-NEXT:    psrld $5, %xmm1
-; CHECK-SSE41-NEXT:    psrld $31, %xmm2
+; CHECK-SSE41-NEXT:    psrld $2, %xmm2
 ; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5,6,7]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
-; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm3
-; CHECK-SSE41-NEXT:    psubd %xmm3, %xmm0
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
 ; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE41-NEXT:    psrld $31, %xmm0
@@ -1798,12 +1776,11 @@ define <4 x i32> @test_urem_odd_even_allones_and_poweroftwo(<4 x i32> %X) nounwi
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
+; CHECK-AVX1-NEXT:    vpsrld $5, %xmm2, %xmm3
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm2, %xmm2
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4,5,6,7]
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm1
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; CHECK-AVX1-NEXT:    vpsrld $5, %xmm1, %xmm2
-; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm3
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
 ; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm3
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
@@ -1903,21 +1880,20 @@ define <4 x i32> @test_urem_even_allones_and_one(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm1
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
 ; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    psrld $2, %xmm2
-; CHECK-SSE2-NEXT:    psrld $31, %xmm1
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[3,3]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [14,4294967295,1,14]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm4
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[0,2,2,3]
-; CHECK-SSE2-NEXT:    movaps %xmm0, %xmm4
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[2,0],xmm2[3,0]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm4[0,2]
-; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    psrld $2, %xmm1
+; CHECK-SSE2-NEXT:    movaps %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,0],xmm1[3,0]
+; CHECK-SSE2-NEXT:    psrld $31, %xmm4
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1],xmm1[3,3]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0,2]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [14,4294967295,1,14]
+; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm2
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    psrld $31, %xmm0
@@ -1935,13 +1911,12 @@ define <4 x i32> @test_urem_even_allones_and_one(<4 x i32> %X) nounwind {
 ; CHECK-SSE41-NEXT:    pmuludq %xmm3, %xmm1
 ; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,3],xmm1[4,5],xmm4[6,7]
-; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE41-NEXT:    psrld $31, %xmm2
 ; CHECK-SSE41-NEXT:    psrld $2, %xmm1
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5,6,7]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm0[4,5],xmm1[6,7]
-; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm1
-; CHECK-SSE41-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    psrld $31, %xmm4
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm1[0,1],xmm4[2,3],xmm1[4,5,6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm0[4,5],xmm4[6,7]
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm4
+; CHECK-SSE41-NEXT:    psubd %xmm4, %xmm0
 ; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE41-NEXT:    psrld $31, %xmm0
@@ -1958,8 +1933,8 @@ define <4 x i32> @test_urem_even_allones_and_one(<4 x i32> %X) nounwind {
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm2
 ; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm2, %xmm2
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5,6,7]
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm0[4,5],xmm1[6,7]
 ; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
@@ -2007,29 +1982,25 @@ define <4 x i32> @test_urem_odd_even_allones_and_one(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-LABEL: test_urem_odd_even_allones_and_one:
 ; CHECK-SSE2:       # %bb.0:
 ; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [3435973837,2147483649,0,1374389535]
-; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
-; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm3
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
-; CHECK-SSE2-NEXT:    psrld $5, %xmm1
-; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    movdqa %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    psrld $5, %xmm2
 ; CHECK-SSE2-NEXT:    psrld $31, %xmm3
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm1[3,3]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [5,4294967295,1,100]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm2[3,3]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [5,4294967295,1,100]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
 ; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm4
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
-; CHECK-SSE2-NEXT:    psrld $2, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm0, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    psrld $2, %xmm1
 ; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm4
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[2,0],xmm2[3,0]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm4[0,2]
-; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[2,0],xmm1[3,0]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm4[0,2]
+; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
 ; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
@@ -2043,14 +2014,12 @@ define <4 x i32> @test_urem_odd_even_allones_and_one(<4 x i32> %X) nounwind {
 ; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
 ; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; CHECK-SSE41-NEXT:    pmuludq %xmm2, %xmm3
-; CHECK-SSE41-NEXT:    pmuludq %xmm0, %xmm1
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
-; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    movdqa %xmm3, %xmm2
 ; CHECK-SSE41-NEXT:    psrld $5, %xmm2
-; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm3
 ; CHECK-SSE41-NEXT:    psrld $31, %xmm3
 ; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; CHECK-SSE41-NEXT:    pmuludq %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; CHECK-SSE41-NEXT:    psrld $2, %xmm1
 ; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
 ; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm0[4,5],xmm1[6,7]
@@ -2067,12 +2036,11 @@ define <4 x i32> @test_urem_odd_even_allones_and_one(<4 x i32> %X) nounwind {
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
+; CHECK-AVX1-NEXT:    vpsrld $5, %xmm2, %xmm3
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm2, %xmm2
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4,5,6,7]
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm1
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; CHECK-AVX1-NEXT:    vpsrld $5, %xmm1, %xmm2
-; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm3
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
 ; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm0[4,5],xmm1[6,7]
@@ -2334,28 +2302,25 @@ define <4 x i32> @test_urem_odd_even_poweroftwo_and_one(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-LABEL: test_urem_odd_even_poweroftwo_and_one:
 ; CHECK-SSE2:       # %bb.0:
 ; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [3435973837,268435456,0,1374389535]
-; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
-; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm3
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
+; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    movdqa %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    psrld $5, %xmm2
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm2[3,3]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [5,16,1,100]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm4
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm0, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; CHECK-SSE2-NEXT:    psrld $2, %xmm1
 ; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm4
 ; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[2,0],xmm1[3,0]
 ; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm4[0,2]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [5,16,1,100]
-; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm1
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; CHECK-SSE2-NEXT:    psrld $5, %xmm2
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm2[3,3]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm2
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
 ; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
@@ -2368,12 +2333,11 @@ define <4 x i32> @test_urem_odd_even_poweroftwo_and_one(<4 x i32> %X) nounwind {
 ; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
 ; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; CHECK-SSE41-NEXT:    pmuludq %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    movdqa %xmm3, %xmm2
+; CHECK-SSE41-NEXT:    psrld $5, %xmm2
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
 ; CHECK-SSE41-NEXT:    pmuludq %xmm0, %xmm1
 ; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
-; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE41-NEXT:    psrld $5, %xmm2
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0,1,2,3],xmm2[4,5,6,7]
 ; CHECK-SSE41-NEXT:    psrld $2, %xmm1
 ; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
 ; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm0[4,5],xmm1[6,7]
@@ -2390,11 +2354,10 @@ define <4 x i32> @test_urem_odd_even_poweroftwo_and_one(<4 x i32> %X) nounwind {
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
+; CHECK-AVX1-NEXT:    vpsrld $5, %xmm2, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4,5,6,7]
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm1
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; CHECK-AVX1-NEXT:    vpsrld $5, %xmm1, %xmm2
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm1[0,1,2,3],xmm2[4,5,6,7]
 ; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm0[4,5],xmm1[6,7]
@@ -2445,25 +2408,23 @@ define <4 x i32> @test_urem_odd_allones_and_poweroftwo_and_one(<4 x i32> %X) nou
 ; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [3435973837,2147483649,268435456,0]
 ; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm3
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
-; CHECK-SSE2-NEXT:    psrld $2, %xmm1
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],xmm2[2,1]
+; CHECK-SSE2-NEXT:    psrld $2, %xmm2
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm3[2,1]
 ; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [5,4294967295,16,1]
-; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm1
+; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm4
+; CHECK-SSE2-NEXT:    psrld $31, %xmm4
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1],xmm0[3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; CHECK-SSE2-NEXT:    psrld $31, %xmm2
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[3,3]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm3
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
 ; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    psrld $31, %xmm0
@@ -2472,20 +2433,20 @@ define <4 x i32> @test_urem_odd_allones_and_poweroftwo_and_one(<4 x i32> %X) nou
 ; CHECK-SSE41-LABEL: test_urem_odd_allones_and_poweroftwo_and_one:
 ; CHECK-SSE41:       # %bb.0:
 ; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [3435973837,2147483649,268435456,0]
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-SSE41-NEXT:    pmuludq %xmm2, %xmm3
-; CHECK-SSE41-NEXT:    pmuludq %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-SSE41-NEXT:    psrld $2, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm2[4,5,6,7]
 ; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
-; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE41-NEXT:    psrld $2, %xmm2
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5,6,7]
-; CHECK-SSE41-NEXT:    psrld $31, %xmm1
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],xmm0[6,7]
-; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm1
-; CHECK-SSE41-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    psrld $31, %xmm2
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5],xmm0[6,7]
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
 ; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE41-NEXT:    psrld $31, %xmm0
@@ -2494,14 +2455,13 @@ define <4 x i32> @test_urem_odd_allones_and_poweroftwo_and_one(<4 x i32> %X) nou
 ; CHECK-AVX1-LABEL: test_urem_odd_allones_and_poweroftwo_and_one:
 ; CHECK-AVX1:       # %bb.0:
 ; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [3435973837,2147483649,268435456,0]
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
-; CHECK-AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpsrld $2, %xmm2, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm2
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
 ; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],xmm0[6,7]
@@ -2547,29 +2507,27 @@ define <4 x i32> @test_urem_odd_allones_and_poweroftwo_and_one(<4 x i32> %X) nou
 define <4 x i32> @test_urem_even_allones_and_poweroftwo_and_one(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-LABEL: test_urem_even_allones_and_poweroftwo_and_one:
 ; CHECK-SSE2:       # %bb.0:
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-SSE2-NEXT:    psrld $1, %xmm1
+; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE2-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
 ; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [2454267027,2147483649,268435456,0]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm3
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,3,2,3]
-; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm3
-; CHECK-SSE2-NEXT:    psrld $1, %xmm3
-; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm4
-; CHECK-SSE2-NEXT:    movss {{.*#+}} xmm4 = xmm3[0],xmm4[1,2,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm4
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; CHECK-SSE2-NEXT:    movdqa %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
 ; CHECK-SSE2-NEXT:    psrld $2, %xmm2
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm1[2,1]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm3[2,1]
 ; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [14,4294967295,16,1]
 ; CHECK-SSE2-NEXT:    pmuludq %xmm3, %xmm2
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; CHECK-SSE2-NEXT:    psrld $31, %xmm1
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[3,3]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm3
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm4
+; CHECK-SSE2-NEXT:    psrld $31, %xmm4
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1],xmm0[3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
 ; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
 ; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
@@ -2579,24 +2537,23 @@ define <4 x i32> @test_urem_even_allones_and_poweroftwo_and_one(<4 x i32> %X) no
 ;
 ; CHECK-SSE41-LABEL: test_urem_even_allones_and_poweroftwo_and_one:
 ; CHECK-SSE41:       # %bb.0:
-; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [2454267027,2147483649,268435456,0]
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-SSE41-NEXT:    pmuludq %xmm2, %xmm3
-; CHECK-SSE41-NEXT:    movdqa %xmm0, %xmm2
-; CHECK-SSE41-NEXT:    psrld $1, %xmm2
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm0[2,3,4,5,6,7]
-; CHECK-SSE41-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    psrld $1, %xmm1
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,3,4,5,6,7]
+; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [2454267027,2147483649,268435456,0]
+; CHECK-SSE41-NEXT:    pmuludq %xmm2, %xmm1
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm3
+; CHECK-SSE41-NEXT:    psrld $2, %xmm3
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm1[4,5,6,7]
 ; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
-; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE41-NEXT:    psrld $2, %xmm2
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5,6,7]
-; CHECK-SSE41-NEXT:    psrld $31, %xmm1
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],xmm0[6,7]
-; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm1
-; CHECK-SSE41-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuludq %xmm1, %xmm2
+; CHECK-SSE41-NEXT:    psrld $31, %xmm2
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5],xmm0[6,7]
+; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
+; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
 ; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE41-NEXT:    psrld $31, %xmm0
@@ -2604,19 +2561,18 @@ define <4 x i32> @test_urem_even_allones_and_poweroftwo_and_one(<4 x i32> %X) no
 ;
 ; CHECK-AVX1-LABEL: test_urem_even_allones_and_poweroftwo_and_one:
 ; CHECK-AVX1:       # %bb.0:
-; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [2454267027,2147483649,268435456,0]
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpsrld $1, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,3,4,5,6,7]
+; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [2454267027,2147483649,268435456,0]
+; CHECK-AVX1-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm3
+; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; CHECK-AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
-; CHECK-AVX1-NEXT:    vpsrld $1, %xmm0, %xmm3
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm0[2,3,4,5,6,7]
-; CHECK-AVX1-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpsrld $31, %xmm2, %xmm2
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; CHECK-AVX1-NEXT:    vpsrld $2, %xmm1, %xmm2
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm1[4,5,6,7]
-; CHECK-AVX1-NEXT:    vpsrld $31, %xmm1, %xmm1
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],xmm0[6,7]
 ; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/urem-seteq-vec-nonzero.ll b/llvm/test/CodeGen/X86/urem-seteq-vec-nonzero.ll
index b3814a1b3f61d..e549b222796d8 100644
--- a/llvm/test/CodeGen/X86/urem-seteq-vec-nonzero.ll
+++ b/llvm/test/CodeGen/X86/urem-seteq-vec-nonzero.ll
@@ -280,24 +280,20 @@ define <4 x i1> @t32_tautological(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [0,0,2147483648,2863311531]
 ; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
 ; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm3
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,1,2,2]
-; CHECK-SSE2-NEXT:    psrld $1, %xmm2
-; CHECK-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; CHECK-SSE2-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [1,1,2,3]
-; CHECK-SSE2-NEXT:    movapd %xmm1, %xmm3
-; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    psrld $1, %xmm3
+; CHECK-SSE2-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; CHECK-SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm0[0],xmm2[1]
+; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [1,1,2,3]
+; CHECK-SSE2-NEXT:    movapd %xmm2, %xmm3
+; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm3
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
-; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; CHECK-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
 ; CHECK-SSE2-NEXT:    psubd %xmm3, %xmm0
 ; CHECK-SSE2-NEXT:    pcmpeqd {{.*}}(%rip), %xmm0
@@ -306,15 +302,13 @@ define <4 x i1> @t32_tautological(<4 x i32> %X) nounwind {
 ; CHECK-SSE41-LABEL: t32_tautological:
 ; CHECK-SSE41:       # %bb.0:
 ; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [0,0,2147483648,2863311531]
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-SSE41-NEXT:    pmuludq %xmm2, %xmm3
-; CHECK-SSE41-NEXT:    pmuludq %xmm0, %xmm1
+; CHECK-SSE41-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-SSE41-NEXT:    pmuludq %xmm1, %xmm2
 ; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
-; CHECK-SSE41-NEXT:    movdqa %xmm1, %xmm2
-; CHECK-SSE41-NEXT:    psrld $1, %xmm2
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0,1,2,3,4,5],xmm2[6,7]
+; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-SSE41-NEXT:    pmuludq %xmm1, %xmm3
+; CHECK-SSE41-NEXT:    psrld $1, %xmm3
+; CHECK-SSE41-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm3[2],xmm2[3],xmm3[3]
 ; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm0[0,1,2,3],xmm2[4,5,6,7]
 ; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
 ; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
@@ -324,14 +318,12 @@ define <4 x i1> @t32_tautological(<4 x i32> %X) nounwind {
 ; CHECK-AVX1-LABEL: t32_tautological:
 ; CHECK-AVX1:       # %bb.0:
 ; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,0,2147483648,2863311531]
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
-; CHECK-AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm1
+; CHECK-AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; CHECK-AVX1-NEXT:    vpsrld $1, %xmm1, %xmm2
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],xmm2[6,7]
+; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpmuludq %xmm1, %xmm3, %xmm1
+; CHECK-AVX1-NEXT:    vpsrld $1, %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1,2,3],xmm1[4,5,6,7]
 ; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/urem-seteq-vec-splat.ll b/llvm/test/CodeGen/X86/urem-seteq-vec-splat.ll
index 409f0bca24cdc..906771ada4d8b 100644
--- a/llvm/test/CodeGen/X86/urem-seteq-vec-splat.ll
+++ b/llvm/test/CodeGen/X86/urem-seteq-vec-splat.ll
@@ -197,22 +197,19 @@ define <4 x i32> @test_urem_even_neg100(<4 x i32> %X) nounwind {
 ; CHECK-SSE2:       # %bb.0:
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm1
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; CHECK-SSE2-NEXT:    psrld $5, %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm1
+; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
 ; CHECK-SSE2-NEXT:    psrld $2, %xmm2
 ; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm2
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
-; CHECK-SSE2-NEXT:    psrld $27, %xmm1
-; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm1
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; CHECK-SSE2-NEXT:    psrld $5, %xmm2
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-SSE2-NEXT:    psrld $27, %xmm2
 ; CHECK-SSE2-NEXT:    pmuludq {{.*}}(%rip), %xmm2
 ; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
+; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; CHECK-SSE2-NEXT:    psubd %xmm2, %xmm0
 ; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
 ; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE2-NEXT:    psrld $31, %xmm0
@@ -222,13 +219,11 @@ define <4 x i32> @test_urem_even_neg100(<4 x i32> %X) nounwind {
 ; CHECK-SSE41:       # %bb.0:
 ; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; CHECK-SSE41-NEXT:    pmuludq {{.*}}(%rip), %xmm1
+; CHECK-SSE41-NEXT:    psrld $5, %xmm1
 ; CHECK-SSE41-NEXT:    movdqa %xmm0, %xmm2
 ; CHECK-SSE41-NEXT:    psrld $2, %xmm2
 ; CHECK-SSE41-NEXT:    pmuludq {{.*}}(%rip), %xmm2
 ; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; CHECK-SSE41-NEXT:    movdqa %xmm2, %xmm1
-; CHECK-SSE41-NEXT:    psrld $5, %xmm1
 ; CHECK-SSE41-NEXT:    psrld $27, %xmm2
 ; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
 ; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
@@ -242,13 +237,12 @@ define <4 x i32> @test_urem_even_neg100(<4 x i32> %X) nounwind {
 ; CHECK-AVX1:       # %bb.0:
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; CHECK-AVX1-NEXT:    vpmuludq {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX1-NEXT:    vpsrld $5, %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpsrld $2, %xmm0, %xmm2
 ; CHECK-AVX1-NEXT:    vpmuludq {{.*}}(%rip), %xmm2, %xmm2
 ; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-AVX1-NEXT:    vpsrld $27, %xmm2, %xmm2
 ; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; CHECK-AVX1-NEXT:    vpsrld $5, %xmm1, %xmm2
-; CHECK-AVX1-NEXT:    vpsrld $27, %xmm1, %xmm1
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
 ; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
diff --git a/llvm/test/CodeGen/X86/vector-fshl-256.ll b/llvm/test/CodeGen/X86/vector-fshl-256.ll
index 12feea7658988..0688107ed5c0b 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-256.ll
@@ -1092,12 +1092,12 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %
 ; AVX2-NEXT:    vpand %xmm3, %xmm2, %xmm2
 ; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
 ; AVX2-NEXT:    vpslld %xmm3, %ymm0, %ymm3
+; AVX2-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
 ; AVX2-NEXT:    vpsubd %xmm2, %xmm4, %xmm4
 ; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
 ; AVX2-NEXT:    vpsrld %xmm4, %ymm1, %ymm1
 ; AVX2-NEXT:    vpor %ymm1, %ymm3, %ymm1
-; AVX2-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm2
 ; AVX2-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
@@ -1110,12 +1110,12 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %
 ; AVX512F-NEXT:    vpand %xmm3, %xmm2, %xmm2
 ; AVX512F-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
 ; AVX512F-NEXT:    vpslld %xmm3, %ymm0, %ymm3
+; AVX512F-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
 ; AVX512F-NEXT:    vpsubd %xmm2, %xmm4, %xmm4
 ; AVX512F-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
 ; AVX512F-NEXT:    vpsrld %xmm4, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpor %ymm1, %ymm3, %ymm1
-; AVX512F-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX512F-NEXT:    vptestnmd %zmm2, %zmm2, %k1
 ; AVX512F-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
 ; AVX512F-NEXT:    vmovdqa %ymm1, %ymm0
@@ -1126,12 +1126,12 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %
 ; AVX512VL-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm2, %xmm2
 ; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
 ; AVX512VL-NEXT:    vpslld %xmm3, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX512VL-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
 ; AVX512VL-NEXT:    vpsubd %xmm2, %xmm4, %xmm4
 ; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
 ; AVX512VL-NEXT:    vpsrld %xmm4, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpor %ymm1, %ymm3, %ymm1
-; AVX512VL-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX512VL-NEXT:    vptestnmd %ymm2, %ymm2, %k1
 ; AVX512VL-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
 ; AVX512VL-NEXT:    vmovdqa %ymm1, %ymm0
@@ -1144,12 +1144,12 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %
 ; AVX512BW-NEXT:    vpand %xmm3, %xmm2, %xmm2
 ; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
 ; AVX512BW-NEXT:    vpslld %xmm3, %ymm0, %ymm3
+; AVX512BW-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
 ; AVX512BW-NEXT:    vpsubd %xmm2, %xmm4, %xmm4
 ; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
 ; AVX512BW-NEXT:    vpsrld %xmm4, %ymm1, %ymm1
 ; AVX512BW-NEXT:    vpor %ymm1, %ymm3, %ymm1
-; AVX512BW-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX512BW-NEXT:    vptestnmd %zmm2, %zmm2, %k1
 ; AVX512BW-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
 ; AVX512BW-NEXT:    vmovdqa %ymm1, %ymm0
@@ -1162,12 +1162,12 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %
 ; AVX512VBMI2-NEXT:    vpand %xmm3, %xmm2, %xmm2
 ; AVX512VBMI2-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
 ; AVX512VBMI2-NEXT:    vpslld %xmm3, %ymm0, %ymm3
+; AVX512VBMI2-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX512VBMI2-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
 ; AVX512VBMI2-NEXT:    vpsubd %xmm2, %xmm4, %xmm4
 ; AVX512VBMI2-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
 ; AVX512VBMI2-NEXT:    vpsrld %xmm4, %ymm1, %ymm1
 ; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm3, %ymm1
-; AVX512VBMI2-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX512VBMI2-NEXT:    vptestnmd %zmm2, %zmm2, %k1
 ; AVX512VBMI2-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
 ; AVX512VBMI2-NEXT:    vmovdqa %ymm1, %ymm0
@@ -1178,12 +1178,12 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %
 ; AVX512VLBW-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm2, %xmm2
 ; AVX512VLBW-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
 ; AVX512VLBW-NEXT:    vpslld %xmm3, %ymm0, %ymm3
+; AVX512VLBW-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX512VLBW-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
 ; AVX512VLBW-NEXT:    vpsubd %xmm2, %xmm4, %xmm4
 ; AVX512VLBW-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
 ; AVX512VLBW-NEXT:    vpsrld %xmm4, %ymm1, %ymm1
 ; AVX512VLBW-NEXT:    vpor %ymm1, %ymm3, %ymm1
-; AVX512VLBW-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX512VLBW-NEXT:    vptestnmd %ymm2, %ymm2, %k1
 ; AVX512VLBW-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
 ; AVX512VLBW-NEXT:    vmovdqa %ymm1, %ymm0
@@ -1224,12 +1224,12 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %
 ; XOPAVX2-NEXT:    vpand %xmm3, %xmm2, %xmm2
 ; XOPAVX2-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
 ; XOPAVX2-NEXT:    vpslld %xmm3, %ymm0, %ymm3
+; XOPAVX2-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; XOPAVX2-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
 ; XOPAVX2-NEXT:    vpsubd %xmm2, %xmm4, %xmm4
 ; XOPAVX2-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
 ; XOPAVX2-NEXT:    vpsrld %xmm4, %ymm1, %ymm1
 ; XOPAVX2-NEXT:    vpor %ymm1, %ymm3, %ymm1
-; XOPAVX2-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; XOPAVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; XOPAVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm2
 ; XOPAVX2-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
@@ -1271,12 +1271,12 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i
 ; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; AVX2-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
+; AVX2-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
 ; AVX2-NEXT:    vpsubw %xmm2, %xmm4, %xmm4
 ; AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
 ; AVX2-NEXT:    vpsrlw %xmm4, %ymm1, %ymm1
 ; AVX2-NEXT:    vpor %ymm1, %ymm3, %ymm1
-; AVX2-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX2-NEXT:    vpcmpeqw %ymm3, %ymm2, %ymm2
 ; AVX2-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
@@ -1287,12 +1287,12 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i
 ; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; AVX512F-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
+; AVX512F-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
 ; AVX512F-NEXT:    vpsubw %xmm2, %xmm4, %xmm4
 ; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
 ; AVX512F-NEXT:    vpsrlw %xmm4, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpor %ymm1, %ymm3, %ymm1
-; AVX512F-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; AVX512F-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX512F-NEXT:    vpcmpeqw %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
@@ -1303,12 +1303,12 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; AVX512VL-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
 ; AVX512VL-NEXT:    vpsubw %xmm2, %xmm4, %xmm4
 ; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
 ; AVX512VL-NEXT:    vpsrlw %xmm4, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpor %ymm1, %ymm3, %ymm1
-; AVX512VL-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; AVX512VL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX512VL-NEXT:    vpcmpeqw %ymm3, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
@@ -1320,12 +1320,12 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i
 ; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; AVX512BW-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
+; AVX512BW-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
 ; AVX512BW-NEXT:    vpsubw %xmm2, %xmm4, %xmm4
 ; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
 ; AVX512BW-NEXT:    vpsrlw %xmm4, %ymm1, %ymm1
 ; AVX512BW-NEXT:    vpor %ymm1, %ymm3, %ymm1
-; AVX512BW-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; AVX512BW-NEXT:    vptestnmw %zmm2, %zmm2, %k1
 ; AVX512BW-NEXT:    vmovdqu16 %zmm0, %zmm1 {%k1}
 ; AVX512BW-NEXT:    vmovdqa %ymm1, %ymm0
@@ -1337,12 +1337,12 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i
 ; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512VBMI2-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; AVX512VBMI2-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
+; AVX512VBMI2-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
 ; AVX512VBMI2-NEXT:    vpsubw %xmm2, %xmm4, %xmm4
 ; AVX512VBMI2-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
 ; AVX512VBMI2-NEXT:    vpsrlw %xmm4, %ymm1, %ymm1
 ; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm3, %ymm1
-; AVX512VBMI2-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; AVX512VBMI2-NEXT:    vptestnmw %zmm2, %zmm2, %k1
 ; AVX512VBMI2-NEXT:    vmovdqu16 %zmm0, %zmm1 {%k1}
 ; AVX512VBMI2-NEXT:    vmovdqa %ymm1, %ymm0
@@ -1353,12 +1353,12 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i
 ; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512VLBW-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; AVX512VLBW-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
+; AVX512VLBW-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
 ; AVX512VLBW-NEXT:    vpsubw %xmm2, %xmm4, %xmm4
 ; AVX512VLBW-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
 ; AVX512VLBW-NEXT:    vpsrlw %xmm4, %ymm1, %ymm1
 ; AVX512VLBW-NEXT:    vpor %ymm1, %ymm3, %ymm1
-; AVX512VLBW-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; AVX512VLBW-NEXT:    vptestnmw %ymm2, %ymm2, %k1
 ; AVX512VLBW-NEXT:    vmovdqu16 %ymm0, %ymm1 {%k1}
 ; AVX512VLBW-NEXT:    vmovdqa %ymm1, %ymm0
@@ -1399,12 +1399,12 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i
 ; XOPAVX2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; XOPAVX2-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; XOPAVX2-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
+; XOPAVX2-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
 ; XOPAVX2-NEXT:    vpsubw %xmm2, %xmm4, %xmm4
 ; XOPAVX2-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
 ; XOPAVX2-NEXT:    vpsrlw %xmm4, %ymm1, %ymm1
 ; XOPAVX2-NEXT:    vpor %ymm1, %ymm3, %ymm1
-; XOPAVX2-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; XOPAVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; XOPAVX2-NEXT:    vpcmpeqw %ymm3, %ymm2, %ymm2
 ; XOPAVX2-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
@@ -1458,6 +1458,7 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX2-NEXT:    vpsllw %xmm3, %xmm5, %xmm3
 ; AVX2-NEXT:    vpbroadcastb %xmm3, %ymm3
 ; AVX2-NEXT:    vpand %ymm3, %ymm4, %ymm3
+; AVX2-NEXT:    vpbroadcastb %xmm2, %ymm2
 ; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; AVX2-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
 ; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
@@ -1467,7 +1468,6 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX2-NEXT:    vpbroadcastb %xmm4, %ymm4
 ; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm1
 ; AVX2-NEXT:    vpor %ymm1, %ymm3, %ymm1
-; AVX2-NEXT:    vpbroadcastb %xmm2, %ymm2
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX2-NEXT:    vpcmpeqb %ymm3, %ymm2, %ymm2
 ; AVX2-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
@@ -1482,6 +1482,7 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX512F-NEXT:    vpsllw %xmm3, %xmm5, %xmm3
 ; AVX512F-NEXT:    vpbroadcastb %xmm3, %ymm3
 ; AVX512F-NEXT:    vpand %ymm3, %ymm4, %ymm3
+; AVX512F-NEXT:    vpbroadcastb %xmm2, %ymm2
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; AVX512F-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
 ; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
@@ -1491,7 +1492,6 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX512F-NEXT:    vpbroadcastb %xmm4, %ymm4
 ; AVX512F-NEXT:    vpand %ymm4, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpor %ymm1, %ymm3, %ymm1
-; AVX512F-NEXT:    vpbroadcastb %xmm2, %ymm2
 ; AVX512F-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX512F-NEXT:    vpcmpeqb %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
@@ -1506,6 +1506,7 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX512VL-NEXT:    vpsllw %xmm3, %xmm5, %xmm3
 ; AVX512VL-NEXT:    vpbroadcastb %xmm3, %ymm3
 ; AVX512VL-NEXT:    vpand %ymm3, %ymm4, %ymm3
+; AVX512VL-NEXT:    vpbroadcastb %xmm2, %ymm2
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; AVX512VL-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
 ; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
@@ -1514,9 +1515,8 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX512VL-NEXT:    vpsrlw $8, %xmm4, %xmm4
 ; AVX512VL-NEXT:    vpbroadcastb %xmm4, %ymm4
 ; AVX512VL-NEXT:    vpternlogq $236, %ymm1, %ymm3, %ymm4
-; AVX512VL-NEXT:    vpbroadcastb %xmm2, %ymm1
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpcmpeqb %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
 ; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm0, %ymm4, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-fshl-512.ll b/llvm/test/CodeGen/X86/vector-fshl-512.ll
index 6e0cb76398df9..088a590a2e07f 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-512.ll
@@ -643,12 +643,12 @@ define <16 x i32> @splatvar_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i
 ; AVX512F-NEXT:    vpand %xmm3, %xmm2, %xmm2
 ; AVX512F-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
 ; AVX512F-NEXT:    vpslld %xmm3, %zmm0, %zmm3
+; AVX512F-NEXT:    vpbroadcastd %xmm2, %zmm2
 ; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
 ; AVX512F-NEXT:    vpsubd %xmm2, %xmm4, %xmm4
 ; AVX512F-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
 ; AVX512F-NEXT:    vpsrld %xmm4, %zmm1, %zmm1
 ; AVX512F-NEXT:    vpord %zmm1, %zmm3, %zmm1
-; AVX512F-NEXT:    vpbroadcastd %xmm2, %zmm2
 ; AVX512F-NEXT:    vptestnmd %zmm2, %zmm2, %k1
 ; AVX512F-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
 ; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0
@@ -659,12 +659,12 @@ define <16 x i32> @splatvar_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i
 ; AVX512VL-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm2, %xmm2
 ; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
 ; AVX512VL-NEXT:    vpslld %xmm3, %zmm0, %zmm3
+; AVX512VL-NEXT:    vpbroadcastd %xmm2, %zmm2
 ; AVX512VL-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
 ; AVX512VL-NEXT:    vpsubd %xmm2, %xmm4, %xmm4
 ; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
 ; AVX512VL-NEXT:    vpsrld %xmm4, %zmm1, %zmm1
 ; AVX512VL-NEXT:    vpord %zmm1, %zmm3, %zmm1
-; AVX512VL-NEXT:    vpbroadcastd %xmm2, %zmm2
 ; AVX512VL-NEXT:    vptestnmd %zmm2, %zmm2, %k1
 ; AVX512VL-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
 ; AVX512VL-NEXT:    vmovdqa64 %zmm1, %zmm0
@@ -676,12 +676,12 @@ define <16 x i32> @splatvar_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i
 ; AVX512BW-NEXT:    vpand %xmm3, %xmm2, %xmm2
 ; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
 ; AVX512BW-NEXT:    vpslld %xmm3, %zmm0, %zmm3
+; AVX512BW-NEXT:    vpbroadcastd %xmm2, %zmm2
 ; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
 ; AVX512BW-NEXT:    vpsubd %xmm2, %xmm4, %xmm4
 ; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
 ; AVX512BW-NEXT:    vpsrld %xmm4, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpord %zmm1, %zmm3, %zmm1
-; AVX512BW-NEXT:    vpbroadcastd %xmm2, %zmm2
 ; AVX512BW-NEXT:    vptestnmd %zmm2, %zmm2, %k1
 ; AVX512BW-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
@@ -698,12 +698,12 @@ define <16 x i32> @splatvar_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i
 ; AVX512VLBW-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm2, %xmm2
 ; AVX512VLBW-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
 ; AVX512VLBW-NEXT:    vpslld %xmm3, %zmm0, %zmm3
+; AVX512VLBW-NEXT:    vpbroadcastd %xmm2, %zmm2
 ; AVX512VLBW-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
 ; AVX512VLBW-NEXT:    vpsubd %xmm2, %xmm4, %xmm4
 ; AVX512VLBW-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
 ; AVX512VLBW-NEXT:    vpsrld %xmm4, %zmm1, %zmm1
 ; AVX512VLBW-NEXT:    vpord %zmm1, %zmm3, %zmm1
-; AVX512VLBW-NEXT:    vpbroadcastd %xmm2, %zmm2
 ; AVX512VLBW-NEXT:    vptestnmd %zmm2, %zmm2, %k1
 ; AVX512VLBW-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
 ; AVX512VLBW-NEXT:    vmovdqa64 %zmm1, %zmm0
@@ -771,12 +771,12 @@ define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i
 ; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; AVX512BW-NEXT:    vpsllw %xmm3, %zmm0, %zmm3
+; AVX512BW-NEXT:    vpbroadcastw %xmm2, %zmm2
 ; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
 ; AVX512BW-NEXT:    vpsubw %xmm2, %xmm4, %xmm4
 ; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
 ; AVX512BW-NEXT:    vpsrlw %xmm4, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vporq %zmm1, %zmm3, %zmm1
-; AVX512BW-NEXT:    vpbroadcastw %xmm2, %zmm2
 ; AVX512BW-NEXT:    vptestnmw %zmm2, %zmm2, %k1
 ; AVX512BW-NEXT:    vmovdqu16 %zmm0, %zmm1 {%k1}
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
@@ -793,12 +793,12 @@ define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i
 ; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512VLBW-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; AVX512VLBW-NEXT:    vpsllw %xmm3, %zmm0, %zmm3
+; AVX512VLBW-NEXT:    vpbroadcastw %xmm2, %zmm2
 ; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
 ; AVX512VLBW-NEXT:    vpsubw %xmm2, %xmm4, %xmm4
 ; AVX512VLBW-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
 ; AVX512VLBW-NEXT:    vpsrlw %xmm4, %zmm1, %zmm1
 ; AVX512VLBW-NEXT:    vporq %zmm1, %zmm3, %zmm1
-; AVX512VLBW-NEXT:    vpbroadcastw %xmm2, %zmm2
 ; AVX512VLBW-NEXT:    vptestnmw %zmm2, %zmm2, %k1
 ; AVX512VLBW-NEXT:    vmovdqu16 %zmm0, %zmm1 {%k1}
 ; AVX512VLBW-NEXT:    vmovdqa64 %zmm1, %zmm0
@@ -890,6 +890,7 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %
 ; AVX512BW-NEXT:    vpsllw %xmm3, %xmm5, %xmm3
 ; AVX512BW-NEXT:    vpbroadcastb %xmm3, %zmm3
 ; AVX512BW-NEXT:    vpandq %zmm3, %zmm4, %zmm3
+; AVX512BW-NEXT:    vpbroadcastb %xmm2, %zmm2
 ; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; AVX512BW-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
 ; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
@@ -898,7 +899,6 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %
 ; AVX512BW-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX512BW-NEXT:    vpbroadcastb %xmm1, %zmm1
 ; AVX512BW-NEXT:    vpternlogq $236, %zmm6, %zmm3, %zmm1
-; AVX512BW-NEXT:    vpbroadcastb %xmm2, %zmm2
 ; AVX512BW-NEXT:    vptestnmb %zmm2, %zmm2, %k1
 ; AVX512BW-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
@@ -913,6 +913,7 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %
 ; AVX512VBMI2-NEXT:    vpsllw %xmm3, %xmm5, %xmm3
 ; AVX512VBMI2-NEXT:    vpbroadcastb %xmm3, %zmm3
 ; AVX512VBMI2-NEXT:    vpandq %zmm3, %zmm4, %zmm3
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm2, %zmm2
 ; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; AVX512VBMI2-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
 ; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
@@ -921,7 +922,6 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %
 ; AVX512VBMI2-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX512VBMI2-NEXT:    vpbroadcastb %xmm1, %zmm1
 ; AVX512VBMI2-NEXT:    vpternlogq $236, %zmm6, %zmm3, %zmm1
-; AVX512VBMI2-NEXT:    vpbroadcastb %xmm2, %zmm2
 ; AVX512VBMI2-NEXT:    vptestnmb %zmm2, %zmm2, %k1
 ; AVX512VBMI2-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
 ; AVX512VBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0
@@ -936,6 +936,7 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %
 ; AVX512VLBW-NEXT:    vpsllw %xmm3, %xmm5, %xmm3
 ; AVX512VLBW-NEXT:    vpbroadcastb %xmm3, %zmm3
 ; AVX512VLBW-NEXT:    vpandq %zmm3, %zmm4, %zmm3
+; AVX512VLBW-NEXT:    vpbroadcastb %xmm2, %zmm2
 ; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; AVX512VLBW-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
 ; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
@@ -944,7 +945,6 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %
 ; AVX512VLBW-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX512VLBW-NEXT:    vpbroadcastb %xmm1, %zmm1
 ; AVX512VLBW-NEXT:    vpternlogq $236, %zmm6, %zmm3, %zmm1
-; AVX512VLBW-NEXT:    vpbroadcastb %xmm2, %zmm2
 ; AVX512VLBW-NEXT:    vptestnmb %zmm2, %zmm2, %k1
 ; AVX512VLBW-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
 ; AVX512VLBW-NEXT:    vmovdqa64 %zmm1, %zmm0
@@ -959,6 +959,7 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %
 ; AVX512VLVBMI2-NEXT:    vpsllw %xmm3, %xmm5, %xmm3
 ; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm3, %zmm3
 ; AVX512VLVBMI2-NEXT:    vpandq %zmm3, %zmm4, %zmm3
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm2, %zmm2
 ; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; AVX512VLVBMI2-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
 ; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
@@ -967,7 +968,6 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %
 ; AVX512VLVBMI2-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm1, %zmm1
 ; AVX512VLVBMI2-NEXT:    vpternlogq $236, %zmm6, %zmm3, %zmm1
-; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm2, %zmm2
 ; AVX512VLVBMI2-NEXT:    vptestnmb %zmm2, %zmm2, %k1
 ; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
 ; AVX512VLVBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
index 49f229bf1d67b..ff177678bed6c 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
@@ -366,13 +366,14 @@ define <16 x i32> @splatvar_funnnel_v16i32(<16 x i32> %x, <16 x i32> %amt) nounw
 define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %amt) nounwind {
 ; AVX512F-LABEL: splatvar_funnnel_v32i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm3
-; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
-; AVX512F-NEXT:    vpsllw %xmm3, %ymm2, %ymm4
-; AVX512F-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm4, %zmm3, %zmm3
 ; AVX512F-NEXT:    vpbroadcastw %xmm1, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm2
+; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vpsllw %xmm2, %ymm3, %ymm4
+; AVX512F-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
 ; AVX512F-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; AVX512F-NEXT:    vpsubw %ymm1, %ymm4, %ymm1
 ; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
@@ -380,22 +381,23 @@ define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %amt) nounw
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; AVX512F-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512F-NEXT:    vpsrlvd %zmm1, %zmm2, %zmm1
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512F-NEXT:    vpsrlvd %zmm1, %zmm3, %zmm1
 ; AVX512F-NEXT:    vpmovdw %zmm1, %ymm1
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm0, %zmm3, %zmm0
+; AVX512F-NEXT:    vporq %zmm0, %zmm2, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_funnnel_v32i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm3
-; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
-; AVX512VL-NEXT:    vpsllw %xmm3, %ymm2, %ymm4
-; AVX512VL-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm4, %zmm3, %zmm3
 ; AVX512VL-NEXT:    vpbroadcastw %xmm1, %ymm1
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm1, %zmm1
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm2
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512VL-NEXT:    vpsllw %xmm2, %ymm3, %ymm4
+; AVX512VL-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
 ; AVX512VL-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; AVX512VL-NEXT:    vpsubw %ymm1, %ymm4, %ymm1
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
@@ -403,11 +405,11 @@ define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %amt) nounw
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; AVX512VL-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512VL-NEXT:    vpsrlvd %zmm1, %zmm2, %zmm1
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512VL-NEXT:    vpsrlvd %zmm1, %zmm3, %zmm1
 ; AVX512VL-NEXT:    vpmovdw %zmm1, %ymm1
 ; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512VL-NEXT:    vporq %zmm0, %zmm3, %zmm0
+; AVX512VL-NEXT:    vporq %zmm0, %zmm2, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v32i16:
@@ -447,16 +449,17 @@ define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %amt) nounw
 define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {
 ; AVX512F-LABEL: splatvar_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; AVX512F-NEXT:    vpsrlw $4, %ymm2, %ymm3
+; AVX512F-NEXT:    vpbroadcastb %xmm1, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm1, %zmm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm3
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
 ; AVX512F-NEXT:    vpand %ymm4, %ymm3, %ymm3
-; AVX512F-NEXT:    vpbroadcastb %xmm1, %ymm5
-; AVX512F-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX512F-NEXT:    vpsubb %ymm5, %ymm6, %ymm5
+; AVX512F-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX512F-NEXT:    vpsubb %ymm2, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm5, %ymm5
 ; AVX512F-NEXT:    vpsllw $5, %ymm5, %ymm5
-; AVX512F-NEXT:    vpblendvb %ymm5, %ymm3, %ymm2, %ymm3
+; AVX512F-NEXT:    vpblendvb %ymm5, %ymm3, %ymm1, %ymm3
 ; AVX512F-NEXT:    vpsrlw $2, %ymm3, %ymm6
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
 ; AVX512F-NEXT:    vpand %ymm7, %ymm6, %ymm6
@@ -477,31 +480,32 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind
 ; AVX512F-NEXT:    vpand %ymm5, %ymm9, %ymm5
 ; AVX512F-NEXT:    vpblendvb %ymm10, %ymm5, %ymm4, %ymm4
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm4, %zmm3
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vpsllw %xmm1, %ymm2, %ymm2
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpsllw %xmm2, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX512F-NEXT:    vpsllw %xmm1, %xmm4, %xmm4
+; AVX512F-NEXT:    vpsllw %xmm2, %xmm4, %xmm4
 ; AVX512F-NEXT:    vpbroadcastb %xmm4, %ymm4
-; AVX512F-NEXT:    vpand %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsllw %xmm2, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpand %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    vporq %zmm3, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; AVX512VL-NEXT:    vpsrlw $4, %ymm2, %ymm3
+; AVX512VL-NEXT:    vpbroadcastb %xmm1, %ymm1
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm1, %zmm2
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm3
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
 ; AVX512VL-NEXT:    vpand %ymm4, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpbroadcastb %xmm1, %ymm5
-; AVX512VL-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX512VL-NEXT:    vpsubb %ymm5, %ymm6, %ymm5
+; AVX512VL-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX512VL-NEXT:    vpsubb %ymm2, %ymm5, %ymm5
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm5, %ymm5
 ; AVX512VL-NEXT:    vpsllw $5, %ymm5, %ymm5
-; AVX512VL-NEXT:    vpblendvb %ymm5, %ymm3, %ymm2, %ymm3
+; AVX512VL-NEXT:    vpblendvb %ymm5, %ymm3, %ymm1, %ymm3
 ; AVX512VL-NEXT:    vpsrlw $2, %ymm3, %ymm6
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
 ; AVX512VL-NEXT:    vpand %ymm7, %ymm6, %ymm6
@@ -522,16 +526,16 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind
 ; AVX512VL-NEXT:    vpand %ymm5, %ymm9, %ymm5
 ; AVX512VL-NEXT:    vpblendvb %ymm10, %ymm5, %ymm4, %ymm4
 ; AVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm4, %zmm3
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpsllw %xmm1, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT:    vpsllw %xmm2, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpsllw %xmm1, %xmm4, %xmm4
+; AVX512VL-NEXT:    vpsllw %xmm2, %xmm4, %xmm4
 ; AVX512VL-NEXT:    vpbroadcastb %xmm4, %ymm4
-; AVX512VL-NEXT:    vpand %ymm4, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpsllw %xmm2, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpand %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vporq %zmm3, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-fshr-256.ll b/llvm/test/CodeGen/X86/vector-fshr-256.ll
index bbeaed5cc725c..e9cb0a0586f09 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-256.ll
@@ -1095,12 +1095,12 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %
 ; AVX2-NEXT:    vpand %xmm3, %xmm2, %xmm2
 ; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
 ; AVX2-NEXT:    vpsrld %xmm3, %ymm1, %ymm3
+; AVX2-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
 ; AVX2-NEXT:    vpsubd %xmm2, %xmm4, %xmm4
 ; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
 ; AVX2-NEXT:    vpslld %xmm4, %ymm0, %ymm0
 ; AVX2-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX2-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm2
 ; AVX2-NEXT:    vblendvps %ymm2, %ymm1, %ymm0, %ymm0
@@ -1113,12 +1113,12 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %
 ; AVX512F-NEXT:    vpand %xmm3, %xmm2, %xmm2
 ; AVX512F-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
 ; AVX512F-NEXT:    vpsrld %xmm3, %ymm1, %ymm3
+; AVX512F-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
 ; AVX512F-NEXT:    vpsubd %xmm2, %xmm4, %xmm4
 ; AVX512F-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
 ; AVX512F-NEXT:    vpslld %xmm4, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX512F-NEXT:    vptestnmd %zmm2, %zmm2, %k1
 ; AVX512F-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
@@ -1129,12 +1129,12 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %
 ; AVX512VL-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm2, %xmm2
 ; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
 ; AVX512VL-NEXT:    vpsrld %xmm3, %ymm1, %ymm3
+; AVX512VL-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX512VL-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
 ; AVX512VL-NEXT:    vpsubd %xmm2, %xmm4, %xmm4
 ; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
 ; AVX512VL-NEXT:    vpslld %xmm4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX512VL-NEXT:    vptestnmd %ymm2, %ymm2, %k1
 ; AVX512VL-NEXT:    vmovdqa32 %ymm1, %ymm0 {%k1}
 ; AVX512VL-NEXT:    retq
@@ -1146,12 +1146,12 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %
 ; AVX512BW-NEXT:    vpand %xmm3, %xmm2, %xmm2
 ; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
 ; AVX512BW-NEXT:    vpsrld %xmm3, %ymm1, %ymm3
+; AVX512BW-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
 ; AVX512BW-NEXT:    vpsubd %xmm2, %xmm4, %xmm4
 ; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
 ; AVX512BW-NEXT:    vpslld %xmm4, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX512BW-NEXT:    vptestnmd %zmm2, %zmm2, %k1
 ; AVX512BW-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
@@ -1164,12 +1164,12 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %
 ; AVX512VBMI2-NEXT:    vpand %xmm3, %xmm2, %xmm2
 ; AVX512VBMI2-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
 ; AVX512VBMI2-NEXT:    vpsrld %xmm3, %ymm1, %ymm3
+; AVX512VBMI2-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX512VBMI2-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
 ; AVX512VBMI2-NEXT:    vpsubd %xmm2, %xmm4, %xmm4
 ; AVX512VBMI2-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
 ; AVX512VBMI2-NEXT:    vpslld %xmm4, %ymm0, %ymm0
 ; AVX512VBMI2-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX512VBMI2-NEXT:    vptestnmd %zmm2, %zmm2, %k1
 ; AVX512VBMI2-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
 ; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
@@ -1180,12 +1180,12 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %
 ; AVX512VLBW-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm2, %xmm2
 ; AVX512VLBW-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
 ; AVX512VLBW-NEXT:    vpsrld %xmm3, %ymm1, %ymm3
+; AVX512VLBW-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX512VLBW-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
 ; AVX512VLBW-NEXT:    vpsubd %xmm2, %xmm4, %xmm4
 ; AVX512VLBW-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
 ; AVX512VLBW-NEXT:    vpslld %xmm4, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX512VLBW-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; AVX512VLBW-NEXT:    vptestnmd %ymm2, %ymm2, %k1
 ; AVX512VLBW-NEXT:    vmovdqa32 %ymm1, %ymm0 {%k1}
 ; AVX512VLBW-NEXT:    retq
@@ -1226,12 +1226,12 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %
 ; XOPAVX2-NEXT:    vpand %xmm3, %xmm2, %xmm2
 ; XOPAVX2-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
 ; XOPAVX2-NEXT:    vpsrld %xmm3, %ymm1, %ymm3
+; XOPAVX2-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; XOPAVX2-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
 ; XOPAVX2-NEXT:    vpsubd %xmm2, %xmm4, %xmm4
 ; XOPAVX2-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
 ; XOPAVX2-NEXT:    vpslld %xmm4, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpbroadcastd %xmm2, %ymm2
 ; XOPAVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; XOPAVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm2
 ; XOPAVX2-NEXT:    vblendvps %ymm2, %ymm1, %ymm0, %ymm0
@@ -1273,12 +1273,12 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i
 ; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; AVX2-NEXT:    vpsrlw %xmm3, %ymm1, %ymm3
+; AVX2-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
 ; AVX2-NEXT:    vpsubw %xmm2, %xmm4, %xmm4
 ; AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
 ; AVX2-NEXT:    vpsllw %xmm4, %ymm0, %ymm0
 ; AVX2-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX2-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX2-NEXT:    vpcmpeqw %ymm3, %ymm2, %ymm2
 ; AVX2-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
@@ -1289,12 +1289,12 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i
 ; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; AVX512F-NEXT:    vpsrlw %xmm3, %ymm1, %ymm3
+; AVX512F-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
 ; AVX512F-NEXT:    vpsubw %xmm2, %xmm4, %xmm4
 ; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
 ; AVX512F-NEXT:    vpsllw %xmm4, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; AVX512F-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX512F-NEXT:    vpcmpeqw %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
@@ -1305,12 +1305,12 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; AVX512VL-NEXT:    vpsrlw %xmm3, %ymm1, %ymm3
+; AVX512VL-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
 ; AVX512VL-NEXT:    vpsubw %xmm2, %xmm4, %xmm4
 ; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
 ; AVX512VL-NEXT:    vpsllw %xmm4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; AVX512VL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX512VL-NEXT:    vpcmpeqw %ymm3, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
@@ -1322,12 +1322,12 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i
 ; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; AVX512BW-NEXT:    vpsrlw %xmm3, %ymm1, %ymm3
+; AVX512BW-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
 ; AVX512BW-NEXT:    vpsubw %xmm2, %xmm4, %xmm4
 ; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
 ; AVX512BW-NEXT:    vpsllw %xmm4, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; AVX512BW-NEXT:    vptestnmw %zmm2, %zmm2, %k1
 ; AVX512BW-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1}
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
@@ -1339,12 +1339,12 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i
 ; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512VBMI2-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; AVX512VBMI2-NEXT:    vpsrlw %xmm3, %ymm1, %ymm3
+; AVX512VBMI2-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
 ; AVX512VBMI2-NEXT:    vpsubw %xmm2, %xmm4, %xmm4
 ; AVX512VBMI2-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
 ; AVX512VBMI2-NEXT:    vpsllw %xmm4, %ymm0, %ymm0
 ; AVX512VBMI2-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; AVX512VBMI2-NEXT:    vptestnmw %zmm2, %zmm2, %k1
 ; AVX512VBMI2-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1}
 ; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
@@ -1355,12 +1355,12 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i
 ; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512VLBW-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; AVX512VLBW-NEXT:    vpsrlw %xmm3, %ymm1, %ymm3
+; AVX512VLBW-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
 ; AVX512VLBW-NEXT:    vpsubw %xmm2, %xmm4, %xmm4
 ; AVX512VLBW-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
 ; AVX512VLBW-NEXT:    vpsllw %xmm4, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX512VLBW-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; AVX512VLBW-NEXT:    vptestnmw %ymm2, %ymm2, %k1
 ; AVX512VLBW-NEXT:    vmovdqu16 %ymm1, %ymm0 {%k1}
 ; AVX512VLBW-NEXT:    retq
@@ -1401,12 +1401,12 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i
 ; XOPAVX2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; XOPAVX2-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; XOPAVX2-NEXT:    vpsrlw %xmm3, %ymm1, %ymm3
+; XOPAVX2-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
 ; XOPAVX2-NEXT:    vpsubw %xmm2, %xmm4, %xmm4
 ; XOPAVX2-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
 ; XOPAVX2-NEXT:    vpsllw %xmm4, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; XOPAVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; XOPAVX2-NEXT:    vpcmpeqw %ymm3, %ymm2, %ymm2
 ; XOPAVX2-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
@@ -1461,6 +1461,7 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX2-NEXT:    vpsrlw $8, %xmm3, %xmm3
 ; AVX2-NEXT:    vpbroadcastb %xmm3, %ymm3
 ; AVX2-NEXT:    vpand %ymm3, %ymm4, %ymm3
+; AVX2-NEXT:    vpbroadcastb %xmm2, %ymm2
 ; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; AVX2-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
 ; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
@@ -1469,7 +1470,6 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX2-NEXT:    vpbroadcastb %xmm4, %ymm4
 ; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX2-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX2-NEXT:    vpbroadcastb %xmm2, %ymm2
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX2-NEXT:    vpcmpeqb %ymm3, %ymm2, %ymm2
 ; AVX2-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
@@ -1485,6 +1485,7 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX512F-NEXT:    vpsrlw $8, %xmm3, %xmm3
 ; AVX512F-NEXT:    vpbroadcastb %xmm3, %ymm3
 ; AVX512F-NEXT:    vpand %ymm3, %ymm4, %ymm3
+; AVX512F-NEXT:    vpbroadcastb %xmm2, %ymm2
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; AVX512F-NEXT:    vpsubb %xmm2, %xmm4, %xmm4
 ; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
@@ -1493,7 +1494,6 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX512F-NEXT:    vpbroadcastb %xmm4, %ymm4
 ; AVX512F-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpbroadcastb %xmm2, %ymm2
 ; AVX512F-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX512F-NEXT:    vpcmpeqb %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
@@ -1502,24 +1502,24 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX512VL-LABEL: splatvar_funnnel_v32i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VL-NEXT:    vpsubb %xmm2, %xmm3, %xmm3
-; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpsllw %xmm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpsllw %xmm3, %xmm4, %xmm3
-; AVX512VL-NEXT:    vpbroadcastb %xmm3, %ymm3
-; AVX512VL-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpsrlw %xmm3, %ymm1, %ymm5
-; AVX512VL-NEXT:    vpsrlw %xmm3, %xmm4, %xmm3
-; AVX512VL-NEXT:    vpsrlw $8, %xmm3, %xmm3
-; AVX512VL-NEXT:    vpbroadcastb %xmm3, %ymm3
-; AVX512VL-NEXT:    vpternlogq $236, %ymm5, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpbroadcastb %xmm2, %ymm0
-; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpblendvb %ymm0, %ymm1, %ymm3, %ymm0
+; AVX512VL-NEXT:    vpbroadcastb %xmm2, %ymm3
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VL-NEXT:    vpsubb %xmm3, %xmm4, %xmm4
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT:    vpsllw %xmm4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
+; AVX512VL-NEXT:    vpsllw %xmm4, %xmm5, %xmm4
+; AVX512VL-NEXT:    vpbroadcastb %xmm4, %ymm4
+; AVX512VL-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT:    vpsrlw %xmm2, %ymm1, %ymm4
+; AVX512VL-NEXT:    vpsrlw %xmm2, %xmm5, %xmm2
+; AVX512VL-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; AVX512VL-NEXT:    vpbroadcastb %xmm2, %ymm2
+; AVX512VL-NEXT:    vpternlogq $236, %ymm4, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpcmpeqb %ymm0, %ymm3, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm0, %ymm1, %ymm2, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v32i8:
diff --git a/llvm/test/CodeGen/X86/vector-fshr-512.ll b/llvm/test/CodeGen/X86/vector-fshr-512.ll
index c89782bc359cc..fa70e840081f5 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-512.ll
@@ -633,12 +633,12 @@ define <16 x i32> @splatvar_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i
 ; AVX512F-NEXT:    vpand %xmm3, %xmm2, %xmm2
 ; AVX512F-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
 ; AVX512F-NEXT:    vpsrld %xmm3, %zmm1, %zmm3
+; AVX512F-NEXT:    vpbroadcastd %xmm2, %zmm2
 ; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
 ; AVX512F-NEXT:    vpsubd %xmm2, %xmm4, %xmm4
 ; AVX512F-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
 ; AVX512F-NEXT:    vpslld %xmm4, %zmm0, %zmm0
 ; AVX512F-NEXT:    vpord %zmm3, %zmm0, %zmm0
-; AVX512F-NEXT:    vpbroadcastd %xmm2, %zmm2
 ; AVX512F-NEXT:    vptestnmd %zmm2, %zmm2, %k1
 ; AVX512F-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
 ; AVX512F-NEXT:    retq
@@ -648,12 +648,12 @@ define <16 x i32> @splatvar_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i
 ; AVX512VL-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm2, %xmm2
 ; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
 ; AVX512VL-NEXT:    vpsrld %xmm3, %zmm1, %zmm3
+; AVX512VL-NEXT:    vpbroadcastd %xmm2, %zmm2
 ; AVX512VL-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
 ; AVX512VL-NEXT:    vpsubd %xmm2, %xmm4, %xmm4
 ; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
 ; AVX512VL-NEXT:    vpslld %xmm4, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vpord %zmm3, %zmm0, %zmm0
-; AVX512VL-NEXT:    vpbroadcastd %xmm2, %zmm2
 ; AVX512VL-NEXT:    vptestnmd %zmm2, %zmm2, %k1
 ; AVX512VL-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
 ; AVX512VL-NEXT:    retq
@@ -664,12 +664,12 @@ define <16 x i32> @splatvar_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i
 ; AVX512BW-NEXT:    vpand %xmm3, %xmm2, %xmm2
 ; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
 ; AVX512BW-NEXT:    vpsrld %xmm3, %zmm1, %zmm3
+; AVX512BW-NEXT:    vpbroadcastd %xmm2, %zmm2
 ; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
 ; AVX512BW-NEXT:    vpsubd %xmm2, %xmm4, %xmm4
 ; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
 ; AVX512BW-NEXT:    vpslld %xmm4, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpord %zmm3, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpbroadcastd %xmm2, %zmm2
 ; AVX512BW-NEXT:    vptestnmd %zmm2, %zmm2, %k1
 ; AVX512BW-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
 ; AVX512BW-NEXT:    retq
@@ -686,12 +686,12 @@ define <16 x i32> @splatvar_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i
 ; AVX512VLBW-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm2, %xmm2
 ; AVX512VLBW-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
 ; AVX512VLBW-NEXT:    vpsrld %xmm3, %zmm1, %zmm3
+; AVX512VLBW-NEXT:    vpbroadcastd %xmm2, %zmm2
 ; AVX512VLBW-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [32,32,32,32]
 ; AVX512VLBW-NEXT:    vpsubd %xmm2, %xmm4, %xmm4
 ; AVX512VLBW-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
 ; AVX512VLBW-NEXT:    vpslld %xmm4, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vpord %zmm3, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vpbroadcastd %xmm2, %zmm2
 ; AVX512VLBW-NEXT:    vptestnmd %zmm2, %zmm2, %k1
 ; AVX512VLBW-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
 ; AVX512VLBW-NEXT:    retq
@@ -759,12 +759,12 @@ define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i
 ; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; AVX512BW-NEXT:    vpsrlw %xmm3, %zmm1, %zmm3
+; AVX512BW-NEXT:    vpbroadcastw %xmm2, %zmm2
 ; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
 ; AVX512BW-NEXT:    vpsubw %xmm2, %xmm4, %xmm4
 ; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
 ; AVX512BW-NEXT:    vpsllw %xmm4, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vporq %zmm3, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpbroadcastw %xmm2, %zmm2
 ; AVX512BW-NEXT:    vptestnmw %zmm2, %zmm2, %k1
 ; AVX512BW-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1}
 ; AVX512BW-NEXT:    retq
@@ -781,12 +781,12 @@ define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i
 ; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
 ; AVX512VLBW-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; AVX512VLBW-NEXT:    vpsrlw %xmm3, %zmm1, %zmm3
+; AVX512VLBW-NEXT:    vpbroadcastw %xmm2, %zmm2
 ; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
 ; AVX512VLBW-NEXT:    vpsubw %xmm2, %xmm4, %xmm4
 ; AVX512VLBW-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
 ; AVX512VLBW-NEXT:    vpsllw %xmm4, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vporq %zmm3, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vpbroadcastw %xmm2, %zmm2
 ; AVX512VLBW-NEXT:    vptestnmw %zmm2, %zmm2, %k1
 ; AVX512VLBW-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1}
 ; AVX512VLBW-NEXT:    retq
@@ -872,88 +872,88 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %
 ; AVX512BW-LABEL: splatvar_funnnel_v64i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512BW-NEXT:    vpsubb %xmm2, %xmm3, %xmm3
-; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-NEXT:    vpsllw %xmm3, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX512BW-NEXT:    vpsllw %xmm3, %xmm4, %xmm3
-; AVX512BW-NEXT:    vpbroadcastb %xmm3, %zmm3
-; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm3
+; AVX512BW-NEXT:    vpbroadcastb %xmm2, %zmm3
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512BW-NEXT:    vpsubb %xmm3, %xmm4, %xmm4
+; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vpsllw %xmm4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
+; AVX512BW-NEXT:    vpsllw %xmm4, %xmm5, %xmm4
+; AVX512BW-NEXT:    vpbroadcastb %xmm4, %zmm4
+; AVX512BW-NEXT:    vpandq %zmm4, %zmm0, %zmm4
 ; AVX512BW-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-NEXT:    vpsrlw %xmm0, %zmm1, %zmm5
-; AVX512BW-NEXT:    vpsrlw %xmm0, %xmm4, %xmm0
+; AVX512BW-NEXT:    vpsrlw %xmm0, %zmm1, %zmm2
+; AVX512BW-NEXT:    vpsrlw %xmm0, %xmm5, %xmm0
 ; AVX512BW-NEXT:    vpsrlw $8, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpbroadcastb %xmm0, %zmm0
-; AVX512BW-NEXT:    vpternlogq $236, %zmm5, %zmm3, %zmm0
-; AVX512BW-NEXT:    vpbroadcastb %xmm2, %zmm2
-; AVX512BW-NEXT:    vptestnmb %zmm2, %zmm2, %k1
+; AVX512BW-NEXT:    vpternlogq $236, %zmm2, %zmm4, %zmm0
+; AVX512BW-NEXT:    vptestnmb %zmm3, %zmm3, %k1
 ; AVX512BW-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VBMI2-LABEL: splatvar_funnnel_v64i8:
 ; AVX512VBMI2:       # %bb.0:
 ; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VBMI2-NEXT:    vpsubb %xmm2, %xmm3, %xmm3
-; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VBMI2-NEXT:    vpsllw %xmm3, %zmm0, %zmm0
-; AVX512VBMI2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX512VBMI2-NEXT:    vpsllw %xmm3, %xmm4, %xmm3
-; AVX512VBMI2-NEXT:    vpbroadcastb %xmm3, %zmm3
-; AVX512VBMI2-NEXT:    vpandq %zmm3, %zmm0, %zmm3
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm2, %zmm3
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VBMI2-NEXT:    vpsubb %xmm3, %xmm4, %xmm4
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsllw %xmm4, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
+; AVX512VBMI2-NEXT:    vpsllw %xmm4, %xmm5, %xmm4
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm4, %zmm4
+; AVX512VBMI2-NEXT:    vpandq %zmm4, %zmm0, %zmm4
 ; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VBMI2-NEXT:    vpsrlw %xmm0, %zmm1, %zmm5
-; AVX512VBMI2-NEXT:    vpsrlw %xmm0, %xmm4, %xmm0
+; AVX512VBMI2-NEXT:    vpsrlw %xmm0, %zmm1, %zmm2
+; AVX512VBMI2-NEXT:    vpsrlw %xmm0, %xmm5, %xmm0
 ; AVX512VBMI2-NEXT:    vpsrlw $8, %xmm0, %xmm0
 ; AVX512VBMI2-NEXT:    vpbroadcastb %xmm0, %zmm0
-; AVX512VBMI2-NEXT:    vpternlogq $236, %zmm5, %zmm3, %zmm0
-; AVX512VBMI2-NEXT:    vpbroadcastb %xmm2, %zmm2
-; AVX512VBMI2-NEXT:    vptestnmb %zmm2, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vpternlogq $236, %zmm2, %zmm4, %zmm0
+; AVX512VBMI2-NEXT:    vptestnmb %zmm3, %zmm3, %k1
 ; AVX512VBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v64i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VLBW-NEXT:    vpsubb %xmm2, %xmm3, %xmm3
-; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VLBW-NEXT:    vpsllw %xmm3, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX512VLBW-NEXT:    vpsllw %xmm3, %xmm4, %xmm3
-; AVX512VLBW-NEXT:    vpbroadcastb %xmm3, %zmm3
-; AVX512VLBW-NEXT:    vpandq %zmm3, %zmm0, %zmm3
+; AVX512VLBW-NEXT:    vpbroadcastb %xmm2, %zmm3
+; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLBW-NEXT:    vpsubb %xmm3, %xmm4, %xmm4
+; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLBW-NEXT:    vpsllw %xmm4, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
+; AVX512VLBW-NEXT:    vpsllw %xmm4, %xmm5, %xmm4
+; AVX512VLBW-NEXT:    vpbroadcastb %xmm4, %zmm4
+; AVX512VLBW-NEXT:    vpandq %zmm4, %zmm0, %zmm4
 ; AVX512VLBW-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VLBW-NEXT:    vpsrlw %xmm0, %zmm1, %zmm5
-; AVX512VLBW-NEXT:    vpsrlw %xmm0, %xmm4, %xmm0
+; AVX512VLBW-NEXT:    vpsrlw %xmm0, %zmm1, %zmm2
+; AVX512VLBW-NEXT:    vpsrlw %xmm0, %xmm5, %xmm0
 ; AVX512VLBW-NEXT:    vpsrlw $8, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    vpbroadcastb %xmm0, %zmm0
-; AVX512VLBW-NEXT:    vpternlogq $236, %zmm5, %zmm3, %zmm0
-; AVX512VLBW-NEXT:    vpbroadcastb %xmm2, %zmm2
-; AVX512VLBW-NEXT:    vptestnmb %zmm2, %zmm2, %k1
+; AVX512VLBW-NEXT:    vpternlogq $236, %zmm2, %zmm4, %zmm0
+; AVX512VLBW-NEXT:    vptestnmb %zmm3, %zmm3, %k1
 ; AVX512VLBW-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
 ; AVX512VLBW-NEXT:    retq
 ;
 ; AVX512VLVBMI2-LABEL: splatvar_funnnel_v64i8:
 ; AVX512VLVBMI2:       # %bb.0:
 ; AVX512VLVBMI2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VLVBMI2-NEXT:    vpsubb %xmm2, %xmm3, %xmm3
-; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VLVBMI2-NEXT:    vpsllw %xmm3, %zmm0, %zmm0
-; AVX512VLVBMI2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX512VLVBMI2-NEXT:    vpsllw %xmm3, %xmm4, %xmm3
-; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm3, %zmm3
-; AVX512VLVBMI2-NEXT:    vpandq %zmm3, %zmm0, %zmm3
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm2, %zmm3
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLVBMI2-NEXT:    vpsubb %xmm3, %xmm4, %xmm4
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpsllw %xmm4, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
+; AVX512VLVBMI2-NEXT:    vpsllw %xmm4, %xmm5, %xmm4
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm4, %zmm4
+; AVX512VLVBMI2-NEXT:    vpandq %zmm4, %zmm0, %zmm4
 ; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VLVBMI2-NEXT:    vpsrlw %xmm0, %zmm1, %zmm5
-; AVX512VLVBMI2-NEXT:    vpsrlw %xmm0, %xmm4, %xmm0
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm0, %zmm1, %zmm2
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm0, %xmm5, %xmm0
 ; AVX512VLVBMI2-NEXT:    vpsrlw $8, %xmm0, %xmm0
 ; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm0, %zmm0
-; AVX512VLVBMI2-NEXT:    vpternlogq $236, %zmm5, %zmm3, %zmm0
-; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm2, %zmm2
-; AVX512VLVBMI2-NEXT:    vptestnmb %zmm2, %zmm2, %k1
+; AVX512VLVBMI2-NEXT:    vpternlogq $236, %zmm2, %zmm4, %zmm0
+; AVX512VLVBMI2-NEXT:    vptestnmb %zmm3, %zmm3, %k1
 ; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
 ; AVX512VLVBMI2-NEXT:    retq
   %splat = shufflevector <64 x i8> %amt, <64 x i8> undef, <64 x i32> zeroinitializer
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
index d642e513c49bf..fda0dacedf85b 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
@@ -366,13 +366,14 @@ define <16 x i32> @splatvar_funnnel_v16i32(<16 x i32> %x, <16 x i32> %amt) nounw
 define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %amt) nounwind {
 ; AVX512F-LABEL: splatvar_funnnel_v32i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm3
-; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
-; AVX512F-NEXT:    vpsrlw %xmm3, %ymm2, %ymm4
-; AVX512F-NEXT:    vpsrlw %xmm3, %ymm0, %ymm3
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm4, %zmm3, %zmm3
 ; AVX512F-NEXT:    vpbroadcastw %xmm1, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm2
+; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vpsrlw %xmm2, %ymm3, %ymm4
+; AVX512F-NEXT:    vpsrlw %xmm2, %ymm0, %ymm2
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
 ; AVX512F-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; AVX512F-NEXT:    vpsubw %ymm1, %ymm4, %ymm1
 ; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
@@ -380,22 +381,23 @@ define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %amt) nounw
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; AVX512F-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512F-NEXT:    vpsllvd %zmm1, %zmm2, %zmm1
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512F-NEXT:    vpsllvd %zmm1, %zmm3, %zmm1
 ; AVX512F-NEXT:    vpmovdw %zmm1, %ymm1
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm3, %zmm0, %zmm0
+; AVX512F-NEXT:    vporq %zmm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_funnnel_v32i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm3
-; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
-; AVX512VL-NEXT:    vpsrlw %xmm3, %ymm2, %ymm4
-; AVX512VL-NEXT:    vpsrlw %xmm3, %ymm0, %ymm3
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm4, %zmm3, %zmm3
 ; AVX512VL-NEXT:    vpbroadcastw %xmm1, %ymm1
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm1, %zmm1
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm2
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
+; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512VL-NEXT:    vpsrlw %xmm2, %ymm3, %ymm4
+; AVX512VL-NEXT:    vpsrlw %xmm2, %ymm0, %ymm2
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
 ; AVX512VL-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; AVX512VL-NEXT:    vpsubw %ymm1, %ymm4, %ymm1
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
@@ -403,11 +405,11 @@ define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %amt) nounw
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; AVX512VL-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vpmovdw %zmm0, %ymm0
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512VL-NEXT:    vpsllvd %zmm1, %zmm2, %zmm1
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512VL-NEXT:    vpsllvd %zmm1, %zmm3, %zmm1
 ; AVX512VL-NEXT:    vpmovdw %zmm1, %ymm1
 ; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512VL-NEXT:    vporq %zmm3, %zmm0, %zmm0
+; AVX512VL-NEXT:    vporq %zmm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v32i16:
@@ -447,13 +449,14 @@ define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %amt) nounw
 define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {
 ; AVX512F-LABEL: splatvar_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpbroadcastb %xmm1, %ymm1
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm1, %zmm1
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
 ; AVX512F-NEXT:    vpsllw $4, %ymm2, %ymm3
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512F-NEXT:    vpand %ymm4, %ymm3, %ymm3
-; AVX512F-NEXT:    vpbroadcastb %xmm1, %ymm5
-; AVX512F-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX512F-NEXT:    vpsubb %ymm5, %ymm6, %ymm5
+; AVX512F-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX512F-NEXT:    vpsubb %ymm1, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm5, %ymm5
 ; AVX512F-NEXT:    vpsllw $5, %ymm5, %ymm5
 ; AVX512F-NEXT:    vpblendvb %ymm5, %ymm3, %ymm2, %ymm3
@@ -490,13 +493,14 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind
 ;
 ; AVX512VL-LABEL: splatvar_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpbroadcastb %xmm1, %ymm1
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm1, %zmm1
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm2, %ymm3
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; AVX512VL-NEXT:    vpand %ymm4, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpbroadcastb %xmm1, %ymm5
-; AVX512VL-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX512VL-NEXT:    vpsubb %ymm5, %ymm6, %ymm5
+; AVX512VL-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX512VL-NEXT:    vpsubb %ymm1, %ymm5, %ymm5
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm5, %ymm5
 ; AVX512VL-NEXT:    vpsllw $5, %ymm5, %ymm5
 ; AVX512VL-NEXT:    vpblendvb %ymm5, %ymm3, %ymm2, %ymm3
diff --git a/llvm/test/CodeGen/X86/vector-rotate-512.ll b/llvm/test/CodeGen/X86/vector-rotate-512.ll
index 05d989ebaa307..831c03f03825a 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-512.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-512.ll
@@ -348,38 +348,34 @@ define <32 x i16> @splatvar_rotate_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind
 ; AVX512F-LABEL: splatvar_rotate_v32i16:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
-; AVX512F-NEXT:    vpsubw %xmm1, %xmm3, %xmm4
-; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
 ; AVX512F-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
 ; AVX512F-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
 ; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
-; AVX512F-NEXT:    vpsllw %xmm2, %ymm3, %ymm5
+; AVX512F-NEXT:    vpsllw %xmm2, %ymm3, %ymm4
 ; AVX512F-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm5, %zmm2, %zmm2
-; AVX512F-NEXT:    vpsrlw %xmm1, %ymm3, %ymm1
-; AVX512F-NEXT:    vpsrlw %xmm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
+; AVX512F-NEXT:    vpsrlw %xmm1, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
 ; AVX512F-NEXT:    vporq %zmm0, %zmm2, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_rotate_v32i16:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
-; AVX512VL-NEXT:    vpsubw %xmm1, %xmm3, %xmm4
-; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
 ; AVX512VL-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
 ; AVX512VL-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
 ; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
-; AVX512VL-NEXT:    vpsllw %xmm2, %ymm3, %ymm5
+; AVX512VL-NEXT:    vpsllw %xmm2, %ymm3, %ymm4
 ; AVX512VL-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm5, %zmm2, %zmm2
-; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm3, %ymm1
-; AVX512VL-NEXT:    vpsrlw %xmm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
+; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vporq %zmm0, %zmm2, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
@@ -416,64 +412,54 @@ define <64 x i8> @splatvar_rotate_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; AVX512F-LABEL: splatvar_rotate_v64i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512F-NEXT:    vpsubb %xmm1, %xmm3, %xmm4
-; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512F-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; AVX512F-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
 ; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
-; AVX512F-NEXT:    vpsllw %xmm2, %ymm3, %ymm5
-; AVX512F-NEXT:    vpcmpeqd %xmm6, %xmm6, %xmm6
-; AVX512F-NEXT:    vpsllw %xmm2, %xmm6, %xmm7
-; AVX512F-NEXT:    vpbroadcastb %xmm7, %ymm7
-; AVX512F-NEXT:    vpand %ymm7, %ymm5, %ymm5
+; AVX512F-NEXT:    vpsllw %xmm2, %ymm3, %ymm4
+; AVX512F-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
+; AVX512F-NEXT:    vpsllw %xmm2, %xmm5, %xmm6
+; AVX512F-NEXT:    vpbroadcastb %xmm6, %ymm6
+; AVX512F-NEXT:    vpand %ymm6, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
-; AVX512F-NEXT:    vpand %ymm7, %ymm2, %ymm2
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm5, %zmm2, %zmm2
+; AVX512F-NEXT:    vpand %ymm6, %ymm2, %ymm2
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
 ; AVX512F-NEXT:    vpsrlw %xmm1, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsrlw %xmm1, %xmm6, %xmm1
-; AVX512F-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX512F-NEXT:    vpbroadcastb %xmm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm1, %ymm3, %ymm1
-; AVX512F-NEXT:    vpsrlw %xmm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw %xmm4, %xmm6, %xmm3
-; AVX512F-NEXT:    vpsrlw $8, %xmm3, %xmm3
-; AVX512F-NEXT:    vpbroadcastb %xmm3, %ymm3
-; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpsrlw %xmm1, %xmm5, %xmm4
+; AVX512F-NEXT:    vpsrlw $8, %xmm4, %xmm4
+; AVX512F-NEXT:    vpbroadcastb %xmm4, %ymm4
+; AVX512F-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
 ; AVX512F-NEXT:    vporq %zmm0, %zmm2, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_rotate_v64i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VL-NEXT:    vpsubb %xmm1, %xmm3, %xmm4
-; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; AVX512VL-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
 ; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
-; AVX512VL-NEXT:    vpsllw %xmm2, %ymm3, %ymm5
-; AVX512VL-NEXT:    vpcmpeqd %xmm6, %xmm6, %xmm6
-; AVX512VL-NEXT:    vpsllw %xmm2, %xmm6, %xmm7
-; AVX512VL-NEXT:    vpbroadcastb %xmm7, %ymm7
-; AVX512VL-NEXT:    vpand %ymm7, %ymm5, %ymm5
+; AVX512VL-NEXT:    vpsllw %xmm2, %ymm3, %ymm4
+; AVX512VL-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
+; AVX512VL-NEXT:    vpsllw %xmm2, %xmm5, %xmm6
+; AVX512VL-NEXT:    vpbroadcastb %xmm6, %ymm6
+; AVX512VL-NEXT:    vpand %ymm6, %ymm4, %ymm4
 ; AVX512VL-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
-; AVX512VL-NEXT:    vpand %ymm7, %ymm2, %ymm2
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm5, %zmm2, %zmm2
+; AVX512VL-NEXT:    vpand %ymm6, %ymm2, %ymm2
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
 ; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpsrlw %xmm1, %xmm6, %xmm1
-; AVX512VL-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpbroadcastb %xmm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm1, %ymm3, %ymm1
-; AVX512VL-NEXT:    vpsrlw %xmm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw %xmm4, %xmm6, %xmm3
-; AVX512VL-NEXT:    vpsrlw $8, %xmm3, %xmm3
-; AVX512VL-NEXT:    vpbroadcastb %xmm3, %ymm3
-; AVX512VL-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpsrlw %xmm1, %xmm5, %xmm4
+; AVX512VL-NEXT:    vpsrlw $8, %xmm4, %xmm4
+; AVX512VL-NEXT:    vpbroadcastb %xmm4, %ymm4
+; AVX512VL-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
 ; AVX512VL-NEXT:    vporq %zmm0, %zmm2, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
diff --git a/llvm/test/DebugInfo/ARM/PR26163.ll b/llvm/test/DebugInfo/ARM/PR26163.ll
index 0beb942b638f8..798812fef201c 100644
--- a/llvm/test/DebugInfo/ARM/PR26163.ll
+++ b/llvm/test/DebugInfo/ARM/PR26163.ll
@@ -8,10 +8,14 @@
 ; but it is what is currently being emitted. Any change here needs to be
 ; intentional, so the test is very specific.
 ;
-; CHECK: DW_TAG_inlined_subroutine
-; CHECK: DW_TAG_variable
-; CHECK:   DW_AT_location ({{.*}}
-; CHECK-NEXT: [0x00000004, 0x00000014): DW_OP_lit0, DW_OP_stack_value, DW_OP_piece 0x4)
+; The variable is given a single location instead of a location list entry
+; because the function validThroughout has a special code path for single
+; locations with a constant value that start in the prologue.
+;
+; CHECK:      DW_TAG_inlined_subroutine
+; CHECK:        DW_TAG_variable
+; CHECK-NEXT:     DW_AT_location (DW_OP_lit0, DW_OP_stack_value, DW_OP_piece 0x4)
+; CHECK-NEXT      DW_AT_name ("i4")
 
 ; Created form the following test case (PR26163) with
 ; clang -cc1 -triple armv4t--freebsd11.0-gnueabi -emit-obj -debug-info-kind=standalone -O2 -x c test.c
diff --git a/llvm/test/DebugInfo/COFF/asm.ll b/llvm/test/DebugInfo/COFF/asm.ll
index c259a53f68a8f..d873df26bf718 100644
--- a/llvm/test/DebugInfo/COFF/asm.ll
+++ b/llvm/test/DebugInfo/COFF/asm.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32 -O0 < %s | FileCheck --check-prefix=X86 %s
-; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32 -o - -O0 < %s | llvm-mc -triple=i686-pc-win32 -filetype=obj | llvm-readobj -S --sr --codeview | FileCheck --check-prefix=OBJ32 %s
+; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32 -o - -O0 < %s | llvm-mc -triple=i686-pc-win32 -filetype=obj | llvm-readobj -S --sr --codeview - | FileCheck --check-prefix=OBJ32 %s
 ; RUN: llc -mcpu=core2 -mtriple=x86_64-pc-win32 -O0 < %s | FileCheck --check-prefix=X64 %s
-; RUN: llc -mcpu=core2 -mtriple=x86_64-pc-win32 -o - -O0 < %s | llvm-mc -triple=x86_64-pc-win32 -filetype=obj | llvm-readobj -S --sr --codeview | FileCheck --check-prefix=OBJ64 %s
+; RUN: llc -mcpu=core2 -mtriple=x86_64-pc-win32 -o - -O0 < %s | llvm-mc -triple=x86_64-pc-win32 -filetype=obj | llvm-readobj -S --sr --codeview - | FileCheck --check-prefix=OBJ64 %s
 
 ; This LL file was generated by running clang on the following code:
 ; D:\asm.c:
diff --git a/llvm/test/DebugInfo/COFF/cpp-mangling.ll b/llvm/test/DebugInfo/COFF/cpp-mangling.ll
index c48dae30ad239..d94de78514e09 100644
--- a/llvm/test/DebugInfo/COFF/cpp-mangling.ll
+++ b/llvm/test/DebugInfo/COFF/cpp-mangling.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32 -o - -O0 -filetype=obj < %s \
-; RUN:   | llvm-readobj --codeview | FileCheck %s
+; RUN:   | llvm-readobj --codeview - | FileCheck %s
 
 ; C++ source to regenerate:
 ; namespace foo {
diff --git a/llvm/test/DebugInfo/COFF/dlang.ll b/llvm/test/DebugInfo/COFF/dlang.ll
index 9c8ce17b4a3d1..c73089b5469ef 100644
--- a/llvm/test/DebugInfo/COFF/dlang.ll
+++ b/llvm/test/DebugInfo/COFF/dlang.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s | FileCheck %s --check-prefix=ASM
-; RUN: llc -filetype=obj < %s | llvm-readobj --codeview | FileCheck %s --check-prefix=OBJ
+; RUN: llc -filetype=obj < %s | llvm-readobj --codeview - | FileCheck %s --check-prefix=OBJ
 
 ; ASM:      .short  4412                    # Record kind: S_COMPILE3
 ; ASM-NEXT: .long   68                      # Flags and language
diff --git a/llvm/test/DebugInfo/COFF/fpo-csrs.ll b/llvm/test/DebugInfo/COFF/fpo-csrs.ll
index 926adab350ba4..c5b1dc996f3ba 100644
--- a/llvm/test/DebugInfo/COFF/fpo-csrs.ll
+++ b/llvm/test/DebugInfo/COFF/fpo-csrs.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s | FileCheck %s --check-prefix=ASM
-; RUN: llc -filetype=obj < %s | llvm-readobj --codeview | FileCheck %s --check-prefix=OBJ
+; RUN: llc -filetype=obj < %s | llvm-readobj --codeview - | FileCheck %s --check-prefix=OBJ
 
 ; C source:
 ; int getval(void);
diff --git a/llvm/test/DebugInfo/COFF/fpo-realign-vframe.ll b/llvm/test/DebugInfo/COFF/fpo-realign-vframe.ll
index 87e80a69edbcb..29e0e6adab377 100644
--- a/llvm/test/DebugInfo/COFF/fpo-realign-vframe.ll
+++ b/llvm/test/DebugInfo/COFF/fpo-realign-vframe.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s | FileCheck %s --check-prefix=ASM
-; RUN: llc -filetype=obj < %s | llvm-readobj --codeview | FileCheck %s --check-prefix=OBJ
+; RUN: llc -filetype=obj < %s | llvm-readobj --codeview - | FileCheck %s --check-prefix=OBJ
 
 ; PR38857
 
diff --git a/llvm/test/DebugInfo/COFF/fpo-shrink-wrap.ll b/llvm/test/DebugInfo/COFF/fpo-shrink-wrap.ll
index 23a1f56fdf409..719d69d16a625 100644
--- a/llvm/test/DebugInfo/COFF/fpo-shrink-wrap.ll
+++ b/llvm/test/DebugInfo/COFF/fpo-shrink-wrap.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -enable-shrink-wrap=true < %s | FileCheck %s --check-prefix=ASM
-; RUN: llc -enable-shrink-wrap=true -filetype=obj < %s | llvm-readobj --codeview | FileCheck %s --check-prefix=OBJ
+; RUN: llc -enable-shrink-wrap=true -filetype=obj < %s | llvm-readobj --codeview - | FileCheck %s --check-prefix=OBJ
 
 ; C source:
 ; int doSomething(int*);
diff --git a/llvm/test/DebugInfo/COFF/inlining-files.ll b/llvm/test/DebugInfo/COFF/inlining-files.ll
index ce92d20e093e5..6b50be90ea2fb 100644
--- a/llvm/test/DebugInfo/COFF/inlining-files.ll
+++ b/llvm/test/DebugInfo/COFF/inlining-files.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32 < %s -filetype=obj | llvm-readobj --codeview | FileCheck %s --check-prefix=OBJ
+; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32 < %s -filetype=obj | llvm-readobj --codeview - | FileCheck %s --check-prefix=OBJ
 
 ; This LL file was generated by running 'clang -O1 -g -gcodeview' on the
 ; volatile int x;
diff --git a/llvm/test/DebugInfo/COFF/inlining-header.ll b/llvm/test/DebugInfo/COFF/inlining-header.ll
index de8ea595b7b8e..3eca5a032f5c4 100644
--- a/llvm/test/DebugInfo/COFF/inlining-header.ll
+++ b/llvm/test/DebugInfo/COFF/inlining-header.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32 < %s | FileCheck %s --check-prefix=ASM
-; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32 < %s -filetype=obj | llvm-readobj --codeview | FileCheck %s --check-prefix=OBJ
+; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32 < %s -filetype=obj | llvm-readobj --codeview - | FileCheck %s --check-prefix=OBJ
 
 ; This C++ source should run and you should be able to step through the volatile
 ; modifications to x in windbg.
diff --git a/llvm/test/DebugInfo/COFF/inlining-levels.ll b/llvm/test/DebugInfo/COFF/inlining-levels.ll
index c1613db874019..72190d4b2e4b4 100644
--- a/llvm/test/DebugInfo/COFF/inlining-levels.ll
+++ b/llvm/test/DebugInfo/COFF/inlining-levels.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32 < %s -filetype=obj | llvm-readobj --codeview | FileCheck %s --check-prefix=OBJ
+; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32 < %s -filetype=obj | llvm-readobj --codeview - | FileCheck %s --check-prefix=OBJ
 
 ; This LL file was generated by running 'clang -O1 -g -gcodeview' on the
 ; following code:
diff --git a/llvm/test/DebugInfo/COFF/inlining-padding.ll b/llvm/test/DebugInfo/COFF/inlining-padding.ll
index 4437cbef95d5f..b29e97f235a21 100644
--- a/llvm/test/DebugInfo/COFF/inlining-padding.ll
+++ b/llvm/test/DebugInfo/COFF/inlining-padding.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -filetype=obj -o - | llvm-readobj --codeview --codeview-subsection-bytes | FileCheck %s
-; RUN: llc < %s -o - | llvm-mc -filetype=obj --triple=x86_64-windows | llvm-readobj --codeview --codeview-subsection-bytes | FileCheck %s
+; RUN: llc < %s -filetype=obj -o - | llvm-readobj --codeview --codeview-subsection-bytes - | FileCheck %s
+; RUN: llc < %s -o - | llvm-mc -filetype=obj --triple=x86_64-windows | llvm-readobj --codeview --codeview-subsection-bytes - | FileCheck %s
 
 ; Check how we pad out the LF_FUNC_ID records. The 00F3F2F1 bytes in LeafData are
 ; what's interesting here.
diff --git a/llvm/test/DebugInfo/COFF/inlining.ll b/llvm/test/DebugInfo/COFF/inlining.ll
index c1b6e69f2b17f..a3c0da76166fa 100644
--- a/llvm/test/DebugInfo/COFF/inlining.ll
+++ b/llvm/test/DebugInfo/COFF/inlining.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32 < %s | FileCheck %s --check-prefix=ASM
-; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32 < %s -filetype=obj | llvm-readobj --codeview | FileCheck %s --check-prefix=OBJ
+; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32 < %s -filetype=obj | llvm-readobj --codeview - | FileCheck %s --check-prefix=OBJ
 
 ; This LL file was generated by running 'clang -O1 -g -gcodeview' on the
 ; following code:
diff --git a/llvm/test/DebugInfo/COFF/lines-difile.ll b/llvm/test/DebugInfo/COFF/lines-difile.ll
index 56be6b3d08d93..74795f164c14e 100644
--- a/llvm/test/DebugInfo/COFF/lines-difile.ll
+++ b/llvm/test/DebugInfo/COFF/lines-difile.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s | FileCheck %s --check-prefix=ASM
-; RUN: llc < %s -filetype=obj | llvm-readobj --codeview | FileCheck %s --check-prefix=OBJ
+; RUN: llc < %s -filetype=obj | llvm-readobj --codeview - | FileCheck %s --check-prefix=OBJ
 
 ; Test what happens when we have two DIFile entries with differing slashes.
 ; Make sure we only emit one file checksum entry.
diff --git a/llvm/test/DebugInfo/COFF/long-name.ll b/llvm/test/DebugInfo/COFF/long-name.ll
index 08be48be8210c..073bd7909cb5e 100644
--- a/llvm/test/DebugInfo/COFF/long-name.ll
+++ b/llvm/test/DebugInfo/COFF/long-name.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -filetype=obj | llvm-readobj --codeview | FileCheck %s
+; RUN: llc < %s -filetype=obj | llvm-readobj --codeview - | FileCheck %s
 
 ; CHECK: {{.*}}Proc{{.*}}Sym {
 ; CHECK:   Kind: S_GPROC32_ID (0x1147)
diff --git a/llvm/test/DebugInfo/COFF/long-type-name.ll b/llvm/test/DebugInfo/COFF/long-type-name.ll
index 1488e0891fc82..8d89bd9232401 100644
--- a/llvm/test/DebugInfo/COFF/long-type-name.ll
+++ b/llvm/test/DebugInfo/COFF/long-type-name.ll
@@ -1,5 +1,5 @@
-; RUN: llc %s -o - -filetype=obj | llvm-readobj --codeview | FileCheck %s
-; RUN: llc %s -o - | llvm-mc -filetype=obj --triple=x86_64-windows | llvm-readobj --codeview | FileCheck %s
+; RUN: llc %s -o - -filetype=obj | llvm-readobj --codeview - | FileCheck %s
+; RUN: llc %s -o - | llvm-mc -filetype=obj --triple=x86_64-windows | llvm-readobj --codeview - | FileCheck %s
 
 ; CHECK: Struct (0x1000) {
 ; CHECK:   TypeLeafKind: LF_STRUCTURE (0x1505)
diff --git a/llvm/test/DebugInfo/COFF/multifile.ll b/llvm/test/DebugInfo/COFF/multifile.ll
index d40f8b7885710..5123b1dece94e 100644
--- a/llvm/test/DebugInfo/COFF/multifile.ll
+++ b/llvm/test/DebugInfo/COFF/multifile.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32 -O0 < %s | FileCheck --check-prefix=X86 %s
-; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32 -o - -O0 < %s | llvm-mc -triple=i686-pc-win32 -filetype=obj | llvm-readobj -S --sr --codeview --section-symbols | FileCheck --check-prefix=OBJ32 %s
+; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32 -o - -O0 < %s | llvm-mc -triple=i686-pc-win32 -filetype=obj | llvm-readobj -S --sr --codeview --section-symbols - | FileCheck --check-prefix=OBJ32 %s
 ; RUN: llc -mcpu=core2 -mtriple=x86_64-pc-win32 -O0 < %s | FileCheck --check-prefix=X64 %s
-; RUN: llc -mcpu=core2 -mtriple=x86_64-pc-win32 -o - -O0 < %s | llvm-mc -triple=x86_64-pc-win32 -filetype=obj | llvm-readobj -S --sr --codeview --section-symbols | FileCheck --check-prefix=OBJ64 %s
+; RUN: llc -mcpu=core2 -mtriple=x86_64-pc-win32 -o - -O0 < %s | llvm-mc -triple=x86_64-pc-win32 -filetype=obj | llvm-readobj -S --sr --codeview --section-symbols - | FileCheck --check-prefix=OBJ64 %s
 
 ; This LL file was generated by running clang on the following code:
 ; D:\input.c:
diff --git a/llvm/test/DebugInfo/COFF/multifunction.ll b/llvm/test/DebugInfo/COFF/multifunction.ll
index b890e6b0b5261..9f99ba05047e6 100644
--- a/llvm/test/DebugInfo/COFF/multifunction.ll
+++ b/llvm/test/DebugInfo/COFF/multifunction.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32 -O0 < %s | FileCheck --check-prefix=X86 %s
-; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32 -o - -O0 < %s | llvm-mc -triple=i686-pc-win32 -filetype=obj | llvm-readobj -S --sr --codeview --section-symbols | FileCheck --check-prefix=OBJ32 %s
+; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32 -o - -O0 < %s | llvm-mc -triple=i686-pc-win32 -filetype=obj | llvm-readobj -S --sr --codeview --section-symbols - | FileCheck --check-prefix=OBJ32 %s
 ; RUN: llc -mcpu=core2 -mtriple=x86_64-pc-win32 -O0 < %s | FileCheck --check-prefix=X64 %s
-; RUN: llc -mcpu=core2 -mtriple=x86_64-pc-win32 -o - -O0 < %s | llvm-mc -triple=x86_64-pc-win32 -filetype=obj | llvm-readobj -S --sr --codeview --section-symbols | FileCheck --check-prefix=OBJ64 %s
+; RUN: llc -mcpu=core2 -mtriple=x86_64-pc-win32 -o - -O0 < %s | llvm-mc -triple=x86_64-pc-win32 -filetype=obj | llvm-readobj -S --sr --codeview --section-symbols - | FileCheck --check-prefix=OBJ64 %s
 
 ; This LL file was generated by running clang on the following code:
 ; D:\source.c:
diff --git a/llvm/test/DebugInfo/COFF/pieces.ll b/llvm/test/DebugInfo/COFF/pieces.ll
index 4e11dcabb4d8e..330c70453b5d2 100644
--- a/llvm/test/DebugInfo/COFF/pieces.ll
+++ b/llvm/test/DebugInfo/COFF/pieces.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s | FileCheck %s --check-prefix=ASM
-; RUN: llc < %s -filetype=obj | llvm-readobj --codeview | FileCheck %s --check-prefix=OBJ
+; RUN: llc < %s -filetype=obj | llvm-readobj --codeview - | FileCheck %s --check-prefix=OBJ
 
 ; Compile with -O1 as C
 
diff --git a/llvm/test/DebugInfo/COFF/register-variables.ll b/llvm/test/DebugInfo/COFF/register-variables.ll
index 60c9bb0d96c3b..4ecb0642e1902 100644
--- a/llvm/test/DebugInfo/COFF/register-variables.ll
+++ b/llvm/test/DebugInfo/COFF/register-variables.ll
@@ -60,13 +60,11 @@
 ; ASM:         .cv_def_range    [[p_ecx_esi]] [[func_end]], reg, 23
 ; ASM:         .short  4414                    # Record kind: S_LOCAL
 ; ASM:         .asciz  "c"
-; ASM:         .cv_def_range    [[after_if]] [[func_finished]], reg, 17
 ; ASM:         .short  4414                    # Record kind: S_LOCAL
 ; ASM:         .asciz  "a"
 ; ASM:         .cv_def_range    [[after_je]] [[after_inc_eax]], reg, 17
 ; ASM:         .short  4414                    # Record kind: S_LOCAL
 ; ASM:         .asciz  "b"
-; ASM:         .cv_def_range    [[after_if]] [[after_if]], reg, 17
 
 ; Note: "b" is a victim of tail de-duplication / branch folding.
 
@@ -109,18 +107,11 @@
 ; OBJ:   }
 ; OBJ:   LocalSym {
 ; OBJ:     Type: int (0x74)
-; OBJ:     Flags [ (0x0)
+; OBJ:     Flags [ (0x100)
+; OBJ:       IsOptimizedOut (0x100)
 ; OBJ:     ]
 ; OBJ:     VarName: c
 ; OBJ:   }
-; OBJ:   DefRangeRegisterSym {
-; OBJ:     Register: EAX (0x11)
-; OBJ:     LocalVariableAddrRange {
-; OBJ:       OffsetStart: .text+0x1A
-; OBJ:       ISectStart: 0x0
-; OBJ:       Range: 0xC
-; OBJ:     }
-; OBJ:   }
 ; OBJ:   LocalSym {
 ; OBJ:     Type: int (0x74)
 ; OBJ:     Flags [ (0x0)
diff --git a/llvm/test/DebugInfo/COFF/simple.ll b/llvm/test/DebugInfo/COFF/simple.ll
index 6f8afb5bf14b1..082640182dc23 100644
--- a/llvm/test/DebugInfo/COFF/simple.ll
+++ b/llvm/test/DebugInfo/COFF/simple.ll
@@ -1,15 +1,15 @@
 ; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32 -O0 < %s | FileCheck --check-prefix=X86 %s
 ; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32-itanium -O0 < %s | FileCheck --check-prefix=X86 %s
 ; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32-gnu -O0 < %s | FileCheck --check-prefix=X86 %s
-; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32 -o - -O0 < %s | llvm-mc -triple=i686-pc-win32 -filetype=obj | llvm-readobj -S --sr --codeview --section-symbols | FileCheck --check-prefix=OBJ32 %s
-; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32-itanium -o - -O0 < %s | llvm-mc -triple=i686-pc-win32 -filetype=obj | llvm-readobj -S --sr --codeview --section-symbols | FileCheck --check-prefix=OBJ32 %s
-; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32-gnu -o - -O0 < %s | llvm-mc -triple=i686-pc-win32 -filetype=obj | llvm-readobj -S --sr --codeview --section-symbols | FileCheck --check-prefix=OBJ32 %s
+; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32 -o - -O0 < %s | llvm-mc -triple=i686-pc-win32 -filetype=obj | llvm-readobj -S --sr --codeview --section-symbols - | FileCheck --check-prefix=OBJ32 %s
+; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32-itanium -o - -O0 < %s | llvm-mc -triple=i686-pc-win32 -filetype=obj | llvm-readobj -S --sr --codeview --section-symbols - | FileCheck --check-prefix=OBJ32 %s
+; RUN: llc -mcpu=core2 -mtriple=i686-pc-win32-gnu -o - -O0 < %s | llvm-mc -triple=i686-pc-win32 -filetype=obj | llvm-readobj -S --sr --codeview --section-symbols - | FileCheck --check-prefix=OBJ32 %s
 ; RUN: llc -mcpu=core2 -mtriple=x86_64-pc-win32 -O0 < %s | FileCheck --check-prefix=X64 %s
 ; RUN: llc -mcpu=core2 -mtriple=x86_64-pc-win32-itanium -O0 < %s | FileCheck --check-prefix=X64 %s
 ; RUN: llc -mcpu=core2 -mtriple=x86_64-pc-win32-gnu -O0 < %s | FileCheck --check-prefix=X64 %s
-; RUN: llc -mcpu=core2 -mtriple=x86_64-pc-win32 -o - -O0 < %s | llvm-mc -triple=x86_64-pc-win32 -filetype=obj | llvm-readobj -S --sr --codeview --section-symbols | FileCheck --check-prefix=OBJ64 %s
-; RUN: llc -mcpu=core2 -mtriple=x86_64-pc-win32-itanium -o - -O0 < %s | llvm-mc -triple=x86_64-pc-win32 -filetype=obj | llvm-readobj -S --sr --codeview --section-symbols | FileCheck --check-prefix=OBJ64 %s
-; RUN: llc -mcpu=core2 -mtriple=x86_64-pc-win32-gnu -o - -O0 < %s | llvm-mc -triple=x86_64-pc-win32 -filetype=obj | llvm-readobj -S --sr --codeview --section-symbols | FileCheck --check-prefix=OBJ64 %s
+; RUN: llc -mcpu=core2 -mtriple=x86_64-pc-win32 -o - -O0 < %s | llvm-mc -triple=x86_64-pc-win32 -filetype=obj | llvm-readobj -S --sr --codeview --section-symbols - | FileCheck --check-prefix=OBJ64 %s
+; RUN: llc -mcpu=core2 -mtriple=x86_64-pc-win32-itanium -o - -O0 < %s | llvm-mc -triple=x86_64-pc-win32 -filetype=obj | llvm-readobj -S --sr --codeview --section-symbols - | FileCheck --check-prefix=OBJ64 %s
+; RUN: llc -mcpu=core2 -mtriple=x86_64-pc-win32-gnu -o - -O0 < %s | llvm-mc -triple=x86_64-pc-win32 -filetype=obj | llvm-readobj -S --sr --codeview --section-symbols - | FileCheck --check-prefix=OBJ64 %s
 
 ; This LL file was generated by running clang on the following code:
 ; D:\test.c:
diff --git a/llvm/test/DebugInfo/COFF/swift.ll b/llvm/test/DebugInfo/COFF/swift.ll
index 4d1ffee2c15a7..81922cc4a3743 100644
--- a/llvm/test/DebugInfo/COFF/swift.ll
+++ b/llvm/test/DebugInfo/COFF/swift.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s | FileCheck %s --check-prefix=ASM
-; RUN: llc -filetype=obj < %s | llvm-readobj --codeview | FileCheck %s --check-prefix=OBJ
+; RUN: llc -filetype=obj < %s | llvm-readobj --codeview - | FileCheck %s --check-prefix=OBJ
 
 ; ASM:      .short  4412                    # Record kind: S_COMPILE3
 ; ASM-NEXT: .long   83                      # Flags and language
diff --git a/llvm/test/DebugInfo/COFF/types-array-unsized.ll b/llvm/test/DebugInfo/COFF/types-array-unsized.ll
index 020ae15bf0080..8bfb0c08ee392 100644
--- a/llvm/test/DebugInfo/COFF/types-array-unsized.ll
+++ b/llvm/test/DebugInfo/COFF/types-array-unsized.ll
@@ -1,5 +1,5 @@
-; RUN: llc -filetype=obj < %s | llvm-readobj --codeview | FileCheck %s
-; RUN: llc < %s | llvm-mc -filetype=obj --triple=x86_64-windows | llvm-readobj --codeview | FileCheck %s
+; RUN: llc -filetype=obj < %s | llvm-readobj --codeview - | FileCheck %s
+; RUN: llc < %s | llvm-mc -filetype=obj --triple=x86_64-windows | llvm-readobj --codeview - | FileCheck %s
 
 ; We should emit two array types: one used to describe the static data member,
 ; and the other used by the S_GDATA32 for the definition.
diff --git a/llvm/test/DebugInfo/COFF/vframe-csr.ll b/llvm/test/DebugInfo/COFF/vframe-csr.ll
index 0d39155db402e..07092df568e4f 100644
--- a/llvm/test/DebugInfo/COFF/vframe-csr.ll
+++ b/llvm/test/DebugInfo/COFF/vframe-csr.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s | FileCheck %s --check-prefix=ASM
-; RUN: llc -filetype=obj < %s | llvm-readobj --codeview | FileCheck %s --check-prefix=OBJ
+; RUN: llc -filetype=obj < %s | llvm-readobj --codeview - | FileCheck %s --check-prefix=OBJ
 
 ; PR38857
 
diff --git a/llvm/test/DebugInfo/COFF/vframe-fpo.ll b/llvm/test/DebugInfo/COFF/vframe-fpo.ll
index 20b3a437d27bf..6b77ac579cb3d 100644
--- a/llvm/test/DebugInfo/COFF/vframe-fpo.ll
+++ b/llvm/test/DebugInfo/COFF/vframe-fpo.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s | FileCheck %s --check-prefix=ASM
-; RUN: llc < %s -filetype=obj | llvm-readobj --codeview | FileCheck %s --check-prefix=CODEVIEW
+; RUN: llc < %s -filetype=obj | llvm-readobj --codeview - | FileCheck %s --check-prefix=CODEVIEW
 
 ; This test checks that for 32-bit x86 we use VFRAME and
 ; S_DEFRANGE_FRAMEPOINTER_REL with the right offsets. The test has two function
diff --git a/llvm/test/DebugInfo/MIR/X86/call-site-gnu-vs-dwarf5-attrs.mir b/llvm/test/DebugInfo/MIR/X86/call-site-gnu-vs-dwarf5-attrs.mir
index b60c10a04e596..fdbebd306f37c 100644
--- a/llvm/test/DebugInfo/MIR/X86/call-site-gnu-vs-dwarf5-attrs.mir
+++ b/llvm/test/DebugInfo/MIR/X86/call-site-gnu-vs-dwarf5-attrs.mir
@@ -21,7 +21,7 @@
 # RUN:     | llvm-dwarfdump - | FileCheck %s -check-prefixes=CHECK-DWARF5 -implicit-check-not=DW_AT_call
 #
 # RUN: llc -emit-call-site-info -dwarf-version 5 -filetype=obj -debugger-tune=sce \
-# RUN:     -emit-debug-entry-values -debug-entry-values -mtriple=x86_64-unknown-unknown \
+# RUN:     -debug-entry-values -mtriple=x86_64-unknown-unknown \
 # RUN:     -start-after=machineverifier -o - %s | llvm-dwarfdump - | FileCheck %s -check-prefixes=CHECK-DWARF5
 #
 # This is based on the following reproducer:
diff --git a/llvm/test/DebugInfo/MIR/X86/DW_OP_entry_value.mir b/llvm/test/DebugInfo/MIR/X86/debug-entry-value-operation.mir
similarity index 91%
rename from llvm/test/DebugInfo/MIR/X86/DW_OP_entry_value.mir
rename to llvm/test/DebugInfo/MIR/X86/debug-entry-value-operation.mir
index f7f74b628d166..a368cc5c9420b 100644
--- a/llvm/test/DebugInfo/MIR/X86/DW_OP_entry_value.mir
+++ b/llvm/test/DebugInfo/MIR/X86/debug-entry-value-operation.mir
@@ -1,16 +1,22 @@
 # RUN: llc -start-before=livedebugvalues -mtriple=x86_64-apple-darwin -o %t %s -filetype=obj
 # RUN: llvm-dwarfdump %t | FileCheck %s
-#
-# int global;
-# int foo(int p, int q, int r) {
-#   global = p + 1;
-#   asm __volatile("" : : : "edi", "esi", "edx");
-#   return 123;
-# }
+
+# RUN: llc -start-before=livedebugvalues -debugger-tune=sce -mtriple=x86_64-sce-ps4 -o %t1 %s -filetype=obj
+# RUN: llvm-dwarfdump %t1 | FileCheck %s -check-prefix=SCE
+
+## Based on:
+## int global;
+## int foo(int p, int q, int r) {
+##   global = p + 1;
+##   asm __volatile("" : : : "edi", "esi", "edx");
+##   return 123;
+## }
 
 # CHECK: DW_TAG_formal_parameter
 # CHECK:   DW_OP_entry_value
 
+# SCE-NOT: DW_OP_{{.*}}entry_value
+
 --- |
   ; ModuleID = 'multiple-param-dbg-value-entry.ll'
   source_filename = "multiple-param-dbg-value-entry.c"
diff --git a/llvm/test/DebugInfo/Mips/eh_frame.ll b/llvm/test/DebugInfo/Mips/eh_frame.ll
index a71ae61f3e632..9dcfa9e413b84 100644
--- a/llvm/test/DebugInfo/Mips/eh_frame.ll
+++ b/llvm/test/DebugInfo/Mips/eh_frame.ll
@@ -1,9 +1,9 @@
 ; RUN: llc -mtriple mips-unknown-linux-gnu -mattr=+micromips \
 ; RUN:     -relocation-model=static -O3 -filetype=obj -o - %s | \
-; RUN:     llvm-readelf -r | FileCheck %s --check-prefixes=STATIC
+; RUN:     llvm-readelf -r - | FileCheck %s --check-prefixes=STATIC
 ; RUN: llc -mtriple mips-unknown-linux-gnu -mattr=+micromips \
 ; RUN:     -relocation-model=pic -O3 -filetype=obj -o - %s | \
-; RUN:     llvm-readelf -r | FileCheck %s --check-prefixes=PIC
+; RUN:     llvm-readelf -r - | FileCheck %s --check-prefixes=PIC
 ; RUN: llc -mtriple mips-unknown-linux-gnu -mattr=+micromips \
 ; RUN:     -relocation-model=static -O3 -filetype=obj -o - %s | \
 ; RUN:     llvm-objdump -s -j .gcc_except_table - | FileCheck %s --check-prefix=EXCEPT-TABLE-STATIC
diff --git a/llvm/test/DebugInfo/NVPTX/packed_bitfields.ll b/llvm/test/DebugInfo/NVPTX/packed_bitfields.ll
index 36433064fa924..63516f5972720 100644
--- a/llvm/test/DebugInfo/NVPTX/packed_bitfields.ll
+++ b/llvm/test/DebugInfo/NVPTX/packed_bitfields.ll
@@ -14,7 +14,8 @@
 ; CHECK:      .b8 3    // DW_AT_decl_line
 ; CHECK-NEXT: .b8 1    // DW_AT_byte_size
 ; CHECK-NEXT: .b8 6    // DW_AT_bit_size
-; CHECK-NEXT: .b64 -1  // DW_AT_bit_offset
+; Negative offset must be encoded as an unsigned integer.
+; CHECK-NEXT: .b64 0xffffffffffffffff // DW_AT_bit_offset
 ; CHECK-NEXT: .b8 2    // DW_AT_data_member_location
 
 %struct.anon = type { i16 }
diff --git a/llvm/test/DebugInfo/RISCV/relax-debug-line.ll b/llvm/test/DebugInfo/RISCV/relax-debug-line.ll
index 814b253fadffe..fa46d5d31ec5c 100644
--- a/llvm/test/DebugInfo/RISCV/relax-debug-line.ll
+++ b/llvm/test/DebugInfo/RISCV/relax-debug-line.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -filetype=obj -mtriple=riscv32 -mattr=+relax %s -o - \
-; RUN:     | llvm-readobj -r | FileCheck -check-prefix=RELAX %s
+; RUN:     | llvm-readobj -r - | FileCheck -check-prefix=RELAX %s
 ;
 ; RELAX: .rela.debug_line {
 ; RELAX: R_RISCV_ADD16
diff --git a/llvm/test/DebugInfo/WebAssembly/dbg-value-live-interval.ll b/llvm/test/DebugInfo/WebAssembly/dbg-value-live-interval.ll
index c216d0d364153..eb005dfe1042d 100644
--- a/llvm/test/DebugInfo/WebAssembly/dbg-value-live-interval.ll
+++ b/llvm/test/DebugInfo/WebAssembly/dbg-value-live-interval.ll
@@ -1,10 +1,10 @@
-; RUN: llc < %s -print-machineinstrs 2>&1 | FileCheck %s
+; FIXME: use -stop-after when MIR serialization output includes needed debug info.
+; RUN: llc < %s -print-after=wasm-optimize-live-intervals 2>&1 | FileCheck %s
 
-; CHECK: After WebAssembly Optimize Live Intervals:
+; CHECK: {{.*}}After WebAssembly Optimize Live Intervals
 ; CHECK: bb.3.for.body.for.body_crit_edge:
 ; CHECK: [[REG:%[0-9]+]]:i32 = nsw ADD_I32 {{.*}} fib.c:7:7
 ; CHECK: DBG_VALUE [[REG]]:i32, $noreg, !"a", {{.*}} fib.c:5:13
-; CHECK: After WebAssembly Memory Intrinsic Results:
 
 ; ModuleID = 'fib.bc'
 source_filename = "fib.c"
diff --git a/llvm/test/DebugInfo/WebAssembly/dbg-value-move-2.ll b/llvm/test/DebugInfo/WebAssembly/dbg-value-move-2.ll
index 90e8b66609bf5..a8d60b591d6d8 100644
--- a/llvm/test/DebugInfo/WebAssembly/dbg-value-move-2.ll
+++ b/llvm/test/DebugInfo/WebAssembly/dbg-value-move-2.ll
@@ -1,10 +1,10 @@
-; RUN: llc < %s -print-machineinstrs 2>&1 | FileCheck %s
+; FIXME: use -stop-after when MIR serialization output includes needed debug info.
+; RUN: llc < %s -print-after=wasm-reg-stackify 2>&1 | FileCheck %s
 
-; CHECK: After WebAssembly Register Stackify:
+; CHECK: {{.*}}After WebAssembly Register Stackify
 ; CHECK: bb.2.for.body:
 ; CHECK: [[REG:%[0-9]+]]:i32 = TEE_I32 {{.*}} fib2.c:6:7
 ; CHECK-NEXT: DBG_VALUE [[REG]]:i32, $noreg, !"a", {{.*}} fib2.c:2:13
-; CHECK: After WebAssembly Register Coloring:
 
 ; ModuleID = 'fib2.bc'
 ; The test generated via: clang --target=wasm32-unknown-unknown-wasm fib2.c -g -O2
diff --git a/llvm/test/DebugInfo/WebAssembly/dbg-value-move.ll b/llvm/test/DebugInfo/WebAssembly/dbg-value-move.ll
index 7ff79c560b90f..28bf0b727e27a 100644
--- a/llvm/test/DebugInfo/WebAssembly/dbg-value-move.ll
+++ b/llvm/test/DebugInfo/WebAssembly/dbg-value-move.ll
@@ -1,10 +1,10 @@
-; RUN: llc < %s -print-machineinstrs 2>&1 | FileCheck %s
+; FIXME: use -stop-after when MIR serialization output includes needed debug info.
+; RUN: llc < %s -print-after=wasm-reg-stackify 2>&1 | FileCheck %s
 
-; CHECK: After WebAssembly Register Stackify:
+; CHECK: {{.*}}After WebAssembly Register Stackify
 ; CHECK: bb.3.for.body.for.body_crit_edge:
 ; CHECK: [[REG:%[0-9]+]]:i32 = nsw ADD_I32 {{.*}} fib.c:7:7
 ; CHECK-NEXT: DBG_VALUE [[REG]]:i32, $noreg, !"a", {{.*}} fib.c:5:13
-; CHECK: After WebAssembly Register Coloring:
 
 ; ModuleID = 'fib.bc'
 ; The test generated via: clang --target=wasm32-unknown-unknown-wasm fib.c -g -O2
diff --git a/llvm/test/DebugInfo/X86/debug-macro-dwo.ll b/llvm/test/DebugInfo/X86/debug-macro-dwo.ll
index 1391795c9cd6b..23e889f5efced 100644
--- a/llvm/test/DebugInfo/X86/debug-macro-dwo.ll
+++ b/llvm/test/DebugInfo/X86/debug-macro-dwo.ll
@@ -2,14 +2,14 @@
 ; -gdwarf-5 -gsplit-dwarf -fdebug-macro is specified.
 
 ; RUN: %llc_dwarf -dwarf-version=5 -O0 -filetype=obj \
-; RUN: -split-dwarf-file=foo.dwo < %s | llvm-dwarfdump -v - | FileCheck %s
+; RUN: -split-dwarf-file=foo.dwo < %s | llvm-dwarfdump -debug-macro -debug-info -debug-line -v - | FileCheck %s
 
 ; CHECK-LABEL:  .debug_info contents:
 ; CHECK: DW_AT_macros [DW_FORM_sec_offset] (0x00000000)
 
 ; CHECK-LABEL:  .debug_macro.dwo contents:
 ; CHECK-NEXT: 0x00000000:
-; CHECK-NEXT: macro header: version = 0x0005, flags = 0x02, format = DWARF32
+; CHECK-NEXT: macro header: version = 0x0005, flags = 0x02, format = DWARF32, debug_line_offset = 0x00000000
 ; CHECK-NEXT: DW_MACRO_start_file - lineno: 0 filenum: 0
 ; CHECK-NEXT:   DW_MACRO_start_file - lineno: 1 filenum: 1
 ; CHECK-NEXT:     DW_MACRO_define_strx - lineno: 1 macro: FOO 5
@@ -20,6 +20,16 @@
 ; CHECK-NEXT:   DW_MACRO_undef_strx - lineno: 14 macro: YEA
 ; CHECK-NEXT: DW_MACRO_end_file
 
+; CHECK-LABEL: .debug_line.dwo contents:
+; CHECK: file_names[  0]:
+; CHECK:            name: "test.c"
+; CHECK: file_names[  1]:
+; CHECK:            name: "foo.h"
+; CHECK: file_names[  2]:
+; CHECK:            name: "bar.h"
+
+
+
 ; ModuleID = 'test.c'
 source_filename = "test.c"
 target datalayout = "e-m:e-p200:32:32-p201:32:32-p202:64:64-i64:64-f80:128-n8:16:32:64-S128"
@@ -29,7 +39,7 @@ target triple = "x86_64-unknown-linux-gnu"
 !llvm.module.flags = !{!14, !15, !16}
 !llvm.ident = !{!17}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0", isOptimized: false, runtimeVersion: 0, splitDebugFilename: "test.dwo", emissionKind: FullDebug, enums: !2, macros: !3, splitDebugInlining: false, nameTableKind: None)
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0", isOptimized: false, runtimeVersion: 0, splitDebugFilename: "test.dwo", emissionKind: FullDebug, enums: !2, macros: !3, globals: !18, splitDebugInlining: false, nameTableKind: None)
 !1 = !DIFile(filename: "test.c", directory: "/home/", checksumkind: CSK_MD5, checksum: "ef6a7032e0c7ceeef614583f2c00dc80")
 !2 = !{}
 !3 = !{!4}
@@ -47,3 +57,8 @@ target triple = "x86_64-unknown-linux-gnu"
 !15 = !{i32 2, !"Debug Info Version", i32 3}
 !16 = !{i32 1, !"wchar_size", i32 4}
 !17 = !{!"clang version 11.0.0"}
+!18 = !{!19}
+!19 = !DIGlobalVariableExpression(var: !20, expr: !DIExpression())
+!20 = distinct !DIGlobalVariable(name: "i", scope: !0, file: !21, line: 1, type: !22, isLocal: true, isDefinition: true)
+!21 = !DIFile(filename: "./not_used_by_macro.h", directory: "/home/", checksumkind: CSK_MD5, checksum: "cf4b34c263eaaa1d7085c18243b8d101")
+!22 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
diff --git a/llvm/test/DebugInfo/X86/dwarfdump-allocatedExp.ll b/llvm/test/DebugInfo/X86/dwarfdump-allocatedExp.ll
new file mode 100644
index 0000000000000..09e4a51283640
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/dwarfdump-allocatedExp.ll
@@ -0,0 +1,56 @@
+; RUN: llc %s -O2 -filetype=obj -o %t.o
+; RUN: llvm-dwarfdump  %t.o | FileCheck %s
+
+; Test whether DW_AT_data_location is generated.
+; CHECK-LABEL:  DW_TAG_array_type
+; CHECK:        DW_AT_allocated (DW_OP_push_object_address, DW_OP_deref)
+; CHECK-NOT: DW_TAG
+; CHECK:       DW_TAG_subrange_type
+
+; Test case is hand written with the help of below testcase
+;------------------------------
+;program main
+;integer, allocatable :: arr(:)
+;allocate(arr(2:20))
+;arr(2)=99
+;print *, arr
+;end program main
+;------------------------------
+
+; ModuleID = 'allocated.ll'
+source_filename = "allocated.ll"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @MAIN_() !dbg !5 {
+L.entry:
+  %.Z0640_333 = alloca i32*, align 8
+  %"arr$sd1_349" = alloca [16 x i64], align 8
+  call void @llvm.dbg.declare(metadata [16 x i64]* %"arr$sd1_349", metadata !8, metadata !DIExpression()), !dbg !13
+  call void @llvm.dbg.declare(metadata i32** %.Z0640_333, metadata !14, metadata !DIExpression()), !dbg !13
+  ret void, !dbg !16
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+!llvm.module.flags = !{!0, !1}
+!llvm.dbg.cu = !{!2}
+
+!0 = !{i32 2, !"Dwarf Version", i32 4}
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+!2 = distinct !DICompileUnit(language: DW_LANG_Fortran90, file: !3, producer: " F90 Flang - 1.5 2017-05-01", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, retainedTypes: !4, globals: !4, imports: !4)
+!3 = !DIFile(filename: "allocated.f90", directory: "/dir")
+!4 = !{}
+!5 = distinct !DISubprogram(name: "main", scope: !2, file: !3, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagMainSubprogram, unit: !2)
+!6 = !DISubroutineType(cc: DW_CC_program, types: !7)
+!7 = !{null}
+!8 = !DILocalVariable(name: "arr", scope: !5, file: !3, type: !9)
+!9 = !DICompositeType(tag: DW_TAG_array_type, baseType: !10, size: 32, align: 32, elements: !11, dataLocation: !DIExpression(DW_OP_push_object_address, DW_OP_deref), allocated: !DIExpression(DW_OP_push_object_address, DW_OP_deref))
+!10 = !DIBasicType(name: "integer", size: 32, align: 32, encoding: DW_ATE_signed)
+!11 = !{!12}
+!12 = !DISubrange(count: 19, lowerBound: 2)
+!13 = !DILocation(line: 0, scope: !5)
+!14 = distinct !DILocalVariable(scope: !5, file: !3, type: !15, flags: DIFlagArtificial)
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !10, size: 32, align: 32)
+!16 = !DILocation(line: 6, column: 1, scope: !5)
diff --git a/llvm/test/DebugInfo/X86/dwarfdump-allocatedVar.ll b/llvm/test/DebugInfo/X86/dwarfdump-allocatedVar.ll
new file mode 100644
index 0000000000000..5636151141a3a
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/dwarfdump-allocatedVar.ll
@@ -0,0 +1,68 @@
+;; This test checks whether DW_AT_data_location attribute
+;; accepts DIVariable
+
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu %s -filetype=obj -o %t.o
+; RUN: llvm-dwarfdump  %t.o | FileCheck %s
+
+;; Test whether DW_AT_data_location is generated.
+; CHECK: [[ALCDIE:0x.+]]:       DW_TAG_variable
+; CHECK:                          DW_AT_type    ({{0x[0-9]+}} "logical")
+; CHECK: [[LOCDIE:0x.+]]:       DW_TAG_variable
+; CHECK:                          DW_AT_type    ({{0x[0-9]+}} "integer*")
+; CHECK:                          DW_AT_artificial    (true)
+; CHECK:  DW_TAG_variable
+; CHECK:    DW_AT_name  ("arr")
+; CHECK:  DW_TAG_array_type
+; CHECK-NEXT:    DW_AT_data_location     ([[LOCDIE]])
+; CHECK-NEXT:    DW_AT_allocated         ([[ALCDIE]])
+
+;; Test case is hand written with the help of below testcase
+;;------------------------------
+;;program main
+;;integer, allocatable :: arr(:)
+;;allocate(arr(2:20))
+;;arr(2)=99
+;;print *, arr
+;;end program main
+;;------------------------------
+
+; ModuleID = 'allocated.ll'
+source_filename = "allocated.ll"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @MAIN_() !dbg !5 {
+L.entry:
+  %.Z0640_333 = alloca i32*, align 8
+  %"arr$sd1_349" = alloca [16 x i64], align 8
+  call void @llvm.dbg.declare(metadata [16 x i64]* %"arr$sd1_349", metadata !8, metadata !DIExpression()), !dbg !17
+  call void @llvm.dbg.declare(metadata i32** %.Z0640_333, metadata !13, metadata !DIExpression()), !dbg !17
+  call void @llvm.dbg.declare(metadata i32** %.Z0640_333, metadata !15, metadata !DIExpression()), !dbg !17
+  ret void, !dbg !18
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+!llvm.module.flags = !{!0, !1}
+!llvm.dbg.cu = !{!2}
+
+!0 = !{i32 2, !"Dwarf Version", i32 4}
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+!2 = distinct !DICompileUnit(language: DW_LANG_Fortran90, file: !3, producer: " F90 Flang - 1.5 2017-05-01", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, retainedTypes: !4, globals: !4, imports: !4)
+!3 = !DIFile(filename: "allocated.f90", directory: "/dir")
+!4 = !{}
+!5 = distinct !DISubprogram(name: "main", scope: !2, file: !3, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagMainSubprogram, unit: !2)
+!6 = !DISubroutineType(cc: DW_CC_program, types: !7)
+!7 = !{null}
+!8 = !DILocalVariable(name: "arr", scope: !5, file: !3, type: !9)
+!9 = !DICompositeType(tag: DW_TAG_array_type, baseType: !10, size: 32, align: 32, elements: !11, dataLocation: !13, allocated: !15)
+!10 = !DIBasicType(name: "integer", size: 32, align: 32, encoding: DW_ATE_signed)
+!11 = !{!12}
+!12 = !DISubrange(count: 19, lowerBound: 2)
+!13 = distinct !DILocalVariable(scope: !5, file: !3, type: !14, flags: DIFlagArtificial)
+!14 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !10, size: 32, align: 32)
+!15 = distinct !DILocalVariable(scope: !5, file: !3, type: !16, flags: DIFlagArtificial)
+!16 = !DIBasicType(name: "logical", size: 32, align: 32, encoding: DW_ATE_signed)
+!17 = !DILocation(line: 0, scope: !5)
+!18 = !DILocation(line: 6, column: 1, scope: !5)
diff --git a/llvm/test/DebugInfo/X86/dwarfdump-associatedExp.ll b/llvm/test/DebugInfo/X86/dwarfdump-associatedExp.ll
new file mode 100644
index 0000000000000..d99c563ddb7c2
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/dwarfdump-associatedExp.ll
@@ -0,0 +1,56 @@
+; RUN: llc %s -O2 -filetype=obj -o %t.o
+; RUN: llvm-dwarfdump  %t.o | FileCheck %s
+
+; Test whether DW_AT_data_location is generated.
+; CHECK-LABEL:  DW_TAG_array_type
+; CHECK-NOT: DW_TAG
+; CHECK:        DW_AT_associated (DW_OP_push_object_address, DW_OP_deref)
+; CHECK:       DW_TAG_subrange_type
+
+; Test case is hand written with the help of below testcase
+;------------------------------
+;program main
+;integer, pointer :: arr(:)
+;allocate(arr(2:20))
+;arr(2)=99
+;print *, arr
+;end program main
+;------------------------------
+
+; ModuleID = 'associated.ll'
+source_filename = "associated.ll"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @MAIN_() !dbg !5 {
+L.entry:
+  %.Z0640_333 = alloca i32*, align 8
+  %"arr$sd1_349" = alloca [16 x i64], align 8
+  call void @llvm.dbg.declare(metadata [16 x i64]* %"arr$sd1_349", metadata !8, metadata !DIExpression()), !dbg !13
+  call void @llvm.dbg.declare(metadata i32** %.Z0640_333, metadata !14, metadata !DIExpression()), !dbg !13
+  ret void, !dbg !16
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+!llvm.module.flags = !{!0, !1}
+!llvm.dbg.cu = !{!2}
+
+!0 = !{i32 2, !"Dwarf Version", i32 4}
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+!2 = distinct !DICompileUnit(language: DW_LANG_Fortran90, file: !3, producer: " F90 Flang - 1.5 2017-05-01", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, retainedTypes: !4, globals: !4, imports: !4)
+!3 = !DIFile(filename: "associated.f90", directory: "/dir")
+!4 = !{}
+!5 = distinct !DISubprogram(name: "main", scope: !2, file: !3, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagMainSubprogram, unit: !2)
+!6 = !DISubroutineType(cc: DW_CC_program, types: !7)
+!7 = !{null}
+!8 = !DILocalVariable(name: "arr", scope: !5, file: !3, type: !9)
+!9 = !DICompositeType(tag: DW_TAG_array_type, baseType: !10, size: 32, align: 32, elements: !11, dataLocation: !DIExpression(DW_OP_push_object_address, DW_OP_deref), associated: !DIExpression(DW_OP_push_object_address, DW_OP_deref))
+!10 = !DIBasicType(name: "integer", size: 32, align: 32, encoding: DW_ATE_signed)
+!11 = !{!12}
+!12 = !DISubrange(count: 19, lowerBound: 2)
+!13 = !DILocation(line: 0, scope: !5)
+!14 = distinct !DILocalVariable(scope: !5, file: !3, type: !15, flags: DIFlagArtificial)
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !10, size: 32, align: 32)
+!16 = !DILocation(line: 6, column: 1, scope: !5)
diff --git a/llvm/test/DebugInfo/X86/dwarfdump-associatedVar.ll b/llvm/test/DebugInfo/X86/dwarfdump-associatedVar.ll
new file mode 100644
index 0000000000000..eda785260c91a
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/dwarfdump-associatedVar.ll
@@ -0,0 +1,68 @@
+;; This test checks whether DW_AT_data_location attribute
+;; accepts DIVariable
+
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu %s -filetype=obj -o %t.o
+; RUN: llvm-dwarfdump  %t.o | FileCheck %s
+
+;; Test whether DW_AT_data_location is generated.
+; CHECK: [[ALCDIE:0x.+]]:       DW_TAG_variable
+; CHECK:                          DW_AT_type    ({{0x[0-9]+}} "logical")
+; CHECK: [[LOCDIE:0x.+]]:       DW_TAG_variable
+; CHECK:                          DW_AT_type    ({{0x[0-9]+}} "integer*")
+; CHECK:                          DW_AT_artificial    (true)
+; CHECK:  DW_TAG_variable
+; CHECK:    DW_AT_name  ("arr")
+; CHECK:  DW_TAG_array_type
+; CHECK-NEXT:    DW_AT_data_location     ([[LOCDIE]])
+; CHECK-NEXT:    DW_AT_associated        ([[ALCDIE]])
+
+;; Test case is hand written with the help of below testcase
+;;------------------------------
+;;program main
+;;integer, pointer :: arr(:)
+;;allocate(arr(2:20))
+;;arr(2)=99
+;;print *, arr
+;;end program main
+;;------------------------------
+
+; ModuleID = 'associated.ll'
+source_filename = "associated.ll"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @MAIN_() !dbg !5 {
+L.entry:
+  %.Z0640_333 = alloca i32*, align 8
+  %"arr$sd1_349" = alloca [16 x i64], align 8
+  call void @llvm.dbg.declare(metadata [16 x i64]* %"arr$sd1_349", metadata !8, metadata !DIExpression()), !dbg !17
+  call void @llvm.dbg.declare(metadata i32** %.Z0640_333, metadata !13, metadata !DIExpression()), !dbg !17
+  call void @llvm.dbg.declare(metadata i32** %.Z0640_333, metadata !15, metadata !DIExpression()), !dbg !17
+  ret void, !dbg !18
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+!llvm.module.flags = !{!0, !1}
+!llvm.dbg.cu = !{!2}
+
+!0 = !{i32 2, !"Dwarf Version", i32 4}
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+!2 = distinct !DICompileUnit(language: DW_LANG_Fortran90, file: !3, producer: " F90 Flang - 1.5 2017-05-01", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, retainedTypes: !4, globals: !4, imports: !4)
+!3 = !DIFile(filename: "associated.f90", directory: "/dir")
+!4 = !{}
+!5 = distinct !DISubprogram(name: "main", scope: !2, file: !3, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagMainSubprogram, unit: !2)
+!6 = !DISubroutineType(cc: DW_CC_program, types: !7)
+!7 = !{null}
+!8 = !DILocalVariable(name: "arr", scope: !5, file: !3, type: !9)
+!9 = !DICompositeType(tag: DW_TAG_array_type, baseType: !10, size: 32, align: 32, elements: !11, dataLocation: !13, associated: !15)
+!10 = !DIBasicType(name: "integer", size: 32, align: 32, encoding: DW_ATE_signed)
+!11 = !{!12}
+!12 = !DISubrange(count: 19, lowerBound: 2)
+!13 = distinct !DILocalVariable(scope: !5, file: !3, type: !14, flags: DIFlagArtificial)
+!14 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !10, size: 32, align: 32)
+!15 = distinct !DILocalVariable(scope: !5, file: !3, type: !16, flags: DIFlagArtificial)
+!16 = !DIBasicType(name: "logical", size: 32, align: 32, encoding: DW_ATE_signed)
+!17 = !DILocation(line: 0, scope: !5)
+!18 = !DILocation(line: 6, column: 1, scope: !5)
diff --git a/llvm/test/DebugInfo/X86/float_const_loclist.ll b/llvm/test/DebugInfo/X86/float_const_loclist.ll
index f9008209e5dfb..24ee16444836a 100644
--- a/llvm/test/DebugInfo/X86/float_const_loclist.ll
+++ b/llvm/test/DebugInfo/X86/float_const_loclist.ll
@@ -20,12 +20,10 @@
 ;
 ; CHECK: .debug_info contents:
 ; CHECK: DW_TAG_variable
-; CHECK-NEXT:  DW_AT_location {{.*}} (
-; CHECK-NEXT:    [0x[[START:.*]], 0x[[END:.*]]): DW_OP_constu 0xc8f5c28f5c28f800, DW_OP_piece 0x8, DW_OP_constu 0x4000, DW_OP_bit_piece 0x10 0x40)
 ; CHECK-NEXT:  DW_AT_name {{.*}}"ld"
 ; CHECK: DW_TAG_variable
 ; CHECK-NEXT:  DW_AT_location {{.*}} (
-; CHECK-NEXT:    [0x[[START]], 0x[[END]]): DW_OP_constu 0x4048f5c3)
+; CHECK-NEXT:    [0x{{.*}}, 0x{{.*}}): DW_OP_constu 0x4048f5c3)
 ; CHECK-NEXT:  DW_AT_name {{.*}}"f"
 
 source_filename = "test.c"
diff --git a/llvm/test/DebugInfo/X86/implicit_value-double.ll b/llvm/test/DebugInfo/X86/implicit_value-double.ll
new file mode 100644
index 0000000000000..3c14c7dfefce4
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/implicit_value-double.ll
@@ -0,0 +1,66 @@
+;; This test checks for emission of DW_OP_implicit_value operation
+;; for double type.
+
+; RUN: llc -debugger-tune=gdb -filetype=obj %s -o - | llvm-dwarfdump - | FileCheck %s
+
+; CHECK: .debug_info contents:
+; CHECK: DW_TAG_variable
+; CHECK-NEXT:  DW_AT_location        ({{.*}}
+; CHECK-NEXT:                     [{{.*}}): DW_OP_implicit_value 0x8 0x1f 0x85 0xeb 0x51 0xb8 0x1e 0x09 0x40)
+; CHECK-NEXT:  DW_AT_name    ("d")
+
+;; Generated from: clang -ggdb -O1
+;;int main() {
+;;        double d = 3.14;
+;;        printf("dummy\n");
+;;        d *= d;
+;;        return 0;
+;;}
+
+; ModuleID = 'implicit_value-double.c'
+source_filename = "implicit_value-double.c"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@str = private unnamed_addr constant [6 x i8] c"dummy\00", align 1
+
+; Function Attrs: nofree nounwind uwtable
+define dso_local i32 @main() local_unnamed_addr #0 !dbg !7 {
+entry:
+  call void @llvm.dbg.value(metadata double 3.140000e+00, metadata !12, metadata !DIExpression()), !dbg !14
+  %puts = call i32 @puts(i8* nonnull dereferenceable(1) getelementptr inbounds ([6 x i8], [6 x i8]* @str, i64 0, i64 0)), !dbg !15
+  call void @llvm.dbg.value(metadata double undef, metadata !12, metadata !DIExpression()), !dbg !14
+  ret i32 0, !dbg !16
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+; Function Attrs: nofree nounwind
+declare i32 @puts(i8* nocapture readonly) local_unnamed_addr #2
+
+attributes #0 = { nofree nounwind uwtable }
+attributes #1 = { nounwind readnone speculatable willreturn }
+attributes #2 = { nofree nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "implicit_value-double.c", directory: "/home/")
+!2 = !{}
+!3 = !{i32 7, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 11.0.0"}
+!7 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !{!12}
+!12 = !DILocalVariable(name: "d", scope: !7, file: !1, line: 2, type: !13)
+!13 = !DIBasicType(name: "double", size: 64, encoding: DW_ATE_float)
+!14 = !DILocation(line: 0, scope: !7)
+!15 = !DILocation(line: 3, column: 2, scope: !7)
+!16 = !DILocation(line: 5, column: 2, scope: !7)
diff --git a/llvm/test/DebugInfo/X86/implicit_value-float.ll b/llvm/test/DebugInfo/X86/implicit_value-float.ll
new file mode 100644
index 0000000000000..8c51b49481773
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/implicit_value-float.ll
@@ -0,0 +1,65 @@
+;; This test checks for emission of DW_OP_implicit_value operation
+;; for float type.
+
+; RUN: llc -debugger-tune=gdb -filetype=obj %s -o - | llvm-dwarfdump - | FileCheck %s
+
+; CHECK: .debug_info contents:
+; CHECK: DW_TAG_variable
+; CHECK-NEXT:  DW_AT_location        ({{.*}}
+; CHECK-NEXT:                     [{{.*}}): DW_OP_implicit_value 0x4 0xc3 0xf5 0x48 0x40)
+; CHECK-NEXT:  DW_AT_name    ("f")
+
+;; Generated from: clang -ggdb -O1
+;;int main() {
+;;        float f = 3.14f;
+;;        printf("dummy\n");
+;;        f *= f;
+;;        return 0;
+;;}
+; ModuleID = 'implicit_value-float.c'
+source_filename = "implicit_value-float.c"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@str = private unnamed_addr constant [6 x i8] c"dummy\00", align 1
+
+; Function Attrs: nofree nounwind uwtable
+define dso_local i32 @main() local_unnamed_addr #0 !dbg !7 {
+entry:
+  call void @llvm.dbg.value(metadata float 0x40091EB860000000, metadata !12, metadata !DIExpression()), !dbg !14
+  %puts = call i32 @puts(i8* nonnull dereferenceable(1) getelementptr inbounds ([6 x i8], [6 x i8]* @str, i64 0, i64 0)), !dbg !15
+  call void @llvm.dbg.value(metadata float undef, metadata !12, metadata !DIExpression()), !dbg !14
+  ret i32 0, !dbg !16
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+; Function Attrs: nofree nounwind
+declare i32 @puts(i8* nocapture readonly) local_unnamed_addr #2
+
+attributes #0 = { nofree nounwind uwtable }
+attributes #1 = { nounwind readnone speculatable willreturn }
+attributes #2 = { nofree nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "implicit_value-float.c", directory: "/home/")
+!2 = !{}
+!3 = !{i32 7, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 11.0.0"}
+!7 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !{!12}
+!12 = !DILocalVariable(name: "f", scope: !7, file: !1, line: 2, type: !13)
+!13 = !DIBasicType(name: "float", size: 32, encoding: DW_ATE_float)
+!14 = !DILocation(line: 0, scope: !7)
+!15 = !DILocation(line: 3, column: 2, scope: !7)
+!16 = !DILocation(line: 5, column: 2, scope: !7)
diff --git a/llvm/test/DebugInfo/X86/implicit_value-ld.ll b/llvm/test/DebugInfo/X86/implicit_value-ld.ll
new file mode 100644
index 0000000000000..5fdaf396c5347
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/implicit_value-ld.ll
@@ -0,0 +1,71 @@
+;; This test checks for emission of DW_OP_implicit_value operation
+;; for long double type.
+
+; RUN: llc -debugger-tune=gdb -filetype=obj %s -o - | llvm-dwarfdump - | FileCheck %s
+
+; CHECK: .debug_info contents:
+; CHECK: DW_TAG_variable
+; CHECK-NEXT:  DW_AT_location        ({{.*}}
+; CHECK-NEXT:                     [{{.*}}): DW_OP_implicit_value 0x10 0x00 0xf8 0x28 0x5c 0x8f 0xc2 0xf5 0xc8 0x00 0x40 0x00 0x00 0x00 0x00 0x00 0x00)
+; CHECK-NEXT:  DW_AT_name    ("ld")
+
+;; Generated from: clang -ggdb -O1
+;;int main() {
+;;        long double ld = 3.14;
+;;        printf("dummy\n");
+;;        ld *= ld;
+;;        return 0;
+;;}
+
+; ModuleID = 'implicit_value-ld.c'
+source_filename = "implicit_value-ld.c"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@str = private unnamed_addr constant [6 x i8] c"dummy\00", align 1
+
+; Function Attrs: nofree nounwind uwtable
+define dso_local i32 @main() local_unnamed_addr #0 !dbg !7 {
+entry:
+  call void @llvm.dbg.declare(metadata [6 x i8]* undef, metadata !12, metadata !DIExpression(DW_OP_LLVM_fragment, 80, 48)), !dbg !14
+  call void @llvm.dbg.value(metadata x86_fp80 0xK4000C8F5C28F5C28F800, metadata !12, metadata !DIExpression()), !dbg !15
+  %puts = call i32 @puts(i8* nonnull dereferenceable(1) getelementptr inbounds ([6 x i8], [6 x i8]* @str, i64 0, i64 0)), !dbg !16
+  call void @llvm.dbg.value(metadata x86_fp80 undef, metadata !12, metadata !DIExpression()), !dbg !15
+  ret i32 0, !dbg !17
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+; Function Attrs: nofree nounwind
+declare i32 @puts(i8* nocapture readonly) local_unnamed_addr #2
+
+attributes #0 = { nofree nounwind uwtable }
+attributes #1 = { nounwind readnone speculatable willreturn }
+attributes #2 = { nofree nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "implicit_value-ld.c", directory: "/home/")
+!2 = !{}
+!3 = !{i32 7, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 11.0.0"}
+!7 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !{!12}
+!12 = !DILocalVariable(name: "ld", scope: !7, file: !1, line: 2, type: !13)
+!13 = !DIBasicType(name: "long double", size: 128, encoding: DW_ATE_float)
+!14 = !DILocation(line: 2, column: 14, scope: !7)
+!15 = !DILocation(line: 0, scope: !7)
+!16 = !DILocation(line: 3, column: 2, scope: !7)
+!17 = !DILocation(line: 5, column: 2, scope: !7)
diff --git a/llvm/test/DebugInfo/X86/live-debug-variables.ll b/llvm/test/DebugInfo/X86/live-debug-variables.ll
index cf65b190947ff..af343227d5a7f 100644
--- a/llvm/test/DebugInfo/X86/live-debug-variables.ll
+++ b/llvm/test/DebugInfo/X86/live-debug-variables.ll
@@ -1,4 +1,5 @@
-; RUN: llc -mtriple=x86_64-linux-gnu -filetype=obj -o - %s | llvm-dwarfdump -debug-loc - | FileCheck %s
+; RUN: llc -mtriple=x86_64-linux-gnu -filetype=obj -o - %s | llvm-dwarfdump -name i4 - \
+; RUN:     | FileCheck %s
 
 ; The test inlines the function F four times, with each inlined variable for
 ; "i4" sharing the same virtual register. This means the live interval of the
@@ -22,12 +23,13 @@
 ;          F(a,b,c,d,e);
 ; }
 
-; CHECK:      .debug_loc contents:
-; CHECK-NEXT: 0x00000000:
-;   We currently emit an entry for the function prologue, too, which could be optimized away.
-; CHECK:              (0x0000000000000018, 0x0000000000000072): DW_OP_reg3 RBX
-;   We should only have one entry inside the function.
-; CHECK-NOT: :
+; Ignore the abstract entry.
+; CHECK: DW_TAG_formal_parameter
+; Check concrete entry has a single location.
+; CHECK:      DW_TAG_formal_parameter
+; CHECK-NEXT:   DW_AT_location (DW_OP_reg3 RBX)
+; CHECK-NEXT:   DW_AT_abstract_origin
+; CHECK-NOT:  DW_TAG_formal_parameter
 
 declare i32 @foobar(i32, i32, i32, i32, i32)
 
diff --git a/llvm/test/DebugInfo/X86/trim-var-locs.mir b/llvm/test/DebugInfo/X86/trim-var-locs.mir
new file mode 100644
index 0000000000000..04ab56d302ae6
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/trim-var-locs.mir
@@ -0,0 +1,121 @@
+# RUN: llc %s --start-after=livedebugvalues -filetype=obj -o - \
+# RUN:     | llvm-dwarfdump - -name local* -regex \
+# RUN:     | FileCheck %s
+#
+# Test that the -trim-var-locs option (enabled by default) works correctly.
+# Test directives and comments inline.
+
+--- |
+  target triple = "x86_64-unknown-linux-gnu"
+  define dso_local i32 @fun() local_unnamed_addr !dbg !7 {
+  entry:
+    ret i32 0
+  }
+
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!3, !4, !5}
+  !llvm.ident = !{!6}
+
+  !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None)
+  !1 = !DIFile(filename: "example.c", directory: "/")
+  !2 = !{}
+  !3 = !{i32 7, !"Dwarf Version", i32 4}
+  !4 = !{i32 2, !"Debug Info Version", i32 3}
+  !5 = !{i32 1, !"wchar_size", i32 4}
+  !6 = !{!"clang version 11.0.0"}
+  !8 = !DISubroutineType(types: !9)
+  !9 = !{!10}
+  !10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+  !11 = !{!12, !13, !25}
+  !22 = !DISubroutineType(types: !23)
+  !23 = !{!10, !10}
+  ; --- Important metadata ---
+  !7 = distinct !DISubprogram(name: "fun", scope: !1, file: !1, line: 2, type: !8, scopeLine: 2, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11)
+  !24 = distinct !DILexicalBlock(scope: !7, file: !1, line: 9, column: 3)
+  !14 = distinct !DILexicalBlock(scope: !7, file: !1, line: 4, column: 3)
+  !12 = !DILocalVariable(name: "locala", scope: !7, file: !1, line: 1, type: !10)
+  !13 = !DILocalVariable(name: "localb", scope: !14, file: !1, line: 2, type: !10)
+  !25 = !DILocalVariable(name: "localc", scope: !24, file: !1, line: 3, type: !10)
+  !15 = !DILocation(line: 1, column: 0, scope: !7)
+  !18 = !DILocation(line: 2, column: 1, scope: !14)
+  !26 = !DILocation(line: 3, column: 1, scope: !24)
+...
+---
+name:            fun
+body:             |
+  bb.0.entry:
+    ; This is the scope and variable structure:
+    ; int fun() {       // scope fun !7
+    ;   int locala;     // scope fun !7,        var locala !12, debug-location !15
+    ;   { int localb; } // scope fun:block !14, var localb !13, debug-location !18
+    ;   { int localc; } // scope fun:block !24, var localc !25, debug-location !26
+    ; }
+    ;
+    ; (1) Check that a variable location range found in implied scope fun !7 is
+    ;     not trimmed.
+    ;
+    ; CHECK:      DW_TAG_variable
+    ; CHECK-NEXT:   DW_AT_location
+    ; CHECK-NEXT:     DW_OP_reg0 RAX
+    ; CHECK-NEXT:   DW_AT_name ("locala")
+    ;
+    ; scope fun !7 is implied as we're in function fun and haven't seen a debug-location
+    $eax = MOV32ri 0
+    ; locala range 1 start in implicit scope fun !7
+    DBG_VALUE $eax, $noreg, !12, !DIExpression(), debug-location !15
+    $edi = MOV32ri 1
+    ; locala range 1 clobber in implicit scope fun !7
+    $eax = MOV32ri 2
+    ; scope fun !7 explicit start
+    $edi = MOV32ri 3, debug-location !15
+
+    ; (2) Check that a variable location range found outside lexical block is
+    ;     trimmed. See check directives for (3).
+    ;
+    ; localb range 1 start in scope fun !7 (outside block !14).
+    DBG_VALUE $eax, $noreg, !13, !DIExpression(), debug-location !18
+    ; localb range 1 clobber in scope fun !7
+    $edi = MOV32ri 4, debug-location !15
+
+    ; (3) Check that a variable location range which overlaps the entire lexical
+    ;     block is not trimmed.
+    ;
+    ; CHECK:      DW_TAG_variable
+    ; CHECK-NEXT:   DW_AT_location
+    ; CHECK-NEXT:     DW_OP_reg5 RDI
+    ; CHECK-NEXT:   DW_AT_name ("localb")
+    ;
+    ; localb range 2 clobber in scope fun !7 (outside block !14)
+    DBG_VALUE $edi, $noreg, !13, !DIExpression(), debug-location !18
+    ; scope block !14 start (and only instruction)
+    $edi = MOV32ri 5, debug-location !18
+
+    ; (4) Check that a variable location range in scope fun !7 (outside block
+    ;     !14) is trimmed. See check directives for (3).
+    ;
+    ; localb range 3 starts after scope !14 (prev instr is last in scope)
+    DBG_VALUE $rax, $noreg, !13, !DIExpression(), debug-location !18
+    ; scope block !14 end
+    $edi = MOV32ri 6, debug-location !15
+
+    ; (5) Check that a variable location range found between disjoint scope
+    ;     ranges is trimmed.
+    ;
+    ; CHECK:      DW_TAG_variable
+    ; CHECK-NOT:    DW_AT_location
+    ; CHECK-NEXT:   DW_AT_name ("localc")
+    ;
+    ; scope fun !7
+    $edi = MOV32ri 6, debug-location !15
+    ; scope block !24 start and end range 1
+    $edi = MOV32ri 7, debug-location !26
+    ; localc range 1 start in scope !7
+    DBG_VALUE $edi, $noreg, !25, !DIExpression(), debug-location !18
+    ; localc range 1 clobber in scope !7
+    $edi = MOV32ri 8, debug-location !15
+    ; scope block !24 start and end range 2
+    $edi = MOV32ri 9, debug-location !26
+
+    ; scope fun !7
+    RETQ debug-location !15
+...
diff --git a/llvm/test/DebugInfo/X86/xray-split-dwarf-interaction.ll b/llvm/test/DebugInfo/X86/xray-split-dwarf-interaction.ll
index 14a44143a419a..c9202d8eeb74b 100644
--- a/llvm/test/DebugInfo/X86/xray-split-dwarf-interaction.ll
+++ b/llvm/test/DebugInfo/X86/xray-split-dwarf-interaction.ll
@@ -7,7 +7,7 @@
 ; RUN:     -function-sections -data-sections \
 ; RUN:     -relocation-model=pic -filetype=obj \
 ; RUN:     -generate-type-units -o - %s | \
-; RUN:     llvm-readelf --sections | \
+; RUN:     llvm-readelf --sections - | \
 ; RUN:     FileCheck %s --check-prefix=CHECK-ELF
 ; Created from `clang++ -fxray-instrument -gsplit-dwarf -fdebug-types-section
 ; -ffunction-sections -fdata-sections -emit-llvm -S input.cc`:
diff --git a/llvm/test/ExecutionEngine/JITLink/X86/ELF_x86-64_relocations.s b/llvm/test/ExecutionEngine/JITLink/X86/ELF_x86-64_relocations.s
index 0eee9a449cea9..ca9c926b32de1 100644
--- a/llvm/test/ExecutionEngine/JITLink/X86/ELF_x86-64_relocations.s
+++ b/llvm/test/ExecutionEngine/JITLink/X86/ELF_x86-64_relocations.s
@@ -1,5 +1,5 @@
 # RUN: rm -rf %t && mkdir -p %t
-# RUN: llvm-mc -triple=x86_64-unknown-linux -filetype=obj -o %t/elf_reloc.o %s
+# RUN: llvm-mc -triple=x86_64-unknown-linux -position-independent -filetype=obj -o %t/elf_reloc.o %s
 # RUN: llvm-jitlink -noexec -check %s %t/elf_reloc.o
 #
 # Test standard ELF relocations.
@@ -20,15 +20,27 @@ main:
 # jitlink-check: decode_operand(test_pcrel32, 4) = named_data - next_pc(test_pcrel32)
         .globl  test_pcrel32
         .p2align       4, 0x90
-        .type  test_pcrel32,@function
+        .type   test_pcrel32,@function
 test_pcrel32:
         movl    named_data(%rip), %eax
-.Ltest_pcrel32_end:
-        .size   test_pcrel32, .Ltest_pcrel32_end-test_pcrel32
+.Lend_test_pcrel32:
+         .size   test_pcrel32, .Lend_test_pcrel32-test_pcrel32
+
+# Test GOTPCREL handling. We want to check both the offset to the GOT entry and its
+# contents.
+# jitlink-check: decode_operand(test_gotpcrel, 4) = got_addr(elf_reloc.o, named_data) - next_pc(test_gotpcrel)
+# jitlink-check: *{8}(got_addr(elf_reloc.o, named_data)) = named_data
+
+        .globl test_gotpcrel
+        .p2align      4, 0x90
+        .type   test_gotpcrel,@function
+test_gotpcrel:
+	movl    named_data@GOTPCREL(%rip), %eax
+.Lend_test_gotpcrel:
+        .size   test_gotpcrel, .Lend_test_gotpcrel-test_gotpcrel
 
         .type   named_data,@object
         .data
-	.globl named_data
         .p2align        2
 named_data:
         .long   42
@@ -36,4 +48,4 @@ named_data:
 
         .ident  "clang version 10.0.0-4ubuntu1 "
         .section        ".note.GNU-stack","",@progbits
-        .addrsig
+        .addrsig
\ No newline at end of file
diff --git a/llvm/test/ExecutionEngine/RuntimeDyld/SystemZ/cfi-relo-pc64.s b/llvm/test/ExecutionEngine/RuntimeDyld/SystemZ/cfi-relo-pc64.s
index 6f4553ff88592..166d46edbd9ed 100644
--- a/llvm/test/ExecutionEngine/RuntimeDyld/SystemZ/cfi-relo-pc64.s
+++ b/llvm/test/ExecutionEngine/RuntimeDyld/SystemZ/cfi-relo-pc64.s
@@ -1,5 +1,5 @@
 // Test that R_390_PC32 and R_390_PC64 relocations can be generated.
-// RUN: llvm-mc -triple s390x-linux-gnu -filetype=obj %s -o - | llvm-readobj -S --sr --sd | FileCheck %s
+// RUN: llvm-mc -triple s390x-linux-gnu -filetype=obj %s -o - | llvm-readobj -S --sr --sd - | FileCheck %s
 
 // Test that RuntimeDyld can fix up such relocations.
 // RUN: rm -rf %t && mkdir -p %t
diff --git a/llvm/test/Feature/optnone-opt.ll b/llvm/test/Feature/optnone-opt.ll
index 27418063d55b4..3af0ce8d0fd05 100644
--- a/llvm/test/Feature/optnone-opt.ll
+++ b/llvm/test/Feature/optnone-opt.ll
@@ -1,9 +1,15 @@
-; RUN: opt     -S -debug %s 2>&1 | FileCheck %s --check-prefix=OPT-O0
-; RUN: opt -O1 -S -debug %s 2>&1 | FileCheck %s --check-prefix=OPT-O1
-; RUN: opt -O2 -S -debug %s 2>&1 | FileCheck %s --check-prefix=OPT-O1 --check-prefix=OPT-O2O3
-; RUN: opt -O3 -S -debug %s 2>&1 | FileCheck %s --check-prefix=OPT-O1 --check-prefix=OPT-O2O3
-; RUN: opt -dce -die -gvn-hoist -loweratomic -S -debug %s 2>&1 | FileCheck %s --check-prefix=OPT-MORE
-; RUN: opt -indvars -licm -loop-deletion -loop-extract -loop-idiom -loop-instsimplify -loop-reduce -loop-reroll -loop-rotate -loop-unroll -loop-unswitch -S -debug %s 2>&1 | FileCheck %s --check-prefix=OPT-LOOP
+; RUN: opt     -S -debug -enable-new-pm=0 %s 2>&1 | FileCheck %s --check-prefix=O0
+; RUN: opt -O1 -S -debug -enable-new-pm=0 %s 2>&1 | FileCheck %s --check-prefix=O1
+; RUN: opt -O2 -S -debug -enable-new-pm=0 %s 2>&1 | FileCheck %s --check-prefix=O1 --check-prefix=O2O3
+; RUN: opt -O3 -S -debug -enable-new-pm=0 %s 2>&1 | FileCheck %s --check-prefix=O1 --check-prefix=O2O3
+; RUN: opt -dce -die -gvn-hoist -loweratomic -S -debug -enable-new-pm=0 %s 2>&1 | FileCheck %s --check-prefix=MORE
+; RUN: opt -indvars -licm -loop-deletion -loop-extract -loop-idiom -loop-instsimplify -loop-reduce -loop-reroll -loop-rotate -loop-unroll -loop-unswitch -enable-new-pm=0 -S -debug %s 2>&1 | FileCheck %s --check-prefix=LOOP
+; RUN: opt -enable-npm-optnone     -S -debug-pass-manager -enable-new-pm %s 2>&1 | FileCheck %s --check-prefix=NPM-O0
+; RUN: opt -enable-npm-optnone -O1 -S -debug-pass-manager -enable-new-pm %s 2>&1 | FileCheck %s --check-prefix=NPM-O1
+; RUN: opt -enable-npm-optnone -O2 -S -debug-pass-manager -enable-new-pm %s 2>&1 | FileCheck %s --check-prefix=NPM-O1 --check-prefix=NPM-O2O3
+; RUN: opt -enable-npm-optnone -O3 -S -debug-pass-manager -enable-new-pm %s 2>&1 | FileCheck %s --check-prefix=NPM-O1 --check-prefix=NPM-O2O3
+; RUN: opt -enable-npm-optnone -dce -gvn-hoist -loweratomic -S -debug-pass-manager -enable-new-pm %s 2>&1 | FileCheck %s --check-prefix=NPM-MORE
+; RUN: opt -enable-npm-optnone -indvars -licm -loop-deletion -loop-idiom -loop-instsimplify -loop-reduce -loop-unroll -simple-loop-unswitch -S -debug-pass-manager -enable-new-pm %s 2>&1 | FileCheck %s --check-prefix=NPM-LOOP
 
 ; REQUIRES: asserts
 
@@ -34,32 +40,53 @@ while.end:                                        ; preds = %while.cond
 attributes #0 = { optnone noinline }
 
 ; Nothing that runs at -O0 gets skipped.
-; OPT-O0-NOT: Skipping pass
+; O0-NOT: Skipping pass
+; NPM-O0-NOT: Skipping pass
 
 ; IR passes run at -O1 and higher.
-; OPT-O1-DAG: Skipping pass 'Aggressive Dead Code Elimination'
-; OPT-O1-DAG: Skipping pass 'Combine redundant instructions'
-; OPT-O1-DAG: Skipping pass 'Early CSE'
-; OPT-O1-DAG: Skipping pass 'Reassociate expressions'
-; OPT-O1-DAG: Skipping pass 'Simplify the CFG'
-; OPT-O1-DAG: Skipping pass 'Sparse Conditional Constant Propagation'
+; O1-DAG: Skipping pass 'Aggressive Dead Code Elimination'
+; O1-DAG: Skipping pass 'Combine redundant instructions'
+; O1-DAG: Skipping pass 'Early CSE'
+; O1-DAG: Skipping pass 'Reassociate expressions'
+; O1-DAG: Skipping pass 'Simplify the CFG'
+; O1-DAG: Skipping pass 'Sparse Conditional Constant Propagation'
+; NPM-O1-DAG: Skipping pass: SimplifyCFGPass
+; NPM-O1-DAG: Skipping pass: SROA
+; NPM-O1-DAG: Skipping pass: EarlyCSEPass
+; NPM-O1-DAG: Skipping pass: LowerExpectIntrinsicPass
+; NPM-O1-DAG: Skipping pass: PromotePass
+; NPM-O1-DAG: Skipping pass: InstCombinePass
 
 ; Additional IR passes run at -O2 and higher.
-; OPT-O2O3-DAG: Skipping pass 'Global Value Numbering'
-; OPT-O2O3-DAG: Skipping pass 'SLP Vectorizer'
+; O2O3-DAG: Skipping pass 'Global Value Numbering'
+; O2O3-DAG: Skipping pass 'SLP Vectorizer'
+; NPM-O2O3-DAG: Skipping pass: GVN
+; NPM-O2O3-DAG: Skipping pass: SLPVectorizerPass
 
 ; Additional IR passes that opt doesn't turn on by default.
-; OPT-MORE-DAG: Skipping pass 'Dead Code Elimination'
-; OPT-MORE-DAG: Skipping pass 'Dead Instruction Elimination'
+; MORE-DAG: Skipping pass 'Dead Code Elimination'
+; MORE-DAG: Skipping pass 'Dead Instruction Elimination'
+; NPM-MORE-DAG: Skipping pass: GVNHoistPass
+; NPM-MORE-DAG: Skipping pass: LowerAtomicPass
 
 ; Loop IR passes that opt doesn't turn on by default.
-; OPT-LOOP-DAG: Skipping pass 'Delete dead loops'
-; OPT-LOOP-DAG: Skipping pass 'Induction Variable Simplification'
-; OPT-LOOP-DAG: Skipping pass 'Loop Invariant Code Motion'
-; OPT-LOOP-DAG: Skipping pass 'Loop Strength Reduction'
-; OPT-LOOP-DAG: Skipping pass 'Recognize loop idioms'
-; OPT-LOOP-DAG: Skipping pass 'Reroll loops'
-; OPT-LOOP-DAG: Skipping pass 'Rotate Loops'
-; OPT-LOOP-DAG: Skipping pass 'Simplify instructions in loops'
-; OPT-LOOP-DAG: Skipping pass 'Unroll loops'
-; OPT-LOOP-DAG: Skipping pass 'Unswitch loops'
+; LOOP-DAG: Skipping pass 'Delete dead loops'
+; LOOP-DAG: Skipping pass 'Induction Variable Simplification'
+; LOOP-DAG: Skipping pass 'Loop Invariant Code Motion'
+; LOOP-DAG: Skipping pass 'Loop Strength Reduction'
+; LOOP-DAG: Skipping pass 'Recognize loop idioms'
+; LOOP-DAG: Skipping pass 'Reroll loops'
+; LOOP-DAG: Skipping pass 'Rotate Loops'
+; LOOP-DAG: Skipping pass 'Simplify instructions in loops'
+; LOOP-DAG: Skipping pass 'Unroll loops'
+; LOOP-DAG: Skipping pass 'Unswitch loops'
+; NPM-LOOP-DAG: Skipping pass: LoopSimplifyPass
+; NPM-LOOP-DAG: Skipping pass: LCSSAPass
+; NPM-LOOP-DAG: Skipping pass: IndVarSimplifyPass
+; NPM-LOOP-DAG: Skipping pass: SimpleLoopUnswitchPass
+; NPM-LOOP-DAG: Skipping pass: LoopUnrollPass
+; NPM-LOOP-DAG: Skipping pass: LoopStrengthReducePass
+; NPM-LOOP-DAG: Skipping pass: LoopDeletionPass
+; NPM-LOOP-DAG: Skipping pass: LICMPass
+; NPM-LOOP-DAG: Skipping pass: LoopIdiomRecognizePass
+; NPM-LOOP-DAG: Skipping pass: LoopInstSimplifyPass
diff --git a/llvm/test/Instrumentation/AddressSanitizer/byref-args.ll b/llvm/test/Instrumentation/AddressSanitizer/byref-args.ll
new file mode 100644
index 0000000000000..df2eeb4c1847c
--- /dev/null
+++ b/llvm/test/Instrumentation/AddressSanitizer/byref-args.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -asan -S | FileCheck %s
+
+; Test that for call instructions, the byref arguments are not
+; instrumented, as no copy is implied.
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.bar = type { %struct.foo }
+%struct.foo = type { i8*, i8*, i8* }
+
+; CHECK-LABEL: @func2
+; CHECK-NEXT: tail call void @func1(
+; CHECK-NEXT: ret void
+define dso_local void @func2(%struct.foo* %foo) sanitize_address {
+  tail call void @func1(%struct.foo* byref(%struct.foo) align 8 %foo) #2
+  ret void
+}
+
+declare dso_local void @func1(%struct.foo* byref(%struct.foo) align 8)
diff --git a/llvm/test/Instrumentation/MemorySanitizer/check-struct.ll b/llvm/test/Instrumentation/MemorySanitizer/check-struct.ll
new file mode 100644
index 0000000000000..344dab45db231
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/check-struct.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -msan-check-access-address=0 -msan-track-origins=1 -S -passes='module(msan-module),function(msan)' 2>&1 | \
+; RUN:   FileCheck -allow-deprecated-dag-overlap -check-prefixes=CHECK,CHECK-ORIGINS %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-LABEL: @main
+define { i32, i8 } @main() sanitize_memory {
+; CHECK: [[P:%.*]] = inttoptr i64 0 to { i32, i8 }*
+  %p = inttoptr i64 0 to { i32, i8 } *
+; CHECK: [[O:%.*]] = load { i32, i8 }, { i32, i8 }* [[P]]
+  %o = load { i32, i8 }, { i32, i8 } *%p
+; CHECK: [[FIELD0:%.+]] = extractvalue { i32, i8 } %_msld, 0
+; CHECK: [[F0_POISONED:%.+]] = icmp ne i32 [[FIELD0]]
+; CHECK: [[FIELD1:%.+]] = extractvalue { i32, i8 } %_msld, 1
+; CHECK: [[F1_POISONED:%.+]] = icmp ne i8 [[FIELD1]]
+; CHECK: [[F1_OR:%.+]] = or i1 [[F0_POISONED]], [[F1_POISONED]]
+; CHECK-NOT: icmp ne i1 {{.*}}, false
+; CHECK: br i1 [[F1_OR]]
+; CHECK: call void @__msan_warning
+; CHECK: ret { i32, i8 } [[O]]
+  ret { i32, i8 } %o
+}
diff --git a/llvm/test/Instrumentation/MemorySanitizer/libatomic.ll b/llvm/test/Instrumentation/MemorySanitizer/libatomic.ll
new file mode 100644
index 0000000000000..f0b1fa4fb9a20
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/libatomic.ll
@@ -0,0 +1,69 @@
+; RUN: opt < %s -msan-check-access-address=0 -S -passes=msan 2>&1 | FileCheck %s
+; RUN: opt < %s -msan-check-access-address=0 -msan-track-origins=2 -S -passes=msan 2>&1 | FileCheck %s -check-prefixes=CHECK,CHECK-ORIGIN
+; RUN: opt < %s -msan -msan-check-access-address=0 -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @__atomic_load(i64, i8*, i8*, i32)
+declare void @__atomic_store(i64, i8*, i8*, i32)
+
+define i24 @odd_sized_load(i24* %ptr) sanitize_memory {
+; CHECK: @odd_sized_load(i24* {{.*}}[[PTR:%.+]])
+; CHECK: [[VAL_PTR:%.*]] = alloca i24, align 1
+; CHECK-ORIGIN: @__msan_set_alloca_origin
+; CHECK: [[VAL_PTR_I8:%.*]] = bitcast i24* [[VAL_PTR]] to i8*
+; CHECK: [[PTR_I8:%.*]] = bitcast i24* [[PTR]] to i8*
+; CHECK: call void @__atomic_load(i64 3, i8* [[PTR_I8]], i8* [[VAL_PTR_I8]], i32 2)
+
+; CHECK: ptrtoint i8* [[PTR_I8]]
+; CHECK: xor
+; CHECK: [[SPTR_I8:%.*]] = inttoptr
+; CHECK-ORIGIN: add
+; CHECK-ORIGIN: and
+; CHECK-ORIGIN: [[OPTR:%.*]] = inttoptr
+
+; CHECK: ptrtoint i8* [[VAL_PTR_I8]]
+; CHECK: xor
+; CHECK: [[VAL_SPTR_I8:%.*]] = inttoptr
+; CHECK-ORIGIN: add
+; CHECK-ORIGIN: and
+; CHECK-ORIGIN: [[VAL_OPTR:%.*]] = inttoptr
+
+; CHECK: call void @llvm.memcpy{{.*}}(i8* align 1 [[VAL_SPTR_I8]], i8* align 1 [[SPTR_I8]], i64 3
+
+; CHECK-ORIGIN: [[ARG_ORIGIN:%.*]] = load i32, i32* [[OPTR]]
+; CHECK-ORIGIN: [[VAL_ORIGIN:%.*]] = call i32 @__msan_chain_origin(i32 [[ARG_ORIGIN]])
+; CHECK-ORIGIN: call void @__msan_set_origin(i8* [[VAL_PTR_I8]], i64 3, i32 [[VAL_ORIGIN]])
+
+; CHECK: [[VAL:%.*]] = load i24, i24* [[VAL_PTR]]
+; CHECK: ret i24 [[VAL]]
+  %val_ptr = alloca i24, align 1
+  %val_ptr_i8 = bitcast i24* %val_ptr to i8*
+  %ptr_i8 = bitcast i24* %ptr to i8*
+  call void @__atomic_load(i64 3, i8* %ptr_i8, i8* %val_ptr_i8, i32 0)
+  %val = load i24, i24* %val_ptr
+  ret i24 %val
+}
+
+define void @odd_sized_store(i24* %ptr, i24 %val) sanitize_memory {
+; CHECK: @odd_sized_store(i24* {{.*}}[[PTR:%.+]], i24 {{.*}}[[VAL:%.+]])
+; CHECK: [[VAL_PTR:%.*]] = alloca i24, align 1
+; CHECK: store i24 [[VAL]], i24* [[VAL_PTR]]
+; CHECK: [[VAL_PTR_I8:%.*]] = bitcast i24* [[VAL_PTR]] to i8*
+; CHECK: [[PTR_I8:%.*]] = bitcast i24* [[PTR]] to i8*
+
+; CHECK: ptrtoint i8* [[PTR_I8]]
+; CHECK: xor
+; CHECK: [[SPTR_I8:%.*]] = inttoptr
+; CHECK: call void @llvm.memset{{.*}}(i8* align 1 [[SPTR_I8]], i8 0, i64 3
+
+; CHECK: call void @__atomic_store(i64 3, i8* [[VAL_PTR_I8]], i8* [[PTR_I8]], i32 3)
+; CHECK: ret void
+  %val_ptr = alloca i24, align 1
+  store i24 %val, i24* %val_ptr
+  %val_ptr_i8 = bitcast i24* %val_ptr to i8*
+  %ptr_i8 = bitcast i24* %ptr to i8*
+  call void @__atomic_store(i64 3, i8* %val_ptr_i8, i8* %ptr_i8, i32 0)
+  ret void
+}
+
diff --git a/llvm/test/Instrumentation/MemorySanitizer/msan_x86_bts_asm.ll b/llvm/test/Instrumentation/MemorySanitizer/msan_x86_bts_asm.ll
index f46917d32d268..d9bf307c2b525 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/msan_x86_bts_asm.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/msan_x86_bts_asm.ll
@@ -92,8 +92,7 @@ if.else:                                          ; preds = %entry
 ; CHECK: [[MSPROP:%.*]] = trunc i8 [[MSLD]] to i1
 
 ; Is the shadow poisoned?
-; CHECK: [[MSCMP:%.*]] = icmp ne i1 [[MSPROP]], false
-; CHECK: br i1 [[MSCMP]], label %[[IFTRUE:.*]], label {{.*}}
+; CHECK: br i1 [[MSPROP]], label %[[IFTRUE:.*]], label {{.*}}
 
 ; If yes, raise a warning.
 ; CHECK: [[IFTRUE]]:
diff --git a/llvm/test/Instrumentation/MemorySanitizer/no-check-rt-unaligned.ll b/llvm/test/Instrumentation/MemorySanitizer/no-check-rt-unaligned.ll
new file mode 100644
index 0000000000000..971d5c38811cd
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/no-check-rt-unaligned.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s -msan-eager-checks -msan-check-access-address=0 -S -passes=msan 2>&1 | FileCheck %s
+; RUN: opt < %s -msan-eager-checks -msan-check-access-address=0 -msan-track-origins=2 -S -passes=msan 2>&1 | FileCheck %s -check-prefixes=CHECK,CHECK-ORIGIN
+; RUN: opt < %s -msan-eager-checks -msan -msan-check-access-address=0 -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare noundef i8 @__sanitizer_unaligned_load8(i8* noundef)
+declare void @__sanitizer_unaligned_store8(i8* noundef, i8 noundef)
+
+define noundef i8 @unaligned_load(i8* noundef %ptr) sanitize_memory {
+; CHECK: @unaligned_load(i8* {{.*}}[[PTR:%.+]])
+; CHECK: store i64 0, {{.*}} @__msan_param_tls
+; CHECK: [[VAL:%.*]] = call noundef i8 @__sanitizer_unaligned_load8(i8* noundef [[PTR]])
+  %val = call noundef i8 @__sanitizer_unaligned_load8(i8* noundef %ptr)
+; CHECK: load {{.*}} @__msan_retval_tls
+; CHECK-ORIGIN: load {{.*}} @__msan_retval_origin_tls
+; CHECK: call void @__msan_warning_{{.*}}noreturn
+; CHECK: ret i8 [[VAL]]
+  ret i8 %val
+}
+
+define void @unaligned_store(i8* noundef %ptr, i8 noundef %val) sanitize_memory {
+; CHECK: @unaligned_store(i8* {{.*}}[[PTR:%.+]], i8 {{.*}}[[VAL:%.+]])
+; CHECK: store i64 0, {{.*}} @__msan_param_tls
+; CHECK: store i8 0, {{.*}} @__msan_param_tls
+; CHECK: call void @__sanitizer_unaligned_store8(i8* noundef [[PTR]], i8 noundef [[VAL]])
+  call void @__sanitizer_unaligned_store8(i8* noundef %ptr, i8 noundef %val)
+; CHECK: ret void
+  ret void
+}
diff --git a/llvm/test/MC/AArch64/arm64-be-datalayout.s b/llvm/test/MC/AArch64/arm64-be-datalayout.s
index 67dd6fb1e5797..f608046e14e09 100644
--- a/llvm/test/MC/AArch64/arm64-be-datalayout.s
+++ b/llvm/test/MC/AArch64/arm64-be-datalayout.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple aarch64_be %s | llvm-readobj --section-data -S | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple aarch64_be %s | llvm-readobj --section-data -S - | FileCheck %s
 
 // CHECK: 0000: 00123456 789ABCDE
 foo:    .xword 0x123456789abcde
diff --git a/llvm/test/MC/AArch64/arm64-elf-reloc-condbr.s b/llvm/test/MC/AArch64/arm64-elf-reloc-condbr.s
index 3552ec2895190..80c94f6df423c 100644
--- a/llvm/test/MC/AArch64/arm64-elf-reloc-condbr.s
+++ b/llvm/test/MC/AArch64/arm64-elf-reloc-condbr.s
@@ -1,8 +1,8 @@
 // RUN: llvm-mc -triple=arm64-none-linux-gnu -filetype=obj %s -o - | \
-// RUN:   llvm-readobj -r | FileCheck -check-prefix=OBJ %s
+// RUN:   llvm-readobj -r - | FileCheck -check-prefix=OBJ %s
 // RUN: llvm-mc -target-abi=ilp32 -triple=arm64-none-linux-gnu -filetype=obj \
 // RUN:   %s -o - | \
-// RUN:   llvm-readobj -r | FileCheck -check-prefix=OBJ-ILP32 %s
+// RUN:   llvm-readobj -r - | FileCheck -check-prefix=OBJ-ILP32 %s
 
         b.eq somewhere
 
diff --git a/llvm/test/MC/AArch64/arm64-leaf-compact-unwind.s b/llvm/test/MC/AArch64/arm64-leaf-compact-unwind.s
index 258b259a3df8b..37d6c456b3227 100644
--- a/llvm/test/MC/AArch64/arm64-leaf-compact-unwind.s
+++ b/llvm/test/MC/AArch64/arm64-leaf-compact-unwind.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -triple=arm64-apple-ios -filetype=obj < %s | \
-// RUN: llvm-readobj --expand-relocs -S --section-relocations --section-data | \
+// RUN: llvm-readobj --expand-relocs -S --section-relocations --section-data - | \
 // RUN: FileCheck %s
 //
 // rdar://13070556
diff --git a/llvm/test/MC/AArch64/arm64-small-data-fixups.s b/llvm/test/MC/AArch64/arm64-small-data-fixups.s
index 6debe0b8fb048..43e0d5b71114d 100644
--- a/llvm/test/MC/AArch64/arm64-small-data-fixups.s
+++ b/llvm/test/MC/AArch64/arm64-small-data-fixups.s
@@ -1,4 +1,4 @@
-; RUN: llvm-mc -triple arm64-apple-darwin -filetype=obj -o - %s | llvm-readobj -r | FileCheck %s
+; RUN: llvm-mc -triple arm64-apple-darwin -filetype=obj -o - %s | llvm-readobj -r - | FileCheck %s
 
 foo:
   .long 0
diff --git a/llvm/test/MC/AArch64/arm64-tls-relocs.s b/llvm/test/MC/AArch64/arm64-tls-relocs.s
index 993eb0ebfe9dc..b838845926051 100644
--- a/llvm/test/MC/AArch64/arm64-tls-relocs.s
+++ b/llvm/test/MC/AArch64/arm64-tls-relocs.s
@@ -1,6 +1,6 @@
 // RUN: llvm-mc -triple=arm64-none-linux-gnu -show-encoding < %s | FileCheck %s
 // RUN: llvm-mc -triple=arm64-none-linux-gnu -filetype=obj < %s -o - | \
-// RUN:   llvm-readobj -r --symbols | FileCheck --check-prefix=CHECK-ELF %s
+// RUN:   llvm-readobj -r --symbols - | FileCheck --check-prefix=CHECK-ELF %s
 
 ////////////////////////////////////////////////////////////////////////////////
 // TLS initial-exec forms
diff --git a/llvm/test/MC/AArch64/coff-align.s b/llvm/test/MC/AArch64/coff-align.s
index b76bd73a629dc..6ef69c50c4903 100644
--- a/llvm/test/MC/AArch64/coff-align.s
+++ b/llvm/test/MC/AArch64/coff-align.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple aarch64-windows-gnu %s | llvm-readobj -S --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple aarch64-windows-gnu %s | llvm-readobj -S --sd - | FileCheck %s
     .text
     .align 5
 f0:
diff --git a/llvm/test/MC/AArch64/coff-function-type-info.ll b/llvm/test/MC/AArch64/coff-function-type-info.ll
index 2c2c46b0ca3e9..21446c50ab275 100644
--- a/llvm/test/MC/AArch64/coff-function-type-info.ll
+++ b/llvm/test/MC/AArch64/coff-function-type-info.ll
@@ -2,7 +2,7 @@
 ; RUN:    | FileCheck %s -check-prefix CHECK-ASM
 
 ; RUN: llc -mtriple arm64-windows -filetype obj -o - %s \
-; RUN:    | llvm-readobj --symbols | FileCheck %s -check-prefix CHECK-OBJECT
+; RUN:    | llvm-readobj --symbols - | FileCheck %s -check-prefix CHECK-OBJECT
 
 define arm_aapcs_vfpcc void @external() {
 entry:
diff --git a/llvm/test/MC/AArch64/cold.s b/llvm/test/MC/AArch64/cold.s
index 812d77254542e..16cb90b49583f 100644
--- a/llvm/test/MC/AArch64/cold.s
+++ b/llvm/test/MC/AArch64/cold.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -triple aarch64-apple-ios -o - -filetype=obj %s | \
-// RUN:   llvm-readobj --symbols | FileCheck %s -check-prefix=READOBJ
+// RUN:   llvm-readobj --symbols - | FileCheck %s -check-prefix=READOBJ
 
 // READOBJ-LABEL: Name: cold_func
 // READOBJ-NEXT: Type: Section
diff --git a/llvm/test/MC/AArch64/elf-extern.s b/llvm/test/MC/AArch64/elf-extern.s
index 5e3b314d570f4..ca77270a3d0ca 100644
--- a/llvm/test/MC/AArch64/elf-extern.s
+++ b/llvm/test/MC/AArch64/elf-extern.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc < %s -triple=aarch64-none-linux-gnu -filetype=obj | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc < %s -triple=aarch64-none-linux-gnu -filetype=obj | llvm-readobj -r - | FileCheck %s
 
 // External symbols are a different concept to global variables but should still
 // get relocations and so on when used.
diff --git a/llvm/test/MC/AArch64/elf-globaladdress.ll b/llvm/test/MC/AArch64/elf-globaladdress.ll
index d8a0b5b8d5e77..37e473b0092cc 100644
--- a/llvm/test/MC/AArch64/elf-globaladdress.ll
+++ b/llvm/test/MC/AArch64/elf-globaladdress.ll
@@ -1,10 +1,10 @@
 ;; RUN: llc -mtriple=aarch64-none-linux-gnu -filetype=obj %s -o - | \
-;; RUN:   llvm-readobj -h -r | FileCheck -check-prefix=OBJ %s
+;; RUN:   llvm-readobj -h -r - | FileCheck -check-prefix=OBJ %s
 
 ; Also take it on a round-trip through llvm-mc to stretch assembly-parsing's legs:
 ;; RUN: llc -mtriple=aarch64-none-linux-gnu %s -o - | \
 ;; RUN:     llvm-mc -triple=arm64-none-linux-gnu -filetype=obj -o - | \
-;; RUN:     llvm-readobj -h -r | FileCheck -check-prefix=OBJ %s
+;; RUN:     llvm-readobj -h -r - | FileCheck -check-prefix=OBJ %s
 
 @var8 = global i8 0
 @var16 = global i16 0
diff --git a/llvm/test/MC/AArch64/elf-reloc-ldrlit.s b/llvm/test/MC/AArch64/elf-reloc-ldrlit.s
index e55902cdf9444..6a23aae410323 100644
--- a/llvm/test/MC/AArch64/elf-reloc-ldrlit.s
+++ b/llvm/test/MC/AArch64/elf-reloc-ldrlit.s
@@ -1,8 +1,8 @@
 // RUN: llvm-mc -triple=aarch64-none-linux-gnu -filetype=obj %s -o - | \
-// RUN:   llvm-readobj -r | FileCheck -check-prefix=OBJ %s
+// RUN:   llvm-readobj -r - | FileCheck -check-prefix=OBJ %s
 // RUN: llvm-mc -target-abi=ilp32 -triple=aarch64-none-linux-gnu \
 // RUN:   -filetype=obj %s -o - | \
-// RUN:   llvm-readobj -r | FileCheck -check-prefix=OBJ-ILP32 %s
+// RUN:   llvm-readobj -r - | FileCheck -check-prefix=OBJ-ILP32 %s
 
         ldr x0, some_label
         ldr w3, some_label
diff --git a/llvm/test/MC/AArch64/elf-reloc-ldstunsimm.s b/llvm/test/MC/AArch64/elf-reloc-ldstunsimm.s
index e68937ceddb87..f3fe4e88fd091 100644
--- a/llvm/test/MC/AArch64/elf-reloc-ldstunsimm.s
+++ b/llvm/test/MC/AArch64/elf-reloc-ldstunsimm.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -triple=aarch64-none-linux-gnu -mattr=+fp-armv8 -filetype=obj %s -o - | \
-// RUN:   llvm-readobj -r | FileCheck -check-prefix=OBJ %s
+// RUN:   llvm-readobj -r - | FileCheck -check-prefix=OBJ %s
 
         ldrb w0, [sp, #:lo12:some_label]
         ldrh w0, [sp, #:lo12:some_label]
diff --git a/llvm/test/MC/AArch64/elf-reloc-movw.s b/llvm/test/MC/AArch64/elf-reloc-movw.s
index e9755f483e5da..7123aa544320f 100644
--- a/llvm/test/MC/AArch64/elf-reloc-movw.s
+++ b/llvm/test/MC/AArch64/elf-reloc-movw.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -triple=aarch64-none-linux-gnu -filetype=obj %s -o - | \
-// RUN:   llvm-readobj -r | FileCheck -check-prefix=OBJ %s
+// RUN:   llvm-readobj -r - | FileCheck -check-prefix=OBJ %s
 
         movz x0, #:abs_g0:some_label
         movk x0, #:abs_g0_nc:some_label
diff --git a/llvm/test/MC/AArch64/elf-reloc-pcreladdressing.s b/llvm/test/MC/AArch64/elf-reloc-pcreladdressing.s
index 3931d543aade5..6064e7fc0e217 100644
--- a/llvm/test/MC/AArch64/elf-reloc-pcreladdressing.s
+++ b/llvm/test/MC/AArch64/elf-reloc-pcreladdressing.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -triple=aarch64-none-linux-gnu -filetype=obj %s -o - | \
-// RUN:   llvm-readobj -r | FileCheck -check-prefix=OBJ %s
+// RUN:   llvm-readobj -r - | FileCheck -check-prefix=OBJ %s
 
         adr x2, some_label
         adrp x5, some_label
diff --git a/llvm/test/MC/AArch64/elf-reloc-plt32.s b/llvm/test/MC/AArch64/elf-reloc-plt32.s
index 5e5e1b33e6c6d..5bde72d5f44cb 100644
--- a/llvm/test/MC/AArch64/elf-reloc-plt32.s
+++ b/llvm/test/MC/AArch64/elf-reloc-plt32.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -triple=aarch64 -filetype=obj %s -o - | \
-// RUN:   llvm-readobj -r | FileCheck %s
+// RUN:   llvm-readobj -r - | FileCheck %s
 
         .section .data
 this:
diff --git a/llvm/test/MC/AArch64/elf-reloc-tstb.s b/llvm/test/MC/AArch64/elf-reloc-tstb.s
index 1070c4d7f3e69..5d8a91ec4707f 100644
--- a/llvm/test/MC/AArch64/elf-reloc-tstb.s
+++ b/llvm/test/MC/AArch64/elf-reloc-tstb.s
@@ -1,8 +1,8 @@
 // RUN: llvm-mc -triple=aarch64-none-linux-gnu -filetype=obj %s -o - | \
-// RUN:   llvm-readobj -r | FileCheck -check-prefix=OBJ %s
+// RUN:   llvm-readobj -r - | FileCheck -check-prefix=OBJ %s
 // RUN: llvm-mc -target-abi=ilp32 -triple=aarch64-none-linux-gnu \
 // RUN:   -filetype=obj %s -o - | \
-// RUN:   llvm-readobj -r | FileCheck -check-prefix=OBJ-ILP32 %s
+// RUN:   llvm-readobj -r - | FileCheck -check-prefix=OBJ-ILP32 %s
 
         tbz x6, #45, somewhere
         tbnz w3, #15, somewhere
diff --git a/llvm/test/MC/AArch64/elf-reloc-uncondbrimm.s b/llvm/test/MC/AArch64/elf-reloc-uncondbrimm.s
index 373779d03d17f..ac788815b789c 100644
--- a/llvm/test/MC/AArch64/elf-reloc-uncondbrimm.s
+++ b/llvm/test/MC/AArch64/elf-reloc-uncondbrimm.s
@@ -1,8 +1,8 @@
 // RUN: llvm-mc -triple=aarch64-none-linux-gnu -filetype=obj %s -o - | \
-// RUN:   llvm-readobj -r | FileCheck -check-prefix=OBJ %s
+// RUN:   llvm-readobj -r - | FileCheck -check-prefix=OBJ %s
 // RUN: llvm-mc -target-abi=ilp32 -triple=aarch64-none-linux-gnu \
 // RUN:   -filetype=obj %s -o - | \
-// RUN:   llvm-readobj -r | FileCheck -check-prefix=OBJ-ILP32 %s
+// RUN:   llvm-readobj -r - | FileCheck -check-prefix=OBJ-ILP32 %s
 
         b somewhere
         bl somewhere
diff --git a/llvm/test/MC/AArch64/elf_osabi_flags.s b/llvm/test/MC/AArch64/elf_osabi_flags.s
index 42c56f043f2b7..0bc022ab5a8c4 100644
--- a/llvm/test/MC/AArch64/elf_osabi_flags.s
+++ b/llvm/test/MC/AArch64/elf_osabi_flags.s
@@ -1,6 +1,6 @@
-# RUN: llvm-mc -filetype=obj -triple aarch64 %s -o -| llvm-readobj -h | FileCheck %s
-# RUN: llvm-mc -filetype=obj -triple aarch64-linux-gnu %s -o -| llvm-readobj -h | FileCheck %s
+# RUN: llvm-mc -filetype=obj -triple aarch64 %s -o -| llvm-readobj -h - | FileCheck %s
+# RUN: llvm-mc -filetype=obj -triple aarch64-linux-gnu %s -o -| llvm-readobj -h - | FileCheck %s
 # CHECK: OS/ABI: SystemV (0x0)
 
-# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-freebsd %s -o -| llvm-readobj -h | FileCheck --check-prefix=AARCH64-FREEBSD-OSABI %s
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-freebsd %s -o -| llvm-readobj -h - | FileCheck --check-prefix=AARCH64-FREEBSD-OSABI %s
 # AARCH64-FREEBSD-OSABI: OS/ABI: FreeBSD (0x9)
diff --git a/llvm/test/MC/AArch64/jump-table.s b/llvm/test/MC/AArch64/jump-table.s
index c4b879e6ef079..32dcad9565213 100644
--- a/llvm/test/MC/AArch64/jump-table.s
+++ b/llvm/test/MC/AArch64/jump-table.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc < %s -triple=aarch64-none-linux-gnu -filetype=obj | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc < %s -triple=aarch64-none-linux-gnu -filetype=obj | llvm-readobj -r - | FileCheck %s
 
 	.file	"<stdin>"
 	.text
diff --git a/llvm/test/MC/AArch64/reloc-directive.s b/llvm/test/MC/AArch64/reloc-directive.s
index ce0e00e193576..88cb48889d184 100644
--- a/llvm/test/MC/AArch64/reloc-directive.s
+++ b/llvm/test/MC/AArch64/reloc-directive.s
@@ -1,6 +1,6 @@
 # RUN: llvm-mc -triple=aarch64-linux-musl %s | FileCheck --check-prefix=PRINT %s
 
-# RUN: llvm-mc -filetype=obj -triple=aarch64-linux-musl %s | llvm-readobj -r | FileCheck %s
+# RUN: llvm-mc -filetype=obj -triple=aarch64-linux-musl %s | llvm-readobj -r - | FileCheck %s
 
 # PRINT: .reloc 8, R_AARCH64_NONE, .data
 # PRINT: .reloc 4, R_AARCH64_NONE, foo+4
diff --git a/llvm/test/MC/AArch64/seh.s b/llvm/test/MC/AArch64/seh.s
index d4acc4fddb9f5..eb6a94a3f7377 100644
--- a/llvm/test/MC/AArch64/seh.s
+++ b/llvm/test/MC/AArch64/seh.s
@@ -1,6 +1,6 @@
 // This test checks that the SEH directives emit the correct unwind data.
 
-// RUN: llvm-mc -triple aarch64-pc-win32 -filetype=obj %s | llvm-readobj -S -r | FileCheck %s
+// RUN: llvm-mc -triple aarch64-pc-win32 -filetype=obj %s | llvm-readobj -S -r - | FileCheck %s
 
 // CHECK:      Sections [
 // CHECK:        Section {
diff --git a/llvm/test/MC/AArch64/tls-relocs.s b/llvm/test/MC/AArch64/tls-relocs.s
index 9f0d5e4a7839f..5194048c5b625 100644
--- a/llvm/test/MC/AArch64/tls-relocs.s
+++ b/llvm/test/MC/AArch64/tls-relocs.s
@@ -1,6 +1,6 @@
 // RUN: llvm-mc -triple=aarch64-none-linux-gnu -show-encoding < %s | FileCheck %s
 // RUN: llvm-mc -triple=aarch64-none-linux-gnu -filetype=obj < %s -o - | \
-// RUN:   llvm-readobj -r --symbols | FileCheck --check-prefix=CHECK-ELF %s
+// RUN:   llvm-readobj -r --symbols - | FileCheck --check-prefix=CHECK-ELF %s
 
         // TLS local-dynamic forms
         movz x1, #:dtprel_g2:var
diff --git a/llvm/test/MC/AMDGPU/elf-lds.s b/llvm/test/MC/AMDGPU/elf-lds.s
index e03f8a2958308..ec1c58e66b215 100644
--- a/llvm/test/MC/AMDGPU/elf-lds.s
+++ b/llvm/test/MC/AMDGPU/elf-lds.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple amdgcn-- -mcpu gfx900 %s -o - | llvm-readobj -t -r | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple amdgcn-- -mcpu gfx900 %s -o - | llvm-readobj -t -r - | FileCheck %s
 
 	.text
 	.globl test_kernel
diff --git a/llvm/test/MC/AMDGPU/hsa-exp.s b/llvm/test/MC/AMDGPU/hsa-exp.s
index cb7c0be8b021f..2324782497c4e 100644
--- a/llvm/test/MC/AMDGPU/hsa-exp.s
+++ b/llvm/test/MC/AMDGPU/hsa-exp.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -triple amdgcn--amdhsa -mcpu=kaveri -mattr=-code-object-v3 -show-encoding %s | FileCheck %s --check-prefix=ASM
-// RUN: llvm-mc -filetype=obj -triple amdgcn--amdhsa -mcpu=kaveri -mattr=-code-object-v3 -show-encoding %s | llvm-readobj --symbols -S --sd | FileCheck %s --check-prefix=ELF
+// RUN: llvm-mc -filetype=obj -triple amdgcn--amdhsa -mcpu=kaveri -mattr=-code-object-v3 -show-encoding %s | llvm-readobj --symbols -S --sd - | FileCheck %s --check-prefix=ELF
 
 // ELF: Section {
 // ELF: Name: .text
diff --git a/llvm/test/MC/AMDGPU/hsa-gfx10.s b/llvm/test/MC/AMDGPU/hsa-gfx10.s
index db868f300f031..707a8be4e8081 100644
--- a/llvm/test/MC/AMDGPU/hsa-gfx10.s
+++ b/llvm/test/MC/AMDGPU/hsa-gfx10.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -triple amdgcn--amdhsa -mcpu=gfx1010 -mattr=-WavefrontSize32,+WavefrontSize64,-code-object-v3 -show-encoding %s | FileCheck %s --check-prefix=ASM
-// RUN: llvm-mc -filetype=obj -triple amdgcn--amdhsa -mcpu=gfx1010 -mattr=-WavefrontSize32,+WavefrontSize64,-code-object-v3 -show-encoding %s | llvm-readobj -symbols -s -sd | FileCheck %s --check-prefix=ELF
+// RUN: llvm-mc -filetype=obj -triple amdgcn--amdhsa -mcpu=gfx1010 -mattr=-WavefrontSize32,+WavefrontSize64,-code-object-v3 -show-encoding %s | llvm-readobj -symbols -s -sd - | FileCheck %s --check-prefix=ELF
 
 // ELF: Section {
 // ELF: Name: .text
diff --git a/llvm/test/MC/AMDGPU/hsa-text.s b/llvm/test/MC/AMDGPU/hsa-text.s
index d4f67d033475d..d917a46b078c2 100644
--- a/llvm/test/MC/AMDGPU/hsa-text.s
+++ b/llvm/test/MC/AMDGPU/hsa-text.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -triple amdgcn--amdhsa -mcpu=kaveri -mattr=-code-object-v3 -show-encoding %s | FileCheck %s --check-prefix=ASM
-// RUN: llvm-mc -filetype=obj -triple amdgcn--amdhsa -mcpu=kaveri -mattr=-code-object-v3 -show-encoding %s | llvm-readobj -S --sd | FileCheck %s --check-prefix=ELF
+// RUN: llvm-mc -filetype=obj -triple amdgcn--amdhsa -mcpu=kaveri -mattr=-code-object-v3 -show-encoding %s | llvm-readobj -S --sd - | FileCheck %s --check-prefix=ELF
 
 // For compatibility reasons we treat convert .text sections to .hsatext
 
diff --git a/llvm/test/MC/AMDGPU/hsa.s b/llvm/test/MC/AMDGPU/hsa.s
index 87c09648e51dc..b031abdd3928e 100644
--- a/llvm/test/MC/AMDGPU/hsa.s
+++ b/llvm/test/MC/AMDGPU/hsa.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -triple amdgcn--amdhsa -mcpu=kaveri -mattr=-code-object-v3 -show-encoding %s | FileCheck %s --check-prefix=ASM
-// RUN: llvm-mc -filetype=obj -triple amdgcn--amdhsa -mcpu=kaveri -mattr=-code-object-v3 -show-encoding %s | llvm-readobj --symbols -S --sd | FileCheck %s --check-prefix=ELF
+// RUN: llvm-mc -filetype=obj -triple amdgcn--amdhsa -mcpu=kaveri -mattr=-code-object-v3 -show-encoding %s | llvm-readobj --symbols -S --sd - | FileCheck %s --check-prefix=ELF
 
 // ELF: Section {
 // ELF: Name: .text
diff --git a/llvm/test/MC/AMDGPU/hsa_code_object_isa_args.s b/llvm/test/MC/AMDGPU/hsa_code_object_isa_args.s
index b84727cf1daa9..ff6cd022c9381 100644
--- a/llvm/test/MC/AMDGPU/hsa_code_object_isa_args.s
+++ b/llvm/test/MC/AMDGPU/hsa_code_object_isa_args.s
@@ -1,9 +1,9 @@
 // RUN: llvm-mc -triple amdgcn--amdhsa -mcpu=kaveri -mattr=-code-object-v3 -show-encoding %s | FileCheck %s --check-prefix=ASM --check-prefix=ASM_700
 // RUN: llvm-mc -triple amdgcn--amdhsa -mcpu=gfx803 -mattr=-code-object-v3 -show-encoding %s | FileCheck %s --check-prefix=ASM --check-prefix=ASM_803
 // RUN: llvm-mc -triple amdgcn--amdhsa -mcpu=stoney -mattr=-code-object-v3 -show-encoding %s | FileCheck %s --check-prefix=ASM --check-prefix=ASM_810
-// RUN: llvm-mc -filetype=obj -triple amdgcn--amdhsa -mcpu=kaveri -mattr=-code-object-v3 -show-encoding %s | llvm-readobj -S --sd | FileCheck %s --check-prefix=ELF --check-prefix=ELF_700
-// RUN: llvm-mc -filetype=obj -triple amdgcn--amdhsa -mcpu=gfx803 -mattr=-code-object-v3 -show-encoding %s | llvm-readobj -S --sd | FileCheck %s --check-prefix=ELF --check-prefix=ELF_803
-// RUN: llvm-mc -filetype=obj -triple amdgcn--amdhsa -mcpu=stoney -mattr=-code-object-v3 -show-encoding %s | llvm-readobj -S --sd | FileCheck %s --check-prefix=ELF --check-prefix=ELF_810
+// RUN: llvm-mc -filetype=obj -triple amdgcn--amdhsa -mcpu=kaveri -mattr=-code-object-v3 -show-encoding %s | llvm-readobj -S --sd - | FileCheck %s --check-prefix=ELF --check-prefix=ELF_700
+// RUN: llvm-mc -filetype=obj -triple amdgcn--amdhsa -mcpu=gfx803 -mattr=-code-object-v3 -show-encoding %s | llvm-readobj -S --sd - | FileCheck %s --check-prefix=ELF --check-prefix=ELF_803
+// RUN: llvm-mc -filetype=obj -triple amdgcn--amdhsa -mcpu=stoney -mattr=-code-object-v3 -show-encoding %s | llvm-readobj -S --sd - | FileCheck %s --check-prefix=ELF --check-prefix=ELF_810
 
 // ELF: SHT_NOTE
 // ELF: 0000: 04000000 08000000 01000000 414D4400
diff --git a/llvm/test/MC/AMDGPU/inline-imm-inv2pi.s b/llvm/test/MC/AMDGPU/inline-imm-inv2pi.s
new file mode 100644
index 0000000000000..e5ecfa293ba4d
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/inline-imm-inv2pi.s
@@ -0,0 +1,10 @@
+// RUN: llvm-mc -arch=amdgcn -mcpu=tahiti -show-encoding %s | FileCheck -check-prefix=SI %s
+// RUN: llvm-mc -arch=amdgcn -mcpu=tonga -show-encoding %s | FileCheck -check-prefix=VI %s
+
+// The value inv2pi should not assert on any targets, but is
+// printed differently depending on whether it's a legal inline
+// immediate or not.
+
+// SI: v_cvt_f32_f16_e32 v0, 0x3118 ; encoding: [0xff,0x16,0x00,0x7e,0x18,0x31,0x00,0x00]
+// VI: v_cvt_f32_f16_e32 v0, 0.15915494 ; encoding: [0xf8,0x16,0x00,0x7e]
+v_cvt_f32_f16_e32 v0, 0x3118
diff --git a/llvm/test/MC/AMDGPU/pal-msgpack.s b/llvm/test/MC/AMDGPU/pal-msgpack.s
index a5599f3a49477..886cc8b0538b4 100644
--- a/llvm/test/MC/AMDGPU/pal-msgpack.s
+++ b/llvm/test/MC/AMDGPU/pal-msgpack.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -triple amdgcn--amdpal -mcpu=kaveri -show-encoding %s | FileCheck %s --check-prefix=ASM
-// RUN: llvm-mc -filetype=obj -triple amdgcn--amdpal -mcpu=kaveri -show-encoding %s | llvm-readobj --symbols -S --sd | FileCheck %s --check-prefix=ELF
+// RUN: llvm-mc -filetype=obj -triple amdgcn--amdpal -mcpu=kaveri -show-encoding %s | llvm-readobj --symbols -S --sd - | FileCheck %s --check-prefix=ELF
 
 	.amdgpu_pal_metadata
 ---
diff --git a/llvm/test/MC/AMDGPU/pal.s b/llvm/test/MC/AMDGPU/pal.s
index 913738727afc5..9d758d30e6e2b 100644
--- a/llvm/test/MC/AMDGPU/pal.s
+++ b/llvm/test/MC/AMDGPU/pal.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -triple amdgcn--amdpal -mcpu=kaveri -show-encoding %s | FileCheck %s --check-prefix=ASM
-// RUN: llvm-mc -filetype=obj -triple amdgcn--amdpal -mcpu=kaveri -show-encoding %s | llvm-readobj --symbols -S --sd | FileCheck %s --check-prefix=ELF
+// RUN: llvm-mc -filetype=obj -triple amdgcn--amdpal -mcpu=kaveri -show-encoding %s | llvm-readobj --symbols -S --sd - | FileCheck %s --check-prefix=ELF
 
 .amd_amdgpu_pal_metadata 0x12345678, 0xfedcba98, 0x2468ace0, 0xfdb97531
 // ASM: .amd_amdgpu_pal_metadata 0x12345678,0xfedcba98,0x2468ace0,0xfdb97531
diff --git a/llvm/test/MC/AMDGPU/reloc.s b/llvm/test/MC/AMDGPU/reloc.s
index e5d38578832d0..f5650d809fe54 100644
--- a/llvm/test/MC/AMDGPU/reloc.s
+++ b/llvm/test/MC/AMDGPU/reloc.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple amdgcn-- -mcpu=kaveri -show-encoding %s | llvm-readobj -relocations | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple amdgcn-- -mcpu=kaveri -show-encoding %s | llvm-readobj -relocations - | FileCheck %s
 
 // CHECK: Relocations [
 // CHECK: .rel.text {
diff --git a/llvm/test/MC/ARM/2010-11-30-reloc-movt.s b/llvm/test/MC/ARM/2010-11-30-reloc-movt.s
index a83465edc0268..d4aefdf35fc26 100644
--- a/llvm/test/MC/ARM/2010-11-30-reloc-movt.s
+++ b/llvm/test/MC/ARM/2010-11-30-reloc-movt.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc  %s -triple=armv7-linux-gnueabi -filetype=obj -o - | \
-// RUN:    llvm-readobj -S --sr --sd | FileCheck  %s
+// RUN:    llvm-readobj -S --sr --sd - | FileCheck  %s
 
 	.syntax unified
 	.eabi_attribute	6, 10
diff --git a/llvm/test/MC/ARM/arm-elf-symver.s b/llvm/test/MC/ARM/arm-elf-symver.s
index 83313c103a1c3..9d7da0c133119 100644
--- a/llvm/test/MC/ARM/arm-elf-symver.s
+++ b/llvm/test/MC/ARM/arm-elf-symver.s
@@ -1,5 +1,5 @@
-@ RUN: llvm-mc -filetype=obj -triple arm-none-linux-gnueabi %s -o - | llvm-readobj -r --symbols | FileCheck %s
-@ RUN: llvm-mc -filetype=obj -triple thumb-none-linux-gnueabi %s -o - | llvm-readobj -r --symbols | FileCheck %s
+@ RUN: llvm-mc -filetype=obj -triple arm-none-linux-gnueabi %s -o - | llvm-readobj -r --symbols - | FileCheck %s
+@ RUN: llvm-mc -filetype=obj -triple thumb-none-linux-gnueabi %s -o - | llvm-readobj -r --symbols - | FileCheck %s
 
 defined1:
 defined2:
diff --git a/llvm/test/MC/ARM/arm-thumb-tail-call.ll b/llvm/test/MC/ARM/arm-thumb-tail-call.ll
index c166719505dfb..4839d2b9741a3 100644
--- a/llvm/test/MC/ARM/arm-thumb-tail-call.ll
+++ b/llvm/test/MC/ARM/arm-thumb-tail-call.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -O0 < %s -mtriple armv7-linux-gnueabi -o - \
 ; RUN:   | llvm-mc -triple armv7-linux-gnueabi -filetype=obj -o - \
-; RUN:    | llvm-readobj -r | FileCheck %s
+; RUN:    | llvm-readobj -r - | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 target triple = "armv7--linux-gnueabihf"
diff --git a/llvm/test/MC/ARM/coff-file.s b/llvm/test/MC/ARM/coff-file.s
index 9caf0d4bcac39..51e75515b988f 100644
--- a/llvm/test/MC/ARM/coff-file.s
+++ b/llvm/test/MC/ARM/coff-file.s
@@ -2,7 +2,7 @@
 // RUN:   | FileCheck %s
 
 // RUN: llvm-mc -triple thumbv7-windows -filetype obj %s -o - \
-// RUN:	  | llvm-readobj --symbols | FileCheck %s -check-prefix CHECK-SCN
+// RUN:	  | llvm-readobj --symbols - | FileCheck %s -check-prefix CHECK-SCN
 
 	.file "null-padded.asm"
 // CHECK: (nx 1) {{0x[0-9]+}} .file
diff --git a/llvm/test/MC/ARM/coff-function-type-info.ll b/llvm/test/MC/ARM/coff-function-type-info.ll
index f7a87e72037a2..07fb8bc0b3b85 100644
--- a/llvm/test/MC/ARM/coff-function-type-info.ll
+++ b/llvm/test/MC/ARM/coff-function-type-info.ll
@@ -2,7 +2,7 @@
 ; RUN:    | FileCheck %s -check-prefix CHECK-ASM
 
 ; RUN: llc -mtriple thumbv7-windows-itanium -filetype obj -o - %s \
-; RUN:    | llvm-readobj --symbols | FileCheck %s -check-prefix CHECK-OBJECT
+; RUN:    | llvm-readobj --symbols - | FileCheck %s -check-prefix CHECK-OBJECT
 
 define arm_aapcs_vfpcc void @external() {
 entry:
diff --git a/llvm/test/MC/ARM/cxx-global-constructor.ll b/llvm/test/MC/ARM/cxx-global-constructor.ll
index 0232433890888..6f0f651505bd5 100644
--- a/llvm/test/MC/ARM/cxx-global-constructor.ll
+++ b/llvm/test/MC/ARM/cxx-global-constructor.ll
@@ -1,5 +1,5 @@
 ; RUN: llc %s -mtriple=armv7-linux-gnueabi -relocation-model=pic \
-; RUN: -filetype=obj -o - | llvm-readobj -r | FileCheck %s
+; RUN: -filetype=obj -o - | llvm-readobj -r - | FileCheck %s
 
 
 @llvm.global_ctors = appending global [1 x { i32, void ()*, i8* }] [{ i32, void ()*, i8* } { i32 65535, void ()* @f, i8* null }]
diff --git a/llvm/test/MC/ARM/data-in-code.ll b/llvm/test/MC/ARM/data-in-code.ll
index 4f368ab3727d5..dcd3947f7e0cc 100644
--- a/llvm/test/MC/ARM/data-in-code.ll
+++ b/llvm/test/MC/ARM/data-in-code.ll
@@ -1,10 +1,10 @@
 ;; RUN: llc -verify-machineinstrs \
 ;; RUN:   -mtriple=armv7-linux-gnueabi -filetype=obj %s -o - | \
-;; RUN:   llvm-readobj --symbols | FileCheck -check-prefix=ARM %s
+;; RUN:   llvm-readobj --symbols - | FileCheck -check-prefix=ARM %s
 
 ;; RUN: llc -verify-machineinstrs \
 ;; RUN:   -mtriple=thumbv7-linux-gnueabi -filetype=obj %s -o - | \
-;; RUN:   llvm-readobj --symbols | FileCheck -check-prefix=TMB %s
+;; RUN:   llvm-readobj --symbols - | FileCheck -check-prefix=TMB %s
 
 ;; Ensure that if a jump table is generated that it has Mapping Symbols
 ;; marking the data-in-code region.
diff --git a/llvm/test/MC/ARM/directive-arch-armv2.s b/llvm/test/MC/ARM/directive-arch-armv2.s
index 8985a0e19ff63..f4edc34a36529 100644
--- a/llvm/test/MC/ARM/directive-arch-armv2.s
+++ b/llvm/test/MC/ARM/directive-arch-armv2.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv2
diff --git a/llvm/test/MC/ARM/directive-arch-armv2a.s b/llvm/test/MC/ARM/directive-arch-armv2a.s
index 7ee434dde61c3..f63f1a57cb9e5 100644
--- a/llvm/test/MC/ARM/directive-arch-armv2a.s
+++ b/llvm/test/MC/ARM/directive-arch-armv2a.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv2a
diff --git a/llvm/test/MC/ARM/directive-arch-armv3.s b/llvm/test/MC/ARM/directive-arch-armv3.s
index 6e514b69c08e9..001b30ad6bfc0 100644
--- a/llvm/test/MC/ARM/directive-arch-armv3.s
+++ b/llvm/test/MC/ARM/directive-arch-armv3.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv3
diff --git a/llvm/test/MC/ARM/directive-arch-armv3m.s b/llvm/test/MC/ARM/directive-arch-armv3m.s
index 08843c834b77d..544445e75c473 100644
--- a/llvm/test/MC/ARM/directive-arch-armv3m.s
+++ b/llvm/test/MC/ARM/directive-arch-armv3m.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv3m
diff --git a/llvm/test/MC/ARM/directive-arch-armv4.s b/llvm/test/MC/ARM/directive-arch-armv4.s
index d49d4fd6c751a..93374efb99f17 100644
--- a/llvm/test/MC/ARM/directive-arch-armv4.s
+++ b/llvm/test/MC/ARM/directive-arch-armv4.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv4
diff --git a/llvm/test/MC/ARM/directive-arch-armv4t.s b/llvm/test/MC/ARM/directive-arch-armv4t.s
index a4eafb1e238aa..f14235ab76683 100644
--- a/llvm/test/MC/ARM/directive-arch-armv4t.s
+++ b/llvm/test/MC/ARM/directive-arch-armv4t.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv4t
diff --git a/llvm/test/MC/ARM/directive-arch-armv5.s b/llvm/test/MC/ARM/directive-arch-armv5.s
index 8f94af66d6fb8..368e88515b95b 100644
--- a/llvm/test/MC/ARM/directive-arch-armv5.s
+++ b/llvm/test/MC/ARM/directive-arch-armv5.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv5
diff --git a/llvm/test/MC/ARM/directive-arch-armv5t.s b/llvm/test/MC/ARM/directive-arch-armv5t.s
index 234180ecb581b..82c7061323dcb 100644
--- a/llvm/test/MC/ARM/directive-arch-armv5t.s
+++ b/llvm/test/MC/ARM/directive-arch-armv5t.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv5t
diff --git a/llvm/test/MC/ARM/directive-arch-armv5te.s b/llvm/test/MC/ARM/directive-arch-armv5te.s
index 2c4c148a363b8..af3e38c025c2e 100644
--- a/llvm/test/MC/ARM/directive-arch-armv5te.s
+++ b/llvm/test/MC/ARM/directive-arch-armv5te.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv5te
diff --git a/llvm/test/MC/ARM/directive-arch-armv6-m.s b/llvm/test/MC/ARM/directive-arch-armv6-m.s
index c37b8b1680c58..52f5800fb13af 100644
--- a/llvm/test/MC/ARM/directive-arch-armv6-m.s
+++ b/llvm/test/MC/ARM/directive-arch-armv6-m.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv6-m
diff --git a/llvm/test/MC/ARM/directive-arch-armv6.s b/llvm/test/MC/ARM/directive-arch-armv6.s
index 2c631500bb9a5..636aa4aa4d3d3 100644
--- a/llvm/test/MC/ARM/directive-arch-armv6.s
+++ b/llvm/test/MC/ARM/directive-arch-armv6.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv6
diff --git a/llvm/test/MC/ARM/directive-arch-armv6k.s b/llvm/test/MC/ARM/directive-arch-armv6k.s
index ba6adea597e0e..d6245e5bb90cc 100644
--- a/llvm/test/MC/ARM/directive-arch-armv6k.s
+++ b/llvm/test/MC/ARM/directive-arch-armv6k.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv6k
diff --git a/llvm/test/MC/ARM/directive-arch-armv6t2.s b/llvm/test/MC/ARM/directive-arch-armv6t2.s
index 0be5169b37769..6ac895546e34d 100644
--- a/llvm/test/MC/ARM/directive-arch-armv6t2.s
+++ b/llvm/test/MC/ARM/directive-arch-armv6t2.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv6t2
diff --git a/llvm/test/MC/ARM/directive-arch-armv6z.s b/llvm/test/MC/ARM/directive-arch-armv6z.s
index efebe951e269c..e757e57aab18d 100644
--- a/llvm/test/MC/ARM/directive-arch-armv6z.s
+++ b/llvm/test/MC/ARM/directive-arch-armv6z.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv6z
diff --git a/llvm/test/MC/ARM/directive-arch-armv7-a.s b/llvm/test/MC/ARM/directive-arch-armv7-a.s
index c3bd86de1349b..84d55a9e28646 100644
--- a/llvm/test/MC/ARM/directive-arch-armv7-a.s
+++ b/llvm/test/MC/ARM/directive-arch-armv7-a.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv7-a
diff --git a/llvm/test/MC/ARM/directive-arch-armv7-m.s b/llvm/test/MC/ARM/directive-arch-armv7-m.s
index 96df23a96b2e0..b5959d53f6943 100644
--- a/llvm/test/MC/ARM/directive-arch-armv7-m.s
+++ b/llvm/test/MC/ARM/directive-arch-armv7-m.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv7-m
diff --git a/llvm/test/MC/ARM/directive-arch-armv7-r.s b/llvm/test/MC/ARM/directive-arch-armv7-r.s
index b67aee3f2ea22..ea58b5d061264 100644
--- a/llvm/test/MC/ARM/directive-arch-armv7-r.s
+++ b/llvm/test/MC/ARM/directive-arch-armv7-r.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv7-r
diff --git a/llvm/test/MC/ARM/directive-arch-armv7.s b/llvm/test/MC/ARM/directive-arch-armv7.s
index 1ea2bcf89fbe1..9485e4b90ab3c 100644
--- a/llvm/test/MC/ARM/directive-arch-armv7.s
+++ b/llvm/test/MC/ARM/directive-arch-armv7.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv7
diff --git a/llvm/test/MC/ARM/directive-arch-armv7a.s b/llvm/test/MC/ARM/directive-arch-armv7a.s
index e17e170fb057b..c3d5e14da6cb6 100644
--- a/llvm/test/MC/ARM/directive-arch-armv7a.s
+++ b/llvm/test/MC/ARM/directive-arch-armv7a.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv7a
diff --git a/llvm/test/MC/ARM/directive-arch-armv7e-m.s b/llvm/test/MC/ARM/directive-arch-armv7e-m.s
index 39f9184fd4213..5664737d1d836 100644
--- a/llvm/test/MC/ARM/directive-arch-armv7e-m.s
+++ b/llvm/test/MC/ARM/directive-arch-armv7e-m.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
   .syntax  unified
   .arch  armv7e-m
diff --git a/llvm/test/MC/ARM/directive-arch-armv7em.s b/llvm/test/MC/ARM/directive-arch-armv7em.s
index 10f9effebcc4d..1b4f66fceebf1 100644
--- a/llvm/test/MC/ARM/directive-arch-armv7em.s
+++ b/llvm/test/MC/ARM/directive-arch-armv7em.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
   .syntax  unified
   .arch  armv7em
diff --git a/llvm/test/MC/ARM/directive-arch-armv7m.s b/llvm/test/MC/ARM/directive-arch-armv7m.s
index 5b623af34e6a2..5c0ca48f23fe0 100644
--- a/llvm/test/MC/ARM/directive-arch-armv7m.s
+++ b/llvm/test/MC/ARM/directive-arch-armv7m.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv7m
diff --git a/llvm/test/MC/ARM/directive-arch-armv7r.s b/llvm/test/MC/ARM/directive-arch-armv7r.s
index f4461483b5353..a2d8060fbd019 100644
--- a/llvm/test/MC/ARM/directive-arch-armv7r.s
+++ b/llvm/test/MC/ARM/directive-arch-armv7r.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv7r
diff --git a/llvm/test/MC/ARM/directive-arch-armv8-a.s b/llvm/test/MC/ARM/directive-arch-armv8-a.s
index fd567c8e4b3a6..a84bcf3d8f228 100644
--- a/llvm/test/MC/ARM/directive-arch-armv8-a.s
+++ b/llvm/test/MC/ARM/directive-arch-armv8-a.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv8-a
diff --git a/llvm/test/MC/ARM/directive-arch-armv8.2-a.s b/llvm/test/MC/ARM/directive-arch-armv8.2-a.s
index 8cd7c2f740a5d..2684ba7a0e94d 100644
--- a/llvm/test/MC/ARM/directive-arch-armv8.2-a.s
+++ b/llvm/test/MC/ARM/directive-arch-armv8.2-a.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv8.2-a
diff --git a/llvm/test/MC/ARM/directive-arch-armv8a.s b/llvm/test/MC/ARM/directive-arch-armv8a.s
index e9a5bef89d546..1eddb69a5996a 100644
--- a/llvm/test/MC/ARM/directive-arch-armv8a.s
+++ b/llvm/test/MC/ARM/directive-arch-armv8a.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv8a
diff --git a/llvm/test/MC/ARM/directive-arch-iwmmxt.s b/llvm/test/MC/ARM/directive-arch-iwmmxt.s
index 2a9317ddf4623..e389b95050de2 100644
--- a/llvm/test/MC/ARM/directive-arch-iwmmxt.s
+++ b/llvm/test/MC/ARM/directive-arch-iwmmxt.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	iwmmxt
diff --git a/llvm/test/MC/ARM/directive-arch-iwmmxt2.s b/llvm/test/MC/ARM/directive-arch-iwmmxt2.s
index 90b2181ee2922..2e1c474bdab80 100644
--- a/llvm/test/MC/ARM/directive-arch-iwmmxt2.s
+++ b/llvm/test/MC/ARM/directive-arch-iwmmxt2.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	iwmmxt2
diff --git a/llvm/test/MC/ARM/directive-cpu.s b/llvm/test/MC/ARM/directive-cpu.s
index 8c95d7b3d86e5..dc99c202c1308 100644
--- a/llvm/test/MC/ARM/directive-cpu.s
+++ b/llvm/test/MC/ARM/directive-cpu.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc < %s -triple armv7-unknown-linux-gnueabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj -S --sd | FileCheck %s
+@ RUN:   | llvm-readobj -S --sd - | FileCheck %s
 
 @ CHECK: Name: .ARM.attribute
 @ CHECK: SectionData (
diff --git a/llvm/test/MC/ARM/directive-eabi_attribute-overwrite.s b/llvm/test/MC/ARM/directive-eabi_attribute-overwrite.s
index 194c77e6a75cf..3a992d3260ca3 100644
--- a/llvm/test/MC/ARM/directive-eabi_attribute-overwrite.s
+++ b/llvm/test/MC/ARM/directive-eabi_attribute-overwrite.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax unified
 	.thumb
diff --git a/llvm/test/MC/ARM/directive-eabi_attribute.s b/llvm/test/MC/ARM/directive-eabi_attribute.s
index 445bc215cfb7e..de858939c5af4 100644
--- a/llvm/test/MC/ARM/directive-eabi_attribute.s
+++ b/llvm/test/MC/ARM/directive-eabi_attribute.s
@@ -1,6 +1,6 @@
 @ RUN: llvm-mc -triple armv7-elf -filetype asm -o - %s | FileCheck %s
 @ RUN: llvm-mc < %s -triple armv7-unknown-linux-gnueabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 
         .syntax unified
         .thumb
diff --git a/llvm/test/MC/ARM/directive-even.s b/llvm/test/MC/ARM/directive-even.s
index e8152e5d20d47..2df0a4cc22497 100644
--- a/llvm/test/MC/ARM/directive-even.s
+++ b/llvm/test/MC/ARM/directive-even.s
@@ -1,4 +1,5 @@
-@ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s | llvm-readobj -S --sd \
+@ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
+@ RUN:   | llvm-readobj -S --sd - \
 @ RUN:   | FileCheck %s
 
 	.syntax unified
diff --git a/llvm/test/MC/ARM/directive-fpu-multiple.s b/llvm/test/MC/ARM/directive-fpu-multiple.s
index 4f474f0a95543..ba407654854cd 100644
--- a/llvm/test/MC/ARM/directive-fpu-multiple.s
+++ b/llvm/test/MC/ARM/directive-fpu-multiple.s
@@ -3,7 +3,8 @@
 @ The later .fpu directive should overwrite the earlier one.
 @ We also check here that all the .fpu directives that we expect to work do work
 
-@ RUN: llvm-mc -triple arm-eabi -filetype obj %s | llvm-readobj --arch-specific \
+@ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
+@ RUN:   | llvm-readobj --arch-specific - \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ATTR
 
 	.fpu none
diff --git a/llvm/test/MC/ARM/directive-fpu.s b/llvm/test/MC/ARM/directive-fpu.s
index f7fea24edcbf9..ba90f1f528619 100644
--- a/llvm/test/MC/ARM/directive-fpu.s
+++ b/llvm/test/MC/ARM/directive-fpu.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc < %s -triple armv7-unknown-linux-gnueabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj -S --sd | FileCheck %s
+@ RUN:   | llvm-readobj -S --sd - | FileCheck %s
 
 @ CHECK: Name: .ARM.attribute
 @ CHECK: SectionData (
diff --git a/llvm/test/MC/ARM/directive-object_arch-2.s b/llvm/test/MC/ARM/directive-object_arch-2.s
index b536f53e7f37b..1fd7bc030ddb7 100644
--- a/llvm/test/MC/ARM/directive-object_arch-2.s
+++ b/llvm/test/MC/ARM/directive-object_arch-2.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s
 
 	.syntax unified
 
diff --git a/llvm/test/MC/ARM/directive-object_arch.s b/llvm/test/MC/ARM/directive-object_arch.s
index 74de1734a40d9..97dbd8d58a786 100644
--- a/llvm/test/MC/ARM/directive-object_arch.s
+++ b/llvm/test/MC/ARM/directive-object_arch.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arch-specific | FileCheck %s
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s
 
 	.syntax unified
 
diff --git a/llvm/test/MC/ARM/directive-tlsdescseq.s b/llvm/test/MC/ARM/directive-tlsdescseq.s
index 649366365b7e4..087bc19b7b6f5 100644
--- a/llvm/test/MC/ARM/directive-tlsdescseq.s
+++ b/llvm/test/MC/ARM/directive-tlsdescseq.s
@@ -1,4 +1,5 @@
-@ RUN: llvm-mc -triple armv7-linux-gnu -filetype obj -o - %s | llvm-readobj -r \
+@ RUN: llvm-mc -triple armv7-linux-gnu -filetype obj -o - %s \
+@ RUN:   | llvm-readobj -r - \
 @ RUN:   | FileCheck %s
 @ RUN: llvm-mc -triple armv7-linux-gnu -filetype asm -o - %s \
 @ RUN:   | FileCheck -check-prefix CHECK-ASM %s
diff --git a/llvm/test/MC/ARM/eh-compact-pr0.s b/llvm/test/MC/ARM/eh-compact-pr0.s
index ac4aa2c2727bc..b96b67d3aaf5c 100644
--- a/llvm/test/MC/ARM/eh-compact-pr0.s
+++ b/llvm/test/MC/ARM/eh-compact-pr0.s
@@ -1,9 +1,9 @@
 @ RUN: llvm-mc %s -triple=armv7-unknown-linux-gnueabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj -S --sd --sr > %t
+@ RUN:   | llvm-readobj -S --sd --sr - > %t
 @ RUN: FileCheck %s < %t
 @ RUN: FileCheck --check-prefixes=RELOC,RELOC-NOAND %s < %t
 @ RUN: llvm-mc %s -triple=armv7-unknown-linux-androideabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj -S --sd --sr | FileCheck --check-prefix=RELOC %s
+@ RUN:   | llvm-readobj -S --sd --sr - | FileCheck --check-prefix=RELOC %s
 
 @ Check the compact pr0 model
 
diff --git a/llvm/test/MC/ARM/eh-compact-pr1.s b/llvm/test/MC/ARM/eh-compact-pr1.s
index d404a4f351b4d..8f3a15c069bcc 100644
--- a/llvm/test/MC/ARM/eh-compact-pr1.s
+++ b/llvm/test/MC/ARM/eh-compact-pr1.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc %s -triple=armv7-unknown-linux-gnueabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj -S --sd --sr | FileCheck %s
+@ RUN:   | llvm-readobj -S --sd --sr - | FileCheck %s
 
 @ Check the compact pr1 model
 
diff --git a/llvm/test/MC/ARM/eh-directive-cantunwind.s b/llvm/test/MC/ARM/eh-directive-cantunwind.s
index 60cbd852f6dfd..86df4b48c414d 100644
--- a/llvm/test/MC/ARM/eh-directive-cantunwind.s
+++ b/llvm/test/MC/ARM/eh-directive-cantunwind.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc %s -triple=armv7-unknown-linux-gnueabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj -S --sd --sr | FileCheck %s
+@ RUN:   | llvm-readobj -S --sd --sr - | FileCheck %s
 
 @ Check the .cantunwind directive
 
diff --git a/llvm/test/MC/ARM/eh-directive-handlerdata.s b/llvm/test/MC/ARM/eh-directive-handlerdata.s
index ce87b4b2d1437..a5f31d4f84f3b 100644
--- a/llvm/test/MC/ARM/eh-directive-handlerdata.s
+++ b/llvm/test/MC/ARM/eh-directive-handlerdata.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc %s -triple=armv7-unknown-linux-gnueabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj -S --sd --sr > %t
+@ RUN:   | llvm-readobj -S --sd --sr - > %t
 @ RUN: FileCheck %s < %t
 @ RUN: FileCheck --check-prefix=RELOC %s < %t
 
diff --git a/llvm/test/MC/ARM/eh-directive-integrated-test.s b/llvm/test/MC/ARM/eh-directive-integrated-test.s
index d69d43a26250f..0560fabce1feb 100644
--- a/llvm/test/MC/ARM/eh-directive-integrated-test.s
+++ b/llvm/test/MC/ARM/eh-directive-integrated-test.s
@@ -20,7 +20,7 @@
 
 
 @ RUN: llvm-mc %s -triple=armv7-unknown-linux-gnueabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj -S --sd | FileCheck %s
+@ RUN:   | llvm-readobj -S --sd - | FileCheck %s
 
 
 @-------------------------------------------------------------------------------
diff --git a/llvm/test/MC/ARM/eh-directive-movsp.s b/llvm/test/MC/ARM/eh-directive-movsp.s
index bbb9e2cfeb2cc..d790c447b0b5f 100644
--- a/llvm/test/MC/ARM/eh-directive-movsp.s
+++ b/llvm/test/MC/ARM/eh-directive-movsp.s
@@ -1,4 +1,5 @@
-@ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s | llvm-readobj -S --sd \
+@ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
+@ RUN:   | llvm-readobj -S --sd - \
 @ RUN:   | FileCheck %s
 
 	.syntax unified
diff --git a/llvm/test/MC/ARM/eh-directive-multiple-offsets.s b/llvm/test/MC/ARM/eh-directive-multiple-offsets.s
index 25c40f9420761..252c731317273 100644
--- a/llvm/test/MC/ARM/eh-directive-multiple-offsets.s
+++ b/llvm/test/MC/ARM/eh-directive-multiple-offsets.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc %s -triple=armv7-unknown-linux-gnueabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj -S --sd | FileCheck %s
+@ RUN:   | llvm-readobj -S --sd - | FileCheck %s
 
 @ Check for different combination of .setfp, .pad, .save and .vsave.
 
diff --git a/llvm/test/MC/ARM/eh-directive-pad.s b/llvm/test/MC/ARM/eh-directive-pad.s
index ef6c34e8d184a..8b400df4b35d6 100644
--- a/llvm/test/MC/ARM/eh-directive-pad.s
+++ b/llvm/test/MC/ARM/eh-directive-pad.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc %s -triple=armv7-unknown-linux-gnueabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj -S --sd | FileCheck %s
+@ RUN:   | llvm-readobj -S --sd - | FileCheck %s
 
 @ Check for different stack pointer offsets.
 
diff --git a/llvm/test/MC/ARM/eh-directive-personality.s b/llvm/test/MC/ARM/eh-directive-personality.s
index 826c10d696a32..9123dc2d14450 100644
--- a/llvm/test/MC/ARM/eh-directive-personality.s
+++ b/llvm/test/MC/ARM/eh-directive-personality.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc %s -triple=armv7-unknown-linux-gnueabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj -S --sd --sr > %t
+@ RUN:   | llvm-readobj -S --sd --sr - > %t
 @ RUN: FileCheck %s < %t
 @ RUN: FileCheck --check-prefix=RELOC %s < %t
 
diff --git a/llvm/test/MC/ARM/eh-directive-personalityindex.s b/llvm/test/MC/ARM/eh-directive-personalityindex.s
index 36c0ac3882f66..03b92a6d82977 100644
--- a/llvm/test/MC/ARM/eh-directive-personalityindex.s
+++ b/llvm/test/MC/ARM/eh-directive-personalityindex.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc -triple armv7-linux-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj -S --sd --sr > %t
+@ RUN:   | llvm-readobj -S --sd --sr - > %t
 @ RUN: FileCheck %s < %t
 @ RUN: FileCheck --check-prefix=RELOC %s < %t
 
diff --git a/llvm/test/MC/ARM/eh-directive-save.s b/llvm/test/MC/ARM/eh-directive-save.s
index 539d73d57a22e..7d32ea3f95e87 100644
--- a/llvm/test/MC/ARM/eh-directive-save.s
+++ b/llvm/test/MC/ARM/eh-directive-save.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc %s -triple=armv7-unknown-linux-gnueabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj -S --sd | FileCheck %s
+@ RUN:   | llvm-readobj -S --sd - | FileCheck %s
 
 @ Check the .save directive
 
diff --git a/llvm/test/MC/ARM/eh-directive-section-comdat.s b/llvm/test/MC/ARM/eh-directive-section-comdat.s
index 10d20f7e0afe0..c41ec8d33b626 100644
--- a/llvm/test/MC/ARM/eh-directive-section-comdat.s
+++ b/llvm/test/MC/ARM/eh-directive-section-comdat.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc %s -triple=armv7-unknown-linux-gnueabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj -S --sd --sr --symbols | FileCheck %s
+@ RUN:   | llvm-readobj -S --sd --sr --symbols - | FileCheck %s
 
 @ Check the .group section for the function in comdat section.
 
diff --git a/llvm/test/MC/ARM/eh-directive-section-multiple-func.s b/llvm/test/MC/ARM/eh-directive-section-multiple-func.s
index 7a281cfb399e4..61629a7858bcd 100644
--- a/llvm/test/MC/ARM/eh-directive-section-multiple-func.s
+++ b/llvm/test/MC/ARM/eh-directive-section-multiple-func.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc %s -triple=armv7-unknown-linux-gnueabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj -S --sd --sr --symbols > %t
+@ RUN:   | llvm-readobj -S --sd --sr --symbols - > %t
 @ RUN: FileCheck %s < %t
 @ RUN: FileCheck --check-prefix=RELOC %s < %t
 
diff --git a/llvm/test/MC/ARM/eh-directive-section.s b/llvm/test/MC/ARM/eh-directive-section.s
index 57816ce28b604..dfdc2147a2f27 100644
--- a/llvm/test/MC/ARM/eh-directive-section.s
+++ b/llvm/test/MC/ARM/eh-directive-section.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc %s -triple=armv7-unknown-linux-gnueabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj -S --sd --sr --symbols > %t
+@ RUN:   | llvm-readobj -S --sd --sr --symbols - > %t
 @ RUN: FileCheck %s < %t
 @ RUN: FileCheck --check-prefix=RELOC %s < %t
 
diff --git a/llvm/test/MC/ARM/eh-directive-setfp.s b/llvm/test/MC/ARM/eh-directive-setfp.s
index 0d9d9e600760d..c24d585474193 100644
--- a/llvm/test/MC/ARM/eh-directive-setfp.s
+++ b/llvm/test/MC/ARM/eh-directive-setfp.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc %s -triple=armv7-unknown-linux-gnueabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj -S --sd | FileCheck %s
+@ RUN:   | llvm-readobj -S --sd - | FileCheck %s
 
 @ Check for .setfp directive.
 
diff --git a/llvm/test/MC/ARM/eh-directive-text-section-multiple-func.s b/llvm/test/MC/ARM/eh-directive-text-section-multiple-func.s
index fe1e6622f07d7..d148f7b61dcaa 100644
--- a/llvm/test/MC/ARM/eh-directive-text-section-multiple-func.s
+++ b/llvm/test/MC/ARM/eh-directive-text-section-multiple-func.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc %s -triple=armv7-unknown-linux-gnueabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj -S --sd --sr -r --symbols | FileCheck %s
+@ RUN:   | llvm-readobj -S --sd --sr -r --symbols - | FileCheck %s
 
 @ Check whether the section is switched back or not.
 
diff --git a/llvm/test/MC/ARM/eh-directive-text-section.s b/llvm/test/MC/ARM/eh-directive-text-section.s
index b665816636534..e1de4e0dabc0d 100644
--- a/llvm/test/MC/ARM/eh-directive-text-section.s
+++ b/llvm/test/MC/ARM/eh-directive-text-section.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc %s -triple=armv7-unknown-linux-gnueabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj -S --sd --sr | FileCheck %s
+@ RUN:   | llvm-readobj -S --sd --sr - | FileCheck %s
 
 @ Check the .fnstart directive and the .fnend directive.
 
diff --git a/llvm/test/MC/ARM/eh-directive-unwind_raw.s b/llvm/test/MC/ARM/eh-directive-unwind_raw.s
index c617aa37c497e..6ec40e94b8f19 100644
--- a/llvm/test/MC/ARM/eh-directive-unwind_raw.s
+++ b/llvm/test/MC/ARM/eh-directive-unwind_raw.s
@@ -1,4 +1,4 @@
-@ RUN: llvm-mc -triple armv7-linux-eabi -filetype obj -o - %s | llvm-readobj -u \
+@ RUN: llvm-mc -triple armv7-linux-eabi -filetype obj -o - %s | llvm-readobj -u - \
 @ RUN:   | FileCheck %s
 
 	.syntax unified
diff --git a/llvm/test/MC/ARM/eh-directive-vsave.s b/llvm/test/MC/ARM/eh-directive-vsave.s
index 1b792650866c5..f4f09f70f4057 100644
--- a/llvm/test/MC/ARM/eh-directive-vsave.s
+++ b/llvm/test/MC/ARM/eh-directive-vsave.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc %s -triple=armv7-unknown-linux-gnueabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj -S --sd --sr | FileCheck %s
+@ RUN:   | llvm-readobj -S --sd --sr - | FileCheck %s
 
 @ Check the .vsave directive
 
diff --git a/llvm/test/MC/ARM/eh-link.s b/llvm/test/MC/ARM/eh-link.s
index bae17d3e3c812..392c12a4ac4db 100644
--- a/llvm/test/MC/ARM/eh-link.s
+++ b/llvm/test/MC/ARM/eh-link.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc %s -triple=armv7-unknown-linux-gnueabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj -S  | FileCheck %s
+@ RUN:   | llvm-readobj -S - | FileCheck %s
 
 @ Test that the ARM_EXIDX sections point (Link) to the corresponding text
 @ sections.
diff --git a/llvm/test/MC/ARM/elf-eflags-eabi.s b/llvm/test/MC/ARM/elf-eflags-eabi.s
index bc4cc3b9942b2..97ab35494c1d6 100644
--- a/llvm/test/MC/ARM/elf-eflags-eabi.s
+++ b/llvm/test/MC/ARM/elf-eflags-eabi.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc %s -triple=armv7-linux-gnueabi -filetype=obj -o - | \
-@ RUN:    llvm-readobj -h | FileCheck -check-prefix=OBJ %s
+@ RUN:    llvm-readobj -h - | FileCheck -check-prefix=OBJ %s
 	.syntax unified
 	.text
 	.globl	barf
diff --git a/llvm/test/MC/ARM/elf-execute-only-section.ll b/llvm/test/MC/ARM/elf-execute-only-section.ll
index 73b088fa41058..fdbcae9544d79 100644
--- a/llvm/test/MC/ARM/elf-execute-only-section.ll
+++ b/llvm/test/MC/ARM/elf-execute-only-section.ll
@@ -1,9 +1,9 @@
 ; RUN: llc < %s -mtriple=thumbv8m.base-eabi -mattr=+execute-only -filetype=obj %s -o - | \
-; RUN: llvm-readelf -S | FileCheck %s
+; RUN: llvm-readelf -S - | FileCheck %s
 ; RUN: llc < %s -mtriple=thumbv8m.main-eabi -mattr=+execute-only -filetype=obj %s -o - | \
-; RUN: llvm-readelf -S | FileCheck %s
+; RUN: llvm-readelf -S - | FileCheck %s
 ; RUN: llc < %s -mtriple=thumbv7m-eabi -mattr=+execute-only -filetype=obj %s -o - | \
-; RUN: llvm-readelf -S | FileCheck %s
+; RUN: llvm-readelf -S - | FileCheck %s
 
 ; CHECK-NOT: {{.text[ ]+PROGBITS[ ]+[0-9]+ [0-9]+ [0-9]+ [0-9]+ AX[^y]}}
 ; CHECK: {{.text[ ]+PROGBITS[ ]+[0-9]+ [0-9]+ [0-9]+ [0-9]+ AXy}}
diff --git a/llvm/test/MC/ARM/elf-reloc-01.s b/llvm/test/MC/ARM/elf-reloc-01.s
index f3019cdff3c6e..0e9195ec4bd6e 100644
--- a/llvm/test/MC/ARM/elf-reloc-01.s
+++ b/llvm/test/MC/ARM/elf-reloc-01.s
@@ -1,7 +1,7 @@
 // RUN: llvm-mc -triple=armv7-linux-gnueabi \
 // RUN:    -mcpu=cortex-a8 -mattr=-neon -mattr=+vfp2  \
 // RUN:    -filetype=obj %s -o - | \
-// RUN:   llvm-readobj -r | FileCheck -check-prefix=OBJ %s
+// RUN:   llvm-readobj -r - | FileCheck -check-prefix=OBJ %s
 
 // Ensure no regression on ARM/gcc compatibility for
 // emitting explicit symbol relocs for nonexternal symbols
diff --git a/llvm/test/MC/ARM/elf-reloc-02.s b/llvm/test/MC/ARM/elf-reloc-02.s
index 24e2bb3b6fdb7..b8a8e3bd44d55 100644
--- a/llvm/test/MC/ARM/elf-reloc-02.s
+++ b/llvm/test/MC/ARM/elf-reloc-02.s
@@ -1,7 +1,7 @@
 // RUN: llvm-mc -triple=armv7-linux-gnueabi \
 // RUN:    -mcpu=cortex-a8 -mattr=-neon -mattr=+vfp2  \
 // RUN:    -filetype=obj %s -o - | \
-// RUN:   llvm-readobj -r | FileCheck -check-prefix=OBJ %s
+// RUN:   llvm-readobj -r - | FileCheck -check-prefix=OBJ %s
 
 // Ensure no regression on ARM/gcc compatibility for
 // emitting explicit symbol relocs for nonexternal symbols
diff --git a/llvm/test/MC/ARM/elf-reloc-03.s b/llvm/test/MC/ARM/elf-reloc-03.s
index e55b1273769c9..5d3947e8a9b04 100644
--- a/llvm/test/MC/ARM/elf-reloc-03.s
+++ b/llvm/test/MC/ARM/elf-reloc-03.s
@@ -1,7 +1,7 @@
 // RUN: llvm-mc -triple=armv7-linux-gnueabi  \
 // RUN:    -mcpu=cortex-a8 -mattr=-neon -mattr=+vfp2   \
 // RUN:    -filetype=obj %s -o - | \
-// RUN:   llvm-readobj -r | FileCheck -check-prefix=OBJ %s
+// RUN:   llvm-readobj -r - | FileCheck -check-prefix=OBJ %s
 
 // Ensure no regression on ARM/gcc compatibility for
 // emitting explicit symbol relocs for nonexternal symbols
diff --git a/llvm/test/MC/ARM/elf-reloc-condcall.s b/llvm/test/MC/ARM/elf-reloc-condcall.s
index c4818b89c23d6..58dc9db241103 100644
--- a/llvm/test/MC/ARM/elf-reloc-condcall.s
+++ b/llvm/test/MC/ARM/elf-reloc-condcall.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -triple=armv7-linux-gnueabi -filetype=obj %s -o - | \
-// RUN:   llvm-readobj -r | FileCheck -check-prefix=OBJ %s
+// RUN:   llvm-readobj -r - | FileCheck -check-prefix=OBJ %s
 
         bleq some_label
         bl some_label
diff --git a/llvm/test/MC/ARM/elf-thumbfunc-reloc.s b/llvm/test/MC/ARM/elf-thumbfunc-reloc.s
index 70aa8c66b96a9..62e084b48fe99 100644
--- a/llvm/test/MC/ARM/elf-thumbfunc-reloc.s
+++ b/llvm/test/MC/ARM/elf-thumbfunc-reloc.s
@@ -1,6 +1,6 @@
 @@ test st_value bit 0 of thumb function
 @ RUN: llvm-mc %s -triple=armv4t-freebsd-eabi -filetype=obj -o - | \
-@ RUN: llvm-readobj -r  | FileCheck %s
+@ RUN: llvm-readobj -r  - | FileCheck %s
 
 
 	.syntax unified
diff --git a/llvm/test/MC/ARM/elf-thumbfunc-reloc2.s b/llvm/test/MC/ARM/elf-thumbfunc-reloc2.s
index cf54b00d48fae..f414a6d10a547 100644
--- a/llvm/test/MC/ARM/elf-thumbfunc-reloc2.s
+++ b/llvm/test/MC/ARM/elf-thumbfunc-reloc2.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc %s -triple=thumbv7-linux-gnueabi \
-// RUN: -filetype=obj -o - | llvm-readobj -S --sd -r --symbols | \
+// RUN: -filetype=obj -o - | llvm-readobj -S --sd -r --symbols - | \
 // RUN: FileCheck %s
 
 // We want to test relocatable thumb function call.
diff --git a/llvm/test/MC/ARM/elf-thumbfunc.s b/llvm/test/MC/ARM/elf-thumbfunc.s
index 6feee8fbb569a..f12b5cb4503cd 100644
--- a/llvm/test/MC/ARM/elf-thumbfunc.s
+++ b/llvm/test/MC/ARM/elf-thumbfunc.s
@@ -1,6 +1,6 @@
 @@ test st_value bit 0 of thumb function
 @ RUN: llvm-mc %s -triple=thumbv7-linux-gnueabi -filetype=obj -o - | \
-@ RUN: llvm-readobj --symbols | FileCheck %s
+@ RUN: llvm-readobj --symbols - | FileCheck %s
 	.syntax unified
 	.text
 	.globl	foo
diff --git a/llvm/test/MC/ARM/inst-directive.s b/llvm/test/MC/ARM/inst-directive.s
index 290bd7d029e41..9d6a1e16884ce 100644
--- a/llvm/test/MC/ARM/inst-directive.s
+++ b/llvm/test/MC/ARM/inst-directive.s
@@ -1,8 +1,8 @@
 @ RUN: llvm-mc %s -triple=armv7-linux-gnueabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj -S --sd | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LE
+@ RUN:   | llvm-readobj -S --sd - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LE
 
 @ RUN: llvm-mc %s -triple=armebv7-linux-gnueabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj -S --sd | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-BE
+@ RUN:   | llvm-readobj -S --sd - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-BE
 
 	.syntax unified
 
diff --git a/llvm/test/MC/ARM/mixed-arm-thumb-bl-fixup.ll b/llvm/test/MC/ARM/mixed-arm-thumb-bl-fixup.ll
index 51a6051e5649e..d109197e98f86 100644
--- a/llvm/test/MC/ARM/mixed-arm-thumb-bl-fixup.ll
+++ b/llvm/test/MC/ARM/mixed-arm-thumb-bl-fixup.ll
@@ -1,15 +1,15 @@
 ; RUN: llc -O0 < %s -mtriple armv7-linux-gnueabi -o - \
 ; RUN:   | llvm-mc -triple armv7-linux-gnueabi -filetype=obj -o - \
-; RUN:    | llvm-readobj -r | FileCheck --check-prefix LINUX %s
+; RUN:    | llvm-readobj -r - | FileCheck --check-prefix LINUX %s
 
 ; RUN: llc -O0 < %s -mtriple armv7-linux-android -o - \
 ; RUN:   | llvm-mc -triple armv7-linux-android -filetype=obj -o - \
-; RUN:    | llvm-readobj -r | FileCheck --check-prefix LINUX %s
+; RUN:    | llvm-readobj -r - | FileCheck --check-prefix LINUX %s
 
 
 ; RUN: llc -O0 < %s -mtriple armv7-apple-ios -o - \
 ; RUN:   | llvm-mc -triple armv7-apple-ios -filetype=obj -o - \
-; RUN:    | llvm-readobj -r | FileCheck --check-prefix IOS %s
+; RUN:    | llvm-readobj -r - | FileCheck --check-prefix IOS %s
 
 
 define void @thumb_caller() #0 {
diff --git a/llvm/test/MC/ARM/target-expressions.s b/llvm/test/MC/ARM/target-expressions.s
index c87cd22b2bad1..ec431201d6f2d 100644
--- a/llvm/test/MC/ARM/target-expressions.s
+++ b/llvm/test/MC/ARM/target-expressions.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc -triple armv7-eabi -filetype asm -o - %s | FileCheck %s
-@ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s | llvm-readobj -r \
+@ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s | llvm-readobj -r - \
 @ RUN:   | FileCheck -check-prefix CHECK-RELOCATIONS %s
 
 	.syntax unified
diff --git a/llvm/test/MC/ARM/thumb-far-jump.s b/llvm/test/MC/ARM/thumb-far-jump.s
index 2fd2c567d1335..1131d08c9f037 100644
--- a/llvm/test/MC/ARM/thumb-far-jump.s
+++ b/llvm/test/MC/ARM/thumb-far-jump.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc < %s -triple thumbv5-linux-gnueabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj -r | FileCheck %s
+@ RUN:   | llvm-readobj -r - | FileCheck %s
 	.syntax	unified
 
 	.text
diff --git a/llvm/test/MC/ARM/thumb-movwt-reloc.s b/llvm/test/MC/ARM/thumb-movwt-reloc.s
index c00cf1c1c9e08..0ca1d85ed574b 100644
--- a/llvm/test/MC/ARM/thumb-movwt-reloc.s
+++ b/llvm/test/MC/ARM/thumb-movwt-reloc.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc -triple thumbv8m.base-eabi -filetype asm -o - %s | FileCheck %s
-@ RUN: llvm-mc -triple thumbv8m.base-eabi -filetype obj -o - %s | llvm-readobj -r \
+@ RUN: llvm-mc -triple thumbv8m.base-eabi -filetype obj -o - %s | llvm-readobj -r - \
 @ RUN:   | FileCheck -check-prefix CHECK-RELOCATIONS %s
 
 	.syntax unified
diff --git a/llvm/test/MC/ARM/thumb-st_other.s b/llvm/test/MC/ARM/thumb-st_other.s
index a9733ba7c0281..f3df31a86aa69 100644
--- a/llvm/test/MC/ARM/thumb-st_other.s
+++ b/llvm/test/MC/ARM/thumb-st_other.s
@@ -4,7 +4,7 @@
 @ for st_other should always be 0.
 
 @ RUN: llvm-mc < %s -triple thumbv5-linux-gnueabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj --symbols | FileCheck %s
+@ RUN:   | llvm-readobj --symbols - | FileCheck %s
 
 	.syntax	unified
 	.text
diff --git a/llvm/test/MC/ARM/thumb-types.s b/llvm/test/MC/ARM/thumb-types.s
index bc915b4954c96..7d17017317d00 100644
--- a/llvm/test/MC/ARM/thumb-types.s
+++ b/llvm/test/MC/ARM/thumb-types.s
@@ -1,4 +1,4 @@
-@ RUN: llvm-mc -triple armv7-elf -filetype obj -o - %s | llvm-readobj --symbols \
+@ RUN: llvm-mc -triple armv7-elf -filetype obj -o - %s | llvm-readobj --symbols - \
 @ RUN:    | FileCheck %s
 
 	.syntax unified
diff --git a/llvm/test/MC/ARM/thumb2-beq-fixup.s b/llvm/test/MC/ARM/thumb2-beq-fixup.s
index 36728081162fc..f89e971a9c0e9 100644
--- a/llvm/test/MC/ARM/thumb2-beq-fixup.s
+++ b/llvm/test/MC/ARM/thumb2-beq-fixup.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc < %s -triple armv7-linux-gnueabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj -r | FileCheck %s
+@ RUN:   | llvm-readobj -r - | FileCheck %s
 
   .code  16
   .thumb_func
diff --git a/llvm/test/MC/ARM/thumb2-diagnostics.s b/llvm/test/MC/ARM/thumb2-diagnostics.s
index ccc9d7b9df6ac..1154e5a9069a1 100644
--- a/llvm/test/MC/ARM/thumb2-diagnostics.s
+++ b/llvm/test/MC/ARM/thumb2-diagnostics.s
@@ -165,3 +165,25 @@ foo2:
 @ CHECK-ERRORS-V7: error: operand must be a register in range [r0, r12] or r14
 @ CHECK-ERRORS-V7: operand must be a register in range [r0, r12] or r14
 @ CHECK-ERRORS-V7: operand must be a register in range [r0, r12] or r14
+
+        tbb [r0, sp]
+        @ v8 allows rm = sp
+@ CHECK-ERRORS-V7: error: instruction variant requires ARMv8 or later
+        tbb [r0, pc]
+        @ rm = pc is always unpredictable
+@ CHECK-ERRORS: error: invalid operand for instruction
+        tbb [sp, r0]
+        @ v8 allows rn = sp
+@ CHECK-ERRORS-V7: error: instruction variant requires ARMv8 or later
+        @ rn = pc is allowed so not included here
+
+        tbh [r0, sp, lsl #1]
+        @ v8 allows rm = sp
+@ CHECK-ERRORS-V7: error: instruction variant requires ARMv8 or later
+        tbh [r0, pc, lsl #1]
+        @ rm = pc is always unpredictable
+@ CHECK-ERRORS: error: invalid operand for instruction
+        tbh [sp, r0, lsl #1]
+        @ v8 allows rn = sp
+@ CHECK-ERRORS-V7: error: instruction variant requires ARMv8 or later
+        @ rn=pc is allowed so not included here
diff --git a/llvm/test/MC/ARM/thumb_set.s b/llvm/test/MC/ARM/thumb_set.s
index f3ee0f4a96204..1404bfd683f74 100644
--- a/llvm/test/MC/ARM/thumb_set.s
+++ b/llvm/test/MC/ARM/thumb_set.s
@@ -1,4 +1,4 @@
-@ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s | llvm-readobj --symbols \
+@ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s | llvm-readobj --symbols - \
 @ RUN:   | FileCheck %s
 
 @ RUN: llvm-mc -triple armv7-eabi -filetype asm -o - %s \
diff --git a/llvm/test/MC/AsmParser/directive_end-2.s b/llvm/test/MC/AsmParser/directive_end-2.s
index 55deaa8ea4177..b326f8e3fe47a 100644
--- a/llvm/test/MC/AsmParser/directive_end-2.s
+++ b/llvm/test/MC/AsmParser/directive_end-2.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -triple i386-unknown-unknown %s -I %p -filetype obj -o - \
-# RUN:   | llvm-readobj --symbols | FileCheck %s
+# RUN:   | llvm-readobj --symbols - | FileCheck %s
 
 rock:
     movl $42, %eax
diff --git a/llvm/test/MC/AsmParser/directive_end.s b/llvm/test/MC/AsmParser/directive_end.s
index f94317191b12c..d26056c1eb149 100644
--- a/llvm/test/MC/AsmParser/directive_end.s
+++ b/llvm/test/MC/AsmParser/directive_end.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -triple i386-unknown-unknown %s -filetype obj -o - \
-# RUN:   | llvm-readobj --symbols | FileCheck %s
+# RUN:   | llvm-readobj --symbols - | FileCheck %s
 
 	.end
 
diff --git a/llvm/test/MC/AsmParser/section.s b/llvm/test/MC/AsmParser/section.s
index e46acee0f0a58..14bb354ec7782 100644
--- a/llvm/test/MC/AsmParser/section.s
+++ b/llvm/test/MC/AsmParser/section.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -triple i386-pc-linux-gnu -filetype=obj -o %t %s
-# RUN: llvm-readobj -S --sd < %t | FileCheck %s
+# RUN: llvm-readobj -S --sd < %t - | FileCheck %s
 .section test1
 .byte 1
 .section test2
diff --git a/llvm/test/MC/AsmParser/section_names.s b/llvm/test/MC/AsmParser/section_names.s
index 8d16c4fb307f3..97cc8fc007267 100644
--- a/llvm/test/MC/AsmParser/section_names.s
+++ b/llvm/test/MC/AsmParser/section_names.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -triple i386-pc-linux-gnu -filetype=obj -o %t %s
-# RUN: llvm-readobj -S < %t | FileCheck %s
+# RUN: llvm-readobj -S - < %t | FileCheck %s
 .section .nobits
 .byte 1
 .section .nobits2
diff --git a/llvm/test/MC/COFF/addrsig.s b/llvm/test/MC/COFF/addrsig.s
index 139f40a37546a..7d0f354e7cdfa 100644
--- a/llvm/test/MC/COFF/addrsig.s
+++ b/llvm/test/MC/COFF/addrsig.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32 %s -o - | llvm-readobj -S --symbols --sd --addrsig | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32 %s -o - | llvm-readobj -S --symbols --sd --addrsig - | FileCheck %s
 
 // CHECK:      Name: .llvm_addrsig
 // CHECK-NEXT: VirtualSize: 0x0
diff --git a/llvm/test/MC/COFF/alias.s b/llvm/test/MC/COFF/alias.s
index 0c225275f6b7b..b6828c658132b 100644
--- a/llvm/test/MC/COFF/alias.s
+++ b/llvm/test/MC/COFF/alias.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -filetype=obj -triple i686-pc-win32 %s -o - \
-// RUN:   | llvm-readobj --symbols -r | FileCheck %s
+// RUN:   | llvm-readobj --symbols -r - | FileCheck %s
 
 local1:
 external_aliased_to_local = local1
diff --git a/llvm/test/MC/COFF/align-nops.s b/llvm/test/MC/COFF/align-nops.s
index e961039749228..d6906d07148b6 100644
--- a/llvm/test/MC/COFF/align-nops.s
+++ b/llvm/test/MC/COFF/align-nops.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple i686-pc-win32 -mcpu=pentiumpro %s | llvm-readobj -S --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple i686-pc-win32 -mcpu=pentiumpro %s | llvm-readobj -S --sd - | FileCheck %s
 
 // Test that we get optimal nops in text
     .text
diff --git a/llvm/test/MC/COFF/basic-coff-64.s b/llvm/test/MC/COFF/basic-coff-64.s
index 5c44b605fe3af..085dfafe4224f 100644
--- a/llvm/test/MC/COFF/basic-coff-64.s
+++ b/llvm/test/MC/COFF/basic-coff-64.s
@@ -1,7 +1,7 @@
 // This test checks that the COFF object emitter works for the most basic
 // programs.
 
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32 %s | llvm-readobj -h -S --sr --sd --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32 %s | llvm-readobj -h -S --sr --sd --symbols - | FileCheck %s
 
 .def	 _main;
 	.scl	2;
diff --git a/llvm/test/MC/COFF/basic-coff.s b/llvm/test/MC/COFF/basic-coff.s
index 265e3f3ac1191..23d74f852db0b 100644
--- a/llvm/test/MC/COFF/basic-coff.s
+++ b/llvm/test/MC/COFF/basic-coff.s
@@ -1,7 +1,7 @@
 // This test checks that the COFF object emitter works for the most basic
 // programs.
 
-// RUN: llvm-mc -filetype=obj -triple i686-pc-win32 %s | llvm-readobj -h -S --sr --sd --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple i686-pc-win32 %s | llvm-readobj -h -S --sr --sd --symbols - | FileCheck %s
 
 .def	 _main;
 	.scl	2;
diff --git a/llvm/test/MC/COFF/bss.s b/llvm/test/MC/COFF/bss.s
index d707703a4c42c..ea61144f8a848 100644
--- a/llvm/test/MC/COFF/bss.s
+++ b/llvm/test/MC/COFF/bss.s
@@ -1,7 +1,7 @@
 // The purpose of this test is to verify that bss sections are emitted correctly.
 
-// RUN: llvm-mc -filetype=obj -triple i686-pc-win32 %s | llvm-readobj -S | FileCheck %s
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32 %s | llvm-readobj -S | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple i686-pc-win32 %s | llvm-readobj -S - | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32 %s | llvm-readobj -S - | FileCheck %s
 
     .bss
     .globl _g0
diff --git a/llvm/test/MC/COFF/comm-align.s b/llvm/test/MC/COFF/comm-align.s
index 048d92e614f6a..441bf17769ea1 100644
--- a/llvm/test/MC/COFF/comm-align.s
+++ b/llvm/test/MC/COFF/comm-align.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -triple i686-windows-gnu -filetype obj -o - %s \
-# RUN:    | llvm-readobj --coff-directives --symbols | FileCheck %s
+# RUN:    | llvm-readobj --coff-directives --symbols - | FileCheck %s
 
 # NOTE: this test checks multiple things:
 # - that -aligncomm is not emitted for 1-byte alignment
diff --git a/llvm/test/MC/COFF/comm.s b/llvm/test/MC/COFF/comm.s
index 5ea620646b4b9..83b5d2e4854a8 100644
--- a/llvm/test/MC/COFF/comm.s
+++ b/llvm/test/MC/COFF/comm.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple i686-pc-win32 %s | llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple i686-pc-win32 %s | llvm-readobj --symbols - | FileCheck %s
 
 .lcomm _a,4,4
 .comm	_b, 4, 2
diff --git a/llvm/test/MC/COFF/cross-section-relative.s b/llvm/test/MC/COFF/cross-section-relative.s
index c448575891dc0..1d8fb709bdd13 100644
--- a/llvm/test/MC/COFF/cross-section-relative.s
+++ b/llvm/test/MC/COFF/cross-section-relative.s
@@ -1,5 +1,5 @@
 // Verify the .fix data section conveys the right offsets and the right relocations
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32 %s -o - | llvm-readobj -r --expand-relocs -S --section-data | FileCheck %s --check-prefix=READOBJ
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32 %s -o - | llvm-readobj -r --expand-relocs -S --section-data - | FileCheck %s --check-prefix=READOBJ
 
         .text
 	.section	.rdata,"dr"
diff --git a/llvm/test/MC/COFF/cv-def-range-gap.s b/llvm/test/MC/COFF/cv-def-range-gap.s
index 1e3703a02a907..95a73e529ede2 100644
--- a/llvm/test/MC/COFF/cv-def-range-gap.s
+++ b/llvm/test/MC/COFF/cv-def-range-gap.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -triple=x86_64-pc-win32 -filetype=obj < %s | llvm-readobj --codeview | FileCheck %s
+# RUN: llvm-mc -triple=x86_64-pc-win32 -filetype=obj < %s | llvm-readobj --codeview - | FileCheck %s
 
 # This tries to test defrange gap edge cases.
 
diff --git a/llvm/test/MC/COFF/cv-def-range.s b/llvm/test/MC/COFF/cv-def-range.s
index 7ce2e5fb455ff..97304759a719e 100644
--- a/llvm/test/MC/COFF/cv-def-range.s
+++ b/llvm/test/MC/COFF/cv-def-range.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -triple=i686-pc-win32 -filetype=obj < %s | llvm-readobj --codeview --codeview-subsection-bytes | FileCheck %s
+# RUN: llvm-mc -triple=i686-pc-win32 -filetype=obj < %s | llvm-readobj --codeview --codeview-subsection-bytes - | FileCheck %s
 	.text
 	.def	 @feat.00;
 	.scl	3;
diff --git a/llvm/test/MC/COFF/cv-fpo-csrs.s b/llvm/test/MC/COFF/cv-fpo-csrs.s
index 1741143ce14b5..76ab8344e653d 100644
--- a/llvm/test/MC/COFF/cv-fpo-csrs.s
+++ b/llvm/test/MC/COFF/cv-fpo-csrs.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -filetype=asm < %s -triple i686-windows-msvc | FileCheck %s --check-prefix=ASM
-# RUN: llvm-mc -filetype=obj < %s -triple i686-windows-msvc | llvm-readobj --codeview | FileCheck %s --check-prefix=OBJ
+# RUN: llvm-mc -filetype=obj < %s -triple i686-windows-msvc | llvm-readobj --codeview - | FileCheck %s --check-prefix=OBJ
 
 .globl _foo
 _foo:
diff --git a/llvm/test/MC/COFF/cv-fpo-realign.s b/llvm/test/MC/COFF/cv-fpo-realign.s
index df94b133acdb1..8b72364b6b3f1 100644
--- a/llvm/test/MC/COFF/cv-fpo-realign.s
+++ b/llvm/test/MC/COFF/cv-fpo-realign.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -triple=i686-windows-msvc -filetype=obj < %s | llvm-readobj --codeview | FileCheck %s
+# RUN: llvm-mc -triple=i686-windows-msvc -filetype=obj < %s | llvm-readobj --codeview - | FileCheck %s
 
 # Test for .cv_fpo_stackalign. We should generate FPO data that restores CSRs
 # at each instruction, and in the last FrameData we should use the '@'
diff --git a/llvm/test/MC/COFF/cv-fpo-setframe.s b/llvm/test/MC/COFF/cv-fpo-setframe.s
index 217248210ce5e..10ac9d6de1025 100644
--- a/llvm/test/MC/COFF/cv-fpo-setframe.s
+++ b/llvm/test/MC/COFF/cv-fpo-setframe.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -filetype=asm < %s -triple i686-windows-msvc | FileCheck %s --check-prefix=ASM
-# RUN: llvm-mc -filetype=obj < %s -triple i686-windows-msvc | llvm-readobj --codeview | FileCheck %s --check-prefix=OBJ
+# RUN: llvm-mc -filetype=obj < %s -triple i686-windows-msvc | llvm-readobj --codeview - | FileCheck %s --check-prefix=OBJ
 
 .globl _foo
 _foo:
diff --git a/llvm/test/MC/COFF/cv-inline-linetable-infloop.s b/llvm/test/MC/COFF/cv-inline-linetable-infloop.s
index 3f5f1f5389842..3802a0eeda7eb 100644
--- a/llvm/test/MC/COFF/cv-inline-linetable-infloop.s
+++ b/llvm/test/MC/COFF/cv-inline-linetable-infloop.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -triple=x86_64-pc-win32 -filetype=obj < %s | llvm-readobj --codeview | FileCheck %s
+# RUN: llvm-mc -triple=x86_64-pc-win32 -filetype=obj < %s | llvm-readobj --codeview - | FileCheck %s
 
 # CHECK:    InlineSiteSym {
 # CHECK:      BinaryAnnotations [
diff --git a/llvm/test/MC/COFF/cv-inline-linetable-unreachable.s b/llvm/test/MC/COFF/cv-inline-linetable-unreachable.s
index b76028e0abfa2..9aaf1f8847814 100644
--- a/llvm/test/MC/COFF/cv-inline-linetable-unreachable.s
+++ b/llvm/test/MC/COFF/cv-inline-linetable-unreachable.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -triple=i686-pc-win32 -filetype=obj < %s | llvm-readobj --codeview | FileCheck %s
+# RUN: llvm-mc -triple=i686-pc-win32 -filetype=obj < %s | llvm-readobj --codeview - | FileCheck %s
 	.text
 	.def	 @feat.00;
 	.scl	3;
diff --git a/llvm/test/MC/COFF/diff.s b/llvm/test/MC/COFF/diff.s
index 2fb48ccc4ee7a..90466b59d0252 100644
--- a/llvm/test/MC/COFF/diff.s
+++ b/llvm/test/MC/COFF/diff.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple i686-pc-mingw32 %s | llvm-readobj -S --sr --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple i686-pc-mingw32 %s | llvm-readobj -S --sr --sd - | FileCheck %s
 
 // COFF resolves differences between labels in the same section, unless that
 // label is declared with function type.
diff --git a/llvm/test/MC/COFF/directive-section-characteristics.ll b/llvm/test/MC/COFF/directive-section-characteristics.ll
index d47103f22cc39..5892755626bcb 100644
--- a/llvm/test/MC/COFF/directive-section-characteristics.ll
+++ b/llvm/test/MC/COFF/directive-section-characteristics.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple i686-windows -filetype obj -o - %s | llvm-readobj --sections \
+; RUN: llc -mtriple i686-windows -filetype obj -o - %s | llvm-readobj --sections - \
 ; RUN:    | FileCheck %s
 
 define dllexport void @function() {
diff --git a/llvm/test/MC/COFF/early-dce.s b/llvm/test/MC/COFF/early-dce.s
index d19743ebb36e7..8398ee930e76a 100644
--- a/llvm/test/MC/COFF/early-dce.s
+++ b/llvm/test/MC/COFF/early-dce.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -triple i686-windows -g -filetype obj -o - %s \
-# RUN:   | llvm-readobj -S --symbols | FileCheck %s
+# RUN:   | llvm-readobj -S --symbols - | FileCheck %s
 
 	.section .rdata
 
diff --git a/llvm/test/MC/COFF/eh-frame.s b/llvm/test/MC/COFF/eh-frame.s
index e51b4c0441059..e464099bbf7c9 100644
--- a/llvm/test/MC/COFF/eh-frame.s
+++ b/llvm/test/MC/COFF/eh-frame.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple i686-pc-win32 %s -o - | llvm-readobj -S | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple i686-pc-win32 %s -o - | llvm-readobj -S - | FileCheck %s
 
 	.def	 _main;
 	.scl	2;
diff --git a/llvm/test/MC/COFF/feat00.s b/llvm/test/MC/COFF/feat00.s
index 3a9e69ddac9f0..5ed340e5b64c4 100644
--- a/llvm/test/MC/COFF/feat00.s
+++ b/llvm/test/MC/COFF/feat00.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple i686-pc-win32 %s -o - | llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple i686-pc-win32 %s -o - | llvm-readobj --symbols - | FileCheck %s
 
 "@feat.00" = 123
 .globl @feat.00
diff --git a/llvm/test/MC/COFF/file.s b/llvm/test/MC/COFF/file.s
index 96a658c073b00..3bbd2daa4f92d 100644
--- a/llvm/test/MC/COFF/file.s
+++ b/llvm/test/MC/COFF/file.s
@@ -7,7 +7,7 @@
 // RUN:   | FileCheck %s
 
 // RUN: llvm-mc -triple i686-windows -filetype obj %s -o - \
-// RUN:	  | llvm-readobj --symbols | FileCheck %s -check-prefix CHECK-SCN
+// RUN:	  | llvm-readobj --symbols - | FileCheck %s -check-prefix CHECK-SCN
 
 	.file "null-padded.asm"
 // CHECK: (nx 1) {{0x[0-9]+}} .file
diff --git a/llvm/test/MC/COFF/linkonce.s b/llvm/test/MC/COFF/linkonce.s
index 225a9e836ac2a..1fe1d6453be91 100644
--- a/llvm/test/MC/COFF/linkonce.s
+++ b/llvm/test/MC/COFF/linkonce.s
@@ -1,7 +1,7 @@
 // Test section manipulation via .linkonce directive.
 //
-// RUN: llvm-mc -triple i386-pc-win32 -filetype=obj %s | llvm-readobj -S --symbols | FileCheck %s
-// RUN: llvm-mc -triple x86_64-pc-win32 -filetype=obj %s | llvm-readobj -S --symbols | FileCheck %s
+// RUN: llvm-mc -triple i386-pc-win32 -filetype=obj %s | llvm-readobj -S --symbols - | FileCheck %s
+// RUN: llvm-mc -triple x86_64-pc-win32 -filetype=obj %s | llvm-readobj -S --symbols - | FileCheck %s
 
 .section s1
 .linkonce
diff --git a/llvm/test/MC/COFF/module-asm.ll b/llvm/test/MC/COFF/module-asm.ll
index 900694e053259..3033b1b482479 100644
--- a/llvm/test/MC/COFF/module-asm.ll
+++ b/llvm/test/MC/COFF/module-asm.ll
@@ -1,8 +1,8 @@
 ; The purpose of this test is to verify that various module level assembly
 ; constructs work.
 
-; RUN: llc -filetype=obj -mtriple i686-pc-win32 %s -o - | llvm-readobj -S --sd | FileCheck %s
-; RUN: llc -filetype=obj -mtriple x86_64-pc-win32 %s -o - | llvm-readobj -S --sd | FileCheck %s
+; RUN: llc -filetype=obj -mtriple i686-pc-win32 %s -o - | llvm-readobj -S --sd - | FileCheck %s
+; RUN: llc -filetype=obj -mtriple x86_64-pc-win32 %s -o - | llvm-readobj -S --sd - | FileCheck %s
 
 module asm ".text"
 module asm "_foo:"
diff --git a/llvm/test/MC/COFF/offset.s b/llvm/test/MC/COFF/offset.s
index bd5e9309f0a63..a7e6da0203ab9 100644
--- a/llvm/test/MC/COFF/offset.s
+++ b/llvm/test/MC/COFF/offset.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple i686-pc-win32 %s -o - | llvm-readobj --symbols -r | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple i686-pc-win32 %s -o - | llvm-readobj --symbols -r - | FileCheck %s
 
 	.data
 	.globl	test1_foo
diff --git a/llvm/test/MC/COFF/pr23025.s b/llvm/test/MC/COFF/pr23025.s
index f0d007543a476..19ce8a014973f 100644
--- a/llvm/test/MC/COFF/pr23025.s
+++ b/llvm/test/MC/COFF/pr23025.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-windows-msvc < %s | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-windows-msvc < %s | llvm-readobj -r - | FileCheck %s
 
 // CHECK:      Relocations [
 // CHECK-NEXT:   Section {{.*}} .text {
diff --git a/llvm/test/MC/COFF/pr28462.s b/llvm/test/MC/COFF/pr28462.s
index 4289d7b2abb3c..ff0eea31f8ae5 100644
--- a/llvm/test/MC/COFF/pr28462.s
+++ b/llvm/test/MC/COFF/pr28462.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple i686-pc-windows-msvc < %s | llvm-readobj -S --section-data | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple i686-pc-windows-msvc < %s | llvm-readobj -S --section-data - | FileCheck %s
 
 .text
 .set var, 42
diff --git a/llvm/test/MC/COFF/secidx.s b/llvm/test/MC/COFF/secidx.s
index 736cb8c80a89c..588dd261265d1 100644
--- a/llvm/test/MC/COFF/secidx.s
+++ b/llvm/test/MC/COFF/secidx.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple i686-pc-win32 %s | llvm-readobj -S --sr | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple i686-pc-win32 %s | llvm-readobj -S --sr - | FileCheck %s
 
 // check that we produce the correct relocation for .secidx
 
diff --git a/llvm/test/MC/COFF/secrel-variant.s b/llvm/test/MC/COFF/secrel-variant.s
index 1061bd404eaa7..0382018d80762 100644
--- a/llvm/test/MC/COFF/secrel-variant.s
+++ b/llvm/test/MC/COFF/secrel-variant.s
@@ -1,6 +1,6 @@
 // COFF section-relative relocations
 
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32 %s | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32 %s | llvm-readobj -r - | FileCheck %s
 
 .data
 values:
diff --git a/llvm/test/MC/COFF/secrel32.s b/llvm/test/MC/COFF/secrel32.s
index d3d59f750f0c4..ec67d79d27181 100644
--- a/llvm/test/MC/COFF/secrel32.s
+++ b/llvm/test/MC/COFF/secrel32.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple i686-pc-win32 %s | llvm-readobj -S --sr | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple i686-pc-win32 %s | llvm-readobj -S --sr - | FileCheck %s
 
 // check that we produce the correct relocation for .secrel32
 
diff --git a/llvm/test/MC/COFF/section-comdat.s b/llvm/test/MC/COFF/section-comdat.s
index 1f59539a3835f..a7803b1f25b95 100644
--- a/llvm/test/MC/COFF/section-comdat.s
+++ b/llvm/test/MC/COFF/section-comdat.s
@@ -1,5 +1,5 @@
-// RUN: llvm-mc -triple i386-pc-win32 -filetype=obj %s | llvm-readobj -S --symbols | FileCheck %s
-// RUN: llvm-mc -triple x86_64-pc-win32 -filetype=obj %s | llvm-readobj -S --symbols | FileCheck %s
+// RUN: llvm-mc -triple i386-pc-win32 -filetype=obj %s | llvm-readobj -S --symbols - | FileCheck %s
+// RUN: llvm-mc -triple x86_64-pc-win32 -filetype=obj %s | llvm-readobj -S --symbols - | FileCheck %s
 
 .section assocSec, "dr", discard, "assocSym"
 .global assocSym
diff --git a/llvm/test/MC/COFF/section-name-encoding.s b/llvm/test/MC/COFF/section-name-encoding.s
index 09dd02000c3da..2a5240e7ad030 100644
--- a/llvm/test/MC/COFF/section-name-encoding.s
+++ b/llvm/test/MC/COFF/section-name-encoding.s
@@ -5,7 +5,7 @@
 //   (8, 999999]:          base 10 string table index (/9999999)
 //   (999999, 0xFFFFFFFF]: base 64 string table index (//AAAAAA)
 //
-// RUN: llvm-mc -triple x86_64-pc-win32 -filetype=obj %s | llvm-readobj -S | FileCheck %s
+// RUN: llvm-mc -triple x86_64-pc-win32 -filetype=obj %s | llvm-readobj -S - | FileCheck %s
 
 // Raw encoding
 
diff --git a/llvm/test/MC/COFF/section.s b/llvm/test/MC/COFF/section.s
index 5116d179f16d9..05203c883f174 100644
--- a/llvm/test/MC/COFF/section.s
+++ b/llvm/test/MC/COFF/section.s
@@ -1,5 +1,5 @@
-// RUN: llvm-mc -triple i386-pc-win32 -filetype=obj %s | llvm-readobj -S | FileCheck %s
-// RUN: llvm-mc -triple x86_64-pc-win32 -filetype=obj %s | llvm-readobj -S | FileCheck %s
+// RUN: llvm-mc -triple i386-pc-win32 -filetype=obj %s | llvm-readobj -S - | FileCheck %s
+// RUN: llvm-mc -triple x86_64-pc-win32 -filetype=obj %s | llvm-readobj -S - | FileCheck %s
 
 .section .foo$bar; .long 1
 .section .foo@bar; .long 1
diff --git a/llvm/test/MC/COFF/seh-align1.s b/llvm/test/MC/COFF/seh-align1.s
index 09ee7697c4feb..ead6a35024680 100644
--- a/llvm/test/MC/COFF/seh-align1.s
+++ b/llvm/test/MC/COFF/seh-align1.s
@@ -1,6 +1,6 @@
 // This test checks the alignment and padding of the unwind info.
 
-// RUN: llvm-mc -triple x86_64-pc-win32 -filetype=obj %s | llvm-readobj -S --sd --sr -u | FileCheck %s
+// RUN: llvm-mc -triple x86_64-pc-win32 -filetype=obj %s | llvm-readobj -S --sd --sr -u - | FileCheck %s
 
 // CHECK:      Sections [
 // CHECK:        Section {
diff --git a/llvm/test/MC/COFF/seh-align2.s b/llvm/test/MC/COFF/seh-align2.s
index fab1c30d0e57c..31b927380f759 100644
--- a/llvm/test/MC/COFF/seh-align2.s
+++ b/llvm/test/MC/COFF/seh-align2.s
@@ -1,6 +1,6 @@
 // This test checks the alignment and padding of the unwind info.
 
-// RUN: llvm-mc -triple x86_64-pc-win32 -filetype=obj %s | llvm-readobj -S --sd --sr -u | FileCheck %s
+// RUN: llvm-mc -triple x86_64-pc-win32 -filetype=obj %s | llvm-readobj -S --sd --sr -u - | FileCheck %s
 
 // CHECK:      Sections [
 // CHECK:        Section {
diff --git a/llvm/test/MC/COFF/seh-align3.s b/llvm/test/MC/COFF/seh-align3.s
index 7b8331a71d679..cc57fbbd220fc 100644
--- a/llvm/test/MC/COFF/seh-align3.s
+++ b/llvm/test/MC/COFF/seh-align3.s
@@ -1,6 +1,6 @@
 // This test checks the alignment and padding of the unwind info.
 
-// RUN: llvm-mc -triple x86_64-pc-win32 -filetype=obj %s | llvm-readobj -S --sd --sr -u | FileCheck %s
+// RUN: llvm-mc -triple x86_64-pc-win32 -filetype=obj %s | llvm-readobj -S --sd --sr -u - | FileCheck %s
 
 // CHECK:      Sections [
 // CHECK:        Section {
diff --git a/llvm/test/MC/COFF/seh-linkonce.s b/llvm/test/MC/COFF/seh-linkonce.s
index 675b5395b72af..52feca020386e 100644
--- a/llvm/test/MC/COFF/seh-linkonce.s
+++ b/llvm/test/MC/COFF/seh-linkonce.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-pc-win32 -filetype=obj %s | llvm-readobj --sections --section-symbols | FileCheck %s
+// RUN: llvm-mc -triple x86_64-pc-win32 -filetype=obj %s | llvm-readobj --sections --section-symbols - | FileCheck %s
 
         .text
         .def     weak_func;
diff --git a/llvm/test/MC/COFF/seh-section-2.s b/llvm/test/MC/COFF/seh-section-2.s
index b910582964829..390a20b3e4f9c 100644
--- a/llvm/test/MC/COFF/seh-section-2.s
+++ b/llvm/test/MC/COFF/seh-section-2.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32 %s | llvm-readobj --symbols | FileCheck %s
+# RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32 %s | llvm-readobj --symbols - | FileCheck %s
 
 # This assembly should make an object with two .text sections, two .xdata
 # sections, and two .pdata sections.
diff --git a/llvm/test/MC/COFF/seh-section.s b/llvm/test/MC/COFF/seh-section.s
index 7d3609d327a1b..7bdc11e7723a2 100644
--- a/llvm/test/MC/COFF/seh-section.s
+++ b/llvm/test/MC/COFF/seh-section.s
@@ -1,6 +1,6 @@
 // This test ensures functions in custom sections get unwind info emitted in a
 // distinct .xdata section.
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32 %s | llvm-readobj -S --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32 %s | llvm-readobj -S --sd - | FileCheck %s
 
 // CHECK:      Name: .xdata
 // CHECK-NEXT: VirtualSize
diff --git a/llvm/test/MC/COFF/seh.s b/llvm/test/MC/COFF/seh.s
index 11e4c9c362bb3..65bc79a55fd71 100644
--- a/llvm/test/MC/COFF/seh.s
+++ b/llvm/test/MC/COFF/seh.s
@@ -1,6 +1,6 @@
 // This test checks that the SEH directives emit the correct unwind data.
 
-// RUN: llvm-mc -triple x86_64-pc-win32 -filetype=obj %s | llvm-readobj -S -u -r | FileCheck %s
+// RUN: llvm-mc -triple x86_64-pc-win32 -filetype=obj %s | llvm-readobj -S -u -r - | FileCheck %s
 
 // CHECK:      Sections [
 // CHECK:        Section {
diff --git a/llvm/test/MC/COFF/simple-fixups.s b/llvm/test/MC/COFF/simple-fixups.s
index 77dd47b3059ac..a6dcadb7fecfa 100644
--- a/llvm/test/MC/COFF/simple-fixups.s
+++ b/llvm/test/MC/COFF/simple-fixups.s
@@ -2,8 +2,8 @@
 // references to functions.  Failing to do so might cause pointer-to-function
 // equality to fail if /INCREMENTAL links are used.
 
-// RUN: llvm-mc -filetype=obj -incremental-linker-compatible -triple i686-pc-win32 %s | llvm-readobj -S | FileCheck %s
-// RUN: llvm-mc -filetype=obj -incremental-linker-compatible -triple x86_64-pc-win32 %s | llvm-readobj -S | FileCheck %s
+// RUN: llvm-mc -filetype=obj -incremental-linker-compatible -triple i686-pc-win32 %s | llvm-readobj -S - | FileCheck %s
+// RUN: llvm-mc -filetype=obj -incremental-linker-compatible -triple x86_64-pc-win32 %s | llvm-readobj -S - | FileCheck %s
 
 	.def	 _foo;
 	.scl	2;
diff --git a/llvm/test/MC/COFF/symbol-alias.s b/llvm/test/MC/COFF/symbol-alias.s
index a20651b690fd6..888da3f7e1283 100644
--- a/llvm/test/MC/COFF/symbol-alias.s
+++ b/llvm/test/MC/COFF/symbol-alias.s
@@ -2,8 +2,8 @@
 // (@foo:  alias <type> @bar) generate the correct entries in the symbol table.
 // They should be identical except for the name.
 
-// RUN: llvm-mc -filetype=obj -triple i686-pc-win32 %s | llvm-readobj --symbols | FileCheck %s
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32 %s | llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple i686-pc-win32 %s | llvm-readobj --symbols - | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32 %s | llvm-readobj --symbols - | FileCheck %s
 
 	.def	 _foo;
 	.scl	2;
diff --git a/llvm/test/MC/COFF/symbol-fragment-offset-64.s b/llvm/test/MC/COFF/symbol-fragment-offset-64.s
index 932d3c46a96ca..634113b40f979 100644
--- a/llvm/test/MC/COFF/symbol-fragment-offset-64.s
+++ b/llvm/test/MC/COFF/symbol-fragment-offset-64.s
@@ -1,7 +1,7 @@
 // The purpose of this test is to see if the COFF object writer is emitting the
 // proper relocations for multiple pieces of data in a single data fragment.
 
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32 %s | llvm-readobj -h -S --sr --sd --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32 %s | llvm-readobj -h -S --sr --sd --symbols - | FileCheck %s
 
 .def	 _main;
 	.scl	2;
diff --git a/llvm/test/MC/COFF/symbol-fragment-offset.s b/llvm/test/MC/COFF/symbol-fragment-offset.s
index 8066229e1dc3e..cfca876bb7b7b 100644
--- a/llvm/test/MC/COFF/symbol-fragment-offset.s
+++ b/llvm/test/MC/COFF/symbol-fragment-offset.s
@@ -1,7 +1,7 @@
 // The purpose of this test is to see if the COFF object writer is emitting the
 // proper relocations for multiple pieces of data in a single data fragment.
 
-// RUN: llvm-mc -filetype=obj -triple i686-pc-win32 %s | llvm-readobj -h -S --sr --sd --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple i686-pc-win32 %s | llvm-readobj -h -S --sr --sd --symbols - | FileCheck %s
 
 .def	 _main;
 	.scl	2;
diff --git a/llvm/test/MC/COFF/timestamp.s b/llvm/test/MC/COFF/timestamp.s
index 140225acf7e85..b624baa429703 100644
--- a/llvm/test/MC/COFF/timestamp.s
+++ b/llvm/test/MC/COFF/timestamp.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple i686-pc-win32 -incremental-linker-compatible %s -o - | llvm-readobj -h | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple i686-pc-win32 -incremental-linker-compatible %s -o - | llvm-readobj -h - | FileCheck %s
 
 // CHECK: ImageFileHeader {
 // CHECK:   TimeDateStamp:
diff --git a/llvm/test/MC/COFF/tricky-names.ll b/llvm/test/MC/COFF/tricky-names.ll
index 55ba880eeafd0..a320cf6dfce26 100644
--- a/llvm/test/MC/COFF/tricky-names.ll
+++ b/llvm/test/MC/COFF/tricky-names.ll
@@ -4,8 +4,8 @@
 
 ; Check that we can roundtrip these names through our assembler,
 ; in both at&t and intel syntax.
-; RUN: llc -mtriple=i686-pc-win32 %s -o - | llvm-mc -triple i686-pc-win32 -filetype=obj | llvm-readobj --symbols | FileCheck %s --check-prefix=READOBJ
-; RUN: llc -mtriple=i686-pc-win32 -x86-asm-syntax=intel %s -o - | llvm-mc -triple i686-pc-win32 -filetype=obj | llvm-readobj --symbols | FileCheck %s --check-prefix=READOBJ
+; RUN: llc -mtriple=i686-pc-win32 %s -o - | llvm-mc -triple i686-pc-win32 -filetype=obj | llvm-readobj --symbols - | FileCheck %s --check-prefix=READOBJ
+; RUN: llc -mtriple=i686-pc-win32 -x86-asm-syntax=intel %s -o - | llvm-mc -triple i686-pc-win32 -filetype=obj | llvm-readobj --symbols - | FileCheck %s --check-prefix=READOBJ
 
 
 @"\01??__E_Generic_object@?$_Error_objects@H@std@@YAXXZ" = global i32 0
diff --git a/llvm/test/MC/COFF/weak.s b/llvm/test/MC/COFF/weak.s
index 6eefdd8dcd197..5a0c3a32192f0 100644
--- a/llvm/test/MC/COFF/weak.s
+++ b/llvm/test/MC/COFF/weak.s
@@ -1,8 +1,8 @@
 // This tests that default-null weak symbols (a GNU extension) are created
 // properly via the .weak directive.
 
-// RUN: llvm-mc -filetype=obj -triple i686-pc-win32 %s | llvm-readobj --symbols | FileCheck %s
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32 %s | llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple i686-pc-win32 %s | llvm-readobj --symbols - | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32 %s | llvm-readobj --symbols - | FileCheck %s
 
     .def    _main;
     .scl    2;
diff --git a/llvm/test/MC/Disassembler/AMDGPU/literal16_vi.txt b/llvm/test/MC/Disassembler/AMDGPU/literal16_vi.txt
index a3cdae33a4cc9..466932e8691e0 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/literal16_vi.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/literal16_vi.txt
@@ -34,16 +34,22 @@
 0xff 0x06 0x02 0x3e 0x00 0x01 0x00 0x00
 
 # non-zero unused bits in constant
-# VI: v_add_f16_e32 v1, 0x10041, v3 ; encoding: [0xff,0x06,0x02,0x3e,0x41,0x00,0x01,0x00]
+# VI: v_add_f16_e32 v1, 0x41, v3 ; encoding: [0xff,0x06,0x02,0x3e,0x41,0x00,0x01,0x00]
 0xff 0x06 0x02 0x3e 0x41 0x00 0x01 0x00
 
-# VI: v_add_f16_e32 v1, 0x1000041, v3 ; encoding: [0xff,0x06,0x02,0x3e,0x41,0x00,0x00,0x01]
+# VI: v_add_f16_e32 v1, 0x41, v3 ; encoding: [0xff,0x06,0x02,0x3e,0x41,0x00,0x00,0x01]
 0xff 0x06 0x02 0x3e 0x41 0x00 0x00 0x01
 
 # FIXME: This should be able to round trip with literal after instruction
 # VI: v_add_f16_e32 v1, 0, v3 ; encoding: [0x80,0x06,0x02,0x3e]
 0xff 0x06 0x02 0x3e 0x00 0x00 0x00 0x00
 
+# VI: v_add_f16_e32 v1, 0xffcd, v3 ; encoding: [0xff,0x06,0x02,0x3e,0xcd,0xff,0xff,0xff]
+0xff 0x06 0x02 0x3e 0xcd 0xff 0xff 0xff
+
+# VI: v_mul_lo_u16_e32 v2, 0xffcd, v2 ; encoding: [0xff,0x04,0x04,0x52,0xcd,0xff,0xff,0xff]
+0xff 0x04 0x04 0x52 0xcd 0xff 0xff 0xff
+
 # VI: v_madmk_f16 v1, v2, 0x41, v3 ; encoding: [0x02,0x07,0x02,0x48,0x41,0x00,0x00,0x00]
 0x02 0x07 0x02 0x48 0x41 0x00 0x00 0x00
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/vop1.txt b/llvm/test/MC/Disassembler/AMDGPU/vop1.txt
index 3e65f6142b752..2d481d245f39a 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/vop1.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/vop1.txt
@@ -243,7 +243,7 @@
 # CHECK: v_cvt_i32_f64_e32 v2, 0x4236b732 ; encoding: [0xff,0x06,0x04,0x7e,0x32,0xb7,0x36,0x42]
 0xff 0x06 0x04 0x7e 0x32 0xb7 0x36 0x42
 
-# CHECK: v_cvt_f16_u16_e32 v123, 0x3ade68b1 ; encoding: [0xff,0x72,0xf6,0x7e,0xb1,0x68,0xde,0x3a]
+# CHECK: v_cvt_f16_u16_e32 v123, 0x68b1 ; encoding: [0xff,0x72,0xf6,0x7e,0xb1,0x68,0xde,0x3a]
 0xff 0x72 0xf6 0x7e 0xb1 0x68 0xde 0x3a
 
 # CHECK: v_cvt_f16_i16_e32 v123, 0x21c2  ; encoding: [0xff,0x74,0xf6,0x7e,0xc2,0x21,0x00,0x00]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/vop3-literal.txt b/llvm/test/MC/Disassembler/AMDGPU/vop3-literal.txt
index 6c4cdea6d9321..d9d8972b88d48 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/vop3-literal.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/vop3-literal.txt
@@ -47,3 +47,12 @@
 
 # GFX10: v_pk_add_u16 v1, 0xffffff9c, v2 ; encoding: [0x01,0x00,0x0a,0xcc,0xff,0x04,0x02,0x18,0x9c,0xff,0xff,0xff]
 0x01,0x00,0x0a,0xcc,0xff,0x04,0x02,0x18,0x9c,0xff,0xff,0xff
+
+# GFX10: v_add_nc_i16 v5, v1, 0xcdab ; encoding: [0x05,0x00,0x0d,0xd7,0x01,0xff,0x01,0x00,0xab,0xcd,0xff,0xff]
+0x05,0x00,0x0d,0xd7,0x01,0xff,0x01,0x00,0xab,0xcd,0xff,0xff
+
+# GFX10: v_ceil_f16_e64 v255, 0xabcd clamp ; encoding: [0xff,0x80,0xdc,0xd5,0xff,0x00,0x00,0x00,0xcd,0xab,0xff,0xff]
+0xff,0x80,0xdc,0xd5,0xff,0x00,0x00,0x00,0xcd,0xab,0xff,0xff
+
+# GFX10: v_min_u16_e64 v5, v1, 0xabcd ; encoding: [0x05,0x00,0x0b,0xd7,0x01,0xff,0x01,0x00,0xcd,0xab,0xff,0xff]
+0x05,0x00,0x0b,0xd7,0x01,0xff,0x01,0x00,0xcd,0xab,0xff,0xff
diff --git a/llvm/test/MC/Disassembler/ARM/thumb2-diagnostic.txt b/llvm/test/MC/Disassembler/ARM/thumb2-diagnostic.txt
new file mode 100644
index 0000000000000..2b49c41cdabc1
--- /dev/null
+++ b/llvm/test/MC/Disassembler/ARM/thumb2-diagnostic.txt
@@ -0,0 +1,49 @@
+# RUN: llvm-mc -triple=thumbv7 -disassemble %s 2>&1 | \
+# RUN: FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-V7
+# RUN: llvm-mc -triple=thumbv8 -disassemble %s 2>&1 | \
+# RUN: FileCheck %s --check-prefix=CHECK
+
+# tbb [r0, sp]
+0xd0 0xe8 0x0d 0xf0
+#      CHECK-V7: warning: potentially undefined instruction encoding
+# CHECK-V7-NEXT: 0xd0 0xe8 0x0d 0xf0
+
+# tbb [r0, pc]
+0xd0 0xe8 0x0f 0xf0
+#      CHECK: warning: potentially undefined instruction encoding
+# CHECK-NEXT: 0xd0 0xe8 0x0f 0xf0
+
+# tbb [sp, r0]
+0xdd 0xe8 0x00 0xf0
+#      CHECK-V7: warning: potentially undefined instruction encoding
+# CHECK-V7-NEXT: 0xdd 0xe8 0x00 0xf0
+
+# tbb [pc, r0]
+0xdf 0xe8 0x00 0xf0
+
+# tbh [r0, sp, lsl #1]
+0xd0 0xe8 0x1d 0xf0
+#      CHECK-V7: warning: potentially undefined instruction encoding
+# CHECK-V7-NEXT: 0xd0 0xe8 0x1d 0xf0
+
+# tbh [r0, pc, lsl #1]
+0xd0 0xe8 0x1f 0xf0
+#      CHECK: warning: potentially undefined instruction encoding
+# CHECK-NEXT: 0xd0 0xe8 0x1f 0xf0
+
+# tbh [sp, r0, lsl #1]
+0xdd 0xe8 0x10 0xf0
+#      CHECK-V7: warning: potentially undefined instruction encoding
+# CHECK-V7-NEXT: 0xdd 0xe8 0x10 0xf0
+
+# tbh [pc, r0, lsl #1]
+0xdf 0xe8 0x10 0xf0
+
+# CHECK: tbb [r0, sp]
+# CHECK: tbb [r0, pc]
+# CHECK: tbb [sp, r0]
+# CHECK: tbb [pc, r0]
+# CHECK: tbh [r0, sp, lsl #1]
+# CHECK: tbh [r0, pc, lsl #1]
+# CHECK: tbh [sp, r0, lsl #1]
+# CHECK: tbh [pc, r0, lsl #1]
diff --git a/llvm/test/MC/Disassembler/PowerPC/ppc64-encoding-ISA31.txt b/llvm/test/MC/Disassembler/PowerPC/ppc64-encoding-ISA31.txt
index 15ea5e9f1138e..038a7f1a84fe2 100644
--- a/llvm/test/MC/Disassembler/PowerPC/ppc64-encoding-ISA31.txt
+++ b/llvm/test/MC/Disassembler/PowerPC/ppc64-encoding-ISA31.txt
@@ -231,6 +231,57 @@
 # CHECK: vsrdbi 2, 3, 4, 5
 0x10 0x43 0x23 0x56
 
+# CHECK: vmulld 1, 2, 3
+0x10 0x22 0x19 0xc9
+
+# CHECK: vmodsw 21, 11, 10
+0x12 0xab 0x57 0x8b
+
+# CHECK: vmoduw 21, 11, 10
+0x12 0xab 0x56 0x8b
+
+# CHECK: vmodsd 21, 11, 10
+0x12 0xab 0x57 0xcb
+
+# CHECK: vmodud 21, 11, 10
+0x12 0xab 0x56 0xcb
+
+# CHECK: vdivsw 21, 11, 10
+0x12 0xab 0x51 0x8b
+
+# CHECK: vdivuw 21, 11, 10
+0x12 0xab 0x50 0x8b
+
+# CHECK: vdivsd 21, 11, 10
+0x12 0xab 0x51 0xcb
+
+# CHECK: vdivud 21, 11, 10
+0x12 0xab 0x50 0xcb
+
+# CHECK: vmulhsw 1, 2, 3
+0x10 0x22 0x1b 0x89
+
+# CHECK: vmulhuw 1, 2, 3
+0x10 0x22 0x1a 0x89
+
+# CHECK: vmulhsd 1, 2, 3
+0x10 0x22 0x1b 0xc9
+
+# CHECK: vmulhud 1, 2, 3
+0x10 0x22 0x1a 0xc9
+
+# CHECK: vdivesw 21, 11, 10
+0x12 0xab 0x53 0x8b
+
+# CHECK: vdiveuw 21, 11, 10
+0x12 0xab 0x52 0x8b
+
+# CHECK: vdivesd 21, 11, 10
+0x12 0xab 0x53 0xcb
+
+# CHECK: vdiveud 21, 11, 10
+0x12 0xab 0x52 0xcb
+
 # CHECK: vinsw 2, 3, 12
 0x10 0x4c 0x18 0xcf
 
diff --git a/llvm/test/MC/ELF/ARM/clang-section.s b/llvm/test/MC/ELF/ARM/clang-section.s
index 12cb57ffcc932..d79aa225d71e4 100644
--- a/llvm/test/MC/ELF/ARM/clang-section.s
+++ b/llvm/test/MC/ELF/ARM/clang-section.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple arm-eabi %s -o - | llvm-readobj -S --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple arm-eabi %s -o - | llvm-readobj -S --symbols - | FileCheck %s
 // Test that global variables and functions are assigned correct section.
 	.text
 	.syntax unified
diff --git a/llvm/test/MC/ELF/ARM/execute-only-populated-text-section.s b/llvm/test/MC/ELF/ARM/execute-only-populated-text-section.s
index 57c80465679fd..ddf647e196389 100644
--- a/llvm/test/MC/ELF/ARM/execute-only-populated-text-section.s
+++ b/llvm/test/MC/ELF/ARM/execute-only-populated-text-section.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -filetype=obj -triple thumbv7m-arm-linux-gnu %s -o - \
-// RUN: | llvm-readobj -S --symbols | FileCheck %s
+// RUN: | llvm-readobj -S --symbols - | FileCheck %s
 
         .text
         bx      lr
diff --git a/llvm/test/MC/ELF/ARM/execute-only-section.s b/llvm/test/MC/ELF/ARM/execute-only-section.s
index 75b7ab8d304fd..12020e030cc04 100644
--- a/llvm/test/MC/ELF/ARM/execute-only-section.s
+++ b/llvm/test/MC/ELF/ARM/execute-only-section.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -filetype=obj -triple thumbv7m-arm-linux-gnu %s -o - \
-// RUN: | llvm-readobj -S --symbols | FileCheck %s
+// RUN: | llvm-readobj -S --symbols - | FileCheck %s
 
         .section        .text,"axy",%progbits,unique,0
         .globl  foo
diff --git a/llvm/test/MC/ELF/ARM/execute-only-text-section-data.s b/llvm/test/MC/ELF/ARM/execute-only-text-section-data.s
index 4845dc2ce1780..c8bd82e7d529e 100644
--- a/llvm/test/MC/ELF/ARM/execute-only-text-section-data.s
+++ b/llvm/test/MC/ELF/ARM/execute-only-text-section-data.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -filetype=obj -triple thumbv7m-arm-linux-gnu %s -o - \
-// RUN: | llvm-readobj -S --symbols | FileCheck %s
+// RUN: | llvm-readobj -S --symbols - | FileCheck %s
 
         .text
         .ascii "test"
diff --git a/llvm/test/MC/ELF/abs.s b/llvm/test/MC/ELF/abs.s
index 259757792aa96..6cb28d56d11d0 100644
--- a/llvm/test/MC/ELF/abs.s
+++ b/llvm/test/MC/ELF/abs.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --symbols - | FileCheck %s
 
 // Test that zed will be an ABS symbol
 
diff --git a/llvm/test/MC/ELF/addrsig.s b/llvm/test/MC/ELF/addrsig.s
index 5d16d5b5a3620..5e5827440c6f8 100644
--- a/llvm/test/MC/ELF/addrsig.s
+++ b/llvm/test/MC/ELF/addrsig.s
@@ -1,6 +1,6 @@
 // RUN: llvm-mc -filetype=asm -triple x86_64-pc-linux-gnu %s -o - | FileCheck --check-prefix=ASM %s
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --symbols --sd --addrsig | FileCheck %s
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -split-dwarf-file %t.dwo -o - | llvm-readobj -S --symbols --sd --addrsig | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --symbols --sd --addrsig - | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -split-dwarf-file %t.dwo -o - | llvm-readobj -S --symbols --sd --addrsig - | FileCheck %s
 // RUN: llvm-readobj -S %t.dwo | FileCheck --check-prefix=DWO %s
 
 // CHECK:        Name: .llvm_addrsig
diff --git a/llvm/test/MC/ELF/alias-reloc.s b/llvm/test/MC/ELF/alias-reloc.s
index 5c78a3b2f180e..aeafe63a7bb3a 100644
--- a/llvm/test/MC/ELF/alias-reloc.s
+++ b/llvm/test/MC/ELF/alias-reloc.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r --symbols - | FileCheck %s
 
 // Test that this produces a R_X86_64_PLT32 with bar.
 
diff --git a/llvm/test/MC/ELF/alias-to-local.s b/llvm/test/MC/ELF/alias-to-local.s
index 4983833b33af2..5b6ac058c1e81 100644
--- a/llvm/test/MC/ELF/alias-to-local.s
+++ b/llvm/test/MC/ELF/alias-to-local.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu < %s | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu < %s | llvm-readobj -r - | FileCheck %s
 
 // CHECK:      Relocations [
 // CHECK-NEXT:   Section {{.*}} .rela.text {
diff --git a/llvm/test/MC/ELF/alias.s b/llvm/test/MC/ELF/alias.s
index c125de3467c09..0aefc522052cb 100644
--- a/llvm/test/MC/ELF/alias.s
+++ b/llvm/test/MC/ELF/alias.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --symbols - | FileCheck %s
 
 foo:
 bar = foo
diff --git a/llvm/test/MC/ELF/align-bss.s b/llvm/test/MC/ELF/align-bss.s
index 1a9191c34efad..9e9cd7d3aaf83 100644
--- a/llvm/test/MC/ELF/align-bss.s
+++ b/llvm/test/MC/ELF/align-bss.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S - | FileCheck %s
 
 // Test that the bss section is correctly aligned
 
diff --git a/llvm/test/MC/ELF/align-nops.s b/llvm/test/MC/ELF/align-nops.s
index 08e8660e4e8a4..41a007111f406 100644
--- a/llvm/test/MC/ELF/align-nops.s
+++ b/llvm/test/MC/ELF/align-nops.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu -mcpu=pentiumpro %s -o - | llvm-readobj -S --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu -mcpu=pentiumpro %s -o - | llvm-readobj -S --sd - | FileCheck %s
 
 // Test that we get optimal nops in text
     .text
diff --git a/llvm/test/MC/ELF/align-size.s b/llvm/test/MC/ELF/align-size.s
index 2f5e74d119235..0e1e9495e65de 100644
--- a/llvm/test/MC/ELF/align-size.s
+++ b/llvm/test/MC/ELF/align-size.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S - | FileCheck %s
 
 // Test that the alignment does contribute to the size of the section.
 
diff --git a/llvm/test/MC/ELF/align-text.s b/llvm/test/MC/ELF/align-text.s
index 278102e26e3c4..893f782ceed7b 100644
--- a/llvm/test/MC/ELF/align-text.s
+++ b/llvm/test/MC/ELF/align-text.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S - | FileCheck %s
 
 // Test that the .text directive doesn't cause alignment.
 
diff --git a/llvm/test/MC/ELF/align.s b/llvm/test/MC/ELF/align.s
index 27c97980e9d48..d97ccb3ee9213 100644
--- a/llvm/test/MC/ELF/align.s
+++ b/llvm/test/MC/ELF/align.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S - | FileCheck %s
 
 // Test that the alignment of rodata doesn't force a alignment of the
 // previous section (.text)
diff --git a/llvm/test/MC/ELF/basic-elf-32.s b/llvm/test/MC/ELF/basic-elf-32.s
index f2303e183da60..effc363d1c6c9 100644
--- a/llvm/test/MC/ELF/basic-elf-32.s
+++ b/llvm/test/MC/ELF/basic-elf-32.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu %s -o - | llvm-readobj -h -S -r --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu %s -o - | llvm-readobj -h -S -r --symbols - | FileCheck %s
 
 	.text
 	.globl	main
diff --git a/llvm/test/MC/ELF/basic-elf-64.s b/llvm/test/MC/ELF/basic-elf-64.s
index 1bc8c5ab6a239..219f14718224a 100644
--- a/llvm/test/MC/ELF/basic-elf-64.s
+++ b/llvm/test/MC/ELF/basic-elf-64.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -h -S -r --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -h -S -r --symbols - | FileCheck %s
 
         .text
 	.globl	main
diff --git a/llvm/test/MC/ELF/call-abs.s b/llvm/test/MC/ELF/call-abs.s
index 145b9a7da2f55..bdb3ba050c0c8 100644
--- a/llvm/test/MC/ELF/call-abs.s
+++ b/llvm/test/MC/ELF/call-abs.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu %s -o - | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu %s -o - | llvm-readobj -r - | FileCheck %s
 
 	.text
 	.globl	f
diff --git a/llvm/test/MC/ELF/cfi-adjust-cfa-offset.s b/llvm/test/MC/ELF/cfi-adjust-cfa-offset.s
index e935dcbefc49a..2b41b2f6a4ff4 100644
--- a/llvm/test/MC/ELF/cfi-adjust-cfa-offset.s
+++ b/llvm/test/MC/ELF/cfi-adjust-cfa-offset.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd - | FileCheck %s
 
 f:
 	.cfi_startproc
diff --git a/llvm/test/MC/ELF/cfi-advance-loc2.s b/llvm/test/MC/ELF/cfi-advance-loc2.s
index aa5f9ddcbf40d..9226200e63a97 100644
--- a/llvm/test/MC/ELF/cfi-advance-loc2.s
+++ b/llvm/test/MC/ELF/cfi-advance-loc2.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd - | FileCheck %s
 
 // test that this produces a correctly encoded cfi_advance_loc2
 
diff --git a/llvm/test/MC/ELF/cfi-def-cfa-offset.s b/llvm/test/MC/ELF/cfi-def-cfa-offset.s
index 5b71c64555a6a..2d788c18a3951 100644
--- a/llvm/test/MC/ELF/cfi-def-cfa-offset.s
+++ b/llvm/test/MC/ELF/cfi-def-cfa-offset.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd - | FileCheck %s
 
 f:
 	.cfi_startproc
diff --git a/llvm/test/MC/ELF/cfi-def-cfa-register.s b/llvm/test/MC/ELF/cfi-def-cfa-register.s
index 220e3cba08f3a..957b56c35cdd7 100644
--- a/llvm/test/MC/ELF/cfi-def-cfa-register.s
+++ b/llvm/test/MC/ELF/cfi-def-cfa-register.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd - | FileCheck %s
 
 f:
 	.cfi_startproc
diff --git a/llvm/test/MC/ELF/cfi-def-cfa.s b/llvm/test/MC/ELF/cfi-def-cfa.s
index 67aa3a65fefdc..6003327da1704 100644
--- a/llvm/test/MC/ELF/cfi-def-cfa.s
+++ b/llvm/test/MC/ELF/cfi-def-cfa.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd - | FileCheck %s
 
 f:
 	.cfi_startproc
diff --git a/llvm/test/MC/ELF/cfi-escape.s b/llvm/test/MC/ELF/cfi-escape.s
index 620d153871640..92229a7a58173 100644
--- a/llvm/test/MC/ELF/cfi-escape.s
+++ b/llvm/test/MC/ELF/cfi-escape.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd - | FileCheck %s
 
 f:
 	.cfi_startproc
diff --git a/llvm/test/MC/ELF/cfi-large-model.s b/llvm/test/MC/ELF/cfi-large-model.s
index 444f38af029b6..cee8d14d6b001 100644
--- a/llvm/test/MC/ELF/cfi-large-model.s
+++ b/llvm/test/MC/ELF/cfi-large-model.s
@@ -1,9 +1,9 @@
 // REQUIRES: powerpc-registered-target
 // REQUIRES: x86-registered-target
 // RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu -large-code-model %s \
-// RUN:   -o - | llvm-readobj -S --sd | FileCheck --check-prefix=CHECK-X86 %s
+// RUN:   -o - | llvm-readobj -S --sd - | FileCheck --check-prefix=CHECK-X86 %s
 // RUN: llvm-mc -filetype=obj -triple powerpc64le-linux-gnu -large-code-model %s \
-// RUN:   -o - | llvm-readobj -S --sd | FileCheck --check-prefix=CHECK-PPC %s
+// RUN:   -o - | llvm-readobj -S --sd - | FileCheck --check-prefix=CHECK-PPC %s
 
 
 // CHECK-X86:      Section {
diff --git a/llvm/test/MC/ELF/cfi-offset.s b/llvm/test/MC/ELF/cfi-offset.s
index be571b4a49b99..efb8dbe84df8a 100644
--- a/llvm/test/MC/ELF/cfi-offset.s
+++ b/llvm/test/MC/ELF/cfi-offset.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd - | FileCheck %s
 
 f:
 	.cfi_startproc
diff --git a/llvm/test/MC/ELF/cfi-register.s b/llvm/test/MC/ELF/cfi-register.s
index 89400cd9a78b6..34aaf4a379863 100644
--- a/llvm/test/MC/ELF/cfi-register.s
+++ b/llvm/test/MC/ELF/cfi-register.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd - | FileCheck %s
 
 f:
 	.cfi_startproc
diff --git a/llvm/test/MC/ELF/cfi-rel-offset.s b/llvm/test/MC/ELF/cfi-rel-offset.s
index 1c877b3e29315..649f8e21b9784 100644
--- a/llvm/test/MC/ELF/cfi-rel-offset.s
+++ b/llvm/test/MC/ELF/cfi-rel-offset.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd - | FileCheck %s
 
 f:
 	.cfi_startproc
diff --git a/llvm/test/MC/ELF/cfi-rel-offset2.s b/llvm/test/MC/ELF/cfi-rel-offset2.s
index 34a667c3091ae..d5909112599f0 100644
--- a/llvm/test/MC/ELF/cfi-rel-offset2.s
+++ b/llvm/test/MC/ELF/cfi-rel-offset2.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd - | FileCheck %s
 
 f:
 	.cfi_startproc
diff --git a/llvm/test/MC/ELF/cfi-remember.s b/llvm/test/MC/ELF/cfi-remember.s
index d1629d1a50829..ae842edc76604 100644
--- a/llvm/test/MC/ELF/cfi-remember.s
+++ b/llvm/test/MC/ELF/cfi-remember.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd - | FileCheck %s
 
 f:
 	.cfi_startproc
diff --git a/llvm/test/MC/ELF/cfi-restore.s b/llvm/test/MC/ELF/cfi-restore.s
index 79be928e72ad1..bbf339f7fdd9f 100644
--- a/llvm/test/MC/ELF/cfi-restore.s
+++ b/llvm/test/MC/ELF/cfi-restore.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd - | FileCheck %s
 
 f:
 	.cfi_startproc
diff --git a/llvm/test/MC/ELF/cfi-same-value.s b/llvm/test/MC/ELF/cfi-same-value.s
index dc343999944ae..8e3abc9a04095 100644
--- a/llvm/test/MC/ELF/cfi-same-value.s
+++ b/llvm/test/MC/ELF/cfi-same-value.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd - | FileCheck %s
 
 f:
 	.cfi_startproc
diff --git a/llvm/test/MC/ELF/cfi-sections.s b/llvm/test/MC/ELF/cfi-sections.s
index ea22eb6e1d1a6..52f01ec933512 100644
--- a/llvm/test/MC/ELF/cfi-sections.s
+++ b/llvm/test/MC/ELF/cfi-sections.s
@@ -1,11 +1,11 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu -dwarf-version 2 %s -o - | llvm-readobj -S --sd | FileCheck -check-prefix=ELF_64 -check-prefix=ELF_64_DWARF_2 %s
-// RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu -dwarf-version 2 %s -o - | llvm-readobj -S --sd | FileCheck -check-prefix=ELF_32 -check-prefix=ELF_32_DWARF_2 %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu -dwarf-version 2 %s -o - | llvm-readobj -S --sd - | FileCheck -check-prefix=ELF_64 -check-prefix=ELF_64_DWARF_2 %s
+// RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu -dwarf-version 2 %s -o - | llvm-readobj -S --sd - | FileCheck -check-prefix=ELF_32 -check-prefix=ELF_32_DWARF_2 %s
 
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu -dwarf-version 3 %s -o - | llvm-readobj -S --sd | FileCheck -check-prefix=ELF_64 -check-prefix=ELF_64_DWARF_3 %s
-// RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu -dwarf-version 3 %s -o - | llvm-readobj -S --sd | FileCheck -check-prefix=ELF_32 -check-prefix=ELF_32_DWARF_3 %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu -dwarf-version 3 %s -o - | llvm-readobj -S --sd - | FileCheck -check-prefix=ELF_64 -check-prefix=ELF_64_DWARF_3 %s
+// RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu -dwarf-version 3 %s -o - | llvm-readobj -S --sd - | FileCheck -check-prefix=ELF_32 -check-prefix=ELF_32_DWARF_3 %s
 
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu -dwarf-version 4 %s -o - | llvm-readobj -S --sd | FileCheck -check-prefix=ELF_64 -check-prefix=ELF_64_DWARF_4 %s
-// RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu -dwarf-version 4 %s -o - | llvm-readobj -S --sd | FileCheck -check-prefix=ELF_32 -check-prefix=ELF_32_DWARF_4 %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu -dwarf-version 4 %s -o - | llvm-readobj -S --sd - | FileCheck -check-prefix=ELF_64 -check-prefix=ELF_64_DWARF_4 %s
+// RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu -dwarf-version 4 %s -o - | llvm-readobj -S --sd - | FileCheck -check-prefix=ELF_32 -check-prefix=ELF_32_DWARF_4 %s
 
 .cfi_sections .debug_frame
 
diff --git a/llvm/test/MC/ELF/cfi-signal-frame.s b/llvm/test/MC/ELF/cfi-signal-frame.s
index 17d3f6afbf020..9844f0b1b7f24 100644
--- a/llvm/test/MC/ELF/cfi-signal-frame.s
+++ b/llvm/test/MC/ELF/cfi-signal-frame.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sd - | FileCheck %s
 
 f:
         .cfi_startproc
diff --git a/llvm/test/MC/ELF/cfi-undefined.s b/llvm/test/MC/ELF/cfi-undefined.s
index dc615bd02bb4f..ebeb16afbd377 100644
--- a/llvm/test/MC/ELF/cfi-undefined.s
+++ b/llvm/test/MC/ELF/cfi-undefined.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd - | FileCheck %s
 
 f:
 	.cfi_startproc
diff --git a/llvm/test/MC/ELF/cfi-window-save.s b/llvm/test/MC/ELF/cfi-window-save.s
index 7eb3ae2f98e54..879c02a3480af 100644
--- a/llvm/test/MC/ELF/cfi-window-save.s
+++ b/llvm/test/MC/ELF/cfi-window-save.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd - | FileCheck %s
 
 # Should use SPARC as the target to test this. However, SPARC does not
 # use MC yet.
diff --git a/llvm/test/MC/ELF/cfi-zero-addr-delta.s b/llvm/test/MC/ELF/cfi-zero-addr-delta.s
index 81d16987d5809..094110515373c 100644
--- a/llvm/test/MC/ELF/cfi-zero-addr-delta.s
+++ b/llvm/test/MC/ELF/cfi-zero-addr-delta.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd - | FileCheck %s
 
 // Test that we don't produce a DW_CFA_advance_loc 0
 
diff --git a/llvm/test/MC/ELF/cfi.s b/llvm/test/MC/ELF/cfi.s
index b11cd8a00b2f0..8f26eaddbbfb3 100644
--- a/llvm/test/MC/ELF/cfi.s
+++ b/llvm/test/MC/ELF/cfi.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd - | FileCheck %s
 
 f1:
         .cfi_startproc
diff --git a/llvm/test/MC/ELF/cgprofile.s b/llvm/test/MC/ELF/cgprofile.s
index ac2a2d2b11170..941bf7731f66b 100644
--- a/llvm/test/MC/ELF/cgprofile.s
+++ b/llvm/test/MC/ELF/cgprofile.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --symbols --sd --cg-profile | FileCheck %s
+# RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --symbols --sd --cg-profile - | FileCheck %s
 
   .section .test,"aw",@progbits
 a: .word b
diff --git a/llvm/test/MC/ELF/comdat-reloc.s b/llvm/test/MC/ELF/comdat-reloc.s
index 7eff043bede92..e4bc84778ecd5 100644
--- a/llvm/test/MC/ELF/comdat-reloc.s
+++ b/llvm/test/MC/ELF/comdat-reloc.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sd - | FileCheck %s
 
   .text
   .globl  hello
diff --git a/llvm/test/MC/ELF/comdat.s b/llvm/test/MC/ELF/comdat.s
index 446f52c54b525..e5862fab9521c 100644
--- a/llvm/test/MC/ELF/comdat.s
+++ b/llvm/test/MC/ELF/comdat.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --symbols --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --symbols --sd - | FileCheck %s
 
 // Test that we produce the group sections and that they are before the members
 
diff --git a/llvm/test/MC/ELF/common.s b/llvm/test/MC/ELF/common.s
index b819f1c692e77..016c1f8ca6f6b 100644
--- a/llvm/test/MC/ELF/common.s
+++ b/llvm/test/MC/ELF/common.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --symbols - | FileCheck %s
 
 
 	.text
diff --git a/llvm/test/MC/ELF/common2.s b/llvm/test/MC/ELF/common2.s
index 54a27b05b2356..ad3998b2cf606 100644
--- a/llvm/test/MC/ELF/common2.s
+++ b/llvm/test/MC/ELF/common2.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S - | FileCheck %s
 
 // Test local common construction.
 // Unlike gas, common symbols are created when found, not at the end of .bss.
diff --git a/llvm/test/MC/ELF/debug-line.s b/llvm/test/MC/ELF/debug-line.s
index 89f07803f4b9a..06cb2dcee2d18 100644
--- a/llvm/test/MC/ELF/debug-line.s
+++ b/llvm/test/MC/ELF/debug-line.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sd - | FileCheck %s
 
 // Test that .debug_line is populated.  TODO: This test should really be using
 // llvm-dwarfdump, but it cannot parse this particular object file.  The content
diff --git a/llvm/test/MC/ELF/debug-line2.s b/llvm/test/MC/ELF/debug-line2.s
index 3c8b73092c521..50fbf4abe6f56 100644
--- a/llvm/test/MC/ELF/debug-line2.s
+++ b/llvm/test/MC/ELF/debug-line2.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sd - | FileCheck %s
 
 // Test that two subsequent .loc directives generate two
 // distinct line table entries.
diff --git a/llvm/test/MC/ELF/debug-loc.s b/llvm/test/MC/ELF/debug-loc.s
index 841c05c2b713f..13cafbce66e7e 100644
--- a/llvm/test/MC/ELF/debug-loc.s
+++ b/llvm/test/MC/ELF/debug-loc.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S - | FileCheck %s
 
 // Test that we don't regress on the size of the line info section. We used
 // to handle negative line diffs incorrectly which manifested as very
diff --git a/llvm/test/MC/ELF/diff.s b/llvm/test/MC/ELF/diff.s
index 5436510669692..aced5ff6da5ce 100644
--- a/llvm/test/MC/ELF/diff.s
+++ b/llvm/test/MC/ELF/diff.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r - | FileCheck %s
 
         .global zed
 foo:
diff --git a/llvm/test/MC/ELF/dot-symbol-assignment.s b/llvm/test/MC/ELF/dot-symbol-assignment.s
index 6f15100ff615a..5b87fe66364b6 100644
--- a/llvm/test/MC/ELF/dot-symbol-assignment.s
+++ b/llvm/test/MC/ELF/dot-symbol-assignment.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --sections --section-data | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --sections --section-data - | FileCheck %s
 
 one:
 	.quad 0xffffffffffffffff
diff --git a/llvm/test/MC/ELF/empty-dwarf-lines.s b/llvm/test/MC/ELF/empty-dwarf-lines.s
index e6ce76280aadf..d694ea9f26b82 100644
--- a/llvm/test/MC/ELF/empty-dwarf-lines.s
+++ b/llvm/test/MC/ELF/empty-dwarf-lines.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S - | FileCheck %s
 
 // Test that the dwarf debug_line section contains no line directives.
 
diff --git a/llvm/test/MC/ELF/empty.s b/llvm/test/MC/ELF/empty.s
index bc5c80a218156..1592f0d7d3cc8 100644
--- a/llvm/test/MC/ELF/empty.s
+++ b/llvm/test/MC/ELF/empty.s
@@ -1,6 +1,6 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S | FileCheck %s
-// RUN: llvm-mc -filetype=obj -triple x86_64-apple-darwin14.0.0-elf %s -o - | llvm-readobj -S | FileCheck %s -check-prefix=DARWIN
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32-elf %s -o - | llvm-readobj -S | FileCheck %s -check-prefix=WINDOWS
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S - | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-apple-darwin14.0.0-elf %s -o - | llvm-readobj -S - | FileCheck %s -check-prefix=DARWIN
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32-elf %s -o - | llvm-readobj -S - | FileCheck %s -check-prefix=WINDOWS
 
 // Check that we can create ELF files for darwin/windows, even though
 // it is not the default file format.
diff --git a/llvm/test/MC/ELF/entsize.ll b/llvm/test/MC/ELF/entsize.ll
index 36f7ab7688d8d..b340fdbcebde7 100644
--- a/llvm/test/MC/ELF/entsize.ll
+++ b/llvm/test/MC/ELF/entsize.ll
@@ -1,4 +1,4 @@
-; RUN: llc -filetype=obj -mtriple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S | FileCheck -check-prefix=64 %s
+; RUN: llc -filetype=obj -mtriple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S - | FileCheck -check-prefix=64 %s
 
 ; Test that constant mergeable strings have sh_entsize set.
 
diff --git a/llvm/test/MC/ELF/entsize.s b/llvm/test/MC/ELF/entsize.s
index 856fdcb339092..a371c3b831227 100644
--- a/llvm/test/MC/ELF/entsize.s
+++ b/llvm/test/MC/ELF/entsize.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S | FileCheck  %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S - | FileCheck  %s
 
 // Test that mergeable constants have sh_entsize set.
 
diff --git a/llvm/test/MC/ELF/file-double.s b/llvm/test/MC/ELF/file-double.s
index 206712f6a15c8..6d3fae27b0b27 100644
--- a/llvm/test/MC/ELF/file-double.s
+++ b/llvm/test/MC/ELF/file-double.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --symbols - | FileCheck %s
 
 // Test that a STT_FILE symbol and a symbol of the same name can coexist.
 
diff --git a/llvm/test/MC/ELF/file.s b/llvm/test/MC/ELF/file.s
index ddb0ed77ff7cd..90b379772f318 100644
--- a/llvm/test/MC/ELF/file.s
+++ b/llvm/test/MC/ELF/file.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --symbols - | FileCheck %s
 
 // Test that the STT_FILE symbol precedes the other local symbols.
 
diff --git a/llvm/test/MC/ELF/gen-dwarf.s b/llvm/test/MC/ELF/gen-dwarf.s
index 2f77c4b73af7e..dc70bf9bc6b0e 100644
--- a/llvm/test/MC/ELF/gen-dwarf.s
+++ b/llvm/test/MC/ELF/gen-dwarf.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -g -dwarf-version 2 -triple  i686-pc-linux-gnu %s -filetype=obj -o - | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -g -dwarf-version 2 -triple  i686-pc-linux-gnu %s -filetype=obj -o - | llvm-readobj -r - | FileCheck %s
 // RUN: not llvm-mc -g -dwarf-version 1  -triple  i686-pc-linux-gnu %s -filetype=asm -o - 2>&1 | FileCheck --check-prefix=DWARF1 %s
 // RUN: llvm-mc -g -dwarf-version 2 -triple  i686-pc-linux-gnu %s -filetype=asm -o - | FileCheck --check-prefix=ASM --check-prefix=DWARF2 %s
 // RUN: llvm-mc -g -dwarf-version 3 -triple  i686-pc-linux-gnu %s -filetype=asm -o - | FileCheck --check-prefix=ASM --check-prefix=DWARF3 %s
diff --git a/llvm/test/MC/ELF/global-offset.s b/llvm/test/MC/ELF/global-offset.s
index 3e5d8ac143187..085e60956b4c5 100644
--- a/llvm/test/MC/ELF/global-offset.s
+++ b/llvm/test/MC/ELF/global-offset.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple i386-pc-linux-gnu %s -o - | llvm-readobj -S --sd | FileCheck  %s
+// RUN: llvm-mc -filetype=obj -triple i386-pc-linux-gnu %s -o - | llvm-readobj -S --sd - | FileCheck  %s
 
 // We test that _GLOBAL_OFFSET_TABLE_ will account for the two bytes at the
 // start of the addl/leal.
diff --git a/llvm/test/MC/ELF/got-relaxed-i386.s b/llvm/test/MC/ELF/got-relaxed-i386.s
index 927066c68f2fe..465b6fa07f6f4 100644
--- a/llvm/test/MC/ELF/got-relaxed-i386.s
+++ b/llvm/test/MC/ELF/got-relaxed-i386.s
@@ -1,5 +1,5 @@
-// RUN: llvm-mc -filetype=obj -triple i386-pc-linux %s -o - | llvm-readobj -r | FileCheck %s
-// RUN: llvm-mc -filetype=obj -relax-relocations=false -triple i386-pc-linux %s -o - | llvm-readobj -r | FileCheck --check-prefix=OLD %s
+// RUN: llvm-mc -filetype=obj -triple i386-pc-linux %s -o - | llvm-readobj -r - | FileCheck %s
+// RUN: llvm-mc -filetype=obj -relax-relocations=false -triple i386-pc-linux %s -o - | llvm-readobj -r - | FileCheck --check-prefix=OLD %s
 
         movl mov@GOT(%ebx), %eax
         mull mul@GOT(%ebx)
diff --git a/llvm/test/MC/ELF/got-relaxed-no-relax.s b/llvm/test/MC/ELF/got-relaxed-no-relax.s
index 69511cd78cd90..382639d10c9ed 100644
--- a/llvm/test/MC/ELF/got-relaxed-no-relax.s
+++ b/llvm/test/MC/ELF/got-relaxed-no-relax.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux %s -o - | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux %s -o - | llvm-readobj -r - | FileCheck %s
 
 // these should not produce relaxable relocations
 
diff --git a/llvm/test/MC/ELF/got-relaxed-rex.s b/llvm/test/MC/ELF/got-relaxed-rex.s
index d35afaab3e52f..e694d7431ba98 100644
--- a/llvm/test/MC/ELF/got-relaxed-rex.s
+++ b/llvm/test/MC/ELF/got-relaxed-rex.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux %s -o - | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux %s -o - | llvm-readobj -r - | FileCheck %s
 
 // these should produce R_X86_64_REX_GOTPCRELX
 
diff --git a/llvm/test/MC/ELF/got-relaxed.s b/llvm/test/MC/ELF/got-relaxed.s
index c89e462095d80..2ac34a8c2137e 100644
--- a/llvm/test/MC/ELF/got-relaxed.s
+++ b/llvm/test/MC/ELF/got-relaxed.s
@@ -1,5 +1,5 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux %s -o - | llvm-readobj -r | FileCheck %s
-// RUN: llvm-mc -filetype=obj -relax-relocations=false -triple x86_64-pc-linux %s -o - | llvm-readobj -r | FileCheck --check-prefix=OLD %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux %s -o - | llvm-readobj -r - | FileCheck %s
+// RUN: llvm-mc -filetype=obj -relax-relocations=false -triple x86_64-pc-linux %s -o - | llvm-readobj -r - | FileCheck --check-prefix=OLD %s
 
 // these should produce R_X86_64_GOTPCRELX
 
diff --git a/llvm/test/MC/ELF/got.s b/llvm/test/MC/ELF/got.s
index 73080e9ad6954..7d8e4d059b9bc 100644
--- a/llvm/test/MC/ELF/got.s
+++ b/llvm/test/MC/ELF/got.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r --symbols - | FileCheck %s
 
 // Test that this produces the correct relocations R_X86_64_GOT32 and that we,
 // unlike gas, don't create a _GLOBAL_OFFSET_TABLE_ symbol as a side effect.
diff --git a/llvm/test/MC/ELF/ident.s b/llvm/test/MC/ELF/ident.s
index e7df90b9b3776..5667aafc551c6 100644
--- a/llvm/test/MC/ELF/ident.s
+++ b/llvm/test/MC/ELF/ident.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sd - | FileCheck %s
 
 // CHECK:        Section {
 // CHECK:          Name: .comment
diff --git a/llvm/test/MC/ELF/ifunc-alias.s b/llvm/test/MC/ELF/ifunc-alias.s
index 1177a9e6ec422..6fd9845548e51 100644
--- a/llvm/test/MC/ELF/ifunc-alias.s
+++ b/llvm/test/MC/ELF/ifunc-alias.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readelf -s | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readelf -s - | FileCheck %s
 .text
 
 .type  foo_impl,@function
diff --git a/llvm/test/MC/ELF/ifunc-reloc.s b/llvm/test/MC/ELF/ifunc-reloc.s
index 101e6b83f84ea..726fa2a3b75b5 100644
--- a/llvm/test/MC/ELF/ifunc-reloc.s
+++ b/llvm/test/MC/ELF/ifunc-reloc.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r | FileCheck %s
+# RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r - | FileCheck %s
 .global alias
 .type alias, @function
 alias:
diff --git a/llvm/test/MC/ELF/lcomm.s b/llvm/test/MC/ELF/lcomm.s
index 88c2fae5a0677..c270027cda0b0 100644
--- a/llvm/test/MC/ELF/lcomm.s
+++ b/llvm/test/MC/ELF/lcomm.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-pc-linux-gnu %s -filetype=obj -o - | llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -triple i386-pc-linux-gnu %s -filetype=obj -o - | llvm-readobj --symbols - | FileCheck %s
 
 .lcomm A, 5
 .lcomm B, 32 << 20
diff --git a/llvm/test/MC/ELF/leb128.s b/llvm/test/MC/ELF/leb128.s
index 87cdd33f4149a..ffde821841380 100644
--- a/llvm/test/MC/ELF/leb128.s
+++ b/llvm/test/MC/ELF/leb128.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sd - | FileCheck %s
 
         .sleb128 .Lfoo - .Lbar
 .Lfoo:
diff --git a/llvm/test/MC/ELF/local-reloc.s b/llvm/test/MC/ELF/local-reloc.s
index 91c4a40bc3e73..5f8a2c816de2d 100644
--- a/llvm/test/MC/ELF/local-reloc.s
+++ b/llvm/test/MC/ELF/local-reloc.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S -r --symbols | FileCheck  %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S -r --symbols - | FileCheck  %s
 
 // Test that relocations with local symbols are represented as relocations
 // with the section. They should be equivalent, but gas behaves like this.
diff --git a/llvm/test/MC/ELF/merge.s b/llvm/test/MC/ELF/merge.s
index e787728ae1978..f70ab3b29cf49 100644
--- a/llvm/test/MC/ELF/merge.s
+++ b/llvm/test/MC/ELF/merge.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r | FileCheck  %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r - | FileCheck  %s
 
 // Test that relocations with local symbols in a mergeable section are done
 // with a reference to the symbol if the offset is non zero.
diff --git a/llvm/test/MC/ELF/n_bytes.s b/llvm/test/MC/ELF/n_bytes.s
index 7ad1b10bb3741..6df6a9fec733d 100644
--- a/llvm/test/MC/ELF/n_bytes.s
+++ b/llvm/test/MC/ELF/n_bytes.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sd | FileCheck  %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sd - | FileCheck  %s
 
         .2byte 42, 1, 2, 3
         .4byte 42, 1, 2, 3
diff --git a/llvm/test/MC/ELF/no-reloc.s b/llvm/test/MC/ELF/no-reloc.s
index 78f1b88cebbe8..b5d62402846ca 100644
--- a/llvm/test/MC/ELF/no-reloc.s
+++ b/llvm/test/MC/ELF/no-reloc.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r - | FileCheck %s
 
 // CHECK: Relocations [
 // CHECK-NEXT: ]
diff --git a/llvm/test/MC/ELF/noexec.s b/llvm/test/MC/ELF/noexec.s
index bfed7990b660f..dbfa45c5f428f 100644
--- a/llvm/test/MC/ELF/noexec.s
+++ b/llvm/test/MC/ELF/noexec.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -no-exec-stack -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S | FileCheck  %s
+// RUN: llvm-mc -no-exec-stack -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S - | FileCheck  %s
 
 // CHECK:        Section {
 // CHECK:          Index:
diff --git a/llvm/test/MC/ELF/norelocation.s b/llvm/test/MC/ELF/norelocation.s
index 054fc5f95cb2f..ac9b961c6119f 100644
--- a/llvm/test/MC/ELF/norelocation.s
+++ b/llvm/test/MC/ELF/norelocation.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r | FileCheck  %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r - | FileCheck  %s
 
         call bar
 bar:
diff --git a/llvm/test/MC/ELF/org.s b/llvm/test/MC/ELF/org.s
index 665bddcc482e7..ec6264f823c27 100644
--- a/llvm/test/MC/ELF/org.s
+++ b/llvm/test/MC/ELF/org.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S - | FileCheck %s
 
         .zero 4
 foo:
diff --git a/llvm/test/MC/ELF/pic-diff.s b/llvm/test/MC/ELF/pic-diff.s
index a3dd8247f310e..a685824bcd746 100644
--- a/llvm/test/MC/ELF/pic-diff.s
+++ b/llvm/test/MC/ELF/pic-diff.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r --symbols - | FileCheck %s
 
 // CHECK:      Relocations [
 // CHECK-NEXT:   Section ({{[^ ]+}}) {{[^ ]+}} {
diff --git a/llvm/test/MC/ELF/plt.s b/llvm/test/MC/ELF/plt.s
index 604a4bf3a38e4..3c3cc76aef5de 100644
--- a/llvm/test/MC/ELF/plt.s
+++ b/llvm/test/MC/ELF/plt.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r - | FileCheck %s
 
 // Test that this produces a R_X86_64_PLT32.
 
diff --git a/llvm/test/MC/ELF/pr19430.s b/llvm/test/MC/ELF/pr19430.s
index 3ba6d729b4be7..8ae0d8349e6d7 100644
--- a/llvm/test/MC/ELF/pr19430.s
+++ b/llvm/test/MC/ELF/pr19430.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-pc-linux-gnu %s -filetype=obj -o - | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -triple x86_64-pc-linux-gnu %s -filetype=obj -o - | llvm-readobj -r - | FileCheck %s
 
 // Test that we can use .cfi_startproc without a global symbol.
 
diff --git a/llvm/test/MC/ELF/pr19582.s b/llvm/test/MC/ELF/pr19582.s
index 304cacb2972cd..551dbd59e212b 100644
--- a/llvm/test/MC/ELF/pr19582.s
+++ b/llvm/test/MC/ELF/pr19582.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r - | FileCheck %s
 
 a:
     .section foo
diff --git a/llvm/test/MC/ELF/pr9292.s b/llvm/test/MC/ELF/pr9292.s
index f2f91fae23c7b..9c346fbe94cb8 100644
--- a/llvm/test/MC/ELF/pr9292.s
+++ b/llvm/test/MC/ELF/pr9292.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --symbols - | FileCheck %s
 
 // Test that both foo and bar are undefined.
 
diff --git a/llvm/test/MC/ELF/reloc-same-name-section.s b/llvm/test/MC/ELF/reloc-same-name-section.s
index 57463c65f8909..a7f5c3cbfb2ee 100644
--- a/llvm/test/MC/ELF/reloc-same-name-section.s
+++ b/llvm/test/MC/ELF/reloc-same-name-section.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux %s -o - | llvm-readobj -r --expand-relocs | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux %s -o - | llvm-readobj -r --expand-relocs - | FileCheck %s
 
 // test that we produce one relocation against each section.
 
diff --git a/llvm/test/MC/ELF/relocation-386.s b/llvm/test/MC/ELF/relocation-386.s
index d06cea9be9fdc..4273750a41926 100644
--- a/llvm/test/MC/ELF/relocation-386.s
+++ b/llvm/test/MC/ELF/relocation-386.s
@@ -1,5 +1,5 @@
-// RUN: llvm-mc -filetype=obj -triple i386-pc-linux-gnu %s -relax-relocations=false -o - | llvm-readobj -r  | FileCheck  %s --check-prefix=CHECK --check-prefix=I386
-// RUN: llvm-mc -filetype=obj -triple i386-pc-elfiamcu %s -relax-relocations=false  -o - | llvm-readobj -r  | FileCheck  %s --check-prefix=CHECK --check-prefix=IAMCU
+// RUN: llvm-mc -filetype=obj -triple i386-pc-linux-gnu %s -relax-relocations=false -o - | llvm-readobj -r  - | FileCheck  %s --check-prefix=CHECK --check-prefix=I386
+// RUN: llvm-mc -filetype=obj -triple i386-pc-elfiamcu %s -relax-relocations=false  -o - | llvm-readobj -r  - | FileCheck  %s --check-prefix=CHECK --check-prefix=IAMCU
 
 // Test that we produce the correct relocation types and that the relocations
 // correctly point to the section or the symbol.
diff --git a/llvm/test/MC/ELF/relocation-pc.s b/llvm/test/MC/ELF/relocation-pc.s
index 48bc55b2e0d21..a74a438002b46 100644
--- a/llvm/test/MC/ELF/relocation-pc.s
+++ b/llvm/test/MC/ELF/relocation-pc.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr | FileCheck  %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr - | FileCheck  %s
 
 // Test that we produce the correct relocation.
 
diff --git a/llvm/test/MC/ELF/relocation.s b/llvm/test/MC/ELF/relocation.s
index d3a53567fe0c3..d55e8acffc1d2 100644
--- a/llvm/test/MC/ELF/relocation.s
+++ b/llvm/test/MC/ELF/relocation.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr  | FileCheck  %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr  - | FileCheck  %s
 
 // Test that we produce the correct relocation.
 
diff --git a/llvm/test/MC/ELF/rename.s b/llvm/test/MC/ELF/rename.s
index 1169cd4a5f79e..c5351a0be558b 100644
--- a/llvm/test/MC/ELF/rename.s
+++ b/llvm/test/MC/ELF/rename.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --symbols - | FileCheck %s
 
 // When doing a rename, all the checks for where the relocation should go
 // should be performed with the original symbol. Only if we decide to relocate
diff --git a/llvm/test/MC/ELF/section-numeric-flag.s b/llvm/test/MC/ELF/section-numeric-flag.s
index ebfffbce31c2f..62d44d49364f7 100644
--- a/llvm/test/MC/ELF/section-numeric-flag.s
+++ b/llvm/test/MC/ELF/section-numeric-flag.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux-gnu %s -o - \
-// RUN: | llvm-readobj -S --symbols | FileCheck %s
+// RUN: | llvm-readobj -S --symbols - | FileCheck %s
 
         .section .text,    "0x806", %progbits, unique, 0
         .section .comment, "0x21"
diff --git a/llvm/test/MC/ELF/section-numeric-invalid-type.s b/llvm/test/MC/ELF/section-numeric-invalid-type.s
index 0e721099ff7c8..fefc364d9917b 100644
--- a/llvm/test/MC/ELF/section-numeric-invalid-type.s
+++ b/llvm/test/MC/ELF/section-numeric-invalid-type.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux-gnu %s -o - \
-// RUN:   | llvm-readobj -S --symbols | FileCheck --check-prefix=OBJ %s
+// RUN:   | llvm-readobj -S --symbols - | FileCheck --check-prefix=OBJ %s
 
 // RUN: not --crash llvm-mc -filetype=asm -triple=x86_64-pc-linux-gnu %s -o - 2>&1 \
 // RUN:   | FileCheck --check-prefix=ASM %s
diff --git a/llvm/test/MC/ELF/section-numeric-type.s b/llvm/test/MC/ELF/section-numeric-type.s
index f8278bc9fb449..062dd6bf2a1ea 100644
--- a/llvm/test/MC/ELF/section-numeric-type.s
+++ b/llvm/test/MC/ELF/section-numeric-type.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux-gnu %s -o - \
-// RUN:   | llvm-readobj -S --symbols | FileCheck --check-prefix=OBJ %s
+// RUN:   | llvm-readobj -S --symbols - | FileCheck --check-prefix=OBJ %s
 
 // RUN: llvm-mc -filetype=asm -triple=x86_64-pc-linux-gnu %s -o - \
 // RUN:   | FileCheck --check-prefix=ASM %s
diff --git a/llvm/test/MC/ELF/section-relro.ll b/llvm/test/MC/ELF/section-relro.ll
index 27fc109dea5ff..ebc8b18e1007d 100644
--- a/llvm/test/MC/ELF/section-relro.ll
+++ b/llvm/test/MC/ELF/section-relro.ll
@@ -1,6 +1,6 @@
 ; Tests that data and relro are correctly placed in sections
 ; specified by "#pragma clang section"
-; RUN: llc -filetype=obj -mtriple x86_64-unknown-linux %s -o - | llvm-readobj -S -t | FileCheck %s
+; RUN: llc -filetype=obj -mtriple x86_64-unknown-linux %s -o - | llvm-readobj -S -t - | FileCheck %s
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux"
diff --git a/llvm/test/MC/ELF/section-sym.s b/llvm/test/MC/ELF/section-sym.s
index c97de12f07f2b..e89b669bcb2c5 100644
--- a/llvm/test/MC/ELF/section-sym.s
+++ b/llvm/test/MC/ELF/section-sym.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --symbols -r --expand-relocs | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --symbols -r --expand-relocs - | FileCheck %s
 
 .section foo, "aG", @progbits, f1, comdat
 .section foo, "G", @progbits, f2, comdat
diff --git a/llvm/test/MC/ELF/section-sym2.s b/llvm/test/MC/ELF/section-sym2.s
index e763c81d96ef3..b404ef79414df 100644
--- a/llvm/test/MC/ELF/section-sym2.s
+++ b/llvm/test/MC/ELF/section-sym2.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj  --symbols -r --expand-relocs | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj  --symbols -r --expand-relocs - | FileCheck %s
 
 // Test that we can forward reference a section.
 
diff --git a/llvm/test/MC/ELF/section-unique.s b/llvm/test/MC/ELF/section-unique.s
index 3566da98f1119..3683ba036f0d0 100644
--- a/llvm/test/MC/ELF/section-unique.s
+++ b/llvm/test/MC/ELF/section-unique.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -triple x86_64-pc-linux-gnu %s -o - | FileCheck %s
-// RUN: llvm-mc -triple x86_64-pc-linux-gnu %s -filetype=obj -o - | llvm-readobj --symbols | FileCheck %s --check-prefix=OBJ
+// RUN: llvm-mc -triple x86_64-pc-linux-gnu %s -filetype=obj -o - | llvm-readobj --symbols - | FileCheck %s --check-prefix=OBJ
 
 	.section	.text,"ax",@progbits,unique, 4294967293
         .globl	f
diff --git a/llvm/test/MC/ELF/section.s b/llvm/test/MC/ELF/section.s
index f4ed666bfc71d..b5c7126c83ea1 100644
--- a/llvm/test/MC/ELF/section.s
+++ b/llvm/test/MC/ELF/section.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S - | FileCheck %s
 // RUN: llvm-mc -filetype=asm -triple x86_64-pc-linux-gnu %s -o - |  FileCheck %s --check-prefix=ASM
 
 // Test that these names are accepted.
diff --git a/llvm/test/MC/ELF/set.s b/llvm/test/MC/ELF/set.s
index 5434b9dac665b..1c8a06a0678bc 100644
--- a/llvm/test/MC/ELF/set.s
+++ b/llvm/test/MC/ELF/set.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --symbols | FileCheck  %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --symbols - | FileCheck  %s
 
 // Test that we emit the correct value.
 
diff --git a/llvm/test/MC/ELF/size.s b/llvm/test/MC/ELF/size.s
index cade2208fdb71..8d516350f7317 100644
--- a/llvm/test/MC/ELF/size.s
+++ b/llvm/test/MC/ELF/size.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux < %s | llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux < %s | llvm-readobj --symbols - | FileCheck %s
 
 foo:
 bar = .
diff --git a/llvm/test/MC/ELF/sleb.s b/llvm/test/MC/ELF/sleb.s
index 600ce19754fb9..6396864134e15 100644
--- a/llvm/test/MC/ELF/sleb.s
+++ b/llvm/test/MC/ELF/sleb.s
@@ -1,7 +1,7 @@
-// RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu %s -o - | llvm-readobj -S --sd | FileCheck -check-prefix=ELF_32 %s
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sd | FileCheck -check-prefix=ELF_64 %s
-// RUN: llvm-mc -filetype=obj -triple i386-apple-darwin9 %s -o - | llvm-readobj -S --sd | FileCheck -check-prefix=MACHO_32 %s
-// RUN: llvm-mc -filetype=obj -triple x86_64-apple-darwin9 %s -o - | llvm-readobj -S --sd | FileCheck -check-prefix=MACHO_64 %s
+// RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu %s -o - | llvm-readobj -S --sd - | FileCheck -check-prefix=ELF_32 %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sd - | FileCheck -check-prefix=ELF_64 %s
+// RUN: llvm-mc -filetype=obj -triple i386-apple-darwin9 %s -o - | llvm-readobj -S --sd - | FileCheck -check-prefix=MACHO_32 %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-apple-darwin9 %s -o - | llvm-readobj -S --sd - | FileCheck -check-prefix=MACHO_64 %s
 
 	.text
 foo:
diff --git a/llvm/test/MC/ELF/strtab-suffix-opt.s b/llvm/test/MC/ELF/strtab-suffix-opt.s
index fb9e895642df6..36fdd66489f12 100644
--- a/llvm/test/MC/ELF/strtab-suffix-opt.s
+++ b/llvm/test/MC/ELF/strtab-suffix-opt.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu %s -o - | llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu %s -o - | llvm-readobj --symbols - | FileCheck %s
 
 	.text
 	.globl	foobar
diff --git a/llvm/test/MC/ELF/symbol-names.s b/llvm/test/MC/ELF/symbol-names.s
index c6b26f36688d9..f605c723d4d4d 100644
--- a/llvm/test/MC/ELF/symbol-names.s
+++ b/llvm/test/MC/ELF/symbol-names.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i686-pc-linux -filetype=obj %s -o - | llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -triple i686-pc-linux -filetype=obj %s -o - | llvm-readobj --symbols - | FileCheck %s
 
 // MC allows ?'s in symbol names as an extension.
 
diff --git a/llvm/test/MC/ELF/symver-msvc.s b/llvm/test/MC/ELF/symver-msvc.s
index 6d3573a4d0eb5..8741053e13580 100644
--- a/llvm/test/MC/ELF/symver-msvc.s
+++ b/llvm/test/MC/ELF/symver-msvc.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-windows-elf %s -o - | llvm-readobj -r --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-windows-elf %s -o - | llvm-readobj -r --symbols - | FileCheck %s
 
 // Verify that MSVC C++ mangled symbols are not affected by the ELF
 // GNU-style symbol versioning. The ELF format is used on Windows by
diff --git a/llvm/test/MC/ELF/symver-pr23914.s b/llvm/test/MC/ELF/symver-pr23914.s
index 69ec02034d828..7ae3895035c31 100644
--- a/llvm/test/MC/ELF/symver-pr23914.s
+++ b/llvm/test/MC/ELF/symver-pr23914.s
@@ -1,5 +1,5 @@
 // Regression test for PR23914.
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r --symbols - | FileCheck %s
 
 defined:
         .symver defined, aaaaaaaaaaaaaaaaaa@@@AAAAAAAAAAAAA
diff --git a/llvm/test/MC/ELF/symver.s b/llvm/test/MC/ELF/symver.s
index a591d5cec7b37..33063accd5bfe 100644
--- a/llvm/test/MC/ELF/symver.s
+++ b/llvm/test/MC/ELF/symver.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r --symbols - | FileCheck %s
 
 defined1:
 defined2:
diff --git a/llvm/test/MC/ELF/tls-i386.s b/llvm/test/MC/ELF/tls-i386.s
index 61ddd7d846276..9508227f9987e 100644
--- a/llvm/test/MC/ELF/tls-i386.s
+++ b/llvm/test/MC/ELF/tls-i386.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple i386-pc-linux-gnu %s -o - | llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple i386-pc-linux-gnu %s -o - | llvm-readobj --symbols - | FileCheck %s
 
 // Test that all symbols are of type STT_TLS.
 
diff --git a/llvm/test/MC/ELF/tls.s b/llvm/test/MC/ELF/tls.s
index 045e552c4b97f..560cf138806ec 100644
--- a/llvm/test/MC/ELF/tls.s
+++ b/llvm/test/MC/ELF/tls.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --symbols - | FileCheck %s
 
 // Test that all symbols are of type STT_TLS.
 
diff --git a/llvm/test/MC/ELF/type.s b/llvm/test/MC/ELF/type.s
index b43ec1cd1104d..dd411a5632fba 100644
--- a/llvm/test/MC/ELF/type.s
+++ b/llvm/test/MC/ELF/type.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --symbols - | FileCheck %s
 
 // Test that both % and @ are accepted.
         .global foo
diff --git a/llvm/test/MC/ELF/uleb-ehtable.s b/llvm/test/MC/ELF/uleb-ehtable.s
index d54392a2578b2..ca3f9e97bffc2 100644
--- a/llvm/test/MC/ELF/uleb-ehtable.s
+++ b/llvm/test/MC/ELF/uleb-ehtable.s
@@ -1,7 +1,7 @@
-// RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu    %s -o - | llvm-readobj -S --sd | FileCheck %s -check-prefix=CHECK -check-prefix=ELF
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu  %s -o - | llvm-readobj -S --sd | FileCheck %s -check-prefix=CHECK -check-prefix=ELF
-// RUN: llvm-mc -filetype=obj -triple i386-apple-darwin9   %s -o - | llvm-readobj -S --sd | FileCheck %s -check-prefix=CHECK -check-prefix=MACHO
-// RUN: llvm-mc -filetype=obj -triple x86_64-apple-darwin9 %s -o - | llvm-readobj -S --sd | FileCheck %s -check-prefix=CHECK -check-prefix=MACHO
+// RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu    %s -o - | llvm-readobj -S --sd - | FileCheck %s -check-prefix=CHECK -check-prefix=ELF
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu  %s -o - | llvm-readobj -S --sd - | FileCheck %s -check-prefix=CHECK -check-prefix=ELF
+// RUN: llvm-mc -filetype=obj -triple i386-apple-darwin9   %s -o - | llvm-readobj -S --sd - | FileCheck %s -check-prefix=CHECK -check-prefix=MACHO
+// RUN: llvm-mc -filetype=obj -triple x86_64-apple-darwin9 %s -o - | llvm-readobj -S --sd - | FileCheck %s -check-prefix=CHECK -check-prefix=MACHO
 
 // Test that we can assemble a GCC-like EH table that has 16381-16383 bytes of
 // non-padding data between .ttbaseref and .ttbase. The assembler must insert
diff --git a/llvm/test/MC/ELF/uleb.s b/llvm/test/MC/ELF/uleb.s
index e09ae6ca1cf13..09479bd9c08da 100644
--- a/llvm/test/MC/ELF/uleb.s
+++ b/llvm/test/MC/ELF/uleb.s
@@ -1,7 +1,7 @@
-// RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu %s -o - | llvm-readobj -S --sd | FileCheck -check-prefix=ELF_32 %s
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sd | FileCheck -check-prefix=ELF_64 %s
-// RUN: llvm-mc -filetype=obj -triple i386-apple-darwin9 %s -o - | llvm-readobj -S --sd | FileCheck -check-prefix=MACHO_32 %s
-// RUN: llvm-mc -filetype=obj -triple x86_64-apple-darwin9 %s -o - | llvm-readobj -S --sd | FileCheck -check-prefix=MACHO_64 %s
+// RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu %s -o - | llvm-readobj -S --sd - | FileCheck -check-prefix=ELF_32 %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sd - | FileCheck -check-prefix=ELF_64 %s
+// RUN: llvm-mc -filetype=obj -triple i386-apple-darwin9 %s -o - | llvm-readobj -S --sd - | FileCheck -check-prefix=MACHO_32 %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-apple-darwin9 %s -o - | llvm-readobj -S --sd - | FileCheck -check-prefix=MACHO_64 %s
 
 	.text
 foo:
diff --git a/llvm/test/MC/ELF/undef.s b/llvm/test/MC/ELF/undef.s
index 9d99b834ffdf8..931fa07868e32 100644
--- a/llvm/test/MC/ELF/undef.s
+++ b/llvm/test/MC/ELF/undef.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --symbols - | FileCheck %s
 
 // Test which symbols should be in the symbol table
 
diff --git a/llvm/test/MC/ELF/version.s b/llvm/test/MC/ELF/version.s
index f138e9ddca567..67526d8b64401 100644
--- a/llvm/test/MC/ELF/version.s
+++ b/llvm/test/MC/ELF/version.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple i386-pc-linux-gnu %s -o - | llvm-readobj -S --sd | FileCheck  %s
+// RUN: llvm-mc -filetype=obj -triple i386-pc-linux-gnu %s -o - | llvm-readobj -S --sd - | FileCheck  %s
 
 .version "1234"
 .version "123"
diff --git a/llvm/test/MC/ELF/weak-diff.s b/llvm/test/MC/ELF/weak-diff.s
index 9785a2befb150..9840bede0b48a 100644
--- a/llvm/test/MC/ELF/weak-diff.s
+++ b/llvm/test/MC/ELF/weak-diff.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu < %s | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu < %s | llvm-readobj -r - | FileCheck %s
 
 // CHECK:      Relocations [
 // CHECK-NEXT:   Section ({{.*}}) .rela.text {
diff --git a/llvm/test/MC/ELF/weak-relocation.s b/llvm/test/MC/ELF/weak-relocation.s
index defaf4e6a38e3..a1f85a9af9400 100644
--- a/llvm/test/MC/ELF/weak-relocation.s
+++ b/llvm/test/MC/ELF/weak-relocation.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r - | FileCheck %s
 
 // Test that weak symbols always produce relocations
 
diff --git a/llvm/test/MC/ELF/weak.s b/llvm/test/MC/ELF/weak.s
index 7d1d0e2d9a018..d78f5db0f10c1 100644
--- a/llvm/test/MC/ELF/weak.s
+++ b/llvm/test/MC/ELF/weak.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --symbols - | FileCheck %s
 
 // Test that this produces a weak undefined symbol.
 
diff --git a/llvm/test/MC/ELF/weakref-plt.s b/llvm/test/MC/ELF/weakref-plt.s
index 0d2be1aed1a22..019d040c1da65 100644
--- a/llvm/test/MC/ELF/weakref-plt.s
+++ b/llvm/test/MC/ELF/weakref-plt.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --symbols - | FileCheck %s
 
 	.weakref	bar,foo
 	call	bar@PLT
diff --git a/llvm/test/MC/ELF/weakref-reloc.s b/llvm/test/MC/ELF/weakref-reloc.s
index f82c29b2fbfc6..5ed9b5696da95 100644
--- a/llvm/test/MC/ELF/weakref-reloc.s
+++ b/llvm/test/MC/ELF/weakref-reloc.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -r --symbols - | FileCheck %s
 
 // Test that the relocations point to the correct symbols.
 
diff --git a/llvm/test/MC/ELF/weakref.s b/llvm/test/MC/ELF/weakref.s
index bca8ec4f28169..b866ffaef744c 100644
--- a/llvm/test/MC/ELF/weakref.s
+++ b/llvm/test/MC/ELF/weakref.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj --symbols - | FileCheck %s
 
 // This is a long test that checks that the aliases created by weakref are
 // never in the symbol table and that the only case it causes a symbol to
diff --git a/llvm/test/MC/ELF/x86_64-reloc-sizetest.s b/llvm/test/MC/ELF/x86_64-reloc-sizetest.s
index 10c13123e6507..2c88ff7aebe53 100644
--- a/llvm/test/MC/ELF/x86_64-reloc-sizetest.s
+++ b/llvm/test/MC/ELF/x86_64-reloc-sizetest.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-linux-gnu -filetype=obj %s | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -triple x86_64-linux-gnu -filetype=obj %s | llvm-readobj -r - | FileCheck %s
 
 // Tests that relocation value fits in the provided size
 // Original bug http://llvm.org/bugs/show_bug.cgi?id=10568
diff --git a/llvm/test/MC/ELF/zero.s b/llvm/test/MC/ELF/zero.s
index bbd2b5604f0f6..8e0d445db359a 100644
--- a/llvm/test/MC/ELF/zero.s
+++ b/llvm/test/MC/ELF/zero.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sd | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sd - | FileCheck %s
 
 .zero 4
 .zero 1,42
diff --git a/llvm/test/MC/Hexagon/basic.ll b/llvm/test/MC/Hexagon/basic.ll
index 5fba44f3a5f9f..b9c9597546d81 100644
--- a/llvm/test/MC/Hexagon/basic.ll
+++ b/llvm/test/MC/Hexagon/basic.ll
@@ -1,5 +1,5 @@
 ;; RUN: llc -mtriple=hexagon-unknown-elf -filetype=obj %s -o - \
-;; RUN: | llvm-readobj -h -r | FileCheck -check-prefix=OBJ %s
+;; RUN: | llvm-readobj -h -r - | FileCheck -check-prefix=OBJ %s
 
 ; OBJ: Format: elf32-hexagon
 ; OBJ: Arch: hexagon
diff --git a/llvm/test/MC/Hexagon/elf-flags.s b/llvm/test/MC/Hexagon/elf-flags.s
index 5f67b37d4ca2c..d392e7e1db24d 100644
--- a/llvm/test/MC/Hexagon/elf-flags.s
+++ b/llvm/test/MC/Hexagon/elf-flags.s
@@ -1,7 +1,7 @@
-# RUN: llvm-mc -arch=hexagon -mcpu=hexagonv5 --filetype=obj %s -o - | llvm-readelf --file-headers | FileCheck --check-prefix=CHECK-V5 %s
-# RUN: llvm-mc -arch=hexagon -mcpu=hexagonv55 --filetype=obj %s -o - | llvm-readelf --file-headers | FileCheck --check-prefix=CHECK-V55 %s
-# RUN: llvm-mc -arch=hexagon -mcpu=hexagonv60 --filetype=obj %s -o - | llvm-readelf --file-headers | FileCheck --check-prefix=CHECK-V60 %s
-# RUN: llvm-mc -arch=hexagon -mcpu=hexagonv62 --filetype=obj %s -o - | llvm-readelf --file-headers | FileCheck --check-prefix=CHECK-V62 %s
+# RUN: llvm-mc -arch=hexagon -mcpu=hexagonv5 --filetype=obj %s -o - | llvm-readelf --file-headers - | FileCheck --check-prefix=CHECK-V5 %s
+# RUN: llvm-mc -arch=hexagon -mcpu=hexagonv55 --filetype=obj %s -o - | llvm-readelf --file-headers - | FileCheck --check-prefix=CHECK-V55 %s
+# RUN: llvm-mc -arch=hexagon -mcpu=hexagonv60 --filetype=obj %s -o - | llvm-readelf --file-headers - | FileCheck --check-prefix=CHECK-V60 %s
+# RUN: llvm-mc -arch=hexagon -mcpu=hexagonv62 --filetype=obj %s -o - | llvm-readelf --file-headers - | FileCheck --check-prefix=CHECK-V62 %s
 
 # CHECK-V5: Flags: 0x4
 # CHECK-V55: Flags: 0x5
diff --git a/llvm/test/MC/MSP430/reloc.s b/llvm/test/MC/MSP430/reloc.s
index 61a25f494735e..3e3d34fa05116 100644
--- a/llvm/test/MC/MSP430/reloc.s
+++ b/llvm/test/MC/MSP430/reloc.s
@@ -21,7 +21,7 @@
 ; CHECK: jmp    foo             ; encoding: [A,0b001111AA]
 ; CHECK:                        ;   fixup A - offset: 0, value: foo, kind: fixup_10_pcrel
 
-; RUN: llvm-mc -filetype=obj -triple msp430 < %s | llvm-readobj -r \
+; RUN: llvm-mc -filetype=obj -triple msp430 < %s | llvm-readobj -r - \
 ; RUN:   | FileCheck -check-prefix=RELOC %s
 .short  _ctype+3
 ; RELOC: R_MSP430_16_BYTE _ctype 0x3
diff --git a/llvm/test/MC/MachO/AArch64/classrefs.s b/llvm/test/MC/MachO/AArch64/classrefs.s
index d92bbb4a57abe..fc308e43ba7df 100644
--- a/llvm/test/MC/MachO/AArch64/classrefs.s
+++ b/llvm/test/MC/MachO/AArch64/classrefs.s
@@ -1,4 +1,4 @@
-; RUN: llvm-mc -triple arm64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj -r --expand-relocs | FileCheck %s
+; RUN: llvm-mc -triple arm64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj -r --expand-relocs - | FileCheck %s
 
 ; Test that we produce an external relocation with Lbar. We could also produce
 ; an internal relocation. We just have to be careful to not use another symbol.
diff --git a/llvm/test/MC/MachO/AArch64/darwin-ARM64-local-label-diff.s b/llvm/test/MC/MachO/AArch64/darwin-ARM64-local-label-diff.s
index 547f5be43f994..4b855ea0df3ff 100644
--- a/llvm/test/MC/MachO/AArch64/darwin-ARM64-local-label-diff.s
+++ b/llvm/test/MC/MachO/AArch64/darwin-ARM64-local-label-diff.s
@@ -1,4 +1,4 @@
-; RUN: llvm-mc -triple arm64-apple-darwin -filetype=obj -o - < %s | llvm-readobj -S --sd | FileCheck %s
+; RUN: llvm-mc -triple arm64-apple-darwin -filetype=obj -o - < %s | llvm-readobj -S --sd - | FileCheck %s
 ; rdar://13028719
 
  .globl context_save0
diff --git a/llvm/test/MC/MachO/AArch64/darwin-ARM64-reloc.s b/llvm/test/MC/MachO/AArch64/darwin-ARM64-reloc.s
index b4d0b082c8a58..a9d988d63015d 100644
--- a/llvm/test/MC/MachO/AArch64/darwin-ARM64-reloc.s
+++ b/llvm/test/MC/MachO/AArch64/darwin-ARM64-reloc.s
@@ -1,4 +1,4 @@
-; RUN: llvm-mc -n -triple arm64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj -r --expand-relocs | FileCheck %s
+; RUN: llvm-mc -n -triple arm64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj -r --expand-relocs - | FileCheck %s
 
 	.text
 _fred:
diff --git a/llvm/test/MC/MachO/AArch64/data-in-code.s b/llvm/test/MC/MachO/AArch64/data-in-code.s
index cf7d860a622f1..484199224f020 100644
--- a/llvm/test/MC/MachO/AArch64/data-in-code.s
+++ b/llvm/test/MC/MachO/AArch64/data-in-code.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple arm64-apple-darwin10 -filetype=obj -o - < %s | llvm-readobj --macho-data-in-code | FileCheck %s
+// RUN: llvm-mc -triple arm64-apple-darwin10 -filetype=obj -o - < %s | llvm-readobj --macho-data-in-code - | FileCheck %s
         .text
 _foo:
 .data_region
diff --git a/llvm/test/MC/MachO/AArch64/ld64-workaround.s b/llvm/test/MC/MachO/AArch64/ld64-workaround.s
index 63f91705599cc..807580c06dbf1 100644
--- a/llvm/test/MC/MachO/AArch64/ld64-workaround.s
+++ b/llvm/test/MC/MachO/AArch64/ld64-workaround.s
@@ -1,4 +1,4 @@
-; RUN: llvm-mc -triple arm64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj -r --expand-relocs | FileCheck %s
+; RUN: llvm-mc -triple arm64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj -r --expand-relocs - | FileCheck %s
 
 ; Test that we produce an external relocation. This is a known and temporary bug
 ; in ld64, where it mishandles pointer-sized internal relocations. We should be
diff --git a/llvm/test/MC/MachO/AArch64/mergeable.s b/llvm/test/MC/MachO/AArch64/mergeable.s
index 4fed04c93346c..8621b25e80d32 100644
--- a/llvm/test/MC/MachO/AArch64/mergeable.s
+++ b/llvm/test/MC/MachO/AArch64/mergeable.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple aarch64-apple-darwin14 %s -filetype=obj -o - | llvm-readobj -r --expand-relocs | FileCheck %s
+// RUN: llvm-mc -triple aarch64-apple-darwin14 %s -filetype=obj -o - | llvm-readobj -r --expand-relocs - | FileCheck %s
 // FIXME: the final relocation should be internal, but the linker doesn't
 // currently handle the it correctly.
 
diff --git a/llvm/test/MC/MachO/AArch64/reloc-crash.s b/llvm/test/MC/MachO/AArch64/reloc-crash.s
index f8ad4c4f751c4..4a6eb0486c75f 100644
--- a/llvm/test/MC/MachO/AArch64/reloc-crash.s
+++ b/llvm/test/MC/MachO/AArch64/reloc-crash.s
@@ -1,4 +1,4 @@
-; RUN: llvm-mc -triple arm64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj -r --expand-relocs | FileCheck %s
+; RUN: llvm-mc -triple arm64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj -r --expand-relocs - | FileCheck %s
 
 ; Test tha we produce an external relocation. There is no apparent need for it, but
 ; ld64 (241.9) crashes if we don't.
diff --git a/llvm/test/MC/MachO/AArch64/reloc-crash2.s b/llvm/test/MC/MachO/AArch64/reloc-crash2.s
index 3aa26281bc026..40bb0347a28d8 100644
--- a/llvm/test/MC/MachO/AArch64/reloc-crash2.s
+++ b/llvm/test/MC/MachO/AArch64/reloc-crash2.s
@@ -1,4 +1,4 @@
-; RUN: llvm-mc -triple arm64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj -r --expand-relocs | FileCheck %s
+; RUN: llvm-mc -triple arm64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj -r --expand-relocs - | FileCheck %s
 
 ; This is a regression test making sure we don't crash.
 
diff --git a/llvm/test/MC/MachO/ARM/build-version-encode.s b/llvm/test/MC/MachO/ARM/build-version-encode.s
index d68f226cc9cc6..4bc6665e58aa8 100644
--- a/llvm/test/MC/MachO/ARM/build-version-encode.s
+++ b/llvm/test/MC/MachO/ARM/build-version-encode.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc %s -triple x86_64-apple-tvos -filetype=obj | llvm-readobj --macho-version-min | FileCheck %s
+// RUN: llvm-mc %s -triple x86_64-apple-tvos -filetype=obj | llvm-readobj --macho-version-min - | FileCheck %s
 
 .build_version tvos,1,2,3
 // CHECK: MinVersion {
diff --git a/llvm/test/MC/MachO/ARM/darwin-ARM-reloc.s b/llvm/test/MC/MachO/ARM/darwin-ARM-reloc.s
index f9a8d539f7678..290818910ebf1 100644
--- a/llvm/test/MC/MachO/ARM/darwin-ARM-reloc.s
+++ b/llvm/test/MC/MachO/ARM/darwin-ARM-reloc.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc -n -triple armv7-apple-darwin10 %s -filetype=obj -o %t.obj
-@ RUN: llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols < %t.obj > %t.dump
+@ RUN: llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols < %t.obj - > %t.dump
 @ RUN: FileCheck < %t.dump %s
 
 	.syntax unified
diff --git a/llvm/test/MC/MachO/ARM/darwin-Thumb-reloc.s b/llvm/test/MC/MachO/ARM/darwin-Thumb-reloc.s
index 8ee19a39e6de0..c387a292ba2fa 100644
--- a/llvm/test/MC/MachO/ARM/darwin-Thumb-reloc.s
+++ b/llvm/test/MC/MachO/ARM/darwin-Thumb-reloc.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc -n -triple thumbv7-apple-darwin10 %s -filetype=obj -o %t.obj
-@ RUN: llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols < %t.obj > %t.dump
+@ RUN: llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - < %t.obj > %t.dump
 @ RUN: FileCheck < %t.dump %s
 
 	.syntax unified
diff --git a/llvm/test/MC/MachO/ARM/data-in-code.s b/llvm/test/MC/MachO/ARM/data-in-code.s
index a7be7e74d4955..8ac4b350a3d60 100644
--- a/llvm/test/MC/MachO/ARM/data-in-code.s
+++ b/llvm/test/MC/MachO/ARM/data-in-code.s
@@ -1,4 +1,4 @@
-@ RUN: llvm-mc -triple armv7-apple-darwin10 -filetype=obj -o - < %s | llvm-readobj --macho-data-in-code | FileCheck %s
+@ RUN: llvm-mc -triple armv7-apple-darwin10 -filetype=obj -o - < %s | llvm-readobj --macho-data-in-code - | FileCheck %s
         .text
 _foo:
 .data_region
diff --git a/llvm/test/MC/MachO/ARM/empty-function-nop.ll b/llvm/test/MC/MachO/ARM/empty-function-nop.ll
index dc09d165318bf..bb7cff09731b4 100644
--- a/llvm/test/MC/MachO/ARM/empty-function-nop.ll
+++ b/llvm/test/MC/MachO/ARM/empty-function-nop.ll
@@ -1,7 +1,7 @@
-; RUN: llc < %s -filetype=obj -mtriple=thumbv6-apple-darwin -o - | llvm-readobj -S --sd | FileCheck -check-prefix=CHECK-T1 %s
-; RUN: llc < %s -filetype=obj -mtriple=thumbv7-apple-darwin -o - | llvm-readobj -S --sd | FileCheck -check-prefix=CHECK-T2 %s
-; RUN: llc < %s -filetype=obj -mtriple=armv6-apple-darwin -o - | llvm-readobj -S --sd | FileCheck -check-prefix=CHECK-ARM %s
-; RUN: llc < %s -filetype=obj -mtriple=armv7-apple-darwin -o - | llvm-readobj -S --sd | FileCheck -check-prefix=CHECK-ARMV7 %s
+; RUN: llc < %s -filetype=obj -mtriple=thumbv6-apple-darwin -o - | llvm-readobj -S --sd - | FileCheck -check-prefix=CHECK-T1 %s
+; RUN: llc < %s -filetype=obj -mtriple=thumbv7-apple-darwin -o - | llvm-readobj -S --sd - | FileCheck -check-prefix=CHECK-T2 %s
+; RUN: llc < %s -filetype=obj -mtriple=armv6-apple-darwin -o - | llvm-readobj -S --sd - | FileCheck -check-prefix=CHECK-ARM %s
+; RUN: llc < %s -filetype=obj -mtriple=armv7-apple-darwin -o - | llvm-readobj -S --sd - | FileCheck -check-prefix=CHECK-ARMV7 %s
 
 ; Empty functions need a NOP in them for MachO to prevent DWARF FDEs from
 ; getting all mucked up. See lib/CodeGen/AsmPrinter/AsmPrinter.cpp for
diff --git a/llvm/test/MC/MachO/ARM/ios-version-min-load-command.s b/llvm/test/MC/MachO/ARM/ios-version-min-load-command.s
index 0fa29da0b4c6b..e7435c6dc198d 100644
--- a/llvm/test/MC/MachO/ARM/ios-version-min-load-command.s
+++ b/llvm/test/MC/MachO/ARM/ios-version-min-load-command.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple armv7-apple-ios %s -filetype=obj -o - | llvm-readobj --macho-version-min | FileCheck %s
+// RUN: llvm-mc -triple armv7-apple-ios %s -filetype=obj -o - | llvm-readobj --macho-version-min - | FileCheck %s
 
 // Test the formation of the version-min load command in the MachO.
 // use a nonsense but well formed version.
diff --git a/llvm/test/MC/MachO/ARM/long-call-branch-island-relocation.s b/llvm/test/MC/MachO/ARM/long-call-branch-island-relocation.s
index e781c6d561886..db6ed200956ca 100644
--- a/llvm/test/MC/MachO/ARM/long-call-branch-island-relocation.s
+++ b/llvm/test/MC/MachO/ARM/long-call-branch-island-relocation.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc -n -triple armv7-apple-darwin10 %s -filetype=obj -o %t.o
-@ RUN: llvm-readobj -r --expand-relocs < %t.o | FileCheck %s
+@ RUN: llvm-readobj -r --expand-relocs - < %t.o | FileCheck %s
 
 @ rdar://12359919
 
diff --git a/llvm/test/MC/MachO/ARM/no-subsections-reloc.s b/llvm/test/MC/MachO/ARM/no-subsections-reloc.s
index 68b9b3865a2d7..31c1d37a66e73 100644
--- a/llvm/test/MC/MachO/ARM/no-subsections-reloc.s
+++ b/llvm/test/MC/MachO/ARM/no-subsections-reloc.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc -n -triple thumbv7-apple-darwin10 %s -filetype=obj -o %t.obj
-@ RUN: llvm-readobj -S --sd < %t.obj > %t.dump
+@ RUN: llvm-readobj -S --sd - < %t.obj > %t.dump
 @ RUN: FileCheck < %t.dump %s
 
 @ When not using subsections-via-symbols, references to non-local symbols
diff --git a/llvm/test/MC/MachO/ARM/nop-armv4-padding.s b/llvm/test/MC/MachO/ARM/nop-armv4-padding.s
index d63e506791e31..58db406ea5cab 100644
--- a/llvm/test/MC/MachO/ARM/nop-armv4-padding.s
+++ b/llvm/test/MC/MachO/ARM/nop-armv4-padding.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc -triple armv4-apple-darwin %s -filetype=obj -o %t.obj
-@ RUN: llvm-readobj -S --sd < %t.obj > %t.dump
+@ RUN: llvm-readobj -S --sd - < %t.obj > %t.dump
 @ RUN: FileCheck %s < %t.dump
 
 x:
diff --git a/llvm/test/MC/MachO/ARM/nop-armv6t2-padding.s b/llvm/test/MC/MachO/ARM/nop-armv6t2-padding.s
index 2b15ca0f54a4f..4cb5abb2a40f8 100644
--- a/llvm/test/MC/MachO/ARM/nop-armv6t2-padding.s
+++ b/llvm/test/MC/MachO/ARM/nop-armv6t2-padding.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc -triple armv6t2-apple-darwin %s -filetype=obj -o %t.obj
-@ RUN: llvm-readobj -S --sd < %t.obj > %t.dump
+@ RUN: llvm-readobj -S --sd - < %t.obj > %t.dump
 @ RUN: FileCheck %s < %t.dump
 
 x:
diff --git a/llvm/test/MC/MachO/ARM/nop-thumb-padding.s b/llvm/test/MC/MachO/ARM/nop-thumb-padding.s
index ac041e110fa73..36b7682802288 100644
--- a/llvm/test/MC/MachO/ARM/nop-thumb-padding.s
+++ b/llvm/test/MC/MachO/ARM/nop-thumb-padding.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc -triple armv6-apple-darwin %s -filetype=obj -o %t.obj
-@ RUN: llvm-readobj -S --sd < %t.obj > %t.dump
+@ RUN: llvm-readobj -S --sd - < %t.obj > %t.dump
 @ RUN: FileCheck %s < %t.dump
 
 .thumb_func x
diff --git a/llvm/test/MC/MachO/ARM/nop-thumb2-padding.s b/llvm/test/MC/MachO/ARM/nop-thumb2-padding.s
index e032c8a7de046..b6daac7ed5ca0 100644
--- a/llvm/test/MC/MachO/ARM/nop-thumb2-padding.s
+++ b/llvm/test/MC/MachO/ARM/nop-thumb2-padding.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc -triple armv7-apple-darwin %s -filetype=obj -o %t.obj
-@ RUN: llvm-readobj -S --sd < %t.obj > %t.dump
+@ RUN: llvm-readobj -S --sd - < %t.obj > %t.dump
 @ RUN: FileCheck %s < %t.dump
 
 .thumb_func x
diff --git a/llvm/test/MC/MachO/ARM/relax-thumb-ldr-literal.s b/llvm/test/MC/MachO/ARM/relax-thumb-ldr-literal.s
index 2ce65de7aab07..e360477c7a304 100644
--- a/llvm/test/MC/MachO/ARM/relax-thumb-ldr-literal.s
+++ b/llvm/test/MC/MachO/ARM/relax-thumb-ldr-literal.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc -n -triple thumbv7-apple-darwin10 %s -filetype=obj -o %t.obj
-@ RUN: llvm-readobj -S --sd < %t.obj > %t.dump
+@ RUN: llvm-readobj -S --sd - < %t.obj > %t.dump
 @ RUN: FileCheck < %t.dump %s
 
 	.syntax unified
diff --git a/llvm/test/MC/MachO/ARM/relax-thumb2-branches.s b/llvm/test/MC/MachO/ARM/relax-thumb2-branches.s
index 7563f4f62ffd4..29a3e79b0dc15 100644
--- a/llvm/test/MC/MachO/ARM/relax-thumb2-branches.s
+++ b/llvm/test/MC/MachO/ARM/relax-thumb2-branches.s
@@ -1,4 +1,4 @@
-@ RUN: llvm-mc -triple=thumbv7-apple-darwin -show-encoding %s -filetype=obj -o - | llvm-readobj -S --sd | FileCheck %s
+@ RUN: llvm-mc -triple=thumbv7-apple-darwin -show-encoding %s -filetype=obj -o - | llvm-readobj -S --sd - | FileCheck %s
 
         ble Lfoo        @ wide encoding
 
diff --git a/llvm/test/MC/MachO/ARM/static-movt-relocs.s b/llvm/test/MC/MachO/ARM/static-movt-relocs.s
index d94be2f4f6c26..47a093c520227 100644
--- a/llvm/test/MC/MachO/ARM/static-movt-relocs.s
+++ b/llvm/test/MC/MachO/ARM/static-movt-relocs.s
@@ -1,4 +1,4 @@
-@ RUN: llvm-mc -mcpu=cortex-a8 -triple thumbv7-apple-darwin10 -filetype=obj -o - < %s | llvm-readobj -r --expand-relocs | FileCheck %s
+@ RUN: llvm-mc -mcpu=cortex-a8 -triple thumbv7-apple-darwin10 -filetype=obj -o - < %s | llvm-readobj -r --expand-relocs - | FileCheck %s
         .thumb
         .thumb_func foo
 foo:
diff --git a/llvm/test/MC/MachO/ARM/thumb-bl-jbits.s b/llvm/test/MC/MachO/ARM/thumb-bl-jbits.s
index b5296b70c82bb..ef80311875079 100644
--- a/llvm/test/MC/MachO/ARM/thumb-bl-jbits.s
+++ b/llvm/test/MC/MachO/ARM/thumb-bl-jbits.s
@@ -1,4 +1,4 @@
-@ RUN: llvm-mc -triple=thumbv7-apple-darwin -filetype=obj -o - < %s | llvm-readobj -S --sd | FileCheck %s
+@ RUN: llvm-mc -triple=thumbv7-apple-darwin -filetype=obj -o - < %s | llvm-readobj -S --sd - | FileCheck %s
 .thumb
 .thumb_func t
 t:	nop
diff --git a/llvm/test/MC/MachO/ARM/thumb2-function-relative-load.s b/llvm/test/MC/MachO/ARM/thumb2-function-relative-load.s
index bdc4aa9c47d55..0f6466ab6e229 100644
--- a/llvm/test/MC/MachO/ARM/thumb2-function-relative-load.s
+++ b/llvm/test/MC/MachO/ARM/thumb2-function-relative-load.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc -n -triple thumbv7-apple-darwin10 %s -filetype=obj -o %t.obj
-@ RUN: llvm-readobj -S --sd < %t.obj > %t.dump
+@ RUN: llvm-readobj -S --sd - < %t.obj > %t.dump
 @ RUN: FileCheck < %t.dump %s
         .syntax unified
         .text
diff --git a/llvm/test/MC/MachO/ARM/thumb2-movt-fixup.s b/llvm/test/MC/MachO/ARM/thumb2-movt-fixup.s
index c946e5c6ea706..3f785938a7950 100644
--- a/llvm/test/MC/MachO/ARM/thumb2-movt-fixup.s
+++ b/llvm/test/MC/MachO/ARM/thumb2-movt-fixup.s
@@ -1,4 +1,4 @@
-@ RUN: llvm-mc -mcpu=cortex-a8 -triple thumbv7-apple-darwin10 -filetype=obj -o - < %s | llvm-readobj -r --expand-relocs | FileCheck %s
+@ RUN: llvm-mc -mcpu=cortex-a8 -triple thumbv7-apple-darwin10 -filetype=obj -o - < %s | llvm-readobj -r --expand-relocs - | FileCheck %s
 
 _fred:
 	movt	r3, :upper16:(_wilma-(LPC0_0+4))
diff --git a/llvm/test/MC/MachO/ARM/thumb2-movw-fixup.s b/llvm/test/MC/MachO/ARM/thumb2-movw-fixup.s
index 3191a90852bb8..18182d1affb06 100644
--- a/llvm/test/MC/MachO/ARM/thumb2-movw-fixup.s
+++ b/llvm/test/MC/MachO/ARM/thumb2-movw-fixup.s
@@ -1,4 +1,4 @@
-@ RUN: llvm-mc -mcpu=cortex-a8 -triple thumbv7-apple-darwin10 -filetype=obj -o - < %s | llvm-readobj -r --expand-relocs | FileCheck %s
+@ RUN: llvm-mc -mcpu=cortex-a8 -triple thumbv7-apple-darwin10 -filetype=obj -o - < %s | llvm-readobj -r --expand-relocs - | FileCheck %s
 
 @ rdar://10038370
 
diff --git a/llvm/test/MC/MachO/ARM/tvos-version-min-load-command.s b/llvm/test/MC/MachO/ARM/tvos-version-min-load-command.s
index 3c9b237d69503..d0bd4dd6b7948 100644
--- a/llvm/test/MC/MachO/ARM/tvos-version-min-load-command.s
+++ b/llvm/test/MC/MachO/ARM/tvos-version-min-load-command.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple armv7-apple-tvos %s -filetype=obj -o - | llvm-readobj --macho-version-min | FileCheck %s
+// RUN: llvm-mc -triple armv7-apple-tvos %s -filetype=obj -o - | llvm-readobj --macho-version-min - | FileCheck %s
 
 
 // Test the formation of the version-min load command in the MachO.
diff --git a/llvm/test/MC/MachO/ARM/watchos-version-min-load-command.s b/llvm/test/MC/MachO/ARM/watchos-version-min-load-command.s
index 3df38fb6a179f..e383ae3da0a81 100644
--- a/llvm/test/MC/MachO/ARM/watchos-version-min-load-command.s
+++ b/llvm/test/MC/MachO/ARM/watchos-version-min-load-command.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple armv7k-apple-watchos %s -filetype=obj -o - | llvm-readobj --macho-version-min | FileCheck %s
+// RUN: llvm-mc -triple armv7k-apple-watchos %s -filetype=obj -o - | llvm-readobj --macho-version-min - | FileCheck %s
 
 
 // Test the formation of the version-min load command in the MachO.
diff --git a/llvm/test/MC/MachO/absolute.s b/llvm/test/MC/MachO/absolute.s
index 7d560d13712f8..7da39bdf14bd3 100644
--- a/llvm/test/MC/MachO/absolute.s
+++ b/llvm/test/MC/MachO/absolute.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj --file-headers -S -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj --file-headers -S -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 
 _bar:
   nop
diff --git a/llvm/test/MC/MachO/absolutize.s b/llvm/test/MC/MachO/absolutize.s
index 1a776d787e3f1..7a2eee903df26 100644
--- a/llvm/test/MC/MachO/absolutize.s
+++ b/llvm/test/MC/MachO/absolutize.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 
 _text_a:
         xorl %eax,%eax
diff --git a/llvm/test/MC/MachO/alias.s b/llvm/test/MC/MachO/alias.s
index aec04c63b68f1..f53fe3507d0f0 100644
--- a/llvm/test/MC/MachO/alias.s
+++ b/llvm/test/MC/MachO/alias.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-macosx10.12.0 %s -filetype=obj | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-macosx10.12.0 %s -filetype=obj | llvm-readobj -r - | FileCheck %s
 
 l_a:
 l_b = l_a
diff --git a/llvm/test/MC/MachO/altentry.s b/llvm/test/MC/MachO/altentry.s
index d2964a1fc190a..8e42cf473bef7 100644
--- a/llvm/test/MC/MachO/altentry.s
+++ b/llvm/test/MC/MachO/altentry.s
@@ -1,4 +1,4 @@
-// RUN:  llvm-mc -triple x86_64-apple-darwin -filetype=obj %s -o - | llvm-readobj --symbols | FileCheck %s
+// RUN:  llvm-mc -triple x86_64-apple-darwin -filetype=obj %s -o - | llvm-readobj --symbols - | FileCheck %s
 
 // CHECK: Symbol {
 // CHECK: Name: _foo
diff --git a/llvm/test/MC/MachO/bss.s b/llvm/test/MC/MachO/bss.s
index 426ae16316548..f0b03242d93b7 100644
--- a/llvm/test/MC/MachO/bss.s
+++ b/llvm/test/MC/MachO/bss.s
@@ -1,7 +1,7 @@
 // The purpose of this test is to verify that bss sections are emitted correctly.
 
-// RUN: llvm-mc -filetype=obj -triple i686-apple-darwin9 %s | llvm-readobj -S | FileCheck %s
-// RUN: llvm-mc -filetype=obj -triple x86_64-apple-darwin9 %s | llvm-readobj -S | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple i686-apple-darwin9 %s | llvm-readobj -S - | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-apple-darwin9 %s | llvm-readobj -S - | FileCheck %s
 
     .bss
     .globl _g0
diff --git a/llvm/test/MC/MachO/build-version-with-sdk-version.s b/llvm/test/MC/MachO/build-version-with-sdk-version.s
index e96c2d520e172..49bdd79f8a02d 100644
--- a/llvm/test/MC/MachO/build-version-with-sdk-version.s
+++ b/llvm/test/MC/MachO/build-version-with-sdk-version.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-macos %s -filetype=obj -o - | llvm-readobj --macho-version-min | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-macos %s -filetype=obj -o - | llvm-readobj --macho-version-min - | FileCheck %s
 
 // Test the formation of the sdk_version component of the version load
 // command in the MachO.
diff --git a/llvm/test/MC/MachO/coal-sections-x86_64.s b/llvm/test/MC/MachO/coal-sections-x86_64.s
index f91373e079357..20d93dfe6b22a 100644
--- a/llvm/test/MC/MachO/coal-sections-x86_64.s
+++ b/llvm/test/MC/MachO/coal-sections-x86_64.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-darwin -filetype=obj %s -o - 2>%t.err | llvm-readobj --sections | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-darwin -filetype=obj %s -o - 2>%t.err | llvm-readobj --sections - | FileCheck %s
 // RUN: FileCheck --check-prefix=WARNING < %t.err %s
 
 // CHECK: Section {
diff --git a/llvm/test/MC/MachO/comm-1.s b/llvm/test/MC/MachO/comm-1.s
index f7c325c13ab01..c8904be7e8ade 100644
--- a/llvm/test/MC/MachO/comm-1.s
+++ b/llvm/test/MC/MachO/comm-1.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 
         .comm           sym_comm_B, 2
         .comm           sym_comm_A, 4
diff --git a/llvm/test/MC/MachO/darwin-complex-difference.s b/llvm/test/MC/MachO/darwin-complex-difference.s
index ae483eff85c67..6bfb880c8747a 100644
--- a/llvm/test/MC/MachO/darwin-complex-difference.s
+++ b/llvm/test/MC/MachO/darwin-complex-difference.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -triple x86_64-apple-darwin10 %s -filetype=obj -o %t.o
-// RUN: llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols < %t.o > %t.dump
+// RUN: llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols < %t.o - > %t.dump
 // RUN: FileCheck < %t.dump %s
         
 _a:
diff --git a/llvm/test/MC/MachO/darwin-x86_64-diff-reloc-assign-2.s b/llvm/test/MC/MachO/darwin-x86_64-diff-reloc-assign-2.s
index e3fa1335a0c43..b01857ed8c38a 100644
--- a/llvm/test/MC/MachO/darwin-x86_64-diff-reloc-assign-2.s
+++ b/llvm/test/MC/MachO/darwin-x86_64-diff-reloc-assign-2.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -r --expand-relocs | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -r --expand-relocs - | FileCheck %s
 
 	.data
 L_var1:
diff --git a/llvm/test/MC/MachO/darwin-x86_64-diff-reloc-assign.s b/llvm/test/MC/MachO/darwin-x86_64-diff-reloc-assign.s
index 7a7919786d411..ae0360fc83a3e 100644
--- a/llvm/test/MC/MachO/darwin-x86_64-diff-reloc-assign.s
+++ b/llvm/test/MC/MachO/darwin-x86_64-diff-reloc-assign.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -r - | FileCheck %s
 
 // Test case for rdar://10743265
 
diff --git a/llvm/test/MC/MachO/darwin-x86_64-diff-relocs.s b/llvm/test/MC/MachO/darwin-x86_64-diff-relocs.s
index c5e022671dcef..8e58565729e23 100644
--- a/llvm/test/MC/MachO/darwin-x86_64-diff-relocs.s
+++ b/llvm/test/MC/MachO/darwin-x86_64-diff-relocs.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 
         .text
 
diff --git a/llvm/test/MC/MachO/darwin-x86_64-nobase-relocs.s b/llvm/test/MC/MachO/darwin-x86_64-nobase-relocs.s
index b974cc673eeae..537aa944f167b 100644
--- a/llvm/test/MC/MachO/darwin-x86_64-nobase-relocs.s
+++ b/llvm/test/MC/MachO/darwin-x86_64-nobase-relocs.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -n -triple x86_64-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --macho-segment | FileCheck %s
+// RUN: llvm-mc -n -triple x86_64-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --macho-segment - | FileCheck %s
 
 // Test case for rdar://10062261
 
diff --git a/llvm/test/MC/MachO/darwin-x86_64-reloc-offsets.s b/llvm/test/MC/MachO/darwin-x86_64-reloc-offsets.s
index fc73ec633c280..e4e48de86bb20 100644
--- a/llvm/test/MC/MachO/darwin-x86_64-reloc-offsets.s
+++ b/llvm/test/MC/MachO/darwin-x86_64-reloc-offsets.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 
         .data
 
diff --git a/llvm/test/MC/MachO/darwin-x86_64-reloc.s b/llvm/test/MC/MachO/darwin-x86_64-reloc.s
index 32e079879eae5..2843b4f39cbe2 100644
--- a/llvm/test/MC/MachO/darwin-x86_64-reloc.s
+++ b/llvm/test/MC/MachO/darwin-x86_64-reloc.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -n -triple x86_64-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -r --expand-relocs | FileCheck %s
+// RUN: llvm-mc -n -triple x86_64-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -r --expand-relocs - | FileCheck %s
 
 // These examples are taken from <mach-o/x86_64/reloc.h>.
 
diff --git a/llvm/test/MC/MachO/data.s b/llvm/test/MC/MachO/data.s
index 299e55e88e9a4..0a02ad20979c0 100644
--- a/llvm/test/MC/MachO/data.s
+++ b/llvm/test/MC/MachO/data.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --macho-segment | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --macho-segment - | FileCheck %s
 
         .data
         .ascii "hello"
diff --git a/llvm/test/MC/MachO/debug_frame.s b/llvm/test/MC/MachO/debug_frame.s
index b6601f78885fe..cc3df008dbb79 100644
--- a/llvm/test/MC/MachO/debug_frame.s
+++ b/llvm/test/MC/MachO/debug_frame.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin %s -filetype=obj -o - | llvm-readobj -S --sd -r | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin %s -filetype=obj -o - | llvm-readobj -S --sd -r - | FileCheck %s
 
 // Make sure MC can handle file level .cfi_startproc and .cfi_endproc that creates
 // an empty frame.
diff --git a/llvm/test/MC/MachO/diff-with-two-sections.s b/llvm/test/MC/MachO/diff-with-two-sections.s
index 6302ede13e10e..87a8ad960e4c8 100644
--- a/llvm/test/MC/MachO/diff-with-two-sections.s
+++ b/llvm/test/MC/MachO/diff-with-two-sections.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --macho-segment --macho-version-min | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --macho-segment --macho-version-min - | FileCheck %s
 
 	.section	__TEXT,__text,regular,pure_instructions
 Leh_func_begin0:
diff --git a/llvm/test/MC/MachO/direction_labels.s b/llvm/test/MC/MachO/direction_labels.s
index 0dac4f7377ead..a5f3af9f27f0e 100644
--- a/llvm/test/MC/MachO/direction_labels.s
+++ b/llvm/test/MC/MachO/direction_labels.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 
 direction_labels:
 10:     nop
diff --git a/llvm/test/MC/MachO/eh-frame-reloc.s b/llvm/test/MC/MachO/eh-frame-reloc.s
index c39ce8479cccb..9da4fb7d1574a 100644
--- a/llvm/test/MC/MachO/eh-frame-reloc.s
+++ b/llvm/test/MC/MachO/eh-frame-reloc.s
@@ -1,10 +1,10 @@
-// RUN: llvm-mc < %s -triple=x86_64-apple-macosx10.7 -filetype=obj | llvm-readobj -r | FileCheck %s
-// RUN: llvm-mc < %s -triple=x86_64-apple-macosx10.6 -filetype=obj | llvm-readobj -r | FileCheck %s
-// RUN: llvm-mc < %s -triple=x86_64-apple-ios7.0.0 -filetype=obj | llvm-readobj -r | FileCheck %s
-// RUN: llvm-mc < %s -triple=x86_64-apple-macosx10.5 -filetype=obj | llvm-readobj -r | FileCheck %s
-// RUN: llvm-mc < %s -triple=i686-apple-macosx10.6 -filetype=obj | llvm-readobj -r | FileCheck %s
-// RUN: llvm-mc < %s -triple=i686-apple-macosx10.5 -filetype=obj | llvm-readobj -r | FileCheck %s
-// RUN: llvm-mc < %s -triple=i686-apple-macosx10.4 -filetype=obj | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc < %s -triple=x86_64-apple-macosx10.7 -filetype=obj | llvm-readobj -r - | FileCheck %s
+// RUN: llvm-mc < %s -triple=x86_64-apple-macosx10.6 -filetype=obj | llvm-readobj -r - | FileCheck %s
+// RUN: llvm-mc < %s -triple=x86_64-apple-ios7.0.0 -filetype=obj | llvm-readobj -r - | FileCheck %s
+// RUN: llvm-mc < %s -triple=x86_64-apple-macosx10.5 -filetype=obj | llvm-readobj -r - | FileCheck %s
+// RUN: llvm-mc < %s -triple=i686-apple-macosx10.6 -filetype=obj | llvm-readobj -r - | FileCheck %s
+// RUN: llvm-mc < %s -triple=i686-apple-macosx10.5 -filetype=obj | llvm-readobj -r - | FileCheck %s
+// RUN: llvm-mc < %s -triple=i686-apple-macosx10.4 -filetype=obj | llvm-readobj -r - | FileCheck %s
 
 	.globl	_bar
 	.align	4, 0x90
diff --git a/llvm/test/MC/MachO/file.s b/llvm/test/MC/MachO/file.s
index 0076fa5647023..cda04b3ba89fd 100644
--- a/llvm/test/MC/MachO/file.s
+++ b/llvm/test/MC/MachO/file.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -S --section-data | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -S --section-data - | FileCheck %s
 
 // This number-less file directive is ignored on MachO.
         .file "bar/baz.s"
diff --git a/llvm/test/MC/MachO/i386-large-relocations.s b/llvm/test/MC/MachO/i386-large-relocations.s
index e8805d37e4a1c..04566d9a70b79 100644
--- a/llvm/test/MC/MachO/i386-large-relocations.s
+++ b/llvm/test/MC/MachO/i386-large-relocations.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin10 %s -filetype=obj -o - | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin10 %s -filetype=obj -o - | llvm-readobj -r - | FileCheck %s
 
 .space 0x1ed280
        .section        __DATA,__const
diff --git a/llvm/test/MC/MachO/indirect-symbols.s b/llvm/test/MC/MachO/indirect-symbols.s
index d5221fb251494..28f60401c4a5e 100644
--- a/llvm/test/MC/MachO/indirect-symbols.s
+++ b/llvm/test/MC/MachO/indirect-symbols.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 
 _b:
         _c = 0
diff --git a/llvm/test/MC/MachO/jcc.s b/llvm/test/MC/MachO/jcc.s
index 9e7936dfec237..c5b80d25d2bfc 100644
--- a/llvm/test/MC/MachO/jcc.s
+++ b/llvm/test/MC/MachO/jcc.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --macho-segment | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --macho-segment - | FileCheck %s
 
    ja 1f
 1: nop
diff --git a/llvm/test/MC/MachO/lcomm-attributes.s b/llvm/test/MC/MachO/lcomm-attributes.s
index c3d2d60764eaf..c553fa5c3da11 100644
--- a/llvm/test/MC/MachO/lcomm-attributes.s
+++ b/llvm/test/MC/MachO/lcomm-attributes.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 
         // Note, this test intentionally mismatches Darwin 'as', which loses the
 	// following global marker.
diff --git a/llvm/test/MC/MachO/linker-option-2.s b/llvm/test/MC/MachO/linker-option-2.s
index 8f9a8ffbb094e..477b44bbdc5f2 100644
--- a/llvm/test/MC/MachO/linker-option-2.s
+++ b/llvm/test/MC/MachO/linker-option-2.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -n -triple x86_64-apple-darwin10 %s -filetype=obj | llvm-readobj --macho-linker-options | FileCheck %s
+// RUN: llvm-mc -n -triple x86_64-apple-darwin10 %s -filetype=obj | llvm-readobj --macho-linker-options - | FileCheck %s
 
 .linker_option "a"
 .linker_option "a", "b"
diff --git a/llvm/test/MC/MachO/linker-options.ll b/llvm/test/MC/MachO/linker-options.ll
index 86c1d8e90b5cb..0f678cd3ad16c 100644
--- a/llvm/test/MC/MachO/linker-options.ll
+++ b/llvm/test/MC/MachO/linker-options.ll
@@ -4,7 +4,7 @@
 ; CHECK-ASM: .linker_option "-lz"
 ; CHECK-ASM-NEXT: .linker_option "-framework", "Cocoa"
 
-; RUN: llc -O0 -mtriple=x86_64-apple-darwin -filetype=obj -o - %s | llvm-readobj --macho-linker-options > %t
+; RUN: llc -O0 -mtriple=x86_64-apple-darwin -filetype=obj -o - %s | llvm-readobj --macho-linker-options - > %t
 ; RUN: FileCheck --check-prefix=CHECK-OBJ < %t %s
 
 ; CHECK-OBJ: Linker Options {
diff --git a/llvm/test/MC/MachO/loc.s b/llvm/test/MC/MachO/loc.s
index 143142027ad44..10dfc5d55d706 100644
--- a/llvm/test/MC/MachO/loc.s
+++ b/llvm/test/MC/MachO/loc.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -r -S --sd | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -r -S --sd - | FileCheck %s
         .file   2 "foo"
         .file   1 "bar"
         .loc    2 64 0
diff --git a/llvm/test/MC/MachO/osx-version-min-load-command-with-sdk.s b/llvm/test/MC/MachO/osx-version-min-load-command-with-sdk.s
index d2ad463d1d6e5..358b4694dca24 100644
--- a/llvm/test/MC/MachO/osx-version-min-load-command-with-sdk.s
+++ b/llvm/test/MC/MachO/osx-version-min-load-command-with-sdk.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-macos %s -filetype=obj -o - | llvm-readobj --macho-version-min | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-macos %s -filetype=obj -o - | llvm-readobj --macho-version-min - | FileCheck %s
 
 // Test the formation of the sdk_version component of the version load
 // command in the MachO.
diff --git a/llvm/test/MC/MachO/osx-version-min-load-command.s b/llvm/test/MC/MachO/osx-version-min-load-command.s
index 2218d556bce51..23886a84da740 100644
--- a/llvm/test/MC/MachO/osx-version-min-load-command.s
+++ b/llvm/test/MC/MachO/osx-version-min-load-command.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-darwin %s -filetype=obj -o - | llvm-readobj --macho-version-min | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-darwin %s -filetype=obj -o - | llvm-readobj --macho-version-min - | FileCheck %s
 
 // Test the formation of the version-min load command in the MachO.
 // use a nonsense but well formed version.
diff --git a/llvm/test/MC/MachO/pcrel-to-other-section.s b/llvm/test/MC/MachO/pcrel-to-other-section.s
index e4f77c8ce3222..e2325ce8a6977 100644
--- a/llvm/test/MC/MachO/pcrel-to-other-section.s
+++ b/llvm/test/MC/MachO/pcrel-to-other-section.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 
 nop
 	.section	__TEXT,__StaticInit,regular,pure_instructions
diff --git a/llvm/test/MC/MachO/relax-jumps.s b/llvm/test/MC/MachO/relax-jumps.s
index 104fd8c849838..6d4eedd03e5ff 100644
--- a/llvm/test/MC/MachO/relax-jumps.s
+++ b/llvm/test/MC/MachO/relax-jumps.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -S --sd | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -S --sd - | FileCheck %s
 
 // FIXME: This is a horrible way of checking the output, we need an llvm-mc
 // based 'otool'. Use:
diff --git a/llvm/test/MC/MachO/relax-recompute-align.s b/llvm/test/MC/MachO/relax-recompute-align.s
index 30a5d82331ac6..bfb17b71274e9 100644
--- a/llvm/test/MC/MachO/relax-recompute-align.s
+++ b/llvm/test/MC/MachO/relax-recompute-align.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -S | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -S - | FileCheck %s
 
 // FIXME: This is a horrible way of checking the output, we need an llvm-mc
 // based 'otool'.
diff --git a/llvm/test/MC/MachO/reloc-diff.s b/llvm/test/MC/MachO/reloc-diff.s
index f9d58d97e529e..8b2e7606b3542 100644
--- a/llvm/test/MC/MachO/reloc-diff.s
+++ b/llvm/test/MC/MachO/reloc-diff.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -r - | FileCheck %s
 
 _local_def:
         .globl _external_def
diff --git a/llvm/test/MC/MachO/reloc-pcrel-offset.s b/llvm/test/MC/MachO/reloc-pcrel-offset.s
index 281d785c8607c..5711069167c47 100644
--- a/llvm/test/MC/MachO/reloc-pcrel-offset.s
+++ b/llvm/test/MC/MachO/reloc-pcrel-offset.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -n -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -r -S --sd | FileCheck %s
+// RUN: llvm-mc -n -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -r -S --sd - | FileCheck %s
 
         .data
         .long 0
diff --git a/llvm/test/MC/MachO/reloc-pcrel.s b/llvm/test/MC/MachO/reloc-pcrel.s
index 88f8828e68740..40688264e3908 100644
--- a/llvm/test/MC/MachO/reloc-pcrel.s
+++ b/llvm/test/MC/MachO/reloc-pcrel.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -r - | FileCheck %s
 
         xorl %eax,%eax
 
diff --git a/llvm/test/MC/MachO/reloc.s b/llvm/test/MC/MachO/reloc.s
index bab5d63d27f45..02863c9c39fa7 100644
--- a/llvm/test/MC/MachO/reloc.s
+++ b/llvm/test/MC/MachO/reloc.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -r --expand-relocs | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -r --expand-relocs - | FileCheck %s
 
         .data
         .long undef
diff --git a/llvm/test/MC/MachO/section-align-1.s b/llvm/test/MC/MachO/section-align-1.s
index fa697af58b5e7..c8ab216868a65 100644
--- a/llvm/test/MC/MachO/section-align-1.s
+++ b/llvm/test/MC/MachO/section-align-1.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --symbols --file-headers -S --macho-indirect-symbols --macho-dysymtab --macho-segment | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --symbols --file-headers -S --macho-indirect-symbols --macho-dysymtab --macho-segment - | FileCheck %s
 
 name:
         .byte 0
diff --git a/llvm/test/MC/MachO/section-align-2.s b/llvm/test/MC/MachO/section-align-2.s
index 39215e20d5755..a32d77e556b92 100644
--- a/llvm/test/MC/MachO/section-align-2.s
+++ b/llvm/test/MC/MachO/section-align-2.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 
         .byte 0
 
diff --git a/llvm/test/MC/MachO/section-flags.s b/llvm/test/MC/MachO/section-flags.s
index 34fb60e9e7f00..8d35d32386ca6 100644
--- a/llvm/test/MC/MachO/section-flags.s
+++ b/llvm/test/MC/MachO/section-flags.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -S --sd | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -S --sd - | FileCheck %s
 
         .text
 
diff --git a/llvm/test/MC/MachO/string-table.s b/llvm/test/MC/MachO/string-table.s
index e43ff03d311ff..e875b2e30ec1a 100644
--- a/llvm/test/MC/MachO/string-table.s
+++ b/llvm/test/MC/MachO/string-table.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
         
 	movl	$a, b
         
diff --git a/llvm/test/MC/MachO/symbol-diff.s b/llvm/test/MC/MachO/symbol-diff.s
index 395e223b58375..fc8292c90af0e 100644
--- a/llvm/test/MC/MachO/symbol-diff.s
+++ b/llvm/test/MC/MachO/symbol-diff.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 _g:
 LFB2:
 	.section __TEXT,__eh_frame,coalesced,no_toc+strip_static_syms+live_support
diff --git a/llvm/test/MC/MachO/symbol-flags.s b/llvm/test/MC/MachO/symbol-flags.s
index 991ea5628c42d..698d152e99af0 100644
--- a/llvm/test/MC/MachO/symbol-flags.s
+++ b/llvm/test/MC/MachO/symbol-flags.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 
         .reference sym_ref_A
         .reference sym_ref_def_A
diff --git a/llvm/test/MC/MachO/symbol-indirect.s b/llvm/test/MC/MachO/symbol-indirect.s
index 286e29283ab77..3f1ecaae62e13 100644
--- a/llvm/test/MC/MachO/symbol-indirect.s
+++ b/llvm/test/MC/MachO/symbol-indirect.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 
 // FIXME: We are missing a lot of diagnostics on this kind of stuff which the
 // assembler has.
diff --git a/llvm/test/MC/MachO/symbols-1.s b/llvm/test/MC/MachO/symbols-1.s
index 496ef3d17f63b..ec2ff7cda34d8 100644
--- a/llvm/test/MC/MachO/symbols-1.s
+++ b/llvm/test/MC/MachO/symbols-1.s
@@ -1,5 +1,5 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck -check-prefix CHECK-X86_32 %s
-// RUN: llvm-mc -triple x86_64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj --file-headers -S -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck -check-prefix CHECK-X86_64 %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck -check-prefix CHECK-X86_32 %s
+// RUN: llvm-mc -triple x86_64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj --file-headers -S -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck -check-prefix CHECK-X86_64 %s
 
 sym_local_B:
 .globl sym_globl_def_B
diff --git a/llvm/test/MC/MachO/tbss.s b/llvm/test/MC/MachO/tbss.s
index 37a089f462751..4df851ab09e56 100644
--- a/llvm/test/MC/MachO/tbss.s
+++ b/llvm/test/MC/MachO/tbss.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-darwin %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-darwin %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 
 .tbss _a$tlv$init, 4
 .tbss _b$tlv$init, 4, 3
diff --git a/llvm/test/MC/MachO/tdata.s b/llvm/test/MC/MachO/tdata.s
index e3d35a7443a6f..a9689ec04740f 100644
--- a/llvm/test/MC/MachO/tdata.s
+++ b/llvm/test/MC/MachO/tdata.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-darwin %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-darwin %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 .tdata
 _a$tlv$init:
 	.long 4
diff --git a/llvm/test/MC/MachO/temp-labels.s b/llvm/test/MC/MachO/temp-labels.s
index 8be5919bf02b4..97b32d8dfdfd2 100644
--- a/llvm/test/MC/MachO/temp-labels.s
+++ b/llvm/test/MC/MachO/temp-labels.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-darwin10 %s -filetype=obj -save-temp-labels -o - | llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-darwin10 %s -filetype=obj -save-temp-labels -o - | llvm-readobj --symbols - | FileCheck %s
 
 _f0:
         .long 0
diff --git a/llvm/test/MC/MachO/thread_init_func.s b/llvm/test/MC/MachO/thread_init_func.s
index 58d3fac801816..166d57702d2da 100644
--- a/llvm/test/MC/MachO/thread_init_func.s
+++ b/llvm/test/MC/MachO/thread_init_func.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-darwin %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --macho-segment | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-darwin %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --macho-segment - | FileCheck %s
 
 	.thread_init_func
 	.quad 0
diff --git a/llvm/test/MC/MachO/tls.s b/llvm/test/MC/MachO/tls.s
index 671184a8406d6..d2fab68632a13 100644
--- a/llvm/test/MC/MachO/tls.s
+++ b/llvm/test/MC/MachO/tls.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-darwin %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-darwin %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 
         .section        __TEXT,__text,regular,pure_instructions
         .section        __DATA,__thread_data,thread_local_regular
diff --git a/llvm/test/MC/MachO/tlv-bss.ll b/llvm/test/MC/MachO/tlv-bss.ll
index 7840367276a3f..58587766d464e 100644
--- a/llvm/test/MC/MachO/tlv-bss.ll
+++ b/llvm/test/MC/MachO/tlv-bss.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -mtriple=x86_64-apple-darwin12 -filetype=obj -o - %s | llvm-readobj -S | FileCheck %s
+; RUN: llc -O0 -mtriple=x86_64-apple-darwin12 -filetype=obj -o - %s | llvm-readobj -S - | FileCheck %s
 ; Test that we emit weak_odr thread_locals correctly into the thread_bss section
 ; PR15972
 
diff --git a/llvm/test/MC/MachO/tlv-reloc.s b/llvm/test/MC/MachO/tlv-reloc.s
index a429fdd2376af..ee8192dc75439 100644
--- a/llvm/test/MC/MachO/tlv-reloc.s
+++ b/llvm/test/MC/MachO/tlv-reloc.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-darwin %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-darwin %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 
 .tdata
 _a$tlv$init:
diff --git a/llvm/test/MC/MachO/tlv.s b/llvm/test/MC/MachO/tlv.s
index 774436bd3192e..70a9d43190d6b 100644
--- a/llvm/test/MC/MachO/tlv.s
+++ b/llvm/test/MC/MachO/tlv.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-darwin %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-darwin %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 
 	.tlv
 	.globl _a
diff --git a/llvm/test/MC/MachO/values.s b/llvm/test/MC/MachO/values.s
index 5a406acfde0b7..b6f71be31e65b 100644
--- a/llvm/test/MC/MachO/values.s
+++ b/llvm/test/MC/MachO/values.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 
         .long 0
 text_def_int:
diff --git a/llvm/test/MC/MachO/variable-exprs.s b/llvm/test/MC/MachO/variable-exprs.s
index b4b57811529d8..6babb97f60fa0 100644
--- a/llvm/test/MC/MachO/variable-exprs.s
+++ b/llvm/test/MC/MachO/variable-exprs.s
@@ -1,9 +1,9 @@
 // RUN: llvm-mc -triple i386-apple-darwin10 %s -filetype=obj -o %t.o
-// RUN: llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols < %t.o > %t.dump
+// RUN: llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols < %t.o - > %t.dump
 // RUN: FileCheck --check-prefix=CHECK-I386 < %t.dump %s
 
 // RUN: llvm-mc -triple x86_64-apple-darwin10 %s -filetype=obj -o %t.o
-// RUN: llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols < %t.o > %t.dump
+// RUN: llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols < %t.o - > %t.dump
 // RUN: FileCheck --check-prefix=CHECK-X86_64 < %t.dump %s
 
 .data
diff --git a/llvm/test/MC/MachO/weakdef.s b/llvm/test/MC/MachO/weakdef.s
index 497aad844282e..200be41cd4c73 100644
--- a/llvm/test/MC/MachO/weakdef.s
+++ b/llvm/test/MC/MachO/weakdef.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 
 	.section	__DATA,__datacoal_nt,coalesced
 	.section	__TEXT,__const_coal,coalesced
diff --git a/llvm/test/MC/MachO/x86-data-in-code.s b/llvm/test/MC/MachO/x86-data-in-code.s
index e7b2406e5d095..346dae2955d21 100644
--- a/llvm/test/MC/MachO/x86-data-in-code.s
+++ b/llvm/test/MC/MachO/x86-data-in-code.s
@@ -1,5 +1,5 @@
-// RUN: llvm-mc -triple x86_64-apple-darwin10 -filetype=obj -o - < %s | llvm-readobj --macho-data-in-code | FileCheck %s
-// RUN: llvm-mc -triple i686-apple-darwin10 -filetype=obj -o - < %s | llvm-readobj --macho-data-in-code | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-darwin10 -filetype=obj -o - < %s | llvm-readobj --macho-data-in-code - | FileCheck %s
+// RUN: llvm-mc -triple i686-apple-darwin10 -filetype=obj -o - < %s | llvm-readobj --macho-data-in-code - | FileCheck %s
 
         .text
 _foo:
diff --git a/llvm/test/MC/MachO/x86_32-optimal_nop.s b/llvm/test/MC/MachO/x86_32-optimal_nop.s
index 23286a50615fe..bb784b9da9954 100644
--- a/llvm/test/MC/MachO/x86_32-optimal_nop.s
+++ b/llvm/test/MC/MachO/x86_32-optimal_nop.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 -mcpu=pentiumpro %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 -mcpu=pentiumpro %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 
 # 1 byte nop test
         .align 4, 0 # start with 16 byte alignment filled with zeros
diff --git a/llvm/test/MC/MachO/x86_32-scattered-reloc-fallback.s b/llvm/test/MC/MachO/x86_32-scattered-reloc-fallback.s
index 909daa2bd2760..c9e25f675a456 100644
--- a/llvm/test/MC/MachO/x86_32-scattered-reloc-fallback.s
+++ b/llvm/test/MC/MachO/x86_32-scattered-reloc-fallback.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -S --sd | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -S --sd - | FileCheck %s
 
 // rdar://15526046
 
diff --git a/llvm/test/MC/MachO/x86_32-sections.s b/llvm/test/MC/MachO/x86_32-sections.s
index 13b8ed3ca940b..a81fa0dcd17f7 100644
--- a/llvm/test/MC/MachO/x86_32-sections.s
+++ b/llvm/test/MC/MachO/x86_32-sections.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd --macho-segment | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd --macho-segment - | FileCheck %s
 
         .text
 	.section	__TEXT,__text,regular,pure_instructions
diff --git a/llvm/test/MC/MachO/x86_32-symbols.s b/llvm/test/MC/MachO/x86_32-symbols.s
index 35bc90491a024..66931fff30a4a 100644
--- a/llvm/test/MC/MachO/x86_32-symbols.s
+++ b/llvm/test/MC/MachO/x86_32-symbols.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 
         .text
 L0:
diff --git a/llvm/test/MC/MachO/x86_64-mergeable.s b/llvm/test/MC/MachO/x86_64-mergeable.s
index b7933f92c0c4f..2c6be44eb35c4 100644
--- a/llvm/test/MC/MachO/x86_64-mergeable.s
+++ b/llvm/test/MC/MachO/x86_64-mergeable.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-darwin14 %s -filetype=obj -o - | llvm-readobj -r --expand-relocs | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-darwin14 %s -filetype=obj -o - | llvm-readobj -r --expand-relocs - | FileCheck %s
 
 // Test that we "S + K" produce a relocation with a symbol, but just S produces
 // a relocation with the section.
diff --git a/llvm/test/MC/MachO/x86_64-reloc-arithmetic.s b/llvm/test/MC/MachO/x86_64-reloc-arithmetic.s
index cbcb77d4194d8..0e3d2712e484a 100644
--- a/llvm/test/MC/MachO/x86_64-reloc-arithmetic.s
+++ b/llvm/test/MC/MachO/x86_64-reloc-arithmetic.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj -r --symbols | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj -r --symbols - | FileCheck %s
 
 // rdar://9906375
 .org 0x100
diff --git a/llvm/test/MC/MachO/x86_64-sections.s b/llvm/test/MC/MachO/x86_64-sections.s
index 450ba3cd887c3..86b65072a9071 100644
--- a/llvm/test/MC/MachO/x86_64-sections.s
+++ b/llvm/test/MC/MachO/x86_64-sections.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --macho-segment | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --macho-segment - | FileCheck %s
 
         .text
 	.section	__TEXT,__text,regular,pure_instructions
diff --git a/llvm/test/MC/MachO/x86_64-symbols.s b/llvm/test/MC/MachO/x86_64-symbols.s
index e781eeb272c7b..789def09b275d 100644
--- a/llvm/test/MC/MachO/x86_64-symbols.s
+++ b/llvm/test/MC/MachO/x86_64-symbols.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-darwin10 %s -filetype=obj -o - | llvm-readobj --symbols - | FileCheck %s
 
         .text
 L0:
diff --git a/llvm/test/MC/MachO/zero-space.s b/llvm/test/MC/MachO/zero-space.s
index e809aa70f7fb2..753bd7851bf0f 100644
--- a/llvm/test/MC/MachO/zero-space.s
+++ b/llvm/test/MC/MachO/zero-space.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 
         .const
         .p2align 6
diff --git a/llvm/test/MC/MachO/zerofill-1.s b/llvm/test/MC/MachO/zerofill-1.s
index 5ca6b24ee1e30..499364b909c36 100644
--- a/llvm/test/MC/MachO/zerofill-1.s
+++ b/llvm/test/MC/MachO/zerofill-1.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 
         .text
         .byte 0                 // Align to 2**3 bytes, not 2**1
diff --git a/llvm/test/MC/MachO/zerofill-2.s b/llvm/test/MC/MachO/zerofill-2.s
index 97bde10fa4347..3ac56a9af5483 100644
--- a/llvm/test/MC/MachO/zerofill-2.s
+++ b/llvm/test/MC/MachO/zerofill-2.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 
         .byte 0
 
diff --git a/llvm/test/MC/MachO/zerofill-3.s b/llvm/test/MC/MachO/zerofill-3.s
index c8fb80f52904a..f894592aa045e 100644
--- a/llvm/test/MC/MachO/zerofill-3.s
+++ b/llvm/test/MC/MachO/zerofill-3.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 
         // FIXME: We don't get the order right currently, the assembler first
         // orders the symbols, then assigns addresses. :(
diff --git a/llvm/test/MC/MachO/zerofill-4.s b/llvm/test/MC/MachO/zerofill-4.s
index a095e9a57b6de..5724fa5745c54 100644
--- a/llvm/test/MC/MachO/zerofill-4.s
+++ b/llvm/test/MC/MachO/zerofill-4.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --symbols - | FileCheck %s
 
 .zerofill __DATA,__bss,_fill0,1,0
 .zerofill __DATA,__bss,_a,4,2
diff --git a/llvm/test/MC/MachO/zerofill-5.s b/llvm/test/MC/MachO/zerofill-5.s
index 2eb494a038b10..b25629171db8b 100644
--- a/llvm/test/MC/MachO/zerofill-5.s
+++ b/llvm/test/MC/MachO/zerofill-5.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --file-headers -S --sd -r --symbols --macho-segment --macho-dysymtab --macho-indirect-symbols - | FileCheck %s
 
 	.text
 	.align	3
diff --git a/llvm/test/MC/MachO/zerofill-sect-align.s b/llvm/test/MC/MachO/zerofill-sect-align.s
index c595bdc785d13..d681b2fe07e11 100644
--- a/llvm/test/MC/MachO/zerofill-sect-align.s
+++ b/llvm/test/MC/MachO/zerofill-sect-align.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj --symbols - | FileCheck %s
 //
 // Check that the section itself is aligned.
 
diff --git a/llvm/test/MC/Mips/cfi-advance-loc.s b/llvm/test/MC/Mips/cfi-advance-loc.s
index 407ad3faffe30..9c4d3c516c42c 100644
--- a/llvm/test/MC/Mips/cfi-advance-loc.s
+++ b/llvm/test/MC/Mips/cfi-advance-loc.s
@@ -1,5 +1,5 @@
-// RUN: llvm-mc -filetype=obj -triple mipsel-pc-Linux-gnu %s -o - | llvm-readobj  -S --sd | FileCheck %s -check-prefix=CHECK-LE
-// RUN: llvm-mc -filetype=obj -triple mips-pc-linux-gnu %s -o - | llvm-readobj -S --sd | FileCheck %s -check-prefix=CHECK-BE
+// RUN: llvm-mc -filetype=obj -triple mipsel-pc-Linux-gnu %s -o - | llvm-readobj  -S --sd - | FileCheck %s -check-prefix=CHECK-LE
+// RUN: llvm-mc -filetype=obj -triple mips-pc-linux-gnu %s -o - | llvm-readobj -S --sd - | FileCheck %s -check-prefix=CHECK-BE
 
 // test that this produces a correctly encoded cfi_advance_loc for both endians.
 
diff --git a/llvm/test/MC/Mips/directive-ent.s b/llvm/test/MC/Mips/directive-ent.s
index 4271c046c0eef..c5166882c6d34 100644
--- a/llvm/test/MC/Mips/directive-ent.s
+++ b/llvm/test/MC/Mips/directive-ent.s
@@ -4,14 +4,14 @@
 # RUN: llvm-mc -mcpu=mips32 -triple mips-unknown-unknown %s | \
 # RUN:     FileCheck -check-prefix=ASM %s
 # RUN: llvm-mc -filetype=obj -mcpu=mips32 -triple mips-unknown-unknown %s | \
-# RUN:     llvm-readobj --symbols | \
+# RUN:     llvm-readobj --symbols - | \
 # RUN:     FileCheck -check-prefixes=OBJ,OBJ-32 %s
 #
 # RUN: llvm-mc -mcpu=mips32 -mattr=micromips -triple mips-unknown-unknown %s | \
 # RUN:     FileCheck -check-prefix=ASM %s
 # RUN: llvm-mc -filetype=obj -mcpu=mips32 -mattr=micromips \
 # RUN:     -triple mips-unknown-unknown %s | \
-# RUN:     llvm-readobj --symbols | \
+# RUN:     llvm-readobj --symbols - | \
 # RUN:     FileCheck -check-prefixes=OBJ,OBJ-MM %s
 #
     .ent a
diff --git a/llvm/test/MC/Mips/elf-N32.s b/llvm/test/MC/Mips/elf-N32.s
index 853973152b3de..df5df392fcce0 100644
--- a/llvm/test/MC/Mips/elf-N32.s
+++ b/llvm/test/MC/Mips/elf-N32.s
@@ -1,11 +1,11 @@
 // Check generation of N32 ABI relocations.
 
 // RUN: llvm-mc -filetype=obj -triple=mips64-linux-gnu -mcpu=mips3 \
-// RUN:         -target-abi=n32  %s -o - | llvm-readobj -r | FileCheck %s
+// RUN:         -target-abi=n32  %s -o - | llvm-readobj -r - | FileCheck %s
 // RUN: llvm-mc -filetype=obj -triple=mips64-linux-gnuabin32 -mcpu=mips3 \
-// RUN:          %s -o - | llvm-readobj -r | FileCheck %s
+// RUN:          %s -o - | llvm-readobj -r - | FileCheck %s
 // RUN: llvm-mc -filetype=obj -triple=mipsn32 -mcpu=mips3 \
-// RUN:          %s -o - | llvm-readobj -r | FileCheck %s
+// RUN:          %s -o - | llvm-readobj -r - | FileCheck %s
 
 // CHECK:      Relocations [
 // CHECK-NEXT:   Section (3) .rela.text {
diff --git a/llvm/test/MC/Mips/elf-N64.s b/llvm/test/MC/Mips/elf-N64.s
index 6be46ddd8a96a..c1d9bd061c210 100644
--- a/llvm/test/MC/Mips/elf-N64.s
+++ b/llvm/test/MC/Mips/elf-N64.s
@@ -1,5 +1,5 @@
-// RUN: llvm-mc -filetype=obj -triple=mips64el-pc-linux -mcpu=mips64  %s -o - | llvm-readobj -r | FileCheck %s
-// RUN: llvm-mc -filetype=obj -triple=mips64-pc-linux -mcpu=mips64  %s -o - | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple=mips64el-pc-linux -mcpu=mips64  %s -o - | llvm-readobj -r - | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple=mips64-pc-linux -mcpu=mips64  %s -o - | llvm-readobj -r - | FileCheck %s
 
 // Check for N64 relocation production.
 // Check that the appropriate relocations were created.
diff --git a/llvm/test/MC/Mips/elf-debug-section.s b/llvm/test/MC/Mips/elf-debug-section.s
index 8e936f7bac5f5..237fef12703ca 100644
--- a/llvm/test/MC/Mips/elf-debug-section.s
+++ b/llvm/test/MC/Mips/elf-debug-section.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -filetype=obj -triple=mips-linux-gnu -g %s -o - \
-# RUN:   | llvm-readobj -S | FileCheck %s
+# RUN:   | llvm-readobj -S - | FileCheck %s
 
 # CHECK:      Section {
 # CHECK:        Name: .debug_line
diff --git a/llvm/test/MC/Mips/elf-gprel-32-64.s b/llvm/test/MC/Mips/elf-gprel-32-64.s
index b590c97b278ad..b16f587ccced8 100644
--- a/llvm/test/MC/Mips/elf-gprel-32-64.s
+++ b/llvm/test/MC/Mips/elf-gprel-32-64.s
@@ -1,8 +1,8 @@
 // RUN: llvm-mc -filetype=obj -triple=mips64el-pc-linux -mcpu=mips64 %s -o - \
-// RUN: | llvm-readobj -r \
+// RUN: | llvm-readobj -r - \
 // RUN: | FileCheck %s
 // RUN: llvm-mc -filetype=obj -triple=mips64-pc-linux -mcpu=mips64 %s -o - \
-// RUN: | llvm-readobj -r \
+// RUN: | llvm-readobj -r - \
 // RUN: | FileCheck %s
 
 // Check that the appropriate relocations were created.
diff --git a/llvm/test/MC/Mips/elf-relsym.s b/llvm/test/MC/Mips/elf-relsym.s
index e0b384ca52790..b8c2f89e82e6d 100644
--- a/llvm/test/MC/Mips/elf-relsym.s
+++ b/llvm/test/MC/Mips/elf-relsym.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux %s -o - | llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux %s -o - | llvm-readobj --symbols - | FileCheck %s
 
 // Check that the appropriate symbols were created.
 
diff --git a/llvm/test/MC/Mips/elf-tls.s b/llvm/test/MC/Mips/elf-tls.s
index 23a52e1ef0537..43730974beeb4 100644
--- a/llvm/test/MC/Mips/elf-tls.s
+++ b/llvm/test/MC/Mips/elf-tls.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux %s -o - | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux %s -o - | llvm-readobj -r - | FileCheck %s
 
 // Check that the appropriate relocations were created.
 
diff --git a/llvm/test/MC/Mips/elf_basic.s b/llvm/test/MC/Mips/elf_basic.s
index 4739247f67fac..be6e24823c621 100644
--- a/llvm/test/MC/Mips/elf_basic.s
+++ b/llvm/test/MC/Mips/elf_basic.s
@@ -1,11 +1,11 @@
 // 32 bit big endian
-// RUN: llvm-mc -filetype=obj -triple mips-unknown-linux %s -o - | llvm-readobj -h | FileCheck -check-prefix=CHECK-BE32 %s
+// RUN: llvm-mc -filetype=obj -triple mips-unknown-linux %s -o - | llvm-readobj -h - | FileCheck -check-prefix=CHECK-BE32 %s
 // 32 bit little endian
-// RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux %s -o - | llvm-readobj -h | FileCheck -check-prefix=CHECK-LE32 %s
+// RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux %s -o - | llvm-readobj -h - | FileCheck -check-prefix=CHECK-LE32 %s
 // 64 bit big endian
-// RUN: llvm-mc -filetype=obj -arch=mips64 -triple mips64-unknown-linux %s -o - | llvm-readobj -h | FileCheck -check-prefix=CHECK-BE64 %s
+// RUN: llvm-mc -filetype=obj -arch=mips64 -triple mips64-unknown-linux %s -o - | llvm-readobj -h - | FileCheck -check-prefix=CHECK-BE64 %s
 // 64 bit little endian
-// RUN: llvm-mc -filetype=obj -arch=mips64el -triple mips64el-unknown-linux %s -o - | llvm-readobj -h | FileCheck -check-prefix=CHECK-LE64 %s
+// RUN: llvm-mc -filetype=obj -arch=mips64el -triple mips64el-unknown-linux %s -o - | llvm-readobj -h - | FileCheck -check-prefix=CHECK-LE64 %s
 
 // Check that we produce 32 bit with each endian.
 
diff --git a/llvm/test/MC/Mips/elf_eflags.s b/llvm/test/MC/Mips/elf_eflags.s
index 161f18562a832..fced1cfe76f94 100644
--- a/llvm/test/MC/Mips/elf_eflags.s
+++ b/llvm/test/MC/Mips/elf_eflags.s
@@ -1,205 +1,205 @@
 # These *MUST* match the output of 'gcc -c' compiled with the same triple and
 # corresponding options (-mcpu=mips32 -> -mips32 for example).
 
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips64r6 -target-abi n64 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS64R6 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips64r6 -target-abi n64 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS64R6 %s
 # MIPSEL-MIPS64R6: Flags [ (0xA0000404)
 
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips64r6 -target-abi n64 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS64R6-NAN2008 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips64r6 -target-abi n64 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS64R6-NAN2008 %s
 # MIPSEL-MIPS64R6-NAN2008: Flags [ (0xA0000404)
 
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips64r2 -target-abi n64 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS64R2 %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips64r3 -target-abi n64 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS64R2 %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips64r5 -target-abi n64 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS64R2 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips64r2 -target-abi n64 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS64R2 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips64r3 -target-abi n64 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS64R2 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips64r5 -target-abi n64 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS64R2 %s
 # MIPSEL-MIPS64R2: Flags [ (0x80000004)
 
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips64r2 -target-abi n64 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS64R2-NAN2008 %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips64r3 -target-abi n64 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS64R2-NAN2008 %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips64r5 -target-abi n64 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS64R2-NAN2008 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips64r2 -target-abi n64 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS64R2-NAN2008 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips64r3 -target-abi n64 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS64R2-NAN2008 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips64r5 -target-abi n64 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS64R2-NAN2008 %s
 # MIPSEL-MIPS64R2-NAN2008: Flags [ (0x80000404)
 
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips64 -target-abi n64 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS64 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips64 -target-abi n64 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS64 %s
 # MIPSEL-MIPS64: Flags [ (0x60000004)
 
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips64 -target-abi n64 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS64-NAN2008 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips64 -target-abi n64 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS64-NAN2008 %s
 # MIPSEL-MIPS64-NAN2008: Flags [ (0x60000404)
 
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -position-independent -mcpu=mips64r6 -target-abi n64 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS64R6-PIC %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -position-independent -mcpu=mips64r6 -target-abi n64 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS64R6-PIC %s
 # MIPSEL-MIPS64R6-PIC: Flags [ (0xA0000406)
 
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -position-independent -mcpu=mips64r6 -target-abi n64 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS64R6-NAN2008-PIC %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -position-independent -mcpu=mips64r6 -target-abi n64 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS64R6-NAN2008-PIC %s
 # MIPSEL-MIPS64R6-NAN2008-PIC: Flags [ (0xA0000406)
 
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -position-independent -mcpu=mips64r2 -target-abi n64 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS64R2-PIC %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -position-independent -mcpu=mips64r3 -target-abi n64 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS64R2-PIC %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -position-independent -mcpu=mips64r5 -target-abi n64 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS64R2-PIC %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -position-independent -mcpu=mips64r2 -target-abi n64 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS64R2-PIC %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -position-independent -mcpu=mips64r3 -target-abi n64 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS64R2-PIC %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -position-independent -mcpu=mips64r5 -target-abi n64 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS64R2-PIC %s
 # MIPSEL-MIPS64R2-PIC: Flags [ (0x80000006)
 
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -position-independent -mcpu=mips64r2 -target-abi n64 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS64R2-NAN2008-PIC %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -position-independent -mcpu=mips64r3 -target-abi n64 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS64R2-NAN2008-PIC %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -position-independent -mcpu=mips64r5 -target-abi n64 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS64R2-NAN2008-PIC %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -position-independent -mcpu=mips64r2 -target-abi n64 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS64R2-NAN2008-PIC %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -position-independent -mcpu=mips64r3 -target-abi n64 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS64R2-NAN2008-PIC %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -position-independent -mcpu=mips64r5 -target-abi n64 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS64R2-NAN2008-PIC %s
 # MIPSEL-MIPS64R2-NAN2008-PIC: Flags [ (0x80000406)
 
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -position-independent -mcpu=mips64 -target-abi n64 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS64-PIC %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -position-independent -mcpu=mips64 -target-abi n64 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS64-PIC %s
 # MIPSEL-MIPS64-PIC: Flags [ (0x60000006)
 
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -position-independent -mcpu=mips64 -target-abi n64 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS64-NAN2008-PIC %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -position-independent -mcpu=mips64 -target-abi n64 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS64-NAN2008-PIC %s
 # MIPSEL-MIPS64-NAN2008-PIC: Flags [ (0x60000406)
 
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32r6 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS32R6 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32r6 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS32R6 %s
 # MIPSEL-MIPS32R6: Flags [ (0x90001404)
 
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32r6 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS32R6-NAN2008 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32r6 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS32R6-NAN2008 %s
 # MIPSEL-MIPS32R6-NAN2008: Flags [ (0x90001404)
 
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32r2 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS32R2 %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32r3 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS32R2 %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32r5 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS32R2 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32r2 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS32R2 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32r3 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS32R2 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32r5 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS32R2 %s
 # MIPSEL-MIPS32R2: Flags [ (0x70001004)
 
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32r2 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS32R2-NAN2008 %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32r3 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS32R2-NAN2008 %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32r5 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS32R2-NAN2008 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32r2 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS32R2-NAN2008 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32r3 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS32R2-NAN2008 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32r5 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS32R2-NAN2008 %s
 # MIPSEL-MIPS32R2-NAN2008: Flags [ (0x70001404)
 
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS32 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS32 %s
 # MIPSEL-MIPS32: Flags [ (0x50001004)
 
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS32-NAN2008 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS32-NAN2008 %s
 # MIPSEL-MIPS32-NAN2008: Flags [ (0x50001404)
 
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips64r2 -target-abi n32 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS64EL-MIPS64R2-N32 %s
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r2 -target-abi n32 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS64EL-MIPS64R2-N32 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips64r2 -target-abi n32 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS64EL-MIPS64R2-N32 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r2 -target-abi n32 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS64EL-MIPS64R2-N32 %s
 # MIPS64EL-MIPS64R2-N32: Flags [ (0x80000024)
 
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r2 -target-abi n32 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS64EL-MIPS64R2-N32-NAN2008 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r2 -target-abi n32 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS64EL-MIPS64R2-N32-NAN2008 %s
 # MIPS64EL-MIPS64R2-N32-NAN2008: Flags [ (0x80000424)
 
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64 -target-abi n32 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS64EL-MIPS64-N32 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64 -target-abi n32 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS64EL-MIPS64-N32 %s
 # MIPS64EL-MIPS64-N32: Flags [ (0x60000024)
 
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64 -target-abi n32 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS64EL-MIPS64-N32-NAN2008 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64 -target-abi n32 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS64EL-MIPS64-N32-NAN2008 %s
 # MIPS64EL-MIPS64-N32-NAN2008: Flags [ (0x60000424)
 
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r2 -target-abi n64 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS64EL-MIPS64R2-N64 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r2 -target-abi n64 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS64EL-MIPS64R2-N64 %s
 # MIPS64EL-MIPS64R2-N64: Flags [ (0x80000004)
 
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r2 -target-abi n64 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS64EL-MIPS64R2-N64-NAN2008 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r2 -target-abi n64 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS64EL-MIPS64R2-N64-NAN2008 %s
 # MIPS64EL-MIPS64R2-N64-NAN2008: Flags [ (0x80000404)
 
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64 %s -target-abi n64 -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS64EL-MIPS64-N64 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64 %s -target-abi n64 -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS64EL-MIPS64-N64 %s
 # MIPS64EL-MIPS64-N64: Flags [ (0x60000004)
 
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64 %s -target-abi n64 -mattr=+nan2008 -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS64EL-MIPS64-N64-NAN2008 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64 %s -target-abi n64 -mattr=+nan2008 -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS64EL-MIPS64-N64-NAN2008 %s
 # MIPS64EL-MIPS64-N64-NAN2008: Flags [ (0x60000404)
 
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -position-independent -mcpu=mips64r2 -target-abi n64 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS64EL-MIPS64R2-N64-PIC %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -position-independent -mcpu=mips64r2 -target-abi n64 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS64EL-MIPS64R2-N64-PIC %s
 # MIPS64EL-MIPS64R2-N64-PIC: Flags [ (0x80000006)
 
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -position-independent -mcpu=mips64r2 -target-abi n64 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS64EL-MIPS64R2-N64-NAN2008-PIC %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -position-independent -mcpu=mips64r2 -target-abi n64 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS64EL-MIPS64R2-N64-NAN2008-PIC %s
 # MIPS64EL-MIPS64R2-N64-NAN2008-PIC: Flags [ (0x80000406)
 
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -position-independent -mcpu=mips64 %s -target-abi n64 -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS64EL-MIPS64-N64-PIC %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -position-independent -mcpu=mips64 %s -target-abi n64 -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS64EL-MIPS64-N64-PIC %s
 # MIPS64EL-MIPS64-N64-PIC: Flags [ (0x60000006)
 
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -position-independent -mcpu=mips64 %s -target-abi n64 -mattr=+nan2008 -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS64EL-MIPS64-N64-NAN2008-PIC %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -position-independent -mcpu=mips64 %s -target-abi n64 -mattr=+nan2008 -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS64EL-MIPS64-N64-NAN2008-PIC %s
 # MIPS64EL-MIPS64-N64-NAN2008-PIC: Flags [ (0x60000406)
 
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r2 -target-abi o32 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS64EL-MIPS64R2-O32 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r2 -target-abi o32 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS64EL-MIPS64R2-O32 %s
 # MIPS64EL-MIPS64R2-O32: Flags [ (0x80001104)
 
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r2 -target-abi o32 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS64EL-MIPS64R2-O32-NAN2008 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r2 -target-abi o32 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS64EL-MIPS64R2-O32-NAN2008 %s
 # MIPS64EL-MIPS64R2-O32-NAN2008: Flags [ (0x80001504)
 
-# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux -mcpu=mips5 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS5 %s
+# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux -mcpu=mips5 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS5 %s
 # MIPS5: Flags [ (0x40000004)
 
-# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux -mcpu=mips5 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS5-NAN2008 %s
+# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux -mcpu=mips5 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS5-NAN2008 %s
 # MIPS5-NAN2008: Flags [ (0x40000404)
 
-# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux -mcpu=mips4 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS4 %s
+# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux -mcpu=mips4 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS4 %s
 # MIPS4: Flags [ (0x30000004)
 
-# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux -mcpu=mips4 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS4-NAN2008 %s
+# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux -mcpu=mips4 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS4-NAN2008 %s
 # MIPS4-NAN2008: Flags [ (0x30000404)
 
-# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux -mcpu=mips3 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS3 %s
+# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux -mcpu=mips3 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS3 %s
 # MIPS3: Flags [ (0x20000004)
 
-# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux -mcpu=mips3 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS3-NAN2008 %s
+# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux -mcpu=mips3 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS3-NAN2008 %s
 # MIPS3-NAN2008: Flags [ (0x20000404)
 
-# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux -position-independent -mcpu=mips5 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS5-PIC %s
+# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux -position-independent -mcpu=mips5 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS5-PIC %s
 # MIPS5-PIC: Flags [ (0x40000006)
 
-# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux -position-independent -mcpu=mips5 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS5-NAN2008-PIC %s
+# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux -position-independent -mcpu=mips5 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS5-NAN2008-PIC %s
 # MIPS5-NAN2008-PIC: Flags [ (0x40000406)
 
-# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux -position-independent -mcpu=mips4 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS4-PIC %s
+# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux -position-independent -mcpu=mips4 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS4-PIC %s
 # MIPS4-PIC: Flags [ (0x30000006)
 
-# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux -position-independent -mcpu=mips4 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS4-NAN2008-PIC %s
+# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux -position-independent -mcpu=mips4 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS4-NAN2008-PIC %s
 # MIPS4-NAN2008-PIC: Flags [ (0x30000406)
 
-# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux -position-independent -mcpu=mips3 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS3-PIC %s
+# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux -position-independent -mcpu=mips3 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS3-PIC %s
 # MIPS3-PIC: Flags [ (0x20000006)
 
-# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux -position-independent -mcpu=mips3 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS3-NAN2008-PIC %s
+# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux -position-independent -mcpu=mips3 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS3-NAN2008-PIC %s
 # MIPS3-NAN2008-PIC: Flags [ (0x20000406)
 
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips2 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS2 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips2 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS2 %s
 # MIPSEL-MIPS2: Flags [ (0x10001004)
 
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips2 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-MIPS2-NAN2008 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips2 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-MIPS2-NAN2008 %s
 # MIPSEL-MIPS2-NAN2008: Flags [ (0x10001404)
 
-# RUN: llvm-mc -filetype=obj -triple mips-unknown-linux -mcpu=mips1 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS1 %s
+# RUN: llvm-mc -filetype=obj -triple mips-unknown-linux -mcpu=mips1 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS1 %s
 # MIPS1: Flags [ (0x1004)
 
-# RUN: llvm-mc -filetype=obj -triple mips-unknown-linux -mcpu=mips1 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS1-NAN2008 %s
+# RUN: llvm-mc -filetype=obj -triple mips-unknown-linux -mcpu=mips1 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS1-NAN2008 %s
 # MIPS1-NAN2008: Flags [ (0x1404)
 
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64 %s -target-abi o32 -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS64EL-MIPS64-O32 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64 %s -target-abi o32 -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS64EL-MIPS64-O32 %s
 # MIPS64EL-MIPS64-O32: Flags [ (0x60001104)
 
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64 %s -target-abi o32 -mattr=+nan2008 -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS64EL-MIPS64-O32-NAN2008 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64 %s -target-abi o32 -mattr=+nan2008 -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS64EL-MIPS64-O32-NAN2008 %s
 # MIPS64EL-MIPS64-O32-NAN2008: Flags [ (0x60001504)
 
 # Default ABI for MIPS64 is N64 as opposed to GCC/GAS (N32)
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r2 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS64EL-MIPS64R2 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r2 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS64EL-MIPS64R2 %s
 # MIPS64EL-MIPS64R2: Flags [ (0x80000004)
 
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r2 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS64EL-MIPS64R2-NAN2008 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r2 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS64EL-MIPS64R2-NAN2008 %s
 # MIPS64EL-MIPS64R2-NAN2008: Flags [ (0x80000404)
 
 # Default ABI for MIPS64 is N64 as opposed to GCC/GAS (N32)
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS64EL-MIPS64 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS64EL-MIPS64 %s
 # MIPS64EL-MIPS64: Flags [ (0x60000004)
 
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS64EL-MIPS64-NAN2008 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS64EL-MIPS64-NAN2008 %s
 # MIPS64EL-MIPS64-NAN2008: Flags [ (0x60000404)
 
 # Default ABI for MIPS64 is N64 as opposed to GCC/GAS (N32)
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -position-independent -mcpu=mips64r2 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS64EL-MIPS64R2-PIC %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -position-independent -mcpu=mips64r2 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS64EL-MIPS64R2-PIC %s
 # MIPS64EL-MIPS64R2-PIC: Flags [ (0x80000006)
 
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -position-independent -mcpu=mips64r2 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS64EL-MIPS64R2-NAN2008-PIC %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -position-independent -mcpu=mips64r2 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS64EL-MIPS64R2-NAN2008-PIC %s
 # MIPS64EL-MIPS64R2-NAN2008-PIC: Flags [ (0x80000406)
 
 # Default ABI for MIPS64 is N64 as opposed to GCC/GAS (N32)
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -position-independent -mcpu=mips64 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS64EL-MIPS64-PIC %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -position-independent -mcpu=mips64 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS64EL-MIPS64-PIC %s
 # MIPS64EL-MIPS64-PIC: Flags [ (0x60000006)
 
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -position-independent -mcpu=mips64 -mattr=+nan2008 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPS64EL-MIPS64-NAN2008-PIC %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -position-independent -mcpu=mips64 -mattr=+nan2008 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPS64EL-MIPS64-NAN2008-PIC %s
 # MIPS64EL-MIPS64-NAN2008-PIC: Flags [ (0x60000406)
 
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=octeon -target-abi n64 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-OCTEON %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=octeon -target-abi n64 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-OCTEON %s
 # RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux \
 # RUN:         -mcpu=octeon+ -target-abi n64 %s -o - \
-# RUN:   | llvm-readobj -h | FileCheck --check-prefix=MIPSEL-OCTEON %s
+# RUN:   | llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-OCTEON %s
 # MIPSEL-OCTEON: Flags [ (0x808B0004)
 
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -position-independent -mcpu=octeon -target-abi n64 %s -o -| llvm-readobj -h | FileCheck --check-prefix=MIPSEL-OCTEON-PIC %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -position-independent -mcpu=octeon -target-abi n64 %s -o -| llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-OCTEON-PIC %s
 # RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux \
 # RUN:         -position-independent -mcpu=octeon+ -target-abi n64 %s -o - \
-# RUN:   | llvm-readobj -h | FileCheck --check-prefix=MIPSEL-OCTEON-PIC %s
+# RUN:   | llvm-readobj -h - | FileCheck --check-prefix=MIPSEL-OCTEON-PIC %s
 # MIPSEL-OCTEON-PIC: Flags [ (0x808B0006)
diff --git a/llvm/test/MC/Mips/elf_eflags_abicalls.s b/llvm/test/MC/Mips/elf_eflags_abicalls.s
index 9e9c013fb3f26..b8b8d6962facf 100644
--- a/llvm/test/MC/Mips/elf_eflags_abicalls.s
+++ b/llvm/test/MC/Mips/elf_eflags_abicalls.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32 %s -o -| llvm-readobj -h | FileCheck %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32 %s -o -| llvm-readobj -h - | FileCheck %s
 
 # This *MUST* match the output of 'gcc -c' compiled with the same triple.
 # CHECK: Flags [ (0x50001006)
diff --git a/llvm/test/MC/Mips/elf_eflags_micromips.s b/llvm/test/MC/Mips/elf_eflags_micromips.s
index 623f5a7329116..9a9f9f783f3e0 100644
--- a/llvm/test/MC/Mips/elf_eflags_micromips.s
+++ b/llvm/test/MC/Mips/elf_eflags_micromips.s
@@ -1,8 +1,8 @@
 # RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32 \
-# RUN:     -mattr=+micromips < /dev/null -o -| llvm-readobj -h | FileCheck \
+# RUN:     -mattr=+micromips < /dev/null -o -| llvm-readobj -h - | FileCheck \
 # RUN:     -check-prefix=NO-MM %s
 # RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32 %s -o - \
-# RUN:     | llvm-readobj -h | FileCheck %s
+# RUN:     | llvm-readobj -h - | FileCheck %s
 
 # This *MUST* match the output of 'gcc -c' compiled with the same triple.
 # CHECK: Flags [ (0x52001004)
diff --git a/llvm/test/MC/Mips/elf_eflags_micromips2.s b/llvm/test/MC/Mips/elf_eflags_micromips2.s
index d44cf7f37a721..06ad9a73dd621 100644
--- a/llvm/test/MC/Mips/elf_eflags_micromips2.s
+++ b/llvm/test/MC/Mips/elf_eflags_micromips2.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32 %s -o - \
-# RUN:     | llvm-readobj -h | FileCheck %s
+# RUN:     | llvm-readobj -h - | FileCheck %s
 
 # This *MUST* match the output of 'gcc -c' compiled with the same triple.
 # The microMIPS flag is not set if no microMIPS code was emitted.
diff --git a/llvm/test/MC/Mips/elf_eflags_mips16.s b/llvm/test/MC/Mips/elf_eflags_mips16.s
index 5143d36df25e8..6582a61ee985a 100644
--- a/llvm/test/MC/Mips/elf_eflags_mips16.s
+++ b/llvm/test/MC/Mips/elf_eflags_mips16.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32 %s -o -| llvm-readobj -h | FileCheck %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32 %s -o -| llvm-readobj -h - | FileCheck %s
 
 # This *MUST* match the output of 'gcc -c' compiled with the same triple.
 # CHECK: Flags [ (0x54001004)
diff --git a/llvm/test/MC/Mips/elf_eflags_nan2008.s b/llvm/test/MC/Mips/elf_eflags_nan2008.s
index f6903429c0df2..2dbe1b3b2736f 100644
--- a/llvm/test/MC/Mips/elf_eflags_nan2008.s
+++ b/llvm/test/MC/Mips/elf_eflags_nan2008.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32 %s -o - | \
-# RUN:   llvm-readobj -h | \
+# RUN:   llvm-readobj -h - | \
 # RUN:     FileCheck %s -check-prefix=CHECK-OBJ
 # RUN: llvm-mc -triple mipsel-unknown-linux -mcpu=mips32 %s -o -| \
 # RUN:   FileCheck %s -check-prefix=CHECK-ASM
diff --git a/llvm/test/MC/Mips/elf_eflags_nanlegacy.s b/llvm/test/MC/Mips/elf_eflags_nanlegacy.s
index 0fa07879d3748..4d93448647d39 100644
--- a/llvm/test/MC/Mips/elf_eflags_nanlegacy.s
+++ b/llvm/test/MC/Mips/elf_eflags_nanlegacy.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32 %s -o - | \
-# RUN:   llvm-readobj -h | \
+# RUN:   llvm-readobj -h - | \
 # RUN:     FileCheck %s -check-prefix=CHECK-OBJ
 # RUN: llvm-mc -triple mipsel-unknown-linux -mcpu=mips32 %s -o -| \
 # RUN:   FileCheck %s -check-prefix=CHECK-ASM
diff --git a/llvm/test/MC/Mips/elf_eflags_noreorder.s b/llvm/test/MC/Mips/elf_eflags_noreorder.s
index fe46b41ae2f50..1fd9fd9f8d22e 100644
--- a/llvm/test/MC/Mips/elf_eflags_noreorder.s
+++ b/llvm/test/MC/Mips/elf_eflags_noreorder.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32 %s -o -| llvm-readobj -h | FileCheck %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32 %s -o -| llvm-readobj -h - | FileCheck %s
 
 # This *MUST* match the output of 'gcc -c' compiled with the same triple.
 # CHECK: Flags [ (0x50001005)
diff --git a/llvm/test/MC/Mips/elf_eflags_pic0.s b/llvm/test/MC/Mips/elf_eflags_pic0.s
index 04115fad1a0ac..1d403c7c2a1e8 100644
--- a/llvm/test/MC/Mips/elf_eflags_pic0.s
+++ b/llvm/test/MC/Mips/elf_eflags_pic0.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32 %s -o -| llvm-readobj -h | FileCheck %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32 %s -o -| llvm-readobj -h - | FileCheck %s
 
 # This *MUST* match the output of 'gcc -c' compiled with the same triple.
 # CHECK: Flags [ (0x50001004)
diff --git a/llvm/test/MC/Mips/elf_eflags_pic2.s b/llvm/test/MC/Mips/elf_eflags_pic2.s
index 692c478d0bcac..278308e866b5e 100644
--- a/llvm/test/MC/Mips/elf_eflags_pic2.s
+++ b/llvm/test/MC/Mips/elf_eflags_pic2.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32 %s -o -| llvm-readobj -h | FileCheck %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32 %s -o -| llvm-readobj -h - | FileCheck %s
 
 # This *MUST* match the output of 'gcc -c' compiled with the same triple.
 # CHECK: Flags [ (0x50001006)
diff --git a/llvm/test/MC/Mips/elf_header.s b/llvm/test/MC/Mips/elf_header.s
index 14136f3112ec3..eb6252061c4b5 100644
--- a/llvm/test/MC/Mips/elf_header.s
+++ b/llvm/test/MC/Mips/elf_header.s
@@ -1,94 +1,94 @@
 # Default ABI for MIPS32 is O32.
-# RUN: llvm-mc -filetype=obj -triple mips-unknown-linux     -mcpu=mips1                                    %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,BE,O32,NAN1985,MIPS1 %s
-# RUN: llvm-mc -filetype=obj -triple mips-unknown-linux     -mcpu=mips2                                    %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,BE,O32,NAN1985,MIPS2 %s
-# RUN: llvm-mc -filetype=obj -triple mips-unknown-linux     -mcpu=mips3                                    %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,BE,O32,NAN1985,MIPS3,32BITMODE %s
-# RUN: llvm-mc -filetype=obj -triple mips-unknown-linux     -mcpu=mips4                                    %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,BE,O32,NAN1985,MIPS4,32BITMODE %s
-# RUN: llvm-mc -filetype=obj -triple mips-unknown-linux     -mcpu=mips5                                    %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,BE,O32,NAN1985,MIPS5,32BITMODE %s
-# RUN: llvm-mc -filetype=obj -triple mips-unknown-linux                                                    %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,BE,O32,NAN1985,MIPS32R1 %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux                                                  %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN1985,MIPS32R1 %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux                                                  %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN1985,MIPS32R1 %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips32r2                                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN1985,MIPS32R2 %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips32r3                                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN1985,MIPS32R3 %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips32r5                                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN1985,MIPS32R5 %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips32r2                 -mattr=+nan2008 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN2008,MIPS32R2 %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips32r3                 -mattr=+nan2008 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN2008,MIPS32R3 %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips32r5                 -mattr=+nan2008 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN2008,MIPS32R5 %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips32r6                                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN2008,MIPS32R6 %s
+# RUN: llvm-mc -filetype=obj -triple mips-unknown-linux     -mcpu=mips1                                    %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,BE,O32,NAN1985,MIPS1 %s
+# RUN: llvm-mc -filetype=obj -triple mips-unknown-linux     -mcpu=mips2                                    %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,BE,O32,NAN1985,MIPS2 %s
+# RUN: llvm-mc -filetype=obj -triple mips-unknown-linux     -mcpu=mips3                                    %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,BE,O32,NAN1985,MIPS3,32BITMODE %s
+# RUN: llvm-mc -filetype=obj -triple mips-unknown-linux     -mcpu=mips4                                    %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,BE,O32,NAN1985,MIPS4,32BITMODE %s
+# RUN: llvm-mc -filetype=obj -triple mips-unknown-linux     -mcpu=mips5                                    %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,BE,O32,NAN1985,MIPS5,32BITMODE %s
+# RUN: llvm-mc -filetype=obj -triple mips-unknown-linux                                                    %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,BE,O32,NAN1985,MIPS32R1 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux                                                  %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN1985,MIPS32R1 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux                                                  %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN1985,MIPS32R1 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips32r2                                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN1985,MIPS32R2 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips32r3                                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN1985,MIPS32R3 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips32r5                                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN1985,MIPS32R5 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips32r2                 -mattr=+nan2008 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN2008,MIPS32R2 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips32r3                 -mattr=+nan2008 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN2008,MIPS32R3 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips32r5                 -mattr=+nan2008 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN2008,MIPS32R5 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips32r6                                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN2008,MIPS32R6 %s
 
 # Selected ABI O32 takes precedence over target triple.
-# FIXME: llvm-mc -filetype=obj -triple mips64-unknown-linux   -mcpu=mips1    -target-abi=o32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,BE,O32,NAN1985,MIPS1 %s
-# FIXME: llvm-mc -filetype=obj -triple mips64-unknown-linux   -mcpu=mips2    -target-abi=o32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,BE,O32,NAN1985,MIPS2 %s
-# FIXME: llvm-mc -filetype=obj -triple mips64-unknown-linux   -mcpu=mips3    -target-abi=o32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,BE,O32,NAN1985,MIPS3,32BITMODE %s
-# FIXME: llvm-mc -filetype=obj -triple mips64-unknown-linux   -mcpu=mips4    -target-abi=o32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,BE,O32,NAN1985,MIPS4,32BITMODE %s
-# FIXME: llvm-mc -filetype=obj -triple mips64-unknown-linux   -mcpu=mips5    -target-abi=o32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,BE,O32,NAN1985,MIPS5,32BITMODE %s
-# FIXME: llvm-mc -filetype=obj -triple mips64-unknown-linux                  -target-abi=o32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,BE,O32,NAN1985,MIPS32R1 %s
-# FIXME: llvm-mc -filetype=obj -triple mips64el-unknown-linux                -target-abi=o32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN1985,MIPS32R1 %s
-# FIXME: llvm-mc -filetype=obj -triple mips64el-unknown-linux                -target-abi=o32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN1985,MIPS32R1 %s
-# FIXME: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips32r2 -target-abi=o32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN1985,MIPS32R2 %s
-# FIXME: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips32r3 -target-abi=o32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN1985,MIPS32R3 %s
-# FIXME: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips32r5 -target-abi=o32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN1985,MIPS32R5 %s
-# FIXME: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips32r2 -target-abi=o32 -mattr=+nan2008 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN2008,MIPS32R2 %s
-# FIXME: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips32r3 -target-abi=o32 -mattr=+nan2008 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN2008,MIPS32R3 %s
-# FIXME: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips32r5 -target-abi=o32 -mattr=+nan2008 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN2008,MIPS32R5 %s
-# FIXME: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips32r6 -target-abi=o32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN2008,MIPS32R6 %s
+# FIXME: llvm-mc -filetype=obj -triple mips64-unknown-linux   -mcpu=mips1    -target-abi=o32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,BE,O32,NAN1985,MIPS1 %s
+# FIXME: llvm-mc -filetype=obj -triple mips64-unknown-linux   -mcpu=mips2    -target-abi=o32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,BE,O32,NAN1985,MIPS2 %s
+# FIXME: llvm-mc -filetype=obj -triple mips64-unknown-linux   -mcpu=mips3    -target-abi=o32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,BE,O32,NAN1985,MIPS3,32BITMODE %s
+# FIXME: llvm-mc -filetype=obj -triple mips64-unknown-linux   -mcpu=mips4    -target-abi=o32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,BE,O32,NAN1985,MIPS4,32BITMODE %s
+# FIXME: llvm-mc -filetype=obj -triple mips64-unknown-linux   -mcpu=mips5    -target-abi=o32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,BE,O32,NAN1985,MIPS5,32BITMODE %s
+# FIXME: llvm-mc -filetype=obj -triple mips64-unknown-linux                  -target-abi=o32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,BE,O32,NAN1985,MIPS32R1 %s
+# FIXME: llvm-mc -filetype=obj -triple mips64el-unknown-linux                -target-abi=o32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN1985,MIPS32R1 %s
+# FIXME: llvm-mc -filetype=obj -triple mips64el-unknown-linux                -target-abi=o32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN1985,MIPS32R1 %s
+# FIXME: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips32r2 -target-abi=o32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN1985,MIPS32R2 %s
+# FIXME: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips32r3 -target-abi=o32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN1985,MIPS32R3 %s
+# FIXME: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips32r5 -target-abi=o32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN1985,MIPS32R5 %s
+# FIXME: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips32r2 -target-abi=o32 -mattr=+nan2008 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN2008,MIPS32R2 %s
+# FIXME: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips32r3 -target-abi=o32 -mattr=+nan2008 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN2008,MIPS32R3 %s
+# FIXME: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips32r5 -target-abi=o32 -mattr=+nan2008 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN2008,MIPS32R5 %s
+# FIXME: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips32r6 -target-abi=o32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,O32,NAN2008,MIPS32R6 %s
 
 # Default ABI for MIPS64 is N64 as opposed to GCC/GAS (N32).
-# RUN: llvm-mc -filetype=obj -triple mips-unknown-linux     -mcpu=mips3    -target-abi=n32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,BE,N32,NAN1985,MIPS3 %s
-# RUN: llvm-mc -filetype=obj -triple mips-unknown-linux     -mcpu=mips4    -target-abi=n32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,BE,N32,NAN1985,MIPS4 %s
-# RUN: llvm-mc -filetype=obj -triple mips-unknown-linux     -mcpu=mips5    -target-abi=n32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,BE,N32,NAN1985,MIPS5 %s
-# FIXME: llvm-mc -filetype=obj -triple mips-unknown-linux                    -target-abi=n32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,BE,N32,NAN1985,MIPS64R1 %s
-# FIXME: llvm-mc -filetype=obj -triple mipsel-unknown-linux                  -target-abi=n32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN1985,MIPS64R1 %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r2 -target-abi=n32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN1985,MIPS64R2 %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r3 -target-abi=n32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN1985,MIPS64R3 %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r5 -target-abi=n32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN1985,MIPS64R5 %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r2 -target-abi=n32 -mattr=+nan2008 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN2008,MIPS64R2 %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r3 -target-abi=n32 -mattr=+nan2008 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN2008,MIPS64R3 %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r5 -target-abi=n32 -mattr=+nan2008 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN2008,MIPS64R5 %s
-# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r6 -target-abi=n32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN2008,MIPS64R6 %s
-# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux   -mcpu=mips3    -target-abi=n32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,BE,N32,NAN1985,MIPS3 %s
-# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux   -mcpu=mips4    -target-abi=n32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,BE,N32,NAN1985,MIPS4 %s
-# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux   -mcpu=mips5    -target-abi=n32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,BE,N32,NAN1985,MIPS5 %s
-# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux                  -target-abi=n32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,BE,N32,NAN1985,MIPS64R1 %s
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux                -target-abi=n32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN1985,MIPS64R1 %s
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r2 -target-abi=n32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN1985,MIPS64R2 %s
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r3 -target-abi=n32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN1985,MIPS64R3 %s
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r5 -target-abi=n32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN1985,MIPS64R5 %s
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r2 -target-abi=n32 -mattr=+nan2008 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN2008,MIPS64R2 %s
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r3 -target-abi=n32 -mattr=+nan2008 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN2008,MIPS64R3 %s
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r5 -target-abi=n32 -mattr=+nan2008 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN2008,MIPS64R5 %s
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r6 -target-abi=n32                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN2008,MIPS64R6 %s
+# RUN: llvm-mc -filetype=obj -triple mips-unknown-linux     -mcpu=mips3    -target-abi=n32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,BE,N32,NAN1985,MIPS3 %s
+# RUN: llvm-mc -filetype=obj -triple mips-unknown-linux     -mcpu=mips4    -target-abi=n32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,BE,N32,NAN1985,MIPS4 %s
+# RUN: llvm-mc -filetype=obj -triple mips-unknown-linux     -mcpu=mips5    -target-abi=n32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,BE,N32,NAN1985,MIPS5 %s
+# FIXME: llvm-mc -filetype=obj -triple mips-unknown-linux                    -target-abi=n32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,BE,N32,NAN1985,MIPS64R1 %s
+# FIXME: llvm-mc -filetype=obj -triple mipsel-unknown-linux                  -target-abi=n32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN1985,MIPS64R1 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r2 -target-abi=n32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN1985,MIPS64R2 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r3 -target-abi=n32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN1985,MIPS64R3 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r5 -target-abi=n32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN1985,MIPS64R5 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r2 -target-abi=n32 -mattr=+nan2008 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN2008,MIPS64R2 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r3 -target-abi=n32 -mattr=+nan2008 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN2008,MIPS64R3 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r5 -target-abi=n32 -mattr=+nan2008 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN2008,MIPS64R5 %s
+# RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r6 -target-abi=n32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN2008,MIPS64R6 %s
+# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux   -mcpu=mips3    -target-abi=n32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,BE,N32,NAN1985,MIPS3 %s
+# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux   -mcpu=mips4    -target-abi=n32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,BE,N32,NAN1985,MIPS4 %s
+# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux   -mcpu=mips5    -target-abi=n32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,BE,N32,NAN1985,MIPS5 %s
+# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux                  -target-abi=n32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,BE,N32,NAN1985,MIPS64R1 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux                -target-abi=n32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN1985,MIPS64R1 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r2 -target-abi=n32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN1985,MIPS64R2 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r3 -target-abi=n32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN1985,MIPS64R3 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r5 -target-abi=n32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN1985,MIPS64R5 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r2 -target-abi=n32 -mattr=+nan2008 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN2008,MIPS64R2 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r3 -target-abi=n32 -mattr=+nan2008 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN2008,MIPS64R3 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r5 -target-abi=n32 -mattr=+nan2008 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN2008,MIPS64R5 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r6 -target-abi=n32                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF32,LE,N32,NAN2008,MIPS64R6 %s
 
 # Default ABI for MIPS64 is N64 as opposed to GCC/GAS (N32).
-# FIXME: llvm-mc -filetype=obj -triple mips-unknown-linux     -mcpu=mips3    -target-abi=n64                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF64,BE,N64,NAN1985,MIPS3    %s
-# FIXME: llvm-mc -filetype=obj -triple mips-unknown-linux     -mcpu=mips4    -target-abi=n64                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF64,BE,N64,NAN1985,MIPS4    %s
-# FIXME: llvm-mc -filetype=obj -triple mips-unknown-linux     -mcpu=mips5    -target-abi=n64                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF64,BE,N64,NAN1985,MIPS5    %s
-# FIXME: llvm-mc -filetype=obj -triple mips-unknown-linux                    -target-abi=n64                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF64,BE,N64,NAN1985,MIPS64R1 %s
-# FIXME: llvm-mc -filetype=obj -triple mipsel-unknown-linux                  -target-abi=n64                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN1985,MIPS64R1 %s
-# FIXME: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r2 -target-abi=n64                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN1985,MIPS64R2 %s
-# FIXME: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r3 -target-abi=n64                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN1985,MIPS64R3 %s
-# FIXME: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r5 -target-abi=n64                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN1985,MIPS64R5 %s
-# FIXME: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r2 -target-abi=n64 -mattr=+nan2008 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN2008,MIPS64R2 %s
-# FIXME: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r3 -target-abi=n64 -mattr=+nan2008 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN2008,MIPS64R3 %s
-# FIXME: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r5 -target-abi=n64 -mattr=+nan2008 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN2008,MIPS64R5 %s
-# FIXME: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r6 -target-abi=n64                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN2008,MIPS64R6 %s
-# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux   -mcpu=mips3    -target-abi=n64                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF64,BE,N64,NAN1985,MIPS3    %s
-# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux   -mcpu=mips4    -target-abi=n64                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF64,BE,N64,NAN1985,MIPS4    %s
-# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux   -mcpu=mips5    -target-abi=n64                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF64,BE,N64,NAN1985,MIPS5    %s
+# FIXME: llvm-mc -filetype=obj -triple mips-unknown-linux     -mcpu=mips3    -target-abi=n64                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF64,BE,N64,NAN1985,MIPS3    %s
+# FIXME: llvm-mc -filetype=obj -triple mips-unknown-linux     -mcpu=mips4    -target-abi=n64                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF64,BE,N64,NAN1985,MIPS4    %s
+# FIXME: llvm-mc -filetype=obj -triple mips-unknown-linux     -mcpu=mips5    -target-abi=n64                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF64,BE,N64,NAN1985,MIPS5    %s
+# FIXME: llvm-mc -filetype=obj -triple mips-unknown-linux                    -target-abi=n64                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF64,BE,N64,NAN1985,MIPS64R1 %s
+# FIXME: llvm-mc -filetype=obj -triple mipsel-unknown-linux                  -target-abi=n64                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN1985,MIPS64R1 %s
+# FIXME: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r2 -target-abi=n64                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN1985,MIPS64R2 %s
+# FIXME: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r3 -target-abi=n64                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN1985,MIPS64R3 %s
+# FIXME: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r5 -target-abi=n64                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN1985,MIPS64R5 %s
+# FIXME: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r2 -target-abi=n64 -mattr=+nan2008 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN2008,MIPS64R2 %s
+# FIXME: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r3 -target-abi=n64 -mattr=+nan2008 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN2008,MIPS64R3 %s
+# FIXME: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r5 -target-abi=n64 -mattr=+nan2008 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN2008,MIPS64R5 %s
+# FIXME: llvm-mc -filetype=obj -triple mipsel-unknown-linux   -mcpu=mips64r6 -target-abi=n64                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN2008,MIPS64R6 %s
+# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux   -mcpu=mips3    -target-abi=n64                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF64,BE,N64,NAN1985,MIPS3    %s
+# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux   -mcpu=mips4    -target-abi=n64                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF64,BE,N64,NAN1985,MIPS4    %s
+# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux   -mcpu=mips5    -target-abi=n64                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF64,BE,N64,NAN1985,MIPS5    %s
 
-# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux                                                  %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF64,BE,N64,NAN1985,MIPS64R1 %s
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux                                                %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN1985,MIPS64R1 %s
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r2                                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN1985,MIPS64R2 %s
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r3                                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN1985,MIPS64R3 %s
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r5                                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN1985,MIPS64R5 %s
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r2                 -mattr=+nan2008 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN2008,MIPS64R2 %s
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r3                 -mattr=+nan2008 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN2008,MIPS64R3 %s
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r5                 -mattr=+nan2008 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN2008,MIPS64R5 %s
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r6                                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN2008,MIPS64R6 %s
+# RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux                                                  %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF64,BE,N64,NAN1985,MIPS64R1 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux                                                %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN1985,MIPS64R1 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r2                                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN1985,MIPS64R2 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r3                                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN1985,MIPS64R3 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r5                                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN1985,MIPS64R5 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r2                 -mattr=+nan2008 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN2008,MIPS64R2 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r3                 -mattr=+nan2008 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN2008,MIPS64R3 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r5                 -mattr=+nan2008 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN2008,MIPS64R5 %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=mips64r6                                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN2008,MIPS64R6 %s
 
-# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=octeon   -target-abi=n64                 %s -o - | llvm-readobj -h | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN1985,OCTEON   %s
+# RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux -mcpu=octeon   -target-abi=n64                 %s -o - | llvm-readobj -h - | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN1985,OCTEON   %s
 # RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux \
 # RUN:         -mcpu=octeon+ -target-abi=n64 %s -o - \
-# RUN:   | llvm-readobj -h \
+# RUN:   | llvm-readobj -h - \
 # RUN:   | FileCheck --check-prefixes=ALL,ELF64,LE,N64,NAN1985,OCTEON %s
 
 # ALL:        ElfHeader {
diff --git a/llvm/test/MC/Mips/elf_reginfo.s b/llvm/test/MC/Mips/elf_reginfo.s
index ea6e39601eff5..5aef6b234abe0 100644
--- a/llvm/test/MC/Mips/elf_reginfo.s
+++ b/llvm/test/MC/Mips/elf_reginfo.s
@@ -2,9 +2,9 @@
 # corresponding options (-mabi=64 -> -mattr=+n64 for example).
 
 # RUN: llvm-mc -filetype=obj -triple=mips64el-linux -target-abi n64 %s -o - \
-# RUN: | llvm-readobj -S | FileCheck --check-prefix=CHECK_64 %s
+# RUN: | llvm-readobj -S - | FileCheck --check-prefix=CHECK_64 %s
 # RUN: llvm-mc -filetype=obj -triple=mipsel %s -target-abi n32 -o - \
-# RUN: | llvm-readobj -S | FileCheck --check-prefix=CHECK_32 %s
+# RUN: | llvm-readobj -S - | FileCheck --check-prefix=CHECK_32 %s
 
 # Check for register information sections.
 #
diff --git a/llvm/test/MC/Mips/elf_st_other.s b/llvm/test/MC/Mips/elf_st_other.s
index 8c78bb45db6b1..6be849343b829 100644
--- a/llvm/test/MC/Mips/elf_st_other.s
+++ b/llvm/test/MC/Mips/elf_st_other.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux %s -o -| llvm-readobj --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux %s -o -| llvm-readobj --symbols - | FileCheck %s
 
 
 .globl f1
diff --git a/llvm/test/MC/Mips/end-directive.s b/llvm/test/MC/Mips/end-directive.s
index a357b35b7be11..6545acb1613e0 100644
--- a/llvm/test/MC/Mips/end-directive.s
+++ b/llvm/test/MC/Mips/end-directive.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -arch=mips -mcpu=mips32 -filetype=obj %s -o - | \
-# RUN:   llvm-readobj --symbols | FileCheck %s
+# RUN:   llvm-readobj --symbols - | FileCheck %s
 
 # Check that the assembler doesn't choke on .align between a symbol and the
 # .end directive.
diff --git a/llvm/test/MC/Mips/higher-highest-addressing.s b/llvm/test/MC/Mips/higher-highest-addressing.s
index 4eb27d4c0d30d..56fd01b5e9aab 100644
--- a/llvm/test/MC/Mips/higher-highest-addressing.s
+++ b/llvm/test/MC/Mips/higher-highest-addressing.s
@@ -2,7 +2,7 @@
 # RUN:   | llvm-objdump -d - | FileCheck %s
 
 # RUN: llvm-mc -filetype=obj -triple=mips64el-unknown-linux -mcpu=mips64r2 %s \
-# RUN:   | llvm-readobj -r | FileCheck %s -check-prefix=CHECK-REL
+# RUN:   | llvm-readobj -r - | FileCheck %s -check-prefix=CHECK-REL
 
 
 # Test that R_MIPS_HIGHER and R_MIPS_HIGHEST relocations are created.  By using
diff --git a/llvm/test/MC/Mips/hilo-addressing.s b/llvm/test/MC/Mips/hilo-addressing.s
index 960f5b705d5b8..cff2d8a6e68eb 100644
--- a/llvm/test/MC/Mips/hilo-addressing.s
+++ b/llvm/test/MC/Mips/hilo-addressing.s
@@ -5,7 +5,7 @@
 # RUN:  | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-INSTR
 
 # RUN: llvm-mc -filetype=obj -triple=mipsel-unknown-linux %s \
-# RUN:  | llvm-readobj -r | FileCheck %s -check-prefix=CHECK-REL
+# RUN:  | llvm-readobj -r - | FileCheck %s -check-prefix=CHECK-REL
 
 
 # Check that 1 is added to the high 16 bits if bit 15 of the low part is 1.
diff --git a/llvm/test/MC/Mips/micromips-alias.s b/llvm/test/MC/Mips/micromips-alias.s
index 9b0dcf50f0367..0a6a586f7ce38 100644
--- a/llvm/test/MC/Mips/micromips-alias.s
+++ b/llvm/test/MC/Mips/micromips-alias.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux -mcpu=mips32 %s -o - \
-# RUN:   | llvm-readobj --symbols | FileCheck %s
+# RUN:   | llvm-readobj --symbols - | FileCheck %s
 
 # Symbol bar must be marked as micromips.
 # CHECK: Name: bar
diff --git a/llvm/test/MC/Mips/micromips-ase-directive.s b/llvm/test/MC/Mips/micromips-ase-directive.s
index fef40ecc3eeb5..9b2e1d42565e3 100644
--- a/llvm/test/MC/Mips/micromips-ase-directive.s
+++ b/llvm/test/MC/Mips/micromips-ase-directive.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -triple=mips-unknown-linux -filetype=obj %s -o - | \
-# RUN:   llvm-readobj -A | \
+# RUN:   llvm-readobj -A - | \
 # RUN:   FileCheck --check-prefix=ASE-MICROMIPS %s
 
 	.set	micromips
diff --git a/llvm/test/MC/Mips/micromips-branch-fixup.s b/llvm/test/MC/Mips/micromips-branch-fixup.s
index 3ad625fa4c13d..5a0f39123f0f6 100644
--- a/llvm/test/MC/Mips/micromips-branch-fixup.s
+++ b/llvm/test/MC/Mips/micromips-branch-fixup.s
@@ -1,7 +1,7 @@
 # RUN: llvm-mc %s -triple=mipsel-unknown-linux -show-encoding \
 # RUN: -mattr=micromips | FileCheck %s -check-prefix=CHECK-FIXUP
 # RUN: llvm-mc %s -filetype=obj -triple=mipsel-unknown-linux \
-# RUN: -mattr=micromips | llvm-readobj -r \
+# RUN: -mattr=micromips | llvm-readobj -r - \
 # RUN: | FileCheck %s -check-prefix=CHECK-ELF
 #------------------------------------------------------------------------------
 # Check that the assembler can handle the documented syntax
diff --git a/llvm/test/MC/Mips/micromips-func-addr.s b/llvm/test/MC/Mips/micromips-func-addr.s
index 0cd49352f3062..4a0e3fdbe4379 100644
--- a/llvm/test/MC/Mips/micromips-func-addr.s
+++ b/llvm/test/MC/Mips/micromips-func-addr.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc %s -filetype=obj -triple=mipsel-unknown-linux \
-# RUN: -mattr=micromips | llvm-readobj -r \
+# RUN: -mattr=micromips | llvm-readobj -r - \
 # RUN: | FileCheck %s
 # CHECK: Relocations [
 # CHECK:     0x0 R_MIPS_32 bar 0x0
diff --git a/llvm/test/MC/Mips/micromips-jump26.s b/llvm/test/MC/Mips/micromips-jump26.s
index 936a9984ab0a2..678a103c58a09 100644
--- a/llvm/test/MC/Mips/micromips-jump26.s
+++ b/llvm/test/MC/Mips/micromips-jump26.s
@@ -1,7 +1,7 @@
 # RUN: llvm-mc %s -triple=mipsel-unknown-linux -show-encoding \
 # RUN: -mattr=micromips | FileCheck %s -check-prefix=CHECK-FIXUP
 # RUN: llvm-mc %s -filetype=obj -triple=mipsel-unknown-linux \
-# RUN: -mattr=micromips | llvm-readobj -r \
+# RUN: -mattr=micromips | llvm-readobj -r - \
 # RUN: | FileCheck %s -check-prefix=CHECK-ELF
 #------------------------------------------------------------------------------
 # Check that the assembler can handle the documented syntax
diff --git a/llvm/test/MC/Mips/micromips-label-test-sections.s b/llvm/test/MC/Mips/micromips-label-test-sections.s
index 092b211de8ed7..561b783b150c8 100644
--- a/llvm/test/MC/Mips/micromips-label-test-sections.s
+++ b/llvm/test/MC/Mips/micromips-label-test-sections.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc %s -triple=mipsel-unknown-linux -mcpu=mips32r2 \
-# RUN:   -mattr=+micromips -filetype=obj -o - | llvm-readobj --symbols | FileCheck %s
+# RUN:   -mattr=+micromips -filetype=obj -o - | llvm-readobj --symbols - | FileCheck %s
   .text
   .set micromips
 f:
diff --git a/llvm/test/MC/Mips/micromips-label-test.s b/llvm/test/MC/Mips/micromips-label-test.s
index 1a8bbdf1ec160..80336d107d733 100644
--- a/llvm/test/MC/Mips/micromips-label-test.s
+++ b/llvm/test/MC/Mips/micromips-label-test.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc %s -triple=mipsel-unknown-linux -mcpu=mips32r2 \
-# RUN:   -mattr=+micromips -filetype=obj -o - | llvm-readobj --symbols | FileCheck %s
+# RUN:   -mattr=+micromips -filetype=obj -o - | llvm-readobj --symbols - | FileCheck %s
   .text
   .set nomicromips
 f:
diff --git a/llvm/test/MC/Mips/micromips-relocations.s b/llvm/test/MC/Mips/micromips-relocations.s
index a8734a06ca895..4f3afe9887cdb 100644
--- a/llvm/test/MC/Mips/micromips-relocations.s
+++ b/llvm/test/MC/Mips/micromips-relocations.s
@@ -1,7 +1,7 @@
 # RUN: llvm-mc %s -triple=mipsel-unknown-linux -show-encoding \
 # RUN: -mattr=micromips | FileCheck %s -check-prefix=CHECK-FIXUP
 # RUN: llvm-mc %s -filetype=obj -triple=mipsel-unknown-linux \
-# RUN: -mattr=micromips | llvm-readobj -r \
+# RUN: -mattr=micromips | llvm-readobj -r - \
 # RUN: | FileCheck %s -check-prefix=CHECK-ELF
 #------------------------------------------------------------------------------
 # Check that the assembler can handle the documented syntax
diff --git a/llvm/test/MC/Mips/micromips-tailr.s b/llvm/test/MC/Mips/micromips-tailr.s
index a11ac61eac2d2..1e4c888cfd2f3 100644
--- a/llvm/test/MC/Mips/micromips-tailr.s
+++ b/llvm/test/MC/Mips/micromips-tailr.s
@@ -1,7 +1,7 @@
 # RUN: llvm-mc %s -triple=mipsel-unknown-linux -show-encoding \
 # RUN: -mattr=micromips | FileCheck %s -check-prefix=CHECK-FIXUP
 # RUN: llvm-mc %s -filetype=obj -triple=mipsel-unknown-linux \
-# RUN: -mattr=micromips | llvm-readobj -r \
+# RUN: -mattr=micromips | llvm-readobj -r - \
 # RUN: | FileCheck %s -check-prefix=CHECK-ELF
 #------------------------------------------------------------------------------
 # Check that the assembler can handle the documented syntax
diff --git a/llvm/test/MC/Mips/micromips32r6/relocations.s b/llvm/test/MC/Mips/micromips32r6/relocations.s
index 615b445a0faad..f5e1748d6c5f0 100644
--- a/llvm/test/MC/Mips/micromips32r6/relocations.s
+++ b/llvm/test/MC/Mips/micromips32r6/relocations.s
@@ -1,7 +1,7 @@
 # RUN: llvm-mc %s -triple=mips-unknown-linux -show-encoding -mcpu=mips32r6 \
 # RUN:   -mattr=micromips | FileCheck %s -check-prefix=CHECK-FIXUP
 # RUN: llvm-mc %s -filetype=obj -triple=mips-unknown-linux -mcpu=mips32r6 \
-# RUN:   -mattr=micromips | llvm-readobj -r | FileCheck %s -check-prefix=CHECK-ELF
+# RUN:   -mattr=micromips | llvm-readobj -r - | FileCheck %s -check-prefix=CHECK-ELF
 #------------------------------------------------------------------------------
 # Check that the assembler can handle the documented syntax for fixups.
 #------------------------------------------------------------------------------
diff --git a/llvm/test/MC/Mips/mips-pdr.s b/llvm/test/MC/Mips/mips-pdr.s
index 267800ebcd501..8769f904e1d28 100644
--- a/llvm/test/MC/Mips/mips-pdr.s
+++ b/llvm/test/MC/Mips/mips-pdr.s
@@ -2,7 +2,7 @@
 # RUN:   FileCheck %s -check-prefix=ASMOUT
 
 # RUN: llvm-mc %s -arch=mips -mcpu=mips32r2 -filetype=obj -o - | \
-# RUN:   llvm-readobj -S --section-data -r | \
+# RUN:   llvm-readobj -S --section-data -r - | \
 # RUN:     FileCheck %s -check-prefix=OBJOUT
 
 # ASMOUT: .text
diff --git a/llvm/test/MC/Mips/mips-rdata.s b/llvm/test/MC/Mips/mips-rdata.s
index d2d8aa8c14a8d..0ed9301d549e0 100644
--- a/llvm/test/MC/Mips/mips-rdata.s
+++ b/llvm/test/MC/Mips/mips-rdata.s
@@ -1,7 +1,7 @@
 # Check that .rdata sections have proper name, flags, and section types.
 
 # RUN: llvm-mc -filetype=obj -triple=mips-unknown-linux %s -o - \
-# RUN:   | llvm-readobj -S | FileCheck %s
+# RUN:   | llvm-readobj -S - | FileCheck %s
 
   .rdata
   .word 0
diff --git a/llvm/test/MC/Mips/mips-reginfo-fp32.s b/llvm/test/MC/Mips/mips-reginfo-fp32.s
index 6b7d94aa57a6d..f37ad7c96b596 100644
--- a/llvm/test/MC/Mips/mips-reginfo-fp32.s
+++ b/llvm/test/MC/Mips/mips-reginfo-fp32.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc %s -arch=mips -mcpu=mips32r2 -filetype=obj -o - | \
-# RUN:   llvm-readobj -S --section-data | \
+# RUN:   llvm-readobj -S --section-data - | \
 # RUN:     FileCheck %s
 
 # CHECK:   Section {
diff --git a/llvm/test/MC/Mips/mips-reginfo-fp64.s b/llvm/test/MC/Mips/mips-reginfo-fp64.s
index 157801d3cec87..b7dee963082a6 100644
--- a/llvm/test/MC/Mips/mips-reginfo-fp64.s
+++ b/llvm/test/MC/Mips/mips-reginfo-fp64.s
@@ -1,13 +1,13 @@
 # RUN: llvm-mc %s -arch=mips -mcpu=mips32r2 -mattr=+msa,+fp64 -filetype=obj -o - | \
-# RUN:   llvm-readobj -S --section-data | \
+# RUN:   llvm-readobj -S --section-data - | \
 # RUN:     FileCheck %s -check-prefix=ELF32
 
 # RUN: llvm-mc %s -arch=mips64 -mcpu=mips64r2 -mattr=+msa,+fp64 -target-abi n32 -filetype=obj -o - | \
-# RUN:   llvm-readobj -S --section-data | \
+# RUN:   llvm-readobj -S --section-data - | \
 # RUN:     FileCheck %s -check-prefix=ELF32
 
 # RUN: llvm-mc %s -arch=mips64 -mcpu=mips64r2 -mattr=+msa,+fp64 -target-abi n64 -filetype=obj -o - | \
-# RUN:   llvm-readobj -S --section-data | \
+# RUN:   llvm-readobj -S --section-data - | \
 # RUN:     FileCheck %s -check-prefix=ELF64
 
 # ELF32:   Section {
diff --git a/llvm/test/MC/Mips/mips32r6/relocations.s b/llvm/test/MC/Mips/mips32r6/relocations.s
index 8095fb156ec93..68553fdf0177a 100644
--- a/llvm/test/MC/Mips/mips32r6/relocations.s
+++ b/llvm/test/MC/Mips/mips32r6/relocations.s
@@ -1,7 +1,7 @@
 # RUN: llvm-mc %s -triple=mips-unknown-linux -show-encoding -mcpu=mips32r6 \
 # RUN:   | FileCheck %s -check-prefix=CHECK-FIXUP
 # RUN: llvm-mc %s -filetype=obj -triple=mips-unknown-linux -mcpu=mips32r6 \
-# RUN:   | llvm-readobj -r | FileCheck %s -check-prefix=CHECK-ELF
+# RUN:   | llvm-readobj -r - | FileCheck %s -check-prefix=CHECK-ELF
 #------------------------------------------------------------------------------
 # Check that the assembler can handle the documented syntax for fixups.
 #------------------------------------------------------------------------------
diff --git a/llvm/test/MC/Mips/mips64eb-fixups.s b/llvm/test/MC/Mips/mips64eb-fixups.s
index 2de804d26e192..f5fadb1c38eab 100644
--- a/llvm/test/MC/Mips/mips64eb-fixups.s
+++ b/llvm/test/MC/Mips/mips64eb-fixups.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -filetype=obj -triple mips64eb-unknown-freebsd %s -o - | llvm-readobj -S --sd | FileCheck %s
+# RUN: llvm-mc -filetype=obj -triple mips64eb-unknown-freebsd %s -o - | llvm-readobj -S --sd - | FileCheck %s
 
         .section        .fixups,"",@progbits
         .byte   0xff
diff --git a/llvm/test/MC/Mips/mips64r6/relocations.s b/llvm/test/MC/Mips/mips64r6/relocations.s
index 5e70f44b96e1a..8353ec019a3ca 100644
--- a/llvm/test/MC/Mips/mips64r6/relocations.s
+++ b/llvm/test/MC/Mips/mips64r6/relocations.s
@@ -1,7 +1,7 @@
 # RUN: llvm-mc %s -triple=mips64-unknown-linux -show-encoding -mcpu=mips64r6 \
 # RUN:   | FileCheck %s -check-prefix=CHECK-FIXUP
 # RUN: llvm-mc %s -filetype=obj -triple=mips64-unknown-linux -mcpu=mips64r6 \
-# RUN:   | llvm-readobj -r | FileCheck %s -check-prefix=CHECK-ELF
+# RUN:   | llvm-readobj -r - | FileCheck %s -check-prefix=CHECK-ELF
 #------------------------------------------------------------------------------
 # Check that the assembler can handle the documented syntax for fixups.
 #------------------------------------------------------------------------------
diff --git a/llvm/test/MC/Mips/mt/abiflag.s b/llvm/test/MC/Mips/mt/abiflag.s
index d067c55587c91..9f8592635bebf 100644
--- a/llvm/test/MC/Mips/mt/abiflag.s
+++ b/llvm/test/MC/Mips/mt/abiflag.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc < %s -arch=mips -mcpu=mips32r2 -mattr=+mt -filetype=obj -o - \
-# RUN:   | llvm-readobj -A | FileCheck %s
+# RUN:   | llvm-readobj -A - | FileCheck %s
 
 # Test that the usage of the MT ASE is recorded in .MIPS.abiflags
 
diff --git a/llvm/test/MC/Mips/mt/module-directive.s b/llvm/test/MC/Mips/mt/module-directive.s
index 1bbe91147545e..0b74308900820 100644
--- a/llvm/test/MC/Mips/mt/module-directive.s
+++ b/llvm/test/MC/Mips/mt/module-directive.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc < %s -arch=mips -mcpu=mips32r2 -filetype=obj -o - | \
-# RUN:   llvm-readobj -A | FileCheck --check-prefix=CHECK-OBJ %s
+# RUN:   llvm-readobj -A - | FileCheck --check-prefix=CHECK-OBJ %s
 # RUN: llvm-mc < %s -arch=mips -mcpu=mips32r2 -filetype=asm -o - | \
 # RUN:   FileCheck --check-prefix=CHECK-ASM %s
 
diff --git a/llvm/test/MC/Mips/mt/set-directive.s b/llvm/test/MC/Mips/mt/set-directive.s
index 5d18486059d49..3593c2ddde522 100644
--- a/llvm/test/MC/Mips/mt/set-directive.s
+++ b/llvm/test/MC/Mips/mt/set-directive.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc < %s -arch=mips -mcpu=mips32r2 -filetype=obj -o - | \
-# RUN:   llvm-readobj -A | FileCheck %s --check-prefix=CHECK-OBJ
+# RUN:   llvm-readobj -A - | FileCheck %s --check-prefix=CHECK-OBJ
 # RUN: llvm-mc < %s -arch=mips -mcpu=mips32r2 -filetype=asm -o - | \
 # RUN:   FileCheck %s --check-prefix=CHECK-ASM
 
diff --git a/llvm/test/MC/Mips/r-mips-got-disp.s b/llvm/test/MC/Mips/r-mips-got-disp.s
index b75cac59217ad..1565b0858a42c 100644
--- a/llvm/test/MC/Mips/r-mips-got-disp.s
+++ b/llvm/test/MC/Mips/r-mips-got-disp.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple=mips64el-pc-linux -filetype=obj -mcpu=mips64r2 < %s -o - | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -triple=mips64el-pc-linux -filetype=obj -mcpu=mips64r2 < %s -o - | llvm-readobj -r - | FileCheck %s
 
 // Check that the R_MIPS_GOT_DISP relocations were created.
 
diff --git a/llvm/test/MC/Mips/reloc-directive-label-offset.s b/llvm/test/MC/Mips/reloc-directive-label-offset.s
index 05ee4dee0f36a..8df9e6ad745a5 100644
--- a/llvm/test/MC/Mips/reloc-directive-label-offset.s
+++ b/llvm/test/MC/Mips/reloc-directive-label-offset.s
@@ -5,11 +5,11 @@
 # RUN: llvm-mc -triple mips64-unknown-linux %s -show-encoding -target-abi=n64 \
 # RUN:     | FileCheck --check-prefixes=ASM,ASM-64 %s
 # RUN: llvm-mc -triple mips-unknown-linux %s -show-encoding -target-abi=o32 \
-# RUN:     -filetype=obj | llvm-readobj -r | FileCheck -check-prefix=OBJ-O32 %s
+# RUN:     -filetype=obj | llvm-readobj -r - | FileCheck -check-prefix=OBJ-O32 %s
 # RUN: llvm-mc -triple mips64-unknown-linux %s -show-encoding -target-abi=n32 \
-# RUN:     -filetype=obj | llvm-readobj -r | FileCheck -check-prefix=OBJ-N32 %s
+# RUN:     -filetype=obj | llvm-readobj -r - | FileCheck -check-prefix=OBJ-N32 %s
 # RUN: llvm-mc -triple mips64-unknown-linux %s -show-encoding -target-abi=n64 \
-# RUN:     -filetype=obj | llvm-readobj -r | FileCheck -check-prefix=OBJ-N64 %s
+# RUN:     -filetype=obj | llvm-readobj -r - | FileCheck -check-prefix=OBJ-N64 %s
 
   .text
 foo: # ASM-LABEL: foo:
diff --git a/llvm/test/MC/Mips/reloc-directive.s b/llvm/test/MC/Mips/reloc-directive.s
index b7498105681b0..b00ab8e1cfaae 100644
--- a/llvm/test/MC/Mips/reloc-directive.s
+++ b/llvm/test/MC/Mips/reloc-directive.s
@@ -5,13 +5,13 @@
 # RUN: llvm-mc -triple mips64-unknown-linux < %s -show-encoding -target-abi=n64 \
 # RUN:     | FileCheck -check-prefix=ASM %s
 # RUN: llvm-mc -triple mips-unknown-linux < %s -show-encoding -target-abi=o32 \
-# RUN:     -filetype=obj | llvm-readobj --sections --section-data -r | \
+# RUN:     -filetype=obj | llvm-readobj --sections --section-data -r - | \
 # RUN:     FileCheck -check-prefix=OBJ-O32 %s
 # RUN: llvm-mc -triple mips64-unknown-linux < %s -show-encoding -target-abi=n32 \
-# RUN:     -filetype=obj | llvm-readobj --sections --section-data -r | \
+# RUN:     -filetype=obj | llvm-readobj --sections --section-data -r - | \
 # RUN:     FileCheck -check-prefix=OBJ-N32 %s
 # RUN: llvm-mc -triple mips64-unknown-linux < %s -show-encoding -target-abi=n64 \
-# RUN:     -filetype=obj | llvm-readobj --sections --section-data -r | \
+# RUN:     -filetype=obj | llvm-readobj --sections --section-data -r - | \
 # RUN:     FileCheck -check-prefix=OBJ-N64 %s
 	.text
 foo:
diff --git a/llvm/test/MC/Mips/relocation-n64.s b/llvm/test/MC/Mips/relocation-n64.s
index 297b1a696c2ba..9aeba3183c640 100644
--- a/llvm/test/MC/Mips/relocation-n64.s
+++ b/llvm/test/MC/Mips/relocation-n64.s
@@ -3,9 +3,9 @@
 // RUN: llvm-mc -triple mips64el-unknown-linux < %s -show-encoding \
 // RUN:     | FileCheck -check-prefixes=ENCLE,FIXUP %s
 // RUN: llvm-mc -filetype=obj -triple mips64el-unknown-linux < %s \
-// RUN:     | llvm-readobj -r | FileCheck -check-prefix=RELOC %s
+// RUN:     | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 // RUN: llvm-mc -filetype=obj -triple mips64-unknown-linux < %s \
-// RUN:     | llvm-readobj --sections --section-data \
+// RUN:     | llvm-readobj --sections --section-data - \
 // RUN:     | FileCheck -check-prefix=DATA %s
 
 // Test that we produce the correct relocation.
diff --git a/llvm/test/MC/Mips/relocation.s b/llvm/test/MC/Mips/relocation.s
index 0ff450b24a004..9c8bb657ea68c 100644
--- a/llvm/test/MC/Mips/relocation.s
+++ b/llvm/test/MC/Mips/relocation.s
@@ -3,9 +3,9 @@
 // RUN: llvm-mc -triple mipsel-unknown-linux < %s -show-encoding \
 // RUN:     | FileCheck -check-prefixes=ENCLE,FIXUP %s
 // RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux < %s \
-// RUN:     | llvm-readobj -r | FileCheck -check-prefix=RELOC %s
+// RUN:     | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 // RUN: llvm-mc -filetype=obj -triple mips-unknown-linux < %s \
-// RUN:     | llvm-readobj --sections --section-data \
+// RUN:     | llvm-readobj --sections --section-data - \
 // RUN:     | FileCheck -check-prefix=DATA %s
 
 // Test that we produce the correct relocation.
diff --git a/llvm/test/MC/Mips/section-size.s b/llvm/test/MC/Mips/section-size.s
index 1d3319cbf091f..b51dad247092d 100644
--- a/llvm/test/MC/Mips/section-size.s
+++ b/llvm/test/MC/Mips/section-size.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -triple mips-unknown-linux -filetype=obj \
-# RUN:     -mips-round-section-sizes %s | llvm-readobj --sections | FileCheck %s
+# RUN:     -mips-round-section-sizes %s | llvm-readobj --sections - | FileCheck %s
 	.section ".talign1", "ax"
 	.p2align 4
 t1:	.byte 1
diff --git a/llvm/test/MC/Mips/sort-relocation-table.s b/llvm/test/MC/Mips/sort-relocation-table.s
index 16368c02d7b94..a6068ed76fc6d 100644
--- a/llvm/test/MC/Mips/sort-relocation-table.s
+++ b/llvm/test/MC/Mips/sort-relocation-table.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -filetype=obj -arch mipsel %s | llvm-readobj -r | FileCheck %s
+# RUN: llvm-mc -filetype=obj -arch mipsel %s | llvm-readobj -r - | FileCheck %s
 
 # Test the order of records in the relocation table.
 #
diff --git a/llvm/test/MC/Mips/tls-symbols.s b/llvm/test/MC/Mips/tls-symbols.s
index 7affb8c4405d3..964d57277ebfd 100644
--- a/llvm/test/MC/Mips/tls-symbols.s
+++ b/llvm/test/MC/Mips/tls-symbols.s
@@ -1,6 +1,6 @@
 # RUN: llvm-mc -arch=mips < %s -position-independent -filetype=obj \
-# RUN:   | llvm-readelf -s | FileCheck %s
-# RUN: llvm-mc -arch=mips < %s -filetype=obj | llvm-readelf -s | FileCheck %s
+# RUN:   | llvm-readelf -s - | FileCheck %s
+# RUN: llvm-mc -arch=mips < %s -filetype=obj | llvm-readelf -s - | FileCheck %s
 
 # Test that TLS relocations cause symbols to be marked as TLS symbols.
 
diff --git a/llvm/test/MC/Mips/xgot.s b/llvm/test/MC/Mips/xgot.s
index 9c64db749fd17..100d25e67223b 100644
--- a/llvm/test/MC/Mips/xgot.s
+++ b/llvm/test/MC/Mips/xgot.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux  %s -o - | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple mipsel-unknown-linux  %s -o - | llvm-readobj -r - | FileCheck %s
 
 // Check that the appropriate relocations were created.
 // For the xgot case we want to see R_MIPS_[GOT|CALL]_[HI|LO]16.
diff --git a/llvm/test/MC/PowerPC/lcomm.s b/llvm/test/MC/PowerPC/lcomm.s
index cc62332378e40..7abc79fb8f33e 100644
--- a/llvm/test/MC/PowerPC/lcomm.s
+++ b/llvm/test/MC/PowerPC/lcomm.s
@@ -1,7 +1,7 @@
 # RUN: llvm-mc -triple powerpc-unknown-unknown -filetype=obj %s | \
-# RUN: llvm-readobj -S --sd | FileCheck %s
+# RUN: llvm-readobj -S --sd - | FileCheck %s
 # RUN: llvm-mc -triple powerpc64-unknown-unknown -filetype=obj %s | \
-# RUN: llvm-readobj -S --sd | FileCheck %s
+# RUN: llvm-readobj -S --sd - | FileCheck %s
 
 .lcomm foo, 16, 16
 
diff --git a/llvm/test/MC/PowerPC/ppc-llong.s b/llvm/test/MC/PowerPC/ppc-llong.s
index 8b7bf133dd0d8..74fbe0bf0b507 100644
--- a/llvm/test/MC/PowerPC/ppc-llong.s
+++ b/llvm/test/MC/PowerPC/ppc-llong.s
@@ -1,10 +1,10 @@
 
 # RUN: llvm-mc -triple powerpc-unknown-unknown -filetype=obj %s | \
-# RUN: llvm-readobj -S --sd | FileCheck %s
+# RUN: llvm-readobj -S --sd - | FileCheck %s
 # RUN: llvm-mc -triple powerpc64-unknown-unknown -filetype=obj %s | \
-# RUN: llvm-readobj -S --sd | FileCheck %s
+# RUN: llvm-readobj -S --sd - | FileCheck %s
 # RUN: llvm-mc -triple powerpc64le-unknown-unknown -filetype=obj %s | \
-# RUN: llvm-readobj -S --sd | FileCheck %s
+# RUN: llvm-readobj -S --sd - | FileCheck %s
 
 .data
 .llong 0
diff --git a/llvm/test/MC/PowerPC/ppc-reloc.s b/llvm/test/MC/PowerPC/ppc-reloc.s
index 999d33ec5759e..62243ca6639a9 100644
--- a/llvm/test/MC/PowerPC/ppc-reloc.s
+++ b/llvm/test/MC/PowerPC/ppc-reloc.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -triple=powerpc-unknown-linux-gnu -filetype=obj %s | \
-# RUN: llvm-readobj -r | FileCheck %s
+# RUN: llvm-readobj -r - | FileCheck %s
 	.section .text
 
 	.globl foo
diff --git a/llvm/test/MC/PowerPC/ppc-word.s b/llvm/test/MC/PowerPC/ppc-word.s
index aade96a64379f..447c4a22f14cf 100644
--- a/llvm/test/MC/PowerPC/ppc-word.s
+++ b/llvm/test/MC/PowerPC/ppc-word.s
@@ -1,10 +1,10 @@
 
 # RUN: llvm-mc -triple powerpc-unknown-unknown -filetype=obj %s | \
-# RUN: llvm-readobj -S --sd | FileCheck %s
+# RUN: llvm-readobj -S --sd - | FileCheck %s
 # RUN: llvm-mc -triple powerpc64-unknown-unknown -filetype=obj %s | \
-# RUN: llvm-readobj -S --sd | FileCheck %s
+# RUN: llvm-readobj -S --sd - | FileCheck %s
 # RUN: llvm-mc -triple powerpc64le-unknown-unknown -filetype=obj %s | \
-# RUN: llvm-readobj -S --sd | FileCheck %s
+# RUN: llvm-readobj -S --sd - | FileCheck %s
 
 .data
 .word 0
diff --git a/llvm/test/MC/PowerPC/ppc32-reloc-directive.s b/llvm/test/MC/PowerPC/ppc32-reloc-directive.s
index 6beeabd4db370..b63e4c91eb08a 100644
--- a/llvm/test/MC/PowerPC/ppc32-reloc-directive.s
+++ b/llvm/test/MC/PowerPC/ppc32-reloc-directive.s
@@ -1,6 +1,6 @@
 # RUN: llvm-mc -triple=powerpc-linux-musl %s | FileCheck --check-prefix=PRINT %s
 
-# RUN: llvm-mc -filetype=obj -triple=powerpc-linux-musl %s | llvm-readobj -r | FileCheck %s
+# RUN: llvm-mc -filetype=obj -triple=powerpc-linux-musl %s | llvm-readobj -r - | FileCheck %s
 
 # PRINT: .reloc 8, R_PPC_NONE, .data
 # PRINT: .reloc 4, R_PPC_NONE, foo+4
diff --git a/llvm/test/MC/PowerPC/ppc64-abiversion.s b/llvm/test/MC/PowerPC/ppc64-abiversion.s
index 49d122772b8c3..37706c1de0509 100644
--- a/llvm/test/MC/PowerPC/ppc64-abiversion.s
+++ b/llvm/test/MC/PowerPC/ppc64-abiversion.s
@@ -1,8 +1,8 @@
 
 # RUN: llvm-mc -triple powerpc64-unknown-unknown -filetype=obj %s | \
-# RUN: llvm-readobj -h | FileCheck %s
+# RUN: llvm-readobj -h - | FileCheck %s
 # RUN: llvm-mc -triple powerpc64le-unknown-unknown -filetype=obj %s | \
-# RUN: llvm-readobj -h | FileCheck %s
+# RUN: llvm-readobj -h - | FileCheck %s
 # RUN: llvm-mc -triple powerpc64le-unknown-unknown -filetype=null %s
 
 	.abiversion 2
diff --git a/llvm/test/MC/PowerPC/ppc64-encoding-ISA31-pcrel-reloc.s b/llvm/test/MC/PowerPC/ppc64-encoding-ISA31-pcrel-reloc.s
index e6d41b061705d..bb849383c0f1e 100644
--- a/llvm/test/MC/PowerPC/ppc64-encoding-ISA31-pcrel-reloc.s
+++ b/llvm/test/MC/PowerPC/ppc64-encoding-ISA31-pcrel-reloc.s
@@ -1,12 +1,12 @@
 # RUN: llvm-mc -triple=powerpc64le-unknown-unknown -filetype=obj %s 2>&1 | \
 # RUN: FileCheck %s -check-prefix=MC
 # RUN: llvm-mc -triple=powerpc64le-unknown-unknown -filetype=obj %s | \
-# RUN: llvm-readobj -r | FileCheck %s -check-prefix=READOBJ
+# RUN: llvm-readobj -r - | FileCheck %s -check-prefix=READOBJ
 
 # RUN: llvm-mc -triple=powerpc64-unknown-unknown -filetype=obj %s 2>&1 | \
 # RUN: FileCheck %s -check-prefix=MC
 # RUN: llvm-mc -triple=powerpc64-unknown-unknown -filetype=obj %s | \
-# RUN: llvm-readobj -r | FileCheck %s -check-prefix=READOBJ
+# RUN: llvm-readobj -r - | FileCheck %s -check-prefix=READOBJ
 
 # This test checks that on Power PC we can correctly convert @pcrel and
 # @got@pcrel into R_PPC64_PCREL34 and R_PPC64_GOT_PCREL34.
diff --git a/llvm/test/MC/PowerPC/ppc64-encoding-ISA31.s b/llvm/test/MC/PowerPC/ppc64-encoding-ISA31.s
index b144493eb2a9e..c9f5547dd4c86 100644
--- a/llvm/test/MC/PowerPC/ppc64-encoding-ISA31.s
+++ b/llvm/test/MC/PowerPC/ppc64-encoding-ISA31.s
@@ -357,6 +357,57 @@
 # CHECK-BE: vsrdbi 2, 3, 4, 5                     # encoding: [0x10,0x43,0x23,0x56]
 # CHECK-LE: vsrdbi 2, 3, 4, 5                     # encoding: [0x56,0x23,0x43,0x10]
             vsrdbi 2, 3, 4, 5
+# CHECK-BE: vmulld 1, 2, 3                        # encoding: [0x10,0x22,0x19,0xc9]
+# CHECK-LE: vmulld 1, 2, 3                        # encoding: [0xc9,0x19,0x22,0x10]
+            vmulld 1, 2, 3
+# CHECK-BE: vmulhsw 1, 2, 3                       # encoding: [0x10,0x22,0x1b,0x89]
+# CHECK-LE: vmulhsw 1, 2, 3                       # encoding: [0x89,0x1b,0x22,0x10]
+            vmulhsw 1, 2, 3
+# CHECK-BE: vmulhuw 1, 2, 3                       # encoding: [0x10,0x22,0x1a,0x89]
+# CHECK-LE: vmulhuw 1, 2, 3                       # encoding: [0x89,0x1a,0x22,0x10]
+            vmulhuw 1, 2, 3
+# CHECK-BE: vmulhsd 1, 2, 3                       # encoding: [0x10,0x22,0x1b,0xc9]
+# CHECK-LE: vmulhsd 1, 2, 3                       # encoding: [0xc9,0x1b,0x22,0x10]
+            vmulhsd 1, 2, 3
+# CHECK-BE: vmulhud 1, 2, 3                       # encoding: [0x10,0x22,0x1a,0xc9]
+# CHECK-LE: vmulhud 1, 2, 3                       # encoding: [0xc9,0x1a,0x22,0x10]
+            vmulhud 1, 2, 3
+# CHECK-BE: vmodsw 21, 11, 10                     # encoding: [0x12,0xab,0x57,0x8b]
+# CHECK-LE: vmodsw 21, 11, 10                     # encoding: [0x8b,0x57,0xab,0x12]
+            vmodsw 21, 11, 10
+# CHECK-BE: vmoduw 21, 11, 10                     # encoding: [0x12,0xab,0x56,0x8b]
+# CHECK-LE: vmoduw 21, 11, 10                     # encoding: [0x8b,0x56,0xab,0x12]
+            vmoduw 21, 11, 10
+# CHECK-BE: vmodsd 21, 11, 10                     # encoding: [0x12,0xab,0x57,0xcb]
+# CHECK-LE: vmodsd 21, 11, 10                     # encoding: [0xcb,0x57,0xab,0x12]
+            vmodsd 21, 11, 10
+# CHECK-BE: vmodud 21, 11, 10                     # encoding: [0x12,0xab,0x56,0xcb]
+# CHECK-LE: vmodud 21, 11, 10                     # encoding: [0xcb,0x56,0xab,0x12]
+            vmodud 21, 11, 10
+# CHECK-BE: vdivsw 21, 11, 10                     # encoding: [0x12,0xab,0x51,0x8b]
+# CHECK-LE: vdivsw 21, 11, 10                     # encoding: [0x8b,0x51,0xab,0x12]
+            vdivsw 21, 11, 10
+# CHECK-BE: vdivuw 21, 11, 10                     # encoding: [0x12,0xab,0x50,0x8b]
+# CHECK-LE: vdivuw 21, 11, 10                     # encoding: [0x8b,0x50,0xab,0x12]
+            vdivuw 21, 11, 10
+# CHECK-BE: vdivsd 21, 11, 10                     # encoding: [0x12,0xab,0x51,0xcb]
+# CHECK-LE: vdivsd 21, 11, 10                     # encoding: [0xcb,0x51,0xab,0x12]
+            vdivsd 21, 11, 10
+# CHECK-BE: vdivud 21, 11, 10                     # encoding: [0x12,0xab,0x50,0xcb]
+# CHECK-LE: vdivud 21, 11, 10                     # encoding: [0xcb,0x50,0xab,0x12]
+            vdivud 21, 11, 10
+# CHECK-BE: vdivesw 21, 11, 10                    # encoding: [0x12,0xab,0x53,0x8b]
+# CHECK-LE: vdivesw 21, 11, 10                    # encoding: [0x8b,0x53,0xab,0x12]
+            vdivesw 21, 11, 10
+# CHECK-BE: vdiveuw 21, 11, 10                    # encoding: [0x12,0xab,0x52,0x8b]
+# CHECK-LE: vdiveuw 21, 11, 10                    # encoding: [0x8b,0x52,0xab,0x12]
+            vdiveuw 21, 11, 10
+# CHECK-BE: vdivesd 21, 11, 10                    # encoding: [0x12,0xab,0x53,0xcb]
+# CHECK-LE: vdivesd 21, 11, 10                    # encoding: [0xcb,0x53,0xab,0x12]
+            vdivesd 21, 11, 10
+# CHECK-BE: vdiveud 21, 11, 10                    # encoding: [0x12,0xab,0x52,0xcb]
+# CHECK-LE: vdiveud 21, 11, 10                    # encoding: [0xcb,0x52,0xab,0x12]
+            vdiveud 21, 11, 10
 # CHECK-BE: vinsw 2, 3, 12                        # encoding: [0x10,0x4c,0x18,0xcf]
 # CHECK-LE: vinsw 2, 3, 12                        # encoding: [0xcf,0x18,0x4c,0x10]
             vinsw 2, 3, 12
diff --git a/llvm/test/MC/PowerPC/ppc64-fixup-apply.s b/llvm/test/MC/PowerPC/ppc64-fixup-apply.s
index fd449c4e6c859..edf03f1cfab36 100644
--- a/llvm/test/MC/PowerPC/ppc64-fixup-apply.s
+++ b/llvm/test/MC/PowerPC/ppc64-fixup-apply.s
@@ -1,8 +1,8 @@
 
 # RUN: llvm-mc -triple powerpc64-unknown-unknown -filetype=obj %s | \
-# RUN: llvm-readobj -S --sd | FileCheck -check-prefix=CHECK -check-prefix=CHECK-BE %s
+# RUN: llvm-readobj -S --sd - | FileCheck -check-prefix=CHECK -check-prefix=CHECK-BE %s
 # RUN: llvm-mc -triple powerpc64le-unknown-unknown -filetype=obj %s | \
-# RUN: llvm-readobj -S --sd | FileCheck -check-prefix=CHECK -check-prefix=CHECK-LE %s
+# RUN: llvm-readobj -S --sd - | FileCheck -check-prefix=CHECK -check-prefix=CHECK-LE %s
 
 # This checks that fixups that can be resolved within the same
 # object file are applied correctly.
diff --git a/llvm/test/MC/PowerPC/ppc64-fixup-explicit.s b/llvm/test/MC/PowerPC/ppc64-fixup-explicit.s
index e0720ab5ea0a2..f73297b167e2f 100644
--- a/llvm/test/MC/PowerPC/ppc64-fixup-explicit.s
+++ b/llvm/test/MC/PowerPC/ppc64-fixup-explicit.s
@@ -3,9 +3,9 @@
 # RUN: llvm-mc -triple powerpc64le-unknown-unknown --show-encoding %s | FileCheck -check-prefix=CHECK-LE %s
 
 # RUN: llvm-mc -triple powerpc64-unknown-unknown -filetype=obj %s | \
-# RUN: llvm-readobj -r | FileCheck %s -check-prefix=CHECK-BE-REL
+# RUN: llvm-readobj -r - | FileCheck %s -check-prefix=CHECK-BE-REL
 # RUN: llvm-mc -triple powerpc64le-unknown-unknown -filetype=obj %s | \
-# RUN: llvm-readobj -r | FileCheck %s -check-prefix=CHECK-LE-REL
+# RUN: llvm-readobj -r - | FileCheck %s -check-prefix=CHECK-LE-REL
 
 # GOT references must result in explicit relocations
 # even if the target symbol is local.
diff --git a/llvm/test/MC/PowerPC/ppc64-fixups.s b/llvm/test/MC/PowerPC/ppc64-fixups.s
index ce19f36a85647..9904ad886e985 100644
--- a/llvm/test/MC/PowerPC/ppc64-fixups.s
+++ b/llvm/test/MC/PowerPC/ppc64-fixups.s
@@ -3,9 +3,9 @@
 # RUN: llvm-mc -triple powerpc64le-unknown-unknown --show-encoding %s | FileCheck -check-prefix=CHECK-LE %s
 
 # RUN: llvm-mc -triple powerpc64-unknown-unknown -filetype=obj %s | \
-# RUN: llvm-readobj -r | FileCheck %s -check-prefix=CHECK-BE-REL
+# RUN: llvm-readobj -r - | FileCheck %s -check-prefix=CHECK-BE-REL
 # RUN: llvm-mc -triple powerpc64le-unknown-unknown -filetype=obj %s | \
-# RUN: llvm-readobj -r | FileCheck %s -check-prefix=CHECK-LE-REL
+# RUN: llvm-readobj -r - | FileCheck %s -check-prefix=CHECK-LE-REL
 
 # CHECK-BE: b target                        # encoding: [0b010010AA,A,A,0bAAAAAA00]
 # CHECK-LE: b target                        # encoding: [0bAAAAAA00,A,A,0b010010AA]
diff --git a/llvm/test/MC/PowerPC/ppc64-initial-cfa.s b/llvm/test/MC/PowerPC/ppc64-initial-cfa.s
index 39567f596597e..e7be1f51ac2c3 100644
--- a/llvm/test/MC/PowerPC/ppc64-initial-cfa.s
+++ b/llvm/test/MC/PowerPC/ppc64-initial-cfa.s
@@ -1,11 +1,11 @@
 # RUN: llvm-mc -triple=powerpc64-unknown-linux-gnu -filetype=obj %s | \
-# RUN: llvm-readobj -S --sr --sd | FileCheck %s -check-prefix=STATIC -check-prefix=STATIC-BE
+# RUN: llvm-readobj -S --sr --sd - | FileCheck %s -check-prefix=STATIC -check-prefix=STATIC-BE
 # RUN: llvm-mc -triple=powerpc64-unknown-linux-gnu -filetype=obj %s | \
-# RUN: llvm-readobj -S --sr --sd | FileCheck %s -check-prefix=PIC -check-prefix=PIC-BE
+# RUN: llvm-readobj -S --sr --sd - | FileCheck %s -check-prefix=PIC -check-prefix=PIC-BE
 # RUN: llvm-mc -triple=powerpc64le-unknown-linux-gnu -filetype=obj %s | \
-# RUN: llvm-readobj -S --sr --sd | FileCheck %s -check-prefix=STATIC -check-prefix=STATIC-LE
+# RUN: llvm-readobj -S --sr --sd - | FileCheck %s -check-prefix=STATIC -check-prefix=STATIC-LE
 # RUN: llvm-mc -triple=powerpc64le-unknown-linux-gnu -filetype=obj %s | \
-# RUN: llvm-readobj -S --sr --sd | FileCheck %s -check-prefix=PIC -check-prefix=PIC-LE
+# RUN: llvm-readobj -S --sr --sd - | FileCheck %s -check-prefix=PIC -check-prefix=PIC-LE
 
 _proc:
         .cfi_startproc
diff --git a/llvm/test/MC/PowerPC/ppc64-localentry.s b/llvm/test/MC/PowerPC/ppc64-localentry.s
index 883b2fcc38efb..0477311549165 100644
--- a/llvm/test/MC/PowerPC/ppc64-localentry.s
+++ b/llvm/test/MC/PowerPC/ppc64-localentry.s
@@ -1,8 +1,8 @@
 
 # RUN: llvm-mc -triple powerpc64-unknown-unknown -filetype=obj %s | \
-# RUN: llvm-readobj -h -r --symbols | FileCheck %s
+# RUN: llvm-readobj -h -r --symbols - | FileCheck %s
 # RUN: llvm-mc -triple powerpc64le-unknown-unknown -filetype=obj %s | \
-# RUN: llvm-readobj -h -r --symbols | FileCheck %s
+# RUN: llvm-readobj -h -r --symbols - | FileCheck %s
 
 	.type callee1, @function
 callee1:
diff --git a/llvm/test/MC/PowerPC/ppc64-reloc-directive-pcrel.s b/llvm/test/MC/PowerPC/ppc64-reloc-directive-pcrel.s
new file mode 100644
index 0000000000000..c17445f0a5022
--- /dev/null
+++ b/llvm/test/MC/PowerPC/ppc64-reloc-directive-pcrel.s
@@ -0,0 +1,401 @@
+# RUN: llvm-mc -triple=powerpc64le -filetype=obj %s | \
+# RUN: llvm-objdump -dr --mcpu=pwr10 - | FileCheck %s
+# RUN: llvm-mc -triple=powerpc64 -filetype=obj %s | \
+# RUN: llvm-objdump -dr --mcpu=pwr10 - | FileCheck %s
+
+
+##
+# This section of tests contains the MCBinaryExpr as the first parameter of the
+# .reloc relocation.
+##
+	.text
+	.abiversion 2
+	.globl	Minimal
+	.p2align	4
+	.type	Minimal,@function
+Minimal:
+.LMinimal$local:
+	pld 3, vec@got@pcrel(0), 1
+.Lpcrel1:
+	.reloc .Lpcrel1-8,R_PPC64_PCREL_OPT,.-(.Lpcrel1-8)
+	lwa 3, 4(3)
+	blr
+	.long	0
+	.quad	0
+# CHECK-LABEL:   Minimal
+# CHECK:         pld 3, 0(0), 1
+# CHECK-NEXT:    R_PPC64_GOT_PCREL34	vec
+# CHECK-NEXT:    R_PPC64_PCREL_OPT	*ABS*+0x8
+# CHECK-NEXT:    lwa 3, 4(3)
+# CHECK-NEXT:    blr
+
+	.globl	SingleInsnBetween
+	.p2align	4
+	.type	SingleInsnBetween,@function
+SingleInsnBetween:
+.LSingleInsnBetween$local:
+	pld 3, vec@got@pcrel(0), 1
+.Lpcrel2:
+        addi 3, 3, 42
+	.reloc .Lpcrel2-8,R_PPC64_PCREL_OPT,.-(.Lpcrel2-8)
+	lwa 3, 4(3)
+	blr
+	.long	0
+	.quad	0
+# CHECK_LABEL:   SingleInsnBetween
+# CHECK:         pld 3, 0(0), 1
+# CHECK-NEXT:    R_PPC64_GOT_PCREL34	vec
+# CHECK-NEXT:    R_PPC64_PCREL_OPT	*ABS*+0xc
+# CHECK-NEXT:    addi 3, 3, 42
+# CHECK-NEXT:    lwa 3, 4(3)
+# CHECK-NEXT:    blr
+
+
+	.globl	MultiInsnBetween                    # -- Begin function
+	.p2align	4
+	.type	MultiInsnBetween,@function
+MultiInsnBetween:
+.LMultiInsnBetween$local:
+	pld 3, vec@got@pcrel(0), 1
+.Lpcrel3:
+        addi 3, 3, 42
+        addi 3, 3, 42
+        addi 3, 3, 42
+        addi 3, 3, 42
+        addi 3, 3, 42
+	.reloc .Lpcrel3-8,R_PPC64_PCREL_OPT,.-(.Lpcrel3-8)
+	lwa 3, 4(3)
+	blr
+	.long	0
+	.quad	0
+# CHECK_LABEL:   MultiInsnBetween
+# CHECK:         pld 3, 0(0), 1
+# CHECK-NEXT:    R_PPC64_GOT_PCREL34	vec
+# CHECK-NEXT:    R_PPC64_PCREL_OPT	*ABS*+0x1c
+# CHECK-NEXT:    addi 3, 3, 42
+# CHECK-NEXT:    addi 3, 3, 42
+# CHECK-NEXT:    addi 3, 3, 42
+# CHECK-NEXT:    addi 3, 3, 42
+# CHECK-NEXT:    addi 3, 3, 42
+# CHECK-NEXT:    lwa 3, 4(3)
+# CHECK-NEXT:    blr
+
+	.globl	PrefixInsnBetween
+	.p2align	6
+	.type	PrefixInsnBetween,@function
+        .space          48       # Add a space to force an alignment of a paddi.
+PrefixInsnBetween:
+.LPrefixInsnBetween$local:
+	pld 3, vec@got@pcrel(0), 1
+.Lpcrel4:
+        addi 3, 3, 42
+        paddi 3, 3, 42, 0
+        addi 3, 3, 42
+        paddi 3, 3, 42, 0
+        addi 3, 3, 42
+	.reloc .Lpcrel4-8,R_PPC64_PCREL_OPT,.-(.Lpcrel4-8)
+	lwa 3, 4(3)
+	blr
+	.long	0
+	.quad	0
+# CHECK_LABEL:   PrefixInsnBetween
+# CHECK:         pld 3, 0(0), 1
+# CHECK-NEXT:    R_PPC64_GOT_PCREL34	vec
+# CHECK-NEXT:    R_PPC64_PCREL_OPT	*ABS*+0x28
+# CHECK-NEXT:    addi 3, 3, 42
+# CHECK-NEXT:    nop
+# CHECK-NEXT:    paddi 3, 3, 42, 0
+# CHECK-NEXT:    addi 3, 3, 42
+# CHECK-NEXT:    paddi 3, 3, 42, 0
+# CHECK-NEXT:    addi 3, 3, 42
+# CHECK-NEXT:    lwa 3, 4(3)
+# CHECK-NEXT:    blr
+
+
+	.globl	SpaceBetween                    # -- Begin function
+	.p2align	4
+	.type	SpaceBetween,@function
+SpaceBetween:
+.LSpaceBetween$local:
+	pld 3, vec@got@pcrel(0), 1
+.Lpcrel5:
+        addi 3, 3, 42
+        paddi 3, 3, 42, 0
+        addi 3, 3, 42
+        .space 40, 0
+        paddi 3, 3, 42, 0
+        addi 3, 3, 42
+	.reloc .Lpcrel5-8,R_PPC64_PCREL_OPT,.-(.Lpcrel5-8)
+	lwa 3, 4(3)
+	blr
+	.long	0
+	.quad	0
+# CHECK_LABEL:   SpaceBetween
+# CHECK:         pld 3, 0(0), 1
+# CHECK-NEXT:    R_PPC64_GOT_PCREL34	vec
+# CHECK-NEXT:    R_PPC64_PCREL_OPT	*ABS*+0x50
+# CHECK-NEXT:    addi 3, 3, 42
+# CHECK-NEXT:    nop
+# CHECK-NEXT:    paddi 3, 3, 42, 0
+# CHECK-NEXT:    addi 3, 3, 42
+# CHECK:         paddi 3, 3, 42, 0
+# CHECK-NEXT:    addi 3, 3, 42
+# CHECK-NEXT:    lwa 3, 4(3)
+# CHECK-NEXT:    blr
+
+
+	.globl	Plus
+	.p2align	4
+	.type	Plus,@function
+Plus:
+.LPlus$local:
+.Lpcrel6:
+        addi 3, 3, 42
+        addi 3, 3, 42
+	pld 3, vec@got@pcrel(0), 1
+	.reloc .Lpcrel6+8,R_PPC64_PCREL_OPT,.-(.Lpcrel6+8)
+	lwa 3, 4(3)
+	blr
+	.long	0
+	.quad	0
+# CHECK-LABEL:   Plus
+# CHECK:         pld 3, 0(0), 1
+# CHECK-NEXT:    R_PPC64_PCREL_OPT	*ABS*+0x8
+# CHECK-NEXT:    R_PPC64_GOT_PCREL34	vec
+# CHECK-NEXT:    lwa 3, 4(3)
+# CHECK-NEXT:    blr
+
+##
+# This section of tests contains the variable MCSymbol as part of the
+# MCSymbolRefExpr for the first parameter of the .reloc relocation.
+##
+	.globl	VarLabelMinimal                    # -- Begin function
+	.p2align	4
+	.type	VarLabelMinimal,@function
+VarLabelMinimal:
+.LVarLabelMinimal$local:
+	pld 3, vec@got@pcrel(0), 1
+.Lpcrel101=.-8
+	.reloc .Lpcrel101,R_PPC64_PCREL_OPT,.-.Lpcrel101
+	lwa 3, 4(3)
+	blr
+	.long	0
+	.quad	0
+# CHECK-LABEL:   VarLabelMinimal
+# CHECK:         pld 3, 0(0), 1
+# CHECK-NEXT:    R_PPC64_GOT_PCREL34	vec
+# CHECK-NEXT:    R_PPC64_PCREL_OPT	*ABS*+0x8
+# CHECK-NEXT:    lwa 3, 4(3)
+# CHECK-NEXT:    blr
+
+
+	.globl	VarLabelSingleInsnBetween
+	.p2align	4
+	.type	VarLabelSingleInsnBetween,@function
+VarLabelSingleInsnBetween:
+.LVarLabelSingleInsnBetween$local:
+	pld 3, vec@got@pcrel(0), 1
+.Lpcrel102=.-8
+        addi 3, 3, 42
+	.reloc .Lpcrel102,R_PPC64_PCREL_OPT,.-.Lpcrel102
+	lwa 3, 4(3)
+	blr
+	.long	0
+	.quad	0
+# CHECK_LABEL:   VarLabelSingleInsnBetween
+# CHECK:         pld 3, 0(0), 1
+# CHECK-NEXT:    R_PPC64_GOT_PCREL34	vec
+# CHECK-NEXT:    R_PPC64_PCREL_OPT	*ABS*+0xc
+# CHECK-NEXT:    addi 3, 3, 42
+# CHECK-NEXT:    lwa 3, 4(3)
+# CHECK-NEXT:    blr
+
+	.globl	VarLabelMultiInsnBetween                    # -- Begin function
+	.p2align	4
+	.type	VarLabelMultiInsnBetween,@function
+VarLabelMultiInsnBetween:
+.LVarLabelMultiInsnBetween$local:
+	pld 3, vec@got@pcrel(0), 1
+.Lpcrel103=.-8
+        addi 3, 3, 42
+        addi 3, 3, 42
+        addi 3, 3, 42
+        addi 3, 3, 42
+        addi 3, 3, 42
+	.reloc .Lpcrel103,R_PPC64_PCREL_OPT,.-.Lpcrel103
+	lwa 3, 4(3)
+	blr
+	.long	0
+	.quad	0
+# CHECK_LABEL:   VarLabelMultiInsnBetween
+# CHECK:         pld 3, 0(0), 1
+# CHECK-NEXT:    R_PPC64_GOT_PCREL34	vec
+# CHECK-NEXT:    R_PPC64_PCREL_OPT	*ABS*+0x1c
+# CHECK-NEXT:    addi 3, 3, 42
+# CHECK-NEXT:    addi 3, 3, 42
+# CHECK-NEXT:    addi 3, 3, 42
+# CHECK-NEXT:    addi 3, 3, 42
+# CHECK-NEXT:    addi 3, 3, 42
+# CHECK-NEXT:    lwa 3, 4(3)
+# CHECK-NEXT:    blr
+
+
+	.globl	VarLabelPrefixInsnBetween                    # -- Begin function
+	.p2align	4
+	.type	VarLabelPrefixInsnBetween,@function
+VarLabelPrefixInsnBetween:
+.LVarLabelPrefixInsnBetween$local:
+	pld 3, vec@got@pcrel(0), 1
+.Lpcrel104=.-8
+        addi 3, 3, 42
+        paddi 3, 3, 42, 0
+        addi 3, 3, 42
+        paddi 3, 3, 42, 0
+        addi 3, 3, 42
+	.reloc .Lpcrel104,R_PPC64_PCREL_OPT,.-.Lpcrel104
+	lwa 3, 4(3)
+	blr
+	.long	0
+	.quad	0
+# CHECK_LABEL:   VarLabelPrefixInsnBetween
+# CHECK:         pld 3, 0(0), 1
+# CHECK-NEXT:    R_PPC64_GOT_PCREL34	vec
+# CHECK-NEXT:    R_PPC64_PCREL_OPT	*ABS*+0x24
+# CHECK-NEXT:    addi 3, 3, 42
+# CHECK-NEXT:    paddi 3, 3, 42, 0
+# CHECK-NEXT:    addi 3, 3, 42
+# CHECK-NEXT:    paddi 3, 3, 42, 0
+# CHECK-NEXT:    addi 3, 3, 42
+# CHECK-NEXT:    lwa 3, 4(3)
+# CHECK-NEXT:    blr
+
+
+	.globl	VarLabelSpaceBetween                    # -- Begin function
+	.p2align	4
+	.type	VarLabelSpaceBetween,@function
+VarLabelSpaceBetween:
+.LVarLabelSpaceBetween$local:
+	pld 3, vec@got@pcrel(0), 1
+.Lpcrel105=.-8
+        addi 3, 3, 42
+        paddi 3, 3, 42, 0
+        addi 3, 3, 42
+        .space 40, 0
+        paddi 3, 3, 42, 0
+        addi 3, 3, 42
+	.reloc .Lpcrel105,R_PPC64_PCREL_OPT,.-.Lpcrel105
+	lwa 3, 4(3)
+	blr
+	.long	0
+	.quad	0
+# CHECK_LABEL:   VarLabelSpaceBetween
+# CHECK:         pld 3, 0(0), 1
+# CHECK-NEXT:    R_PPC64_GOT_PCREL34	vec
+# CHECK-NEXT:    R_PPC64_PCREL_OPT	*ABS*+0x4c
+# CHECK-NEXT:    addi 3, 3, 42
+# CHECK-NEXT:    paddi 3, 3, 42, 0
+# CHECK-NEXT:    addi 3, 3, 42
+# CHECK:         paddi 3, 3, 42, 0
+# CHECK-NEXT:    addi 3, 3, 42
+# CHECK-NEXT:    lwa 3, 4(3)
+# CHECK-NEXT:    blr
+
+
+	.globl	VarLabelPlus
+	.p2align	4
+	.type	VarLabelPlus,@function
+VarLabelPlus:
+.LVarLabelPlus$local:
+.Lpcrel106:
+        addi 3, 3, 42
+        addi 3, 3, 42
+	pld 3, vec@got@pcrel(0), 1
+	.reloc .Lpcrel106+8,R_PPC64_PCREL_OPT,.-(.Lpcrel106+8)
+	lwa 3, 4(3)
+	blr
+	.long	0
+	.quad	0
+# CHECK-LABEL:   VarLabelPlus
+# CHECK:         pld 3, 0(0), 1
+# CHECK-NEXT:    R_PPC64_PCREL_OPT	*ABS*+0x8
+# CHECK-NEXT:    R_PPC64_GOT_PCREL34	vec
+# CHECK-NEXT:    lwa 3, 4(3)
+# CHECK-NEXT:    blr
+
+# Check the situation where the PLD requires an alignment nop.
+	.globl	AlignPLD
+	.p2align	6
+	.type	AlignPLD,@function
+        .space          60      # Force the pld to require an alignment nop.
+AlignPLD:
+.LAlignPLD$local:
+	pld 3, vec@got@pcrel(0), 1
+.Lpcrel201:
+	.reloc .Lpcrel201-8,R_PPC64_PCREL_OPT,.-(.Lpcrel201-8)
+	lwa 3, 4(3)
+	blr
+# CHECK-LABEL:   AlignPLD
+# CHECK:         nop
+# CHECK-NEXT:    pld 3, 0(0), 1
+# CHECK-NEXT:    R_PPC64_GOT_PCREL34	vec
+# CHECK-NEXT:    R_PPC64_PCREL_OPT	*ABS*+0x8
+# CHECK-NEXT:    lwa 3, 4(3)
+# CHECK-NEXT:    blr
+
+# The label and the pld are on the same line and so the nop is inserted before
+# the label and the relocation should work.
+	.globl	AlignPLDSameLine
+	.p2align	6
+	.type	AlignPLDSameLine,@function
+        .space          60      # Force the pld to require an alignment nop.
+AlignPLDSameLine:
+.LAlignPLDSameLine$local:
+.Lpcrel202: pld 3, vec@got@pcrel(0), 1
+	.reloc .Lpcrel202,R_PPC64_PCREL_OPT,.-.Lpcrel202
+	lwa 3, 4(3)
+	blr
+# CHECK-LABEL:   AlignPLDSameLine
+# CHECK:         nop
+# CHECK-NEXT:    pld 3, 0(0), 1
+# CHECK-NEXT:    R_PPC64_GOT_PCREL34	vec
+# CHECK-NEXT:    R_PPC64_PCREL_OPT	*ABS*+0x8
+# CHECK-NEXT:    lwa 3, 4(3)
+# CHECK-NEXT:    blr
+
+	.globl	AlignPLDLabelBefore
+	.p2align	6
+	.type	AlignPLDLabelBefore,@function
+        .space          60      # Force the pld to require an alignment nop.
+AlignPLDLabelBefore:
+.LAlignPLDLabelBefore$local:
+.Label:
+	pld 3, vec@got@pcrel(0), 1
+.Lpcrel203:
+	.reloc .Lpcrel203-8,R_PPC64_PCREL_OPT,.-(.Lpcrel203-8)
+	lwa 3, 4(3)
+	blr
+# CHECK-LABEL:   AlignPLDLabelBefore
+# CHECK:         nop
+# CHECK-NEXT:    pld 3, 0(0), 1
+# CHECK-NEXT:    R_PPC64_GOT_PCREL34	vec
+# CHECK-NEXT:    R_PPC64_PCREL_OPT	*ABS*+0x8
+# CHECK-NEXT:    lwa 3, 4(3)
+# CHECK-NEXT:    blr
+
+	.globl	AlignPLDLabelSameLine
+	.p2align	6
+	.type	AlignPLDLabelSameLine,@function
+        .space          60      # Force the pld to require an alignment nop.
+AlignPLDLabelSameLine:
+.Label2: pld 3, vec@got@pcrel(0), 1
+.Lpcrel204:
+	.reloc .Lpcrel204-8,R_PPC64_PCREL_OPT,.-(.Lpcrel204-8)
+	lwa 3, 4(3)
+	blr
+# CHECK-LABEL:   AlignPLDLabelSameLine
+# CHECK:         nop
+# CHECK-NEXT:    pld 3, 0(0), 1
+# CHECK-NEXT:    R_PPC64_GOT_PCREL34	vec
+# CHECK-NEXT:    R_PPC64_PCREL_OPT	*ABS*+0x8
+# CHECK-NEXT:    lwa 3, 4(3)
+# CHECK-NEXT:    blr
diff --git a/llvm/test/MC/PowerPC/ppc64-reloc-directive.s b/llvm/test/MC/PowerPC/ppc64-reloc-directive.s
index 2a03a92786b7e..e683dbeb75978 100644
--- a/llvm/test/MC/PowerPC/ppc64-reloc-directive.s
+++ b/llvm/test/MC/PowerPC/ppc64-reloc-directive.s
@@ -1,8 +1,8 @@
 # RUN: llvm-mc -triple=powerpc64-linux-musl %s | FileCheck --check-prefix=PRINT %s
 # RUN: llvm-mc -triple=powerpc64le-linux-musl %s | FileCheck --check-prefix=PRINT %s
 
-# RUN: llvm-mc -filetype=obj -triple=powerpc64-linux-musl %s | llvm-readobj -r | FileCheck %s
-# RUN: llvm-mc -filetype=obj -triple=powerpc64le-linux-musl %s | llvm-readobj -r | FileCheck %s
+# RUN: llvm-mc -filetype=obj -triple=powerpc64-linux-musl %s | llvm-readobj -r - | FileCheck %s
+# RUN: llvm-mc -filetype=obj -triple=powerpc64le-linux-musl %s | llvm-readobj -r - | FileCheck %s
 
 # PRINT: .reloc 8, R_PPC64_NONE, .data
 # PRINT: .reloc 4, R_PPC64_NONE, foo+4
diff --git a/llvm/test/MC/PowerPC/ppc64-relocs-01.s b/llvm/test/MC/PowerPC/ppc64-relocs-01.s
index f2e899b83430d..0cc0c106717ae 100644
--- a/llvm/test/MC/PowerPC/ppc64-relocs-01.s
+++ b/llvm/test/MC/PowerPC/ppc64-relocs-01.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -triple=powerpc64-unknown-linux-gnu -filetype=obj %s | \
-# RUN: llvm-readobj -r | FileCheck %s
+# RUN: llvm-readobj -r - | FileCheck %s
 
         .section .opd,"aw",@progbits
 access_int64:
diff --git a/llvm/test/MC/PowerPC/ppc64-tls-relocs-01.s b/llvm/test/MC/PowerPC/ppc64-tls-relocs-01.s
index a37dd97fec67e..e9e8466f86b17 100644
--- a/llvm/test/MC/PowerPC/ppc64-tls-relocs-01.s
+++ b/llvm/test/MC/PowerPC/ppc64-tls-relocs-01.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -triple=powerpc64-unknown-linux-gnu -filetype=obj %s | \
-# RUN: llvm-readobj -r | FileCheck %s
+# RUN: llvm-readobj -r - | FileCheck %s
 
         .text
         addis 3, 13, t@tprel@ha
diff --git a/llvm/test/MC/PowerPC/st-other-crash.s b/llvm/test/MC/PowerPC/st-other-crash.s
index 93a547a2ebbf8..50fbd5186d55a 100644
--- a/llvm/test/MC/PowerPC/st-other-crash.s
+++ b/llvm/test/MC/PowerPC/st-other-crash.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc < %s -filetype=obj -triple powerpc64le-pc-linux | \
-// RUN:   llvm-readobj --symbols | FileCheck %s
+// RUN:   llvm-readobj --symbols - | FileCheck %s
 
 // This used to crash. Make sure it produce the correct symbol.
 
diff --git a/llvm/test/MC/PowerPC/tls-gd-obj.s b/llvm/test/MC/PowerPC/tls-gd-obj.s
index 66c8fa0138bd6..8cbda288cec70 100644
--- a/llvm/test/MC/PowerPC/tls-gd-obj.s
+++ b/llvm/test/MC/PowerPC/tls-gd-obj.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -triple=powerpc64-pc-linux -filetype=obj %s -o - | \
-// RUN: llvm-readobj -r | FileCheck %s
+// RUN: llvm-readobj -r - | FileCheck %s
 
 // Test correct relocation generation for thread-local storage using
 // the general dynamic model and integrated assembly.
diff --git a/llvm/test/MC/PowerPC/tls-ie-obj.s b/llvm/test/MC/PowerPC/tls-ie-obj.s
index f3bb7ee033540..b90a45e60ff1a 100644
--- a/llvm/test/MC/PowerPC/tls-ie-obj.s
+++ b/llvm/test/MC/PowerPC/tls-ie-obj.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -triple=powerpc64-pc-linux -filetype=obj %s -o - | \
-// RUN: llvm-readobj -r | FileCheck %s
+// RUN: llvm-readobj -r - | FileCheck %s
 
 // Test correct relocation generation for thread-local storage
 // using the initial-exec model and integrated assembly.
diff --git a/llvm/test/MC/PowerPC/tls-ld-obj.s b/llvm/test/MC/PowerPC/tls-ld-obj.s
index 3538d70a30a66..2631d95145f47 100644
--- a/llvm/test/MC/PowerPC/tls-ld-obj.s
+++ b/llvm/test/MC/PowerPC/tls-ld-obj.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -triple=powerpc64-pc-linux -filetype=obj %s -o - | \
-// RUN: llvm-readobj -r | FileCheck %s
+// RUN: llvm-readobj -r - | FileCheck %s
 
 // Test correct relocation generation for thread-local storage using
 // the local dynamic model.
diff --git a/llvm/test/MC/PowerPC/tls-ld-v2-abi.s b/llvm/test/MC/PowerPC/tls-ld-v2-abi.s
index 6eaa9888168b5..9f2cea6f3a755 100644
--- a/llvm/test/MC/PowerPC/tls-ld-v2-abi.s
+++ b/llvm/test/MC/PowerPC/tls-ld-v2-abi.s
@@ -1,8 +1,8 @@
 // RUN: llvm-mc -triple=powerpc64le-pc-linux -filetype=obj %s -o - | \
-// RUN: llvm-readobj -r | FileCheck %s
+// RUN: llvm-readobj -r - | FileCheck %s
 
 // RUN: llvm-mc -triple=powerpc64-pc-linux -filetype=obj %s -o - | \
-// RUN: llvm-readobj -r | FileCheck %s
+// RUN: llvm-readobj -r - | FileCheck %s
 
 // Verify we can handle all the dtprel symbol modifiers for local-dynamic tls.
 // Tests a 16 bit offset on both DS-form and D-form instructions, 32 bit
diff --git a/llvm/test/MC/PowerPC/tls-le-v2-abi.s b/llvm/test/MC/PowerPC/tls-le-v2-abi.s
index db122262ee20b..d8a7f34930ddc 100644
--- a/llvm/test/MC/PowerPC/tls-le-v2-abi.s
+++ b/llvm/test/MC/PowerPC/tls-le-v2-abi.s
@@ -1,8 +1,8 @@
 // RUN: llvm-mc -triple=powerpc64le-pc-linux -filetype=obj %s -o - | \
-// RUN: llvm-readobj -r | FileCheck %s
+// RUN: llvm-readobj -r - | FileCheck %s
 
 // RUN: llvm-mc -triple=powerpc64-pc-linux -filetype=obj %s -o - | \
-// RUN: llvm-readobj -r | FileCheck %s
+// RUN: llvm-readobj -r - | FileCheck %s
 
 // Verify we can handle all the tprel symbol modifiers for local exec tls.
 // Tests 16 bit offsets on both DS-form and D-form instructions, 32 bit
diff --git a/llvm/test/MC/RISCV/align.s b/llvm/test/MC/RISCV/align.s
index d378e3fb1f926..5b9f3dc8baa9e 100644
--- a/llvm/test/MC/RISCV/align.s
+++ b/llvm/test/MC/RISCV/align.s
@@ -5,28 +5,28 @@
 # RUN:     | llvm-objdump -d -M no-aliases - \
 # RUN:     | FileCheck -check-prefix=RELAX-INST %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+relax < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=RELAX-RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=RELAX-RELOC %s
 
 # Relaxation disabled:
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=-relax < %s \
 # RUN:     | llvm-objdump -d -M no-aliases - \
 # RUN:     | FileCheck -check-prefix=NORELAX-INST %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=-relax < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=NORELAX-RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=NORELAX-RELOC %s
 
 # Relaxation enabled with C extension:
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+c,+relax < %s \
 # RUN:     | llvm-objdump -d -M no-aliases - \
 # RUN:     | FileCheck -check-prefix=C-EXT-RELAX-INST %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+c,+relax < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=C-EXT-RELAX-RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=C-EXT-RELAX-RELOC %s
 
 # Relaxation disabled with C extension:
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+c,-relax < %s \
 # RUN:     | llvm-objdump -d -M no-aliases - \
 # RUN:     | FileCheck -check-prefix=C-EXT-NORELAX-INST %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+c,-relax < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=C-EXT-NORELAX-RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=C-EXT-NORELAX-RELOC %s
 
 # We need to insert N-MinNopSize bytes NOPs and R_RISCV_ALIGN relocation
 # type for .align N directive when linker relaxation enabled.
diff --git a/llvm/test/MC/RISCV/compressed-relocations.s b/llvm/test/MC/RISCV/compressed-relocations.s
index 31c398ef84ed5..03b9993f98612 100644
--- a/llvm/test/MC/RISCV/compressed-relocations.s
+++ b/llvm/test/MC/RISCV/compressed-relocations.s
@@ -1,9 +1,9 @@
 # RUN: llvm-mc -triple riscv32 -mattr=+c -riscv-no-aliases < %s -show-encoding \
 # RUN:     | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+c < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+c,+relax < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 
 # Check prefixes:
 # RELOC - Check the relocation in the object.
diff --git a/llvm/test/MC/RISCV/elf-header.s b/llvm/test/MC/RISCV/elf-header.s
index 491d3bea4ce4b..17fbb2fcf84fc 100644
--- a/llvm/test/MC/RISCV/elf-header.s
+++ b/llvm/test/MC/RISCV/elf-header.s
@@ -1,6 +1,6 @@
-# RUN: llvm-mc %s -filetype=obj -triple=riscv32 | llvm-readobj -h \
+# RUN: llvm-mc %s -filetype=obj -triple=riscv32 | llvm-readobj -h - \
 # RUN:     | FileCheck -check-prefix=RV32 %s
-# RUN: llvm-mc %s -filetype=obj -triple=riscv64 | llvm-readobj -h \
+# RUN: llvm-mc %s -filetype=obj -triple=riscv64 | llvm-readobj -h - \
 # RUN:     | FileCheck -check-prefix=RV64 %s
 
 # RV32: Format: elf32-littleriscv
diff --git a/llvm/test/MC/RISCV/fde-reloc.s b/llvm/test/MC/RISCV/fde-reloc.s
index 335594de3aba6..98793f1552da2 100644
--- a/llvm/test/MC/RISCV/fde-reloc.s
+++ b/llvm/test/MC/RISCV/fde-reloc.s
@@ -1,7 +1,7 @@
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+relax < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=RELAX-RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=RELAX-RELOC %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=-relax < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=NORELAX-RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=NORELAX-RELOC %s
 
 func:
 	.cfi_startproc
diff --git a/llvm/test/MC/RISCV/fixups-compressed.s b/llvm/test/MC/RISCV/fixups-compressed.s
index 5556b3d4eb2d1..3c5fcffb567c2 100644
--- a/llvm/test/MC/RISCV/fixups-compressed.s
+++ b/llvm/test/MC/RISCV/fixups-compressed.s
@@ -3,7 +3,7 @@
 # RUN: llvm-mc -triple riscv32 -filetype=obj -mattr=+c < %s \
 # RUN:     | llvm-objdump -d -M no-aliases - | FileCheck --check-prefix=CHECK-INSTR %s
 # RUN: llvm-mc -filetype=obj -mattr=+c -triple=riscv32 %s \
-# RUN:     | llvm-readobj -r | FileCheck %s -check-prefix=CHECK-REL
+# RUN:     | llvm-readobj -r - | FileCheck %s -check-prefix=CHECK-REL
 
 .LBB0_2:
 # CHECK-FIXUP:   fixup A - offset: 0, value: .LBB0_2, kind: fixup_riscv_rvc_jump
diff --git a/llvm/test/MC/RISCV/fixups-expr.s b/llvm/test/MC/RISCV/fixups-expr.s
index 303b25c4832e5..3d025fdd34362 100644
--- a/llvm/test/MC/RISCV/fixups-expr.s
+++ b/llvm/test/MC/RISCV/fixups-expr.s
@@ -1,12 +1,12 @@
 # RUN: llvm-mc -filetype=obj -triple=riscv32 -mattr=+relax %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=RELAX %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=RELAX %s
 # RUN: llvm-mc -filetype=obj -triple=riscv32 -mattr=-relax %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=NORELAX %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=NORELAX %s
 
 # RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+relax %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=RELAX %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=RELAX %s
 # RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=-relax %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=NORELAX %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=NORELAX %s
 
 # Check that subtraction expressions are emitted as two relocations
 # only when relaxation is enabled
diff --git a/llvm/test/MC/RISCV/fixups.s b/llvm/test/MC/RISCV/fixups.s
index a4aad46bdb33b..45d46d84be9a1 100644
--- a/llvm/test/MC/RISCV/fixups.s
+++ b/llvm/test/MC/RISCV/fixups.s
@@ -4,7 +4,7 @@
 # RUN:     | llvm-objdump -M no-aliases -d - \
 # RUN:     | FileCheck -check-prefix=CHECK-INSTR %s
 # RUN: llvm-mc -filetype=obj -triple=riscv32 %s \
-# RUN:     | llvm-readobj -r | FileCheck %s -check-prefix=CHECK-REL
+# RUN:     | llvm-readobj -r - | FileCheck %s -check-prefix=CHECK-REL
 
 # Checks that fixups that can be resolved within the same object file are
 # applied correctly
diff --git a/llvm/test/MC/RISCV/function-call.s b/llvm/test/MC/RISCV/function-call.s
index 01cae43bf235f..1c1bac6eea52b 100644
--- a/llvm/test/MC/RISCV/function-call.s
+++ b/llvm/test/MC/RISCV/function-call.s
@@ -1,7 +1,7 @@
 # RUN: llvm-mc -filetype=obj -triple riscv32 < %s \
 # RUN:     | llvm-objdump -d - | FileCheck --check-prefix=INSTR %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 # RUN: llvm-mc -triple riscv32 < %s -show-encoding \
 # RUN:     | FileCheck -check-prefix=FIXUP %s
 
diff --git a/llvm/test/MC/RISCV/hilo-constaddr.s b/llvm/test/MC/RISCV/hilo-constaddr.s
index 7c7c03129ba59..36217822bca9e 100644
--- a/llvm/test/MC/RISCV/hilo-constaddr.s
+++ b/llvm/test/MC/RISCV/hilo-constaddr.s
@@ -2,7 +2,7 @@
 # RUN:  | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-INSTR
 
 # RUN: llvm-mc -filetype=obj -triple=riscv32 %s \
-# RUN:  | llvm-readobj -r | FileCheck %s -check-prefix=CHECK-REL
+# RUN:  | llvm-readobj -r - | FileCheck %s -check-prefix=CHECK-REL
 
 # Check the assembler can handle hi and lo expressions with a constant
 # address. Test case derived from test/MC/Mips/hilo-addressing.s
diff --git a/llvm/test/MC/RISCV/linker-relaxation.s b/llvm/test/MC/RISCV/linker-relaxation.s
index 411c7f8887027..eed9d1e104c2f 100644
--- a/llvm/test/MC/RISCV/linker-relaxation.s
+++ b/llvm/test/MC/RISCV/linker-relaxation.s
@@ -1,13 +1,13 @@
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+relax < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=RELAX-RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=RELAX-RELOC %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=-relax < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=NORELAX-RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=NORELAX-RELOC %s
 # RUN: llvm-mc -triple riscv32 -mattr=+relax < %s -show-encoding \
 # RUN:     | FileCheck -check-prefix=RELAX-FIXUP %s
 # RUN: llvm-mc -filetype=obj -triple riscv64 -mattr=+relax < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=RELAX-RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=RELAX-RELOC %s
 # RUN: llvm-mc -filetype=obj -triple riscv64 -mattr=-relax < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=NORELAX-RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=NORELAX-RELOC %s
 # RUN: llvm-mc -triple riscv64 -mattr=+relax < %s -show-encoding \
 # RUN:     | FileCheck -check-prefix=RELAX-FIXUP %s
 
diff --git a/llvm/test/MC/RISCV/option-nopic.s b/llvm/test/MC/RISCV/option-nopic.s
index 7673b04556196..db0cf1db8e115 100644
--- a/llvm/test/MC/RISCV/option-nopic.s
+++ b/llvm/test/MC/RISCV/option-nopic.s
@@ -1,22 +1,22 @@
 # RUN: llvm-mc -triple riscv32 -mattr=-relax -riscv-no-aliases < %s \
 # RUN:     | FileCheck -check-prefix=CHECK-INST %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=CHECK-RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=CHECK-RELOC %s
 
 # RUN: llvm-mc -triple riscv32 -mattr=-relax -riscv-no-aliases \
 # RUN:     -position-independent < %s | FileCheck -check-prefix=CHECK-INST %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 -position-independent < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=CHECK-RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=CHECK-RELOC %s
 
 # RUN: llvm-mc -triple riscv64 -mattr=-relax -riscv-no-aliases < %s \
 # RUN:     | FileCheck -check-prefix=CHECK-INST %s
 # RUN: llvm-mc -filetype=obj -triple riscv64 < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=CHECK-RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=CHECK-RELOC %s
 
 # RUN: llvm-mc -triple riscv64 -mattr=-relax -riscv-no-aliases \
 # RUN:     -position-independent < %s | FileCheck -check-prefix=CHECK-INST %s
 # RUN: llvm-mc -filetype=obj -triple riscv64 -position-independent < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=CHECK-RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=CHECK-RELOC %s
 
 .option nopic
 # CHECK-INST: .option nopic
diff --git a/llvm/test/MC/RISCV/option-pic.s b/llvm/test/MC/RISCV/option-pic.s
index 37d643c324830..ef456e01c4ba8 100644
--- a/llvm/test/MC/RISCV/option-pic.s
+++ b/llvm/test/MC/RISCV/option-pic.s
@@ -1,22 +1,22 @@
 # RUN: llvm-mc -triple riscv32 -mattr=-relax -riscv-no-aliases < %s \
 # RUN:     | FileCheck -check-prefix=CHECK-INST %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=CHECK-RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=CHECK-RELOC %s
 
 # RUN: llvm-mc -triple riscv32 -mattr=-relax -riscv-no-aliases \
 # RUN:     -position-independent < %s | FileCheck -check-prefix=CHECK-INST %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 -position-independent < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=CHECK-RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=CHECK-RELOC %s
 
 # RUN: llvm-mc -triple riscv64 -mattr=-relax -riscv-no-aliases < %s \
 # RUN:     | FileCheck -check-prefix=CHECK-INST %s
 # RUN: llvm-mc -filetype=obj -triple riscv64 < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=CHECK-RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=CHECK-RELOC %s
 
 # RUN: llvm-mc -triple riscv64 -mattr=-relax -riscv-no-aliases \
 # RUN:     -position-independent < %s | FileCheck -check-prefix=CHECK-INST %s
 # RUN: llvm-mc -filetype=obj -triple riscv64 -position-independent < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=CHECK-RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=CHECK-RELOC %s
 
 .option pic
 # CHECK-INST: .option pic
diff --git a/llvm/test/MC/RISCV/option-pushpop.s b/llvm/test/MC/RISCV/option-pushpop.s
index 96101dbc63768..909f3d027b85b 100644
--- a/llvm/test/MC/RISCV/option-pushpop.s
+++ b/llvm/test/MC/RISCV/option-pushpop.s
@@ -1,7 +1,7 @@
 # RUN: llvm-mc -triple riscv32 -mattr=-relax -riscv-no-aliases < %s \
 # RUN:     | FileCheck -check-prefix=CHECK-INST %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=CHECK-RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=CHECK-RELOC %s
 # RUN: llvm-mc -triple riscv32 -filetype=obj < %s \
 # RUN:     | llvm-objdump  --triple=riscv32 --mattr=+c -d - \
 # RUN:     | FileCheck -check-prefixes=CHECK-BYTES,CHECK-ALIAS %s
@@ -9,7 +9,7 @@
 # RUN: llvm-mc -triple riscv64 -mattr=-relax -riscv-no-aliases < %s \
 # RUN:     | FileCheck -check-prefix=CHECK-INST %s
 # RUN: llvm-mc -filetype=obj -triple riscv64 < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=CHECK-RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=CHECK-RELOC %s
 # RUN: llvm-mc -triple riscv64 -filetype=obj < %s \
 # RUN:     | llvm-objdump  --triple=riscv64 --mattr=+c -d - \
 # RUN:     | FileCheck -check-prefixes=CHECK-BYTES,CHECK-ALIAS %s
diff --git a/llvm/test/MC/RISCV/option-relax.s b/llvm/test/MC/RISCV/option-relax.s
index 1ee567a27820d..7a231d152f766 100644
--- a/llvm/test/MC/RISCV/option-relax.s
+++ b/llvm/test/MC/RISCV/option-relax.s
@@ -1,12 +1,12 @@
 # RUN: llvm-mc -triple riscv32 < %s \
 # RUN:     | FileCheck -check-prefix=CHECK-INST %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=CHECK-RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=CHECK-RELOC %s
 
 # RUN: llvm-mc -triple riscv64 < %s \
 # RUN:     | FileCheck -check-prefix=CHECK-INST %s
 # RUN: llvm-mc -filetype=obj -triple riscv64 < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=CHECK-RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=CHECK-RELOC %s
 
 # Check .option relax causes R_RISCV_RELAX to be emitted, and .option
 # norelax suppresses it. Also check that if .option relax was enabled
diff --git a/llvm/test/MC/RISCV/pseudo-jump.s b/llvm/test/MC/RISCV/pseudo-jump.s
index 1cc46cc0efa8a..0a2e2faa1b65a 100644
--- a/llvm/test/MC/RISCV/pseudo-jump.s
+++ b/llvm/test/MC/RISCV/pseudo-jump.s
@@ -1,7 +1,7 @@
 # RUN: llvm-mc -filetype=obj -triple riscv32 < %s \
 # RUN:     | llvm-objdump -d - | FileCheck --check-prefix=INSTR %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 # RUN: llvm-mc -triple riscv32 < %s -show-encoding \
 # RUN:     | FileCheck -check-prefix=FIXUP %s
 
diff --git a/llvm/test/MC/RISCV/reloc-directive.s b/llvm/test/MC/RISCV/reloc-directive.s
index 7f22a0ba61d38..d025a64243f5c 100644
--- a/llvm/test/MC/RISCV/reloc-directive.s
+++ b/llvm/test/MC/RISCV/reloc-directive.s
@@ -1,7 +1,7 @@
 # RUN: llvm-mc -triple=riscv32 %s | FileCheck --check-prefix=PRINT %s
 # RUN: llvm-mc -triple=riscv64 %s | FileCheck --check-prefix=PRINT %s
-# RUN: llvm-mc -filetype=obj -triple=riscv32 %s | llvm-readobj -r | FileCheck %s
-# RUN: llvm-mc -filetype=obj -triple=riscv64 %s | llvm-readobj -r | FileCheck %s
+# RUN: llvm-mc -filetype=obj -triple=riscv32 %s | llvm-readobj -r - | FileCheck %s
+# RUN: llvm-mc -filetype=obj -triple=riscv64 %s | llvm-readobj -r - | FileCheck %s
 
 # PRINT: .reloc 8, R_RISCV_NONE, .data
 # PRINT: .reloc 4, R_RISCV_NONE, foo+4
diff --git a/llvm/test/MC/RISCV/relocations.s b/llvm/test/MC/RISCV/relocations.s
index 10a28903906ae..d35bdc887e9f8 100644
--- a/llvm/test/MC/RISCV/relocations.s
+++ b/llvm/test/MC/RISCV/relocations.s
@@ -1,7 +1,7 @@
 # RUN: llvm-mc -triple riscv32 -riscv-no-aliases < %s -show-encoding \
 # RUN:     | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+c < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 
 # Check prefixes:
 # RELOC - Check the relocation in the object.
diff --git a/llvm/test/MC/RISCV/rv32-relaxation.s b/llvm/test/MC/RISCV/rv32-relaxation.s
index a1a2172bf5a1e..609fa7c83e84b 100644
--- a/llvm/test/MC/RISCV/rv32-relaxation.s
+++ b/llvm/test/MC/RISCV/rv32-relaxation.s
@@ -3,7 +3,7 @@
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+c,+relax < %s \
 # RUN:     | llvm-objdump -d -M no-aliases - | FileCheck --check-prefix=RELAX-INSTR %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+c,+relax < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=RELAX-RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=RELAX-RELOC %s
 
 FAR_JUMP_NEGATIVE:
   c.nop
diff --git a/llvm/test/MC/RISCV/rv64-relaxation.s b/llvm/test/MC/RISCV/rv64-relaxation.s
index 94d3e3379f54a..0e057e36afbf9 100644
--- a/llvm/test/MC/RISCV/rv64-relaxation.s
+++ b/llvm/test/MC/RISCV/rv64-relaxation.s
@@ -3,7 +3,7 @@
 # RUN: llvm-mc -filetype=obj -triple riscv64 -mattr=+c,+relax < %s \
 # RUN:     | llvm-objdump -d -M no-aliases - | FileCheck --check-prefix=RELAX-INSTR %s
 # RUN: llvm-mc -filetype=obj -triple riscv64 -mattr=+c,+relax < %s \
-# RUN:     | llvm-readobj -r | FileCheck -check-prefix=RELAX-RELOC %s
+# RUN:     | llvm-readobj -r - | FileCheck -check-prefix=RELAX-RELOC %s
 
 FAR_JUMP_NEGATIVE:
   c.nop
diff --git a/llvm/test/MC/RISCV/tail-call.s b/llvm/test/MC/RISCV/tail-call.s
index 411ab7dd853b1..63e9d420b112e 100644
--- a/llvm/test/MC/RISCV/tail-call.s
+++ b/llvm/test/MC/RISCV/tail-call.s
@@ -1,14 +1,14 @@
 # RUN: llvm-mc -filetype=obj -triple riscv32 < %s \
 # RUN:   | llvm-objdump -d - | FileCheck --check-prefix=INSTR %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 < %s \
-# RUN:   | llvm-readobj -r | FileCheck -check-prefix=RELOC %s
+# RUN:   | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 # RUN: llvm-mc -triple riscv32 < %s -show-encoding \
 # RUN:   | FileCheck -check-prefix=FIXUP %s
 
 # RUN: llvm-mc -filetype=obj -triple riscv64 < %s \
 # RUN:   | llvm-objdump -d - | FileCheck --check-prefix=INSTR %s
 # RUN: llvm-mc -filetype=obj -triple riscv64 < %s \
-# RUN:   | llvm-readobj -r | FileCheck -check-prefix=RELOC %s
+# RUN:   | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 # RUN: llvm-mc -triple riscv64 < %s -show-encoding \
 # RUN:   | FileCheck -check-prefix=FIXUP %s
 
diff --git a/llvm/test/MC/Sparc/sparc-nop-data.s b/llvm/test/MC/Sparc/sparc-nop-data.s
index 2ba8cd43b36b8..1987dfd288580 100644
--- a/llvm/test/MC/Sparc/sparc-nop-data.s
+++ b/llvm/test/MC/Sparc/sparc-nop-data.s
@@ -1,5 +1,5 @@
-! RUN: llvm-mc %s -arch=sparc  -filetype=obj | llvm-readobj -S --sd | FileCheck %s
-! RUN: llvm-mc %s -arch=sparcv9  -filetype=obj | llvm-readobj -S --sd | FileCheck %s
+! RUN: llvm-mc %s -arch=sparc  -filetype=obj | llvm-readobj -S --sd - | FileCheck %s
+! RUN: llvm-mc %s -arch=sparcv9  -filetype=obj | llvm-readobj -S --sd - | FileCheck %s
 
 ! CHECK: 0000: BA1F401D 01000000 01000000 01000000
 ! CHECK: 0010: BA1F401D
diff --git a/llvm/test/MC/Sparc/sparc-pic.s b/llvm/test/MC/Sparc/sparc-pic.s
index a16ed807afcc1..859c6efc12ae7 100644
--- a/llvm/test/MC/Sparc/sparc-pic.s
+++ b/llvm/test/MC/Sparc/sparc-pic.s
@@ -1,5 +1,5 @@
-! RUN: llvm-mc %s -arch=sparcv9 --position-independent -filetype=obj | llvm-readobj -r | FileCheck --check-prefix=PIC %s
-! RUN: llvm-mc %s -arch=sparcv9 -filetype=obj | llvm-readobj -r | FileCheck --check-prefix=NOPIC %s
+! RUN: llvm-mc %s -arch=sparcv9 --position-independent -filetype=obj | llvm-readobj -r - | FileCheck --check-prefix=PIC %s
+! RUN: llvm-mc %s -arch=sparcv9 -filetype=obj | llvm-readobj -r - | FileCheck --check-prefix=NOPIC %s
 
 
 ! PIC:      Relocations [
diff --git a/llvm/test/MC/Sparc/sparc-relocations.s b/llvm/test/MC/Sparc/sparc-relocations.s
index 5c090a6f2edd4..a0a7b7084d2cc 100644
--- a/llvm/test/MC/Sparc/sparc-relocations.s
+++ b/llvm/test/MC/Sparc/sparc-relocations.s
@@ -1,5 +1,5 @@
 ! RUN: llvm-mc %s -arch=sparcv9 -show-encoding | FileCheck %s
-! RUN: llvm-mc %s -arch=sparcv9 -filetype=obj | llvm-readobj -r | FileCheck %s --check-prefix=CHECK-OBJ
+! RUN: llvm-mc %s -arch=sparcv9 -filetype=obj | llvm-readobj -r - | FileCheck %s --check-prefix=CHECK-OBJ
 
         ! CHECK-OBJ: Format: elf64-sparc
         ! CHECK-OBJ: Relocations [
diff --git a/llvm/test/MC/Sparc/sparc-tls-relocations.s b/llvm/test/MC/Sparc/sparc-tls-relocations.s
index 6e0149e6414bb..257fda74120c5 100644
--- a/llvm/test/MC/Sparc/sparc-tls-relocations.s
+++ b/llvm/test/MC/Sparc/sparc-tls-relocations.s
@@ -3,8 +3,8 @@
 !
 ! RUN: llvm-mc %s -arch=sparc -show-encoding | FileCheck %s --check-prefix=ASM
 ! RUN: llvm-mc %s -arch=sparcv9 -show-encoding | FileCheck %s --check-prefix=ASM
-! RUN: llvm-mc %s -arch=sparc -filetype=obj | llvm-readobj -r | FileCheck %s --check-prefix=REL
-! RUN: llvm-mc %s -arch=sparcv9 -filetype=obj | llvm-readobj -r | FileCheck %s --check-prefix=REL
+! RUN: llvm-mc %s -arch=sparc -filetype=obj | llvm-readobj -r - | FileCheck %s --check-prefix=REL
+! RUN: llvm-mc %s -arch=sparcv9 -filetype=obj | llvm-readobj -r - | FileCheck %s --check-prefix=REL
 ! RUN: llvm-mc %s -arch=sparc -filetype=obj | llvm-objdump -r -d - | FileCheck %s --check-prefix=OBJDUMP
 ! RUN: llvm-mc %s -arch=sparcv9 -filetype=obj | llvm-objdump -r -d - | FileCheck %s --check-prefix=OBJDUMP
 
diff --git a/llvm/test/MC/SystemZ/fixups-zEC12.s b/llvm/test/MC/SystemZ/fixups-zEC12.s
index a2a64a84347ee..f1fd43261a812 100644
--- a/llvm/test/MC/SystemZ/fixups-zEC12.s
+++ b/llvm/test/MC/SystemZ/fixups-zEC12.s
@@ -2,7 +2,7 @@
 # RUN: llvm-mc -triple s390x-unknown-unknown -mcpu=zEC12 --show-encoding %s | FileCheck %s
 
 # RUN: llvm-mc -triple s390x-unknown-unknown -mcpu=zEC12 -filetype=obj %s | \
-# RUN: llvm-readobj -r | FileCheck %s -check-prefix=CHECK-REL
+# RUN: llvm-readobj -r - | FileCheck %s -check-prefix=CHECK-REL
 
 # CHECK: bpp 12, branch, 0                      # encoding: [0xc7,0xc0,0x00,0x00,A,A]
 # CHECK:                                        # fixup A - offset: 4, value: branch+4, kind: FK_390_PC16DBL
diff --git a/llvm/test/MC/SystemZ/fixups.s b/llvm/test/MC/SystemZ/fixups.s
index ea3b690d253be..fc8cf49c0d825 100644
--- a/llvm/test/MC/SystemZ/fixups.s
+++ b/llvm/test/MC/SystemZ/fixups.s
@@ -2,7 +2,7 @@
 # RUN: llvm-mc -triple s390x-unknown-unknown --show-encoding %s | FileCheck %s
 
 # RUN: llvm-mc -triple s390x-unknown-unknown -filetype=obj %s | \
-# RUN: llvm-readobj -r | FileCheck %s -check-prefix=CHECK-REL
+# RUN: llvm-readobj -r - | FileCheck %s -check-prefix=CHECK-REL
 
 # CHECK: larl %r14, target                      # encoding: [0xc0,0xe0,A,A,A,A]
 # CHECK-NEXT:                                   # fixup A - offset: 2, value: target+2, kind: FK_390_PC32DBL
diff --git a/llvm/test/MC/SystemZ/word.s b/llvm/test/MC/SystemZ/word.s
index 6299101bc3cc8..c0eae957e1c23 100644
--- a/llvm/test/MC/SystemZ/word.s
+++ b/llvm/test/MC/SystemZ/word.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -triple s390x-linux-gnu -filetype=obj %s | \
-# RUN: llvm-readobj -S --sd | FileCheck %s
+# RUN: llvm-readobj -S --sd - | FileCheck %s
 
 .section word, "aw"
 .word 0xabcd
diff --git a/llvm/test/MC/WebAssembly/custom-sections.ll b/llvm/test/MC/WebAssembly/custom-sections.ll
index d0fc5df99770a..d92399947cb1f 100644
--- a/llvm/test/MC/WebAssembly/custom-sections.ll
+++ b/llvm/test/MC/WebAssembly/custom-sections.ll
@@ -1,4 +1,4 @@
-; RUN: llc -filetype=obj %s -o - | llvm-readobj -S | FileCheck %s
+; RUN: llc -filetype=obj %s -o - | llvm-readobj -S - | FileCheck %s
 
 ; Test the mechanism for defining user custom sections.
 
diff --git a/llvm/test/MC/WebAssembly/debug-info.ll b/llvm/test/MC/WebAssembly/debug-info.ll
index 1da20a8a12e4f..0277daf2bcc98 100644
--- a/llvm/test/MC/WebAssembly/debug-info.ll
+++ b/llvm/test/MC/WebAssembly/debug-info.ll
@@ -1,4 +1,4 @@
-; RUN: llc -filetype=obj %s -o - | llvm-readobj -r -S --symbols | FileCheck %s
+; RUN: llc -filetype=obj %s -o - | llvm-readobj -r -S --symbols - | FileCheck %s
 
 ; CHECK: Format: WASM
 ; CHECK-NEXT:Arch: wasm32
diff --git a/llvm/test/MC/WebAssembly/event-section.ll b/llvm/test/MC/WebAssembly/event-section.ll
index b662c290d3c7e..35928fbcfe944 100644
--- a/llvm/test/MC/WebAssembly/event-section.ll
+++ b/llvm/test/MC/WebAssembly/event-section.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -filetype=obj -exception-model=wasm -mattr=+exception-handling %s -o - | obj2yaml | FileCheck %s
-; RUN: llc -filetype=obj -exception-model=wasm -mattr=+exception-handling %s -o - | llvm-readobj -S | FileCheck -check-prefix=SEC %s
+; RUN: llc -filetype=obj -exception-model=wasm -mattr=+exception-handling %s -o - | llvm-readobj -S - | FileCheck -check-prefix=SEC %s
 
 target triple = "wasm32-unknown-unknown"
 
diff --git a/llvm/test/MC/WebAssembly/file-headers.ll b/llvm/test/MC/WebAssembly/file-headers.ll
index ff738f8066c23..676494e8b8144 100644
--- a/llvm/test/MC/WebAssembly/file-headers.ll
+++ b/llvm/test/MC/WebAssembly/file-headers.ll
@@ -1,4 +1,4 @@
-; RUN: llc -filetype=obj %s -o - | llvm-readobj --file-headers | FileCheck %s
+; RUN: llc -filetype=obj %s -o - | llvm-readobj --file-headers - | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
diff --git a/llvm/test/MC/WebAssembly/func-address.ll b/llvm/test/MC/WebAssembly/func-address.ll
index c5eb0ea5938a0..281325e6b9fd9 100644
--- a/llvm/test/MC/WebAssembly/func-address.ll
+++ b/llvm/test/MC/WebAssembly/func-address.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O2 -filetype=obj %s -o - | llvm-readobj -r -S --expand-relocs | FileCheck %s
+; RUN: llc -O2 -filetype=obj %s -o - | llvm-readobj -r -S --expand-relocs - | FileCheck %s
 
 target triple = "wasm32-unknown-unknown"
 
diff --git a/llvm/test/MC/WebAssembly/function-alias.ll b/llvm/test/MC/WebAssembly/function-alias.ll
index a6a44ab0bd049..ec571083344bc 100644
--- a/llvm/test/MC/WebAssembly/function-alias.ll
+++ b/llvm/test/MC/WebAssembly/function-alias.ll
@@ -1,4 +1,4 @@
-; RUN: llc -filetype=obj %s -o - | llvm-readobj --symbols | FileCheck %s
+; RUN: llc -filetype=obj %s -o - | llvm-readobj --symbols - | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown-wasm"
diff --git a/llvm/test/MC/WebAssembly/no-dead-strip.ll b/llvm/test/MC/WebAssembly/no-dead-strip.ll
index f35be2c7f1cae..e828c41b6e363 100644
--- a/llvm/test/MC/WebAssembly/no-dead-strip.ll
+++ b/llvm/test/MC/WebAssembly/no-dead-strip.ll
@@ -1,4 +1,4 @@
-; RUN: llc -filetype=obj -wasm-keep-registers %s -o - | llvm-readobj --symbols | FileCheck %s
+; RUN: llc -filetype=obj -wasm-keep-registers %s -o - | llvm-readobj --symbols - | FileCheck %s
 
 target triple = "wasm32-unknown-unknown"
 
diff --git a/llvm/test/MC/WebAssembly/reloc-code.ll b/llvm/test/MC/WebAssembly/reloc-code.ll
index 0a9a3aeda6a56..0aafeabec5caa 100644
--- a/llvm/test/MC/WebAssembly/reloc-code.ll
+++ b/llvm/test/MC/WebAssembly/reloc-code.ll
@@ -1,4 +1,4 @@
-; RUN: llc -filetype=obj %s -o - | llvm-readobj -r --expand-relocs | FileCheck %s
+; RUN: llc -filetype=obj %s -o - | llvm-readobj -r --expand-relocs - | FileCheck %s
 
 target triple = "wasm32-unknown-unknown"
 
diff --git a/llvm/test/MC/WebAssembly/reloc-data.ll b/llvm/test/MC/WebAssembly/reloc-data.ll
index 124db881b2f72..78176326f2451 100644
--- a/llvm/test/MC/WebAssembly/reloc-data.ll
+++ b/llvm/test/MC/WebAssembly/reloc-data.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -filetype=obj %s -o - | llvm-readobj -r --expand-relocs | FileCheck %s
+; RUN: llc -O0 -filetype=obj %s -o - | llvm-readobj -r --expand-relocs - | FileCheck %s
 
 target triple = "wasm32-unknown-unknown"
 
diff --git a/llvm/test/MC/WebAssembly/sections.ll b/llvm/test/MC/WebAssembly/sections.ll
index abc1fe486074e..690a64f11e71c 100644
--- a/llvm/test/MC/WebAssembly/sections.ll
+++ b/llvm/test/MC/WebAssembly/sections.ll
@@ -1,4 +1,4 @@
-; RUN: llc -filetype=obj %s -o - | llvm-readobj -S | FileCheck %s
+; RUN: llc -filetype=obj %s -o - | llvm-readobj -S - | FileCheck %s
 
 target triple = "wasm32-unknown-unknown"
 
diff --git a/llvm/test/MC/X86/AlignedBundling/section-alignment.s b/llvm/test/MC/X86/AlignedBundling/section-alignment.s
index 5cbe58e5675ac..d5277d194576e 100644
--- a/llvm/test/MC/X86/AlignedBundling/section-alignment.s
+++ b/llvm/test/MC/X86/AlignedBundling/section-alignment.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - \
-# RUN:   | llvm-readobj --sections | FileCheck %s
+# RUN:   | llvm-readobj --sections - | FileCheck %s
 
 # Test that bundle-aligned sections with instructions are aligned
 
diff --git a/llvm/test/MC/X86/align-branch-section-size.s b/llvm/test/MC/X86/align-branch-section-size.s
index 6ed9540b679d7..06c92373b1794 100644
--- a/llvm/test/MC/X86/align-branch-section-size.s
+++ b/llvm/test/MC/X86/align-branch-section-size.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -filetype=obj -triple x86_64 --x86-align-branch-boundary=32 --x86-align-branch=ret %s | llvm-readobj -S | FileCheck %s
+# RUN: llvm-mc -filetype=obj -triple x86_64 --x86-align-branch-boundary=32 --x86-align-branch=ret %s | llvm-readobj -S - | FileCheck %s
 
   # Check the aligment of section that contains instructions to be aligned
   # is correctly set.
diff --git a/llvm/test/MC/X86/align-branch-section-type.s b/llvm/test/MC/X86/align-branch-section-type.s
index 7a5c10b83d9ea..c2152fd77bdb7 100644
--- a/llvm/test/MC/X86/align-branch-section-type.s
+++ b/llvm/test/MC/X86/align-branch-section-type.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -filetype=obj -triple x86_64 --x86-align-branch-boundary=32 --x86-align-branch=ret %s | llvm-readobj -S | FileCheck %s
+# RUN: llvm-mc -filetype=obj -triple x86_64 --x86-align-branch-boundary=32 --x86-align-branch=ret %s | llvm-readobj -S - | FileCheck %s
 
 ## Check we only pad in a text section
 
diff --git a/llvm/test/MC/X86/cfi_def_cfa-crash.s b/llvm/test/MC/X86/cfi_def_cfa-crash.s
index b67db658dd5d4..164737d1499f2 100644
--- a/llvm/test/MC/X86/cfi_def_cfa-crash.s
+++ b/llvm/test/MC/X86/cfi_def_cfa-crash.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-darwin -filetype=obj %s -o - | llvm-readobj --sections | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-darwin -filetype=obj %s -o - | llvm-readobj --sections - | FileCheck %s
 
 // We were trying to generate compact unwind info for assembly like this.
 // The .cfi_def_cfa directive, however, throws a wrench into that and was
diff --git a/llvm/test/MC/X86/eval-fill.s b/llvm/test/MC/X86/eval-fill.s
index 838eedf5e1771..bdc534f954aca 100644
--- a/llvm/test/MC/X86/eval-fill.s
+++ b/llvm/test/MC/X86/eval-fill.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj %s -o - -triple x86_64-pc-linux | llvm-readobj -S | FileCheck %s
+// RUN: llvm-mc -filetype=obj %s -o - -triple x86_64-pc-linux | llvm-readobj -S - | FileCheck %s
 
 // CHECK:      Name: .text
 // CHECK-NEXT: Type: SHT_PROGBITS
diff --git a/llvm/test/MC/X86/expand-var.s b/llvm/test/MC/X86/expand-var.s
index 26209424351de..2cd27ec16c77d 100644
--- a/llvm/test/MC/X86/expand-var.s
+++ b/llvm/test/MC/X86/expand-var.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux < %s | llvm-readobj -r --symbols | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux < %s | llvm-readobj -r --symbols - | FileCheck %s
 
 // CHECK:       Section {{.*}} .rela.text {
 // CHECK-NEXT:    0x0 R_X86_64_32 d 0x0
diff --git a/llvm/test/MC/X86/fp-setup-macho.s b/llvm/test/MC/X86/fp-setup-macho.s
index 4bfff129f19cb..0067b02786137 100644
--- a/llvm/test/MC/X86/fp-setup-macho.s
+++ b/llvm/test/MC/X86/fp-setup-macho.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-apple-macho -filetype obj -o - %s | llvm-readobj --sections | FileCheck %s
+// RUN: llvm-mc -triple x86_64-apple-macho -filetype obj -o - %s | llvm-readobj --sections - | FileCheck %s
 
 _label:
 	.cfi_startproc
diff --git a/llvm/test/MC/X86/large-bss.s b/llvm/test/MC/X86/large-bss.s
index 97fcc5de704ab..779425092fe8a 100644
--- a/llvm/test/MC/X86/large-bss.s
+++ b/llvm/test/MC/X86/large-bss.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-pc-linux -filetype=obj %s -o - | llvm-readobj -S | FileCheck %s
+// RUN: llvm-mc -triple x86_64-pc-linux -filetype=obj %s -o - | llvm-readobj -S - | FileCheck %s
 
 .bss
 .zero 0x10000000000000
diff --git a/llvm/test/MC/X86/no-elf-compact-unwind.s b/llvm/test/MC/X86/no-elf-compact-unwind.s
index e61c24c141a6b..ade412d7964cf 100644
--- a/llvm/test/MC/X86/no-elf-compact-unwind.s
+++ b/llvm/test/MC/X86/no-elf-compact-unwind.s
@@ -1,6 +1,6 @@
-// RUN: llvm-mc < %s -filetype=obj -triple x86_64-apple-macosx10.8.0 | llvm-readobj -S | FileCheck -check-prefix=MACHO %s
-// RUN: llvm-mc < %s -filetype=obj -triple x86_64-apple-ios7.0.0 | llvm-readobj -S | FileCheck -check-prefix=MACHO %s
-// RUN: llvm-mc < %s -filetype=obj -triple x86_64-unknown-linux | llvm-readobj -S | FileCheck -check-prefix=ELF %s
+// RUN: llvm-mc < %s -filetype=obj -triple x86_64-apple-macosx10.8.0 | llvm-readobj -S - | FileCheck -check-prefix=MACHO %s
+// RUN: llvm-mc < %s -filetype=obj -triple x86_64-apple-ios7.0.0 | llvm-readobj -S - | FileCheck -check-prefix=MACHO %s
+// RUN: llvm-mc < %s -filetype=obj -triple x86_64-unknown-linux | llvm-readobj -S - | FileCheck -check-prefix=ELF %s
 
 	.globl	__Z3barv
 	.align	4, 0x90
diff --git a/llvm/test/MC/X86/reloc-directive.s b/llvm/test/MC/X86/reloc-directive.s
index 124f520962812..5f4fc2394f5e7 100644
--- a/llvm/test/MC/X86/reloc-directive.s
+++ b/llvm/test/MC/X86/reloc-directive.s
@@ -1,10 +1,10 @@
 # RUN: llvm-mc -triple i686-pc-win32 < %s -show-encoding \
 # RUN:     | FileCheck -check-prefix=ASM %s
 # RUN: llvm-mc -triple i686-pc-win32 < %s -show-encoding \
-# RUN:     -filetype=obj | llvm-readobj --sections --section-data -r | \
+# RUN:     -filetype=obj | llvm-readobj --sections --section-data -r - | \
 # RUN:     FileCheck -check-prefix=OBJ-32 %s
 # RUN: llvm-mc -triple x86_64-pc-win32 < %s -show-encoding \
-# RUN:     -filetype=obj | llvm-readobj --sections --section-data -r | \
+# RUN:     -filetype=obj | llvm-readobj --sections --section-data -r - | \
 # RUN:     FileCheck -check-prefix=OBJ-64 %s
 	.text
 foo:
diff --git a/llvm/test/MC/X86/reloc-macho.s b/llvm/test/MC/X86/reloc-macho.s
index 9297b1ba8b319..82ca0de8706d9 100644
--- a/llvm/test/MC/X86/reloc-macho.s
+++ b/llvm/test/MC/X86/reloc-macho.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-apple-darwin %s -o - | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-apple-darwin %s -o - | llvm-readobj -r - | FileCheck %s
 
 // CHECK:      Relocations [
 // CHECK-NEXT: ]
diff --git a/llvm/test/MC/X86/reloc-undef-global.s b/llvm/test/MC/X86/reloc-undef-global.s
index 24de904525260..d89a5f86ca15b 100644
--- a/llvm/test/MC/X86/reloc-undef-global.s
+++ b/llvm/test/MC/X86/reloc-undef-global.s
@@ -1,5 +1,5 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux %s -o - | llvm-readobj -r | FileCheck --check-prefix=ELF %s
-// RUN: llvm-mc -filetype=obj -triple x86_64-apple-darwin %s -o - | llvm-readobj -r | FileCheck --check-prefix=MACHO %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux %s -o - | llvm-readobj -r - | FileCheck --check-prefix=ELF %s
+// RUN: llvm-mc -filetype=obj -triple x86_64-apple-darwin %s -o - | llvm-readobj -r - | FileCheck --check-prefix=MACHO %s
 
 
 bar = foo + 4
diff --git a/llvm/test/MC/X86/x86-evenDirective.s b/llvm/test/MC/X86/x86-evenDirective.s
index d59ae903e2d46..df810383ec4ba 100644
--- a/llvm/test/MC/X86/x86-evenDirective.s
+++ b/llvm/test/MC/X86/x86-evenDirective.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -triple -x86_64-unknown-unknown -filetype obj -o - %s | llvm-readobj -S --sd \
+# RUN: llvm-mc -triple -x86_64-unknown-unknown -filetype obj -o - %s | llvm-readobj -S --sd - \
 # RUN:   | FileCheck %s
 
 	.text
diff --git a/llvm/test/MC/X86/x86_64-signed-reloc.s b/llvm/test/MC/X86/x86_64-signed-reloc.s
index e8a2720df1d36..c31e9932fb6f9 100644
--- a/llvm/test/MC/X86/x86_64-signed-reloc.s
+++ b/llvm/test/MC/X86/x86_64-signed-reloc.s
@@ -1,4 +1,4 @@
-// RUN: llvm-mc -triple x86_64-linux-gnu -filetype=obj %s | llvm-readobj -r | FileCheck %s
+// RUN: llvm-mc -triple x86_64-linux-gnu -filetype=obj %s | llvm-readobj -r - | FileCheck %s
 
 
 				// CHECK:      Relocations [
diff --git a/llvm/test/Object/invalid.test b/llvm/test/Object/invalid.test
index a930a6a9dbe68..bc95c5bc6e905 100644
--- a/llvm/test/Object/invalid.test
+++ b/llvm/test/Object/invalid.test
@@ -137,7 +137,7 @@ Symbols:
 # RUN: yaml2obj %s --docnum=7 -o %t7
 # RUN: llvm-readobj --dyn-symbols %t7 2>&1 | FileCheck -DFILE=%t7 --check-prefix=INVALID-DYNSYM-SIZE %s
 
-# INVALID-DYNSYM-SIZE: warning: '[[FILE]]': section with index 1 has invalid size (0x30) or entry size (0x20)
+# INVALID-DYNSYM-SIZE: warning: '[[FILE]]': SHT_DYNSYM section with index 1 has invalid size (0x30) or entry size (0x20)
 
 --- !ELF
 FileHeader:
@@ -276,13 +276,22 @@ Sections:
     Type:     SHT_RELA
     ShOffset: 0x10000
 
-## Check that llvm-objdump reports a warning when we try to print symbols and
+## Check that llvm-readobj reports a warning when we try to print section symbols and
 ## .shstrtab has a broken sh_offset so large that sh_offset + sh_size overflows the platform address size type.
 
 # RUN: yaml2obj %s --docnum=14 -o %t14
 # RUN: llvm-readobj --symbols %t14 2>&1 | FileCheck -DFILE=%t14 --check-prefix=INVALID-SECTION-SIZE2 %s
 
-# INVALID-SECTION-SIZE2: warning: '[[FILE]]': unable to get the name of the SHT_SYMTAB section: section [index 1] has a sh_offset (0xffffffff) + sh_size (0x1b) that cannot be represented
+# INVALID-SECTION-SIZE2:      Symbol {
+# INVALID-SECTION-SIZE2:       Name: symbol (1)
+# INVALID-SECTION-SIZE2-NEXT:  Value: 0x123
+# INVALID-SECTION-SIZE2-NEXT:  Size: 0
+# INVALID-SECTION-SIZE2-NEXT:  Binding: Local (0x0)
+# INVALID-SECTION-SIZE2-NEXT:  Type: None (0x0)
+# INVALID-SECTION-SIZE2-NEXT:  Other: 0
+# INVALID-SECTION-SIZE2-NEXT: warning: '[[FILE]]': section [index 1] has a sh_offset (0xffffffff) + sh_size (0x1b) that cannot be represented
+# INVALID-SECTION-SIZE2-NEXT:  Section: <?> (0x1)
+# INVALID-SECTION-SIZE2-NEXT: }
 
 --- !ELF
 FileHeader:
@@ -294,7 +303,10 @@ Sections:
   - Name:     .shstrtab
     Type:     SHT_STRTAB
     ShOffset: 0xFFFFFFFF
-Symbols: []
+Symbols:
+  - Name:    symbol
+    Section: .shstrtab
+    Value:   0x123
 
 ## Check that llvm-readobj reports an error when trying to dump sections
 ## when the e_shnum field is broken (is greater than the actual number of sections).
@@ -458,7 +470,7 @@ ProgramHeaders:
 # RUN: yaml2obj %s --docnum=21 -o %t21
 # RUN: llvm-readobj --dyn-relocations %t21 2>&1 | FileCheck -DFILE=%t21 --check-prefix=DYN-TABLE-SIZE %s
 
-# DYN-TABLE-SIZE: warning: '[[FILE]]': section with index 1 has invalid size (0x1){{$}}
+# DYN-TABLE-SIZE: warning: '[[FILE]]': SHT_DYNAMIC section with index 1 has invalid size (0x1){{$}}
 
 --- !ELF
 FileHeader:
diff --git a/llvm/test/ObjectYAML/MachO/DWARF-debug_info.yaml b/llvm/test/ObjectYAML/MachO/DWARF-debug_info.yaml
index 98af99b86e0a5..1ca45f5dadfed 100644
--- a/llvm/test/ObjectYAML/MachO/DWARF-debug_info.yaml
+++ b/llvm/test/ObjectYAML/MachO/DWARF-debug_info.yaml
@@ -513,7 +513,7 @@ DWARF:
 
 #      DWARF32: DWARF:
 #      DWARF32:   debug_info:
-# DWARF32-NEXT:     - Length:     117
+# DWARF32-NEXT:     - Length:     0x0000000000000075
 # DWARF32-NEXT:       Version:    4
 # DWARF32-NEXT:       AbbrOffset: 0
 # DWARF32-NEXT:       AddrSize:   8
@@ -584,7 +584,7 @@ DWARF:
 # RUN:   obj2yaml | FileCheck %s --check-prefix=DWARF32-YAML
 
 #      DWARF32-YAML: debug_info:
-# DWARF32-YAML-NEXT:   - Length:     12
+# DWARF32-YAML-NEXT:   - Length:     0x000000000000000C
 # DWARF32-YAML-NEXT:     Version:    4
 # DWARF32-YAML-NEXT:     AbbrOffset: 0
 # DWARF32-YAML-NEXT:     AddrSize:   8
@@ -666,7 +666,7 @@ DWARF:
 
 #      DWARF64-YAML: debug_info:
 # DWARF64-YAML-NEXT:   - Format:     DWARF64
-# DWARF64-YAML-NEXT:     Length:     12
+# DWARF64-YAML-NEXT:     Length:     0x000000000000000C
 # DWARF64-YAML-NEXT:     Version:    4
 # DWARF64-YAML-NEXT:     AbbrOffset: 0
 # DWARF64-YAML-NEXT:     AddrSize:   8
diff --git a/llvm/test/ObjectYAML/MachO/DWARF2-AddrSize8-FormValues.yaml b/llvm/test/ObjectYAML/MachO/DWARF2-AddrSize8-FormValues.yaml
index 142ed0ac02eae..b8e361620663b 100644
--- a/llvm/test/ObjectYAML/MachO/DWARF2-AddrSize8-FormValues.yaml
+++ b/llvm/test/ObjectYAML/MachO/DWARF2-AddrSize8-FormValues.yaml
@@ -415,7 +415,7 @@ DWARF:
 ...
 
 #CHECK:   debug_info:      
-#CHECK:     - Length:          168
+#CHECK:     - Length:          0x00000000000000A8
 #CHECK:       Version:         2
 #CHECK:       AbbrOffset:      0
 #CHECK:       AddrSize:        8
diff --git a/llvm/test/ObjectYAML/MachO/DWARF5-debug_info.yaml b/llvm/test/ObjectYAML/MachO/DWARF5-debug_info.yaml
index 1ed156a63599b..fc2280066c0c0 100644
--- a/llvm/test/ObjectYAML/MachO/DWARF5-debug_info.yaml
+++ b/llvm/test/ObjectYAML/MachO/DWARF5-debug_info.yaml
@@ -513,7 +513,7 @@ DWARF:
 
 #      DWARF32: DWARF:
 #      DWARF32:   debug_info:
-# DWARF32-NEXT:     - Length:     118
+# DWARF32-NEXT:     - Length:     0x0000000000000076
 # DWARF32-NEXT:       Version:    5
 # DWARF32-NEXT:       UnitType:   DW_UT_compile
 # DWARF32-NEXT:       AbbrOffset: 0
@@ -585,7 +585,7 @@ DWARF:
 # RUN:   obj2yaml | FileCheck %s --check-prefix=DWARF32-YAML
 
 #      DWARF32-YAML: debug_info:
-# DWARF32-YAML-NEXT:   - Length:     12
+# DWARF32-YAML-NEXT:   - Length:     0x000000000000000C
 # DWARF32-YAML-NEXT:     Version:    5
 # DWARF32-YAML-NEXT:     UnitType:   DW_UT_compile
 # DWARF32-YAML-NEXT:     AbbrOffset: 0x0000000000000000
@@ -669,7 +669,7 @@ DWARF:
 
 #      DWARF64-YAML: debug_info:
 # DWARF64-YAML-NEXT:   - Format:     DWARF64
-# DWARF64-YAML-NEXT:     Length:     20
+# DWARF64-YAML-NEXT:     Length:     0x0000000000000014
 # DWARF64-YAML-NEXT:     Version:    5
 # DWARF64-YAML-NEXT:     UnitType:   DW_UT_compile
 # DWARF64-YAML-NEXT:     AbbrOffset: 0x0000000000000000
diff --git a/llvm/test/Other/opt-O0-pipeline-enable-matrix.ll b/llvm/test/Other/opt-O0-pipeline-enable-matrix.ll
index 5e2d2726eb874..401cbb976a416 100644
--- a/llvm/test/Other/opt-O0-pipeline-enable-matrix.ll
+++ b/llvm/test/Other/opt-O0-pipeline-enable-matrix.ll
@@ -4,19 +4,10 @@
 
 ; CHECK:      Pass Arguments:
 ; CHECK-NEXT: Target Transform Information
-; CHECK-NEXT: Target Library Information
-; CHECK-NEXT: Assumption Cache Tracker
 ; CHECK-NEXT:   FunctionPass Manager
 ; CHECK-NEXT:     Module Verifier
 ; CHECK-NEXT:     Instrument function entry/exit with calls to e.g. mcount() (pre inlining)
-; CHECK-NEXT:     Dominator Tree Construction
-; CHECK-NEXT:     Natural Loop Information
-; CHECK-NEXT:     Lazy Branch Probability Analysis
-; CHECK-NEXT:     Lazy Block Frequency Analysis
-; CHECK-NEXT:     Optimization Remark Emitter
-; CHECK-NEXT:     Basic Alias Analysis (stateless AA impl)
-; CHECK-NEXT:     Function Alias Analysis Results
-; CHECK-NEXT:     Lower the matrix intrinsics
+; CHECK-NEXT:     Lower the matrix intrinsics (minimal)
 
 
 define void @f() {
diff --git a/llvm/test/Reduce/Inputs/remove-funcs.py b/llvm/test/Reduce/Inputs/remove-funcs.py
deleted file mode 100755
index 1d726c4e9ee75..0000000000000
--- a/llvm/test/Reduce/Inputs/remove-funcs.py
+++ /dev/null
@@ -1,9 +0,0 @@
-
-import sys
-
-input = open(sys.argv[1], "r")
-for line in input:
-  if "@interesting" in line:
-    sys.exit(0)
-
-sys.exit(1) # IR isn't interesting
diff --git a/llvm/test/Reduce/remove-all-of-multiple-args.ll b/llvm/test/Reduce/remove-all-of-multiple-args.ll
new file mode 100644
index 0000000000000..671a0f652aaf9
--- /dev/null
+++ b/llvm/test/Reduce/remove-all-of-multiple-args.ll
@@ -0,0 +1,12 @@
+; RUN: llvm-reduce --test FileCheck --test-arg --check-prefixes=CHECK-ALL,CHECK-INTERESTINGNESS --test-arg %s --test-arg --input-file %s -o %t
+; RUN: cat %t | FileCheck --check-prefixes=CHECK-ALL,CHECK-FINAL %s
+
+define i32 @t(i32 %a0, i32 %a1, i32 %a2) {
+; CHECK-ALL-LABEL: @t
+; CHECK-FINAL: () {
+;
+; CHECK-INTERESTINGNESS: ret i32
+; CHECK-FINAL: ret i32 undef
+
+  ret i32 %a1
+}
diff --git a/llvm/test/Reduce/remove-args-2.ll b/llvm/test/Reduce/remove-args-2.ll
index fddcfc75195cf..25b5b32be38d1 100644
--- a/llvm/test/Reduce/remove-args-2.ll
+++ b/llvm/test/Reduce/remove-args-2.ll
@@ -1,19 +1,22 @@
 ; Test that llvm-reduce can remove uninteresting function arguments from function definitions as well as their calls.
 ; This test checks that functions with different argument types are handled correctly
 ;
-; RUN: llvm-reduce --test %python --test-arg %p/Inputs/remove-args.py %s -o %t
-; RUN: cat %t | FileCheck -implicit-check-not=uninteresting %s
+; RUN: llvm-reduce --test FileCheck --test-arg --check-prefixes=CHECK-ALL,CHECK-INTERESTINGNESS --test-arg %s --test-arg --input-file %s -o %t
+; RUN: FileCheck --check-prefixes=CHECK-ALL,CHECK-FINAL %s --input-file %t
 
 %struct.foo = type { %struct.foo*, i32, i32, i8* }
 
 define dso_local void @bar() {
 entry:
-  ; CHECK: call void @interesting(%struct.foo* null)
+  ; CHECK-INTERESTINGNESS: call void @interesting(
+  ; CHECK-FINAL: call void @interesting(%struct.foo* null)
   call void @interesting(i32 0, i8* null, %struct.foo* null, i8* null, i64 0)
   ret void
 }
 
-; CHECK: define internal void @interesting(%struct.foo* %interesting) {
+; CHECK-ALL: define internal void @interesting
+; CHECK-INTERESTINGNESS-SAME: ({{.*}}%interesting{{.*}}) {
+; CHECK-FINAL-SAME: (%struct.foo* %interesting) {
 define internal void @interesting(i32 %uninteresting1, i8* %uninteresting2, %struct.foo* %interesting, i8* %uninteresting3, i64 %uninteresting4) {
 entry:
   ret void
diff --git a/llvm/test/Reduce/remove-args-used-by-ret.ll b/llvm/test/Reduce/remove-args-used-by-ret.ll
new file mode 100644
index 0000000000000..13b7fe58e5f01
--- /dev/null
+++ b/llvm/test/Reduce/remove-args-used-by-ret.ll
@@ -0,0 +1,14 @@
+; RUN: llvm-reduce --test FileCheck --test-arg --check-prefixes=CHECK-ALL,CHECK-INTERESTINGNESS --test-arg %s --test-arg --input-file %s -o %t
+; RUN: cat %t | FileCheck --check-prefixes=CHECK-ALL,CHECK-FINAL %s
+
+; We can't drop arguments if they are used by terminator instructions.
+
+define i32 @t(i32 %a0, i32 %a1, i32 %a2) {
+; CHECK-ALL-LABEL: @t
+; CHECK-FINAL-NOT: %a1
+;
+; CHECK-INTERESTINGNESS: ret i32
+; CHECK-FINAL: ret i32 undef
+
+  ret i32 %a1
+}
diff --git a/llvm/test/Reduce/remove-funcs.ll b/llvm/test/Reduce/remove-funcs.ll
index 59ffd849193db..a9b525de03725 100644
--- a/llvm/test/Reduce/remove-funcs.ll
+++ b/llvm/test/Reduce/remove-funcs.ll
@@ -1,23 +1,27 @@
 ; Test that llvm-reduce can remove uninteresting functions as well as
 ; their InstCalls.
 ;
-; RUN: llvm-reduce --test %python --test-arg %p/Inputs/remove-funcs.py %s -o %t
-; RUN: cat %t | FileCheck -implicit-check-not=uninteresting %s
+; RUN: llvm-reduce --test FileCheck --test-arg --check-prefixes=CHECK-ALL,CHECK-INTERESTINGNESS --test-arg %s --test-arg --input-file %s -o %t
+; RUN: cat %t | FileCheck -implicit-check-not=uninteresting --check-prefixes=CHECK-ALL,CHECK-FINAL %s
 
 define i32 @uninteresting1() {
 entry:
   ret i32 0
 }
 
-; CHECK: interesting()
+; CHECK-ALL-LABEL: interesting()
 define i32 @interesting() {
 entry:
-  ; CHECK: call i32 @interesting()
+  ; CHECK-INTERESTINGNESS: call i32 @interesting()
   %call2 = call i32 @interesting()
   %call = call i32 @uninteresting1()
   ret i32 5
 }
 
+; CHECK-FINAL-NEXT: entry:
+; CHECK-FINAL-NEXT:   %call2 = call i32 @interesting()
+; CHECK-FINAL-NEXT: }
+
 define i32 @uninteresting2() {
 entry:
   ret i32 0
diff --git a/llvm/test/Reduce/remove-single-arg.ll b/llvm/test/Reduce/remove-single-arg.ll
new file mode 100644
index 0000000000000..4e33a6eebb671
--- /dev/null
+++ b/llvm/test/Reduce/remove-single-arg.ll
@@ -0,0 +1,12 @@
+; RUN: llvm-reduce --test FileCheck --test-arg --check-prefixes=CHECK-ALL,CHECK-INTERESTINGNESS --test-arg %s --test-arg --input-file %s -o %t
+; RUN: cat %t | FileCheck --check-prefixes=CHECK-ALL,CHECK-FINAL %s
+
+define i32 @t(i32 %a0) {
+; CHECK-ALL-LABEL: @t
+; CHECK-FINAL: () {
+;
+; CHECK-INTERESTINGNESS: ret i32
+; CHECK-FINAL: ret i32 42
+
+  ret i32 42
+}
diff --git a/llvm/test/SYCLLowerIR/byval_arg.ll b/llvm/test/SYCLLowerIR/byval_arg.ll
index 7be1b277c16b1..df3cc8062a796 100644
--- a/llvm/test/SYCLLowerIR/byval_arg.ll
+++ b/llvm/test/SYCLLowerIR/byval_arg.ll
@@ -11,6 +11,7 @@
 define internal spir_func void @wibble(%struct.baz* byval(%struct.baz) %arg1) !work_group_scope !0 {
 ; CHECK-LABEL: @wibble(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, i64 addrspace(1)* @__spirv_BuiltInLocalInvocationIndex
+; CHECK-NEXT:    call void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
 ; CHECK-NEXT:    [[CMPZ:%.*]] = icmp eq i64 [[TMP1]], 0
 ; CHECK-NEXT:    br i1 [[CMPZ]], label [[LEADER:%.*]], label [[MERGE:%.*]]
 ; CHECK:       leader:
@@ -19,6 +20,8 @@ define internal spir_func void @wibble(%struct.baz* byval(%struct.baz) %arg1) !w
 ; CHECK-NEXT:    br label [[MERGE]]
 ; CHECK:       merge:
 ; CHECK-NEXT:    call void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast %struct.baz* [[ARG1]] to i8*
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p3i8.i64(i8* [[TMP3]], i8 addrspace(3)* align 8 bitcast (%struct.baz addrspace(3)* @[[SHADOW]] to i8 addrspace(3)*), i64 8, i1 false)
 ; CHECK-NEXT:    ret void
 ;
   ret void
diff --git a/llvm/test/SYCLLowerIR/byval_arg_cast.ll b/llvm/test/SYCLLowerIR/byval_arg_cast.ll
new file mode 100644
index 0000000000000..ef94ccd8e9461
--- /dev/null
+++ b/llvm/test/SYCLLowerIR/byval_arg_cast.ll
@@ -0,0 +1,52 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -LowerWGScope -S | FileCheck %s
+
+; Test to check that shadow local variable for byval argument is
+; materialized before use. Otherwise invalid cast between address
+; spaces is generated.
+
+%struct.widget = type { %struct.baz, %struct.baz, %struct.baz, %struct.spam }
+%struct.baz = type { %struct.snork }
+%struct.snork = type { [1 x i64] }
+%struct.spam = type { %struct.snork }
+
+
+declare dso_local spir_func void @zot(i8*)
+
+; CHECK: @[[SHADOW:[a-zA-Z0-9]+]] = internal unnamed_addr addrspace(3) global %struct.widget undef, align 16
+
+; Function Attrs: inlinehint norecurse
+define dso_local spir_func void @wombat(%struct.widget* byval(%struct.widget) align 8 %arg) align 2 !work_group_scope !1 {
+; CHECK-LABEL: @wombat(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64 addrspace(1)* @__spirv_BuiltInLocalInvocationIndex, align 4
+; CHECK-NEXT:    call void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+; CHECK-NEXT:    [[CMPZ1:%.*]] = icmp eq i64 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[CMPZ1]], label [[LEADER:%.*]], label [[MERGE:%.*]]
+; CHECK:       leader:
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast %struct.widget* [[ARG:%.*]] to i8*
+; CHECK-NEXT:    call void @llvm.memcpy.p3i8.p0i8.i64(i8 addrspace(3)* align 16 bitcast (%struct.widget addrspace(3)* @[[SHADOW]] to i8 addrspace(3)*), i8* align 8 [[TMP1]], i64 32, i1 false)
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    call void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272) #0
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast %struct.widget* [[ARG]] to i8*
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p3i8.i64(i8* align 8 [[TMP2]], i8 addrspace(3)* align 16 bitcast (%struct.widget addrspace(3)* @[[SHADOW]] to i8 addrspace(3)*), i64 32, i1 false)
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, i64 addrspace(1)* @__spirv_BuiltInLocalInvocationIndex, align 4
+; CHECK-NEXT:    call void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+; CHECK-NEXT:    [[CMPZ:%.*]] = icmp eq i64 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[CMPZ]], label [[WG_LEADER:%.*]], label [[WG_CF:%.*]]
+; CHECK:       wg_leader:
+; CHECK-NEXT:    [[TMP:%.*]] = bitcast %struct.widget* [[ARG]] to i8*
+; CHECK-NEXT:    call void @zot(i8* [[TMP]])
+; CHECK-NEXT:    br label [[WG_CF]]
+; CHECK:       wg_cf:
+; CHECK-NEXT:    call void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272) #0
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = bitcast %struct.widget* %arg to i8*
+  call void @zot(i8* %tmp)
+  ret void
+}
+
+!1 = !{}
diff --git a/llvm/test/SYCLLowerIR/cast_shadow.ll b/llvm/test/SYCLLowerIR/cast_shadow.ll
index 1822f1c5e8148..56e8e9e0288bc 100644
--- a/llvm/test/SYCLLowerIR/cast_shadow.ll
+++ b/llvm/test/SYCLLowerIR/cast_shadow.ll
@@ -15,7 +15,10 @@ target triple = "nvptx64-nvidia-cuda-sycldevice"
 define internal void @wobble(%struct.baz* %arg, %struct.spam* byval(%struct.spam) %arg1) !work_group_scope !0 {
 ; CHECK:    [[TMP10:%.*]] = bitcast %struct.spam* [[ARG1:%.*]] to i8*
 ; CHECK:    call void @llvm.memcpy.p3i8.p0i8.i64(i8 addrspace(3)* align 16 bitcast (%struct.spam addrspace(3)* @[[SHADOW]] to i8 addrspace(3)*), i8* [[TMP10]], i64 32, i1 false)
-; CHECK:    call void @widget(%struct.spam* addrspacecast (%struct.spam addrspace(3)* @[[SHADOW]] to %struct.spam*), %struct.quux* byval(%struct.quux) [[TMP2:%.*]])
+; CHECK:    call void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272) #0
+; CHECK:    [[TMP11:%.*]] = bitcast %struct.spam* %arg1 to i8*
+; CHECK:    call void @llvm.memcpy.p0i8.p3i8.i64(i8* [[TMP11:%.*]], i8 addrspace(3)* align 16 bitcast (%struct.spam addrspace(3)* @[[SHADOW]] to i8
+; CHECK:    call void @widget(%struct.spam* %arg1, %struct.quux* byval(%struct.quux) [[TMP2:%.*]])
 ;
 bb:
   %tmp = alloca %struct.baz*
diff --git a/llvm/test/SYCLLowerIR/convergent.ll b/llvm/test/SYCLLowerIR/convergent.ll
index 7c8d70c08d2a9..be9c9bd687776 100644
--- a/llvm/test/SYCLLowerIR/convergent.ll
+++ b/llvm/test/SYCLLowerIR/convergent.ll
@@ -19,8 +19,8 @@ define internal spir_func void @wibble(%struct.baz* byval(%struct.baz) %arg1) !w
 ; CHECK-PTX: declare i64 @_Z27__spirv_LocalInvocationId_zv()
 
 ; CHECK: ; Function Attrs: convergent
-; CHECK: declare void @_Z22__spirv_ControlBarrierjjj(i32, i32, i32) #1
+; CHECK: declare void @_Z22__spirv_ControlBarrierjjj(i32, i32, i32) #[[ATTR_NUM:[0-9]+]]
 
-; CHECK: attributes #1 = { convergent }
+; CHECK: attributes #[[ATTR_NUM]] = { convergent }
 
 !0 = !{}
diff --git a/llvm/test/SYCLLowerIR/esimd_global.ll b/llvm/test/SYCLLowerIR/esimd_global.ll
new file mode 100644
index 0000000000000..ab86858fe4da7
--- /dev/null
+++ b/llvm/test/SYCLLowerIR/esimd_global.ll
@@ -0,0 +1,166 @@
+; This test checks whether globals are converted
+; correctly to llvm's native vector type.
+;
+; RUN: opt < %s -ESIMDLowerVecArg -S | FileCheck %s                            
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
+target triple = "spir64-unknown-unknown-sycldevice"
+
+%"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd" = type { <16 x i32> }
+
+$"_ZTSZZ4mainENK3$_0clERN2cl4sycl7handlerEE4Test" = comdat any
+
+; CHECK: [[NEWGLOBAL:[@a-zA-Z0-9_]*]] = dso_local global <16 x i32> zeroinitializer, align 64 #0
+@0 = dso_local global %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd" zeroinitializer, align 64 #0
+
+; Function Attrs: norecurse
+define weak_odr dso_local spir_kernel void @"_ZTSZZ4mainENK3$_0clERN2cl4sycl7handlerEE4Test"(i32 addrspace(1)* %_arg_) local_unnamed_addr #1 comdat !kernel_arg_addr_space !8 !kernel_arg_access_qual !9 !kernel_arg_type !10 !kernel_arg_base_type !10 !kernel_arg_type_qual !11 !sycl_explicit_simd !12 !intel_reqd_sub_group_size !8 {
+entry:
+  %vc.i = alloca %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd", align 64
+  %agg.tmp.i = alloca %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd", align 64
+  %call.esimd.i.i.i.i.i = call <3 x i32> @llvm.genx.local.id.v3i32() #5
+  %local_id.y.i.i.i.i.i = extractelement <3 x i32> %call.esimd.i.i.i.i.i, i32 1
+  %local_id.y.cast.ty.i.i.i.i.i = zext i32 %local_id.y.i.i.i.i.i to i64
+  %call.esimd1.i.i.i.i.i = call <3 x i32> @llvm.genx.local.size.v3i32() #5
+  %wgsize.y.i.i.i.i.i = extractelement <3 x i32> %call.esimd1.i.i.i.i.i, i32 1
+  %wgsize.y.cast.ty.i.i.i.i.i = zext i32 %wgsize.y.i.i.i.i.i to i64
+  %group.id.y.i.i.i.i.i = call i32 @llvm.genx.group.id.y() #5
+  %group.id.y.cast.ty.i.i.i.i.i = zext i32 %group.id.y.i.i.i.i.i to i64
+  %mul.i.i.i.i.i = mul nuw i64 %wgsize.y.cast.ty.i.i.i.i.i, %group.id.y.cast.ty.i.i.i.i.i
+  %add.i.i.i.i.i = add i64 %mul.i.i.i.i.i, %local_id.y.cast.ty.i.i.i.i.i
+  %local_id.x.i.i.i.i.i = extractelement <3 x i32> %call.esimd.i.i.i.i.i, i32 0
+  %local_id.x.cast.ty.i.i.i.i.i = zext i32 %local_id.x.i.i.i.i.i to i64
+  %wgsize.x.i.i.i.i.i = extractelement <3 x i32> %call.esimd1.i.i.i.i.i, i32 0
+  %wgsize.x.cast.ty.i.i.i.i.i = zext i32 %wgsize.x.i.i.i.i.i to i64
+  %group.id.x.i.i.i.i.i = call i32 @llvm.genx.group.id.x() #5
+  %group.id.x.cast.ty.i.i.i.i.i = zext i32 %group.id.x.i.i.i.i.i to i64
+  %mul.i4.i.i.i.i = mul nuw i64 %group.id.x.cast.ty.i.i.i.i.i, %wgsize.x.cast.ty.i.i.i.i.i
+  %add.i5.i.i.i.i = add i64 %mul.i4.i.i.i.i, %local_id.x.cast.ty.i.i.i.i.i
+  %0 = bitcast %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd"* %agg.tmp.i to i8*
+  call void @llvm.lifetime.start.p0i8(i64 64, i8* nonnull %0)
+  %1 = bitcast %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd"* %vc.i to i8*
+  call void @llvm.lifetime.start.p0i8(i64 64, i8* nonnull %1) #5
+  %conv.i = trunc i64 %add.i5.i.i.i.i to i32
+  %2 = addrspacecast %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd"* %vc.i to %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd" addrspace(4)*
+  %splat.splatinsert.i.i = insertelement <16 x i32> undef, i32 %conv.i, i32 0
+  %splat.splat.i.i = shufflevector <16 x i32> %splat.splatinsert.i.i, <16 x i32> undef, <16 x i32> zeroinitializer
+  %M_data.i13.i = getelementptr inbounds %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd", %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd" addrspace(4)* %2, i64 0, i32 0
+  store <16 x i32> %splat.splat.i.i, <16 x i32> addrspace(4)* %M_data.i13.i, align 64, !tbaa !13
+  %conv3.i = trunc i64 %add.i.i.i.i.i to i32
+  %splat.splatinsert.i20.i = insertelement <8 x i32> undef, i32 %conv3.i, i32 0
+  %splat.splat.i21.i = shufflevector <8 x i32> %splat.splatinsert.i20.i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %call.esimd.i.i.i.i.i2 = call <16 x i32> @llvm.genx.vload.v16i32.p4v16i32(<16 x i32> addrspace(4)* %M_data.i13.i) #5
+  %call4.esimd.i.i.i.i = call <16 x i32> @llvm.genx.wrregioni.v16i32.v8i32.i16.v8i1(<16 x i32> %call.esimd.i.i.i.i.i2, <8 x i32> %splat.splat.i21.i, i32 0, i32 8, i32 1, i16 0, i32 0, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>) #5
+  call void @llvm.genx.vstore.v16i32.p4v16i32(<16 x i32> %call4.esimd.i.i.i.i, <16 x i32> addrspace(4)* %M_data.i13.i) #5
+  %cmp.i = icmp eq i64 %add.i.i.i.i.i, 0
+  %..i = select i1 %cmp.i, i64 %add.i5.i.i.i.i, i64 %add.i.i.i.i.i
+  %conv9.i = trunc i64 %..i to i32
+; CHECK: store <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, <16 x i32> addrspace(4)* addrspacecast (<16 x i32>* getelementptr inbounds ({{.+}}, {{.+}}* bitcast (<16 x i32>* [[NEWGLOBAL]] to {{.+}}*), i64 0, i32 0) to <16 x i32> addrspace(4)*), align 64, !tbaa.struct !16
+  store <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, <16 x i32> addrspace(4)* addrspacecast (<16 x i32>* getelementptr inbounds (%"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd", %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd"* @0, i64 0, i32 0) to <16 x i32> addrspace(4)*), align 64, !tbaa.struct !16
+  %mul.i = shl nsw i32 %conv9.i, 4
+  %idx.ext.i = sext i32 %mul.i to i64
+  %add.ptr.i16 = getelementptr inbounds i32, i32 addrspace(1)* %_arg_, i64 %idx.ext.i
+  %add.ptr.i = addrspacecast i32 addrspace(1)* %add.ptr.i16 to i32 addrspace(4)*
+  %3 = addrspacecast %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd"* %agg.tmp.i to %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd" addrspace(4)*
+  %call.esimd.i.i.i = call <16 x i32> @llvm.genx.vload.v16i32.p4v16i32(<16 x i32> addrspace(4)* %M_data.i13.i) #5
+  %M_data.i2.i.i = getelementptr inbounds %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd", %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd" addrspace(4)* %3, i64 0, i32 0
+  call void @llvm.genx.vstore.v16i32.p4v16i32(<16 x i32> %call.esimd.i.i.i, <16 x i32> addrspace(4)* %M_data.i2.i.i) #5
+  call spir_func void @_Z3fooPiN2cl4sycl5intel3gpu4simdIiLi16EEE(i32 addrspace(4)* %add.ptr.i, %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd"* nonnull %agg.tmp.i) #5
+  store <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>, <16 x i32> addrspace(4)* addrspacecast (<16 x i32>* getelementptr inbounds (%"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd", %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd"* @0, i64 0, i32 0) to <16 x i32> addrspace(4)*), align 64, !tbaa.struct !16
+  call void @llvm.lifetime.end.p0i8(i64 64, i8* nonnull %1) #5
+  call void @llvm.lifetime.end.p0i8(i64 64, i8* nonnull %0)
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg %0, i8* nocapture %1) #2
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg %0, i8* nocapture %1) #2
+
+; Function Attrs: noinline norecurse nounwind
+define dso_local spir_func void @_Z3fooPiN2cl4sycl5intel3gpu4simdIiLi16EEE(i32 addrspace(4)* %C, %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd"* %v) local_unnamed_addr #3 !sycl_explicit_simd !12 {
+entry:
+  %agg.tmp = alloca %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd", align 64
+  %0 = addrspacecast %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd"* %v to %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd" addrspace(4)*
+  %1 = addrspacecast %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd"* %agg.tmp to %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd" addrspace(4)*
+  %M_data.i.i = getelementptr inbounds %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd", %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd" addrspace(4)* %0, i64 0, i32 0
+  %call.esimd.i.i = call <16 x i32> @llvm.genx.vload.v16i32.p4v16i32(<16 x i32> addrspace(4)* %M_data.i.i), !noalias !17
+; CHECK: {{.+}} = call <16 x i32> @llvm.genx.vload.v16i32.p4v16i32(<16 x i32> addrspace(4)* getelementptr ({{.+}}, {{.+}} addrspace(4)* addrspacecast ({{.+}}* bitcast (<16 x i32>* [[NEWGLOBAL]] to {{.+}}*) to {{.+}} addrspace(4)*), i64 0, i32 0)), !noalias !17
+  %call.esimd.i8.i = call <16 x i32> @llvm.genx.vload.v16i32.p4v16i32(<16 x i32> addrspace(4)* getelementptr (%"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd", %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd" addrspace(4)* addrspacecast (%"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd"* @0 to %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd" addrspace(4)*), i64 0, i32 0)), !noalias !17
+  %add.i = add <16 x i32> %call.esimd.i8.i, %call.esimd.i.i
+  %M_data.i.i.i = getelementptr inbounds %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd", %"class._ZTSN2cl4sycl5intel3gpu4simdIiLi16EEE.cl::sycl::intel::gpu::simd" addrspace(4)* %1, i64 0, i32 0
+  call void @llvm.genx.vstore.v16i32.p4v16i32(<16 x i32> %add.i, <16 x i32> addrspace(4)* %M_data.i.i.i)
+  %2 = ptrtoint i32 addrspace(4)* %C to i64
+  %call.esimd.i.i2 = call <16 x i32> @llvm.genx.vload.v16i32.p4v16i32(<16 x i32> addrspace(4)* %M_data.i.i.i)
+  call void @llvm.genx.svm.block.st.v16i32(i64 %2, <16 x i32> %call.esimd.i.i2)
+  ret void
+}
+
+; Function Attrs: nounwind readnone
+declare !genx_intrinsic_id !20 <16 x i32> @llvm.genx.wrregioni.v16i32.v8i32.i16.v8i1(<16 x i32> %0, <8 x i32> %1, i32 %2, i32 %3, i32 %4, i16 %5, i32 %6, <8 x i1> %7) #4
+
+; Function Attrs: nounwind
+declare !genx_intrinsic_id !21 <16 x i32> @llvm.genx.vload.v16i32.p4v16i32(<16 x i32> addrspace(4)* %0) #5
+
+; Function Attrs: nounwind
+declare !genx_intrinsic_id !22 void @llvm.genx.vstore.v16i32.p4v16i32(<16 x i32> %0, <16 x i32> addrspace(4)* %1) #5
+
+; Function Attrs: nounwind
+declare !genx_intrinsic_id !23 void @llvm.genx.svm.block.st.v16i32(i64 %0, <16 x i32> %1) #5
+
+; Function Attrs: nounwind readnone
+declare !genx_intrinsic_id !24 <3 x i32> @llvm.genx.local.id.v3i32() #4
+
+; Function Attrs: nounwind readnone
+declare !genx_intrinsic_id !25 <3 x i32> @llvm.genx.local.size.v3i32() #4
+
+; Function Attrs: nounwind readnone
+declare !genx_intrinsic_id !26 i32 @llvm.genx.group.id.y() #4
+
+; Function Attrs: nounwind readnone
+declare !genx_intrinsic_id !27 i32 @llvm.genx.group.id.x() #4
+
+attributes #0 = { "genx_byte_offset"="192" "genx_volatile" }
+attributes #1 = { norecurse "CMGenxMain" "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="512" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "oclrt"="1" "stack-protector-buffer-size"="8" "sycl-module-id"="subroutine.cpp" "uniform-work-group-size"="true" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { argmemonly nounwind willreturn }
+attributes #3 = { noinline norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="512" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { nounwind readnone }
+attributes #5 = { nounwind }
+
+!llvm.dependent-libraries = !{!0}
+!llvm.module.flags = !{!1}
+!opencl.spir.version = !{!2}
+!spirv.Source = !{!3}
+!llvm.ident = !{!4}
+!genx.kernels = !{!5}
+
+!0 = !{!"libcpmt"}
+!1 = !{i32 1, !"wchar_size", i32 2}
+!2 = !{i32 1, i32 2}
+!3 = !{i32 6, i32 100000}
+!4 = !{!"clang version 11.0.0"}
+!5 = !{void (i32 addrspace(1)*)* @"_ZTSZZ4mainENK3$_0clERN2cl4sycl7handlerEE4Test", !"_ZTSZZ4mainENK3$_0clERN2cl4sycl7handlerEE4Test", !6, i32 0, i32 0, !6, !7, i32 0, i32 0}
+!6 = !{i32 0}
+!7 = !{!"svmptr_t"}
+!8 = !{i32 1}
+!9 = !{!"none"}
+!10 = !{!"int*"}
+!11 = !{!""}
+!12 = !{}
+!13 = !{!14, !14, i64 0}
+!14 = !{!"omnipotent char", !15, i64 0}
+!15 = !{!"Simple C++ TBAA"}
+!16 = !{i64 0, i64 64, !13}
+!17 = !{!18}
+!18 = distinct !{!18, !19, !"_ZNK2cl4sycl5intel3gpu4simdIiLi16EEplERKS4_: %agg.result"}
+!19 = distinct !{!19, !"_ZNK2cl4sycl5intel3gpu4simdIiLi16EEplERKS4_"}
+!20 = !{i32 8275}
+!21 = !{i32 8268}
+!22 = !{i32 8269}
+!23 = !{i32 8166}
+!24 = !{i32 8029}
+!25 = !{i32 8034}
+!26 = !{i32 8020}
+!27 = !{i32 8019}
+
diff --git a/llvm/test/SYCLLowerIR/esimd_lower_intrins.ll b/llvm/test/SYCLLowerIR/esimd_lower_intrins.ll
index f29503b1bfce3..5ed90614a675b 100644
--- a/llvm/test/SYCLLowerIR/esimd_lower_intrins.ll
+++ b/llvm/test/SYCLLowerIR/esimd_lower_intrins.ll
@@ -170,6 +170,17 @@ define dso_local spir_kernel void  @FUNC_30() !sycl_explicit_simd !1 {
 ; CHECK-NEXT: ret void
 }
 
+define dso_local spir_kernel void  @FUNC_31() !sycl_explicit_simd !1 {
+; CHECK: define dso_local spir_kernel void  @FUNC_31() !sycl_explicit_simd !1
+  %call = call spir_func i64 @_Z27__spirv_LocalInvocationId_xv()
+; CHECK-NEXT: %call.esimd = call <3 x i32> @llvm.genx.local.id.v3i32()
+; CHECK-NEXT: %local_id.x = extractelement <3 x i32> %call.esimd, i32 0
+; CHECK-NEXT: %local_id.x.cast.ty = zext i32 %local_id.x to i64
+  ret void
+; CHECK-NEXT: ret void
+}
+
+declare dso_local spir_func i64 @_Z27__spirv_LocalInvocationId_xv()
 declare dso_local spir_func <32 x i32> @_Z20__esimd_flat_atomic0ILN2cm3gen14CmAtomicOpTypeE2EjLi32ELNS1_9CacheHintE0ELS3_0EENS1_13__vector_typeIT0_XT1_EE4typeENS4_IyXT1_EE4typeENS4_ItXT1_EE4typeE(<32 x i64> %0, <32 x i16> %1)
 declare dso_local spir_func <32 x i32> @_Z20__esimd_flat_atomic1ILN2cm3gen14CmAtomicOpTypeE0EjLi32ELNS1_9CacheHintE0ELS3_0EENS1_13__vector_typeIT0_XT1_EE4typeENS4_IyXT1_EE4typeES7_NS4_ItXT1_EE4typeE(<32 x i64> %0, <32 x i32> %1, <32 x i16> %2)
 declare dso_local spir_func <32 x i32> @_Z20__esimd_flat_atomic2ILN2cm3gen14CmAtomicOpTypeE7EjLi32ELNS1_9CacheHintE0ELS3_0EENS1_13__vector_typeIT0_XT1_EE4typeENS4_IyXT1_EE4typeES7_S7_NS4_ItXT1_EE4typeE(<32 x i64> %0, <32 x i32> %1, <32 x i32> %2, <32 x i16> %3)
diff --git a/llvm/test/SYCLLowerIR/esimd_subroutine.ll b/llvm/test/SYCLLowerIR/esimd_subroutine.ll
new file mode 100644
index 0000000000000..6b5019bcc868e
--- /dev/null
+++ b/llvm/test/SYCLLowerIR/esimd_subroutine.ll
@@ -0,0 +1,107 @@
+; This test checks whether subroutine arguments are converted
+; correctly to llvm's native vector type.
+;
+; RUN: opt < %s -ESIMDLowerVecArg -S | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
+target triple = "spir64-unknown-unknown-sycldevice"
+
+%class._ZTS4simdIiLi16EE.simd = type { <16 x i32> }
+
+$_ZN4simdIiLi16EEC1ERS0_ = comdat any
+
+$_ZN4simdIiLi16EEC2ERS0_ = comdat any
+
+; Function Attrs: norecurse nounwind
+define spir_func void @_Z3fooi(i32 %x) #0 {
+entry:
+  %x.addr = alloca i32, align 4
+; CHECK: {{.+}} = alloca {{.+}}
+; CHECK-NEXT: [[A:%[a-zA-Z0-9_]*]] = alloca {{.+}}
+  %a = alloca %class._ZTS4simdIiLi16EE.simd, align 64
+  %agg.tmp = alloca %class._ZTS4simdIiLi16EE.simd, align 64
+  store i32 %x, i32* %x.addr, align 4, !tbaa !4
+  %0 = bitcast %class._ZTS4simdIiLi16EE.simd* %a to i8*
+  call void @llvm.lifetime.start.p0i8(i64 64, i8* %0) #2
+; CHECK: [[ADDRSPCAST1:%[a-zA-Z0-9_]*]] = addrspacecast {{.+}} [[A]] to {{.+}}
+  %1 = addrspacecast %class._ZTS4simdIiLi16EE.simd* %agg.tmp to %class._ZTS4simdIiLi16EE.simd addrspace(4)*
+  %2 = addrspacecast %class._ZTS4simdIiLi16EE.simd* %a to %class._ZTS4simdIiLi16EE.simd addrspace(4)*
+; CHECK: [[BITCASTRESULT1:%[a-zA-Z0-9_]*]] = bitcast {{.+}} addrspace(4)* [[ADDRSPCAST1]] to <16 x i32> addrspace(4)*
+; CHECK-NEXT: call spir_func void @_ZN4simdIiLi16EEC1ERS0_(<16 x i32> addrspace(4)* {{.+}}, <16 x i32> addrspace(4)* [[BITCASTRESULT1]])
+  call spir_func void @_ZN4simdIiLi16EEC1ERS0_(%class._ZTS4simdIiLi16EE.simd addrspace(4)* %1, %class._ZTS4simdIiLi16EE.simd addrspace(4)* align 64 dereferenceable(64) %2)
+; CHECK: [[BITCASTRESULT2:%[a-zA-Z0-9_]*]] = bitcast {{.+}} to <16 x i32>*
+; CHECK-NEXT: {{.+}} = call spir_func i32 {{.+}}bar{{.+}}(<16 x i32>* [[BITCASTRESULT2]])
+  %call = call spir_func i32 @_Z3bar4simdIiLi16EE(%class._ZTS4simdIiLi16EE.simd* %agg.tmp)
+  %3 = bitcast %class._ZTS4simdIiLi16EE.simd* %a to i8*
+  call void @llvm.lifetime.end.p0i8(i64 64, i8* %3) #2
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+; Function Attrs: norecurse nounwind
+; CHECK: define spir_func i32 @_Z3bar4simdIiLi16EE(<16 x i32>* {{.+}}
+define spir_func i32 @_Z3bar4simdIiLi16EE(%class._ZTS4simdIiLi16EE.simd* %v) #0 {
+entry:
+; CHECK: {{.+}} = bitcast <16 x i32>* {{.+}}
+  ret i32 1
+}
+
+; Function Attrs: norecurse nounwind
+; CHECK: define linkonce_odr spir_func void @_ZN4simdIiLi16EEC1ERS0_(<16 x i32> addrspace(4)* [[OLDARG0:%[a-zA-Z0-9_]*]], <16 x i32> addrspace(4)*{{.*}} [[OLDARG1:%[a-zA-Z0-9_]*]]) unnamed_addr {{.+}}
+define linkonce_odr spir_func void @_ZN4simdIiLi16EEC1ERS0_(%class._ZTS4simdIiLi16EE.simd addrspace(4)* %this, %class._ZTS4simdIiLi16EE.simd addrspace(4)* align 64 dereferenceable(64) %other) unnamed_addr #0 comdat align 2 {
+entry:
+; CHECK: [[NEWARG1:%[a-zA-Z0-9_]*]] = bitcast <16 x i32> addrspace(4)* [[OLDARG1]] to {{.+}}
+; CHECK-NEXT: [[NEWARG0:%[a-zA-Z0-9_]*]] = bitcast <16 x i32> addrspace(4)* [[OLDARG0]] to {{.+}}
+  %this.addr = alloca %class._ZTS4simdIiLi16EE.simd addrspace(4)*, align 8
+  %other.addr = alloca %class._ZTS4simdIiLi16EE.simd addrspace(4)*, align 8
+; CHECK: store {{.+}} addrspace(4)* [[NEWARG0]], {{.+}}
+  store %class._ZTS4simdIiLi16EE.simd addrspace(4)* %this, %class._ZTS4simdIiLi16EE.simd addrspace(4)** %this.addr, align 8, !tbaa !8
+; CHECK-NEXT: store {{.+}} addrspace(4)* [[NEWARG1]], {{.+}}
+  store %class._ZTS4simdIiLi16EE.simd addrspace(4)* %other, %class._ZTS4simdIiLi16EE.simd addrspace(4)** %other.addr, align 8, !tbaa !8
+  %this1 = load %class._ZTS4simdIiLi16EE.simd addrspace(4)*, %class._ZTS4simdIiLi16EE.simd addrspace(4)** %this.addr, align 8
+  %0 = load %class._ZTS4simdIiLi16EE.simd addrspace(4)*, %class._ZTS4simdIiLi16EE.simd addrspace(4)** %other.addr, align 8
+  call spir_func void @_ZN4simdIiLi16EEC2ERS0_(%class._ZTS4simdIiLi16EE.simd addrspace(4)* %this1, %class._ZTS4simdIiLi16EE.simd addrspace(4)* align 64 dereferenceable(64) %0)
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
+
+; Function Attrs: norecurse nounwind
+define linkonce_odr spir_func void @_ZN4simdIiLi16EEC2ERS0_(%class._ZTS4simdIiLi16EE.simd addrspace(4)* %this, %class._ZTS4simdIiLi16EE.simd addrspace(4)* align 64 dereferenceable(64) %other) unnamed_addr #0 comdat align 2 {
+entry:
+  %this.addr = alloca %class._ZTS4simdIiLi16EE.simd addrspace(4)*, align 8
+  %other.addr = alloca %class._ZTS4simdIiLi16EE.simd addrspace(4)*, align 8
+  store %class._ZTS4simdIiLi16EE.simd addrspace(4)* %this, %class._ZTS4simdIiLi16EE.simd addrspace(4)** %this.addr, align 8, !tbaa !8
+  store %class._ZTS4simdIiLi16EE.simd addrspace(4)* %other, %class._ZTS4simdIiLi16EE.simd addrspace(4)** %other.addr, align 8, !tbaa !8
+  %this1 = load %class._ZTS4simdIiLi16EE.simd addrspace(4)*, %class._ZTS4simdIiLi16EE.simd addrspace(4)** %this.addr, align 8
+  %0 = load %class._ZTS4simdIiLi16EE.simd addrspace(4)*, %class._ZTS4simdIiLi16EE.simd addrspace(4)** %other.addr, align 8, !tbaa !8
+  %__M_data = getelementptr inbounds %class._ZTS4simdIiLi16EE.simd, %class._ZTS4simdIiLi16EE.simd addrspace(4)* %0, i32 0, i32 0
+  %1 = load <16 x i32>, <16 x i32> addrspace(4)* %__M_data, align 64, !tbaa !10
+  %__M_data2 = getelementptr inbounds %class._ZTS4simdIiLi16EE.simd, %class._ZTS4simdIiLi16EE.simd addrspace(4)* %this1, i32 0, i32 0
+  store <16 x i32> %1, <16 x i32> addrspace(4)* %__M_data2, align 64, !tbaa !10
+  ret void
+}
+
+attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind willreturn }
+attributes #2 = { nounwind }
+
+!llvm.module.flags = !{!0}
+!opencl.spir.version = !{!1}
+!spirv.Source = !{!2}
+!llvm.ident = !{!3}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 1, i32 2}
+!2 = !{i32 4, i32 100000}
+!3 = !{!"clang version 11.0.0 (https://github.com/kbobrovs/llvm.git fb752d6351dc6785f5438b137a86fa39a3493225)"}
+!4 = !{!5, !5, i64 0}
+!5 = !{!"int", !6, i64 0}
+!6 = !{!"omnipotent char", !7, i64 0}
+!7 = !{!"Simple C++ TBAA"}
+!8 = !{!9, !9, i64 0}
+!9 = !{!"any pointer", !6, i64 0}
+!10 = !{!6, !6, i64 0}
diff --git a/llvm/test/SYCLLowerIR/pfwg_and_pfwi.ll b/llvm/test/SYCLLowerIR/pfwg_and_pfwi.ll
index 09ba788316dee..fdbdca4686f57 100644
--- a/llvm/test/SYCLLowerIR/pfwg_and_pfwi.ll
+++ b/llvm/test/SYCLLowerIR/pfwg_and_pfwi.ll
@@ -25,6 +25,7 @@ define internal spir_func void @wibble(%struct.bar addrspace(4)* %arg, %struct.z
 ; CHECK-NEXT:    [[TMP:%.*]] = alloca [[STRUCT_BAR:%.*]] addrspace(4)*, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = alloca [[STRUCT_FOO_0:%.*]], align 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64 addrspace(1)* @__spirv_BuiltInLocalInvocationIndex
+; CHECK-NEXT:    call void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
 ; CHECK-NEXT:    [[CMPZ3:%.*]] = icmp eq i64 [[TMP0]], 0
 ; CHECK-NEXT:    br i1 [[CMPZ3]], label [[LEADER:%.*]], label [[MERGE:%.*]]
 ; CHECK:       leader:
@@ -35,17 +36,23 @@ define internal spir_func void @wibble(%struct.bar addrspace(4)* %arg, %struct.z
 ; CHECK-NEXT:    br label [[MERGE]]
 ; CHECK:       merge:
 ; CHECK-NEXT:    call void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272) #0
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, i64 addrspace(1)* @__spirv_BuiltInLocalInvocationIndex
-; CHECK-NEXT:    [[CMPZ:%.*]] = icmp eq i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast %struct.zot* [[ARG1]] to i8*
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p3i8.i64(i8* align 8 [[TMP3]], i8 addrspace(3)* align 16 bitcast (%struct.zot addrspace(3)* @[[GROUP_SHADOW]] to i8 addrspace(3)*), i64 96, i1 false)
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast [[STRUCT_BAR]] addrspace(4)* [[ARG]] to i8 addrspace(4)*
+; CHECK-NEXT:    call void @llvm.memcpy.p4i8.p3i8.i64(i8 addrspace(4)* align 8 [[TMP4]], i8 addrspace(3)* align 8 getelementptr inbounds (%struct.bar, [[STRUCT_BAR]] addrspace(3)* @[[PFWG_SHADOW]], i32 0, i32 0), i64 1, i1 false)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i64, i64 addrspace(1)* @__spirv_BuiltInLocalInvocationIndex
+; CHECK-NEXT:    call void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+; CHECK-NEXT:    [[CMPZ:%.*]] = icmp eq i64 [[TMP5]], 0
 ; CHECK-NEXT:    br i1 [[CMPZ]], label [[WG_LEADER:%.*]], label [[WG_CF:%.*]]
 ; CHECK:       wg_leader:
-; CHECK-NEXT:    store [[STRUCT_BAR]] addrspace(4)* addrspacecast (%struct.bar addrspace(3)* @[[PFWG_SHADOW]] to [[STRUCT_BAR]] addrspace(4)*), [[STRUCT_BAR]] addrspace(4)** [[TMP]], align 8
+; CHECK-NEXT:    store [[STRUCT_BAR]] addrspace(4)* [[ARG]], [[STRUCT_BAR]] addrspace(4)** [[TMP]], align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load [[STRUCT_BAR]] addrspace(4)*, [[STRUCT_BAR]] addrspace(4)** [[TMP]], align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = addrspacecast [[STRUCT_ZOT:%.*]] addrspace(3)* @[[GROUP_SHADOW]] to [[STRUCT_ZOT]] addrspace(4)*
+; CHECK-NEXT:    [[TMP4:%.*]] = addrspacecast %struct.zot* [[ARG1]] to [[STRUCT_ZOT:%.*]] addrspace(4)*
 ; CHECK-NEXT:    store [[STRUCT_ZOT]] addrspace(4)* [[TMP4]], [[STRUCT_ZOT]] addrspace(4)* addrspace(3)* @[[GROUP_SHADOW_PTR]]
 ; CHECK-NEXT:    br label [[WG_CF]]
 ; CHECK:       wg_cf:
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i64, i64 addrspace(1)* @__spirv_BuiltInLocalInvocationIndex
+; CHECK-NEXT:    call void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
 ; CHECK-NEXT:    [[CMPZ2:%.*]] = icmp eq i64 [[TMP4]], 0
 ; CHECK-NEXT:    br i1 [[CMPZ2]], label [[TESTMAT:%.*]], label [[LEADERMAT:%.*]]
 ; CHECK:       TestMat:
diff --git a/llvm/test/SYCLLowerIR/wg_scope_ctor_loop.ll b/llvm/test/SYCLLowerIR/wg_scope_ctor_loop.ll
new file mode 100644
index 0000000000000..84575dc77f27c
--- /dev/null
+++ b/llvm/test/SYCLLowerIR/wg_scope_ctor_loop.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -LowerWGScope -S | FileCheck %s
+
+%struct.snork = type { i32 }
+%struct.eggs = type { i8 }
+%struct.snork.0 = type { %struct.widget, %struct.widget, %struct.widget, %struct.ham }
+%struct.widget = type { %struct.wibble }
+%struct.wibble = type { [3 x i64] }
+%struct.ham = type { %struct.wibble }
+
+@global = internal addrspace(3) global [12 x %struct.snork] zeroinitializer, align 4
+
+; CHECK: @[[WG_NEXT:[a-zA-Z0-9_.]+]] = internal unnamed_addr addrspace(3) global %struct.snork addrspace(4)* undef, align 8
+; CHECK: @[[WG_DONE:[a-zA-Z0-9_.]+]] = internal unnamed_addr addrspace(3) global i1 undef, align 1
+
+define internal spir_func void @spam(%struct.eggs addrspace(4)* %arg, %struct.snork.0* byval(%struct.snork.0) align 8 %arg1) align 2 !work_group_scope !0 {
+; CHECK:       arrayctor.loop:
+; CHECK-NEXT:    [[ARRAYCTOR_CUR:%.*]] = phi [[STRUCT_SNORK:%.*]] addrspace(4)* [ getelementptr inbounds ([12 x %struct.snork], [12 x %struct.snork] addrspace(4)* addrspacecast ([12 x %struct.snork] addrspace(3)* @global to [12 x %struct.snork] addrspace(4)*), i32 0, i32 0), [[WG_CF:%.*]] ], [ [[WG_VAL_ARRAYCTOR_NEXT:%.*]], [[WG_CF2:%.*]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, i64 addrspace(1)* @__spirv_BuiltInLocalInvocationIndex, align 4
+; CHECK-NEXT:    call void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272)
+; CHECK-NEXT:    [[CMPZ3:%.*]] = icmp eq i64 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[CMPZ3]], label [[WG_LEADER1:%.*]], label [[WG_CF2]]
+; CHECK:       wg_leader1:
+; CHECK-NEXT:    call spir_func void @bar(%struct.snork addrspace(4)* [[ARRAYCTOR_CUR]])
+; CHECK-NEXT:    [[ARRAYCTOR_NEXT:%.*]] = getelementptr inbounds [[STRUCT_SNORK]], [[STRUCT_SNORK]] addrspace(4)* [[ARRAYCTOR_CUR]], i64 1
+; CHECK-NEXT:    store [[STRUCT_SNORK]] addrspace(4)* [[ARRAYCTOR_NEXT]], [[STRUCT_SNORK]] addrspace(4)* addrspace(3)* @[[WG_NEXT]], align 8
+; CHECK-NEXT:    [[ARRAYCTOR_DONE:%.*]] = icmp eq [[STRUCT_SNORK]] addrspace(4)* [[ARRAYCTOR_NEXT]], getelementptr inbounds (%struct.snork, [[STRUCT_SNORK]] addrspace(4)* getelementptr inbounds ([12 x %struct.snork], [12 x %struct.snork] addrspace(4)* addrspacecast ([12 x %struct.snork] addrspace(3)* @global to [12 x %struct.snork] addrspace(4)*), i32 0, i32 0), i64 12)
+; CHECK-NEXT:    store i1 [[ARRAYCTOR_DONE]], i1 addrspace(3)* @[[WG_DONE]], align 1
+; CHECK-NEXT:    br label [[WG_CF2]]
+; CHECK:       wg_cf2:
+; CHECK-NEXT:    call void @_Z22__spirv_ControlBarrierjjj(i32 2, i32 2, i32 272) #0
+; CHECK-NEXT:    [[WG_VAL_ARRAYCTOR_DONE:%.*]] = load i1, i1 addrspace(3)* @[[WG_DONE]], align 1
+; CHECK-NEXT:    [[WG_VAL_ARRAYCTOR_NEXT]] = load [[STRUCT_SNORK]] addrspace(4)*, [[STRUCT_SNORK]] addrspace(4)* addrspace(3)* @[[WG_NEXT]], align 8
+; CHECK-NEXT:    br i1 [[WG_VAL_ARRAYCTOR_DONE]], label [[ARRAYCTOR_CONT:%.*]], label [[ARRAYCTOR_LOOP:%.*]]
+; CHECK:       arrayctor.cont:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %tmp = alloca %struct.eggs addrspace(4)*, align 8
+  store %struct.eggs addrspace(4)* %arg, %struct.eggs addrspace(4)** %tmp, align 8
+  %tmp2 = load %struct.eggs addrspace(4)*, %struct.eggs addrspace(4)** %tmp, align 8
+  br label %arrayctor.loop
+
+arrayctor.loop:                                              ; preds = %arrayctor.loop, %entry
+  %arrayctor.cur = phi %struct.snork addrspace(4)* [ getelementptr inbounds ([12 x %struct.snork], [12 x %struct.snork] addrspace(4)* addrspacecast ([12 x %struct.snork] addrspace(3)* @global to [12 x %struct.snork] addrspace(4)*), i32 0, i32 0), %entry ], [ %arrayctor.next, %arrayctor.loop ]
+  call spir_func void @bar(%struct.snork addrspace(4)* %arrayctor.cur)
+  %arrayctor.next = getelementptr inbounds %struct.snork, %struct.snork addrspace(4)* %arrayctor.cur, i64 1
+  %arrayctor.done = icmp eq %struct.snork addrspace(4)* %arrayctor.next, getelementptr inbounds (%struct.snork, %struct.snork addrspace(4)* getelementptr inbounds ([12 x %struct.snork], [12 x %struct.snork] addrspace(4)* addrspacecast ([12 x %struct.snork] addrspace(3)* @global to [12 x %struct.snork] addrspace(4)*), i32 0, i32 0), i64 12)
+  br i1 %arrayctor.done, label %arrayctor.cont, label %arrayctor.loop
+
+arrayctor.cont:                                   ; preds = %arrayctor.loop
+  ret void
+}
+
+define linkonce_odr dso_local spir_func void @bar(%struct.snork addrspace(4)* %arg) unnamed_addr align 2 {
+bb:
+  %tmp = alloca %struct.snork addrspace(4)*, align 8
+  store %struct.snork addrspace(4)* %arg, %struct.snork addrspace(4)** %tmp, align 8
+  %tmp1 = load %struct.snork addrspace(4)*, %struct.snork addrspace(4)** %tmp, align 8
+  %tmp2 = getelementptr inbounds %struct.snork, %struct.snork addrspace(4)* %tmp1, i32 0, i32 0
+  store i32 0, i32 addrspace(4)* %tmp2, align 4
+  ret void
+}
+
+!0 = !{}
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-02-01-ReturnAttrs.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-02-01-ReturnAttrs.ll
index b943af621940a..58135b8d22707 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-02-01-ReturnAttrs.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-02-01-ReturnAttrs.ll
@@ -1,16 +1,18 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 define internal i32 @deref(i32* %x) nounwind {
+; IS__TUNIT_OPM: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
 ; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@deref
 ; IS__TUNIT_OPM-SAME: (i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[X:%.*]])
 ; IS__TUNIT_OPM-NEXT:  entry:
 ; IS__TUNIT_OPM-NEXT:    [[TMP2:%.*]] = load i32, i32* [[X]], align 4
 ; IS__TUNIT_OPM-NEXT:    ret i32 [[TMP2]]
 ;
+; IS__TUNIT_NPM: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@deref
 ; IS__TUNIT_NPM-SAME: (i32 [[TMP0:%.*]])
 ; IS__TUNIT_NPM-NEXT:  entry:
@@ -19,6 +21,7 @@ define internal i32 @deref(i32* %x) nounwind {
 ; IS__TUNIT_NPM-NEXT:    [[TMP2:%.*]] = load i32, i32* [[X_PRIV]], align 4
 ; IS__TUNIT_NPM-NEXT:    ret i32 [[TMP2]]
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@deref
 ; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[X:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -31,14 +34,16 @@ entry:
 }
 
 define i32 @f(i32 %x) {
-; NOT_TUNIT_NPM-LABEL: define {{[^@]+}}@f
-; NOT_TUNIT_NPM-SAME: (i32 [[X:%.*]])
-; NOT_TUNIT_NPM-NEXT:  entry:
-; NOT_TUNIT_NPM-NEXT:    [[X_ADDR:%.*]] = alloca i32, align 4
-; NOT_TUNIT_NPM-NEXT:    store i32 [[X]], i32* [[X_ADDR]], align 4
-; NOT_TUNIT_NPM-NEXT:    [[TMP1:%.*]] = call i32 @deref(i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[X_ADDR]])
-; NOT_TUNIT_NPM-NEXT:    ret i32 [[TMP1]]
+; IS__TUNIT_OPM: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@f
+; IS__TUNIT_OPM-SAME: (i32 [[X:%.*]])
+; IS__TUNIT_OPM-NEXT:  entry:
+; IS__TUNIT_OPM-NEXT:    [[X_ADDR:%.*]] = alloca i32, align 4
+; IS__TUNIT_OPM-NEXT:    store i32 [[X]], i32* [[X_ADDR]], align 4
+; IS__TUNIT_OPM-NEXT:    [[TMP1:%.*]] = call i32 @deref(i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[X_ADDR]])
+; IS__TUNIT_OPM-NEXT:    ret i32 [[TMP1]]
 ;
+; IS__TUNIT_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@f
 ; IS__TUNIT_NPM-SAME: (i32 [[X:%.*]])
 ; IS__TUNIT_NPM-NEXT:  entry:
@@ -48,6 +53,15 @@ define i32 @f(i32 %x) {
 ; IS__TUNIT_NPM-NEXT:    [[TMP1:%.*]] = call i32 @deref(i32 [[TMP0]])
 ; IS__TUNIT_NPM-NEXT:    ret i32 [[TMP1]]
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@f
+; IS__CGSCC____-SAME: (i32 [[X:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[X_ADDR:%.*]] = alloca i32, align 4
+; IS__CGSCC____-NEXT:    store i32 [[X]], i32* [[X_ADDR]], align 4
+; IS__CGSCC____-NEXT:    [[TMP1:%.*]] = call i32 @deref(i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[X_ADDR]])
+; IS__CGSCC____-NEXT:    ret i32 [[TMP1]]
+;
 entry:
   %x_addr = alloca i32
   store i32 %x, i32* %x_addr, align 4
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-07-02-array-indexing.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-07-02-array-indexing.ll
index e357272536b9d..d0817a05a5fdb 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-07-02-array-indexing.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-07-02-array-indexing.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -9,17 +9,31 @@
 ; because there is a load of %A in the entry block
 define internal i32 @callee(i1 %C, i32* %A) {
 ;
-; CHECK-LABEL: define {{[^@]+}}@callee
-; CHECK-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[A_0:%.*]] = load i32, i32* [[A]], align 4
-; CHECK-NEXT:    br label [[F:%.*]]
-; CHECK:       T:
-; CHECK-NEXT:    unreachable
-; CHECK:       F:
-; CHECK-NEXT:    [[A_2:%.*]] = getelementptr i32, i32* [[A]], i32 2
-; CHECK-NEXT:    [[R:%.*]] = load i32, i32* [[A_2]], align 4
-; CHECK-NEXT:    ret i32 [[R]]
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@callee
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[A_0:%.*]] = load i32, i32* [[A]], align 4
+; IS__TUNIT____-NEXT:    br label [[F:%.*]]
+; IS__TUNIT____:       T:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       F:
+; IS__TUNIT____-NEXT:    [[A_2:%.*]] = getelementptr i32, i32* [[A]], i32 2
+; IS__TUNIT____-NEXT:    [[R:%.*]] = load i32, i32* [[A_2]], align 4
+; IS__TUNIT____-NEXT:    ret i32 [[R]]
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@callee
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[A_0:%.*]] = load i32, i32* [[A]], align 4
+; IS__CGSCC____-NEXT:    br label [[F:%.*]]
+; IS__CGSCC____:       T:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       F:
+; IS__CGSCC____-NEXT:    [[A_2:%.*]] = getelementptr i32, i32* [[A]], i32 2
+; IS__CGSCC____-NEXT:    [[R:%.*]] = load i32, i32* [[A_2]], align 4
+; IS__CGSCC____-NEXT:    ret i32 [[R]]
 ;
 entry:
   ; Unconditonally load the element at %A
@@ -37,11 +51,13 @@ F:
 }
 
 define i32 @foo(i32* %A) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@foo
 ; IS__TUNIT____-SAME: (i32* nocapture nofree readonly [[A:%.*]])
 ; IS__TUNIT____-NEXT:    [[X:%.*]] = call i32 @callee(i32* nocapture nofree readonly align 4 [[A]])
 ; IS__TUNIT____-NEXT:    ret i32 [[X]]
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@foo
 ; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A:%.*]])
 ; IS__CGSCC____-NEXT:    [[X:%.*]] = call i32 @callee(i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A]])
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-07-CGUpdate.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-07-CGUpdate.ll
index fb80ad91c716a..68fc3d62e5d2e 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-07-CGUpdate.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-07-CGUpdate.ll
@@ -1,10 +1,11 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 define internal fastcc i32 @hash(i32* %ts, i32 %mod) nounwind {
+; IS__CGSCC____: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@hash()
 ; IS__CGSCC____-NEXT:  entry:
 ; IS__CGSCC____-NEXT:    unreachable
@@ -14,10 +15,17 @@ entry:
 }
 
 define void @encode(i32* %m, i32* %ts, i32* %new) nounwind {
-; CHECK-LABEL: define {{[^@]+}}@encode
-; CHECK-SAME: (i32* nocapture nofree readnone [[M:%.*]], i32* nocapture nofree readnone [[TS:%.*]], i32* nocapture nofree readnone [[NEW:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree noreturn nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@encode
+; IS__TUNIT____-SAME: (i32* nocapture nofree readnone [[M:%.*]], i32* nocapture nofree readnone [[TS:%.*]], i32* nocapture nofree readnone [[NEW:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@encode
+; IS__CGSCC____-SAME: (i32* nocapture nofree readnone [[M:%.*]], i32* nocapture nofree readnone [[TS:%.*]], i32* nocapture nofree readnone [[NEW:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    unreachable
 ;
 entry:
   %0 = call fastcc i32 @hash( i32* %ts, i32 0 ) nounwind		; <i32> [#uses=0]
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-08-CGUpdateSelfEdge.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-08-CGUpdateSelfEdge.ll
index 2d0c4d66aa43c..5f56e670aab91 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-08-CGUpdateSelfEdge.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-09-08-CGUpdateSelfEdge.ll
@@ -1,10 +1,11 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 define internal fastcc i32 @term_SharingList(i32* %Term, i32* %List) nounwind {
+; IS__TUNIT____: Function Attrs: nounwind
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@term_SharingList
 ; IS__TUNIT____-SAME: (i32* [[TERM:%.*]], i32* [[LIST:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
@@ -15,6 +16,7 @@ define internal fastcc i32 @term_SharingList(i32* %Term, i32* %List) nounwind {
 ; IS__TUNIT____:       bb5:
 ; IS__TUNIT____-NEXT:    ret i32 0
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@term_SharingList()
 ; IS__CGSCC____-NEXT:  entry:
 ; IS__CGSCC____-NEXT:    br i1 false, label [[BB:%.*]], label [[BB5:%.*]]
@@ -35,14 +37,25 @@ bb5:		; preds = %entry
 }
 
 define i32 @term_Sharing(i32* %Term) nounwind {
-; CHECK-LABEL: define {{[^@]+}}@term_Sharing
-; CHECK-SAME: (i32* nocapture nofree readnone [[TERM:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 false, label [[BB_I:%.*]], label [[BB14:%.*]]
-; CHECK:       bb.i:
-; CHECK-NEXT:    unreachable
-; CHECK:       bb14:
-; CHECK-NEXT:    ret i32 0
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@term_Sharing
+; IS__TUNIT____-SAME: (i32* nocapture nofree readnone [[TERM:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    br i1 false, label [[BB_I:%.*]], label [[BB14:%.*]]
+; IS__TUNIT____:       bb.i:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       bb14:
+; IS__TUNIT____-NEXT:    ret i32 0
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@term_Sharing
+; IS__CGSCC____-SAME: (i32* nocapture nofree readnone [[TERM:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    br i1 false, label [[BB_I:%.*]], label [[BB14:%.*]]
+; IS__CGSCC____:       bb.i:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       bb14:
+; IS__CGSCC____-NEXT:    ret i32 0
 ;
 entry:
   br i1 false, label %bb.i, label %bb14
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/aggregate-promote.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/aggregate-promote.ll
index 8196d18c0e68b..7728b12af97a9 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/aggregate-promote.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/aggregate-promote.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -8,14 +8,25 @@
 @G = constant %T { i32 0, i32 0, i32 17, i32 25 }
 
 define internal i32 @test(%T* %p) {
-; CHECK-LABEL: define {{[^@]+}}@test()
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[A_GEP:%.*]] = getelementptr [[T:%.*]], %T* @G, i64 0, i32 3
-; CHECK-NEXT:    [[B_GEP:%.*]] = getelementptr [[T]], %T* @G, i64 0, i32 2
-; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[A_GEP]], align 4
-; CHECK-NEXT:    [[B:%.*]] = load i32, i32* [[B_GEP]], align 4
-; CHECK-NEXT:    [[V:%.*]] = add i32 [[A]], [[B]]
-; CHECK-NEXT:    ret i32 [[V]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test()
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[A_GEP:%.*]] = getelementptr [[T:%.*]], %T* @G, i64 0, i32 3
+; IS__TUNIT____-NEXT:    [[B_GEP:%.*]] = getelementptr [[T]], %T* @G, i64 0, i32 2
+; IS__TUNIT____-NEXT:    [[A:%.*]] = load i32, i32* [[A_GEP]], align 4
+; IS__TUNIT____-NEXT:    [[B:%.*]] = load i32, i32* [[B_GEP]], align 4
+; IS__TUNIT____-NEXT:    [[V:%.*]] = add i32 [[A]], [[B]]
+; IS__TUNIT____-NEXT:    ret i32 [[V]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test()
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[A_GEP:%.*]] = getelementptr [[T:%.*]], %T* @G, i64 0, i32 3
+; IS__CGSCC____-NEXT:    [[B_GEP:%.*]] = getelementptr [[T]], %T* @G, i64 0, i32 2
+; IS__CGSCC____-NEXT:    [[A:%.*]] = load i32, i32* [[A_GEP]], align 4
+; IS__CGSCC____-NEXT:    [[B:%.*]] = load i32, i32* [[B_GEP]], align 4
+; IS__CGSCC____-NEXT:    [[V:%.*]] = add i32 [[A]], [[B]]
+; IS__CGSCC____-NEXT:    ret i32 [[V]]
 ;
 entry:
   %a.gep = getelementptr %T, %T* %p, i64 0, i32 3
@@ -27,10 +38,17 @@ entry:
 }
 
 define i32 @caller() {
-; CHECK-LABEL: define {{[^@]+}}@caller()
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[V:%.*]] = call i32 @test()
-; CHECK-NEXT:    ret i32 [[V]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@caller()
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[V:%.*]] = call i32 @test()
+; IS__TUNIT____-NEXT:    ret i32 [[V]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@caller()
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[V:%.*]] = call i32 @test()
+; IS__CGSCC____-NEXT:    ret i32 [[V]]
 ;
 entry:
   %v = call i32 @test(%T* @G)
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/alignment.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/alignment.ll
index 0367a5e42ec16..98f3a5943e37b 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/alignment.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/alignment.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=11 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=11 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -55,6 +55,7 @@ declare void @z(i32)
 ; Test2
 ; Different alignemnt privatizable arguments
 define internal i32 @test(i32* %X, i64* %Y) {
+; IS__TUNIT_OPM: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
 ; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@test
 ; IS__TUNIT_OPM-SAME: (i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[X:%.*]], i64* noalias nocapture nofree nonnull readonly align 8 dereferenceable(8) [[Y:%.*]])
 ; IS__TUNIT_OPM-NEXT:    [[A:%.*]] = load i32, i32* [[X]], align 4
@@ -68,6 +69,7 @@ define internal i32 @test(i32* %X, i64* %Y) {
 ; IS__TUNIT_OPM:       Return2:
 ; IS__TUNIT_OPM-NEXT:    ret i32 [[A]]
 ;
+; IS__TUNIT_NPM: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@test
 ; IS__TUNIT_NPM-SAME: (i32 [[TMP0:%.*]], i64 [[TMP1:%.*]])
 ; IS__TUNIT_NPM-NEXT:    [[Y_PRIV:%.*]] = alloca i64, align 8
@@ -85,6 +87,7 @@ define internal i32 @test(i32* %X, i64* %Y) {
 ; IS__TUNIT_NPM:       Return2:
 ; IS__TUNIT_NPM-NEXT:    ret i32 [[A]]
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@test
 ; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[X:%.*]], i64* nocapture nofree nonnull readonly align 8 dereferenceable(8) [[Y:%.*]])
 ; IS__CGSCC____-NEXT:    [[A:%.*]] = load i32, i32* [[X]], align 4
@@ -111,6 +114,7 @@ Return2:
 }
 
 define internal i32 @caller(i32* %A) {
+; IS__TUNIT_OPM: Function Attrs: argmemonly nofree nosync nounwind willreturn
 ; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@caller
 ; IS__TUNIT_OPM-SAME: (i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A:%.*]])
 ; IS__TUNIT_OPM-NEXT:    [[B:%.*]] = alloca i64, align 8
@@ -118,6 +122,7 @@ define internal i32 @caller(i32* %A) {
 ; IS__TUNIT_OPM-NEXT:    [[C:%.*]] = call i32 @test(i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A]], i64* noalias nocapture nofree nonnull readonly align 8 dereferenceable(8) [[B]])
 ; IS__TUNIT_OPM-NEXT:    ret i32 [[C]]
 ;
+; IS__TUNIT_NPM: Function Attrs: argmemonly nofree nosync nounwind willreturn
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@caller
 ; IS__TUNIT_NPM-SAME: (i32 [[TMP0:%.*]])
 ; IS__TUNIT_NPM-NEXT:    [[A_PRIV:%.*]] = alloca i32, align 4
@@ -129,6 +134,7 @@ define internal i32 @caller(i32* %A) {
 ; IS__TUNIT_NPM-NEXT:    [[C:%.*]] = call i32 @test(i32 [[TMP2]], i64 [[TMP3]])
 ; IS__TUNIT_NPM-NEXT:    ret i32 [[C]]
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@caller
 ; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A:%.*]])
 ; IS__CGSCC____-NEXT:    [[B:%.*]] = alloca i64, align 8
@@ -143,18 +149,27 @@ define internal i32 @caller(i32* %A) {
 }
 
 define i32 @callercaller() {
-; NOT_TUNIT_NPM-LABEL: define {{[^@]+}}@callercaller()
-; NOT_TUNIT_NPM-NEXT:    [[B:%.*]] = alloca i32, align 4
-; NOT_TUNIT_NPM-NEXT:    store i32 2, i32* [[B]], align 4
-; NOT_TUNIT_NPM-NEXT:    [[X:%.*]] = call i32 @caller(i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B]])
-; NOT_TUNIT_NPM-NEXT:    ret i32 [[X]]
+; IS__TUNIT_OPM: Function Attrs: nofree nosync nounwind readnone
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@callercaller()
+; IS__TUNIT_OPM-NEXT:    [[B:%.*]] = alloca i32, align 4
+; IS__TUNIT_OPM-NEXT:    store i32 2, i32* [[B]], align 4
+; IS__TUNIT_OPM-NEXT:    [[X:%.*]] = call i32 @caller(i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B]])
+; IS__TUNIT_OPM-NEXT:    ret i32 [[X]]
 ;
+; IS__TUNIT_NPM: Function Attrs: nofree nosync nounwind readnone
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@callercaller()
 ; IS__TUNIT_NPM-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; IS__TUNIT_NPM-NEXT:    store i32 2, i32* [[B]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[TMP1:%.*]] = load i32, i32* [[B]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[X:%.*]] = call i32 @caller(i32 [[TMP1]])
 ; IS__TUNIT_NPM-NEXT:    ret i32 [[X]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@callercaller()
+; IS__CGSCC____-NEXT:    [[B:%.*]] = alloca i32, align 4
+; IS__CGSCC____-NEXT:    store i32 2, i32* [[B]], align 4
+; IS__CGSCC____-NEXT:    [[X:%.*]] = call i32 @caller(i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B]])
+; IS__CGSCC____-NEXT:    ret i32 [[X]]
 ;
   %B = alloca i32
   store i32 2, i32* %B
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/attrs.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/attrs.ll
index 524b26c2b128b..8ecc12b171af5 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/attrs.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/attrs.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -9,6 +9,7 @@
 ; Don't drop 'byval' on %X here.
 define internal i32 @f(%struct.ss* byval %b, i32* byval %X, i32 %i) nounwind {
 ;
+; IS__TUNIT_OPM: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@f
 ; IS__TUNIT_OPM-SAME: (%struct.ss* noalias nocapture nofree nonnull byval align 8 dereferenceable(12) [[B:%.*]], i32* noalias nocapture nofree nonnull byval align 4 dereferenceable(4) [[X:%.*]], i32 [[I:%.*]])
 ; IS__TUNIT_OPM-NEXT:  entry:
@@ -21,6 +22,7 @@ define internal i32 @f(%struct.ss* byval %b, i32* byval %X, i32 %i) nounwind {
 ; IS__TUNIT_OPM-NEXT:    [[A:%.*]] = add i32 [[L]], [[TMP2]]
 ; IS__TUNIT_OPM-NEXT:    ret i32 [[A]]
 ;
+; IS__TUNIT_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@f
 ; IS__TUNIT_NPM-SAME: (i32 [[TMP0:%.*]], i64 [[TMP1:%.*]], i32 [[TMP2:%.*]], i32 [[I:%.*]])
 ; IS__TUNIT_NPM-NEXT:  entry:
@@ -40,6 +42,7 @@ define internal i32 @f(%struct.ss* byval %b, i32* byval %X, i32 %i) nounwind {
 ; IS__TUNIT_NPM-NEXT:    [[A:%.*]] = add i32 [[L]], [[TMP2]]
 ; IS__TUNIT_NPM-NEXT:    ret i32 [[A]]
 ;
+; IS__CGSCC_OPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@f
 ; IS__CGSCC_OPM-SAME: (%struct.ss* noalias nocapture nofree nonnull byval align 8 dereferenceable(12) [[B:%.*]], i32* noalias nocapture nofree nonnull byval align 4 dereferenceable(4) [[X:%.*]])
 ; IS__CGSCC_OPM-NEXT:  entry:
@@ -52,6 +55,7 @@ define internal i32 @f(%struct.ss* byval %b, i32* byval %X, i32 %i) nounwind {
 ; IS__CGSCC_OPM-NEXT:    [[A:%.*]] = add i32 [[L]], [[TMP2]]
 ; IS__CGSCC_OPM-NEXT:    ret i32 [[A]]
 ;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@f
 ; IS__CGSCC_NPM-SAME: (i32 [[TMP0:%.*]], i64 [[TMP1:%.*]], i32 [[TMP2:%.*]])
 ; IS__CGSCC_NPM-NEXT:  entry:
@@ -87,6 +91,7 @@ entry:
 ; Also make sure we don't drop the call zeroext attribute.
 define i32 @test(i32* %X) {
 ;
+; IS__TUNIT_OPM: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@test
 ; IS__TUNIT_OPM-SAME: (i32* nocapture nofree readonly align 4 [[X:%.*]])
 ; IS__TUNIT_OPM-NEXT:  entry:
@@ -98,6 +103,7 @@ define i32 @test(i32* %X) {
 ; IS__TUNIT_OPM-NEXT:    [[C:%.*]] = call i32 @f(%struct.ss* noalias nocapture nofree nonnull readonly byval align 8 dereferenceable(12) [[S]], i32* nocapture nofree readonly byval align 4 [[X]], i32 zeroext 0)
 ; IS__TUNIT_OPM-NEXT:    ret i32 [[C]]
 ;
+; IS__TUNIT_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@test
 ; IS__TUNIT_NPM-SAME: (i32* nocapture nofree readonly align 4 [[X:%.*]])
 ; IS__TUNIT_NPM-NEXT:  entry:
@@ -114,6 +120,7 @@ define i32 @test(i32* %X) {
 ; IS__TUNIT_NPM-NEXT:    [[C:%.*]] = call i32 @f(i32 [[TMP0]], i64 [[TMP1]], i32 [[TMP2]], i32 zeroext 0)
 ; IS__TUNIT_NPM-NEXT:    ret i32 [[C]]
 ;
+; IS__CGSCC_OPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@test
 ; IS__CGSCC_OPM-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(4) [[X:%.*]])
 ; IS__CGSCC_OPM-NEXT:  entry:
@@ -125,6 +132,7 @@ define i32 @test(i32* %X) {
 ; IS__CGSCC_OPM-NEXT:    [[C:%.*]] = call i32 @f(%struct.ss* noalias nocapture nofree nonnull readnone byval align 8 dereferenceable(12) [[S]], i32* noalias nocapture nofree nonnull readnone byval align 4 dereferenceable(4) [[X]])
 ; IS__CGSCC_OPM-NEXT:    ret i32 [[C]]
 ;
+; IS__CGSCC_NPM: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn
 ; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@test
 ; IS__CGSCC_NPM-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[X:%.*]])
 ; IS__CGSCC_NPM-NEXT:  entry:
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/basictest.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/basictest.ll
index 3519c96a731a4..13a4187830c1a 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/basictest.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/basictest.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=8 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=8 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -6,6 +6,7 @@
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
 define internal i32 @test(i32* %X, i32* %Y) {
+; IS__TUNIT_OPM: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
 ; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@test
 ; IS__TUNIT_OPM-SAME: (i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[X:%.*]], i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[Y:%.*]])
 ; IS__TUNIT_OPM-NEXT:    [[A:%.*]] = load i32, i32* [[X]], align 4
@@ -13,6 +14,7 @@ define internal i32 @test(i32* %X, i32* %Y) {
 ; IS__TUNIT_OPM-NEXT:    [[C:%.*]] = add i32 [[A]], [[B]]
 ; IS__TUNIT_OPM-NEXT:    ret i32 [[C]]
 ;
+; IS__TUNIT_NPM: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@test
 ; IS__TUNIT_NPM-SAME: (i32 [[TMP0:%.*]], i32 [[TMP1:%.*]])
 ; IS__TUNIT_NPM-NEXT:    [[Y_PRIV:%.*]] = alloca i32, align 4
@@ -24,6 +26,7 @@ define internal i32 @test(i32* %X, i32* %Y) {
 ; IS__TUNIT_NPM-NEXT:    [[C:%.*]] = add i32 [[A]], [[B]]
 ; IS__TUNIT_NPM-NEXT:    ret i32 [[C]]
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@test
 ; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[X:%.*]], i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[Y:%.*]])
 ; IS__CGSCC____-NEXT:    [[A:%.*]] = load i32, i32* [[X]], align 4
@@ -38,6 +41,7 @@ define internal i32 @test(i32* %X, i32* %Y) {
 }
 
 define internal i32 @caller(i32* %B) {
+; IS__TUNIT_OPM: Function Attrs: argmemonly nofree nosync nounwind willreturn
 ; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@caller
 ; IS__TUNIT_OPM-SAME: (i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B:%.*]])
 ; IS__TUNIT_OPM-NEXT:    [[A:%.*]] = alloca i32, align 4
@@ -45,6 +49,7 @@ define internal i32 @caller(i32* %B) {
 ; IS__TUNIT_OPM-NEXT:    [[C:%.*]] = call i32 @test(i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A]], i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B]])
 ; IS__TUNIT_OPM-NEXT:    ret i32 [[C]]
 ;
+; IS__TUNIT_NPM: Function Attrs: argmemonly nofree nosync nounwind willreturn
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@caller
 ; IS__TUNIT_NPM-SAME: (i32 [[TMP0:%.*]])
 ; IS__TUNIT_NPM-NEXT:    [[B_PRIV:%.*]] = alloca i32, align 4
@@ -56,6 +61,7 @@ define internal i32 @caller(i32* %B) {
 ; IS__TUNIT_NPM-NEXT:    [[C:%.*]] = call i32 @test(i32 [[TMP2]], i32 [[TMP3]])
 ; IS__TUNIT_NPM-NEXT:    ret i32 [[C]]
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@caller
 ; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B:%.*]])
 ; IS__CGSCC____-NEXT:    [[A:%.*]] = alloca i32, align 4
@@ -70,18 +76,27 @@ define internal i32 @caller(i32* %B) {
 }
 
 define i32 @callercaller() {
-; NOT_TUNIT_NPM-LABEL: define {{[^@]+}}@callercaller()
-; NOT_TUNIT_NPM-NEXT:    [[B:%.*]] = alloca i32, align 4
-; NOT_TUNIT_NPM-NEXT:    store i32 2, i32* [[B]], align 4
-; NOT_TUNIT_NPM-NEXT:    [[X:%.*]] = call i32 @caller(i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B]])
-; NOT_TUNIT_NPM-NEXT:    ret i32 [[X]]
+; IS__TUNIT_OPM: Function Attrs: nofree nosync nounwind readnone
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@callercaller()
+; IS__TUNIT_OPM-NEXT:    [[B:%.*]] = alloca i32, align 4
+; IS__TUNIT_OPM-NEXT:    store i32 2, i32* [[B]], align 4
+; IS__TUNIT_OPM-NEXT:    [[X:%.*]] = call i32 @caller(i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B]])
+; IS__TUNIT_OPM-NEXT:    ret i32 [[X]]
 ;
+; IS__TUNIT_NPM: Function Attrs: nofree nosync nounwind readnone
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@callercaller()
 ; IS__TUNIT_NPM-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; IS__TUNIT_NPM-NEXT:    store i32 2, i32* [[B]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[TMP1:%.*]] = load i32, i32* [[B]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[X:%.*]] = call i32 @caller(i32 [[TMP1]])
 ; IS__TUNIT_NPM-NEXT:    ret i32 [[X]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@callercaller()
+; IS__CGSCC____-NEXT:    [[B:%.*]] = alloca i32, align 4
+; IS__CGSCC____-NEXT:    store i32 2, i32* [[B]], align 4
+; IS__CGSCC____-NEXT:    [[X:%.*]] = call i32 @caller(i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B]])
+; IS__CGSCC____-NEXT:    ret i32 [[X]]
 ;
   %B = alloca i32
   store i32 2, i32* %B
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/byval-2.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/byval-2.ll
index ccfbb9463f900..bd1dfb5a11b7d 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/byval-2.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/byval-2.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -7,6 +7,7 @@
 %struct.ss = type { i32, i64 }
 
 define internal void @f(%struct.ss* byval  %b, i32* byval %X) nounwind  {
+; IS__CGSCC_OPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@f
 ; IS__CGSCC_OPM-SAME: (%struct.ss* noalias nocapture nofree nonnull byval align 8 dereferenceable(12) [[B:%.*]], i32* noalias nocapture nofree nonnull writeonly byval align 4 dereferenceable(4) [[X:%.*]])
 ; IS__CGSCC_OPM-NEXT:  entry:
@@ -17,6 +18,7 @@ define internal void @f(%struct.ss* byval  %b, i32* byval %X) nounwind  {
 ; IS__CGSCC_OPM-NEXT:    store i32 0, i32* [[X]], align 4
 ; IS__CGSCC_OPM-NEXT:    ret void
 ;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@f
 ; IS__CGSCC_NPM-SAME: (i32 [[TMP0:%.*]], i64 [[TMP1:%.*]], i32 [[TMP2:%.*]])
 ; IS__CGSCC_NPM-NEXT:  entry:
@@ -46,6 +48,7 @@ entry:
 
 define i32 @test(i32* %X) {
 ;
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@test
 ; IS__TUNIT____-SAME: (i32* nocapture nofree readonly align 4 [[X:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
@@ -56,6 +59,7 @@ define i32 @test(i32* %X) {
 ; IS__TUNIT____-NEXT:    store i64 2, i64* [[TMP4]], align 4
 ; IS__TUNIT____-NEXT:    ret i32 0
 ;
+; IS__CGSCC_OPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@test
 ; IS__CGSCC_OPM-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(4) [[X:%.*]])
 ; IS__CGSCC_OPM-NEXT:  entry:
@@ -66,6 +70,7 @@ define i32 @test(i32* %X) {
 ; IS__CGSCC_OPM-NEXT:    store i64 2, i64* [[TMP4]], align 4
 ; IS__CGSCC_OPM-NEXT:    ret i32 0
 ;
+; IS__CGSCC_NPM: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn
 ; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@test
 ; IS__CGSCC_NPM-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[X:%.*]])
 ; IS__CGSCC_NPM-NEXT:  entry:
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/byval.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/byval.ll
index a74151b777a5a..f05d8883be5b9 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/byval.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/byval.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -9,28 +9,55 @@ target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:1
 %struct.ss = type { i32, i64 }
 
 define internal i32 @f(%struct.ss* byval  %b) nounwind  {
-; IS________OPM-LABEL: define {{[^@]+}}@f
-; IS________OPM-SAME: (%struct.ss* noalias nocapture nofree nonnull byval align 8 dereferenceable(12) [[B:%.*]])
-; IS________OPM-NEXT:  entry:
-; IS________OPM-NEXT:    [[TMP:%.*]] = getelementptr [[STRUCT_SS:%.*]], %struct.ss* [[B]], i32 0, i32 0
-; IS________OPM-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP]], align 8
-; IS________OPM-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], 1
-; IS________OPM-NEXT:    store i32 [[TMP2]], i32* [[TMP]], align 8
-; IS________OPM-NEXT:    ret i32 [[TMP1]]
+; IS__TUNIT_OPM: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@f
+; IS__TUNIT_OPM-SAME: (%struct.ss* noalias nocapture nofree nonnull byval align 8 dereferenceable(12) [[B:%.*]])
+; IS__TUNIT_OPM-NEXT:  entry:
+; IS__TUNIT_OPM-NEXT:    [[TMP:%.*]] = getelementptr [[STRUCT_SS:%.*]], %struct.ss* [[B]], i32 0, i32 0
+; IS__TUNIT_OPM-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP]], align 8
+; IS__TUNIT_OPM-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], 1
+; IS__TUNIT_OPM-NEXT:    store i32 [[TMP2]], i32* [[TMP]], align 8
+; IS__TUNIT_OPM-NEXT:    ret i32 [[TMP1]]
+;
+; IS__TUNIT_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@f
+; IS__TUNIT_NPM-SAME: (i32 [[TMP0:%.*]], i64 [[TMP1:%.*]])
+; IS__TUNIT_NPM-NEXT:  entry:
+; IS__TUNIT_NPM-NEXT:    [[B_PRIV:%.*]] = alloca [[STRUCT_SS:%.*]], align 4
+; IS__TUNIT_NPM-NEXT:    [[B_PRIV_CAST:%.*]] = bitcast %struct.ss* [[B_PRIV]] to i32*
+; IS__TUNIT_NPM-NEXT:    store i32 [[TMP0]], i32* [[B_PRIV_CAST]], align 4
+; IS__TUNIT_NPM-NEXT:    [[B_PRIV_0_1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[B_PRIV]], i32 0, i32 1
+; IS__TUNIT_NPM-NEXT:    store i64 [[TMP1]], i64* [[B_PRIV_0_1]], align 4
+; IS__TUNIT_NPM-NEXT:    [[TMP:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[B_PRIV]], i32 0, i32 0
+; IS__TUNIT_NPM-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP]], align 8
+; IS__TUNIT_NPM-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], 1
+; IS__TUNIT_NPM-NEXT:    store i32 [[TMP2]], i32* [[TMP]], align 8
+; IS__TUNIT_NPM-NEXT:    ret i32 [[TMP1]]
 ;
-; IS________NPM-LABEL: define {{[^@]+}}@f
-; IS________NPM-SAME: (i32 [[TMP0:%.*]], i64 [[TMP1:%.*]])
-; IS________NPM-NEXT:  entry:
-; IS________NPM-NEXT:    [[B_PRIV:%.*]] = alloca [[STRUCT_SS:%.*]], align 4
-; IS________NPM-NEXT:    [[B_PRIV_CAST:%.*]] = bitcast %struct.ss* [[B_PRIV]] to i32*
-; IS________NPM-NEXT:    store i32 [[TMP0]], i32* [[B_PRIV_CAST]], align 4
-; IS________NPM-NEXT:    [[B_PRIV_0_1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[B_PRIV]], i32 0, i32 1
-; IS________NPM-NEXT:    store i64 [[TMP1]], i64* [[B_PRIV_0_1]], align 4
-; IS________NPM-NEXT:    [[TMP:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[B_PRIV]], i32 0, i32 0
-; IS________NPM-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP]], align 8
-; IS________NPM-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], 1
-; IS________NPM-NEXT:    store i32 [[TMP2]], i32* [[TMP]], align 8
-; IS________NPM-NEXT:    ret i32 [[TMP1]]
+; IS__CGSCC_OPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@f
+; IS__CGSCC_OPM-SAME: (%struct.ss* noalias nocapture nofree nonnull byval align 8 dereferenceable(12) [[B:%.*]])
+; IS__CGSCC_OPM-NEXT:  entry:
+; IS__CGSCC_OPM-NEXT:    [[TMP:%.*]] = getelementptr [[STRUCT_SS:%.*]], %struct.ss* [[B]], i32 0, i32 0
+; IS__CGSCC_OPM-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP]], align 8
+; IS__CGSCC_OPM-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], 1
+; IS__CGSCC_OPM-NEXT:    store i32 [[TMP2]], i32* [[TMP]], align 8
+; IS__CGSCC_OPM-NEXT:    ret i32 [[TMP1]]
+;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@f
+; IS__CGSCC_NPM-SAME: (i32 [[TMP0:%.*]], i64 [[TMP1:%.*]])
+; IS__CGSCC_NPM-NEXT:  entry:
+; IS__CGSCC_NPM-NEXT:    [[B_PRIV:%.*]] = alloca [[STRUCT_SS:%.*]], align 4
+; IS__CGSCC_NPM-NEXT:    [[B_PRIV_CAST:%.*]] = bitcast %struct.ss* [[B_PRIV]] to i32*
+; IS__CGSCC_NPM-NEXT:    store i32 [[TMP0]], i32* [[B_PRIV_CAST]], align 4
+; IS__CGSCC_NPM-NEXT:    [[B_PRIV_0_1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[B_PRIV]], i32 0, i32 1
+; IS__CGSCC_NPM-NEXT:    store i64 [[TMP1]], i64* [[B_PRIV_0_1]], align 4
+; IS__CGSCC_NPM-NEXT:    [[TMP:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[B_PRIV]], i32 0, i32 0
+; IS__CGSCC_NPM-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP]], align 8
+; IS__CGSCC_NPM-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], 1
+; IS__CGSCC_NPM-NEXT:    store i32 [[TMP2]], i32* [[TMP]], align 8
+; IS__CGSCC_NPM-NEXT:    ret i32 [[TMP1]]
 ;
 entry:
   %tmp = getelementptr %struct.ss, %struct.ss* %b, i32 0, i32 0
@@ -42,28 +69,55 @@ entry:
 
 
 define internal i32 @g(%struct.ss* byval align 32 %b) nounwind {
-; IS________OPM-LABEL: define {{[^@]+}}@g
-; IS________OPM-SAME: (%struct.ss* noalias nocapture nofree nonnull byval align 32 dereferenceable(12) [[B:%.*]])
-; IS________OPM-NEXT:  entry:
-; IS________OPM-NEXT:    [[TMP:%.*]] = getelementptr [[STRUCT_SS:%.*]], %struct.ss* [[B]], i32 0, i32 0
-; IS________OPM-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP]], align 32
-; IS________OPM-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], 1
-; IS________OPM-NEXT:    store i32 [[TMP2]], i32* [[TMP]], align 32
-; IS________OPM-NEXT:    ret i32 [[TMP2]]
+; IS__TUNIT_OPM: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@g
+; IS__TUNIT_OPM-SAME: (%struct.ss* noalias nocapture nofree nonnull byval align 32 dereferenceable(12) [[B:%.*]])
+; IS__TUNIT_OPM-NEXT:  entry:
+; IS__TUNIT_OPM-NEXT:    [[TMP:%.*]] = getelementptr [[STRUCT_SS:%.*]], %struct.ss* [[B]], i32 0, i32 0
+; IS__TUNIT_OPM-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP]], align 32
+; IS__TUNIT_OPM-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], 1
+; IS__TUNIT_OPM-NEXT:    store i32 [[TMP2]], i32* [[TMP]], align 32
+; IS__TUNIT_OPM-NEXT:    ret i32 [[TMP2]]
+;
+; IS__TUNIT_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@g
+; IS__TUNIT_NPM-SAME: (i32 [[TMP0:%.*]], i64 [[TMP1:%.*]])
+; IS__TUNIT_NPM-NEXT:  entry:
+; IS__TUNIT_NPM-NEXT:    [[B_PRIV:%.*]] = alloca [[STRUCT_SS:%.*]], align 4
+; IS__TUNIT_NPM-NEXT:    [[B_PRIV_CAST:%.*]] = bitcast %struct.ss* [[B_PRIV]] to i32*
+; IS__TUNIT_NPM-NEXT:    store i32 [[TMP0]], i32* [[B_PRIV_CAST]], align 4
+; IS__TUNIT_NPM-NEXT:    [[B_PRIV_0_1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[B_PRIV]], i32 0, i32 1
+; IS__TUNIT_NPM-NEXT:    store i64 [[TMP1]], i64* [[B_PRIV_0_1]], align 4
+; IS__TUNIT_NPM-NEXT:    [[TMP:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[B_PRIV]], i32 0, i32 0
+; IS__TUNIT_NPM-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP]], align 32
+; IS__TUNIT_NPM-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], 1
+; IS__TUNIT_NPM-NEXT:    store i32 [[TMP2]], i32* [[TMP]], align 32
+; IS__TUNIT_NPM-NEXT:    ret i32 [[TMP2]]
 ;
-; IS________NPM-LABEL: define {{[^@]+}}@g
-; IS________NPM-SAME: (i32 [[TMP0:%.*]], i64 [[TMP1:%.*]])
-; IS________NPM-NEXT:  entry:
-; IS________NPM-NEXT:    [[B_PRIV:%.*]] = alloca [[STRUCT_SS:%.*]], align 4
-; IS________NPM-NEXT:    [[B_PRIV_CAST:%.*]] = bitcast %struct.ss* [[B_PRIV]] to i32*
-; IS________NPM-NEXT:    store i32 [[TMP0]], i32* [[B_PRIV_CAST]], align 4
-; IS________NPM-NEXT:    [[B_PRIV_0_1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[B_PRIV]], i32 0, i32 1
-; IS________NPM-NEXT:    store i64 [[TMP1]], i64* [[B_PRIV_0_1]], align 4
-; IS________NPM-NEXT:    [[TMP:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[B_PRIV]], i32 0, i32 0
-; IS________NPM-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP]], align 32
-; IS________NPM-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], 1
-; IS________NPM-NEXT:    store i32 [[TMP2]], i32* [[TMP]], align 32
-; IS________NPM-NEXT:    ret i32 [[TMP2]]
+; IS__CGSCC_OPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@g
+; IS__CGSCC_OPM-SAME: (%struct.ss* noalias nocapture nofree nonnull byval align 32 dereferenceable(12) [[B:%.*]])
+; IS__CGSCC_OPM-NEXT:  entry:
+; IS__CGSCC_OPM-NEXT:    [[TMP:%.*]] = getelementptr [[STRUCT_SS:%.*]], %struct.ss* [[B]], i32 0, i32 0
+; IS__CGSCC_OPM-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP]], align 32
+; IS__CGSCC_OPM-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], 1
+; IS__CGSCC_OPM-NEXT:    store i32 [[TMP2]], i32* [[TMP]], align 32
+; IS__CGSCC_OPM-NEXT:    ret i32 [[TMP2]]
+;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@g
+; IS__CGSCC_NPM-SAME: (i32 [[TMP0:%.*]], i64 [[TMP1:%.*]])
+; IS__CGSCC_NPM-NEXT:  entry:
+; IS__CGSCC_NPM-NEXT:    [[B_PRIV:%.*]] = alloca [[STRUCT_SS:%.*]], align 4
+; IS__CGSCC_NPM-NEXT:    [[B_PRIV_CAST:%.*]] = bitcast %struct.ss* [[B_PRIV]] to i32*
+; IS__CGSCC_NPM-NEXT:    store i32 [[TMP0]], i32* [[B_PRIV_CAST]], align 4
+; IS__CGSCC_NPM-NEXT:    [[B_PRIV_0_1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[B_PRIV]], i32 0, i32 1
+; IS__CGSCC_NPM-NEXT:    store i64 [[TMP1]], i64* [[B_PRIV_0_1]], align 4
+; IS__CGSCC_NPM-NEXT:    [[TMP:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[B_PRIV]], i32 0, i32 0
+; IS__CGSCC_NPM-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP]], align 32
+; IS__CGSCC_NPM-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], 1
+; IS__CGSCC_NPM-NEXT:    store i32 [[TMP2]], i32* [[TMP]], align 32
+; IS__CGSCC_NPM-NEXT:    ret i32 [[TMP2]]
 ;
 entry:
   %tmp = getelementptr %struct.ss, %struct.ss* %b, i32 0, i32 0
@@ -75,6 +129,7 @@ entry:
 
 
 define i32 @main() nounwind  {
+; IS__TUNIT_OPM: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@main()
 ; IS__TUNIT_OPM-NEXT:  entry:
 ; IS__TUNIT_OPM-NEXT:    [[S:%.*]] = alloca [[STRUCT_SS:%.*]], align 4
@@ -87,6 +142,7 @@ define i32 @main() nounwind  {
 ; IS__TUNIT_OPM-NEXT:    [[A:%.*]] = add i32 [[C0]], [[C1]]
 ; IS__TUNIT_OPM-NEXT:    ret i32 [[A]]
 ;
+; IS__TUNIT_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@main()
 ; IS__TUNIT_NPM-NEXT:  entry:
 ; IS__TUNIT_NPM-NEXT:    [[S:%.*]] = alloca [[STRUCT_SS:%.*]], align 4
@@ -107,6 +163,7 @@ define i32 @main() nounwind  {
 ; IS__TUNIT_NPM-NEXT:    [[A:%.*]] = add i32 [[C0]], [[C1]]
 ; IS__TUNIT_NPM-NEXT:    ret i32 [[A]]
 ;
+; IS__CGSCC_OPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@main()
 ; IS__CGSCC_OPM-NEXT:  entry:
 ; IS__CGSCC_OPM-NEXT:    [[S:%.*]] = alloca [[STRUCT_SS:%.*]], align 4
@@ -119,6 +176,7 @@ define i32 @main() nounwind  {
 ; IS__CGSCC_OPM-NEXT:    [[A:%.*]] = add i32 [[C0]], [[C1]]
 ; IS__CGSCC_OPM-NEXT:    ret i32 [[A]]
 ;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@main()
 ; IS__CGSCC_NPM-NEXT:  entry:
 ; IS__CGSCC_NPM-NEXT:    [[S:%.*]] = alloca [[STRUCT_SS:%.*]], align 4
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/chained.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/chained.ll
index 823edbbd7783f..75e44569407aa 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/chained.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/chained.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -9,11 +9,19 @@
 
 define internal i32 @test(i32** %x) {
 ;
-; CHECK-LABEL: define {{[^@]+}}@test()
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[Y:%.*]] = load i32*, i32** @G2, align 8
-; CHECK-NEXT:    [[Z:%.*]] = load i32, i32* [[Y]], align 4
-; CHECK-NEXT:    ret i32 [[Z]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test()
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[Y:%.*]] = load i32*, i32** @G2, align 8
+; IS__TUNIT____-NEXT:    [[Z:%.*]] = load i32, i32* [[Y]], align 4
+; IS__TUNIT____-NEXT:    ret i32 [[Z]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test()
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[Y:%.*]] = load i32*, i32** @G2, align 8
+; IS__CGSCC____-NEXT:    [[Z:%.*]] = load i32, i32* [[Y]], align 4
+; IS__CGSCC____-NEXT:    ret i32 [[Z]]
 ;
 entry:
   %y = load i32*, i32** %x
@@ -22,10 +30,17 @@ entry:
 }
 
 define i32 @caller() {
-; CHECK-LABEL: define {{[^@]+}}@caller()
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[X:%.*]] = call i32 @test()
-; CHECK-NEXT:    ret i32 [[X]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@caller()
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[X:%.*]] = call i32 @test()
+; IS__TUNIT____-NEXT:    ret i32 [[X]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@caller()
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[X:%.*]] = call i32 @test()
+; IS__CGSCC____-NEXT:    ret i32 [[X]]
 ;
 entry:
   %x = call i32 @test(i32** @G2)
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow.ll
index b301e226169be..cc66b88e391f2 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -6,15 +6,27 @@
 
 ; Don't promote around control flow.
 define internal i32 @callee(i1 %C, i32* %P) {
-; CHECK-LABEL: define {{[^@]+}}@callee
-; CHECK-SAME: (i1 [[C:%.*]], i32* nocapture nofree readonly [[P:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
-; CHECK:       T:
-; CHECK-NEXT:    ret i32 17
-; CHECK:       F:
-; CHECK-NEXT:    [[X:%.*]] = load i32, i32* [[P]], align 4
-; CHECK-NEXT:    ret i32 [[X]]
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@callee
+; IS__TUNIT____-SAME: (i1 [[C:%.*]], i32* nocapture nofree readonly [[P:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
+; IS__TUNIT____:       T:
+; IS__TUNIT____-NEXT:    ret i32 17
+; IS__TUNIT____:       F:
+; IS__TUNIT____-NEXT:    [[X:%.*]] = load i32, i32* [[P]], align 4
+; IS__TUNIT____-NEXT:    ret i32 [[X]]
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@callee
+; IS__CGSCC____-SAME: (i1 [[C:%.*]], i32* nocapture nofree readonly [[P:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
+; IS__CGSCC____:       T:
+; IS__CGSCC____-NEXT:    ret i32 17
+; IS__CGSCC____:       F:
+; IS__CGSCC____-NEXT:    [[X:%.*]] = load i32, i32* [[P]], align 4
+; IS__CGSCC____-NEXT:    ret i32 [[X]]
 ;
 entry:
   br i1 %C, label %T, label %F
@@ -28,11 +40,19 @@ F:
 }
 
 define i32 @foo(i1 %C, i32* %P) {
-; CHECK-LABEL: define {{[^@]+}}@foo
-; CHECK-SAME: (i1 [[C:%.*]], i32* nocapture nofree readonly [[P:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[X:%.*]] = call i32 @callee(i1 [[C]], i32* nocapture nofree readonly [[P]])
-; CHECK-NEXT:    ret i32 [[X]]
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@foo
+; IS__TUNIT____-SAME: (i1 [[C:%.*]], i32* nocapture nofree readonly [[P:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[X:%.*]] = call i32 @callee(i1 [[C]], i32* nocapture nofree readonly [[P]])
+; IS__TUNIT____-NEXT:    ret i32 [[X]]
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@foo
+; IS__CGSCC____-SAME: (i1 [[C:%.*]], i32* nocapture nofree readonly [[P:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[X:%.*]] = call i32 @callee(i1 [[C]], i32* nocapture nofree readonly [[P]])
+; IS__CGSCC____-NEXT:    ret i32 [[X]]
 ;
 entry:
   %X = call i32 @callee(i1 %C, i32* %P)
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow2.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow2.ll
index eaeebf14ae0ad..bba3e92959d94 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow2.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow2.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=8 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=8 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -7,6 +7,7 @@
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
 define internal i32 @callee(i1 %C, i32* %P) {
+; IS__TUNIT_OPM: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
 ; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@callee
 ; IS__TUNIT_OPM-SAME: (i1 [[C:%.*]], i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P:%.*]])
 ; IS__TUNIT_OPM-NEXT:    br label [[F:%.*]]
@@ -16,6 +17,7 @@ define internal i32 @callee(i1 %C, i32* %P) {
 ; IS__TUNIT_OPM-NEXT:    [[X:%.*]] = load i32, i32* [[P]], align 4
 ; IS__TUNIT_OPM-NEXT:    ret i32 [[X]]
 ;
+; IS__TUNIT_NPM: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@callee
 ; IS__TUNIT_NPM-SAME: (i1 [[C:%.*]], i32 [[TMP0:%.*]])
 ; IS__TUNIT_NPM-NEXT:    [[P_PRIV:%.*]] = alloca i32, align 4
@@ -27,6 +29,7 @@ define internal i32 @callee(i1 %C, i32* %P) {
 ; IS__TUNIT_NPM-NEXT:    [[X:%.*]] = load i32, i32* [[P_PRIV]], align 4
 ; IS__TUNIT_NPM-NEXT:    ret i32 [[X]]
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@callee
 ; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P:%.*]])
 ; IS__CGSCC____-NEXT:    br label [[F:%.*]]
@@ -47,12 +50,14 @@ F:              ; preds = %0
 }
 
 define i32 @foo() {
+; IS__TUNIT_OPM: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@foo()
 ; IS__TUNIT_OPM-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; IS__TUNIT_OPM-NEXT:    store i32 17, i32* [[A]], align 4
 ; IS__TUNIT_OPM-NEXT:    [[X:%.*]] = call i32 @callee(i1 false, i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A]])
 ; IS__TUNIT_OPM-NEXT:    ret i32 [[X]]
 ;
+; IS__TUNIT_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@foo()
 ; IS__TUNIT_NPM-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; IS__TUNIT_NPM-NEXT:    store i32 17, i32* [[A]], align 4
@@ -60,6 +65,7 @@ define i32 @foo() {
 ; IS__TUNIT_NPM-NEXT:    [[X:%.*]] = call i32 @callee(i1 false, i32 [[TMP1]])
 ; IS__TUNIT_NPM-NEXT:    ret i32 [[X]]
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@foo()
 ; IS__CGSCC____-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; IS__CGSCC____-NEXT:    store i32 17, i32* [[A]], align 4
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/crash.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/crash.ll
index 5a03026c2dc2c..dfd40d0b32a0a 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/crash.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/crash.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -9,14 +9,25 @@
 ; Inlining should nuke the invoke (and any inlined calls) here even with
 ; argument promotion running along with it.
 define void @zot() personality i32 (...)* @wibble {
-; CHECK-LABEL: define {{[^@]+}}@zot() #0 personality i32 (...)* @wibble
-; CHECK-NEXT:  bb:
-; CHECK-NEXT:    call void @hoge()
-; CHECK-NEXT:    unreachable
-; CHECK:       bb1:
-; CHECK-NEXT:    unreachable
-; CHECK:       bb2:
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree noreturn nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@zot() #0 personality i32 (...)* @wibble
+; IS__TUNIT____-NEXT:  bb:
+; IS__TUNIT____-NEXT:    call void @hoge()
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       bb1:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       bb2:
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@zot() #0 personality i32 (...)* @wibble
+; IS__CGSCC____-NEXT:  bb:
+; IS__CGSCC____-NEXT:    call void @hoge()
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       bb1:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       bb2:
+; IS__CGSCC____-NEXT:    unreachable
 ;
 bb:
   invoke void @hoge()
@@ -32,9 +43,15 @@ bb2:
 }
 
 define internal void @hoge() {
-; CHECK-LABEL: define {{[^@]+}}@hoge()
-; CHECK-NEXT:  bb:
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree noreturn nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@hoge()
+; IS__TUNIT____-NEXT:  bb:
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@hoge()
+; IS__CGSCC____-NEXT:  bb:
+; IS__CGSCC____-NEXT:    unreachable
 ;
 bb:
   %tmp = call fastcc i8* @spam(i1 (i8*)* @eggs)
@@ -43,6 +60,7 @@ bb:
 }
 
 define internal fastcc i8* @spam(i1 (i8*)* %arg) {
+; IS__CGSCC____: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@spam()
 ; IS__CGSCC____-NEXT:  bb:
 ; IS__CGSCC____-NEXT:    unreachable
@@ -58,6 +76,7 @@ bb:
 }
 
 define internal i1 @barney(i8* %arg) {
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@barney()
 ; IS__CGSCC____-NEXT:  bb:
 ; IS__CGSCC____-NEXT:    ret i1 undef
@@ -67,11 +86,13 @@ bb:
 }
 
 define i32 @test_inf_promote_caller(i32 %arg) {
+; IS__TUNIT____: Function Attrs: nofree noreturn nosync nounwind readnone willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@test_inf_promote_caller
 ; IS__TUNIT____-SAME: (i32 [[ARG:%.*]])
 ; IS__TUNIT____-NEXT:  bb:
 ; IS__TUNIT____-NEXT:    unreachable
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@test_inf_promote_caller
 ; IS__CGSCC____-SAME: (i32 [[ARG:%.*]])
 ; IS__CGSCC____-NEXT:  bb:
@@ -88,6 +109,7 @@ bb:
 }
 
 define internal i32 @test_inf_promote_callee(%S* %arg, %S* %arg1) {
+; IS__CGSCC____: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@test_inf_promote_callee()
 ; IS__CGSCC____-NEXT:  bb:
 ; IS__CGSCC____-NEXT:    unreachable
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/dbg.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/dbg.ll
index bf544c09cd325..051e486ea70b1 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/dbg.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/dbg.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/fp80.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/fp80.ll
index 5c2ed40cf5908..295907cbca0b2 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/fp80.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/fp80.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; TODO: The old pass manager cgscc run is disabled as it causes a crash on windows which is under investigation: http://lab.llvm.org:8011/builders/llvm-clang-x86_64-expensive-checks-win/builds/23151
@@ -17,9 +17,15 @@ target triple = "x86_64-unknown-linux-gnu"
 @a = internal global %struct.Foo { i32 1, i64 2 }, align 8
 
 define void @run() {
-; CHECK-LABEL: define {{[^@]+}}@run()
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    unreachable
+; NOT_CGSCC_NPM: Function Attrs: nofree noreturn nosync nounwind readnone
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@run()
+; NOT_CGSCC_NPM-NEXT:  entry:
+; NOT_CGSCC_NPM-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse noreturn nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@run()
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    unreachable
 ;
 entry:
   tail call i8 @UseLongDoubleUnsafely(%union.u* byval align 16 bitcast (%struct.s* @b to %union.u*))
@@ -30,6 +36,7 @@ entry:
 }
 
 define internal i8 @UseLongDoubleUnsafely(%union.u* byval align 16 %arg) {
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@UseLongDoubleUnsafely()
 ; IS__CGSCC____-NEXT:  entry:
 ; IS__CGSCC____-NEXT:    ret i8 undef
@@ -42,6 +49,7 @@ entry:
 }
 
 define internal x86_fp80 @UseLongDoubleSafely(%union.u* byval align 16 %arg) {
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@UseLongDoubleSafely()
 ; IS__CGSCC____-NEXT:    ret x86_fp80 undef
 ;
@@ -51,6 +59,7 @@ define internal x86_fp80 @UseLongDoubleSafely(%union.u* byval align 16 %arg) {
 }
 
 define internal i64 @AccessPaddingOfStruct(%struct.Foo* byval %a) {
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@AccessPaddingOfStruct()
 ; IS__CGSCC____-NEXT:    ret i64 undef
 ;
@@ -72,6 +81,7 @@ define internal i64 @CaptureAStruct(%struct.Foo* byval %a) {
 ; IS__CGSCC_OPM-NEXT:    [[GEP]] = getelementptr [[STRUCT_FOO:%.*]], %struct.Foo* [[A]], i64 0
 ; IS__CGSCC_OPM-NEXT:    br label [[LOOP]]
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse noreturn nosync nounwind readnone
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@CaptureAStruct
 ; IS__CGSCC____-SAME: (i32 [[TMP0:%.*]], i64 [[TMP1:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/inalloca.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/inalloca.ll
index f8625b2562369..ea13554f587b8 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/inalloca.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/inalloca.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -10,6 +10,7 @@ target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:1
 
 ; Argpromote + sroa should change this to passing the two integers by value.
 define internal i32 @f(%struct.ss* inalloca  %s) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@f
 ; IS__TUNIT____-SAME: (%struct.ss* inalloca noalias nocapture nofree nonnull align 4 dereferenceable(8) [[S:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
@@ -20,6 +21,7 @@ define internal i32 @f(%struct.ss* inalloca  %s) {
 ; IS__TUNIT____-NEXT:    [[R:%.*]] = add i32 [[A]], [[B]]
 ; IS__TUNIT____-NEXT:    ret i32 [[R]]
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@f
 ; IS__CGSCC____-SAME: (%struct.ss* inalloca nocapture nofree nonnull align 4 dereferenceable(8) [[S:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -40,15 +42,27 @@ entry:
 }
 
 define i32 @main() {
-; CHECK-LABEL: define {{[^@]+}}@main()
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[S:%.*]] = alloca inalloca [[STRUCT_SS:%.*]], align 4
-; CHECK-NEXT:    [[F0:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 0
-; CHECK-NEXT:    [[F1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 1
-; CHECK-NEXT:    store i32 1, i32* [[F0]], align 4
-; CHECK-NEXT:    store i32 2, i32* [[F1]], align 4
-; CHECK-NEXT:    [[R:%.*]] = call i32 @f(%struct.ss* inalloca noalias nocapture nofree nonnull align 4 dereferenceable(8) [[S]])
-; CHECK-NEXT:    ret i32 [[R]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@main()
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[S:%.*]] = alloca inalloca [[STRUCT_SS:%.*]], align 4
+; IS__TUNIT____-NEXT:    [[F0:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 0
+; IS__TUNIT____-NEXT:    [[F1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 1
+; IS__TUNIT____-NEXT:    store i32 1, i32* [[F0]], align 4
+; IS__TUNIT____-NEXT:    store i32 2, i32* [[F1]], align 4
+; IS__TUNIT____-NEXT:    [[R:%.*]] = call i32 @f(%struct.ss* inalloca noalias nocapture nofree nonnull align 4 dereferenceable(8) [[S]])
+; IS__TUNIT____-NEXT:    ret i32 [[R]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@main()
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[S:%.*]] = alloca inalloca [[STRUCT_SS:%.*]], align 4
+; IS__CGSCC____-NEXT:    [[F0:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 0
+; IS__CGSCC____-NEXT:    [[F1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 1
+; IS__CGSCC____-NEXT:    store i32 1, i32* [[F0]], align 4
+; IS__CGSCC____-NEXT:    store i32 2, i32* [[F1]], align 4
+; IS__CGSCC____-NEXT:    [[R:%.*]] = call i32 @f(%struct.ss* inalloca noalias nocapture nofree nonnull align 4 dereferenceable(8) [[S]])
+; IS__CGSCC____-NEXT:    ret i32 [[R]]
 ;
 entry:
   %S = alloca inalloca %struct.ss
@@ -62,6 +76,7 @@ entry:
 
 ; Argpromote can't promote %a because of the icmp use.
 define internal i1 @g(%struct.ss* %a, %struct.ss* inalloca %b) nounwind  {
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@g
 ; IS__CGSCC____-SAME: (%struct.ss* nocapture nofree nonnull readnone align 4 dereferenceable(8) [[A:%.*]], %struct.ss* inalloca nocapture nofree nonnull writeonly align 4 dereferenceable(8) [[B:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -73,9 +88,15 @@ entry:
 }
 
 define i32 @test() {
-; CHECK-LABEL: define {{[^@]+}}@test()
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i32 0
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test()
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    ret i32 0
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test()
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    ret i32 0
 ;
 entry:
   %S = alloca inalloca %struct.ss
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/invalidation.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/invalidation.ll
index 5f6e9d10888ed..ef0064ed68c46 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/invalidation.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/invalidation.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; Check that when argument promotion changes a function in some parent node of
 ; the call graph, any analyses that happened to be cached for that function are
 ; actually invalidated. We are using `demanded-bits` here because when printed
@@ -12,6 +12,7 @@
 @G = constant i32 0
 
 define internal i32 @a(i32* %x) {
+; CHECK: Function Attrs: nofree nosync nounwind readonly willreturn
 ; CHECK-LABEL: define {{[^@]+}}@a()
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[V:%.*]] = load i32, i32* @G, align 4
@@ -23,6 +24,7 @@ entry:
 }
 
 define i32 @b() {
+; CHECK: Function Attrs: nofree nosync nounwind readonly willreturn
 ; CHECK-LABEL: define {{[^@]+}}@b()
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[V:%.*]] = call i32 @a()
@@ -34,6 +36,7 @@ entry:
 }
 
 define i32 @c() {
+; CHECK: Function Attrs: nofree nosync nounwind readonly willreturn
 ; CHECK-LABEL: define {{[^@]+}}@c()
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[V1:%.*]] = call i32 @a()
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead.ll
index 4da401f1db296..a53ab08d60ec3 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -11,6 +11,7 @@ define internal void @dead() {
 }
 
 define internal i32 @test(i32* %X, i32* %Y) {
+; IS__CGSCC_OPM: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@test
 ; IS__CGSCC_OPM-SAME: (i32* noalias nocapture nofree writeonly align 4 [[X:%.*]])
 ; IS__CGSCC_OPM-NEXT:    br i1 true, label [[LIVE:%.*]], label [[DEAD:%.*]]
@@ -20,6 +21,7 @@ define internal i32 @test(i32* %X, i32* %Y) {
 ; IS__CGSCC_OPM:       dead:
 ; IS__CGSCC_OPM-NEXT:    unreachable
 ;
+; IS__CGSCC_NPM: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
 ; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@test
 ; IS__CGSCC_NPM-SAME: (i32* noalias nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[X:%.*]])
 ; IS__CGSCC_NPM-NEXT:    br i1 true, label [[LIVE:%.*]], label [[DEAD:%.*]]
@@ -40,12 +42,14 @@ dead:
 }
 
 define internal i32 @caller(i32* %B) {
+; IS__CGSCC_OPM: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@caller()
 ; IS__CGSCC_OPM-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; IS__CGSCC_OPM-NEXT:    store i32 1, i32* [[A]], align 4
 ; IS__CGSCC_OPM-NEXT:    [[C:%.*]] = call i32 @test(i32* noalias nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[A]])
 ; IS__CGSCC_OPM-NEXT:    ret i32 0
 ;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@caller()
 ; IS__CGSCC_NPM-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; IS__CGSCC_NPM-NEXT:    store i32 1, i32* [[A]], align 4
@@ -59,10 +63,17 @@ define internal i32 @caller(i32* %B) {
 }
 
 define i32 @callercaller() {
-; CHECK-LABEL: define {{[^@]+}}@callercaller()
-; CHECK-NEXT:    [[B:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    store i32 2, i32* [[B]], align 4
-; CHECK-NEXT:    ret i32 0
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@callercaller()
+; IS__TUNIT____-NEXT:    [[B:%.*]] = alloca i32, align 4
+; IS__TUNIT____-NEXT:    store i32 2, i32* [[B]], align 4
+; IS__TUNIT____-NEXT:    ret i32 0
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@callercaller()
+; IS__CGSCC____-NEXT:    [[B:%.*]] = alloca i32, align 4
+; IS__CGSCC____-NEXT:    store i32 2, i32* [[B]], align 4
+; IS__CGSCC____-NEXT:    ret i32 0
 ;
   %B = alloca i32
   store i32 2, i32* %B
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead_2.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead_2.ll
index 9d8d5e5decc39..6b2b7def5b956 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead_2.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead_2.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=8 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -11,6 +11,7 @@ define internal void @dead() {
 }
 
 define internal i32 @test(i32* %X, i32* %Y) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@test
 ; IS__TUNIT____-SAME: (i32* noalias nocapture nofree writeonly align 4 [[X:%.*]])
 ; IS__TUNIT____-NEXT:    br i1 true, label [[LIVE:%.*]], label [[DEAD:%.*]]
@@ -20,6 +21,7 @@ define internal i32 @test(i32* %X, i32* %Y) {
 ; IS__TUNIT____:       dead:
 ; IS__TUNIT____-NEXT:    unreachable
 ;
+; IS__CGSCC_OPM: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@test
 ; IS__CGSCC_OPM-SAME: (i32* nocapture nofree writeonly align 4 [[X:%.*]])
 ; IS__CGSCC_OPM-NEXT:    br i1 true, label [[LIVE:%.*]], label [[DEAD:%.*]]
@@ -29,6 +31,7 @@ define internal i32 @test(i32* %X, i32* %Y) {
 ; IS__CGSCC_OPM:       dead:
 ; IS__CGSCC_OPM-NEXT:    unreachable
 ;
+; IS__CGSCC_NPM: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
 ; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@test
 ; IS__CGSCC_NPM-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[X:%.*]])
 ; IS__CGSCC_NPM-NEXT:    br i1 true, label [[LIVE:%.*]], label [[DEAD:%.*]]
@@ -49,6 +52,7 @@ dead:
 }
 
 define internal i32 @caller(i32* %B) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@caller
 ; IS__TUNIT____-SAME: (i32* noalias nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[B:%.*]])
 ; IS__TUNIT____-NEXT:    [[A:%.*]] = alloca i32, align 4
@@ -56,6 +60,7 @@ define internal i32 @caller(i32* %B) {
 ; IS__TUNIT____-NEXT:    [[C:%.*]] = call i32 @test(i32* noalias nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[B]])
 ; IS__TUNIT____-NEXT:    ret i32 0
 ;
+; IS__CGSCC_OPM: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@caller
 ; IS__CGSCC_OPM-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[B:%.*]])
 ; IS__CGSCC_OPM-NEXT:    [[A:%.*]] = alloca i32, align 4
@@ -63,6 +68,7 @@ define internal i32 @caller(i32* %B) {
 ; IS__CGSCC_OPM-NEXT:    [[C:%.*]] = call i32 @test(i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[B]])
 ; IS__CGSCC_OPM-NEXT:    ret i32 0
 ;
+; IS__CGSCC_NPM: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
 ; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@caller
 ; IS__CGSCC_NPM-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[B:%.*]])
 ; IS__CGSCC_NPM-NEXT:    [[A:%.*]] = alloca i32, align 4
@@ -77,11 +83,19 @@ define internal i32 @caller(i32* %B) {
 }
 
 define i32 @callercaller() {
-; CHECK-LABEL: define {{[^@]+}}@callercaller()
-; CHECK-NEXT:    [[B:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    store i32 2, i32* [[B]], align 4
-; CHECK-NEXT:    [[X:%.*]] = call i32 @caller(i32* noalias nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[B]])
-; CHECK-NEXT:    ret i32 0
+; NOT_CGSCC_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@callercaller()
+; NOT_CGSCC_NPM-NEXT:    [[B:%.*]] = alloca i32, align 4
+; NOT_CGSCC_NPM-NEXT:    store i32 2, i32* [[B]], align 4
+; NOT_CGSCC_NPM-NEXT:    [[X:%.*]] = call i32 @caller(i32* noalias nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[B]])
+; NOT_CGSCC_NPM-NEXT:    ret i32 0
+;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@callercaller()
+; IS__CGSCC_NPM-NEXT:    [[B:%.*]] = alloca i32, align 4
+; IS__CGSCC_NPM-NEXT:    store i32 2, i32* [[B]], align 4
+; IS__CGSCC_NPM-NEXT:    [[X:%.*]] = call i32 @caller(i32* noalias nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[B]])
+; IS__CGSCC_NPM-NEXT:    ret i32 0
 ;
   %B = alloca i32
   store i32 2, i32* %B
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/musttail.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/musttail.ll
index b70eff3abede1..4b266d8a5d17c 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/musttail.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/musttail.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -10,14 +10,25 @@
 %T = type { i32, i32, i32, i32 }
 
 define internal i32 @test(%T* %p) {
-; CHECK-LABEL: define {{[^@]+}}@test
-; CHECK-SAME: (%T* nocapture nofree readonly [[P:%.*]])
-; CHECK-NEXT:    [[A_GEP:%.*]] = getelementptr [[T:%.*]], %T* [[P]], i64 0, i32 3
-; CHECK-NEXT:    [[B_GEP:%.*]] = getelementptr [[T]], %T* [[P]], i64 0, i32 2
-; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[A_GEP]], align 4
-; CHECK-NEXT:    [[B:%.*]] = load i32, i32* [[B_GEP]], align 4
-; CHECK-NEXT:    [[V:%.*]] = add i32 [[A]], [[B]]
-; CHECK-NEXT:    ret i32 [[V]]
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test
+; IS__TUNIT____-SAME: (%T* nocapture nofree readonly [[P:%.*]])
+; IS__TUNIT____-NEXT:    [[A_GEP:%.*]] = getelementptr [[T:%.*]], %T* [[P]], i64 0, i32 3
+; IS__TUNIT____-NEXT:    [[B_GEP:%.*]] = getelementptr [[T]], %T* [[P]], i64 0, i32 2
+; IS__TUNIT____-NEXT:    [[A:%.*]] = load i32, i32* [[A_GEP]], align 4
+; IS__TUNIT____-NEXT:    [[B:%.*]] = load i32, i32* [[B_GEP]], align 4
+; IS__TUNIT____-NEXT:    [[V:%.*]] = add i32 [[A]], [[B]]
+; IS__TUNIT____-NEXT:    ret i32 [[V]]
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test
+; IS__CGSCC____-SAME: (%T* nocapture nofree readonly [[P:%.*]])
+; IS__CGSCC____-NEXT:    [[A_GEP:%.*]] = getelementptr [[T:%.*]], %T* [[P]], i64 0, i32 3
+; IS__CGSCC____-NEXT:    [[B_GEP:%.*]] = getelementptr [[T]], %T* [[P]], i64 0, i32 2
+; IS__CGSCC____-NEXT:    [[A:%.*]] = load i32, i32* [[A_GEP]], align 4
+; IS__CGSCC____-NEXT:    [[B:%.*]] = load i32, i32* [[B_GEP]], align 4
+; IS__CGSCC____-NEXT:    [[V:%.*]] = add i32 [[A]], [[B]]
+; IS__CGSCC____-NEXT:    ret i32 [[V]]
 ;
   %a.gep = getelementptr %T, %T* %p, i64 0, i32 3
   %b.gep = getelementptr %T, %T* %p, i64 0, i32 2
@@ -28,10 +39,17 @@ define internal i32 @test(%T* %p) {
 }
 
 define i32 @caller(%T* %p) {
-; CHECK-LABEL: define {{[^@]+}}@caller
-; CHECK-SAME: (%T* nocapture nofree readonly [[P:%.*]])
-; CHECK-NEXT:    [[V:%.*]] = musttail call i32 @test(%T* nocapture nofree readonly [[P]])
-; CHECK-NEXT:    ret i32 [[V]]
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@caller
+; IS__TUNIT____-SAME: (%T* nocapture nofree readonly [[P:%.*]])
+; IS__TUNIT____-NEXT:    [[V:%.*]] = musttail call i32 @test(%T* nocapture nofree readonly [[P]])
+; IS__TUNIT____-NEXT:    ret i32 [[V]]
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@caller
+; IS__CGSCC____-SAME: (%T* nocapture nofree readonly [[P:%.*]])
+; IS__CGSCC____-NEXT:    [[V:%.*]] = musttail call i32 @test(%T* nocapture nofree readonly [[P]])
+; IS__CGSCC____-NEXT:    ret i32 [[V]]
 ;
   %v = musttail call i32 @test(%T* %p)
   ret i32 %v
@@ -40,14 +58,21 @@ define i32 @caller(%T* %p) {
 ; Don't promote arguments of musttail caller
 
 define i32 @foo(%T* %p, i32 %v) {
-; CHECK-LABEL: define {{[^@]+}}@foo
-; CHECK-SAME: (%T* nocapture nofree readnone [[P:%.*]], i32 [[V:%.*]])
-; CHECK-NEXT:    ret i32 0
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@foo
+; IS__TUNIT____-SAME: (%T* nocapture nofree readnone [[P:%.*]], i32 [[V:%.*]])
+; IS__TUNIT____-NEXT:    ret i32 0
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@foo
+; IS__CGSCC____-SAME: (%T* nocapture nofree readnone [[P:%.*]], i32 [[V:%.*]])
+; IS__CGSCC____-NEXT:    ret i32 0
 ;
   ret i32 0
 }
 
 define internal i32 @test2(%T* %p, i32 %p2) {
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@test2
 ; IS__CGSCC____-SAME: (%T* nocapture nofree readonly [[P:%.*]], i32 [[P2:%.*]])
 ; IS__CGSCC____-NEXT:    [[A_GEP:%.*]] = getelementptr [[T:%.*]], %T* [[P]], i64 0, i32 3
@@ -68,10 +93,12 @@ define internal i32 @test2(%T* %p, i32 %p2) {
 }
 
 define i32 @caller2(%T* %g) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@caller2
 ; IS__TUNIT____-SAME: (%T* nocapture nofree readnone [[G:%.*]])
 ; IS__TUNIT____-NEXT:    ret i32 0
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@caller2
 ; IS__CGSCC____-SAME: (%T* nocapture nofree readonly align 4 [[G:%.*]])
 ; IS__CGSCC____-NEXT:    [[V:%.*]] = call i32 @test2(%T* nocapture nofree readonly [[G]], i32 0)
@@ -86,11 +113,19 @@ define i32 @caller2(%T* %g) {
 ; is kept as well.
 
 define i32 @bar(%T* %p, i32 %v) {
-; CHECK-LABEL: define {{[^@]+}}@bar
-; CHECK-SAME: (%T* nocapture nofree nonnull writeonly dereferenceable(4) [[P:%.*]], i32 [[V:%.*]])
-; CHECK-NEXT:    [[I32PTR:%.*]] = getelementptr [[T:%.*]], %T* [[P]], i64 0, i32 0
-; CHECK-NEXT:    store i32 [[V]], i32* [[I32PTR]], align 4
-; CHECK-NEXT:    ret i32 0
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@bar
+; IS__TUNIT____-SAME: (%T* nocapture nofree nonnull writeonly dereferenceable(4) [[P:%.*]], i32 [[V:%.*]])
+; IS__TUNIT____-NEXT:    [[I32PTR:%.*]] = getelementptr [[T:%.*]], %T* [[P]], i64 0, i32 0
+; IS__TUNIT____-NEXT:    store i32 [[V]], i32* [[I32PTR]], align 4
+; IS__TUNIT____-NEXT:    ret i32 0
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@bar
+; IS__CGSCC____-SAME: (%T* nocapture nofree nonnull writeonly dereferenceable(4) [[P:%.*]], i32 [[V:%.*]])
+; IS__CGSCC____-NEXT:    [[I32PTR:%.*]] = getelementptr [[T:%.*]], %T* [[P]], i64 0, i32 0
+; IS__CGSCC____-NEXT:    store i32 [[V]], i32* [[I32PTR]], align 4
+; IS__CGSCC____-NEXT:    ret i32 0
 ;
   %i32ptr = getelementptr %T, %T* %p, i64 0, i32 0
   store i32 %v, i32* %i32ptr
@@ -98,15 +133,27 @@ define i32 @bar(%T* %p, i32 %v) {
 }
 
 define internal i32 @test2b(%T* %p, i32 %p2) {
-; CHECK-LABEL: define {{[^@]+}}@test2b
-; CHECK-SAME: (%T* nocapture nofree readonly [[P:%.*]], i32 [[P2:%.*]])
-; CHECK-NEXT:    [[A_GEP:%.*]] = getelementptr [[T:%.*]], %T* [[P]], i64 0, i32 3
-; CHECK-NEXT:    [[B_GEP:%.*]] = getelementptr [[T]], %T* [[P]], i64 0, i32 2
-; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[A_GEP]], align 4
-; CHECK-NEXT:    [[B:%.*]] = load i32, i32* [[B_GEP]], align 4
-; CHECK-NEXT:    [[V:%.*]] = add i32 [[A]], [[B]]
-; CHECK-NEXT:    [[CA:%.*]] = musttail call i32 @bar(%T* undef, i32 [[V]])
-; CHECK-NEXT:    ret i32 [[CA]]
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test2b
+; IS__TUNIT____-SAME: (%T* nocapture nofree readonly [[P:%.*]], i32 [[P2:%.*]])
+; IS__TUNIT____-NEXT:    [[A_GEP:%.*]] = getelementptr [[T:%.*]], %T* [[P]], i64 0, i32 3
+; IS__TUNIT____-NEXT:    [[B_GEP:%.*]] = getelementptr [[T]], %T* [[P]], i64 0, i32 2
+; IS__TUNIT____-NEXT:    [[A:%.*]] = load i32, i32* [[A_GEP]], align 4
+; IS__TUNIT____-NEXT:    [[B:%.*]] = load i32, i32* [[B_GEP]], align 4
+; IS__TUNIT____-NEXT:    [[V:%.*]] = add i32 [[A]], [[B]]
+; IS__TUNIT____-NEXT:    [[CA:%.*]] = musttail call i32 @bar(%T* undef, i32 [[V]])
+; IS__TUNIT____-NEXT:    ret i32 [[CA]]
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test2b
+; IS__CGSCC____-SAME: (%T* nocapture nofree readonly [[P:%.*]], i32 [[P2:%.*]])
+; IS__CGSCC____-NEXT:    [[A_GEP:%.*]] = getelementptr [[T:%.*]], %T* [[P]], i64 0, i32 3
+; IS__CGSCC____-NEXT:    [[B_GEP:%.*]] = getelementptr [[T]], %T* [[P]], i64 0, i32 2
+; IS__CGSCC____-NEXT:    [[A:%.*]] = load i32, i32* [[A_GEP]], align 4
+; IS__CGSCC____-NEXT:    [[B:%.*]] = load i32, i32* [[B_GEP]], align 4
+; IS__CGSCC____-NEXT:    [[V:%.*]] = add i32 [[A]], [[B]]
+; IS__CGSCC____-NEXT:    [[CA:%.*]] = musttail call i32 @bar(%T* undef, i32 [[V]])
+; IS__CGSCC____-NEXT:    ret i32 [[CA]]
 ;
   %a.gep = getelementptr %T, %T* %p, i64 0, i32 3
   %b.gep = getelementptr %T, %T* %p, i64 0, i32 2
@@ -118,11 +165,13 @@ define internal i32 @test2b(%T* %p, i32 %p2) {
 }
 
 define i32 @caller2b(%T* %g) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@caller2b
 ; IS__TUNIT____-SAME: (%T* nocapture nofree readonly [[G:%.*]])
 ; IS__TUNIT____-NEXT:    [[V:%.*]] = call i32 @test2b(%T* nocapture nofree readonly [[G]], i32 undef)
 ; IS__TUNIT____-NEXT:    ret i32 0
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@caller2b
 ; IS__CGSCC____-SAME: (%T* nocapture nofree readonly align 4 [[G:%.*]])
 ; IS__CGSCC____-NEXT:    [[V:%.*]] = call i32 @test2b(%T* nocapture nofree readonly [[G]], i32 0)
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/naked_functions.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/naked_functions.ll
index b952ac6dc9a2b..f566358cef61e 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/naked_functions.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/naked_functions.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -20,6 +20,7 @@ entry:
 }
 
 define internal i32 @foo(i32*) #0 {
+; CHECK: Function Attrs: naked
 ; CHECK-LABEL: define {{[^@]+}}@foo
 ; CHECK-SAME: (i32* [[TMP0:%.*]])
 ; CHECK-NEXT:  entry:
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/nonzero-address-spaces.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/nonzero-address-spaces.ll
index 1e9e45ed37e8f..96acd981a162a 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/nonzero-address-spaces.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/nonzero-address-spaces.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -12,6 +12,7 @@ target datalayout = "e-P1-p:16:8-i8:8-i16:8-i32:8-i64:8-f32:8-f64:8-n8-a:8"
 @g = common global i32 0, align 4
 
 define i32 @bar() {
+; CHECK: Function Attrs: noreturn
 ; CHECK-LABEL: define {{[^@]+}}@bar() addrspace(1)
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = call addrspace(1) i32 @foo()
@@ -24,6 +25,7 @@ entry:
 }
 
 define internal i32 @foo(i32*) {
+; CHECK: Function Attrs: noreturn
 ; CHECK-LABEL: define {{[^@]+}}@foo() addrspace(1)
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/pr27568.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr27568.ll
index a60d18f5e6cb7..546bd149a4f0a 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/pr27568.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr27568.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/pr32917.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr32917.ll
index 0f1f7a23ec2c6..40edeef6db927 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/pr32917.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr32917.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -9,6 +9,7 @@
 @a = common local_unnamed_addr global i32 0, align 4
 
 define i32 @fn2() local_unnamed_addr {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@fn2() local_unnamed_addr
 ; IS__TUNIT____-NEXT:    [[TMP1:%.*]] = load i32, i32* @b, align 4
 ; IS__TUNIT____-NEXT:    [[TMP2:%.*]] = sext i32 [[TMP1]] to i64
@@ -16,6 +17,7 @@ define i32 @fn2() local_unnamed_addr {
 ; IS__TUNIT____-NEXT:    call fastcc void @fn1(i32* nocapture nofree readonly align 4 [[TMP3]])
 ; IS__TUNIT____-NEXT:    ret i32 undef
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@fn2() local_unnamed_addr
 ; IS__CGSCC____-NEXT:    [[TMP1:%.*]] = load i32, i32* @b, align 4
 ; IS__CGSCC____-NEXT:    [[TMP2:%.*]] = sext i32 [[TMP1]] to i64
@@ -31,12 +33,21 @@ define i32 @fn2() local_unnamed_addr {
 }
 
 define internal fastcc void @fn1(i32* nocapture readonly) unnamed_addr {
-; CHECK-LABEL: define {{[^@]+}}@fn1
-; CHECK-SAME: (i32* nocapture nofree nonnull readonly align 4 [[TMP0:%.*]]) unnamed_addr
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 -1
-; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP2]], align 4
-; CHECK-NEXT:    store i32 [[TMP3]], i32* @a, align 4
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@fn1
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull readonly align 4 [[TMP0:%.*]]) unnamed_addr
+; IS__TUNIT____-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 -1
+; IS__TUNIT____-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP2]], align 4
+; IS__TUNIT____-NEXT:    store i32 [[TMP3]], i32* @a, align 4
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@fn1
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readonly align 4 [[TMP0:%.*]]) unnamed_addr
+; IS__CGSCC____-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 -1
+; IS__CGSCC____-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP2]], align 4
+; IS__CGSCC____-NEXT:    store i32 [[TMP3]], i32* @a, align 4
+; IS__CGSCC____-NEXT:    ret void
 ;
   %2 = getelementptr inbounds i32, i32* %0, i64 -1
   %3 = load i32, i32* %2, align 4
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
index a372028311a63..302f0e2b3b3fa 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -14,10 +14,17 @@
 %fun_t = type void (%p_t)*
 
 define void @foo() {
-; CHECK-LABEL: define {{[^@]+}}@foo()
-; CHECK-NEXT:    [[TMP:%.*]] = alloca void (i16*)*, align 8
-; CHECK-NEXT:    store void (i16*)* @bar, void (i16*)** [[TMP]], align 8
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@foo()
+; IS__TUNIT____-NEXT:    [[TMP:%.*]] = alloca void (i16*)*, align 8
+; IS__TUNIT____-NEXT:    store void (i16*)* @bar, void (i16*)** [[TMP]], align 8
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@foo()
+; IS__CGSCC____-NEXT:    [[TMP:%.*]] = alloca void (i16*)*, align 8
+; IS__CGSCC____-NEXT:    store void (i16*)* @bar, void (i16*)** [[TMP]], align 8
+; IS__CGSCC____-NEXT:    ret void
 ;
   %tmp = alloca %fun_t
   store %fun_t @bar, %fun_t* %tmp
@@ -25,6 +32,7 @@ define void @foo() {
 }
 
 define internal void @bar(%p_t %p)  {
+; CHECK: Function Attrs: nosync nounwind readnone willreturn
 ; CHECK-LABEL: define {{[^@]+}}@bar
 ; CHECK-SAME: (i16* nocapture nofree readnone [[P:%.*]])
 ; CHECK-NEXT:    call void @llvm.dbg.value(metadata i16* [[P]], metadata !3, metadata !DIExpression()) #3, !dbg !5
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/profile.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/profile.ll
index cacc6b95d2638..bf18851fb7c2c 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/profile.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/profile.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/reserve-tbaa.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/reserve-tbaa.ll
index a646b258a0fcc..a649971612297 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/reserve-tbaa.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/reserve-tbaa.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -17,6 +17,7 @@
 @d = global i8 0, align 1
 
 define internal fastcc void @fn(i32* nocapture readonly %p1, i64* nocapture readonly %p2) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@fn
 ; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P1:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
@@ -25,6 +26,7 @@ define internal fastcc void @fn(i32* nocapture readonly %p1, i64* nocapture read
 ; IS__TUNIT____-NEXT:    store i8 [[CONV1]], i8* @d, align 1, !tbaa !4
 ; IS__TUNIT____-NEXT:    ret void
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@fn()
 ; IS__CGSCC____-NEXT:  entry:
 ; IS__CGSCC____-NEXT:    [[TMP0:%.*]] = load i32, i32* @g, align 4, !tbaa !0
@@ -42,6 +44,7 @@ entry:
 }
 
 define i32 @main() {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@main()
 ; IS__TUNIT____-NEXT:  entry:
 ; IS__TUNIT____-NEXT:    [[TMP0:%.*]] = load i32**, i32*** @e, align 8, !tbaa !5
@@ -51,6 +54,7 @@ define i32 @main() {
 ; IS__TUNIT____-NEXT:    call fastcc void @fn(i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) @g)
 ; IS__TUNIT____-NEXT:    ret i32 0
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@main()
 ; IS__CGSCC____-NEXT:  entry:
 ; IS__CGSCC____-NEXT:    [[TMP0:%.*]] = load i32**, i32*** @e, align 8, !tbaa !5
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll
index a84ab1f4ef4b2..2c4312c6159c0 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -9,16 +9,18 @@ target triple = "x86_64-pc-windows-msvc"
 
 define internal void @add({i32, i32}* %this, i32* sret %r) {
 ;
-; NOT_TUNIT_NPM-LABEL: define {{[^@]+}}@add
-; NOT_TUNIT_NPM-SAME: ({ i32, i32 }* nocapture nofree nonnull readonly align 8 dereferenceable(8) [[THIS:%.*]], i32* nocapture nofree nonnull sret writeonly align 4 dereferenceable(4) [[R:%.*]])
-; NOT_TUNIT_NPM-NEXT:    [[AP:%.*]] = getelementptr { i32, i32 }, { i32, i32 }* [[THIS]], i32 0, i32 0
-; NOT_TUNIT_NPM-NEXT:    [[BP:%.*]] = getelementptr { i32, i32 }, { i32, i32 }* [[THIS]], i32 0, i32 1
-; NOT_TUNIT_NPM-NEXT:    [[A:%.*]] = load i32, i32* [[AP]], align 8
-; NOT_TUNIT_NPM-NEXT:    [[B:%.*]] = load i32, i32* [[BP]], align 4
-; NOT_TUNIT_NPM-NEXT:    [[AB:%.*]] = add i32 [[A]], [[B]]
-; NOT_TUNIT_NPM-NEXT:    store i32 [[AB]], i32* [[R]], align 4
-; NOT_TUNIT_NPM-NEXT:    ret void
+; IS__TUNIT_OPM: Function Attrs: argmemonly nofree nosync nounwind willreturn
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@add
+; IS__TUNIT_OPM-SAME: ({ i32, i32 }* nocapture nofree nonnull readonly align 8 dereferenceable(8) [[THIS:%.*]], i32* nocapture nofree nonnull sret writeonly align 4 dereferenceable(4) [[R:%.*]])
+; IS__TUNIT_OPM-NEXT:    [[AP:%.*]] = getelementptr { i32, i32 }, { i32, i32 }* [[THIS]], i32 0, i32 0
+; IS__TUNIT_OPM-NEXT:    [[BP:%.*]] = getelementptr { i32, i32 }, { i32, i32 }* [[THIS]], i32 0, i32 1
+; IS__TUNIT_OPM-NEXT:    [[A:%.*]] = load i32, i32* [[AP]], align 8
+; IS__TUNIT_OPM-NEXT:    [[B:%.*]] = load i32, i32* [[BP]], align 4
+; IS__TUNIT_OPM-NEXT:    [[AB:%.*]] = add i32 [[A]], [[B]]
+; IS__TUNIT_OPM-NEXT:    store i32 [[AB]], i32* [[R]], align 4
+; IS__TUNIT_OPM-NEXT:    ret void
 ;
+; IS__TUNIT_NPM: Function Attrs: argmemonly nofree nosync nounwind willreturn
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@add
 ; IS__TUNIT_NPM-SAME: ({ i32, i32 }* noalias nocapture nofree nonnull readonly align 8 dereferenceable(8) [[THIS:%.*]], i32* noalias nocapture nofree nonnull sret writeonly align 4 dereferenceable(4) [[R:%.*]])
 ; IS__TUNIT_NPM-NEXT:    [[AP:%.*]] = getelementptr { i32, i32 }, { i32, i32 }* [[THIS]], i32 0, i32 0
@@ -28,6 +30,17 @@ define internal void @add({i32, i32}* %this, i32* sret %r) {
 ; IS__TUNIT_NPM-NEXT:    [[AB:%.*]] = add i32 [[A]], [[B]]
 ; IS__TUNIT_NPM-NEXT:    store i32 [[AB]], i32* [[R]], align 4
 ; IS__TUNIT_NPM-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@add
+; IS__CGSCC____-SAME: ({ i32, i32 }* nocapture nofree nonnull readonly align 8 dereferenceable(8) [[THIS:%.*]], i32* nocapture nofree nonnull sret writeonly align 4 dereferenceable(4) [[R:%.*]])
+; IS__CGSCC____-NEXT:    [[AP:%.*]] = getelementptr { i32, i32 }, { i32, i32 }* [[THIS]], i32 0, i32 0
+; IS__CGSCC____-NEXT:    [[BP:%.*]] = getelementptr { i32, i32 }, { i32, i32 }* [[THIS]], i32 0, i32 1
+; IS__CGSCC____-NEXT:    [[A:%.*]] = load i32, i32* [[AP]], align 8
+; IS__CGSCC____-NEXT:    [[B:%.*]] = load i32, i32* [[BP]], align 4
+; IS__CGSCC____-NEXT:    [[AB:%.*]] = add i32 [[A]], [[B]]
+; IS__CGSCC____-NEXT:    store i32 [[AB]], i32* [[R]], align 4
+; IS__CGSCC____-NEXT:    ret void
 ;
   %ap = getelementptr {i32, i32}, {i32, i32}* %this, i32 0, i32 0
   %bp = getelementptr {i32, i32}, {i32, i32}* %this, i32 0, i32 1
@@ -39,17 +52,33 @@ define internal void @add({i32, i32}* %this, i32* sret %r) {
 }
 
 define void @f() {
-; IS________OPM-LABEL: define {{[^@]+}}@f()
-; IS________OPM-NEXT:    [[R:%.*]] = alloca i32, align 4
-; IS________OPM-NEXT:    [[PAIR:%.*]] = alloca { i32, i32 }, align 8
-; IS________OPM-NEXT:    call void @add({ i32, i32 }* nocapture nofree nonnull readonly align 8 dereferenceable(8) [[PAIR]], i32* nocapture nofree nonnull sret writeonly align 4 dereferenceable(4) [[R]])
-; IS________OPM-NEXT:    ret void
-;
-; IS________NPM-LABEL: define {{[^@]+}}@f()
-; IS________NPM-NEXT:    [[R:%.*]] = alloca i32, align 4
-; IS________NPM-NEXT:    [[PAIR:%.*]] = alloca { i32, i32 }, align 8
-; IS________NPM-NEXT:    call void @add({ i32, i32 }* noalias nocapture nofree nonnull readonly align 8 dereferenceable(8) [[PAIR]], i32* noalias nocapture nofree nonnull sret writeonly align 4 dereferenceable(4) [[R]])
-; IS________NPM-NEXT:    ret void
+; IS__TUNIT_OPM: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@f()
+; IS__TUNIT_OPM-NEXT:    [[R:%.*]] = alloca i32, align 4
+; IS__TUNIT_OPM-NEXT:    [[PAIR:%.*]] = alloca { i32, i32 }, align 8
+; IS__TUNIT_OPM-NEXT:    call void @add({ i32, i32 }* nocapture nofree nonnull readonly align 8 dereferenceable(8) [[PAIR]], i32* nocapture nofree nonnull sret writeonly align 4 dereferenceable(4) [[R]])
+; IS__TUNIT_OPM-NEXT:    ret void
+;
+; IS__TUNIT_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@f()
+; IS__TUNIT_NPM-NEXT:    [[R:%.*]] = alloca i32, align 4
+; IS__TUNIT_NPM-NEXT:    [[PAIR:%.*]] = alloca { i32, i32 }, align 8
+; IS__TUNIT_NPM-NEXT:    call void @add({ i32, i32 }* noalias nocapture nofree nonnull readonly align 8 dereferenceable(8) [[PAIR]], i32* noalias nocapture nofree nonnull sret writeonly align 4 dereferenceable(4) [[R]])
+; IS__TUNIT_NPM-NEXT:    ret void
+;
+; IS__CGSCC_OPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@f()
+; IS__CGSCC_OPM-NEXT:    [[R:%.*]] = alloca i32, align 4
+; IS__CGSCC_OPM-NEXT:    [[PAIR:%.*]] = alloca { i32, i32 }, align 8
+; IS__CGSCC_OPM-NEXT:    call void @add({ i32, i32 }* nocapture nofree nonnull readonly align 8 dereferenceable(8) [[PAIR]], i32* nocapture nofree nonnull sret writeonly align 4 dereferenceable(4) [[R]])
+; IS__CGSCC_OPM-NEXT:    ret void
+;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@f()
+; IS__CGSCC_NPM-NEXT:    [[R:%.*]] = alloca i32, align 4
+; IS__CGSCC_NPM-NEXT:    [[PAIR:%.*]] = alloca { i32, i32 }, align 8
+; IS__CGSCC_NPM-NEXT:    call void @add({ i32, i32 }* noalias nocapture nofree nonnull readonly align 8 dereferenceable(8) [[PAIR]], i32* noalias nocapture nofree nonnull sret writeonly align 4 dereferenceable(4) [[R]])
+; IS__CGSCC_NPM-NEXT:    ret void
 ;
   %r = alloca i32
   %pair = alloca {i32, i32}
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/tail.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/tail.ll
index 1920e0108f512..9d5dab8f3be09 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/tail.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/tail.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/variadic.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/variadic.ll
index 4cc49336fa6af..df9b420d11a27 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/variadic.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/variadic.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/2008-06-09-WeakProp.ll b/llvm/test/Transforms/Attributor/IPConstantProp/2008-06-09-WeakProp.ll
index 8ff87ae3bf336..ebd06c15bb63d 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/2008-06-09-WeakProp.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/2008-06-09-WeakProp.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -8,6 +8,7 @@
 ; PR2411
 
 define weak i32 @foo() nounwind  {
+; CHECK: Function Attrs: nounwind
 ; CHECK-LABEL: define {{[^@]+}}@foo()
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    ret i32 1
@@ -17,6 +18,7 @@ entry:
 }
 
 define i32 @main() nounwind  {
+; CHECK: Function Attrs: nounwind
 ; CHECK-LABEL: define {{[^@]+}}@main()
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[R:%.*]] = call i32 @foo()
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/2009-09-24-byval-ptr.ll b/llvm/test/Transforms/Attributor/IPConstantProp/2009-09-24-byval-ptr.ll
index e7d9a63f7076f..2801725d3662c 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/2009-09-24-byval-ptr.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/2009-09-24-byval-ptr.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -9,6 +9,7 @@
 %struct.MYstr = type { i8, i32 }
 @mystr = internal global %struct.MYstr zeroinitializer ; <%struct.MYstr*> [#uses=3]
 define internal void @vfu1(%struct.MYstr* byval align 4 %u) nounwind {
+; IS__CGSCC_OPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@vfu1
 ; IS__CGSCC_OPM-SAME: (%struct.MYstr* noalias nocapture nofree nonnull writeonly byval align 8 dereferenceable(8) [[U:%.*]])
 ; IS__CGSCC_OPM-NEXT:  entry:
@@ -20,6 +21,7 @@ define internal void @vfu1(%struct.MYstr* byval align 4 %u) nounwind {
 ; IS__CGSCC_OPM:       return:
 ; IS__CGSCC_OPM-NEXT:    ret void
 ;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@vfu1
 ; IS__CGSCC_NPM-SAME: (i8 [[TMP0:%.*]], i32 [[TMP1:%.*]])
 ; IS__CGSCC_NPM-NEXT:  entry:
@@ -48,6 +50,7 @@ return:                                           ; preds = %entry
 }
 
 define internal i32 @vfu2(%struct.MYstr* byval align 4 %u) nounwind readonly {
+; IS__TUNIT_OPM: Function Attrs: nofree nosync nounwind readonly willreturn
 ; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@vfu2
 ; IS__TUNIT_OPM-SAME: (%struct.MYstr* noalias nocapture nofree nonnull readonly byval align 8 dereferenceable(8) [[U:%.*]])
 ; IS__TUNIT_OPM-NEXT:  entry:
@@ -59,6 +62,7 @@ define internal i32 @vfu2(%struct.MYstr* byval align 4 %u) nounwind readonly {
 ; IS__TUNIT_OPM-NEXT:    [[TMP5:%.*]] = add i32 [[TMP4]], [[TMP1]]
 ; IS__TUNIT_OPM-NEXT:    ret i32 [[TMP5]]
 ;
+; IS__TUNIT_NPM: Function Attrs: nofree nosync nounwind readonly willreturn
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@vfu2
 ; IS__TUNIT_NPM-SAME: (i8 [[TMP0:%.*]], i32 [[TMP1:%.*]])
 ; IS__TUNIT_NPM-NEXT:  entry:
@@ -75,6 +79,7 @@ define internal i32 @vfu2(%struct.MYstr* byval align 4 %u) nounwind readonly {
 ; IS__TUNIT_NPM-NEXT:    [[TMP7:%.*]] = add i32 [[TMP6]], [[TMP3]]
 ; IS__TUNIT_NPM-NEXT:    ret i32 [[TMP7]]
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readonly willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@vfu2()
 ; IS__CGSCC____-NEXT:  entry:
 ; IS__CGSCC____-NEXT:    [[TMP0:%.*]] = getelementptr [[STRUCT_MYSTR:%.*]], %struct.MYstr* @mystr, i32 0, i32 1
@@ -96,11 +101,13 @@ entry:
 }
 
 define i32 @unions() nounwind {
+; IS__TUNIT_OPM: Function Attrs: nofree nosync nounwind willreturn
 ; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@unions()
 ; IS__TUNIT_OPM-NEXT:  entry:
 ; IS__TUNIT_OPM-NEXT:    [[RESULT:%.*]] = call i32 @vfu2(%struct.MYstr* nocapture nofree nonnull readonly byval align 8 dereferenceable(8) @mystr)
 ; IS__TUNIT_OPM-NEXT:    ret i32 [[RESULT]]
 ;
+; IS__TUNIT_NPM: Function Attrs: nofree nosync nounwind willreturn
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@unions()
 ; IS__TUNIT_NPM-NEXT:  entry:
 ; IS__TUNIT_NPM-NEXT:    [[MYSTR_CAST:%.*]] = bitcast %struct.MYstr* @mystr to i8*
@@ -110,6 +117,7 @@ define i32 @unions() nounwind {
 ; IS__TUNIT_NPM-NEXT:    [[RESULT:%.*]] = call i32 @vfu2(i8 [[TMP0]], i32 [[TMP1]])
 ; IS__TUNIT_NPM-NEXT:    ret i32 [[RESULT]]
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readonly willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@unions()
 ; IS__CGSCC____-NEXT:  entry:
 ; IS__CGSCC____-NEXT:    [[RESULT:%.*]] = call i32 @vfu2()
@@ -122,36 +130,71 @@ entry:
 }
 
 define internal i32 @vfu2_v2(%struct.MYstr* byval align 4 %u) nounwind readonly {
-; IS________OPM-LABEL: define {{[^@]+}}@vfu2_v2
-; IS________OPM-SAME: (%struct.MYstr* noalias nocapture nofree nonnull byval align 8 dereferenceable(8) [[U:%.*]])
-; IS________OPM-NEXT:  entry:
-; IS________OPM-NEXT:    [[Z:%.*]] = getelementptr [[STRUCT_MYSTR:%.*]], %struct.MYstr* [[U]], i32 0, i32 1
-; IS________OPM-NEXT:    store i32 99, i32* [[Z]], align 4
-; IS________OPM-NEXT:    [[TMP0:%.*]] = getelementptr [[STRUCT_MYSTR]], %struct.MYstr* [[U]], i32 0, i32 1
-; IS________OPM-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP0]], align 4
-; IS________OPM-NEXT:    [[TMP2:%.*]] = getelementptr [[STRUCT_MYSTR]], %struct.MYstr* [[U]], i32 0, i32 0
-; IS________OPM-NEXT:    [[TMP3:%.*]] = load i8, i8* [[TMP2]], align 8
-; IS________OPM-NEXT:    [[TMP4:%.*]] = zext i8 [[TMP3]] to i32
-; IS________OPM-NEXT:    [[TMP5:%.*]] = add i32 [[TMP4]], [[TMP1]]
-; IS________OPM-NEXT:    ret i32 [[TMP5]]
-;
-; IS________NPM-LABEL: define {{[^@]+}}@vfu2_v2
-; IS________NPM-SAME: (i8 [[TMP0:%.*]], i32 [[TMP1:%.*]])
-; IS________NPM-NEXT:  entry:
-; IS________NPM-NEXT:    [[U_PRIV:%.*]] = alloca [[STRUCT_MYSTR:%.*]], align 8
-; IS________NPM-NEXT:    [[U_PRIV_CAST:%.*]] = bitcast %struct.MYstr* [[U_PRIV]] to i8*
-; IS________NPM-NEXT:    store i8 [[TMP0]], i8* [[U_PRIV_CAST]], align 1
-; IS________NPM-NEXT:    [[U_PRIV_0_1:%.*]] = getelementptr [[STRUCT_MYSTR]], %struct.MYstr* [[U_PRIV]], i32 0, i32 1
-; IS________NPM-NEXT:    store i32 [[TMP1]], i32* [[U_PRIV_0_1]], align 4
-; IS________NPM-NEXT:    [[Z:%.*]] = getelementptr [[STRUCT_MYSTR]], %struct.MYstr* [[U_PRIV]], i32 0, i32 1
-; IS________NPM-NEXT:    store i32 99, i32* [[Z]], align 4
-; IS________NPM-NEXT:    [[TMP2:%.*]] = getelementptr [[STRUCT_MYSTR]], %struct.MYstr* [[U_PRIV]], i32 0, i32 1
-; IS________NPM-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP2]], align 4
-; IS________NPM-NEXT:    [[TMP4:%.*]] = getelementptr [[STRUCT_MYSTR]], %struct.MYstr* [[U_PRIV]], i32 0, i32 0
-; IS________NPM-NEXT:    [[TMP5:%.*]] = load i8, i8* [[TMP4]], align 8
-; IS________NPM-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i32
-; IS________NPM-NEXT:    [[TMP7:%.*]] = add i32 [[TMP6]], [[TMP3]]
-; IS________NPM-NEXT:    ret i32 [[TMP7]]
+; IS__TUNIT_OPM: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@vfu2_v2
+; IS__TUNIT_OPM-SAME: (%struct.MYstr* noalias nocapture nofree nonnull byval align 8 dereferenceable(8) [[U:%.*]])
+; IS__TUNIT_OPM-NEXT:  entry:
+; IS__TUNIT_OPM-NEXT:    [[Z:%.*]] = getelementptr [[STRUCT_MYSTR:%.*]], %struct.MYstr* [[U]], i32 0, i32 1
+; IS__TUNIT_OPM-NEXT:    store i32 99, i32* [[Z]], align 4
+; IS__TUNIT_OPM-NEXT:    [[TMP0:%.*]] = getelementptr [[STRUCT_MYSTR]], %struct.MYstr* [[U]], i32 0, i32 1
+; IS__TUNIT_OPM-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP0]], align 4
+; IS__TUNIT_OPM-NEXT:    [[TMP2:%.*]] = getelementptr [[STRUCT_MYSTR]], %struct.MYstr* [[U]], i32 0, i32 0
+; IS__TUNIT_OPM-NEXT:    [[TMP3:%.*]] = load i8, i8* [[TMP2]], align 8
+; IS__TUNIT_OPM-NEXT:    [[TMP4:%.*]] = zext i8 [[TMP3]] to i32
+; IS__TUNIT_OPM-NEXT:    [[TMP5:%.*]] = add i32 [[TMP4]], [[TMP1]]
+; IS__TUNIT_OPM-NEXT:    ret i32 [[TMP5]]
+;
+; IS__TUNIT_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@vfu2_v2
+; IS__TUNIT_NPM-SAME: (i8 [[TMP0:%.*]], i32 [[TMP1:%.*]])
+; IS__TUNIT_NPM-NEXT:  entry:
+; IS__TUNIT_NPM-NEXT:    [[U_PRIV:%.*]] = alloca [[STRUCT_MYSTR:%.*]], align 8
+; IS__TUNIT_NPM-NEXT:    [[U_PRIV_CAST:%.*]] = bitcast %struct.MYstr* [[U_PRIV]] to i8*
+; IS__TUNIT_NPM-NEXT:    store i8 [[TMP0]], i8* [[U_PRIV_CAST]], align 1
+; IS__TUNIT_NPM-NEXT:    [[U_PRIV_0_1:%.*]] = getelementptr [[STRUCT_MYSTR]], %struct.MYstr* [[U_PRIV]], i32 0, i32 1
+; IS__TUNIT_NPM-NEXT:    store i32 [[TMP1]], i32* [[U_PRIV_0_1]], align 4
+; IS__TUNIT_NPM-NEXT:    [[Z:%.*]] = getelementptr [[STRUCT_MYSTR]], %struct.MYstr* [[U_PRIV]], i32 0, i32 1
+; IS__TUNIT_NPM-NEXT:    store i32 99, i32* [[Z]], align 4
+; IS__TUNIT_NPM-NEXT:    [[TMP2:%.*]] = getelementptr [[STRUCT_MYSTR]], %struct.MYstr* [[U_PRIV]], i32 0, i32 1
+; IS__TUNIT_NPM-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP2]], align 4
+; IS__TUNIT_NPM-NEXT:    [[TMP4:%.*]] = getelementptr [[STRUCT_MYSTR]], %struct.MYstr* [[U_PRIV]], i32 0, i32 0
+; IS__TUNIT_NPM-NEXT:    [[TMP5:%.*]] = load i8, i8* [[TMP4]], align 8
+; IS__TUNIT_NPM-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i32
+; IS__TUNIT_NPM-NEXT:    [[TMP7:%.*]] = add i32 [[TMP6]], [[TMP3]]
+; IS__TUNIT_NPM-NEXT:    ret i32 [[TMP7]]
+;
+; IS__CGSCC_OPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@vfu2_v2
+; IS__CGSCC_OPM-SAME: (%struct.MYstr* noalias nocapture nofree nonnull byval align 8 dereferenceable(8) [[U:%.*]])
+; IS__CGSCC_OPM-NEXT:  entry:
+; IS__CGSCC_OPM-NEXT:    [[Z:%.*]] = getelementptr [[STRUCT_MYSTR:%.*]], %struct.MYstr* [[U]], i32 0, i32 1
+; IS__CGSCC_OPM-NEXT:    store i32 99, i32* [[Z]], align 4
+; IS__CGSCC_OPM-NEXT:    [[TMP0:%.*]] = getelementptr [[STRUCT_MYSTR]], %struct.MYstr* [[U]], i32 0, i32 1
+; IS__CGSCC_OPM-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP0]], align 4
+; IS__CGSCC_OPM-NEXT:    [[TMP2:%.*]] = getelementptr [[STRUCT_MYSTR]], %struct.MYstr* [[U]], i32 0, i32 0
+; IS__CGSCC_OPM-NEXT:    [[TMP3:%.*]] = load i8, i8* [[TMP2]], align 8
+; IS__CGSCC_OPM-NEXT:    [[TMP4:%.*]] = zext i8 [[TMP3]] to i32
+; IS__CGSCC_OPM-NEXT:    [[TMP5:%.*]] = add i32 [[TMP4]], [[TMP1]]
+; IS__CGSCC_OPM-NEXT:    ret i32 [[TMP5]]
+;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@vfu2_v2
+; IS__CGSCC_NPM-SAME: (i8 [[TMP0:%.*]], i32 [[TMP1:%.*]])
+; IS__CGSCC_NPM-NEXT:  entry:
+; IS__CGSCC_NPM-NEXT:    [[U_PRIV:%.*]] = alloca [[STRUCT_MYSTR:%.*]], align 8
+; IS__CGSCC_NPM-NEXT:    [[U_PRIV_CAST:%.*]] = bitcast %struct.MYstr* [[U_PRIV]] to i8*
+; IS__CGSCC_NPM-NEXT:    store i8 [[TMP0]], i8* [[U_PRIV_CAST]], align 1
+; IS__CGSCC_NPM-NEXT:    [[U_PRIV_0_1:%.*]] = getelementptr [[STRUCT_MYSTR]], %struct.MYstr* [[U_PRIV]], i32 0, i32 1
+; IS__CGSCC_NPM-NEXT:    store i32 [[TMP1]], i32* [[U_PRIV_0_1]], align 4
+; IS__CGSCC_NPM-NEXT:    [[Z:%.*]] = getelementptr [[STRUCT_MYSTR]], %struct.MYstr* [[U_PRIV]], i32 0, i32 1
+; IS__CGSCC_NPM-NEXT:    store i32 99, i32* [[Z]], align 4
+; IS__CGSCC_NPM-NEXT:    [[TMP2:%.*]] = getelementptr [[STRUCT_MYSTR]], %struct.MYstr* [[U_PRIV]], i32 0, i32 1
+; IS__CGSCC_NPM-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP2]], align 4
+; IS__CGSCC_NPM-NEXT:    [[TMP4:%.*]] = getelementptr [[STRUCT_MYSTR]], %struct.MYstr* [[U_PRIV]], i32 0, i32 0
+; IS__CGSCC_NPM-NEXT:    [[TMP5:%.*]] = load i8, i8* [[TMP4]], align 8
+; IS__CGSCC_NPM-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i32
+; IS__CGSCC_NPM-NEXT:    [[TMP7:%.*]] = add i32 [[TMP6]], [[TMP3]]
+; IS__CGSCC_NPM-NEXT:    ret i32 [[TMP7]]
 ;
 entry:
   %z = getelementptr %struct.MYstr, %struct.MYstr* %u, i32 0, i32 1
@@ -166,11 +209,13 @@ entry:
 }
 
 define i32 @unions_v2() nounwind {
+; IS__TUNIT_OPM: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@unions_v2()
 ; IS__TUNIT_OPM-NEXT:  entry:
 ; IS__TUNIT_OPM-NEXT:    [[RESULT:%.*]] = call i32 @vfu2_v2(%struct.MYstr* nocapture nofree nonnull readonly byval align 8 dereferenceable(8) @mystr)
 ; IS__TUNIT_OPM-NEXT:    ret i32 [[RESULT]]
 ;
+; IS__TUNIT_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@unions_v2()
 ; IS__TUNIT_NPM-NEXT:  entry:
 ; IS__TUNIT_NPM-NEXT:    [[MYSTR_CAST:%.*]] = bitcast %struct.MYstr* @mystr to i8*
@@ -180,11 +225,13 @@ define i32 @unions_v2() nounwind {
 ; IS__TUNIT_NPM-NEXT:    [[RESULT:%.*]] = call i32 @vfu2_v2(i8 [[TMP0]], i32 [[TMP1]])
 ; IS__TUNIT_NPM-NEXT:    ret i32 [[RESULT]]
 ;
+; IS__CGSCC_OPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@unions_v2()
 ; IS__CGSCC_OPM-NEXT:  entry:
 ; IS__CGSCC_OPM-NEXT:    [[RESULT:%.*]] = call i32 @vfu2_v2(%struct.MYstr* noalias nocapture nofree nonnull readnone byval align 8 dereferenceable(8) @mystr)
 ; IS__CGSCC_OPM-NEXT:    ret i32 [[RESULT]]
 ;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readonly willreturn
 ; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@unions_v2()
 ; IS__CGSCC_NPM-NEXT:  entry:
 ; IS__CGSCC_NPM-NEXT:    [[MYSTR_CAST1:%.*]] = bitcast %struct.MYstr* @mystr to i8*
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/PR16052.ll b/llvm/test/Transforms/Attributor/IPConstantProp/PR16052.ll
index 57912e7490465..8e66dcf04c183 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/PR16052.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/PR16052.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -8,11 +8,13 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i64 @fn2() {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@fn2()
 ; IS__TUNIT____-NEXT:  entry:
 ; IS__TUNIT____-NEXT:    [[CALL2:%.*]] = call i64 @fn1(i64 undef) #0, !range !0
 ; IS__TUNIT____-NEXT:    ret i64 [[CALL2]]
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@fn2()
 ; IS__CGSCC____-NEXT:  entry:
 ; IS__CGSCC____-NEXT:    [[CALL2:%.*]] = call i64 @fn1(i64 undef)
@@ -26,6 +28,7 @@ entry:
 }
 
 define i64 @fn2b(i32 %arg) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@fn2b
 ; IS__TUNIT____-SAME: (i32 [[ARG:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
@@ -34,6 +37,7 @@ define i64 @fn2b(i32 %arg) {
 ; IS__TUNIT____-NEXT:    [[CALL2:%.*]] = call i64 @fn1(i64 [[DIV]]) #0, !range !0
 ; IS__TUNIT____-NEXT:    ret i64 [[CALL2]]
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@fn2b
 ; IS__CGSCC____-SAME: (i32 [[ARG:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -50,11 +54,13 @@ entry:
 }
 
 define i64 @fn2c() {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@fn2c()
 ; IS__TUNIT____-NEXT:  entry:
 ; IS__TUNIT____-NEXT:    [[CALL2:%.*]] = call i64 @fn1(i64 42) #0, !range !0
 ; IS__TUNIT____-NEXT:    ret i64 [[CALL2]]
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@fn2c()
 ; IS__CGSCC____-NEXT:  entry:
 ; IS__CGSCC____-NEXT:    ret i64 42
@@ -67,12 +73,21 @@ entry:
 }
 
 define internal i64 @fn1(i64 %p1) {
-; CHECK-LABEL: define {{[^@]+}}@fn1
-; CHECK-SAME: (i64 returned [[P1:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i64 [[P1]], 0
-; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i64 [[P1]], i64 [[P1]]
-; CHECK-NEXT:    ret i64 [[COND]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@fn1
+; IS__TUNIT____-SAME: (i64 returned [[P1:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[TOBOOL:%.*]] = icmp ne i64 [[P1]], 0
+; IS__TUNIT____-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i64 [[P1]], i64 [[P1]]
+; IS__TUNIT____-NEXT:    ret i64 [[COND]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@fn1
+; IS__CGSCC____-SAME: (i64 returned [[P1:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[TOBOOL:%.*]] = icmp ne i64 [[P1]], 0
+; IS__CGSCC____-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i64 [[P1]], i64 [[P1]]
+; IS__CGSCC____-NEXT:    ret i64 [[COND]]
 ;
 entry:
   %tobool = icmp ne i64 %p1, 0
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/PR26044.ll b/llvm/test/Transforms/Attributor/IPConstantProp/PR26044.ll
index 05217a23ccf9a..ae1071032e984 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/PR26044.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/PR26044.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -8,6 +8,7 @@ target triple = "x86_64-unknown-linux-gnu"
 
 define void @fn2(i32* %P, i1 %C) {
 ;
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@fn2
 ; IS__TUNIT____-SAME: (i32* nocapture nofree [[P:%.*]], i1 [[C:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
@@ -23,6 +24,7 @@ define void @fn2(i32* %P, i1 %C) {
 ; IS__TUNIT____:       exit:
 ; IS__TUNIT____-NEXT:    ret void
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@fn2
 ; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull align 4 dereferenceable(4) [[P:%.*]], i1 [[C:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -55,12 +57,21 @@ exit:
 }
 
 define internal i32 @fn1(i32 %p1) {
-; CHECK-LABEL: define {{[^@]+}}@fn1
-; CHECK-SAME: (i32 returned [[P1:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[P1]], 0
-; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 [[P1]], i32 [[P1]]
-; CHECK-NEXT:    ret i32 [[COND]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@fn1
+; IS__TUNIT____-SAME: (i32 returned [[P1:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[P1]], 0
+; IS__TUNIT____-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 [[P1]], i32 [[P1]]
+; IS__TUNIT____-NEXT:    ret i32 [[COND]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@fn1
+; IS__CGSCC____-SAME: (i32 returned [[P1:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[P1]], 0
+; IS__CGSCC____-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 [[P1]], i32 [[P1]]
+; IS__CGSCC____-NEXT:    ret i32 [[COND]]
 ;
 entry:
   %tobool = icmp ne i32 %p1, 0
@@ -70,6 +81,7 @@ entry:
 
 define void @fn_no_null_opt(i32* %P, i1 %C) null_pointer_is_valid {
 ;
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind null_pointer_is_valid
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@fn_no_null_opt
 ; IS__TUNIT____-SAME: (i32* nocapture nofree writeonly [[P:%.*]], i1 [[C:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
@@ -85,6 +97,7 @@ define void @fn_no_null_opt(i32* %P, i1 %C) null_pointer_is_valid {
 ; IS__TUNIT____:       exit:
 ; IS__TUNIT____-NEXT:    ret void
 ;
+; IS__CGSCC_OPM: Function Attrs: nofree norecurse nosync nounwind null_pointer_is_valid
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@fn_no_null_opt
 ; IS__CGSCC_OPM-SAME: (i32* nocapture nofree writeonly align 4 dereferenceable_or_null(4) [[P:%.*]], i1 [[C:%.*]])
 ; IS__CGSCC_OPM-NEXT:  entry:
@@ -100,6 +113,7 @@ define void @fn_no_null_opt(i32* %P, i1 %C) null_pointer_is_valid {
 ; IS__CGSCC_OPM:       exit:
 ; IS__CGSCC_OPM-NEXT:    ret void
 ;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind null_pointer_is_valid
 ; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@fn_no_null_opt
 ; IS__CGSCC_NPM-SAME: (i32* nocapture nofree writeonly align 4 dereferenceable_or_null(4) [[P:%.*]], i1 [[C:%.*]])
 ; IS__CGSCC_NPM-NEXT:  entry:
@@ -132,12 +146,21 @@ exit:
 }
 
 define internal i32 @fn0(i32 %p1) {
-; CHECK-LABEL: define {{[^@]+}}@fn0
-; CHECK-SAME: (i32 returned [[P1:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[P1]], 0
-; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 [[P1]], i32 [[P1]]
-; CHECK-NEXT:    ret i32 [[COND]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@fn0
+; IS__TUNIT____-SAME: (i32 returned [[P1:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[P1]], 0
+; IS__TUNIT____-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 [[P1]], i32 [[P1]]
+; IS__TUNIT____-NEXT:    ret i32 [[COND]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@fn0
+; IS__CGSCC____-SAME: (i32 returned [[P1:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[P1]], 0
+; IS__CGSCC____-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 [[P1]], i32 [[P1]]
+; IS__CGSCC____-NEXT:    ret i32 [[COND]]
 ;
 entry:
   %tobool = icmp ne i32 %p1, 0
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/PR43857.ll b/llvm/test/Transforms/Attributor/IPConstantProp/PR43857.ll
index c894dc9bd767e..08e664239f318 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/PR43857.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/PR43857.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -10,21 +10,30 @@
 declare dso_local fastcc float @bar(%struct.wobble* noalias, <8 x i32>) unnamed_addr
 
 define %struct.zot @widget(<8 x i32> %arg) local_unnamed_addr {
-; CHECK-LABEL: define {{[^@]+}}@widget
-; CHECK-SAME: (<8 x i32> [[ARG:%.*]]) local_unnamed_addr
-; CHECK-NEXT:  bb:
-; CHECK-NEXT:    ret [[STRUCT_ZOT:%.*]] undef
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@widget
+; IS__TUNIT____-SAME: (<8 x i32> [[ARG:%.*]]) local_unnamed_addr
+; IS__TUNIT____-NEXT:  bb:
+; IS__TUNIT____-NEXT:    ret [[STRUCT_ZOT:%.*]] undef
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@widget
+; IS__CGSCC____-SAME: (<8 x i32> [[ARG:%.*]]) local_unnamed_addr
+; IS__CGSCC____-NEXT:  bb:
+; IS__CGSCC____-NEXT:    ret [[STRUCT_ZOT:%.*]] undef
 ;
 bb:
   ret %struct.zot undef
 }
 
 define void @baz(<8 x i32> %arg) local_unnamed_addr {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@baz
 ; IS__TUNIT____-SAME: (<8 x i32> [[ARG:%.*]]) local_unnamed_addr
 ; IS__TUNIT____-NEXT:  bb:
 ; IS__TUNIT____-NEXT:    ret void
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@baz
 ; IS__CGSCC____-SAME: (<8 x i32> [[ARG:%.*]]) local_unnamed_addr
 ; IS__CGSCC____-NEXT:  bb:
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/arg-count-mismatch.ll b/llvm/test/Transforms/Attributor/IPConstantProp/arg-count-mismatch.ll
index 4f177224d3143..1124b5d012800 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/arg-count-mismatch.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/arg-count-mismatch.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -36,18 +36,29 @@
 ; FIXME we should recognize this as UB and make it an unreachable.
 
 define dso_local i16 @foo(i16 %a) {
-; CHECK-LABEL: define {{[^@]+}}@foo
-; CHECK-SAME: (i16 [[A:%.*]])
-; CHECK-NEXT:    [[CALL:%.*]] = call i16 @bar()
-; CHECK-NEXT:    ret i16 [[CALL]]
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@foo
+; NOT_CGSCC_NPM-SAME: (i16 [[A:%.*]])
+; NOT_CGSCC_NPM-NEXT:    [[CALL:%.*]] = call i16 @bar()
+; NOT_CGSCC_NPM-NEXT:    ret i16 [[CALL]]
+;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@foo
+; IS__CGSCC_NPM-SAME: (i16 [[A:%.*]])
+; IS__CGSCC_NPM-NEXT:    [[CALL:%.*]] = call i16 @bar()
+; IS__CGSCC_NPM-NEXT:    ret i16 [[CALL]]
 ;
   %call = call i16 bitcast (i16 (i16, i16) * @bar to i16 (i16) *)(i16 %a)
   ret i16 %call
 }
 
 define internal i16 @bar(i16 %p1, i16 %p2) {
-; CHECK-LABEL: define {{[^@]+}}@bar()
-; CHECK-NEXT:    ret i16 0
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@bar()
+; IS__TUNIT____-NEXT:    ret i16 0
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@bar()
+; IS__CGSCC____-NEXT:    ret i16 0
 ;
   ret i16 0
 }
@@ -63,10 +74,17 @@ define dso_local i16 @foo2(i16 %a) {
 }
 
 define internal i16 @bar2(i16 %p1, i16 %p2) {
-; CHECK-LABEL: define {{[^@]+}}@bar2
-; CHECK-SAME: (i16 [[P1:%.*]], i16 [[P2:%.*]])
-; CHECK-NEXT:    [[A:%.*]] = add i16 [[P1]], [[P2]]
-; CHECK-NEXT:    ret i16 [[A]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@bar2
+; IS__TUNIT____-SAME: (i16 [[P1:%.*]], i16 [[P2:%.*]])
+; IS__TUNIT____-NEXT:    [[A:%.*]] = add i16 [[P1]], [[P2]]
+; IS__TUNIT____-NEXT:    ret i16 [[A]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@bar2
+; IS__CGSCC____-SAME: (i16 [[P1:%.*]], i16 [[P2:%.*]])
+; IS__CGSCC____-NEXT:    [[A:%.*]] = add i16 [[P1]], [[P2]]
+; IS__CGSCC____-NEXT:    ret i16 [[A]]
 ;
   %a = add i16 %p1, %p2
   ret i16 %a
@@ -99,6 +117,7 @@ define dso_local i16 @vararg_tests(i16 %a) {
 }
 
 define internal i16 @vararg_prop(i16 %p1, ...) {
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@vararg_prop
 ; IS__CGSCC____-SAME: (i16 returned [[P1:%.*]], ...)
 ; IS__CGSCC____-NEXT:    ret i16 7
@@ -107,9 +126,15 @@ define internal i16 @vararg_prop(i16 %p1, ...) {
 }
 
 define internal i16 @vararg_no_prop(i16 %p1, i16 %p2, ...) {
-; CHECK-LABEL: define {{[^@]+}}@vararg_no_prop
-; CHECK-SAME: (i16 returned [[P1:%.*]], i16 [[P2:%.*]], ...)
-; CHECK-NEXT:    ret i16 7
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@vararg_no_prop
+; IS__TUNIT____-SAME: (i16 returned [[P1:%.*]], i16 [[P2:%.*]], ...)
+; IS__TUNIT____-NEXT:    ret i16 7
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@vararg_no_prop
+; IS__CGSCC____-SAME: (i16 returned [[P1:%.*]], i16 [[P2:%.*]], ...)
+; IS__CGSCC____-NEXT:    ret i16 7
 ;
   ret i16 %p1
 }
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/arg-type-mismatch.ll b/llvm/test/Transforms/Attributor/IPConstantProp/arg-type-mismatch.ll
index 2d38afb5dfc98..adc52ab707d17 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/arg-type-mismatch.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/arg-type-mismatch.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -18,9 +18,15 @@ define dso_local i16 @foo(i16 %a) {
 }
 
 define internal i16 @bar(i16 %p1, i16 %p2) {
-; CHECK-LABEL: define {{[^@]+}}@bar
-; CHECK-SAME: (i16 [[P1:%.*]], i16 returned [[P2:%.*]])
-; CHECK-NEXT:    ret i16 [[P2]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@bar
+; IS__TUNIT____-SAME: (i16 [[P1:%.*]], i16 returned [[P2:%.*]])
+; IS__TUNIT____-NEXT:    ret i16 [[P2]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@bar
+; IS__CGSCC____-SAME: (i16 [[P1:%.*]], i16 returned [[P2:%.*]])
+; IS__CGSCC____-NEXT:    ret i16 [[P2]]
 ;
   ret i16 %p2
 }
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/comdat-ipo.ll b/llvm/test/Transforms/Attributor/IPConstantProp/comdat-ipo.ll
index 9ba9f69f4e006..0a57b47508ed0 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/comdat-ipo.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/comdat-ipo.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -7,8 +7,13 @@
 ; See PR26774
 
 define i32 @baz() {
-; CHECK-LABEL: define {{[^@]+}}@baz()
-; CHECK-NEXT:    ret i32 10
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@baz()
+; IS__TUNIT____-NEXT:    ret i32 10
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@baz()
+; IS__CGSCC____-NEXT:    ret i32 10
 ;
   ret i32 10
 }
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/dangling-block-address.ll b/llvm/test/Transforms/Attributor/IPConstantProp/dangling-block-address.ll
index aa1374a7c3c51..381e2d58bc796 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/dangling-block-address.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/dangling-block-address.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -15,6 +15,7 @@
 @bar.l = internal constant [2 x i8*] [i8* blockaddress(@bar, %lab0), i8* blockaddress(@bar, %end)] ; <[2 x i8*]*> [#uses=1]
 
 define internal void @foo(i32 %x) nounwind readnone {
+; IS__CGSCC____: Function Attrs: nounwind readnone
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@foo
 ; IS__CGSCC____-SAME: (i32 [[X:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -29,6 +30,7 @@ entry:
 }
 
 define internal void @bar(i32* nocapture %pc) nounwind readonly {
+; IS__CGSCC_OPM: Function Attrs: nounwind readonly
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@bar
 ; IS__CGSCC_OPM-SAME: (i32* nocapture [[PC:%.*]])
 ; IS__CGSCC_OPM-NEXT:  entry:
@@ -66,9 +68,15 @@ indirectgoto:                                     ; preds = %lab0, %entry
 }
 
 define i32 @main() nounwind readnone {
-; CHECK-LABEL: define {{[^@]+}}@main()
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i32 0
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@main()
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    ret i32 0
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@main()
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    ret i32 0
 ;
 entry:
   ret i32 0
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/deadarg.ll b/llvm/test/Transforms/Attributor/IPConstantProp/deadarg.ll
index c46e9d103d28f..4d78ab0906d3f 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/deadarg.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/deadarg.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/fp-bc-icmp-const-fold.ll b/llvm/test/Transforms/Attributor/IPConstantProp/fp-bc-icmp-const-fold.ll
index 4cd030b408f5e..c45c48d502343 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/fp-bc-icmp-const-fold.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/fp-bc-icmp-const-fold.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -7,32 +7,61 @@ target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64-bgq-linux"
 
 define void @test(i32 signext %n) {
-; CHECK-LABEL: define {{[^@]+}}@test
-; CHECK-SAME: (i32 signext [[N:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    unreachable
-; CHECK:       if.then:
-; CHECK-NEXT:    unreachable
-; CHECK:       if.end:
-; CHECK-NEXT:    unreachable
-; CHECK:       if.then2:
-; CHECK-NEXT:    unreachable
-; CHECK:       if.end4:
-; CHECK-NEXT:    unreachable
-; CHECK:       if.then9:
-; CHECK-NEXT:    unreachable
-; CHECK:       if.then12:
-; CHECK-NEXT:    unreachable
-; CHECK:       if.else14:
-; CHECK-NEXT:    unreachable
-; CHECK:       do.body:
-; CHECK-NEXT:    unreachable
-; CHECK:       if.then33:
-; CHECK-NEXT:    unreachable
-; CHECK:       cond.false.i28:
-; CHECK-NEXT:    unreachable
-; CHECK:       _ZN5boost4math4signIgEEiRKT_.exit30:
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree noreturn nosync nounwind readnone
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test
+; IS__TUNIT____-SAME: (i32 signext [[N:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       if.then:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       if.end:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       if.then2:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       if.end4:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       if.then9:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       if.then12:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       if.else14:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       do.body:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       if.then33:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       cond.false.i28:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       _ZN5boost4math4signIgEEiRKT_.exit30:
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse noreturn nosync nounwind readnone
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test
+; IS__CGSCC____-SAME: (i32 signext [[N:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       if.then:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       if.end:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       if.then2:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       if.end4:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       if.then9:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       if.then12:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       if.else14:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       do.body:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       if.then33:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       cond.false.i28:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       _ZN5boost4math4signIgEEiRKT_.exit30:
+; IS__CGSCC____-NEXT:    unreachable
 ;
 
 entry:
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/global.ll b/llvm/test/Transforms/Attributor/IPConstantProp/global.ll
index fbd0d9f74d579..d30c764e5258b 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/global.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/global.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -7,16 +7,29 @@
 @_ZL6test1g = internal global i32 42, align 4
 
 define void @_Z7test1f1v() nounwind {
-; CHECK-LABEL: define {{[^@]+}}@_Z7test1f1v()
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP:%.*]] = load i32, i32* @_ZL6test1g, align 4
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP]], 0
-; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    store i32 0, i32* @_ZL6test1g, align 4
-; CHECK-NEXT:    br label [[IF_END]]
-; CHECK:       if.end:
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@_Z7test1f1v()
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[TMP:%.*]] = load i32, i32* @_ZL6test1g, align 4
+; IS__TUNIT____-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP]], 0
+; IS__TUNIT____-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; IS__TUNIT____:       if.then:
+; IS__TUNIT____-NEXT:    store i32 0, i32* @_ZL6test1g, align 4
+; IS__TUNIT____-NEXT:    br label [[IF_END]]
+; IS__TUNIT____:       if.end:
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@_Z7test1f1v()
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[TMP:%.*]] = load i32, i32* @_ZL6test1g, align 4
+; IS__CGSCC____-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP]], 0
+; IS__CGSCC____-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; IS__CGSCC____:       if.then:
+; IS__CGSCC____-NEXT:    store i32 0, i32* @_ZL6test1g, align 4
+; IS__CGSCC____-NEXT:    br label [[IF_END]]
+; IS__CGSCC____:       if.end:
+; IS__CGSCC____-NEXT:    ret void
 ;
 entry:
   %tmp = load i32, i32* @_ZL6test1g, align 4
@@ -32,10 +45,17 @@ if.end:                                           ; preds = %if.then, %entry
 }
 
 define i32 @_Z7test1f2v() nounwind {
-; CHECK-LABEL: define {{[^@]+}}@_Z7test1f2v()
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP:%.*]] = load i32, i32* @_ZL6test1g, align 4
-; CHECK-NEXT:    ret i32 [[TMP]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@_Z7test1f2v()
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[TMP:%.*]] = load i32, i32* @_ZL6test1g, align 4
+; IS__TUNIT____-NEXT:    ret i32 [[TMP]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@_Z7test1f2v()
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[TMP:%.*]] = load i32, i32* @_ZL6test1g, align 4
+; IS__CGSCC____-NEXT:    ret i32 [[TMP]]
 ;
 entry:
   %tmp = load i32, i32* @_ZL6test1g, align 4
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/multiple_callbacks.ll b/llvm/test/Transforms/Attributor/IPConstantProp/multiple_callbacks.ll
index 7232d969667d2..c1a50c55e8477 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/multiple_callbacks.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/multiple_callbacks.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -40,30 +40,51 @@
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 define internal i32 @cb0(i32 %zero) {
-; CHECK-LABEL: define {{[^@]+}}@cb0
-; CHECK-SAME: (i32 returned [[ZERO:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i32 0
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@cb0
+; IS__TUNIT____-SAME: (i32 returned [[ZERO:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    ret i32 0
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@cb0
+; IS__CGSCC____-SAME: (i32 returned [[ZERO:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    ret i32 0
 ;
 entry:
   ret i32 %zero
 }
 
 define internal i32 @cb1(i32 %unknown) {
-; CHECK-LABEL: define {{[^@]+}}@cb1
-; CHECK-SAME: (i32 returned [[UNKNOWN:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i32 [[UNKNOWN]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@cb1
+; IS__TUNIT____-SAME: (i32 returned [[UNKNOWN:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    ret i32 [[UNKNOWN]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@cb1
+; IS__CGSCC____-SAME: (i32 returned [[UNKNOWN:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    ret i32 [[UNKNOWN]]
 ;
 entry:
   ret i32 %unknown
 }
 
 define internal i32 @cb2(i32 %unknown) {
-; CHECK-LABEL: define {{[^@]+}}@cb2
-; CHECK-SAME: (i32 returned [[UNKNOWN:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i32 [[UNKNOWN]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@cb2
+; IS__TUNIT____-SAME: (i32 returned [[UNKNOWN:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    ret i32 [[UNKNOWN]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@cb2
+; IS__CGSCC____-SAME: (i32 returned [[UNKNOWN:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    ret i32 [[UNKNOWN]]
 ;
 entry:
   %call = call i32 @cb0(i32 0)
@@ -71,20 +92,34 @@ entry:
 }
 
 define internal i32 @cb3(i32 %unknown) {
-; CHECK-LABEL: define {{[^@]+}}@cb3
-; CHECK-SAME: (i32 returned [[UNKNOWN:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i32 [[UNKNOWN]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@cb3
+; IS__TUNIT____-SAME: (i32 returned [[UNKNOWN:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    ret i32 [[UNKNOWN]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@cb3
+; IS__CGSCC____-SAME: (i32 returned [[UNKNOWN:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    ret i32 [[UNKNOWN]]
 ;
 entry:
   ret i32 %unknown
 }
 
 define internal i32 @cb4(i32 %unknown) {
-; CHECK-LABEL: define {{[^@]+}}@cb4
-; CHECK-SAME: (i32 returned [[UNKNOWN:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i32 [[UNKNOWN]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@cb4
+; IS__TUNIT____-SAME: (i32 returned [[UNKNOWN:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    ret i32 [[UNKNOWN]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@cb4
+; IS__CGSCC____-SAME: (i32 returned [[UNKNOWN:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    ret i32 [[UNKNOWN]]
 ;
 entry:
   ret i32 %unknown
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/musttail-call.ll b/llvm/test/Transforms/Attributor/IPConstantProp/musttail-call.ll
index 8fcbbba977ed1..467d641bac07a 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/musttail-call.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/musttail-call.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -86,6 +86,7 @@ define internal i8* @side_effects(i8 %v) {
 }
 
 define internal i8* @no_side_effects(i8 %v) readonly nounwind {
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@no_side_effects
 ; IS__CGSCC____-SAME: (i8 [[V:%.*]])
 ; IS__CGSCC____-NEXT:    ret i8* null
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/naked-return.ll b/llvm/test/Transforms/Attributor/IPConstantProp/naked-return.ll
index e31670ab773bd..e954d34652425 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/naked-return.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/naked-return.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -8,6 +8,7 @@ target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
 target triple = "i686-pc-windows-msvc19.0.24215"
 
 define i32 @dipsy(i32, i32) local_unnamed_addr #0 {
+; CHECK: Function Attrs: naked
 ; CHECK-LABEL: define {{[^@]+}}@dipsy
 ; CHECK-SAME: (i32 [[TMP0:%.*]], i32 [[TMP1:%.*]]) local_unnamed_addr
 ; CHECK-NEXT:  BasicBlock0:
@@ -20,6 +21,7 @@ BasicBlock0:
 }
 
 define void @tinkywinky(i32, i32, i32) local_unnamed_addr #0 {
+; CHECK: Function Attrs: naked
 ; CHECK-LABEL: define {{[^@]+}}@tinkywinky
 ; CHECK-SAME: (i32 [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]]) local_unnamed_addr
 ; CHECK-NEXT:  BasicBlock1:
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/openmp_parallel_for.ll b/llvm/test/Transforms/Attributor/IPConstantProp/openmp_parallel_for.ll
index 4b19fbebbcb1e..00ca0cc92e8b3 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/openmp_parallel_for.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/openmp_parallel_for.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=7 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/pthreads.ll b/llvm/test/Transforms/Attributor/IPConstantProp/pthreads.ll
index b28b38f896877..90021fdd57560 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/pthreads.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/pthreads.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=7 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=7 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -67,21 +67,30 @@ entry:
 declare !callback !0 dso_local i32 @pthread_create(i64*, %union.pthread_attr_t*, i8* (i8*)*, i8*)
 
 define internal i8* @foo(i8* %arg) {
-; CHECK-LABEL: define {{[^@]+}}@foo
-; CHECK-SAME: (i8* noalias nofree readnone returned align 536870912 "no-capture-maybe-returned" [[ARG:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i8* null
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@foo
+; IS__TUNIT____-SAME: (i8* noalias nofree readnone returned align 536870912 "no-capture-maybe-returned" [[ARG:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    ret i8* null
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@foo
+; IS__CGSCC____-SAME: (i8* noalias nofree readnone returned align 536870912 "no-capture-maybe-returned" [[ARG:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    ret i8* null
 ;
 entry:
   ret i8* %arg
 }
 
 define internal i8* @bar(i8* %arg) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@bar
 ; IS__TUNIT____-SAME: (i8* noalias nofree nonnull readnone returned align 8 dereferenceable(8) "no-capture-maybe-returned" [[ARG:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
 ; IS__TUNIT____-NEXT:    ret i8* bitcast (i8** @GlobalVPtr to i8*)
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@bar
 ; IS__CGSCC____-SAME: (i8* nofree readnone returned "no-capture-maybe-returned" [[ARG:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -92,11 +101,13 @@ entry:
 }
 
 define internal i8* @baz(i8* %arg) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@baz
 ; IS__TUNIT____-SAME: (i8* noalias nofree nonnull readnone returned align 8 dereferenceable(1) "no-capture-maybe-returned" [[ARG:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
 ; IS__TUNIT____-NEXT:    ret i8* [[ARG]]
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@baz
 ; IS__CGSCC____-SAME: (i8* nofree nonnull readnone returned align 8 dereferenceable(1) "no-capture-maybe-returned" [[ARG:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -107,11 +118,13 @@ entry:
 }
 
 define internal i8* @buz(i8* %arg) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@buz
 ; IS__TUNIT____-SAME: (i8* noalias nofree nonnull readnone returned align 8 dereferenceable(1) "no-capture-maybe-returned" [[ARG:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
 ; IS__TUNIT____-NEXT:    ret i8* [[ARG]]
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@buz
 ; IS__CGSCC____-SAME: (i8* nofree nonnull readnone returned align 8 dereferenceable(1) "no-capture-maybe-returned" [[ARG:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/recursion.ll b/llvm/test/Transforms/Attributor/IPConstantProp/recursion.ll
index 717552aee1c3e..4d9a6ce4600f6 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/recursion.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/recursion.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -7,6 +7,7 @@
 ; CHECK-NOT: %X
 
 define internal i32 @foo(i32 %X) {
+; IS__CGSCC____: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@foo()
 ; IS__CGSCC____-NEXT:    unreachable
 ;
@@ -16,8 +17,13 @@ define internal i32 @foo(i32 %X) {
 }
 
 define void @bar() {
-; CHECK-LABEL: define {{[^@]+}}@bar()
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree noreturn nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@bar()
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@bar()
+; IS__CGSCC____-NEXT:    unreachable
 ;
   call i32 @foo( i32 17 )         ; <i32>:1 [#uses=0]
   ret void
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/remove-call-inst.ll b/llvm/test/Transforms/Attributor/IPConstantProp/remove-call-inst.ll
index a0d71566c3ba7..63115d831883c 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/remove-call-inst.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/remove-call-inst.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -11,9 +11,15 @@
 ; FIXME: Remove obsolete calls/instructions
 
 define i32 @main() noreturn nounwind {
-; CHECK-LABEL: define {{[^@]+}}@main()
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i32 123
+; IS__TUNIT____: Function Attrs: nofree noreturn nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@main()
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    ret i32 123
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@main()
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    ret i32 123
 ;
 entry:
   %call2 = tail call i32 @wwrite(i64 0) nounwind
@@ -21,6 +27,7 @@ entry:
 }
 
 define internal i32 @wwrite(i64 %i) nounwind readnone {
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@wwrite()
 ; IS__CGSCC____-NEXT:  entry:
 ; IS__CGSCC____-NEXT:    switch i64 0, label [[SW_DEFAULT:%.*]] [
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/return-argument.ll b/llvm/test/Transforms/Attributor/IPConstantProp/return-argument.ll
index 040be04ec3eee..933887fb3cc04 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/return-argument.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/return-argument.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -6,6 +6,7 @@
 
 ;; This function returns its second argument on all return statements
 define internal i32* @incdec(i1 %C, i32* %V) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@incdec
 ; IS__TUNIT____-SAME: (i1 [[C:%.*]], i32* noalias nofree nonnull returned align 4 dereferenceable(4) "no-capture-maybe-returned" [[V:%.*]])
 ; IS__TUNIT____-NEXT:    [[X:%.*]] = load i32, i32* [[V]], align 4
@@ -19,6 +20,7 @@ define internal i32* @incdec(i1 %C, i32* %V) {
 ; IS__TUNIT____-NEXT:    store i32 [[X2]], i32* [[V]], align 4
 ; IS__TUNIT____-NEXT:    ret i32* [[V]]
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@incdec
 ; IS__CGSCC____-SAME: (i1 [[C:%.*]], i32* nofree nonnull returned align 4 dereferenceable(4) "no-capture-maybe-returned" [[V:%.*]])
 ; IS__CGSCC____-NEXT:    [[X:%.*]] = load i32, i32* [[V]], align 4
@@ -49,12 +51,21 @@ F:              ; preds = %0
 ;; This function returns its first argument as a part of a multiple return
 ;; value
 define internal { i32, i32 } @foo(i32 %A, i32 %B) {
-; CHECK-LABEL: define {{[^@]+}}@foo
-; CHECK-SAME: (i32 [[A:%.*]], i32 [[B:%.*]])
-; CHECK-NEXT:    [[X:%.*]] = add i32 [[A]], [[B]]
-; CHECK-NEXT:    [[Y:%.*]] = insertvalue { i32, i32 } undef, i32 [[A]], 0
-; CHECK-NEXT:    [[Z:%.*]] = insertvalue { i32, i32 } [[Y]], i32 [[X]], 1
-; CHECK-NEXT:    ret { i32, i32 } [[Z]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@foo
+; IS__TUNIT____-SAME: (i32 [[A:%.*]], i32 [[B:%.*]])
+; IS__TUNIT____-NEXT:    [[X:%.*]] = add i32 [[A]], [[B]]
+; IS__TUNIT____-NEXT:    [[Y:%.*]] = insertvalue { i32, i32 } undef, i32 [[A]], 0
+; IS__TUNIT____-NEXT:    [[Z:%.*]] = insertvalue { i32, i32 } [[Y]], i32 [[X]], 1
+; IS__TUNIT____-NEXT:    ret { i32, i32 } [[Z]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@foo
+; IS__CGSCC____-SAME: (i32 [[A:%.*]], i32 [[B:%.*]])
+; IS__CGSCC____-NEXT:    [[X:%.*]] = add i32 [[A]], [[B]]
+; IS__CGSCC____-NEXT:    [[Y:%.*]] = insertvalue { i32, i32 } undef, i32 [[A]], 0
+; IS__CGSCC____-NEXT:    [[Z:%.*]] = insertvalue { i32, i32 } [[Y]], i32 [[X]], 1
+; IS__CGSCC____-NEXT:    ret { i32, i32 } [[Z]]
 ;
   %X = add i32 %A, %B
   %Y = insertvalue { i32, i32 } undef, i32 %A, 0
@@ -63,6 +74,7 @@ define internal { i32, i32 } @foo(i32 %A, i32 %B) {
 }
 
 define void @caller(i1 %C) personality i32 (...)* @__gxx_personality_v0 {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@caller
 ; IS__TUNIT____-SAME: (i1 [[C:%.*]]) #2 personality i32 (...)* @__gxx_personality_v0
 ; IS__TUNIT____-NEXT:    [[Q:%.*]] = alloca i32, align 4
@@ -81,6 +93,7 @@ define void @caller(i1 %C) personality i32 (...)* @__gxx_personality_v0 {
 ; IS__TUNIT____:       RET:
 ; IS__TUNIT____-NEXT:    ret void
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@caller
 ; IS__CGSCC____-SAME: (i1 [[C:%.*]]) #1 personality i32 (...)* @__gxx_personality_v0
 ; IS__CGSCC____-NEXT:    [[Q:%.*]] = alloca i32, align 4
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/return-constant.ll b/llvm/test/Transforms/Attributor/IPConstantProp/return-constant.ll
index c827669374711..e598d759dc4e9 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/return-constant.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/return-constant.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -7,14 +7,25 @@
 ; FIXME: icmp folding is missing
 
 define i1 @invokecaller(i1 %C) personality i32 (...)* @__gxx_personality_v0 {
-; CHECK-LABEL: define {{[^@]+}}@invokecaller
-; CHECK-SAME: (i1 [[C:%.*]]) #0 personality i32 (...)* @__gxx_personality_v0
-; CHECK-NEXT:    [[X:%.*]] = call i32 @foo(i1 [[C]])
-; CHECK-NEXT:    br label [[OK:%.*]]
-; CHECK:       OK:
-; CHECK-NEXT:    ret i1 true
-; CHECK:       FAIL:
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@invokecaller
+; IS__TUNIT____-SAME: (i1 [[C:%.*]]) #0 personality i32 (...)* @__gxx_personality_v0
+; IS__TUNIT____-NEXT:    [[X:%.*]] = call i32 @foo(i1 [[C]])
+; IS__TUNIT____-NEXT:    br label [[OK:%.*]]
+; IS__TUNIT____:       OK:
+; IS__TUNIT____-NEXT:    ret i1 true
+; IS__TUNIT____:       FAIL:
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@invokecaller
+; IS__CGSCC____-SAME: (i1 [[C:%.*]]) #0 personality i32 (...)* @__gxx_personality_v0
+; IS__CGSCC____-NEXT:    [[X:%.*]] = call i32 @foo(i1 [[C]])
+; IS__CGSCC____-NEXT:    br label [[OK:%.*]]
+; IS__CGSCC____:       OK:
+; IS__CGSCC____-NEXT:    ret i1 true
+; IS__CGSCC____:       FAIL:
+; IS__CGSCC____-NEXT:    unreachable
 ;
   %X = invoke i32 @foo( i1 %C ) to label %OK unwind label %FAIL             ; <i32> [#uses=1]
 OK:
@@ -27,6 +38,7 @@ FAIL:
 }
 
 define internal i32 @foo(i1 %C) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@foo
 ; IS__TUNIT____-SAME: (i1 [[C:%.*]])
 ; IS__TUNIT____-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
@@ -35,6 +47,7 @@ define internal i32 @foo(i1 %C) {
 ; IS__TUNIT____:       F:
 ; IS__TUNIT____-NEXT:    ret i32 undef
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@foo
 ; IS__CGSCC____-SAME: (i1 [[C:%.*]])
 ; IS__CGSCC____-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
@@ -53,9 +66,15 @@ F:              ; preds = %0
 }
 
 define i1 @caller(i1 %C) {
-; CHECK-LABEL: define {{[^@]+}}@caller
-; CHECK-SAME: (i1 [[C:%.*]])
-; CHECK-NEXT:    ret i1 true
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@caller
+; IS__TUNIT____-SAME: (i1 [[C:%.*]])
+; IS__TUNIT____-NEXT:    ret i1 true
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@caller
+; IS__CGSCC____-SAME: (i1 [[C:%.*]])
+; IS__CGSCC____-NEXT:    ret i1 true
 ;
   %X = call i32 @foo( i1 %C )             ; <i32> [#uses=1]
   %Y = icmp ne i32 %X, 0          ; <i1> [#uses=1]
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/return-constants.ll b/llvm/test/Transforms/Attributor/IPConstantProp/return-constants.ll
index 26d4206dcdf02..0934412e9112a 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/return-constants.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/return-constants.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=8 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=8 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -9,17 +9,31 @@
 %0 = type { i32, i32 }
 
 define internal %0 @foo(i1 %Q) {
-; CHECK-LABEL: define {{[^@]+}}@foo
-; CHECK-SAME: (i1 [[Q:%.*]])
-; CHECK-NEXT:    br i1 [[Q]], label [[T:%.*]], label [[F:%.*]]
-; CHECK:       T:
-; CHECK-NEXT:    [[MRV:%.*]] = insertvalue [[TMP0:%.*]] undef, i32 21, 0
-; CHECK-NEXT:    [[MRV1:%.*]] = insertvalue [[TMP0]] %mrv, i32 22, 1
-; CHECK-NEXT:    ret [[TMP0]] %mrv1
-; CHECK:       F:
-; CHECK-NEXT:    [[MRV2:%.*]] = insertvalue [[TMP0]] undef, i32 21, 0
-; CHECK-NEXT:    [[MRV3:%.*]] = insertvalue [[TMP0]] %mrv2, i32 23, 1
-; CHECK-NEXT:    ret [[TMP0]] %mrv3
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@foo
+; IS__TUNIT____-SAME: (i1 [[Q:%.*]])
+; IS__TUNIT____-NEXT:    br i1 [[Q]], label [[T:%.*]], label [[F:%.*]]
+; IS__TUNIT____:       T:
+; IS__TUNIT____-NEXT:    [[MRV:%.*]] = insertvalue [[TMP0:%.*]] undef, i32 21, 0
+; IS__TUNIT____-NEXT:    [[MRV1:%.*]] = insertvalue [[TMP0]] %mrv, i32 22, 1
+; IS__TUNIT____-NEXT:    ret [[TMP0]] %mrv1
+; IS__TUNIT____:       F:
+; IS__TUNIT____-NEXT:    [[MRV2:%.*]] = insertvalue [[TMP0]] undef, i32 21, 0
+; IS__TUNIT____-NEXT:    [[MRV3:%.*]] = insertvalue [[TMP0]] %mrv2, i32 23, 1
+; IS__TUNIT____-NEXT:    ret [[TMP0]] %mrv3
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@foo
+; IS__CGSCC____-SAME: (i1 [[Q:%.*]])
+; IS__CGSCC____-NEXT:    br i1 [[Q]], label [[T:%.*]], label [[F:%.*]]
+; IS__CGSCC____:       T:
+; IS__CGSCC____-NEXT:    [[MRV:%.*]] = insertvalue [[TMP0:%.*]] undef, i32 21, 0
+; IS__CGSCC____-NEXT:    [[MRV1:%.*]] = insertvalue [[TMP0]] %mrv, i32 22, 1
+; IS__CGSCC____-NEXT:    ret [[TMP0]] %mrv1
+; IS__CGSCC____:       F:
+; IS__CGSCC____-NEXT:    [[MRV2:%.*]] = insertvalue [[TMP0]] undef, i32 21, 0
+; IS__CGSCC____-NEXT:    [[MRV3:%.*]] = insertvalue [[TMP0]] %mrv2, i32 23, 1
+; IS__CGSCC____-NEXT:    ret [[TMP0]] %mrv3
 ;
   br i1 %Q, label %T, label %F
 
@@ -35,16 +49,29 @@ F:                                                ; preds = %0
 }
 
 define internal %0 @bar(i1 %Q) {
-; CHECK-LABEL: define {{[^@]+}}@bar
-; CHECK-SAME: (i1 [[Q:%.*]])
-; CHECK-NEXT:    [[A:%.*]] = insertvalue [[TMP0:%.*]] undef, i32 21, 0
-; CHECK-NEXT:    br i1 [[Q]], label [[T:%.*]], label [[F:%.*]]
-; CHECK:       T:
-; CHECK-NEXT:    [[B:%.*]] = insertvalue [[TMP0]] %A, i32 22, 1
-; CHECK-NEXT:    ret [[TMP0]] %B
-; CHECK:       F:
-; CHECK-NEXT:    [[C:%.*]] = insertvalue [[TMP0]] %A, i32 23, 1
-; CHECK-NEXT:    ret [[TMP0]] %C
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@bar
+; IS__TUNIT____-SAME: (i1 [[Q:%.*]])
+; IS__TUNIT____-NEXT:    [[A:%.*]] = insertvalue [[TMP0:%.*]] undef, i32 21, 0
+; IS__TUNIT____-NEXT:    br i1 [[Q]], label [[T:%.*]], label [[F:%.*]]
+; IS__TUNIT____:       T:
+; IS__TUNIT____-NEXT:    [[B:%.*]] = insertvalue [[TMP0]] %A, i32 22, 1
+; IS__TUNIT____-NEXT:    ret [[TMP0]] %B
+; IS__TUNIT____:       F:
+; IS__TUNIT____-NEXT:    [[C:%.*]] = insertvalue [[TMP0]] %A, i32 23, 1
+; IS__TUNIT____-NEXT:    ret [[TMP0]] %C
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@bar
+; IS__CGSCC____-SAME: (i1 [[Q:%.*]])
+; IS__CGSCC____-NEXT:    [[A:%.*]] = insertvalue [[TMP0:%.*]] undef, i32 21, 0
+; IS__CGSCC____-NEXT:    br i1 [[Q]], label [[T:%.*]], label [[F:%.*]]
+; IS__CGSCC____:       T:
+; IS__CGSCC____-NEXT:    [[B:%.*]] = insertvalue [[TMP0]] %A, i32 22, 1
+; IS__CGSCC____-NEXT:    ret [[TMP0]] %B
+; IS__CGSCC____:       F:
+; IS__CGSCC____-NEXT:    [[C:%.*]] = insertvalue [[TMP0]] %A, i32 23, 1
+; IS__CGSCC____-NEXT:    ret [[TMP0]] %C
 ;
   %A = insertvalue %0 undef, i32 21, 0
   br i1 %Q, label %T, label %F
@@ -59,10 +86,17 @@ F:                                                ; preds = %0
 }
 
 define %0 @caller(i1 %Q) {
-; CHECK-LABEL: define {{[^@]+}}@caller
-; CHECK-SAME: (i1 [[Q:%.*]])
-; CHECK-NEXT:    [[X:%.*]] = call [[TMP0:%.*]] @foo(i1 [[Q]])
-; CHECK-NEXT:    ret [[TMP0]] %X
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@caller
+; IS__TUNIT____-SAME: (i1 [[Q:%.*]])
+; IS__TUNIT____-NEXT:    [[X:%.*]] = call [[TMP0:%.*]] @foo(i1 [[Q]])
+; IS__TUNIT____-NEXT:    ret [[TMP0]] %X
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@caller
+; IS__CGSCC____-SAME: (i1 [[Q:%.*]])
+; IS__CGSCC____-NEXT:    [[X:%.*]] = call [[TMP0:%.*]] @foo(i1 [[Q]])
+; IS__CGSCC____-NEXT:    ret [[TMP0]] %X
 ;
   %X = call %0 @foo(i1 %Q)
   %A = extractvalue %0 %X, 0
@@ -77,18 +111,33 @@ define %0 @caller(i1 %Q) {
 
 ; Similar to @caller but the result of both calls are actually used.
 define i32 @caller2(i1 %Q) {
-; CHECK-LABEL: define {{[^@]+}}@caller2
-; CHECK-SAME: (i1 [[Q:%.*]])
-; CHECK-NEXT:    [[X:%.*]] = call [[TMP0:%.*]] @foo(i1 [[Q]])
-; CHECK-NEXT:    [[A:%.*]] = extractvalue [[TMP0]] %X, 0
-; CHECK-NEXT:    [[B:%.*]] = extractvalue [[TMP0]] %X, 1
-; CHECK-NEXT:    [[Y:%.*]] = call [[TMP0]] @bar(i1 [[Q]])
-; CHECK-NEXT:    [[C:%.*]] = extractvalue [[TMP0]] %Y, 0
-; CHECK-NEXT:    [[D:%.*]] = extractvalue [[TMP0]] %Y, 1
-; CHECK-NEXT:    [[M:%.*]] = add i32 [[A]], [[C]]
-; CHECK-NEXT:    [[N:%.*]] = add i32 [[B]], [[D]]
-; CHECK-NEXT:    [[R:%.*]] = add i32 [[N]], [[M]]
-; CHECK-NEXT:    ret i32 [[R]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@caller2
+; IS__TUNIT____-SAME: (i1 [[Q:%.*]])
+; IS__TUNIT____-NEXT:    [[X:%.*]] = call [[TMP0:%.*]] @foo(i1 [[Q]])
+; IS__TUNIT____-NEXT:    [[A:%.*]] = extractvalue [[TMP0]] %X, 0
+; IS__TUNIT____-NEXT:    [[B:%.*]] = extractvalue [[TMP0]] %X, 1
+; IS__TUNIT____-NEXT:    [[Y:%.*]] = call [[TMP0]] @bar(i1 [[Q]])
+; IS__TUNIT____-NEXT:    [[C:%.*]] = extractvalue [[TMP0]] %Y, 0
+; IS__TUNIT____-NEXT:    [[D:%.*]] = extractvalue [[TMP0]] %Y, 1
+; IS__TUNIT____-NEXT:    [[M:%.*]] = add i32 [[A]], [[C]]
+; IS__TUNIT____-NEXT:    [[N:%.*]] = add i32 [[B]], [[D]]
+; IS__TUNIT____-NEXT:    [[R:%.*]] = add i32 [[N]], [[M]]
+; IS__TUNIT____-NEXT:    ret i32 [[R]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@caller2
+; IS__CGSCC____-SAME: (i1 [[Q:%.*]])
+; IS__CGSCC____-NEXT:    [[X:%.*]] = call [[TMP0:%.*]] @foo(i1 [[Q]])
+; IS__CGSCC____-NEXT:    [[A:%.*]] = extractvalue [[TMP0]] %X, 0
+; IS__CGSCC____-NEXT:    [[B:%.*]] = extractvalue [[TMP0]] %X, 1
+; IS__CGSCC____-NEXT:    [[Y:%.*]] = call [[TMP0]] @bar(i1 [[Q]])
+; IS__CGSCC____-NEXT:    [[C:%.*]] = extractvalue [[TMP0]] %Y, 0
+; IS__CGSCC____-NEXT:    [[D:%.*]] = extractvalue [[TMP0]] %Y, 1
+; IS__CGSCC____-NEXT:    [[M:%.*]] = add i32 [[A]], [[C]]
+; IS__CGSCC____-NEXT:    [[N:%.*]] = add i32 [[B]], [[D]]
+; IS__CGSCC____-NEXT:    [[R:%.*]] = add i32 [[N]], [[M]]
+; IS__CGSCC____-NEXT:    ret i32 [[R]]
 ;
   %X = call %0 @foo(i1 %Q)
   %A = extractvalue %0 %X, 0
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/solve-after-each-resolving-undefs-for-function.ll b/llvm/test/Transforms/Attributor/IPConstantProp/solve-after-each-resolving-undefs-for-function.ll
index f958e068da170..11b65dfe51dc8 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/solve-after-each-resolving-undefs-for-function.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/solve-after-each-resolving-undefs-for-function.ll
@@ -1,10 +1,11 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 define internal i32 @testf(i1 %c) {
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@testf
 ; IS__CGSCC____-SAME: (i1 [[C:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -30,6 +31,7 @@ if.end:                                          ; preds = %if.then1, %entry
 }
 
 define internal i32 @test1(i1 %c) {
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@test1
 ; IS__CGSCC____-SAME: (i1 [[C:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -57,9 +59,15 @@ ret2:                                           ; preds = %if.then, %entry
 }
 
 define i32 @main(i1 %c) {
-; CHECK-LABEL: define {{[^@]+}}@main
-; CHECK-SAME: (i1 [[C:%.*]])
-; CHECK-NEXT:    ret i32 99
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@main
+; IS__TUNIT____-SAME: (i1 [[C:%.*]])
+; IS__TUNIT____-NEXT:    ret i32 99
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@main
+; IS__CGSCC____-SAME: (i1 [[C:%.*]])
+; IS__CGSCC____-NEXT:    ret i32 99
 ;
   %res = call i32 @test1(i1 %c)
   ret i32 %res
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/thread_local_acs.ll b/llvm/test/Transforms/Attributor/IPConstantProp/thread_local_acs.ll
index 12dd77a61b986..58dfbaefaa98b 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/thread_local_acs.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/thread_local_acs.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -24,13 +24,23 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 @gsh = dso_local global i32 0, align 4
 
 define internal i32 @callee(i32* %thread_local_ptr, i32* %shared_ptr) {
-; CHECK-LABEL: define {{[^@]+}}@callee
-; CHECK-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[THREAD_LOCAL_PTR:%.*]], i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[SHARED_PTR:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP:%.*]] = load i32, i32* [[THREAD_LOCAL_PTR]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* @gsh, align 4
-; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP]], [[TMP1]]
-; CHECK-NEXT:    ret i32 [[ADD]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@callee
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[THREAD_LOCAL_PTR:%.*]], i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[SHARED_PTR:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[TMP:%.*]] = load i32, i32* [[THREAD_LOCAL_PTR]], align 4
+; IS__TUNIT____-NEXT:    [[TMP1:%.*]] = load i32, i32* @gsh, align 4
+; IS__TUNIT____-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP]], [[TMP1]]
+; IS__TUNIT____-NEXT:    ret i32 [[ADD]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@callee
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[THREAD_LOCAL_PTR:%.*]], i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[SHARED_PTR:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[TMP:%.*]] = load i32, i32* [[THREAD_LOCAL_PTR]], align 4
+; IS__CGSCC____-NEXT:    [[TMP1:%.*]] = load i32, i32* @gsh, align 4
+; IS__CGSCC____-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP]], [[TMP1]]
+; IS__CGSCC____-NEXT:    ret i32 [[ADD]]
 ;
 entry:
   %tmp = load i32, i32* %thread_local_ptr, align 4
diff --git a/llvm/test/Transforms/Attributor/align.ll b/llvm/test/Transforms/Attributor/align.ll
index 858c8eba16c95..0ee08d76f5416 100644
--- a/llvm/test/Transforms/Attributor/align.ll
+++ b/llvm/test/Transforms/Attributor/align.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=11 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=11 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -12,28 +12,47 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 ; TEST 1
 define i32* @test1(i32* align 8 %0) #0 {
-; CHECK-LABEL: define {{[^@]+}}@test1
-; CHECK-SAME: (i32* nofree readnone returned align 8 "no-capture-maybe-returned" [[TMP0:%.*]])
-; CHECK-NEXT:    ret i32* [[TMP0]]
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test1
+; IS__TUNIT____-SAME: (i32* nofree readnone returned align 8 "no-capture-maybe-returned" [[TMP0:%.*]])
+; IS__TUNIT____-NEXT:    ret i32* [[TMP0]]
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test1
+; IS__CGSCC____-SAME: (i32* nofree readnone returned align 8 "no-capture-maybe-returned" [[TMP0:%.*]])
+; IS__CGSCC____-NEXT:    ret i32* [[TMP0]]
 ;
   ret i32* %0
 }
 
 ; TEST 2
 define i32* @test2(i32* %0) #0 {
-; CHECK-LABEL: define {{[^@]+}}@test2
-; CHECK-SAME: (i32* nofree readnone returned "no-capture-maybe-returned" [[TMP0:%.*]])
-; CHECK-NEXT:    ret i32* [[TMP0]]
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test2
+; IS__TUNIT____-SAME: (i32* nofree readnone returned "no-capture-maybe-returned" [[TMP0:%.*]])
+; IS__TUNIT____-NEXT:    ret i32* [[TMP0]]
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test2
+; IS__CGSCC____-SAME: (i32* nofree readnone returned "no-capture-maybe-returned" [[TMP0:%.*]])
+; IS__CGSCC____-NEXT:    ret i32* [[TMP0]]
 ;
   ret i32* %0
 }
 
 ; TEST 3
 define i32* @test3(i32* align 8 %0, i32* align 4 %1, i1 %2) #0 {
-; CHECK-LABEL: define {{[^@]+}}@test3
-; CHECK-SAME: (i32* nofree readnone align 8 "no-capture-maybe-returned" [[TMP0:%.*]], i32* nofree readnone align 4 "no-capture-maybe-returned" [[TMP1:%.*]], i1 [[TMP2:%.*]])
-; CHECK-NEXT:    [[RET:%.*]] = select i1 [[TMP2]], i32* [[TMP0]], i32* [[TMP1]]
-; CHECK-NEXT:    ret i32* [[RET]]
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test3
+; IS__TUNIT____-SAME: (i32* nofree readnone align 8 "no-capture-maybe-returned" [[TMP0:%.*]], i32* nofree readnone align 4 "no-capture-maybe-returned" [[TMP1:%.*]], i1 [[TMP2:%.*]])
+; IS__TUNIT____-NEXT:    [[RET:%.*]] = select i1 [[TMP2]], i32* [[TMP0]], i32* [[TMP1]]
+; IS__TUNIT____-NEXT:    ret i32* [[RET]]
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test3
+; IS__CGSCC____-SAME: (i32* nofree readnone align 8 "no-capture-maybe-returned" [[TMP0:%.*]], i32* nofree readnone align 4 "no-capture-maybe-returned" [[TMP1:%.*]], i1 [[TMP2:%.*]])
+; IS__CGSCC____-NEXT:    [[RET:%.*]] = select i1 [[TMP2]], i32* [[TMP0]], i32* [[TMP1]]
+; IS__CGSCC____-NEXT:    ret i32* [[RET]]
 ;
   %ret = select i1 %2, i32* %0, i32* %1
   ret i32* %ret
@@ -41,10 +60,17 @@ define i32* @test3(i32* align 8 %0, i32* align 4 %1, i1 %2) #0 {
 
 ; TEST 4
 define i32* @test4(i32* align 32 %0, i32* align 32 %1, i1 %2) #0 {
-; CHECK-LABEL: define {{[^@]+}}@test4
-; CHECK-SAME: (i32* nofree readnone align 32 "no-capture-maybe-returned" [[TMP0:%.*]], i32* nofree readnone align 32 "no-capture-maybe-returned" [[TMP1:%.*]], i1 [[TMP2:%.*]])
-; CHECK-NEXT:    [[RET:%.*]] = select i1 [[TMP2]], i32* [[TMP0]], i32* [[TMP1]]
-; CHECK-NEXT:    ret i32* [[RET]]
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test4
+; IS__TUNIT____-SAME: (i32* nofree readnone align 32 "no-capture-maybe-returned" [[TMP0:%.*]], i32* nofree readnone align 32 "no-capture-maybe-returned" [[TMP1:%.*]], i1 [[TMP2:%.*]])
+; IS__TUNIT____-NEXT:    [[RET:%.*]] = select i1 [[TMP2]], i32* [[TMP0]], i32* [[TMP1]]
+; IS__TUNIT____-NEXT:    ret i32* [[RET]]
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test4
+; IS__CGSCC____-SAME: (i32* nofree readnone align 32 "no-capture-maybe-returned" [[TMP0:%.*]], i32* nofree readnone align 32 "no-capture-maybe-returned" [[TMP1:%.*]], i1 [[TMP2:%.*]])
+; IS__CGSCC____-NEXT:    [[RET:%.*]] = select i1 [[TMP2]], i32* [[TMP0]], i32* [[TMP1]]
+; IS__CGSCC____-NEXT:    ret i32* [[RET]]
 ;
   %ret = select i1 %2, i32* %0, i32* %1
   ret i32* %ret
@@ -76,16 +102,26 @@ define i32* @test5_2() {
 ; TEST 6
 ; SCC
 define i32* @test6_1() #0 {
-; CHECK-LABEL: define {{[^@]+}}@test6_1()
-; CHECK-NEXT:    unreachable
+; NOT_CGSCC_NPM: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@test6_1()
+; NOT_CGSCC_NPM-NEXT:    unreachable
+;
+; IS__CGSCC_NPM: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readnone uwtable willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@test6_1()
+; IS__CGSCC_NPM-NEXT:    unreachable
 ;
   %ret = tail call i32* @test6_2()
   ret i32* %ret
 }
 
 define i32* @test6_2() #0 {
-; CHECK-LABEL: define {{[^@]+}}@test6_2()
-; CHECK-NEXT:    unreachable
+; NOT_CGSCC_NPM: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@test6_2()
+; NOT_CGSCC_NPM-NEXT:    unreachable
+;
+; IS__CGSCC_NPM: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readnone uwtable willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@test6_2()
+; IS__CGSCC_NPM-NEXT:    unreachable
 ;
   %ret = tail call i32* @test6_1()
   ret i32* %ret
@@ -111,6 +147,7 @@ define i32* @test6_2() #0 {
 
 ; Function Attrs: nounwind readnone ssp uwtable
 define internal i8* @f1(i8* readnone %0) local_unnamed_addr #0 {
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@f1
 ; IS__TUNIT____-SAME: (i8* noalias nofree nonnull readnone align 8 dereferenceable(1) "no-capture-maybe-returned" [[TMP0:%.*]]) local_unnamed_addr
 ; IS__TUNIT____-NEXT:    [[TMP2:%.*]] = icmp eq i8* [[TMP0]], null
@@ -122,6 +159,7 @@ define internal i8* @f1(i8* readnone %0) local_unnamed_addr #0 {
 ; IS__TUNIT____-NEXT:    [[TMP6:%.*]] = phi i8* [ [[TMP4]], [[TMP3]] ], [ [[TMP0]], [[TMP1:%.*]] ]
 ; IS__TUNIT____-NEXT:    ret i8* [[TMP6]]
 ;
+; IS__CGSCC____: Function Attrs: nofree noinline nosync nounwind readnone uwtable
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@f1
 ; IS__CGSCC____-SAME: (i8* nofree nonnull readnone align 8 dereferenceable(1) "no-capture-maybe-returned" [[TMP0:%.*]]) local_unnamed_addr
 ; IS__CGSCC____-NEXT:    [[TMP2:%.*]] = icmp eq i8* [[TMP0]], null
@@ -148,6 +186,7 @@ define internal i8* @f1(i8* readnone %0) local_unnamed_addr #0 {
 
 ; Function Attrs: nounwind readnone ssp uwtable
 define internal i8* @f2(i8* readnone %0) local_unnamed_addr #0 {
+; CHECK: Function Attrs: nofree noinline nosync nounwind readnone uwtable
 ; CHECK-LABEL: define {{[^@]+}}@f2() local_unnamed_addr
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8* @a1, null
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP4:%.*]], label [[TMP2:%.*]]
@@ -180,6 +219,7 @@ define internal i8* @f2(i8* readnone %0) local_unnamed_addr #0 {
 
 ; Function Attrs: nounwind readnone ssp uwtable
 define internal i8* @f3(i8* readnone %0) local_unnamed_addr #0 {
+; CHECK: Function Attrs: nofree noinline nosync nounwind readnone uwtable
 ; CHECK-LABEL: define {{[^@]+}}@f3() local_unnamed_addr
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8* @a2, null
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP4:%.*]]
@@ -205,10 +245,12 @@ define internal i8* @f3(i8* readnone %0) local_unnamed_addr #0 {
 ; TEST 7
 ; Better than IR information
 define align 4 i8* @test7() #0 {
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@test7()
 ; IS__TUNIT____-NEXT:    [[C:%.*]] = tail call i8* @f1(i8* noalias nofree nonnull readnone align 8 dereferenceable(1) "no-capture-maybe-returned" @a1)
 ; IS__TUNIT____-NEXT:    ret i8* [[C]]
 ;
+; IS__CGSCC____: Function Attrs: nofree noinline nosync nounwind readnone uwtable
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@test7()
 ; IS__CGSCC____-NEXT:    [[C:%.*]] = tail call nonnull align 8 dereferenceable(1) i8* @f1(i8* noalias nofree nonnull readnone align 8 dereferenceable(1) @a1)
 ; IS__CGSCC____-NEXT:    ret i8* [[C]]
@@ -220,6 +262,7 @@ define align 4 i8* @test7() #0 {
 ; TEST 7b
 ; Function Attrs: nounwind readnone ssp uwtable
 define internal i8* @f1b(i8* readnone %0) local_unnamed_addr #0 {
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind uwtable
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@f1b
 ; IS__TUNIT____-SAME: (i8* noalias nofree nonnull readnone align 8 dereferenceable(1) "no-capture-maybe-returned" [[TMP0:%.*]]) local_unnamed_addr
 ; IS__TUNIT____-NEXT:    [[TMP2:%.*]] = icmp eq i8* [[TMP0]], null
@@ -233,6 +276,7 @@ define internal i8* @f1b(i8* readnone %0) local_unnamed_addr #0 {
 ; IS__TUNIT____-NEXT:    [[TMP6:%.*]] = phi i8* [ [[TMP4]], [[TMP3]] ], [ [[TMP0]], [[TMP1:%.*]] ]
 ; IS__TUNIT____-NEXT:    ret i8* [[TMP6]]
 ;
+; IS__CGSCC____: Function Attrs: nofree noinline nosync nounwind uwtable
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@f1b
 ; IS__CGSCC____-SAME: (i8* nofree nonnull readnone align 8 dereferenceable(1) "no-capture-maybe-returned" [[TMP0:%.*]]) local_unnamed_addr
 ; IS__CGSCC____-NEXT:    [[TMP2:%.*]] = icmp eq i8* [[TMP0]], null
@@ -263,6 +307,7 @@ define internal i8* @f1b(i8* readnone %0) local_unnamed_addr #0 {
 ; Function Attrs: nounwind readnone ssp uwtable
 define internal i8* @f2b(i8* readnone %0) local_unnamed_addr #0 {
 ;
+; CHECK: Function Attrs: nofree noinline nosync nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@f2b() local_unnamed_addr
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8* @a1, null
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP4:%.*]], label [[TMP2:%.*]]
@@ -296,6 +341,7 @@ define internal i8* @f2b(i8* readnone %0) local_unnamed_addr #0 {
 ; Function Attrs: nounwind readnone ssp uwtable
 define internal i8* @f3b(i8* readnone %0) local_unnamed_addr #0 {
 ;
+; CHECK: Function Attrs: nofree noinline nosync nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@f3b() local_unnamed_addr
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8* @a2, null
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP4:%.*]]
@@ -319,11 +365,13 @@ define internal i8* @f3b(i8* readnone %0) local_unnamed_addr #0 {
 }
 
 define align 4 i32* @test7b(i32* align 32 %p) #0 {
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind uwtable
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@test7b
 ; IS__TUNIT____-SAME: (i32* nofree readnone returned align 32 "no-capture-maybe-returned" [[P:%.*]])
 ; IS__TUNIT____-NEXT:    [[TMP1:%.*]] = tail call i8* @f1b(i8* noalias nofree nonnull readnone align 8 dereferenceable(1) "no-capture-maybe-returned" @a1)
 ; IS__TUNIT____-NEXT:    ret i32* [[P]]
 ;
+; IS__CGSCC____: Function Attrs: nofree noinline nosync nounwind uwtable
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@test7b
 ; IS__CGSCC____-SAME: (i32* nofree readnone returned align 32 "no-capture-maybe-returned" [[P:%.*]])
 ; IS__CGSCC____-NEXT:    [[TMP1:%.*]] = tail call i8* @f1b(i8* noalias nofree nonnull readnone align 8 dereferenceable(1) @a1)
@@ -356,6 +404,7 @@ define void @test8_helper() {
 
 declare void @user_i32_ptr(i32* nocapture readnone) nounwind
 define internal void @test8(i32* %a, i32* %b, i32* %c) {
+; IS__TUNIT____: Function Attrs: nounwind
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@test8
 ; IS__TUNIT____-SAME: (i32* noalias nocapture readnone align 4 [[A:%.*]], i32* noalias nocapture readnone align 4 [[B:%.*]], i32* noalias nocapture readnone [[C:%.*]])
 ; IS__TUNIT____-NEXT:    call void @user_i32_ptr(i32* noalias nocapture readnone align 4 [[A]])
@@ -363,6 +412,7 @@ define internal void @test8(i32* %a, i32* %b, i32* %c) {
 ; IS__TUNIT____-NEXT:    call void @user_i32_ptr(i32* noalias nocapture readnone [[C]])
 ; IS__TUNIT____-NEXT:    ret void
 ;
+; IS__CGSCC____: Function Attrs: nounwind
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@test8
 ; IS__CGSCC____-SAME: (i32* nocapture readnone align 4 [[A:%.*]], i32* nocapture readnone align 4 [[B:%.*]], i32* nocapture readnone [[C:%.*]])
 ; IS__CGSCC____-NEXT:    call void @user_i32_ptr(i32* noalias nocapture readnone align 4 [[A]])
@@ -396,6 +446,7 @@ define void @test9_traversal(i1 %cnd, i32* align 4 %B, i32* align 8 %C) {
 ; FIXME: This will work with an upcoming patch (D66618 or similar)
 ;             store i32 -1, i32* %g1, align 32
 define i32* @test10a(i32* align 32 %p) {
+; CHECK: Function Attrs: nofree nosync nounwind
 ; CHECK-LABEL: define {{[^@]+}}@test10a
 ; CHECK-SAME: (i32* nofree nonnull align 32 dereferenceable(4) "no-capture-maybe-returned" [[P:%.*]])
 ; CHECK-NEXT:    [[L:%.*]] = load i32, i32* [[P]], align 32
@@ -438,6 +489,7 @@ e:
 ; FIXME: This will work with an upcoming patch (D66618 or similar)
 ;             store i32 -1, i32* %g1, align 32
 define i32* @test10b(i32* align 32 %p) {
+; CHECK: Function Attrs: nofree nosync nounwind
 ; CHECK-LABEL: define {{[^@]+}}@test10b
 ; CHECK-SAME: (i32* nofree nonnull align 32 dereferenceable(4) "no-capture-maybe-returned" [[P:%.*]])
 ; CHECK-NEXT:    [[L:%.*]] = load i32, i32* [[P]], align 32
@@ -475,11 +527,19 @@ e:
 
 
 define i64 @test11(i32* %p) {
-; CHECK-LABEL: define {{[^@]+}}@test11
-; CHECK-SAME: (i32* nocapture nofree nonnull readonly align 8 dereferenceable(8) [[P:%.*]])
-; CHECK-NEXT:    [[P_CAST:%.*]] = bitcast i32* [[P]] to i64*
-; CHECK-NEXT:    [[RET:%.*]] = load i64, i64* [[P_CAST]], align 8
-; CHECK-NEXT:    ret i64 [[RET]]
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test11
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull readonly align 8 dereferenceable(8) [[P:%.*]])
+; IS__TUNIT____-NEXT:    [[P_CAST:%.*]] = bitcast i32* [[P]] to i64*
+; IS__TUNIT____-NEXT:    [[RET:%.*]] = load i64, i64* [[P_CAST]], align 8
+; IS__TUNIT____-NEXT:    ret i64 [[RET]]
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test11
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readonly align 8 dereferenceable(8) [[P:%.*]])
+; IS__CGSCC____-NEXT:    [[P_CAST:%.*]] = bitcast i32* [[P]] to i64*
+; IS__CGSCC____-NEXT:    [[RET:%.*]] = load i64, i64* [[P_CAST]], align 8
+; IS__CGSCC____-NEXT:    ret i64 [[RET]]
 ;
   %p-cast = bitcast i32* %p to i64*
   %ret = load i64, i64* %p-cast, align 8
@@ -491,13 +551,23 @@ define i64 @test11(i32* %p) {
 
 ; FXIME: %p should have nonnull
 define i64 @test12-1(i32* align 4 %p) {
-; CHECK-LABEL: define {{[^@]+}}@test12-1
-; CHECK-SAME: (i32* nocapture nofree readonly align 16 [[P:%.*]])
-; CHECK-NEXT:    [[P_CAST:%.*]] = bitcast i32* [[P]] to i64*
-; CHECK-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i64, i64* [[P_CAST]], i64 1
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr i64, i64* [[ARRAYIDX0]], i64 3
-; CHECK-NEXT:    [[RET:%.*]] = load i64, i64* [[ARRAYIDX1]], align 16
-; CHECK-NEXT:    ret i64 [[RET]]
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test12-1
+; IS__TUNIT____-SAME: (i32* nocapture nofree readonly align 16 [[P:%.*]])
+; IS__TUNIT____-NEXT:    [[P_CAST:%.*]] = bitcast i32* [[P]] to i64*
+; IS__TUNIT____-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i64, i64* [[P_CAST]], i64 1
+; IS__TUNIT____-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr i64, i64* [[ARRAYIDX0]], i64 3
+; IS__TUNIT____-NEXT:    [[RET:%.*]] = load i64, i64* [[ARRAYIDX1]], align 16
+; IS__TUNIT____-NEXT:    ret i64 [[RET]]
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test12-1
+; IS__CGSCC____-SAME: (i32* nocapture nofree readonly align 16 [[P:%.*]])
+; IS__CGSCC____-NEXT:    [[P_CAST:%.*]] = bitcast i32* [[P]] to i64*
+; IS__CGSCC____-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i64, i64* [[P_CAST]], i64 1
+; IS__CGSCC____-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr i64, i64* [[ARRAYIDX0]], i64 3
+; IS__CGSCC____-NEXT:    [[RET:%.*]] = load i64, i64* [[ARRAYIDX1]], align 16
+; IS__CGSCC____-NEXT:    ret i64 [[RET]]
 ;
   %p-cast = bitcast i32* %p to i64*
   %arrayidx0 = getelementptr i64, i64* %p-cast, i64 1
@@ -507,12 +577,21 @@ define i64 @test12-1(i32* align 4 %p) {
 }
 
 define i64 @test12-2(i32* align 4 %p) {
-; CHECK-LABEL: define {{[^@]+}}@test12-2
-; CHECK-SAME: (i32* nocapture nofree nonnull readonly align 16 dereferenceable(8) [[P:%.*]])
-; CHECK-NEXT:    [[P_CAST:%.*]] = bitcast i32* [[P]] to i64*
-; CHECK-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i64, i64* [[P_CAST]], i64 0
-; CHECK-NEXT:    [[RET:%.*]] = load i64, i64* [[ARRAYIDX0]], align 16
-; CHECK-NEXT:    ret i64 [[RET]]
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test12-2
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull readonly align 16 dereferenceable(8) [[P:%.*]])
+; IS__TUNIT____-NEXT:    [[P_CAST:%.*]] = bitcast i32* [[P]] to i64*
+; IS__TUNIT____-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i64, i64* [[P_CAST]], i64 0
+; IS__TUNIT____-NEXT:    [[RET:%.*]] = load i64, i64* [[ARRAYIDX0]], align 16
+; IS__TUNIT____-NEXT:    ret i64 [[RET]]
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test12-2
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readonly align 16 dereferenceable(8) [[P:%.*]])
+; IS__CGSCC____-NEXT:    [[P_CAST:%.*]] = bitcast i32* [[P]] to i64*
+; IS__CGSCC____-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i64, i64* [[P_CAST]], i64 0
+; IS__CGSCC____-NEXT:    [[RET:%.*]] = load i64, i64* [[ARRAYIDX0]], align 16
+; IS__CGSCC____-NEXT:    ret i64 [[RET]]
 ;
   %p-cast = bitcast i32* %p to i64*
   %arrayidx0 = getelementptr i64, i64* %p-cast, i64 0
@@ -522,13 +601,23 @@ define i64 @test12-2(i32* align 4 %p) {
 
 ; FXIME: %p should have nonnull
 define void @test12-3(i32* align 4 %p) {
-; CHECK-LABEL: define {{[^@]+}}@test12-3
-; CHECK-SAME: (i32* nocapture nofree writeonly align 16 [[P:%.*]])
-; CHECK-NEXT:    [[P_CAST:%.*]] = bitcast i32* [[P]] to i64*
-; CHECK-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i64, i64* [[P_CAST]], i64 1
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr i64, i64* [[ARRAYIDX0]], i64 3
-; CHECK-NEXT:    store i64 0, i64* [[ARRAYIDX1]], align 16
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test12-3
+; IS__TUNIT____-SAME: (i32* nocapture nofree writeonly align 16 [[P:%.*]])
+; IS__TUNIT____-NEXT:    [[P_CAST:%.*]] = bitcast i32* [[P]] to i64*
+; IS__TUNIT____-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i64, i64* [[P_CAST]], i64 1
+; IS__TUNIT____-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr i64, i64* [[ARRAYIDX0]], i64 3
+; IS__TUNIT____-NEXT:    store i64 0, i64* [[ARRAYIDX1]], align 16
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test12-3
+; IS__CGSCC____-SAME: (i32* nocapture nofree writeonly align 16 [[P:%.*]])
+; IS__CGSCC____-NEXT:    [[P_CAST:%.*]] = bitcast i32* [[P]] to i64*
+; IS__CGSCC____-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i64, i64* [[P_CAST]], i64 1
+; IS__CGSCC____-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr i64, i64* [[ARRAYIDX0]], i64 3
+; IS__CGSCC____-NEXT:    store i64 0, i64* [[ARRAYIDX1]], align 16
+; IS__CGSCC____-NEXT:    ret void
 ;
   %p-cast = bitcast i32* %p to i64*
   %arrayidx0 = getelementptr i64, i64* %p-cast, i64 1
@@ -538,12 +627,21 @@ define void @test12-3(i32* align 4 %p) {
 }
 
 define void @test12-4(i32* align 4 %p) {
-; CHECK-LABEL: define {{[^@]+}}@test12-4
-; CHECK-SAME: (i32* nocapture nofree nonnull writeonly align 16 dereferenceable(8) [[P:%.*]])
-; CHECK-NEXT:    [[P_CAST:%.*]] = bitcast i32* [[P]] to i64*
-; CHECK-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i64, i64* [[P_CAST]], i64 0
-; CHECK-NEXT:    store i64 0, i64* [[ARRAYIDX0]], align 16
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test12-4
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull writeonly align 16 dereferenceable(8) [[P:%.*]])
+; IS__TUNIT____-NEXT:    [[P_CAST:%.*]] = bitcast i32* [[P]] to i64*
+; IS__TUNIT____-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i64, i64* [[P_CAST]], i64 0
+; IS__TUNIT____-NEXT:    store i64 0, i64* [[ARRAYIDX0]], align 16
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test12-4
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull writeonly align 16 dereferenceable(8) [[P:%.*]])
+; IS__CGSCC____-NEXT:    [[P_CAST:%.*]] = bitcast i32* [[P]] to i64*
+; IS__CGSCC____-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i64, i64* [[P_CAST]], i64 0
+; IS__CGSCC____-NEXT:    store i64 0, i64* [[ARRAYIDX0]], align 16
+; IS__CGSCC____-NEXT:    ret void
 ;
   %p-cast = bitcast i32* %p to i64*
   %arrayidx0 = getelementptr i64, i64* %p-cast, i64 0
@@ -554,6 +652,7 @@ define void @test12-4(i32* align 4 %p) {
 declare void @use(i64*) willreturn nounwind
 
 define void @test12-5(i32* align 4 %p) {
+; CHECK: Function Attrs: nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@test12-5
 ; CHECK-SAME: (i32* align 16 [[P:%.*]])
 ; CHECK-NEXT:    [[P_CAST:%.*]] = bitcast i32* [[P]] to i64*
@@ -570,6 +669,7 @@ define void @test12-5(i32* align 4 %p) {
 }
 
 define void @test12-6(i32* align 4 %p) {
+; CHECK: Function Attrs: nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@test12-6
 ; CHECK-SAME: (i32* align 16 [[P:%.*]])
 ; CHECK-NEXT:    [[P_CAST:%.*]] = bitcast i32* [[P]] to i64*
@@ -584,17 +684,31 @@ define void @test12-6(i32* align 4 %p) {
 }
 
 define void @test13(i1 %c, i32* align 32 %dst) #0 {
-; CHECK-LABEL: define {{[^@]+}}@test13
-; CHECK-SAME: (i1 [[C:%.*]], i32* nocapture nofree writeonly align 32 [[DST:%.*]])
-; CHECK-NEXT:    br i1 [[C]], label [[TRUEBB:%.*]], label [[FALSEBB:%.*]]
-; CHECK:       truebb:
-; CHECK-NEXT:    br label [[END:%.*]]
-; CHECK:       falsebb:
-; CHECK-NEXT:    br label [[END]]
-; CHECK:       end:
-; CHECK-NEXT:    [[PTR:%.*]] = phi i32* [ [[DST]], [[TRUEBB]] ], [ null, [[FALSEBB]] ]
-; CHECK-NEXT:    store i32 0, i32* [[PTR]], align 32
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: argmemonly nofree noinline nosync nounwind uwtable willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test13
+; IS__TUNIT____-SAME: (i1 [[C:%.*]], i32* nocapture nofree writeonly align 32 [[DST:%.*]])
+; IS__TUNIT____-NEXT:    br i1 [[C]], label [[TRUEBB:%.*]], label [[FALSEBB:%.*]]
+; IS__TUNIT____:       truebb:
+; IS__TUNIT____-NEXT:    br label [[END:%.*]]
+; IS__TUNIT____:       falsebb:
+; IS__TUNIT____-NEXT:    br label [[END]]
+; IS__TUNIT____:       end:
+; IS__TUNIT____-NEXT:    [[PTR:%.*]] = phi i32* [ [[DST]], [[TRUEBB]] ], [ null, [[FALSEBB]] ]
+; IS__TUNIT____-NEXT:    store i32 0, i32* [[PTR]], align 32
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree noinline norecurse nosync nounwind uwtable willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test13
+; IS__CGSCC____-SAME: (i1 [[C:%.*]], i32* nocapture nofree writeonly align 32 [[DST:%.*]])
+; IS__CGSCC____-NEXT:    br i1 [[C]], label [[TRUEBB:%.*]], label [[FALSEBB:%.*]]
+; IS__CGSCC____:       truebb:
+; IS__CGSCC____-NEXT:    br label [[END:%.*]]
+; IS__CGSCC____:       falsebb:
+; IS__CGSCC____-NEXT:    br label [[END]]
+; IS__CGSCC____:       end:
+; IS__CGSCC____-NEXT:    [[PTR:%.*]] = phi i32* [ [[DST]], [[TRUEBB]] ], [ null, [[FALSEBB]] ]
+; IS__CGSCC____-NEXT:    store i32 0, i32* [[PTR]], align 32
+; IS__CGSCC____-NEXT:    ret void
 ;
   br i1 %c, label %truebb, label %falsebb
 truebb:
@@ -608,17 +722,31 @@ end:
 }
 
 define void @test13-1(i1 %c, i32* align 32 %dst) {
-; CHECK-LABEL: define {{[^@]+}}@test13-1
-; CHECK-SAME: (i1 [[C:%.*]], i32* nocapture nofree writeonly align 32 [[DST:%.*]])
-; CHECK-NEXT:    br i1 [[C]], label [[TRUEBB:%.*]], label [[FALSEBB:%.*]]
-; CHECK:       truebb:
-; CHECK-NEXT:    br label [[END:%.*]]
-; CHECK:       falsebb:
-; CHECK-NEXT:    br label [[END]]
-; CHECK:       end:
-; CHECK-NEXT:    [[PTR:%.*]] = phi i32* [ [[DST]], [[TRUEBB]] ], [ inttoptr (i64 48 to i32*), [[FALSEBB]] ]
-; CHECK-NEXT:    store i32 0, i32* [[PTR]], align 16
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test13-1
+; IS__TUNIT____-SAME: (i1 [[C:%.*]], i32* nocapture nofree writeonly align 32 [[DST:%.*]])
+; IS__TUNIT____-NEXT:    br i1 [[C]], label [[TRUEBB:%.*]], label [[FALSEBB:%.*]]
+; IS__TUNIT____:       truebb:
+; IS__TUNIT____-NEXT:    br label [[END:%.*]]
+; IS__TUNIT____:       falsebb:
+; IS__TUNIT____-NEXT:    br label [[END]]
+; IS__TUNIT____:       end:
+; IS__TUNIT____-NEXT:    [[PTR:%.*]] = phi i32* [ [[DST]], [[TRUEBB]] ], [ inttoptr (i64 48 to i32*), [[FALSEBB]] ]
+; IS__TUNIT____-NEXT:    store i32 0, i32* [[PTR]], align 16
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test13-1
+; IS__CGSCC____-SAME: (i1 [[C:%.*]], i32* nocapture nofree writeonly align 32 [[DST:%.*]])
+; IS__CGSCC____-NEXT:    br i1 [[C]], label [[TRUEBB:%.*]], label [[FALSEBB:%.*]]
+; IS__CGSCC____:       truebb:
+; IS__CGSCC____-NEXT:    br label [[END:%.*]]
+; IS__CGSCC____:       falsebb:
+; IS__CGSCC____-NEXT:    br label [[END]]
+; IS__CGSCC____:       end:
+; IS__CGSCC____-NEXT:    [[PTR:%.*]] = phi i32* [ [[DST]], [[TRUEBB]] ], [ inttoptr (i64 48 to i32*), [[FALSEBB]] ]
+; IS__CGSCC____-NEXT:    store i32 0, i32* [[PTR]], align 16
+; IS__CGSCC____-NEXT:    ret void
 ;
   br i1 %c, label %truebb, label %falsebb
 truebb:
@@ -632,17 +760,31 @@ end:
 }
 
 define void @test13-2(i1 %c, i32* align 32 %dst) {
-; CHECK-LABEL: define {{[^@]+}}@test13-2
-; CHECK-SAME: (i1 [[C:%.*]], i32* nocapture nofree writeonly align 32 [[DST:%.*]])
-; CHECK-NEXT:    br i1 [[C]], label [[TRUEBB:%.*]], label [[FALSEBB:%.*]]
-; CHECK:       truebb:
-; CHECK-NEXT:    br label [[END:%.*]]
-; CHECK:       falsebb:
-; CHECK-NEXT:    br label [[END]]
-; CHECK:       end:
-; CHECK-NEXT:    [[PTR:%.*]] = phi i32* [ [[DST]], [[TRUEBB]] ], [ inttoptr (i64 160 to i32*), [[FALSEBB]] ]
-; CHECK-NEXT:    store i32 0, i32* [[PTR]], align 32
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test13-2
+; IS__TUNIT____-SAME: (i1 [[C:%.*]], i32* nocapture nofree writeonly align 32 [[DST:%.*]])
+; IS__TUNIT____-NEXT:    br i1 [[C]], label [[TRUEBB:%.*]], label [[FALSEBB:%.*]]
+; IS__TUNIT____:       truebb:
+; IS__TUNIT____-NEXT:    br label [[END:%.*]]
+; IS__TUNIT____:       falsebb:
+; IS__TUNIT____-NEXT:    br label [[END]]
+; IS__TUNIT____:       end:
+; IS__TUNIT____-NEXT:    [[PTR:%.*]] = phi i32* [ [[DST]], [[TRUEBB]] ], [ inttoptr (i64 160 to i32*), [[FALSEBB]] ]
+; IS__TUNIT____-NEXT:    store i32 0, i32* [[PTR]], align 32
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test13-2
+; IS__CGSCC____-SAME: (i1 [[C:%.*]], i32* nocapture nofree writeonly align 32 [[DST:%.*]])
+; IS__CGSCC____-NEXT:    br i1 [[C]], label [[TRUEBB:%.*]], label [[FALSEBB:%.*]]
+; IS__CGSCC____:       truebb:
+; IS__CGSCC____-NEXT:    br label [[END:%.*]]
+; IS__CGSCC____:       falsebb:
+; IS__CGSCC____-NEXT:    br label [[END]]
+; IS__CGSCC____:       end:
+; IS__CGSCC____-NEXT:    [[PTR:%.*]] = phi i32* [ [[DST]], [[TRUEBB]] ], [ inttoptr (i64 160 to i32*), [[FALSEBB]] ]
+; IS__CGSCC____-NEXT:    store i32 0, i32* [[PTR]], align 32
+; IS__CGSCC____-NEXT:    ret void
 ;
   br i1 %c, label %truebb, label %falsebb
 truebb:
@@ -656,17 +798,31 @@ end:
 }
 
 define void @test13-3(i1 %c, i32* align 32 %dst) {
-; CHECK-LABEL: define {{[^@]+}}@test13-3
-; CHECK-SAME: (i1 [[C:%.*]], i32* nocapture nofree writeonly align 32 [[DST:%.*]])
-; CHECK-NEXT:    br i1 [[C]], label [[TRUEBB:%.*]], label [[FALSEBB:%.*]]
-; CHECK:       truebb:
-; CHECK-NEXT:    br label [[END:%.*]]
-; CHECK:       falsebb:
-; CHECK-NEXT:    br label [[END]]
-; CHECK:       end:
-; CHECK-NEXT:    [[PTR:%.*]] = phi i32* [ [[DST]], [[TRUEBB]] ], [ inttoptr (i64 128 to i32*), [[FALSEBB]] ]
-; CHECK-NEXT:    store i32 0, i32* [[PTR]], align 32
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test13-3
+; IS__TUNIT____-SAME: (i1 [[C:%.*]], i32* nocapture nofree writeonly align 32 [[DST:%.*]])
+; IS__TUNIT____-NEXT:    br i1 [[C]], label [[TRUEBB:%.*]], label [[FALSEBB:%.*]]
+; IS__TUNIT____:       truebb:
+; IS__TUNIT____-NEXT:    br label [[END:%.*]]
+; IS__TUNIT____:       falsebb:
+; IS__TUNIT____-NEXT:    br label [[END]]
+; IS__TUNIT____:       end:
+; IS__TUNIT____-NEXT:    [[PTR:%.*]] = phi i32* [ [[DST]], [[TRUEBB]] ], [ inttoptr (i64 128 to i32*), [[FALSEBB]] ]
+; IS__TUNIT____-NEXT:    store i32 0, i32* [[PTR]], align 32
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test13-3
+; IS__CGSCC____-SAME: (i1 [[C:%.*]], i32* nocapture nofree writeonly align 32 [[DST:%.*]])
+; IS__CGSCC____-NEXT:    br i1 [[C]], label [[TRUEBB:%.*]], label [[FALSEBB:%.*]]
+; IS__CGSCC____:       truebb:
+; IS__CGSCC____-NEXT:    br label [[END:%.*]]
+; IS__CGSCC____:       falsebb:
+; IS__CGSCC____-NEXT:    br label [[END]]
+; IS__CGSCC____:       end:
+; IS__CGSCC____-NEXT:    [[PTR:%.*]] = phi i32* [ [[DST]], [[TRUEBB]] ], [ inttoptr (i64 128 to i32*), [[FALSEBB]] ]
+; IS__CGSCC____-NEXT:    store i32 0, i32* [[PTR]], align 32
+; IS__CGSCC____-NEXT:    ret void
 ;
   br i1 %c, label %truebb, label %falsebb
 truebb:
@@ -681,19 +837,33 @@ end:
 
 ; Don't crash on ptr2int/int2ptr uses.
 define i64 @ptr2int(i32* %p) {
-; CHECK-LABEL: define {{[^@]+}}@ptr2int
-; CHECK-SAME: (i32* nofree readnone [[P:%.*]])
-; CHECK-NEXT:    [[P2I:%.*]] = ptrtoint i32* [[P]] to i64
-; CHECK-NEXT:    ret i64 [[P2I]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@ptr2int
+; IS__TUNIT____-SAME: (i32* nofree readnone [[P:%.*]])
+; IS__TUNIT____-NEXT:    [[P2I:%.*]] = ptrtoint i32* [[P]] to i64
+; IS__TUNIT____-NEXT:    ret i64 [[P2I]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@ptr2int
+; IS__CGSCC____-SAME: (i32* nofree readnone [[P:%.*]])
+; IS__CGSCC____-NEXT:    [[P2I:%.*]] = ptrtoint i32* [[P]] to i64
+; IS__CGSCC____-NEXT:    ret i64 [[P2I]]
 ;
   %p2i = ptrtoint i32* %p to i64
   ret i64 %p2i
 }
 define i64* @int2ptr(i64 %i) {
-; CHECK-LABEL: define {{[^@]+}}@int2ptr
-; CHECK-SAME: (i64 [[I:%.*]])
-; CHECK-NEXT:    [[I2P:%.*]] = inttoptr i64 [[I]] to i64*
-; CHECK-NEXT:    ret i64* [[I2P]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@int2ptr
+; IS__TUNIT____-SAME: (i64 [[I:%.*]])
+; IS__TUNIT____-NEXT:    [[I2P:%.*]] = inttoptr i64 [[I]] to i64*
+; IS__TUNIT____-NEXT:    ret i64* [[I2P]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@int2ptr
+; IS__CGSCC____-SAME: (i64 [[I:%.*]])
+; IS__CGSCC____-NEXT:    [[I2P:%.*]] = inttoptr i64 [[I]] to i64*
+; IS__CGSCC____-NEXT:    ret i64* [[I2P]]
 ;
   %i2p = inttoptr i64 %i to i64*
   ret i64* %i2p
@@ -701,10 +871,17 @@ define i64* @int2ptr(i64 %i) {
 
 ; Use the store alignment only for the pointer operand.
 define void @aligned_store(i8* %Value, i8** %Ptr) {
-; CHECK-LABEL: define {{[^@]+}}@aligned_store
-; CHECK-SAME: (i8* nofree writeonly [[VALUE:%.*]], i8** nocapture nofree nonnull writeonly align 32 dereferenceable(8) [[PTR:%.*]])
-; CHECK-NEXT:    store i8* [[VALUE]], i8** [[PTR]], align 32
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@aligned_store
+; IS__TUNIT____-SAME: (i8* nofree writeonly [[VALUE:%.*]], i8** nocapture nofree nonnull writeonly align 32 dereferenceable(8) [[PTR:%.*]])
+; IS__TUNIT____-NEXT:    store i8* [[VALUE]], i8** [[PTR]], align 32
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@aligned_store
+; IS__CGSCC____-SAME: (i8* nofree writeonly [[VALUE:%.*]], i8** nocapture nofree nonnull writeonly align 32 dereferenceable(8) [[PTR:%.*]])
+; IS__CGSCC____-NEXT:    store i8* [[VALUE]], i8** [[PTR]], align 32
+; IS__CGSCC____-NEXT:    ret void
 ;
   store i8* %Value, i8** %Ptr, align 32
   ret void
@@ -724,11 +901,19 @@ define void @align_call_op_not_store(i8* align 2048 %arg) {
 }
 define void @align_store_after_bc(i32* align 2048 %arg) {
 ;
-; CHECK-LABEL: define {{[^@]+}}@align_store_after_bc
-; CHECK-SAME: (i32* nocapture nofree nonnull writeonly align 2048 dereferenceable(1) [[ARG:%.*]])
-; CHECK-NEXT:    [[BC:%.*]] = bitcast i32* [[ARG]] to i8*
-; CHECK-NEXT:    store i8 0, i8* [[BC]], align 2048
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@align_store_after_bc
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull writeonly align 2048 dereferenceable(1) [[ARG:%.*]])
+; IS__TUNIT____-NEXT:    [[BC:%.*]] = bitcast i32* [[ARG]] to i8*
+; IS__TUNIT____-NEXT:    store i8 0, i8* [[BC]], align 2048
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@align_store_after_bc
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull writeonly align 2048 dereferenceable(1) [[ARG:%.*]])
+; IS__CGSCC____-NEXT:    [[BC:%.*]] = bitcast i32* [[ARG]] to i8*
+; IS__CGSCC____-NEXT:    store i8 0, i8* [[BC]], align 2048
+; IS__CGSCC____-NEXT:    ret void
 ;
   %bc = bitcast i32* %arg to i8*
   store i8 0, i8* %bc
@@ -739,15 +924,23 @@ define void @align_store_after_bc(i32* align 2048 %arg) {
 ; we cannot also put on the caller.
 @cnd = external global i1
 define i32 @musttail_callee_1(i32* %p) {
-; CHECK-LABEL: define {{[^@]+}}@musttail_callee_1
-; CHECK-SAME: (i32* nocapture nofree nonnull readonly dereferenceable(4) [[P:%.*]])
-; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P]], align 32
-; CHECK-NEXT:    ret i32 [[V]]
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@musttail_callee_1
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull readonly dereferenceable(4) [[P:%.*]])
+; IS__TUNIT____-NEXT:    [[V:%.*]] = load i32, i32* [[P]], align 32
+; IS__TUNIT____-NEXT:    ret i32 [[V]]
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@musttail_callee_1
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readonly dereferenceable(4) [[P:%.*]])
+; IS__CGSCC____-NEXT:    [[V:%.*]] = load i32, i32* [[P]], align 32
+; IS__CGSCC____-NEXT:    ret i32 [[V]]
 ;
   %v = load i32, i32* %p, align 32
   ret i32 %v
 }
 define i32 @musttail_caller_1(i32* %p) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readonly willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@musttail_caller_1
 ; IS__TUNIT____-SAME: (i32* nocapture nofree readonly [[P:%.*]])
 ; IS__TUNIT____-NEXT:    [[C:%.*]] = load i1, i1* @cnd, align 1
@@ -758,6 +951,7 @@ define i32 @musttail_caller_1(i32* %p) {
 ; IS__TUNIT____:       exit:
 ; IS__TUNIT____-NEXT:    ret i32 0
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readonly willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@musttail_caller_1
 ; IS__CGSCC____-SAME: (i32* nocapture nofree readonly [[P:%.*]])
 ; IS__CGSCC____-NEXT:    [[C:%.*]] = load i1, i1* @cnd, align 1
diff --git a/llvm/test/Transforms/Attributor/allow_list.ll b/llvm/test/Transforms/Attributor/allow_list.ll
index 7670090cb03b7..3e483dd1d4d76 100644
--- a/llvm/test/Transforms/Attributor/allow_list.ll
+++ b/llvm/test/Transforms/Attributor/allow_list.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -S -passes=attributor --attributor-seed-allow-list asd < %s | FileCheck %s --check-prefixes=CHECK_DISABLED
 ; RUN: opt -S -passes=attributor --attributor-seed-allow-list AAValueSimplify < %s | FileCheck %s --check-prefixes=CHECK_ENABLED
 
@@ -6,6 +6,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 ; Function Attrs: nounwind uwtable
 define internal i32 @range_test(i32 %a) #0 {
+; CHECK_DISABLED: Function Attrs: noinline nounwind uwtable
 ; CHECK_DISABLED-LABEL: define {{[^@]+}}@range_test
 ; CHECK_DISABLED-SAME: (i32 [[A:%.*]])
 ; CHECK_DISABLED-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[A]], 100
@@ -19,10 +20,12 @@ define internal i32 @range_test(i32 %a) #0 {
 
 ; Function Attrs: nounwind uwtable
 define i32 @range_use() #0 {
+; CHECK_DISABLED: Function Attrs: noinline nounwind uwtable
 ; CHECK_DISABLED-LABEL: define {{[^@]+}}@range_use()
 ; CHECK_DISABLED-NEXT:    [[TMP1:%.*]] = call i32 @range_test(i32 123)
 ; CHECK_DISABLED-NEXT:    ret i32 [[TMP1]]
 ;
+; CHECK_ENABLED: Function Attrs: noinline nounwind uwtable
 ; CHECK_ENABLED-LABEL: define {{[^@]+}}@range_use()
 ; CHECK_ENABLED-NEXT:    ret i32 1
 ;
@@ -30,4 +33,4 @@ define i32 @range_use() #0 {
   ret i32 %1
 }
 
-attributes #0 = { nounwind uwtable noinline }
\ No newline at end of file
+attributes #0 = { nounwind uwtable noinline }
diff --git a/llvm/test/Transforms/Attributor/alwaysinline.ll b/llvm/test/Transforms/Attributor/alwaysinline.ll
index 49fc2983fde68..a3aaa3782645f 100644
--- a/llvm/test/Transforms/Attributor/alwaysinline.ll
+++ b/llvm/test/Transforms/Attributor/alwaysinline.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -9,23 +9,31 @@
 
 ; the function is not exactly defined, and marked alwaysinline and can be inlined,
 ; so the function can be analyzed
-; CHECK: Function Attrs: alwaysinline
-; CHECK-SAME: willreturn
 define linkonce void @inner1() alwaysinline {
-; CHECK-LABEL: define {{[^@]+}}@inner1()
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: alwaysinline nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@inner1()
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: alwaysinline nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@inner1()
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    ret void
 ;
 entry:
   ret void
 }
 
-; CHECK: Function Attrs:
-; CHECK-SAME: willreturn
 define void @outer1() {
-; CHECK-LABEL: define {{[^@]+}}@outer1()
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@outer1()
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@outer1()
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    ret void
 ;
 entry:
   call void @inner1()
@@ -34,7 +42,6 @@ entry:
 
 ; The function is not alwaysinline and is not exactly defined
 ; so it will not be analyzed
-; CHECK-NOT: Function Attrs:
 define linkonce i32 @inner2() {
 ; CHECK-LABEL: define {{[^@]+}}@inner2()
 ; CHECK-NEXT:  entry:
@@ -59,12 +66,8 @@ entry:
 ; This function cannot be inlined although it is marked alwaysinline
 ; it is `unexactly defined` and alwaysinline but cannot be inlined.
 ; so it will not be analyzed
-; CHECK: Function Attrs:
-; CHECK-NOT: nofree
-; CHECK-NOT: nosync
-; CHECK-NOT: nounwind
-; CHECK-NOT: readnone
 define linkonce i32 @inner3(i8* %addr) alwaysinline {
+; CHECK: Function Attrs: alwaysinline
 ; CHECK-LABEL: define {{[^@]+}}@inner3
 ; CHECK-SAME: (i8* [[ADDR:%.*]])
 ; CHECK-NEXT:  entry:
@@ -84,7 +87,6 @@ two:
   ret i32 44
 }
 
-; CHECK-NOT: Function Attrs:
 define i32 @outer3(i32 %x) {
 ; CHECK-LABEL: define {{[^@]+}}@outer3
 ; CHECK-SAME: (i32 [[X:%.*]])
diff --git a/llvm/test/Transforms/Attributor/callbacks.ll b/llvm/test/Transforms/Attributor/callbacks.ll
index 249b9fd0276e8..d7fe9dd842614 100644
--- a/llvm/test/Transforms/Attributor/callbacks.ll
+++ b/llvm/test/Transforms/Attributor/callbacks.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -178,6 +178,7 @@ entry:
 ; The others are annotated with alignment information, amongst others, or even replaced by the constants passed to the call.
 define internal void @t1_callback_callee(i32* %is_not_null, i32* %ptr, i32* %a, i64 %b, i32** %c) {
 ;
+; NOT_TUNIT_NPM: Function Attrs: nosync
 ; NOT_TUNIT_NPM-LABEL: define {{[^@]+}}@t1_callback_callee
 ; NOT_TUNIT_NPM-SAME: (i32* nocapture nonnull writeonly align 4 dereferenceable(4) [[IS_NOT_NULL:%.*]], i32* nocapture nonnull readonly align 8 dereferenceable(4) [[PTR:%.*]], i32* nocapture align 256 [[A:%.*]], i64 [[B:%.*]], i32** nocapture nonnull readonly align 64 dereferenceable(8) [[C:%.*]])
 ; NOT_TUNIT_NPM-NEXT:  entry:
@@ -187,6 +188,7 @@ define internal void @t1_callback_callee(i32* %is_not_null, i32* %ptr, i32* %a,
 ; NOT_TUNIT_NPM-NEXT:    tail call void @t1_check(i32* nocapture align 256 [[A]], i64 99, i32* nocapture [[TMP0]])
 ; NOT_TUNIT_NPM-NEXT:    ret void
 ;
+; IS__TUNIT_NPM: Function Attrs: nosync
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@t1_callback_callee
 ; IS__TUNIT_NPM-SAME: (i32* nocapture nonnull writeonly align 4 dereferenceable(4) [[IS_NOT_NULL:%.*]], i32* nocapture nonnull readonly align 8 dereferenceable(4) [[PTR:%.*]], i32* noalias nocapture align 256 [[A:%.*]], i64 [[B:%.*]], i32** noalias nocapture nonnull readonly align 64 dereferenceable(8) [[C:%.*]])
 ; IS__TUNIT_NPM-NEXT:  entry:
diff --git a/llvm/test/Transforms/Attributor/dereferenceable-1.ll b/llvm/test/Transforms/Attributor/dereferenceable-1.ll
index dea81f9c3d14a..402ed41f91fa6 100644
--- a/llvm/test/Transforms/Attributor/dereferenceable-1.ll
+++ b/llvm/test/Transforms/Attributor/dereferenceable-1.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=16 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=16 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -11,11 +11,19 @@ declare void @deref_phi_user(i32* %a);
 ; take mininimum of return values
 ;
 define i32* @test1(i32* dereferenceable(4) %0, double* dereferenceable(8) %1, i1 zeroext %2) local_unnamed_addr {
-; CHECK-LABEL: define {{[^@]+}}@test1
-; CHECK-SAME: (i32* nofree nonnull readnone dereferenceable(4) "no-capture-maybe-returned" [[TMP0:%.*]], double* nofree nonnull readnone dereferenceable(8) "no-capture-maybe-returned" [[TMP1:%.*]], i1 zeroext [[TMP2:%.*]]) local_unnamed_addr
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[TMP1]] to i32*
-; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP2]], i32* [[TMP0]], i32* [[TMP4]]
-; CHECK-NEXT:    ret i32* [[TMP5]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test1
+; IS__TUNIT____-SAME: (i32* nofree nonnull readnone dereferenceable(4) "no-capture-maybe-returned" [[TMP0:%.*]], double* nofree nonnull readnone dereferenceable(8) "no-capture-maybe-returned" [[TMP1:%.*]], i1 zeroext [[TMP2:%.*]]) local_unnamed_addr
+; IS__TUNIT____-NEXT:    [[TMP4:%.*]] = bitcast double* [[TMP1]] to i32*
+; IS__TUNIT____-NEXT:    [[TMP5:%.*]] = select i1 [[TMP2]], i32* [[TMP0]], i32* [[TMP4]]
+; IS__TUNIT____-NEXT:    ret i32* [[TMP5]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test1
+; IS__CGSCC____-SAME: (i32* nofree nonnull readnone dereferenceable(4) "no-capture-maybe-returned" [[TMP0:%.*]], double* nofree nonnull readnone dereferenceable(8) "no-capture-maybe-returned" [[TMP1:%.*]], i1 zeroext [[TMP2:%.*]]) local_unnamed_addr
+; IS__CGSCC____-NEXT:    [[TMP4:%.*]] = bitcast double* [[TMP1]] to i32*
+; IS__CGSCC____-NEXT:    [[TMP5:%.*]] = select i1 [[TMP2]], i32* [[TMP0]], i32* [[TMP4]]
+; IS__CGSCC____-NEXT:    ret i32* [[TMP5]]
 ;
   %4 = bitcast double* %1 to i32*
   %5 = select i1 %2, i32* %0, i32* %4
@@ -24,11 +32,19 @@ define i32* @test1(i32* dereferenceable(4) %0, double* dereferenceable(8) %1, i1
 
 ; TEST 2
 define i32* @test2(i32* dereferenceable_or_null(4) %0, double* dereferenceable(8) %1, i1 zeroext %2) local_unnamed_addr {
-; CHECK-LABEL: define {{[^@]+}}@test2
-; CHECK-SAME: (i32* nofree readnone dereferenceable_or_null(4) "no-capture-maybe-returned" [[TMP0:%.*]], double* nofree nonnull readnone dereferenceable(8) "no-capture-maybe-returned" [[TMP1:%.*]], i1 zeroext [[TMP2:%.*]]) local_unnamed_addr
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[TMP1]] to i32*
-; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP2]], i32* [[TMP0]], i32* [[TMP4]]
-; CHECK-NEXT:    ret i32* [[TMP5]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test2
+; IS__TUNIT____-SAME: (i32* nofree readnone dereferenceable_or_null(4) "no-capture-maybe-returned" [[TMP0:%.*]], double* nofree nonnull readnone dereferenceable(8) "no-capture-maybe-returned" [[TMP1:%.*]], i1 zeroext [[TMP2:%.*]]) local_unnamed_addr
+; IS__TUNIT____-NEXT:    [[TMP4:%.*]] = bitcast double* [[TMP1]] to i32*
+; IS__TUNIT____-NEXT:    [[TMP5:%.*]] = select i1 [[TMP2]], i32* [[TMP0]], i32* [[TMP4]]
+; IS__TUNIT____-NEXT:    ret i32* [[TMP5]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test2
+; IS__CGSCC____-SAME: (i32* nofree readnone dereferenceable_or_null(4) "no-capture-maybe-returned" [[TMP0:%.*]], double* nofree nonnull readnone dereferenceable(8) "no-capture-maybe-returned" [[TMP1:%.*]], i1 zeroext [[TMP2:%.*]]) local_unnamed_addr
+; IS__CGSCC____-NEXT:    [[TMP4:%.*]] = bitcast double* [[TMP1]] to i32*
+; IS__CGSCC____-NEXT:    [[TMP5:%.*]] = select i1 [[TMP2]], i32* [[TMP0]], i32* [[TMP4]]
+; IS__CGSCC____-NEXT:    ret i32* [[TMP5]]
 ;
   %4 = bitcast double* %1 to i32*
   %5 = select i1 %2, i32* %0, i32* %4
@@ -38,32 +54,55 @@ define i32* @test2(i32* dereferenceable_or_null(4) %0, double* dereferenceable(8
 ; TEST 3
 ; GEP inbounds
 define i32* @test3_1(i32* dereferenceable(8) %0) local_unnamed_addr {
-; CHECK-LABEL: define {{[^@]+}}@test3_1
-; CHECK-SAME: (i32* nofree nonnull readnone dereferenceable(8) "no-capture-maybe-returned" [[TMP0:%.*]]) local_unnamed_addr
-; CHECK-NEXT:    [[RET:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 1
-; CHECK-NEXT:    ret i32* [[RET]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test3_1
+; IS__TUNIT____-SAME: (i32* nofree nonnull readnone dereferenceable(8) "no-capture-maybe-returned" [[TMP0:%.*]]) local_unnamed_addr
+; IS__TUNIT____-NEXT:    [[RET:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 1
+; IS__TUNIT____-NEXT:    ret i32* [[RET]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test3_1
+; IS__CGSCC____-SAME: (i32* nofree nonnull readnone dereferenceable(8) "no-capture-maybe-returned" [[TMP0:%.*]]) local_unnamed_addr
+; IS__CGSCC____-NEXT:    [[RET:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 1
+; IS__CGSCC____-NEXT:    ret i32* [[RET]]
 ;
   %ret = getelementptr inbounds i32, i32* %0, i64 1
   ret i32* %ret
 }
 
 define i32* @test3_2(i32* dereferenceable_or_null(32) %0) local_unnamed_addr {
-; CHECK-LABEL: define {{[^@]+}}@test3_2
-; CHECK-SAME: (i32* nofree readnone dereferenceable_or_null(32) "no-capture-maybe-returned" [[TMP0:%.*]]) local_unnamed_addr
-; CHECK-NEXT:    [[RET:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 4
-; CHECK-NEXT:    ret i32* [[RET]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test3_2
+; IS__TUNIT____-SAME: (i32* nofree readnone dereferenceable_or_null(32) "no-capture-maybe-returned" [[TMP0:%.*]]) local_unnamed_addr
+; IS__TUNIT____-NEXT:    [[RET:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 4
+; IS__TUNIT____-NEXT:    ret i32* [[RET]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test3_2
+; IS__CGSCC____-SAME: (i32* nofree readnone dereferenceable_or_null(32) "no-capture-maybe-returned" [[TMP0:%.*]]) local_unnamed_addr
+; IS__CGSCC____-NEXT:    [[RET:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 4
+; IS__CGSCC____-NEXT:    ret i32* [[RET]]
 ;
   %ret = getelementptr inbounds i32, i32* %0, i64 4
   ret i32* %ret
 }
 
 define i32* @test3_3(i32* dereferenceable(8) %0, i32* dereferenceable(16) %1, i1 %2) local_unnamed_addr {
-; CHECK-LABEL: define {{[^@]+}}@test3_3
-; CHECK-SAME: (i32* nofree nonnull readnone dereferenceable(8) "no-capture-maybe-returned" [[TMP0:%.*]], i32* nofree nonnull readnone dereferenceable(16) "no-capture-maybe-returned" [[TMP1:%.*]], i1 [[TMP2:%.*]]) local_unnamed_addr
-; CHECK-NEXT:    [[RET1:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 1
-; CHECK-NEXT:    [[RET2:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 2
-; CHECK-NEXT:    [[RET:%.*]] = select i1 [[TMP2]], i32* [[RET1]], i32* [[RET2]]
-; CHECK-NEXT:    ret i32* [[RET]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test3_3
+; IS__TUNIT____-SAME: (i32* nofree nonnull readnone dereferenceable(8) "no-capture-maybe-returned" [[TMP0:%.*]], i32* nofree nonnull readnone dereferenceable(16) "no-capture-maybe-returned" [[TMP1:%.*]], i1 [[TMP2:%.*]]) local_unnamed_addr
+; IS__TUNIT____-NEXT:    [[RET1:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 1
+; IS__TUNIT____-NEXT:    [[RET2:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 2
+; IS__TUNIT____-NEXT:    [[RET:%.*]] = select i1 [[TMP2]], i32* [[RET1]], i32* [[RET2]]
+; IS__TUNIT____-NEXT:    ret i32* [[RET]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test3_3
+; IS__CGSCC____-SAME: (i32* nofree nonnull readnone dereferenceable(8) "no-capture-maybe-returned" [[TMP0:%.*]], i32* nofree nonnull readnone dereferenceable(16) "no-capture-maybe-returned" [[TMP1:%.*]], i1 [[TMP2:%.*]]) local_unnamed_addr
+; IS__CGSCC____-NEXT:    [[RET1:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 1
+; IS__CGSCC____-NEXT:    [[RET2:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 2
+; IS__CGSCC____-NEXT:    [[RET:%.*]] = select i1 [[TMP2]], i32* [[RET1]], i32* [[RET2]]
+; IS__CGSCC____-NEXT:    ret i32* [[RET]]
 ;
   %ret1 = getelementptr inbounds i32, i32* %0, i64 1
   %ret2 = getelementptr inbounds i32, i32* %1, i64 2
@@ -75,9 +114,15 @@ define i32* @test3_3(i32* dereferenceable(8) %0, i32* dereferenceable(16) %1, i1
 ; Better than known in IR.
 
 define dereferenceable(4) i32* @test4(i32* dereferenceable(8) %0) local_unnamed_addr {
-; CHECK-LABEL: define {{[^@]+}}@test4
-; CHECK-SAME: (i32* nofree nonnull readnone returned dereferenceable(8) "no-capture-maybe-returned" [[TMP0:%.*]]) local_unnamed_addr
-; CHECK-NEXT:    ret i32* [[TMP0]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test4
+; IS__TUNIT____-SAME: (i32* nofree nonnull readnone returned dereferenceable(8) "no-capture-maybe-returned" [[TMP0:%.*]]) local_unnamed_addr
+; IS__TUNIT____-NEXT:    ret i32* [[TMP0]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test4
+; IS__CGSCC____-SAME: (i32* nofree nonnull readnone returned dereferenceable(8) "no-capture-maybe-returned" [[TMP0:%.*]]) local_unnamed_addr
+; IS__CGSCC____-NEXT:    ret i32* [[TMP0]]
 ;
   ret i32* %0
 }
@@ -189,6 +234,7 @@ for.end:                                          ; preds = %for.cond.cleanup
 declare i32* @unkown_ptr() willreturn nounwind
 declare i32 @unkown_f(i32*) willreturn nounwind
 define i32* @f7_0(i32* %ptr) {
+; CHECK: Function Attrs: nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@f7_0
 ; CHECK-SAME: (i32* nonnull returned dereferenceable(8) [[PTR:%.*]])
 ; CHECK-NEXT:    [[T:%.*]] = tail call i32 @unkown_f(i32* nonnull dereferenceable(8) [[PTR]])
@@ -199,6 +245,7 @@ define i32* @f7_0(i32* %ptr) {
 }
 
 define void @f7_1(i32* %ptr, i1 %c) {
+; CHECK: Function Attrs: nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@f7_1
 ; CHECK-SAME: (i32* nonnull align 4 dereferenceable(4) [[PTR:%.*]], i1 [[C:%.*]])
 ; CHECK-NEXT:    [[A:%.*]] = tail call i32 @unkown_f(i32* nonnull align 4 dereferenceable(4) [[PTR]])
@@ -228,6 +275,7 @@ if.false:
 }
 
 define void @f7_2(i1 %c) {
+; CHECK: Function Attrs: nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@f7_2
 ; CHECK-SAME: (i1 [[C:%.*]])
 ; CHECK-NEXT:    [[PTR:%.*]] = tail call nonnull align 4 dereferenceable(4) i32* @unkown_ptr()
@@ -258,6 +306,7 @@ if.false:
 }
 
 define i32* @f7_3() {
+; CHECK: Function Attrs: nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@f7_3()
 ; CHECK-NEXT:    [[PTR:%.*]] = tail call nonnull align 16 dereferenceable(4) i32* @unkown_ptr()
 ; CHECK-NEXT:    store i32 10, i32* [[PTR]], align 16
@@ -270,11 +319,19 @@ define i32* @f7_3() {
 
 ; FIXME: This should have a return dereferenceable(8) but we need to make sure it will work in loops as well.
 define i32* @test_for_minus_index(i32* %p) {
-; CHECK-LABEL: define {{[^@]+}}@test_for_minus_index
-; CHECK-SAME: (i32* nofree nonnull writeonly align 4 "no-capture-maybe-returned" [[P:%.*]])
-; CHECK-NEXT:    [[Q:%.*]] = getelementptr inbounds i32, i32* [[P]], i32 -2
-; CHECK-NEXT:    store i32 1, i32* [[Q]], align 4
-; CHECK-NEXT:    ret i32* [[Q]]
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test_for_minus_index
+; IS__TUNIT____-SAME: (i32* nofree nonnull writeonly align 4 "no-capture-maybe-returned" [[P:%.*]])
+; IS__TUNIT____-NEXT:    [[Q:%.*]] = getelementptr inbounds i32, i32* [[P]], i32 -2
+; IS__TUNIT____-NEXT:    store i32 1, i32* [[Q]], align 4
+; IS__TUNIT____-NEXT:    ret i32* [[Q]]
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test_for_minus_index
+; IS__CGSCC____-SAME: (i32* nofree nonnull writeonly align 4 "no-capture-maybe-returned" [[P:%.*]])
+; IS__CGSCC____-NEXT:    [[Q:%.*]] = getelementptr inbounds i32, i32* [[P]], i32 -2
+; IS__CGSCC____-NEXT:    store i32 1, i32* [[Q]], align 4
+; IS__CGSCC____-NEXT:    ret i32* [[Q]]
 ;
   %q = getelementptr inbounds i32, i32* %p, i32 -2
   store i32 1, i32* %q
@@ -282,10 +339,17 @@ define i32* @test_for_minus_index(i32* %p) {
 }
 
 define void @deref_or_null_and_nonnull(i32* dereferenceable_or_null(100) %0) {
-; CHECK-LABEL: define {{[^@]+}}@deref_or_null_and_nonnull
-; CHECK-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(100) [[TMP0:%.*]])
-; CHECK-NEXT:    store i32 1, i32* [[TMP0]], align 4
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@deref_or_null_and_nonnull
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(100) [[TMP0:%.*]])
+; IS__TUNIT____-NEXT:    store i32 1, i32* [[TMP0]], align 4
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@deref_or_null_and_nonnull
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(100) [[TMP0:%.*]])
+; IS__CGSCC____-NEXT:    store i32 1, i32* [[TMP0]], align 4
+; IS__CGSCC____-NEXT:    ret void
 ;
   store i32 1, i32* %0
   ret void
@@ -300,37 +364,73 @@ define void @deref_or_null_and_nonnull(i32* dereferenceable_or_null(100) %0) {
 
 ; FIXME: %ptr should be dereferenceable(31)
 define void @test8(i8* %ptr) #0 {
-; IS________OPM-LABEL: define {{[^@]+}}@test8
-; IS________OPM-SAME: (i8* nocapture nofree nonnull writeonly [[PTR:%.*]])
-; IS________OPM-NEXT:    br label [[TMP1:%.*]]
-; IS________OPM:       1:
-; IS________OPM-NEXT:    [[I_0:%.*]] = phi i32 [ 20, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[TMP5:%.*]] ]
-; IS________OPM-NEXT:    [[TMP2:%.*]] = sext i32 [[I_0]] to i64
-; IS________OPM-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, i8* [[PTR]], i64 [[TMP2]]
-; IS________OPM-NEXT:    store i8 32, i8* [[TMP3]], align 1
-; IS________OPM-NEXT:    [[TMP4]] = add nsw i32 [[I_0]], 1
-; IS________OPM-NEXT:    br label [[TMP5]]
-; IS________OPM:       5:
-; IS________OPM-NEXT:    [[TMP6:%.*]] = icmp slt i32 [[TMP4]], 30
-; IS________OPM-NEXT:    br i1 [[TMP6]], label [[TMP1]], label [[TMP7:%.*]]
-; IS________OPM:       7:
-; IS________OPM-NEXT:    ret void
-;
-; IS________NPM-LABEL: define {{[^@]+}}@test8
-; IS________NPM-SAME: (i8* nocapture nofree nonnull writeonly dereferenceable(21) [[PTR:%.*]])
-; IS________NPM-NEXT:    br label [[TMP1:%.*]]
-; IS________NPM:       1:
-; IS________NPM-NEXT:    [[I_0:%.*]] = phi i32 [ 20, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[TMP5:%.*]] ]
-; IS________NPM-NEXT:    [[TMP2:%.*]] = sext i32 [[I_0]] to i64
-; IS________NPM-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, i8* [[PTR]], i64 [[TMP2]]
-; IS________NPM-NEXT:    store i8 32, i8* [[TMP3]], align 1
-; IS________NPM-NEXT:    [[TMP4]] = add nsw i32 [[I_0]], 1
-; IS________NPM-NEXT:    br label [[TMP5]]
-; IS________NPM:       5:
-; IS________NPM-NEXT:    [[TMP6:%.*]] = icmp slt i32 [[TMP4]], 30
-; IS________NPM-NEXT:    br i1 [[TMP6]], label [[TMP1]], label [[TMP7:%.*]]
-; IS________NPM:       7:
-; IS________NPM-NEXT:    ret void
+; IS__TUNIT_OPM: Function Attrs: argmemonly nofree nosync nounwind writeonly
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@test8
+; IS__TUNIT_OPM-SAME: (i8* nocapture nofree nonnull writeonly [[PTR:%.*]])
+; IS__TUNIT_OPM-NEXT:    br label [[TMP1:%.*]]
+; IS__TUNIT_OPM:       1:
+; IS__TUNIT_OPM-NEXT:    [[I_0:%.*]] = phi i32 [ 20, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[TMP5:%.*]] ]
+; IS__TUNIT_OPM-NEXT:    [[TMP2:%.*]] = sext i32 [[I_0]] to i64
+; IS__TUNIT_OPM-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, i8* [[PTR]], i64 [[TMP2]]
+; IS__TUNIT_OPM-NEXT:    store i8 32, i8* [[TMP3]], align 1
+; IS__TUNIT_OPM-NEXT:    [[TMP4]] = add nsw i32 [[I_0]], 1
+; IS__TUNIT_OPM-NEXT:    br label [[TMP5]]
+; IS__TUNIT_OPM:       5:
+; IS__TUNIT_OPM-NEXT:    [[TMP6:%.*]] = icmp slt i32 [[TMP4]], 30
+; IS__TUNIT_OPM-NEXT:    br i1 [[TMP6]], label [[TMP1]], label [[TMP7:%.*]]
+; IS__TUNIT_OPM:       7:
+; IS__TUNIT_OPM-NEXT:    ret void
+;
+; IS__TUNIT_NPM: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@test8
+; IS__TUNIT_NPM-SAME: (i8* nocapture nofree nonnull writeonly dereferenceable(21) [[PTR:%.*]])
+; IS__TUNIT_NPM-NEXT:    br label [[TMP1:%.*]]
+; IS__TUNIT_NPM:       1:
+; IS__TUNIT_NPM-NEXT:    [[I_0:%.*]] = phi i32 [ 20, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[TMP5:%.*]] ]
+; IS__TUNIT_NPM-NEXT:    [[TMP2:%.*]] = sext i32 [[I_0]] to i64
+; IS__TUNIT_NPM-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, i8* [[PTR]], i64 [[TMP2]]
+; IS__TUNIT_NPM-NEXT:    store i8 32, i8* [[TMP3]], align 1
+; IS__TUNIT_NPM-NEXT:    [[TMP4]] = add nsw i32 [[I_0]], 1
+; IS__TUNIT_NPM-NEXT:    br label [[TMP5]]
+; IS__TUNIT_NPM:       5:
+; IS__TUNIT_NPM-NEXT:    [[TMP6:%.*]] = icmp slt i32 [[TMP4]], 30
+; IS__TUNIT_NPM-NEXT:    br i1 [[TMP6]], label [[TMP1]], label [[TMP7:%.*]]
+; IS__TUNIT_NPM:       7:
+; IS__TUNIT_NPM-NEXT:    ret void
+;
+; IS__CGSCC_OPM: Function Attrs: argmemonly nofree norecurse nosync nounwind writeonly
+; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@test8
+; IS__CGSCC_OPM-SAME: (i8* nocapture nofree nonnull writeonly [[PTR:%.*]])
+; IS__CGSCC_OPM-NEXT:    br label [[TMP1:%.*]]
+; IS__CGSCC_OPM:       1:
+; IS__CGSCC_OPM-NEXT:    [[I_0:%.*]] = phi i32 [ 20, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[TMP5:%.*]] ]
+; IS__CGSCC_OPM-NEXT:    [[TMP2:%.*]] = sext i32 [[I_0]] to i64
+; IS__CGSCC_OPM-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, i8* [[PTR]], i64 [[TMP2]]
+; IS__CGSCC_OPM-NEXT:    store i8 32, i8* [[TMP3]], align 1
+; IS__CGSCC_OPM-NEXT:    [[TMP4]] = add nsw i32 [[I_0]], 1
+; IS__CGSCC_OPM-NEXT:    br label [[TMP5]]
+; IS__CGSCC_OPM:       5:
+; IS__CGSCC_OPM-NEXT:    [[TMP6:%.*]] = icmp slt i32 [[TMP4]], 30
+; IS__CGSCC_OPM-NEXT:    br i1 [[TMP6]], label [[TMP1]], label [[TMP7:%.*]]
+; IS__CGSCC_OPM:       7:
+; IS__CGSCC_OPM-NEXT:    ret void
+;
+; IS__CGSCC_NPM: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@test8
+; IS__CGSCC_NPM-SAME: (i8* nocapture nofree nonnull writeonly dereferenceable(21) [[PTR:%.*]])
+; IS__CGSCC_NPM-NEXT:    br label [[TMP1:%.*]]
+; IS__CGSCC_NPM:       1:
+; IS__CGSCC_NPM-NEXT:    [[I_0:%.*]] = phi i32 [ 20, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[TMP5:%.*]] ]
+; IS__CGSCC_NPM-NEXT:    [[TMP2:%.*]] = sext i32 [[I_0]] to i64
+; IS__CGSCC_NPM-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, i8* [[PTR]], i64 [[TMP2]]
+; IS__CGSCC_NPM-NEXT:    store i8 32, i8* [[TMP3]], align 1
+; IS__CGSCC_NPM-NEXT:    [[TMP4]] = add nsw i32 [[I_0]], 1
+; IS__CGSCC_NPM-NEXT:    br label [[TMP5]]
+; IS__CGSCC_NPM:       5:
+; IS__CGSCC_NPM-NEXT:    [[TMP6:%.*]] = icmp slt i32 [[TMP4]], 30
+; IS__CGSCC_NPM-NEXT:    br i1 [[TMP6]], label [[TMP1]], label [[TMP7:%.*]]
+; IS__CGSCC_NPM:       7:
+; IS__CGSCC_NPM-NEXT:    ret void
 ;
   br label %1
 1:                                                ; preds = %5, %0
@@ -350,12 +450,21 @@ define void @test8(i8* %ptr) #0 {
 
 ; 8.2 (negative case)
 define void @test8_neg(i32 %i, i8* %ptr) #0 {
-; CHECK-LABEL: define {{[^@]+}}@test8_neg
-; CHECK-SAME: (i32 [[I:%.*]], i8* nocapture nofree nonnull writeonly [[PTR:%.*]])
-; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[I]] to i64
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[PTR]], i64 [[TMP1]]
-; CHECK-NEXT:    store i8 65, i8* [[TMP2]], align 1
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test8_neg
+; IS__TUNIT____-SAME: (i32 [[I:%.*]], i8* nocapture nofree nonnull writeonly [[PTR:%.*]])
+; IS__TUNIT____-NEXT:    [[TMP1:%.*]] = sext i32 [[I]] to i64
+; IS__TUNIT____-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[PTR]], i64 [[TMP1]]
+; IS__TUNIT____-NEXT:    store i8 65, i8* [[TMP2]], align 1
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test8_neg
+; IS__CGSCC____-SAME: (i32 [[I:%.*]], i8* nocapture nofree nonnull writeonly [[PTR:%.*]])
+; IS__CGSCC____-NEXT:    [[TMP1:%.*]] = sext i32 [[I]] to i64
+; IS__CGSCC____-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[PTR]], i64 [[TMP1]]
+; IS__CGSCC____-NEXT:    store i8 65, i8* [[TMP2]], align 1
+; IS__CGSCC____-NEXT:    ret void
 ;
   %1 = sext i32 %i to i64
   %2 = getelementptr inbounds i8, i8* %ptr, i64 %1
@@ -372,21 +481,73 @@ define void @test8_neg(i32 %i, i8* %ptr) #0 {
 
 ; NOTE: %p should not be dereferenceable
 define internal void @fill_range_not_inbounds(i32* %p, i64 %start){
-; CHECK-LABEL: define {{[^@]+}}@fill_range_not_inbounds
-; CHECK-SAME: (i32* nocapture nofree writeonly [[P:%.*]], i64 [[START:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i64 [[START]], 9
-; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
-; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    ret void
-; CHECK:       for.body:
-; CHECK-NEXT:    [[I_06:%.*]] = phi i64 [ [[START]], [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[CONV:%.*]] = trunc i64 [[I_06]] to i32
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr i32, i32* [[P]], i64 [[I_06]]
-; CHECK-NEXT:    store i32 [[CONV]], i32* [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[INC]] = add nsw i64 [[I_06]], 1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[I_06]], [[TMP0]]
-; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
+; IS__TUNIT_OPM: Function Attrs: argmemonly nofree nosync nounwind writeonly
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@fill_range_not_inbounds
+; IS__TUNIT_OPM-SAME: (i32* nocapture nofree writeonly [[P:%.*]], i64 [[START:%.*]])
+; IS__TUNIT_OPM-NEXT:  entry:
+; IS__TUNIT_OPM-NEXT:    [[TMP0:%.*]] = add nsw i64 [[START]], 9
+; IS__TUNIT_OPM-NEXT:    br label [[FOR_BODY:%.*]]
+; IS__TUNIT_OPM:       for.cond.cleanup:
+; IS__TUNIT_OPM-NEXT:    ret void
+; IS__TUNIT_OPM:       for.body:
+; IS__TUNIT_OPM-NEXT:    [[I_06:%.*]] = phi i64 [ [[START]], [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; IS__TUNIT_OPM-NEXT:    [[CONV:%.*]] = trunc i64 [[I_06]] to i32
+; IS__TUNIT_OPM-NEXT:    [[ARRAYIDX:%.*]] = getelementptr i32, i32* [[P]], i64 [[I_06]]
+; IS__TUNIT_OPM-NEXT:    store i32 [[CONV]], i32* [[ARRAYIDX]], align 4
+; IS__TUNIT_OPM-NEXT:    [[INC]] = add nsw i64 [[I_06]], 1
+; IS__TUNIT_OPM-NEXT:    [[CMP:%.*]] = icmp slt i64 [[I_06]], [[TMP0]]
+; IS__TUNIT_OPM-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
+;
+; IS__TUNIT_NPM: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@fill_range_not_inbounds
+; IS__TUNIT_NPM-SAME: (i32* nocapture nofree writeonly [[P:%.*]], i64 [[START:%.*]])
+; IS__TUNIT_NPM-NEXT:  entry:
+; IS__TUNIT_NPM-NEXT:    [[TMP0:%.*]] = add nsw i64 [[START]], 9
+; IS__TUNIT_NPM-NEXT:    br label [[FOR_BODY:%.*]]
+; IS__TUNIT_NPM:       for.cond.cleanup:
+; IS__TUNIT_NPM-NEXT:    ret void
+; IS__TUNIT_NPM:       for.body:
+; IS__TUNIT_NPM-NEXT:    [[I_06:%.*]] = phi i64 [ [[START]], [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; IS__TUNIT_NPM-NEXT:    [[CONV:%.*]] = trunc i64 [[I_06]] to i32
+; IS__TUNIT_NPM-NEXT:    [[ARRAYIDX:%.*]] = getelementptr i32, i32* [[P]], i64 [[I_06]]
+; IS__TUNIT_NPM-NEXT:    store i32 [[CONV]], i32* [[ARRAYIDX]], align 4
+; IS__TUNIT_NPM-NEXT:    [[INC]] = add nsw i64 [[I_06]], 1
+; IS__TUNIT_NPM-NEXT:    [[CMP:%.*]] = icmp slt i64 [[I_06]], [[TMP0]]
+; IS__TUNIT_NPM-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
+;
+; IS__CGSCC_OPM: Function Attrs: argmemonly nofree norecurse nosync nounwind writeonly
+; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@fill_range_not_inbounds
+; IS__CGSCC_OPM-SAME: (i32* nocapture nofree writeonly [[P:%.*]], i64 [[START:%.*]])
+; IS__CGSCC_OPM-NEXT:  entry:
+; IS__CGSCC_OPM-NEXT:    [[TMP0:%.*]] = add nsw i64 [[START]], 9
+; IS__CGSCC_OPM-NEXT:    br label [[FOR_BODY:%.*]]
+; IS__CGSCC_OPM:       for.cond.cleanup:
+; IS__CGSCC_OPM-NEXT:    ret void
+; IS__CGSCC_OPM:       for.body:
+; IS__CGSCC_OPM-NEXT:    [[I_06:%.*]] = phi i64 [ [[START]], [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; IS__CGSCC_OPM-NEXT:    [[CONV:%.*]] = trunc i64 [[I_06]] to i32
+; IS__CGSCC_OPM-NEXT:    [[ARRAYIDX:%.*]] = getelementptr i32, i32* [[P]], i64 [[I_06]]
+; IS__CGSCC_OPM-NEXT:    store i32 [[CONV]], i32* [[ARRAYIDX]], align 4
+; IS__CGSCC_OPM-NEXT:    [[INC]] = add nsw i64 [[I_06]], 1
+; IS__CGSCC_OPM-NEXT:    [[CMP:%.*]] = icmp slt i64 [[I_06]], [[TMP0]]
+; IS__CGSCC_OPM-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
+;
+; IS__CGSCC_NPM: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@fill_range_not_inbounds
+; IS__CGSCC_NPM-SAME: (i32* nocapture nofree writeonly [[P:%.*]], i64 [[START:%.*]])
+; IS__CGSCC_NPM-NEXT:  entry:
+; IS__CGSCC_NPM-NEXT:    [[TMP0:%.*]] = add nsw i64 [[START]], 9
+; IS__CGSCC_NPM-NEXT:    br label [[FOR_BODY:%.*]]
+; IS__CGSCC_NPM:       for.cond.cleanup:
+; IS__CGSCC_NPM-NEXT:    ret void
+; IS__CGSCC_NPM:       for.body:
+; IS__CGSCC_NPM-NEXT:    [[I_06:%.*]] = phi i64 [ [[START]], [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; IS__CGSCC_NPM-NEXT:    [[CONV:%.*]] = trunc i64 [[I_06]] to i32
+; IS__CGSCC_NPM-NEXT:    [[ARRAYIDX:%.*]] = getelementptr i32, i32* [[P]], i64 [[I_06]]
+; IS__CGSCC_NPM-NEXT:    store i32 [[CONV]], i32* [[ARRAYIDX]], align 4
+; IS__CGSCC_NPM-NEXT:    [[INC]] = add nsw i64 [[I_06]], 1
+; IS__CGSCC_NPM-NEXT:    [[CMP:%.*]] = icmp slt i64 [[I_06]], [[TMP0]]
+; IS__CGSCC_NPM-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
 ;
 entry:
   %0 = add nsw i64 %start, 9
@@ -407,21 +568,73 @@ for.body:                                         ; preds = %entry, %for.body
 
 ; FIXME: %p should be dereferenceable(40)
 define internal void @fill_range_inbounds(i32* %p, i64 %start){
-; CHECK-LABEL: define {{[^@]+}}@fill_range_inbounds
-; CHECK-SAME: (i32* nocapture nofree writeonly [[P:%.*]], i64 [[START:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i64 [[START]], 9
-; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
-; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    ret void
-; CHECK:       for.body:
-; CHECK-NEXT:    [[I_06:%.*]] = phi i64 [ [[START]], [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[CONV:%.*]] = trunc i64 [[I_06]] to i32
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 [[I_06]]
-; CHECK-NEXT:    store i32 [[CONV]], i32* [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[INC]] = add nsw i64 [[I_06]], 1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[I_06]], [[TMP0]]
-; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
+; IS__TUNIT_OPM: Function Attrs: argmemonly nofree nosync nounwind writeonly
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@fill_range_inbounds
+; IS__TUNIT_OPM-SAME: (i32* nocapture nofree writeonly [[P:%.*]], i64 [[START:%.*]])
+; IS__TUNIT_OPM-NEXT:  entry:
+; IS__TUNIT_OPM-NEXT:    [[TMP0:%.*]] = add nsw i64 [[START]], 9
+; IS__TUNIT_OPM-NEXT:    br label [[FOR_BODY:%.*]]
+; IS__TUNIT_OPM:       for.cond.cleanup:
+; IS__TUNIT_OPM-NEXT:    ret void
+; IS__TUNIT_OPM:       for.body:
+; IS__TUNIT_OPM-NEXT:    [[I_06:%.*]] = phi i64 [ [[START]], [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; IS__TUNIT_OPM-NEXT:    [[CONV:%.*]] = trunc i64 [[I_06]] to i32
+; IS__TUNIT_OPM-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 [[I_06]]
+; IS__TUNIT_OPM-NEXT:    store i32 [[CONV]], i32* [[ARRAYIDX]], align 4
+; IS__TUNIT_OPM-NEXT:    [[INC]] = add nsw i64 [[I_06]], 1
+; IS__TUNIT_OPM-NEXT:    [[CMP:%.*]] = icmp slt i64 [[I_06]], [[TMP0]]
+; IS__TUNIT_OPM-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
+;
+; IS__TUNIT_NPM: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@fill_range_inbounds
+; IS__TUNIT_NPM-SAME: (i32* nocapture nofree writeonly [[P:%.*]], i64 [[START:%.*]])
+; IS__TUNIT_NPM-NEXT:  entry:
+; IS__TUNIT_NPM-NEXT:    [[TMP0:%.*]] = add nsw i64 [[START]], 9
+; IS__TUNIT_NPM-NEXT:    br label [[FOR_BODY:%.*]]
+; IS__TUNIT_NPM:       for.cond.cleanup:
+; IS__TUNIT_NPM-NEXT:    ret void
+; IS__TUNIT_NPM:       for.body:
+; IS__TUNIT_NPM-NEXT:    [[I_06:%.*]] = phi i64 [ [[START]], [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; IS__TUNIT_NPM-NEXT:    [[CONV:%.*]] = trunc i64 [[I_06]] to i32
+; IS__TUNIT_NPM-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 [[I_06]]
+; IS__TUNIT_NPM-NEXT:    store i32 [[CONV]], i32* [[ARRAYIDX]], align 4
+; IS__TUNIT_NPM-NEXT:    [[INC]] = add nsw i64 [[I_06]], 1
+; IS__TUNIT_NPM-NEXT:    [[CMP:%.*]] = icmp slt i64 [[I_06]], [[TMP0]]
+; IS__TUNIT_NPM-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
+;
+; IS__CGSCC_OPM: Function Attrs: argmemonly nofree norecurse nosync nounwind writeonly
+; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@fill_range_inbounds
+; IS__CGSCC_OPM-SAME: (i32* nocapture nofree writeonly [[P:%.*]], i64 [[START:%.*]])
+; IS__CGSCC_OPM-NEXT:  entry:
+; IS__CGSCC_OPM-NEXT:    [[TMP0:%.*]] = add nsw i64 [[START]], 9
+; IS__CGSCC_OPM-NEXT:    br label [[FOR_BODY:%.*]]
+; IS__CGSCC_OPM:       for.cond.cleanup:
+; IS__CGSCC_OPM-NEXT:    ret void
+; IS__CGSCC_OPM:       for.body:
+; IS__CGSCC_OPM-NEXT:    [[I_06:%.*]] = phi i64 [ [[START]], [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; IS__CGSCC_OPM-NEXT:    [[CONV:%.*]] = trunc i64 [[I_06]] to i32
+; IS__CGSCC_OPM-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 [[I_06]]
+; IS__CGSCC_OPM-NEXT:    store i32 [[CONV]], i32* [[ARRAYIDX]], align 4
+; IS__CGSCC_OPM-NEXT:    [[INC]] = add nsw i64 [[I_06]], 1
+; IS__CGSCC_OPM-NEXT:    [[CMP:%.*]] = icmp slt i64 [[I_06]], [[TMP0]]
+; IS__CGSCC_OPM-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
+;
+; IS__CGSCC_NPM: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@fill_range_inbounds
+; IS__CGSCC_NPM-SAME: (i32* nocapture nofree writeonly [[P:%.*]], i64 [[START:%.*]])
+; IS__CGSCC_NPM-NEXT:  entry:
+; IS__CGSCC_NPM-NEXT:    [[TMP0:%.*]] = add nsw i64 [[START]], 9
+; IS__CGSCC_NPM-NEXT:    br label [[FOR_BODY:%.*]]
+; IS__CGSCC_NPM:       for.cond.cleanup:
+; IS__CGSCC_NPM-NEXT:    ret void
+; IS__CGSCC_NPM:       for.body:
+; IS__CGSCC_NPM-NEXT:    [[I_06:%.*]] = phi i64 [ [[START]], [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; IS__CGSCC_NPM-NEXT:    [[CONV:%.*]] = trunc i64 [[I_06]] to i32
+; IS__CGSCC_NPM-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 [[I_06]]
+; IS__CGSCC_NPM-NEXT:    store i32 [[CONV]], i32* [[ARRAYIDX]], align 4
+; IS__CGSCC_NPM-NEXT:    [[INC]] = add nsw i64 [[I_06]], 1
+; IS__CGSCC_NPM-NEXT:    [[CMP:%.*]] = icmp slt i64 [[I_06]], [[TMP0]]
+; IS__CGSCC_NPM-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
 ;
 entry:
   %0 = add nsw i64 %start, 9
@@ -441,13 +654,41 @@ for.body:                                         ; preds = %entry, %for.body
 }
 
 define void @call_fill_range(i32* nocapture %p, i64* nocapture readonly %range) {
-; CHECK-LABEL: define {{[^@]+}}@call_fill_range
-; CHECK-SAME: (i32* nocapture nofree writeonly [[P:%.*]], i64* nocapture nofree nonnull readonly align 8 dereferenceable(8) [[RANGE:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[RANGE]], align 8, !range !0
-; CHECK-NEXT:    tail call void @fill_range_inbounds(i32* nocapture nofree writeonly [[P]], i64 [[TMP0]])
-; CHECK-NEXT:    tail call void @fill_range_not_inbounds(i32* nocapture nofree writeonly [[P]], i64 [[TMP0]])
-; CHECK-NEXT:    ret void
+; IS__TUNIT_OPM: Function Attrs: argmemonly nofree nosync nounwind
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@call_fill_range
+; IS__TUNIT_OPM-SAME: (i32* nocapture nofree writeonly [[P:%.*]], i64* nocapture nofree nonnull readonly align 8 dereferenceable(8) [[RANGE:%.*]])
+; IS__TUNIT_OPM-NEXT:  entry:
+; IS__TUNIT_OPM-NEXT:    [[TMP0:%.*]] = load i64, i64* [[RANGE]], align 8, !range !0
+; IS__TUNIT_OPM-NEXT:    tail call void @fill_range_inbounds(i32* nocapture nofree writeonly [[P]], i64 [[TMP0]])
+; IS__TUNIT_OPM-NEXT:    tail call void @fill_range_not_inbounds(i32* nocapture nofree writeonly [[P]], i64 [[TMP0]])
+; IS__TUNIT_OPM-NEXT:    ret void
+;
+; IS__TUNIT_NPM: Function Attrs: argmemonly nofree nosync nounwind willreturn
+; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@call_fill_range
+; IS__TUNIT_NPM-SAME: (i32* nocapture nofree writeonly [[P:%.*]], i64* nocapture nofree nonnull readonly align 8 dereferenceable(8) [[RANGE:%.*]])
+; IS__TUNIT_NPM-NEXT:  entry:
+; IS__TUNIT_NPM-NEXT:    [[TMP0:%.*]] = load i64, i64* [[RANGE]], align 8, !range !0
+; IS__TUNIT_NPM-NEXT:    tail call void @fill_range_inbounds(i32* nocapture nofree writeonly [[P]], i64 [[TMP0]])
+; IS__TUNIT_NPM-NEXT:    tail call void @fill_range_not_inbounds(i32* nocapture nofree writeonly [[P]], i64 [[TMP0]])
+; IS__TUNIT_NPM-NEXT:    ret void
+;
+; IS__CGSCC_OPM: Function Attrs: argmemonly nofree norecurse nosync nounwind
+; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@call_fill_range
+; IS__CGSCC_OPM-SAME: (i32* nocapture nofree writeonly [[P:%.*]], i64* nocapture nofree nonnull readonly align 8 dereferenceable(8) [[RANGE:%.*]])
+; IS__CGSCC_OPM-NEXT:  entry:
+; IS__CGSCC_OPM-NEXT:    [[TMP0:%.*]] = load i64, i64* [[RANGE]], align 8, !range !0
+; IS__CGSCC_OPM-NEXT:    tail call void @fill_range_inbounds(i32* nocapture nofree writeonly [[P]], i64 [[TMP0]])
+; IS__CGSCC_OPM-NEXT:    tail call void @fill_range_not_inbounds(i32* nocapture nofree writeonly [[P]], i64 [[TMP0]])
+; IS__CGSCC_OPM-NEXT:    ret void
+;
+; IS__CGSCC_NPM: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@call_fill_range
+; IS__CGSCC_NPM-SAME: (i32* nocapture nofree writeonly [[P:%.*]], i64* nocapture nofree nonnull readonly align 8 dereferenceable(8) [[RANGE:%.*]])
+; IS__CGSCC_NPM-NEXT:  entry:
+; IS__CGSCC_NPM-NEXT:    [[TMP0:%.*]] = load i64, i64* [[RANGE]], align 8, !range !0
+; IS__CGSCC_NPM-NEXT:    tail call void @fill_range_inbounds(i32* nocapture nofree writeonly [[P]], i64 [[TMP0]])
+; IS__CGSCC_NPM-NEXT:    tail call void @fill_range_not_inbounds(i32* nocapture nofree writeonly [[P]], i64 [[TMP0]])
+; IS__CGSCC_NPM-NEXT:    ret void
 ;
 entry:
   %0 = load i64, i64* %range, align 8, !range !0
@@ -468,6 +709,7 @@ declare void @use3(i8*, i8*, i8*) willreturn nounwind
 ;   fun2(dereferenceable(4) %a, %b)
 ; We can say that %a is dereferenceable(4) but %b is not.
 define void @simple-path(i8* %a, i8 * %b, i8 %c) {
+; CHECK: Function Attrs: nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@simple-path
 ; CHECK-SAME: (i8* nonnull dereferenceable(4) [[A:%.*]], i8* [[B:%.*]], i8 [[C:%.*]])
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[C]], 0
@@ -502,6 +744,7 @@ if.else:
 ; }
 ; %a is dereferenceable(12)
 define void @complex-path(i8* %a, i8* %b, i8 %c) {
+; CHECK: Function Attrs: nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@complex-path
 ; CHECK-SAME: (i8* nonnull dereferenceable(12) [[A:%.*]], i8* nocapture nofree readnone [[B:%.*]], i8 [[C:%.*]])
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[C]], 0
@@ -547,31 +790,59 @@ cont2:
 ;
 ; FIXME: %ptr should be dereferenceable(4)
 define dso_local void @rec-branch-1(i32 %a, i32 %b, i32 %c, i32* %ptr) {
-; CHECK-LABEL: define {{[^@]+}}@rec-branch-1
-; CHECK-SAME: (i32 [[A:%.*]], i32 [[B:%.*]], i32 [[C:%.*]], i32* nocapture nofree writeonly [[PTR:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[A]], 0
-; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_ELSE3:%.*]], label [[IF_THEN:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    [[TOBOOL1:%.*]] = icmp eq i32 [[B]], 0
-; CHECK-NEXT:    br i1 [[TOBOOL1]], label [[IF_ELSE:%.*]], label [[IF_THEN2:%.*]]
-; CHECK:       if.then2:
-; CHECK-NEXT:    store i32 1, i32* [[PTR]], align 4
-; CHECK-NEXT:    br label [[IF_END8:%.*]]
-; CHECK:       if.else:
-; CHECK-NEXT:    store i32 2, i32* [[PTR]], align 4
-; CHECK-NEXT:    br label [[IF_END8]]
-; CHECK:       if.else3:
-; CHECK-NEXT:    [[TOBOOL4:%.*]] = icmp eq i32 [[C]], 0
-; CHECK-NEXT:    br i1 [[TOBOOL4]], label [[IF_ELSE6:%.*]], label [[IF_THEN5:%.*]]
-; CHECK:       if.then5:
-; CHECK-NEXT:    store i32 3, i32* [[PTR]], align 4
-; CHECK-NEXT:    br label [[IF_END8]]
-; CHECK:       if.else6:
-; CHECK-NEXT:    store i32 4, i32* [[PTR]], align 4
-; CHECK-NEXT:    br label [[IF_END8]]
-; CHECK:       if.end8:
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@rec-branch-1
+; IS__TUNIT____-SAME: (i32 [[A:%.*]], i32 [[B:%.*]], i32 [[C:%.*]], i32* nocapture nofree writeonly [[PTR:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[A]], 0
+; IS__TUNIT____-NEXT:    br i1 [[TOBOOL]], label [[IF_ELSE3:%.*]], label [[IF_THEN:%.*]]
+; IS__TUNIT____:       if.then:
+; IS__TUNIT____-NEXT:    [[TOBOOL1:%.*]] = icmp eq i32 [[B]], 0
+; IS__TUNIT____-NEXT:    br i1 [[TOBOOL1]], label [[IF_ELSE:%.*]], label [[IF_THEN2:%.*]]
+; IS__TUNIT____:       if.then2:
+; IS__TUNIT____-NEXT:    store i32 1, i32* [[PTR]], align 4
+; IS__TUNIT____-NEXT:    br label [[IF_END8:%.*]]
+; IS__TUNIT____:       if.else:
+; IS__TUNIT____-NEXT:    store i32 2, i32* [[PTR]], align 4
+; IS__TUNIT____-NEXT:    br label [[IF_END8]]
+; IS__TUNIT____:       if.else3:
+; IS__TUNIT____-NEXT:    [[TOBOOL4:%.*]] = icmp eq i32 [[C]], 0
+; IS__TUNIT____-NEXT:    br i1 [[TOBOOL4]], label [[IF_ELSE6:%.*]], label [[IF_THEN5:%.*]]
+; IS__TUNIT____:       if.then5:
+; IS__TUNIT____-NEXT:    store i32 3, i32* [[PTR]], align 4
+; IS__TUNIT____-NEXT:    br label [[IF_END8]]
+; IS__TUNIT____:       if.else6:
+; IS__TUNIT____-NEXT:    store i32 4, i32* [[PTR]], align 4
+; IS__TUNIT____-NEXT:    br label [[IF_END8]]
+; IS__TUNIT____:       if.end8:
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@rec-branch-1
+; IS__CGSCC____-SAME: (i32 [[A:%.*]], i32 [[B:%.*]], i32 [[C:%.*]], i32* nocapture nofree writeonly [[PTR:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[A]], 0
+; IS__CGSCC____-NEXT:    br i1 [[TOBOOL]], label [[IF_ELSE3:%.*]], label [[IF_THEN:%.*]]
+; IS__CGSCC____:       if.then:
+; IS__CGSCC____-NEXT:    [[TOBOOL1:%.*]] = icmp eq i32 [[B]], 0
+; IS__CGSCC____-NEXT:    br i1 [[TOBOOL1]], label [[IF_ELSE:%.*]], label [[IF_THEN2:%.*]]
+; IS__CGSCC____:       if.then2:
+; IS__CGSCC____-NEXT:    store i32 1, i32* [[PTR]], align 4
+; IS__CGSCC____-NEXT:    br label [[IF_END8:%.*]]
+; IS__CGSCC____:       if.else:
+; IS__CGSCC____-NEXT:    store i32 2, i32* [[PTR]], align 4
+; IS__CGSCC____-NEXT:    br label [[IF_END8]]
+; IS__CGSCC____:       if.else3:
+; IS__CGSCC____-NEXT:    [[TOBOOL4:%.*]] = icmp eq i32 [[C]], 0
+; IS__CGSCC____-NEXT:    br i1 [[TOBOOL4]], label [[IF_ELSE6:%.*]], label [[IF_THEN5:%.*]]
+; IS__CGSCC____:       if.then5:
+; IS__CGSCC____-NEXT:    store i32 3, i32* [[PTR]], align 4
+; IS__CGSCC____-NEXT:    br label [[IF_END8]]
+; IS__CGSCC____:       if.else6:
+; IS__CGSCC____-NEXT:    store i32 4, i32* [[PTR]], align 4
+; IS__CGSCC____-NEXT:    br label [[IF_END8]]
+; IS__CGSCC____:       if.end8:
+; IS__CGSCC____-NEXT:    ret void
 ;
 entry:
   %tobool = icmp eq i32 %a, 0
@@ -620,6 +891,7 @@ if.end8:                                          ; preds = %if.then5, %if.else6
 ;  }
 ; FIXME: %ptr should be dereferenceable(4)
 define dso_local void @rec-branch-2(i32 %a, i32 %b, i32 %c, i32* %ptr) {
+; CHECK: Function Attrs: argmemonly nofree nosync nounwind writeonly
 ; CHECK-LABEL: define {{[^@]+}}@rec-branch-2
 ; CHECK-SAME: (i32 [[A:%.*]], i32 [[B:%.*]], i32 [[C:%.*]], i32* nocapture nofree writeonly [[PTR:%.*]])
 ; CHECK-NEXT:  entry:
diff --git a/llvm/test/Transforms/Attributor/dereferenceable-2.ll b/llvm/test/Transforms/Attributor/dereferenceable-2.ll
index b4abb2bb0c338..af38f5dfd9bcc 100644
--- a/llvm/test/Transforms/Attributor/dereferenceable-2.ll
+++ b/llvm/test/Transforms/Attributor/dereferenceable-2.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -8,21 +8,39 @@
 ; https://bugs.llvm.org/show_bug.cgi?id=21780
 
 define <4 x double> @PR21780(double* %ptr) {
-; CHECK-LABEL: define {{[^@]+}}@PR21780
-; CHECK-SAME: (double* nocapture nofree nonnull readonly align 8 dereferenceable(32) [[PTR:%.*]])
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds double, double* [[PTR]], i64 1
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, double* [[PTR]], i64 2
-; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[PTR]], i64 3
-; CHECK-NEXT:    [[T0:%.*]] = load double, double* [[PTR]], align 8
-; CHECK-NEXT:    [[T1:%.*]] = load double, double* [[ARRAYIDX1]], align 8
-; CHECK-NEXT:    [[T2:%.*]] = load double, double* [[ARRAYIDX2]], align 8
-; CHECK-NEXT:    [[T3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
-; CHECK-NEXT:    [[VECINIT0:%.*]] = insertelement <4 x double> undef, double [[T0]], i32 0
-; CHECK-NEXT:    [[VECINIT1:%.*]] = insertelement <4 x double> [[VECINIT0]], double [[T1]], i32 1
-; CHECK-NEXT:    [[VECINIT2:%.*]] = insertelement <4 x double> [[VECINIT1]], double [[T2]], i32 2
-; CHECK-NEXT:    [[VECINIT3:%.*]] = insertelement <4 x double> [[VECINIT2]], double [[T3]], i32 3
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x double> [[VECINIT3]], <4 x double> [[VECINIT3]], <4 x i32> <i32 0, i32 0, i32 2, i32 2>
-; CHECK-NEXT:    ret <4 x double> [[SHUFFLE]]
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@PR21780
+; IS__TUNIT____-SAME: (double* nocapture nofree nonnull readonly align 8 dereferenceable(32) [[PTR:%.*]])
+; IS__TUNIT____-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds double, double* [[PTR]], i64 1
+; IS__TUNIT____-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, double* [[PTR]], i64 2
+; IS__TUNIT____-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[PTR]], i64 3
+; IS__TUNIT____-NEXT:    [[T0:%.*]] = load double, double* [[PTR]], align 8
+; IS__TUNIT____-NEXT:    [[T1:%.*]] = load double, double* [[ARRAYIDX1]], align 8
+; IS__TUNIT____-NEXT:    [[T2:%.*]] = load double, double* [[ARRAYIDX2]], align 8
+; IS__TUNIT____-NEXT:    [[T3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; IS__TUNIT____-NEXT:    [[VECINIT0:%.*]] = insertelement <4 x double> undef, double [[T0]], i32 0
+; IS__TUNIT____-NEXT:    [[VECINIT1:%.*]] = insertelement <4 x double> [[VECINIT0]], double [[T1]], i32 1
+; IS__TUNIT____-NEXT:    [[VECINIT2:%.*]] = insertelement <4 x double> [[VECINIT1]], double [[T2]], i32 2
+; IS__TUNIT____-NEXT:    [[VECINIT3:%.*]] = insertelement <4 x double> [[VECINIT2]], double [[T3]], i32 3
+; IS__TUNIT____-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x double> [[VECINIT3]], <4 x double> [[VECINIT3]], <4 x i32> <i32 0, i32 0, i32 2, i32 2>
+; IS__TUNIT____-NEXT:    ret <4 x double> [[SHUFFLE]]
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@PR21780
+; IS__CGSCC____-SAME: (double* nocapture nofree nonnull readonly align 8 dereferenceable(32) [[PTR:%.*]])
+; IS__CGSCC____-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds double, double* [[PTR]], i64 1
+; IS__CGSCC____-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, double* [[PTR]], i64 2
+; IS__CGSCC____-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[PTR]], i64 3
+; IS__CGSCC____-NEXT:    [[T0:%.*]] = load double, double* [[PTR]], align 8
+; IS__CGSCC____-NEXT:    [[T1:%.*]] = load double, double* [[ARRAYIDX1]], align 8
+; IS__CGSCC____-NEXT:    [[T2:%.*]] = load double, double* [[ARRAYIDX2]], align 8
+; IS__CGSCC____-NEXT:    [[T3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; IS__CGSCC____-NEXT:    [[VECINIT0:%.*]] = insertelement <4 x double> undef, double [[T0]], i32 0
+; IS__CGSCC____-NEXT:    [[VECINIT1:%.*]] = insertelement <4 x double> [[VECINIT0]], double [[T1]], i32 1
+; IS__CGSCC____-NEXT:    [[VECINIT2:%.*]] = insertelement <4 x double> [[VECINIT1]], double [[T2]], i32 2
+; IS__CGSCC____-NEXT:    [[VECINIT3:%.*]] = insertelement <4 x double> [[VECINIT2]], double [[T3]], i32 3
+; IS__CGSCC____-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x double> [[VECINIT3]], <4 x double> [[VECINIT3]], <4 x i32> <i32 0, i32 0, i32 2, i32 2>
+; IS__CGSCC____-NEXT:    ret <4 x double> [[SHUFFLE]]
 ;
 
   ; GEP of index 0 is simplified away.
@@ -45,11 +63,19 @@ define <4 x double> @PR21780(double* %ptr) {
 
 
 define double @PR21780_only_access3_with_inbounds(double* %ptr) {
-; CHECK-LABEL: define {{[^@]+}}@PR21780_only_access3_with_inbounds
-; CHECK-SAME: (double* nocapture nofree nonnull readonly align 8 dereferenceable(32) [[PTR:%.*]])
-; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[PTR]], i64 3
-; CHECK-NEXT:    [[T3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
-; CHECK-NEXT:    ret double [[T3]]
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@PR21780_only_access3_with_inbounds
+; IS__TUNIT____-SAME: (double* nocapture nofree nonnull readonly align 8 dereferenceable(32) [[PTR:%.*]])
+; IS__TUNIT____-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[PTR]], i64 3
+; IS__TUNIT____-NEXT:    [[T3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; IS__TUNIT____-NEXT:    ret double [[T3]]
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@PR21780_only_access3_with_inbounds
+; IS__CGSCC____-SAME: (double* nocapture nofree nonnull readonly align 8 dereferenceable(32) [[PTR:%.*]])
+; IS__CGSCC____-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[PTR]], i64 3
+; IS__CGSCC____-NEXT:    [[T3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; IS__CGSCC____-NEXT:    ret double [[T3]]
 ;
 
   %arrayidx3 = getelementptr inbounds double, double* %ptr, i64 3
@@ -58,11 +84,19 @@ define double @PR21780_only_access3_with_inbounds(double* %ptr) {
 }
 
 define double @PR21780_only_access3_without_inbounds(double* %ptr) {
-; CHECK-LABEL: define {{[^@]+}}@PR21780_only_access3_without_inbounds
-; CHECK-SAME: (double* nocapture nofree readonly align 8 [[PTR:%.*]])
-; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr double, double* [[PTR]], i64 3
-; CHECK-NEXT:    [[T3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
-; CHECK-NEXT:    ret double [[T3]]
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@PR21780_only_access3_without_inbounds
+; IS__TUNIT____-SAME: (double* nocapture nofree readonly align 8 [[PTR:%.*]])
+; IS__TUNIT____-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr double, double* [[PTR]], i64 3
+; IS__TUNIT____-NEXT:    [[T3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; IS__TUNIT____-NEXT:    ret double [[T3]]
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@PR21780_only_access3_without_inbounds
+; IS__CGSCC____-SAME: (double* nocapture nofree readonly align 8 [[PTR:%.*]])
+; IS__CGSCC____-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr double, double* [[PTR]], i64 3
+; IS__CGSCC____-NEXT:    [[T3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; IS__CGSCC____-NEXT:    ret double [[T3]]
 ;
   %arrayidx3 = getelementptr double, double* %ptr, i64 3
   %t3 = load double, double* %arrayidx3, align 8
@@ -70,11 +104,19 @@ define double @PR21780_only_access3_without_inbounds(double* %ptr) {
 }
 
 define double @PR21780_without_inbounds(double* %ptr) {
-; CHECK-LABEL: define {{[^@]+}}@PR21780_without_inbounds
-; CHECK-SAME: (double* nocapture nofree nonnull readonly align 8 dereferenceable(32) [[PTR:%.*]])
-; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr double, double* [[PTR]], i64 3
-; CHECK-NEXT:    [[T3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
-; CHECK-NEXT:    ret double [[T3]]
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@PR21780_without_inbounds
+; IS__TUNIT____-SAME: (double* nocapture nofree nonnull readonly align 8 dereferenceable(32) [[PTR:%.*]])
+; IS__TUNIT____-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr double, double* [[PTR]], i64 3
+; IS__TUNIT____-NEXT:    [[T3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; IS__TUNIT____-NEXT:    ret double [[T3]]
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@PR21780_without_inbounds
+; IS__CGSCC____-SAME: (double* nocapture nofree nonnull readonly align 8 dereferenceable(32) [[PTR:%.*]])
+; IS__CGSCC____-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr double, double* [[PTR]], i64 3
+; IS__CGSCC____-NEXT:    [[T3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; IS__CGSCC____-NEXT:    ret double [[T3]]
 ;
 
   %arrayidx1 = getelementptr double, double* %ptr, i64 1
@@ -92,12 +134,21 @@ define double @PR21780_without_inbounds(double* %ptr) {
 ; Unsimplified, but still valid. Also, throw in some bogus arguments.
 
 define void @gep0(i8* %unused, i8* %other, i8* %ptr) {
-; CHECK-LABEL: define {{[^@]+}}@gep0
-; CHECK-SAME: (i8* nocapture nofree readnone [[UNUSED:%.*]], i8* nocapture nofree nonnull writeonly dereferenceable(1) [[OTHER:%.*]], i8* nocapture nofree nonnull readonly dereferenceable(3) [[PTR:%.*]])
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr i8, i8* [[PTR]], i64 2
-; CHECK-NEXT:    [[T2:%.*]] = load i8, i8* [[ARRAYIDX2]], align 1
-; CHECK-NEXT:    store i8 [[T2]], i8* [[OTHER]], align 1
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@gep0
+; IS__TUNIT____-SAME: (i8* nocapture nofree readnone [[UNUSED:%.*]], i8* nocapture nofree nonnull writeonly dereferenceable(1) [[OTHER:%.*]], i8* nocapture nofree nonnull readonly dereferenceable(3) [[PTR:%.*]])
+; IS__TUNIT____-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr i8, i8* [[PTR]], i64 2
+; IS__TUNIT____-NEXT:    [[T2:%.*]] = load i8, i8* [[ARRAYIDX2]], align 1
+; IS__TUNIT____-NEXT:    store i8 [[T2]], i8* [[OTHER]], align 1
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@gep0
+; IS__CGSCC____-SAME: (i8* nocapture nofree readnone [[UNUSED:%.*]], i8* nocapture nofree nonnull writeonly dereferenceable(1) [[OTHER:%.*]], i8* nocapture nofree nonnull readonly dereferenceable(3) [[PTR:%.*]])
+; IS__CGSCC____-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr i8, i8* [[PTR]], i64 2
+; IS__CGSCC____-NEXT:    [[T2:%.*]] = load i8, i8* [[ARRAYIDX2]], align 1
+; IS__CGSCC____-NEXT:    store i8 [[T2]], i8* [[OTHER]], align 1
+; IS__CGSCC____-NEXT:    ret void
 ;
   %arrayidx0 = getelementptr i8, i8* %ptr, i64 0
   %arrayidx1 = getelementptr i8, i8* %ptr, i64 1
@@ -113,9 +164,15 @@ define void @gep0(i8* %unused, i8* %other, i8* %ptr) {
 ; Multiple arguments may be dereferenceable.
 
 define void @ordering(i8* %ptr1, i32* %ptr2) {
-; CHECK-LABEL: define {{[^@]+}}@ordering
-; CHECK-SAME: (i8* nocapture nofree nonnull readnone dereferenceable(3) [[PTR1:%.*]], i32* nocapture nofree nonnull readnone align 4 dereferenceable(8) [[PTR2:%.*]])
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@ordering
+; IS__TUNIT____-SAME: (i8* nocapture nofree nonnull readnone dereferenceable(3) [[PTR1:%.*]], i32* nocapture nofree nonnull readnone align 4 dereferenceable(8) [[PTR2:%.*]])
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@ordering
+; IS__CGSCC____-SAME: (i8* nocapture nofree nonnull readnone dereferenceable(3) [[PTR1:%.*]], i32* nocapture nofree nonnull readnone align 4 dereferenceable(8) [[PTR2:%.*]])
+; IS__CGSCC____-NEXT:    ret void
 ;
   %a20 = getelementptr i32, i32* %ptr2, i64 0
   %a12 = getelementptr i8, i8* %ptr1, i64 2
@@ -133,12 +190,21 @@ define void @ordering(i8* %ptr1, i32* %ptr2) {
 ; Not in entry block.
 
 define void @not_entry_but_guaranteed_to_execute(i8* %ptr) {
-; CHECK-LABEL: define {{[^@]+}}@not_entry_but_guaranteed_to_execute
-; CHECK-SAME: (i8* nocapture nofree nonnull readnone dereferenceable(3) [[PTR:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[EXIT:%.*]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@not_entry_but_guaranteed_to_execute
+; IS__TUNIT____-SAME: (i8* nocapture nofree nonnull readnone dereferenceable(3) [[PTR:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    br label [[EXIT:%.*]]
+; IS__TUNIT____:       exit:
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@not_entry_but_guaranteed_to_execute
+; IS__CGSCC____-SAME: (i8* nocapture nofree nonnull readnone dereferenceable(3) [[PTR:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    br label [[EXIT:%.*]]
+; IS__CGSCC____:       exit:
+; IS__CGSCC____-NEXT:    ret void
 ;
 entry:
   br label %exit
@@ -155,14 +221,25 @@ exit:
 ; Not in entry block and not guaranteed to execute.
 
 define void @not_entry_not_guaranteed_to_execute(i8* %ptr, i1 %cond) {
-; CHECK-LABEL: define {{[^@]+}}@not_entry_not_guaranteed_to_execute
-; CHECK-SAME: (i8* nocapture nofree readnone [[PTR:%.*]], i1 [[COND:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND]], label [[LOADS:%.*]], label [[EXIT:%.*]]
-; CHECK:       loads:
-; CHECK-NEXT:    ret void
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@not_entry_not_guaranteed_to_execute
+; IS__TUNIT____-SAME: (i8* nocapture nofree readnone [[PTR:%.*]], i1 [[COND:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    br i1 [[COND]], label [[LOADS:%.*]], label [[EXIT:%.*]]
+; IS__TUNIT____:       loads:
+; IS__TUNIT____-NEXT:    ret void
+; IS__TUNIT____:       exit:
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@not_entry_not_guaranteed_to_execute
+; IS__CGSCC____-SAME: (i8* nocapture nofree readnone [[PTR:%.*]], i1 [[COND:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    br i1 [[COND]], label [[LOADS:%.*]], label [[EXIT:%.*]]
+; IS__CGSCC____:       loads:
+; IS__CGSCC____-NEXT:    ret void
+; IS__CGSCC____:       exit:
+; IS__CGSCC____-NEXT:    ret void
 ;
 entry:
   br i1 %cond, label %loads, label %exit
@@ -181,14 +258,25 @@ exit:
 ; The last load may not execute, so derefenceable bytes only covers the 1st two loads.
 
 define void @partial_in_entry(i16* %ptr, i1 %cond) {
-; CHECK-LABEL: define {{[^@]+}}@partial_in_entry
-; CHECK-SAME: (i16* nocapture nofree nonnull readnone align 2 dereferenceable(4) [[PTR:%.*]], i1 [[COND:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND]], label [[LOADS:%.*]], label [[EXIT:%.*]]
-; CHECK:       loads:
-; CHECK-NEXT:    ret void
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@partial_in_entry
+; IS__TUNIT____-SAME: (i16* nocapture nofree nonnull readnone align 2 dereferenceable(4) [[PTR:%.*]], i1 [[COND:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    br i1 [[COND]], label [[LOADS:%.*]], label [[EXIT:%.*]]
+; IS__TUNIT____:       loads:
+; IS__TUNIT____-NEXT:    ret void
+; IS__TUNIT____:       exit:
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@partial_in_entry
+; IS__CGSCC____-SAME: (i16* nocapture nofree nonnull readnone align 2 dereferenceable(4) [[PTR:%.*]], i1 [[COND:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    br i1 [[COND]], label [[LOADS:%.*]], label [[EXIT:%.*]]
+; IS__CGSCC____:       loads:
+; IS__CGSCC____-NEXT:    ret void
+; IS__CGSCC____:       exit:
+; IS__CGSCC____-NEXT:    ret void
 ;
 entry:
   %arrayidx0 = getelementptr i16, i16* %ptr, i64 0
@@ -208,11 +296,19 @@ exit:
 ; The 2nd and 3rd loads may never execute.
 
 define void @volatile_is_not_dereferenceable(i16* %ptr) {
-; CHECK-LABEL: define {{[^@]+}}@volatile_is_not_dereferenceable
-; CHECK-SAME: (i16* nofree align 2 [[PTR:%.*]])
-; CHECK-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i16, i16* [[PTR]], i64 0
-; CHECK-NEXT:    [[T0:%.*]] = load volatile i16, i16* [[ARRAYIDX0]], align 2
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: argmemonly nofree nounwind willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@volatile_is_not_dereferenceable
+; IS__TUNIT____-SAME: (i16* nofree align 2 [[PTR:%.*]])
+; IS__TUNIT____-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i16, i16* [[PTR]], i64 0
+; IS__TUNIT____-NEXT:    [[T0:%.*]] = load volatile i16, i16* [[ARRAYIDX0]], align 2
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nounwind willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@volatile_is_not_dereferenceable
+; IS__CGSCC____-SAME: (i16* nofree align 2 [[PTR:%.*]])
+; IS__CGSCC____-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr i16, i16* [[PTR]], i64 0
+; IS__CGSCC____-NEXT:    [[T0:%.*]] = load volatile i16, i16* [[ARRAYIDX0]], align 2
+; IS__CGSCC____-NEXT:    ret void
 ;
   %arrayidx0 = getelementptr i16, i16* %ptr, i64 0
   %arrayidx1 = getelementptr i16, i16* %ptr, i64 1
@@ -226,9 +322,15 @@ define void @volatile_is_not_dereferenceable(i16* %ptr) {
 ; TODO: We should allow inference for atomic (but not volatile) ops.
 
 define void @atomic_is_alright(i16* %ptr) {
-; CHECK-LABEL: define {{[^@]+}}@atomic_is_alright
-; CHECK-SAME: (i16* nocapture nofree nonnull readnone align 2 dereferenceable(6) [[PTR:%.*]])
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@atomic_is_alright
+; IS__TUNIT____-SAME: (i16* nocapture nofree nonnull readnone align 2 dereferenceable(6) [[PTR:%.*]])
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@atomic_is_alright
+; IS__CGSCC____-SAME: (i16* nocapture nofree nonnull readnone align 2 dereferenceable(6) [[PTR:%.*]])
+; IS__CGSCC____-NEXT:    ret void
 ;
   %arrayidx0 = getelementptr i16, i16* %ptr, i64 0
   %arrayidx1 = getelementptr i16, i16* %ptr, i64 1
@@ -260,9 +362,15 @@ define void @not_guaranteed_to_transfer_execution(i16* %ptr) {
 ; We must have consecutive accesses.
 
 define void @variable_gep_index(i8* %unused, i8* %ptr, i64 %variable_index) {
-; CHECK-LABEL: define {{[^@]+}}@variable_gep_index
-; CHECK-SAME: (i8* nocapture nofree readnone [[UNUSED:%.*]], i8* nocapture nofree nonnull readnone dereferenceable(1) [[PTR:%.*]], i64 [[VARIABLE_INDEX:%.*]])
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@variable_gep_index
+; IS__TUNIT____-SAME: (i8* nocapture nofree readnone [[UNUSED:%.*]], i8* nocapture nofree nonnull readnone dereferenceable(1) [[PTR:%.*]], i64 [[VARIABLE_INDEX:%.*]])
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@variable_gep_index
+; IS__CGSCC____-SAME: (i8* nocapture nofree readnone [[UNUSED:%.*]], i8* nocapture nofree nonnull readnone dereferenceable(1) [[PTR:%.*]], i64 [[VARIABLE_INDEX:%.*]])
+; IS__CGSCC____-NEXT:    ret void
 ;
   %arrayidx1 = getelementptr i8, i8* %ptr, i64 %variable_index
   %arrayidx2 = getelementptr i8, i8* %ptr, i64 2
@@ -276,9 +384,15 @@ define void @variable_gep_index(i8* %unused, i8* %ptr, i64 %variable_index) {
 
 define void @multi_index_gep(<4 x i8>* %ptr) {
 ; FIXME: %ptr should be dereferenceable(4)
-; CHECK-LABEL: define {{[^@]+}}@multi_index_gep
-; CHECK-SAME: (<4 x i8>* nocapture nofree nonnull readnone dereferenceable(1) [[PTR:%.*]])
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@multi_index_gep
+; IS__TUNIT____-SAME: (<4 x i8>* nocapture nofree nonnull readnone dereferenceable(1) [[PTR:%.*]])
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@multi_index_gep
+; IS__CGSCC____-SAME: (<4 x i8>* nocapture nofree nonnull readnone dereferenceable(1) [[PTR:%.*]])
+; IS__CGSCC____-NEXT:    ret void
 ;
   %arrayidx00 = getelementptr <4 x i8>, <4 x i8>* %ptr, i64 0, i64 0
   %t0 = load i8, i8* %arrayidx00
@@ -288,9 +402,15 @@ define void @multi_index_gep(<4 x i8>* %ptr) {
 ; Could round weird bitwidths down?
 
 define void @not_byte_multiple(i9* %ptr) {
-; CHECK-LABEL: define {{[^@]+}}@not_byte_multiple
-; CHECK-SAME: (i9* nocapture nofree nonnull readnone align 2 dereferenceable(2) [[PTR:%.*]])
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@not_byte_multiple
+; IS__TUNIT____-SAME: (i9* nocapture nofree nonnull readnone align 2 dereferenceable(2) [[PTR:%.*]])
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@not_byte_multiple
+; IS__CGSCC____-SAME: (i9* nocapture nofree nonnull readnone align 2 dereferenceable(2) [[PTR:%.*]])
+; IS__CGSCC____-NEXT:    ret void
 ;
   %arrayidx0 = getelementptr i9, i9* %ptr, i64 0
   %t0 = load i9, i9* %arrayidx0
@@ -300,9 +420,15 @@ define void @not_byte_multiple(i9* %ptr) {
 ; Missing direct access from the pointer.
 
 define void @no_pointer_deref(i16* %ptr) {
-; CHECK-LABEL: define {{[^@]+}}@no_pointer_deref
-; CHECK-SAME: (i16* nocapture nofree readnone align 2 [[PTR:%.*]])
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@no_pointer_deref
+; IS__TUNIT____-SAME: (i16* nocapture nofree readnone align 2 [[PTR:%.*]])
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@no_pointer_deref
+; IS__CGSCC____-SAME: (i16* nocapture nofree readnone align 2 [[PTR:%.*]])
+; IS__CGSCC____-NEXT:    ret void
 ;
   %arrayidx1 = getelementptr i16, i16* %ptr, i64 1
   %arrayidx2 = getelementptr i16, i16* %ptr, i64 2
@@ -314,9 +440,15 @@ define void @no_pointer_deref(i16* %ptr) {
 ; Out-of-order is ok, but missing access concludes dereferenceable range.
 
 define void @non_consecutive(i32* %ptr) {
-; CHECK-LABEL: define {{[^@]+}}@non_consecutive
-; CHECK-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(8) [[PTR:%.*]])
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@non_consecutive
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(8) [[PTR:%.*]])
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@non_consecutive
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(8) [[PTR:%.*]])
+; IS__CGSCC____-NEXT:    ret void
 ;
   %arrayidx1 = getelementptr i32, i32* %ptr, i64 1
   %arrayidx0 = getelementptr i32, i32* %ptr, i64 0
@@ -330,9 +462,15 @@ define void @non_consecutive(i32* %ptr) {
 ; Improve on existing dereferenceable attribute.
 
 define void @more_bytes(i32* dereferenceable(8) %ptr) {
-; CHECK-LABEL: define {{[^@]+}}@more_bytes
-; CHECK-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(16) [[PTR:%.*]])
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@more_bytes
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(16) [[PTR:%.*]])
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@more_bytes
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(16) [[PTR:%.*]])
+; IS__CGSCC____-NEXT:    ret void
 ;
   %arrayidx3 = getelementptr i32, i32* %ptr, i64 3
   %arrayidx1 = getelementptr i32, i32* %ptr, i64 1
@@ -348,9 +486,15 @@ define void @more_bytes(i32* dereferenceable(8) %ptr) {
 ; Improve on existing dereferenceable_or_null attribute.
 
 define void @more_bytes_and_not_null(i32* dereferenceable_or_null(8) %ptr) {
-; CHECK-LABEL: define {{[^@]+}}@more_bytes_and_not_null
-; CHECK-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(16) [[PTR:%.*]])
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@more_bytes_and_not_null
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(16) [[PTR:%.*]])
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@more_bytes_and_not_null
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(16) [[PTR:%.*]])
+; IS__CGSCC____-NEXT:    ret void
 ;
   %arrayidx3 = getelementptr i32, i32* %ptr, i64 3
   %arrayidx1 = getelementptr i32, i32* %ptr, i64 1
@@ -366,9 +510,15 @@ define void @more_bytes_and_not_null(i32* dereferenceable_or_null(8) %ptr) {
 ; But don't pessimize existing dereferenceable attribute.
 
 define void @better_bytes(i32* dereferenceable(100) %ptr) {
-; CHECK-LABEL: define {{[^@]+}}@better_bytes
-; CHECK-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(100) [[PTR:%.*]])
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@better_bytes
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(100) [[PTR:%.*]])
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@better_bytes
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(100) [[PTR:%.*]])
+; IS__CGSCC____-NEXT:    ret void
 ;
   %arrayidx3 = getelementptr i32, i32* %ptr, i64 3
   %arrayidx1 = getelementptr i32, i32* %ptr, i64 1
@@ -382,9 +532,15 @@ define void @better_bytes(i32* dereferenceable(100) %ptr) {
 }
 
 define void @bitcast(i32* %arg) {
-; CHECK-LABEL: define {{[^@]+}}@bitcast
-; CHECK-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(8) [[ARG:%.*]])
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@bitcast
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(8) [[ARG:%.*]])
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@bitcast
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(8) [[ARG:%.*]])
+; IS__CGSCC____-NEXT:    ret void
 ;
   %ptr = bitcast i32* %arg to float*
   %arrayidx0 = getelementptr float, float* %ptr, i64 0
@@ -395,9 +551,15 @@ define void @bitcast(i32* %arg) {
 }
 
 define void @bitcast_different_sizes(double* %arg1, i8* %arg2) {
-; CHECK-LABEL: define {{[^@]+}}@bitcast_different_sizes
-; CHECK-SAME: (double* nocapture nofree nonnull readnone align 4 dereferenceable(12) [[ARG1:%.*]], i8* nocapture nofree nonnull readnone align 4 dereferenceable(16) [[ARG2:%.*]])
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@bitcast_different_sizes
+; IS__TUNIT____-SAME: (double* nocapture nofree nonnull readnone align 4 dereferenceable(12) [[ARG1:%.*]], i8* nocapture nofree nonnull readnone align 4 dereferenceable(16) [[ARG2:%.*]])
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@bitcast_different_sizes
+; IS__CGSCC____-SAME: (double* nocapture nofree nonnull readnone align 4 dereferenceable(12) [[ARG1:%.*]], i8* nocapture nofree nonnull readnone align 4 dereferenceable(16) [[ARG2:%.*]])
+; IS__CGSCC____-NEXT:    ret void
 ;
   %ptr1 = bitcast double* %arg1 to float*
   %a10 = getelementptr float, float* %ptr1, i64 0
@@ -416,9 +578,15 @@ define void @bitcast_different_sizes(double* %arg1, i8* %arg2) {
 }
 
 define void @negative_offset(i32* %arg) {
-; CHECK-LABEL: define {{[^@]+}}@negative_offset
-; CHECK-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(4) [[ARG:%.*]])
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@negative_offset
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(4) [[ARG:%.*]])
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@negative_offset
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(4) [[ARG:%.*]])
+; IS__CGSCC____-NEXT:    ret void
 ;
   %ptr = bitcast i32* %arg to float*
   %arrayidx0 = getelementptr float, float* %ptr, i64 0
@@ -429,14 +597,25 @@ define void @negative_offset(i32* %arg) {
 }
 
 define void @stores(i32* %arg) {
-; CHECK-LABEL: define {{[^@]+}}@stores
-; CHECK-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(8) [[ARG:%.*]])
-; CHECK-NEXT:    [[PTR:%.*]] = bitcast i32* [[ARG]] to float*
-; CHECK-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr float, float* [[PTR]], i64 0
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr float, float* [[PTR]], i64 1
-; CHECK-NEXT:    store float 1.000000e+00, float* [[ARRAYIDX0]], align 4
-; CHECK-NEXT:    store float 2.000000e+00, float* [[ARRAYIDX1]], align 4
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@stores
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(8) [[ARG:%.*]])
+; IS__TUNIT____-NEXT:    [[PTR:%.*]] = bitcast i32* [[ARG]] to float*
+; IS__TUNIT____-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr float, float* [[PTR]], i64 0
+; IS__TUNIT____-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr float, float* [[PTR]], i64 1
+; IS__TUNIT____-NEXT:    store float 1.000000e+00, float* [[ARRAYIDX0]], align 4
+; IS__TUNIT____-NEXT:    store float 2.000000e+00, float* [[ARRAYIDX1]], align 4
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@stores
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(8) [[ARG:%.*]])
+; IS__CGSCC____-NEXT:    [[PTR:%.*]] = bitcast i32* [[ARG]] to float*
+; IS__CGSCC____-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr float, float* [[PTR]], i64 0
+; IS__CGSCC____-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr float, float* [[PTR]], i64 1
+; IS__CGSCC____-NEXT:    store float 1.000000e+00, float* [[ARRAYIDX0]], align 4
+; IS__CGSCC____-NEXT:    store float 2.000000e+00, float* [[ARRAYIDX1]], align 4
+; IS__CGSCC____-NEXT:    ret void
 ;
   %ptr = bitcast i32* %arg to float*
   %arrayidx0 = getelementptr float, float* %ptr, i64 0
@@ -447,12 +626,21 @@ define void @stores(i32* %arg) {
 }
 
 define void @load_store(i32* %arg) {
-; CHECK-LABEL: define {{[^@]+}}@load_store
-; CHECK-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(8) [[ARG:%.*]])
-; CHECK-NEXT:    [[PTR:%.*]] = bitcast i32* [[ARG]] to float*
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr float, float* [[PTR]], i64 1
-; CHECK-NEXT:    store float 2.000000e+00, float* [[ARRAYIDX1]], align 4
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@load_store
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(8) [[ARG:%.*]])
+; IS__TUNIT____-NEXT:    [[PTR:%.*]] = bitcast i32* [[ARG]] to float*
+; IS__TUNIT____-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr float, float* [[PTR]], i64 1
+; IS__TUNIT____-NEXT:    store float 2.000000e+00, float* [[ARRAYIDX1]], align 4
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@load_store
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(8) [[ARG:%.*]])
+; IS__CGSCC____-NEXT:    [[PTR:%.*]] = bitcast i32* [[ARG]] to float*
+; IS__CGSCC____-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr float, float* [[PTR]], i64 1
+; IS__CGSCC____-NEXT:    store float 2.000000e+00, float* [[ARRAYIDX1]], align 4
+; IS__CGSCC____-NEXT:    ret void
 ;
   %ptr = bitcast i32* %arg to float*
   %arrayidx0 = getelementptr float, float* %ptr, i64 0
@@ -463,12 +651,21 @@ define void @load_store(i32* %arg) {
 }
 
 define void @different_size1(i32* %arg) {
-; CHECK-LABEL: define {{[^@]+}}@different_size1
-; CHECK-SAME: (i32* nocapture nofree nonnull writeonly align 8 dereferenceable(8) [[ARG:%.*]])
-; CHECK-NEXT:    [[ARG_CAST:%.*]] = bitcast i32* [[ARG]] to double*
-; CHECK-NEXT:    store double 0.000000e+00, double* [[ARG_CAST]], align 8
-; CHECK-NEXT:    store i32 0, i32* [[ARG]], align 8
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@different_size1
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull writeonly align 8 dereferenceable(8) [[ARG:%.*]])
+; IS__TUNIT____-NEXT:    [[ARG_CAST:%.*]] = bitcast i32* [[ARG]] to double*
+; IS__TUNIT____-NEXT:    store double 0.000000e+00, double* [[ARG_CAST]], align 8
+; IS__TUNIT____-NEXT:    store i32 0, i32* [[ARG]], align 8
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@different_size1
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull writeonly align 8 dereferenceable(8) [[ARG:%.*]])
+; IS__CGSCC____-NEXT:    [[ARG_CAST:%.*]] = bitcast i32* [[ARG]] to double*
+; IS__CGSCC____-NEXT:    store double 0.000000e+00, double* [[ARG_CAST]], align 8
+; IS__CGSCC____-NEXT:    store i32 0, i32* [[ARG]], align 8
+; IS__CGSCC____-NEXT:    ret void
 ;
   %arg-cast = bitcast i32* %arg to double*
   store double 0.000000e+00, double* %arg-cast
@@ -477,12 +674,21 @@ define void @different_size1(i32* %arg) {
 }
 
 define void @different_size2(i32* %arg) {
-; CHECK-LABEL: define {{[^@]+}}@different_size2
-; CHECK-SAME: (i32* nocapture nofree nonnull writeonly align 8 dereferenceable(8) [[ARG:%.*]])
-; CHECK-NEXT:    store i32 0, i32* [[ARG]], align 8
-; CHECK-NEXT:    [[ARG_CAST:%.*]] = bitcast i32* [[ARG]] to double*
-; CHECK-NEXT:    store double 0.000000e+00, double* [[ARG_CAST]], align 8
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@different_size2
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull writeonly align 8 dereferenceable(8) [[ARG:%.*]])
+; IS__TUNIT____-NEXT:    store i32 0, i32* [[ARG]], align 8
+; IS__TUNIT____-NEXT:    [[ARG_CAST:%.*]] = bitcast i32* [[ARG]] to double*
+; IS__TUNIT____-NEXT:    store double 0.000000e+00, double* [[ARG_CAST]], align 8
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@different_size2
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull writeonly align 8 dereferenceable(8) [[ARG:%.*]])
+; IS__CGSCC____-NEXT:    store i32 0, i32* [[ARG]], align 8
+; IS__CGSCC____-NEXT:    [[ARG_CAST:%.*]] = bitcast i32* [[ARG]] to double*
+; IS__CGSCC____-NEXT:    store double 0.000000e+00, double* [[ARG_CAST]], align 8
+; IS__CGSCC____-NEXT:    ret void
 ;
   store i32 0, i32* %arg
   %arg-cast = bitcast i32* %arg to double*
@@ -509,56 +715,111 @@ define void @different_size2(i32* %arg) {
 ;
 ; ATTRIBUTOR_CGSCC_NPM-LABEL: define i32 @require_cfg_analysis(i32 %c, i32* {{.*}} dereferenceable(4) %p)
 define i32 @require_cfg_analysis(i32 %c, i32* %p) {
-; IS________OPM-LABEL: define {{[^@]+}}@require_cfg_analysis
-; IS________OPM-SAME: (i32 [[C:%.*]], i32* nocapture nofree writeonly [[P:%.*]])
-; IS________OPM-NEXT:    [[TOBOOL1:%.*]] = icmp eq i32 [[C]], 0
-; IS________OPM-NEXT:    br i1 [[TOBOOL1]], label [[L1:%.*]], label [[L2:%.*]]
-; IS________OPM:       l1:
-; IS________OPM-NEXT:    [[TOBOOL2:%.*]] = icmp eq i32 [[C]], 1
-; IS________OPM-NEXT:    br i1 [[TOBOOL2]], label [[L3:%.*]], label [[L4:%.*]]
-; IS________OPM:       l2:
-; IS________OPM-NEXT:    [[TOBOOL3:%.*]] = icmp eq i32 [[C]], 2
-; IS________OPM-NEXT:    br i1 [[TOBOOL3]], label [[L3]], label [[L4]]
-; IS________OPM:       l3:
-; IS________OPM-NEXT:    br label [[L5:%.*]]
-; IS________OPM:       l4:
-; IS________OPM-NEXT:    br label [[L5]]
-; IS________OPM:       l5:
-; IS________OPM-NEXT:    [[TOBOOL4:%.*]] = icmp eq i32 [[C]], 4
-; IS________OPM-NEXT:    br i1 [[TOBOOL4]], label [[L6:%.*]], label [[L7:%.*]]
-; IS________OPM:       l6:
-; IS________OPM-NEXT:    store i32 0, i32* [[P]], align 4
-; IS________OPM-NEXT:    br label [[END:%.*]]
-; IS________OPM:       l7:
-; IS________OPM-NEXT:    store i32 1, i32* [[P]], align 4
-; IS________OPM-NEXT:    br label [[END]]
-; IS________OPM:       end:
-; IS________OPM-NEXT:    ret i32 1
-;
-; IS________NPM-LABEL: define {{[^@]+}}@require_cfg_analysis
-; IS________NPM-SAME: (i32 [[C:%.*]], i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[P:%.*]])
-; IS________NPM-NEXT:    [[TOBOOL1:%.*]] = icmp eq i32 [[C]], 0
-; IS________NPM-NEXT:    br i1 [[TOBOOL1]], label [[L1:%.*]], label [[L2:%.*]]
-; IS________NPM:       l1:
-; IS________NPM-NEXT:    br label [[L4:%.*]]
-; IS________NPM:       l2:
-; IS________NPM-NEXT:    [[TOBOOL3:%.*]] = icmp eq i32 [[C]], 2
-; IS________NPM-NEXT:    br i1 [[TOBOOL3]], label [[L3:%.*]], label [[L4]]
-; IS________NPM:       l3:
-; IS________NPM-NEXT:    br label [[L5:%.*]]
-; IS________NPM:       l4:
-; IS________NPM-NEXT:    br label [[L5]]
-; IS________NPM:       l5:
-; IS________NPM-NEXT:    [[TOBOOL4:%.*]] = icmp eq i32 [[C]], 4
-; IS________NPM-NEXT:    br i1 [[TOBOOL4]], label [[L6:%.*]], label [[L7:%.*]]
-; IS________NPM:       l6:
-; IS________NPM-NEXT:    store i32 0, i32* [[P]], align 4
-; IS________NPM-NEXT:    br label [[END:%.*]]
-; IS________NPM:       l7:
-; IS________NPM-NEXT:    store i32 1, i32* [[P]], align 4
-; IS________NPM-NEXT:    br label [[END]]
-; IS________NPM:       end:
-; IS________NPM-NEXT:    ret i32 1
+; IS__TUNIT_OPM: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@require_cfg_analysis
+; IS__TUNIT_OPM-SAME: (i32 [[C:%.*]], i32* nocapture nofree writeonly [[P:%.*]])
+; IS__TUNIT_OPM-NEXT:    [[TOBOOL1:%.*]] = icmp eq i32 [[C]], 0
+; IS__TUNIT_OPM-NEXT:    br i1 [[TOBOOL1]], label [[L1:%.*]], label [[L2:%.*]]
+; IS__TUNIT_OPM:       l1:
+; IS__TUNIT_OPM-NEXT:    [[TOBOOL2:%.*]] = icmp eq i32 [[C]], 1
+; IS__TUNIT_OPM-NEXT:    br i1 [[TOBOOL2]], label [[L3:%.*]], label [[L4:%.*]]
+; IS__TUNIT_OPM:       l2:
+; IS__TUNIT_OPM-NEXT:    [[TOBOOL3:%.*]] = icmp eq i32 [[C]], 2
+; IS__TUNIT_OPM-NEXT:    br i1 [[TOBOOL3]], label [[L3]], label [[L4]]
+; IS__TUNIT_OPM:       l3:
+; IS__TUNIT_OPM-NEXT:    br label [[L5:%.*]]
+; IS__TUNIT_OPM:       l4:
+; IS__TUNIT_OPM-NEXT:    br label [[L5]]
+; IS__TUNIT_OPM:       l5:
+; IS__TUNIT_OPM-NEXT:    [[TOBOOL4:%.*]] = icmp eq i32 [[C]], 4
+; IS__TUNIT_OPM-NEXT:    br i1 [[TOBOOL4]], label [[L6:%.*]], label [[L7:%.*]]
+; IS__TUNIT_OPM:       l6:
+; IS__TUNIT_OPM-NEXT:    store i32 0, i32* [[P]], align 4
+; IS__TUNIT_OPM-NEXT:    br label [[END:%.*]]
+; IS__TUNIT_OPM:       l7:
+; IS__TUNIT_OPM-NEXT:    store i32 1, i32* [[P]], align 4
+; IS__TUNIT_OPM-NEXT:    br label [[END]]
+; IS__TUNIT_OPM:       end:
+; IS__TUNIT_OPM-NEXT:    ret i32 1
+;
+; IS__TUNIT_NPM: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@require_cfg_analysis
+; IS__TUNIT_NPM-SAME: (i32 [[C:%.*]], i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[P:%.*]])
+; IS__TUNIT_NPM-NEXT:    [[TOBOOL1:%.*]] = icmp eq i32 [[C]], 0
+; IS__TUNIT_NPM-NEXT:    br i1 [[TOBOOL1]], label [[L1:%.*]], label [[L2:%.*]]
+; IS__TUNIT_NPM:       l1:
+; IS__TUNIT_NPM-NEXT:    br label [[L4:%.*]]
+; IS__TUNIT_NPM:       l2:
+; IS__TUNIT_NPM-NEXT:    [[TOBOOL3:%.*]] = icmp eq i32 [[C]], 2
+; IS__TUNIT_NPM-NEXT:    br i1 [[TOBOOL3]], label [[L3:%.*]], label [[L4]]
+; IS__TUNIT_NPM:       l3:
+; IS__TUNIT_NPM-NEXT:    br label [[L5:%.*]]
+; IS__TUNIT_NPM:       l4:
+; IS__TUNIT_NPM-NEXT:    br label [[L5]]
+; IS__TUNIT_NPM:       l5:
+; IS__TUNIT_NPM-NEXT:    [[TOBOOL4:%.*]] = icmp eq i32 [[C]], 4
+; IS__TUNIT_NPM-NEXT:    br i1 [[TOBOOL4]], label [[L6:%.*]], label [[L7:%.*]]
+; IS__TUNIT_NPM:       l6:
+; IS__TUNIT_NPM-NEXT:    store i32 0, i32* [[P]], align 4
+; IS__TUNIT_NPM-NEXT:    br label [[END:%.*]]
+; IS__TUNIT_NPM:       l7:
+; IS__TUNIT_NPM-NEXT:    store i32 1, i32* [[P]], align 4
+; IS__TUNIT_NPM-NEXT:    br label [[END]]
+; IS__TUNIT_NPM:       end:
+; IS__TUNIT_NPM-NEXT:    ret i32 1
+;
+; IS__CGSCC_OPM: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@require_cfg_analysis
+; IS__CGSCC_OPM-SAME: (i32 [[C:%.*]], i32* nocapture nofree writeonly [[P:%.*]])
+; IS__CGSCC_OPM-NEXT:    [[TOBOOL1:%.*]] = icmp eq i32 [[C]], 0
+; IS__CGSCC_OPM-NEXT:    br i1 [[TOBOOL1]], label [[L1:%.*]], label [[L2:%.*]]
+; IS__CGSCC_OPM:       l1:
+; IS__CGSCC_OPM-NEXT:    [[TOBOOL2:%.*]] = icmp eq i32 [[C]], 1
+; IS__CGSCC_OPM-NEXT:    br i1 [[TOBOOL2]], label [[L3:%.*]], label [[L4:%.*]]
+; IS__CGSCC_OPM:       l2:
+; IS__CGSCC_OPM-NEXT:    [[TOBOOL3:%.*]] = icmp eq i32 [[C]], 2
+; IS__CGSCC_OPM-NEXT:    br i1 [[TOBOOL3]], label [[L3]], label [[L4]]
+; IS__CGSCC_OPM:       l3:
+; IS__CGSCC_OPM-NEXT:    br label [[L5:%.*]]
+; IS__CGSCC_OPM:       l4:
+; IS__CGSCC_OPM-NEXT:    br label [[L5]]
+; IS__CGSCC_OPM:       l5:
+; IS__CGSCC_OPM-NEXT:    [[TOBOOL4:%.*]] = icmp eq i32 [[C]], 4
+; IS__CGSCC_OPM-NEXT:    br i1 [[TOBOOL4]], label [[L6:%.*]], label [[L7:%.*]]
+; IS__CGSCC_OPM:       l6:
+; IS__CGSCC_OPM-NEXT:    store i32 0, i32* [[P]], align 4
+; IS__CGSCC_OPM-NEXT:    br label [[END:%.*]]
+; IS__CGSCC_OPM:       l7:
+; IS__CGSCC_OPM-NEXT:    store i32 1, i32* [[P]], align 4
+; IS__CGSCC_OPM-NEXT:    br label [[END]]
+; IS__CGSCC_OPM:       end:
+; IS__CGSCC_OPM-NEXT:    ret i32 1
+;
+; IS__CGSCC_NPM: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@require_cfg_analysis
+; IS__CGSCC_NPM-SAME: (i32 [[C:%.*]], i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[P:%.*]])
+; IS__CGSCC_NPM-NEXT:    [[TOBOOL1:%.*]] = icmp eq i32 [[C]], 0
+; IS__CGSCC_NPM-NEXT:    br i1 [[TOBOOL1]], label [[L1:%.*]], label [[L2:%.*]]
+; IS__CGSCC_NPM:       l1:
+; IS__CGSCC_NPM-NEXT:    br label [[L4:%.*]]
+; IS__CGSCC_NPM:       l2:
+; IS__CGSCC_NPM-NEXT:    [[TOBOOL3:%.*]] = icmp eq i32 [[C]], 2
+; IS__CGSCC_NPM-NEXT:    br i1 [[TOBOOL3]], label [[L3:%.*]], label [[L4]]
+; IS__CGSCC_NPM:       l3:
+; IS__CGSCC_NPM-NEXT:    br label [[L5:%.*]]
+; IS__CGSCC_NPM:       l4:
+; IS__CGSCC_NPM-NEXT:    br label [[L5]]
+; IS__CGSCC_NPM:       l5:
+; IS__CGSCC_NPM-NEXT:    [[TOBOOL4:%.*]] = icmp eq i32 [[C]], 4
+; IS__CGSCC_NPM-NEXT:    br i1 [[TOBOOL4]], label [[L6:%.*]], label [[L7:%.*]]
+; IS__CGSCC_NPM:       l6:
+; IS__CGSCC_NPM-NEXT:    store i32 0, i32* [[P]], align 4
+; IS__CGSCC_NPM-NEXT:    br label [[END:%.*]]
+; IS__CGSCC_NPM:       l7:
+; IS__CGSCC_NPM-NEXT:    store i32 1, i32* [[P]], align 4
+; IS__CGSCC_NPM-NEXT:    br label [[END]]
+; IS__CGSCC_NPM:       end:
+; IS__CGSCC_NPM-NEXT:    ret i32 1
 ;
   %tobool1 = icmp eq i32 %c, 0
   br i1 %tobool1, label %l1, label %l2
diff --git a/llvm/test/Transforms/Attributor/heap_to_stack.ll b/llvm/test/Transforms/Attributor/heap_to_stack.ll
index 909cda54fc868..f46da51e2dfdb 100644
--- a/llvm/test/Transforms/Attributor/heap_to_stack.ll
+++ b/llvm/test/Transforms/Attributor/heap_to_stack.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -287,11 +287,13 @@ define void @test6(i32) {
 ; TEST 7 - free is dead.
 
 define void @test7() {
+; IS________OPM: Function Attrs: noreturn
 ; IS________OPM-LABEL: define {{[^@]+}}@test7()
 ; IS________OPM-NEXT:    [[TMP1:%.*]] = tail call noalias i8* @malloc(i64 4)
 ; IS________OPM-NEXT:    [[TMP2:%.*]] = tail call i32 @no_return_call()
 ; IS________OPM-NEXT:    unreachable
 ;
+; IS________NPM: Function Attrs: noreturn
 ; IS________NPM-LABEL: define {{[^@]+}}@test7()
 ; IS________NPM-NEXT:    [[TMP1:%.*]] = alloca i8, i64 4, align 1
 ; IS________NPM-NEXT:    [[TMP2:%.*]] = tail call i32 @no_return_call()
diff --git a/llvm/test/Transforms/Attributor/internal-noalias.ll b/llvm/test/Transforms/Attributor/internal-noalias.ll
index 182cadf48a0b0..67aa1ec985e95 100644
--- a/llvm/test/Transforms/Attributor/internal-noalias.ll
+++ b/llvm/test/Transforms/Attributor/internal-noalias.ll
@@ -1,10 +1,11 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 define dso_local i32 @visible(i32* noalias %A, i32* noalias %B) #0 {
+; IS__TUNIT____: Function Attrs: argmemonly nofree noinline nosync nounwind readonly uwtable willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@visible
 ; IS__TUNIT____-SAME: (i32* noalias nocapture nofree readonly [[A:%.*]], i32* noalias nocapture nofree readonly [[B:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
@@ -13,6 +14,7 @@ define dso_local i32 @visible(i32* noalias %A, i32* noalias %B) #0 {
 ; IS__TUNIT____-NEXT:    [[ADD:%.*]] = add nsw i32 [[CALL1]], [[CALL2]]
 ; IS__TUNIT____-NEXT:    ret i32 [[ADD]]
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree noinline norecurse nosync nounwind readonly uwtable willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@visible
 ; IS__CGSCC____-SAME: (i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A:%.*]], i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -29,6 +31,7 @@ entry:
 }
 
 define private i32 @noalias_args(i32* %A, i32* %B) #0 {
+; IS__TUNIT____: Function Attrs: argmemonly nofree noinline nosync nounwind readonly uwtable willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@noalias_args
 ; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A:%.*]], i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
@@ -39,6 +42,7 @@ define private i32 @noalias_args(i32* %A, i32* %B) #0 {
 ; IS__TUNIT____-NEXT:    [[ADD2:%.*]] = add nsw i32 [[ADD]], [[CALL]]
 ; IS__TUNIT____-NEXT:    ret i32 [[ADD2]]
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree noinline norecurse nosync nounwind readonly uwtable willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@noalias_args
 ; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A:%.*]], i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -60,6 +64,7 @@ entry:
 
 
 define internal i32 @noalias_args_argmem(i32* %A, i32* %B) #1 {
+; IS__TUNIT____: Function Attrs: argmemonly nofree noinline nosync nounwind readonly uwtable willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@noalias_args_argmem
 ; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A:%.*]], i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
@@ -68,6 +73,7 @@ define internal i32 @noalias_args_argmem(i32* %A, i32* %B) #1 {
 ; IS__TUNIT____-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[TMP1]]
 ; IS__TUNIT____-NEXT:    ret i32 [[ADD]]
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree noinline norecurse nosync nounwind readonly uwtable willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@noalias_args_argmem
 ; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A:%.*]], i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -84,6 +90,7 @@ entry:
 }
 
 define dso_local i32 @visible_local(i32* %A) #0 {
+; IS__TUNIT____: Function Attrs: argmemonly nofree noinline nosync nounwind uwtable willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@visible_local
 ; IS__TUNIT____-SAME: (i32* nocapture nofree readonly [[A:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
@@ -94,6 +101,7 @@ define dso_local i32 @visible_local(i32* %A) #0 {
 ; IS__TUNIT____-NEXT:    [[ADD:%.*]] = add nsw i32 [[CALL1]], [[CALL2]]
 ; IS__TUNIT____-NEXT:    ret i32 [[ADD]]
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree noinline norecurse nosync nounwind uwtable willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@visible_local
 ; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -114,6 +122,7 @@ entry:
 }
 
 define internal i32 @noalias_args_argmem_ro(i32* %A, i32* %B) #1 {
+; IS__TUNIT_OPM: Function Attrs: argmemonly nofree noinline nosync nounwind readonly uwtable willreturn
 ; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@noalias_args_argmem_ro
 ; IS__TUNIT_OPM-SAME: (i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A:%.*]], i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B:%.*]])
 ; IS__TUNIT_OPM-NEXT:    [[T0:%.*]] = load i32, i32* [[A]], align 4
@@ -121,6 +130,7 @@ define internal i32 @noalias_args_argmem_ro(i32* %A, i32* %B) #1 {
 ; IS__TUNIT_OPM-NEXT:    [[ADD:%.*]] = add nsw i32 [[T0]], [[T1]]
 ; IS__TUNIT_OPM-NEXT:    ret i32 [[ADD]]
 ;
+; IS__TUNIT_NPM: Function Attrs: argmemonly nofree noinline nosync nounwind readonly uwtable willreturn
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@noalias_args_argmem_ro
 ; IS__TUNIT_NPM-SAME: (i32 [[TMP0:%.*]], i32 [[TMP1:%.*]])
 ; IS__TUNIT_NPM-NEXT:    [[B_PRIV:%.*]] = alloca i32, align 4
@@ -132,6 +142,7 @@ define internal i32 @noalias_args_argmem_ro(i32* %A, i32* %B) #1 {
 ; IS__TUNIT_NPM-NEXT:    [[ADD:%.*]] = add nsw i32 [[T0]], [[T1]]
 ; IS__TUNIT_NPM-NEXT:    ret i32 [[ADD]]
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree noinline norecurse nosync nounwind readonly uwtable willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@noalias_args_argmem_ro
 ; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[A:%.*]], i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B:%.*]])
 ; IS__CGSCC____-NEXT:    [[T0:%.*]] = load i32, i32* [[A]], align 4
@@ -146,12 +157,14 @@ define internal i32 @noalias_args_argmem_ro(i32* %A, i32* %B) #1 {
 }
 
 define i32 @visible_local_2() {
-; NOT_TUNIT_NPM-LABEL: define {{[^@]+}}@visible_local_2()
-; NOT_TUNIT_NPM-NEXT:    [[B:%.*]] = alloca i32, align 4
-; NOT_TUNIT_NPM-NEXT:    store i32 5, i32* [[B]], align 4
-; NOT_TUNIT_NPM-NEXT:    [[CALL:%.*]] = call i32 @noalias_args_argmem_ro(i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B]], i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B]])
-; NOT_TUNIT_NPM-NEXT:    ret i32 [[CALL]]
-;
+; IS__TUNIT_OPM: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@visible_local_2()
+; IS__TUNIT_OPM-NEXT:    [[B:%.*]] = alloca i32, align 4
+; IS__TUNIT_OPM-NEXT:    store i32 5, i32* [[B]], align 4
+; IS__TUNIT_OPM-NEXT:    [[CALL:%.*]] = call i32 @noalias_args_argmem_ro(i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B]], i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B]])
+; IS__TUNIT_OPM-NEXT:    ret i32 [[CALL]]
+;
+; IS__TUNIT_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@visible_local_2()
 ; IS__TUNIT_NPM-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; IS__TUNIT_NPM-NEXT:    store i32 5, i32* [[B]], align 4
@@ -159,6 +172,13 @@ define i32 @visible_local_2() {
 ; IS__TUNIT_NPM-NEXT:    [[TMP2:%.*]] = load i32, i32* [[B]], align 4
 ; IS__TUNIT_NPM-NEXT:    [[CALL:%.*]] = call i32 @noalias_args_argmem_ro(i32 [[TMP1]], i32 [[TMP2]])
 ; IS__TUNIT_NPM-NEXT:    ret i32 [[CALL]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@visible_local_2()
+; IS__CGSCC____-NEXT:    [[B:%.*]] = alloca i32, align 4
+; IS__CGSCC____-NEXT:    store i32 5, i32* [[B]], align 4
+; IS__CGSCC____-NEXT:    [[CALL:%.*]] = call i32 @noalias_args_argmem_ro(i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B]], i32* noalias nocapture nofree nonnull readonly align 4 dereferenceable(4) [[B]])
+; IS__CGSCC____-NEXT:    ret i32 [[CALL]]
 ;
   %B = alloca i32, align 4
   store i32 5, i32* %B, align 4
@@ -167,12 +187,14 @@ define i32 @visible_local_2() {
 }
 
 define internal i32 @noalias_args_argmem_rn(i32* %A, i32* %B) #1 {
+; IS__TUNIT____: Function Attrs: argmemonly nofree noinline nosync nounwind uwtable willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@noalias_args_argmem_rn
 ; IS__TUNIT____-SAME: (i32* noalias nocapture nofree nonnull align 4 dereferenceable(4) [[B:%.*]])
 ; IS__TUNIT____-NEXT:    [[T0:%.*]] = load i32, i32* [[B]], align 4
 ; IS__TUNIT____-NEXT:    store i32 0, i32* [[B]], align 4
 ; IS__TUNIT____-NEXT:    ret i32 [[T0]]
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree noinline norecurse nosync nounwind uwtable willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@noalias_args_argmem_rn
 ; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull align 4 dereferenceable(4) [[B:%.*]])
 ; IS__CGSCC____-NEXT:    [[T0:%.*]] = load i32, i32* [[B]], align 4
@@ -185,11 +207,19 @@ define internal i32 @noalias_args_argmem_rn(i32* %A, i32* %B) #1 {
 }
 
 define i32 @visible_local_3() {
-; CHECK-LABEL: define {{[^@]+}}@visible_local_3()
-; CHECK-NEXT:    [[B:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    store i32 5, i32* [[B]], align 4
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @noalias_args_argmem_rn(i32* noalias nocapture nofree nonnull align 4 dereferenceable(4) [[B]])
-; CHECK-NEXT:    ret i32 [[CALL]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@visible_local_3()
+; IS__TUNIT____-NEXT:    [[B:%.*]] = alloca i32, align 4
+; IS__TUNIT____-NEXT:    store i32 5, i32* [[B]], align 4
+; IS__TUNIT____-NEXT:    [[CALL:%.*]] = call i32 @noalias_args_argmem_rn(i32* noalias nocapture nofree nonnull align 4 dereferenceable(4) [[B]])
+; IS__TUNIT____-NEXT:    ret i32 [[CALL]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@visible_local_3()
+; IS__CGSCC____-NEXT:    [[B:%.*]] = alloca i32, align 4
+; IS__CGSCC____-NEXT:    store i32 5, i32* [[B]], align 4
+; IS__CGSCC____-NEXT:    [[CALL:%.*]] = call i32 @noalias_args_argmem_rn(i32* noalias nocapture nofree nonnull align 4 dereferenceable(4) [[B]])
+; IS__CGSCC____-NEXT:    ret i32 [[CALL]]
 ;
   %B = alloca i32, align 4
   store i32 5, i32* %B, align 4
diff --git a/llvm/test/Transforms/Attributor/liveness.ll b/llvm/test/Transforms/Attributor/liveness.ll
index 1ce3ff007092b..04899eb1f7121 100644
--- a/llvm/test/Transforms/Attributor/liveness.ll
+++ b/llvm/test/Transforms/Attributor/liveness.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=17 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=17 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; TODO: The old pass manager cgscc run is disabled as it causes a crash on windows which is under investigation: http://lab.llvm.org:8011/builders/llvm-clang-x86_64-expensive-checks-win/builds/23151
@@ -27,6 +27,7 @@ declare i32 @bar() nosync readnone
 ; and nothing should be deduced for it.
 
 define internal i32 @dead_internal_func(i32 %0) {
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@dead_internal_func()
 ; IS__CGSCC____-NEXT:    br label [[TMP2:%.*]]
 ; IS__CGSCC____:       1:
@@ -55,8 +56,8 @@ define internal i32 @dead_internal_func(i32 %0) {
   br i1 %10, label %3, label %5
 }
 
-; CHECK: Function Attrs: argmemonly nofree norecurse nounwind uwtable willreturn
 define i32 @volatile_load(i32*) norecurse nounwind uwtable {
+; CHECK: Function Attrs: argmemonly nofree norecurse nounwind uwtable willreturn
 ; CHECK-LABEL: define {{[^@]+}}@volatile_load
 ; CHECK-SAME: (i32* nofree align 4 [[TMP0:%.*]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = load volatile i32, i32* [[TMP0]], align 4
@@ -67,6 +68,7 @@ define i32 @volatile_load(i32*) norecurse nounwind uwtable {
 }
 
 define internal i32 @internal_load(i32*) norecurse nounwind uwtable {
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone uwtable willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@internal_load()
 ; IS__CGSCC____-NEXT:    ret i32 undef
 ;
@@ -75,8 +77,8 @@ define internal i32 @internal_load(i32*) norecurse nounwind uwtable {
 }
 ; TEST 1: Only first block is live.
 
-; CHECK: Function Attrs: nofree noreturn nosync nounwind
 define i32 @first_block_no_return(i32 %a, i32* nonnull %ptr1, i32* %ptr2) #0 {
+; CHECK: Function Attrs: nofree noreturn nosync nounwind
 ; CHECK-LABEL: define {{[^@]+}}@first_block_no_return
 ; CHECK-SAME: (i32 [[A:%.*]], i32* nocapture nofree nonnull readnone [[PTR1:%.*]], i32* nocapture nofree readnone [[PTR2:%.*]])
 ; CHECK-NEXT:  entry:
@@ -118,8 +120,8 @@ cond.end:                                         ; preds = %cond.false, %cond.t
 ; This is just an example. For example we can put a sync call in a
 ; dead block and check if it is deduced.
 
-; CHECK: Function Attrs: nosync
 define i32 @dead_block_present(i32 %a, i32* %ptr1) #0 {
+; CHECK: Function Attrs: nosync
 ; CHECK-LABEL: define {{[^@]+}}@dead_block_present
 ; CHECK-SAME: (i32 [[A:%.*]], i32* nocapture nofree readnone [[PTR1:%.*]])
 ; CHECK-NEXT:  entry:
@@ -157,6 +159,7 @@ cond.end:                                         ; preds = %cond.false, %cond.t
 ; TEST 3: both cond.true and cond.false are dead, therfore cond.end is dead as well.
 
 define i32 @all_dead(i32 %a) #0 {
+; CHECK: Function Attrs: nofree noreturn nosync nounwind
 ; CHECK-LABEL: define {{[^@]+}}@all_dead
 ; CHECK-SAME: (i32 [[A:%.*]])
 ; CHECK-NEXT:  entry:
@@ -478,13 +481,23 @@ cleanup:
 ; FIXME: Should be able to detect undefined behavior.
 
 define void @ub(i32* %0) {
-; CHECK-LABEL: define {{[^@]+}}@ub
-; CHECK-SAME: (i32* nocapture nofree writeonly [[TMP0:%.*]])
-; CHECK-NEXT:    [[POISON:%.*]] = sub nuw i32 0, 1
-; CHECK-NEXT:    [[STILL_POISON:%.*]] = and i32 [[POISON]], 0
-; CHECK-NEXT:    [[POISON_YET_AGAIN:%.*]] = getelementptr i32, i32* [[TMP0]], i32 [[STILL_POISON]]
-; CHECK-NEXT:    store i32 0, i32* [[POISON_YET_AGAIN]], align 4
-; CHECK-NEXT:    ret void
+; NOT_CGSCC_NPM: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@ub
+; NOT_CGSCC_NPM-SAME: (i32* nocapture nofree writeonly [[TMP0:%.*]])
+; NOT_CGSCC_NPM-NEXT:    [[POISON:%.*]] = sub nuw i32 0, 1
+; NOT_CGSCC_NPM-NEXT:    [[STILL_POISON:%.*]] = and i32 [[POISON]], 0
+; NOT_CGSCC_NPM-NEXT:    [[POISON_YET_AGAIN:%.*]] = getelementptr i32, i32* [[TMP0]], i32 [[STILL_POISON]]
+; NOT_CGSCC_NPM-NEXT:    store i32 0, i32* [[POISON_YET_AGAIN]], align 4
+; NOT_CGSCC_NPM-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@ub
+; IS__CGSCC____-SAME: (i32* nocapture nofree writeonly [[TMP0:%.*]])
+; IS__CGSCC____-NEXT:    [[POISON:%.*]] = sub nuw i32 0, 1
+; IS__CGSCC____-NEXT:    [[STILL_POISON:%.*]] = and i32 [[POISON]], 0
+; IS__CGSCC____-NEXT:    [[POISON_YET_AGAIN:%.*]] = getelementptr i32, i32* [[TMP0]], i32 [[STILL_POISON]]
+; IS__CGSCC____-NEXT:    store i32 0, i32* [[POISON_YET_AGAIN]], align 4
+; IS__CGSCC____-NEXT:    ret void
 ;
   %poison = sub nuw i32 0, 1           ; Results in a poison value.
   %still_poison = and i32 %poison, 0   ; 0, but also poison.
@@ -494,11 +507,19 @@ define void @ub(i32* %0) {
 }
 
 define void @inf_loop() #0 {
-; CHECK-LABEL: define {{[^@]+}}@inf_loop()
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
-; CHECK:       while.body:
-; CHECK-NEXT:    br label [[WHILE_BODY]]
+; NOT_CGSCC_NPM: Function Attrs: nofree noreturn nosync nounwind readnone
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@inf_loop()
+; NOT_CGSCC_NPM-NEXT:  entry:
+; NOT_CGSCC_NPM-NEXT:    br label [[WHILE_BODY:%.*]]
+; NOT_CGSCC_NPM:       while.body:
+; NOT_CGSCC_NPM-NEXT:    br label [[WHILE_BODY]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse noreturn nosync nounwind readnone
+; IS__CGSCC____-LABEL: define {{[^@]+}}@inf_loop()
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    br label [[WHILE_BODY:%.*]]
+; IS__CGSCC____:       while.body:
+; IS__CGSCC____-NEXT:    br label [[WHILE_BODY]]
 ;
 entry:
   br label %while.body
@@ -511,6 +532,7 @@ while.body:                                       ; preds = %entry, %while.body
 ; FIXME: Detect infloops, and mark affected blocks dead.
 
 define i32 @test5(i32, i32) #0 {
+; CHECK: Function Attrs: nosync readnone
 ; CHECK-LABEL: define {{[^@]+}}@test5
 ; CHECK-SAME: (i32 [[TMP0:%.*]], i32 [[TMP1:%.*]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i32 [[TMP0]], [[TMP1]]
@@ -547,9 +569,15 @@ cond.end:                                               ; preds = %cond.if, %con
 }
 
 define void @rec() #0 {
-; CHECK-LABEL: define {{[^@]+}}@rec()
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    unreachable
+; NOT_CGSCC_NPM: Function Attrs: nofree noreturn nosync nounwind readnone willreturn
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@rec()
+; NOT_CGSCC_NPM-NEXT:  entry:
+; NOT_CGSCC_NPM-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@rec()
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    unreachable
 ;
 entry:
   call void @rec()
@@ -561,17 +589,31 @@ entry:
 ; and unreachable should be put after call to @rec().
 
 define i32 @test6(i32, i32) #0 {
-; CHECK-LABEL: define {{[^@]+}}@test6
-; CHECK-SAME: (i32 [[TMP0:%.*]], i32 [[TMP1:%.*]])
-; CHECK-NEXT:    unreachable
-; CHECK:       cond.if:
-; CHECK-NEXT:    unreachable
-; CHECK:       cond.elseif:
-; CHECK-NEXT:    unreachable
-; CHECK:       cond.else:
-; CHECK-NEXT:    unreachable
-; CHECK:       cond.end:
-; CHECK-NEXT:    unreachable
+; NOT_CGSCC_NPM: Function Attrs: nofree noreturn nosync nounwind readnone willreturn
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@test6
+; NOT_CGSCC_NPM-SAME: (i32 [[TMP0:%.*]], i32 [[TMP1:%.*]])
+; NOT_CGSCC_NPM-NEXT:    unreachable
+; NOT_CGSCC_NPM:       cond.if:
+; NOT_CGSCC_NPM-NEXT:    unreachable
+; NOT_CGSCC_NPM:       cond.elseif:
+; NOT_CGSCC_NPM-NEXT:    unreachable
+; NOT_CGSCC_NPM:       cond.else:
+; NOT_CGSCC_NPM-NEXT:    unreachable
+; NOT_CGSCC_NPM:       cond.end:
+; NOT_CGSCC_NPM-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test6
+; IS__CGSCC____-SAME: (i32 [[TMP0:%.*]], i32 [[TMP1:%.*]])
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       cond.if:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       cond.elseif:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       cond.else:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       cond.end:
+; IS__CGSCC____-NEXT:    unreachable
 ;
   call void @rec()
   %3 = icmp sgt i32 %0, %1
@@ -738,6 +780,7 @@ define internal i8* @f3(i8* readnone %0) local_unnamed_addr #0 {
 
 declare void @sink() nofree nosync nounwind willreturn
 define void @test_unreachable() {
+; CHECK: Function Attrs: nofree noreturn nosync nounwind
 ; CHECK-LABEL: define {{[^@]+}}@test_unreachable()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    call void @test_unreachable()
@@ -929,6 +972,7 @@ define linkonce_odr void @non_exact3() {
 }
 
 define internal void @non_dead_a0() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_a0()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -937,6 +981,7 @@ define internal void @non_dead_a0() {
   ret void
 }
 define internal void @non_dead_a1() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_a1()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -945,6 +990,7 @@ define internal void @non_dead_a1() {
   ret void
 }
 define internal void @non_dead_a2() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_a2()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -953,6 +999,7 @@ define internal void @non_dead_a2() {
   ret void
 }
 define internal void @non_dead_a3() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_a3()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -961,6 +1008,7 @@ define internal void @non_dead_a3() {
   ret void
 }
 define internal void @non_dead_a4() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_a4()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -969,6 +1017,7 @@ define internal void @non_dead_a4() {
   ret void
 }
 define internal void @non_dead_a5() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_a5()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -977,6 +1026,7 @@ define internal void @non_dead_a5() {
   ret void
 }
 define internal void @non_dead_a6() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_a6()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -985,6 +1035,7 @@ define internal void @non_dead_a6() {
   ret void
 }
 define internal void @non_dead_a7() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_a7()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -993,6 +1044,7 @@ define internal void @non_dead_a7() {
   ret void
 }
 define internal void @non_dead_a8() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_a8()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1001,6 +1053,7 @@ define internal void @non_dead_a8() {
   ret void
 }
 define internal void @non_dead_a9() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_a9()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1009,6 +1062,7 @@ define internal void @non_dead_a9() {
   ret void
 }
 define internal void @non_dead_a10() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_a10()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1017,6 +1071,7 @@ define internal void @non_dead_a10() {
   ret void
 }
 define internal void @non_dead_a11() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_a11()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1025,6 +1080,7 @@ define internal void @non_dead_a11() {
   ret void
 }
 define internal void @non_dead_a12() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_a12()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1033,6 +1089,7 @@ define internal void @non_dead_a12() {
   ret void
 }
 define internal void @non_dead_a13() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_a13()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1041,6 +1098,7 @@ define internal void @non_dead_a13() {
   ret void
 }
 define internal void @non_dead_a14() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_a14()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1049,6 +1107,7 @@ define internal void @non_dead_a14() {
   ret void
 }
 define internal void @non_dead_a15() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_a15()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1057,6 +1116,7 @@ define internal void @non_dead_a15() {
   ret void
 }
 define internal void @non_dead_b0() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_b0()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1065,6 +1125,7 @@ define internal void @non_dead_b0() {
   ret void
 }
 define internal void @non_dead_b1() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_b1()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1073,6 +1134,7 @@ define internal void @non_dead_b1() {
   ret void
 }
 define internal void @non_dead_b2() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_b2()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1081,6 +1143,7 @@ define internal void @non_dead_b2() {
   ret void
 }
 define internal void @non_dead_b3() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_b3()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1089,6 +1152,7 @@ define internal void @non_dead_b3() {
   ret void
 }
 define internal void @non_dead_b4() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_b4()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1097,6 +1161,7 @@ define internal void @non_dead_b4() {
   ret void
 }
 define internal void @non_dead_b5() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_b5()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1105,6 +1170,7 @@ define internal void @non_dead_b5() {
   ret void
 }
 define internal void @non_dead_b6() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_b6()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1113,6 +1179,7 @@ define internal void @non_dead_b6() {
   ret void
 }
 define internal void @non_dead_b7() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_b7()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1121,6 +1188,7 @@ define internal void @non_dead_b7() {
   ret void
 }
 define internal void @non_dead_b8() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_b8()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1129,6 +1197,7 @@ define internal void @non_dead_b8() {
   ret void
 }
 define internal void @non_dead_b9() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_b9()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1137,6 +1206,7 @@ define internal void @non_dead_b9() {
   ret void
 }
 define internal void @non_dead_b10() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_b10()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1145,6 +1215,7 @@ define internal void @non_dead_b10() {
   ret void
 }
 define internal void @non_dead_b11() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_b11()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1153,6 +1224,7 @@ define internal void @non_dead_b11() {
   ret void
 }
 define internal void @non_dead_b12() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_b12()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1161,6 +1233,7 @@ define internal void @non_dead_b12() {
   ret void
 }
 define internal void @non_dead_b13() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_b13()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1169,6 +1242,7 @@ define internal void @non_dead_b13() {
   ret void
 }
 define internal void @non_dead_b14() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_b14()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1177,6 +1251,7 @@ define internal void @non_dead_b14() {
   ret void
 }
 define internal void @non_dead_b15() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_b15()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1185,6 +1260,7 @@ define internal void @non_dead_b15() {
   ret void
 }
 define internal void @non_dead_c0() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_c0()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1193,6 +1269,7 @@ define internal void @non_dead_c0() {
   ret void
 }
 define internal void @non_dead_c1() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_c1()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1201,6 +1278,7 @@ define internal void @non_dead_c1() {
   ret void
 }
 define internal void @non_dead_c2() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_c2()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1209,6 +1287,7 @@ define internal void @non_dead_c2() {
   ret void
 }
 define internal void @non_dead_c3() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_c3()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1217,6 +1296,7 @@ define internal void @non_dead_c3() {
   ret void
 }
 define internal void @non_dead_c4() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_c4()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1225,6 +1305,7 @@ define internal void @non_dead_c4() {
   ret void
 }
 define internal void @non_dead_c5() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_c5()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1233,6 +1314,7 @@ define internal void @non_dead_c5() {
   ret void
 }
 define internal void @non_dead_c6() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_c6()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1241,6 +1323,7 @@ define internal void @non_dead_c6() {
   ret void
 }
 define internal void @non_dead_c7() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_c7()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1249,6 +1332,7 @@ define internal void @non_dead_c7() {
   ret void
 }
 define internal void @non_dead_c8() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_c8()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1257,6 +1341,7 @@ define internal void @non_dead_c8() {
   ret void
 }
 define internal void @non_dead_c9() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_c9()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1265,6 +1350,7 @@ define internal void @non_dead_c9() {
   ret void
 }
 define internal void @non_dead_c10() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_c10()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1273,6 +1359,7 @@ define internal void @non_dead_c10() {
   ret void
 }
 define internal void @non_dead_c11() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_c11()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1281,6 +1368,7 @@ define internal void @non_dead_c11() {
   ret void
 }
 define internal void @non_dead_c12() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_c12()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1289,6 +1377,7 @@ define internal void @non_dead_c12() {
   ret void
 }
 define internal void @non_dead_c13() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_c13()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1297,6 +1386,7 @@ define internal void @non_dead_c13() {
   ret void
 }
 define internal void @non_dead_c14() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_c14()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1305,6 +1395,7 @@ define internal void @non_dead_c14() {
   ret void
 }
 define internal void @non_dead_c15() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_c15()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1313,6 +1404,7 @@ define internal void @non_dead_c15() {
   ret void
 }
 define internal void @non_dead_d0() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_d0()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1321,6 +1413,7 @@ define internal void @non_dead_d0() {
   ret void
 }
 define internal void @non_dead_d1() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_d1()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1329,6 +1422,7 @@ define internal void @non_dead_d1() {
   ret void
 }
 define internal void @non_dead_d2() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_d2()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1337,6 +1431,7 @@ define internal void @non_dead_d2() {
   ret void
 }
 define internal void @non_dead_d3() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_d3()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1345,6 +1440,7 @@ define internal void @non_dead_d3() {
   ret void
 }
 define internal void @non_dead_d4() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_d4()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1353,6 +1449,7 @@ define internal void @non_dead_d4() {
   ret void
 }
 define internal void @non_dead_d5() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_d5()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1361,6 +1458,7 @@ define internal void @non_dead_d5() {
   ret void
 }
 define internal void @non_dead_d6() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_d6()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1369,6 +1467,7 @@ define internal void @non_dead_d6() {
   ret void
 }
 define internal void @non_dead_d7() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_d7()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1377,6 +1476,7 @@ define internal void @non_dead_d7() {
   ret void
 }
 define internal void @non_dead_d8() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_d8()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1385,6 +1485,7 @@ define internal void @non_dead_d8() {
   ret void
 }
 define internal void @non_dead_d9() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_d9()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1393,6 +1494,7 @@ define internal void @non_dead_d9() {
   ret void
 }
 define internal void @non_dead_d10() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_d10()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1401,6 +1503,7 @@ define internal void @non_dead_d10() {
   ret void
 }
 define internal void @non_dead_d11() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_d11()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1409,6 +1512,7 @@ define internal void @non_dead_d11() {
   ret void
 }
 define internal void @non_dead_d12() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_d12()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1417,6 +1521,7 @@ define internal void @non_dead_d12() {
   ret void
 }
 define internal void @non_dead_d13() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_d13()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1425,6 +1530,7 @@ define internal void @non_dead_d13() {
   ret void
 }
 define internal void @non_dead_d14() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_d14()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1433,6 +1539,7 @@ define internal void @non_dead_d14() {
   ret void
 }
 define internal void @non_dead_d15() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@non_dead_d15()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1462,6 +1569,7 @@ live_with_dead_entry:
 }
 
 define void @live_with_dead_entry_lp() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK: Function Attrs: nounwind
 ; CHECK-LABEL: define {{[^@]+}}@live_with_dead_entry_lp() #2 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    invoke void @blowup()
@@ -1495,6 +1603,7 @@ live_with_dead_entry:
 }
 
 define internal void @useless_arg_sink(i32* %a) {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@useless_arg_sink()
 ; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret void
@@ -1504,6 +1613,7 @@ define internal void @useless_arg_sink(i32* %a) {
 }
 
 define internal void @useless_arg_almost_sink(i32* %a) {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@useless_arg_almost_sink()
 ; CHECK-NEXT:    call void @useless_arg_sink()
 ; CHECK-NEXT:    ret void
@@ -1546,6 +1656,7 @@ define void @useless_arg_ext_int_ext(i32* %a) {
 ; FIXME: We should fold terminators.
 
 define internal i32 @switch_default(i64 %i) nounwind {
+; NOT_CGSCC_NPM: Function Attrs: nofree nosync nounwind willreturn
 ; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@switch_default()
 ; NOT_CGSCC_NPM-NEXT:  entry:
 ; NOT_CGSCC_NPM-NEXT:    switch i64 0, label [[SW_DEFAULT:%.*]] [
@@ -1558,6 +1669,7 @@ define internal i32 @switch_default(i64 %i) nounwind {
 ; NOT_CGSCC_NPM:       return:
 ; NOT_CGSCC_NPM-NEXT:    unreachable
 ;
+; IS__CGSCC____: Function Attrs: nofree nosync nounwind willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@switch_default()
 ; IS__CGSCC____-NEXT:  entry:
 ; IS__CGSCC____-NEXT:    switch i64 0, label [[SW_DEFAULT:%.*]] [
@@ -1585,6 +1697,7 @@ return:
 }
 
 define i32 @switch_default_caller() {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@switch_default_caller()
 ; CHECK-NEXT:    [[CALL2:%.*]] = tail call i32 @switch_default()
 ; CHECK-NEXT:    ret i32 123
@@ -1594,6 +1707,7 @@ define i32 @switch_default_caller() {
 }
 
 define internal i32 @switch_default_dead(i64 %i) nounwind {
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@switch_default_dead()
 ; IS__CGSCC____-NEXT:  entry:
 ; IS__CGSCC____-NEXT:    switch i64 0, label [[SW_DEFAULT:%.*]] [
@@ -1619,8 +1733,13 @@ return:
 }
 
 define i32 @switch_default_dead_caller() {
-; CHECK-LABEL: define {{[^@]+}}@switch_default_dead_caller()
-; CHECK-NEXT:    ret i32 123
+; NOT_CGSCC_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@switch_default_dead_caller()
+; NOT_CGSCC_NPM-NEXT:    ret i32 123
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@switch_default_dead_caller()
+; IS__CGSCC____-NEXT:    ret i32 123
 ;
   %call2 = tail call i32 @switch_default_dead(i64 0)
   ret i32 %call2
@@ -1841,9 +1960,15 @@ for.end.1:
 declare noalias i8* @malloc(i64)
 
 define i32 @h(i32 %i) {
-; CHECK-LABEL: define {{[^@]+}}@h
-; CHECK-SAME: (i32 [[I:%.*]])
-; CHECK-NEXT:    ret i32 0
+; NOT_CGSCC_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@h
+; NOT_CGSCC_NPM-SAME: (i32 [[I:%.*]])
+; NOT_CGSCC_NPM-NEXT:    ret i32 0
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@h
+; IS__CGSCC____-SAME: (i32 [[I:%.*]])
+; IS__CGSCC____-NEXT:    ret i32 0
 ;
   ret i32 0
 }
@@ -1854,6 +1979,7 @@ define i32 @h(i32 %i) {
 @p = global i8 0
 
 define void @bad_gep() {
+; CHECK: Function Attrs: nounwind readnone willreturn
 ; CHECK-LABEL: define {{[^@]+}}@bad_gep()
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[N:%.*]] = alloca i8, align 1
diff --git a/llvm/test/Transforms/Attributor/liveness_chains.ll b/llvm/test/Transforms/Attributor/liveness_chains.ll
index 53c05bf2d7a77..5339ea839e337 100644
--- a/llvm/test/Transforms/Attributor/liveness_chains.ll
+++ b/llvm/test/Transforms/Attributor/liveness_chains.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -9,9 +9,15 @@
 declare i32 @source() nounwind readonly
 
 define i32 @chain_dead(i32 %arg) {
-; CHECK-LABEL: define {{[^@]+}}@chain_dead
-; CHECK-SAME: (i32 [[ARG:%.*]])
-; CHECK-NEXT:    ret i32 0
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@chain_dead
+; IS__TUNIT____-SAME: (i32 [[ARG:%.*]])
+; IS__TUNIT____-NEXT:    ret i32 0
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@chain_dead
+; IS__CGSCC____-SAME: (i32 [[ARG:%.*]])
+; IS__CGSCC____-NEXT:    ret i32 0
 ;
   %init = call i32 @source()
   %v0 = add i32 %arg, %init
@@ -28,6 +34,7 @@ define i32 @chain_dead(i32 %arg) {
 }
 
 define i32 @chain_alive(i32 %arg) {
+; CHECK: Function Attrs: nounwind readonly
 ; CHECK-LABEL: define {{[^@]+}}@chain_alive
 ; CHECK-SAME: (i32 [[ARG:%.*]])
 ; CHECK-NEXT:    [[INIT:%.*]] = call i32 @source()
diff --git a/llvm/test/Transforms/Attributor/lvi-after-jumpthreading.ll b/llvm/test/Transforms/Attributor/lvi-after-jumpthreading.ll
index 846a3615d0b4c..6e74cb2bed613 100644
--- a/llvm/test/Transforms/Attributor/lvi-after-jumpthreading.ll
+++ b/llvm/test/Transforms/Attributor/lvi-after-jumpthreading.ll
@@ -1,38 +1,71 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 define i8 @test1(i32 %a, i32 %length) {
-; IS________OPM-LABEL: define {{[^@]+}}@test1
-; IS________OPM-SAME: (i32 [[A:%.*]], i32 [[LENGTH:%.*]])
-; IS________OPM-NEXT:  entry:
-; IS________OPM-NEXT:    br label [[LOOP:%.*]]
-; IS________OPM:       loop:
-; IS________OPM-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
-; IS________OPM-NEXT:    [[CND:%.*]] = icmp sge i32 [[IV]], 0
-; IS________OPM-NEXT:    br i1 [[CND]], label [[BACKEDGE]], label [[EXIT:%.*]]
-; IS________OPM:       backedge:
-; IS________OPM-NEXT:    [[IV_NEXT]] = add nsw i32 [[IV]], 1
-; IS________OPM-NEXT:    [[CONT:%.*]] = icmp slt i32 [[IV_NEXT]], 400
-; IS________OPM-NEXT:    br i1 [[CONT]], label [[LOOP]], label [[EXIT]]
-; IS________OPM:       exit:
-; IS________OPM-NEXT:    ret i8 0
+; IS__TUNIT_OPM: Function Attrs: nofree nosync nounwind readnone
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@test1
+; IS__TUNIT_OPM-SAME: (i32 [[A:%.*]], i32 [[LENGTH:%.*]])
+; IS__TUNIT_OPM-NEXT:  entry:
+; IS__TUNIT_OPM-NEXT:    br label [[LOOP:%.*]]
+; IS__TUNIT_OPM:       loop:
+; IS__TUNIT_OPM-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; IS__TUNIT_OPM-NEXT:    [[CND:%.*]] = icmp sge i32 [[IV]], 0
+; IS__TUNIT_OPM-NEXT:    br i1 [[CND]], label [[BACKEDGE]], label [[EXIT:%.*]]
+; IS__TUNIT_OPM:       backedge:
+; IS__TUNIT_OPM-NEXT:    [[IV_NEXT]] = add nsw i32 [[IV]], 1
+; IS__TUNIT_OPM-NEXT:    [[CONT:%.*]] = icmp slt i32 [[IV_NEXT]], 400
+; IS__TUNIT_OPM-NEXT:    br i1 [[CONT]], label [[LOOP]], label [[EXIT]]
+; IS__TUNIT_OPM:       exit:
+; IS__TUNIT_OPM-NEXT:    ret i8 0
 ;
-; IS________NPM-LABEL: define {{[^@]+}}@test1
-; IS________NPM-SAME: (i32 [[A:%.*]], i32 [[LENGTH:%.*]])
-; IS________NPM-NEXT:  entry:
-; IS________NPM-NEXT:    br label [[LOOP:%.*]]
-; IS________NPM:       loop:
-; IS________NPM-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
-; IS________NPM-NEXT:    br label [[BACKEDGE]]
-; IS________NPM:       backedge:
-; IS________NPM-NEXT:    [[IV_NEXT]] = add nsw i32 [[IV]], 1
-; IS________NPM-NEXT:    [[CONT:%.*]] = icmp slt i32 [[IV_NEXT]], 400
-; IS________NPM-NEXT:    br i1 [[CONT]], label [[LOOP]], label [[EXIT:%.*]]
-; IS________NPM:       exit:
-; IS________NPM-NEXT:    ret i8 0
+; IS__TUNIT_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@test1
+; IS__TUNIT_NPM-SAME: (i32 [[A:%.*]], i32 [[LENGTH:%.*]])
+; IS__TUNIT_NPM-NEXT:  entry:
+; IS__TUNIT_NPM-NEXT:    br label [[LOOP:%.*]]
+; IS__TUNIT_NPM:       loop:
+; IS__TUNIT_NPM-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; IS__TUNIT_NPM-NEXT:    br label [[BACKEDGE]]
+; IS__TUNIT_NPM:       backedge:
+; IS__TUNIT_NPM-NEXT:    [[IV_NEXT]] = add nsw i32 [[IV]], 1
+; IS__TUNIT_NPM-NEXT:    [[CONT:%.*]] = icmp slt i32 [[IV_NEXT]], 400
+; IS__TUNIT_NPM-NEXT:    br i1 [[CONT]], label [[LOOP]], label [[EXIT:%.*]]
+; IS__TUNIT_NPM:       exit:
+; IS__TUNIT_NPM-NEXT:    ret i8 0
+;
+; IS__CGSCC_OPM: Function Attrs: nofree norecurse nosync nounwind readnone
+; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@test1
+; IS__CGSCC_OPM-SAME: (i32 [[A:%.*]], i32 [[LENGTH:%.*]])
+; IS__CGSCC_OPM-NEXT:  entry:
+; IS__CGSCC_OPM-NEXT:    br label [[LOOP:%.*]]
+; IS__CGSCC_OPM:       loop:
+; IS__CGSCC_OPM-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; IS__CGSCC_OPM-NEXT:    [[CND:%.*]] = icmp sge i32 [[IV]], 0
+; IS__CGSCC_OPM-NEXT:    br i1 [[CND]], label [[BACKEDGE]], label [[EXIT:%.*]]
+; IS__CGSCC_OPM:       backedge:
+; IS__CGSCC_OPM-NEXT:    [[IV_NEXT]] = add nsw i32 [[IV]], 1
+; IS__CGSCC_OPM-NEXT:    [[CONT:%.*]] = icmp slt i32 [[IV_NEXT]], 400
+; IS__CGSCC_OPM-NEXT:    br i1 [[CONT]], label [[LOOP]], label [[EXIT]]
+; IS__CGSCC_OPM:       exit:
+; IS__CGSCC_OPM-NEXT:    ret i8 0
+;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@test1
+; IS__CGSCC_NPM-SAME: (i32 [[A:%.*]], i32 [[LENGTH:%.*]])
+; IS__CGSCC_NPM-NEXT:  entry:
+; IS__CGSCC_NPM-NEXT:    br label [[LOOP:%.*]]
+; IS__CGSCC_NPM:       loop:
+; IS__CGSCC_NPM-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; IS__CGSCC_NPM-NEXT:    br label [[BACKEDGE]]
+; IS__CGSCC_NPM:       backedge:
+; IS__CGSCC_NPM-NEXT:    [[IV_NEXT]] = add nsw i32 [[IV]], 1
+; IS__CGSCC_NPM-NEXT:    [[CONT:%.*]] = icmp slt i32 [[IV_NEXT]], 400
+; IS__CGSCC_NPM-NEXT:    br i1 [[CONT]], label [[LOOP]], label [[EXIT:%.*]]
+; IS__CGSCC_NPM:       exit:
+; IS__CGSCC_NPM-NEXT:    ret i8 0
 ;
 entry:
   br label %loop
@@ -52,26 +85,93 @@ exit:
 }
 
 define i8 @test2(i32 %n) {
-; CHECK-LABEL: define {{[^@]+}}@test2
-; CHECK-SAME: (i32 [[N:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
-; CHECK-NEXT:    [[IV2:%.*]] = phi i32 [ [[N]], [[ENTRY]] ], [ [[IV2_NEXT:%.*]], [[BACKEDGE]] ]
-; CHECK-NEXT:    [[CND1:%.*]] = icmp sge i32 [[IV]], 0
-; CHECK-NEXT:    [[CND2:%.*]] = icmp sgt i32 [[IV2]], 0
-; CHECK-NEXT:    [[CND:%.*]] = and i1 [[CND1]], [[CND2]]
-; CHECK-NEXT:    br i1 [[CND]], label [[BACKEDGE]], label [[EXIT:%.*]]
-; CHECK:       backedge:
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i32 [[IV]], 1
-; CHECK-NEXT:    [[IV2_NEXT]] = sub nsw i32 [[IV2]], 1
-; CHECK-NEXT:    [[CONT1:%.*]] = icmp slt i32 [[IV_NEXT]], 400
-; CHECK-NEXT:    [[CONT2:%.*]] = icmp sgt i32 [[IV2_NEXT]], 0
-; CHECK-NEXT:    [[CONT:%.*]] = and i1 [[CONT1]], [[CONT2]]
-; CHECK-NEXT:    br i1 [[CONT]], label [[LOOP]], label [[EXIT]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret i8 0
+; IS__TUNIT_OPM: Function Attrs: nofree nosync nounwind readnone
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@test2
+; IS__TUNIT_OPM-SAME: (i32 [[N:%.*]])
+; IS__TUNIT_OPM-NEXT:  entry:
+; IS__TUNIT_OPM-NEXT:    br label [[LOOP:%.*]]
+; IS__TUNIT_OPM:       loop:
+; IS__TUNIT_OPM-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; IS__TUNIT_OPM-NEXT:    [[IV2:%.*]] = phi i32 [ [[N]], [[ENTRY]] ], [ [[IV2_NEXT:%.*]], [[BACKEDGE]] ]
+; IS__TUNIT_OPM-NEXT:    [[CND1:%.*]] = icmp sge i32 [[IV]], 0
+; IS__TUNIT_OPM-NEXT:    [[CND2:%.*]] = icmp sgt i32 [[IV2]], 0
+; IS__TUNIT_OPM-NEXT:    [[CND:%.*]] = and i1 [[CND1]], [[CND2]]
+; IS__TUNIT_OPM-NEXT:    br i1 [[CND]], label [[BACKEDGE]], label [[EXIT:%.*]]
+; IS__TUNIT_OPM:       backedge:
+; IS__TUNIT_OPM-NEXT:    [[IV_NEXT]] = add nsw i32 [[IV]], 1
+; IS__TUNIT_OPM-NEXT:    [[IV2_NEXT]] = sub nsw i32 [[IV2]], 1
+; IS__TUNIT_OPM-NEXT:    [[CONT1:%.*]] = icmp slt i32 [[IV_NEXT]], 400
+; IS__TUNIT_OPM-NEXT:    [[CONT2:%.*]] = icmp sgt i32 [[IV2_NEXT]], 0
+; IS__TUNIT_OPM-NEXT:    [[CONT:%.*]] = and i1 [[CONT1]], [[CONT2]]
+; IS__TUNIT_OPM-NEXT:    br i1 [[CONT]], label [[LOOP]], label [[EXIT]]
+; IS__TUNIT_OPM:       exit:
+; IS__TUNIT_OPM-NEXT:    ret i8 0
+;
+; IS__TUNIT_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@test2
+; IS__TUNIT_NPM-SAME: (i32 [[N:%.*]])
+; IS__TUNIT_NPM-NEXT:  entry:
+; IS__TUNIT_NPM-NEXT:    br label [[LOOP:%.*]]
+; IS__TUNIT_NPM:       loop:
+; IS__TUNIT_NPM-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; IS__TUNIT_NPM-NEXT:    [[IV2:%.*]] = phi i32 [ [[N]], [[ENTRY]] ], [ [[IV2_NEXT:%.*]], [[BACKEDGE]] ]
+; IS__TUNIT_NPM-NEXT:    [[CND1:%.*]] = icmp sge i32 [[IV]], 0
+; IS__TUNIT_NPM-NEXT:    [[CND2:%.*]] = icmp sgt i32 [[IV2]], 0
+; IS__TUNIT_NPM-NEXT:    [[CND:%.*]] = and i1 [[CND1]], [[CND2]]
+; IS__TUNIT_NPM-NEXT:    br i1 [[CND]], label [[BACKEDGE]], label [[EXIT:%.*]]
+; IS__TUNIT_NPM:       backedge:
+; IS__TUNIT_NPM-NEXT:    [[IV_NEXT]] = add nsw i32 [[IV]], 1
+; IS__TUNIT_NPM-NEXT:    [[IV2_NEXT]] = sub nsw i32 [[IV2]], 1
+; IS__TUNIT_NPM-NEXT:    [[CONT1:%.*]] = icmp slt i32 [[IV_NEXT]], 400
+; IS__TUNIT_NPM-NEXT:    [[CONT2:%.*]] = icmp sgt i32 [[IV2_NEXT]], 0
+; IS__TUNIT_NPM-NEXT:    [[CONT:%.*]] = and i1 [[CONT1]], [[CONT2]]
+; IS__TUNIT_NPM-NEXT:    br i1 [[CONT]], label [[LOOP]], label [[EXIT]]
+; IS__TUNIT_NPM:       exit:
+; IS__TUNIT_NPM-NEXT:    ret i8 0
+;
+; IS__CGSCC_OPM: Function Attrs: nofree norecurse nosync nounwind readnone
+; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@test2
+; IS__CGSCC_OPM-SAME: (i32 [[N:%.*]])
+; IS__CGSCC_OPM-NEXT:  entry:
+; IS__CGSCC_OPM-NEXT:    br label [[LOOP:%.*]]
+; IS__CGSCC_OPM:       loop:
+; IS__CGSCC_OPM-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; IS__CGSCC_OPM-NEXT:    [[IV2:%.*]] = phi i32 [ [[N]], [[ENTRY]] ], [ [[IV2_NEXT:%.*]], [[BACKEDGE]] ]
+; IS__CGSCC_OPM-NEXT:    [[CND1:%.*]] = icmp sge i32 [[IV]], 0
+; IS__CGSCC_OPM-NEXT:    [[CND2:%.*]] = icmp sgt i32 [[IV2]], 0
+; IS__CGSCC_OPM-NEXT:    [[CND:%.*]] = and i1 [[CND1]], [[CND2]]
+; IS__CGSCC_OPM-NEXT:    br i1 [[CND]], label [[BACKEDGE]], label [[EXIT:%.*]]
+; IS__CGSCC_OPM:       backedge:
+; IS__CGSCC_OPM-NEXT:    [[IV_NEXT]] = add nsw i32 [[IV]], 1
+; IS__CGSCC_OPM-NEXT:    [[IV2_NEXT]] = sub nsw i32 [[IV2]], 1
+; IS__CGSCC_OPM-NEXT:    [[CONT1:%.*]] = icmp slt i32 [[IV_NEXT]], 400
+; IS__CGSCC_OPM-NEXT:    [[CONT2:%.*]] = icmp sgt i32 [[IV2_NEXT]], 0
+; IS__CGSCC_OPM-NEXT:    [[CONT:%.*]] = and i1 [[CONT1]], [[CONT2]]
+; IS__CGSCC_OPM-NEXT:    br i1 [[CONT]], label [[LOOP]], label [[EXIT]]
+; IS__CGSCC_OPM:       exit:
+; IS__CGSCC_OPM-NEXT:    ret i8 0
+;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@test2
+; IS__CGSCC_NPM-SAME: (i32 [[N:%.*]])
+; IS__CGSCC_NPM-NEXT:  entry:
+; IS__CGSCC_NPM-NEXT:    br label [[LOOP:%.*]]
+; IS__CGSCC_NPM:       loop:
+; IS__CGSCC_NPM-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; IS__CGSCC_NPM-NEXT:    [[IV2:%.*]] = phi i32 [ [[N]], [[ENTRY]] ], [ [[IV2_NEXT:%.*]], [[BACKEDGE]] ]
+; IS__CGSCC_NPM-NEXT:    [[CND1:%.*]] = icmp sge i32 [[IV]], 0
+; IS__CGSCC_NPM-NEXT:    [[CND2:%.*]] = icmp sgt i32 [[IV2]], 0
+; IS__CGSCC_NPM-NEXT:    [[CND:%.*]] = and i1 [[CND1]], [[CND2]]
+; IS__CGSCC_NPM-NEXT:    br i1 [[CND]], label [[BACKEDGE]], label [[EXIT:%.*]]
+; IS__CGSCC_NPM:       backedge:
+; IS__CGSCC_NPM-NEXT:    [[IV_NEXT]] = add nsw i32 [[IV]], 1
+; IS__CGSCC_NPM-NEXT:    [[IV2_NEXT]] = sub nsw i32 [[IV2]], 1
+; IS__CGSCC_NPM-NEXT:    [[CONT1:%.*]] = icmp slt i32 [[IV_NEXT]], 400
+; IS__CGSCC_NPM-NEXT:    [[CONT2:%.*]] = icmp sgt i32 [[IV2_NEXT]], 0
+; IS__CGSCC_NPM-NEXT:    [[CONT:%.*]] = and i1 [[CONT1]], [[CONT2]]
+; IS__CGSCC_NPM-NEXT:    br i1 [[CONT]], label [[LOOP]], label [[EXIT]]
+; IS__CGSCC_NPM:       exit:
+; IS__CGSCC_NPM-NEXT:    ret i8 0
 ;
 entry:
   br label %loop
diff --git a/llvm/test/Transforms/Attributor/lvi-for-ashr.ll b/llvm/test/Transforms/Attributor/lvi-for-ashr.ll
index a9d46b97b2287..a6be5bcc9740f 100644
--- a/llvm/test/Transforms/Attributor/lvi-for-ashr.ll
+++ b/llvm/test/Transforms/Attributor/lvi-for-ashr.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -7,25 +7,47 @@
 ; FIXME: DOT should be replaced with 3
 
 define i32 @test-ashr(i32 %c) {
-; CHECK-LABEL: define {{[^@]+}}@test-ashr
-; CHECK-SAME: (i32 [[C:%.*]])
-; CHECK-NEXT:  chk65:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[C]], 65
-; CHECK-NEXT:    br i1 [[CMP]], label [[RETURN:%.*]], label [[CHK0:%.*]]
-; CHECK:       chk0:
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[C]], 0
-; CHECK-NEXT:    br i1 [[CMP]], label [[RETURN]], label [[BB_IF:%.*]]
-; CHECK:       bb_if:
-; CHECK-NEXT:    [[ASHR_VAL:%.*]] = ashr exact i32 [[C]], 2
-; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[ASHR_VAL]], 15
-; CHECK-NEXT:    br i1 [[CMP2]], label [[BB_THEN:%.*]], label [[RETURN]]
-; CHECK:       bb_then:
-; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i32 [[ASHR_VAL]], 16
-; CHECK-NEXT:    [[DOT:%.*]] = select i1 [[CMP3]], i32 3, i32 2
-; CHECK-NEXT:    br label [[RETURN]]
-; CHECK:       return:
-; CHECK-NEXT:    [[RETVAL:%.*]] = phi i32 [ 0, [[CHK65:%.*]] ], [ 1, [[CHK0]] ], [ [[DOT]], [[BB_THEN]] ], [ 4, [[BB_IF]] ]
-; CHECK-NEXT:    ret i32 [[RETVAL]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test-ashr
+; IS__TUNIT____-SAME: (i32 [[C:%.*]])
+; IS__TUNIT____-NEXT:  chk65:
+; IS__TUNIT____-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[C]], 65
+; IS__TUNIT____-NEXT:    br i1 [[CMP]], label [[RETURN:%.*]], label [[CHK0:%.*]]
+; IS__TUNIT____:       chk0:
+; IS__TUNIT____-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[C]], 0
+; IS__TUNIT____-NEXT:    br i1 [[CMP]], label [[RETURN]], label [[BB_IF:%.*]]
+; IS__TUNIT____:       bb_if:
+; IS__TUNIT____-NEXT:    [[ASHR_VAL:%.*]] = ashr exact i32 [[C]], 2
+; IS__TUNIT____-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[ASHR_VAL]], 15
+; IS__TUNIT____-NEXT:    br i1 [[CMP2]], label [[BB_THEN:%.*]], label [[RETURN]]
+; IS__TUNIT____:       bb_then:
+; IS__TUNIT____-NEXT:    [[CMP3:%.*]] = icmp eq i32 [[ASHR_VAL]], 16
+; IS__TUNIT____-NEXT:    [[DOT:%.*]] = select i1 [[CMP3]], i32 3, i32 2
+; IS__TUNIT____-NEXT:    br label [[RETURN]]
+; IS__TUNIT____:       return:
+; IS__TUNIT____-NEXT:    [[RETVAL:%.*]] = phi i32 [ 0, [[CHK65:%.*]] ], [ 1, [[CHK0]] ], [ [[DOT]], [[BB_THEN]] ], [ 4, [[BB_IF]] ]
+; IS__TUNIT____-NEXT:    ret i32 [[RETVAL]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test-ashr
+; IS__CGSCC____-SAME: (i32 [[C:%.*]])
+; IS__CGSCC____-NEXT:  chk65:
+; IS__CGSCC____-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[C]], 65
+; IS__CGSCC____-NEXT:    br i1 [[CMP]], label [[RETURN:%.*]], label [[CHK0:%.*]]
+; IS__CGSCC____:       chk0:
+; IS__CGSCC____-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[C]], 0
+; IS__CGSCC____-NEXT:    br i1 [[CMP]], label [[RETURN]], label [[BB_IF:%.*]]
+; IS__CGSCC____:       bb_if:
+; IS__CGSCC____-NEXT:    [[ASHR_VAL:%.*]] = ashr exact i32 [[C]], 2
+; IS__CGSCC____-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[ASHR_VAL]], 15
+; IS__CGSCC____-NEXT:    br i1 [[CMP2]], label [[BB_THEN:%.*]], label [[RETURN]]
+; IS__CGSCC____:       bb_then:
+; IS__CGSCC____-NEXT:    [[CMP3:%.*]] = icmp eq i32 [[ASHR_VAL]], 16
+; IS__CGSCC____-NEXT:    [[DOT:%.*]] = select i1 [[CMP3]], i32 3, i32 2
+; IS__CGSCC____-NEXT:    br label [[RETURN]]
+; IS__CGSCC____:       return:
+; IS__CGSCC____-NEXT:    [[RETVAL:%.*]] = phi i32 [ 0, [[CHK65:%.*]] ], [ 1, [[CHK0]] ], [ [[DOT]], [[BB_THEN]] ], [ 4, [[BB_IF]] ]
+; IS__CGSCC____-NEXT:    ret i32 [[RETVAL]]
 ;
 chk65:
   %cmp = icmp sgt i32 %c, 65
diff --git a/llvm/test/Transforms/Attributor/memory_locations.ll b/llvm/test/Transforms/Attributor/memory_locations.ll
index 42dc2e51be3a6..be3f9a0f829bd 100644
--- a/llvm/test/Transforms/Attributor/memory_locations.ll
+++ b/llvm/test/Transforms/Attributor/memory_locations.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=11 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=11 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -10,8 +10,8 @@ target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16
 ; CHECK: Function Attrs: inaccessiblememonly
 declare noalias i8* @malloc(i64) inaccessiblememonly
 
-; CHECK: Function Attrs: inaccessiblememonly
 define dso_local i8* @internal_only(i32 %arg) {
+; CHECK: Function Attrs: inaccessiblememonly
 ; CHECK-LABEL: define {{[^@]+}}@internal_only
 ; CHECK-SAME: (i32 [[ARG:%.*]])
 ; CHECK-NEXT:  entry:
@@ -25,8 +25,8 @@ entry:
   ret i8* %call
 }
 
-; CHECK: Function Attrs: inaccessiblememonly
 define dso_local i8* @internal_only_rec(i32 %arg) {
+; CHECK: Function Attrs: inaccessiblememonly
 ; CHECK-LABEL: define {{[^@]+}}@internal_only_rec
 ; CHECK-SAME: (i32 [[ARG:%.*]])
 ; CHECK-NEXT:  entry:
@@ -65,8 +65,8 @@ return:                                           ; preds = %if.end, %if.then
   ret i8* %retval.0
 }
 
-; CHECK: Function Attrs: inaccessiblememonly
 define dso_local i8* @internal_only_rec_static_helper(i32 %arg) {
+; CHECK: Function Attrs: inaccessiblememonly
 ; CHECK-LABEL: define {{[^@]+}}@internal_only_rec_static_helper
 ; CHECK-SAME: (i32 [[ARG:%.*]])
 ; CHECK-NEXT:  entry:
@@ -78,8 +78,8 @@ entry:
   ret i8* %call
 }
 
-; CHECK: Function Attrs: inaccessiblememonly
 define internal i8* @internal_only_rec_static(i32 %arg) {
+; CHECK: Function Attrs: inaccessiblememonly
 ; CHECK-LABEL: define {{[^@]+}}@internal_only_rec_static
 ; CHECK-SAME: (i32 [[ARG:%.*]])
 ; CHECK-NEXT:  entry:
@@ -173,8 +173,8 @@ return:                                           ; preds = %if.end, %if.then
   ret i8* %retval.0
 }
 
-; CHECK: Function Attrs: inaccessiblemem_or_argmemonly
 define dso_local i8* @internal_argmem_only_read(i32* %arg) {
+; CHECK: Function Attrs: inaccessiblemem_or_argmemonly
 ; CHECK-LABEL: define {{[^@]+}}@internal_argmem_only_read
 ; CHECK-SAME: (i32* nocapture nonnull readonly align 4 dereferenceable(4) [[ARG:%.*]])
 ; CHECK-NEXT:  entry:
@@ -190,8 +190,8 @@ entry:
   ret i8* %call
 }
 
-; CHECK: Function Attrs: inaccessiblemem_or_argmemonly
 define dso_local i8* @internal_argmem_only_write(i32* %arg) {
+; CHECK: Function Attrs: inaccessiblemem_or_argmemonly
 ; CHECK-LABEL: define {{[^@]+}}@internal_argmem_only_write
 ; CHECK-SAME: (i32* nocapture nonnull writeonly align 4 dereferenceable(4) [[ARG:%.*]])
 ; CHECK-NEXT:  entry:
@@ -205,14 +205,15 @@ entry:
   ret i8* %call
 }
 
-; CHECK: Function Attrs: inaccessiblemem_or_argmemonly
 define dso_local i8* @internal_argmem_only_rec(i32* %arg) {
+; IS__TUNIT____: Function Attrs: inaccessiblemem_or_argmemonly
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@internal_argmem_only_rec
 ; IS__TUNIT____-SAME: (i32* nocapture [[ARG:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
 ; IS__TUNIT____-NEXT:    [[CALL:%.*]] = call noalias i8* @internal_argmem_only_rec_1(i32* nocapture align 4 [[ARG]])
 ; IS__TUNIT____-NEXT:    ret i8* [[CALL]]
 ;
+; IS__CGSCC____: Function Attrs: inaccessiblemem_or_argmemonly
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@internal_argmem_only_rec
 ; IS__CGSCC____-SAME: (i32* nocapture nonnull align 4 dereferenceable(4) [[ARG:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -224,8 +225,8 @@ entry:
   ret i8* %call
 }
 
-; CHECK: Function Attrs: inaccessiblemem_or_argmemonly
 define internal i8* @internal_argmem_only_rec_1(i32* %arg) {
+; CHECK: Function Attrs: inaccessiblemem_or_argmemonly
 ; CHECK-LABEL: define {{[^@]+}}@internal_argmem_only_rec_1
 ; CHECK-SAME: (i32* nocapture nonnull align 4 dereferenceable(4) [[ARG:%.*]])
 ; CHECK-NEXT:  entry:
@@ -280,8 +281,8 @@ return:                                           ; preds = %if.end3, %if.then2,
   ret i8* %retval.0
 }
 
-; CHECK: Function Attrs: inaccessiblemem_or_argmemonly
 define internal i8* @internal_argmem_only_rec_2(i32* %arg) {
+; CHECK: Function Attrs: inaccessiblemem_or_argmemonly
 ; CHECK-LABEL: define {{[^@]+}}@internal_argmem_only_rec_2
 ; CHECK-SAME: (i32* nocapture nonnull align 4 dereferenceable(4) [[ARG:%.*]])
 ; CHECK-NEXT:  entry:
@@ -302,8 +303,8 @@ declare i8* @argmem_only(i8* %arg) argmemonly
 declare i8* @inaccesible_argmem_only_decl(i8* %arg) inaccessiblemem_or_argmemonly
 declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) nounwind argmemonly willreturn
 
-; CHECK: Function Attrs: argmemonly
 define void @callerA1(i8* %arg) {
+; CHECK: Function Attrs: argmemonly
 ; CHECK-LABEL: define {{[^@]+}}@callerA1
 ; CHECK-SAME: (i8* [[ARG:%.*]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i8* @argmem_only(i8* [[ARG]])
@@ -312,8 +313,8 @@ define void @callerA1(i8* %arg) {
   call i8* @argmem_only(i8* %arg)
   ret void
 }
-; CHECK: Function Attrs: inaccessiblemem_or_argmemonly
 define void @callerA2(i8* %arg) {
+; CHECK: Function Attrs: inaccessiblemem_or_argmemonly
 ; CHECK-LABEL: define {{[^@]+}}@callerA2
 ; CHECK-SAME: (i8* [[ARG:%.*]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i8* @inaccesible_argmem_only_decl(i8* [[ARG]])
@@ -322,8 +323,8 @@ define void @callerA2(i8* %arg) {
   call i8* @inaccesible_argmem_only_decl(i8* %arg)
   ret void
 }
-; CHECK: Function Attrs: readnone
 define void @callerB1() {
+; CHECK: Function Attrs: readnone
 ; CHECK-LABEL: define {{[^@]+}}@callerB1()
 ; CHECK-NEXT:    [[STACK:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i8* @argmem_only(i8* nonnull dereferenceable(1) [[STACK]])
@@ -333,8 +334,8 @@ define void @callerB1() {
   call i8* @argmem_only(i8* %stack)
   ret void
 }
-; CHECK: Function Attrs: inaccessiblememonly
 define void @callerB2() {
+; CHECK: Function Attrs: inaccessiblememonly
 ; CHECK-LABEL: define {{[^@]+}}@callerB2()
 ; CHECK-NEXT:    [[STACK:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i8* @inaccesible_argmem_only_decl(i8* nonnull dereferenceable(1) [[STACK]])
@@ -344,7 +345,6 @@ define void @callerB2() {
   call i8* @inaccesible_argmem_only_decl(i8* %stack)
   ret void
 }
-; CHECK-NOT: Function Attrs
 define void @callerC1() {
 ; CHECK-LABEL: define {{[^@]+}}@callerC1()
 ; CHECK-NEXT:    [[UNKNOWN:%.*]] = call i8* @unknown_ptr()
@@ -355,7 +355,6 @@ define void @callerC1() {
   call i8* @argmem_only(i8* %unknown)
   ret void
 }
-; CHECK-NOT: Function Attrs
 define void @callerC2() {
 ; CHECK-LABEL: define {{[^@]+}}@callerC2()
 ; CHECK-NEXT:    [[UNKNOWN:%.*]] = call i8* @unknown_ptr()
@@ -366,7 +365,6 @@ define void @callerC2() {
   call i8* @inaccesible_argmem_only_decl(i8* %unknown)
   ret void
 }
-; CHECK-NOT: Function Attrs
 define void @callerD1() {
 ; CHECK-LABEL: define {{[^@]+}}@callerD1()
 ; CHECK-NEXT:    [[UNKNOWN:%.*]] = call i8* @argmem_only(i8* noalias nocapture align 536870912 null)
@@ -377,7 +375,6 @@ define void @callerD1() {
   store i8 0, i8* %unknown
   ret void
 }
-; CHECK-NOT: Function Attrs
 define void @callerD2() {
 ; CHECK-LABEL: define {{[^@]+}}@callerD2()
 ; CHECK-NEXT:    [[UNKNOWN:%.*]] = call i8* @inaccesible_argmem_only_decl(i8* noalias nocapture align 536870912 null)
@@ -389,91 +386,114 @@ define void @callerD2() {
   ret void
 }
 
-; CHECK: Function Attrs: nofree {{(norecurse )?}}nosync nounwind readnone willreturn
 define void @callerE(i8* %arg) {
-; CHECK-LABEL: define {{[^@]+}}@callerE
-; CHECK-SAME: (i8* nocapture nofree readnone [[ARG:%.*]])
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@callerE
+; IS__TUNIT____-SAME: (i8* nocapture nofree readnone [[ARG:%.*]])
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@callerE
+; IS__CGSCC____-SAME: (i8* nocapture nofree readnone [[ARG:%.*]])
+; IS__CGSCC____-NEXT:    ret void
 ;
   call void @llvm.lifetime.start.p0i8(i64 4, i8* %arg)
   ret void
 }
 
 
-; CHECK: Function Attrs:
-; CHECK-SAME: writeonly
 define void @write_global() {
-; CHECK-LABEL: define {{[^@]+}}@write_global()
-; CHECK-NEXT:    store i32 0, i32* @G, align 4
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@write_global()
+; IS__TUNIT____-NEXT:    store i32 0, i32* @G, align 4
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@write_global()
+; IS__CGSCC____-NEXT:    store i32 0, i32* @G, align 4
+; IS__CGSCC____-NEXT:    ret void
 ;
   store i32 0, i32* @G, align 4
   ret void
 }
-; CHECK: Function Attrs: argmemonly
-; CHECK-SAME: writeonly
 define void @write_global_via_arg(i32* %GPtr) {
-; CHECK-LABEL: define {{[^@]+}}@write_global_via_arg
-; CHECK-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[GPTR:%.*]])
-; CHECK-NEXT:    store i32 0, i32* [[GPTR]], align 4
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@write_global_via_arg
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[GPTR:%.*]])
+; IS__TUNIT____-NEXT:    store i32 0, i32* [[GPTR]], align 4
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@write_global_via_arg
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[GPTR:%.*]])
+; IS__CGSCC____-NEXT:    store i32 0, i32* [[GPTR]], align 4
+; IS__CGSCC____-NEXT:    ret void
 ;
   store i32 0, i32* %GPtr, align 4
   ret void
 }
-; CHECK: Function Attrs:
-; CHECK-NOT: argmemonly
-; CHECK-SAME: writeonly
-; CHECK-NOT: argmemonly
 define internal void @write_global_via_arg_internal(i32* %GPtr) {
-; CHECK-LABEL: define {{[^@]+}}@write_global_via_arg_internal()
-; CHECK-NEXT:    store i32 0, i32* @G, align 4
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@write_global_via_arg_internal()
+; IS__TUNIT____-NEXT:    store i32 0, i32* @G, align 4
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@write_global_via_arg_internal()
+; IS__CGSCC____-NEXT:    store i32 0, i32* @G, align 4
+; IS__CGSCC____-NEXT:    ret void
 ;
   store i32 0, i32* %GPtr, align 4
   ret void
 }
 
-
-; CHECK: Function Attrs:
-; CHECK-SAME: writeonly
 define void @writeonly_global() {
-; CHECK-LABEL: define {{[^@]+}}@writeonly_global()
-; CHECK-NEXT:    call void @write_global()
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@writeonly_global()
+; IS__TUNIT____-NEXT:    call void @write_global()
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@writeonly_global()
+; IS__CGSCC____-NEXT:    call void @write_global()
+; IS__CGSCC____-NEXT:    ret void
 ;
   call void @write_global()
   ret void
 }
-; CHECK: Function Attrs:
-; CHECK-SAME: writeonly
 define void @writeonly_global_via_arg() {
-; CHECK-LABEL: define {{[^@]+}}@writeonly_global_via_arg()
-; CHECK-NEXT:    call void @write_global_via_arg(i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) @G)
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@writeonly_global_via_arg()
+; IS__TUNIT____-NEXT:    call void @write_global_via_arg(i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) @G)
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@writeonly_global_via_arg()
+; IS__CGSCC____-NEXT:    call void @write_global_via_arg(i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) @G)
+; IS__CGSCC____-NEXT:    ret void
 ;
   call void @write_global_via_arg(i32* @G)
   ret void
 }
 
-; CHECK: Function Attrs:
-; CHECK-SAME: writeonly
 define void @writeonly_global_via_arg_internal() {
 ;
-; CHECK-LABEL: define {{[^@]+}}@writeonly_global_via_arg_internal()
-; CHECK-NEXT:    call void @write_global_via_arg_internal()
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@writeonly_global_via_arg_internal()
+; IS__TUNIT____-NEXT:    call void @write_global_via_arg_internal()
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@writeonly_global_via_arg_internal()
+; IS__CGSCC____-NEXT:    call void @write_global_via_arg_internal()
+; IS__CGSCC____-NEXT:    ret void
 ;
   call void @write_global_via_arg_internal(i32* @G)
   ret void
 }
 
-
-; CHECK: Function Attrs:
-; CHECK-NOT: readnone
-; CHECK-SAME: argmemonly
-; CHECK-NOT: readnone
 define i8 @recursive_not_readnone(i8* %ptr, i1 %c) {
+; CHECK: Function Attrs: argmemonly nofree nosync nounwind
 ; CHECK-LABEL: define {{[^@]+}}@recursive_not_readnone
 ; CHECK-SAME: (i8* nocapture nofree writeonly [[PTR:%.*]], i1 [[C:%.*]])
 ; CHECK-NEXT:    [[ALLOC:%.*]] = alloca i8, align 1
@@ -497,11 +517,8 @@ f:
   ret i8 0
 }
 
-; CHECK: Function Attrs:
-; CHECK-NOT: readnone
-; CHECK-SAME: argmemonly
-; CHECK-NOT: readnone
 define internal i8 @recursive_not_readnone_internal(i8* %ptr, i1 %c) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@recursive_not_readnone_internal
 ; IS__TUNIT____-SAME: (i8* noalias nocapture nofree nonnull writeonly dereferenceable(1) [[PTR:%.*]], i1 [[C:%.*]])
 ; IS__TUNIT____-NEXT:    [[ALLOC:%.*]] = alloca i8, align 1
@@ -514,6 +531,7 @@ define internal i8 @recursive_not_readnone_internal(i8* %ptr, i1 %c) {
 ; IS__TUNIT____-NEXT:    store i8 1, i8* [[PTR]], align 1
 ; IS__TUNIT____-NEXT:    ret i8 0
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree nosync nounwind
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@recursive_not_readnone_internal
 ; IS__CGSCC____-SAME: (i8* nocapture nofree nonnull writeonly dereferenceable(1) [[PTR:%.*]], i1 [[C:%.*]])
 ; IS__CGSCC____-NEXT:    [[ALLOC:%.*]] = alloca i8, align 1
@@ -537,9 +555,8 @@ f:
   ret i8 0
 }
 
-; CHECK: Function Attrs:
-; CHECK-SAME: readnone
 define i8 @readnone_caller(i1 %c) {
+; CHECK: Function Attrs: nofree nosync nounwind readnone
 ; CHECK-LABEL: define {{[^@]+}}@readnone_caller
 ; CHECK-SAME: (i1 [[C:%.*]])
 ; CHECK-NEXT:    [[A:%.*]] = alloca i8, align 1
@@ -551,11 +568,8 @@ define i8 @readnone_caller(i1 %c) {
   ret i8 %r
 }
 
-; CHECK: Function Attrs:
-; CHECK-NOT: readnone
-; CHECK-SAME: argmemonly
-; CHECK-NOT: readnone
 define internal i8 @recursive_not_readnone_internal2(i8* %ptr, i1 %c) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@recursive_not_readnone_internal2
 ; IS__TUNIT____-SAME: (i8* noalias nocapture nofree nonnull writeonly [[PTR:%.*]], i1 [[C:%.*]])
 ; IS__TUNIT____-NEXT:    [[ALLOC:%.*]] = alloca i8, align 1
@@ -568,6 +582,7 @@ define internal i8 @recursive_not_readnone_internal2(i8* %ptr, i1 %c) {
 ; IS__TUNIT____-NEXT:    store i8 1, i8* [[PTR]], align 1
 ; IS__TUNIT____-NEXT:    ret i8 0
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree nosync nounwind
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@recursive_not_readnone_internal2
 ; IS__CGSCC____-SAME: (i8* nocapture nofree nonnull writeonly [[PTR:%.*]], i1 [[C:%.*]])
 ; IS__CGSCC____-NEXT:    [[ALLOC:%.*]] = alloca i8, align 1
@@ -591,9 +606,8 @@ f:
   ret i8 0
 }
 
-; CHECK: Function Attrs:
-; CHECK-SAME: readnone
 define i8 @readnone_caller2(i1 %c) {
+; CHECK: Function Attrs: nofree nosync nounwind readnone
 ; CHECK-LABEL: define {{[^@]+}}@readnone_caller2
 ; CHECK-SAME: (i1 [[C:%.*]])
 ; CHECK-NEXT:    [[R:%.*]] = call i8 @recursive_not_readnone_internal2(i8* undef, i1 [[C]])
@@ -603,23 +617,31 @@ define i8 @readnone_caller2(i1 %c) {
   ret i8 %r
 }
 
-; CHECK: Function Attrs:
-; CHECK-NOT: argmemonly
 define internal void @argmemonly_before_ipconstprop(i32* %p) argmemonly {
-; CHECK-LABEL: define {{[^@]+}}@argmemonly_before_ipconstprop()
-; CHECK-NEXT:    store i32 0, i32* @G, align 4
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@argmemonly_before_ipconstprop()
+; IS__TUNIT____-NEXT:    store i32 0, i32* @G, align 4
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@argmemonly_before_ipconstprop()
+; IS__CGSCC____-NEXT:    store i32 0, i32* @G, align 4
+; IS__CGSCC____-NEXT:    ret void
 ;
   store i32 0, i32* %p
   ret void
 }
 
-; CHECK: Function Attrs:
-; CHECK-NOT: argmemonly
 define void @argmemonky_caller() {
-; CHECK-LABEL: define {{[^@]+}}@argmemonky_caller()
-; CHECK-NEXT:    call void @argmemonly_before_ipconstprop()
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@argmemonky_caller()
+; IS__TUNIT____-NEXT:    call void @argmemonly_before_ipconstprop()
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@argmemonky_caller()
+; IS__CGSCC____-NEXT:    call void @argmemonly_before_ipconstprop()
+; IS__CGSCC____-NEXT:    ret void
 ;
   call void @argmemonly_before_ipconstprop(i32* @G)
   ret void
diff --git a/llvm/test/Transforms/Attributor/misc.ll b/llvm/test/Transforms/Attributor/misc.ll
index 5c8ddc16a9a0b..6101380c4bae5 100644
--- a/llvm/test/Transforms/Attributor/misc.ll
+++ b/llvm/test/Transforms/Attributor/misc.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -95,11 +95,19 @@ entry:
 
 define internal void @foo(i32* %a) {
 ;
-; CHECK-LABEL: define {{[^@]+}}@foo
-; CHECK-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[A:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    store i32 0, i32* [[A]], align 4
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@foo
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[A:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    store i32 0, i32* [[A]], align 4
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@foo
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[A:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    store i32 0, i32* [[A]], align 4
+; IS__CGSCC____-NEXT:    ret void
 ;
 entry:
   store i32 0, i32* %a
diff --git a/llvm/test/Transforms/Attributor/misc_crash.ll b/llvm/test/Transforms/Attributor/misc_crash.ll
index f8e10fa445454..afb3093aeeb77 100644
--- a/llvm/test/Transforms/Attributor/misc_crash.ll
+++ b/llvm/test/Transforms/Attributor/misc_crash.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -S %s | FileCheck %s
 ; RUN: opt -passes=attributor -S %s | FileCheck %s
 
@@ -6,8 +6,9 @@
 @var2 = internal global i32 0
 
 define i32 addrspace(1)* @foo(i32 addrspace(4)* %arg) {
+; CHECK: Function Attrs: nofree nosync nounwind readnone willreturn
 ; CHECK-LABEL: define {{[^@]+}}@foo
-; CHECK-SAME: (i32 addrspace(4)* nofree readnone [[ARG:%.*]]) #0
+; CHECK-SAME: (i32 addrspace(4)* nofree readnone [[ARG:%.*]])
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast i32 addrspace(4)* [[ARG]] to i32 addrspace(1)*
 ; CHECK-NEXT:    ret i32 addrspace(1)* [[TMP0]]
@@ -18,8 +19,9 @@ entry:
 }
 
 define i32* @func1() {
-; CHECK-LABEL: define {{[^@]+}}@func1() #0
-; CHECK-NEXT:    [[PTR:%.*]] = call i32* @func1a() #0
+; CHECK: Function Attrs: nofree nosync nounwind readnone willreturn
+; CHECK-LABEL: define {{[^@]+}}@func1()
+; CHECK-NEXT:    [[PTR:%.*]] = call i32* @func1a()
 ; CHECK-NEXT:    ret i32* [[PTR]]
 ;
   %ptr = call i32* @func1a([1 x i32]* @var1)
@@ -36,8 +38,9 @@ define internal i32* @func1a([1 x i32]* %arg) {
 ; UTC_ARGS: --enable
 
 define internal void @func2a(i32* %0) {
+; CHECK: Function Attrs: nofree nosync nounwind willreturn writeonly
 ; CHECK-LABEL: define {{[^@]+}}@func2a
-; CHECK-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[TMP0:%.*]]) #1
+; CHECK-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[TMP0:%.*]])
 ; CHECK-NEXT:    store i32 0, i32* @var2, align 4
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/Attributor/new_attributes.ll b/llvm/test/Transforms/Attributor/new_attributes.ll
index 5b6934b0c8997..f944db358f614 100644
--- a/llvm/test/Transforms/Attributor/new_attributes.ll
+++ b/llvm/test/Transforms/Attributor/new_attributes.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt < %s -attributor -attributor-annotate-decl-cs  -attributor-max-iterations=2 -S | FileCheck %s
 ; RUN: opt < %s -attributor -attributor-annotate-decl-cs  -attributor-max-iterations=3 -S | FileCheck %s
 ; RUN: opt < %s -attributor -attributor-annotate-decl-cs  -attributor-max-iterations=4 -S | FileCheck %s
@@ -13,26 +14,24 @@ declare i32 @foo1()
 declare i32 @foo2()
 declare i32 @foo3()
 
-; CHECK-NOT: Function
-; CHECK:      define internal i32 @bar() {
-; CHECK-NEXT:   %1 = call i32 @foo1()
-; CHECK-NEXT:   %2 = call i32 @foo2()
-; CHECK-NEXT:   %3 = call i32 @foo3()
-; CHECK-NEXT:   ret i32 undef
-; CHECK-NEXT: }
 define internal i32 @bar() {
+; CHECK-LABEL: define {{[^@]+}}@bar()
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @foo1()
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @foo2()
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @foo3()
+; CHECK-NEXT:    ret i32 undef
+;
   %1 = call i32 @foo1()
   %2 = call i32 @foo2()
   %3 = call i32 @foo3()
   ret i32 1
 }
 
-; CHECK-NOT: Function
-; CHECK:      define i32 @baz() {
-; CHECK-NEXT:   %1 = call i32 @bar()
-; CHECK-NEXT:   ret i32 0
-; CHECK-NEXT: }
 define i32 @baz() {
+; CHECK-LABEL: define {{[^@]+}}@baz()
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @bar()
+; CHECK-NEXT:    ret i32 0
+;
   %1 = call i32 @bar()
   ret i32 0
 }
diff --git a/llvm/test/Transforms/Attributor/noalias.ll b/llvm/test/Transforms/Attributor/noalias.ll
index 9831d52ca8487..71cf2263867cb 100644
--- a/llvm/test/Transforms/Attributor/noalias.ll
+++ b/llvm/test/Transforms/Attributor/noalias.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; TODO: The old pass manager cgscc run is disabled as it causes a crash on windows which is under investigation: http://lab.llvm.org:8011/builders/llvm-clang-x86_64-expensive-checks-win/builds/23151
@@ -42,9 +42,15 @@ define i8* @return_noalias(){
 }
 
 define void @nocapture(i8* %a){
-; CHECK-LABEL: define {{[^@]+}}@nocapture
-; CHECK-SAME: (i8* nocapture nofree readnone [[A:%.*]])
-; CHECK-NEXT:    ret void
+; NOT_CGSCC_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@nocapture
+; NOT_CGSCC_NPM-SAME: (i8* nocapture nofree readnone [[A:%.*]])
+; NOT_CGSCC_NPM-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@nocapture
+; IS__CGSCC____-SAME: (i8* nocapture nofree readnone [[A:%.*]])
+; IS__CGSCC____-NEXT:    ret void
 ;
   ret void
 }
@@ -98,6 +104,7 @@ define i8* @call_alias(){
 ; }
 
 define i8* @bar() nounwind uwtable {
+; CHECK: Function Attrs: nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@bar()
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call i8* (...) @baz()
 ; CHECK-NEXT:    ret i8* [[TMP1]]
@@ -107,6 +114,7 @@ define i8* @bar() nounwind uwtable {
 }
 
 define i8* @foo1(i32 %0) nounwind uwtable {
+; CHECK: Function Attrs: nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@foo1
 ; CHECK-SAME: (i32 [[TMP0:%.*]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP0]], 0
@@ -136,16 +144,26 @@ declare i8* @baz(...) nounwind uwtable
 
 ; Returning global pointer. Should not be noalias.
 define i8** @getter() {
-; CHECK-LABEL: define {{[^@]+}}@getter()
-; CHECK-NEXT:    ret i8** @G
+; NOT_CGSCC_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@getter()
+; NOT_CGSCC_NPM-NEXT:    ret i8** @G
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@getter()
+; IS__CGSCC____-NEXT:    ret i8** @G
 ;
   ret i8** @G
 }
 
 ; Returning global pointer. Should not be noalias.
 define i8** @calle1(){
-; CHECK-LABEL: define {{[^@]+}}@calle1()
-; CHECK-NEXT:    ret i8** @G
+; NOT_CGSCC_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@calle1()
+; NOT_CGSCC_NPM-NEXT:    ret i8** @G
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@calle1()
+; IS__CGSCC____-NEXT:    ret i8** @G
 ;
   %1 = call i8** @getter()
   ret i8** %1
@@ -155,6 +173,7 @@ define i8** @calle1(){
 declare noalias i8* @strdup(i8* nocapture) nounwind
 
 define i8* @test6() nounwind uwtable ssp {
+; CHECK: Function Attrs: nounwind ssp uwtable
 ; CHECK-LABEL: define {{[^@]+}}@test6()
 ; CHECK-NEXT:    [[X:%.*]] = alloca [2 x i8], align 1
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i8], [2 x i8]* [[X]], i64 0, i64 0
@@ -176,6 +195,7 @@ define i8* @test6() nounwind uwtable ssp {
 ; TEST 7
 
 define i8* @test7() nounwind {
+; CHECK: Function Attrs: nounwind
 ; CHECK-LABEL: define {{[^@]+}}@test7()
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[A:%.*]] = call noalias i8* @malloc(i64 4)
@@ -205,6 +225,7 @@ return:
 ; TEST 8
 
 define i8* @test8(i32* %0) nounwind uwtable {
+; CHECK: Function Attrs: nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@test8
 ; CHECK-SAME: (i32* [[TMP0:%.*]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call noalias i8* @malloc(i64 4)
@@ -476,11 +497,13 @@ define void @test13_use_alias(){
 
 ; TEST 14 i2p casts
 define internal i32 @p2i(i32* %arg) {
+; NOT_CGSCC_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
 ; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@p2i
 ; NOT_CGSCC_NPM-SAME: (i32* noalias nofree readnone [[ARG:%.*]])
 ; NOT_CGSCC_NPM-NEXT:    [[P2I:%.*]] = ptrtoint i32* [[ARG]] to i32
 ; NOT_CGSCC_NPM-NEXT:    ret i32 [[P2I]]
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@p2i
 ; IS__CGSCC____-SAME: (i32* nofree readnone [[ARG:%.*]])
 ; IS__CGSCC____-NEXT:    [[P2I:%.*]] = ptrtoint i32* [[ARG]] to i32
@@ -491,6 +514,7 @@ define internal i32 @p2i(i32* %arg) {
 }
 
 define i32 @i2p(i32* %arg) {
+; NOT_CGSCC_NPM: Function Attrs: nofree nosync nounwind readonly willreturn
 ; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@i2p
 ; NOT_CGSCC_NPM-SAME: (i32* nofree readonly [[ARG:%.*]])
 ; NOT_CGSCC_NPM-NEXT:    [[C:%.*]] = call i32 @p2i(i32* noalias nofree readnone [[ARG]])
@@ -499,6 +523,7 @@ define i32 @i2p(i32* %arg) {
 ; NOT_CGSCC_NPM-NEXT:    [[CALL:%.*]] = call i32 @ret(i32* nocapture nofree readonly align 4 [[BC]])
 ; NOT_CGSCC_NPM-NEXT:    ret i32 [[CALL]]
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readonly willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@i2p
 ; IS__CGSCC____-SAME: (i32* nofree readonly [[ARG:%.*]])
 ; IS__CGSCC____-NEXT:    [[C:%.*]] = call i32 @p2i(i32* noalias nofree readnone [[ARG]])
@@ -514,10 +539,17 @@ define i32 @i2p(i32* %arg) {
   ret i32 %call
 }
 define internal i32 @ret(i32* %arg) {
-; CHECK-LABEL: define {{[^@]+}}@ret
-; CHECK-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[ARG:%.*]])
-; CHECK-NEXT:    [[L:%.*]] = load i32, i32* [[ARG]], align 4
-; CHECK-NEXT:    ret i32 [[L]]
+; NOT_CGSCC_NPM: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@ret
+; NOT_CGSCC_NPM-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[ARG:%.*]])
+; NOT_CGSCC_NPM-NEXT:    [[L:%.*]] = load i32, i32* [[ARG]], align 4
+; NOT_CGSCC_NPM-NEXT:    ret i32 [[L]]
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@ret
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[ARG:%.*]])
+; IS__CGSCC____-NEXT:    [[L:%.*]] = load i32, i32* [[ARG]], align 4
+; IS__CGSCC____-NEXT:    ret i32 [[L]]
 ;
   %l = load i32, i32* %arg
   ret i32 %l
@@ -591,26 +623,41 @@ declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture)
 @alias_of_p = external global i32*
 
 define void @make_alias(i32* %p) {
-; CHECK-LABEL: define {{[^@]+}}@make_alias
-; CHECK-SAME: (i32* nofree writeonly [[P:%.*]])
-; CHECK-NEXT:    store i32* [[P]], i32** @alias_of_p, align 8
-; CHECK-NEXT:    ret void
+; NOT_CGSCC_NPM: Function Attrs: nofree nosync nounwind willreturn writeonly
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@make_alias
+; NOT_CGSCC_NPM-SAME: (i32* nofree writeonly [[P:%.*]])
+; NOT_CGSCC_NPM-NEXT:    store i32* [[P]], i32** @alias_of_p, align 8
+; NOT_CGSCC_NPM-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@make_alias
+; IS__CGSCC____-SAME: (i32* nofree writeonly [[P:%.*]])
+; IS__CGSCC____-NEXT:    store i32* [[P]], i32** @alias_of_p, align 8
+; IS__CGSCC____-NEXT:    ret void
 ;
   store i32* %p, i32** @alias_of_p
   ret void
 }
 
 define void @only_store(i32* %p) {
-; CHECK-LABEL: define {{[^@]+}}@only_store
-; CHECK-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[P:%.*]])
-; CHECK-NEXT:    store i32 0, i32* [[P]], align 4
-; CHECK-NEXT:    ret void
+; NOT_CGSCC_NPM: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@only_store
+; NOT_CGSCC_NPM-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[P:%.*]])
+; NOT_CGSCC_NPM-NEXT:    store i32 0, i32* [[P]], align 4
+; NOT_CGSCC_NPM-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@only_store
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[P:%.*]])
+; IS__CGSCC____-NEXT:    store i32 0, i32* [[P]], align 4
+; IS__CGSCC____-NEXT:    ret void
 ;
   store i32 0, i32* %p
   ret void
 }
 
 define void @test15_caller(i32* noalias %p, i32 %c) {
+; NOT_CGSCC_NPM: Function Attrs: nofree nosync nounwind willreturn writeonly
 ; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@test15_caller
 ; NOT_CGSCC_NPM-SAME: (i32* noalias nofree writeonly [[P:%.*]], i32 [[C:%.*]])
 ; NOT_CGSCC_NPM-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[C]], 0
@@ -622,6 +669,7 @@ define void @test15_caller(i32* noalias %p, i32 %c) {
 ; NOT_CGSCC_NPM-NEXT:    tail call void @make_alias(i32* nofree writeonly [[P]])
 ; NOT_CGSCC_NPM-NEXT:    ret void
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn writeonly
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@test15_caller
 ; IS__CGSCC____-SAME: (i32* noalias nofree writeonly [[P:%.*]], i32 [[C:%.*]])
 ; IS__CGSCC____-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[C]], 0
@@ -666,6 +714,7 @@ if.end:
 ;        Therefore, only one of the two conditions of if statementes will be fulfilled.
 
 define internal void @test16_sub(i32* noalias %p, i32 %c1, i32 %c2) {
+; NOT_CGSCC_NPM: Function Attrs: nofree nosync nounwind willreturn writeonly
 ; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@test16_sub
 ; NOT_CGSCC_NPM-SAME: (i32* noalias nofree writeonly [[P:%.*]], i32 [[C1:%.*]], i32 [[C2:%.*]])
 ; NOT_CGSCC_NPM-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[C1]], 0
@@ -683,6 +732,7 @@ define internal void @test16_sub(i32* noalias %p, i32 %c1, i32 %c2) {
 ; NOT_CGSCC_NPM:       if.end3:
 ; NOT_CGSCC_NPM-NEXT:    ret void
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn writeonly
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@test16_sub
 ; IS__CGSCC____-SAME: (i32* noalias nofree writeonly [[P:%.*]], i32 [[C1:%.*]], i32 [[C2:%.*]])
 ; IS__CGSCC____-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[C1]], 0
@@ -721,10 +771,17 @@ if.end3:
 }
 
 define void @test16_caller(i32* %p, i32 %c) {
-; CHECK-LABEL: define {{[^@]+}}@test16_caller
-; CHECK-SAME: (i32* nofree writeonly [[P:%.*]], i32 [[C:%.*]])
-; CHECK-NEXT:    tail call void @test16_sub(i32* noalias nofree writeonly [[P]], i32 [[C]], i32 [[C]])
-; CHECK-NEXT:    ret void
+; NOT_CGSCC_NPM: Function Attrs: nofree nosync nounwind willreturn writeonly
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@test16_caller
+; NOT_CGSCC_NPM-SAME: (i32* nofree writeonly [[P:%.*]], i32 [[C:%.*]])
+; NOT_CGSCC_NPM-NEXT:    tail call void @test16_sub(i32* noalias nofree writeonly [[P]], i32 [[C]], i32 [[C]])
+; NOT_CGSCC_NPM-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test16_caller
+; IS__CGSCC____-SAME: (i32* nofree writeonly [[P:%.*]], i32 [[C:%.*]])
+; IS__CGSCC____-NEXT:    tail call void @test16_sub(i32* noalias nofree writeonly [[P]], i32 [[C]], i32 [[C]])
+; IS__CGSCC____-NEXT:    ret void
 ;
   tail call void @test16_sub(i32* %p, i32 %c, i32 %c)
   ret void
diff --git a/llvm/test/Transforms/Attributor/nocapture-1.ll b/llvm/test/Transforms/Attributor/nocapture-1.ll
index 559c626b5741a..45606b09bec84 100644
--- a/llvm/test/Transforms/Attributor/nocapture-1.ll
+++ b/llvm/test/Transforms/Attributor/nocapture-1.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -7,45 +7,78 @@
 @g = global i32* null		; <i32**> [#uses=1]
 
 define i32* @c1(i32* %q) {
-; CHECK-LABEL: define {{[^@]+}}@c1
-; CHECK-SAME: (i32* nofree readnone returned "no-capture-maybe-returned" [[Q:%.*]])
-; CHECK-NEXT:    ret i32* [[Q]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@c1
+; IS__TUNIT____-SAME: (i32* nofree readnone returned "no-capture-maybe-returned" [[Q:%.*]])
+; IS__TUNIT____-NEXT:    ret i32* [[Q]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@c1
+; IS__CGSCC____-SAME: (i32* nofree readnone returned "no-capture-maybe-returned" [[Q:%.*]])
+; IS__CGSCC____-NEXT:    ret i32* [[Q]]
 ;
   ret i32* %q
 }
 
 ; It would also be acceptable to mark %q as readnone. Update @c3 too.
 define void @c2(i32* %q) {
-; CHECK-LABEL: define {{[^@]+}}@c2
-; CHECK-SAME: (i32* nofree writeonly [[Q:%.*]])
-; CHECK-NEXT:    store i32* [[Q]], i32** @g, align 8
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@c2
+; IS__TUNIT____-SAME: (i32* nofree writeonly [[Q:%.*]])
+; IS__TUNIT____-NEXT:    store i32* [[Q]], i32** @g, align 8
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@c2
+; IS__CGSCC____-SAME: (i32* nofree writeonly [[Q:%.*]])
+; IS__CGSCC____-NEXT:    store i32* [[Q]], i32** @g, align 8
+; IS__CGSCC____-NEXT:    ret void
 ;
   store i32* %q, i32** @g
   ret void
 }
 
 define void @c3(i32* %q) {
-; CHECK-LABEL: define {{[^@]+}}@c3
-; CHECK-SAME: (i32* nofree writeonly [[Q:%.*]])
-; CHECK-NEXT:    call void @c2(i32* nofree writeonly [[Q]])
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@c3
+; IS__TUNIT____-SAME: (i32* nofree writeonly [[Q:%.*]])
+; IS__TUNIT____-NEXT:    call void @c2(i32* nofree writeonly [[Q]])
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@c3
+; IS__CGSCC____-SAME: (i32* nofree writeonly [[Q:%.*]])
+; IS__CGSCC____-NEXT:    call void @c2(i32* nofree writeonly [[Q]])
+; IS__CGSCC____-NEXT:    ret void
 ;
   call void @c2(i32* %q)
   ret void
 }
 
 define i1 @c4(i32* %q, i32 %bitno) {
-; CHECK-LABEL: define {{[^@]+}}@c4
-; CHECK-SAME: (i32* nofree readnone [[Q:%.*]], i32 [[BITNO:%.*]])
-; CHECK-NEXT:    [[TMP:%.*]] = ptrtoint i32* [[Q]] to i32
-; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP]], [[BITNO]]
-; CHECK-NEXT:    [[BIT:%.*]] = trunc i32 [[TMP2]] to i1
-; CHECK-NEXT:    br i1 [[BIT]], label [[L1:%.*]], label [[L0:%.*]]
-; CHECK:       l0:
-; CHECK-NEXT:    ret i1 false
-; CHECK:       l1:
-; CHECK-NEXT:    ret i1 true
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@c4
+; IS__TUNIT____-SAME: (i32* nofree readnone [[Q:%.*]], i32 [[BITNO:%.*]])
+; IS__TUNIT____-NEXT:    [[TMP:%.*]] = ptrtoint i32* [[Q]] to i32
+; IS__TUNIT____-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP]], [[BITNO]]
+; IS__TUNIT____-NEXT:    [[BIT:%.*]] = trunc i32 [[TMP2]] to i1
+; IS__TUNIT____-NEXT:    br i1 [[BIT]], label [[L1:%.*]], label [[L0:%.*]]
+; IS__TUNIT____:       l0:
+; IS__TUNIT____-NEXT:    ret i1 false
+; IS__TUNIT____:       l1:
+; IS__TUNIT____-NEXT:    ret i1 true
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@c4
+; IS__CGSCC____-SAME: (i32* nofree readnone [[Q:%.*]], i32 [[BITNO:%.*]])
+; IS__CGSCC____-NEXT:    [[TMP:%.*]] = ptrtoint i32* [[Q]] to i32
+; IS__CGSCC____-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP]], [[BITNO]]
+; IS__CGSCC____-NEXT:    [[BIT:%.*]] = trunc i32 [[TMP2]] to i1
+; IS__CGSCC____-NEXT:    br i1 [[BIT]], label [[L1:%.*]], label [[L0:%.*]]
+; IS__CGSCC____:       l0:
+; IS__CGSCC____-NEXT:    ret i1 false
+; IS__CGSCC____:       l1:
+; IS__CGSCC____-NEXT:    ret i1 true
 ;
   %tmp = ptrtoint i32* %q to i32
   %tmp2 = lshr i32 %tmp, %bitno
@@ -59,16 +92,29 @@ l1:
 
 ; c4b is c4 but without the escaping part
 define i1 @c4b(i32* %q, i32 %bitno) {
-; CHECK-LABEL: define {{[^@]+}}@c4b
-; CHECK-SAME: (i32* nocapture nofree readnone [[Q:%.*]], i32 [[BITNO:%.*]])
-; CHECK-NEXT:    [[TMP:%.*]] = ptrtoint i32* [[Q]] to i32
-; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP]], [[BITNO]]
-; CHECK-NEXT:    [[BIT:%.*]] = trunc i32 [[TMP2]] to i1
-; CHECK-NEXT:    br i1 [[BIT]], label [[L1:%.*]], label [[L0:%.*]]
-; CHECK:       l0:
-; CHECK-NEXT:    ret i1 false
-; CHECK:       l1:
-; CHECK-NEXT:    ret i1 false
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@c4b
+; IS__TUNIT____-SAME: (i32* nocapture nofree readnone [[Q:%.*]], i32 [[BITNO:%.*]])
+; IS__TUNIT____-NEXT:    [[TMP:%.*]] = ptrtoint i32* [[Q]] to i32
+; IS__TUNIT____-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP]], [[BITNO]]
+; IS__TUNIT____-NEXT:    [[BIT:%.*]] = trunc i32 [[TMP2]] to i1
+; IS__TUNIT____-NEXT:    br i1 [[BIT]], label [[L1:%.*]], label [[L0:%.*]]
+; IS__TUNIT____:       l0:
+; IS__TUNIT____-NEXT:    ret i1 false
+; IS__TUNIT____:       l1:
+; IS__TUNIT____-NEXT:    ret i1 false
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@c4b
+; IS__CGSCC____-SAME: (i32* nocapture nofree readnone [[Q:%.*]], i32 [[BITNO:%.*]])
+; IS__CGSCC____-NEXT:    [[TMP:%.*]] = ptrtoint i32* [[Q]] to i32
+; IS__CGSCC____-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP]], [[BITNO]]
+; IS__CGSCC____-NEXT:    [[BIT:%.*]] = trunc i32 [[TMP2]] to i1
+; IS__CGSCC____-NEXT:    br i1 [[BIT]], label [[L1:%.*]], label [[L0:%.*]]
+; IS__CGSCC____:       l0:
+; IS__CGSCC____-NEXT:    ret i1 false
+; IS__CGSCC____:       l1:
+; IS__CGSCC____-NEXT:    ret i1 false
 ;
   %tmp = ptrtoint i32* %q to i32
   %tmp2 = lshr i32 %tmp, %bitno
@@ -83,14 +129,25 @@ l1:
 @lookup_table = global [2 x i1] [ i1 0, i1 1 ]
 
 define i1 @c5(i32* %q, i32 %bitno) {
-; CHECK-LABEL: define {{[^@]+}}@c5
-; CHECK-SAME: (i32* nofree readonly [[Q:%.*]], i32 [[BITNO:%.*]])
-; CHECK-NEXT:    [[TMP:%.*]] = ptrtoint i32* [[Q]] to i32
-; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP]], [[BITNO]]
-; CHECK-NEXT:    [[BIT:%.*]] = and i32 [[TMP2]], 1
-; CHECK-NEXT:    [[LOOKUP:%.*]] = getelementptr [2 x i1], [2 x i1]* @lookup_table, i32 0, i32 [[BIT]]
-; CHECK-NEXT:    [[VAL:%.*]] = load i1, i1* [[LOOKUP]], align 1
-; CHECK-NEXT:    ret i1 [[VAL]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@c5
+; IS__TUNIT____-SAME: (i32* nofree readonly [[Q:%.*]], i32 [[BITNO:%.*]])
+; IS__TUNIT____-NEXT:    [[TMP:%.*]] = ptrtoint i32* [[Q]] to i32
+; IS__TUNIT____-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP]], [[BITNO]]
+; IS__TUNIT____-NEXT:    [[BIT:%.*]] = and i32 [[TMP2]], 1
+; IS__TUNIT____-NEXT:    [[LOOKUP:%.*]] = getelementptr [2 x i1], [2 x i1]* @lookup_table, i32 0, i32 [[BIT]]
+; IS__TUNIT____-NEXT:    [[VAL:%.*]] = load i1, i1* [[LOOKUP]], align 1
+; IS__TUNIT____-NEXT:    ret i1 [[VAL]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@c5
+; IS__CGSCC____-SAME: (i32* nofree readonly [[Q:%.*]], i32 [[BITNO:%.*]])
+; IS__CGSCC____-NEXT:    [[TMP:%.*]] = ptrtoint i32* [[Q]] to i32
+; IS__CGSCC____-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP]], [[BITNO]]
+; IS__CGSCC____-NEXT:    [[BIT:%.*]] = and i32 [[TMP2]], 1
+; IS__CGSCC____-NEXT:    [[LOOKUP:%.*]] = getelementptr [2 x i1], [2 x i1]* @lookup_table, i32 0, i32 [[BIT]]
+; IS__CGSCC____-NEXT:    [[VAL:%.*]] = load i1, i1* [[LOOKUP]], align 1
+; IS__CGSCC____-NEXT:    ret i1 [[VAL]]
 ;
   %tmp = ptrtoint i32* %q to i32
   %tmp2 = lshr i32 %tmp, %bitno
@@ -104,6 +161,7 @@ define i1 @c5(i32* %q, i32 %bitno) {
 declare void @throw_if_bit_set(i8*, i8) readonly
 
 define i1 @c6(i8* %q, i8 %bit) personality i32 (...)* @__gxx_personality_v0 {
+; CHECK: Function Attrs: nounwind readonly
 ; CHECK-LABEL: define {{[^@]+}}@c6
 ; CHECK-SAME: (i8* readonly [[Q:%.*]], i8 [[BIT:%.*]]) #4 personality i32 (...)* @__gxx_personality_v0
 ; CHECK-NEXT:    invoke void @throw_if_bit_set(i8* readonly [[Q]], i8 [[BIT]])
@@ -128,13 +186,23 @@ ret1:
 declare i32 @__gxx_personality_v0(...)
 
 define i1* @lookup_bit(i32* %q, i32 %bitno) readnone nounwind {
-; CHECK-LABEL: define {{[^@]+}}@lookup_bit
-; CHECK-SAME: (i32* nofree readnone [[Q:%.*]], i32 [[BITNO:%.*]])
-; CHECK-NEXT:    [[TMP:%.*]] = ptrtoint i32* [[Q]] to i32
-; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP]], [[BITNO]]
-; CHECK-NEXT:    [[BIT:%.*]] = and i32 [[TMP2]], 1
-; CHECK-NEXT:    [[LOOKUP:%.*]] = getelementptr [2 x i1], [2 x i1]* @lookup_table, i32 0, i32 [[BIT]]
-; CHECK-NEXT:    ret i1* [[LOOKUP]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@lookup_bit
+; IS__TUNIT____-SAME: (i32* nofree readnone [[Q:%.*]], i32 [[BITNO:%.*]])
+; IS__TUNIT____-NEXT:    [[TMP:%.*]] = ptrtoint i32* [[Q]] to i32
+; IS__TUNIT____-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP]], [[BITNO]]
+; IS__TUNIT____-NEXT:    [[BIT:%.*]] = and i32 [[TMP2]], 1
+; IS__TUNIT____-NEXT:    [[LOOKUP:%.*]] = getelementptr [2 x i1], [2 x i1]* @lookup_table, i32 0, i32 [[BIT]]
+; IS__TUNIT____-NEXT:    ret i1* [[LOOKUP]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@lookup_bit
+; IS__CGSCC____-SAME: (i32* nofree readnone [[Q:%.*]], i32 [[BITNO:%.*]])
+; IS__CGSCC____-NEXT:    [[TMP:%.*]] = ptrtoint i32* [[Q]] to i32
+; IS__CGSCC____-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP]], [[BITNO]]
+; IS__CGSCC____-NEXT:    [[BIT:%.*]] = and i32 [[TMP2]], 1
+; IS__CGSCC____-NEXT:    [[LOOKUP:%.*]] = getelementptr [2 x i1], [2 x i1]* @lookup_table, i32 0, i32 [[BIT]]
+; IS__CGSCC____-NEXT:    ret i1* [[LOOKUP]]
 ;
   %tmp = ptrtoint i32* %q to i32
   %tmp2 = lshr i32 %tmp, %bitno
@@ -144,11 +212,19 @@ define i1* @lookup_bit(i32* %q, i32 %bitno) readnone nounwind {
 }
 
 define i1 @c7(i32* %q, i32 %bitno) {
-; CHECK-LABEL: define {{[^@]+}}@c7
-; CHECK-SAME: (i32* nofree readonly [[Q:%.*]], i32 [[BITNO:%.*]])
-; CHECK-NEXT:    [[PTR:%.*]] = call i1* @lookup_bit(i32* noalias nofree readnone [[Q]], i32 [[BITNO]])
-; CHECK-NEXT:    [[VAL:%.*]] = load i1, i1* [[PTR]], align 1
-; CHECK-NEXT:    ret i1 [[VAL]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@c7
+; IS__TUNIT____-SAME: (i32* nofree readonly [[Q:%.*]], i32 [[BITNO:%.*]])
+; IS__TUNIT____-NEXT:    [[PTR:%.*]] = call i1* @lookup_bit(i32* noalias nofree readnone [[Q]], i32 [[BITNO]])
+; IS__TUNIT____-NEXT:    [[VAL:%.*]] = load i1, i1* [[PTR]], align 1
+; IS__TUNIT____-NEXT:    ret i1 [[VAL]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@c7
+; IS__CGSCC____-SAME: (i32* nofree readonly [[Q:%.*]], i32 [[BITNO:%.*]])
+; IS__CGSCC____-NEXT:    [[PTR:%.*]] = call i1* @lookup_bit(i32* noalias nofree readnone [[Q]], i32 [[BITNO]])
+; IS__CGSCC____-NEXT:    [[VAL:%.*]] = load i1, i1* [[PTR]], align 1
+; IS__CGSCC____-NEXT:    ret i1 [[VAL]]
 ;
   %ptr = call i1* @lookup_bit(i32* %q, i32 %bitno)
   %val = load i1, i1* %ptr
@@ -157,19 +233,35 @@ define i1 @c7(i32* %q, i32 %bitno) {
 
 
 define i32 @nc1(i32* %q, i32* %p, i1 %b) {
-; CHECK-LABEL: define {{[^@]+}}@nc1
-; CHECK-SAME: (i32* nofree [[Q:%.*]], i32* nocapture nofree [[P:%.*]], i1 [[B:%.*]])
-; CHECK-NEXT:  e:
-; CHECK-NEXT:    br label [[L:%.*]]
-; CHECK:       l:
-; CHECK-NEXT:    [[X:%.*]] = phi i32* [ [[P]], [[E:%.*]] ]
-; CHECK-NEXT:    [[Y:%.*]] = phi i32* [ [[Q]], [[E]] ]
-; CHECK-NEXT:    [[TMP:%.*]] = bitcast i32* [[X]] to i32*
-; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[B]], i32* [[TMP]], i32* [[Y]]
-; CHECK-NEXT:    [[VAL:%.*]] = load i32, i32* [[TMP2]], align 4
-; CHECK-NEXT:    store i32 0, i32* [[TMP]], align 4
-; CHECK-NEXT:    store i32* [[Y]], i32** @g, align 8
-; CHECK-NEXT:    ret i32 [[VAL]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@nc1
+; IS__TUNIT____-SAME: (i32* nofree [[Q:%.*]], i32* nocapture nofree [[P:%.*]], i1 [[B:%.*]])
+; IS__TUNIT____-NEXT:  e:
+; IS__TUNIT____-NEXT:    br label [[L:%.*]]
+; IS__TUNIT____:       l:
+; IS__TUNIT____-NEXT:    [[X:%.*]] = phi i32* [ [[P]], [[E:%.*]] ]
+; IS__TUNIT____-NEXT:    [[Y:%.*]] = phi i32* [ [[Q]], [[E]] ]
+; IS__TUNIT____-NEXT:    [[TMP:%.*]] = bitcast i32* [[X]] to i32*
+; IS__TUNIT____-NEXT:    [[TMP2:%.*]] = select i1 [[B]], i32* [[TMP]], i32* [[Y]]
+; IS__TUNIT____-NEXT:    [[VAL:%.*]] = load i32, i32* [[TMP2]], align 4
+; IS__TUNIT____-NEXT:    store i32 0, i32* [[TMP]], align 4
+; IS__TUNIT____-NEXT:    store i32* [[Y]], i32** @g, align 8
+; IS__TUNIT____-NEXT:    ret i32 [[VAL]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@nc1
+; IS__CGSCC____-SAME: (i32* nofree [[Q:%.*]], i32* nocapture nofree [[P:%.*]], i1 [[B:%.*]])
+; IS__CGSCC____-NEXT:  e:
+; IS__CGSCC____-NEXT:    br label [[L:%.*]]
+; IS__CGSCC____:       l:
+; IS__CGSCC____-NEXT:    [[X:%.*]] = phi i32* [ [[P]], [[E:%.*]] ]
+; IS__CGSCC____-NEXT:    [[Y:%.*]] = phi i32* [ [[Q]], [[E]] ]
+; IS__CGSCC____-NEXT:    [[TMP:%.*]] = bitcast i32* [[X]] to i32*
+; IS__CGSCC____-NEXT:    [[TMP2:%.*]] = select i1 [[B]], i32* [[TMP]], i32* [[Y]]
+; IS__CGSCC____-NEXT:    [[VAL:%.*]] = load i32, i32* [[TMP2]], align 4
+; IS__CGSCC____-NEXT:    store i32 0, i32* [[TMP]], align 4
+; IS__CGSCC____-NEXT:    store i32* [[Y]], i32** @g, align 8
+; IS__CGSCC____-NEXT:    ret i32 [[VAL]]
 ;
 e:
   br label %l
@@ -185,19 +277,35 @@ l:
 }
 
 define i32 @nc1_addrspace(i32* %q, i32 addrspace(1)* %p, i1 %b) {
-; CHECK-LABEL: define {{[^@]+}}@nc1_addrspace
-; CHECK-SAME: (i32* nofree [[Q:%.*]], i32 addrspace(1)* nocapture nofree [[P:%.*]], i1 [[B:%.*]])
-; CHECK-NEXT:  e:
-; CHECK-NEXT:    br label [[L:%.*]]
-; CHECK:       l:
-; CHECK-NEXT:    [[X:%.*]] = phi i32 addrspace(1)* [ [[P]], [[E:%.*]] ]
-; CHECK-NEXT:    [[Y:%.*]] = phi i32* [ [[Q]], [[E]] ]
-; CHECK-NEXT:    [[TMP:%.*]] = addrspacecast i32 addrspace(1)* [[X]] to i32*
-; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[B]], i32* [[TMP]], i32* [[Y]]
-; CHECK-NEXT:    [[VAL:%.*]] = load i32, i32* [[TMP2]], align 4
-; CHECK-NEXT:    store i32 0, i32* [[TMP]], align 4
-; CHECK-NEXT:    store i32* [[Y]], i32** @g, align 8
-; CHECK-NEXT:    ret i32 [[VAL]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@nc1_addrspace
+; IS__TUNIT____-SAME: (i32* nofree [[Q:%.*]], i32 addrspace(1)* nocapture nofree [[P:%.*]], i1 [[B:%.*]])
+; IS__TUNIT____-NEXT:  e:
+; IS__TUNIT____-NEXT:    br label [[L:%.*]]
+; IS__TUNIT____:       l:
+; IS__TUNIT____-NEXT:    [[X:%.*]] = phi i32 addrspace(1)* [ [[P]], [[E:%.*]] ]
+; IS__TUNIT____-NEXT:    [[Y:%.*]] = phi i32* [ [[Q]], [[E]] ]
+; IS__TUNIT____-NEXT:    [[TMP:%.*]] = addrspacecast i32 addrspace(1)* [[X]] to i32*
+; IS__TUNIT____-NEXT:    [[TMP2:%.*]] = select i1 [[B]], i32* [[TMP]], i32* [[Y]]
+; IS__TUNIT____-NEXT:    [[VAL:%.*]] = load i32, i32* [[TMP2]], align 4
+; IS__TUNIT____-NEXT:    store i32 0, i32* [[TMP]], align 4
+; IS__TUNIT____-NEXT:    store i32* [[Y]], i32** @g, align 8
+; IS__TUNIT____-NEXT:    ret i32 [[VAL]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@nc1_addrspace
+; IS__CGSCC____-SAME: (i32* nofree [[Q:%.*]], i32 addrspace(1)* nocapture nofree [[P:%.*]], i1 [[B:%.*]])
+; IS__CGSCC____-NEXT:  e:
+; IS__CGSCC____-NEXT:    br label [[L:%.*]]
+; IS__CGSCC____:       l:
+; IS__CGSCC____-NEXT:    [[X:%.*]] = phi i32 addrspace(1)* [ [[P]], [[E:%.*]] ]
+; IS__CGSCC____-NEXT:    [[Y:%.*]] = phi i32* [ [[Q]], [[E]] ]
+; IS__CGSCC____-NEXT:    [[TMP:%.*]] = addrspacecast i32 addrspace(1)* [[X]] to i32*
+; IS__CGSCC____-NEXT:    [[TMP2:%.*]] = select i1 [[B]], i32* [[TMP]], i32* [[Y]]
+; IS__CGSCC____-NEXT:    [[VAL:%.*]] = load i32, i32* [[TMP2]], align 4
+; IS__CGSCC____-NEXT:    store i32 0, i32* [[TMP]], align 4
+; IS__CGSCC____-NEXT:    store i32* [[Y]], i32** @g, align 8
+; IS__CGSCC____-NEXT:    ret i32 [[VAL]]
 ;
 e:
   br label %l
@@ -213,10 +321,17 @@ l:
 }
 
 define void @nc2(i32* %p, i32* %q) {
-; CHECK-LABEL: define {{[^@]+}}@nc2
-; CHECK-SAME: (i32* nocapture nofree [[P:%.*]], i32* nofree [[Q:%.*]])
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @nc1(i32* nofree [[Q]], i32* nocapture nofree [[P]], i1 false)
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@nc2
+; IS__TUNIT____-SAME: (i32* nocapture nofree [[P:%.*]], i32* nofree [[Q:%.*]])
+; IS__TUNIT____-NEXT:    [[TMP1:%.*]] = call i32 @nc1(i32* nofree [[Q]], i32* nocapture nofree [[P]], i1 false)
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@nc2
+; IS__CGSCC____-SAME: (i32* nocapture nofree [[P:%.*]], i32* nofree [[Q:%.*]])
+; IS__CGSCC____-NEXT:    [[TMP1:%.*]] = call i32 @nc1(i32* nofree [[Q]], i32* nocapture nofree [[P]], i1 false)
+; IS__CGSCC____-NEXT:    ret void
 ;
   %1 = call i32 @nc1(i32* %q, i32* %p, i1 0)		; <i32> [#uses=0]
   ret void
@@ -237,6 +352,7 @@ define void @nc3(void ()* %p) {
 ; FIXME: readonly and nocapture missing on the pointer.
 declare void @external(i8* readonly) nounwind argmemonly
 define void @nc4(i8* %p) {
+; CHECK: Function Attrs: argmemonly nounwind
 ; CHECK-LABEL: define {{[^@]+}}@nc4
 ; CHECK-SAME: (i8* [[P:%.*]])
 ; CHECK-NEXT:    call void @external(i8* readonly [[P]])
@@ -259,6 +375,7 @@ define void @nc5(void (i8*)* %f, i8* %p) {
 
 ; It would be acceptable to add readnone to %y1_1 and %y1_2.
 define void @test1_1(i8* %x1_1, i8* %y1_1, i1 %c) {
+; CHECK: Function Attrs: nofree nosync nounwind writeonly
 ; CHECK-LABEL: define {{[^@]+}}@test1_1
 ; CHECK-SAME: (i8* nocapture nofree readnone [[X1_1:%.*]], i8* nocapture nofree readnone [[Y1_1:%.*]], i1 [[C:%.*]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i8* @test1_2(i8* noalias nocapture nofree readnone undef, i8* noalias nofree readnone "no-capture-maybe-returned" [[Y1_1]], i1 [[C]])
@@ -271,6 +388,7 @@ define void @test1_1(i8* %x1_1, i8* %y1_1, i1 %c) {
 }
 
 define i8* @test1_2(i8* %x1_2, i8* %y1_2, i1 %c) {
+; CHECK: Function Attrs: nofree nosync nounwind writeonly
 ; CHECK-LABEL: define {{[^@]+}}@test1_2
 ; CHECK-SAME: (i8* nocapture nofree readnone [[X1_2:%.*]], i8* nofree readnone returned "no-capture-maybe-returned" [[Y1_2:%.*]], i1 [[C:%.*]])
 ; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
@@ -291,9 +409,15 @@ f:
 }
 
 define void @test2(i8* %x2) {
-; CHECK-LABEL: define {{[^@]+}}@test2
-; CHECK-SAME: (i8* nocapture nofree readnone [[X2:%.*]])
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree noreturn nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test2
+; IS__TUNIT____-SAME: (i8* nocapture nofree readnone [[X2:%.*]])
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test2
+; IS__CGSCC____-SAME: (i8* nocapture nofree readnone [[X2:%.*]])
+; IS__CGSCC____-NEXT:    unreachable
 ;
   call void @test2(i8* %x2)
   store i32* null, i32** @g
@@ -301,9 +425,15 @@ define void @test2(i8* %x2) {
 }
 
 define void @test3(i8* %x3, i8* %y3, i8* %z3) {
-; CHECK-LABEL: define {{[^@]+}}@test3
-; CHECK-SAME: (i8* nocapture nofree readnone [[X3:%.*]], i8* nocapture nofree readnone [[Y3:%.*]], i8* nocapture nofree readnone [[Z3:%.*]])
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree noreturn nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test3
+; IS__TUNIT____-SAME: (i8* nocapture nofree readnone [[X3:%.*]], i8* nocapture nofree readnone [[Y3:%.*]], i8* nocapture nofree readnone [[Z3:%.*]])
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test3
+; IS__CGSCC____-SAME: (i8* nocapture nofree readnone [[X3:%.*]], i8* nocapture nofree readnone [[Y3:%.*]], i8* nocapture nofree readnone [[Z3:%.*]])
+; IS__CGSCC____-NEXT:    unreachable
 ;
   call void @test3(i8* %z3, i8* %y3, i8* %x3)
   store i32* null, i32** @g
@@ -311,6 +441,7 @@ define void @test3(i8* %x3, i8* %y3, i8* %z3) {
 }
 
 define void @test4_1(i8* %x4_1, i1 %c) {
+; CHECK: Function Attrs: nofree nosync nounwind writeonly
 ; CHECK-LABEL: define {{[^@]+}}@test4_1
 ; CHECK-SAME: (i8* nocapture nofree readnone [[X4_1:%.*]], i1 [[C:%.*]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i8* @test4_2(i8* noalias nocapture nofree readnone undef, i8* noalias nofree readnone "no-capture-maybe-returned" [[X4_1]], i8* noalias nocapture nofree readnone undef, i1 [[C]])
@@ -323,6 +454,7 @@ define void @test4_1(i8* %x4_1, i1 %c) {
 }
 
 define i8* @test4_2(i8* %x4_2, i8* %y4_2, i8* %z4_2, i1 %c) {
+; CHECK: Function Attrs: nofree nosync nounwind writeonly
 ; CHECK-LABEL: define {{[^@]+}}@test4_2
 ; CHECK-SAME: (i8* nocapture nofree readnone [[X4_2:%.*]], i8* nofree readnone returned "no-capture-maybe-returned" [[Y4_2:%.*]], i8* nocapture nofree readnone [[Z4_2:%.*]], i1 [[C:%.*]])
 ; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]]
@@ -371,42 +503,72 @@ define void @test6_2(i8* %x6_2, i8* %y6_2, i8* %z6_2) {
 }
 
 define void @test_cmpxchg(i32* %p) {
-; CHECK-LABEL: define {{[^@]+}}@test_cmpxchg
-; CHECK-SAME: (i32* nocapture nofree nonnull dereferenceable(4) [[P:%.*]])
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[P]], i32 0, i32 1 acquire monotonic
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: argmemonly nofree nounwind willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test_cmpxchg
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull dereferenceable(4) [[P:%.*]])
+; IS__TUNIT____-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[P]], i32 0, i32 1 acquire monotonic
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nounwind willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test_cmpxchg
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull dereferenceable(4) [[P:%.*]])
+; IS__CGSCC____-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[P]], i32 0, i32 1 acquire monotonic
+; IS__CGSCC____-NEXT:    ret void
 ;
   cmpxchg i32* %p, i32 0, i32 1 acquire monotonic
   ret void
 }
 
 define void @test_cmpxchg_ptr(i32** %p, i32* %q) {
-; CHECK-LABEL: define {{[^@]+}}@test_cmpxchg_ptr
-; CHECK-SAME: (i32** nocapture nofree nonnull dereferenceable(8) [[P:%.*]], i32* nofree [[Q:%.*]])
-; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32** [[P]], i32* null, i32* [[Q]] acquire monotonic
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: argmemonly nofree nounwind willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test_cmpxchg_ptr
+; IS__TUNIT____-SAME: (i32** nocapture nofree nonnull dereferenceable(8) [[P:%.*]], i32* nofree [[Q:%.*]])
+; IS__TUNIT____-NEXT:    [[TMP1:%.*]] = cmpxchg i32** [[P]], i32* null, i32* [[Q]] acquire monotonic
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nounwind willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test_cmpxchg_ptr
+; IS__CGSCC____-SAME: (i32** nocapture nofree nonnull dereferenceable(8) [[P:%.*]], i32* nofree [[Q:%.*]])
+; IS__CGSCC____-NEXT:    [[TMP1:%.*]] = cmpxchg i32** [[P]], i32* null, i32* [[Q]] acquire monotonic
+; IS__CGSCC____-NEXT:    ret void
 ;
   cmpxchg i32** %p, i32* null, i32* %q acquire monotonic
   ret void
 }
 
 define void @test_atomicrmw(i32* %p) {
-; CHECK-LABEL: define {{[^@]+}}@test_atomicrmw
-; CHECK-SAME: (i32* nocapture nofree nonnull dereferenceable(4) [[P:%.*]])
-; CHECK-NEXT:    [[TMP1:%.*]] = atomicrmw add i32* [[P]], i32 1 seq_cst
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: argmemonly nofree nounwind willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test_atomicrmw
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull dereferenceable(4) [[P:%.*]])
+; IS__TUNIT____-NEXT:    [[TMP1:%.*]] = atomicrmw add i32* [[P]], i32 1 seq_cst
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nounwind willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test_atomicrmw
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull dereferenceable(4) [[P:%.*]])
+; IS__CGSCC____-NEXT:    [[TMP1:%.*]] = atomicrmw add i32* [[P]], i32 1 seq_cst
+; IS__CGSCC____-NEXT:    ret void
 ;
   atomicrmw add i32* %p, i32 1 seq_cst
   ret void
 }
 
 define void @test_volatile(i32* %x) {
-; CHECK-LABEL: define {{[^@]+}}@test_volatile
-; CHECK-SAME: (i32* nofree align 4 [[X:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, i32* [[X]], i64 1
-; CHECK-NEXT:    store volatile i32 0, i32* [[GEP]], align 4
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: argmemonly nofree nounwind willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test_volatile
+; IS__TUNIT____-SAME: (i32* nofree align 4 [[X:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[GEP:%.*]] = getelementptr i32, i32* [[X]], i64 1
+; IS__TUNIT____-NEXT:    store volatile i32 0, i32* [[GEP]], align 4
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nounwind willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test_volatile
+; IS__CGSCC____-SAME: (i32* nofree align 4 [[X:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[GEP:%.*]] = getelementptr i32, i32* [[X]], i64 1
+; IS__CGSCC____-NEXT:    store volatile i32 0, i32* [[GEP]], align 4
+; IS__CGSCC____-NEXT:    ret void
 ;
 entry:
   %gep = getelementptr i32, i32* %x, i64 1
@@ -415,6 +577,7 @@ entry:
 }
 
 define void @nocaptureLaunder(i8* %p) {
+; CHECK: Function Attrs: nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@nocaptureLaunder
 ; CHECK-SAME: (i8* nocapture [[P:%.*]])
 ; CHECK-NEXT:  entry:
@@ -430,6 +593,7 @@ entry:
 
 @g2 = global i8* null
 define void @captureLaunder(i8* %p) {
+; CHECK: Function Attrs: nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@captureLaunder
 ; CHECK-SAME: (i8* [[P:%.*]])
 ; CHECK-NEXT:    [[B:%.*]] = call i8* @llvm.launder.invariant.group.p0i8(i8* [[P]])
@@ -442,6 +606,7 @@ define void @captureLaunder(i8* %p) {
 }
 
 define void @nocaptureStrip(i8* %p) {
+; CHECK: Function Attrs: nosync nounwind willreturn writeonly
 ; CHECK-LABEL: define {{[^@]+}}@nocaptureStrip
 ; CHECK-SAME: (i8* nocapture writeonly [[P:%.*]])
 ; CHECK-NEXT:  entry:
@@ -457,6 +622,7 @@ entry:
 
 @g3 = global i8* null
 define void @captureStrip(i8* %p) {
+; CHECK: Function Attrs: nosync nounwind willreturn writeonly
 ; CHECK-LABEL: define {{[^@]+}}@captureStrip
 ; CHECK-SAME: (i8* writeonly [[P:%.*]])
 ; CHECK-NEXT:    [[B:%.*]] = call i8* @llvm.strip.invariant.group.p0i8(i8* noalias readnone [[P]])
@@ -469,32 +635,55 @@ define void @captureStrip(i8* %p) {
 }
 
 define i1 @captureICmp(i32* %x) {
-; CHECK-LABEL: define {{[^@]+}}@captureICmp
-; CHECK-SAME: (i32* nofree readnone [[X:%.*]])
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32* [[X]], null
-; CHECK-NEXT:    ret i1 [[TMP1]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@captureICmp
+; IS__TUNIT____-SAME: (i32* nofree readnone [[X:%.*]])
+; IS__TUNIT____-NEXT:    [[TMP1:%.*]] = icmp eq i32* [[X]], null
+; IS__TUNIT____-NEXT:    ret i1 [[TMP1]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@captureICmp
+; IS__CGSCC____-SAME: (i32* nofree readnone [[X:%.*]])
+; IS__CGSCC____-NEXT:    [[TMP1:%.*]] = icmp eq i32* [[X]], null
+; IS__CGSCC____-NEXT:    ret i1 [[TMP1]]
 ;
   %1 = icmp eq i32* %x, null
   ret i1 %1
 }
 
 define i1 @captureICmpRev(i32* %x) {
-; CHECK-LABEL: define {{[^@]+}}@captureICmpRev
-; CHECK-SAME: (i32* nofree readnone [[X:%.*]])
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32* null, [[X]]
-; CHECK-NEXT:    ret i1 [[TMP1]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@captureICmpRev
+; IS__TUNIT____-SAME: (i32* nofree readnone [[X:%.*]])
+; IS__TUNIT____-NEXT:    [[TMP1:%.*]] = icmp eq i32* null, [[X]]
+; IS__TUNIT____-NEXT:    ret i1 [[TMP1]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@captureICmpRev
+; IS__CGSCC____-SAME: (i32* nofree readnone [[X:%.*]])
+; IS__CGSCC____-NEXT:    [[TMP1:%.*]] = icmp eq i32* null, [[X]]
+; IS__CGSCC____-NEXT:    ret i1 [[TMP1]]
 ;
   %1 = icmp eq i32* null, %x
   ret i1 %1
 }
 
 define i1 @nocaptureInboundsGEPICmp(i32* %x) {
-; CHECK-LABEL: define {{[^@]+}}@nocaptureInboundsGEPICmp
-; CHECK-SAME: (i32* nocapture nofree readnone [[X:%.*]])
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 5
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i8* [[TMP2]], null
-; CHECK-NEXT:    ret i1 [[TMP3]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@nocaptureInboundsGEPICmp
+; IS__TUNIT____-SAME: (i32* nocapture nofree readnone [[X:%.*]])
+; IS__TUNIT____-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 5
+; IS__TUNIT____-NEXT:    [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
+; IS__TUNIT____-NEXT:    [[TMP3:%.*]] = icmp eq i8* [[TMP2]], null
+; IS__TUNIT____-NEXT:    ret i1 [[TMP3]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@nocaptureInboundsGEPICmp
+; IS__CGSCC____-SAME: (i32* nocapture nofree readnone [[X:%.*]])
+; IS__CGSCC____-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 5
+; IS__CGSCC____-NEXT:    [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
+; IS__CGSCC____-NEXT:    [[TMP3:%.*]] = icmp eq i8* [[TMP2]], null
+; IS__CGSCC____-NEXT:    ret i1 [[TMP3]]
 ;
   %1 = getelementptr inbounds i32, i32* %x, i32 5
   %2 = bitcast i32* %1 to i8*
@@ -503,12 +692,21 @@ define i1 @nocaptureInboundsGEPICmp(i32* %x) {
 }
 
 define i1 @nocaptureInboundsGEPICmpRev(i32* %x) {
-; CHECK-LABEL: define {{[^@]+}}@nocaptureInboundsGEPICmpRev
-; CHECK-SAME: (i32* nocapture nofree readnone [[X:%.*]])
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 5
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i8* null, [[TMP2]]
-; CHECK-NEXT:    ret i1 [[TMP3]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@nocaptureInboundsGEPICmpRev
+; IS__TUNIT____-SAME: (i32* nocapture nofree readnone [[X:%.*]])
+; IS__TUNIT____-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 5
+; IS__TUNIT____-NEXT:    [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
+; IS__TUNIT____-NEXT:    [[TMP3:%.*]] = icmp eq i8* null, [[TMP2]]
+; IS__TUNIT____-NEXT:    ret i1 [[TMP3]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@nocaptureInboundsGEPICmpRev
+; IS__CGSCC____-SAME: (i32* nocapture nofree readnone [[X:%.*]])
+; IS__CGSCC____-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[X]], i32 5
+; IS__CGSCC____-NEXT:    [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
+; IS__CGSCC____-NEXT:    [[TMP3:%.*]] = icmp eq i8* null, [[TMP2]]
+; IS__CGSCC____-NEXT:    ret i1 [[TMP3]]
 ;
   %1 = getelementptr inbounds i32, i32* %x, i32 5
   %2 = bitcast i32* %1 to i8*
@@ -517,11 +715,19 @@ define i1 @nocaptureInboundsGEPICmpRev(i32* %x) {
 }
 
 define i1 @nocaptureDereferenceableOrNullICmp(i32* dereferenceable_or_null(4) %x) {
-; CHECK-LABEL: define {{[^@]+}}@nocaptureDereferenceableOrNullICmp
-; CHECK-SAME: (i32* nocapture nofree readnone dereferenceable_or_null(4) [[X:%.*]])
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[X]] to i8*
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8* [[TMP1]], null
-; CHECK-NEXT:    ret i1 [[TMP2]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@nocaptureDereferenceableOrNullICmp
+; IS__TUNIT____-SAME: (i32* nocapture nofree readnone dereferenceable_or_null(4) [[X:%.*]])
+; IS__TUNIT____-NEXT:    [[TMP1:%.*]] = bitcast i32* [[X]] to i8*
+; IS__TUNIT____-NEXT:    [[TMP2:%.*]] = icmp eq i8* [[TMP1]], null
+; IS__TUNIT____-NEXT:    ret i1 [[TMP2]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@nocaptureDereferenceableOrNullICmp
+; IS__CGSCC____-SAME: (i32* nocapture nofree readnone dereferenceable_or_null(4) [[X:%.*]])
+; IS__CGSCC____-NEXT:    [[TMP1:%.*]] = bitcast i32* [[X]] to i8*
+; IS__CGSCC____-NEXT:    [[TMP2:%.*]] = icmp eq i8* [[TMP1]], null
+; IS__CGSCC____-NEXT:    ret i1 [[TMP2]]
 ;
   %1 = bitcast i32* %x to i8*
   %2 = icmp eq i8* %1, null
@@ -529,11 +735,19 @@ define i1 @nocaptureDereferenceableOrNullICmp(i32* dereferenceable_or_null(4) %x
 }
 
 define i1 @captureDereferenceableOrNullICmp(i32* dereferenceable_or_null(4) %x) null_pointer_is_valid {
-; CHECK-LABEL: define {{[^@]+}}@captureDereferenceableOrNullICmp
-; CHECK-SAME: (i32* nofree readnone dereferenceable_or_null(4) [[X:%.*]])
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[X]] to i8*
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8* [[TMP1]], null
-; CHECK-NEXT:    ret i1 [[TMP2]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind null_pointer_is_valid readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@captureDereferenceableOrNullICmp
+; IS__TUNIT____-SAME: (i32* nofree readnone dereferenceable_or_null(4) [[X:%.*]])
+; IS__TUNIT____-NEXT:    [[TMP1:%.*]] = bitcast i32* [[X]] to i8*
+; IS__TUNIT____-NEXT:    [[TMP2:%.*]] = icmp eq i8* [[TMP1]], null
+; IS__TUNIT____-NEXT:    ret i1 [[TMP2]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind null_pointer_is_valid readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@captureDereferenceableOrNullICmp
+; IS__CGSCC____-SAME: (i32* nofree readnone dereferenceable_or_null(4) [[X:%.*]])
+; IS__CGSCC____-NEXT:    [[TMP1:%.*]] = bitcast i32* [[X]] to i8*
+; IS__CGSCC____-NEXT:    [[TMP2:%.*]] = icmp eq i8* [[TMP1]], null
+; IS__CGSCC____-NEXT:    ret i1 [[TMP2]]
 ;
   %1 = bitcast i32* %x to i8*
   %2 = icmp eq i8* %1, null
@@ -555,6 +769,7 @@ entry:
 
 declare i8* @unknownpi8pi8(i8*,i8* returned)
 define i8* @test_returned1(i8* %A, i8* returned %B) nounwind readonly {
+; CHECK: Function Attrs: nounwind readonly
 ; CHECK-LABEL: define {{[^@]+}}@test_returned1
 ; CHECK-SAME: (i8* nocapture readonly [[A:%.*]], i8* readonly returned [[B:%.*]])
 ; CHECK-NEXT:  entry:
@@ -567,6 +782,7 @@ entry:
 }
 
 define i8* @test_returned2(i8* %A, i8* %B) {
+; CHECK: Function Attrs: nounwind readonly
 ; CHECK-LABEL: define {{[^@]+}}@test_returned2
 ; CHECK-SAME: (i8* nocapture readonly [[A:%.*]], i8* readonly returned [[B:%.*]])
 ; CHECK-NEXT:  entry:
@@ -584,6 +800,7 @@ declare void @val_use(i8 %ptr) readonly nounwind
 
 ; FIXME: Both pointers should be nocapture
 define void @ptr_uses(i8* %ptr, i8* %wptr) {
+; CHECK: Function Attrs: nounwind
 ; CHECK-LABEL: define {{[^@]+}}@ptr_uses
 ; CHECK-SAME: (i8* [[PTR:%.*]], i8* nocapture nonnull writeonly dereferenceable(1) [[WPTR:%.*]])
 ; CHECK-NEXT:    store i8 0, i8* [[WPTR]], align 1
diff --git a/llvm/test/Transforms/Attributor/nocapture-2.ll b/llvm/test/Transforms/Attributor/nocapture-2.ll
index 5e97dd0a461a9..c16047de065d4 100644
--- a/llvm/test/Transforms/Attributor/nocapture-2.ll
+++ b/llvm/test/Transforms/Attributor/nocapture-2.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=10 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=10 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -18,12 +18,21 @@ declare i32* @unknown()
 ;
 ; no-capture is missing on %p because it is not dereferenceable
 define i32 @is_null_return(i32* %p) #0 {
-; CHECK-LABEL: define {{[^@]+}}@is_null_return
-; CHECK-SAME: (i32* nofree readnone [[P:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32* [[P]], null
-; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
-; CHECK-NEXT:    ret i32 [[CONV]]
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@is_null_return
+; IS__TUNIT____-SAME: (i32* nofree readnone [[P:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[CMP:%.*]] = icmp eq i32* [[P]], null
+; IS__TUNIT____-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; IS__TUNIT____-NEXT:    ret i32 [[CONV]]
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@is_null_return
+; IS__CGSCC____-SAME: (i32* nofree readnone [[P:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[CMP:%.*]] = icmp eq i32* [[P]], null
+; IS__CGSCC____-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; IS__CGSCC____-NEXT:    ret i32 [[CONV]]
 ;
 entry:
   %cmp = icmp eq i32* %p, null
@@ -43,27 +52,51 @@ entry:
 ;
 ; no-capture is missing on %p because it is not dereferenceable
 define i32 @is_null_control(i32* %p) #0 {
-; CHECK-LABEL: define {{[^@]+}}@is_null_control
-; CHECK-SAME: (i32* nofree [[P:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32* [[P]], null
-; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    store i32 1, i32* [[RETVAL]], align 4
-; CHECK-NEXT:    br label [[RETURN:%.*]]
-; CHECK:       if.end:
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32* null, [[P]]
-; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN2:%.*]], label [[IF_END3:%.*]]
-; CHECK:       if.then2:
-; CHECK-NEXT:    store i32 1, i32* [[RETVAL]], align 4
-; CHECK-NEXT:    br label [[RETURN]]
-; CHECK:       if.end3:
-; CHECK-NEXT:    store i32 0, i32* [[RETVAL]], align 4
-; CHECK-NEXT:    br label [[RETURN]]
-; CHECK:       return:
-; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[RETVAL]], align 4
-; CHECK-NEXT:    ret i32 [[TMP0]]
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@is_null_control
+; IS__TUNIT____-SAME: (i32* nofree [[P:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
+; IS__TUNIT____-NEXT:    [[CMP:%.*]] = icmp eq i32* [[P]], null
+; IS__TUNIT____-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; IS__TUNIT____:       if.then:
+; IS__TUNIT____-NEXT:    store i32 1, i32* [[RETVAL]], align 4
+; IS__TUNIT____-NEXT:    br label [[RETURN:%.*]]
+; IS__TUNIT____:       if.end:
+; IS__TUNIT____-NEXT:    [[CMP1:%.*]] = icmp eq i32* null, [[P]]
+; IS__TUNIT____-NEXT:    br i1 [[CMP1]], label [[IF_THEN2:%.*]], label [[IF_END3:%.*]]
+; IS__TUNIT____:       if.then2:
+; IS__TUNIT____-NEXT:    store i32 1, i32* [[RETVAL]], align 4
+; IS__TUNIT____-NEXT:    br label [[RETURN]]
+; IS__TUNIT____:       if.end3:
+; IS__TUNIT____-NEXT:    store i32 0, i32* [[RETVAL]], align 4
+; IS__TUNIT____-NEXT:    br label [[RETURN]]
+; IS__TUNIT____:       return:
+; IS__TUNIT____-NEXT:    [[TMP0:%.*]] = load i32, i32* [[RETVAL]], align 4
+; IS__TUNIT____-NEXT:    ret i32 [[TMP0]]
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@is_null_control
+; IS__CGSCC____-SAME: (i32* nofree [[P:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
+; IS__CGSCC____-NEXT:    [[CMP:%.*]] = icmp eq i32* [[P]], null
+; IS__CGSCC____-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; IS__CGSCC____:       if.then:
+; IS__CGSCC____-NEXT:    store i32 1, i32* [[RETVAL]], align 4
+; IS__CGSCC____-NEXT:    br label [[RETURN:%.*]]
+; IS__CGSCC____:       if.end:
+; IS__CGSCC____-NEXT:    [[CMP1:%.*]] = icmp eq i32* null, [[P]]
+; IS__CGSCC____-NEXT:    br i1 [[CMP1]], label [[IF_THEN2:%.*]], label [[IF_END3:%.*]]
+; IS__CGSCC____:       if.then2:
+; IS__CGSCC____-NEXT:    store i32 1, i32* [[RETVAL]], align 4
+; IS__CGSCC____-NEXT:    br label [[RETURN]]
+; IS__CGSCC____:       if.end3:
+; IS__CGSCC____-NEXT:    store i32 0, i32* [[RETVAL]], align 4
+; IS__CGSCC____-NEXT:    br label [[RETURN]]
+; IS__CGSCC____:       return:
+; IS__CGSCC____-NEXT:    [[TMP0:%.*]] = load i32, i32* [[RETVAL]], align 4
+; IS__CGSCC____-NEXT:    ret i32 [[TMP0]]
 ;
 entry:
   %retval = alloca i32, align 4
@@ -99,10 +132,17 @@ return:                                           ; preds = %if.end3, %if.then2,
 ; }
 ;
 define double* @srec0(double* %a) #0 {
-; CHECK-LABEL: define {{[^@]+}}@srec0
-; CHECK-SAME: (double* nocapture nofree readnone [[A:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@srec0
+; IS__TUNIT____-SAME: (double* nocapture nofree readnone [[A:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@srec0
+; IS__CGSCC____-SAME: (double* nocapture nofree readnone [[A:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    unreachable
 ;
 entry:
   %call = call double* @srec0(double* %a)
@@ -123,10 +163,17 @@ entry:
 ; Other arguments are possible here due to the no-return behavior.
 ;
 define i32* @srec16(i32* %a) #0 {
-; CHECK-LABEL: define {{[^@]+}}@srec16
-; CHECK-SAME: (i32* nocapture nofree readnone [[A:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@srec16
+; IS__TUNIT____-SAME: (i32* nocapture nofree readnone [[A:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@srec16
+; IS__CGSCC____-SAME: (i32* nocapture nofree readnone [[A:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    unreachable
 ;
 entry:
   %call = call i32* @srec16(i32* %a)
@@ -162,6 +209,7 @@ entry:
 ;   return scc_A((int*)(scc_A(a) ? scc_B((double*)a) : scc_C(a)));
 ; }
 define float* @scc_A(i32* dereferenceable_or_null(4) %a) {
+; CHECK: Function Attrs: nofree nosync nounwind readnone
 ; CHECK-LABEL: define {{[^@]+}}@scc_A
 ; CHECK-SAME: (i32* nofree readnone returned dereferenceable_or_null(4) "no-capture-maybe-returned" [[A:%.*]])
 ; CHECK-NEXT:  entry:
@@ -207,6 +255,7 @@ cond.end:                                         ; preds = %cond.false, %cond.t
 }
 
 define i64* @scc_B(double* dereferenceable_or_null(8) %a) {
+; CHECK: Function Attrs: nofree nosync nounwind readnone
 ; CHECK-LABEL: define {{[^@]+}}@scc_B
 ; CHECK-SAME: (double* nofree readnone returned dereferenceable_or_null(8) "no-capture-maybe-returned" [[A:%.*]])
 ; CHECK-NEXT:  entry:
@@ -252,6 +301,7 @@ cond.end:                                         ; preds = %cond.false, %cond.t
 }
 
 define i8* @scc_C(i16* dereferenceable_or_null(2) %a) {
+; CHECK: Function Attrs: nofree nosync nounwind readnone
 ; CHECK-LABEL: define {{[^@]+}}@scc_C
 ; CHECK-SAME: (i16* nofree readnone returned dereferenceable_or_null(4) "no-capture-maybe-returned" [[A:%.*]])
 ; CHECK-NEXT:  entry:
@@ -311,6 +361,7 @@ cond.end:                                         ; preds = %cond.false, %cond.t
 declare void @external_no_capture(i32* nocapture)
 
 define void @test_external_no_capture(i32* %p) #0 {
+; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@test_external_no_capture
 ; CHECK-SAME: (i32* nocapture [[P:%.*]])
 ; CHECK-NEXT:  entry:
@@ -329,6 +380,7 @@ entry:
 ; }
 ;
 define void @test_var_arg_call(i8* %p, i32 %a) #0 {
+; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@test_var_arg_call
 ; CHECK-SAME: (i8* nocapture [[P:%.*]], i32 [[A:%.*]])
 ; CHECK-NEXT:  entry:
@@ -352,11 +404,19 @@ declare i32 @printf(i8* nocapture, ...)
 ;
 ; There should *not* be a no-capture attribute on %a
 define i64* @not_captured_but_returned_0(i64* %a) #0 {
-; CHECK-LABEL: define {{[^@]+}}@not_captured_but_returned_0
-; CHECK-SAME: (i64* nofree nonnull returned writeonly align 8 dereferenceable(8) "no-capture-maybe-returned" [[A:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    store i64 0, i64* [[A]], align 8
-; CHECK-NEXT:    ret i64* [[A]]
+; IS__TUNIT____: Function Attrs: argmemonly nofree noinline nosync nounwind uwtable willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@not_captured_but_returned_0
+; IS__TUNIT____-SAME: (i64* nofree nonnull returned writeonly align 8 dereferenceable(8) "no-capture-maybe-returned" [[A:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    store i64 0, i64* [[A]], align 8
+; IS__TUNIT____-NEXT:    ret i64* [[A]]
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree noinline norecurse nosync nounwind uwtable willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@not_captured_but_returned_0
+; IS__CGSCC____-SAME: (i64* nofree nonnull returned writeonly align 8 dereferenceable(8) "no-capture-maybe-returned" [[A:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    store i64 0, i64* [[A]], align 8
+; IS__CGSCC____-NEXT:    ret i64* [[A]]
 ;
 entry:
   store i64 0, i64* %a, align 8
@@ -372,12 +432,21 @@ entry:
 ;
 ; There should *not* be a no-capture attribute on %a
 define i64* @not_captured_but_returned_1(i64* %a) #0 {
-; CHECK-LABEL: define {{[^@]+}}@not_captured_but_returned_1
-; CHECK-SAME: (i64* nofree nonnull writeonly align 8 dereferenceable(16) "no-capture-maybe-returned" [[A:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i64, i64* [[A]], i64 1
-; CHECK-NEXT:    store i64 1, i64* [[ADD_PTR]], align 8
-; CHECK-NEXT:    ret i64* [[ADD_PTR]]
+; IS__TUNIT____: Function Attrs: argmemonly nofree noinline nosync nounwind uwtable willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@not_captured_but_returned_1
+; IS__TUNIT____-SAME: (i64* nofree nonnull writeonly align 8 dereferenceable(16) "no-capture-maybe-returned" [[A:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i64, i64* [[A]], i64 1
+; IS__TUNIT____-NEXT:    store i64 1, i64* [[ADD_PTR]], align 8
+; IS__TUNIT____-NEXT:    ret i64* [[ADD_PTR]]
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree noinline norecurse nosync nounwind uwtable willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@not_captured_but_returned_1
+; IS__CGSCC____-SAME: (i64* nofree nonnull writeonly align 8 dereferenceable(16) "no-capture-maybe-returned" [[A:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i64, i64* [[A]], i64 1
+; IS__CGSCC____-NEXT:    store i64 1, i64* [[ADD_PTR]], align 8
+; IS__CGSCC____-NEXT:    ret i64* [[ADD_PTR]]
 ;
 entry:
   %add.ptr = getelementptr inbounds i64, i64* %a, i64 1
@@ -393,6 +462,7 @@ entry:
 ; }
 ;
 define void @test_not_captured_but_returned_calls(i64* %a) #0 {
+; IS__TUNIT____: Function Attrs: argmemonly nofree noinline nosync nounwind uwtable willreturn writeonly
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@test_not_captured_but_returned_calls
 ; IS__TUNIT____-SAME: (i64* nocapture nofree writeonly align 8 [[A:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
@@ -400,6 +470,7 @@ define void @test_not_captured_but_returned_calls(i64* %a) #0 {
 ; IS__TUNIT____-NEXT:    [[CALL1:%.*]] = call i64* @not_captured_but_returned_1(i64* nofree writeonly align 8 "no-capture-maybe-returned" [[A]])
 ; IS__TUNIT____-NEXT:    ret void
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree noinline norecurse nosync nounwind uwtable willreturn writeonly
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@test_not_captured_but_returned_calls
 ; IS__CGSCC____-SAME: (i64* nofree nonnull writeonly align 8 dereferenceable(16) [[A:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -421,12 +492,14 @@ entry:
 ;
 ; There should *not* be a no-capture attribute on %a
 define i64* @negative_test_not_captured_but_returned_call_0a(i64* %a) #0 {
+; IS__TUNIT____: Function Attrs: argmemonly nofree noinline nosync nounwind uwtable willreturn writeonly
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@negative_test_not_captured_but_returned_call_0a
 ; IS__TUNIT____-SAME: (i64* nofree returned writeonly align 8 "no-capture-maybe-returned" [[A:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
 ; IS__TUNIT____-NEXT:    [[CALL:%.*]] = call i64* @not_captured_but_returned_0(i64* nofree writeonly align 8 "no-capture-maybe-returned" [[A]])
 ; IS__TUNIT____-NEXT:    ret i64* [[CALL]]
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree noinline norecurse nosync nounwind uwtable willreturn writeonly
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@negative_test_not_captured_but_returned_call_0a
 ; IS__CGSCC____-SAME: (i64* nofree nonnull returned writeonly align 8 dereferenceable(8) [[A:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -446,6 +519,7 @@ entry:
 ;
 ; There should *not* be a no-capture attribute on %a
 define void @negative_test_not_captured_but_returned_call_0b(i64* %a) #0 {
+; IS__TUNIT____: Function Attrs: argmemonly nofree noinline nosync nounwind uwtable willreturn writeonly
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@negative_test_not_captured_but_returned_call_0b
 ; IS__TUNIT____-SAME: (i64* nofree writeonly align 8 [[A:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
@@ -454,6 +528,7 @@ define void @negative_test_not_captured_but_returned_call_0b(i64* %a) #0 {
 ; IS__TUNIT____-NEXT:    store i64 [[TMP0]], i64* [[A]], align 8
 ; IS__TUNIT____-NEXT:    ret void
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree noinline norecurse nosync nounwind uwtable willreturn writeonly
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@negative_test_not_captured_but_returned_call_0b
 ; IS__CGSCC____-SAME: (i64* nofree nonnull writeonly align 8 dereferenceable(8) [[A:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -477,12 +552,14 @@ entry:
 ;
 ; There should *not* be a no-capture attribute on %a
 define i64* @negative_test_not_captured_but_returned_call_1a(i64* %a) #0 {
+; IS__TUNIT____: Function Attrs: argmemonly nofree noinline nosync nounwind uwtable willreturn writeonly
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@negative_test_not_captured_but_returned_call_1a
 ; IS__TUNIT____-SAME: (i64* nofree writeonly align 8 "no-capture-maybe-returned" [[A:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
 ; IS__TUNIT____-NEXT:    [[CALL:%.*]] = call nonnull align 8 dereferenceable(8) i64* @not_captured_but_returned_1(i64* nofree writeonly align 8 "no-capture-maybe-returned" [[A]])
 ; IS__TUNIT____-NEXT:    ret i64* [[CALL]]
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree noinline norecurse nosync nounwind uwtable willreturn writeonly
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@negative_test_not_captured_but_returned_call_1a
 ; IS__CGSCC____-SAME: (i64* nofree nonnull writeonly align 8 dereferenceable(16) [[A:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -502,6 +579,7 @@ entry:
 ;
 ; There should *not* be a no-capture attribute on %a
 define void @negative_test_not_captured_but_returned_call_1b(i64* %a) #0 {
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind uwtable willreturn writeonly
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@negative_test_not_captured_but_returned_call_1b
 ; IS__TUNIT____-SAME: (i64* nofree writeonly align 8 [[A:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
@@ -510,6 +588,7 @@ define void @negative_test_not_captured_but_returned_call_1b(i64* %a) #0 {
 ; IS__TUNIT____-NEXT:    store i64 [[TMP0]], i64* [[CALL]], align 8
 ; IS__TUNIT____-NEXT:    ret void
 ;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind uwtable willreturn writeonly
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@negative_test_not_captured_but_returned_call_1b
 ; IS__CGSCC____-SAME: (i64* nofree nonnull writeonly align 8 dereferenceable(16) [[A:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -537,6 +616,7 @@ entry:
 ;
 
 define i32* @ret_arg_or_unknown(i32* %b) #0 {
+; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@ret_arg_or_unknown
 ; CHECK-SAME: (i32* [[B:%.*]])
 ; CHECK-NEXT:  entry:
@@ -561,6 +641,7 @@ ret_unknown:
 }
 
 define i32* @ret_arg_or_unknown_through_phi(i32* %b) #0 {
+; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@ret_arg_or_unknown_through_phi
 ; CHECK-SAME: (i32* [[B:%.*]])
 ; CHECK-NEXT:  entry:
@@ -597,6 +678,7 @@ r:
 declare i32* @readonly_unknown(i32*, i32*) readonly
 
 define void @not_captured_by_readonly_call(i32* %b) #0 {
+; CHECK: Function Attrs: noinline nounwind readonly uwtable
 ; CHECK-LABEL: define {{[^@]+}}@not_captured_by_readonly_call
 ; CHECK-SAME: (i32* nocapture readonly [[B:%.*]])
 ; CHECK-NEXT:  entry:
@@ -614,6 +696,7 @@ entry:
 ; Make sure the returned flag on %r is strong enough to justify nocapture on %b but **not** on %r.
 ;
 define i32* @not_captured_by_readonly_call_not_returned_either1(i32* %b, i32* returned %r) {
+; CHECK: Function Attrs: nounwind readonly
 ; CHECK-LABEL: define {{[^@]+}}@not_captured_by_readonly_call_not_returned_either1
 ; CHECK-SAME: (i32* nocapture readonly [[B:%.*]], i32* readonly returned [[R:%.*]])
 ; CHECK-NEXT:  entry:
@@ -627,6 +710,7 @@ entry:
 
 declare i32* @readonly_unknown_r1a(i32*, i32* returned) readonly
 define i32* @not_captured_by_readonly_call_not_returned_either2(i32* %b, i32* %r) {
+; CHECK: Function Attrs: nounwind readonly
 ; CHECK-LABEL: define {{[^@]+}}@not_captured_by_readonly_call_not_returned_either2
 ; CHECK-SAME: (i32* nocapture readonly [[B:%.*]], i32* readonly returned [[R:%.*]])
 ; CHECK-NEXT:  entry:
@@ -640,6 +724,7 @@ entry:
 
 declare i32* @readonly_unknown_r1b(i32*, i32* returned) readonly nounwind
 define i32* @not_captured_by_readonly_call_not_returned_either3(i32* %b, i32* %r) {
+; CHECK: Function Attrs: nounwind readonly
 ; CHECK-LABEL: define {{[^@]+}}@not_captured_by_readonly_call_not_returned_either3
 ; CHECK-SAME: (i32* nocapture readonly [[B:%.*]], i32* readonly returned [[R:%.*]])
 ; CHECK-NEXT:  entry:
@@ -652,6 +737,7 @@ entry:
 }
 
 define i32* @not_captured_by_readonly_call_not_returned_either4(i32* %b, i32* %r) nounwind {
+; CHECK: Function Attrs: nounwind readonly
 ; CHECK-LABEL: define {{[^@]+}}@not_captured_by_readonly_call_not_returned_either4
 ; CHECK-SAME: (i32* nocapture readonly [[B:%.*]], i32* readonly returned [[R:%.*]])
 ; CHECK-NEXT:  entry:
diff --git a/llvm/test/Transforms/Attributor/nofree.ll b/llvm/test/Transforms/Attributor/nofree.ll
index fd7324dc9b3de..d19e174797650 100644
--- a/llvm/test/Transforms/Attributor/nofree.ll
+++ b/llvm/test/Transforms/Attributor/nofree.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=11 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=11 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -16,11 +16,14 @@ declare void @_ZdaPv(i8*) local_unnamed_addr #2
 
 
 ; TEST 1 (positive case)
-; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable
-; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
 define void @only_return() #0 {
-; CHECK-LABEL: define {{[^@]+}}@only_return()
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@only_return()
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@only_return()
+; IS__CGSCC____-NEXT:    ret void
 ;
   ret void
 }
@@ -32,9 +35,8 @@ define void @only_return() #0 {
 ;    free(p);
 ; }
 
-; CHECK: Function Attrs: noinline nounwind uwtable
-; CHECK-NOT: nofree
 define void @only_free(i8* nocapture %0) local_unnamed_addr #0 {
+; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@only_free
 ; CHECK-SAME: (i8* nocapture [[TMP0:%.*]]) local_unnamed_addr
 ; CHECK-NEXT:    tail call void @free(i8* nocapture [[TMP0]])
@@ -57,6 +59,7 @@ define void @only_free(i8* nocapture %0) local_unnamed_addr #0 {
 
 
 define void @free_in_scc1(i8* nocapture %0) local_unnamed_addr #0 {
+; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@free_in_scc1
 ; CHECK-SAME: (i8* nocapture [[TMP0:%.*]]) local_unnamed_addr
 ; CHECK-NEXT:    tail call void @free_in_scc2(i8* nocapture [[TMP0]])
@@ -67,9 +70,8 @@ define void @free_in_scc1(i8* nocapture %0) local_unnamed_addr #0 {
 }
 
 
-; CHECK: Function Attrs: noinline nounwind uwtable
-; CHECK-NOT: nofree
 define void @free_in_scc2(i8* nocapture %0) local_unnamed_addr #0 {
+; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@free_in_scc2
 ; CHECK-SAME: (i8* nocapture [[TMP0:%.*]]) local_unnamed_addr
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8* [[TMP0]], null
@@ -106,21 +108,27 @@ end:
 ; }
 
 
-; NOT_CGSCC_NPM: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable
-; IS__CGSCC_NPM: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readnone uwtable
 define void @mutual_recursion1() #0 {
-; CHECK-LABEL: define {{[^@]+}}@mutual_recursion1()
-; CHECK-NEXT:    unreachable
+; NOT_CGSCC_NPM: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable willreturn
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@mutual_recursion1()
+; NOT_CGSCC_NPM-NEXT:    unreachable
+;
+; IS__CGSCC_NPM: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readnone uwtable willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@mutual_recursion1()
+; IS__CGSCC_NPM-NEXT:    unreachable
 ;
   call void @mutual_recursion2()
   ret void
 }
 
-; NOT_CGSCC_NPM: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable
-; IS__CGSCC_NPM: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readnone uwtable
 define void @mutual_recursion2() #0 {
-; CHECK-LABEL: define {{[^@]+}}@mutual_recursion2()
-; CHECK-NEXT:    unreachable
+; NOT_CGSCC_NPM: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable willreturn
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@mutual_recursion2()
+; NOT_CGSCC_NPM-NEXT:    unreachable
+;
+; IS__CGSCC_NPM: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readnone uwtable willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@mutual_recursion2()
+; IS__CGSCC_NPM-NEXT:    unreachable
 ;
   call void @mutual_recursion1()
   ret void
@@ -133,9 +141,8 @@ define void @mutual_recursion2() #0 {
 ;     delete [] p;
 ; }
 
-; CHECK: Function Attrs: noinline nounwind uwtable
-; CHECK-NOT: nofree
 define void @_Z9delete_opPc(i8* %0) local_unnamed_addr #0 {
+; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@_Z9delete_opPc
 ; CHECK-SAME: (i8* [[TMP0:%.*]]) local_unnamed_addr
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8* [[TMP0]], null
@@ -160,9 +167,8 @@ define void @_Z9delete_opPc(i8* %0) local_unnamed_addr #0 {
 
 ; TEST 6 (negative case)
 ; Call realloc
-; CHECK: Function Attrs: noinline nounwind uwtable
-; CHECK-NOT: nofree
 define noalias i8* @call_realloc(i8* nocapture %0, i64 %1) local_unnamed_addr #0 {
+; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@call_realloc
 ; CHECK-SAME: (i8* nocapture [[TMP0:%.*]], i64 [[TMP1:%.*]]) local_unnamed_addr
 ; CHECK-NEXT:    [[RET:%.*]] = tail call i8* @realloc(i8* nocapture [[TMP0]], i64 [[TMP1]])
@@ -181,11 +187,14 @@ define noalias i8* @call_realloc(i8* nocapture %0, i64 %1) local_unnamed_addr #0
 ; CHECK-NEXT: declare void @nofree_function()
 declare void @nofree_function() nofree readnone #0
 
-; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable
-; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
 define void @call_nofree_function() #0 {
-; CHECK-LABEL: define {{[^@]+}}@call_nofree_function()
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@call_nofree_function()
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@call_nofree_function()
+; IS__CGSCC____-NEXT:    ret void
 ;
   tail call void @nofree_function()
   ret void
@@ -200,9 +209,8 @@ define void @call_nofree_function() #0 {
 declare void @maybe_free() #0
 
 
-; CHECK: Function Attrs: noinline nounwind uwtable
-; CHECK-NOT: nofree
 define void @call_maybe_free() #0 {
+; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@call_maybe_free()
 ; CHECK-NEXT:    tail call void @maybe_free()
 ; CHECK-NEXT:    ret void
@@ -215,9 +223,8 @@ define void @call_maybe_free() #0 {
 ; TEST 9 (negative case)
 ; Call both of above functions
 
-; CHECK: Function Attrs: noinline nounwind uwtable
-; CHECK-NOT: nofree
 define void @call_both() #0 {
+; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@call_both()
 ; CHECK-NEXT:    tail call void @maybe_free()
 ; CHECK-NEXT:    ret void
@@ -234,20 +241,24 @@ define void @call_both() #0 {
 ; CHECK-NEXT: declare float @llvm.floor.f32(float)
 declare float @llvm.floor.f32(float)
 
-; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable
-; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
 define void @call_floor(float %a) #0 {
-; CHECK-LABEL: define {{[^@]+}}@call_floor
-; CHECK-SAME: (float [[A:%.*]])
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@call_floor
+; IS__TUNIT____-SAME: (float [[A:%.*]])
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@call_floor
+; IS__CGSCC____-SAME: (float [[A:%.*]])
+; IS__CGSCC____-NEXT:    ret void
 ;
   tail call float @llvm.floor.f32(float %a)
   ret void
 }
 
 ; FIXME: missing nofree
-; CHECK: Function Attrs: noinline nosync nounwind readnone uwtable
 define float @call_floor2(float %a) #0 {
+; CHECK: Function Attrs: noinline nosync nounwind readnone uwtable willreturn
 ; CHECK-LABEL: define {{[^@]+}}@call_floor2
 ; CHECK-SAME: (float [[A:%.*]])
 ; CHECK-NEXT:    [[C:%.*]] = tail call float @llvm.floor.f32(float [[A]])
@@ -260,21 +271,27 @@ define float @call_floor2(float %a) #0 {
 ; TEST 11 (positive case)
 ; Check propagation.
 
-; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable
-; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
 define void @f1() #0 {
-; CHECK-LABEL: define {{[^@]+}}@f1()
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@f1()
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@f1()
+; IS__CGSCC____-NEXT:    ret void
 ;
   tail call void @nofree_function()
   ret void
 }
 
-; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable
-; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
 define void @f2() #0 {
-; CHECK-LABEL: define {{[^@]+}}@f2()
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@f2()
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@f2()
+; IS__CGSCC____-NEXT:    ret void
 ;
   tail call void @f1()
   ret void
@@ -282,6 +299,7 @@ define void @f2() #0 {
 
 ; TEST 12 NoFree argument - positive.
 define double @test12(double* nocapture readonly %a) {
+; CHECK: Function Attrs: nofree nounwind
 ; CHECK-LABEL: define {{[^@]+}}@test12
 ; CHECK-SAME: (double* nocapture nofree nonnull readonly align 8 dereferenceable(8) [[A:%.*]])
 ; CHECK-NEXT:  entry:
@@ -300,6 +318,7 @@ declare double @cos(double) nobuiltin nounwind nofree
 ; FIXME: %a should be nofree.
 ; TEST 13 NoFree argument - positive.
 define noalias i32* @test13(i64* nocapture readonly %a) {
+; CHECK: Function Attrs: nounwind
 ; CHECK-LABEL: define {{[^@]+}}@test13
 ; CHECK-SAME: (i64* nocapture nonnull readonly align 8 dereferenceable(8) [[A:%.*]])
 ; CHECK-NEXT:  entry:
@@ -316,6 +335,7 @@ entry:
 }
 
 define void @test14(i8* nocapture %0, i8* nocapture %1) {
+; CHECK: Function Attrs: nounwind
 ; CHECK-LABEL: define {{[^@]+}}@test14
 ; CHECK-SAME: (i8* nocapture [[TMP0:%.*]], i8* nocapture nofree readnone [[TMP1:%.*]])
 ; CHECK-NEXT:    tail call void @free(i8* nocapture [[TMP0]])
diff --git a/llvm/test/Transforms/Attributor/nonnull.ll b/llvm/test/Transforms/Attributor/nonnull.ll
index 9e4a4ac4bb18b..c67bc3e4f17b7 100644
--- a/llvm/test/Transforms/Attributor/nonnull.ll
+++ b/llvm/test/Transforms/Attributor/nonnull.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -22,15 +22,22 @@ define i8* @test1() {
 
 ; Return a pointer trivially nonnull (argument attribute)
 define i8* @test2(i8* nonnull %p) {
-; CHECK-LABEL: define {{[^@]+}}@test2
-; CHECK-SAME: (i8* nofree nonnull readnone returned "no-capture-maybe-returned" [[P:%.*]])
-; CHECK-NEXT:    ret i8* [[P]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test2
+; IS__TUNIT____-SAME: (i8* nofree nonnull readnone returned "no-capture-maybe-returned" [[P:%.*]])
+; IS__TUNIT____-NEXT:    ret i8* [[P]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test2
+; IS__CGSCC____-SAME: (i8* nofree nonnull readnone returned "no-capture-maybe-returned" [[P:%.*]])
+; IS__CGSCC____-NEXT:    ret i8* [[P]]
 ;
   ret i8* %p
 }
 
 define i8* @test2A(i1 %c, i8* %ret) {
 ; ATTRIBUTOR: define nonnull i8* @test2A(i1 %c, i8* nofree nonnull readnone returned %ret)
+; NOT_CGSCC_OPM: Function Attrs: nounwind willreturn
 ; NOT_CGSCC_OPM-LABEL: define {{[^@]+}}@test2A
 ; NOT_CGSCC_OPM-SAME: (i1 [[C:%.*]], i8* nofree nonnull readnone returned "no-capture-maybe-returned" [[RET:%.*]])
 ; NOT_CGSCC_OPM-NEXT:    br i1 [[C]], label [[A:%.*]], label [[B:%.*]]
@@ -41,6 +48,7 @@ define i8* @test2A(i1 %c, i8* %ret) {
 ; NOT_CGSCC_OPM-NEXT:    call void @llvm.assume(i1 true) #11 [ "nonnull"(i8* [[RET]]) ]
 ; NOT_CGSCC_OPM-NEXT:    ret i8* [[RET]]
 ;
+; IS__CGSCC_OPM: Function Attrs: nounwind willreturn
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@test2A
 ; IS__CGSCC_OPM-SAME: (i1 [[C:%.*]], i8* nofree nonnull readnone returned "no-capture-maybe-returned" [[RET:%.*]])
 ; IS__CGSCC_OPM-NEXT:    br i1 [[C]], label [[A:%.*]], label [[B:%.*]]
@@ -62,6 +70,7 @@ B:
 
 define i8* @test2B(i1 %c, i8* %ret) {
 ; ATTRIBUTOR: define nonnull dereferenceable(4) i8* @test2B(i1 %c, i8* nofree nonnull readnone returned dereferenceable(4) %ret)
+; NOT_CGSCC_OPM: Function Attrs: nounwind willreturn
 ; NOT_CGSCC_OPM-LABEL: define {{[^@]+}}@test2B
 ; NOT_CGSCC_OPM-SAME: (i1 [[C:%.*]], i8* nofree nonnull readnone returned dereferenceable(4) "no-capture-maybe-returned" [[RET:%.*]])
 ; NOT_CGSCC_OPM-NEXT:    br i1 [[C]], label [[A:%.*]], label [[B:%.*]]
@@ -72,6 +81,7 @@ define i8* @test2B(i1 %c, i8* %ret) {
 ; NOT_CGSCC_OPM-NEXT:    call void @llvm.assume(i1 true) #11 [ "dereferenceable"(i8* [[RET]], i32 4) ]
 ; NOT_CGSCC_OPM-NEXT:    ret i8* [[RET]]
 ;
+; IS__CGSCC_OPM: Function Attrs: nounwind willreturn
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@test2B
 ; IS__CGSCC_OPM-SAME: (i1 [[C:%.*]], i8* nofree nonnull readnone returned dereferenceable(4) "no-capture-maybe-returned" [[RET:%.*]])
 ; IS__CGSCC_OPM-NEXT:    br i1 [[C]], label [[A:%.*]], label [[B:%.*]]
@@ -127,16 +137,26 @@ define i8* @test3(i1 %c) {
 ; nonnull if neither can ever return null.  (In this case, they
 ; just never return period.)
 define i8* @test4_helper() {
-; CHECK-LABEL: define {{[^@]+}}@test4_helper()
-; CHECK-NEXT:    unreachable
+; NOT_CGSCC_NPM: Function Attrs: nofree noreturn nosync nounwind readnone
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@test4_helper()
+; NOT_CGSCC_NPM-NEXT:    unreachable
+;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@test4_helper()
+; IS__CGSCC_NPM-NEXT:    unreachable
 ;
   %ret = call i8* @test4()
   ret i8* %ret
 }
 
 define i8* @test4() {
-; CHECK-LABEL: define {{[^@]+}}@test4()
-; CHECK-NEXT:    unreachable
+; NOT_CGSCC_NPM: Function Attrs: nofree noreturn nosync nounwind readnone
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@test4()
+; NOT_CGSCC_NPM-NEXT:    unreachable
+;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@test4()
+; IS__CGSCC_NPM-NEXT:    unreachable
 ;
   %ret = call i8* @test4_helper()
   ret i8* %ret
@@ -145,13 +165,23 @@ define i8* @test4() {
 ; Given a mutual recursive set of functions which *can* return null
 ; make sure we haven't marked them as nonnull.
 define i8* @test5_helper(i1 %c) {
-; CHECK-LABEL: define {{[^@]+}}@test5_helper
-; CHECK-SAME: (i1 [[C:%.*]])
-; CHECK-NEXT:    br i1 [[C]], label [[REC:%.*]], label [[END:%.*]]
-; CHECK:       rec:
-; CHECK-NEXT:    br label [[END]]
-; CHECK:       end:
-; CHECK-NEXT:    ret i8* null
+; NOT_CGSCC_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@test5_helper
+; NOT_CGSCC_NPM-SAME: (i1 [[C:%.*]])
+; NOT_CGSCC_NPM-NEXT:    br i1 [[C]], label [[REC:%.*]], label [[END:%.*]]
+; NOT_CGSCC_NPM:       rec:
+; NOT_CGSCC_NPM-NEXT:    br label [[END]]
+; NOT_CGSCC_NPM:       end:
+; NOT_CGSCC_NPM-NEXT:    ret i8* null
+;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@test5_helper
+; IS__CGSCC_NPM-SAME: (i1 [[C:%.*]])
+; IS__CGSCC_NPM-NEXT:    br i1 [[C]], label [[REC:%.*]], label [[END:%.*]]
+; IS__CGSCC_NPM:       rec:
+; IS__CGSCC_NPM-NEXT:    br label [[END]]
+; IS__CGSCC_NPM:       end:
+; IS__CGSCC_NPM-NEXT:    ret i8* null
 ;
   br i1 %c, label %rec, label %end
 rec:
@@ -162,17 +192,23 @@ end:
 }
 
 define i8* @test5(i1 %c) {
-; CHECK-LABEL: define {{[^@]+}}@test5
-; CHECK-SAME: (i1 [[C:%.*]])
-; CHECK-NEXT:    ret i8* null
+; NOT_CGSCC_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@test5
+; NOT_CGSCC_NPM-SAME: (i1 [[C:%.*]])
+; NOT_CGSCC_NPM-NEXT:    ret i8* null
+;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@test5
+; IS__CGSCC_NPM-SAME: (i1 [[C:%.*]])
+; IS__CGSCC_NPM-NEXT:    ret i8* null
 ;
   %ret = call i8* @test5_helper(i1 %c)
   ret i8* %ret
 }
 
 ; Local analysis, but going through a self recursive phi
-; CHECK: Function Attrs: noreturn
 define i8* @test6a() {
+; CHECK: Function Attrs: noreturn
 ; CHECK-LABEL: define {{[^@]+}}@test6a()
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[RET:%.*]] = call i8* @ret_nonnull()
@@ -215,30 +251,51 @@ exit:
 }
 
 define i8* @test7(i8* %a) {
-; CHECK-LABEL: define {{[^@]+}}@test7
-; CHECK-SAME: (i8* nofree readnone returned "no-capture-maybe-returned" [[A:%.*]])
-; CHECK-NEXT:    [[B:%.*]] = getelementptr inbounds i8, i8* [[A]], i64 0
-; CHECK-NEXT:    ret i8* [[B]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test7
+; IS__TUNIT____-SAME: (i8* nofree readnone returned "no-capture-maybe-returned" [[A:%.*]])
+; IS__TUNIT____-NEXT:    [[B:%.*]] = getelementptr inbounds i8, i8* [[A]], i64 0
+; IS__TUNIT____-NEXT:    ret i8* [[B]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test7
+; IS__CGSCC____-SAME: (i8* nofree readnone returned "no-capture-maybe-returned" [[A:%.*]])
+; IS__CGSCC____-NEXT:    [[B:%.*]] = getelementptr inbounds i8, i8* [[A]], i64 0
+; IS__CGSCC____-NEXT:    ret i8* [[B]]
 ;
   %b = getelementptr inbounds i8, i8* %a, i64 0
   ret i8* %b
 }
 
 define i8* @test8(i8* %a) {
-; CHECK-LABEL: define {{[^@]+}}@test8
-; CHECK-SAME: (i8* nofree readnone "no-capture-maybe-returned" [[A:%.*]])
-; CHECK-NEXT:    [[B:%.*]] = getelementptr inbounds i8, i8* [[A]], i64 1
-; CHECK-NEXT:    ret i8* [[B]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test8
+; IS__TUNIT____-SAME: (i8* nofree readnone "no-capture-maybe-returned" [[A:%.*]])
+; IS__TUNIT____-NEXT:    [[B:%.*]] = getelementptr inbounds i8, i8* [[A]], i64 1
+; IS__TUNIT____-NEXT:    ret i8* [[B]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test8
+; IS__CGSCC____-SAME: (i8* nofree readnone "no-capture-maybe-returned" [[A:%.*]])
+; IS__CGSCC____-NEXT:    [[B:%.*]] = getelementptr inbounds i8, i8* [[A]], i64 1
+; IS__CGSCC____-NEXT:    ret i8* [[B]]
 ;
   %b = getelementptr inbounds i8, i8* %a, i64 1
   ret i8* %b
 }
 
 define i8* @test9(i8* %a, i64 %n) {
-; CHECK-LABEL: define {{[^@]+}}@test9
-; CHECK-SAME: (i8* nofree readnone "no-capture-maybe-returned" [[A:%.*]], i64 [[N:%.*]])
-; CHECK-NEXT:    [[B:%.*]] = getelementptr inbounds i8, i8* [[A]], i64 [[N]]
-; CHECK-NEXT:    ret i8* [[B]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test9
+; IS__TUNIT____-SAME: (i8* nofree readnone "no-capture-maybe-returned" [[A:%.*]], i64 [[N:%.*]])
+; IS__TUNIT____-NEXT:    [[B:%.*]] = getelementptr inbounds i8, i8* [[A]], i64 [[N]]
+; IS__TUNIT____-NEXT:    ret i8* [[B]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test9
+; IS__CGSCC____-SAME: (i8* nofree readnone "no-capture-maybe-returned" [[A:%.*]], i64 [[N:%.*]])
+; IS__CGSCC____-NEXT:    [[B:%.*]] = getelementptr inbounds i8, i8* [[A]], i64 [[N]]
+; IS__CGSCC____-NEXT:    ret i8* [[B]]
 ;
   %b = getelementptr inbounds i8, i8* %a, i64 %n
   ret i8* %b
@@ -247,6 +304,7 @@ define i8* @test9(i8* %a, i64 %n) {
 ; ATTRIBUTOR_OPM: define i8* @test10
 ; ATTRIBUTOR_NPM: define nonnull i8* @test10
 define i8* @test10(i8* %a, i64 %n) {
+; CHECK: Function Attrs: nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@test10
 ; CHECK-SAME: (i8* nofree readnone "no-capture-maybe-returned" [[A:%.*]], i64 [[N:%.*]])
 ; CHECK-NEXT:    call void @llvm.assume(i1 true)
@@ -319,6 +377,7 @@ define void @test13_helper() {
   ret void
 }
 define internal void @test13(i8* %a, i8* %b, i8* %c) {
+; IS__TUNIT____: Function Attrs: nounwind
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@test13
 ; IS__TUNIT____-SAME: (i8* noalias nocapture nofree nonnull readnone [[A:%.*]], i8* noalias nocapture nofree readnone [[B:%.*]], i8* noalias nocapture nofree readnone [[C:%.*]])
 ; IS__TUNIT____-NEXT:    call void @use_i8_ptr(i8* noalias nocapture nofree nonnull readnone [[A]])
@@ -326,6 +385,7 @@ define internal void @test13(i8* %a, i8* %b, i8* %c) {
 ; IS__TUNIT____-NEXT:    call void @use_i8_ptr(i8* noalias nocapture nofree readnone [[C]])
 ; IS__TUNIT____-NEXT:    ret void
 ;
+; IS__CGSCC____: Function Attrs: nounwind
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@test13
 ; IS__CGSCC____-SAME: (i8* nocapture nofree readnone [[A:%.*]], i8* nocapture nofree readnone [[B:%.*]], i8* nocapture nofree readnone [[C:%.*]])
 ; IS__CGSCC____-NEXT:    call void @use_i8_ptr(i8* noalias nocapture nofree readnone [[A]])
@@ -355,6 +415,7 @@ declare nonnull i8* @nonnull()
 
 define internal i32* @f1(i32* %arg) {
 ; FIXME: missing nonnull It should be nonnull @f1(i32* nonnull readonly %arg)
+; CHECK: Function Attrs: argmemonly nofree nosync nounwind readonly
 ; CHECK-LABEL: define {{[^@]+}}@f1
 ; CHECK-SAME: (i32* nofree readonly [[ARG:%.*]])
 ; CHECK-NEXT:  bb:
@@ -402,6 +463,7 @@ bb9:                                              ; preds = %bb4, %bb
 }
 
 define internal i32* @f2(i32* %arg) {
+; CHECK: Function Attrs: argmemonly nofree nosync nounwind readonly
 ; CHECK-LABEL: define {{[^@]+}}@f2
 ; CHECK-SAME: (i32* nofree nonnull readonly align 4 dereferenceable(4) [[ARG:%.*]])
 ; CHECK-NEXT:  bb:
@@ -415,6 +477,7 @@ bb:
 
 define dso_local noalias i32* @f3(i32* %arg) {
 ; FIXME: missing nonnull. It should be nonnull @f3(i32* nonnull readonly %arg)
+; CHECK: Function Attrs: argmemonly nofree nosync nounwind readonly
 ; CHECK-LABEL: define {{[^@]+}}@f3
 ; CHECK-SAME: (i32* nofree readonly [[ARG:%.*]])
 ; CHECK-NEXT:  bb:
@@ -450,6 +513,7 @@ declare void @fun3(i8*, i8*, i8*) #1
 ;   fun2(nonnull %a, %b)
 ; We can say that %a is nonnull but %b is not.
 define void @f16(i8* %a, i8 * %b, i8 %c) {
+; CHECK: Function Attrs: nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@f16
 ; CHECK-SAME: (i8* nonnull [[A:%.*]], i8* [[B:%.*]], i8 [[C:%.*]])
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[C]], 0
@@ -478,6 +542,7 @@ if.else:
 ; fun1(nonnull %a)
 ; We can say that %a is nonnull
 define void @f17(i8* %a, i8 %c) {
+; CHECK: Function Attrs: nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@f17
 ; CHECK-SAME: (i8* nonnull [[A:%.*]], i8 [[C:%.*]])
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[C]], 0
@@ -516,6 +581,7 @@ cont:
 ; fun1(nonnull %a)
 
 define void @f18(i8* %a, i8* %b, i8 %c) {
+; CHECK: Function Attrs: nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@f18
 ; CHECK-SAME: (i8* nonnull [[A:%.*]], i8* [[B:%.*]], i8 [[C:%.*]])
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[C]], 0
@@ -564,6 +630,7 @@ cont2:
 ; TEST 19: Loop
 
 define void @f19(i8* %a, i8* %b, i8 %c) {
+; CHECK: Function Attrs: nounwind
 ; CHECK-LABEL: define {{[^@]+}}@f19
 ; CHECK-SAME: (i8* [[A:%.*]], i8* nonnull [[B:%.*]], i8 [[C:%.*]])
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
@@ -727,6 +794,7 @@ define i8 @parent7(i8* %a) {
 declare i32 @esfp(...)
 
 define i1 @parent8(i8* %a, i8* %bogus1, i8* %b) personality i8* bitcast (i32 (...)* @esfp to i8*){
+; NOT_CGSCC_OPM: Function Attrs: nounwind
 ; NOT_CGSCC_OPM-LABEL: define {{[^@]+}}@parent8
 ; NOT_CGSCC_OPM-SAME: (i8* nonnull [[A:%.*]], i8* nocapture nofree readnone [[BOGUS1:%.*]], i8* nonnull [[B:%.*]]) #4 personality i8* bitcast (i32 (...)* @esfp to i8*)
 ; NOT_CGSCC_OPM-NEXT:  entry:
@@ -740,6 +808,7 @@ define i1 @parent8(i8* %a, i8* %bogus1, i8* %b) personality i8* bitcast (i32 (..
 ; NOT_CGSCC_OPM-NEXT:    filter [0 x i8*] zeroinitializer
 ; NOT_CGSCC_OPM-NEXT:    unreachable
 ;
+; IS__CGSCC_OPM: Function Attrs: nounwind
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@parent8
 ; IS__CGSCC_OPM-SAME: (i8* nonnull [[A:%.*]], i8* nocapture nofree readnone [[BOGUS1:%.*]], i8* nonnull [[B:%.*]]) #5 personality i8* bitcast (i32 (...)* @esfp to i8*)
 ; IS__CGSCC_OPM-NEXT:  entry:
@@ -769,10 +838,17 @@ exc:
 }
 
 define i32* @gep1(i32* %p) {
-; CHECK-LABEL: define {{[^@]+}}@gep1
-; CHECK-SAME: (i32* nofree readnone "no-capture-maybe-returned" [[P:%.*]])
-; CHECK-NEXT:    [[Q:%.*]] = getelementptr inbounds i32, i32* [[P]], i32 1
-; CHECK-NEXT:    ret i32* [[Q]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@gep1
+; IS__TUNIT____-SAME: (i32* nofree readnone "no-capture-maybe-returned" [[P:%.*]])
+; IS__TUNIT____-NEXT:    [[Q:%.*]] = getelementptr inbounds i32, i32* [[P]], i32 1
+; IS__TUNIT____-NEXT:    ret i32* [[Q]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@gep1
+; IS__CGSCC____-SAME: (i32* nofree readnone "no-capture-maybe-returned" [[P:%.*]])
+; IS__CGSCC____-NEXT:    [[Q:%.*]] = getelementptr inbounds i32, i32* [[P]], i32 1
+; IS__CGSCC____-NEXT:    ret i32* [[Q]]
 ;
   %q = getelementptr inbounds i32, i32* %p, i32 1
   ret i32* %q
@@ -780,20 +856,34 @@ define i32* @gep1(i32* %p) {
 
 define i32* @gep1_no_null_opt(i32* %p) #0 {
 ; Should't be able to derive nonnull based on gep.
-; CHECK-LABEL: define {{[^@]+}}@gep1_no_null_opt
-; CHECK-SAME: (i32* nofree readnone "no-capture-maybe-returned" [[P:%.*]])
-; CHECK-NEXT:    [[Q:%.*]] = getelementptr inbounds i32, i32* [[P]], i32 1
-; CHECK-NEXT:    ret i32* [[Q]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind null_pointer_is_valid readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@gep1_no_null_opt
+; IS__TUNIT____-SAME: (i32* nofree readnone "no-capture-maybe-returned" [[P:%.*]])
+; IS__TUNIT____-NEXT:    [[Q:%.*]] = getelementptr inbounds i32, i32* [[P]], i32 1
+; IS__TUNIT____-NEXT:    ret i32* [[Q]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind null_pointer_is_valid readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@gep1_no_null_opt
+; IS__CGSCC____-SAME: (i32* nofree readnone "no-capture-maybe-returned" [[P:%.*]])
+; IS__CGSCC____-NEXT:    [[Q:%.*]] = getelementptr inbounds i32, i32* [[P]], i32 1
+; IS__CGSCC____-NEXT:    ret i32* [[Q]]
 ;
   %q = getelementptr inbounds i32, i32* %p, i32 1
   ret i32* %q
 }
 
 define i32 addrspace(3)* @gep2(i32 addrspace(3)* %p) {
-; CHECK-LABEL: define {{[^@]+}}@gep2
-; CHECK-SAME: (i32 addrspace(3)* nofree readnone "no-capture-maybe-returned" [[P:%.*]])
-; CHECK-NEXT:    [[Q:%.*]] = getelementptr inbounds i32, i32 addrspace(3)* [[P]], i32 1
-; CHECK-NEXT:    ret i32 addrspace(3)* [[Q]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@gep2
+; IS__TUNIT____-SAME: (i32 addrspace(3)* nofree readnone "no-capture-maybe-returned" [[P:%.*]])
+; IS__TUNIT____-NEXT:    [[Q:%.*]] = getelementptr inbounds i32, i32 addrspace(3)* [[P]], i32 1
+; IS__TUNIT____-NEXT:    ret i32 addrspace(3)* [[Q]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@gep2
+; IS__CGSCC____-SAME: (i32 addrspace(3)* nofree readnone "no-capture-maybe-returned" [[P:%.*]])
+; IS__CGSCC____-NEXT:    [[Q:%.*]] = getelementptr inbounds i32, i32 addrspace(3)* [[P]], i32 1
+; IS__CGSCC____-NEXT:    ret i32 addrspace(3)* [[Q]]
 ;
   %q = getelementptr inbounds i32, i32 addrspace(3)* %p, i32 1
   ret i32 addrspace(3)* %q
@@ -801,15 +891,22 @@ define i32 addrspace(3)* @gep2(i32 addrspace(3)* %p) {
 
 ; FIXME: We should propagate dereferenceable here but *not* nonnull
 define i32 addrspace(3)* @as(i32 addrspace(3)* dereferenceable(4) %p) {
-; CHECK-LABEL: define {{[^@]+}}@as
-; CHECK-SAME: (i32 addrspace(3)* nofree nonnull readnone returned dereferenceable(4) "no-capture-maybe-returned" [[P:%.*]])
-; CHECK-NEXT:    ret i32 addrspace(3)* [[P]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@as
+; IS__TUNIT____-SAME: (i32 addrspace(3)* nofree nonnull readnone returned dereferenceable(4) "no-capture-maybe-returned" [[P:%.*]])
+; IS__TUNIT____-NEXT:    ret i32 addrspace(3)* [[P]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@as
+; IS__CGSCC____-SAME: (i32 addrspace(3)* nofree nonnull readnone returned dereferenceable(4) "no-capture-maybe-returned" [[P:%.*]])
+; IS__CGSCC____-NEXT:    ret i32 addrspace(3)* [[P]]
 ;
   ret i32 addrspace(3)* %p
 }
 
 ; CHECK-NOT: @g2()
 define internal i32* @g2() {
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@g2()
 ; IS__CGSCC____-NEXT:    ret i32* inttoptr (i64 4 to i32*)
 ;
@@ -817,8 +914,13 @@ define internal i32* @g2() {
 }
 
 define i32* @g1() {
-; CHECK-LABEL: define {{[^@]+}}@g1()
-; CHECK-NEXT:    ret i32* inttoptr (i64 4 to i32*)
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@g1()
+; IS__TUNIT____-NEXT:    ret i32* inttoptr (i64 4 to i32*)
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@g1()
+; IS__CGSCC____-NEXT:    ret i32* inttoptr (i64 4 to i32*)
 ;
   %c = call i32* @g2()
   ret i32* %c
@@ -826,11 +928,13 @@ define i32* @g1() {
 
 declare void @use_i32_ptr(i32* readnone nocapture) nounwind
 define internal void @called_by_weak(i32* %a) {
+; IS__TUNIT____: Function Attrs: nounwind
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@called_by_weak
 ; IS__TUNIT____-SAME: (i32* noalias nocapture nonnull readnone [[A:%.*]])
 ; IS__TUNIT____-NEXT:    call void @use_i32_ptr(i32* noalias nocapture nonnull readnone [[A]])
 ; IS__TUNIT____-NEXT:    ret void
 ;
+; IS__CGSCC____: Function Attrs: nounwind
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@called_by_weak
 ; IS__CGSCC____-SAME: (i32* nocapture nonnull readnone [[A:%.*]])
 ; IS__CGSCC____-NEXT:    call void @use_i32_ptr(i32* noalias nocapture nonnull readnone [[A]])
@@ -853,11 +957,13 @@ define weak_odr void @weak_caller(i32* nonnull %a) {
 
 ; Expect nonnull
 define internal void @control(i32* dereferenceable(4) %a) {
+; IS__TUNIT____: Function Attrs: nounwind
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@control
 ; IS__TUNIT____-SAME: (i32* noalias nocapture nonnull readnone align 16 dereferenceable(8) [[A:%.*]])
 ; IS__TUNIT____-NEXT:    call void @use_i32_ptr(i32* noalias nocapture nonnull readnone align 16 dereferenceable(8) [[A]])
 ; IS__TUNIT____-NEXT:    ret void
 ;
+; IS__CGSCC____: Function Attrs: nounwind
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@control
 ; IS__CGSCC____-SAME: (i32* nocapture nonnull readnone align 16 dereferenceable(8) [[A:%.*]])
 ; IS__CGSCC____-NEXT:    call void @use_i32_ptr(i32* noalias nocapture nonnull readnone align 16 dereferenceable(8) [[A]])
@@ -868,6 +974,7 @@ define internal void @control(i32* dereferenceable(4) %a) {
 }
 ; Avoid nonnull as we do not touch naked functions
 define internal void @naked(i32* dereferenceable(4) %a) naked {
+; CHECK: Function Attrs: naked
 ; CHECK-LABEL: define {{[^@]+}}@naked
 ; CHECK-SAME: (i32* dereferenceable(4) [[A:%.*]])
 ; CHECK-NEXT:    call void @use_i32_ptr(i32* [[A]])
@@ -878,6 +985,7 @@ define internal void @naked(i32* dereferenceable(4) %a) naked {
 }
 ; Avoid nonnull as we do not touch optnone
 define internal void @optnone(i32* dereferenceable(4) %a) optnone noinline {
+; CHECK: Function Attrs: noinline optnone
 ; CHECK-LABEL: define {{[^@]+}}@optnone
 ; CHECK-SAME: (i32* dereferenceable(4) [[A:%.*]])
 ; CHECK-NEXT:    call void @use_i32_ptr(i32* [[A]])
@@ -911,20 +1019,37 @@ declare void @h(i32*) willreturn nounwind
 declare i32 @g(i32*) willreturn nounwind
 define i32 @nonnull_exec_ctx_1(i32* %a, i32 %b) {
 ;
-; CHECK-LABEL: define {{[^@]+}}@nonnull_exec_ctx_1
-; CHECK-SAME: (i32* [[A:%.*]], i32 [[B:%.*]])
-; CHECK-NEXT:  en:
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[B]], 0
-; CHECK-NEXT:    br i1 [[TMP3]], label [[EX:%.*]], label [[HD:%.*]]
-; CHECK:       ex:
-; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @g(i32* nonnull [[A]])
-; CHECK-NEXT:    ret i32 [[TMP5]]
-; CHECK:       hd:
-; CHECK-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP8:%.*]], [[HD]] ], [ 0, [[EN:%.*]] ]
-; CHECK-NEXT:    tail call void @h(i32* [[A]])
-; CHECK-NEXT:    [[TMP8]] = add nuw i32 [[TMP7]], 1
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], [[B]]
-; CHECK-NEXT:    br i1 [[TMP9]], label [[EX]], label [[HD]]
+; IS________OPM: Function Attrs: nounwind
+; IS________OPM-LABEL: define {{[^@]+}}@nonnull_exec_ctx_1
+; IS________OPM-SAME: (i32* [[A:%.*]], i32 [[B:%.*]])
+; IS________OPM-NEXT:  en:
+; IS________OPM-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[B]], 0
+; IS________OPM-NEXT:    br i1 [[TMP3]], label [[EX:%.*]], label [[HD:%.*]]
+; IS________OPM:       ex:
+; IS________OPM-NEXT:    [[TMP5:%.*]] = tail call i32 @g(i32* nonnull [[A]])
+; IS________OPM-NEXT:    ret i32 [[TMP5]]
+; IS________OPM:       hd:
+; IS________OPM-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP8:%.*]], [[HD]] ], [ 0, [[EN:%.*]] ]
+; IS________OPM-NEXT:    tail call void @h(i32* [[A]])
+; IS________OPM-NEXT:    [[TMP8]] = add nuw i32 [[TMP7]], 1
+; IS________OPM-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], [[B]]
+; IS________OPM-NEXT:    br i1 [[TMP9]], label [[EX]], label [[HD]]
+;
+; IS________NPM: Function Attrs: nounwind willreturn
+; IS________NPM-LABEL: define {{[^@]+}}@nonnull_exec_ctx_1
+; IS________NPM-SAME: (i32* [[A:%.*]], i32 [[B:%.*]])
+; IS________NPM-NEXT:  en:
+; IS________NPM-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[B]], 0
+; IS________NPM-NEXT:    br i1 [[TMP3]], label [[EX:%.*]], label [[HD:%.*]]
+; IS________NPM:       ex:
+; IS________NPM-NEXT:    [[TMP5:%.*]] = tail call i32 @g(i32* nonnull [[A]])
+; IS________NPM-NEXT:    ret i32 [[TMP5]]
+; IS________NPM:       hd:
+; IS________NPM-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP8:%.*]], [[HD]] ], [ 0, [[EN:%.*]] ]
+; IS________NPM-NEXT:    tail call void @h(i32* [[A]])
+; IS________NPM-NEXT:    [[TMP8]] = add nuw i32 [[TMP7]], 1
+; IS________NPM-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], [[B]]
+; IS________NPM-NEXT:    br i1 [[TMP9]], label [[EX]], label [[HD]]
 ;
 en:
   %tmp3 = icmp eq i32 %b, 0
@@ -944,22 +1069,41 @@ hd:
 
 define i32 @nonnull_exec_ctx_1b(i32* %a, i32 %b) {
 ;
-; CHECK-LABEL: define {{[^@]+}}@nonnull_exec_ctx_1b
-; CHECK-SAME: (i32* [[A:%.*]], i32 [[B:%.*]])
-; CHECK-NEXT:  en:
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[B]], 0
-; CHECK-NEXT:    br i1 [[TMP3]], label [[EX:%.*]], label [[HD:%.*]]
-; CHECK:       ex:
-; CHECK-NEXT:    [[TMP5:%.*]] = tail call i32 @g(i32* nonnull [[A]])
-; CHECK-NEXT:    ret i32 [[TMP5]]
-; CHECK:       hd:
-; CHECK-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP8:%.*]], [[HD2:%.*]] ], [ 0, [[EN:%.*]] ]
-; CHECK-NEXT:    tail call void @h(i32* [[A]])
-; CHECK-NEXT:    br label [[HD2]]
-; CHECK:       hd2:
-; CHECK-NEXT:    [[TMP8]] = add nuw i32 [[TMP7]], 1
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], [[B]]
-; CHECK-NEXT:    br i1 [[TMP9]], label [[EX]], label [[HD]]
+; IS________OPM: Function Attrs: nounwind
+; IS________OPM-LABEL: define {{[^@]+}}@nonnull_exec_ctx_1b
+; IS________OPM-SAME: (i32* [[A:%.*]], i32 [[B:%.*]])
+; IS________OPM-NEXT:  en:
+; IS________OPM-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[B]], 0
+; IS________OPM-NEXT:    br i1 [[TMP3]], label [[EX:%.*]], label [[HD:%.*]]
+; IS________OPM:       ex:
+; IS________OPM-NEXT:    [[TMP5:%.*]] = tail call i32 @g(i32* nonnull [[A]])
+; IS________OPM-NEXT:    ret i32 [[TMP5]]
+; IS________OPM:       hd:
+; IS________OPM-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP8:%.*]], [[HD2:%.*]] ], [ 0, [[EN:%.*]] ]
+; IS________OPM-NEXT:    tail call void @h(i32* [[A]])
+; IS________OPM-NEXT:    br label [[HD2]]
+; IS________OPM:       hd2:
+; IS________OPM-NEXT:    [[TMP8]] = add nuw i32 [[TMP7]], 1
+; IS________OPM-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], [[B]]
+; IS________OPM-NEXT:    br i1 [[TMP9]], label [[EX]], label [[HD]]
+;
+; IS________NPM: Function Attrs: nounwind willreturn
+; IS________NPM-LABEL: define {{[^@]+}}@nonnull_exec_ctx_1b
+; IS________NPM-SAME: (i32* [[A:%.*]], i32 [[B:%.*]])
+; IS________NPM-NEXT:  en:
+; IS________NPM-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[B]], 0
+; IS________NPM-NEXT:    br i1 [[TMP3]], label [[EX:%.*]], label [[HD:%.*]]
+; IS________NPM:       ex:
+; IS________NPM-NEXT:    [[TMP5:%.*]] = tail call i32 @g(i32* nonnull [[A]])
+; IS________NPM-NEXT:    ret i32 [[TMP5]]
+; IS________NPM:       hd:
+; IS________NPM-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP8:%.*]], [[HD2:%.*]] ], [ 0, [[EN:%.*]] ]
+; IS________NPM-NEXT:    tail call void @h(i32* [[A]])
+; IS________NPM-NEXT:    br label [[HD2]]
+; IS________NPM:       hd2:
+; IS________NPM-NEXT:    [[TMP8]] = add nuw i32 [[TMP7]], 1
+; IS________NPM-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], [[B]]
+; IS________NPM-NEXT:    br i1 [[TMP9]], label [[EX]], label [[HD]]
 ;
 en:
   %tmp3 = icmp eq i32 %b, 0
@@ -982,6 +1126,7 @@ hd2:
 
 define i32 @nonnull_exec_ctx_2(i32* %a, i32 %b) willreturn nounwind {
 ;
+; CHECK: Function Attrs: nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@nonnull_exec_ctx_2
 ; CHECK-SAME: (i32* nonnull [[A:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:  en:
@@ -1015,6 +1160,7 @@ hd:
 
 define i32 @nonnull_exec_ctx_2b(i32* %a, i32 %b) willreturn nounwind {
 ;
+; CHECK: Function Attrs: nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@nonnull_exec_ctx_2b
 ; CHECK-SAME: (i32* nonnull [[A:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:  en:
@@ -1151,6 +1297,7 @@ declare i8* @strrchr(i8* %0, i32 %1) nofree nounwind readonly
 
 ; We should not mark the return of @strrchr as `nonnull`, it may well be NULL!
 define i8* @mybasename(i8* nofree readonly %str) {
+; CHECK: Function Attrs: nofree nounwind readonly
 ; CHECK-LABEL: define {{[^@]+}}@mybasename
 ; CHECK-SAME: (i8* nofree readonly [[STR:%.*]])
 ; CHECK-NEXT:    [[CALL:%.*]] = call i8* @strrchr(i8* nofree readonly [[STR]], i32 47)
diff --git a/llvm/test/Transforms/Attributor/norecurse.ll b/llvm/test/Transforms/Attributor/norecurse.ll
index 41e8d7c0bb116..da40ea6c23215 100644
--- a/llvm/test/Transforms/Attributor/norecurse.ll
+++ b/llvm/test/Transforms/Attributor/norecurse.ll
@@ -1,54 +1,61 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
-; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
-; IS__CGSSA____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 define i32 @leaf() {
-; CHECK-LABEL: define {{[^@]+}}@leaf()
-; CHECK-NEXT:    ret i32 1
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@leaf()
+; IS__TUNIT____-NEXT:    ret i32 1
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@leaf()
+; IS__CGSCC____-NEXT:    ret i32 1
 ;
   ret i32 1
 }
 
-; CHECK: Function Attrs
-; CHECK-SAME: readnone
-; CHECK-NOT: norecurse
 define i32 @self_rec() {
-; CHECK-LABEL: define {{[^@]+}}@self_rec()
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree noreturn nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@self_rec()
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@self_rec()
+; IS__CGSCC____-NEXT:    unreachable
 ;
   %a = call i32 @self_rec()
   ret i32 4
 }
 
-; CHECK: Function Attrs
-; CHECK-SAME: readnone
-; CHECK-NOT: norecurse
 define i32 @indirect_rec() {
-; CHECK-LABEL: define {{[^@]+}}@indirect_rec()
-; CHECK-NEXT:    unreachable
+; NOT_CGSCC_NPM: Function Attrs: nofree noreturn nosync nounwind readnone
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@indirect_rec()
+; NOT_CGSCC_NPM-NEXT:    unreachable
+;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@indirect_rec()
+; IS__CGSCC_NPM-NEXT:    unreachable
 ;
   %a = call i32 @indirect_rec2()
   ret i32 %a
 }
-; CHECK: Function Attrs
-; CHECK-SAME: readnone
-; CHECK-NOT: norecurse
 define i32 @indirect_rec2() {
-; CHECK-LABEL: define {{[^@]+}}@indirect_rec2()
-; CHECK-NEXT:    unreachable
+; NOT_CGSCC_NPM: Function Attrs: nofree noreturn nosync nounwind readnone
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@indirect_rec2()
+; NOT_CGSCC_NPM-NEXT:    unreachable
+;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@indirect_rec2()
+; IS__CGSCC_NPM-NEXT:    unreachable
 ;
   %a = call i32 @indirect_rec()
   ret i32 %a
 }
 
-; CHECK: Function Attrs
-; CHECK-SAME: readnone
-; CHECK-NOT: norecurse
 define i32 @extern() {
+; CHECK: Function Attrs: nosync readnone
 ; CHECK-LABEL: define {{[^@]+}}@extern()
 ; CHECK-NEXT:    [[A:%.*]] = call i32 @k()
 ; CHECK-NEXT:    ret i32 [[A]]
@@ -61,9 +68,8 @@ define i32 @extern() {
 ; CHECK-NEXT: declare i32 @k()
 declare i32 @k() readnone
 
-; CHECK: Function Attrs
-; CHECK-NOT: norecurse
 define void @intrinsic(i8* %dest, i8* %src, i32 %len) {
+; CHECK: Function Attrs: argmemonly nosync nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@intrinsic
 ; CHECK-SAME: (i8* nocapture writeonly [[DEST:%.*]], i8* nocapture readonly [[SRC:%.*]], i32 [[LEN:%.*]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* noalias nocapture writeonly [[DEST]], i8* noalias nocapture readonly [[SRC]], i32 [[LEN]], i1 false)
@@ -77,20 +83,22 @@ define void @intrinsic(i8* %dest, i8* %src, i32 %len) {
 ; CHECK-NEXT: declare void @llvm.memcpy.p0i8.p0i8.i32
 declare void @llvm.memcpy.p0i8.p0i8.i32(i8*, i8*, i32, i1)
 
-; IS__TUNIT____: Function Attrs
-; IS__TUNIT____-SAME: nosync readnone
-; IS__CGSSA____: Function Attrs
-; IS__CGSSA____-SAME: norecurse nosync readnone
 define internal i32 @called_by_norecurse() {
-; CHECK-LABEL: define {{[^@]+}}@called_by_norecurse()
-; CHECK-NEXT:    [[A:%.*]] = call i32 @k()
-; CHECK-NEXT:    ret i32 undef
+; IS__TUNIT____: Function Attrs: nosync readnone
+; IS__TUNIT____-LABEL: define {{[^@]+}}@called_by_norecurse()
+; IS__TUNIT____-NEXT:    [[A:%.*]] = call i32 @k()
+; IS__TUNIT____-NEXT:    ret i32 undef
+;
+; IS__CGSCC____: Function Attrs: norecurse nosync readnone
+; IS__CGSCC____-LABEL: define {{[^@]+}}@called_by_norecurse()
+; IS__CGSCC____-NEXT:    [[A:%.*]] = call i32 @k()
+; IS__CGSCC____-NEXT:    ret i32 undef
 ;
   %a = call i32 @k()
   ret i32 %a
 }
-; CHECK: Function Attrs
 define void @m() norecurse {
+; CHECK: Function Attrs: norecurse nosync readnone
 ; CHECK-LABEL: define {{[^@]+}}@m()
 ; CHECK-NEXT:    [[A:%.*]] = call i32 @called_by_norecurse()
 ; CHECK-NEXT:    ret void
@@ -99,10 +107,8 @@ define void @m() norecurse {
   ret void
 }
 
-; CHECK: Function Attrs
-; FIXME: norecurse missing
-; CHECK-SAME: nosync readnone
 define internal i32 @called_by_norecurse_indirectly() {
+; CHECK: Function Attrs: nosync readnone
 ; CHECK-LABEL: define {{[^@]+}}@called_by_norecurse_indirectly()
 ; CHECK-NEXT:    [[A:%.*]] = call i32 @k()
 ; CHECK-NEXT:    ret i32 [[A]]
@@ -110,19 +116,22 @@ define internal i32 @called_by_norecurse_indirectly() {
   %a = call i32 @k()
   ret i32 %a
 }
-; IS__TUNIT____: Function Attrs
-; IS__TUNIT____-SAME: nosync readnone
-; IS__CGSSA____: Function Attrs
-; IS__CGSSA____-SAME: norecurse nosync readnone
 define internal i32 @o() {
-; CHECK-LABEL: define {{[^@]+}}@o()
-; CHECK-NEXT:    [[A:%.*]] = call i32 @called_by_norecurse_indirectly()
-; CHECK-NEXT:    ret i32 [[A]]
+; IS__TUNIT____: Function Attrs: nosync readnone
+; IS__TUNIT____-LABEL: define {{[^@]+}}@o()
+; IS__TUNIT____-NEXT:    [[A:%.*]] = call i32 @called_by_norecurse_indirectly()
+; IS__TUNIT____-NEXT:    ret i32 [[A]]
+;
+; IS__CGSCC____: Function Attrs: norecurse nosync readnone
+; IS__CGSCC____-LABEL: define {{[^@]+}}@o()
+; IS__CGSCC____-NEXT:    [[A:%.*]] = call i32 @called_by_norecurse_indirectly()
+; IS__CGSCC____-NEXT:    ret i32 [[A]]
 ;
   %a = call i32 @called_by_norecurse_indirectly()
   ret i32 %a
 }
 define i32 @p() norecurse {
+; CHECK: Function Attrs: norecurse nosync readnone
 ; CHECK-LABEL: define {{[^@]+}}@p()
 ; CHECK-NEXT:    [[A:%.*]] = call i32 @o()
 ; CHECK-NEXT:    ret i32 [[A]]
@@ -131,9 +140,8 @@ define i32 @p() norecurse {
   ret i32 %a
 }
 
-; NOT_CGSCC_NPM: Function Attrs: nofree nosync nounwind
-; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 define void @f(i32 %x)  {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@f
 ; IS__TUNIT____-SAME: (i32 [[X:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
@@ -148,18 +156,33 @@ define void @f(i32 %x)  {
 ; IS__TUNIT____:       if.end:
 ; IS__TUNIT____-NEXT:    ret void
 ;
-; IS__CGSCC____-LABEL: define {{[^@]+}}@f
-; IS__CGSCC____-SAME: (i32 [[X:%.*]])
-; IS__CGSCC____-NEXT:  entry:
-; IS__CGSCC____-NEXT:    [[X_ADDR:%.*]] = alloca i32, align 4
-; IS__CGSCC____-NEXT:    store i32 [[X]], i32* [[X_ADDR]], align 4
-; IS__CGSCC____-NEXT:    [[TMP0:%.*]] = load i32, i32* [[X_ADDR]], align 4
-; IS__CGSCC____-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[TMP0]], 0
-; IS__CGSCC____-NEXT:    br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-; IS__CGSCC____:       if.then:
-; IS__CGSCC____-NEXT:    br label [[IF_END]]
-; IS__CGSCC____:       if.end:
-; IS__CGSCC____-NEXT:    ret void
+; IS__CGSCC_OPM: Function Attrs: nofree nosync nounwind readnone
+; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@f
+; IS__CGSCC_OPM-SAME: (i32 [[X:%.*]])
+; IS__CGSCC_OPM-NEXT:  entry:
+; IS__CGSCC_OPM-NEXT:    [[X_ADDR:%.*]] = alloca i32, align 4
+; IS__CGSCC_OPM-NEXT:    store i32 [[X]], i32* [[X_ADDR]], align 4
+; IS__CGSCC_OPM-NEXT:    [[TMP0:%.*]] = load i32, i32* [[X_ADDR]], align 4
+; IS__CGSCC_OPM-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[TMP0]], 0
+; IS__CGSCC_OPM-NEXT:    br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; IS__CGSCC_OPM:       if.then:
+; IS__CGSCC_OPM-NEXT:    br label [[IF_END]]
+; IS__CGSCC_OPM:       if.end:
+; IS__CGSCC_OPM-NEXT:    ret void
+;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@f
+; IS__CGSCC_NPM-SAME: (i32 [[X:%.*]])
+; IS__CGSCC_NPM-NEXT:  entry:
+; IS__CGSCC_NPM-NEXT:    [[X_ADDR:%.*]] = alloca i32, align 4
+; IS__CGSCC_NPM-NEXT:    store i32 [[X]], i32* [[X_ADDR]], align 4
+; IS__CGSCC_NPM-NEXT:    [[TMP0:%.*]] = load i32, i32* [[X_ADDR]], align 4
+; IS__CGSCC_NPM-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[TMP0]], 0
+; IS__CGSCC_NPM-NEXT:    br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; IS__CGSCC_NPM:       if.then:
+; IS__CGSCC_NPM-NEXT:    br label [[IF_END]]
+; IS__CGSCC_NPM:       if.end:
+; IS__CGSCC_NPM-NEXT:    ret void
 ;
 entry:
   %x.addr = alloca i32, align 4
@@ -177,21 +200,27 @@ if.end:
 }
 
 define void @g() norecurse {
-; NOT_CGSCC_OPM-LABEL: define {{[^@]+}}@g()
-; NOT_CGSCC_OPM-NEXT:  entry:
-; NOT_CGSCC_OPM-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree norecurse nosync nounwind readnone
+; IS__TUNIT____-LABEL: define {{[^@]+}}@g()
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    ret void
 ;
+; IS__CGSCC_OPM: Function Attrs: nofree norecurse nosync nounwind readnone
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@g()
 ; IS__CGSCC_OPM-NEXT:  entry:
 ; IS__CGSCC_OPM-NEXT:    call void @f(i32 0)
 ; IS__CGSCC_OPM-NEXT:    ret void
 ;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@g()
+; IS__CGSCC_NPM-NEXT:  entry:
+; IS__CGSCC_NPM-NEXT:    ret void
+;
 entry:
   call void @f(i32 0)
   ret void
 }
 
-; CHECK-NOT: Function Attrs
 define linkonce_odr i32 @leaf_redefinable() {
 ; CHECK-LABEL: define {{[^@]+}}@leaf_redefinable()
 ; CHECK-NEXT:    ret i32 1
@@ -200,7 +229,6 @@ define linkonce_odr i32 @leaf_redefinable() {
 }
 
 ; Call through a function pointer
-; CHECK-NOT: Function Attrs
 define i32 @eval_func1(i32 (i32)* , i32) local_unnamed_addr {
 ; CHECK-LABEL: define {{[^@]+}}@eval_func1
 ; CHECK-SAME: (i32 (i32)* nocapture nofree nonnull [[TMP0:%.*]], i32 [[TMP1:%.*]]) local_unnamed_addr
@@ -211,8 +239,8 @@ define i32 @eval_func1(i32 (i32)* , i32) local_unnamed_addr {
   ret i32 %3
 }
 
-; CHECK: Function Attrs: null_pointer_is_valid
 define i32 @eval_func2(i32 (i32)* , i32) local_unnamed_addr null_pointer_is_valid{
+; CHECK: Function Attrs: null_pointer_is_valid
 ; CHECK-LABEL: define {{[^@]+}}@eval_func2
 ; CHECK-SAME: (i32 (i32)* nocapture nofree [[TMP0:%.*]], i32 [[TMP1:%.*]]) local_unnamed_addr
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 [[TMP0]](i32 [[TMP1]])
@@ -223,13 +251,19 @@ define i32 @eval_func2(i32 (i32)* , i32) local_unnamed_addr null_pointer_is_vali
 }
 
 ; Call an unknown function in a dead block.
-; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 declare void @unknown()
 define i32 @call_unknown_in_dead_block() local_unnamed_addr {
-; CHECK-LABEL: define {{[^@]+}}@call_unknown_in_dead_block() local_unnamed_addr
-; CHECK-NEXT:    ret i32 0
-; CHECK:       Dead:
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@call_unknown_in_dead_block() local_unnamed_addr
+; IS__TUNIT____-NEXT:    ret i32 0
+; IS__TUNIT____:       Dead:
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@call_unknown_in_dead_block() local_unnamed_addr
+; IS__CGSCC____-NEXT:    ret i32 0
+; IS__CGSCC____:       Dead:
+; IS__CGSCC____-NEXT:    unreachable
 ;
   ret i32 0
 Dead:
diff --git a/llvm/test/Transforms/Attributor/noreturn.ll b/llvm/test/Transforms/Attributor/noreturn.ll
index 0520a52a47b96..ea0e2bd6103f7 100644
--- a/llvm/test/Transforms/Attributor/noreturn.ll
+++ b/llvm/test/Transforms/Attributor/noreturn.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -16,12 +16,16 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 ;   return srec0();
 ; }
 ;
-; IS__TUNIT____: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable
-; IS__CGSCC____: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readnone uwtable
 define void @srec0() #0 {
-; CHECK-LABEL: define {{[^@]+}}@srec0()
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@srec0()
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@srec0()
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    unreachable
 ;
 entry:
   call void @srec0()
@@ -35,15 +39,22 @@ entry:
 ;   return srec16(srec16(srec16(srec16(srec16(srec16(srec16(srec16(srec16(srec16(srec16(srec16(srec16(srec16(srec16(srec16(a))))))))))))))));
 ; }
 ;
-; IS__TUNIT____: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable
-; IS__CGSCC____: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readnone uwtable
 define i32 @srec16(i32 %a) #0 {
-; CHECK-LABEL: define {{[^@]+}}@srec16
-; CHECK-SAME: (i32 [[A:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    unreachable
-; CHECK:       exit:
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@srec16
+; IS__TUNIT____-SAME: (i32 [[A:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       exit:
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@srec16
+; IS__CGSCC____-SAME: (i32 [[A:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       exit:
+; IS__CGSCC____-NEXT:    unreachable
 ;
 entry:
   %call = call i32 @srec16(i32 %a)
@@ -75,15 +86,22 @@ exit:
 ;   while (1);
 ; }
 ;
+define i32 @endless_loop(i32 %a) #0 {
 ; IS__TUNIT____: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable
+; IS__TUNIT____-LABEL: define {{[^@]+}}@endless_loop
+; IS__TUNIT____-SAME: (i32 [[A:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    br label [[WHILE_BODY:%.*]]
+; IS__TUNIT____:       while.body:
+; IS__TUNIT____-NEXT:    br label [[WHILE_BODY]]
+;
 ; IS__CGSCC____: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readnone uwtable
-define i32 @endless_loop(i32 %a) #0 {
-; CHECK-LABEL: define {{[^@]+}}@endless_loop
-; CHECK-SAME: (i32 [[A:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
-; CHECK:       while.body:
-; CHECK-NEXT:    br label [[WHILE_BODY]]
+; IS__CGSCC____-LABEL: define {{[^@]+}}@endless_loop
+; IS__CGSCC____-SAME: (i32 [[A:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    br label [[WHILE_BODY:%.*]]
+; IS__CGSCC____:       while.body:
+; IS__CGSCC____-NEXT:    br label [[WHILE_BODY]]
 ;
 entry:
   br label %while.body
@@ -101,17 +119,26 @@ while.body:                                       ; preds = %entry, %while.body
 ; }
 ;
 ; FIXME: no-return missing (D65243 should fix this)
+define i32 @dead_return(i32 %a) #0 {
 ; IS__TUNIT____: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable
+; IS__TUNIT____-LABEL: define {{[^@]+}}@dead_return
+; IS__TUNIT____-SAME: (i32 [[A:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    br label [[WHILE_BODY:%.*]]
+; IS__TUNIT____:       while.body:
+; IS__TUNIT____-NEXT:    br label [[WHILE_BODY]]
+; IS__TUNIT____:       return:
+; IS__TUNIT____-NEXT:    unreachable
+;
 ; IS__CGSCC____: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readnone uwtable
-define i32 @dead_return(i32 %a) #0 {
-; CHECK-LABEL: define {{[^@]+}}@dead_return
-; CHECK-SAME: (i32 [[A:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
-; CHECK:       while.body:
-; CHECK-NEXT:    br label [[WHILE_BODY]]
-; CHECK:       return:
-; CHECK-NEXT:    unreachable
+; IS__CGSCC____-LABEL: define {{[^@]+}}@dead_return
+; IS__CGSCC____-SAME: (i32 [[A:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    br label [[WHILE_BODY:%.*]]
+; IS__CGSCC____:       while.body:
+; IS__CGSCC____-NEXT:    br label [[WHILE_BODY]]
+; IS__CGSCC____:       return:
+; IS__CGSCC____-NEXT:    unreachable
 ;
 entry:
   br label %while.body
@@ -130,20 +157,32 @@ return:                                           ; No predecessors!
 ;   return a == 0 ? endless_loop(a) : srec16(a);
 ; }
 ;
-; IS__TUNIT____: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable
-; IS__CGSCC____: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readnone uwtable
 define i32 @multiple_noreturn_calls(i32 %a) #0 {
-; CHECK-LABEL: define {{[^@]+}}@multiple_noreturn_calls
-; CHECK-SAME: (i32 [[A:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A]], 0
-; CHECK-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-; CHECK:       cond.true:
-; CHECK-NEXT:    unreachable
-; CHECK:       cond.false:
-; CHECK-NEXT:    unreachable
-; CHECK:       cond.end:
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@multiple_noreturn_calls
+; IS__TUNIT____-SAME: (i32 [[A:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A]], 0
+; IS__TUNIT____-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+; IS__TUNIT____:       cond.true:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       cond.false:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       cond.end:
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@multiple_noreturn_calls
+; IS__CGSCC____-SAME: (i32 [[A:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A]], 0
+; IS__CGSCC____-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+; IS__CGSCC____:       cond.true:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       cond.false:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       cond.end:
+; IS__CGSCC____-NEXT:    unreachable
 ;
 entry:
   %cmp = icmp eq i32 %a, 0
@@ -166,14 +205,20 @@ cond.end:                                         ; preds = %cond.false, %cond.t
 ; TEST 6a: willreturn means *not* no-return or UB
 ; FIXME: we should derive "UB" as an argument and report it to the user on request.
 
+define i32 @endless_loop_but_willreturn() willreturn {
 ; IS__TUNIT____: Function Attrs: nofree noreturn nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@endless_loop_but_willreturn()
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    br label [[WHILE_BODY:%.*]]
+; IS__TUNIT____:       while.body:
+; IS__TUNIT____-NEXT:    br label [[WHILE_BODY]]
+;
 ; IS__CGSCC____: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
-define i32 @endless_loop_but_willreturn() willreturn {
-; CHECK-LABEL: define {{[^@]+}}@endless_loop_but_willreturn()
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
-; CHECK:       while.body:
-; CHECK-NEXT:    br label [[WHILE_BODY]]
+; IS__CGSCC____-LABEL: define {{[^@]+}}@endless_loop_but_willreturn()
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    br label [[WHILE_BODY:%.*]]
+; IS__CGSCC____:       while.body:
+; IS__CGSCC____-NEXT:    br label [[WHILE_BODY]]
 ;
 entry:
   br label %while.body
@@ -183,12 +228,16 @@ while.body:                                       ; preds = %entry, %while.body
 }
 
 ; TEST 6b: willreturn means *not* no-return or UB
+define i32 @UB_and_willreturn() willreturn {
 ; IS__TUNIT____: Function Attrs: nofree noreturn nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@UB_and_willreturn()
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    unreachable
+;
 ; IS__CGSCC____: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
-define i32 @UB_and_willreturn() willreturn {
-; CHECK-LABEL: define {{[^@]+}}@UB_and_willreturn()
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    unreachable
+; IS__CGSCC____-LABEL: define {{[^@]+}}@UB_and_willreturn()
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    unreachable
 ;
 entry:
   unreachable
diff --git a/llvm/test/Transforms/Attributor/noreturn_async.ll b/llvm/test/Transforms/Attributor/noreturn_async.ll
index c632a4e6d1181..4c0fc203eb095 100644
--- a/llvm/test/Transforms/Attributor/noreturn_async.ll
+++ b/llvm/test/Transforms/Attributor/noreturn_async.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s
 ;
 ; This file is the same as noreturn_sync.ll but with a personality which
diff --git a/llvm/test/Transforms/Attributor/noreturn_sync.ll b/llvm/test/Transforms/Attributor/noreturn_sync.ll
index 055a324e02ba0..22b675427cf01 100644
--- a/llvm/test/Transforms/Attributor/noreturn_sync.ll
+++ b/llvm/test/Transforms/Attributor/noreturn_sync.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s
 ;
 ; This file is the same as noreturn_async.ll but with a personality which
@@ -41,7 +42,7 @@ define dso_local i32 @"?catchoverflow@@YAHXZ"()  personality i8* bitcast (i32 (.
 entry:
   %retval = alloca i32, align 4
   %__exception_code = alloca i32, align 4
-  invoke void @"?overflow@@YAXXZ"() 
+  invoke void @"?overflow@@YAXXZ"()
           to label %invoke.cont unwind label %catch.dispatch
 ; CHECK:      call void @"?overflow@@YAXXZ"()
 ; CHECK-NEXT: unreachable
@@ -98,9 +99,9 @@ define dso_local i32 @"?catchoverflow@@YAHXZ_may_throw"()  personality i8* bitca
 entry:
   %retval = alloca i32, align 4
   %__exception_code = alloca i32, align 4
-; CHECK: invoke void @"?overflow@@YAXXZ_may_throw"() 
+; CHECK: invoke void @"?overflow@@YAXXZ_may_throw"()
 ; CHECK:          to label %invoke.cont unwind label %catch.dispatch
-  invoke void @"?overflow@@YAXXZ_may_throw"() 
+  invoke void @"?overflow@@YAXXZ_may_throw"()
           to label %invoke.cont unwind label %catch.dispatch
 
 invoke.cont:                                      ; preds = %entry
diff --git a/llvm/test/Transforms/Attributor/nosync.ll b/llvm/test/Transforms/Attributor/nosync.ll
index fc01770ee47e9..33614b636f53e 100644
--- a/llvm/test/Transforms/Attributor/nosync.ll
+++ b/llvm/test/Transforms/Attributor/nosync.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -28,14 +28,20 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 %struct.RT = type { i8, [10 x [20 x i32]], i8 }
 %struct.ST = type { i32, double, %struct.RT }
 
-; IS__TUNIT____: Function Attrs: nofree nosync nounwind optsize readnone ssp uwtable
-; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind optsize readnone ssp uwtable
 define i32* @foo(%struct.ST* %s) nounwind uwtable readnone optsize ssp {
-; CHECK-LABEL: define {{[^@]+}}@foo
-; CHECK-SAME: (%struct.ST* nofree readnone "no-capture-maybe-returned" [[S:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [[STRUCT_ST:%.*]], %struct.ST* [[S]], i64 1, i32 2, i32 1, i64 5, i64 13
-; CHECK-NEXT:    ret i32* [[ARRAYIDX]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind optsize readnone ssp uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@foo
+; IS__TUNIT____-SAME: (%struct.ST* nofree readnone "no-capture-maybe-returned" [[S:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [[STRUCT_ST:%.*]], %struct.ST* [[S]], i64 1, i32 2, i32 1, i64 5, i64 13
+; IS__TUNIT____-NEXT:    ret i32* [[ARRAYIDX]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind optsize readnone ssp uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@foo
+; IS__CGSCC____-SAME: (%struct.ST* nofree readnone "no-capture-maybe-returned" [[S:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [[STRUCT_ST:%.*]], %struct.ST* [[S]], i64 1, i32 2, i32 1, i64 5, i64 13
+; IS__CGSCC____-NEXT:    ret i32* [[ARRAYIDX]]
 ;
 entry:
   %arrayidx = getelementptr inbounds %struct.ST, %struct.ST* %s, i64 1, i32 2, i32 1, i64 5, i64 13
@@ -49,9 +55,8 @@ entry:
 ;   return n;
 ; }
 
-; IS__TUNIT____: Function Attrs: argmemonly nofree norecurse nosync nounwind uwtable
-; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind uwtable
 define i32 @load_monotonic(i32* nocapture readonly %0) norecurse nounwind uwtable {
+; CHECK: Function Attrs: argmemonly nofree norecurse nosync nounwind uwtable willreturn
 ; CHECK-LABEL: define {{[^@]+}}@load_monotonic
 ; CHECK-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[TMP0:%.*]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = load atomic i32, i32* [[TMP0]] monotonic, align 4
@@ -68,9 +73,8 @@ define i32 @load_monotonic(i32* nocapture readonly %0) norecurse nounwind uwtabl
 ;   atomic_load_explicit(num, memory_order_relaxed);
 ; }
 
-; IS__TUNIT____: Function Attrs: argmemonly nofree norecurse nosync nounwind uwtable
-; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind uwtable
 define void @store_monotonic(i32* nocapture %0) norecurse nounwind uwtable {
+; CHECK: Function Attrs: argmemonly nofree norecurse nosync nounwind uwtable willreturn
 ; CHECK-LABEL: define {{[^@]+}}@store_monotonic
 ; CHECK-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[TMP0:%.*]])
 ; CHECK-NEXT:    store atomic i32 10, i32* [[TMP0]] monotonic, align 4
@@ -87,10 +91,8 @@ define void @store_monotonic(i32* nocapture %0) norecurse nounwind uwtable {
 ;   return n;
 ; }
 
-; IS__TUNIT____: Function Attrs: argmemonly nofree norecurse nounwind uwtable
-; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nounwind uwtable
-; CHECK-NOT: nosync
 define i32 @load_acquire(i32* nocapture readonly %0) norecurse nounwind uwtable {
+; CHECK: Function Attrs: argmemonly nofree norecurse nounwind uwtable willreturn
 ; CHECK-LABEL: define {{[^@]+}}@load_acquire
 ; CHECK-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[TMP0:%.*]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = load atomic i32, i32* [[TMP0]] acquire, align 4
@@ -106,10 +108,8 @@ define i32 @load_acquire(i32* nocapture readonly %0) norecurse nounwind uwtable
 ;   atomic_store_explicit(num, 10, memory_order_release);
 ; }
 
-; IS__TUNIT____: Function Attrs: argmemonly nofree norecurse nounwind uwtable
-; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nounwind uwtable
-; CHECK-NOT: nosync
 define void @load_release(i32* nocapture %0) norecurse nounwind uwtable {
+; CHECK: Function Attrs: argmemonly nofree norecurse nounwind uwtable willreturn
 ; CHECK-LABEL: define {{[^@]+}}@load_release
 ; CHECK-SAME: (i32* nocapture nofree writeonly align 4 [[TMP0:%.*]])
 ; CHECK-NEXT:    store atomic volatile i32 10, i32* [[TMP0]] release, align 4
@@ -121,10 +121,8 @@ define void @load_release(i32* nocapture %0) norecurse nounwind uwtable {
 
 ; TEST 6 - negative volatile, relaxed atomic
 
-; IS__TUNIT____: Function Attrs: argmemonly nofree norecurse nounwind uwtable
-; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nounwind uwtable
-; CHECK-NOT: nosync
 define void @load_volatile_release(i32* nocapture %0) norecurse nounwind uwtable {
+; CHECK: Function Attrs: argmemonly nofree norecurse nounwind uwtable willreturn
 ; CHECK-LABEL: define {{[^@]+}}@load_volatile_release
 ; CHECK-SAME: (i32* nocapture nofree writeonly align 4 [[TMP0:%.*]])
 ; CHECK-NEXT:    store atomic volatile i32 10, i32* [[TMP0]] release, align 4
@@ -140,10 +138,8 @@ define void @load_volatile_release(i32* nocapture %0) norecurse nounwind uwtable
 ;   *num = 14;
 ; }
 
-; IS__TUNIT____: Function Attrs: argmemonly nofree norecurse nounwind uwtable
-; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nounwind uwtable
-; CHECK-NOT: nosync
 define void @volatile_store(i32* %0) norecurse nounwind uwtable {
+; CHECK: Function Attrs: argmemonly nofree norecurse nounwind uwtable willreturn
 ; CHECK-LABEL: define {{[^@]+}}@volatile_store
 ; CHECK-SAME: (i32* nofree align 4 [[TMP0:%.*]])
 ; CHECK-NEXT:    store volatile i32 14, i32* [[TMP0]], align 4
@@ -160,10 +156,8 @@ define void @volatile_store(i32* %0) norecurse nounwind uwtable {
 ;   return n;
 ; }
 
-; IS__TUNIT____: Function Attrs: argmemonly nofree norecurse nounwind uwtable
-; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nounwind uwtable
-; CHECK-NOT: nosync
 define i32 @volatile_load(i32* %0) norecurse nounwind uwtable {
+; CHECK: Function Attrs: argmemonly nofree norecurse nounwind uwtable willreturn
 ; CHECK-LABEL: define {{[^@]+}}@volatile_load
 ; CHECK-SAME: (i32* nofree align 4 [[TMP0:%.*]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = load volatile i32, i32* [[TMP0]], align 4
@@ -179,9 +173,8 @@ define i32 @volatile_load(i32* %0) norecurse nounwind uwtable {
 ; CHECK-NEXT: declare void @nosync_function()
 declare void @nosync_function() noinline nounwind uwtable nosync
 
-; IS__TUNIT____: Function Attrs: noinline nosync nounwind uwtable
-; IS__CGSCC____: Function Attrs: noinline nosync nounwind uwtable
 define void @call_nosync_function() nounwind uwtable noinline {
+; CHECK: Function Attrs: noinline nosync nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@call_nosync_function()
 ; CHECK-NEXT:    tail call void @nosync_function()
 ; CHECK-NEXT:    ret void
@@ -196,10 +189,8 @@ define void @call_nosync_function() nounwind uwtable noinline {
 ; CHECK-NEXT: declare void @might_sync()
 declare void @might_sync() noinline nounwind uwtable
 
-; IS__TUNIT____: Function Attrs: noinline nounwind uwtable
-; IS__CGSCC____: Function Attrs: noinline nounwind uwtable
-; CHECK-NOT: nosync
 define void @call_might_sync() nounwind uwtable noinline {
+; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@call_might_sync()
 ; CHECK-NEXT:    tail call void @might_sync()
 ; CHECK-NEXT:    ret void
@@ -211,26 +202,32 @@ define void @call_might_sync() nounwind uwtable noinline {
 ; TEST 11 - positive, should deduce nosync
 ; volatile operation in same scc but dead. Call volatile_load defined in TEST 8.
 
-; IS__TUNIT____: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable
-; IS__CGSCC_OPM: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable
-; IS__CGSCC_NPM: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readnone uwtable
 define i32 @scc1(i32* %0) noinline nounwind uwtable {
-; CHECK-LABEL: define {{[^@]+}}@scc1
-; CHECK-SAME: (i32* nocapture nofree readnone [[TMP0:%.*]])
-; CHECK-NEXT:    unreachable
+; NOT_CGSCC_NPM: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable willreturn
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@scc1
+; NOT_CGSCC_NPM-SAME: (i32* nocapture nofree readnone [[TMP0:%.*]])
+; NOT_CGSCC_NPM-NEXT:    unreachable
+;
+; IS__CGSCC_NPM: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readnone uwtable willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@scc1
+; IS__CGSCC_NPM-SAME: (i32* nocapture nofree readnone [[TMP0:%.*]])
+; IS__CGSCC_NPM-NEXT:    unreachable
 ;
   tail call void @scc2(i32* %0);
   %val = tail call i32 @volatile_load(i32* %0);
   ret i32 %val;
 }
 
-; IS__TUNIT____: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable
-; IS__CGSCC_OPM: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable
-; IS__CGSCC_NPM: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readnone uwtable
 define void @scc2(i32* %0) noinline nounwind uwtable {
-; CHECK-LABEL: define {{[^@]+}}@scc2
-; CHECK-SAME: (i32* nocapture nofree readnone [[TMP0:%.*]])
-; CHECK-NEXT:    unreachable
+; NOT_CGSCC_NPM: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable willreturn
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@scc2
+; NOT_CGSCC_NPM-SAME: (i32* nocapture nofree readnone [[TMP0:%.*]])
+; NOT_CGSCC_NPM-NEXT:    unreachable
+;
+; IS__CGSCC_NPM: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readnone uwtable willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@scc2
+; IS__CGSCC_NPM-SAME: (i32* nocapture nofree readnone [[TMP0:%.*]])
+; IS__CGSCC_NPM-NEXT:    unreachable
 ;
   tail call i32 @scc1(i32* %0);
   ret void;
@@ -255,15 +252,24 @@ define void @scc2(i32* %0) noinline nounwind uwtable {
 %"struct.std::atomic" = type { %"struct.std::__atomic_base" }
 %"struct.std::__atomic_base" = type { i8 }
 
-; CHECK-NOT: nosync
 define void @foo1(i32* %0, %"struct.std::atomic"* %1) {
-; CHECK-LABEL: define {{[^@]+}}@foo1
-; CHECK-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[TMP0:%.*]], %"struct.std::atomic"* nocapture nofree nonnull writeonly dereferenceable(1) [[TMP1:%.*]])
-; CHECK-NEXT:    store i32 100, i32* [[TMP0]], align 4
-; CHECK-NEXT:    fence release
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds %"struct.std::atomic", %"struct.std::atomic"* [[TMP1]], i64 0, i32 0, i32 0
-; CHECK-NEXT:    store atomic i8 1, i8* [[TMP3]] monotonic, align 1
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nounwind willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@foo1
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[TMP0:%.*]], %"struct.std::atomic"* nocapture nofree nonnull writeonly dereferenceable(1) [[TMP1:%.*]])
+; IS__TUNIT____-NEXT:    store i32 100, i32* [[TMP0]], align 4
+; IS__TUNIT____-NEXT:    fence release
+; IS__TUNIT____-NEXT:    [[TMP3:%.*]] = getelementptr inbounds %"struct.std::atomic", %"struct.std::atomic"* [[TMP1]], i64 0, i32 0, i32 0
+; IS__TUNIT____-NEXT:    store atomic i8 1, i8* [[TMP3]] monotonic, align 1
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nounwind willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@foo1
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[TMP0:%.*]], %"struct.std::atomic"* nocapture nofree nonnull writeonly dereferenceable(1) [[TMP1:%.*]])
+; IS__CGSCC____-NEXT:    store i32 100, i32* [[TMP0]], align 4
+; IS__CGSCC____-NEXT:    fence release
+; IS__CGSCC____-NEXT:    [[TMP3:%.*]] = getelementptr inbounds %"struct.std::atomic", %"struct.std::atomic"* [[TMP1]], i64 0, i32 0, i32 0
+; IS__CGSCC____-NEXT:    store atomic i8 1, i8* [[TMP3]] monotonic, align 1
+; IS__CGSCC____-NEXT:    ret void
 ;
   store i32 100, i32* %0, align 4
   fence release
@@ -272,20 +278,34 @@ define void @foo1(i32* %0, %"struct.std::atomic"* %1) {
   ret void
 }
 
-; CHECK-NOT: nosync
 define void @bar(i32* %0, %"struct.std::atomic"* %1) {
-; CHECK-LABEL: define {{[^@]+}}@bar
-; CHECK-SAME: (i32* nocapture nofree readnone [[TMP0:%.*]], %"struct.std::atomic"* nocapture nofree nonnull readonly dereferenceable(1) [[TMP1:%.*]])
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds %"struct.std::atomic", %"struct.std::atomic"* [[TMP1]], i64 0, i32 0, i32 0
-; CHECK-NEXT:    br label [[TMP4:%.*]]
-; CHECK:       4:
-; CHECK-NEXT:    [[TMP5:%.*]] = load atomic i8, i8* [[TMP3]] monotonic, align 1
-; CHECK-NEXT:    [[TMP6:%.*]] = and i8 [[TMP5]], 1
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i8 [[TMP6]], 0
-; CHECK-NEXT:    br i1 [[TMP7]], label [[TMP4]], label [[TMP8:%.*]]
-; CHECK:       8:
-; CHECK-NEXT:    fence acquire
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nounwind
+; IS__TUNIT____-LABEL: define {{[^@]+}}@bar
+; IS__TUNIT____-SAME: (i32* nocapture nofree readnone [[TMP0:%.*]], %"struct.std::atomic"* nocapture nofree nonnull readonly dereferenceable(1) [[TMP1:%.*]])
+; IS__TUNIT____-NEXT:    [[TMP3:%.*]] = getelementptr inbounds %"struct.std::atomic", %"struct.std::atomic"* [[TMP1]], i64 0, i32 0, i32 0
+; IS__TUNIT____-NEXT:    br label [[TMP4:%.*]]
+; IS__TUNIT____:       4:
+; IS__TUNIT____-NEXT:    [[TMP5:%.*]] = load atomic i8, i8* [[TMP3]] monotonic, align 1
+; IS__TUNIT____-NEXT:    [[TMP6:%.*]] = and i8 [[TMP5]], 1
+; IS__TUNIT____-NEXT:    [[TMP7:%.*]] = icmp eq i8 [[TMP6]], 0
+; IS__TUNIT____-NEXT:    br i1 [[TMP7]], label [[TMP4]], label [[TMP8:%.*]]
+; IS__TUNIT____:       8:
+; IS__TUNIT____-NEXT:    fence acquire
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nounwind
+; IS__CGSCC____-LABEL: define {{[^@]+}}@bar
+; IS__CGSCC____-SAME: (i32* nocapture nofree readnone [[TMP0:%.*]], %"struct.std::atomic"* nocapture nofree nonnull readonly dereferenceable(1) [[TMP1:%.*]])
+; IS__CGSCC____-NEXT:    [[TMP3:%.*]] = getelementptr inbounds %"struct.std::atomic", %"struct.std::atomic"* [[TMP1]], i64 0, i32 0, i32 0
+; IS__CGSCC____-NEXT:    br label [[TMP4:%.*]]
+; IS__CGSCC____:       4:
+; IS__CGSCC____-NEXT:    [[TMP5:%.*]] = load atomic i8, i8* [[TMP3]] monotonic, align 1
+; IS__CGSCC____-NEXT:    [[TMP6:%.*]] = and i8 [[TMP5]], 1
+; IS__CGSCC____-NEXT:    [[TMP7:%.*]] = icmp eq i8 [[TMP6]], 0
+; IS__CGSCC____-NEXT:    br i1 [[TMP7]], label [[TMP4]], label [[TMP8:%.*]]
+; IS__CGSCC____:       8:
+; IS__CGSCC____-NEXT:    fence acquire
+; IS__CGSCC____-NEXT:    ret void
 ;
   %3 = getelementptr inbounds %"struct.std::atomic", %"struct.std::atomic"* %1, i64 0, i32 0, i32 0
   br label %4
@@ -302,16 +322,24 @@ define void @bar(i32* %0, %"struct.std::atomic"* %1) {
 }
 
 ; TEST 13 - Fence syncscope("singlethread") seq_cst
+define void @foo1_singlethread(i32* %0, %"struct.std::atomic"* %1) {
 ; IS__TUNIT____: Function Attrs: nofree nosync nounwind willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@foo1_singlethread
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[TMP0:%.*]], %"struct.std::atomic"* nocapture nofree nonnull writeonly dereferenceable(1) [[TMP1:%.*]])
+; IS__TUNIT____-NEXT:    store i32 100, i32* [[TMP0]], align 4
+; IS__TUNIT____-NEXT:    fence syncscope("singlethread") release
+; IS__TUNIT____-NEXT:    [[TMP3:%.*]] = getelementptr inbounds %"struct.std::atomic", %"struct.std::atomic"* [[TMP1]], i64 0, i32 0, i32 0
+; IS__TUNIT____-NEXT:    store atomic i8 1, i8* [[TMP3]] monotonic, align 1
+; IS__TUNIT____-NEXT:    ret void
+;
 ; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn
-define void @foo1_singlethread(i32* %0, %"struct.std::atomic"* %1) {
-; CHECK-LABEL: define {{[^@]+}}@foo1_singlethread
-; CHECK-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[TMP0:%.*]], %"struct.std::atomic"* nocapture nofree nonnull writeonly dereferenceable(1) [[TMP1:%.*]])
-; CHECK-NEXT:    store i32 100, i32* [[TMP0]], align 4
-; CHECK-NEXT:    fence syncscope("singlethread") release
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds %"struct.std::atomic", %"struct.std::atomic"* [[TMP1]], i64 0, i32 0, i32 0
-; CHECK-NEXT:    store atomic i8 1, i8* [[TMP3]] monotonic, align 1
-; CHECK-NEXT:    ret void
+; IS__CGSCC____-LABEL: define {{[^@]+}}@foo1_singlethread
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[TMP0:%.*]], %"struct.std::atomic"* nocapture nofree nonnull writeonly dereferenceable(1) [[TMP1:%.*]])
+; IS__CGSCC____-NEXT:    store i32 100, i32* [[TMP0]], align 4
+; IS__CGSCC____-NEXT:    fence syncscope("singlethread") release
+; IS__CGSCC____-NEXT:    [[TMP3:%.*]] = getelementptr inbounds %"struct.std::atomic", %"struct.std::atomic"* [[TMP1]], i64 0, i32 0, i32 0
+; IS__CGSCC____-NEXT:    store atomic i8 1, i8* [[TMP3]] monotonic, align 1
+; IS__CGSCC____-NEXT:    ret void
 ;
   store i32 100, i32* %0, align 4
   fence syncscope("singlethread") release
@@ -320,21 +348,34 @@ define void @foo1_singlethread(i32* %0, %"struct.std::atomic"* %1) {
   ret void
 }
 
+define void @bar_singlethread(i32* %0, %"struct.std::atomic"* %1) {
 ; IS__TUNIT____: Function Attrs: nofree nosync nounwind
+; IS__TUNIT____-LABEL: define {{[^@]+}}@bar_singlethread
+; IS__TUNIT____-SAME: (i32* nocapture nofree readnone [[TMP0:%.*]], %"struct.std::atomic"* nocapture nofree nonnull readonly dereferenceable(1) [[TMP1:%.*]])
+; IS__TUNIT____-NEXT:    [[TMP3:%.*]] = getelementptr inbounds %"struct.std::atomic", %"struct.std::atomic"* [[TMP1]], i64 0, i32 0, i32 0
+; IS__TUNIT____-NEXT:    br label [[TMP4:%.*]]
+; IS__TUNIT____:       4:
+; IS__TUNIT____-NEXT:    [[TMP5:%.*]] = load atomic i8, i8* [[TMP3]] monotonic, align 1
+; IS__TUNIT____-NEXT:    [[TMP6:%.*]] = and i8 [[TMP5]], 1
+; IS__TUNIT____-NEXT:    [[TMP7:%.*]] = icmp eq i8 [[TMP6]], 0
+; IS__TUNIT____-NEXT:    br i1 [[TMP7]], label [[TMP4]], label [[TMP8:%.*]]
+; IS__TUNIT____:       8:
+; IS__TUNIT____-NEXT:    fence syncscope("singlethread") acquire
+; IS__TUNIT____-NEXT:    ret void
+;
 ; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind
-define void @bar_singlethread(i32* %0, %"struct.std::atomic"* %1) {
-; CHECK-LABEL: define {{[^@]+}}@bar_singlethread
-; CHECK-SAME: (i32* nocapture nofree readnone [[TMP0:%.*]], %"struct.std::atomic"* nocapture nofree nonnull readonly dereferenceable(1) [[TMP1:%.*]])
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds %"struct.std::atomic", %"struct.std::atomic"* [[TMP1]], i64 0, i32 0, i32 0
-; CHECK-NEXT:    br label [[TMP4:%.*]]
-; CHECK:       4:
-; CHECK-NEXT:    [[TMP5:%.*]] = load atomic i8, i8* [[TMP3]] monotonic, align 1
-; CHECK-NEXT:    [[TMP6:%.*]] = and i8 [[TMP5]], 1
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i8 [[TMP6]], 0
-; CHECK-NEXT:    br i1 [[TMP7]], label [[TMP4]], label [[TMP8:%.*]]
-; CHECK:       8:
-; CHECK-NEXT:    fence syncscope("singlethread") acquire
-; CHECK-NEXT:    ret void
+; IS__CGSCC____-LABEL: define {{[^@]+}}@bar_singlethread
+; IS__CGSCC____-SAME: (i32* nocapture nofree readnone [[TMP0:%.*]], %"struct.std::atomic"* nocapture nofree nonnull readonly dereferenceable(1) [[TMP1:%.*]])
+; IS__CGSCC____-NEXT:    [[TMP3:%.*]] = getelementptr inbounds %"struct.std::atomic", %"struct.std::atomic"* [[TMP1]], i64 0, i32 0, i32 0
+; IS__CGSCC____-NEXT:    br label [[TMP4:%.*]]
+; IS__CGSCC____:       4:
+; IS__CGSCC____-NEXT:    [[TMP5:%.*]] = load atomic i8, i8* [[TMP3]] monotonic, align 1
+; IS__CGSCC____-NEXT:    [[TMP6:%.*]] = and i8 [[TMP5]], 1
+; IS__CGSCC____-NEXT:    [[TMP7:%.*]] = icmp eq i8 [[TMP6]], 0
+; IS__CGSCC____-NEXT:    br i1 [[TMP7]], label [[TMP4]], label [[TMP8:%.*]]
+; IS__CGSCC____:       8:
+; IS__CGSCC____-NEXT:    fence syncscope("singlethread") acquire
+; IS__CGSCC____-NEXT:    ret void
 ;
   %3 = getelementptr inbounds %"struct.std::atomic", %"struct.std::atomic"* %1, i64 0, i32 0, i32 0
   br label %4
@@ -357,10 +398,8 @@ declare void @llvm.memset(i8* %dest, i8 %val, i32 %len, i1 %isvolatile)
 
 ; It is odd to add nocapture but a result of the llvm.memcpy nocapture.
 ;
-; IS__TUNIT____: Function Attrs: argmemonly nounwind
-; IS__CGSCC____: Function Attrs: argmemonly nounwind
-; CHECK-NOT: nosync
 define i32 @memcpy_volatile(i8* %ptr1, i8* %ptr2) {
+; CHECK: Function Attrs: argmemonly nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@memcpy_volatile
 ; CHECK-SAME: (i8* nocapture writeonly [[PTR1:%.*]], i8* nocapture readonly [[PTR2:%.*]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* noalias nocapture writeonly [[PTR1]], i8* noalias nocapture readonly [[PTR2]], i32 8, i1 true)
@@ -374,9 +413,8 @@ define i32 @memcpy_volatile(i8* %ptr1, i8* %ptr2) {
 
 ; It is odd to add nocapture but a result of the llvm.memset nocapture.
 ;
-; IS__TUNIT____: Function Attrs: argmemonly nosync
-; IS__CGSCC____: Function Attrs: argmemonly nosync
 define i32 @memset_non_volatile(i8* %ptr1, i8 %val) {
+; CHECK: Function Attrs: argmemonly nosync nounwind willreturn writeonly
 ; CHECK-LABEL: define {{[^@]+}}@memset_non_volatile
 ; CHECK-SAME: (i8* nocapture writeonly [[PTR1:%.*]], i8 [[VAL:%.*]])
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nocapture writeonly [[PTR1]], i8 [[VAL]], i32 8, i1 false)
@@ -402,6 +440,7 @@ declare void @readnone_test() convergent readnone
 
 ; TEST 17 - negative. Convergent
 define void @convergent_readnone(){
+; CHECK: Function Attrs: readnone
 ; CHECK-LABEL: define {{[^@]+}}@convergent_readnone()
 ; CHECK-NEXT:    call void @readnone_test()
 ; CHECK-NEXT:    ret void
@@ -417,10 +456,8 @@ declare void @llvm.x86.sse2.clflush(i8*)
 
 ; TEST 18 - negative. Synchronizing intrinsic
 
-; IS__TUNIT____: Function Attrs: nounwind
-; IS__CGSCC____: Function Attrs: nounwind
-; CHECK-NOT: nosync
 define void @i_totally_sync() {
+; CHECK: Function Attrs: nounwind
 ; CHECK-LABEL: define {{[^@]+}}@i_totally_sync()
 ; CHECK-NEXT:    tail call void @llvm.x86.sse2.clflush(i8* nonnull align 4 dereferenceable(4) bitcast (i32* @a to i8*))
 ; CHECK-NEXT:    ret void
@@ -433,20 +470,23 @@ declare float @llvm.cos(float %val) readnone
 
 ; TEST 19 - positive, readnone & non-convergent intrinsic.
 
+define i32 @cos_test(float %x) {
 ; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@cos_test
+; IS__TUNIT____-SAME: (float [[X:%.*]])
+; IS__TUNIT____-NEXT:    ret i32 4
+;
 ; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
-define i32 @cos_test(float %x) {
-; CHECK-LABEL: define {{[^@]+}}@cos_test
-; CHECK-SAME: (float [[X:%.*]])
-; CHECK-NEXT:    ret i32 4
+; IS__CGSCC____-LABEL: define {{[^@]+}}@cos_test
+; IS__CGSCC____-SAME: (float [[X:%.*]])
+; IS__CGSCC____-NEXT:    ret i32 4
 ;
   call float @llvm.cos(float %x)
   ret i32 4
 }
 
-; IS__TUNIT____: Function Attrs: nosync nounwind
-; IS__CGSCC____: Function Attrs: nosync nounwind
 define float @cos_test2(float %x) {
+; CHECK: Function Attrs: nosync nounwind readnone willreturn
 ; CHECK-LABEL: define {{[^@]+}}@cos_test2
 ; CHECK-SAME: (float [[X:%.*]])
 ; CHECK-NEXT:    [[C:%.*]] = call float @llvm.cos.f32(float [[X]])
diff --git a/llvm/test/Transforms/Attributor/nounwind.ll b/llvm/test/Transforms/Attributor/nounwind.ll
index 7f5287a74d749..3eb525a015e7f 100644
--- a/llvm/test/Transforms/Attributor/nounwind.ll
+++ b/llvm/test/Transforms/Attributor/nounwind.ll
@@ -1,26 +1,31 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
 
 ; TEST 1
-; IS__TUNIT____: Function Attrs: nofree nosync nounwind
-; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind
 define i32 @foo1() {
-; CHECK-LABEL: define {{[^@]+}}@foo1()
-; CHECK-NEXT:    ret i32 1
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@foo1()
+; IS__TUNIT____-NEXT:    ret i32 1
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@foo1()
+; IS__CGSCC____-NEXT:    ret i32 1
 ;
   ret i32 1
 }
 
 ; TEST 2
-; IS__TUNIT____: Function Attrs: nofree noreturn nosync nounwind
-; IS__CGSCC_OPM: Function Attrs: nofree noreturn nosync nounwind
-; IS__CGSCC_NPM: Function Attrs: nofree norecurse noreturn nosync nounwind
 define i32 @scc1_foo() {
-; CHECK-LABEL: define {{[^@]+}}@scc1_foo()
-; CHECK-NEXT:    unreachable
+; NOT_CGSCC_NPM: Function Attrs: nofree noreturn nosync nounwind readnone willreturn
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@scc1_foo()
+; NOT_CGSCC_NPM-NEXT:    unreachable
+;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@scc1_foo()
+; IS__CGSCC_NPM-NEXT:    unreachable
 ;
   %1 = call i32 @scc1_bar()
   ret i32 1
@@ -28,12 +33,14 @@ define i32 @scc1_foo() {
 
 
 ; TEST 3
-; IS__TUNIT____: Function Attrs: nofree noreturn nosync nounwind
-; IS__CGSCC_OPM: Function Attrs: nofree noreturn nosync nounwind
-; IS__CGSCC_NPM: Function Attrs: nofree norecurse noreturn nosync nounwind
 define i32 @scc1_bar() {
-; CHECK-LABEL: define {{[^@]+}}@scc1_bar()
-; CHECK-NEXT:    unreachable
+; NOT_CGSCC_NPM: Function Attrs: nofree noreturn nosync nounwind readnone willreturn
+; NOT_CGSCC_NPM-LABEL: define {{[^@]+}}@scc1_bar()
+; NOT_CGSCC_NPM-NEXT:    unreachable
+;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@scc1_bar()
+; IS__CGSCC_NPM-NEXT:    unreachable
 ;
   %1 = call i32 @scc1_foo()
   ret i32 1
diff --git a/llvm/test/Transforms/Attributor/range.ll b/llvm/test/Transforms/Attributor/range.ll
index f105bb3fad0e1..8f6ccf4c1e5d6 100644
--- a/llvm/test/Transforms/Attributor/range.ll
+++ b/llvm/test/Transforms/Attributor/range.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=17 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=17 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -7,26 +7,36 @@
 ; FIXME: CGSCC is not looking at callees and calleers even though it could be allowed.
 
 define i32 @test0(i32* %p) {
-; CHECK-LABEL: define {{[^@]+}}@test0
-; CHECK-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P:%.*]])
-; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[P]], align 4, !range !0
-; CHECK-NEXT:    ret i32 [[A]]
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test0
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P:%.*]])
+; IS__TUNIT____-NEXT:    [[A:%.*]] = load i32, i32* [[P]], align 4, !range !0
+; IS__TUNIT____-NEXT:    ret i32 [[A]]
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test0
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P:%.*]])
+; IS__CGSCC____-NEXT:    [[A:%.*]] = load i32, i32* [[P]], align 4, !range !0
+; IS__CGSCC____-NEXT:    ret i32 [[A]]
 ;
   %a = load i32, i32* %p, !range !0
   ret i32 %a
 }
 
 define i32 @test0-range-check(i32* %p) {
+; IS__TUNIT_OPM: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
 ; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@test0-range-check
 ; IS__TUNIT_OPM-SAME: (i32* nocapture nofree readonly align 4 [[P:%.*]])
 ; IS__TUNIT_OPM-NEXT:    [[A:%.*]] = tail call i32 @test0(i32* nocapture nofree readonly align 4 [[P]]) #3, !range !0
 ; IS__TUNIT_OPM-NEXT:    ret i32 [[A]]
 ;
+; IS__TUNIT_NPM: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@test0-range-check
 ; IS__TUNIT_NPM-SAME: (i32* nocapture nofree readonly align 4 [[P:%.*]])
 ; IS__TUNIT_NPM-NEXT:    [[A:%.*]] = tail call i32 @test0(i32* nocapture nofree readonly align 4 [[P]]) #2, !range !0
 ; IS__TUNIT_NPM-NEXT:    ret i32 [[A]]
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@test0-range-check
 ; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P:%.*]])
 ; IS__CGSCC____-NEXT:    [[A:%.*]] = tail call i32 @test0(i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P]])
@@ -296,12 +306,21 @@ define void @test0-icmp-check(i32* %p){
   ret void
 }
 define i32 @test1(i32* %p) {
-; CHECK-LABEL: define {{[^@]+}}@test1
-; CHECK-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P:%.*]])
-; CHECK-NEXT:    [[LOAD_10_100:%.*]] = load i32, i32* [[P]], align 4, !range !1
-; CHECK-NEXT:    [[ADD_10_THEN_20_110:%.*]] = add i32 [[LOAD_10_100]], 10
-; CHECK-NEXT:    [[MUL_10_THEN_200_1091:%.*]] = mul i32 [[ADD_10_THEN_20_110]], 10
-; CHECK-NEXT:    ret i32 [[MUL_10_THEN_200_1091]]
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test1
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P:%.*]])
+; IS__TUNIT____-NEXT:    [[LOAD_10_100:%.*]] = load i32, i32* [[P]], align 4, !range !1
+; IS__TUNIT____-NEXT:    [[ADD_10_THEN_20_110:%.*]] = add i32 [[LOAD_10_100]], 10
+; IS__TUNIT____-NEXT:    [[MUL_10_THEN_200_1091:%.*]] = mul i32 [[ADD_10_THEN_20_110]], 10
+; IS__TUNIT____-NEXT:    ret i32 [[MUL_10_THEN_200_1091]]
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test1
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P:%.*]])
+; IS__CGSCC____-NEXT:    [[LOAD_10_100:%.*]] = load i32, i32* [[P]], align 4, !range !1
+; IS__CGSCC____-NEXT:    [[ADD_10_THEN_20_110:%.*]] = add i32 [[LOAD_10_100]], 10
+; IS__CGSCC____-NEXT:    [[MUL_10_THEN_200_1091:%.*]] = mul i32 [[ADD_10_THEN_20_110]], 10
+; IS__CGSCC____-NEXT:    ret i32 [[MUL_10_THEN_200_1091]]
 ;
   %load-10-100 = load i32, i32* %p, !range !1
   %add-10-then-20-110 = add i32 %load-10-100, 10
@@ -311,18 +330,21 @@ define i32 @test1(i32* %p) {
 
 define i1 @test1-check(i32* %p) {
 ;
+; IS__TUNIT_OPM: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
 ; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@test1-check
 ; IS__TUNIT_OPM-SAME: (i32* nocapture nofree readonly align 4 [[P:%.*]])
 ; IS__TUNIT_OPM-NEXT:    [[RES:%.*]] = tail call i32 @test1(i32* nocapture nofree readonly align 4 [[P]]) #3, !range !2
 ; IS__TUNIT_OPM-NEXT:    [[CMP:%.*]] = icmp eq i32 [[RES]], 500
 ; IS__TUNIT_OPM-NEXT:    ret i1 [[CMP]]
 ;
+; IS__TUNIT_NPM: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@test1-check
 ; IS__TUNIT_NPM-SAME: (i32* nocapture nofree readonly align 4 [[P:%.*]])
 ; IS__TUNIT_NPM-NEXT:    [[RES:%.*]] = tail call i32 @test1(i32* nocapture nofree readonly align 4 [[P]]) #2, !range !2
 ; IS__TUNIT_NPM-NEXT:    [[CMP:%.*]] = icmp eq i32 [[RES]], 500
 ; IS__TUNIT_NPM-NEXT:    ret i1 [[CMP]]
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@test1-check
 ; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P:%.*]])
 ; IS__CGSCC____-NEXT:    [[RES:%.*]] = tail call i32 @test1(i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P]])
@@ -347,13 +369,23 @@ define i1 @test1-check(i32* %p) {
 ;  }
 
 define i32 @test2(i32* %p) {
-; CHECK-LABEL: define {{[^@]+}}@test2
-; CHECK-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[P]], align 4
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TMP0]], 0
-; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 4, i32 3
-; CHECK-NEXT:    ret i32 [[COND]]
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test2
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[TMP0:%.*]] = load i32, i32* [[P]], align 4
+; IS__TUNIT____-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TMP0]], 0
+; IS__TUNIT____-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 4, i32 3
+; IS__TUNIT____-NEXT:    ret i32 [[COND]]
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test2
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[TMP0:%.*]] = load i32, i32* [[P]], align 4
+; IS__CGSCC____-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TMP0]], 0
+; IS__CGSCC____-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 4, i32 3
+; IS__CGSCC____-NEXT:    ret i32 [[COND]]
 ;
 entry:
   %0 = load i32, i32* %p, align 4
@@ -363,6 +395,7 @@ entry:
 }
 
 define i32 @test2_check(i32* %p) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@test2_check
 ; IS__TUNIT____-SAME: (i32* nocapture nofree readonly align 4 [[P:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
@@ -374,6 +407,7 @@ define i32 @test2_check(i32* %p) {
 ; IS__TUNIT____:       return:
 ; IS__TUNIT____-NEXT:    ret i32 2
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@test2_check
 ; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -432,23 +466,43 @@ return:                                           ; preds = %if.end, %if.then
 declare dso_local void @unkown()
 
 define internal i32 @r1(i32) local_unnamed_addr {
-; IS________OPM-LABEL: define {{[^@]+}}@r1() local_unnamed_addr
-; IS________OPM-NEXT:    br label [[TMP4:%.*]]
-; IS________OPM:       1:
-; IS________OPM-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[TMP7:%.*]], 10000
-; IS________OPM-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[F:%.*]]
-; IS________OPM:       3:
-; IS________OPM-NEXT:    ret i32 20
-; IS________OPM:       f:
-; IS________OPM-NEXT:    ret i32 10
-; IS________OPM:       4:
-; IS________OPM-NEXT:    [[TMP5:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP8:%.*]], [[TMP4]] ]
-; IS________OPM-NEXT:    [[TMP6:%.*]] = phi i32 [ 0, [[TMP0]] ], [ [[TMP7]], [[TMP4]] ]
-; IS________OPM-NEXT:    [[TMP7]] = add nuw nsw i32 [[TMP5]], [[TMP6]]
-; IS________OPM-NEXT:    [[TMP8]] = add nuw nsw i32 [[TMP5]], 1
-; IS________OPM-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 100
-; IS________OPM-NEXT:    br i1 [[TMP9]], label [[TMP1:%.*]], label [[TMP4]]
-;
+; IS__TUNIT_OPM: Function Attrs: nofree nosync nounwind readnone
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@r1() local_unnamed_addr
+; IS__TUNIT_OPM-NEXT:    br label [[TMP4:%.*]]
+; IS__TUNIT_OPM:       1:
+; IS__TUNIT_OPM-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[TMP7:%.*]], 10000
+; IS__TUNIT_OPM-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[F:%.*]]
+; IS__TUNIT_OPM:       3:
+; IS__TUNIT_OPM-NEXT:    ret i32 20
+; IS__TUNIT_OPM:       f:
+; IS__TUNIT_OPM-NEXT:    ret i32 10
+; IS__TUNIT_OPM:       4:
+; IS__TUNIT_OPM-NEXT:    [[TMP5:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP8:%.*]], [[TMP4]] ]
+; IS__TUNIT_OPM-NEXT:    [[TMP6:%.*]] = phi i32 [ 0, [[TMP0]] ], [ [[TMP7]], [[TMP4]] ]
+; IS__TUNIT_OPM-NEXT:    [[TMP7]] = add nuw nsw i32 [[TMP5]], [[TMP6]]
+; IS__TUNIT_OPM-NEXT:    [[TMP8]] = add nuw nsw i32 [[TMP5]], 1
+; IS__TUNIT_OPM-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 100
+; IS__TUNIT_OPM-NEXT:    br i1 [[TMP9]], label [[TMP1:%.*]], label [[TMP4]]
+;
+; IS__CGSCC_OPM: Function Attrs: nofree norecurse nosync nounwind readnone
+; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@r1() local_unnamed_addr
+; IS__CGSCC_OPM-NEXT:    br label [[TMP4:%.*]]
+; IS__CGSCC_OPM:       1:
+; IS__CGSCC_OPM-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[TMP7:%.*]], 10000
+; IS__CGSCC_OPM-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[F:%.*]]
+; IS__CGSCC_OPM:       3:
+; IS__CGSCC_OPM-NEXT:    ret i32 20
+; IS__CGSCC_OPM:       f:
+; IS__CGSCC_OPM-NEXT:    ret i32 10
+; IS__CGSCC_OPM:       4:
+; IS__CGSCC_OPM-NEXT:    [[TMP5:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP8:%.*]], [[TMP4]] ]
+; IS__CGSCC_OPM-NEXT:    [[TMP6:%.*]] = phi i32 [ 0, [[TMP0]] ], [ [[TMP7]], [[TMP4]] ]
+; IS__CGSCC_OPM-NEXT:    [[TMP7]] = add nuw nsw i32 [[TMP5]], [[TMP6]]
+; IS__CGSCC_OPM-NEXT:    [[TMP8]] = add nuw nsw i32 [[TMP5]], 1
+; IS__CGSCC_OPM-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 100
+; IS__CGSCC_OPM-NEXT:    br i1 [[TMP9]], label [[TMP1:%.*]], label [[TMP4]]
+;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@r1() local_unnamed_addr
 ; IS__CGSCC_NPM-NEXT:    br label [[TMP3:%.*]]
 ; IS__CGSCC_NPM:       1:
@@ -495,13 +549,23 @@ define void @f1(i32){
 ; IS________OPM:       5:
 ; IS________OPM-NEXT:    ret void
 ;
-; IS________NPM-LABEL: define {{[^@]+}}@f1
-; IS________NPM-SAME: (i32 [[TMP0:%.*]])
-; IS________NPM-NEXT:    br label [[TMP3:%.*]]
-; IS________NPM:       2:
-; IS________NPM-NEXT:    unreachable
-; IS________NPM:       3:
-; IS________NPM-NEXT:    ret void
+; IS__TUNIT_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@f1
+; IS__TUNIT_NPM-SAME: (i32 [[TMP0:%.*]])
+; IS__TUNIT_NPM-NEXT:    br label [[TMP3:%.*]]
+; IS__TUNIT_NPM:       2:
+; IS__TUNIT_NPM-NEXT:    unreachable
+; IS__TUNIT_NPM:       3:
+; IS__TUNIT_NPM-NEXT:    ret void
+;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@f1
+; IS__CGSCC_NPM-SAME: (i32 [[TMP0:%.*]])
+; IS__CGSCC_NPM-NEXT:    br label [[TMP3:%.*]]
+; IS__CGSCC_NPM:       2:
+; IS__CGSCC_NPM-NEXT:    unreachable
+; IS__CGSCC_NPM:       3:
+; IS__CGSCC_NPM-NEXT:    ret void
 ;
   %2 = tail call i32 @r1(i32 %0)
   %3 = icmp sgt i32 %2, 15
@@ -526,16 +590,29 @@ define void @f1(i32){
 ;   }
 ; }
 define dso_local i32 @test4-f1(i32 %u) {
-; CHECK-LABEL: define {{[^@]+}}@test4-f1
-; CHECK-SAME: (i32 [[U:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[U]], -1
-; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[RETURN:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    br label [[RETURN]]
-; CHECK:       return:
-; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[U]], [[IF_THEN]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    ret i32 [[RETVAL_0]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test4-f1
+; IS__TUNIT____-SAME: (i32 [[U:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[U]], -1
+; IS__TUNIT____-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[RETURN:%.*]]
+; IS__TUNIT____:       if.then:
+; IS__TUNIT____-NEXT:    br label [[RETURN]]
+; IS__TUNIT____:       return:
+; IS__TUNIT____-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[U]], [[IF_THEN]] ], [ 0, [[ENTRY:%.*]] ]
+; IS__TUNIT____-NEXT:    ret i32 [[RETVAL_0]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test4-f1
+; IS__CGSCC____-SAME: (i32 [[U:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[U]], -1
+; IS__CGSCC____-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[RETURN:%.*]]
+; IS__CGSCC____:       if.then:
+; IS__CGSCC____-NEXT:    br label [[RETURN]]
+; IS__CGSCC____:       return:
+; IS__CGSCC____-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[U]], [[IF_THEN]] ], [ 0, [[ENTRY:%.*]] ]
+; IS__CGSCC____-NEXT:    ret i32 [[RETVAL_0]]
 ;
 ; FIXME: RETVAL_0 >= 0
 entry:
@@ -552,11 +629,19 @@ return:                                           ; preds = %entry, %if.then
 
 
 define dso_local i32 @test4-g1(i32 %u) {
-; CHECK-LABEL: define {{[^@]+}}@test4-g1
-; CHECK-SAME: (i32 [[U:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @test4-f1(i32 [[U]])
-; CHECK-NEXT:    ret i32 [[CALL]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test4-g1
+; IS__TUNIT____-SAME: (i32 [[U:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[CALL:%.*]] = tail call i32 @test4-f1(i32 [[U]])
+; IS__TUNIT____-NEXT:    ret i32 [[CALL]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test4-g1
+; IS__CGSCC____-SAME: (i32 [[U:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[CALL:%.*]] = tail call i32 @test4-f1(i32 [[U]])
+; IS__CGSCC____-NEXT:    ret i32 [[CALL]]
 ;
 ; FIXME: %call should have range [0, inf]
 
@@ -574,19 +659,35 @@ entry:
 ;   }
 ; }
 define dso_local i32 @test4-f2(i32 %u) {
-; CHECK-LABEL: define {{[^@]+}}@test4-f2
-; CHECK-SAME: (i32 [[U:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[U]], -1
-; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[U]], 1
-; CHECK-NEXT:    br label [[RETURN:%.*]]
-; CHECK:       if.else:
-; CHECK-NEXT:    br label [[RETURN]]
-; CHECK:       return:
-; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ 1, [[IF_ELSE]] ]
-; CHECK-NEXT:    ret i32 [[RETVAL_0]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test4-f2
+; IS__TUNIT____-SAME: (i32 [[U:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[U]], -1
+; IS__TUNIT____-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; IS__TUNIT____:       if.then:
+; IS__TUNIT____-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[U]], 1
+; IS__TUNIT____-NEXT:    br label [[RETURN:%.*]]
+; IS__TUNIT____:       if.else:
+; IS__TUNIT____-NEXT:    br label [[RETURN]]
+; IS__TUNIT____:       return:
+; IS__TUNIT____-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ 1, [[IF_ELSE]] ]
+; IS__TUNIT____-NEXT:    ret i32 [[RETVAL_0]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test4-f2
+; IS__CGSCC____-SAME: (i32 [[U:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[U]], -1
+; IS__CGSCC____-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; IS__CGSCC____:       if.then:
+; IS__CGSCC____-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[U]], 1
+; IS__CGSCC____-NEXT:    br label [[RETURN:%.*]]
+; IS__CGSCC____:       if.else:
+; IS__CGSCC____-NEXT:    br label [[RETURN]]
+; IS__CGSCC____:       return:
+; IS__CGSCC____-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ 1, [[IF_ELSE]] ]
+; IS__CGSCC____-NEXT:    ret i32 [[RETVAL_0]]
 ;
 entry:
   %cmp = icmp sgt i32 %u, -1
@@ -606,18 +707,27 @@ return:                                           ; preds = %if.else, %if.then
 
 
 define dso_local i32 @test4-g2(i32 %u) {
-; NOT_TUNIT_NPM-LABEL: define {{[^@]+}}@test4-g2
-; NOT_TUNIT_NPM-SAME: (i32 [[U:%.*]])
-; NOT_TUNIT_NPM-NEXT:  entry:
-; NOT_TUNIT_NPM-NEXT:    [[CALL:%.*]] = tail call i32 @test4-f2(i32 [[U]])
-; NOT_TUNIT_NPM-NEXT:    ret i32 [[CALL]]
+; IS__TUNIT_OPM: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@test4-g2
+; IS__TUNIT_OPM-SAME: (i32 [[U:%.*]])
+; IS__TUNIT_OPM-NEXT:  entry:
+; IS__TUNIT_OPM-NEXT:    [[CALL:%.*]] = tail call i32 @test4-f2(i32 [[U]])
+; IS__TUNIT_OPM-NEXT:    ret i32 [[CALL]]
 ;
+; IS__TUNIT_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@test4-g2
 ; IS__TUNIT_NPM-SAME: (i32 [[U:%.*]])
 ; IS__TUNIT_NPM-NEXT:  entry:
 ; IS__TUNIT_NPM-NEXT:    [[CALL:%.*]] = tail call i32 @test4-f2(i32 [[U]]) #1, !range !3
 ; IS__TUNIT_NPM-NEXT:    ret i32 [[CALL]]
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test4-g2
+; IS__CGSCC____-SAME: (i32 [[U:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[CALL:%.*]] = tail call i32 @test4-f2(i32 [[U]])
+; IS__CGSCC____-NEXT:    ret i32 [[CALL]]
+;
 entry:
   %call = tail call i32 @test4-f2(i32 %u)
   ret i32 %call
@@ -700,30 +810,57 @@ declare dso_local i32 @foo(i32)
 
 ; FIXME: All but the return is not needed anymore
 define dso_local zeroext i1 @phi(i32 %arg) {
-; CHECK-LABEL: define {{[^@]+}}@phi
-; CHECK-SAME: (i32 [[ARG:%.*]])
-; CHECK-NEXT:  bb:
-; CHECK-NEXT:    [[TMP:%.*]] = icmp sgt i32 [[ARG]], 5
-; CHECK-NEXT:    br i1 [[TMP]], label [[BB1:%.*]], label [[BB2:%.*]]
-; CHECK:       bb1:
-; CHECK-NEXT:    br label [[BB3:%.*]]
-; CHECK:       bb2:
-; CHECK-NEXT:    br label [[BB3]]
-; CHECK:       bb3:
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[ARG]], 10
-; CHECK-NEXT:    br i1 [[TMP4]], label [[BB5:%.*]], label [[BB7:%.*]]
-; CHECK:       bb5:
-; CHECK-NEXT:    br label [[BB9:%.*]]
-; CHECK:       bb7:
-; CHECK-NEXT:    br label [[BB9]]
-; CHECK:       bb9:
-; CHECK-NEXT:    br label [[BB12:%.*]]
-; CHECK:       bb11:
-; CHECK-NEXT:    unreachable
-; CHECK:       bb12:
-; CHECK-NEXT:    br label [[BB13:%.*]]
-; CHECK:       bb13:
-; CHECK-NEXT:    ret i1 false
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@phi
+; IS__TUNIT____-SAME: (i32 [[ARG:%.*]])
+; IS__TUNIT____-NEXT:  bb:
+; IS__TUNIT____-NEXT:    [[TMP:%.*]] = icmp sgt i32 [[ARG]], 5
+; IS__TUNIT____-NEXT:    br i1 [[TMP]], label [[BB1:%.*]], label [[BB2:%.*]]
+; IS__TUNIT____:       bb1:
+; IS__TUNIT____-NEXT:    br label [[BB3:%.*]]
+; IS__TUNIT____:       bb2:
+; IS__TUNIT____-NEXT:    br label [[BB3]]
+; IS__TUNIT____:       bb3:
+; IS__TUNIT____-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[ARG]], 10
+; IS__TUNIT____-NEXT:    br i1 [[TMP4]], label [[BB5:%.*]], label [[BB7:%.*]]
+; IS__TUNIT____:       bb5:
+; IS__TUNIT____-NEXT:    br label [[BB9:%.*]]
+; IS__TUNIT____:       bb7:
+; IS__TUNIT____-NEXT:    br label [[BB9]]
+; IS__TUNIT____:       bb9:
+; IS__TUNIT____-NEXT:    br label [[BB12:%.*]]
+; IS__TUNIT____:       bb11:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       bb12:
+; IS__TUNIT____-NEXT:    br label [[BB13:%.*]]
+; IS__TUNIT____:       bb13:
+; IS__TUNIT____-NEXT:    ret i1 false
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@phi
+; IS__CGSCC____-SAME: (i32 [[ARG:%.*]])
+; IS__CGSCC____-NEXT:  bb:
+; IS__CGSCC____-NEXT:    [[TMP:%.*]] = icmp sgt i32 [[ARG]], 5
+; IS__CGSCC____-NEXT:    br i1 [[TMP]], label [[BB1:%.*]], label [[BB2:%.*]]
+; IS__CGSCC____:       bb1:
+; IS__CGSCC____-NEXT:    br label [[BB3:%.*]]
+; IS__CGSCC____:       bb2:
+; IS__CGSCC____-NEXT:    br label [[BB3]]
+; IS__CGSCC____:       bb3:
+; IS__CGSCC____-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[ARG]], 10
+; IS__CGSCC____-NEXT:    br i1 [[TMP4]], label [[BB5:%.*]], label [[BB7:%.*]]
+; IS__CGSCC____:       bb5:
+; IS__CGSCC____-NEXT:    br label [[BB9:%.*]]
+; IS__CGSCC____:       bb7:
+; IS__CGSCC____-NEXT:    br label [[BB9]]
+; IS__CGSCC____:       bb9:
+; IS__CGSCC____-NEXT:    br label [[BB12:%.*]]
+; IS__CGSCC____:       bb11:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       bb12:
+; IS__CGSCC____-NEXT:    br label [[BB13:%.*]]
+; IS__CGSCC____:       bb13:
+; IS__CGSCC____-NEXT:    ret i1 false
 ;
 bb:
   %tmp = icmp sgt i32 %arg, 5
@@ -765,10 +902,17 @@ bb13:                                             ; preds = %bb12, %bb11
 }
 
 define dso_local i1 @select(i32 %a) local_unnamed_addr #0 {
-; CHECK-LABEL: define {{[^@]+}}@select
-; CHECK-SAME: (i32 [[A:%.*]]) local_unnamed_addr
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i1 false
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@select
+; IS__TUNIT____-SAME: (i32 [[A:%.*]]) local_unnamed_addr
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    ret i1 false
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@select
+; IS__CGSCC____-SAME: (i32 [[A:%.*]]) local_unnamed_addr
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    ret i1 false
 ;
 entry:
   %cmp = icmp sgt i32 %a, 5
@@ -781,10 +925,17 @@ entry:
 }
 
 define dso_local i32 @select_zext(i32 %a) local_unnamed_addr #0 {
-; CHECK-LABEL: define {{[^@]+}}@select_zext
-; CHECK-SAME: (i32 [[A:%.*]]) local_unnamed_addr
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i32 0
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@select_zext
+; IS__TUNIT____-SAME: (i32 [[A:%.*]]) local_unnamed_addr
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    ret i32 0
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@select_zext
+; IS__CGSCC____-SAME: (i32 [[A:%.*]]) local_unnamed_addr
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    ret i32 0
 ;
 entry:
   %cmp = icmp sgt i32 %a, 5
@@ -799,13 +950,23 @@ entry:
 
 ; FIXME: We do not look through the ptr casts here.
 define dso_local i64 @select_int2ptr_bitcast_ptr2int(i32 %a) local_unnamed_addr #0 {
-; CHECK-LABEL: define {{[^@]+}}@select_int2ptr_bitcast_ptr2int
-; CHECK-SAME: (i32 [[A:%.*]]) local_unnamed_addr
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[I2P:%.*]] = inttoptr i1 false to i1*
-; CHECK-NEXT:    [[BC:%.*]] = bitcast i1* [[I2P]] to i32*
-; CHECK-NEXT:    [[P2I:%.*]] = ptrtoint i32* [[BC]] to i64
-; CHECK-NEXT:    ret i64 [[P2I]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@select_int2ptr_bitcast_ptr2int
+; IS__TUNIT____-SAME: (i32 [[A:%.*]]) local_unnamed_addr
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[I2P:%.*]] = inttoptr i1 false to i1*
+; IS__TUNIT____-NEXT:    [[BC:%.*]] = bitcast i1* [[I2P]] to i32*
+; IS__TUNIT____-NEXT:    [[P2I:%.*]] = ptrtoint i32* [[BC]] to i64
+; IS__TUNIT____-NEXT:    ret i64 [[P2I]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@select_int2ptr_bitcast_ptr2int
+; IS__CGSCC____-SAME: (i32 [[A:%.*]]) local_unnamed_addr
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[I2P:%.*]] = inttoptr i1 false to i1*
+; IS__CGSCC____-NEXT:    [[BC:%.*]] = bitcast i1* [[I2P]] to i32*
+; IS__CGSCC____-NEXT:    [[P2I:%.*]] = ptrtoint i32* [[BC]] to i64
+; IS__CGSCC____-NEXT:    ret i64 [[P2I]]
 ;
 entry:
   %cmp = icmp sgt i32 %a, 5
@@ -823,60 +984,105 @@ entry:
 ; }
 
 define i1 @f_fcmp(float %a, float %b) {
-; CHECK-LABEL: define {{[^@]+}}@f_fcmp
-; CHECK-SAME: (float [[A:%.*]], float [[B:%.*]])
-; CHECK-NEXT:    [[R:%.*]] = fcmp uge float [[A]], [[B]]
-; CHECK-NEXT:    [[S:%.*]] = select i1 [[R]], i1 [[R]], i1 false
-; CHECK-NEXT:    ret i1 [[S]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@f_fcmp
+; IS__TUNIT____-SAME: (float [[A:%.*]], float [[B:%.*]])
+; IS__TUNIT____-NEXT:    [[R:%.*]] = fcmp uge float [[A]], [[B]]
+; IS__TUNIT____-NEXT:    [[S:%.*]] = select i1 [[R]], i1 [[R]], i1 false
+; IS__TUNIT____-NEXT:    ret i1 [[S]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@f_fcmp
+; IS__CGSCC____-SAME: (float [[A:%.*]], float [[B:%.*]])
+; IS__CGSCC____-NEXT:    [[R:%.*]] = fcmp uge float [[A]], [[B]]
+; IS__CGSCC____-NEXT:    [[S:%.*]] = select i1 [[R]], i1 [[R]], i1 false
+; IS__CGSCC____-NEXT:    ret i1 [[S]]
 ;
   %r = fcmp uge float %a, %b
   %s = select i1 %r, i1 %r, i1 0
   ret i1 %s
 }
 define i1 @d_fcmp(double %a, double %b) {
-; CHECK-LABEL: define {{[^@]+}}@d_fcmp
-; CHECK-SAME: (double [[A:%.*]], double [[B:%.*]])
-; CHECK-NEXT:    [[R:%.*]] = fcmp oeq double [[A]], [[B]]
-; CHECK-NEXT:    [[S:%.*]] = select i1 [[R]], i1 [[R]], i1 false
-; CHECK-NEXT:    ret i1 [[S]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@d_fcmp
+; IS__TUNIT____-SAME: (double [[A:%.*]], double [[B:%.*]])
+; IS__TUNIT____-NEXT:    [[R:%.*]] = fcmp oeq double [[A]], [[B]]
+; IS__TUNIT____-NEXT:    [[S:%.*]] = select i1 [[R]], i1 [[R]], i1 false
+; IS__TUNIT____-NEXT:    ret i1 [[S]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@d_fcmp
+; IS__CGSCC____-SAME: (double [[A:%.*]], double [[B:%.*]])
+; IS__CGSCC____-NEXT:    [[R:%.*]] = fcmp oeq double [[A]], [[B]]
+; IS__CGSCC____-NEXT:    [[S:%.*]] = select i1 [[R]], i1 [[R]], i1 false
+; IS__CGSCC____-NEXT:    ret i1 [[S]]
 ;
   %r = fcmp oeq double %a, %b
   %s = select i1 %r, i1 %r, i1 0
   ret i1 %s
 }
 define i1 @dp_icmp(double* %a, double* %b) {
-; CHECK-LABEL: define {{[^@]+}}@dp_icmp
-; CHECK-SAME: (double* nofree readnone [[A:%.*]], double* nofree readnone [[B:%.*]])
-; CHECK-NEXT:    [[R:%.*]] = icmp sge double* [[A]], [[B]]
-; CHECK-NEXT:    [[S:%.*]] = select i1 [[R]], i1 [[R]], i1 false
-; CHECK-NEXT:    ret i1 [[S]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@dp_icmp
+; IS__TUNIT____-SAME: (double* nofree readnone [[A:%.*]], double* nofree readnone [[B:%.*]])
+; IS__TUNIT____-NEXT:    [[R:%.*]] = icmp sge double* [[A]], [[B]]
+; IS__TUNIT____-NEXT:    [[S:%.*]] = select i1 [[R]], i1 [[R]], i1 false
+; IS__TUNIT____-NEXT:    ret i1 [[S]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@dp_icmp
+; IS__CGSCC____-SAME: (double* nofree readnone [[A:%.*]], double* nofree readnone [[B:%.*]])
+; IS__CGSCC____-NEXT:    [[R:%.*]] = icmp sge double* [[A]], [[B]]
+; IS__CGSCC____-NEXT:    [[S:%.*]] = select i1 [[R]], i1 [[R]], i1 false
+; IS__CGSCC____-NEXT:    ret i1 [[S]]
 ;
   %r = icmp sge double* %a, %b
   %s = select i1 %r, i1 %r, i1 0
   ret i1 %s
 }
 define i1 @ip_icmp(i8* %a, i8* %b) {
-; CHECK-LABEL: define {{[^@]+}}@ip_icmp
-; CHECK-SAME: (i8* nofree readnone [[A:%.*]], i8* nofree readnone [[B:%.*]])
-; CHECK-NEXT:    [[R:%.*]] = icmp ult i8* [[A]], [[B]]
-; CHECK-NEXT:    [[S:%.*]] = select i1 [[R]], i1 [[R]], i1 false
-; CHECK-NEXT:    ret i1 [[S]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@ip_icmp
+; IS__TUNIT____-SAME: (i8* nofree readnone [[A:%.*]], i8* nofree readnone [[B:%.*]])
+; IS__TUNIT____-NEXT:    [[R:%.*]] = icmp ult i8* [[A]], [[B]]
+; IS__TUNIT____-NEXT:    [[S:%.*]] = select i1 [[R]], i1 [[R]], i1 false
+; IS__TUNIT____-NEXT:    ret i1 [[S]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@ip_icmp
+; IS__CGSCC____-SAME: (i8* nofree readnone [[A:%.*]], i8* nofree readnone [[B:%.*]])
+; IS__CGSCC____-NEXT:    [[R:%.*]] = icmp ult i8* [[A]], [[B]]
+; IS__CGSCC____-NEXT:    [[S:%.*]] = select i1 [[R]], i1 [[R]], i1 false
+; IS__CGSCC____-NEXT:    ret i1 [[S]]
 ;
   %r = icmp ult i8* %a, %b
   %s = select i1 %r, i1 %r, i1 0
   ret i1 %s
 }
 define i1 @fcmp_caller(float %fa, float %fb, double %da, double %db, double* %dpa, double* %dpb, i8* %ipa, i8* %ipb) {
-; CHECK-LABEL: define {{[^@]+}}@fcmp_caller
-; CHECK-SAME: (float [[FA:%.*]], float [[FB:%.*]], double [[DA:%.*]], double [[DB:%.*]], double* nofree readnone [[DPA:%.*]], double* nofree readnone [[DPB:%.*]], i8* nofree readnone [[IPA:%.*]], i8* nofree readnone [[IPB:%.*]])
-; CHECK-NEXT:    [[R1:%.*]] = call i1 @f_fcmp(float [[FA]], float [[FB]])
-; CHECK-NEXT:    [[R2:%.*]] = call i1 @d_fcmp(double [[DA]], double [[DB]])
-; CHECK-NEXT:    [[R3:%.*]] = call i1 @dp_icmp(double* noalias nofree readnone [[DPA]], double* noalias nofree readnone [[DPB]])
-; CHECK-NEXT:    [[R4:%.*]] = call i1 @ip_icmp(i8* noalias nofree readnone [[IPA]], i8* noalias nofree readnone [[IPB]])
-; CHECK-NEXT:    [[O1:%.*]] = or i1 [[R1]], [[R2]]
-; CHECK-NEXT:    [[O2:%.*]] = or i1 [[R3]], [[R4]]
-; CHECK-NEXT:    [[O3:%.*]] = or i1 [[O1]], [[O2]]
-; CHECK-NEXT:    ret i1 [[O3]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@fcmp_caller
+; IS__TUNIT____-SAME: (float [[FA:%.*]], float [[FB:%.*]], double [[DA:%.*]], double [[DB:%.*]], double* nofree readnone [[DPA:%.*]], double* nofree readnone [[DPB:%.*]], i8* nofree readnone [[IPA:%.*]], i8* nofree readnone [[IPB:%.*]])
+; IS__TUNIT____-NEXT:    [[R1:%.*]] = call i1 @f_fcmp(float [[FA]], float [[FB]])
+; IS__TUNIT____-NEXT:    [[R2:%.*]] = call i1 @d_fcmp(double [[DA]], double [[DB]])
+; IS__TUNIT____-NEXT:    [[R3:%.*]] = call i1 @dp_icmp(double* noalias nofree readnone [[DPA]], double* noalias nofree readnone [[DPB]])
+; IS__TUNIT____-NEXT:    [[R4:%.*]] = call i1 @ip_icmp(i8* noalias nofree readnone [[IPA]], i8* noalias nofree readnone [[IPB]])
+; IS__TUNIT____-NEXT:    [[O1:%.*]] = or i1 [[R1]], [[R2]]
+; IS__TUNIT____-NEXT:    [[O2:%.*]] = or i1 [[R3]], [[R4]]
+; IS__TUNIT____-NEXT:    [[O3:%.*]] = or i1 [[O1]], [[O2]]
+; IS__TUNIT____-NEXT:    ret i1 [[O3]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@fcmp_caller
+; IS__CGSCC____-SAME: (float [[FA:%.*]], float [[FB:%.*]], double [[DA:%.*]], double [[DB:%.*]], double* nofree readnone [[DPA:%.*]], double* nofree readnone [[DPB:%.*]], i8* nofree readnone [[IPA:%.*]], i8* nofree readnone [[IPB:%.*]])
+; IS__CGSCC____-NEXT:    [[R1:%.*]] = call i1 @f_fcmp(float [[FA]], float [[FB]])
+; IS__CGSCC____-NEXT:    [[R2:%.*]] = call i1 @d_fcmp(double [[DA]], double [[DB]])
+; IS__CGSCC____-NEXT:    [[R3:%.*]] = call i1 @dp_icmp(double* noalias nofree readnone [[DPA]], double* noalias nofree readnone [[DPB]])
+; IS__CGSCC____-NEXT:    [[R4:%.*]] = call i1 @ip_icmp(i8* noalias nofree readnone [[IPA]], i8* noalias nofree readnone [[IPB]])
+; IS__CGSCC____-NEXT:    [[O1:%.*]] = or i1 [[R1]], [[R2]]
+; IS__CGSCC____-NEXT:    [[O2:%.*]] = or i1 [[R3]], [[R4]]
+; IS__CGSCC____-NEXT:    [[O3:%.*]] = or i1 [[O1]], [[O2]]
+; IS__CGSCC____-NEXT:    ret i1 [[O3]]
 ;
   %r1 = call i1 @f_fcmp(float %fa, float %fb)
   %r2 = call i1 @d_fcmp(double %da, double %db)
@@ -889,22 +1095,37 @@ define i1 @fcmp_caller(float %fa, float %fb, double %da, double %db, double* %dp
 }
 
 define i8 @ret_two() {
-; CHECK-LABEL: define {{[^@]+}}@ret_two()
-; CHECK-NEXT:    ret i8 2
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@ret_two()
+; IS__TUNIT____-NEXT:    ret i8 2
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@ret_two()
+; IS__CGSCC____-NEXT:    ret i8 2
 ;
   ret i8 2
 }
 define i8 @ret_undef() {
-; CHECK-LABEL: define {{[^@]+}}@ret_undef()
-; CHECK-NEXT:    ret i8 undef
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@ret_undef()
+; IS__TUNIT____-NEXT:    ret i8 undef
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@ret_undef()
+; IS__CGSCC____-NEXT:    ret i8 undef
 ;
   ret i8 undef
 }
 
 ; Verify we collapse undef to a value and return something non-undef here.
 define i8 @undef_collapse_1() {
-; CHECK-LABEL: define {{[^@]+}}@undef_collapse_1()
-; CHECK-NEXT:    ret i8 0
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@undef_collapse_1()
+; IS__TUNIT____-NEXT:    ret i8 0
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@undef_collapse_1()
+; IS__CGSCC____-NEXT:    ret i8 0
 ;
   %c = call i8 @ret_undef()
   %s = shl i8 %c, 2
@@ -913,8 +1134,13 @@ define i8 @undef_collapse_1() {
 
 ; Verify we collapse undef to a value and return something non-undef here.
 define i8 @undef_collapse_2() {
-; CHECK-LABEL: define {{[^@]+}}@undef_collapse_2()
-; CHECK-NEXT:    ret i8 0
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@undef_collapse_2()
+; IS__TUNIT____-NEXT:    ret i8 0
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@undef_collapse_2()
+; IS__CGSCC____-NEXT:    ret i8 0
 ;
   %c = call i8 @ret_two()
   %s = shl i8 undef, %c
@@ -923,9 +1149,11 @@ define i8 @undef_collapse_2() {
 
 define i8 @undef_collapse_caller() {
 ;
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@undef_collapse_caller()
 ; IS__TUNIT____-NEXT:    ret i8 0
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@undef_collapse_caller()
 ; IS__CGSCC____-NEXT:    [[C1:%.*]] = call i8 @undef_collapse_1()
 ; IS__CGSCC____-NEXT:    [[C2:%.*]] = call i8 @undef_collapse_2()
@@ -939,20 +1167,29 @@ define i8 @undef_collapse_caller() {
 }
 
 define i32 @ret1or2(i1 %c) {
-; CHECK-LABEL: define {{[^@]+}}@ret1or2
-; CHECK-SAME: (i1 [[C:%.*]])
-; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i32 1, i32 2
-; CHECK-NEXT:    ret i32 [[S]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@ret1or2
+; IS__TUNIT____-SAME: (i1 [[C:%.*]])
+; IS__TUNIT____-NEXT:    [[S:%.*]] = select i1 [[C]], i32 1, i32 2
+; IS__TUNIT____-NEXT:    ret i32 [[S]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@ret1or2
+; IS__CGSCC____-SAME: (i1 [[C:%.*]])
+; IS__CGSCC____-NEXT:    [[S:%.*]] = select i1 [[C]], i32 1, i32 2
+; IS__CGSCC____-NEXT:    ret i32 [[S]]
 ;
   %s = select i1 %c, i32 1, i32 2
   ret i32 %s
 }
 define i1 @callee_range_1(i1 %c1, i1 %c2, i1 %c3) {
 ;
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@callee_range_1
 ; IS__TUNIT____-SAME: (i1 [[C1:%.*]], i1 [[C2:%.*]], i1 [[C3:%.*]])
 ; IS__TUNIT____-NEXT:    ret i1 true
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@callee_range_1
 ; IS__CGSCC____-SAME: (i1 [[C1:%.*]], i1 [[C2:%.*]], i1 [[C3:%.*]])
 ; IS__CGSCC____-NEXT:    [[R1:%.*]] = call i32 @ret1or2(i1 [[C1]])
@@ -976,6 +1213,7 @@ define i1 @callee_range_1(i1 %c1, i1 %c2, i1 %c3) {
 
 define i1 @callee_range_2(i1 %c1, i1 %c2) {
 ;
+; IS__TUNIT_OPM: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@callee_range_2
 ; IS__TUNIT_OPM-SAME: (i1 [[C1:%.*]], i1 [[C2:%.*]])
 ; IS__TUNIT_OPM-NEXT:    [[R1:%.*]] = call i32 @ret1or2(i1 [[C1]]) #2, !range !4
@@ -986,6 +1224,7 @@ define i1 @callee_range_2(i1 %c1, i1 %c2) {
 ; IS__TUNIT_OPM-NEXT:    [[F:%.*]] = and i1 [[I1]], [[I2]]
 ; IS__TUNIT_OPM-NEXT:    ret i1 [[F]]
 ;
+; IS__TUNIT_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@callee_range_2
 ; IS__TUNIT_NPM-SAME: (i1 [[C1:%.*]], i1 [[C2:%.*]])
 ; IS__TUNIT_NPM-NEXT:    [[R1:%.*]] = call i32 @ret1or2(i1 [[C1]]) #1, !range !5
@@ -996,6 +1235,7 @@ define i1 @callee_range_2(i1 %c1, i1 %c2) {
 ; IS__TUNIT_NPM-NEXT:    [[F:%.*]] = and i1 [[I1]], [[I2]]
 ; IS__TUNIT_NPM-NEXT:    ret i1 [[F]]
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@callee_range_2
 ; IS__CGSCC____-SAME: (i1 [[C1:%.*]], i1 [[C2:%.*]])
 ; IS__CGSCC____-NEXT:    [[R1:%.*]] = call i32 @ret1or2(i1 [[C1]])
@@ -1017,37 +1257,70 @@ define i1 @callee_range_2(i1 %c1, i1 %c2) {
 
 
 define i32 @ret100() {
-; CHECK-LABEL: define {{[^@]+}}@ret100()
-; CHECK-NEXT:    ret i32 100
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@ret100()
+; IS__TUNIT____-NEXT:    ret i32 100
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@ret100()
+; IS__CGSCC____-NEXT:    ret i32 100
 ;
   ret i32 100
 }
 
 define i1 @ctx_adjustment(i32 %V) {
 ;
-; IS________OPM-LABEL: define {{[^@]+}}@ctx_adjustment
-; IS________OPM-SAME: (i32 [[V:%.*]])
-; IS________OPM-NEXT:    [[C1:%.*]] = icmp sge i32 [[V]], 100
-; IS________OPM-NEXT:    br i1 [[C1]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
-; IS________OPM:       if.true:
-; IS________OPM-NEXT:    br label [[END:%.*]]
-; IS________OPM:       if.false:
-; IS________OPM-NEXT:    br label [[END]]
-; IS________OPM:       end:
-; IS________OPM-NEXT:    [[PHI:%.*]] = phi i32 [ [[V]], [[IF_TRUE]] ], [ 100, [[IF_FALSE]] ]
-; IS________OPM-NEXT:    [[C2:%.*]] = icmp sge i32 [[PHI]], 100
-; IS________OPM-NEXT:    ret i1 [[C2]]
-;
-; IS________NPM-LABEL: define {{[^@]+}}@ctx_adjustment
-; IS________NPM-SAME: (i32 [[V:%.*]])
-; IS________NPM-NEXT:    [[C1:%.*]] = icmp sge i32 [[V]], 100
-; IS________NPM-NEXT:    br i1 [[C1]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
-; IS________NPM:       if.true:
-; IS________NPM-NEXT:    br label [[END:%.*]]
-; IS________NPM:       if.false:
-; IS________NPM-NEXT:    br label [[END]]
-; IS________NPM:       end:
-; IS________NPM-NEXT:    ret i1 true
+; IS__TUNIT_OPM: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@ctx_adjustment
+; IS__TUNIT_OPM-SAME: (i32 [[V:%.*]])
+; IS__TUNIT_OPM-NEXT:    [[C1:%.*]] = icmp sge i32 [[V]], 100
+; IS__TUNIT_OPM-NEXT:    br i1 [[C1]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; IS__TUNIT_OPM:       if.true:
+; IS__TUNIT_OPM-NEXT:    br label [[END:%.*]]
+; IS__TUNIT_OPM:       if.false:
+; IS__TUNIT_OPM-NEXT:    br label [[END]]
+; IS__TUNIT_OPM:       end:
+; IS__TUNIT_OPM-NEXT:    [[PHI:%.*]] = phi i32 [ [[V]], [[IF_TRUE]] ], [ 100, [[IF_FALSE]] ]
+; IS__TUNIT_OPM-NEXT:    [[C2:%.*]] = icmp sge i32 [[PHI]], 100
+; IS__TUNIT_OPM-NEXT:    ret i1 [[C2]]
+;
+; IS__TUNIT_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@ctx_adjustment
+; IS__TUNIT_NPM-SAME: (i32 [[V:%.*]])
+; IS__TUNIT_NPM-NEXT:    [[C1:%.*]] = icmp sge i32 [[V]], 100
+; IS__TUNIT_NPM-NEXT:    br i1 [[C1]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; IS__TUNIT_NPM:       if.true:
+; IS__TUNIT_NPM-NEXT:    br label [[END:%.*]]
+; IS__TUNIT_NPM:       if.false:
+; IS__TUNIT_NPM-NEXT:    br label [[END]]
+; IS__TUNIT_NPM:       end:
+; IS__TUNIT_NPM-NEXT:    ret i1 true
+;
+; IS__CGSCC_OPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@ctx_adjustment
+; IS__CGSCC_OPM-SAME: (i32 [[V:%.*]])
+; IS__CGSCC_OPM-NEXT:    [[C1:%.*]] = icmp sge i32 [[V]], 100
+; IS__CGSCC_OPM-NEXT:    br i1 [[C1]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; IS__CGSCC_OPM:       if.true:
+; IS__CGSCC_OPM-NEXT:    br label [[END:%.*]]
+; IS__CGSCC_OPM:       if.false:
+; IS__CGSCC_OPM-NEXT:    br label [[END]]
+; IS__CGSCC_OPM:       end:
+; IS__CGSCC_OPM-NEXT:    [[PHI:%.*]] = phi i32 [ [[V]], [[IF_TRUE]] ], [ 100, [[IF_FALSE]] ]
+; IS__CGSCC_OPM-NEXT:    [[C2:%.*]] = icmp sge i32 [[PHI]], 100
+; IS__CGSCC_OPM-NEXT:    ret i1 [[C2]]
+;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@ctx_adjustment
+; IS__CGSCC_NPM-SAME: (i32 [[V:%.*]])
+; IS__CGSCC_NPM-NEXT:    [[C1:%.*]] = icmp sge i32 [[V]], 100
+; IS__CGSCC_NPM-NEXT:    br i1 [[C1]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; IS__CGSCC_NPM:       if.true:
+; IS__CGSCC_NPM-NEXT:    br label [[END:%.*]]
+; IS__CGSCC_NPM:       if.false:
+; IS__CGSCC_NPM-NEXT:    br label [[END]]
+; IS__CGSCC_NPM:       end:
+; IS__CGSCC_NPM-NEXT:    ret i1 true
 ;
   %c1 = icmp sge i32 %V, 100
   br i1 %c1, label %if.true, label %if.false
@@ -1064,16 +1337,24 @@ end:
 
 
 define i32 @func(i1 %c) {
-; CHECK-LABEL: define {{[^@]+}}@func
-; CHECK-SAME: (i1 [[C:%.*]])
-; CHECK-NEXT:    [[RET:%.*]] = select i1 [[C]], i32 0, i32 1
-; CHECK-NEXT:    ret i32 [[RET]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@func
+; IS__TUNIT____-SAME: (i1 [[C:%.*]])
+; IS__TUNIT____-NEXT:    [[RET:%.*]] = select i1 [[C]], i32 0, i32 1
+; IS__TUNIT____-NEXT:    ret i32 [[RET]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@func
+; IS__CGSCC____-SAME: (i1 [[C:%.*]])
+; IS__CGSCC____-NEXT:    [[RET:%.*]] = select i1 [[C]], i32 0, i32 1
+; IS__CGSCC____-NEXT:    ret i32 [[RET]]
 ;
   %ret = select i1 %c, i32 0, i32 1
   ret i32 %ret
 }
 
 define i32 @simplify_callsite_argument(i1 %d) {
+; IS__TUNIT_OPM: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@simplify_callsite_argument
 ; IS__TUNIT_OPM-SAME: (i1 [[D:%.*]])
 ; IS__TUNIT_OPM-NEXT:    [[C:%.*]] = select i1 [[D]], i1 true, i1 false
@@ -1085,6 +1366,7 @@ define i32 @simplify_callsite_argument(i1 %d) {
 ; IS__TUNIT_OPM-NEXT:    [[RET2:%.*]] = call i32 @func(i1 false) #2, !range !3
 ; IS__TUNIT_OPM-NEXT:    ret i32 [[RET2]]
 ;
+; IS__TUNIT_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@simplify_callsite_argument
 ; IS__TUNIT_NPM-SAME: (i1 [[D:%.*]])
 ; IS__TUNIT_NPM-NEXT:    [[C:%.*]] = select i1 [[D]], i1 true, i1 false
@@ -1096,6 +1378,7 @@ define i32 @simplify_callsite_argument(i1 %d) {
 ; IS__TUNIT_NPM-NEXT:    [[RET2:%.*]] = call i32 @func(i1 false) #1, !range !4
 ; IS__TUNIT_NPM-NEXT:    ret i32 [[RET2]]
 ;
+; IS__CGSCC_OPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@simplify_callsite_argument
 ; IS__CGSCC_OPM-SAME: (i1 [[D:%.*]])
 ; IS__CGSCC_OPM-NEXT:    [[C:%.*]] = select i1 [[D]], i1 true, i1 false
@@ -1107,6 +1390,7 @@ define i32 @simplify_callsite_argument(i1 %d) {
 ; IS__CGSCC_OPM-NEXT:    [[RET2:%.*]] = call i32 @func(i1 false)
 ; IS__CGSCC_OPM-NEXT:    ret i32 [[RET2]]
 ;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@simplify_callsite_argument
 ; IS__CGSCC_NPM-SAME: (i1 [[D:%.*]])
 ; IS__CGSCC_NPM-NEXT:    [[C:%.*]] = select i1 [[D]], i1 true, i1 false
diff --git a/llvm/test/Transforms/Attributor/read_write_returned_arguments_scc.ll b/llvm/test/Transforms/Attributor/read_write_returned_arguments_scc.ll
index d42997f05b419..3c7e8c5ff4d79 100644
--- a/llvm/test/Transforms/Attributor/read_write_returned_arguments_scc.ll
+++ b/llvm/test/Transforms/Attributor/read_write_returned_arguments_scc.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=16 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=16 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -34,8 +34,8 @@
 ;
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
-; CHECK: Function Attrs: argmemonly nofree nosync nounwind
 define i32* @external_ret2_nrw(i32* %n0, i32* %r0, i32* %w0) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@external_ret2_nrw
 ; IS__TUNIT____-SAME: (i32* nofree [[N0:%.*]], i32* nofree [[R0:%.*]], i32* nofree returned [[W0:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
@@ -45,6 +45,7 @@ define i32* @external_ret2_nrw(i32* %n0, i32* %r0, i32* %w0) {
 ; IS__TUNIT____-NEXT:    [[CALL3:%.*]] = call i32* @internal_ret1_rw(i32* nofree align 4 [[R0]], i32* nofree [[W0]])
 ; IS__TUNIT____-NEXT:    ret i32* [[CALL3]]
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree nosync nounwind
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@external_ret2_nrw
 ; IS__CGSCC____-SAME: (i32* nofree [[N0:%.*]], i32* nofree align 4 [[R0:%.*]], i32* nofree returned [[W0:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -62,8 +63,8 @@ entry:
   ret i32* %call3
 }
 
-; CHECK: Function Attrs: argmemonly nofree nosync nounwind
 define internal i32* @internal_ret0_nw(i32* %n0, i32* %w0) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@internal_ret0_nw
 ; IS__TUNIT____-SAME: (i32* nofree returned [[N0:%.*]], i32* nofree [[W0:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
@@ -88,6 +89,7 @@ define internal i32* @internal_ret0_nw(i32* %n0, i32* %w0) {
 ; IS__TUNIT____-NEXT:    [[RETVAL_0:%.*]] = phi i32* [ [[CALL5]], [[IF_END]] ], [ [[N0]], [[IF_THEN]] ]
 ; IS__TUNIT____-NEXT:    ret i32* [[RETVAL_0]]
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree nosync nounwind
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@internal_ret0_nw
 ; IS__CGSCC____-SAME: (i32* nofree returned [[N0:%.*]], i32* nofree [[W0:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -138,8 +140,8 @@ return:                                           ; preds = %if.end, %if.then
   ret i32* %retval.0
 }
 
-; CHECK: Function Attrs: argmemonly nofree nosync nounwind
 define internal i32* @internal_ret1_rrw(i32* %r0, i32* %r1, i32* %w0) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@internal_ret1_rrw
 ; IS__TUNIT____-SAME: (i32* nofree nonnull align 4 dereferenceable(4) [[R0:%.*]], i32* nofree returned [[R1:%.*]], i32* nofree [[W0:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
@@ -167,6 +169,7 @@ define internal i32* @internal_ret1_rrw(i32* %r0, i32* %r1, i32* %w0) {
 ; IS__TUNIT____-NEXT:    [[RETVAL_0:%.*]] = phi i32* [ [[CALL8]], [[IF_END]] ], [ [[R1]], [[IF_THEN]] ]
 ; IS__TUNIT____-NEXT:    ret i32* undef
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree nosync nounwind
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@internal_ret1_rrw
 ; IS__CGSCC____-SAME: (i32* nofree nonnull align 4 dereferenceable(4) [[R0:%.*]], i32* nofree returned [[R1:%.*]], i32* nofree [[W0:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -223,22 +226,36 @@ return:                                           ; preds = %if.end, %if.then
   ret i32* %retval.0
 }
 
+define i32* @external_sink_ret2_nrw(i32* %n0, i32* %r0, i32* %w0) {
 ; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@external_sink_ret2_nrw
+; IS__TUNIT____-SAME: (i32* nofree [[N0:%.*]], i32* nocapture nofree readonly [[R0:%.*]], i32* nofree returned writeonly "no-capture-maybe-returned" [[W0:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[TOBOOL:%.*]] = icmp ne i32* [[N0]], null
+; IS__TUNIT____-NEXT:    br i1 [[TOBOOL]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
+; IS__TUNIT____:       if.then:
+; IS__TUNIT____-NEXT:    br label [[RETURN:%.*]]
+; IS__TUNIT____:       if.end:
+; IS__TUNIT____-NEXT:    [[TMP0:%.*]] = load i32, i32* [[R0]], align 4
+; IS__TUNIT____-NEXT:    store i32 [[TMP0]], i32* [[W0]], align 4
+; IS__TUNIT____-NEXT:    br label [[RETURN]]
+; IS__TUNIT____:       return:
+; IS__TUNIT____-NEXT:    ret i32* [[W0]]
+;
 ; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn
-define i32* @external_sink_ret2_nrw(i32* %n0, i32* %r0, i32* %w0) {
-; CHECK-LABEL: define {{[^@]+}}@external_sink_ret2_nrw
-; CHECK-SAME: (i32* nofree [[N0:%.*]], i32* nocapture nofree readonly [[R0:%.*]], i32* nofree returned writeonly "no-capture-maybe-returned" [[W0:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32* [[N0]], null
-; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    br label [[RETURN:%.*]]
-; CHECK:       if.end:
-; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[R0]], align 4
-; CHECK-NEXT:    store i32 [[TMP0]], i32* [[W0]], align 4
-; CHECK-NEXT:    br label [[RETURN]]
-; CHECK:       return:
-; CHECK-NEXT:    ret i32* [[W0]]
+; IS__CGSCC____-LABEL: define {{[^@]+}}@external_sink_ret2_nrw
+; IS__CGSCC____-SAME: (i32* nofree [[N0:%.*]], i32* nocapture nofree readonly [[R0:%.*]], i32* nofree returned writeonly "no-capture-maybe-returned" [[W0:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[TOBOOL:%.*]] = icmp ne i32* [[N0]], null
+; IS__CGSCC____-NEXT:    br i1 [[TOBOOL]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
+; IS__CGSCC____:       if.then:
+; IS__CGSCC____-NEXT:    br label [[RETURN:%.*]]
+; IS__CGSCC____:       if.end:
+; IS__CGSCC____-NEXT:    [[TMP0:%.*]] = load i32, i32* [[R0]], align 4
+; IS__CGSCC____-NEXT:    store i32 [[TMP0]], i32* [[W0]], align 4
+; IS__CGSCC____-NEXT:    br label [[RETURN]]
+; IS__CGSCC____:       return:
+; IS__CGSCC____-NEXT:    ret i32* [[W0]]
 ;
 entry:
   %tobool = icmp ne i32* %n0, null
@@ -256,8 +273,8 @@ return:                                           ; preds = %if.end, %if.then
   ret i32* %w0
 }
 
-; CHECK: Function Attrs: argmemonly nofree nosync nounwind
 define internal i32* @internal_ret1_rw(i32* %r0, i32* %w0) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@internal_ret1_rw
 ; IS__TUNIT____-SAME: (i32* nofree nonnull align 4 dereferenceable(4) [[R0:%.*]], i32* nofree returned [[W0:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
@@ -279,6 +296,7 @@ define internal i32* @internal_ret1_rw(i32* %r0, i32* %w0) {
 ; IS__TUNIT____-NEXT:    [[RETVAL_0:%.*]] = phi i32* [ [[CALL4]], [[IF_END]] ], [ [[W0]], [[IF_THEN]] ]
 ; IS__TUNIT____-NEXT:    ret i32* [[RETVAL_0]]
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree nosync nounwind
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@internal_ret1_rw
 ; IS__CGSCC____-SAME: (i32* nofree nonnull align 4 dereferenceable(4) [[R0:%.*]], i32* nofree returned [[W0:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -323,8 +341,8 @@ return:                                           ; preds = %if.end, %if.then
   ret i32* %retval.0
 }
 
-; CHECK: Function Attrs: argmemonly nofree nosync nounwind
 define i32* @external_source_ret2_nrw(i32* %n0, i32* %r0, i32* %w0) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@external_source_ret2_nrw
 ; IS__TUNIT____-SAME: (i32* nofree [[N0:%.*]], i32* nofree [[R0:%.*]], i32* nofree returned [[W0:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
@@ -332,6 +350,7 @@ define i32* @external_source_ret2_nrw(i32* %n0, i32* %r0, i32* %w0) {
 ; IS__TUNIT____-NEXT:    [[CALL1:%.*]] = call i32* @external_ret2_nrw(i32* nofree [[N0]], i32* nofree [[R0]], i32* nofree [[W0]])
 ; IS__TUNIT____-NEXT:    ret i32* [[CALL1]]
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree nosync nounwind
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@external_source_ret2_nrw
 ; IS__CGSCC____-SAME: (i32* nofree [[N0:%.*]], i32* nofree align 4 [[R0:%.*]], i32* nofree returned [[W0:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
diff --git a/llvm/test/Transforms/Attributor/readattrs.ll b/llvm/test/Transforms/Attributor/readattrs.ll
index a78ffb0653b69..058fa0d83d001 100644
--- a/llvm/test/Transforms/Attributor/readattrs.ll
+++ b/llvm/test/Transforms/Attributor/readattrs.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -24,20 +24,34 @@ define void @test1_2(i8* %x1_2, i8* %y1_2, i8* %z1_2) {
 }
 
 define i8* @test2(i8* %p) {
-; CHECK-LABEL: define {{[^@]+}}@test2
-; CHECK-SAME: (i8* nofree readnone returned "no-capture-maybe-returned" [[P:%.*]])
-; CHECK-NEXT:    store i32 0, i32* @x, align 4
-; CHECK-NEXT:    ret i8* [[P]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test2
+; IS__TUNIT____-SAME: (i8* nofree readnone returned "no-capture-maybe-returned" [[P:%.*]])
+; IS__TUNIT____-NEXT:    store i32 0, i32* @x, align 4
+; IS__TUNIT____-NEXT:    ret i8* [[P]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test2
+; IS__CGSCC____-SAME: (i8* nofree readnone returned "no-capture-maybe-returned" [[P:%.*]])
+; IS__CGSCC____-NEXT:    store i32 0, i32* @x, align 4
+; IS__CGSCC____-NEXT:    ret i8* [[P]]
 ;
   store i32 0, i32* @x
   ret i8* %p
 }
 
 define i1 @test3(i8* %p, i8* %q) {
-; CHECK-LABEL: define {{[^@]+}}@test3
-; CHECK-SAME: (i8* nofree readnone [[P:%.*]], i8* nofree readnone [[Q:%.*]])
-; CHECK-NEXT:    [[A:%.*]] = icmp ult i8* [[P]], [[Q]]
-; CHECK-NEXT:    ret i1 [[A]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test3
+; IS__TUNIT____-SAME: (i8* nofree readnone [[P:%.*]], i8* nofree readnone [[Q:%.*]])
+; IS__TUNIT____-NEXT:    [[A:%.*]] = icmp ult i8* [[P]], [[Q]]
+; IS__TUNIT____-NEXT:    ret i1 [[A]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test3
+; IS__CGSCC____-SAME: (i8* nofree readnone [[P:%.*]], i8* nofree readnone [[Q:%.*]])
+; IS__CGSCC____-NEXT:    [[A:%.*]] = icmp ult i8* [[P]], [[Q]]
+; IS__CGSCC____-NEXT:    ret i1 [[A]]
 ;
   %A = icmp ult i8* %p, %q
   ret i1 %A
@@ -46,6 +60,7 @@ define i1 @test3(i8* %p, i8* %q) {
 declare void @test4_1(i8* nocapture) readonly
 
 define void @test4_2(i8* %p) {
+; CHECK: Function Attrs: readonly
 ; CHECK-LABEL: define {{[^@]+}}@test4_2
 ; CHECK-SAME: (i8* nocapture readonly [[P:%.*]])
 ; CHECK-NEXT:    call void @test4_1(i8* nocapture readonly [[P]])
@@ -57,10 +72,17 @@ define void @test4_2(i8* %p) {
 
 ; Missed optz'n: we could make %q readnone, but don't break test6!
 define void @test5(i8** %p, i8* %q) {
-; CHECK-LABEL: define {{[^@]+}}@test5
-; CHECK-SAME: (i8** nocapture nofree nonnull writeonly align 8 dereferenceable(8) [[P:%.*]], i8* nofree writeonly [[Q:%.*]])
-; CHECK-NEXT:    store i8* [[Q]], i8** [[P]], align 8
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test5
+; IS__TUNIT____-SAME: (i8** nocapture nofree nonnull writeonly align 8 dereferenceable(8) [[P:%.*]], i8* nofree writeonly [[Q:%.*]])
+; IS__TUNIT____-NEXT:    store i8* [[Q]], i8** [[P]], align 8
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test5
+; IS__CGSCC____-SAME: (i8** nocapture nofree nonnull writeonly align 8 dereferenceable(8) [[P:%.*]], i8* nofree writeonly [[Q:%.*]])
+; IS__CGSCC____-NEXT:    store i8* [[Q]], i8** [[P]], align 8
+; IS__CGSCC____-NEXT:    ret void
 ;
   store i8* %q, i8** %p
   ret void
@@ -82,24 +104,38 @@ define void @test6_2(i8** %p, i8* %q) {
 
 ; inalloca parameters are always considered written
 define void @test7_1(i32* inalloca %a) {
-; CHECK-LABEL: define {{[^@]+}}@test7_1
-; CHECK-SAME: (i32* inalloca nocapture nofree writeonly [[A:%.*]])
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test7_1
+; IS__TUNIT____-SAME: (i32* inalloca nocapture nofree writeonly [[A:%.*]])
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test7_1
+; IS__CGSCC____-SAME: (i32* inalloca nocapture nofree writeonly [[A:%.*]])
+; IS__CGSCC____-NEXT:    ret void
 ;
   ret void
 }
 
 define i32* @test8_1(i32* %p) {
-; CHECK-LABEL: define {{[^@]+}}@test8_1
-; CHECK-SAME: (i32* nofree readnone returned "no-capture-maybe-returned" [[P:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i32* [[P]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test8_1
+; IS__TUNIT____-SAME: (i32* nofree readnone returned "no-capture-maybe-returned" [[P:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    ret i32* [[P]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test8_1
+; IS__CGSCC____-SAME: (i32* nofree readnone returned "no-capture-maybe-returned" [[P:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    ret i32* [[P]]
 ;
 entry:
   ret i32* %p
 }
 
 define void @test8_2(i32* %p) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind willreturn writeonly
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@test8_2
 ; IS__TUNIT____-SAME: (i32* nocapture nofree writeonly [[P:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
@@ -107,6 +143,7 @@ define void @test8_2(i32* %p) {
 ; IS__TUNIT____-NEXT:    store i32 10, i32* [[CALL]], align 4
 ; IS__TUNIT____-NEXT:    ret void
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@test8_2
 ; IS__CGSCC____-SAME: (i32* nofree writeonly [[P:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -126,6 +163,7 @@ declare void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32>%val, <4 x i32*>, i32,
 ; CHECK-NOT: readnone
 ; CHECK-NOT: readonly
 define void @test9(<4 x i32*> %ptrs, <4 x i32>%val) {
+; CHECK: Function Attrs: nounwind willreturn
 ; CHECK-LABEL: define {{[^@]+}}@test9
 ; CHECK-SAME: (<4 x i32*> [[PTRS:%.*]], <4 x i32> [[VAL:%.*]])
 ; CHECK-NEXT:    call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> [[VAL]], <4 x i32*> [[PTRS]], i32 4, <4 x i1> <i1 true, i1 false, i1 true, i1 false>)
@@ -137,8 +175,8 @@ define void @test9(<4 x i32*> %ptrs, <4 x i32>%val) {
 
 ; CHECK: declare <4 x i32> @llvm.masked.gather
 declare <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*>, i32, <4 x i1>, <4 x i32>)
-; CHECK: readonly
 define <4 x i32> @test10(<4 x i32*> %ptrs) {
+; CHECK: Function Attrs: nounwind readonly willreturn
 ; CHECK-LABEL: define {{[^@]+}}@test10
 ; CHECK-SAME: (<4 x i32*> [[PTRS:%.*]])
 ; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[PTRS]], i32 4, <4 x i1> <i1 true, i1 false, i1 true, i1 false>, <4 x i32> undef)
@@ -150,9 +188,8 @@ define <4 x i32> @test10(<4 x i32*> %ptrs) {
 
 ; CHECK: declare <4 x i32> @test11_1
 declare <4 x i32> @test11_1(<4 x i32*>) argmemonly nounwind readonly
-; CHECK: readonly
-; CHECK-NOT: readnone
 define <4 x i32> @test11_2(<4 x i32*> %ptrs) {
+; CHECK: Function Attrs: argmemonly nounwind readonly
 ; CHECK-LABEL: define {{[^@]+}}@test11_2
 ; CHECK-SAME: (<4 x i32*> [[PTRS:%.*]])
 ; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @test11_1(<4 x i32*> [[PTRS]])
@@ -165,6 +202,7 @@ define <4 x i32> @test11_2(<4 x i32*> %ptrs) {
 declare <4 x i32> @test12_1(<4 x i32*>) argmemonly nounwind
 ; CHECK-NOT: readnone
 define <4 x i32> @test12_2(<4 x i32*> %ptrs) {
+; CHECK: Function Attrs: argmemonly nounwind
 ; CHECK-LABEL: define {{[^@]+}}@test12_2
 ; CHECK-SAME: (<4 x i32*> [[PTRS:%.*]])
 ; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @test12_1(<4 x i32*> [[PTRS]])
@@ -175,10 +213,17 @@ define <4 x i32> @test12_2(<4 x i32*> %ptrs) {
 }
 
 define i32 @volatile_load(i32* %p) {
-; CHECK-LABEL: define {{[^@]+}}@volatile_load
-; CHECK-SAME: (i32* nofree align 4 [[P:%.*]])
-; CHECK-NEXT:    [[LOAD:%.*]] = load volatile i32, i32* [[P]], align 4
-; CHECK-NEXT:    ret i32 [[LOAD]]
+; IS__TUNIT____: Function Attrs: argmemonly nofree nounwind willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@volatile_load
+; IS__TUNIT____-SAME: (i32* nofree align 4 [[P:%.*]])
+; IS__TUNIT____-NEXT:    [[LOAD:%.*]] = load volatile i32, i32* [[P]], align 4
+; IS__TUNIT____-NEXT:    ret i32 [[LOAD]]
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nounwind willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@volatile_load
+; IS__CGSCC____-SAME: (i32* nofree align 4 [[P:%.*]])
+; IS__CGSCC____-NEXT:    [[LOAD:%.*]] = load volatile i32, i32* [[P]], align 4
+; IS__CGSCC____-NEXT:    ret i32 [[LOAD]]
 ;
   %load = load volatile i32, i32* %p
   ret i32 %load
@@ -230,6 +275,7 @@ define void @unsound_readonly(i8* %ignored, i8* %escaped_then_written) {
 declare void @escape_i8(i8* %ptr)
 
 define void @byval_not_readonly_1(i8* byval %written) readonly {
+; CHECK: Function Attrs: readonly
 ; CHECK-LABEL: define {{[^@]+}}@byval_not_readonly_1
 ; CHECK-SAME: (i8* noalias nonnull byval dereferenceable(1) [[WRITTEN:%.*]])
 ; CHECK-NEXT:    call void @escape_i8(i8* nonnull dereferenceable(1) [[WRITTEN]])
@@ -240,16 +286,24 @@ define void @byval_not_readonly_1(i8* byval %written) readonly {
 }
 
 define void @byval_not_readonly_2(i8* byval %written) readonly {
-; CHECK-LABEL: define {{[^@]+}}@byval_not_readonly_2
-; CHECK-SAME: (i8* noalias nocapture nofree nonnull writeonly byval dereferenceable(1) [[WRITTEN:%.*]])
-; CHECK-NEXT:    store i8 0, i8* [[WRITTEN]], align 1
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@byval_not_readonly_2
+; IS__TUNIT____-SAME: (i8* noalias nocapture nofree nonnull writeonly byval dereferenceable(1) [[WRITTEN:%.*]])
+; IS__TUNIT____-NEXT:    store i8 0, i8* [[WRITTEN]], align 1
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@byval_not_readonly_2
+; IS__CGSCC____-SAME: (i8* noalias nocapture nofree nonnull writeonly byval dereferenceable(1) [[WRITTEN:%.*]])
+; IS__CGSCC____-NEXT:    store i8 0, i8* [[WRITTEN]], align 1
+; IS__CGSCC____-NEXT:    ret void
 ;
   store i8 0, i8* %written
   ret void
 }
 
 define void @byval_not_readnone_1(i8* byval %written) readnone {
+; CHECK: Function Attrs: readnone
 ; CHECK-LABEL: define {{[^@]+}}@byval_not_readnone_1
 ; CHECK-SAME: (i8* noalias nonnull byval dereferenceable(1) [[WRITTEN:%.*]])
 ; CHECK-NEXT:    call void @escape_i8(i8* nonnull dereferenceable(1) [[WRITTEN]])
@@ -260,20 +314,34 @@ define void @byval_not_readnone_1(i8* byval %written) readnone {
 }
 
 define void @byval_not_readnone_2(i8* byval %written) readnone {
-; CHECK-LABEL: define {{[^@]+}}@byval_not_readnone_2
-; CHECK-SAME: (i8* noalias nocapture nofree nonnull writeonly byval dereferenceable(1) [[WRITTEN:%.*]])
-; CHECK-NEXT:    store i8 0, i8* [[WRITTEN]], align 1
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@byval_not_readnone_2
+; IS__TUNIT____-SAME: (i8* noalias nocapture nofree nonnull writeonly byval dereferenceable(1) [[WRITTEN:%.*]])
+; IS__TUNIT____-NEXT:    store i8 0, i8* [[WRITTEN]], align 1
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@byval_not_readnone_2
+; IS__CGSCC____-SAME: (i8* noalias nocapture nofree nonnull writeonly byval dereferenceable(1) [[WRITTEN:%.*]])
+; IS__CGSCC____-NEXT:    store i8 0, i8* [[WRITTEN]], align 1
+; IS__CGSCC____-NEXT:    ret void
 ;
   store i8 0, i8* %written
   ret void
 }
 
 define void @byval_no_fnarg(i8* byval %written) {
-; CHECK-LABEL: define {{[^@]+}}@byval_no_fnarg
-; CHECK-SAME: (i8* noalias nocapture nofree nonnull writeonly byval dereferenceable(1) [[WRITTEN:%.*]])
-; CHECK-NEXT:    store i8 0, i8* [[WRITTEN]], align 1
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@byval_no_fnarg
+; IS__TUNIT____-SAME: (i8* noalias nocapture nofree nonnull writeonly byval dereferenceable(1) [[WRITTEN:%.*]])
+; IS__TUNIT____-NEXT:    store i8 0, i8* [[WRITTEN]], align 1
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@byval_no_fnarg
+; IS__CGSCC____-SAME: (i8* noalias nocapture nofree nonnull writeonly byval dereferenceable(1) [[WRITTEN:%.*]])
+; IS__CGSCC____-NEXT:    store i8 0, i8* [[WRITTEN]], align 1
+; IS__CGSCC____-NEXT:    ret void
 ;
   store i8 0, i8* %written
   ret void
@@ -286,6 +354,7 @@ define void @testbyval(i8* %read_only) {
 ; IS__TUNIT____-NEXT:    call void @byval_not_readnone_1(i8* noalias nocapture readnone [[READ_ONLY]])
 ; IS__TUNIT____-NEXT:    ret void
 ;
+; IS__CGSCC____: Function Attrs: readonly
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@testbyval
 ; IS__CGSCC____-SAME: (i8* nocapture nonnull readonly dereferenceable(1) [[READ_ONLY:%.*]])
 ; IS__CGSCC____-NEXT:    call void @byval_not_readonly_1(i8* noalias nocapture nonnull readonly dereferenceable(1) [[READ_ONLY]])
@@ -306,6 +375,7 @@ declare i8 @maybe_returned_val(i8* %ptr) readonly nounwind
 declare void @val_use(i8 %ptr) readonly nounwind
 
 define void @ptr_uses(i8* %ptr) {
+; CHECK: Function Attrs: nounwind readonly
 ; CHECK-LABEL: define {{[^@]+}}@ptr_uses
 ; CHECK-SAME: (i8* nocapture readonly [[PTR:%.*]])
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/Attributor/reduced/register_benchmark_test.ll b/llvm/test/Transforms/Attributor/reduced/register_benchmark_test.ll
index d983bc947cac3..462d333911bae 100644
--- a/llvm/test/Transforms/Attributor/reduced/register_benchmark_test.ll
+++ b/llvm/test/Transforms/Attributor/reduced/register_benchmark_test.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s
 ; RUN: opt -attributor-cgscc  -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s
 ; RUN: opt -passes=attributor  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s
diff --git a/llvm/test/Transforms/Attributor/returned.ll b/llvm/test/Transforms/Attributor/returned.ll
index 1d14685ed1fc6..ab6f73acdcdb0 100644
--- a/llvm/test/Transforms/Attributor/returned.ll
+++ b/llvm/test/Transforms/Attributor/returned.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=14 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=14 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -38,16 +38,24 @@
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 define i32 @sink_r0(i32 %r) #0 {
-; CHECK-LABEL: define {{[^@]+}}@sink_r0
-; CHECK-SAME: (i32 returned [[R:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i32 [[R]]
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@sink_r0
+; IS__TUNIT____-SAME: (i32 returned [[R:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    ret i32 [[R]]
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@sink_r0
+; IS__CGSCC____-SAME: (i32 returned [[R:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    ret i32 [[R]]
 ;
 entry:
   ret i32 %r
 }
 
 define i32 @scc_r1(i32 %a, i32 %r, i32 %b) #0 {
+; CHECK: Function Attrs: nofree noinline nosync nounwind readnone uwtable
 ; CHECK-LABEL: define {{[^@]+}}@scc_r1
 ; CHECK-SAME: (i32 [[A:%.*]], i32 returned [[R:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:  entry:
@@ -62,6 +70,7 @@ entry:
 }
 
 define i32 @scc_r2(i32 %a, i32 %b, i32 %r) #0 {
+; CHECK: Function Attrs: nofree noinline nosync nounwind readnone uwtable
 ; CHECK-LABEL: define {{[^@]+}}@scc_r2
 ; CHECK-SAME: (i32 [[A:%.*]], i32 [[B:%.*]], i32 returned [[R:%.*]])
 ; CHECK-NEXT:  entry:
@@ -143,6 +152,7 @@ return:                                           ; preds = %cond.end, %if.then3
 }
 
 define i32 @scc_rX(i32 %a, i32 %b, i32 %r) #0 {
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@scc_rX
 ; IS__TUNIT____-SAME: (i32 [[A:%.*]], i32 [[B:%.*]], i32 [[R:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
@@ -179,6 +189,7 @@ define i32 @scc_rX(i32 %a, i32 %b, i32 %r) #0 {
 ; IS__TUNIT____-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[CALL1]], [[IF_THEN]] ], [ [[CALL11]], [[IF_THEN3]] ], [ [[COND]], [[COND_END]] ]
 ; IS__TUNIT____-NEXT:    ret i32 [[RETVAL_0]]
 ;
+; IS__CGSCC____: Function Attrs: nofree noinline nosync nounwind readnone uwtable
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@scc_rX
 ; IS__CGSCC____-SAME: (i32 [[A:%.*]], i32 [[B:%.*]], i32 [[R:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -282,16 +293,24 @@ return:                                           ; preds = %cond.end, %if.then3
 ;   return a == b ? r : ptr_scc_r2(a, b, r);
 ; }
 define double* @ptr_sink_r0(double* %r) #0 {
-; CHECK-LABEL: define {{[^@]+}}@ptr_sink_r0
-; CHECK-SAME: (double* nofree readnone returned "no-capture-maybe-returned" [[R:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret double* [[R]]
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@ptr_sink_r0
+; IS__TUNIT____-SAME: (double* nofree readnone returned "no-capture-maybe-returned" [[R:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    ret double* [[R]]
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@ptr_sink_r0
+; IS__CGSCC____-SAME: (double* nofree readnone returned "no-capture-maybe-returned" [[R:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    ret double* [[R]]
 ;
 entry:
   ret double* %r
 }
 
 define double* @ptr_scc_r1(double* %a, double* %r, double* %b) #0 {
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@ptr_scc_r1
 ; IS__TUNIT____-SAME: (double* nofree readnone [[A:%.*]], double* nofree readnone returned [[R:%.*]], double* nocapture nofree readnone [[B:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
@@ -299,6 +318,7 @@ define double* @ptr_scc_r1(double* %a, double* %r, double* %b) #0 {
 ; IS__TUNIT____-NEXT:    [[CALL1:%.*]] = call double* @ptr_scc_r2(double* noalias nofree readnone [[R]], double* noalias nofree readnone [[A]], double* noalias nofree readnone [[CALL]])
 ; IS__TUNIT____-NEXT:    ret double* [[CALL1]]
 ;
+; IS__CGSCC____: Function Attrs: nofree noinline nosync nounwind readnone uwtable
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@ptr_scc_r1
 ; IS__CGSCC____-SAME: (double* nofree readnone [[A:%.*]], double* nofree readnone returned [[R:%.*]], double* nocapture nofree readnone [[B:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -313,6 +333,7 @@ entry:
 }
 
 define double* @ptr_scc_r2(double* %a, double* %b, double* %r) #0 {
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@ptr_scc_r2
 ; IS__TUNIT____-SAME: (double* nofree readnone [[A:%.*]], double* nofree readnone [[B:%.*]], double* nofree readnone returned [[R:%.*]])
 ; IS__TUNIT____-NEXT:  entry:
@@ -349,6 +370,7 @@ define double* @ptr_scc_r2(double* %a, double* %b, double* %r) #0 {
 ; IS__TUNIT____-NEXT:    [[RETVAL_0:%.*]] = phi double* [ [[CALL1]], [[IF_THEN]] ], [ [[CALL11]], [[IF_THEN3]] ], [ [[COND]], [[COND_END]] ]
 ; IS__TUNIT____-NEXT:    ret double* [[RETVAL_0]]
 ;
+; IS__CGSCC____: Function Attrs: nofree noinline nosync nounwind readnone uwtable
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@ptr_scc_r2
 ; IS__CGSCC____-SAME: (double* nofree readnone [[A:%.*]], double* nofree readnone [[B:%.*]], double* nofree readnone returned [[R:%.*]])
 ; IS__CGSCC____-NEXT:  entry:
@@ -437,10 +459,17 @@ return:                                           ; preds = %cond.end, %if.then3
 ; }
 ;
 define i32* @rt0(i32* %a) #0 {
-; CHECK-LABEL: define {{[^@]+}}@rt0
-; CHECK-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(4) [[A:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@rt0
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(4) [[A:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@rt0
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(4) [[A:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    unreachable
 ;
 entry:
   %v = load i32, i32* %a, align 4
@@ -457,10 +486,17 @@ entry:
 ; }
 ;
 define i32* @rt1(i32* %a) #0 {
-; CHECK-LABEL: define {{[^@]+}}@rt1
-; CHECK-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(4) [[A:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@rt1
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(4) [[A:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@rt1
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(4) [[A:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    unreachable
 ;
 entry:
   %v = load i32, i32* %a, align 4
@@ -473,6 +509,7 @@ entry:
 ; TEST another SCC test
 ;
 define i32* @rt2_helper(i32* %a) #0 {
+; CHECK: Function Attrs: nofree noinline nosync nounwind readnone uwtable
 ; CHECK-LABEL: define {{[^@]+}}@rt2_helper
 ; CHECK-SAME: (i32* nofree readnone returned [[A:%.*]])
 ; CHECK-NEXT:  entry:
@@ -485,6 +522,7 @@ entry:
 }
 
 define i32* @rt2(i32* %a, i32 *%b) #0 {
+; CHECK: Function Attrs: nofree noinline nosync nounwind readnone uwtable
 ; CHECK-LABEL: define {{[^@]+}}@rt2
 ; CHECK-SAME: (i32* nofree readnone [[A:%.*]], i32* nofree readnone "no-capture-maybe-returned" [[B:%.*]])
 ; CHECK-NEXT:  entry:
@@ -513,6 +551,7 @@ if.end:
 ; TEST another SCC test
 ;
 define i32* @rt3_helper(i32* %a, i32* %b) #0 {
+; CHECK: Function Attrs: nofree noinline nosync nounwind readnone uwtable
 ; CHECK-LABEL: define {{[^@]+}}@rt3_helper
 ; CHECK-SAME: (i32* nofree readnone [[A:%.*]], i32* nofree readnone returned "no-capture-maybe-returned" [[B:%.*]])
 ; CHECK-NEXT:  entry:
@@ -525,6 +564,7 @@ entry:
 }
 
 define i32* @rt3(i32* %a, i32 *%b) #0 {
+; CHECK: Function Attrs: nofree noinline nosync nounwind readnone uwtable
 ; CHECK-LABEL: define {{[^@]+}}@rt3
 ; CHECK-SAME: (i32* nofree readnone [[A:%.*]], i32* nofree readnone returned "no-capture-maybe-returned" [[B:%.*]])
 ; CHECK-NEXT:  entry:
@@ -562,6 +602,7 @@ if.end:
 declare void @unknown_fn(i32* (i32*)*) #0
 
 define i32* @calls_unknown_fn(i32* %r) #0 {
+; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@calls_unknown_fn
 ; CHECK-SAME: (i32* nofree readnone returned "no-capture-maybe-returned" [[R:%.*]])
 ; CHECK-NEXT:    tail call void @unknown_fn(i32* (i32*)* nonnull @calls_unknown_fn)
@@ -586,6 +627,7 @@ define i32* @calls_unknown_fn(i32* %r) #0 {
 ; Verify the maybe-redefined function is not annotated:
 ;
 define linkonce_odr i32* @maybe_redefined_fn(i32* %r) #0 {
+; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@maybe_redefined_fn
 ; CHECK-SAME: (i32* [[R:%.*]])
 ; CHECK-NEXT:  entry:
@@ -596,6 +638,7 @@ entry:
 }
 
 define i32* @calls_maybe_redefined_fn(i32* %r) #0 {
+; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@calls_maybe_redefined_fn
 ; CHECK-SAME: (i32* returned [[R:%.*]])
 ; CHECK-NEXT:  entry:
@@ -620,6 +663,7 @@ entry:
 ; Verify the maybe-redefined function is not annotated:
 ;
 define linkonce_odr i32* @maybe_redefined_fn2(i32* %r) #0 {
+; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@maybe_redefined_fn2
 ; CHECK-SAME: (i32* [[R:%.*]])
 ; CHECK-NEXT:  entry:
@@ -630,6 +674,7 @@ entry:
 }
 
 define i32* @calls_maybe_redefined_fn2(i32* %r) #0 {
+; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@calls_maybe_redefined_fn2
 ; CHECK-SAME: (i32* [[R:%.*]])
 ; CHECK-NEXT:  entry:
@@ -652,18 +697,33 @@ entry:
 ; }
 ;
 define double @select_and_phi(double %b) #0 {
-; CHECK-LABEL: define {{[^@]+}}@select_and_phi
-; CHECK-SAME: (double returned [[B:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt double [[B]], 0.000000e+00
-; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    br label [[IF_END]]
-; CHECK:       if.end:
-; CHECK-NEXT:    [[PHI:%.*]] = phi double [ [[B]], [[IF_THEN]] ], [ [[B]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[CMP1:%.*]] = fcmp oeq double [[B]], 0.000000e+00
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP1]], double [[B]], double [[PHI]]
-; CHECK-NEXT:    ret double [[SEL]]
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@select_and_phi
+; IS__TUNIT____-SAME: (double returned [[B:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[CMP:%.*]] = fcmp ogt double [[B]], 0.000000e+00
+; IS__TUNIT____-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; IS__TUNIT____:       if.then:
+; IS__TUNIT____-NEXT:    br label [[IF_END]]
+; IS__TUNIT____:       if.end:
+; IS__TUNIT____-NEXT:    [[PHI:%.*]] = phi double [ [[B]], [[IF_THEN]] ], [ [[B]], [[ENTRY:%.*]] ]
+; IS__TUNIT____-NEXT:    [[CMP1:%.*]] = fcmp oeq double [[B]], 0.000000e+00
+; IS__TUNIT____-NEXT:    [[SEL:%.*]] = select i1 [[CMP1]], double [[B]], double [[PHI]]
+; IS__TUNIT____-NEXT:    ret double [[SEL]]
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@select_and_phi
+; IS__CGSCC____-SAME: (double returned [[B:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[CMP:%.*]] = fcmp ogt double [[B]], 0.000000e+00
+; IS__CGSCC____-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; IS__CGSCC____:       if.then:
+; IS__CGSCC____-NEXT:    br label [[IF_END]]
+; IS__CGSCC____:       if.end:
+; IS__CGSCC____-NEXT:    [[PHI:%.*]] = phi double [ [[B]], [[IF_THEN]] ], [ [[B]], [[ENTRY:%.*]] ]
+; IS__CGSCC____-NEXT:    [[CMP1:%.*]] = fcmp oeq double [[B]], 0.000000e+00
+; IS__CGSCC____-NEXT:    [[SEL:%.*]] = select i1 [[CMP1]], double [[B]], double [[PHI]]
+; IS__CGSCC____-NEXT:    ret double [[SEL]]
 ;
 entry:
   %cmp = fcmp ogt double %b, 0.000000e+00
@@ -690,6 +750,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; }
 ;
 define double @recursion_select_and_phi(i32 %a, double %b) #0 {
+; CHECK: Function Attrs: nofree noinline nosync nounwind readnone uwtable
 ; CHECK-LABEL: define {{[^@]+}}@recursion_select_and_phi
 ; CHECK-SAME: (i32 [[A:%.*]], double returned [[B:%.*]])
 ; CHECK-NEXT:  entry:
@@ -729,11 +790,19 @@ if.end:                                           ; preds = %if.then, %entry
 ; }
 ;
 define double* @bitcast(i32* %b) #0 {
-; CHECK-LABEL: define {{[^@]+}}@bitcast
-; CHECK-SAME: (i32* nofree readnone returned "no-capture-maybe-returned" [[B:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[BC0:%.*]] = bitcast i32* [[B]] to double*
-; CHECK-NEXT:    ret double* [[BC0]]
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@bitcast
+; IS__TUNIT____-SAME: (i32* nofree readnone returned "no-capture-maybe-returned" [[B:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[BC0:%.*]] = bitcast i32* [[B]] to double*
+; IS__TUNIT____-NEXT:    ret double* [[BC0]]
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@bitcast
+; IS__CGSCC____-SAME: (i32* nofree readnone returned "no-capture-maybe-returned" [[B:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[BC0:%.*]] = bitcast i32* [[B]] to double*
+; IS__CGSCC____-NEXT:    ret double* [[BC0]]
 ;
 entry:
   %bc0 = bitcast i32* %b to double*
@@ -751,23 +820,43 @@ entry:
 ; }
 ;
 define double* @bitcasts_select_and_phi(i32* %b) #0 {
-; CHECK-LABEL: define {{[^@]+}}@bitcasts_select_and_phi
-; CHECK-SAME: (i32* nofree readnone returned [[B:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[BC0:%.*]] = bitcast i32* [[B]] to double*
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq double* [[BC0]], null
-; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    [[BC1:%.*]] = bitcast i32* [[B]] to double*
-; CHECK-NEXT:    br label [[IF_END]]
-; CHECK:       if.end:
-; CHECK-NEXT:    [[PHI:%.*]] = phi double* [ [[BC1]], [[IF_THEN]] ], [ [[BC0]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC2:%.*]] = bitcast double* [[PHI]] to i8*
-; CHECK-NEXT:    [[BC3:%.*]] = bitcast i32* [[B]] to i8*
-; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne double* [[BC0]], null
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP2]], i8* [[BC2]], i8* [[BC3]]
-; CHECK-NEXT:    [[BC4:%.*]] = bitcast i8* [[SEL]] to double*
-; CHECK-NEXT:    ret double* [[BC4]]
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@bitcasts_select_and_phi
+; IS__TUNIT____-SAME: (i32* nofree readnone returned [[B:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[BC0:%.*]] = bitcast i32* [[B]] to double*
+; IS__TUNIT____-NEXT:    [[CMP:%.*]] = icmp eq double* [[BC0]], null
+; IS__TUNIT____-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; IS__TUNIT____:       if.then:
+; IS__TUNIT____-NEXT:    [[BC1:%.*]] = bitcast i32* [[B]] to double*
+; IS__TUNIT____-NEXT:    br label [[IF_END]]
+; IS__TUNIT____:       if.end:
+; IS__TUNIT____-NEXT:    [[PHI:%.*]] = phi double* [ [[BC1]], [[IF_THEN]] ], [ [[BC0]], [[ENTRY:%.*]] ]
+; IS__TUNIT____-NEXT:    [[BC2:%.*]] = bitcast double* [[PHI]] to i8*
+; IS__TUNIT____-NEXT:    [[BC3:%.*]] = bitcast i32* [[B]] to i8*
+; IS__TUNIT____-NEXT:    [[CMP2:%.*]] = icmp ne double* [[BC0]], null
+; IS__TUNIT____-NEXT:    [[SEL:%.*]] = select i1 [[CMP2]], i8* [[BC2]], i8* [[BC3]]
+; IS__TUNIT____-NEXT:    [[BC4:%.*]] = bitcast i8* [[SEL]] to double*
+; IS__TUNIT____-NEXT:    ret double* [[BC4]]
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@bitcasts_select_and_phi
+; IS__CGSCC____-SAME: (i32* nofree readnone returned [[B:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[BC0:%.*]] = bitcast i32* [[B]] to double*
+; IS__CGSCC____-NEXT:    [[CMP:%.*]] = icmp eq double* [[BC0]], null
+; IS__CGSCC____-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; IS__CGSCC____:       if.then:
+; IS__CGSCC____-NEXT:    [[BC1:%.*]] = bitcast i32* [[B]] to double*
+; IS__CGSCC____-NEXT:    br label [[IF_END]]
+; IS__CGSCC____:       if.end:
+; IS__CGSCC____-NEXT:    [[PHI:%.*]] = phi double* [ [[BC1]], [[IF_THEN]] ], [ [[BC0]], [[ENTRY:%.*]] ]
+; IS__CGSCC____-NEXT:    [[BC2:%.*]] = bitcast double* [[PHI]] to i8*
+; IS__CGSCC____-NEXT:    [[BC3:%.*]] = bitcast i32* [[B]] to i8*
+; IS__CGSCC____-NEXT:    [[CMP2:%.*]] = icmp ne double* [[BC0]], null
+; IS__CGSCC____-NEXT:    [[SEL:%.*]] = select i1 [[CMP2]], i8* [[BC2]], i8* [[BC3]]
+; IS__CGSCC____-NEXT:    [[BC4:%.*]] = bitcast i8* [[SEL]] to double*
+; IS__CGSCC____-NEXT:    ret double* [[BC4]]
 ;
 entry:
   %bc0 = bitcast i32* %b to double*
@@ -800,21 +889,39 @@ if.end:                                           ; preds = %if.then, %entry
 ; }
 ;
 define double* @ret_arg_arg_undef(i32* %b) #0 {
-; CHECK-LABEL: define {{[^@]+}}@ret_arg_arg_undef
-; CHECK-SAME: (i32* nofree readnone returned [[B:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[BC0:%.*]] = bitcast i32* [[B]] to double*
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq double* [[BC0]], null
-; CHECK-NEXT:    br i1 [[CMP]], label [[RET_ARG0:%.*]], label [[IF_END:%.*]]
-; CHECK:       ret_arg0:
-; CHECK-NEXT:    [[BC1:%.*]] = bitcast i32* [[B]] to double*
-; CHECK-NEXT:    ret double* [[BC1]]
-; CHECK:       if.end:
-; CHECK-NEXT:    br i1 [[CMP]], label [[RET_ARG1:%.*]], label [[RET_UNDEF:%.*]]
-; CHECK:       ret_arg1:
-; CHECK-NEXT:    ret double* [[BC0]]
-; CHECK:       ret_undef:
-; CHECK-NEXT:    ret double* undef
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@ret_arg_arg_undef
+; IS__TUNIT____-SAME: (i32* nofree readnone returned [[B:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[BC0:%.*]] = bitcast i32* [[B]] to double*
+; IS__TUNIT____-NEXT:    [[CMP:%.*]] = icmp eq double* [[BC0]], null
+; IS__TUNIT____-NEXT:    br i1 [[CMP]], label [[RET_ARG0:%.*]], label [[IF_END:%.*]]
+; IS__TUNIT____:       ret_arg0:
+; IS__TUNIT____-NEXT:    [[BC1:%.*]] = bitcast i32* [[B]] to double*
+; IS__TUNIT____-NEXT:    ret double* [[BC1]]
+; IS__TUNIT____:       if.end:
+; IS__TUNIT____-NEXT:    br i1 [[CMP]], label [[RET_ARG1:%.*]], label [[RET_UNDEF:%.*]]
+; IS__TUNIT____:       ret_arg1:
+; IS__TUNIT____-NEXT:    ret double* [[BC0]]
+; IS__TUNIT____:       ret_undef:
+; IS__TUNIT____-NEXT:    ret double* undef
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@ret_arg_arg_undef
+; IS__CGSCC____-SAME: (i32* nofree readnone returned [[B:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[BC0:%.*]] = bitcast i32* [[B]] to double*
+; IS__CGSCC____-NEXT:    [[CMP:%.*]] = icmp eq double* [[BC0]], null
+; IS__CGSCC____-NEXT:    br i1 [[CMP]], label [[RET_ARG0:%.*]], label [[IF_END:%.*]]
+; IS__CGSCC____:       ret_arg0:
+; IS__CGSCC____-NEXT:    [[BC1:%.*]] = bitcast i32* [[B]] to double*
+; IS__CGSCC____-NEXT:    ret double* [[BC1]]
+; IS__CGSCC____:       if.end:
+; IS__CGSCC____-NEXT:    br i1 [[CMP]], label [[RET_ARG1:%.*]], label [[RET_UNDEF:%.*]]
+; IS__CGSCC____:       ret_arg1:
+; IS__CGSCC____-NEXT:    ret double* [[BC0]]
+; IS__CGSCC____:       ret_undef:
+; IS__CGSCC____-NEXT:    ret double* undef
 ;
 entry:
   %bc0 = bitcast i32* %b to double*
@@ -847,21 +954,39 @@ ret_undef:
 ; }
 ;
 define double* @ret_undef_arg_arg(i32* %b) #0 {
-; CHECK-LABEL: define {{[^@]+}}@ret_undef_arg_arg
-; CHECK-SAME: (i32* nofree readnone returned [[B:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[BC0:%.*]] = bitcast i32* [[B]] to double*
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq double* [[BC0]], null
-; CHECK-NEXT:    br i1 [[CMP]], label [[RET_UNDEF:%.*]], label [[IF_END:%.*]]
-; CHECK:       ret_undef:
-; CHECK-NEXT:    ret double* undef
-; CHECK:       if.end:
-; CHECK-NEXT:    br i1 [[CMP]], label [[RET_ARG0:%.*]], label [[RET_ARG1:%.*]]
-; CHECK:       ret_arg0:
-; CHECK-NEXT:    ret double* [[BC0]]
-; CHECK:       ret_arg1:
-; CHECK-NEXT:    [[BC1:%.*]] = bitcast i32* [[B]] to double*
-; CHECK-NEXT:    ret double* [[BC1]]
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@ret_undef_arg_arg
+; IS__TUNIT____-SAME: (i32* nofree readnone returned [[B:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[BC0:%.*]] = bitcast i32* [[B]] to double*
+; IS__TUNIT____-NEXT:    [[CMP:%.*]] = icmp eq double* [[BC0]], null
+; IS__TUNIT____-NEXT:    br i1 [[CMP]], label [[RET_UNDEF:%.*]], label [[IF_END:%.*]]
+; IS__TUNIT____:       ret_undef:
+; IS__TUNIT____-NEXT:    ret double* undef
+; IS__TUNIT____:       if.end:
+; IS__TUNIT____-NEXT:    br i1 [[CMP]], label [[RET_ARG0:%.*]], label [[RET_ARG1:%.*]]
+; IS__TUNIT____:       ret_arg0:
+; IS__TUNIT____-NEXT:    ret double* [[BC0]]
+; IS__TUNIT____:       ret_arg1:
+; IS__TUNIT____-NEXT:    [[BC1:%.*]] = bitcast i32* [[B]] to double*
+; IS__TUNIT____-NEXT:    ret double* [[BC1]]
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@ret_undef_arg_arg
+; IS__CGSCC____-SAME: (i32* nofree readnone returned [[B:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[BC0:%.*]] = bitcast i32* [[B]] to double*
+; IS__CGSCC____-NEXT:    [[CMP:%.*]] = icmp eq double* [[BC0]], null
+; IS__CGSCC____-NEXT:    br i1 [[CMP]], label [[RET_UNDEF:%.*]], label [[IF_END:%.*]]
+; IS__CGSCC____:       ret_undef:
+; IS__CGSCC____-NEXT:    ret double* undef
+; IS__CGSCC____:       if.end:
+; IS__CGSCC____-NEXT:    br i1 [[CMP]], label [[RET_ARG0:%.*]], label [[RET_ARG1:%.*]]
+; IS__CGSCC____:       ret_arg0:
+; IS__CGSCC____-NEXT:    ret double* [[BC0]]
+; IS__CGSCC____:       ret_arg1:
+; IS__CGSCC____-NEXT:    [[BC1:%.*]] = bitcast i32* [[B]] to double*
+; IS__CGSCC____-NEXT:    ret double* [[BC1]]
 ;
 entry:
   %bc0 = bitcast i32* %b to double*
@@ -894,20 +1019,37 @@ ret_arg1:
 ; }
 ;
 define double* @ret_undef_arg_undef(i32* %b) #0 {
-; CHECK-LABEL: define {{[^@]+}}@ret_undef_arg_undef
-; CHECK-SAME: (i32* nofree readnone returned [[B:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[BC0:%.*]] = bitcast i32* [[B]] to double*
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq double* [[BC0]], null
-; CHECK-NEXT:    br i1 [[CMP]], label [[RET_UNDEF0:%.*]], label [[IF_END:%.*]]
-; CHECK:       ret_undef0:
-; CHECK-NEXT:    ret double* undef
-; CHECK:       if.end:
-; CHECK-NEXT:    br i1 [[CMP]], label [[RET_ARG:%.*]], label [[RET_UNDEF1:%.*]]
-; CHECK:       ret_arg:
-; CHECK-NEXT:    ret double* [[BC0]]
-; CHECK:       ret_undef1:
-; CHECK-NEXT:    ret double* undef
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@ret_undef_arg_undef
+; IS__TUNIT____-SAME: (i32* nofree readnone returned [[B:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[BC0:%.*]] = bitcast i32* [[B]] to double*
+; IS__TUNIT____-NEXT:    [[CMP:%.*]] = icmp eq double* [[BC0]], null
+; IS__TUNIT____-NEXT:    br i1 [[CMP]], label [[RET_UNDEF0:%.*]], label [[IF_END:%.*]]
+; IS__TUNIT____:       ret_undef0:
+; IS__TUNIT____-NEXT:    ret double* undef
+; IS__TUNIT____:       if.end:
+; IS__TUNIT____-NEXT:    br i1 [[CMP]], label [[RET_ARG:%.*]], label [[RET_UNDEF1:%.*]]
+; IS__TUNIT____:       ret_arg:
+; IS__TUNIT____-NEXT:    ret double* [[BC0]]
+; IS__TUNIT____:       ret_undef1:
+; IS__TUNIT____-NEXT:    ret double* undef
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@ret_undef_arg_undef
+; IS__CGSCC____-SAME: (i32* nofree readnone returned [[B:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[BC0:%.*]] = bitcast i32* [[B]] to double*
+; IS__CGSCC____-NEXT:    [[CMP:%.*]] = icmp eq double* [[BC0]], null
+; IS__CGSCC____-NEXT:    br i1 [[CMP]], label [[RET_UNDEF0:%.*]], label [[IF_END:%.*]]
+; IS__CGSCC____:       ret_undef0:
+; IS__CGSCC____-NEXT:    ret double* undef
+; IS__CGSCC____:       if.end:
+; IS__CGSCC____-NEXT:    br i1 [[CMP]], label [[RET_ARG:%.*]], label [[RET_UNDEF1:%.*]]
+; IS__CGSCC____:       ret_arg:
+; IS__CGSCC____-NEXT:    ret double* [[BC0]]
+; IS__CGSCC____:       ret_undef1:
+; IS__CGSCC____-NEXT:    ret double* undef
 ;
 entry:
   %bc0 = bitcast i32* %b to double*
@@ -940,6 +1082,7 @@ ret_undef1:
 declare i32* @unknown(i32*)
 
 define i32* @ret_arg_or_unknown(i32* %b) #0 {
+; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@ret_arg_or_unknown
 ; CHECK-SAME: (i32* [[B:%.*]])
 ; CHECK-NEXT:  entry:
@@ -964,6 +1107,7 @@ ret_unknown:
 }
 
 define i32* @ret_arg_or_unknown_through_phi(i32* %b) #0 {
+; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@ret_arg_or_unknown_through_phi
 ; CHECK-SAME: (i32* [[B:%.*]])
 ; CHECK-NEXT:  entry:
@@ -997,12 +1141,21 @@ r:
 ; TEST inconsistent IR in dead code.
 ;
 define i32 @deadblockcall1(i32 %A) #0 {
-; CHECK-LABEL: define {{[^@]+}}@deadblockcall1
-; CHECK-SAME: (i32 returned [[A:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i32 [[A]]
-; CHECK:       unreachableblock:
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@deadblockcall1
+; IS__TUNIT____-SAME: (i32 returned [[A:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    ret i32 [[A]]
+; IS__TUNIT____:       unreachableblock:
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@deadblockcall1
+; IS__CGSCC____-SAME: (i32 returned [[A:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    ret i32 [[A]]
+; IS__CGSCC____:       unreachableblock:
+; IS__CGSCC____-NEXT:    unreachable
 ;
 entry:
   ret i32 %A
@@ -1014,14 +1167,25 @@ unreachableblock:
 declare i32 @deadblockcall_helper(i32 returned %A);
 
 define i32 @deadblockcall2(i32 %A) #0 {
-; CHECK-LABEL: define {{[^@]+}}@deadblockcall2
-; CHECK-SAME: (i32 returned [[A:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    ret i32 [[A]]
-; CHECK:       unreachableblock1:
-; CHECK-NEXT:    unreachable
-; CHECK:       unreachableblock2:
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@deadblockcall2
+; IS__TUNIT____-SAME: (i32 returned [[A:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    ret i32 [[A]]
+; IS__TUNIT____:       unreachableblock1:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       unreachableblock2:
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@deadblockcall2
+; IS__CGSCC____-SAME: (i32 returned [[A:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    ret i32 [[A]]
+; IS__CGSCC____:       unreachableblock1:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       unreachableblock2:
+; IS__CGSCC____-NEXT:    unreachable
 ;
 entry:
   ret i32 %A
@@ -1034,16 +1198,29 @@ unreachableblock2:
 }
 
 define i32 @deadblockphi1(i32 %A) #0 {
-; CHECK-LABEL: define {{[^@]+}}@deadblockphi1
-; CHECK-SAME: (i32 returned [[A:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[R:%.*]]
-; CHECK:       unreachableblock1:
-; CHECK-NEXT:    unreachable
-; CHECK:       unreachableblock2:
-; CHECK-NEXT:    unreachable
-; CHECK:       r:
-; CHECK-NEXT:    ret i32 [[A]]
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@deadblockphi1
+; IS__TUNIT____-SAME: (i32 returned [[A:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    br label [[R:%.*]]
+; IS__TUNIT____:       unreachableblock1:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       unreachableblock2:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       r:
+; IS__TUNIT____-NEXT:    ret i32 [[A]]
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@deadblockphi1
+; IS__CGSCC____-SAME: (i32 returned [[A:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    br label [[R:%.*]]
+; IS__CGSCC____:       unreachableblock1:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       unreachableblock2:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       r:
+; IS__CGSCC____-NEXT:    ret i32 [[A]]
 ;
 entry:
   br label %r
@@ -1059,18 +1236,33 @@ r:
 }
 
 define i32 @deadblockphi2(i32 %A) #0 {
-; CHECK-LABEL: define {{[^@]+}}@deadblockphi2
-; CHECK-SAME: (i32 returned [[A:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[R:%.*]]
-; CHECK:       unreachableblock1:
-; CHECK-NEXT:    unreachable
-; CHECK:       unreachableblock2:
-; CHECK-NEXT:    unreachable
-; CHECK:       unreachableblock3:
-; CHECK-NEXT:    unreachable
-; CHECK:       r:
-; CHECK-NEXT:    ret i32 [[A]]
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@deadblockphi2
+; IS__TUNIT____-SAME: (i32 returned [[A:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    br label [[R:%.*]]
+; IS__TUNIT____:       unreachableblock1:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       unreachableblock2:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       unreachableblock3:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       r:
+; IS__TUNIT____-NEXT:    ret i32 [[A]]
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@deadblockphi2
+; IS__CGSCC____-SAME: (i32 returned [[A:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    br label [[R:%.*]]
+; IS__CGSCC____:       unreachableblock1:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       unreachableblock2:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       unreachableblock3:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       r:
+; IS__CGSCC____-NEXT:    ret i32 [[A]]
 ;
 entry:
   br label %r
@@ -1091,6 +1283,7 @@ r:
 declare void @noreturn() noreturn;
 
 define i32 @deadblockphi3(i32 %A, i1 %c) #0 {
+; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@deadblockphi3
 ; CHECK-SAME: (i32 returned [[A:%.*]], i1 [[C:%.*]])
 ; CHECK-NEXT:  entry:
@@ -1207,25 +1400,38 @@ define i32 @exact(i32* align 8 %a, i32* align 8 %b) {
 
 @G = external global i8
 define i32* @ret_const() #0 {
-; CHECK-LABEL: define {{[^@]+}}@ret_const()
-; CHECK-NEXT:    [[BC:%.*]] = bitcast i8* @G to i32*
-; CHECK-NEXT:    ret i32* [[BC]]
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@ret_const()
+; IS__TUNIT____-NEXT:    [[BC:%.*]] = bitcast i8* @G to i32*
+; IS__TUNIT____-NEXT:    ret i32* [[BC]]
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@ret_const()
+; IS__CGSCC____-NEXT:    [[BC:%.*]] = bitcast i8* @G to i32*
+; IS__CGSCC____-NEXT:    ret i32* [[BC]]
 ;
   %bc = bitcast i8* @G to i32*
   ret i32* %bc
 }
 define i32* @use_const() #0 {
-; CHECK-LABEL: define {{[^@]+}}@use_const()
-; CHECK-NEXT:    ret i32* bitcast (i8* @G to i32*)
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@use_const()
+; IS__TUNIT____-NEXT:    ret i32* bitcast (i8* @G to i32*)
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@use_const()
+; IS__CGSCC____-NEXT:    ret i32* bitcast (i8* @G to i32*)
 ;
   %c = call i32* @ret_const()
   ret i32* %c
 }
 define i32* @dont_use_const() #0 {
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@dont_use_const()
 ; IS__TUNIT____-NEXT:    [[C:%.*]] = musttail call i32* @ret_const()
 ; IS__TUNIT____-NEXT:    ret i32* [[C]]
 ;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@dont_use_const()
 ; IS__CGSCC____-NEXT:    [[C:%.*]] = musttail call nonnull dereferenceable(1) i32* @ret_const()
 ; IS__CGSCC____-NEXT:    ret i32* [[C]]
diff --git a/llvm/test/Transforms/Attributor/undefined_behavior.ll b/llvm/test/Transforms/Attributor/undefined_behavior.ll
index b49c26e8a2f7a..6894037681749 100644
--- a/llvm/test/Transforms/Attributor/undefined_behavior.ll
+++ b/llvm/test/Transforms/Attributor/undefined_behavior.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=5 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -13,16 +13,26 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 ; -- Load tests --
 
 define void @load_wholly_unreachable() {
-; CHECK-LABEL: define {{[^@]+}}@load_wholly_unreachable()
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@load_wholly_unreachable()
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@load_wholly_unreachable()
+; IS__CGSCC____-NEXT:    unreachable
 ;
   %a = load i32, i32* null
   ret void
 }
 
 define void @loads_wholly_unreachable() {
-; CHECK-LABEL: define {{[^@]+}}@loads_wholly_unreachable()
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@loads_wholly_unreachable()
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@loads_wholly_unreachable()
+; IS__CGSCC____-NEXT:    unreachable
 ;
   %a = load i32, i32* null
   %b = load i32, i32* null
@@ -31,13 +41,23 @@ define void @loads_wholly_unreachable() {
 
 
 define void @load_single_bb_unreachable(i1 %cond) {
-; CHECK-LABEL: define {{[^@]+}}@load_single_bb_unreachable
-; CHECK-SAME: (i1 [[COND:%.*]])
-; CHECK-NEXT:    br i1 [[COND]], label [[T:%.*]], label [[E:%.*]]
-; CHECK:       t:
-; CHECK-NEXT:    unreachable
-; CHECK:       e:
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@load_single_bb_unreachable
+; IS__TUNIT____-SAME: (i1 [[COND:%.*]])
+; IS__TUNIT____-NEXT:    br i1 [[COND]], label [[T:%.*]], label [[E:%.*]]
+; IS__TUNIT____:       t:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       e:
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@load_single_bb_unreachable
+; IS__CGSCC____-SAME: (i1 [[COND:%.*]])
+; IS__CGSCC____-NEXT:    br i1 [[COND]], label [[T:%.*]], label [[E:%.*]]
+; IS__CGSCC____:       t:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       e:
+; IS__CGSCC____-NEXT:    ret void
 ;
   br i1 %cond, label %t, label %e
 t:
@@ -50,14 +70,20 @@ e:
 ; Note that while the load is removed (because it's unused), the block
 ; is not changed to unreachable
 define void @load_null_pointer_is_defined() null_pointer_is_valid {
-; CHECK-LABEL: define {{[^@]+}}@load_null_pointer_is_defined()
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind null_pointer_is_valid readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@load_null_pointer_is_defined()
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind null_pointer_is_valid readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@load_null_pointer_is_defined()
+; IS__CGSCC____-NEXT:    ret void
 ;
   %a = load i32, i32* null
   ret void
 }
 
 define internal i32* @ret_null() {
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@ret_null()
 ; IS__CGSCC____-NEXT:    ret i32* null
 ;
@@ -65,8 +91,13 @@ define internal i32* @ret_null() {
 }
 
 define void @load_null_propagated() {
-; CHECK-LABEL: define {{[^@]+}}@load_null_propagated()
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@load_null_propagated()
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@load_null_propagated()
+; IS__CGSCC____-NEXT:    unreachable
 ;
   %ptr = call i32* @ret_null()
   %a = load i32, i32* %ptr
@@ -76,21 +107,36 @@ define void @load_null_propagated() {
 ; -- Store tests --
 
 define void @store_wholly_unreachable() {
-; CHECK-LABEL: define {{[^@]+}}@store_wholly_unreachable()
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@store_wholly_unreachable()
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@store_wholly_unreachable()
+; IS__CGSCC____-NEXT:    unreachable
 ;
   store i32 5, i32* null
   ret void
 }
 
 define void @store_single_bb_unreachable(i1 %cond) {
-; CHECK-LABEL: define {{[^@]+}}@store_single_bb_unreachable
-; CHECK-SAME: (i1 [[COND:%.*]])
-; CHECK-NEXT:    br i1 [[COND]], label [[T:%.*]], label [[E:%.*]]
-; CHECK:       t:
-; CHECK-NEXT:    unreachable
-; CHECK:       e:
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@store_single_bb_unreachable
+; IS__TUNIT____-SAME: (i1 [[COND:%.*]])
+; IS__TUNIT____-NEXT:    br i1 [[COND]], label [[T:%.*]], label [[E:%.*]]
+; IS__TUNIT____:       t:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       e:
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@store_single_bb_unreachable
+; IS__CGSCC____-SAME: (i1 [[COND:%.*]])
+; IS__CGSCC____-NEXT:    br i1 [[COND]], label [[T:%.*]], label [[E:%.*]]
+; IS__CGSCC____:       t:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       e:
+; IS__CGSCC____-NEXT:    ret void
 ;
   br i1 %cond, label %t, label %e
 t:
@@ -101,9 +147,15 @@ e:
 }
 
 define void @store_null_pointer_is_defined() null_pointer_is_valid {
-; CHECK-LABEL: define {{[^@]+}}@store_null_pointer_is_defined()
-; CHECK-NEXT:    store i32 5, i32* null, align 536870912
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind null_pointer_is_valid willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@store_null_pointer_is_defined()
+; IS__TUNIT____-NEXT:    store i32 5, i32* null, align 536870912
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind null_pointer_is_valid willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@store_null_pointer_is_defined()
+; IS__CGSCC____-NEXT:    store i32 5, i32* null, align 536870912
+; IS__CGSCC____-NEXT:    ret void
 ;
   store i32 5, i32* null
   ret void
@@ -113,8 +165,13 @@ define void @store_null_propagated() {
 ; ATTRIBUTOR-LABEL: @store_null_propagated(
 ; ATTRIBUTOR-NEXT:    unreachable
 ;
-; CHECK-LABEL: define {{[^@]+}}@store_null_propagated()
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@store_null_propagated()
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@store_null_propagated()
+; IS__CGSCC____-NEXT:    unreachable
 ;
   %ptr = call i32* @ret_null()
   store i32 5, i32* %ptr
@@ -124,21 +181,36 @@ define void @store_null_propagated() {
 ; -- AtomicRMW tests --
 
 define void @atomicrmw_wholly_unreachable() {
-; CHECK-LABEL: define {{[^@]+}}@atomicrmw_wholly_unreachable()
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@atomicrmw_wholly_unreachable()
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@atomicrmw_wholly_unreachable()
+; IS__CGSCC____-NEXT:    unreachable
 ;
   %a = atomicrmw add i32* null, i32 1 acquire
   ret void
 }
 
 define void @atomicrmw_single_bb_unreachable(i1 %cond) {
-; CHECK-LABEL: define {{[^@]+}}@atomicrmw_single_bb_unreachable
-; CHECK-SAME: (i1 [[COND:%.*]])
-; CHECK-NEXT:    br i1 [[COND]], label [[T:%.*]], label [[E:%.*]]
-; CHECK:       t:
-; CHECK-NEXT:    unreachable
-; CHECK:       e:
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@atomicrmw_single_bb_unreachable
+; IS__TUNIT____-SAME: (i1 [[COND:%.*]])
+; IS__TUNIT____-NEXT:    br i1 [[COND]], label [[T:%.*]], label [[E:%.*]]
+; IS__TUNIT____:       t:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       e:
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@atomicrmw_single_bb_unreachable
+; IS__CGSCC____-SAME: (i1 [[COND:%.*]])
+; IS__CGSCC____-NEXT:    br i1 [[COND]], label [[T:%.*]], label [[E:%.*]]
+; IS__CGSCC____:       t:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       e:
+; IS__CGSCC____-NEXT:    ret void
 ;
   br i1 %cond, label %t, label %e
 t:
@@ -149,9 +221,15 @@ e:
 }
 
 define void @atomicrmw_null_pointer_is_defined() null_pointer_is_valid {
-; CHECK-LABEL: define {{[^@]+}}@atomicrmw_null_pointer_is_defined()
-; CHECK-NEXT:    [[A:%.*]] = atomicrmw add i32* null, i32 1 acquire
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nounwind null_pointer_is_valid willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@atomicrmw_null_pointer_is_defined()
+; IS__TUNIT____-NEXT:    [[A:%.*]] = atomicrmw add i32* null, i32 1 acquire
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nounwind null_pointer_is_valid willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@atomicrmw_null_pointer_is_defined()
+; IS__CGSCC____-NEXT:    [[A:%.*]] = atomicrmw add i32* null, i32 1 acquire
+; IS__CGSCC____-NEXT:    ret void
 ;
   %a = atomicrmw add i32* null, i32 1 acquire
   ret void
@@ -161,8 +239,13 @@ define void @atomicrmw_null_propagated() {
 ; ATTRIBUTOR-LABEL: @atomicrmw_null_propagated(
 ; ATTRIBUTOR-NEXT:    unreachable
 ;
-; CHECK-LABEL: define {{[^@]+}}@atomicrmw_null_propagated()
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@atomicrmw_null_propagated()
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@atomicrmw_null_propagated()
+; IS__CGSCC____-NEXT:    unreachable
 ;
   %ptr = call i32* @ret_null()
   %a = atomicrmw add i32* %ptr, i32 1 acquire
@@ -172,21 +255,36 @@ define void @atomicrmw_null_propagated() {
 ; -- AtomicCmpXchg tests --
 
 define void @atomiccmpxchg_wholly_unreachable() {
-; CHECK-LABEL: define {{[^@]+}}@atomiccmpxchg_wholly_unreachable()
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@atomiccmpxchg_wholly_unreachable()
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@atomiccmpxchg_wholly_unreachable()
+; IS__CGSCC____-NEXT:    unreachable
 ;
   %a = cmpxchg i32* null, i32 2, i32 3 acq_rel monotonic
   ret void
 }
 
 define void @atomiccmpxchg_single_bb_unreachable(i1 %cond) {
-; CHECK-LABEL: define {{[^@]+}}@atomiccmpxchg_single_bb_unreachable
-; CHECK-SAME: (i1 [[COND:%.*]])
-; CHECK-NEXT:    br i1 [[COND]], label [[T:%.*]], label [[E:%.*]]
-; CHECK:       t:
-; CHECK-NEXT:    unreachable
-; CHECK:       e:
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@atomiccmpxchg_single_bb_unreachable
+; IS__TUNIT____-SAME: (i1 [[COND:%.*]])
+; IS__TUNIT____-NEXT:    br i1 [[COND]], label [[T:%.*]], label [[E:%.*]]
+; IS__TUNIT____:       t:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       e:
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@atomiccmpxchg_single_bb_unreachable
+; IS__CGSCC____-SAME: (i1 [[COND:%.*]])
+; IS__CGSCC____-NEXT:    br i1 [[COND]], label [[T:%.*]], label [[E:%.*]]
+; IS__CGSCC____:       t:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       e:
+; IS__CGSCC____-NEXT:    ret void
 ;
   br i1 %cond, label %t, label %e
 t:
@@ -197,9 +295,15 @@ e:
 }
 
 define void @atomiccmpxchg_null_pointer_is_defined() null_pointer_is_valid {
-; CHECK-LABEL: define {{[^@]+}}@atomiccmpxchg_null_pointer_is_defined()
-; CHECK-NEXT:    [[A:%.*]] = cmpxchg i32* null, i32 2, i32 3 acq_rel monotonic
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nounwind null_pointer_is_valid willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@atomiccmpxchg_null_pointer_is_defined()
+; IS__TUNIT____-NEXT:    [[A:%.*]] = cmpxchg i32* null, i32 2, i32 3 acq_rel monotonic
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nounwind null_pointer_is_valid willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@atomiccmpxchg_null_pointer_is_defined()
+; IS__CGSCC____-NEXT:    [[A:%.*]] = cmpxchg i32* null, i32 2, i32 3 acq_rel monotonic
+; IS__CGSCC____-NEXT:    ret void
 ;
   %a = cmpxchg i32* null, i32 2, i32 3 acq_rel monotonic
   ret void
@@ -209,8 +313,13 @@ define void @atomiccmpxchg_null_propagated() {
 ; ATTRIBUTOR-LABEL: @atomiccmpxchg_null_propagated(
 ; ATTRIBUTOR-NEXT:    unreachable
 ;
-; CHECK-LABEL: define {{[^@]+}}@atomiccmpxchg_null_propagated()
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@atomiccmpxchg_null_propagated()
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@atomiccmpxchg_null_propagated()
+; IS__CGSCC____-NEXT:    unreachable
 ;
   %ptr = call i32* @ret_null()
   %a = cmpxchg i32* %ptr, i32 2, i32 3 acq_rel monotonic
@@ -222,12 +331,21 @@ define void @atomiccmpxchg_null_propagated() {
 ; Note: The unreachable on %t and %e is _not_ from AAUndefinedBehavior
 
 define i32 @cond_br_on_undef() {
-; CHECK-LABEL: define {{[^@]+}}@cond_br_on_undef()
-; CHECK-NEXT:    unreachable
-; CHECK:       t:
-; CHECK-NEXT:    unreachable
-; CHECK:       e:
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree noreturn nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@cond_br_on_undef()
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       t:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       e:
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@cond_br_on_undef()
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       t:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       e:
+; IS__CGSCC____-NEXT:    unreachable
 ;
   br i1 undef, label %t, label %e
 t:
@@ -240,17 +358,31 @@ e:
   ; Valid branch - verify that this is not converted
   ; to unreachable.
 define void @cond_br_on_undef2(i1 %cond) {
-; CHECK-LABEL: define {{[^@]+}}@cond_br_on_undef2
-; CHECK-SAME: (i1 [[COND:%.*]])
-; CHECK-NEXT:    br i1 [[COND]], label [[T1:%.*]], label [[E1:%.*]]
-; CHECK:       t1:
-; CHECK-NEXT:    unreachable
-; CHECK:       t2:
-; CHECK-NEXT:    unreachable
-; CHECK:       e2:
-; CHECK-NEXT:    unreachable
-; CHECK:       e1:
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@cond_br_on_undef2
+; IS__TUNIT____-SAME: (i1 [[COND:%.*]])
+; IS__TUNIT____-NEXT:    br i1 [[COND]], label [[T1:%.*]], label [[E1:%.*]]
+; IS__TUNIT____:       t1:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       t2:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       e2:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       e1:
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@cond_br_on_undef2
+; IS__CGSCC____-SAME: (i1 [[COND:%.*]])
+; IS__CGSCC____-NEXT:    br i1 [[COND]], label [[T1:%.*]], label [[E1:%.*]]
+; IS__CGSCC____:       t1:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       t2:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       e2:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       e1:
+; IS__CGSCC____-NEXT:    ret void
 ;
   br i1 %cond, label %t1, label %e1
 t1:
@@ -264,19 +396,33 @@ e1:
 }
 
 define i1 @ret_undef() {
-; CHECK-LABEL: define {{[^@]+}}@ret_undef()
-; CHECK-NEXT:    ret i1 undef
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@ret_undef()
+; IS__TUNIT____-NEXT:    ret i1 undef
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@ret_undef()
+; IS__CGSCC____-NEXT:    ret i1 undef
 ;
   ret i1 undef
 }
 
 define void @cond_br_on_undef_interproc() {
-; CHECK-LABEL: define {{[^@]+}}@cond_br_on_undef_interproc()
-; CHECK-NEXT:    unreachable
-; CHECK:       t:
-; CHECK-NEXT:    unreachable
-; CHECK:       e:
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree noreturn nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@cond_br_on_undef_interproc()
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       t:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       e:
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@cond_br_on_undef_interproc()
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       t:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       e:
+; IS__CGSCC____-NEXT:    unreachable
 ;
   %cond = call i1 @ret_undef()
   br i1 %cond, label %t, label %e
@@ -287,12 +433,21 @@ e:
 }
 
 define i1 @ret_undef2() {
-; CHECK-LABEL: define {{[^@]+}}@ret_undef2()
-; CHECK-NEXT:    br i1 true, label [[T:%.*]], label [[E:%.*]]
-; CHECK:       t:
-; CHECK-NEXT:    ret i1 undef
-; CHECK:       e:
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@ret_undef2()
+; IS__TUNIT____-NEXT:    br i1 true, label [[T:%.*]], label [[E:%.*]]
+; IS__TUNIT____:       t:
+; IS__TUNIT____-NEXT:    ret i1 undef
+; IS__TUNIT____:       e:
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@ret_undef2()
+; IS__CGSCC____-NEXT:    br i1 true, label [[T:%.*]], label [[E:%.*]]
+; IS__CGSCC____:       t:
+; IS__CGSCC____-NEXT:    ret i1 undef
+; IS__CGSCC____:       e:
+; IS__CGSCC____-NEXT:    unreachable
 ;
   br i1 true, label %t, label %e
 t:
@@ -303,12 +458,21 @@ e:
 
 ; More complicated interproc deduction of undef
 define void @cond_br_on_undef_interproc2() {
-; CHECK-LABEL: define {{[^@]+}}@cond_br_on_undef_interproc2()
-; CHECK-NEXT:    unreachable
-; CHECK:       t:
-; CHECK-NEXT:    unreachable
-; CHECK:       e:
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree noreturn nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@cond_br_on_undef_interproc2()
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       t:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       e:
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse noreturn nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@cond_br_on_undef_interproc2()
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       t:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       e:
+; IS__CGSCC____-NEXT:    unreachable
 ;
   %cond = call i1 @ret_undef2()
   br i1 %cond, label %t, label %e
@@ -321,12 +485,21 @@ e:
 ; Branch on undef that depends on propagation of
 ; undef of a previous instruction.
 define i32 @cond_br_on_undef3() {
-; CHECK-LABEL: define {{[^@]+}}@cond_br_on_undef3()
-; CHECK-NEXT:    br label [[T:%.*]]
-; CHECK:       t:
-; CHECK-NEXT:    ret i32 1
-; CHECK:       e:
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@cond_br_on_undef3()
+; IS__TUNIT____-NEXT:    br label [[T:%.*]]
+; IS__TUNIT____:       t:
+; IS__TUNIT____-NEXT:    ret i32 1
+; IS__TUNIT____:       e:
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@cond_br_on_undef3()
+; IS__CGSCC____-NEXT:    br label [[T:%.*]]
+; IS__CGSCC____:       t:
+; IS__CGSCC____-NEXT:    ret i32 1
+; IS__CGSCC____:       e:
+; IS__CGSCC____-NEXT:    unreachable
 ;
   %cond = icmp ne i32 1, undef
   br i1 %cond, label %t, label %e
@@ -339,14 +512,25 @@ e:
 ; Branch on undef because of uninitialized value.
 ; FIXME: Currently it doesn't propagate the undef.
 define i32 @cond_br_on_undef_uninit() {
-; CHECK-LABEL: define {{[^@]+}}@cond_br_on_undef_uninit()
-; CHECK-NEXT:    [[ALLOC:%.*]] = alloca i1, align 1
-; CHECK-NEXT:    [[COND:%.*]] = load i1, i1* [[ALLOC]], align 1
-; CHECK-NEXT:    br i1 [[COND]], label [[T:%.*]], label [[E:%.*]]
-; CHECK:       t:
-; CHECK-NEXT:    ret i32 1
-; CHECK:       e:
-; CHECK-NEXT:    ret i32 2
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@cond_br_on_undef_uninit()
+; IS__TUNIT____-NEXT:    [[ALLOC:%.*]] = alloca i1, align 1
+; IS__TUNIT____-NEXT:    [[COND:%.*]] = load i1, i1* [[ALLOC]], align 1
+; IS__TUNIT____-NEXT:    br i1 [[COND]], label [[T:%.*]], label [[E:%.*]]
+; IS__TUNIT____:       t:
+; IS__TUNIT____-NEXT:    ret i32 1
+; IS__TUNIT____:       e:
+; IS__TUNIT____-NEXT:    ret i32 2
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@cond_br_on_undef_uninit()
+; IS__CGSCC____-NEXT:    [[ALLOC:%.*]] = alloca i1, align 1
+; IS__CGSCC____-NEXT:    [[COND:%.*]] = load i1, i1* [[ALLOC]], align 1
+; IS__CGSCC____-NEXT:    br i1 [[COND]], label [[T:%.*]], label [[E:%.*]]
+; IS__CGSCC____:       t:
+; IS__CGSCC____-NEXT:    ret i32 1
+; IS__CGSCC____:       e:
+; IS__CGSCC____-NEXT:    ret i32 2
 ;
   %alloc = alloca i1
   %cond = load i1, i1* %alloc
@@ -363,6 +547,7 @@ e:
 ; MODULE-NOT: @callee(
 define internal i32 @callee(i1 %C, i32* %A) {
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@callee()
 ; IS__CGSCC____-NEXT:  entry:
 ; IS__CGSCC____-NEXT:    unreachable
@@ -383,8 +568,13 @@ F:
 }
 
 define i32 @foo() {
-; CHECK-LABEL: define {{[^@]+}}@foo()
-; CHECK-NEXT:    ret i32 1
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@foo()
+; IS__TUNIT____-NEXT:    ret i32 1
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@foo()
+; IS__CGSCC____-NEXT:    ret i32 1
 ;
   %X = call i32 @callee(i1 false, i32* null)
   ret i32 %X
diff --git a/llvm/test/Transforms/Attributor/value-simplify.ll b/llvm/test/Transforms/Attributor/value-simplify.ll
index febfaba5b19a1..24f3954b81652 100644
--- a/llvm/test/Transforms/Attributor/value-simplify.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -30,31 +30,55 @@ define void @test1_helper() {
 
 ; TEST 2 : Simplify return value
 define i32 @return0() {
-; CHECK-LABEL: define {{[^@]+}}@return0()
-; CHECK-NEXT:    ret i32 0
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@return0()
+; IS__TUNIT____-NEXT:    ret i32 0
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@return0()
+; IS__CGSCC____-NEXT:    ret i32 0
 ;
   ret i32 0
 }
 
 define i32 @return1() {
-; CHECK-LABEL: define {{[^@]+}}@return1()
-; CHECK-NEXT:    ret i32 1
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@return1()
+; IS__TUNIT____-NEXT:    ret i32 1
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@return1()
+; IS__CGSCC____-NEXT:    ret i32 1
 ;
   ret i32 1
 }
 
 define i32 @test2_1(i1 %c) {
-; CHECK-LABEL: define {{[^@]+}}@test2_1
-; CHECK-SAME: (i1 [[C:%.*]])
-; CHECK-NEXT:    br i1 [[C]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
-; CHECK:       if.true:
-; CHECK-NEXT:    [[RET0:%.*]] = add i32 0, 1
-; CHECK-NEXT:    br label [[END:%.*]]
-; CHECK:       if.false:
-; CHECK-NEXT:    br label [[END]]
-; CHECK:       end:
-; CHECK-NEXT:    [[RET:%.*]] = phi i32 [ [[RET0]], [[IF_TRUE]] ], [ 1, [[IF_FALSE]] ]
-; CHECK-NEXT:    ret i32 1
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test2_1
+; IS__TUNIT____-SAME: (i1 [[C:%.*]])
+; IS__TUNIT____-NEXT:    br i1 [[C]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; IS__TUNIT____:       if.true:
+; IS__TUNIT____-NEXT:    [[RET0:%.*]] = add i32 0, 1
+; IS__TUNIT____-NEXT:    br label [[END:%.*]]
+; IS__TUNIT____:       if.false:
+; IS__TUNIT____-NEXT:    br label [[END]]
+; IS__TUNIT____:       end:
+; IS__TUNIT____-NEXT:    [[RET:%.*]] = phi i32 [ [[RET0]], [[IF_TRUE]] ], [ 1, [[IF_FALSE]] ]
+; IS__TUNIT____-NEXT:    ret i32 1
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test2_1
+; IS__CGSCC____-SAME: (i1 [[C:%.*]])
+; IS__CGSCC____-NEXT:    br i1 [[C]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; IS__CGSCC____:       if.true:
+; IS__CGSCC____-NEXT:    [[RET0:%.*]] = add i32 0, 1
+; IS__CGSCC____-NEXT:    br label [[END:%.*]]
+; IS__CGSCC____:       if.false:
+; IS__CGSCC____-NEXT:    br label [[END]]
+; IS__CGSCC____:       end:
+; IS__CGSCC____-NEXT:    [[RET:%.*]] = phi i32 [ [[RET0]], [[IF_TRUE]] ], [ 1, [[IF_FALSE]] ]
+; IS__CGSCC____-NEXT:    ret i32 1
 ;
   br i1 %c, label %if.true, label %if.false
 if.true:
@@ -74,9 +98,15 @@ end:
 
 
 define i32 @test2_2(i1 %c) {
-; CHECK-LABEL: define {{[^@]+}}@test2_2
-; CHECK-SAME: (i1 [[C:%.*]])
-; CHECK-NEXT:    ret i32 1
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test2_2
+; IS__TUNIT____-SAME: (i1 [[C:%.*]])
+; IS__TUNIT____-NEXT:    ret i32 1
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test2_2
+; IS__CGSCC____-SAME: (i1 [[C:%.*]])
+; IS__CGSCC____-NEXT:    ret i32 1
 ;
   %ret = tail call i32 @test2_1(i1 %c)
   ret i32 %ret
@@ -167,13 +197,23 @@ end:
 }
 
 define i32 @ipccp1(i32 %a) {
-; CHECK-LABEL: define {{[^@]+}}@ipccp1
-; CHECK-SAME: (i32 returned [[A:%.*]])
-; CHECK-NEXT:    br i1 true, label [[T:%.*]], label [[F:%.*]]
-; CHECK:       t:
-; CHECK-NEXT:    ret i32 [[A]]
-; CHECK:       f:
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@ipccp1
+; IS__TUNIT____-SAME: (i32 returned [[A:%.*]])
+; IS__TUNIT____-NEXT:    br i1 true, label [[T:%.*]], label [[F:%.*]]
+; IS__TUNIT____:       t:
+; IS__TUNIT____-NEXT:    ret i32 [[A]]
+; IS__TUNIT____:       f:
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@ipccp1
+; IS__CGSCC____-SAME: (i32 returned [[A:%.*]])
+; IS__CGSCC____-NEXT:    br i1 true, label [[T:%.*]], label [[F:%.*]]
+; IS__CGSCC____:       t:
+; IS__CGSCC____-NEXT:    ret i32 [[A]]
+; IS__CGSCC____:       f:
+; IS__CGSCC____-NEXT:    unreachable
 ;
   br i1 true, label %t, label %f
 t:
@@ -184,6 +224,7 @@ f:
 }
 
 define internal i1 @ipccp2i(i1 %a) {
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@ipccp2i()
 ; IS__CGSCC____-NEXT:    br label [[T:%.*]]
 ; IS__CGSCC____:       t:
@@ -200,9 +241,11 @@ f:
 }
 
 define i1 @ipccp2() {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@ipccp2()
 ; IS__TUNIT____-NEXT:    ret i1 true
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@ipccp2()
 ; IS__CGSCC____-NEXT:    [[R:%.*]] = call i1 @ipccp2i()
 ; IS__CGSCC____-NEXT:    ret i1 [[R]]
@@ -212,6 +255,7 @@ define i1 @ipccp2() {
 }
 
 define internal i1 @ipccp2ib(i1 %a) {
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@ipccp2ib()
 ; IS__CGSCC____-NEXT:    br label [[T:%.*]]
 ; IS__CGSCC____:       t:
@@ -228,14 +272,20 @@ f:
 }
 
 define i1 @ipccp2b() {
-; CHECK-LABEL: define {{[^@]+}}@ipccp2b()
-; CHECK-NEXT:    ret i1 true
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@ipccp2b()
+; IS__TUNIT____-NEXT:    ret i1 true
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@ipccp2b()
+; IS__CGSCC____-NEXT:    ret i1 true
 ;
   %r = call i1 @ipccp2ib(i1 true)
   ret i1 %r
 }
 
 define internal i32 @ipccp3i(i32 %a) {
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@ipccp3i()
 ; IS__CGSCC____-NEXT:    br label [[T:%.*]]
 ; IS__CGSCC____:       t:
@@ -253,9 +303,11 @@ f:
 }
 
 define i32 @ipccp3() {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@ipccp3()
 ; IS__TUNIT____-NEXT:    ret i32 7
 ;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@ipccp3()
 ; IS__CGSCC____-NEXT:    [[R:%.*]] = call i32 @ipccp3i()
 ; IS__CGSCC____-NEXT:    ret i32 [[R]]
@@ -267,44 +319,66 @@ define i32 @ipccp3() {
 ; Do not touch complicated arguments (for now)
 %struct.X = type { i8* }
 define internal i32* @test_inalloca(i32* inalloca %a) {
-; CHECK-LABEL: define {{[^@]+}}@test_inalloca
-; CHECK-SAME: (i32* inalloca noalias nofree returned writeonly align 536870912 "no-capture-maybe-returned" [[A:%.*]])
-; CHECK-NEXT:    ret i32* [[A]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test_inalloca
+; IS__TUNIT____-SAME: (i32* inalloca noalias nofree returned writeonly align 536870912 "no-capture-maybe-returned" [[A:%.*]])
+; IS__TUNIT____-NEXT:    ret i32* [[A]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test_inalloca
+; IS__CGSCC____-SAME: (i32* inalloca noalias nofree returned writeonly align 536870912 "no-capture-maybe-returned" [[A:%.*]])
+; IS__CGSCC____-NEXT:    ret i32* [[A]]
 ;
   ret i32* %a
 }
 define i32* @complicated_args_inalloca() {
-; CHECK-LABEL: define {{[^@]+}}@complicated_args_inalloca()
-; CHECK-NEXT:    [[CALL:%.*]] = call i32* @test_inalloca(i32* noalias nocapture nofree writeonly align 536870912 null)
-; CHECK-NEXT:    ret i32* [[CALL]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@complicated_args_inalloca()
+; IS__TUNIT____-NEXT:    [[CALL:%.*]] = call i32* @test_inalloca(i32* noalias nocapture nofree writeonly align 536870912 null)
+; IS__TUNIT____-NEXT:    ret i32* [[CALL]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@complicated_args_inalloca()
+; IS__CGSCC____-NEXT:    [[CALL:%.*]] = call i32* @test_inalloca(i32* noalias nocapture nofree writeonly align 536870912 null)
+; IS__CGSCC____-NEXT:    ret i32* [[CALL]]
 ;
   %call = call i32* @test_inalloca(i32* null)
   ret i32* %call
 }
 
 define internal i32* @test_preallocated(i32* preallocated(i32) %a) {
-; CHECK-LABEL: define {{[^@]+}}@test_preallocated
-; CHECK-SAME: (i32* noalias nofree returned writeonly preallocated(i32) align 536870912 "no-capture-maybe-returned" [[A:%.*]])
-; CHECK-NEXT:    ret i32* [[A]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test_preallocated
+; IS__TUNIT____-SAME: (i32* noalias nofree returned writeonly preallocated(i32) align 536870912 "no-capture-maybe-returned" [[A:%.*]])
+; IS__TUNIT____-NEXT:    ret i32* [[A]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test_preallocated
+; IS__CGSCC____-SAME: (i32* noalias nofree returned writeonly preallocated(i32) align 536870912 "no-capture-maybe-returned" [[A:%.*]])
+; IS__CGSCC____-NEXT:    ret i32* [[A]]
 ;
   ret i32* %a
 }
 define i32* @complicated_args_preallocated() {
+; IS__TUNIT_OPM: Function Attrs: nounwind
 ; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@complicated_args_preallocated()
 ; IS__TUNIT_OPM-NEXT:    [[C:%.*]] = call token @llvm.call.preallocated.setup(i32 1)
 ; IS__TUNIT_OPM-NEXT:    [[CALL:%.*]] = call i32* @test_preallocated(i32* noalias nocapture nofree writeonly preallocated(i32) align 536870912 null) #5 [ "preallocated"(token [[C]]) ]
 ; IS__TUNIT_OPM-NEXT:    ret i32* [[CALL]]
 ;
+; IS__TUNIT_NPM: Function Attrs: nounwind
 ; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@complicated_args_preallocated()
 ; IS__TUNIT_NPM-NEXT:    [[C:%.*]] = call token @llvm.call.preallocated.setup(i32 1)
 ; IS__TUNIT_NPM-NEXT:    [[CALL:%.*]] = call i32* @test_preallocated(i32* noalias nocapture nofree writeonly preallocated(i32) align 536870912 null) #4 [ "preallocated"(token [[C]]) ]
 ; IS__TUNIT_NPM-NEXT:    ret i32* [[CALL]]
 ;
+; IS__CGSCC_OPM: Function Attrs: nounwind
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@complicated_args_preallocated()
 ; IS__CGSCC_OPM-NEXT:    [[C:%.*]] = call token @llvm.call.preallocated.setup(i32 1)
 ; IS__CGSCC_OPM-NEXT:    [[CALL:%.*]] = call i32* @test_preallocated(i32* noalias nocapture nofree writeonly preallocated(i32) align 536870912 null) #6 [ "preallocated"(token [[C]]) ]
 ; IS__CGSCC_OPM-NEXT:    ret i32* [[CALL]]
 ;
+; IS__CGSCC_NPM: Function Attrs: nounwind
 ; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@complicated_args_preallocated()
 ; IS__CGSCC_NPM-NEXT:    [[C:%.*]] = call token @llvm.call.preallocated.setup(i32 1)
 ; IS__CGSCC_NPM-NEXT:    [[CALL:%.*]] = call i32* @test_preallocated(i32* noalias nocapture nofree writeonly preallocated(i32) align 536870912 null) #5 [ "preallocated"(token [[C]]) ]
@@ -317,10 +391,17 @@ define i32* @complicated_args_preallocated() {
 
 define internal void @test_sret(%struct.X* sret %a, %struct.X** %b) {
 ;
-; CHECK-LABEL: define {{[^@]+}}@test_sret
-; CHECK-SAME: (%struct.X* noalias nofree nonnull sret writeonly align 536870912 dereferenceable(8) [[A:%.*]], %struct.X** nocapture nofree nonnull writeonly align 8 dereferenceable(8) [[B:%.*]])
-; CHECK-NEXT:    store %struct.X* [[A]], %struct.X** [[B]], align 8
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test_sret
+; IS__TUNIT____-SAME: (%struct.X* noalias nofree nonnull sret writeonly align 536870912 dereferenceable(8) [[A:%.*]], %struct.X** nocapture nofree nonnull writeonly align 8 dereferenceable(8) [[B:%.*]])
+; IS__TUNIT____-NEXT:    store %struct.X* [[A]], %struct.X** [[B]], align 8
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test_sret
+; IS__CGSCC____-SAME: (%struct.X* noalias nofree nonnull sret writeonly align 536870912 dereferenceable(8) [[A:%.*]], %struct.X** nocapture nofree nonnull writeonly align 8 dereferenceable(8) [[B:%.*]])
+; IS__CGSCC____-NEXT:    store %struct.X* [[A]], %struct.X** [[B]], align 8
+; IS__CGSCC____-NEXT:    ret void
 ;
   store %struct.X* %a, %struct.X** %b
   ret void
@@ -328,11 +409,13 @@ define internal void @test_sret(%struct.X* sret %a, %struct.X** %b) {
 ; FIXME: Alignment and dereferenceability are not propagated to the argument
 define void @complicated_args_sret(%struct.X** %b) {
 ;
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@complicated_args_sret
 ; IS__TUNIT____-SAME: (%struct.X** nocapture nofree writeonly [[B:%.*]])
 ; IS__TUNIT____-NEXT:    call void @test_sret(%struct.X* noalias nocapture nofree writeonly align 536870912 null, %struct.X** nocapture nofree writeonly align 8 [[B]])
 ; IS__TUNIT____-NEXT:    ret void
 ;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@complicated_args_sret
 ; IS__CGSCC____-SAME: (%struct.X** nocapture nofree nonnull writeonly align 8 dereferenceable(8) [[B:%.*]])
 ; IS__CGSCC____-NEXT:    call void @test_sret(%struct.X* noalias nocapture nofree nonnull writeonly align 536870912 dereferenceable(8) null, %struct.X** nocapture nofree nonnull writeonly align 8 dereferenceable(8) [[B]])
@@ -343,16 +426,28 @@ define void @complicated_args_sret(%struct.X** %b) {
 }
 
 define internal %struct.X* @test_nest(%struct.X* nest %a) {
-; CHECK-LABEL: define {{[^@]+}}@test_nest
-; CHECK-SAME: (%struct.X* nest noalias nofree readnone returned align 536870912 "no-capture-maybe-returned" [[A:%.*]])
-; CHECK-NEXT:    ret %struct.X* [[A]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test_nest
+; IS__TUNIT____-SAME: (%struct.X* nest noalias nofree readnone returned align 536870912 "no-capture-maybe-returned" [[A:%.*]])
+; IS__TUNIT____-NEXT:    ret %struct.X* [[A]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test_nest
+; IS__CGSCC____-SAME: (%struct.X* nest noalias nofree readnone returned align 536870912 "no-capture-maybe-returned" [[A:%.*]])
+; IS__CGSCC____-NEXT:    ret %struct.X* [[A]]
 ;
   ret %struct.X* %a
 }
 define %struct.X* @complicated_args_nest() {
-; CHECK-LABEL: define {{[^@]+}}@complicated_args_nest()
-; CHECK-NEXT:    [[CALL:%.*]] = call %struct.X* @test_nest(%struct.X* noalias nocapture nofree readnone align 536870912 null)
-; CHECK-NEXT:    ret %struct.X* [[CALL]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@complicated_args_nest()
+; IS__TUNIT____-NEXT:    [[CALL:%.*]] = call %struct.X* @test_nest(%struct.X* noalias nocapture nofree readnone align 536870912 null)
+; IS__TUNIT____-NEXT:    ret %struct.X* [[CALL]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@complicated_args_nest()
+; IS__CGSCC____-NEXT:    [[CALL:%.*]] = call %struct.X* @test_nest(%struct.X* noalias nocapture nofree readnone align 536870912 null)
+; IS__CGSCC____-NEXT:    ret %struct.X* [[CALL]]
 ;
   %call = call %struct.X* @test_nest(%struct.X* null)
   ret %struct.X* %call
@@ -360,12 +455,14 @@ define %struct.X* @complicated_args_nest() {
 
 @S = external global %struct.X
 define internal void @test_byval(%struct.X* byval %a) {
+; IS__CGSCC_OPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@test_byval
 ; IS__CGSCC_OPM-SAME: (%struct.X* noalias nocapture nofree nonnull writeonly byval align 8 dereferenceable(8) [[A:%.*]])
 ; IS__CGSCC_OPM-NEXT:    [[G0:%.*]] = getelementptr [[STRUCT_X:%.*]], %struct.X* [[A]], i32 0, i32 0
 ; IS__CGSCC_OPM-NEXT:    store i8* null, i8** [[G0]], align 8
 ; IS__CGSCC_OPM-NEXT:    ret void
 ;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
 ; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@test_byval
 ; IS__CGSCC_NPM-SAME: (i8* nocapture nofree readnone [[TMP0:%.*]])
 ; IS__CGSCC_NPM-NEXT:    [[A_PRIV:%.*]] = alloca [[STRUCT_X:%.*]], align 8
@@ -380,54 +477,146 @@ define internal void @test_byval(%struct.X* byval %a) {
   ret void
 }
 define void @complicated_args_byval() {
-; CHECK-LABEL: define {{[^@]+}}@complicated_args_byval()
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@complicated_args_byval()
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC_OPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@complicated_args_byval()
+; IS__CGSCC_OPM-NEXT:    ret void
+;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@complicated_args_byval()
+; IS__CGSCC_NPM-NEXT:    ret void
 ;
   call void @test_byval(%struct.X* @S)
   ret void
 }
 
 define internal i8*@test_byval2(%struct.X* byval %a) {
-; CHECK-LABEL: define {{[^@]+}}@test_byval2()
-; CHECK-NEXT:    [[G0:%.*]] = getelementptr [[STRUCT_X:%.*]], %struct.X* @S, i32 0, i32 0
-; CHECK-NEXT:    [[L:%.*]] = load i8*, i8** [[G0]], align 8
-; CHECK-NEXT:    ret i8* [[L]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@test_byval2()
+; IS__TUNIT____-NEXT:    [[G0:%.*]] = getelementptr [[STRUCT_X:%.*]], %struct.X* @S, i32 0, i32 0
+; IS__TUNIT____-NEXT:    [[L:%.*]] = load i8*, i8** [[G0]], align 8
+; IS__TUNIT____-NEXT:    ret i8* [[L]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@test_byval2()
+; IS__CGSCC____-NEXT:    [[G0:%.*]] = getelementptr [[STRUCT_X:%.*]], %struct.X* @S, i32 0, i32 0
+; IS__CGSCC____-NEXT:    [[L:%.*]] = load i8*, i8** [[G0]], align 8
+; IS__CGSCC____-NEXT:    ret i8* [[L]]
 ;
   %g0 = getelementptr %struct.X, %struct.X* %a, i32 0, i32 0
   %l = load i8*, i8** %g0
   ret i8* %l
 }
 define i8* @complicated_args_byval2() {
-; CHECK-LABEL: define {{[^@]+}}@complicated_args_byval2()
-; CHECK-NEXT:    [[C:%.*]] = call i8* @test_byval2()
-; CHECK-NEXT:    ret i8* [[C]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@complicated_args_byval2()
+; IS__TUNIT____-NEXT:    [[C:%.*]] = call i8* @test_byval2()
+; IS__TUNIT____-NEXT:    ret i8* [[C]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@complicated_args_byval2()
+; IS__CGSCC____-NEXT:    [[C:%.*]] = call i8* @test_byval2()
+; IS__CGSCC____-NEXT:    ret i8* [[C]]
 ;
   %c = call i8* @test_byval2(%struct.X* @S)
   ret i8* %c
 }
 
 define void @fixpoint_changed(i32* %p) {
-; CHECK-LABEL: define {{[^@]+}}@fixpoint_changed
-; CHECK-SAME: (i32* nocapture nofree writeonly [[P:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[FOR_COND:%.*]]
-; CHECK:       for.cond:
-; CHECK-NEXT:    [[J_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[SW_EPILOG:%.*]] ]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[J_0]], 30
-; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
-; CHECK:       for.body:
-; CHECK-NEXT:    switch i32 [[J_0]], label [[SW_EPILOG]] [
-; CHECK-NEXT:    i32 1, label [[SW_BB:%.*]]
-; CHECK-NEXT:    ]
-; CHECK:       sw.bb:
-; CHECK-NEXT:    br label [[SW_EPILOG]]
-; CHECK:       sw.epilog:
-; CHECK-NEXT:    [[X_0:%.*]] = phi i32 [ 255, [[FOR_BODY]] ], [ 253, [[SW_BB]] ]
-; CHECK-NEXT:    store i32 [[X_0]], i32* [[P]], align 4
-; CHECK-NEXT:    [[INC]] = add nsw i32 [[J_0]], 1
-; CHECK-NEXT:    br label [[FOR_COND]]
-; CHECK:       for.end:
-; CHECK-NEXT:    ret void
+; IS__TUNIT_OPM: Function Attrs: argmemonly nofree nosync nounwind writeonly
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@fixpoint_changed
+; IS__TUNIT_OPM-SAME: (i32* nocapture nofree writeonly [[P:%.*]])
+; IS__TUNIT_OPM-NEXT:  entry:
+; IS__TUNIT_OPM-NEXT:    br label [[FOR_COND:%.*]]
+; IS__TUNIT_OPM:       for.cond:
+; IS__TUNIT_OPM-NEXT:    [[J_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[SW_EPILOG:%.*]] ]
+; IS__TUNIT_OPM-NEXT:    [[CMP:%.*]] = icmp slt i32 [[J_0]], 30
+; IS__TUNIT_OPM-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+; IS__TUNIT_OPM:       for.body:
+; IS__TUNIT_OPM-NEXT:    switch i32 [[J_0]], label [[SW_EPILOG]] [
+; IS__TUNIT_OPM-NEXT:    i32 1, label [[SW_BB:%.*]]
+; IS__TUNIT_OPM-NEXT:    ]
+; IS__TUNIT_OPM:       sw.bb:
+; IS__TUNIT_OPM-NEXT:    br label [[SW_EPILOG]]
+; IS__TUNIT_OPM:       sw.epilog:
+; IS__TUNIT_OPM-NEXT:    [[X_0:%.*]] = phi i32 [ 255, [[FOR_BODY]] ], [ 253, [[SW_BB]] ]
+; IS__TUNIT_OPM-NEXT:    store i32 [[X_0]], i32* [[P]], align 4
+; IS__TUNIT_OPM-NEXT:    [[INC]] = add nsw i32 [[J_0]], 1
+; IS__TUNIT_OPM-NEXT:    br label [[FOR_COND]]
+; IS__TUNIT_OPM:       for.end:
+; IS__TUNIT_OPM-NEXT:    ret void
+;
+; IS__TUNIT_NPM: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@fixpoint_changed
+; IS__TUNIT_NPM-SAME: (i32* nocapture nofree writeonly [[P:%.*]])
+; IS__TUNIT_NPM-NEXT:  entry:
+; IS__TUNIT_NPM-NEXT:    br label [[FOR_COND:%.*]]
+; IS__TUNIT_NPM:       for.cond:
+; IS__TUNIT_NPM-NEXT:    [[J_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[SW_EPILOG:%.*]] ]
+; IS__TUNIT_NPM-NEXT:    [[CMP:%.*]] = icmp slt i32 [[J_0]], 30
+; IS__TUNIT_NPM-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+; IS__TUNIT_NPM:       for.body:
+; IS__TUNIT_NPM-NEXT:    switch i32 [[J_0]], label [[SW_EPILOG]] [
+; IS__TUNIT_NPM-NEXT:    i32 1, label [[SW_BB:%.*]]
+; IS__TUNIT_NPM-NEXT:    ]
+; IS__TUNIT_NPM:       sw.bb:
+; IS__TUNIT_NPM-NEXT:    br label [[SW_EPILOG]]
+; IS__TUNIT_NPM:       sw.epilog:
+; IS__TUNIT_NPM-NEXT:    [[X_0:%.*]] = phi i32 [ 255, [[FOR_BODY]] ], [ 253, [[SW_BB]] ]
+; IS__TUNIT_NPM-NEXT:    store i32 [[X_0]], i32* [[P]], align 4
+; IS__TUNIT_NPM-NEXT:    [[INC]] = add nsw i32 [[J_0]], 1
+; IS__TUNIT_NPM-NEXT:    br label [[FOR_COND]]
+; IS__TUNIT_NPM:       for.end:
+; IS__TUNIT_NPM-NEXT:    ret void
+;
+; IS__CGSCC_OPM: Function Attrs: argmemonly nofree norecurse nosync nounwind writeonly
+; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@fixpoint_changed
+; IS__CGSCC_OPM-SAME: (i32* nocapture nofree writeonly [[P:%.*]])
+; IS__CGSCC_OPM-NEXT:  entry:
+; IS__CGSCC_OPM-NEXT:    br label [[FOR_COND:%.*]]
+; IS__CGSCC_OPM:       for.cond:
+; IS__CGSCC_OPM-NEXT:    [[J_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[SW_EPILOG:%.*]] ]
+; IS__CGSCC_OPM-NEXT:    [[CMP:%.*]] = icmp slt i32 [[J_0]], 30
+; IS__CGSCC_OPM-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+; IS__CGSCC_OPM:       for.body:
+; IS__CGSCC_OPM-NEXT:    switch i32 [[J_0]], label [[SW_EPILOG]] [
+; IS__CGSCC_OPM-NEXT:    i32 1, label [[SW_BB:%.*]]
+; IS__CGSCC_OPM-NEXT:    ]
+; IS__CGSCC_OPM:       sw.bb:
+; IS__CGSCC_OPM-NEXT:    br label [[SW_EPILOG]]
+; IS__CGSCC_OPM:       sw.epilog:
+; IS__CGSCC_OPM-NEXT:    [[X_0:%.*]] = phi i32 [ 255, [[FOR_BODY]] ], [ 253, [[SW_BB]] ]
+; IS__CGSCC_OPM-NEXT:    store i32 [[X_0]], i32* [[P]], align 4
+; IS__CGSCC_OPM-NEXT:    [[INC]] = add nsw i32 [[J_0]], 1
+; IS__CGSCC_OPM-NEXT:    br label [[FOR_COND]]
+; IS__CGSCC_OPM:       for.end:
+; IS__CGSCC_OPM-NEXT:    ret void
+;
+; IS__CGSCC_NPM: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@fixpoint_changed
+; IS__CGSCC_NPM-SAME: (i32* nocapture nofree writeonly [[P:%.*]])
+; IS__CGSCC_NPM-NEXT:  entry:
+; IS__CGSCC_NPM-NEXT:    br label [[FOR_COND:%.*]]
+; IS__CGSCC_NPM:       for.cond:
+; IS__CGSCC_NPM-NEXT:    [[J_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[SW_EPILOG:%.*]] ]
+; IS__CGSCC_NPM-NEXT:    [[CMP:%.*]] = icmp slt i32 [[J_0]], 30
+; IS__CGSCC_NPM-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+; IS__CGSCC_NPM:       for.body:
+; IS__CGSCC_NPM-NEXT:    switch i32 [[J_0]], label [[SW_EPILOG]] [
+; IS__CGSCC_NPM-NEXT:    i32 1, label [[SW_BB:%.*]]
+; IS__CGSCC_NPM-NEXT:    ]
+; IS__CGSCC_NPM:       sw.bb:
+; IS__CGSCC_NPM-NEXT:    br label [[SW_EPILOG]]
+; IS__CGSCC_NPM:       sw.epilog:
+; IS__CGSCC_NPM-NEXT:    [[X_0:%.*]] = phi i32 [ 255, [[FOR_BODY]] ], [ 253, [[SW_BB]] ]
+; IS__CGSCC_NPM-NEXT:    store i32 [[X_0]], i32* [[P]], align 4
+; IS__CGSCC_NPM-NEXT:    [[INC]] = add nsw i32 [[J_0]], 1
+; IS__CGSCC_NPM-NEXT:    br label [[FOR_COND]]
+; IS__CGSCC_NPM:       for.end:
+; IS__CGSCC_NPM-NEXT:    ret void
 ;
 entry:
   br label %for.cond
@@ -458,57 +647,99 @@ for.end:
 ; Check we merge undef and a constant properly.
 ; FIXME fold the addition and return the constant.
 define i8 @caller0() {
-; CHECK-LABEL: define {{[^@]+}}@caller0()
-; CHECK-NEXT:    [[C:%.*]] = call i8 @callee()
-; CHECK-NEXT:    ret i8 [[C]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@caller0()
+; IS__TUNIT____-NEXT:    [[C:%.*]] = call i8 @callee()
+; IS__TUNIT____-NEXT:    ret i8 [[C]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@caller0()
+; IS__CGSCC____-NEXT:    [[C:%.*]] = call i8 @callee()
+; IS__CGSCC____-NEXT:    ret i8 [[C]]
 ;
   %c = call i8 @callee(i8 undef)
   ret i8 %c
 }
 define i8 @caller1() {
-; CHECK-LABEL: define {{[^@]+}}@caller1()
-; CHECK-NEXT:    [[C:%.*]] = call i8 @callee()
-; CHECK-NEXT:    ret i8 [[C]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@caller1()
+; IS__TUNIT____-NEXT:    [[C:%.*]] = call i8 @callee()
+; IS__TUNIT____-NEXT:    ret i8 [[C]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@caller1()
+; IS__CGSCC____-NEXT:    [[C:%.*]] = call i8 @callee()
+; IS__CGSCC____-NEXT:    ret i8 [[C]]
 ;
   %c = call i8 @callee(i8 undef)
   ret i8 %c
 }
 define i8 @caller2() {
-; CHECK-LABEL: define {{[^@]+}}@caller2()
-; CHECK-NEXT:    [[C:%.*]] = call i8 @callee()
-; CHECK-NEXT:    ret i8 [[C]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@caller2()
+; IS__TUNIT____-NEXT:    [[C:%.*]] = call i8 @callee()
+; IS__TUNIT____-NEXT:    ret i8 [[C]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@caller2()
+; IS__CGSCC____-NEXT:    [[C:%.*]] = call i8 @callee()
+; IS__CGSCC____-NEXT:    ret i8 [[C]]
 ;
   %c = call i8 @callee(i8 undef)
   ret i8 %c
 }
 define i8 @caller_middle() {
-; CHECK-LABEL: define {{[^@]+}}@caller_middle()
-; CHECK-NEXT:    [[C:%.*]] = call i8 @callee()
-; CHECK-NEXT:    ret i8 [[C]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@caller_middle()
+; IS__TUNIT____-NEXT:    [[C:%.*]] = call i8 @callee()
+; IS__TUNIT____-NEXT:    ret i8 [[C]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@caller_middle()
+; IS__CGSCC____-NEXT:    [[C:%.*]] = call i8 @callee()
+; IS__CGSCC____-NEXT:    ret i8 [[C]]
 ;
   %c = call i8 @callee(i8 42)
   ret i8 %c
 }
 define i8 @caller3() {
-; CHECK-LABEL: define {{[^@]+}}@caller3()
-; CHECK-NEXT:    [[C:%.*]] = call i8 @callee()
-; CHECK-NEXT:    ret i8 [[C]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@caller3()
+; IS__TUNIT____-NEXT:    [[C:%.*]] = call i8 @callee()
+; IS__TUNIT____-NEXT:    ret i8 [[C]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@caller3()
+; IS__CGSCC____-NEXT:    [[C:%.*]] = call i8 @callee()
+; IS__CGSCC____-NEXT:    ret i8 [[C]]
 ;
   %c = call i8 @callee(i8 undef)
   ret i8 %c
 }
 define i8 @caller4() {
-; CHECK-LABEL: define {{[^@]+}}@caller4()
-; CHECK-NEXT:    [[C:%.*]] = call i8 @callee()
-; CHECK-NEXT:    ret i8 [[C]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@caller4()
+; IS__TUNIT____-NEXT:    [[C:%.*]] = call i8 @callee()
+; IS__TUNIT____-NEXT:    ret i8 [[C]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@caller4()
+; IS__CGSCC____-NEXT:    [[C:%.*]] = call i8 @callee()
+; IS__CGSCC____-NEXT:    ret i8 [[C]]
 ;
   %c = call i8 @callee(i8 undef)
   ret i8 %c
 }
 define internal i8 @callee(i8 %a) {
-; CHECK-LABEL: define {{[^@]+}}@callee()
-; CHECK-NEXT:    [[C:%.*]] = add i8 42, 7
-; CHECK-NEXT:    ret i8 [[C]]
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@callee()
+; IS__TUNIT____-NEXT:    [[C:%.*]] = add i8 42, 7
+; IS__TUNIT____-NEXT:    ret i8 [[C]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@callee()
+; IS__CGSCC____-NEXT:    [[C:%.*]] = add i8 42, 7
+; IS__CGSCC____-NEXT:    ret i8 [[C]]
 ;
   %c = add i8 %a, 7
   ret i8 %c
diff --git a/llvm/test/Transforms/Attributor/willreturn.ll b/llvm/test/Transforms/Attributor/willreturn.ll
index 296339ce99b0f..c568b2a956d5c 100644
--- a/llvm/test/Transforms/Attributor/willreturn.ll
+++ b/llvm/test/Transforms/Attributor/willreturn.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
 ; RUN: opt -attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=8 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal  -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=8 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
 ; RUN: opt -attributor-cgscc -attributor-manifest-internal  -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
@@ -11,11 +11,14 @@ target datalayout = "e-m:e-i54:64-f80:128-n8:16:32:64-S128"
 
 
 ; TEST 1 (positive case)
+define void @only_return() #0 {
 ; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@only_return()
+; IS__TUNIT____-NEXT:    ret void
+;
 ; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
-define void @only_return() #0 {
-; CHECK-LABEL: define {{[^@]+}}@only_return()
-; CHECK-NEXT:    ret void
+; IS__CGSCC____-LABEL: define {{[^@]+}}@only_return()
+; IS__CGSCC____-NEXT:    ret void
 ;
   ret void
 }
@@ -29,8 +32,8 @@ define void @only_return() #0 {
 ; }
 
 ; FIXME: missing willreturn
-; CHECK: Function Attrs: nofree noinline nosync nounwind readnone uwtable
 define i32 @fib(i32 %0) local_unnamed_addr #0 {
+; CHECK: Function Attrs: nofree noinline nosync nounwind readnone uwtable
 ; CHECK-LABEL: define {{[^@]+}}@fib
 ; CHECK-SAME: (i32 [[TMP0:%.*]]) local_unnamed_addr
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP0]], 2
@@ -70,26 +73,42 @@ define i32 @fib(i32 %0) local_unnamed_addr #0 {
 ; }
 ; fact_maybe_not(-1) doesn't stop.
 
+define i32 @fact_maybe_not_halt(i32 %0) local_unnamed_addr #0 {
 ; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable
+; IS__TUNIT____-LABEL: define {{[^@]+}}@fact_maybe_not_halt
+; IS__TUNIT____-SAME: (i32 [[TMP0:%.*]]) local_unnamed_addr
+; IS__TUNIT____-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP0]], 0
+; IS__TUNIT____-NEXT:    br i1 [[TMP2]], label [[TMP11:%.*]], label [[TMP3:%.*]]
+; IS__TUNIT____:       3:
+; IS__TUNIT____-NEXT:    [[TMP4:%.*]] = phi i32 [ [[TMP8:%.*]], [[TMP3]] ], [ [[TMP0]], [[TMP1:%.*]] ]
+; IS__TUNIT____-NEXT:    [[TMP5:%.*]] = phi i32 [ [[TMP9:%.*]], [[TMP3]] ], [ 1, [[TMP1]] ]
+; IS__TUNIT____-NEXT:    [[TMP6:%.*]] = icmp sgt i32 [[TMP4]], 0
+; IS__TUNIT____-NEXT:    [[TMP7:%.*]] = sext i1 [[TMP6]] to i32
+; IS__TUNIT____-NEXT:    [[TMP8]] = add nsw i32 [[TMP4]], [[TMP7]]
+; IS__TUNIT____-NEXT:    [[TMP9]] = mul nsw i32 [[TMP4]], [[TMP5]]
+; IS__TUNIT____-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IS__TUNIT____-NEXT:    br i1 [[TMP10]], label [[TMP11]], label [[TMP3]]
+; IS__TUNIT____:       11:
+; IS__TUNIT____-NEXT:    [[TMP12:%.*]] = phi i32 [ 1, [[TMP1]] ], [ [[TMP9]], [[TMP3]] ]
+; IS__TUNIT____-NEXT:    ret i32 [[TMP12]]
+;
 ; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
-; CHECK-NOT: willreturn
-define i32 @fact_maybe_not_halt(i32 %0) local_unnamed_addr #0 {
-; CHECK-LABEL: define {{[^@]+}}@fact_maybe_not_halt
-; CHECK-SAME: (i32 [[TMP0:%.*]]) local_unnamed_addr
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP0]], 0
-; CHECK-NEXT:    br i1 [[TMP2]], label [[TMP11:%.*]], label [[TMP3:%.*]]
-; CHECK:       3:
-; CHECK-NEXT:    [[TMP4:%.*]] = phi i32 [ [[TMP8:%.*]], [[TMP3]] ], [ [[TMP0]], [[TMP1:%.*]] ]
-; CHECK-NEXT:    [[TMP5:%.*]] = phi i32 [ [[TMP9:%.*]], [[TMP3]] ], [ 1, [[TMP1]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp sgt i32 [[TMP4]], 0
-; CHECK-NEXT:    [[TMP7:%.*]] = sext i1 [[TMP6]] to i32
-; CHECK-NEXT:    [[TMP8]] = add nsw i32 [[TMP4]], [[TMP7]]
-; CHECK-NEXT:    [[TMP9]] = mul nsw i32 [[TMP4]], [[TMP5]]
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
-; CHECK-NEXT:    br i1 [[TMP10]], label [[TMP11]], label [[TMP3]]
-; CHECK:       11:
-; CHECK-NEXT:    [[TMP12:%.*]] = phi i32 [ 1, [[TMP1]] ], [ [[TMP9]], [[TMP3]] ]
-; CHECK-NEXT:    ret i32 [[TMP12]]
+; IS__CGSCC____-LABEL: define {{[^@]+}}@fact_maybe_not_halt
+; IS__CGSCC____-SAME: (i32 [[TMP0:%.*]]) local_unnamed_addr
+; IS__CGSCC____-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP0]], 0
+; IS__CGSCC____-NEXT:    br i1 [[TMP2]], label [[TMP11:%.*]], label [[TMP3:%.*]]
+; IS__CGSCC____:       3:
+; IS__CGSCC____-NEXT:    [[TMP4:%.*]] = phi i32 [ [[TMP8:%.*]], [[TMP3]] ], [ [[TMP0]], [[TMP1:%.*]] ]
+; IS__CGSCC____-NEXT:    [[TMP5:%.*]] = phi i32 [ [[TMP9:%.*]], [[TMP3]] ], [ 1, [[TMP1]] ]
+; IS__CGSCC____-NEXT:    [[TMP6:%.*]] = icmp sgt i32 [[TMP4]], 0
+; IS__CGSCC____-NEXT:    [[TMP7:%.*]] = sext i1 [[TMP6]] to i32
+; IS__CGSCC____-NEXT:    [[TMP8]] = add nsw i32 [[TMP4]], [[TMP7]]
+; IS__CGSCC____-NEXT:    [[TMP9]] = mul nsw i32 [[TMP4]], [[TMP5]]
+; IS__CGSCC____-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IS__CGSCC____-NEXT:    br i1 [[TMP10]], label [[TMP11]], label [[TMP3]]
+; IS__CGSCC____:       11:
+; IS__CGSCC____-NEXT:    [[TMP12:%.*]] = phi i32 [ 1, [[TMP1]] ], [ [[TMP9]], [[TMP3]] ]
+; IS__CGSCC____-NEXT:    ret i32 [[TMP12]]
 ;
   %2 = icmp eq i32 %0, 0
   br i1 %2, label %11, label %3
@@ -120,23 +139,70 @@ define i32 @fact_maybe_not_halt(i32 %0) local_unnamed_addr #0 {
 ;   return ans;
 ; }
 
-; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable
-; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
 define i32 @fact_loop(i32 %0) local_unnamed_addr #0 {
-; CHECK-LABEL: define {{[^@]+}}@fact_loop
-; CHECK-SAME: (i32 [[TMP0:%.*]]) local_unnamed_addr
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP0]], 1
-; CHECK-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP5:%.*]]
-; CHECK:       3:
-; CHECK-NEXT:    [[TMP4:%.*]] = phi i32 [ 1, [[TMP1:%.*]] ], [ [[TMP8:%.*]], [[TMP5]] ]
-; CHECK-NEXT:    ret i32 [[TMP4]]
-; CHECK:       5:
-; CHECK-NEXT:    [[TMP6:%.*]] = phi i32 [ [[TMP9:%.*]], [[TMP5]] ], [ 1, [[TMP1]] ]
-; CHECK-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP8]], [[TMP5]] ], [ 1, [[TMP1]] ]
-; CHECK-NEXT:    [[TMP8]] = mul nsw i32 [[TMP6]], [[TMP7]]
-; CHECK-NEXT:    [[TMP9]] = add nuw nsw i32 [[TMP6]], 1
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[TMP6]], [[TMP0]]
-; CHECK-NEXT:    br i1 [[TMP10]], label [[TMP3]], label [[TMP5]]
+; IS__TUNIT_OPM: Function Attrs: nofree noinline nosync nounwind readnone uwtable
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@fact_loop
+; IS__TUNIT_OPM-SAME: (i32 [[TMP0:%.*]]) local_unnamed_addr
+; IS__TUNIT_OPM-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP0]], 1
+; IS__TUNIT_OPM-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP5:%.*]]
+; IS__TUNIT_OPM:       3:
+; IS__TUNIT_OPM-NEXT:    [[TMP4:%.*]] = phi i32 [ 1, [[TMP1:%.*]] ], [ [[TMP8:%.*]], [[TMP5]] ]
+; IS__TUNIT_OPM-NEXT:    ret i32 [[TMP4]]
+; IS__TUNIT_OPM:       5:
+; IS__TUNIT_OPM-NEXT:    [[TMP6:%.*]] = phi i32 [ [[TMP9:%.*]], [[TMP5]] ], [ 1, [[TMP1]] ]
+; IS__TUNIT_OPM-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP8]], [[TMP5]] ], [ 1, [[TMP1]] ]
+; IS__TUNIT_OPM-NEXT:    [[TMP8]] = mul nsw i32 [[TMP6]], [[TMP7]]
+; IS__TUNIT_OPM-NEXT:    [[TMP9]] = add nuw nsw i32 [[TMP6]], 1
+; IS__TUNIT_OPM-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[TMP6]], [[TMP0]]
+; IS__TUNIT_OPM-NEXT:    br i1 [[TMP10]], label [[TMP3]], label [[TMP5]]
+;
+; IS__TUNIT_NPM: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@fact_loop
+; IS__TUNIT_NPM-SAME: (i32 [[TMP0:%.*]]) local_unnamed_addr
+; IS__TUNIT_NPM-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP0]], 1
+; IS__TUNIT_NPM-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP5:%.*]]
+; IS__TUNIT_NPM:       3:
+; IS__TUNIT_NPM-NEXT:    [[TMP4:%.*]] = phi i32 [ 1, [[TMP1:%.*]] ], [ [[TMP8:%.*]], [[TMP5]] ]
+; IS__TUNIT_NPM-NEXT:    ret i32 [[TMP4]]
+; IS__TUNIT_NPM:       5:
+; IS__TUNIT_NPM-NEXT:    [[TMP6:%.*]] = phi i32 [ [[TMP9:%.*]], [[TMP5]] ], [ 1, [[TMP1]] ]
+; IS__TUNIT_NPM-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP8]], [[TMP5]] ], [ 1, [[TMP1]] ]
+; IS__TUNIT_NPM-NEXT:    [[TMP8]] = mul nsw i32 [[TMP6]], [[TMP7]]
+; IS__TUNIT_NPM-NEXT:    [[TMP9]] = add nuw nsw i32 [[TMP6]], 1
+; IS__TUNIT_NPM-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[TMP6]], [[TMP0]]
+; IS__TUNIT_NPM-NEXT:    br i1 [[TMP10]], label [[TMP3]], label [[TMP5]]
+;
+; IS__CGSCC_OPM: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
+; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@fact_loop
+; IS__CGSCC_OPM-SAME: (i32 [[TMP0:%.*]]) local_unnamed_addr
+; IS__CGSCC_OPM-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP0]], 1
+; IS__CGSCC_OPM-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP5:%.*]]
+; IS__CGSCC_OPM:       3:
+; IS__CGSCC_OPM-NEXT:    [[TMP4:%.*]] = phi i32 [ 1, [[TMP1:%.*]] ], [ [[TMP8:%.*]], [[TMP5]] ]
+; IS__CGSCC_OPM-NEXT:    ret i32 [[TMP4]]
+; IS__CGSCC_OPM:       5:
+; IS__CGSCC_OPM-NEXT:    [[TMP6:%.*]] = phi i32 [ [[TMP9:%.*]], [[TMP5]] ], [ 1, [[TMP1]] ]
+; IS__CGSCC_OPM-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP8]], [[TMP5]] ], [ 1, [[TMP1]] ]
+; IS__CGSCC_OPM-NEXT:    [[TMP8]] = mul nsw i32 [[TMP6]], [[TMP7]]
+; IS__CGSCC_OPM-NEXT:    [[TMP9]] = add nuw nsw i32 [[TMP6]], 1
+; IS__CGSCC_OPM-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[TMP6]], [[TMP0]]
+; IS__CGSCC_OPM-NEXT:    br i1 [[TMP10]], label [[TMP3]], label [[TMP5]]
+;
+; IS__CGSCC_NPM: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@fact_loop
+; IS__CGSCC_NPM-SAME: (i32 [[TMP0:%.*]]) local_unnamed_addr
+; IS__CGSCC_NPM-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP0]], 1
+; IS__CGSCC_NPM-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP5:%.*]]
+; IS__CGSCC_NPM:       3:
+; IS__CGSCC_NPM-NEXT:    [[TMP4:%.*]] = phi i32 [ 1, [[TMP1:%.*]] ], [ [[TMP8:%.*]], [[TMP5]] ]
+; IS__CGSCC_NPM-NEXT:    ret i32 [[TMP4]]
+; IS__CGSCC_NPM:       5:
+; IS__CGSCC_NPM-NEXT:    [[TMP6:%.*]] = phi i32 [ [[TMP9:%.*]], [[TMP5]] ], [ 1, [[TMP1]] ]
+; IS__CGSCC_NPM-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP8]], [[TMP5]] ], [ 1, [[TMP1]] ]
+; IS__CGSCC_NPM-NEXT:    [[TMP8]] = mul nsw i32 [[TMP6]], [[TMP7]]
+; IS__CGSCC_NPM-NEXT:    [[TMP9]] = add nuw nsw i32 [[TMP6]], 1
+; IS__CGSCC_NPM-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[TMP6]], [[TMP0]]
+; IS__CGSCC_NPM-NEXT:    br i1 [[TMP10]], label [[TMP3]], label [[TMP5]]
 ;
   %2 = icmp slt i32 %0, 1
   br i1 %2, label %3, label %5
@@ -165,9 +231,8 @@ define i32 @fact_loop(i32 %0) local_unnamed_addr #0 {
 
 declare void @sink() nounwind willreturn nosync nofree
 
-; CHECK: Function Attrs: nofree noinline nosync nounwind uwtable
-; CHECK-NOT: willreturn
 define void @mutual_recursion1(i1 %c) #0 {
+; CHECK: Function Attrs: nofree noinline nosync nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@mutual_recursion1
 ; CHECK-SAME: (i1 [[C:%.*]])
 ; CHECK-NEXT:    br i1 [[C]], label [[REC:%.*]], label [[END:%.*]]
@@ -188,9 +253,8 @@ end:
 }
 
 
-; CHECK: Function Attrs: nofree noinline nosync nounwind uwtable
-; CHECK-NOT: willreturn
 define void @mutual_recursion2(i1 %c) #0 {
+; CHECK: Function Attrs: nofree noinline nosync nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@mutual_recursion2
 ; CHECK-SAME: (i1 [[C:%.*]])
 ; CHECK-NEXT:    call void @mutual_recursion1(i1 [[C]])
@@ -207,9 +271,8 @@ define void @mutual_recursion2(i1 %c) #0 {
 ; CHECK-NEXT: declare void @exit(i32) local_unnamed_add
 declare void @exit(i32 %0) local_unnamed_addr noreturn
 
-; CHECK: Function Attrs: noinline noreturn nounwind uwtable
-; CHECK-NOT: willreturn
 define void @only_exit() local_unnamed_addr #0 {
+; CHECK: Function Attrs: noinline noreturn nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@only_exit() local_unnamed_addr
 ; CHECK-NEXT:    tail call void @exit(i32 0)
 ; CHECK-NEXT:    unreachable
@@ -228,9 +291,8 @@ define void @only_exit() local_unnamed_addr #0 {
 ;     }
 ;     return;
 ; }
-; CHECK: Function Attrs: noinline nounwind uwtable
-; CHECK-NOT: willreturn
 define void @conditional_exit(i32 %0, i32* nocapture readonly %1) local_unnamed_addr #0 {
+; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@conditional_exit
 ; CHECK-SAME: (i32 [[TMP0:%.*]], i32* nocapture readonly [[TMP1:%.*]]) local_unnamed_addr
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP0]], 0
@@ -274,19 +336,23 @@ define void @conditional_exit(i32 %0, i32* nocapture readonly %1) local_unnamed_
 ; CHECK-NEXT: declare float @llvm.floor.f32(float)
 declare float @llvm.floor.f32(float)
 
-; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
-; IS__CGSCC____:  Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
 define void @call_floor(float %a) #0 {
-; CHECK-LABEL: define {{[^@]+}}@call_floor
-; CHECK-SAME: (float [[A:%.*]])
-; CHECK-NEXT:    ret void
+; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@call_floor
+; IS__TUNIT____-SAME: (float [[A:%.*]])
+; IS__TUNIT____-NEXT:    ret void
+;
+; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@call_floor
+; IS__CGSCC____-SAME: (float [[A:%.*]])
+; IS__CGSCC____-NEXT:    ret void
 ;
   tail call float @llvm.floor.f32(float %a)
   ret void
 }
 
-; CHECK: Function Attrs: noinline nosync nounwind readnone uwtable willreturn
 define float @call_floor2(float %a) #0 {
+; CHECK: Function Attrs: noinline nosync nounwind readnone uwtable willreturn
 ; CHECK-LABEL: define {{[^@]+}}@call_floor2
 ; CHECK-SAME: (float [[A:%.*]])
 ; CHECK-NEXT:    [[C:%.*]] = tail call float @llvm.floor.f32(float [[A]])
@@ -305,9 +371,8 @@ define float @call_floor2(float %a) #0 {
 ; CHECK-NEXT: declare void @maybe_noreturn()
 declare void @maybe_noreturn() #0
 
-; CHECK: Function Attrs: noinline nounwind uwtable
-; CHECK-NOT: willreturn
 define void @call_maybe_noreturn() #0 {
+; CHECK: Function Attrs: noinline nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@call_maybe_noreturn()
 ; CHECK-NEXT:    tail call void @maybe_noreturn()
 ; CHECK-NEXT:    ret void
@@ -324,23 +389,31 @@ define void @call_maybe_noreturn() #0 {
 ; CHECK-NEXT: declare void @will_return()
 declare void @will_return() willreturn norecurse
 
+define void @f1() #0 {
 ; IS__TUNIT____: Function Attrs: noinline nounwind uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@f1()
+; IS__TUNIT____-NEXT:    tail call void @will_return()
+; IS__TUNIT____-NEXT:    ret void
+;
 ; IS__CGSCC____: Function Attrs: noinline norecurse nounwind uwtable willreturn
-define void @f1() #0 {
-; CHECK-LABEL: define {{[^@]+}}@f1()
-; CHECK-NEXT:    tail call void @will_return()
-; CHECK-NEXT:    ret void
+; IS__CGSCC____-LABEL: define {{[^@]+}}@f1()
+; IS__CGSCC____-NEXT:    tail call void @will_return()
+; IS__CGSCC____-NEXT:    ret void
 ;
   tail call void @will_return()
   ret void
 }
 
+define void @f2() #0 {
 ; IS__TUNIT____: Function Attrs: noinline nounwind uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@f2()
+; IS__TUNIT____-NEXT:    tail call void @f1()
+; IS__TUNIT____-NEXT:    ret void
+;
 ; IS__CGSCC____: Function Attrs: noinline norecurse nounwind uwtable willreturn
-define void @f2() #0 {
-; CHECK-LABEL: define {{[^@]+}}@f2()
-; CHECK-NEXT:    tail call void @f1()
-; CHECK-NEXT:    ret void
+; IS__CGSCC____-LABEL: define {{[^@]+}}@f2()
+; IS__CGSCC____-NEXT:    tail call void @f1()
+; IS__CGSCC____-NEXT:    ret void
 ;
   tail call void @f1()
   ret void
@@ -350,17 +423,24 @@ define void @f2() #0 {
 ; TEST 9 (negative case)
 ; call willreturn function in endless loop.
 
+define void @call_will_return_but_has_loop() #0 {
 ; IS__TUNIT____: Function Attrs: noinline noreturn nounwind uwtable
+; IS__TUNIT____-LABEL: define {{[^@]+}}@call_will_return_but_has_loop()
+; IS__TUNIT____-NEXT:    br label [[LABEL1:%.*]]
+; IS__TUNIT____:       label1:
+; IS__TUNIT____-NEXT:    tail call void @will_return()
+; IS__TUNIT____-NEXT:    br label [[LABEL2:%.*]]
+; IS__TUNIT____:       label2:
+; IS__TUNIT____-NEXT:    br label [[LABEL1]]
+;
 ; IS__CGSCC____: Function Attrs: noinline norecurse noreturn nounwind uwtable
-; CHECK-NOT: willreturn
-define void @call_will_return_but_has_loop() #0 {
-; CHECK-LABEL: define {{[^@]+}}@call_will_return_but_has_loop()
-; CHECK-NEXT:    br label [[LABEL1:%.*]]
-; CHECK:       label1:
-; CHECK-NEXT:    tail call void @will_return()
-; CHECK-NEXT:    br label [[LABEL2:%.*]]
-; CHECK:       label2:
-; CHECK-NEXT:    br label [[LABEL1]]
+; IS__CGSCC____-LABEL: define {{[^@]+}}@call_will_return_but_has_loop()
+; IS__CGSCC____-NEXT:    br label [[LABEL1:%.*]]
+; IS__CGSCC____:       label1:
+; IS__CGSCC____-NEXT:    tail call void @will_return()
+; IS__CGSCC____-NEXT:    br label [[LABEL2:%.*]]
+; IS__CGSCC____:       label2:
+; IS__CGSCC____-NEXT:    br label [[LABEL1]]
 ;
   br label %label1
 label1:
@@ -378,8 +458,8 @@ label2:
 ; CHECK-NEXT: declare i1 @maybe_raise_exception()
 declare i1 @maybe_raise_exception() #1 willreturn
 
-; CHECK: Function Attrs: nounwind willreturn
 define void @invoke_test() personality i32 (...)* @__gxx_personality_v0 {
+; IS__TUNIT____: Function Attrs: nounwind willreturn
 ; IS__TUNIT____-LABEL: define {{[^@]+}}@invoke_test() #12 personality i32 (...)* @__gxx_personality_v0
 ; IS__TUNIT____-NEXT:    [[TMP1:%.*]] = invoke i1 @maybe_raise_exception()
 ; IS__TUNIT____-NEXT:    to label [[N:%.*]] unwind label [[F:%.*]]
@@ -390,6 +470,7 @@ define void @invoke_test() personality i32 (...)* @__gxx_personality_v0 {
 ; IS__TUNIT____-NEXT:    catch i8* null
 ; IS__TUNIT____-NEXT:    ret void
 ;
+; IS__CGSCC____: Function Attrs: nounwind willreturn
 ; IS__CGSCC____-LABEL: define {{[^@]+}}@invoke_test() #14 personality i32 (...)* @__gxx_personality_v0
 ; IS__CGSCC____-NEXT:    [[TMP1:%.*]] = invoke i1 @maybe_raise_exception()
 ; IS__CGSCC____-NEXT:    to label [[N:%.*]] unwind label [[F:%.*]]
@@ -423,38 +504,70 @@ declare i32 @__gxx_personality_v0(...)
 ;    return ans;
 ; }
 
-; IS__TUNIT____: Function Attrs: argmemonly nofree noinline nosync nounwind readonly uwtable
-; IS__CGSCC____: Function Attrs: argmemonly nofree noinline norecurse nosync nounwind readonly uwtable
 define i32 @loop_constant_trip_count(i32* nocapture readonly %0) #0 {
-; IS________OPM-LABEL: define {{[^@]+}}@loop_constant_trip_count
-; IS________OPM-SAME: (i32* nocapture nofree readonly [[TMP0:%.*]])
-; IS________OPM-NEXT:    br label [[TMP3:%.*]]
-; IS________OPM:       2:
-; IS________OPM-NEXT:    ret i32 [[TMP8:%.*]]
-; IS________OPM:       3:
-; IS________OPM-NEXT:    [[TMP4:%.*]] = phi i64 [ 0, [[TMP1:%.*]] ], [ [[TMP9:%.*]], [[TMP3]] ]
-; IS________OPM-NEXT:    [[TMP5:%.*]] = phi i32 [ 0, [[TMP1]] ], [ [[TMP8]], [[TMP3]] ]
-; IS________OPM-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 [[TMP4]]
-; IS________OPM-NEXT:    [[TMP7:%.*]] = load i32, i32* [[TMP6]], align 4
-; IS________OPM-NEXT:    [[TMP8]] = add nsw i32 [[TMP7]], [[TMP5]]
-; IS________OPM-NEXT:    [[TMP9]] = add nuw nsw i64 [[TMP4]], 1
-; IS________OPM-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 10
-; IS________OPM-NEXT:    br i1 [[TMP10]], label [[TMP2:%.*]], label [[TMP3]]
+; IS__TUNIT_OPM: Function Attrs: argmemonly nofree noinline nosync nounwind readonly uwtable
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@loop_constant_trip_count
+; IS__TUNIT_OPM-SAME: (i32* nocapture nofree readonly [[TMP0:%.*]])
+; IS__TUNIT_OPM-NEXT:    br label [[TMP3:%.*]]
+; IS__TUNIT_OPM:       2:
+; IS__TUNIT_OPM-NEXT:    ret i32 [[TMP8:%.*]]
+; IS__TUNIT_OPM:       3:
+; IS__TUNIT_OPM-NEXT:    [[TMP4:%.*]] = phi i64 [ 0, [[TMP1:%.*]] ], [ [[TMP9:%.*]], [[TMP3]] ]
+; IS__TUNIT_OPM-NEXT:    [[TMP5:%.*]] = phi i32 [ 0, [[TMP1]] ], [ [[TMP8]], [[TMP3]] ]
+; IS__TUNIT_OPM-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 [[TMP4]]
+; IS__TUNIT_OPM-NEXT:    [[TMP7:%.*]] = load i32, i32* [[TMP6]], align 4
+; IS__TUNIT_OPM-NEXT:    [[TMP8]] = add nsw i32 [[TMP7]], [[TMP5]]
+; IS__TUNIT_OPM-NEXT:    [[TMP9]] = add nuw nsw i64 [[TMP4]], 1
+; IS__TUNIT_OPM-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 10
+; IS__TUNIT_OPM-NEXT:    br i1 [[TMP10]], label [[TMP2:%.*]], label [[TMP3]]
+;
+; IS__TUNIT_NPM: Function Attrs: argmemonly nofree noinline nosync nounwind readonly uwtable willreturn
+; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@loop_constant_trip_count
+; IS__TUNIT_NPM-SAME: (i32* nocapture nofree nonnull readonly dereferenceable(4) [[TMP0:%.*]])
+; IS__TUNIT_NPM-NEXT:    br label [[TMP3:%.*]]
+; IS__TUNIT_NPM:       2:
+; IS__TUNIT_NPM-NEXT:    ret i32 [[TMP8:%.*]]
+; IS__TUNIT_NPM:       3:
+; IS__TUNIT_NPM-NEXT:    [[TMP4:%.*]] = phi i64 [ 0, [[TMP1:%.*]] ], [ [[TMP9:%.*]], [[TMP3]] ]
+; IS__TUNIT_NPM-NEXT:    [[TMP5:%.*]] = phi i32 [ 0, [[TMP1]] ], [ [[TMP8]], [[TMP3]] ]
+; IS__TUNIT_NPM-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 [[TMP4]]
+; IS__TUNIT_NPM-NEXT:    [[TMP7:%.*]] = load i32, i32* [[TMP6]], align 4
+; IS__TUNIT_NPM-NEXT:    [[TMP8]] = add nsw i32 [[TMP7]], [[TMP5]]
+; IS__TUNIT_NPM-NEXT:    [[TMP9]] = add nuw nsw i64 [[TMP4]], 1
+; IS__TUNIT_NPM-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 10
+; IS__TUNIT_NPM-NEXT:    br i1 [[TMP10]], label [[TMP2:%.*]], label [[TMP3]]
+;
+; IS__CGSCC_OPM: Function Attrs: argmemonly nofree noinline norecurse nosync nounwind readonly uwtable
+; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@loop_constant_trip_count
+; IS__CGSCC_OPM-SAME: (i32* nocapture nofree readonly [[TMP0:%.*]])
+; IS__CGSCC_OPM-NEXT:    br label [[TMP3:%.*]]
+; IS__CGSCC_OPM:       2:
+; IS__CGSCC_OPM-NEXT:    ret i32 [[TMP8:%.*]]
+; IS__CGSCC_OPM:       3:
+; IS__CGSCC_OPM-NEXT:    [[TMP4:%.*]] = phi i64 [ 0, [[TMP1:%.*]] ], [ [[TMP9:%.*]], [[TMP3]] ]
+; IS__CGSCC_OPM-NEXT:    [[TMP5:%.*]] = phi i32 [ 0, [[TMP1]] ], [ [[TMP8]], [[TMP3]] ]
+; IS__CGSCC_OPM-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 [[TMP4]]
+; IS__CGSCC_OPM-NEXT:    [[TMP7:%.*]] = load i32, i32* [[TMP6]], align 4
+; IS__CGSCC_OPM-NEXT:    [[TMP8]] = add nsw i32 [[TMP7]], [[TMP5]]
+; IS__CGSCC_OPM-NEXT:    [[TMP9]] = add nuw nsw i64 [[TMP4]], 1
+; IS__CGSCC_OPM-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 10
+; IS__CGSCC_OPM-NEXT:    br i1 [[TMP10]], label [[TMP2:%.*]], label [[TMP3]]
 ;
-; IS________NPM-LABEL: define {{[^@]+}}@loop_constant_trip_count
-; IS________NPM-SAME: (i32* nocapture nofree nonnull readonly dereferenceable(4) [[TMP0:%.*]])
-; IS________NPM-NEXT:    br label [[TMP3:%.*]]
-; IS________NPM:       2:
-; IS________NPM-NEXT:    ret i32 [[TMP8:%.*]]
-; IS________NPM:       3:
-; IS________NPM-NEXT:    [[TMP4:%.*]] = phi i64 [ 0, [[TMP1:%.*]] ], [ [[TMP9:%.*]], [[TMP3]] ]
-; IS________NPM-NEXT:    [[TMP5:%.*]] = phi i32 [ 0, [[TMP1]] ], [ [[TMP8]], [[TMP3]] ]
-; IS________NPM-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 [[TMP4]]
-; IS________NPM-NEXT:    [[TMP7:%.*]] = load i32, i32* [[TMP6]], align 4
-; IS________NPM-NEXT:    [[TMP8]] = add nsw i32 [[TMP7]], [[TMP5]]
-; IS________NPM-NEXT:    [[TMP9]] = add nuw nsw i64 [[TMP4]], 1
-; IS________NPM-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 10
-; IS________NPM-NEXT:    br i1 [[TMP10]], label [[TMP2:%.*]], label [[TMP3]]
+; IS__CGSCC_NPM: Function Attrs: argmemonly nofree noinline norecurse nosync nounwind readonly uwtable willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@loop_constant_trip_count
+; IS__CGSCC_NPM-SAME: (i32* nocapture nofree nonnull readonly dereferenceable(4) [[TMP0:%.*]])
+; IS__CGSCC_NPM-NEXT:    br label [[TMP3:%.*]]
+; IS__CGSCC_NPM:       2:
+; IS__CGSCC_NPM-NEXT:    ret i32 [[TMP8:%.*]]
+; IS__CGSCC_NPM:       3:
+; IS__CGSCC_NPM-NEXT:    [[TMP4:%.*]] = phi i64 [ 0, [[TMP1:%.*]] ], [ [[TMP9:%.*]], [[TMP3]] ]
+; IS__CGSCC_NPM-NEXT:    [[TMP5:%.*]] = phi i32 [ 0, [[TMP1]] ], [ [[TMP8]], [[TMP3]] ]
+; IS__CGSCC_NPM-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 [[TMP4]]
+; IS__CGSCC_NPM-NEXT:    [[TMP7:%.*]] = load i32, i32* [[TMP6]], align 4
+; IS__CGSCC_NPM-NEXT:    [[TMP8]] = add nsw i32 [[TMP7]], [[TMP5]]
+; IS__CGSCC_NPM-NEXT:    [[TMP9]] = add nuw nsw i64 [[TMP4]], 1
+; IS__CGSCC_NPM-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 10
+; IS__CGSCC_NPM-NEXT:    br i1 [[TMP10]], label [[TMP2:%.*]], label [[TMP3]]
 ;
   br label %3
 
@@ -483,27 +596,44 @@ define i32 @loop_constant_trip_count(i32* nocapture readonly %0) #0 {
 ;     }
 ;     return ans;
 ; }
+define i32 @loop_trip_count_unbound(i32 %0, i32 %1, i32* nocapture readonly %2, i32 %3) local_unnamed_addr #0 {
 ; IS__TUNIT____: Function Attrs: argmemonly nofree noinline nosync nounwind readonly uwtable
+; IS__TUNIT____-LABEL: define {{[^@]+}}@loop_trip_count_unbound
+; IS__TUNIT____-SAME: (i32 [[TMP0:%.*]], i32 [[TMP1:%.*]], i32* nocapture nofree readonly [[TMP2:%.*]], i32 [[TMP3:%.*]]) local_unnamed_addr
+; IS__TUNIT____-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[TMP0]], [[TMP1]]
+; IS__TUNIT____-NEXT:    br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP8:%.*]]
+; IS__TUNIT____:       6:
+; IS__TUNIT____-NEXT:    [[TMP7:%.*]] = phi i32 [ 0, [[TMP4:%.*]] ], [ [[TMP14:%.*]], [[TMP8]] ]
+; IS__TUNIT____-NEXT:    ret i32 [[TMP7]]
+; IS__TUNIT____:       8:
+; IS__TUNIT____-NEXT:    [[TMP9:%.*]] = phi i32 [ [[TMP15:%.*]], [[TMP8]] ], [ [[TMP0]], [[TMP4]] ]
+; IS__TUNIT____-NEXT:    [[TMP10:%.*]] = phi i32 [ [[TMP14]], [[TMP8]] ], [ 0, [[TMP4]] ]
+; IS__TUNIT____-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP9]] to i64
+; IS__TUNIT____-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 [[TMP11]]
+; IS__TUNIT____-NEXT:    [[TMP13:%.*]] = load i32, i32* [[TMP12]], align 4
+; IS__TUNIT____-NEXT:    [[TMP14]] = add nsw i32 [[TMP13]], [[TMP10]]
+; IS__TUNIT____-NEXT:    [[TMP15]] = add i32 [[TMP9]], [[TMP3]]
+; IS__TUNIT____-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[TMP15]], [[TMP1]]
+; IS__TUNIT____-NEXT:    br i1 [[TMP16]], label [[TMP6]], label [[TMP8]]
+;
 ; IS__CGSCC____: Function Attrs: argmemonly nofree noinline norecurse nosync nounwind readonly uwtable
-; CHECK-NOT: willreturn
-define i32 @loop_trip_count_unbound(i32 %0, i32 %1, i32* nocapture readonly %2, i32 %3) local_unnamed_addr #0 {
-; CHECK-LABEL: define {{[^@]+}}@loop_trip_count_unbound
-; CHECK-SAME: (i32 [[TMP0:%.*]], i32 [[TMP1:%.*]], i32* nocapture nofree readonly [[TMP2:%.*]], i32 [[TMP3:%.*]]) local_unnamed_addr
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[TMP0]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP8:%.*]]
-; CHECK:       6:
-; CHECK-NEXT:    [[TMP7:%.*]] = phi i32 [ 0, [[TMP4:%.*]] ], [ [[TMP14:%.*]], [[TMP8]] ]
-; CHECK-NEXT:    ret i32 [[TMP7]]
-; CHECK:       8:
-; CHECK-NEXT:    [[TMP9:%.*]] = phi i32 [ [[TMP15:%.*]], [[TMP8]] ], [ [[TMP0]], [[TMP4]] ]
-; CHECK-NEXT:    [[TMP10:%.*]] = phi i32 [ [[TMP14]], [[TMP8]] ], [ 0, [[TMP4]] ]
-; CHECK-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP9]] to i64
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 [[TMP11]]
-; CHECK-NEXT:    [[TMP13:%.*]] = load i32, i32* [[TMP12]], align 4
-; CHECK-NEXT:    [[TMP14]] = add nsw i32 [[TMP13]], [[TMP10]]
-; CHECK-NEXT:    [[TMP15]] = add i32 [[TMP9]], [[TMP3]]
-; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[TMP15]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[TMP16]], label [[TMP6]], label [[TMP8]]
+; IS__CGSCC____-LABEL: define {{[^@]+}}@loop_trip_count_unbound
+; IS__CGSCC____-SAME: (i32 [[TMP0:%.*]], i32 [[TMP1:%.*]], i32* nocapture nofree readonly [[TMP2:%.*]], i32 [[TMP3:%.*]]) local_unnamed_addr
+; IS__CGSCC____-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[TMP0]], [[TMP1]]
+; IS__CGSCC____-NEXT:    br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP8:%.*]]
+; IS__CGSCC____:       6:
+; IS__CGSCC____-NEXT:    [[TMP7:%.*]] = phi i32 [ 0, [[TMP4:%.*]] ], [ [[TMP14:%.*]], [[TMP8]] ]
+; IS__CGSCC____-NEXT:    ret i32 [[TMP7]]
+; IS__CGSCC____:       8:
+; IS__CGSCC____-NEXT:    [[TMP9:%.*]] = phi i32 [ [[TMP15:%.*]], [[TMP8]] ], [ [[TMP0]], [[TMP4]] ]
+; IS__CGSCC____-NEXT:    [[TMP10:%.*]] = phi i32 [ [[TMP14]], [[TMP8]] ], [ 0, [[TMP4]] ]
+; IS__CGSCC____-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP9]] to i64
+; IS__CGSCC____-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 [[TMP11]]
+; IS__CGSCC____-NEXT:    [[TMP13:%.*]] = load i32, i32* [[TMP12]], align 4
+; IS__CGSCC____-NEXT:    [[TMP14]] = add nsw i32 [[TMP13]], [[TMP10]]
+; IS__CGSCC____-NEXT:    [[TMP15]] = add i32 [[TMP9]], [[TMP3]]
+; IS__CGSCC____-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[TMP15]], [[TMP1]]
+; IS__CGSCC____-NEXT:    br i1 [[TMP16]], label [[TMP6]], label [[TMP8]]
 ;
   %5 = icmp eq i32 %0, %1
   br i1 %5, label %6, label %8
@@ -536,28 +666,90 @@ define i32 @loop_trip_count_unbound(i32 %0, i32 %1, i32* nocapture readonly %2,
 ;  }
 
 
-; IS__TUNIT____: Function Attrs: argmemonly nofree noinline nosync nounwind readonly uwtable
-; IS__CGSCC____: Function Attrs: argmemonly nofree noinline norecurse nosync nounwind readonly uwtable
 define i32 @loop_trip_dec(i32 %0, i32* nocapture readonly %1) local_unnamed_addr #0 {
-; CHECK-LABEL: define {{[^@]+}}@loop_trip_dec
-; CHECK-SAME: (i32 [[TMP0:%.*]], i32* nocapture nofree readonly [[TMP1:%.*]]) local_unnamed_addr
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i32 [[TMP0]], -1
-; CHECK-NEXT:    br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP14:%.*]]
-; CHECK:       4:
-; CHECK-NEXT:    [[TMP5:%.*]] = sext i32 [[TMP0]] to i64
-; CHECK-NEXT:    br label [[TMP6:%.*]]
-; CHECK:       6:
-; CHECK-NEXT:    [[TMP7:%.*]] = phi i64 [ [[TMP5]], [[TMP4]] ], [ [[TMP12:%.*]], [[TMP6]] ]
-; CHECK-NEXT:    [[TMP8:%.*]] = phi i32 [ 0, [[TMP4]] ], [ [[TMP11:%.*]], [[TMP6]] ]
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 [[TMP7]]
-; CHECK-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP9]], align 4
-; CHECK-NEXT:    [[TMP11]] = add nsw i32 [[TMP10]], [[TMP8]]
-; CHECK-NEXT:    [[TMP12]] = add nsw i64 [[TMP7]], -1
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp sgt i64 [[TMP7]], 0
-; CHECK-NEXT:    br i1 [[TMP13]], label [[TMP6]], label [[TMP14]]
-; CHECK:       14:
-; CHECK-NEXT:    [[TMP15:%.*]] = phi i32 [ 0, [[TMP2:%.*]] ], [ [[TMP11]], [[TMP6]] ]
-; CHECK-NEXT:    ret i32 [[TMP15]]
+; IS__TUNIT_OPM: Function Attrs: argmemonly nofree noinline nosync nounwind readonly uwtable
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@loop_trip_dec
+; IS__TUNIT_OPM-SAME: (i32 [[TMP0:%.*]], i32* nocapture nofree readonly [[TMP1:%.*]]) local_unnamed_addr
+; IS__TUNIT_OPM-NEXT:    [[TMP3:%.*]] = icmp sgt i32 [[TMP0]], -1
+; IS__TUNIT_OPM-NEXT:    br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP14:%.*]]
+; IS__TUNIT_OPM:       4:
+; IS__TUNIT_OPM-NEXT:    [[TMP5:%.*]] = sext i32 [[TMP0]] to i64
+; IS__TUNIT_OPM-NEXT:    br label [[TMP6:%.*]]
+; IS__TUNIT_OPM:       6:
+; IS__TUNIT_OPM-NEXT:    [[TMP7:%.*]] = phi i64 [ [[TMP5]], [[TMP4]] ], [ [[TMP12:%.*]], [[TMP6]] ]
+; IS__TUNIT_OPM-NEXT:    [[TMP8:%.*]] = phi i32 [ 0, [[TMP4]] ], [ [[TMP11:%.*]], [[TMP6]] ]
+; IS__TUNIT_OPM-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 [[TMP7]]
+; IS__TUNIT_OPM-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP9]], align 4
+; IS__TUNIT_OPM-NEXT:    [[TMP11]] = add nsw i32 [[TMP10]], [[TMP8]]
+; IS__TUNIT_OPM-NEXT:    [[TMP12]] = add nsw i64 [[TMP7]], -1
+; IS__TUNIT_OPM-NEXT:    [[TMP13:%.*]] = icmp sgt i64 [[TMP7]], 0
+; IS__TUNIT_OPM-NEXT:    br i1 [[TMP13]], label [[TMP6]], label [[TMP14]]
+; IS__TUNIT_OPM:       14:
+; IS__TUNIT_OPM-NEXT:    [[TMP15:%.*]] = phi i32 [ 0, [[TMP2:%.*]] ], [ [[TMP11]], [[TMP6]] ]
+; IS__TUNIT_OPM-NEXT:    ret i32 [[TMP15]]
+;
+; IS__TUNIT_NPM: Function Attrs: argmemonly nofree noinline nosync nounwind readonly uwtable willreturn
+; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@loop_trip_dec
+; IS__TUNIT_NPM-SAME: (i32 [[TMP0:%.*]], i32* nocapture nofree readonly [[TMP1:%.*]]) local_unnamed_addr
+; IS__TUNIT_NPM-NEXT:    [[TMP3:%.*]] = icmp sgt i32 [[TMP0]], -1
+; IS__TUNIT_NPM-NEXT:    br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP14:%.*]]
+; IS__TUNIT_NPM:       4:
+; IS__TUNIT_NPM-NEXT:    [[TMP5:%.*]] = sext i32 [[TMP0]] to i64
+; IS__TUNIT_NPM-NEXT:    br label [[TMP6:%.*]]
+; IS__TUNIT_NPM:       6:
+; IS__TUNIT_NPM-NEXT:    [[TMP7:%.*]] = phi i64 [ [[TMP5]], [[TMP4]] ], [ [[TMP12:%.*]], [[TMP6]] ]
+; IS__TUNIT_NPM-NEXT:    [[TMP8:%.*]] = phi i32 [ 0, [[TMP4]] ], [ [[TMP11:%.*]], [[TMP6]] ]
+; IS__TUNIT_NPM-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 [[TMP7]]
+; IS__TUNIT_NPM-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP9]], align 4
+; IS__TUNIT_NPM-NEXT:    [[TMP11]] = add nsw i32 [[TMP10]], [[TMP8]]
+; IS__TUNIT_NPM-NEXT:    [[TMP12]] = add nsw i64 [[TMP7]], -1
+; IS__TUNIT_NPM-NEXT:    [[TMP13:%.*]] = icmp sgt i64 [[TMP7]], 0
+; IS__TUNIT_NPM-NEXT:    br i1 [[TMP13]], label [[TMP6]], label [[TMP14]]
+; IS__TUNIT_NPM:       14:
+; IS__TUNIT_NPM-NEXT:    [[TMP15:%.*]] = phi i32 [ 0, [[TMP2:%.*]] ], [ [[TMP11]], [[TMP6]] ]
+; IS__TUNIT_NPM-NEXT:    ret i32 [[TMP15]]
+;
+; IS__CGSCC_OPM: Function Attrs: argmemonly nofree noinline norecurse nosync nounwind readonly uwtable
+; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@loop_trip_dec
+; IS__CGSCC_OPM-SAME: (i32 [[TMP0:%.*]], i32* nocapture nofree readonly [[TMP1:%.*]]) local_unnamed_addr
+; IS__CGSCC_OPM-NEXT:    [[TMP3:%.*]] = icmp sgt i32 [[TMP0]], -1
+; IS__CGSCC_OPM-NEXT:    br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP14:%.*]]
+; IS__CGSCC_OPM:       4:
+; IS__CGSCC_OPM-NEXT:    [[TMP5:%.*]] = sext i32 [[TMP0]] to i64
+; IS__CGSCC_OPM-NEXT:    br label [[TMP6:%.*]]
+; IS__CGSCC_OPM:       6:
+; IS__CGSCC_OPM-NEXT:    [[TMP7:%.*]] = phi i64 [ [[TMP5]], [[TMP4]] ], [ [[TMP12:%.*]], [[TMP6]] ]
+; IS__CGSCC_OPM-NEXT:    [[TMP8:%.*]] = phi i32 [ 0, [[TMP4]] ], [ [[TMP11:%.*]], [[TMP6]] ]
+; IS__CGSCC_OPM-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 [[TMP7]]
+; IS__CGSCC_OPM-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP9]], align 4
+; IS__CGSCC_OPM-NEXT:    [[TMP11]] = add nsw i32 [[TMP10]], [[TMP8]]
+; IS__CGSCC_OPM-NEXT:    [[TMP12]] = add nsw i64 [[TMP7]], -1
+; IS__CGSCC_OPM-NEXT:    [[TMP13:%.*]] = icmp sgt i64 [[TMP7]], 0
+; IS__CGSCC_OPM-NEXT:    br i1 [[TMP13]], label [[TMP6]], label [[TMP14]]
+; IS__CGSCC_OPM:       14:
+; IS__CGSCC_OPM-NEXT:    [[TMP15:%.*]] = phi i32 [ 0, [[TMP2:%.*]] ], [ [[TMP11]], [[TMP6]] ]
+; IS__CGSCC_OPM-NEXT:    ret i32 [[TMP15]]
+;
+; IS__CGSCC_NPM: Function Attrs: argmemonly nofree noinline norecurse nosync nounwind readonly uwtable willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@loop_trip_dec
+; IS__CGSCC_NPM-SAME: (i32 [[TMP0:%.*]], i32* nocapture nofree readonly [[TMP1:%.*]]) local_unnamed_addr
+; IS__CGSCC_NPM-NEXT:    [[TMP3:%.*]] = icmp sgt i32 [[TMP0]], -1
+; IS__CGSCC_NPM-NEXT:    br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP14:%.*]]
+; IS__CGSCC_NPM:       4:
+; IS__CGSCC_NPM-NEXT:    [[TMP5:%.*]] = sext i32 [[TMP0]] to i64
+; IS__CGSCC_NPM-NEXT:    br label [[TMP6:%.*]]
+; IS__CGSCC_NPM:       6:
+; IS__CGSCC_NPM-NEXT:    [[TMP7:%.*]] = phi i64 [ [[TMP5]], [[TMP4]] ], [ [[TMP12:%.*]], [[TMP6]] ]
+; IS__CGSCC_NPM-NEXT:    [[TMP8:%.*]] = phi i32 [ 0, [[TMP4]] ], [ [[TMP11:%.*]], [[TMP6]] ]
+; IS__CGSCC_NPM-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 [[TMP7]]
+; IS__CGSCC_NPM-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP9]], align 4
+; IS__CGSCC_NPM-NEXT:    [[TMP11]] = add nsw i32 [[TMP10]], [[TMP8]]
+; IS__CGSCC_NPM-NEXT:    [[TMP12]] = add nsw i64 [[TMP7]], -1
+; IS__CGSCC_NPM-NEXT:    [[TMP13:%.*]] = icmp sgt i64 [[TMP7]], 0
+; IS__CGSCC_NPM-NEXT:    br i1 [[TMP13]], label [[TMP6]], label [[TMP14]]
+; IS__CGSCC_NPM:       14:
+; IS__CGSCC_NPM-NEXT:    [[TMP15:%.*]] = phi i32 [ 0, [[TMP2:%.*]] ], [ [[TMP11]], [[TMP6]] ]
+; IS__CGSCC_NPM-NEXT:    ret i32 [[TMP15]]
 ;
   %3 = icmp sgt i32 %0, -1
   br i1 %3, label %4, label %14
@@ -584,17 +776,26 @@ define i32 @loop_trip_dec(i32 %0, i32* nocapture readonly %1) local_unnamed_addr
 ; TEST 14 (positive case)
 ; multiple return
 
+define i32 @multiple_return(i32 %a) #0 {
 ; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@multiple_return
+; IS__TUNIT____-SAME: (i32 [[A:%.*]])
+; IS__TUNIT____-NEXT:    [[B:%.*]] = icmp eq i32 [[A]], 0
+; IS__TUNIT____-NEXT:    br i1 [[B]], label [[T:%.*]], label [[F:%.*]]
+; IS__TUNIT____:       t:
+; IS__TUNIT____-NEXT:    ret i32 1
+; IS__TUNIT____:       f:
+; IS__TUNIT____-NEXT:    ret i32 0
+;
 ; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
-define i32 @multiple_return(i32 %a) #0 {
-; CHECK-LABEL: define {{[^@]+}}@multiple_return
-; CHECK-SAME: (i32 [[A:%.*]])
-; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[A]], 0
-; CHECK-NEXT:    br i1 [[B]], label [[T:%.*]], label [[F:%.*]]
-; CHECK:       t:
-; CHECK-NEXT:    ret i32 1
-; CHECK:       f:
-; CHECK-NEXT:    ret i32 0
+; IS__CGSCC____-LABEL: define {{[^@]+}}@multiple_return
+; IS__CGSCC____-SAME: (i32 [[A:%.*]])
+; IS__CGSCC____-NEXT:    [[B:%.*]] = icmp eq i32 [[A]], 0
+; IS__CGSCC____-NEXT:    br i1 [[B]], label [[T:%.*]], label [[F:%.*]]
+; IS__CGSCC____:       t:
+; IS__CGSCC____-NEXT:    ret i32 1
+; IS__CGSCC____:       f:
+; IS__CGSCC____-NEXT:    ret i32 0
 ;
   %b =  icmp eq i32 %a, 0
   br i1 %b, label %t, label %f
@@ -609,14 +810,20 @@ f:
 ; unreachable exit
 
 ; 15.1 (positive case)
-; IS__TUNIT____: Function Attrs: noinline nounwind uwtable
-; IS__CGSCC____: Function Attrs: noinline norecurse nounwind uwtable willreturn
 define void @unreachable_exit_positive1() #0 {
-; CHECK-LABEL: define {{[^@]+}}@unreachable_exit_positive1()
-; CHECK-NEXT:    tail call void @will_return()
-; CHECK-NEXT:    ret void
-; CHECK:       unreachable_label:
-; CHECK-NEXT:    unreachable
+; IS__TUNIT____: Function Attrs: noinline nounwind uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@unreachable_exit_positive1()
+; IS__TUNIT____-NEXT:    tail call void @will_return()
+; IS__TUNIT____-NEXT:    ret void
+; IS__TUNIT____:       unreachable_label:
+; IS__TUNIT____-NEXT:    unreachable
+;
+; IS__CGSCC____: Function Attrs: noinline norecurse nounwind uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@unreachable_exit_positive1()
+; IS__CGSCC____-NEXT:    tail call void @will_return()
+; IS__CGSCC____-NEXT:    ret void
+; IS__CGSCC____:       unreachable_label:
+; IS__CGSCC____-NEXT:    unreachable
 ;
   tail call void @will_return()
   ret void
@@ -626,25 +833,78 @@ unreachable_label:
   unreachable
 }
 
-; IS__TUNIT____: Function Attrs: nofree noinline nosync nounwind readnone uwtable
-; IS__CGSCC____: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
 define i32 @unreachable_exit_positive2(i32) local_unnamed_addr #0 {
-; CHECK-LABEL: define {{[^@]+}}@unreachable_exit_positive2
-; CHECK-SAME: (i32 [[TMP0:%.*]]) local_unnamed_addr
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP0]], 1
-; CHECK-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP5:%.*]]
-; CHECK:       3:
-; CHECK-NEXT:    [[TMP4:%.*]] = phi i32 [ 1, [[TMP1:%.*]] ], [ [[TMP8:%.*]], [[TMP5]] ]
-; CHECK-NEXT:    ret i32 [[TMP4]]
-; CHECK:       5:
-; CHECK-NEXT:    [[TMP6:%.*]] = phi i32 [ [[TMP9:%.*]], [[TMP5]] ], [ 1, [[TMP1]] ]
-; CHECK-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP8]], [[TMP5]] ], [ 1, [[TMP1]] ]
-; CHECK-NEXT:    [[TMP8]] = mul nsw i32 [[TMP6]], [[TMP7]]
-; CHECK-NEXT:    [[TMP9]] = add nuw nsw i32 [[TMP6]], 1
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[TMP6]], [[TMP0]]
-; CHECK-NEXT:    br i1 [[TMP10]], label [[TMP3]], label [[TMP5]]
-; CHECK:       unreachable_label:
-; CHECK-NEXT:    unreachable
+; IS__TUNIT_OPM: Function Attrs: nofree noinline nosync nounwind readnone uwtable
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@unreachable_exit_positive2
+; IS__TUNIT_OPM-SAME: (i32 [[TMP0:%.*]]) local_unnamed_addr
+; IS__TUNIT_OPM-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP0]], 1
+; IS__TUNIT_OPM-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP5:%.*]]
+; IS__TUNIT_OPM:       3:
+; IS__TUNIT_OPM-NEXT:    [[TMP4:%.*]] = phi i32 [ 1, [[TMP1:%.*]] ], [ [[TMP8:%.*]], [[TMP5]] ]
+; IS__TUNIT_OPM-NEXT:    ret i32 [[TMP4]]
+; IS__TUNIT_OPM:       5:
+; IS__TUNIT_OPM-NEXT:    [[TMP6:%.*]] = phi i32 [ [[TMP9:%.*]], [[TMP5]] ], [ 1, [[TMP1]] ]
+; IS__TUNIT_OPM-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP8]], [[TMP5]] ], [ 1, [[TMP1]] ]
+; IS__TUNIT_OPM-NEXT:    [[TMP8]] = mul nsw i32 [[TMP6]], [[TMP7]]
+; IS__TUNIT_OPM-NEXT:    [[TMP9]] = add nuw nsw i32 [[TMP6]], 1
+; IS__TUNIT_OPM-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[TMP6]], [[TMP0]]
+; IS__TUNIT_OPM-NEXT:    br i1 [[TMP10]], label [[TMP3]], label [[TMP5]]
+; IS__TUNIT_OPM:       unreachable_label:
+; IS__TUNIT_OPM-NEXT:    unreachable
+;
+; IS__TUNIT_NPM: Function Attrs: nofree noinline nosync nounwind readnone uwtable willreturn
+; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@unreachable_exit_positive2
+; IS__TUNIT_NPM-SAME: (i32 [[TMP0:%.*]]) local_unnamed_addr
+; IS__TUNIT_NPM-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP0]], 1
+; IS__TUNIT_NPM-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP5:%.*]]
+; IS__TUNIT_NPM:       3:
+; IS__TUNIT_NPM-NEXT:    [[TMP4:%.*]] = phi i32 [ 1, [[TMP1:%.*]] ], [ [[TMP8:%.*]], [[TMP5]] ]
+; IS__TUNIT_NPM-NEXT:    ret i32 [[TMP4]]
+; IS__TUNIT_NPM:       5:
+; IS__TUNIT_NPM-NEXT:    [[TMP6:%.*]] = phi i32 [ [[TMP9:%.*]], [[TMP5]] ], [ 1, [[TMP1]] ]
+; IS__TUNIT_NPM-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP8]], [[TMP5]] ], [ 1, [[TMP1]] ]
+; IS__TUNIT_NPM-NEXT:    [[TMP8]] = mul nsw i32 [[TMP6]], [[TMP7]]
+; IS__TUNIT_NPM-NEXT:    [[TMP9]] = add nuw nsw i32 [[TMP6]], 1
+; IS__TUNIT_NPM-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[TMP6]], [[TMP0]]
+; IS__TUNIT_NPM-NEXT:    br i1 [[TMP10]], label [[TMP3]], label [[TMP5]]
+; IS__TUNIT_NPM:       unreachable_label:
+; IS__TUNIT_NPM-NEXT:    unreachable
+;
+; IS__CGSCC_OPM: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable
+; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@unreachable_exit_positive2
+; IS__CGSCC_OPM-SAME: (i32 [[TMP0:%.*]]) local_unnamed_addr
+; IS__CGSCC_OPM-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP0]], 1
+; IS__CGSCC_OPM-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP5:%.*]]
+; IS__CGSCC_OPM:       3:
+; IS__CGSCC_OPM-NEXT:    [[TMP4:%.*]] = phi i32 [ 1, [[TMP1:%.*]] ], [ [[TMP8:%.*]], [[TMP5]] ]
+; IS__CGSCC_OPM-NEXT:    ret i32 [[TMP4]]
+; IS__CGSCC_OPM:       5:
+; IS__CGSCC_OPM-NEXT:    [[TMP6:%.*]] = phi i32 [ [[TMP9:%.*]], [[TMP5]] ], [ 1, [[TMP1]] ]
+; IS__CGSCC_OPM-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP8]], [[TMP5]] ], [ 1, [[TMP1]] ]
+; IS__CGSCC_OPM-NEXT:    [[TMP8]] = mul nsw i32 [[TMP6]], [[TMP7]]
+; IS__CGSCC_OPM-NEXT:    [[TMP9]] = add nuw nsw i32 [[TMP6]], 1
+; IS__CGSCC_OPM-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[TMP6]], [[TMP0]]
+; IS__CGSCC_OPM-NEXT:    br i1 [[TMP10]], label [[TMP3]], label [[TMP5]]
+; IS__CGSCC_OPM:       unreachable_label:
+; IS__CGSCC_OPM-NEXT:    unreachable
+;
+; IS__CGSCC_NPM: Function Attrs: nofree noinline norecurse nosync nounwind readnone uwtable willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@unreachable_exit_positive2
+; IS__CGSCC_NPM-SAME: (i32 [[TMP0:%.*]]) local_unnamed_addr
+; IS__CGSCC_NPM-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP0]], 1
+; IS__CGSCC_NPM-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP5:%.*]]
+; IS__CGSCC_NPM:       3:
+; IS__CGSCC_NPM-NEXT:    [[TMP4:%.*]] = phi i32 [ 1, [[TMP1:%.*]] ], [ [[TMP8:%.*]], [[TMP5]] ]
+; IS__CGSCC_NPM-NEXT:    ret i32 [[TMP4]]
+; IS__CGSCC_NPM:       5:
+; IS__CGSCC_NPM-NEXT:    [[TMP6:%.*]] = phi i32 [ [[TMP9:%.*]], [[TMP5]] ], [ 1, [[TMP1]] ]
+; IS__CGSCC_NPM-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP8]], [[TMP5]] ], [ 1, [[TMP1]] ]
+; IS__CGSCC_NPM-NEXT:    [[TMP8]] = mul nsw i32 [[TMP6]], [[TMP7]]
+; IS__CGSCC_NPM-NEXT:    [[TMP9]] = add nuw nsw i32 [[TMP6]], 1
+; IS__CGSCC_NPM-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[TMP6]], [[TMP0]]
+; IS__CGSCC_NPM-NEXT:    br i1 [[TMP10]], label [[TMP3]], label [[TMP5]]
+; IS__CGSCC_NPM:       unreachable_label:
+; IS__CGSCC_NPM-NEXT:    unreachable
 ;
   %2 = icmp slt i32 %0, 1
   br i1 %2, label %3, label %5
@@ -669,9 +929,8 @@ unreachable_label:
 
 ;15.2
 
-; CHECK: Function Attrs: noinline noreturn nounwind uwtable
-; CHECK-NOT: willreturn
 define void @unreachable_exit_negative1() #0 {
+; CHECK: Function Attrs: noinline noreturn nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@unreachable_exit_negative1()
 ; CHECK-NEXT:    tail call void @exit(i32 0)
 ; CHECK-NEXT:    unreachable
@@ -686,18 +945,26 @@ unreachable_label:
   unreachable
 }
 
+define void @unreachable_exit_negative2() #0 {
 ; IS__TUNIT____: Function Attrs: nofree noinline noreturn nosync nounwind readnone uwtable
+; IS__TUNIT____-LABEL: define {{[^@]+}}@unreachable_exit_negative2()
+; IS__TUNIT____-NEXT:    br label [[L1:%.*]]
+; IS__TUNIT____:       L1:
+; IS__TUNIT____-NEXT:    br label [[L2:%.*]]
+; IS__TUNIT____:       L2:
+; IS__TUNIT____-NEXT:    br label [[L1]]
+; IS__TUNIT____:       unreachable_label:
+; IS__TUNIT____-NEXT:    unreachable
+;
 ; IS__CGSCC____: Function Attrs: nofree noinline norecurse noreturn nosync nounwind readnone uwtable
-; CHECK-NOT: willreturn
-define void @unreachable_exit_negative2() #0 {
-; CHECK-LABEL: define {{[^@]+}}@unreachable_exit_negative2()
-; CHECK-NEXT:    br label [[L1:%.*]]
-; CHECK:       L1:
-; CHECK-NEXT:    br label [[L2:%.*]]
-; CHECK:       L2:
-; CHECK-NEXT:    br label [[L1]]
-; CHECK:       unreachable_label:
-; CHECK-NEXT:    unreachable
+; IS__CGSCC____-LABEL: define {{[^@]+}}@unreachable_exit_negative2()
+; IS__CGSCC____-NEXT:    br label [[L1:%.*]]
+; IS__CGSCC____:       L1:
+; IS__CGSCC____-NEXT:    br label [[L2:%.*]]
+; IS__CGSCC____:       L2:
+; IS__CGSCC____-NEXT:    br label [[L1]]
+; IS__CGSCC____:       unreachable_label:
+; IS__CGSCC____-NEXT:    unreachable
 ;
   br label %L1
 L1:
@@ -714,9 +981,8 @@ unreachable_label:
 ; CHECK-NEXT: declare void @llvm.eh.sjlj.longjmp(i8*)
 declare void @llvm.eh.sjlj.longjmp(i8*)
 
-; CHECK: Function Attrs: noinline noreturn nounwind uwtable
-; CHECK-NOT: willreturn
 define void @call_longjmp(i8* nocapture readnone %0) local_unnamed_addr #0 {
+; CHECK: Function Attrs: noinline noreturn nounwind uwtable
 ; CHECK-LABEL: define {{[^@]+}}@call_longjmp
 ; CHECK-SAME: (i8* nocapture readnone [[TMP0:%.*]]) local_unnamed_addr
 ; CHECK-NEXT:    tail call void @llvm.eh.sjlj.longjmp(i8* noalias readnone [[TMP0]])
@@ -737,29 +1003,48 @@ define void @call_longjmp(i8* nocapture readnone %0) local_unnamed_addr #0 {
 ;   return ans;
 ; }
 
+define i32 @infinite_loop_inside_bounded_loop(i32 %n) {
 ; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone
+; IS__TUNIT____-LABEL: define {{[^@]+}}@infinite_loop_inside_bounded_loop
+; IS__TUNIT____-SAME: (i32 [[N:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    br label [[FOR_COND:%.*]]
+; IS__TUNIT____:       for.cond:
+; IS__TUNIT____-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[N]], 0
+; IS__TUNIT____-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; IS__TUNIT____:       for.cond.cleanup:
+; IS__TUNIT____-NEXT:    br label [[FOR_END:%.*]]
+; IS__TUNIT____:       for.body:
+; IS__TUNIT____-NEXT:    br label [[WHILE_COND:%.*]]
+; IS__TUNIT____:       while.cond:
+; IS__TUNIT____-NEXT:    br label [[WHILE_BODY:%.*]]
+; IS__TUNIT____:       while.body:
+; IS__TUNIT____-NEXT:    br label [[WHILE_COND]]
+; IS__TUNIT____:       for.inc:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       for.end:
+; IS__TUNIT____-NEXT:    ret i32 0
+;
 ; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone
-; CHECK-NOT: willreturn
-define i32 @infinite_loop_inside_bounded_loop(i32 %n) {
-; CHECK-LABEL: define {{[^@]+}}@infinite_loop_inside_bounded_loop
-; CHECK-SAME: (i32 [[N:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[FOR_COND:%.*]]
-; CHECK:       for.cond:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[N]], 0
-; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    br label [[FOR_END:%.*]]
-; CHECK:       for.body:
-; CHECK-NEXT:    br label [[WHILE_COND:%.*]]
-; CHECK:       while.cond:
-; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
-; CHECK:       while.body:
-; CHECK-NEXT:    br label [[WHILE_COND]]
-; CHECK:       for.inc:
-; CHECK-NEXT:    unreachable
-; CHECK:       for.end:
-; CHECK-NEXT:    ret i32 0
+; IS__CGSCC____-LABEL: define {{[^@]+}}@infinite_loop_inside_bounded_loop
+; IS__CGSCC____-SAME: (i32 [[N:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    br label [[FOR_COND:%.*]]
+; IS__CGSCC____:       for.cond:
+; IS__CGSCC____-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[N]], 0
+; IS__CGSCC____-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; IS__CGSCC____:       for.cond.cleanup:
+; IS__CGSCC____-NEXT:    br label [[FOR_END:%.*]]
+; IS__CGSCC____:       for.body:
+; IS__CGSCC____-NEXT:    br label [[WHILE_COND:%.*]]
+; IS__CGSCC____:       while.cond:
+; IS__CGSCC____-NEXT:    br label [[WHILE_BODY:%.*]]
+; IS__CGSCC____:       while.body:
+; IS__CGSCC____-NEXT:    br label [[WHILE_COND]]
+; IS__CGSCC____:       for.inc:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       for.end:
+; IS__CGSCC____-NEXT:    ret i32 0
 ;
 entry:
   br label %for.cond
@@ -800,36 +1085,122 @@ for.end:                                          ; preds = %for.cond.cleanup
 ;   return ans;
 ; }
 
-; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone
-; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone
 define i32 @bounded_nested_loops(i32 %n) {
-; CHECK-LABEL: define {{[^@]+}}@bounded_nested_loops
-; CHECK-SAME: (i32 [[N:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[FOR_COND:%.*]]
-; CHECK:       for.cond:
-; CHECK-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC1:%.*]], [[FOR_INC:%.*]] ]
-; CHECK-NEXT:    [[ANS_0:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP:%.*]], [[FOR_INC]] ]
-; CHECK-NEXT:    [[N_ADDR_0:%.*]] = phi i32 [ [[N]], [[ENTRY]] ], [ -1, [[FOR_INC]] ]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_0]], [[N_ADDR_0]]
-; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    [[ANS_0_LCSSA:%.*]] = phi i32 [ [[ANS_0]], [[FOR_COND]] ]
-; CHECK-NEXT:    br label [[FOR_END:%.*]]
-; CHECK:       for.body:
-; CHECK-NEXT:    br label [[WHILE_COND:%.*]]
-; CHECK:       while.cond:
-; CHECK-NEXT:    br i1 true, label [[WHILE_END:%.*]], label [[WHILE_BODY:%.*]]
-; CHECK:       while.body:
-; CHECK-NEXT:    unreachable
-; CHECK:       while.end:
-; CHECK-NEXT:    [[TMP]] = add i32 [[N_ADDR_0]], [[ANS_0]]
-; CHECK-NEXT:    br label [[FOR_INC]]
-; CHECK:       for.inc:
-; CHECK-NEXT:    [[INC1]] = add nuw nsw i32 [[I_0]], 1
-; CHECK-NEXT:    br label [[FOR_COND]]
-; CHECK:       for.end:
-; CHECK-NEXT:    ret i32 [[ANS_0_LCSSA]]
+; IS__TUNIT_OPM: Function Attrs: nofree nosync nounwind readnone
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@bounded_nested_loops
+; IS__TUNIT_OPM-SAME: (i32 [[N:%.*]])
+; IS__TUNIT_OPM-NEXT:  entry:
+; IS__TUNIT_OPM-NEXT:    br label [[FOR_COND:%.*]]
+; IS__TUNIT_OPM:       for.cond:
+; IS__TUNIT_OPM-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC1:%.*]], [[FOR_INC:%.*]] ]
+; IS__TUNIT_OPM-NEXT:    [[ANS_0:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP:%.*]], [[FOR_INC]] ]
+; IS__TUNIT_OPM-NEXT:    [[N_ADDR_0:%.*]] = phi i32 [ [[N]], [[ENTRY]] ], [ -1, [[FOR_INC]] ]
+; IS__TUNIT_OPM-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_0]], [[N_ADDR_0]]
+; IS__TUNIT_OPM-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; IS__TUNIT_OPM:       for.cond.cleanup:
+; IS__TUNIT_OPM-NEXT:    [[ANS_0_LCSSA:%.*]] = phi i32 [ [[ANS_0]], [[FOR_COND]] ]
+; IS__TUNIT_OPM-NEXT:    br label [[FOR_END:%.*]]
+; IS__TUNIT_OPM:       for.body:
+; IS__TUNIT_OPM-NEXT:    br label [[WHILE_COND:%.*]]
+; IS__TUNIT_OPM:       while.cond:
+; IS__TUNIT_OPM-NEXT:    br i1 true, label [[WHILE_END:%.*]], label [[WHILE_BODY:%.*]]
+; IS__TUNIT_OPM:       while.body:
+; IS__TUNIT_OPM-NEXT:    unreachable
+; IS__TUNIT_OPM:       while.end:
+; IS__TUNIT_OPM-NEXT:    [[TMP]] = add i32 [[N_ADDR_0]], [[ANS_0]]
+; IS__TUNIT_OPM-NEXT:    br label [[FOR_INC]]
+; IS__TUNIT_OPM:       for.inc:
+; IS__TUNIT_OPM-NEXT:    [[INC1]] = add nuw nsw i32 [[I_0]], 1
+; IS__TUNIT_OPM-NEXT:    br label [[FOR_COND]]
+; IS__TUNIT_OPM:       for.end:
+; IS__TUNIT_OPM-NEXT:    ret i32 [[ANS_0_LCSSA]]
+;
+; IS__TUNIT_NPM: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@bounded_nested_loops
+; IS__TUNIT_NPM-SAME: (i32 [[N:%.*]])
+; IS__TUNIT_NPM-NEXT:  entry:
+; IS__TUNIT_NPM-NEXT:    br label [[FOR_COND:%.*]]
+; IS__TUNIT_NPM:       for.cond:
+; IS__TUNIT_NPM-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC1:%.*]], [[FOR_INC:%.*]] ]
+; IS__TUNIT_NPM-NEXT:    [[ANS_0:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP:%.*]], [[FOR_INC]] ]
+; IS__TUNIT_NPM-NEXT:    [[N_ADDR_0:%.*]] = phi i32 [ [[N]], [[ENTRY]] ], [ -1, [[FOR_INC]] ]
+; IS__TUNIT_NPM-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_0]], [[N_ADDR_0]]
+; IS__TUNIT_NPM-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; IS__TUNIT_NPM:       for.cond.cleanup:
+; IS__TUNIT_NPM-NEXT:    [[ANS_0_LCSSA:%.*]] = phi i32 [ [[ANS_0]], [[FOR_COND]] ]
+; IS__TUNIT_NPM-NEXT:    br label [[FOR_END:%.*]]
+; IS__TUNIT_NPM:       for.body:
+; IS__TUNIT_NPM-NEXT:    br label [[WHILE_COND:%.*]]
+; IS__TUNIT_NPM:       while.cond:
+; IS__TUNIT_NPM-NEXT:    br i1 true, label [[WHILE_END:%.*]], label [[WHILE_BODY:%.*]]
+; IS__TUNIT_NPM:       while.body:
+; IS__TUNIT_NPM-NEXT:    unreachable
+; IS__TUNIT_NPM:       while.end:
+; IS__TUNIT_NPM-NEXT:    [[TMP]] = add i32 [[N_ADDR_0]], [[ANS_0]]
+; IS__TUNIT_NPM-NEXT:    br label [[FOR_INC]]
+; IS__TUNIT_NPM:       for.inc:
+; IS__TUNIT_NPM-NEXT:    [[INC1]] = add nuw nsw i32 [[I_0]], 1
+; IS__TUNIT_NPM-NEXT:    br label [[FOR_COND]]
+; IS__TUNIT_NPM:       for.end:
+; IS__TUNIT_NPM-NEXT:    ret i32 [[ANS_0_LCSSA]]
+;
+; IS__CGSCC_OPM: Function Attrs: nofree norecurse nosync nounwind readnone
+; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@bounded_nested_loops
+; IS__CGSCC_OPM-SAME: (i32 [[N:%.*]])
+; IS__CGSCC_OPM-NEXT:  entry:
+; IS__CGSCC_OPM-NEXT:    br label [[FOR_COND:%.*]]
+; IS__CGSCC_OPM:       for.cond:
+; IS__CGSCC_OPM-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC1:%.*]], [[FOR_INC:%.*]] ]
+; IS__CGSCC_OPM-NEXT:    [[ANS_0:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP:%.*]], [[FOR_INC]] ]
+; IS__CGSCC_OPM-NEXT:    [[N_ADDR_0:%.*]] = phi i32 [ [[N]], [[ENTRY]] ], [ -1, [[FOR_INC]] ]
+; IS__CGSCC_OPM-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_0]], [[N_ADDR_0]]
+; IS__CGSCC_OPM-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; IS__CGSCC_OPM:       for.cond.cleanup:
+; IS__CGSCC_OPM-NEXT:    [[ANS_0_LCSSA:%.*]] = phi i32 [ [[ANS_0]], [[FOR_COND]] ]
+; IS__CGSCC_OPM-NEXT:    br label [[FOR_END:%.*]]
+; IS__CGSCC_OPM:       for.body:
+; IS__CGSCC_OPM-NEXT:    br label [[WHILE_COND:%.*]]
+; IS__CGSCC_OPM:       while.cond:
+; IS__CGSCC_OPM-NEXT:    br i1 true, label [[WHILE_END:%.*]], label [[WHILE_BODY:%.*]]
+; IS__CGSCC_OPM:       while.body:
+; IS__CGSCC_OPM-NEXT:    unreachable
+; IS__CGSCC_OPM:       while.end:
+; IS__CGSCC_OPM-NEXT:    [[TMP]] = add i32 [[N_ADDR_0]], [[ANS_0]]
+; IS__CGSCC_OPM-NEXT:    br label [[FOR_INC]]
+; IS__CGSCC_OPM:       for.inc:
+; IS__CGSCC_OPM-NEXT:    [[INC1]] = add nuw nsw i32 [[I_0]], 1
+; IS__CGSCC_OPM-NEXT:    br label [[FOR_COND]]
+; IS__CGSCC_OPM:       for.end:
+; IS__CGSCC_OPM-NEXT:    ret i32 [[ANS_0_LCSSA]]
+;
+; IS__CGSCC_NPM: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@bounded_nested_loops
+; IS__CGSCC_NPM-SAME: (i32 [[N:%.*]])
+; IS__CGSCC_NPM-NEXT:  entry:
+; IS__CGSCC_NPM-NEXT:    br label [[FOR_COND:%.*]]
+; IS__CGSCC_NPM:       for.cond:
+; IS__CGSCC_NPM-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC1:%.*]], [[FOR_INC:%.*]] ]
+; IS__CGSCC_NPM-NEXT:    [[ANS_0:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP:%.*]], [[FOR_INC]] ]
+; IS__CGSCC_NPM-NEXT:    [[N_ADDR_0:%.*]] = phi i32 [ [[N]], [[ENTRY]] ], [ -1, [[FOR_INC]] ]
+; IS__CGSCC_NPM-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_0]], [[N_ADDR_0]]
+; IS__CGSCC_NPM-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; IS__CGSCC_NPM:       for.cond.cleanup:
+; IS__CGSCC_NPM-NEXT:    [[ANS_0_LCSSA:%.*]] = phi i32 [ [[ANS_0]], [[FOR_COND]] ]
+; IS__CGSCC_NPM-NEXT:    br label [[FOR_END:%.*]]
+; IS__CGSCC_NPM:       for.body:
+; IS__CGSCC_NPM-NEXT:    br label [[WHILE_COND:%.*]]
+; IS__CGSCC_NPM:       while.cond:
+; IS__CGSCC_NPM-NEXT:    br i1 true, label [[WHILE_END:%.*]], label [[WHILE_BODY:%.*]]
+; IS__CGSCC_NPM:       while.body:
+; IS__CGSCC_NPM-NEXT:    unreachable
+; IS__CGSCC_NPM:       while.end:
+; IS__CGSCC_NPM-NEXT:    [[TMP]] = add i32 [[N_ADDR_0]], [[ANS_0]]
+; IS__CGSCC_NPM-NEXT:    br label [[FOR_INC]]
+; IS__CGSCC_NPM:       for.inc:
+; IS__CGSCC_NPM-NEXT:    [[INC1]] = add nuw nsw i32 [[I_0]], 1
+; IS__CGSCC_NPM-NEXT:    br label [[FOR_COND]]
+; IS__CGSCC_NPM:       for.end:
+; IS__CGSCC_NPM-NEXT:    ret i32 [[ANS_0_LCSSA]]
 ;
 entry:
   br label %for.cond
@@ -878,39 +1249,68 @@ for.end:                                          ; preds = %for.cond.cleanup
 ;   return ans;
 ; }
 
+define i32 @bounded_loop_inside_unbounded_loop(i32 %n) {
 ; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone
+; IS__TUNIT____-LABEL: define {{[^@]+}}@bounded_loop_inside_unbounded_loop
+; IS__TUNIT____-SAME: (i32 [[N:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    br label [[WHILE_COND:%.*]]
+; IS__TUNIT____:       while.cond:
+; IS__TUNIT____-NEXT:    [[ANS_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP2:%.*]], [[FOR_END:%.*]] ]
+; IS__TUNIT____-NEXT:    [[N_ADDR_0:%.*]] = phi i32 [ [[N]], [[ENTRY]] ], [ [[INC:%.*]], [[FOR_END]] ]
+; IS__TUNIT____-NEXT:    [[TMP:%.*]] = icmp sgt i32 [[N_ADDR_0]], -1
+; IS__TUNIT____-NEXT:    [[SMAX:%.*]] = select i1 [[TMP]], i32 [[N_ADDR_0]], i32 -1
+; IS__TUNIT____-NEXT:    [[INC]] = add nsw i32 [[N_ADDR_0]], 1
+; IS__TUNIT____-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[N_ADDR_0]], 0
+; IS__TUNIT____-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END:%.*]], label [[WHILE_BODY:%.*]]
+; IS__TUNIT____:       while.body:
+; IS__TUNIT____-NEXT:    [[TMP1:%.*]] = add i32 [[ANS_0]], 1
+; IS__TUNIT____-NEXT:    br label [[FOR_COND:%.*]]
+; IS__TUNIT____:       for.cond:
+; IS__TUNIT____-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY:%.*]]
+; IS__TUNIT____:       for.cond.cleanup:
+; IS__TUNIT____-NEXT:    [[TMP2]] = add i32 [[TMP1]], [[SMAX]]
+; IS__TUNIT____-NEXT:    br label [[FOR_END]]
+; IS__TUNIT____:       for.body:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       for.inc:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       for.end:
+; IS__TUNIT____-NEXT:    br label [[WHILE_COND]]
+; IS__TUNIT____:       while.end:
+; IS__TUNIT____-NEXT:    [[ANS_0_LCSSA:%.*]] = phi i32 [ [[ANS_0]], [[WHILE_COND]] ]
+; IS__TUNIT____-NEXT:    ret i32 [[ANS_0_LCSSA]]
+;
 ; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone
-; CHECK-NOT: willreturn
-define i32 @bounded_loop_inside_unbounded_loop(i32 %n) {
-; CHECK-LABEL: define {{[^@]+}}@bounded_loop_inside_unbounded_loop
-; CHECK-SAME: (i32 [[N:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[WHILE_COND:%.*]]
-; CHECK:       while.cond:
-; CHECK-NEXT:    [[ANS_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP2:%.*]], [[FOR_END:%.*]] ]
-; CHECK-NEXT:    [[N_ADDR_0:%.*]] = phi i32 [ [[N]], [[ENTRY]] ], [ [[INC:%.*]], [[FOR_END]] ]
-; CHECK-NEXT:    [[TMP:%.*]] = icmp sgt i32 [[N_ADDR_0]], -1
-; CHECK-NEXT:    [[SMAX:%.*]] = select i1 [[TMP]], i32 [[N_ADDR_0]], i32 -1
-; CHECK-NEXT:    [[INC]] = add nsw i32 [[N_ADDR_0]], 1
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[N_ADDR_0]], 0
-; CHECK-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END:%.*]], label [[WHILE_BODY:%.*]]
-; CHECK:       while.body:
-; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[ANS_0]], 1
-; CHECK-NEXT:    br label [[FOR_COND:%.*]]
-; CHECK:       for.cond:
-; CHECK-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY:%.*]]
-; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    [[TMP2]] = add i32 [[TMP1]], [[SMAX]]
-; CHECK-NEXT:    br label [[FOR_END]]
-; CHECK:       for.body:
-; CHECK-NEXT:    unreachable
-; CHECK:       for.inc:
-; CHECK-NEXT:    unreachable
-; CHECK:       for.end:
-; CHECK-NEXT:    br label [[WHILE_COND]]
-; CHECK:       while.end:
-; CHECK-NEXT:    [[ANS_0_LCSSA:%.*]] = phi i32 [ [[ANS_0]], [[WHILE_COND]] ]
-; CHECK-NEXT:    ret i32 [[ANS_0_LCSSA]]
+; IS__CGSCC____-LABEL: define {{[^@]+}}@bounded_loop_inside_unbounded_loop
+; IS__CGSCC____-SAME: (i32 [[N:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    br label [[WHILE_COND:%.*]]
+; IS__CGSCC____:       while.cond:
+; IS__CGSCC____-NEXT:    [[ANS_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP2:%.*]], [[FOR_END:%.*]] ]
+; IS__CGSCC____-NEXT:    [[N_ADDR_0:%.*]] = phi i32 [ [[N]], [[ENTRY]] ], [ [[INC:%.*]], [[FOR_END]] ]
+; IS__CGSCC____-NEXT:    [[TMP:%.*]] = icmp sgt i32 [[N_ADDR_0]], -1
+; IS__CGSCC____-NEXT:    [[SMAX:%.*]] = select i1 [[TMP]], i32 [[N_ADDR_0]], i32 -1
+; IS__CGSCC____-NEXT:    [[INC]] = add nsw i32 [[N_ADDR_0]], 1
+; IS__CGSCC____-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[N_ADDR_0]], 0
+; IS__CGSCC____-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END:%.*]], label [[WHILE_BODY:%.*]]
+; IS__CGSCC____:       while.body:
+; IS__CGSCC____-NEXT:    [[TMP1:%.*]] = add i32 [[ANS_0]], 1
+; IS__CGSCC____-NEXT:    br label [[FOR_COND:%.*]]
+; IS__CGSCC____:       for.cond:
+; IS__CGSCC____-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY:%.*]]
+; IS__CGSCC____:       for.cond.cleanup:
+; IS__CGSCC____-NEXT:    [[TMP2]] = add i32 [[TMP1]], [[SMAX]]
+; IS__CGSCC____-NEXT:    br label [[FOR_END]]
+; IS__CGSCC____:       for.body:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       for.inc:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       for.end:
+; IS__CGSCC____-NEXT:    br label [[WHILE_COND]]
+; IS__CGSCC____:       while.end:
+; IS__CGSCC____-NEXT:    [[ANS_0_LCSSA:%.*]] = phi i32 [ [[ANS_0]], [[WHILE_COND]] ]
+; IS__CGSCC____-NEXT:    ret i32 [[ANS_0_LCSSA]]
 ;
 entry:
   br label %while.cond
@@ -964,38 +1364,66 @@ while.end:                                        ; preds = %while.cond
 ;   return ans;
 ; }
 
+define i32 @nested_unbounded_loops(i32 %n) {
 ; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone
+; IS__TUNIT____-LABEL: define {{[^@]+}}@nested_unbounded_loops
+; IS__TUNIT____-SAME: (i32 [[N:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    br label [[WHILE_COND:%.*]]
+; IS__TUNIT____:       while.cond:
+; IS__TUNIT____-NEXT:    [[ANS_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP1:%.*]], [[WHILE_END10:%.*]] ]
+; IS__TUNIT____-NEXT:    [[N_ADDR_0:%.*]] = phi i32 [ [[N]], [[ENTRY]] ], [ -1, [[WHILE_END10]] ]
+; IS__TUNIT____-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[N_ADDR_0]], 0
+; IS__TUNIT____-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END11:%.*]], label [[WHILE_BODY:%.*]]
+; IS__TUNIT____:       while.body:
+; IS__TUNIT____-NEXT:    br label [[WHILE_COND1:%.*]]
+; IS__TUNIT____:       while.cond1:
+; IS__TUNIT____-NEXT:    br i1 true, label [[WHILE_END:%.*]], label [[WHILE_BODY4:%.*]]
+; IS__TUNIT____:       while.body4:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       while.end:
+; IS__TUNIT____-NEXT:    [[TMP:%.*]] = add i32 [[N_ADDR_0]], -2
+; IS__TUNIT____-NEXT:    br label [[WHILE_COND5:%.*]]
+; IS__TUNIT____:       while.cond5:
+; IS__TUNIT____-NEXT:    br i1 true, label [[WHILE_END10]], label [[WHILE_BODY8:%.*]]
+; IS__TUNIT____:       while.body8:
+; IS__TUNIT____-NEXT:    unreachable
+; IS__TUNIT____:       while.end10:
+; IS__TUNIT____-NEXT:    [[TMP1]] = add i32 [[TMP]], [[ANS_0]]
+; IS__TUNIT____-NEXT:    br label [[WHILE_COND]]
+; IS__TUNIT____:       while.end11:
+; IS__TUNIT____-NEXT:    [[ANS_0_LCSSA:%.*]] = phi i32 [ [[ANS_0]], [[WHILE_COND]] ]
+; IS__TUNIT____-NEXT:    ret i32 [[ANS_0_LCSSA]]
+;
 ; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone
-; CHECK-NOT: willreturn
-define i32 @nested_unbounded_loops(i32 %n) {
-; CHECK-LABEL: define {{[^@]+}}@nested_unbounded_loops
-; CHECK-SAME: (i32 [[N:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[WHILE_COND:%.*]]
-; CHECK:       while.cond:
-; CHECK-NEXT:    [[ANS_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP1:%.*]], [[WHILE_END10:%.*]] ]
-; CHECK-NEXT:    [[N_ADDR_0:%.*]] = phi i32 [ [[N]], [[ENTRY]] ], [ -1, [[WHILE_END10]] ]
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[N_ADDR_0]], 0
-; CHECK-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END11:%.*]], label [[WHILE_BODY:%.*]]
-; CHECK:       while.body:
-; CHECK-NEXT:    br label [[WHILE_COND1:%.*]]
-; CHECK:       while.cond1:
-; CHECK-NEXT:    br i1 true, label [[WHILE_END:%.*]], label [[WHILE_BODY4:%.*]]
-; CHECK:       while.body4:
-; CHECK-NEXT:    unreachable
-; CHECK:       while.end:
-; CHECK-NEXT:    [[TMP:%.*]] = add i32 [[N_ADDR_0]], -2
-; CHECK-NEXT:    br label [[WHILE_COND5:%.*]]
-; CHECK:       while.cond5:
-; CHECK-NEXT:    br i1 true, label [[WHILE_END10]], label [[WHILE_BODY8:%.*]]
-; CHECK:       while.body8:
-; CHECK-NEXT:    unreachable
-; CHECK:       while.end10:
-; CHECK-NEXT:    [[TMP1]] = add i32 [[TMP]], [[ANS_0]]
-; CHECK-NEXT:    br label [[WHILE_COND]]
-; CHECK:       while.end11:
-; CHECK-NEXT:    [[ANS_0_LCSSA:%.*]] = phi i32 [ [[ANS_0]], [[WHILE_COND]] ]
-; CHECK-NEXT:    ret i32 [[ANS_0_LCSSA]]
+; IS__CGSCC____-LABEL: define {{[^@]+}}@nested_unbounded_loops
+; IS__CGSCC____-SAME: (i32 [[N:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    br label [[WHILE_COND:%.*]]
+; IS__CGSCC____:       while.cond:
+; IS__CGSCC____-NEXT:    [[ANS_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP1:%.*]], [[WHILE_END10:%.*]] ]
+; IS__CGSCC____-NEXT:    [[N_ADDR_0:%.*]] = phi i32 [ [[N]], [[ENTRY]] ], [ -1, [[WHILE_END10]] ]
+; IS__CGSCC____-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[N_ADDR_0]], 0
+; IS__CGSCC____-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END11:%.*]], label [[WHILE_BODY:%.*]]
+; IS__CGSCC____:       while.body:
+; IS__CGSCC____-NEXT:    br label [[WHILE_COND1:%.*]]
+; IS__CGSCC____:       while.cond1:
+; IS__CGSCC____-NEXT:    br i1 true, label [[WHILE_END:%.*]], label [[WHILE_BODY4:%.*]]
+; IS__CGSCC____:       while.body4:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       while.end:
+; IS__CGSCC____-NEXT:    [[TMP:%.*]] = add i32 [[N_ADDR_0]], -2
+; IS__CGSCC____-NEXT:    br label [[WHILE_COND5:%.*]]
+; IS__CGSCC____:       while.cond5:
+; IS__CGSCC____-NEXT:    br i1 true, label [[WHILE_END10]], label [[WHILE_BODY8:%.*]]
+; IS__CGSCC____:       while.body8:
+; IS__CGSCC____-NEXT:    unreachable
+; IS__CGSCC____:       while.end10:
+; IS__CGSCC____-NEXT:    [[TMP1]] = add i32 [[TMP]], [[ANS_0]]
+; IS__CGSCC____-NEXT:    br label [[WHILE_COND]]
+; IS__CGSCC____:       while.end11:
+; IS__CGSCC____-NEXT:    [[ANS_0_LCSSA:%.*]] = phi i32 [ [[ANS_0]], [[WHILE_COND]] ]
+; IS__CGSCC____-NEXT:    ret i32 [[ANS_0_LCSSA]]
 ;
 entry:
   br label %while.cond
@@ -1056,38 +1484,66 @@ while.end11:                                      ; preds = %while.cond
 ;      return;
 ;    }
 
+define void @non_loop_cycle(i32 %n) {
 ; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone
+; IS__TUNIT____-LABEL: define {{[^@]+}}@non_loop_cycle
+; IS__TUNIT____-SAME: (i32 [[N:%.*]])
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[CALL:%.*]] = call i32 @fact_loop(i32 [[N]])
+; IS__TUNIT____-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[CALL]], 5
+; IS__TUNIT____-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; IS__TUNIT____:       if.then:
+; IS__TUNIT____-NEXT:    br label [[ENTRY1:%.*]]
+; IS__TUNIT____:       if.else:
+; IS__TUNIT____-NEXT:    br label [[ENTRY2:%.*]]
+; IS__TUNIT____:       entry1:
+; IS__TUNIT____-NEXT:    [[CALL1:%.*]] = call i32 @fact_loop(i32 [[N]])
+; IS__TUNIT____-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[CALL1]], 5
+; IS__TUNIT____-NEXT:    br i1 [[CMP2]], label [[IF_THEN3:%.*]], label [[IF_ELSE4:%.*]]
+; IS__TUNIT____:       if.then3:
+; IS__TUNIT____-NEXT:    br label [[EXIT:%.*]]
+; IS__TUNIT____:       if.else4:
+; IS__TUNIT____-NEXT:    br label [[ENTRY2]]
+; IS__TUNIT____:       entry2:
+; IS__TUNIT____-NEXT:    [[CALL5:%.*]] = call i32 @fact_loop(i32 [[N]])
+; IS__TUNIT____-NEXT:    [[CMP6:%.*]] = icmp sgt i32 [[CALL5]], 5
+; IS__TUNIT____-NEXT:    br i1 [[CMP6]], label [[IF_THEN7:%.*]], label [[IF_ELSE8:%.*]]
+; IS__TUNIT____:       if.then7:
+; IS__TUNIT____-NEXT:    br label [[EXIT]]
+; IS__TUNIT____:       if.else8:
+; IS__TUNIT____-NEXT:    br label [[ENTRY1]]
+; IS__TUNIT____:       exit:
+; IS__TUNIT____-NEXT:    ret void
+;
 ; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone
-; CHECK-NOT: willreturn
-define void @non_loop_cycle(i32 %n) {
-; CHECK-LABEL: define {{[^@]+}}@non_loop_cycle
-; CHECK-SAME: (i32 [[N:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @fact_loop(i32 [[N]])
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[CALL]], 5
-; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    br label [[ENTRY1:%.*]]
-; CHECK:       if.else:
-; CHECK-NEXT:    br label [[ENTRY2:%.*]]
-; CHECK:       entry1:
-; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @fact_loop(i32 [[N]])
-; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[CALL1]], 5
-; CHECK-NEXT:    br i1 [[CMP2]], label [[IF_THEN3:%.*]], label [[IF_ELSE4:%.*]]
-; CHECK:       if.then3:
-; CHECK-NEXT:    br label [[EXIT:%.*]]
-; CHECK:       if.else4:
-; CHECK-NEXT:    br label [[ENTRY2]]
-; CHECK:       entry2:
-; CHECK-NEXT:    [[CALL5:%.*]] = call i32 @fact_loop(i32 [[N]])
-; CHECK-NEXT:    [[CMP6:%.*]] = icmp sgt i32 [[CALL5]], 5
-; CHECK-NEXT:    br i1 [[CMP6]], label [[IF_THEN7:%.*]], label [[IF_ELSE8:%.*]]
-; CHECK:       if.then7:
-; CHECK-NEXT:    br label [[EXIT]]
-; CHECK:       if.else8:
-; CHECK-NEXT:    br label [[ENTRY1]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
+; IS__CGSCC____-LABEL: define {{[^@]+}}@non_loop_cycle
+; IS__CGSCC____-SAME: (i32 [[N:%.*]])
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[CALL:%.*]] = call i32 @fact_loop(i32 [[N]])
+; IS__CGSCC____-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[CALL]], 5
+; IS__CGSCC____-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; IS__CGSCC____:       if.then:
+; IS__CGSCC____-NEXT:    br label [[ENTRY1:%.*]]
+; IS__CGSCC____:       if.else:
+; IS__CGSCC____-NEXT:    br label [[ENTRY2:%.*]]
+; IS__CGSCC____:       entry1:
+; IS__CGSCC____-NEXT:    [[CALL1:%.*]] = call i32 @fact_loop(i32 [[N]])
+; IS__CGSCC____-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[CALL1]], 5
+; IS__CGSCC____-NEXT:    br i1 [[CMP2]], label [[IF_THEN3:%.*]], label [[IF_ELSE4:%.*]]
+; IS__CGSCC____:       if.then3:
+; IS__CGSCC____-NEXT:    br label [[EXIT:%.*]]
+; IS__CGSCC____:       if.else4:
+; IS__CGSCC____-NEXT:    br label [[ENTRY2]]
+; IS__CGSCC____:       entry2:
+; IS__CGSCC____-NEXT:    [[CALL5:%.*]] = call i32 @fact_loop(i32 [[N]])
+; IS__CGSCC____-NEXT:    [[CMP6:%.*]] = icmp sgt i32 [[CALL5]], 5
+; IS__CGSCC____-NEXT:    br i1 [[CMP6]], label [[IF_THEN7:%.*]], label [[IF_ELSE8:%.*]]
+; IS__CGSCC____:       if.then7:
+; IS__CGSCC____-NEXT:    br label [[EXIT]]
+; IS__CGSCC____:       if.else8:
+; IS__CGSCC____-NEXT:    br label [[ENTRY1]]
+; IS__CGSCC____:       exit:
+; IS__CGSCC____-NEXT:    ret void
 ;
 entry:
   %call = call i32 @fact_loop(i32 %n)
diff --git a/llvm/test/Transforms/DeadArgElim/byref.ll b/llvm/test/Transforms/DeadArgElim/byref.ll
new file mode 100644
index 0000000000000..95e839c04aa28
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/byref.ll
@@ -0,0 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -deadargelim -S | FileCheck %s
+
+declare void @sideeffect()
+
+define void @unused_byref_arg(i32* byref(i32) %dead_arg) {
+; CHECK-LABEL: @unused_byref_arg(
+; CHECK-NEXT:    tail call void @sideeffect()
+; CHECK-NEXT:    ret void
+;
+  tail call void @sideeffect()
+  ret void
+}
+
+define void @dont_replace_by_undef(i32* %ptr) {
+; CHECK-LABEL: @dont_replace_by_undef(
+; CHECK-NEXT:    call void @unused_byref_arg(i32* undef)
+; CHECK-NEXT:    ret void
+;
+  call void @unused_byref_arg(i32* %ptr)
+  ret void
+}
diff --git a/llvm/test/Transforms/DivRemPairs/PowerPC/div-expanded-rem-pair.ll b/llvm/test/Transforms/DivRemPairs/PowerPC/div-expanded-rem-pair.ll
index 4b640b98e302d..277d3de7c7126 100644
--- a/llvm/test/Transforms/DivRemPairs/PowerPC/div-expanded-rem-pair.ll
+++ b/llvm/test/Transforms/DivRemPairs/PowerPC/div-expanded-rem-pair.ll
@@ -106,12 +106,11 @@ define i32 @srem_of_srem_unexpanded(i32 %X, i32 %Y, i32 %Z) {
 ; CHECK-NEXT:    [[T2:%.*]] = mul nsw i32 [[T0]], [[T1]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[T1]], [[T0_FROZEN]]
 ; CHECK-NEXT:    [[T3_DECOMPOSED:%.*]] = sub i32 [[X_FROZEN]], [[TMP1]]
-; CHECK-NEXT:    [[T3_DECOMPOSED_FROZEN:%.*]] = freeze i32 [[T3_DECOMPOSED]]
 ; CHECK-NEXT:    [[Y_FROZEN:%.*]] = freeze i32 [[Y]]
-; CHECK-NEXT:    [[T4:%.*]] = sdiv i32 [[T3_DECOMPOSED_FROZEN]], [[Y_FROZEN]]
+; CHECK-NEXT:    [[T4:%.*]] = sdiv i32 [[T3_DECOMPOSED]], [[Y_FROZEN]]
 ; CHECK-NEXT:    [[T5:%.*]] = mul nsw i32 [[T4]], [[Y]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = mul i32 [[T4]], [[Y_FROZEN]]
-; CHECK-NEXT:    [[T6_DECOMPOSED:%.*]] = sub i32 [[T3_DECOMPOSED_FROZEN]], [[TMP2]]
+; CHECK-NEXT:    [[T6_DECOMPOSED:%.*]] = sub i32 [[T3_DECOMPOSED]], [[TMP2]]
 ; CHECK-NEXT:    ret i32 [[T6_DECOMPOSED]]
 ;
   %t0 = mul nsw i32 %Z, %Y
diff --git a/llvm/test/Transforms/DivRemPairs/PowerPC/div-rem-pairs.ll b/llvm/test/Transforms/DivRemPairs/PowerPC/div-rem-pairs.ll
index 3692d7d224f10..9e6d85d9920a1 100644
--- a/llvm/test/Transforms/DivRemPairs/PowerPC/div-rem-pairs.ll
+++ b/llvm/test/Transforms/DivRemPairs/PowerPC/div-rem-pairs.ll
@@ -175,12 +175,11 @@ define i32 @srem_of_srem_unexpanded(i32 %X, i32 %Y, i32 %Z) {
 ; CHECK-NEXT:    [[T2:%.*]] = mul nsw i32 [[T0]], [[T1]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[T1]], [[T0_FROZEN]]
 ; CHECK-NEXT:    [[T3_DECOMPOSED:%.*]] = sub i32 [[X_FROZEN]], [[TMP1]]
-; CHECK-NEXT:    [[T3_DECOMPOSED_FROZEN:%.*]] = freeze i32 [[T3_DECOMPOSED]]
 ; CHECK-NEXT:    [[Y_FROZEN:%.*]] = freeze i32 [[Y]]
-; CHECK-NEXT:    [[T4:%.*]] = sdiv i32 [[T3_DECOMPOSED_FROZEN]], [[Y_FROZEN]]
+; CHECK-NEXT:    [[T4:%.*]] = sdiv i32 [[T3_DECOMPOSED]], [[Y_FROZEN]]
 ; CHECK-NEXT:    [[T5:%.*]] = mul nsw i32 [[T4]], [[Y]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = mul i32 [[T4]], [[Y_FROZEN]]
-; CHECK-NEXT:    [[T6_DECOMPOSED:%.*]] = sub i32 [[T3_DECOMPOSED_FROZEN]], [[TMP2]]
+; CHECK-NEXT:    [[T6_DECOMPOSED:%.*]] = sub i32 [[T3_DECOMPOSED]], [[TMP2]]
 ; CHECK-NEXT:    ret i32 [[T6_DECOMPOSED]]
 ;
   %t0 = mul nsw i32 %Z, %Y
diff --git a/llvm/test/Transforms/GVN/loadpre-missed-opportunity.ll b/llvm/test/Transforms/GVN/loadpre-missed-opportunity.ll
new file mode 100644
index 0000000000000..dae22d18334b8
--- /dev/null
+++ b/llvm/test/Transforms/GVN/loadpre-missed-opportunity.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -gvn -S | FileCheck %s
+
+define void @loadpre_opportunity(i32** %arg, i1 %arg1, i1 %arg2, i1 %arg3) {
+; CHECK-LABEL: @loadpre_opportunity(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label [[BB9:%.*]]
+; CHECK:       bb6:
+; CHECK-NEXT:    br label [[BB9]]
+; CHECK:       bb9:
+; CHECK-NEXT:    br i1 [[ARG1:%.*]], label [[BB6:%.*]], label [[BB10:%.*]]
+; CHECK:       bb10:
+; CHECK-NEXT:    call void @somecall()
+; CHECK-NEXT:    br i1 [[ARG2:%.*]], label [[BB12:%.*]], label [[BB15:%.*]]
+; CHECK:       bb12:
+; CHECK-NEXT:    br label [[BB13:%.*]]
+; CHECK:       bb13:
+; CHECK-NEXT:    br i1 [[ARG3:%.*]], label [[BB14:%.*]], label [[BB13]]
+; CHECK:       bb14:
+; CHECK-NEXT:    br label [[BB15]]
+; CHECK:       bb15:
+; CHECK-NEXT:    br label [[BB6]]
+;
+bb:
+  %i = load i32*, i32** %arg, align 8
+  %i4 = getelementptr inbounds i32, i32* %i, i64 0
+  br label %bb5
+
+bb5:
+  br label %bb9
+
+bb6:
+  %i7 = load i32*, i32** %arg, align 8
+  %i8 = getelementptr inbounds i32, i32* %i7, i64 0
+  br label %bb9
+
+bb9:
+  br i1 %arg1, label %bb6, label %bb10
+
+bb10:
+  call void @somecall()
+  br i1 %arg2, label %bb12, label %bb15
+
+bb12:
+  br label %bb13
+
+bb13:
+  br i1 %arg3, label %bb14, label %bb13
+
+bb14:
+  br label %bb15
+
+bb15:
+  br label %bb6
+}
+
+declare void @somecall()
diff --git a/llvm/test/Transforms/GVNSink/indirect-call.ll b/llvm/test/Transforms/GVNSink/indirect-call.ll
index 57b7297c84bd6..66c1a4f647a98 100644
--- a/llvm/test/Transforms/GVNSink/indirect-call.ll
+++ b/llvm/test/Transforms/GVNSink/indirect-call.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -gvn-sink -simplifycfg -simplifycfg-sink-common=false -S | FileCheck %s
+; RUN: opt < %s -gvn-sink -simplifycfg -hoist-common-insts=true -simplifycfg-sink-common=false -S | FileCheck %s
 
 declare i8 @ext(i1)
 
diff --git a/llvm/test/Transforms/GVNSink/sink-common-code.ll b/llvm/test/Transforms/GVNSink/sink-common-code.ll
index 293e0daff5fb1..4131fb37485fb 100644
--- a/llvm/test/Transforms/GVNSink/sink-common-code.ll
+++ b/llvm/test/Transforms/GVNSink/sink-common-code.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -gvn-sink -simplifycfg -simplifycfg-sink-common=false -S | FileCheck %s
+; RUN: opt < %s -gvn-sink -simplifycfg -hoist-common-insts=true -simplifycfg-sink-common=false -S | FileCheck %s
 
 define zeroext i1 @test1(i1 zeroext %flag, i32 %blksA, i32 %blksB, i32 %nblks) {
 entry:
diff --git a/llvm/test/Transforms/Inline/ML/bounds-checks-rewards.ll b/llvm/test/Transforms/Inline/ML/bounds-checks-rewards.ll
new file mode 100644
index 0000000000000..73668c39439e2
--- /dev/null
+++ b/llvm/test/Transforms/Inline/ML/bounds-checks-rewards.ll
@@ -0,0 +1,45 @@
+; Test behavior when inlining policy grows size out of control.
+; In all cases, the end result is the same: mandatory inlinings must happen.
+; However, when we discover we 'trip' over the artificially-low size increase 
+; factor, we penalize the 'bad' decision.
+; REQUIRES: have_tf_api
+; RUN: opt -passes=scc-oz-module-inliner -ml-inliner-ir2native-model=%S/../../../../unittests/Analysis/Inputs/ir2native_x86_64_model -ml-inliner-model-under-training=%S/../../../../lib/Analysis/models/inliner -training-log=- -enable-ml-inliner=development -ml-advisor-size-increase-threshold=10.0 -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=NOBOUNDS
+; RUN: opt -passes=scc-oz-module-inliner -ml-inliner-ir2native-model=%S/../../../../unittests/Analysis/Inputs/ir2native_x86_64_model -ml-inliner-model-under-training=%S/../../../../lib/Analysis/models/inliner -training-log=- -enable-ml-inliner=development -ml-advisor-size-increase-threshold=1.0 -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=BOUNDS
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+declare i64 @f1()
+
+define i64 @f2() #0 {
+  %r = call i64 @f1()
+  %r2 = add i64 13, %r
+  ret i64 %r2
+}
+
+define i64 @some_function() {
+  %r = call i64 @f1()
+  %r2 = add i64 13, %r
+  ret i64 %r2
+}
+
+define i64 @top() {
+  %r = call i64 @f2()
+  %r2 = call i64 @some_function()
+  %r3 = add i64 %r, %r2
+  ret i64 %r3
+}
+
+attributes #0 = { alwaysinline }
+; CHECK: key: "delta_size" value: {
+; NOBOUNDS-NEXT: feature: { int64_list: { value: [10] } }
+; NOBOUNDS-NEXT: feature: { int64_list: { value: [6] } }
+; BOUNDS-NEXT: feature: { int64_list: { value: [2147483647] } }
+; CHECK-NEXT: }
+; CHECK-LABEL: @top
+; f2 must always be inlined, so we won't find a call to it in @top()
+; CHECK-NOT: call i64 @f2
+; @some-function isn't mandatory, and when we set the increase threshold too low,
+; it won't be inlined.
+; NOBOUNDS-NOT: @some_function
+; BOUNDS: call i64 @some_function
\ No newline at end of file
diff --git a/llvm/test/Transforms/Inline/ML/development-training-log.ll b/llvm/test/Transforms/Inline/ML/development-training-log.ll
new file mode 100644
index 0000000000000..4bf6259f7e482
--- /dev/null
+++ b/llvm/test/Transforms/Inline/ML/development-training-log.ll
@@ -0,0 +1,51 @@
+; Test that we can produce a log if we have or do not have a model, in development mode.
+; REQUIRES: have_tf_api
+; RUN: opt -enable-ml-inliner=development -passes=scc-oz-module-inliner -training-log=- -ml-inliner-model-under-training=%S/../../../../lib/Analysis/models/inliner -ml-inliner-ir2native-model=%S/../../../../unittests/Analysis/Inputs/ir2native_x86_64_model -S < %s | FileCheck %s 
+; RUN: opt -enable-ml-inliner=development -passes=scc-oz-module-inliner -training-log=- -ml-inliner-ir2native-model=%S/../../../../unittests/Analysis/Inputs/ir2native_x86_64_model -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-linux-gnu"
+
+declare i32 @f1(i32)
+declare i32 @f2(i32)
+
+define dso_local i32 @branches(i32) {
+  %cond = icmp slt i32 %0, 3
+  br i1 %cond, label %then, label %else
+
+then:
+  %ret.1 = call i32 @f1(i32 %0)
+  br label %last.block
+
+else:
+  %ret.2 = call i32 @f2(i32 %0)
+  br label %last.block
+
+last.block:
+  %ret = phi i32 [%ret.1, %then], [%ret.2, %else]
+  ret i32 %ret
+}
+
+define dso_local i32 @top() {
+  %1 = call i32 @branches(i32 2)
+  ret i32 %1
+}
+
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 7.0.0-6 (tags/RELEASE_700/final)"}
+
+; Check we produce a protobuf that has inlining decisions and rewards.
+; CHECK:     feature_lists: {
+; CHECK:          key: "inlining_decision" value: {
+; CHECK-NEXT:       feature: { int64_list: { value: [1] } }
+; CHECK-NEXT:     }
+; CHECK-NEXT:   }
+; CHECK-NEXT:   feature_list: {
+; CHECK-NEXT:     key: "delta_size" value: {
+; CHECK-NEXT:       feature: { int64_list: { value: [0] } }
+; CHECK-NEXT:     }
+; CHECK-NEXT:   }
\ No newline at end of file
diff --git a/llvm/test/Transforms/Inline/ML/ml-test-development-mode.ll b/llvm/test/Transforms/Inline/ML/ml-test-development-mode.ll
new file mode 100644
index 0000000000000..0546c9a23d4e2
--- /dev/null
+++ b/llvm/test/Transforms/Inline/ML/ml-test-development-mode.ll
@@ -0,0 +1,10 @@
+; The default inliner doesn't elide @adder, it believes it's too costly to inline 
+; adder into switcher. The ML inliner carries out that inlining, resulting in
+; a smaller result (part of it is that adder gets elided).
+;
+; This test uses Inputs/test-module.ll, as it shares it with a similar test
+; for the 'release' mode.
+;
+; REQUIRES: have_tf_api
+; RUN: opt -passes=scc-oz-module-inliner -enable-ml-inliner=default -S < %S/Inputs/test-module.ll 2>&1 | FileCheck %S/Inputs/test-module.ll --check-prefix=DEFAULT
+; RUN: opt -passes=scc-oz-module-inliner -enable-ml-inliner=development -ml-inliner-model-under-training=%S/../../../../lib/Analysis/models/inliner -S < %S/Inputs/test-module.ll 2>&1 | FileCheck %S/Inputs/test-module.ll --check-prefix=CHECK
diff --git a/llvm/test/Transforms/Inline/byref-align.ll b/llvm/test/Transforms/Inline/byref-align.ll
new file mode 100644
index 0000000000000..fb70db2af449d
--- /dev/null
+++ b/llvm/test/Transforms/Inline/byref-align.ll
@@ -0,0 +1,52 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature
+; RUN: opt -inline -preserve-alignment-assumptions-during-inlining -S < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Test behavior of inserted alignment assumptions with byref. There is
+; no implied copy to a higher alignment, so an alignment assume call
+; should be inserted.
+define void @byref_callee(float* align(128) byref(float) nocapture %a, float* %b) #0 {
+; CHECK-LABEL: define {{[^@]+}}@byref_callee
+; CHECK-SAME: (float* nocapture byref(float) align 128 [[A:%.*]], float* [[B:%.*]]) #0
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOAD:%.*]] = load float, float* [[A]], align 4
+; CHECK-NEXT:    [[B_IDX:%.*]] = getelementptr inbounds float, float* [[B]], i64 8
+; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[LOAD]], 2.000000e+00
+; CHECK-NEXT:    store float [[ADD]], float* [[B_IDX]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %load = load float, float* %a, align 4
+  %b.idx = getelementptr inbounds float, float* %b, i64 8
+  %add = fadd float %load, 2.0
+  store float %add, float* %b.idx, align 4
+  ret void
+}
+
+define void @byref_caller(float* nocapture align 64 %a, float* %b) #0 {
+; CHECK-LABEL: define {{[^@]+}}@byref_caller
+; CHECK-SAME: (float* nocapture align 64 [[A:%.*]], float* [[B:%.*]]) #0
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint float* [[A]] to i64
+; CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 127
+; CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[MASKCOND]])
+; CHECK-NEXT:    [[LOAD_I:%.*]] = load float, float* [[A]], align 4
+; CHECK-NEXT:    [[B_IDX_I:%.*]] = getelementptr inbounds float, float* [[B]], i64 8
+; CHECK-NEXT:    [[ADD_I:%.*]] = fadd float [[LOAD_I]], 2.000000e+00
+; CHECK-NEXT:    store float [[ADD_I]], float* [[B_IDX_I]], align 4
+; CHECK-NEXT:    [[CALLER_LOAD:%.*]] = load float, float* [[B]], align 4
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[A]], i64 7
+; CHECK-NEXT:    store float [[CALLER_LOAD]], float* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void @byref_callee(float* align(128) byref(float) %a, float* %b)
+  %caller.load = load float, float* %b, align 4
+  %arrayidx = getelementptr inbounds float, float* %a, i64 7
+  store float %caller.load, float* %arrayidx, align 4
+  ret void
+}
+
+attributes #0 = { nounwind uwtable }
diff --git a/llvm/test/Transforms/Inline/inlining-advisor-default.ll b/llvm/test/Transforms/Inline/inlining-advisor-default.ll
index 7ee3e176bbeab..8a760ff5b3a8a 100644
--- a/llvm/test/Transforms/Inline/inlining-advisor-default.ll
+++ b/llvm/test/Transforms/Inline/inlining-advisor-default.ll
@@ -1,6 +1,7 @@
 ; Check that, in the absence of dependencies, we emit an error message when
 ; trying to use ML-driven inlining.
 ; REQUIRES: !have_tf_aot
+; REQUIRES: !have_tf_api
 ; RUN: not opt -passes=scc-oz-module-inliner -enable-ml-inliner=development -S < %s 2>&1 | FileCheck %s
 ; RUN: not opt -passes=scc-oz-module-inliner -enable-ml-inliner=release -S < %s 2>&1 | FileCheck %s
 
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll
index c2f5e2857be08..1969056311f8c 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -instcombine %s | FileCheck %s
+; RUN: opt -S -instcombine -mtriple=amdgcn-amd-amdhsa %s | FileCheck %s
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.buffer.load
@@ -2965,6 +2965,64 @@ declare <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f32.f32(i32, float, fl
 ; llvm.amdgcn.image.sample.cd.cl
 ; --------------------------------------------------------------------
 
+; CHECK-LABEL: @extract_elt3_image_sample_cd_cl_1d_v4f16_f32_f32(
+; CHECK-NEXT: %data = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32(i32 8, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret half %data
+define amdgpu_ps half @extract_elt3_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x half> %data, i32 3
+  ret half %elt0
+}
+
+; CHECK-LABEL: @extract_elt2_image_sample_cd_cl_1d_v4f16_f32_f32(
+; CHECK-NEXT: %data = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32(i32 4, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret half %data
+define amdgpu_ps half @extract_elt2_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x half> %data, i32 2
+  ret half %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_image_sample_cd_cl_1d_v4f16_f32_f32(
+; CHECK-NEXT: %data = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32(i32 2, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret half %data
+define amdgpu_ps half @extract_elt1_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x half> %data, i32 1
+  ret half %elt0
+}
+
+; FIXME: Enable load shortening when full support for v3f16 has been added (should expect call <3 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v3f16.f32.f32).
+; CHECK-LABEL: @extract_elt_to3_image_sample_cd_cl_1d_v4f16_f32_f32(
+; CHECK-NEXT: %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: %res = shufflevector <4 x half> %data, <4 x half> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT: ret <4 x half> %res
+define amdgpu_ps <4 x half> @extract_elt_to3_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %res = shufflevector <4 x half> %data, <4 x half> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+  ret <4 x half> %res
+}
+
+; CHECK-LABEL: @extract_elt_to2_image_sample_cd_cl_1d_v4f16_f32_f32(
+; CHECK-NEXT: %data = call <2 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v2f16.f32.f32(i32 3, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: %res = shufflevector <2 x half> %data, <2 x half> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT: ret <4 x half> %res
+define amdgpu_ps <4 x half> @extract_elt_to2_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %res = shufflevector <4 x half> %data, <4 x half> undef, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+  ret <4 x half> %res
+}
+
+; CHECK-LABEL: @extract_elt_to1_image_sample_cd_cl_1d_v4f16_f32_f32(
+; CHECK-NEXT: %data = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32(i32 1, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: %res = insertelement <4 x half> undef, half %data, i64 0
+; CHECK-NEXT: ret <4 x half> %res
+define amdgpu_ps <4 x half> @extract_elt_to1_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %res = shufflevector <4 x half> %data, <4 x half> undef, <4 x i32> <i32 0, i32 4, i32 5, i32 6>
+  ret <4 x half> %res
+}
+
 ; CHECK-LABEL: @extract_elt0_image_sample_cd_cl_1d_v4f16_f32_f32(
 ; CHECK-NEXT: %data = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32(i32 1, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
 ; CHECK-NEXT: ret half %data
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
index 13b7bd87f8dd4..3e02b73293573 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -instcombine -S < %s | FileCheck %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -instcombine -S < %s | FileCheck %s
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.rcp
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/ldexp.ll b/llvm/test/Transforms/InstCombine/AMDGPU/ldexp.ll
index c9982f1e03c33..75f9b74007bf1 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/ldexp.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/ldexp.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -mtriple=amdgcn-amd-amdhsa -instcombine -S | FileCheck %s
 
 define float @ldexp_f32_undef_undef() {
 ; CHECK-LABEL: @ldexp_f32_undef_undef(
diff --git a/llvm/test/Transforms/InstCombine/ARM/mve-v2i2v.ll b/llvm/test/Transforms/InstCombine/ARM/mve-v2i2v.ll
index 7c511135ba151..0b1f9fdaad9ac 100644
--- a/llvm/test/Transforms/InstCombine/ARM/mve-v2i2v.ll
+++ b/llvm/test/Transforms/InstCombine/ARM/mve-v2i2v.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -instcombine -S -o - %s | FileCheck %s
+; RUN: opt -instcombine -S -mtriple=arm -o - %s | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
 
diff --git a/llvm/test/Transforms/InstCombine/ARM/neon-intrinsics.ll b/llvm/test/Transforms/InstCombine/ARM/neon-intrinsics.ll
index d22fa9c811dc8..564b45a432a1a 100644
--- a/llvm/test/Transforms/InstCombine/ARM/neon-intrinsics.ll
+++ b/llvm/test/Transforms/InstCombine/ARM/neon-intrinsics.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -instcombine -mtriple=arm -S | FileCheck %s
 
 ; The alignment arguments for NEON load/store intrinsics can be increased
 ; by instcombine.  Check for this.
diff --git a/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll b/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
index d713a95e3a0b1..9e94aee713981 100644
--- a/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
+++ b/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
@@ -6,11 +6,11 @@
 
 ; RUN: cat %s > %t.ftz
 ; RUN: echo 'attributes #0 = { "denormal-fp-math-f32" = "preserve-sign" }' >> %t.ftz
-; RUN: opt < %t.ftz -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=FTZ
+; RUN: opt < %t.ftz -instcombine -mtriple=nvptx64-nvidia-cuda -S | FileCheck %s --check-prefix=CHECK --check-prefix=FTZ
 
 ; RUN: cat %s > %t.noftz
 ; RUN: echo 'attributes #0 = { "denormal-fp-math-f32" = "ieee" }' >> %t.noftz
-; RUN: opt < %t.noftz -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=NOFTZ
+; RUN: opt < %t.noftz -instcombine -mtriple=nvptx64-nvidia-cuda -S | FileCheck %s --check-prefix=CHECK --check-prefix=NOFTZ
 
 ; We handle nvvm intrinsics with ftz variants as follows:
 ;  - If the module is in ftz mode, the ftz variant is transformed into the
diff --git a/llvm/test/Transforms/InstCombine/X86/X86FsubCmpCombine.ll b/llvm/test/Transforms/InstCombine/X86/X86FsubCmpCombine.ll
index fc1034a8d48f1..c08865034359e 100644
--- a/llvm/test/Transforms/InstCombine/X86/X86FsubCmpCombine.ll
+++ b/llvm/test/Transforms/InstCombine/X86/X86FsubCmpCombine.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
 
 ; The test checks the folding of cmp(sub(a,b),0) into cmp(a,b).
 
diff --git a/llvm/test/Transforms/InstCombine/X86/addcarry.ll b/llvm/test/Transforms/InstCombine/X86/addcarry.ll
index d762b4e96da5a..25a6cf216b077 100644
--- a/llvm/test/Transforms/InstCombine/X86/addcarry.ll
+++ b/llvm/test/Transforms/InstCombine/X86/addcarry.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
 
 declare { i8, i32 } @llvm.x86.addcarry.32(i8, i32, i32)
 declare { i8, i64 } @llvm.x86.addcarry.64(i8, i64, i64)
@@ -35,4 +35,3 @@ define i64 @no_carryin_i64(i64 %x, i64 %y, i8* %p) {
   %r = extractvalue { i8, i64 } %s, 1
   ret i64 %r
 }
-
diff --git a/llvm/test/Transforms/InstCombine/X86/clmulqdq.ll b/llvm/test/Transforms/InstCombine/X86/clmulqdq.ll
index 12429e169096f..7ec7e1a20d5d9 100644
--- a/llvm/test/Transforms/InstCombine/X86/clmulqdq.ll
+++ b/llvm/test/Transforms/InstCombine/X86/clmulqdq.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
 
 declare <2 x i64> @llvm.x86.pclmulqdq(<2 x i64>, <2 x i64>, i8)
 declare <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64>, <4 x i64>, i8)
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-avx2.ll b/llvm/test/Transforms/InstCombine/X86/x86-avx2.ll
index 9a6a0a5a4cd73..8ed36a44ba7a4 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-avx2.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-avx2.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 ; Verify that instcombine is able to fold identity shuffles.
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-avx512.ll b/llvm/test/Transforms/InstCombine/X86/x86-avx512.ll
index 3ead926f8df79..7eabd4165beaf 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-avx512.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-avx512.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 declare <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>, <4 x float>, <4 x float>, i8, i32)
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-bmi-tbm.ll b/llvm/test/Transforms/InstCombine/X86/x86-bmi-tbm.ll
index 669f813306d1f..8d04eca11485d 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-bmi-tbm.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-bmi-tbm.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
 
 declare i32 @llvm.x86.tbm.bextri.u32(i32, i32) nounwind readnone
 declare i64 @llvm.x86.tbm.bextri.u64(i64, i64) nounwind readnone
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-insertps.ll b/llvm/test/Transforms/InstCombine/X86/x86-insertps.ll
index 54f00644f10b8..634de193c023c 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-insertps.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-insertps.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
 
 declare <4 x float> @llvm.x86.sse41.insertps(<4 x float>, <4 x float>, i8) nounwind readnone
 
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-masked-memops.ll b/llvm/test/Transforms/InstCombine/X86/x86-masked-memops.ll
index be190007327d8..d845dcb5cac4d 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-masked-memops.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-masked-memops.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
 
 ;; MASKED LOADS
 
@@ -325,4 +325,3 @@ declare void @llvm.x86.avx2.maskstore.d.256(i8*, <8 x i32>, <8 x i32>)
 declare void @llvm.x86.avx2.maskstore.q.256(i8*, <4 x i64>, <4 x i64>)
 
 declare void @llvm.x86.sse2.maskmov.dqu(<16 x i8>, <16 x i8>, i8*)
-
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-movmsk.ll b/llvm/test/Transforms/InstCombine/X86/x86-movmsk.ll
index 7be8f08dc6338..60704d0e9d62a 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-movmsk.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-movmsk.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-pack.ll b/llvm/test/Transforms/InstCombine/X86/x86-pack.ll
index 5f643bcc0dc8a..2a0f2cc5cd4e3 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-pack.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-pack.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
 
 ;
 ; UNDEF Elts
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll b/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll
index 92122a10d4486..b37e6049f2183 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
 
 ; Verify that instcombine is able to fold identity shuffles.
 
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-sse.ll b/llvm/test/Transforms/InstCombine/X86/x86-sse.ll
index d7eb0dfe716de..a08c9f93079fb 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-sse.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-sse.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 define float @test_rcp_ss_0(float %a) {
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-sse2.ll b/llvm/test/Transforms/InstCombine/X86/x86-sse2.ll
index dee3da0b0e1c1..ec74be9807b54 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-sse2.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-sse2.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 define double @test_sqrt_sd_0(double %a) {
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-sse41.ll b/llvm/test/Transforms/InstCombine/X86/x86-sse41.ll
index eb4a011b13052..98ed7bed0d50f 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-sse41.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-sse41.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 define <2 x double> @test_round_sd(<2 x double> %a, <2 x double> %b) {
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-sse4a.ll b/llvm/test/Transforms/InstCombine/X86/x86-sse4a.ll
index 95b02b2243770..2ee0c3462c19e 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-sse4a.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-sse4a.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
 
 ;
 ; EXTRQ
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-vec_demanded_elts.ll b/llvm/test/Transforms/InstCombine/X86/x86-vec_demanded_elts.ll
index 5ad8e767d767a..c0add457e1e39 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-vec_demanded_elts.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-vec_demanded_elts.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 define i16 @test1(float %f) {
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll b/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll
index 8d75cc34be38b..de67b1868b10e 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 ;
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll b/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll
index c367321b6f1f9..b227b83fee9b8 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 ; Verify that instcombine is able to fold identity shuffles.
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-xop.ll b/llvm/test/Transforms/InstCombine/X86/x86-xop.ll
index 6cc93698f7fd5..bef9861ea6d91 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-xop.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-xop.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
 
 define <2 x double> @test_vfrcz_sd(<2 x double> %a) {
 ; CHECK-LABEL: @test_vfrcz_sd(
diff --git a/llvm/test/Transforms/InstCombine/rotate.ll b/llvm/test/Transforms/InstCombine/rotate.ll
index a11fcd974246a..2b8d9fc1e3d15 100644
--- a/llvm/test/Transforms/InstCombine/rotate.ll
+++ b/llvm/test/Transforms/InstCombine/rotate.ll
@@ -438,8 +438,8 @@ define i9 @rotate9_not_safe(i9 %v, i32 %shamt) {
 
 define i16 @rotateleft_16_neg_mask(i16 %v, i16 %shamt) {
 ; CHECK-LABEL: @rotateleft_16_neg_mask(
-; CHECK-NEXT:    [[RET:%.*]] = call i16 @llvm.fshl.i16(i16 [[V:%.*]], i16 [[V]], i16 [[SHAMT:%.*]])
-; CHECK-NEXT:    ret i16 [[RET]]
+; CHECK-NEXT:    [[OR:%.*]] = call i16 @llvm.fshl.i16(i16 [[V:%.*]], i16 [[V]], i16 [[SHAMT:%.*]])
+; CHECK-NEXT:    ret i16 [[OR]]
 ;
   %neg = sub i16 0, %shamt
   %lshamt = and i16 %shamt, 15
@@ -456,8 +456,8 @@ define i16 @rotateleft_16_neg_mask(i16 %v, i16 %shamt) {
 
 define i16 @rotateleft_16_neg_mask_commute(i16 %v, i16 %shamt) {
 ; CHECK-LABEL: @rotateleft_16_neg_mask_commute(
-; CHECK-NEXT:    [[RET:%.*]] = call i16 @llvm.fshl.i16(i16 [[V:%.*]], i16 [[V]], i16 [[SHAMT:%.*]])
-; CHECK-NEXT:    ret i16 [[RET]]
+; CHECK-NEXT:    [[OR:%.*]] = call i16 @llvm.fshl.i16(i16 [[V:%.*]], i16 [[V]], i16 [[SHAMT:%.*]])
+; CHECK-NEXT:    ret i16 [[OR]]
 ;
   %neg = sub i16 0, %shamt
   %lshamt = and i16 %shamt, 15
@@ -474,8 +474,8 @@ define i16 @rotateleft_16_neg_mask_commute(i16 %v, i16 %shamt) {
 
 define i8 @rotateright_8_neg_mask(i8 %v, i8 %shamt) {
 ; CHECK-LABEL: @rotateright_8_neg_mask(
-; CHECK-NEXT:    [[RET:%.*]] = call i8 @llvm.fshr.i8(i8 [[V:%.*]], i8 [[V]], i8 [[SHAMT:%.*]])
-; CHECK-NEXT:    ret i8 [[RET]]
+; CHECK-NEXT:    [[OR:%.*]] = call i8 @llvm.fshr.i8(i8 [[V:%.*]], i8 [[V]], i8 [[SHAMT:%.*]])
+; CHECK-NEXT:    ret i8 [[OR]]
 ;
   %neg = sub i8 0, %shamt
   %rshamt = and i8 %shamt, 7
@@ -492,8 +492,8 @@ define i8 @rotateright_8_neg_mask(i8 %v, i8 %shamt) {
 
 define i8 @rotateright_8_neg_mask_commute(i8 %v, i8 %shamt) {
 ; CHECK-LABEL: @rotateright_8_neg_mask_commute(
-; CHECK-NEXT:    [[RET:%.*]] = call i8 @llvm.fshr.i8(i8 [[V:%.*]], i8 [[V]], i8 [[SHAMT:%.*]])
-; CHECK-NEXT:    ret i8 [[RET]]
+; CHECK-NEXT:    [[OR:%.*]] = call i8 @llvm.fshr.i8(i8 [[V:%.*]], i8 [[V]], i8 [[SHAMT:%.*]])
+; CHECK-NEXT:    ret i8 [[OR]]
 ;
   %neg = sub i8 0, %shamt
   %rshamt = and i8 %shamt, 7
@@ -689,6 +689,42 @@ define i24 @rotl_select_weird_type(i24 %x, i24 %shamt) {
   ret i24 %r
 }
 
+define i32 @rotl_select_zext_shamt(i32 %x, i8 %y) {
+; CHECK-LABEL: @rotl_select_zext_shamt(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[Y:%.*]] to i32
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.fshl.i32(i32 [[X:%.*]], i32 [[X]], i32 [[TMP1]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %rem = and i8 %y, 31
+  %cmp = icmp eq i8 %rem, 0
+  %sh_prom = zext i8 %rem to i32
+  %sub = sub nuw nsw i8 32, %rem
+  %sh_prom1 = zext i8 %sub to i32
+  %shr = lshr i32 %x, %sh_prom1
+  %shl = shl i32 %x, %sh_prom
+  %or = or i32 %shl, %shr
+  %r = select i1 %cmp, i32 %x, i32 %or
+  ret i32 %r
+}
+
+define i64 @rotr_select_zext_shamt(i64 %x, i32 %y) {
+; CHECK-LABEL: @rotr_select_zext_shamt(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[Y:%.*]] to i64
+; CHECK-NEXT:    [[R:%.*]] = call i64 @llvm.fshr.i64(i64 [[X:%.*]], i64 [[X]], i64 [[TMP1]])
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %rem = and i32 %y, 63
+  %cmp = icmp eq i32 %rem, 0
+  %sh_prom = zext i32 %rem to i64
+  %shr = lshr i64 %x, %sh_prom
+  %sub = sub nuw nsw i32 64, %rem
+  %sh_prom1 = zext i32 %sub to i64
+  %shl = shl i64 %x, %sh_prom1
+  %or = or i64 %shl, %shr
+  %r = select i1 %cmp, i64 %x, i64 %or
+  ret i64 %r
+}
+
 ; Test that the transform doesn't crash when there's an "or" with a ConstantExpr operand.
 
 @external_global = external global i8
diff --git a/llvm/test/Transforms/InstCombine/trunc-extractelement-spir.ll b/llvm/test/Transforms/InstCombine/trunc-extractelement-spir.ll
new file mode 100644
index 0000000000000..4203aa55e6f0b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/trunc-extractelement-spir.ll
@@ -0,0 +1,14 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024"
+target triple = "spir"
+
+define i32 @shrinkExtractElt_i64_to_i32_0(<3 x i64> %x) {
+; CHECK-LABEL: @shrinkExtractElt_i64_to_i32_0(
+; CHECK-NOT:    [[TMP1:%.*]] = bitcast <3 x i64> [[X:%.*]] to <6 x i32>
+; CHECK-NOT:    [[T:%.*]] = extractelement <6 x i32> [[TMP1]], i32 0
+
+  %e = extractelement <3 x i64> %x, i32 0
+  %t = trunc i64 %e to i32
+  ret i32 %t
+}
diff --git a/llvm/test/Transforms/InstSimplify/freeze-noundef.ll b/llvm/test/Transforms/InstSimplify/freeze-noundef.ll
index 4244778910587..f2e897b507e3d 100644
--- a/llvm/test/Transforms/InstSimplify/freeze-noundef.ll
+++ b/llvm/test/Transforms/InstSimplify/freeze-noundef.ll
@@ -19,12 +19,10 @@ define i1 @icmp(i8 noundef %x, i8 noundef %y) {
   ret i1 %f
 }
 
-; TODO: should look into binary operations
 define i1 @or(i1 noundef %x, i1 noundef %x2) {
 ; CHECK-LABEL: @or(
 ; CHECK-NEXT:    [[Y:%.*]] = or i1 [[X:%.*]], [[X2:%.*]]
-; CHECK-NEXT:    [[Z:%.*]] = freeze i1 [[Y]]
-; CHECK-NEXT:    ret i1 [[Z]]
+; CHECK-NEXT:    ret i1 [[Y]]
 ;
   %y = or i1 %x, %x2
   %z = freeze i1 %y
@@ -42,12 +40,10 @@ define i1 @or2(i1 noundef %x, i1 %x2) {
   ret i1 %z
 }
 
-; TODO: should look into binary operations
 define i8 @add(i8 noundef %x) {
 ; CHECK-LABEL: @add(
 ; CHECK-NEXT:    [[Y:%.*]] = add i8 [[X:%.*]], 1
-; CHECK-NEXT:    [[Z:%.*]] = freeze i8 [[Y]]
-; CHECK-NEXT:    ret i8 [[Z]]
+; CHECK-NEXT:    ret i8 [[Y]]
 ;
   %y = add i8 %x, 1
   %z = freeze i8 %y
@@ -77,21 +73,18 @@ define {i8, i32} @aggr({i8, i32} noundef %x) {
 define i32 @extract({i8, i32} noundef %x) {
 ; CHECK-LABEL: @extract(
 ; CHECK-NEXT:    [[Y:%.*]] = extractvalue { i8, i32 } [[X:%.*]], 1
-; CHECK-NEXT:    [[Z:%.*]] = freeze i32 [[Y]]
-; CHECK-NEXT:    ret i32 [[Z]]
+; CHECK-NEXT:    ret i32 [[Y]]
 ;
   %y = extractvalue {i8, i32} %x, 1
   %z = freeze i32 %y
   ret i32 %z
 }
 
-; TODO: should look into extract operations
 define i32 @extract2({i8, {i8, i32}} noundef %x) {
 ; CHECK-LABEL: @extract2(
 ; CHECK-NEXT:    [[Y:%.*]] = extractvalue { i8, { i8, i32 } } [[X:%.*]], 1
 ; CHECK-NEXT:    [[Z:%.*]] = extractvalue { i8, i32 } [[Y]], 1
-; CHECK-NEXT:    [[W:%.*]] = freeze i32 [[Z]]
-; CHECK-NEXT:    ret i32 [[W]]
+; CHECK-NEXT:    ret i32 [[Z]]
 ;
   %y = extractvalue {i8, {i8, i32}} %x, 1
   %z = extractvalue {i8, i32} %y, 1
diff --git a/llvm/test/Transforms/InstSimplify/freeze.ll b/llvm/test/Transforms/InstSimplify/freeze.ll
index 306f293f7be71..3c4ca7a1afb95 100644
--- a/llvm/test/Transforms/InstSimplify/freeze.ll
+++ b/llvm/test/Transforms/InstSimplify/freeze.ll
@@ -116,13 +116,28 @@ define <2 x float> @constvector_FP_noopt() {
 
 define float @constant_expr() {
 ; CHECK-LABEL: @constant_expr(
-; CHECK-NEXT:    [[R:%.*]] = freeze float bitcast (i32 ptrtoint (i16* @g to i32) to float)
-; CHECK-NEXT:    ret float [[R]]
+; CHECK-NEXT:    ret float bitcast (i32 ptrtoint (i16* @g to i32) to float)
 ;
   %r = freeze float bitcast (i32 ptrtoint (i16* @g to i32) to float)
   ret float %r
 }
 
+define i8* @constant_expr2() {
+; CHECK-LABEL: @constant_expr2(
+; CHECK-NEXT:    ret i8* bitcast (i16* @g to i8*)
+;
+  %r = freeze i8* bitcast (i16* @g to i8*)
+  ret i8* %r
+}
+
+define i32* @constant_expr3() {
+; CHECK-LABEL: @constant_expr3(
+; CHECK-NEXT:    ret i32* getelementptr (i32, i32* @glb, i64 3)
+;
+  %r = freeze i32* getelementptr (i32, i32* @glb, i64 3)
+  ret i32* %r
+}
+
 ; Negative test
 
 define <2 x i31> @vector_element_constant_expr() {
@@ -136,7 +151,7 @@ define <2 x i31> @vector_element_constant_expr() {
 
 define void @alloca() {
 ; CHECK-LABEL: @alloca(
-; CHECK-NEXT:    [[P:%.*]] = alloca i8
+; CHECK-NEXT:    [[P:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    call void @f3(i8* [[P]])
 ; CHECK-NEXT:    ret void
 ;
@@ -148,7 +163,7 @@ define void @alloca() {
 
 define i8* @gep() {
 ; CHECK-LABEL: @gep(
-; CHECK-NEXT:    [[P:%.*]] = alloca [4 x i8]
+; CHECK-NEXT:    [[P:%.*]] = alloca [4 x i8], align 1
 ; CHECK-NEXT:    [[Q:%.*]] = getelementptr [4 x i8], [4 x i8]* [[P]], i32 0, i32 6
 ; CHECK-NEXT:    ret i8* [[Q]]
 ;
@@ -171,7 +186,7 @@ define i8* @gep_noopt(i32 %arg) {
 
 define i8* @gep_inbounds() {
 ; CHECK-LABEL: @gep_inbounds(
-; CHECK-NEXT:    [[P:%.*]] = alloca [4 x i8]
+; CHECK-NEXT:    [[P:%.*]] = alloca [4 x i8], align 1
 ; CHECK-NEXT:    [[Q:%.*]] = getelementptr inbounds [4 x i8], [4 x i8]* [[P]], i32 0, i32 0
 ; CHECK-NEXT:    ret i8* [[Q]]
 ;
@@ -183,7 +198,7 @@ define i8* @gep_inbounds() {
 
 define i8* @gep_inbounds_noopt(i32 %arg) {
 ; CHECK-LABEL: @gep_inbounds_noopt(
-; CHECK-NEXT:    [[P:%.*]] = alloca [4 x i8]
+; CHECK-NEXT:    [[P:%.*]] = alloca [4 x i8], align 1
 ; CHECK-NEXT:    [[Q:%.*]] = getelementptr inbounds [4 x i8], [4 x i8]* [[P]], i32 0, i32 [[ARG:%.*]]
 ; CHECK-NEXT:    [[Q2:%.*]] = freeze i8* [[Q]]
 ; CHECK-NEXT:    ret i8* [[Q2]]
@@ -366,6 +381,23 @@ EXIT:
   ret i32 %fr2
 }
 
+declare i32 @any_num()
+
+define i32 @brcond_call() {
+; CHECK-LABEL: @brcond_call(
+; CHECK-NEXT:    [[X:%.*]] = call i32 @any_num()
+; CHECK-NEXT:    switch i32 [[X]], label [[EXIT:%.*]] [
+; CHECK-NEXT:    ]
+; CHECK:       EXIT:
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %x = call i32 @any_num()
+  switch i32 %x, label %EXIT []
+EXIT:
+  %y = freeze i32 %x
+  ret i32 %y
+}
+
 define i1 @brcond_noopt(i1 %c, i1 %c2) {
 ; CHECK-LABEL: @brcond_noopt(
 ; CHECK-NEXT:    [[F:%.*]] = freeze i1 [[C:%.*]]
diff --git a/llvm/test/Transforms/LoopFusion/guarded_peel.ll b/llvm/test/Transforms/LoopFusion/guarded_peel.ll
new file mode 100644
index 0000000000000..44b2f18b2d3c5
--- /dev/null
+++ b/llvm/test/Transforms/LoopFusion/guarded_peel.ll
@@ -0,0 +1,84 @@
+; RUN: opt -S -loop-fusion -loop-fusion-peel-max-count=3 < %s | FileCheck %s
+
+; Tests if we are able to fuse two guarded loops which have constant but
+; different trip counts. The first two iterations of the first loop should be
+; peeled off, and then the loops should be fused together.
+
+@B = common global [1024 x i32] zeroinitializer, align 16
+
+; CHECK-LABEL: void @main(i32* noalias %A)
+; CHECK-NEXT:  entry:
+; CHECK:         br i1 %cmp4, label %for.first.entry, label %for.end
+; CHECK:       for.first.entry
+; CHECK-NEXT:    br label %for.first.peel.begin
+; CHECK:       for.first.peel.begin:
+; CHECK-NEXT:    br label %for.first.peel
+; CHECK:       for.first.peel:
+; CHECK:         br label %for.first.peel.next
+; CHECK:       for.first.peel.next:
+; CHECK-NEXT:    br label %for.first.peel2
+; CHECK:       for.first.peel2:
+; CHECK:         br label %for.first.peel.next1
+; CHECK:       for.first.peel.next1:
+; CHECK-NEXT:    br label %for.first.peel.next11
+; CHECK:       for.first.peel.next11:
+; CHECK-NEXT:    br label %for.first.entry.peel.newph
+; CHECK:       for.first.entry.peel.newph:
+; CHECK:         br label %for.first
+; CHECK:       for.first:
+; CHECK:         br i1 %cmp3, label %for.first, label %for.second.exit
+; CHECK:       for.second.exit:
+; CHECK:         br label %for.end
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+
+define void @main(i32* noalias %A) {
+entry:
+  %cmp4 = icmp slt i64 0, 45
+  br i1 %cmp4, label %for.first.entry, label %for.second.guard
+
+for.first.entry:                                 ; preds = %entry
+  br label %for.first
+
+for.first:                                       ; preds = %for.first.entry, %for.first
+  %i.05 = phi i64 [ %inc, %for.first ], [ 0, %for.first.entry ]
+  %sub = sub nsw i64 %i.05, 3
+  %add = add nsw i64 %i.05, 3
+  %mul = mul nsw i64 %sub, %add
+  %rem = srem i64 %mul, %i.05
+  %conv = trunc i64 %rem to i32
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %i.05
+  store i32 %conv, i32* %arrayidx, align 4
+  %inc = add nsw i64 %i.05, 1
+  %cmp = icmp slt i64 %inc, 45
+  br i1 %cmp, label %for.first, label %for.first.exit
+
+for.first.exit:                                  ; preds = %for.first
+  br label %for.second.guard
+
+for.second.guard:                                ; preds = %for.first.exit, %entry
+  %cmp31 = icmp slt i64 2, 45
+  br i1 %cmp31, label %for.second.entry, label %for.end
+
+for.second.entry:                                ; preds = %for.second.guard
+  br label %for.second
+
+for.second:                                      ; preds = %for.second.entry, %for.second
+  %i1.02 = phi i64 [ %inc14, %for.second ], [ 2, %for.second.entry ]
+  %sub7 = sub nsw i64 %i1.02, 3
+  %add8 = add nsw i64 %i1.02, 3
+  %mul9 = mul nsw i64 %sub7, %add8
+  %rem10 = srem i64 %mul9, %i1.02
+  %conv11 = trunc i64 %rem10 to i32
+  %arrayidx12 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %i1.02
+  store i32 %conv11, i32* %arrayidx12, align 4
+  %inc14 = add nsw i64 %i1.02, 1
+  %cmp3 = icmp slt i64 %inc14, 45
+  br i1 %cmp3, label %for.second, label %for.second.exit
+
+for.second.exit:                                 ; preds = %for.second
+  br label %for.end
+
+for.end:                                         ; preds = %for.second.exit, %for.second.guard
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopFusion/guarded_unsafeblock_peel.ll b/llvm/test/Transforms/LoopFusion/guarded_unsafeblock_peel.ll
new file mode 100644
index 0000000000000..c7b8996aaf4ed
--- /dev/null
+++ b/llvm/test/Transforms/LoopFusion/guarded_unsafeblock_peel.ll
@@ -0,0 +1,72 @@
+; RUN: opt -S -loop-fusion -loop-fusion-peel-max-count=3 < %s | FileCheck %s
+
+; Tests that we do not fuse two guarded loops together.
+; These loops do not have the same trip count, and the first loop meets the
+; requirements for peeling. However, the exit block of the first loop makes the
+; loops unsafe to fuse together.
+; The expected output of this test is the function as below.
+
+; CHECK-LABEL: void @unsafe_exitblock(i32* noalias %A, i32* noalias %B)
+; CHECK:       for.first.guard
+; CHECK:         br i1 %cmp3, label %for.first.preheader, label %for.second.guard
+; CHECK:       for.first.preheader:
+; CHECK-NEXT:    br label %for.first
+; CHECK:       for.first:
+; CHECK:         br i1 %cmp, label %for.first, label %for.first.exit
+; CHECK:       for.first.exit:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label %for.second.guard
+; CHECK:       for.second.guard:
+; CHECK:         br i1 %cmp21, label %for.second.preheader, label %for.end
+; CHECK:       for.second.preheader:
+; CHECK-NEXT:    br label %for.second
+; CHECK:       for.second:
+; CHECK:         br i1 %cmp2, label %for.second, label %for.second.exit
+; CHECK:       for.second.exit:
+; CHECK-NEXT:    br label %for.end
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+
+define void @unsafe_exitblock(i32* noalias %A, i32* noalias %B) {
+for.first.guard:
+  %cmp3 = icmp slt i64 0, 45
+  br i1 %cmp3, label %for.first.preheader, label %for.second.guard
+
+for.first.preheader:                             ; preds = %for.first.guard
+  br label %for.first
+
+for.first:                                       ; preds = %for.first.preheader, %for.first
+  %i.04 = phi i64 [ %inc, %for.first ], [ 0, %for.first.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %i.04
+  store i32 0, i32* %arrayidx, align 4
+  %inc = add nsw i64 %i.04, 1
+  %cmp = icmp slt i64 %inc, 45
+  br i1 %cmp, label %for.first, label %for.first.exit
+
+for.first.exit:                                  ; preds = %for.first
+  call void @bar()
+  br label %for.second.guard
+
+for.second.guard:                                ; preds = %for.first.exit, %for.first.guard
+  %cmp21 = icmp slt i64 2,45
+  br i1 %cmp21, label %for.second.preheader, label %for.end
+
+for.second.preheader:                            ; preds = %for.second.guard
+  br label %for.second
+
+for.second:                                      ; preds = %for.second.preheader, %for.second
+  %j.02 = phi i64 [ %inc6, %for.second ], [ 2, %for.second.preheader ]
+  %arrayidx4 = getelementptr inbounds i32, i32* %B, i64 %j.02
+  store i32 0, i32* %arrayidx4, align 4
+  %inc6 = add nsw i64 %j.02, 1
+  %cmp2 = icmp slt i64 %inc6, 45
+  br i1 %cmp2, label %for.second, label %for.second.exit
+
+for.second.exit:                                 ; preds = %for.second
+  br label %for.end
+
+for.end:                                         ; preds = %for.second.exit, %for.second.guard
+  ret void
+}
+
+declare void @bar()
diff --git a/llvm/test/Transforms/LoopFusion/nonadjacent_peel.ll b/llvm/test/Transforms/LoopFusion/nonadjacent_peel.ll
new file mode 100644
index 0000000000000..d5256c31c8b5e
--- /dev/null
+++ b/llvm/test/Transforms/LoopFusion/nonadjacent_peel.ll
@@ -0,0 +1,84 @@
+; RUN: opt -S -loop-fusion -loop-fusion-peel-max-count=3 < %s | FileCheck %s
+
+; Tests that we do not fuse these two loops together. These loops do not have
+; the same tripcount, and the first loop is valid candiate for peeling; however
+; the loops are not adjacent, hence they are not valid to be fused (after
+; peeling).
+; The expected output of this test is the function below.
+
+; CHECK-LABEL: void @function(i32* noalias %arg)
+; CHECK-NEXT:  for.first.preheader:
+; CHECK-NEXT:    br label %for.first
+; CHECK:       for.first:
+; CHECK:         br label %for.first.latch
+; CHECK:       for.first.latch:
+; CHECK:         br i1 %exitcond4, label %for.first, label %for.first.exit
+; CHECK:       for.first.exit:
+; CHECK-NEXT:    br label %for.next
+; CHECK:       for.next:
+; CHECK-NEXT:    br label %for.second.preheader
+; CHECK:       for.second.preheader:
+; CHECK:         br label %for.second
+; CHECK:       for.second:
+; CHECK:         br label %for.second.latch
+; CHECK:       for.second.latch:
+; CHECK:         br i1 %exitcond, label %for.second, label %for.end
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+
+@B = common global [1024 x i32] zeroinitializer, align 16
+
+define void @function(i32* noalias %arg) {
+for.first.preheader:
+  br label %for.first
+
+for.first:                                       ; preds = %for.first.preheader, %for.first.latch
+  %.014 = phi i32 [ 0, %for.first.preheader ], [ %tmp15, %for.first.latch ]
+  %indvars.iv23 = phi i64 [ 0, %for.first.preheader ], [ %indvars.iv.next3, %for.first.latch ]
+  %tmp = add nsw i32 %.014, -3
+  %tmp8 = add nuw nsw i64 %indvars.iv23, 3
+  %tmp9 = trunc i64 %tmp8 to i32
+  %tmp10 = mul nsw i32 %tmp, %tmp9
+  %tmp11 = trunc i64 %indvars.iv23 to i32
+  %tmp12 = srem i32 %tmp10, %tmp11
+  %tmp13 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv23
+  store i32 %tmp12, i32* %tmp13, align 4
+  br label %for.first.latch
+
+for.first.latch:                                 ; preds = %for.first
+  %indvars.iv.next3 = add nuw nsw i64 %indvars.iv23, 1
+  %tmp15 = add nuw nsw i32 %.014, 1
+  %exitcond4 = icmp ne i64 %indvars.iv.next3, 100
+  br i1 %exitcond4, label %for.first, label %for.first.exit
+
+for.first.exit:                                  ; preds: %for.first.latch
+  br label %for.next
+
+for.next:                                        ; preds = %for.first.exit
+  br label %for.second.preheader
+
+for.second.preheader:                            ; preds = %for.next
+  br label %for.second
+
+for.second:                                      ; preds = %for.second.preheader, %for.second.latch
+  %.02 = phi i32 [ 0, %for.second.preheader ], [ %tmp28, %for.second.latch ]
+  %indvars.iv1 = phi i64 [ 3, %for.second.preheader ], [ %indvars.iv.next, %for.second.latch ]
+  %tmp20 = add nsw i32 %.02, -3
+  %tmp21 = add nuw nsw i64 %indvars.iv1, 3
+  %tmp22 = trunc i64 %tmp21 to i32
+  %tmp23 = mul nsw i32 %tmp20, %tmp22
+  %tmp24 = trunc i64 %indvars.iv1 to i32
+  %tmp25 = srem i32 %tmp23, %tmp24
+  %tmp26 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv1
+  store i32 %tmp25, i32* %tmp26, align 4
+  br label %for.second.latch
+
+for.second.latch:                                ; preds = %for.second
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv1, 1
+  %tmp28 = add nuw nsw i32 %.02, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, 100
+  br i1 %exitcond, label %for.second, label %for.end
+
+for.end:                                         ; preds = %for.second.latch
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopFusion/peel.ll b/llvm/test/Transforms/LoopFusion/peel.ll
new file mode 100644
index 0000000000000..6044df91ef0ab
--- /dev/null
+++ b/llvm/test/Transforms/LoopFusion/peel.ll
@@ -0,0 +1,105 @@
+; RUN: opt -S -loop-fusion -loop-fusion-peel-max-count=3 < %s | FileCheck %s
+
+; Tests whether we can fuse two loops together if they have constant but a
+; different tripcount.
+; The first three iterations of the first loop should be peeled, and then the
+; two loops should be fused together in this example.
+
+; C Code
+;
+; int B[1024];
+;
+; void function(int *arg) {
+;   for (int i = 0; i != 100; ++i)
+;     arg[i] = ((i - 3)*(i+3)) % i;
+;
+;   for (int i = 3; i != 100; ++i)
+;     B[i] = ((i-6)*(i+3)) % i;
+; }
+
+; CHECK-LABEL: void @function(i32* noalias %arg)
+; CHECK-NEXT:  for.first.preheader:
+; CHECK-NEXT:    br label %for.first.peel.begin
+; CHECK:       for.first.peel.begin:
+; CHECK-NEXT:    br label %for.first.peel
+; CHECK:       for.first.peel:
+; CHECK:         br label %for.first.latch.peel
+; CHECK:       for.first.latch.peel:
+; CHECK:         br label %for.first.peel.next
+; CHECK:       for.first.peel.next:
+; CHECK-NEXT:    br label %for.first.peel2
+; CHECK:       for.first.peel2:
+; CHECK:         br label %for.first.latch.peel10
+; CHECK:       for.first.latch.peel10:
+; CHECK:         br label %for.first.peel.next1
+; CHECK:       for.first.peel.next1:
+; CHECK-NEXT:    br label %for.first.peel15
+; CHECK:       for.first.peel15:
+; CHECK:         br label %for.first.latch.peel23
+; CHECK:       for.first.latch.peel23:
+; CHECK:         br label %for.first.peel.next14
+; CHECK:       for.first.peel.next14:
+; CHECK-NEXT:    br label %for.first.peel.next27
+; CHECK:       for.first.peel.next27:
+; CHECK-NEXT:    br label %for.first.preheader.peel.newph
+; CHECK:       for.first.preheader.peel.newph:
+; CHECK-NEXT:    br label %for.first
+; CHECK:       for.first:
+; CHECK:         br label %for.first.latch
+; CHECK:       for.first.latch:
+; CHECK:         br label %for.second.latch
+; CHECK:       for.second.latch:
+; CHECK:         br i1 %exitcond, label %for.first, label %for.end
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+
+@B = common global [1024 x i32] zeroinitializer, align 16
+
+define void @function(i32* noalias %arg) {
+for.first.preheader:
+  br label %for.first
+
+for.first:                                       ; preds = %for.first.preheader, %for.first.latch
+  %.014 = phi i32 [ 0, %for.first.preheader ], [ %tmp15, %for.first.latch ]
+  %indvars.iv23 = phi i64 [ 0, %for.first.preheader ], [ %indvars.iv.next3, %for.first.latch ]
+  %tmp = add nsw i32 %.014, -3
+  %tmp8 = add nuw nsw i64 %indvars.iv23, 3
+  %tmp9 = trunc i64 %tmp8 to i32
+  %tmp10 = mul nsw i32 %tmp, %tmp9
+  %tmp11 = trunc i64 %indvars.iv23 to i32
+  %tmp12 = srem i32 %tmp10, %tmp11
+  %tmp13 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv23
+  store i32 %tmp12, i32* %tmp13, align 4
+  br label %for.first.latch
+
+for.first.latch:                                 ; preds = %for.first
+  %indvars.iv.next3 = add nuw nsw i64 %indvars.iv23, 1
+  %tmp15 = add nuw nsw i32 %.014, 1
+  %exitcond4 = icmp ne i64 %indvars.iv.next3, 100
+  br i1 %exitcond4, label %for.first, label %for.second.preheader
+
+for.second.preheader:                            ; preds = %for.first.latch
+  br label %for.second
+
+for.second:                                      ; preds = %for.second.preheader, %for.second.latch
+  %.02 = phi i32 [ 0, %for.second.preheader ], [ %tmp28, %for.second.latch ]
+  %indvars.iv1 = phi i64 [ 3, %for.second.preheader ], [ %indvars.iv.next, %for.second.latch ]
+  %tmp20 = add nsw i32 %.02, -3
+  %tmp21 = add nuw nsw i64 %indvars.iv1, 3
+  %tmp22 = trunc i64 %tmp21 to i32
+  %tmp23 = mul nsw i32 %tmp20, %tmp22
+  %tmp24 = trunc i64 %indvars.iv1 to i32
+  %tmp25 = srem i32 %tmp23, %tmp24
+  %tmp26 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv1
+  store i32 %tmp25, i32* %tmp26, align 4
+  br label %for.second.latch
+
+for.second.latch:                                ; preds = %for.second
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv1, 1
+  %tmp28 = add nuw nsw i32 %.02, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, 100
+  br i1 %exitcond, label %for.second, label %for.end
+
+for.end:                                        ; preds = %for.second.latch
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopIdiom/phi-insertion.ll b/llvm/test/Transforms/LoopIdiom/phi-insertion.ll
new file mode 100644
index 0000000000000..1087a709357d6
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/phi-insertion.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -loop-idiom -S %s | FileCheck %s
+
+define void @phi_insertion(i1 %c, i32* %ptr.start, i32* %ptr.end, i64 %offset) {
+; CHECK-LABEL: @phi_insertion(
+; CHECK-NEXT:  bb5:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[LOOP_0_PH:%.*]], label [[EXIT:%.*]]
+; CHECK:       loop.0.ph:
+; CHECK-NEXT:    br label [[LOOP_0:%.*]]
+; CHECK:       loop.0:
+; CHECK-NEXT:    [[IV_0:%.*]] = phi i32* [ [[IV_0_NEXT:%.*]], [[LOOP_0_LATCH:%.*]] ], [ [[PTR_START:%.*]], [[LOOP_0_PH]] ]
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32* [[IV_0]], [[PTR_END:%.*]]
+; CHECK-NEXT:    br i1 [[COND]], label [[LOOP_1_PREHEADER:%.*]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       loop.1.preheader:
+; CHECK-NEXT:    br label [[LOOP_1:%.*]]
+; CHECK:       loop.1:
+; CHECK-NEXT:    [[PTR_1:%.*]] = getelementptr inbounds i32, i32* [[IV_0]], i64 0
+; CHECK-NEXT:    [[LV:%.*]] = load i32, i32* [[PTR_1]], align 8
+; CHECK-NEXT:    [[PTR_2:%.*]] = getelementptr inbounds i32, i32* [[IV_0]], i64 0
+; CHECK-NEXT:    store i32 [[LV]], i32* [[PTR_2]], align 8
+; CHECK-NEXT:    [[IV_1_NEXT:%.*]] = add nuw nsw i64 0, 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_1_NEXT]], 100
+; CHECK-NEXT:    br i1 [[EC]], label [[LOOP_0_LATCH]], label [[LOOP_1]]
+; CHECK:       loop.0.latch:
+; CHECK-NEXT:    [[IV_0_NEXT]] = getelementptr inbounds i32, i32* [[IV_0]], i64 [[OFFSET:%.*]]
+; CHECK-NEXT:    br label [[LOOP_0]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+bb5:
+  br i1 %c, label %loop.0.ph, label %exit
+
+loop.0.ph:                                        ; preds = %bb5
+  br label %loop.0
+
+loop.0:                                           ; preds = %loop.0.latch, %loop.0.ph
+  %iv.0 = phi i32* [ %iv.0.next, %loop.0.latch ], [ %ptr.start, %loop.0.ph ]
+  %cond = icmp eq i32* %iv.0, %ptr.end
+  br i1 %cond, label %loop.1, label %exit
+
+loop.1:                                           ; preds = %loop.1, %loop.0
+  %iv.1 = phi i64 [ 0, %loop.0 ], [ %iv.1, %loop.1 ]
+  %ptr.1 = getelementptr inbounds i32, i32* %iv.0, i64 %iv.1
+  %lv = load i32, i32* %ptr.1, align 8
+  %ptr.2 = getelementptr inbounds i32, i32* %iv.0, i64 %iv.1
+  store i32 %lv, i32* %ptr.2, align 8
+  %iv.1.next = add nuw nsw i64 %iv.1, 1
+  %ec = icmp eq i64 %iv.1.next, 100
+  br i1 %ec, label %loop.0.latch, label %loop.1
+
+loop.0.latch:                                     ; preds = %loop.1
+  %iv.0.next = getelementptr inbounds i32, i32* %iv.0, i64 %offset
+  br label %loop.0
+
+exit:                                             ; preds = %loop.0, %bb5
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopIdiom/reuse-cast.ll b/llvm/test/Transforms/LoopIdiom/reuse-cast.ll
new file mode 100644
index 0000000000000..0b1bfeaa27302
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/reuse-cast.ll
@@ -0,0 +1,131 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -loop-idiom -S %s | FileCheck %s
+
+define void @reuse_cast_1(float** %ptr, i1 %c) {
+; CHECK-LABEL: @reuse_cast_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP_0:%.*]]
+; CHECK:       loop.0:
+; CHECK-NEXT:    [[TMP:%.*]] = load float*, float** [[PTR:%.*]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[TMP]] to i8*
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[LOOP_2_PREHEADER:%.*]], label [[LOOP_1_PREHEADER:%.*]]
+; CHECK:       loop.1.preheader:
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 4 [[TMP2]], i8 0, i64 400, i1 false)
+; CHECK-NEXT:    br label [[LOOP_1:%.*]]
+; CHECK:       loop.2.preheader:
+; CHECK-NEXT:    br label [[LOOP_2:%.*]]
+; CHECK:       loop.1:
+; CHECK-NEXT:    [[IV_1:%.*]] = phi i64 [ [[IV_1_NEXT:%.*]], [[LOOP_1]] ], [ 0, [[LOOP_1_PREHEADER]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, float* [[TMP]], i64 [[IV_1]]
+; CHECK-NEXT:    [[IV_1_NEXT]] = add nuw nsw i64 [[IV_1]], 1
+; CHECK-NEXT:    [[C_1:%.*]] = icmp ne i64 [[IV_1_NEXT]], 100
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_1]], label [[LOOP_0_LATCH_LOOPEXIT1:%.*]]
+; CHECK:       loop.2:
+; CHECK-NEXT:    [[IV_2:%.*]] = phi i64 [ [[IV_2_NEXT:%.*]], [[LOOP_2]] ], [ 0, [[LOOP_2_PREHEADER]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, float* [[TMP]], i64 [[IV_2]]
+; CHECK-NEXT:    store float 0.000000e+00, float* [[TMP10]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = load float*, float** [[PTR]], align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, float* [[TMP11]], i64 [[IV_2]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load float, float* [[TMP12]], align 4
+; CHECK-NEXT:    [[IV_2_NEXT]] = add nuw nsw i64 [[IV_2]], 1
+; CHECK-NEXT:    [[C_2:%.*]] = icmp ne i64 [[IV_2_NEXT]], 100
+; CHECK-NEXT:    br i1 [[C_2]], label [[LOOP_2]], label [[LOOP_0_LATCH_LOOPEXIT:%.*]]
+; CHECK:       loop.0.latch.loopexit:
+; CHECK-NEXT:    br label [[LOOP_0_LATCH:%.*]]
+; CHECK:       loop.0.latch.loopexit1:
+; CHECK-NEXT:    br label [[LOOP_0_LATCH]]
+; CHECK:       loop.0.latch:
+; CHECK-NEXT:    [[C_0:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_0]], label [[LOOP_0]], label [[END:%.*]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.0
+
+loop.0:                                           ; preds = %loop.0.latch, %entry
+  %tmp = load float*, float** %ptr, align 8
+  br i1 %c, label %loop.2, label %loop.1
+
+loop.1:                                           ; preds = %loop.1, %loop.0
+  %iv.1 = phi i64 [ %iv.1.next, %loop.1 ], [ 0, %loop.0 ]
+  %tmp4 = getelementptr inbounds float, float* %tmp, i64 %iv.1
+  store float 0.000000e+00, float* %tmp4, align 4
+  %iv.1.next = add nuw nsw i64 %iv.1, 1
+  %c.1 = icmp ne i64 %iv.1.next, 100
+  br i1 %c.1, label %loop.1, label %loop.0.latch
+
+loop.2:                                           ; preds = %loop.2, %loop.0
+  %iv.2 = phi i64 [ %iv.2.next, %loop.2 ], [ 0, %loop.0 ]
+  %tmp10 = getelementptr inbounds float, float* %tmp, i64 %iv.2
+  store float 0.000000e+00, float* %tmp10, align 4
+  %tmp11 = load float*, float** %ptr, align 8
+  %tmp12 = getelementptr inbounds float, float* %tmp11, i64 %iv.2
+  %tmp13 = load float, float* %tmp12, align 4
+  %iv.2.next = add nuw nsw i64 %iv.2, 1
+  %c.2 = icmp ne i64 %iv.2.next, 100
+  br i1 %c.2, label %loop.2, label %loop.0.latch
+
+loop.0.latch:                                     ; preds = %loop.2, %loop.1
+  %c.0 = call i1 @cond()
+  br i1 %c.0, label %loop.0, label %end
+
+end:                                              ; preds = %loop.0.latch
+  ret void
+}
+
+declare i1 @cond()
+
+declare void @use.i8(i8*)
+
+declare void @use.i1(i1)
+
+define void @reuse_cast_2(i32 %x, i32* %ptr.1.start) {
+; CHECK-LABEL: @reuse_cast_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[STACK:%.*]] = alloca [2 x i32], align 4
+; CHECK-NEXT:    [[CAST_TO_REUSE:%.*]] = bitcast [2 x i32]* [[STACK]] to i8*
+; CHECK-NEXT:    [[C_0:%.*]] = icmp sgt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast [2 x i32]* [[STACK]] to i8*
+; CHECK-NEXT:    [[PTR_2_START:%.*]] = getelementptr inbounds [2 x i32], [2 x i32]* [[STACK]], i64 0, i64 0
+; CHECK-NEXT:    call void @use.i8(i8* [[CAST_TO_REUSE]])
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[PTR_1:%.*]] = phi i32* [ [[PTR_1_START:%.*]], [[ENTRY]] ], [ [[PTR_1_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[PTR_2:%.*]] = phi i32* [ [[PTR_2_START]], [[ENTRY]] ], [ [[PTR_2_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[PTR_1_NEXT]] = getelementptr inbounds i32, i32* [[PTR_1]], i64 1
+; CHECK-NEXT:    [[LV:%.*]] = load i32, i32* [[PTR_1]], align 4
+; CHECK-NEXT:    [[PTR_2_NEXT]] = getelementptr inbounds i32, i32* [[PTR_2]], i64 1
+; CHECK-NEXT:    store i32 [[LV]], i32* [[PTR_2]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    [[C_1:%.*]] = icmp eq i32 [[IV]], 0
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    call void @use.i1(i1 [[C_0]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %stack = alloca [2 x i32], align 4
+  %c.0 = icmp sgt i32 %x, 0
+  %cast.to.reuse = bitcast [2 x i32]* %stack to i8*
+  %ptr.2.start = getelementptr inbounds [2 x i32], [2 x i32]* %stack, i64 0, i64 0
+  call void @use.i8(i8* %cast.to.reuse)
+  br label %loop
+
+loop:                                             ; preds = %loop, %entry
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %ptr.1 = phi i32* [ %ptr.1.start, %entry ], [ %ptr.1.next, %loop ]
+  %ptr.2 = phi i32* [ %ptr.2.start, %entry ], [ %ptr.2.next, %loop ]
+  %ptr.1.next = getelementptr inbounds i32, i32* %ptr.1, i64 1
+  %lv = load i32, i32* %ptr.1, align 4
+  %ptr.2.next = getelementptr inbounds i32, i32* %ptr.2, i64 1
+  store i32 %lv, i32* %ptr.2, align 4
+  %iv.next = add nuw nsw i32 %iv, 1
+  %c.1 = icmp eq i32 %iv, 0
+  br i1 %c.1, label %loop, label %exit
+
+exit:                                             ; preds = %loop
+  call void @use.i1(i1 %c.0)
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopReroll/nonconst_lb.ll b/llvm/test/Transforms/LoopReroll/nonconst_lb.ll
index 46a5805d63706..92d893fabbd22 100644
--- a/llvm/test/Transforms/LoopReroll/nonconst_lb.ll
+++ b/llvm/test/Transforms/LoopReroll/nonconst_lb.ll
@@ -48,7 +48,7 @@ for.end:                                          ; preds = %for.body, %entry
   ret void
 }
 ; CHECK-LABEL: @foo
-; CHECK: for.body.preheader:                               ; preds = %entry
+; CHECK: for.body.preheader:
 ; CHECK:   %0 = add i32 %n, -1
 ; CHECK:   %1 = sub i32 %0, %m
 ; CHECK:   %2 = lshr i32 %1, 2
@@ -56,7 +56,7 @@ for.end:                                          ; preds = %for.body, %entry
 ; CHECK:   %4 = add i32 %3, 3
 ; CHECK:   br label %for.body
 
-; CHECK: for.body:                                         ; preds = %for.body, %for.body.preheader
+; CHECK: for.body:
 ; CHECK:   %indvar = phi i32 [ 0, %for.body.preheader ], [ %indvar.next, %for.body ]
 ; CHECK:   %5 = add i32 %m, %indvar
 ; CHECK:   %arrayidx = getelementptr inbounds i32, i32* %B, i32 %5
diff --git a/llvm/test/Transforms/LoopStrengthReduce/pr27056.ll b/llvm/test/Transforms/LoopStrengthReduce/pr27056.ll
index 57dc8cd45fe73..a1e37443b3a72 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/pr27056.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/pr27056.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -loop-reduce -S | FileCheck %s
 target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-windows-msvc18.0.0"
@@ -10,6 +11,41 @@ declare i32 @__CxxFrameHandler3(...)
 @GV2 = external global %struct.L
 
 define void @b_copy_ctor() personality i32 (...)* @__CxxFrameHandler3 {
+; CHECK-LABEL: @b_copy_ctor(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load %struct.L*, %struct.L** @GV1, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint %struct.L* [[TMP0]] to i64
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i64 [ [[LSR_IV_NEXT:%.*]], [[CALL_I_NOEXC:%.*]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP:%.*]] = inttoptr i64 [[LSR_IV]] to %struct.L*
+; CHECK-NEXT:    invoke void @a_copy_ctor()
+; CHECK-NEXT:    to label [[CALL_I_NOEXC]] unwind label [[CATCH_DISPATCH:%.*]]
+; CHECK:       call.i.noexc:
+; CHECK-NEXT:    [[LSR_IV_NEXT]] = add i64 [[LSR_IV]], 16
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       catch.dispatch:
+; CHECK-NEXT:    [[TMP2:%.*]] = catchswitch within none [label %catch] unwind to caller
+; CHECK:       catch:
+; CHECK-NEXT:    [[TMP3:%.*]] = catchpad within [[TMP2]] [i8* null, i32 64, i8* null]
+; CHECK-NEXT:    [[CMP16:%.*]] = icmp eq %struct.L* [[TMP]], null
+; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 0, [[TMP1]]
+; CHECK-NEXT:    [[UGLYGEP:%.*]] = getelementptr i8, i8* getelementptr inbounds (%struct.L, %struct.L* @GV2, i32 0, i32 0), i64 [[TMP4]]
+; CHECK-NEXT:    [[UGLYGEP1:%.*]] = bitcast i8* [[UGLYGEP]] to %struct.L*
+; CHECK-NEXT:    br i1 [[CMP16]], label [[FOR_END:%.*]], label [[FOR_BODY_PREHEADER:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[LSR_IV]] to %struct.L*
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq %struct.L* [[UGLYGEP1]], [[TMP]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    catchret from [[TMP3]] to label [[TRY_CONT:%.*]]
+; CHECK:       try.cont:
+; CHECK-NEXT:    ret void
+;
 entry:
   %0 = load %struct.L*, %struct.L** @GV1, align 8
   br label %for.cond
@@ -17,7 +53,7 @@ entry:
 for.cond:                                         ; preds = %call.i.noexc, %entry
   %d.0 = phi %struct.L* [ %0, %entry ], [ %incdec.ptr, %call.i.noexc ]
   invoke void @a_copy_ctor()
-          to label %call.i.noexc unwind label %catch.dispatch
+  to label %call.i.noexc unwind label %catch.dispatch
 
 call.i.noexc:                                     ; preds = %for.cond
   %incdec.ptr = getelementptr inbounds %struct.L, %struct.L* %d.0, i64 1
@@ -42,10 +78,4 @@ try.cont:                                         ; preds = %for.end
   ret void
 }
 
-; CHECK-LABEL: define void @b_copy_ctor(
-; CHECK:       catchpad
-; CHECK-NEXT:  icmp eq %struct.L
-; CHECK-NEXT:  %4 = sub i64 0, %1
-; CHECK-NEXT:  getelementptr {{.*}} getelementptr inbounds (%struct.L, %struct.L* @GV2, i32 0, i32 0), i64 %4
-
 declare void @a_copy_ctor()
diff --git a/llvm/test/Transforms/LoopVectorize/optsize.ll b/llvm/test/Transforms/LoopVectorize/optsize.ll
index 0e88f362746fb..b4233e6751cbc 100644
--- a/llvm/test/Transforms/LoopVectorize/optsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/optsize.ll
@@ -121,6 +121,38 @@ for.body29:
   br i1 %cmp26, label %for.body29, label %for.cond.cleanup28
 }
 
+define void @pr43371_pgso() !prof !14 {
+;
+; CHECK-LABEL: @pr43371_pgso
+; CHECK-NOT:   vector.scevcheck
+;
+; We do not want to generate SCEV predicates when optimising for size, because
+; that will lead to extra code generation such as the SCEV overflow runtime
+; checks. Not generating SCEV predicates can still result in vectorisation as
+; the non-consecutive loads/stores can be scalarized:
+;
+; CHECK: vector.body:
+; CHECK: store i16 0, i16* %{{.*}}, align 1
+; CHECK: store i16 0, i16* %{{.*}}, align 1
+; CHECK: br i1 {{.*}}, label %vector.body
+;
+entry:
+  br label %for.body29
+
+for.cond.cleanup28:
+  unreachable
+
+for.body29:
+  %i24.0170 = phi i16 [ 0, %entry], [ %inc37, %for.body29]
+  %add33 = add i16 undef, %i24.0170
+  %idxprom34 = zext i16 %add33 to i32
+  %arrayidx35 = getelementptr [2592 x i16], [2592 x i16] * @cm_array, i32 0, i32 %idxprom34
+  store i16 0, i16 * %arrayidx35, align 1
+  %inc37 = add i16 %i24.0170, 1
+  %cmp26 = icmp ult i16 %inc37, 756
+  br i1 %cmp26, label %for.body29, label %for.cond.cleanup28
+}
+
 ; PR45526: don't vectorize with fold-tail if first-order-recurrence is live-out.
 ;
 define i32 @pr45526() optsize {
@@ -154,6 +186,37 @@ exit:
   ret i32 %for
 }
 
+define i32 @pr45526_pgso() !prof !14 {
+;
+; CHECK-LABEL: @pr45526_pgso
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   br label %loop
+; CHECK-EMPTY:
+; CHECK-NEXT: loop:
+; CHECK-NEXT:   %piv = phi i32 [ 0, %entry ], [ %pivPlus1, %loop ]
+; CHECK-NEXT:   %for = phi i32 [ 5, %entry ], [ %pivPlus1, %loop ]
+; CHECK-NEXT:   %pivPlus1 = add nuw nsw i32 %piv, 1
+; CHECK-NEXT:   %cond = icmp ult i32 %piv, 510
+; CHECK-NEXT:   br i1 %cond, label %loop, label %exit
+; CHECK-EMPTY:
+; CHECK-NEXT: exit:
+; CHECK-NEXT:   %for.lcssa = phi i32 [ %for, %loop ]
+; CHECK-NEXT:   ret i32 %for.lcssa
+;
+entry:
+  br label %loop
+
+loop:
+  %piv = phi i32 [ 0, %entry ], [ %pivPlus1, %loop ]
+  %for = phi i32 [ 5, %entry ], [ %pivPlus1, %loop ]
+  %pivPlus1 = add nuw nsw i32 %piv, 1
+  %cond = icmp ult i32 %piv, 510
+  br i1 %cond, label %loop, label %exit
+
+exit:
+  ret i32 %for
+}
+
 ; PR46228: Vectorize w/o versioning for unit stride under optsize and enabled
 ; vectorization.
 
@@ -190,7 +253,7 @@ define void @stride1(i16* noalias %B, i32 %BStride) optsize {
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], <i32 2, i32 2>
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1026
-; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !19
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !21
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
diff --git a/llvm/test/Transforms/LowerConstantIntrinsics/objectsize_basic.ll b/llvm/test/Transforms/LowerConstantIntrinsics/objectsize_basic.ll
index ee5d792d53299..ab59b8950d700 100644
--- a/llvm/test/Transforms/LowerConstantIntrinsics/objectsize_basic.ll
+++ b/llvm/test/Transforms/LowerConstantIntrinsics/objectsize_basic.ll
@@ -89,3 +89,11 @@ define i64 @test_objectsize_byval_arg([42 x i8]* byval([42 x i8]) %ptr) {
   %size = tail call i64 @llvm.objectsize.i64(i8* %cast, i1 true, i1 false, i1 false)
   ret i64 %size
 }
+
+; CHECK-LABEL: @test_objectsize_byref_arg(
+; CHECK: ret i64 42
+define i64 @test_objectsize_byref_arg([42 x i8]* byref([42 x i8]) %ptr) {
+  %cast = bitcast [42 x i8]* %ptr to i8*
+  %size = tail call i64 @llvm.objectsize.i64(i8* %cast, i1 true, i1 false, i1 false)
+  ret i64 %size
+}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-loops.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-loops.ll
new file mode 100644
index 0000000000000..d0503b7371d42
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-loops.ll
@@ -0,0 +1,397 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-matrix-intrinsics -fuse-matrix-use-loops -fuse-matrix-tile-size=2 -matrix-allow-contract -force-fuse-matrix -instcombine -verify-dom-info %s -S | FileCheck %s
+
+; REQUIRES: aarch64-registered-target
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "aarch64-apple-ios"
+
+define void @multiply_noalias_4x4(<16 x double>* noalias %A, <16 x double>* noalias %B, <16 x double>* noalias %C) {
+; CHECK-LABEL: @multiply_noalias_4x4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[COLS_HEADER:%.*]]
+; CHECK:       cols.header:
+; CHECK-NEXT:    [[COLS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[COLS_STEP:%.*]], [[COLS_LATCH:%.*]] ]
+; CHECK-NEXT:    br label [[COLS_BODY:%.*]]
+; CHECK:       cols.body:
+; CHECK-NEXT:    br label [[ROWS_HEADER:%.*]]
+; CHECK:       rows.header:
+; CHECK-NEXT:    [[ROWS_IV:%.*]] = phi i64 [ 0, [[COLS_BODY]] ], [ [[ROWS_STEP:%.*]], [[ROWS_LATCH:%.*]] ]
+; CHECK-NEXT:    br label [[ROWS_BODY:%.*]]
+; CHECK:       rows.body:
+; CHECK-NEXT:    br label [[INNER_HEADER:%.*]]
+; CHECK:       inner.header:
+; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, [[ROWS_BODY]] ], [ [[INNER_STEP:%.*]], [[INNER_LATCH:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP9:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP11:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT:    br label [[INNER_BODY:%.*]]
+; CHECK:       inner.body:
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[INNER_IV]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[TMP2]], [[ROWS_IV]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr <16 x double>, <16 x double>* [[A:%.*]], i64 0, i64 [[TMP3]]
+; CHECK-NEXT:    [[VEC_CAST:%.*]] = bitcast double* [[TMP4]] to <2 x double>*
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST]], align 8
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP4]], i64 4
+; CHECK-NEXT:    [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
+; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST1]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[COLS_IV]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP5]], [[INNER_IV]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr <16 x double>, <16 x double>* [[B:%.*]], i64 0, i64 [[TMP6]]
+; CHECK-NEXT:    [[VEC_CAST4:%.*]] = bitcast double* [[TMP7]] to <2 x double>*
+; CHECK-NEXT:    [[COL_LOAD5:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST4]], align 8
+; CHECK-NEXT:    [[VEC_GEP6:%.*]] = getelementptr double, double* [[TMP7]], i64 4
+; CHECK-NEXT:    [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <2 x double>*
+; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST7]], align 8
+; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD5]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD]], <2 x double> [[SPLAT_SPLAT]], <2 x double> [[TMP0]])
+; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x double> [[COL_LOAD5]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP9]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD2]], <2 x double> [[SPLAT_SPLAT12]], <2 x double> [[TMP8]])
+; CHECK-NEXT:    [[SPLAT_SPLAT16:%.*]] = shufflevector <2 x double> [[COL_LOAD8]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD]], <2 x double> [[SPLAT_SPLAT16]], <2 x double> [[TMP1]])
+; CHECK-NEXT:    [[SPLAT_SPLAT19:%.*]] = shufflevector <2 x double> [[COL_LOAD8]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP11]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD2]], <2 x double> [[SPLAT_SPLAT19]], <2 x double> [[TMP10]])
+; CHECK-NEXT:    br label [[INNER_LATCH]]
+; CHECK:       inner.latch:
+; CHECK-NEXT:    [[INNER_STEP]] = add i64 [[INNER_IV]], 2
+; CHECK-NEXT:    [[INNER_COND_NOT:%.*]] = icmp eq i64 [[INNER_STEP]], 4
+; CHECK-NEXT:    br i1 [[INNER_COND_NOT]], label [[ROWS_LATCH]], label [[INNER_HEADER]], !llvm.loop !0
+; CHECK:       rows.latch:
+; CHECK-NEXT:    [[ROWS_STEP]] = add i64 [[ROWS_IV]], 2
+; CHECK-NEXT:    [[ROWS_COND_NOT:%.*]] = icmp eq i64 [[ROWS_STEP]], 4
+; CHECK-NEXT:    [[TMP12:%.*]] = shl i64 [[COLS_IV]], 2
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP12]], [[ROWS_IV]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr <16 x double>, <16 x double>* [[C:%.*]], i64 0, i64 [[TMP13]]
+; CHECK-NEXT:    [[VEC_CAST21:%.*]] = bitcast double* [[TMP14]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP9]], <2 x double>* [[VEC_CAST21]], align 8
+; CHECK-NEXT:    [[VEC_GEP22:%.*]] = getelementptr double, double* [[TMP14]], i64 4
+; CHECK-NEXT:    [[VEC_CAST23:%.*]] = bitcast double* [[VEC_GEP22]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP11]], <2 x double>* [[VEC_CAST23]], align 8
+; CHECK-NEXT:    br i1 [[ROWS_COND_NOT]], label [[COLS_LATCH]], label [[ROWS_HEADER]]
+; CHECK:       cols.latch:
+; CHECK-NEXT:    [[COLS_STEP]] = add i64 [[COLS_IV]], 2
+; CHECK-NEXT:    [[COLS_COND_NOT:%.*]] = icmp eq i64 [[COLS_STEP]], 4
+; CHECK-NEXT:    br i1 [[COLS_COND_NOT]], label [[CONTINUE:%.*]], label [[COLS_HEADER]]
+; CHECK:       continue:
+; CHECK-NEXT:    ret void
+;
+
+entry:
+  %a = load <16 x double>, <16 x double>* %A, align 8
+  %b = load <16 x double>, <16 x double>* %B, align 8
+
+  %c = call <16 x double> @llvm.matrix.multiply.v16f64.v16f64.v16f64(<16 x double> %a, <16 x double> %b, i32 4, i32 4, i32 4)
+
+  store <16 x double> %c, <16 x double>* %C, align 8
+  ret void
+}
+
+
+declare <16 x double> @llvm.matrix.multiply.v16f64.v16f64.v16f64(<16 x double>, <16 x double>, i32, i32, i32)
+
+define void @multiply_noalias_2x4(<8 x i64>* noalias %A, <8 x i64>* noalias %B, <4 x i64>* noalias %C) {
+; CHECK-LABEL: @multiply_noalias_2x4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[COLS_HEADER:%.*]]
+; CHECK:       cols.header:
+; CHECK-NEXT:    [[COLS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[COLS_STEP:%.*]], [[COLS_LATCH:%.*]] ]
+; CHECK-NEXT:    br label [[COLS_BODY:%.*]]
+; CHECK:       cols.body:
+; CHECK-NEXT:    br label [[ROWS_HEADER:%.*]]
+; CHECK:       rows.header:
+; CHECK-NEXT:    [[ROWS_IV:%.*]] = phi i64 [ 0, [[COLS_BODY]] ], [ [[ROWS_STEP:%.*]], [[ROWS_LATCH:%.*]] ]
+; CHECK-NEXT:    br label [[ROWS_BODY:%.*]]
+; CHECK:       rows.body:
+; CHECK-NEXT:    br label [[INNER_HEADER:%.*]]
+; CHECK:       inner.header:
+; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, [[ROWS_BODY]] ], [ [[INNER_STEP:%.*]], [[INNER_LATCH:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x i64> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP11:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x i64> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP15:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT:    br label [[INNER_BODY:%.*]]
+; CHECK:       inner.body:
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[INNER_IV]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[TMP2]], [[ROWS_IV]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr <8 x i64>, <8 x i64>* [[A:%.*]], i64 0, i64 [[TMP3]]
+; CHECK-NEXT:    [[VEC_CAST:%.*]] = bitcast i64* [[TMP4]] to <2 x i64>*
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x i64>, <2 x i64>* [[VEC_CAST]], align 8
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr i64, i64* [[TMP4]], i64 2
+; CHECK-NEXT:    [[VEC_CAST1:%.*]] = bitcast i64* [[VEC_GEP]] to <2 x i64>*
+; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <2 x i64>, <2 x i64>* [[VEC_CAST1]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[COLS_IV]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP5]], [[INNER_IV]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr <8 x i64>, <8 x i64>* [[B:%.*]], i64 0, i64 [[TMP6]]
+; CHECK-NEXT:    [[VEC_CAST4:%.*]] = bitcast i64* [[TMP7]] to <2 x i64>*
+; CHECK-NEXT:    [[COL_LOAD5:%.*]] = load <2 x i64>, <2 x i64>* [[VEC_CAST4]], align 8
+; CHECK-NEXT:    [[VEC_GEP6:%.*]] = getelementptr i64, i64* [[TMP7]], i64 4
+; CHECK-NEXT:    [[VEC_CAST7:%.*]] = bitcast i64* [[VEC_GEP6]] to <2 x i64>*
+; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <2 x i64>, <2 x i64>* [[VEC_CAST7]], align 8
+; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <2 x i64> [[COL_LOAD5]], <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = mul <2 x i64> [[COL_LOAD]], [[SPLAT_SPLAT]]
+; CHECK-NEXT:    [[TMP9:%.*]] = add <2 x i64> [[TMP0]], [[TMP8]]
+; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x i64> [[COL_LOAD5]], <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP10:%.*]] = mul <2 x i64> [[COL_LOAD2]], [[SPLAT_SPLAT12]]
+; CHECK-NEXT:    [[TMP11]] = add <2 x i64> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[SPLAT_SPLAT16:%.*]] = shufflevector <2 x i64> [[COL_LOAD8]], <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = mul <2 x i64> [[COL_LOAD]], [[SPLAT_SPLAT16]]
+; CHECK-NEXT:    [[TMP13:%.*]] = add <2 x i64> [[TMP1]], [[TMP12]]
+; CHECK-NEXT:    [[SPLAT_SPLAT19:%.*]] = shufflevector <2 x i64> [[COL_LOAD8]], <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP14:%.*]] = mul <2 x i64> [[COL_LOAD2]], [[SPLAT_SPLAT19]]
+; CHECK-NEXT:    [[TMP15]] = add <2 x i64> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    br label [[INNER_LATCH]]
+; CHECK:       inner.latch:
+; CHECK-NEXT:    [[INNER_STEP]] = add i64 [[INNER_IV]], 2
+; CHECK-NEXT:    [[INNER_COND_NOT:%.*]] = icmp eq i64 [[INNER_STEP]], 4
+; CHECK-NEXT:    br i1 [[INNER_COND_NOT]], label [[ROWS_LATCH]], label [[INNER_HEADER]], !llvm.loop !2
+; CHECK:       rows.latch:
+; CHECK-NEXT:    [[ROWS_STEP]] = add i64 [[ROWS_IV]], 2
+; CHECK-NEXT:    [[ROWS_COND_NOT:%.*]] = icmp eq i64 [[ROWS_IV]], 0
+; CHECK-NEXT:    [[TMP16:%.*]] = shl i64 [[COLS_IV]], 1
+; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[TMP16]], [[ROWS_IV]]
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr <4 x i64>, <4 x i64>* [[C:%.*]], i64 0, i64 [[TMP17]]
+; CHECK-NEXT:    [[VEC_CAST21:%.*]] = bitcast i64* [[TMP18]] to <2 x i64>*
+; CHECK-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* [[VEC_CAST21]], align 8
+; CHECK-NEXT:    [[VEC_GEP22:%.*]] = getelementptr i64, i64* [[TMP18]], i64 2
+; CHECK-NEXT:    [[VEC_CAST23:%.*]] = bitcast i64* [[VEC_GEP22]] to <2 x i64>*
+; CHECK-NEXT:    store <2 x i64> [[TMP15]], <2 x i64>* [[VEC_CAST23]], align 8
+; CHECK-NEXT:    br i1 [[ROWS_COND_NOT]], label [[COLS_LATCH]], label [[ROWS_HEADER]]
+; CHECK:       cols.latch:
+; CHECK-NEXT:    [[COLS_STEP]] = add i64 [[COLS_IV]], 2
+; CHECK-NEXT:    [[COLS_COND_NOT:%.*]] = icmp eq i64 [[COLS_IV]], 0
+; CHECK-NEXT:    br i1 [[COLS_COND_NOT]], label [[CONTINUE:%.*]], label [[COLS_HEADER]]
+; CHECK:       continue:
+; CHECK-NEXT:    ret void
+;
+
+; In the inner loop, compute
+;   Result += Load(A, ROWS_IV, INNER_IV) * Load(B, INNER_IV, COLS_IV)
+
+
+; Store the current 2x2 tile.
+
+entry:
+  %a = load <8 x i64>, <8 x i64>* %A, align 8
+  %b = load <8 x i64>, <8 x i64>* %B, align 8
+
+  %c = call <4 x i64> @llvm.matrix.multiply.v4i64.v8i64.v8i64(<8 x i64> %a, <8 x i64> %b, i32 2, i32 4, i32 2)
+
+  store <4 x i64> %c, <4 x i64>* %C, align 8
+  ret void
+}
+
+
+declare <4 x i64> @llvm.matrix.multiply.v4i64.v8i64.v8i64(<8 x i64>, <8 x i64>, i32, i32, i32)
+
+define void @multiply_noalias_4x2_2x8(<8 x i64>* noalias %A, <16 x i64>* noalias %B, <32 x i64>* noalias %C) {
+; CHECK-LABEL: @multiply_noalias_4x2_2x8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[COLS_HEADER:%.*]]
+; CHECK:       cols.header:
+; CHECK-NEXT:    [[COLS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[COLS_STEP:%.*]], [[COLS_LATCH:%.*]] ]
+; CHECK-NEXT:    br label [[COLS_BODY:%.*]]
+; CHECK:       cols.body:
+; CHECK-NEXT:    br label [[ROWS_HEADER:%.*]]
+; CHECK:       rows.header:
+; CHECK-NEXT:    [[ROWS_IV:%.*]] = phi i64 [ 0, [[COLS_BODY]] ], [ [[ROWS_STEP:%.*]], [[ROWS_LATCH:%.*]] ]
+; CHECK-NEXT:    br label [[ROWS_BODY:%.*]]
+; CHECK:       rows.body:
+; CHECK-NEXT:    br label [[INNER_HEADER:%.*]]
+; CHECK:       inner.header:
+; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, [[ROWS_BODY]] ], [ [[INNER_STEP:%.*]], [[INNER_LATCH:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x i64> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP11:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x i64> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP15:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT:    br label [[INNER_BODY:%.*]]
+; CHECK:       inner.body:
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[INNER_IV]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[TMP2]], [[ROWS_IV]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr <8 x i64>, <8 x i64>* [[A:%.*]], i64 0, i64 [[TMP3]]
+; CHECK-NEXT:    [[VEC_CAST:%.*]] = bitcast i64* [[TMP4]] to <2 x i64>*
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x i64>, <2 x i64>* [[VEC_CAST]], align 8
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr i64, i64* [[TMP4]], i64 4
+; CHECK-NEXT:    [[VEC_CAST1:%.*]] = bitcast i64* [[VEC_GEP]] to <2 x i64>*
+; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <2 x i64>, <2 x i64>* [[VEC_CAST1]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[COLS_IV]], 1
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP5]], [[INNER_IV]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr <16 x i64>, <16 x i64>* [[B:%.*]], i64 0, i64 [[TMP6]]
+; CHECK-NEXT:    [[VEC_CAST4:%.*]] = bitcast i64* [[TMP7]] to <2 x i64>*
+; CHECK-NEXT:    [[COL_LOAD5:%.*]] = load <2 x i64>, <2 x i64>* [[VEC_CAST4]], align 8
+; CHECK-NEXT:    [[VEC_GEP6:%.*]] = getelementptr i64, i64* [[TMP7]], i64 2
+; CHECK-NEXT:    [[VEC_CAST7:%.*]] = bitcast i64* [[VEC_GEP6]] to <2 x i64>*
+; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <2 x i64>, <2 x i64>* [[VEC_CAST7]], align 8
+; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <2 x i64> [[COL_LOAD5]], <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = mul <2 x i64> [[COL_LOAD]], [[SPLAT_SPLAT]]
+; CHECK-NEXT:    [[TMP9:%.*]] = add <2 x i64> [[TMP0]], [[TMP8]]
+; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x i64> [[COL_LOAD5]], <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP10:%.*]] = mul <2 x i64> [[COL_LOAD2]], [[SPLAT_SPLAT12]]
+; CHECK-NEXT:    [[TMP11]] = add <2 x i64> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[SPLAT_SPLAT16:%.*]] = shufflevector <2 x i64> [[COL_LOAD8]], <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = mul <2 x i64> [[COL_LOAD]], [[SPLAT_SPLAT16]]
+; CHECK-NEXT:    [[TMP13:%.*]] = add <2 x i64> [[TMP1]], [[TMP12]]
+; CHECK-NEXT:    [[SPLAT_SPLAT19:%.*]] = shufflevector <2 x i64> [[COL_LOAD8]], <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP14:%.*]] = mul <2 x i64> [[COL_LOAD2]], [[SPLAT_SPLAT19]]
+; CHECK-NEXT:    [[TMP15]] = add <2 x i64> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    br label [[INNER_LATCH]]
+; CHECK:       inner.latch:
+; CHECK-NEXT:    [[INNER_STEP]] = add i64 [[INNER_IV]], 2
+; CHECK-NEXT:    [[INNER_COND_NOT:%.*]] = icmp eq i64 [[INNER_IV]], 0
+; CHECK-NEXT:    br i1 [[INNER_COND_NOT]], label [[ROWS_LATCH]], label [[INNER_HEADER]], !llvm.loop !3
+; CHECK:       rows.latch:
+; CHECK-NEXT:    [[ROWS_STEP]] = add i64 [[ROWS_IV]], 2
+; CHECK-NEXT:    [[ROWS_COND_NOT:%.*]] = icmp eq i64 [[ROWS_STEP]], 4
+; CHECK-NEXT:    [[TMP16:%.*]] = shl i64 [[COLS_IV]], 2
+; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[TMP16]], [[ROWS_IV]]
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr <32 x i64>, <32 x i64>* [[C:%.*]], i64 0, i64 [[TMP17]]
+; CHECK-NEXT:    [[VEC_CAST21:%.*]] = bitcast i64* [[TMP18]] to <2 x i64>*
+; CHECK-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* [[VEC_CAST21]], align 8
+; CHECK-NEXT:    [[VEC_GEP22:%.*]] = getelementptr i64, i64* [[TMP18]], i64 4
+; CHECK-NEXT:    [[VEC_CAST23:%.*]] = bitcast i64* [[VEC_GEP22]] to <2 x i64>*
+; CHECK-NEXT:    store <2 x i64> [[TMP15]], <2 x i64>* [[VEC_CAST23]], align 8
+; CHECK-NEXT:    br i1 [[ROWS_COND_NOT]], label [[COLS_LATCH]], label [[ROWS_HEADER]]
+; CHECK:       cols.latch:
+; CHECK-NEXT:    [[COLS_STEP]] = add i64 [[COLS_IV]], 2
+; CHECK-NEXT:    [[COLS_COND_NOT:%.*]] = icmp eq i64 [[COLS_STEP]], 8
+; CHECK-NEXT:    br i1 [[COLS_COND_NOT]], label [[CONTINUE:%.*]], label [[COLS_HEADER]]
+; CHECK:       continue:
+; CHECK-NEXT:    ret void
+;
+
+; In the inner loop, compute
+;   Result += Load(A, ROWS_IV, INNER_IV) * Load(B, INNER_IV, COLS_IV)
+
+
+; Store the current 2x2 tile.
+
+entry:
+  %a = load <8 x i64>, <8 x i64>* %A, align 8
+  %b = load <16 x i64>, <16 x i64>* %B, align 8
+
+  %c = call <32 x i64> @llvm.matrix.multiply.v32i64.v8i64.v16i64(<8 x i64> %a, <16 x i64> %b, i32 4, i32 2, i32 8)
+
+  store <32 x i64> %c, <32 x i64>* %C, align 8
+  ret void
+}
+
+declare <32 x i64> @llvm.matrix.multiply.v32i64.v8i64.v16i64(<8 x i64>, <16 x i64>, i32, i32, i32)
+
+
+; Check the runtime aliasing checks.
+define void @multiply_alias_2x2(<4 x float>* %A, <4 x float>* %B, <4 x float>* %C) {
+; CHECK-LABEL: @multiply_alias_2x2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[STORE_BEGIN:%.*]] = ptrtoint <4 x float>* [[C:%.*]] to i64
+; CHECK-NEXT:    [[STORE_END:%.*]] = add nuw nsw i64 [[STORE_BEGIN]], 16
+; CHECK-NEXT:    [[LOAD_BEGIN:%.*]] = ptrtoint <4 x float>* [[A:%.*]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ugt i64 [[STORE_END]], [[LOAD_BEGIN]]
+; CHECK-NEXT:    br i1 [[TMP0]], label [[ALIAS_CONT:%.*]], label [[NO_ALIAS:%.*]]
+; CHECK:       alias_cont:
+; CHECK-NEXT:    [[LOAD_END:%.*]] = add nuw nsw i64 [[LOAD_BEGIN]], 16
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i64 [[LOAD_END]], [[STORE_BEGIN]]
+; CHECK-NEXT:    br i1 [[TMP1]], label [[COPY:%.*]], label [[NO_ALIAS]]
+; CHECK:       copy:
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca <4 x float>, align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x float>* [[TMP2]] to i8*
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x float>* [[A]] to i8*
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 16 dereferenceable(16) [[TMP3]], i8* nonnull align 8 dereferenceable(16) [[TMP4]], i64 16, i1 false)
+; CHECK-NEXT:    br label [[NO_ALIAS]]
+; CHECK:       no_alias:
+; CHECK-NEXT:    [[TMP5:%.*]] = phi <4 x float>* [ [[A]], [[ENTRY:%.*]] ], [ [[A]], [[ALIAS_CONT]] ], [ [[TMP2]], [[COPY]] ]
+; CHECK-NEXT:    [[STORE_BEGIN4:%.*]] = ptrtoint <4 x float>* [[C]] to i64
+; CHECK-NEXT:    [[STORE_END5:%.*]] = add nuw nsw i64 [[STORE_BEGIN4]], 16
+; CHECK-NEXT:    [[LOAD_BEGIN6:%.*]] = ptrtoint <4 x float>* [[B:%.*]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ugt i64 [[STORE_END5]], [[LOAD_BEGIN6]]
+; CHECK-NEXT:    br i1 [[TMP6]], label [[ALIAS_CONT1:%.*]], label [[NO_ALIAS3:%.*]]
+; CHECK:       alias_cont1:
+; CHECK-NEXT:    [[LOAD_END7:%.*]] = add nuw nsw i64 [[LOAD_BEGIN6]], 16
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ugt i64 [[LOAD_END7]], [[STORE_BEGIN4]]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[COPY2:%.*]], label [[NO_ALIAS3]]
+; CHECK:       copy2:
+; CHECK-NEXT:    [[TMP8:%.*]] = alloca <4 x float>, align 16
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x float>* [[TMP8]] to i8*
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x float>* [[B]] to i8*
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 16 dereferenceable(16) [[TMP9]], i8* nonnull align 8 dereferenceable(16) [[TMP10]], i64 16, i1 false)
+; CHECK-NEXT:    br label [[NO_ALIAS3]]
+; CHECK:       no_alias3:
+; CHECK-NEXT:    [[TMP11:%.*]] = phi <4 x float>* [ [[B]], [[NO_ALIAS]] ], [ [[B]], [[ALIAS_CONT1]] ], [ [[TMP8]], [[COPY2]] ]
+; CHECK-NEXT:    br label [[COLS_HEADER:%.*]]
+; CHECK:       cols.header:
+; CHECK-NEXT:    [[COLS_IV:%.*]] = phi i64 [ 0, [[NO_ALIAS3]] ], [ [[COLS_STEP:%.*]], [[COLS_LATCH:%.*]] ]
+; CHECK-NEXT:    br label [[COLS_BODY:%.*]]
+; CHECK:       cols.body:
+; CHECK-NEXT:    br label [[ROWS_HEADER:%.*]]
+; CHECK:       rows.header:
+; CHECK-NEXT:    [[ROWS_IV:%.*]] = phi i64 [ 0, [[COLS_BODY]] ], [ [[ROWS_STEP:%.*]], [[ROWS_LATCH:%.*]] ]
+; CHECK-NEXT:    br label [[ROWS_BODY:%.*]]
+; CHECK:       rows.body:
+; CHECK-NEXT:    br label [[INNER_HEADER:%.*]]
+; CHECK:       inner.header:
+; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, [[ROWS_BODY]] ], [ [[INNER_STEP:%.*]], [[INNER_LATCH:%.*]] ]
+; CHECK-NEXT:    [[TMP12:%.*]] = phi <2 x float> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP21:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = phi <2 x float> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP23:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT:    br label [[INNER_BODY:%.*]]
+; CHECK:       inner.body:
+; CHECK-NEXT:    [[TMP14:%.*]] = shl i64 [[INNER_IV]], 1
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[TMP14]], [[ROWS_IV]]
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr <4 x float>, <4 x float>* [[TMP5]], i64 0, i64 [[TMP15]]
+; CHECK-NEXT:    [[VEC_CAST:%.*]] = bitcast float* [[TMP16]] to <2 x float>*
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x float>, <2 x float>* [[VEC_CAST]], align 4
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr float, float* [[TMP16]], i64 2
+; CHECK-NEXT:    [[VEC_CAST8:%.*]] = bitcast float* [[VEC_GEP]] to <2 x float>*
+; CHECK-NEXT:    [[COL_LOAD9:%.*]] = load <2 x float>, <2 x float>* [[VEC_CAST8]], align 4
+; CHECK-NEXT:    [[TMP17:%.*]] = shl i64 [[COLS_IV]], 1
+; CHECK-NEXT:    [[TMP18:%.*]] = add i64 [[TMP17]], [[INNER_IV]]
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr <4 x float>, <4 x float>* [[TMP11]], i64 0, i64 [[TMP18]]
+; CHECK-NEXT:    [[VEC_CAST11:%.*]] = bitcast float* [[TMP19]] to <2 x float>*
+; CHECK-NEXT:    [[COL_LOAD12:%.*]] = load <2 x float>, <2 x float>* [[VEC_CAST11]], align 4
+; CHECK-NEXT:    [[VEC_GEP13:%.*]] = getelementptr float, float* [[TMP19]], i64 2
+; CHECK-NEXT:    [[VEC_CAST14:%.*]] = bitcast float* [[VEC_GEP13]] to <2 x float>*
+; CHECK-NEXT:    [[COL_LOAD15:%.*]] = load <2 x float>, <2 x float>* [[VEC_CAST14]], align 4
+; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <2 x float> [[COL_LOAD12]], <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[COL_LOAD]], <2 x float> [[SPLAT_SPLAT]], <2 x float> [[TMP12]])
+; CHECK-NEXT:    [[SPLAT_SPLAT19:%.*]] = shufflevector <2 x float> [[COL_LOAD12]], <2 x float> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP21]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[COL_LOAD9]], <2 x float> [[SPLAT_SPLAT19]], <2 x float> [[TMP20]])
+; CHECK-NEXT:    [[SPLAT_SPLAT23:%.*]] = shufflevector <2 x float> [[COL_LOAD15]], <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[COL_LOAD]], <2 x float> [[SPLAT_SPLAT23]], <2 x float> [[TMP13]])
+; CHECK-NEXT:    [[SPLAT_SPLAT26:%.*]] = shufflevector <2 x float> [[COL_LOAD15]], <2 x float> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP23]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[COL_LOAD9]], <2 x float> [[SPLAT_SPLAT26]], <2 x float> [[TMP22]])
+; CHECK-NEXT:    br label [[INNER_LATCH]]
+; CHECK:       inner.latch:
+; CHECK-NEXT:    [[INNER_STEP]] = add i64 [[INNER_IV]], 2
+; CHECK-NEXT:    [[INNER_COND_NOT:%.*]] = icmp eq i64 [[INNER_IV]], 0
+; CHECK-NEXT:    br i1 [[INNER_COND_NOT]], label [[ROWS_LATCH]], label [[INNER_HEADER]], !llvm.loop !5
+; CHECK:       rows.latch:
+; CHECK-NEXT:    [[ROWS_STEP]] = add i64 [[ROWS_IV]], 2
+; CHECK-NEXT:    [[ROWS_COND_NOT:%.*]] = icmp eq i64 [[ROWS_IV]], 0
+; CHECK-NEXT:    [[TMP24:%.*]] = shl i64 [[COLS_IV]], 1
+; CHECK-NEXT:    [[TMP25:%.*]] = add i64 [[TMP24]], [[ROWS_IV]]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr <4 x float>, <4 x float>* [[C]], i64 0, i64 [[TMP25]]
+; CHECK-NEXT:    [[VEC_CAST28:%.*]] = bitcast float* [[TMP26]] to <2 x float>*
+; CHECK-NEXT:    store <2 x float> [[TMP21]], <2 x float>* [[VEC_CAST28]], align 8
+; CHECK-NEXT:    [[VEC_GEP29:%.*]] = getelementptr float, float* [[TMP26]], i64 2
+; CHECK-NEXT:    [[VEC_CAST30:%.*]] = bitcast float* [[VEC_GEP29]] to <2 x float>*
+; CHECK-NEXT:    store <2 x float> [[TMP23]], <2 x float>* [[VEC_CAST30]], align 8
+; CHECK-NEXT:    br i1 [[ROWS_COND_NOT]], label [[COLS_LATCH]], label [[ROWS_HEADER]]
+; CHECK:       cols.latch:
+; CHECK-NEXT:    [[COLS_STEP]] = add i64 [[COLS_IV]], 2
+; CHECK-NEXT:    [[COLS_COND_NOT:%.*]] = icmp eq i64 [[COLS_IV]], 0
+; CHECK-NEXT:    br i1 [[COLS_COND_NOT]], label [[CONTINUE:%.*]], label [[COLS_HEADER]]
+; CHECK:       continue:
+; CHECK-NEXT:    ret void
+;
+
+; First, check for aliasing at runtime, create non-aliasing copies if required.
+entry:
+  %a = load <4 x float>, <4 x float>* %A, align 8
+  %b = load <4 x float>, <4 x float>* %B, align 8
+
+  %c = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %a, <4 x float> %b, i32 2, i32 2, i32 2)
+
+  store <4 x float> %c, <4 x float>* %C, align 8
+  ret void
+}
+
+declare <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float>, <4 x float>, i32, i32, i32)
+
+; CHECK:      !0 = distinct !{!0, !1}
+; CHECK-NEXT: !1 = !{!"llvm.loop.unroll.count", i32 2}
+; CHECK-NEXT: !2 = distinct !{!2, !1}
+; CHECK-NEXT: !3 = distinct !{!3, !4}
+; CHECK-NEXT: !4 = !{!"llvm.loop.unroll.count", i32 1}
+; CHECK-NEXT: !5 = distinct !{!5, !4}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-volatile.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-volatile.ll
index 21365202bf19d..3eb1bd40387cc 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-volatile.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-volatile.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -lower-matrix-intrinsics -fuse-matrix-tile-size=2 -matrix-allow-contract -force-fuse-matrix -verify-dom-info %s -S | FileCheck %s
+; RUN: opt -lower-matrix-intrinsics -fuse-matrix-use-loops -fuse-matrix-tile-size=2 -matrix-allow-contract -force-fuse-matrix -verify-dom-info %s -S | FileCheck %s
 
 ; REQUIRES: aarch64-registered-target
 
@@ -9,37 +9,100 @@ target triple = "aarch64-apple-ios"
 define void @multiply_all_volatile(<4 x double>* noalias %A, <4 x double>* noalias %B, <4 x double>* noalias %C) {
 ; CHECK-LABEL: @multiply_all_volatile(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x double>* [[A:%.*]] to double*
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr double, double* [[TMP0]], i64 0
-; CHECK-NEXT:    [[COL_CAST:%.*]] = bitcast double* [[TMP1]] to <4 x double>*
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x double>* [[COL_CAST]] to double*
-; CHECK-NEXT:    [[VEC_CAST:%.*]] = bitcast double* [[TMP2]] to <2 x double>*
-; CHECK-NEXT:    load volatile <2 x double>, <2 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP2]], i64 2
+; CHECK-NEXT:    br label [[COLS_HEADER:%.*]]
+; CHECK:       cols.header:
+; CHECK-NEXT:    [[COLS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[COLS_STEP:%.*]], [[COLS_LATCH:%.*]] ]
+; CHECK-NEXT:    br label [[COLS_BODY:%.*]]
+; CHECK:       cols.body:
+; CHECK-NEXT:    br label [[ROWS_HEADER:%.*]]
+; CHECK:       rows.header:
+; CHECK-NEXT:    [[ROWS_IV:%.*]] = phi i64 [ 0, [[COLS_BODY]] ], [ [[ROWS_STEP:%.*]], [[ROWS_LATCH:%.*]] ]
+; CHECK-NEXT:    br label [[ROWS_BODY:%.*]]
+; CHECK:       rows.body:
+; CHECK-NEXT:    br label [[INNER_HEADER:%.*]]
+; CHECK:       inner.header:
+; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, [[ROWS_BODY]] ], [ [[INNER_STEP:%.*]], [[INNER_LATCH:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP17:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP23:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT:    br label [[INNER_BODY:%.*]]
+; CHECK:       inner.body:
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[INNER_IV]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[TMP2]], [[ROWS_IV]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x double>* [[A:%.*]] to double*
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr double, double* [[TMP4]], i64 [[TMP3]]
+; CHECK-NEXT:    [[COL_CAST:%.*]] = bitcast double* [[TMP5]] to <4 x double>*
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x double>* [[COL_CAST]] to double*
+; CHECK-NEXT:    [[VEC_CAST:%.*]] = bitcast double* [[TMP6]] to <2 x double>*
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST]], align 8
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP6]], i64 2
 ; CHECK-NEXT:    [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
-; CHECK-NEXT:    load volatile <2 x double>, <2 x double>* [[VEC_CAST1]], align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x double>* [[B:%.*]] to double*
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr double, double* [[TMP3]], i64 0
-; CHECK-NEXT:    [[COL_CAST3:%.*]] = bitcast double* [[TMP4]] to <4 x double>*
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x double>* [[COL_CAST3]] to double*
-; CHECK-NEXT:    [[VEC_CAST4:%.*]] = bitcast double* [[TMP5]] to <2 x double>*
-; CHECK-NEXT:    load volatile <2 x double>, <2 x double>* [[VEC_CAST4]], align 8
-; CHECK-NEXT:    [[VEC_GEP6:%.*]] = getelementptr double, double* [[TMP5]], i64 2
+; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST1]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[COLS_IV]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[TMP7]], [[INNER_IV]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x double>* [[B:%.*]] to double*
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr double, double* [[TMP9]], i64 [[TMP8]]
+; CHECK-NEXT:    [[COL_CAST3:%.*]] = bitcast double* [[TMP10]] to <4 x double>*
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x double>* [[COL_CAST3]] to double*
+; CHECK-NEXT:    [[VEC_CAST4:%.*]] = bitcast double* [[TMP11]] to <2 x double>*
+; CHECK-NEXT:    [[COL_LOAD5:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST4]], align 8
+; CHECK-NEXT:    [[VEC_GEP6:%.*]] = getelementptr double, double* [[TMP11]], i64 2
 ; CHECK-NEXT:    [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <2 x double>*
-; CHECK-NEXT:    load volatile <2 x double>, <2 x double>* [[VEC_CAST7]], align 8
-
-; CHECK:         [[TMP18:%.*]] = bitcast <4 x double>* [[C:%.*]] to double*
-; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr double, double* [[TMP18]], i64 0
-; CHECK-NEXT:    [[COL_CAST18:%.*]] = bitcast double* [[TMP19]] to <4 x double>*
-; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <4 x double>* [[COL_CAST18]] to double*
-; CHECK-NEXT:    [[VEC_CAST19:%.*]] = bitcast double* [[TMP20]] to <2 x double>*
-; CHECK-NEXT:    store volatile <2 x double> {{.*}}, <2 x double>* [[VEC_CAST19]], align 8
-; CHECK-NEXT:    [[VEC_GEP20:%.*]] = getelementptr double, double* [[TMP20]], i64 2
-; CHECK-NEXT:    [[VEC_CAST21:%.*]] = bitcast double* [[VEC_GEP20]] to <2 x double>*
-; CHECK-NEXT:    store volatile <2 x double> {{.*}}, <2 x double>* [[VEC_CAST21]], align 8
+; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST7]], align 8
+; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[BLOCK9:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x double> [[COL_LOAD5]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <2 x double> undef, double [[TMP12]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK9]], <2 x double> [[SPLAT_SPLAT]], <2 x double> [[BLOCK]])
+; CHECK-NEXT:    [[BLOCK10:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[COL_LOAD5]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT11:%.*]] = insertelement <2 x double> undef, double [[TMP14]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT11]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK10]], <2 x double> [[SPLAT_SPLAT12]], <2 x double> [[TMP13]])
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <2 x double> [[TMP15]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP17]] = shufflevector <2 x double> [[TMP0]], <2 x double> [[TMP16]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[BLOCK13:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[BLOCK14:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x double> [[COL_LOAD8]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT15:%.*]] = insertelement <2 x double> undef, double [[TMP18]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT16:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT15]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK14]], <2 x double> [[SPLAT_SPLAT16]], <2 x double> [[BLOCK13]])
+; CHECK-NEXT:    [[BLOCK17:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <2 x double> [[COL_LOAD8]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT18:%.*]] = insertelement <2 x double> undef, double [[TMP20]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT19:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT18]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK17]], <2 x double> [[SPLAT_SPLAT19]], <2 x double> [[TMP19]])
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <2 x double> [[TMP21]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP23]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP22]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    br label [[INNER_LATCH]]
+; CHECK:       inner.latch:
+; CHECK-NEXT:    [[INNER_STEP]] = add i64 [[INNER_IV]], 2
+; CHECK-NEXT:    [[INNER_COND:%.*]] = icmp ne i64 [[INNER_STEP]], 2
+; CHECK-NEXT:    br i1 [[INNER_COND]], label [[INNER_HEADER]], label [[ROWS_LATCH]], !llvm.loop !0
+; CHECK:       rows.latch:
+; CHECK-NEXT:    [[ROWS_STEP]] = add i64 [[ROWS_IV]], 2
+; CHECK-NEXT:    [[ROWS_COND:%.*]] = icmp ne i64 [[ROWS_STEP]], 2
+; CHECK-NEXT:    [[TMP24:%.*]] = mul i64 [[COLS_IV]], 2
+; CHECK-NEXT:    [[TMP25:%.*]] = add i64 [[TMP24]], [[ROWS_IV]]
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast <4 x double>* [[C:%.*]] to double*
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr double, double* [[TMP26]], i64 [[TMP25]]
+; CHECK-NEXT:    [[COL_CAST20:%.*]] = bitcast double* [[TMP27]] to <4 x double>*
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast <4 x double>* [[COL_CAST20]] to double*
+; CHECK-NEXT:    [[VEC_CAST21:%.*]] = bitcast double* [[TMP28]] to <2 x double>*
+; CHECK-NEXT:    store volatile <2 x double> [[TMP17]], <2 x double>* [[VEC_CAST21]], align 8
+; CHECK-NEXT:    [[VEC_GEP22:%.*]] = getelementptr double, double* [[TMP28]], i64 2
+; CHECK-NEXT:    [[VEC_CAST23:%.*]] = bitcast double* [[VEC_GEP22]] to <2 x double>*
+; CHECK-NEXT:    store volatile <2 x double> [[TMP23]], <2 x double>* [[VEC_CAST23]], align 8
+; CHECK-NEXT:    br i1 [[ROWS_COND]], label [[ROWS_HEADER]], label [[COLS_LATCH]]
+; CHECK:       cols.latch:
+; CHECK-NEXT:    [[COLS_STEP]] = add i64 [[COLS_IV]], 2
+; CHECK-NEXT:    [[COLS_COND:%.*]] = icmp ne i64 [[COLS_STEP]], 2
+; CHECK-NEXT:    br i1 [[COLS_COND]], label [[COLS_HEADER]], label [[CONTINUE:%.*]]
+; CHECK:       continue:
 ; CHECK-NEXT:    ret void
 ;
 
+
 entry:
   %a = load volatile <4 x double>, <4 x double>* %A, align 8
   %b = load volatile <4 x double>, <4 x double>* %B, align 8
@@ -54,37 +117,100 @@ entry:
 define void @multiply_load0_volatile(<4 x double>* noalias %A, <4 x double>* noalias %B, <4 x double>* noalias %C) {
 ; CHECK-LABEL: @multiply_load0_volatile(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x double>* [[A:%.*]] to double*
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr double, double* [[TMP0]], i64 0
-; CHECK-NEXT:    [[COL_CAST:%.*]] = bitcast double* [[TMP1]] to <4 x double>*
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x double>* [[COL_CAST]] to double*
-; CHECK-NEXT:    [[VEC_CAST:%.*]] = bitcast double* [[TMP2]] to <2 x double>*
-; CHECK-NEXT:    load volatile <2 x double>, <2 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP2]], i64 2
+; CHECK-NEXT:    br label [[COLS_HEADER:%.*]]
+; CHECK:       cols.header:
+; CHECK-NEXT:    [[COLS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[COLS_STEP:%.*]], [[COLS_LATCH:%.*]] ]
+; CHECK-NEXT:    br label [[COLS_BODY:%.*]]
+; CHECK:       cols.body:
+; CHECK-NEXT:    br label [[ROWS_HEADER:%.*]]
+; CHECK:       rows.header:
+; CHECK-NEXT:    [[ROWS_IV:%.*]] = phi i64 [ 0, [[COLS_BODY]] ], [ [[ROWS_STEP:%.*]], [[ROWS_LATCH:%.*]] ]
+; CHECK-NEXT:    br label [[ROWS_BODY:%.*]]
+; CHECK:       rows.body:
+; CHECK-NEXT:    br label [[INNER_HEADER:%.*]]
+; CHECK:       inner.header:
+; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, [[ROWS_BODY]] ], [ [[INNER_STEP:%.*]], [[INNER_LATCH:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP17:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP23:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT:    br label [[INNER_BODY:%.*]]
+; CHECK:       inner.body:
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[INNER_IV]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[TMP2]], [[ROWS_IV]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x double>* [[A:%.*]] to double*
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr double, double* [[TMP4]], i64 [[TMP3]]
+; CHECK-NEXT:    [[COL_CAST:%.*]] = bitcast double* [[TMP5]] to <4 x double>*
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x double>* [[COL_CAST]] to double*
+; CHECK-NEXT:    [[VEC_CAST:%.*]] = bitcast double* [[TMP6]] to <2 x double>*
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST]], align 8
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP6]], i64 2
 ; CHECK-NEXT:    [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
-; CHECK-NEXT:    load volatile <2 x double>, <2 x double>* [[VEC_CAST1]], align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x double>* [[B:%.*]] to double*
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr double, double* [[TMP3]], i64 0
-; CHECK-NEXT:    [[COL_CAST3:%.*]] = bitcast double* [[TMP4]] to <4 x double>*
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x double>* [[COL_CAST3]] to double*
-; CHECK-NEXT:    [[VEC_CAST4:%.*]] = bitcast double* [[TMP5]] to <2 x double>*
-; CHECK-NEXT:    load <2 x double>, <2 x double>* [[VEC_CAST4]], align 8
-; CHECK-NEXT:    [[VEC_GEP6:%.*]] = getelementptr double, double* [[TMP5]], i64 2
+; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST1]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[COLS_IV]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[TMP7]], [[INNER_IV]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x double>* [[B:%.*]] to double*
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr double, double* [[TMP9]], i64 [[TMP8]]
+; CHECK-NEXT:    [[COL_CAST3:%.*]] = bitcast double* [[TMP10]] to <4 x double>*
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x double>* [[COL_CAST3]] to double*
+; CHECK-NEXT:    [[VEC_CAST4:%.*]] = bitcast double* [[TMP11]] to <2 x double>*
+; CHECK-NEXT:    [[COL_LOAD5:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST4]], align 8
+; CHECK-NEXT:    [[VEC_GEP6:%.*]] = getelementptr double, double* [[TMP11]], i64 2
 ; CHECK-NEXT:    [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <2 x double>*
-; CHECK-NEXT:    load <2 x double>, <2 x double>* [[VEC_CAST7]], align 8
-
-; CHECK:         [[TMP18:%.*]] = bitcast <4 x double>* [[C:%.*]] to double*
-; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr double, double* [[TMP18]], i64 0
-; CHECK-NEXT:    [[COL_CAST18:%.*]] = bitcast double* [[TMP19]] to <4 x double>*
-; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <4 x double>* [[COL_CAST18]] to double*
-; CHECK-NEXT:    [[VEC_CAST19:%.*]] = bitcast double* [[TMP20]] to <2 x double>*
-; CHECK-NEXT:    store <2 x double> {{.*}}, <2 x double>* [[VEC_CAST19]], align 8
-; CHECK-NEXT:    [[VEC_GEP20:%.*]] = getelementptr double, double* [[TMP20]], i64 2
-; CHECK-NEXT:    [[VEC_CAST21:%.*]] = bitcast double* [[VEC_GEP20]] to <2 x double>*
-; CHECK-NEXT:    store <2 x double> {{.*}}, <2 x double>* [[VEC_CAST21]], align 8
+; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST7]], align 8
+; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[BLOCK9:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x double> [[COL_LOAD5]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <2 x double> undef, double [[TMP12]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK9]], <2 x double> [[SPLAT_SPLAT]], <2 x double> [[BLOCK]])
+; CHECK-NEXT:    [[BLOCK10:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[COL_LOAD5]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT11:%.*]] = insertelement <2 x double> undef, double [[TMP14]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT11]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK10]], <2 x double> [[SPLAT_SPLAT12]], <2 x double> [[TMP13]])
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <2 x double> [[TMP15]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP17]] = shufflevector <2 x double> [[TMP0]], <2 x double> [[TMP16]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[BLOCK13:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[BLOCK14:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x double> [[COL_LOAD8]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT15:%.*]] = insertelement <2 x double> undef, double [[TMP18]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT16:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT15]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK14]], <2 x double> [[SPLAT_SPLAT16]], <2 x double> [[BLOCK13]])
+; CHECK-NEXT:    [[BLOCK17:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <2 x double> [[COL_LOAD8]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT18:%.*]] = insertelement <2 x double> undef, double [[TMP20]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT19:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT18]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK17]], <2 x double> [[SPLAT_SPLAT19]], <2 x double> [[TMP19]])
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <2 x double> [[TMP21]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP23]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP22]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    br label [[INNER_LATCH]]
+; CHECK:       inner.latch:
+; CHECK-NEXT:    [[INNER_STEP]] = add i64 [[INNER_IV]], 2
+; CHECK-NEXT:    [[INNER_COND:%.*]] = icmp ne i64 [[INNER_STEP]], 2
+; CHECK-NEXT:    br i1 [[INNER_COND]], label [[INNER_HEADER]], label [[ROWS_LATCH]], !llvm.loop !2
+; CHECK:       rows.latch:
+; CHECK-NEXT:    [[ROWS_STEP]] = add i64 [[ROWS_IV]], 2
+; CHECK-NEXT:    [[ROWS_COND:%.*]] = icmp ne i64 [[ROWS_STEP]], 2
+; CHECK-NEXT:    [[TMP24:%.*]] = mul i64 [[COLS_IV]], 2
+; CHECK-NEXT:    [[TMP25:%.*]] = add i64 [[TMP24]], [[ROWS_IV]]
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast <4 x double>* [[C:%.*]] to double*
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr double, double* [[TMP26]], i64 [[TMP25]]
+; CHECK-NEXT:    [[COL_CAST20:%.*]] = bitcast double* [[TMP27]] to <4 x double>*
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast <4 x double>* [[COL_CAST20]] to double*
+; CHECK-NEXT:    [[VEC_CAST21:%.*]] = bitcast double* [[TMP28]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP17]], <2 x double>* [[VEC_CAST21]], align 8
+; CHECK-NEXT:    [[VEC_GEP22:%.*]] = getelementptr double, double* [[TMP28]], i64 2
+; CHECK-NEXT:    [[VEC_CAST23:%.*]] = bitcast double* [[VEC_GEP22]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP23]], <2 x double>* [[VEC_CAST23]], align 8
+; CHECK-NEXT:    br i1 [[ROWS_COND]], label [[ROWS_HEADER]], label [[COLS_LATCH]]
+; CHECK:       cols.latch:
+; CHECK-NEXT:    [[COLS_STEP]] = add i64 [[COLS_IV]], 2
+; CHECK-NEXT:    [[COLS_COND:%.*]] = icmp ne i64 [[COLS_STEP]], 2
+; CHECK-NEXT:    br i1 [[COLS_COND]], label [[COLS_HEADER]], label [[CONTINUE:%.*]]
+; CHECK:       continue:
 ; CHECK-NEXT:    ret void
 ;
 
+
 entry:
   %a = load volatile <4 x double>, <4 x double>* %A, align 8
   %b = load <4 x double>, <4 x double>* %B, align 8
@@ -98,37 +224,100 @@ entry:
 define void @multiply_load1_volatile(<4 x double>* noalias %A, <4 x double>* noalias %B, <4 x double>* noalias %C) {
 ; CHECK-LABEL: @multiply_load1_volatile(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x double>* [[A:%.*]] to double*
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr double, double* [[TMP0]], i64 0
-; CHECK-NEXT:    [[COL_CAST:%.*]] = bitcast double* [[TMP1]] to <4 x double>*
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x double>* [[COL_CAST]] to double*
-; CHECK-NEXT:    [[VEC_CAST:%.*]] = bitcast double* [[TMP2]] to <2 x double>*
-; CHECK-NEXT:    load <2 x double>, <2 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP2]], i64 2
+; CHECK-NEXT:    br label [[COLS_HEADER:%.*]]
+; CHECK:       cols.header:
+; CHECK-NEXT:    [[COLS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[COLS_STEP:%.*]], [[COLS_LATCH:%.*]] ]
+; CHECK-NEXT:    br label [[COLS_BODY:%.*]]
+; CHECK:       cols.body:
+; CHECK-NEXT:    br label [[ROWS_HEADER:%.*]]
+; CHECK:       rows.header:
+; CHECK-NEXT:    [[ROWS_IV:%.*]] = phi i64 [ 0, [[COLS_BODY]] ], [ [[ROWS_STEP:%.*]], [[ROWS_LATCH:%.*]] ]
+; CHECK-NEXT:    br label [[ROWS_BODY:%.*]]
+; CHECK:       rows.body:
+; CHECK-NEXT:    br label [[INNER_HEADER:%.*]]
+; CHECK:       inner.header:
+; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, [[ROWS_BODY]] ], [ [[INNER_STEP:%.*]], [[INNER_LATCH:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP17:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP23:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT:    br label [[INNER_BODY:%.*]]
+; CHECK:       inner.body:
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[INNER_IV]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[TMP2]], [[ROWS_IV]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x double>* [[A:%.*]] to double*
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr double, double* [[TMP4]], i64 [[TMP3]]
+; CHECK-NEXT:    [[COL_CAST:%.*]] = bitcast double* [[TMP5]] to <4 x double>*
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x double>* [[COL_CAST]] to double*
+; CHECK-NEXT:    [[VEC_CAST:%.*]] = bitcast double* [[TMP6]] to <2 x double>*
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST]], align 8
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP6]], i64 2
 ; CHECK-NEXT:    [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
-; CHECK-NEXT:    load <2 x double>, <2 x double>* [[VEC_CAST1]], align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x double>* [[B:%.*]] to double*
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr double, double* [[TMP3]], i64 0
-; CHECK-NEXT:    [[COL_CAST3:%.*]] = bitcast double* [[TMP4]] to <4 x double>*
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x double>* [[COL_CAST3]] to double*
-; CHECK-NEXT:    [[VEC_CAST4:%.*]] = bitcast double* [[TMP5]] to <2 x double>*
-; CHECK-NEXT:    load volatile <2 x double>, <2 x double>* [[VEC_CAST4]], align 8
-; CHECK-NEXT:    [[VEC_GEP6:%.*]] = getelementptr double, double* [[TMP5]], i64 2
+; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST1]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[COLS_IV]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[TMP7]], [[INNER_IV]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x double>* [[B:%.*]] to double*
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr double, double* [[TMP9]], i64 [[TMP8]]
+; CHECK-NEXT:    [[COL_CAST3:%.*]] = bitcast double* [[TMP10]] to <4 x double>*
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x double>* [[COL_CAST3]] to double*
+; CHECK-NEXT:    [[VEC_CAST4:%.*]] = bitcast double* [[TMP11]] to <2 x double>*
+; CHECK-NEXT:    [[COL_LOAD5:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST4]], align 8
+; CHECK-NEXT:    [[VEC_GEP6:%.*]] = getelementptr double, double* [[TMP11]], i64 2
 ; CHECK-NEXT:    [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <2 x double>*
-; CHECK-NEXT:    load volatile <2 x double>, <2 x double>* [[VEC_CAST7]], align 8
-
-; CHECK:         [[TMP18:%.*]] = bitcast <4 x double>* [[C:%.*]] to double*
-; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr double, double* [[TMP18]], i64 0
-; CHECK-NEXT:    [[COL_CAST18:%.*]] = bitcast double* [[TMP19]] to <4 x double>*
-; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <4 x double>* [[COL_CAST18]] to double*
-; CHECK-NEXT:    [[VEC_CAST19:%.*]] = bitcast double* [[TMP20]] to <2 x double>*
-; CHECK-NEXT:    store <2 x double> {{.*}}, <2 x double>* [[VEC_CAST19]], align 8
-; CHECK-NEXT:    [[VEC_GEP20:%.*]] = getelementptr double, double* [[TMP20]], i64 2
-; CHECK-NEXT:    [[VEC_CAST21:%.*]] = bitcast double* [[VEC_GEP20]] to <2 x double>*
-; CHECK-NEXT:    store <2 x double> {{.*}}, <2 x double>* [[VEC_CAST21]], align 8
+; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST7]], align 8
+; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[BLOCK9:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x double> [[COL_LOAD5]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <2 x double> undef, double [[TMP12]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK9]], <2 x double> [[SPLAT_SPLAT]], <2 x double> [[BLOCK]])
+; CHECK-NEXT:    [[BLOCK10:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[COL_LOAD5]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT11:%.*]] = insertelement <2 x double> undef, double [[TMP14]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT11]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK10]], <2 x double> [[SPLAT_SPLAT12]], <2 x double> [[TMP13]])
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <2 x double> [[TMP15]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP17]] = shufflevector <2 x double> [[TMP0]], <2 x double> [[TMP16]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[BLOCK13:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[BLOCK14:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x double> [[COL_LOAD8]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT15:%.*]] = insertelement <2 x double> undef, double [[TMP18]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT16:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT15]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK14]], <2 x double> [[SPLAT_SPLAT16]], <2 x double> [[BLOCK13]])
+; CHECK-NEXT:    [[BLOCK17:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <2 x double> [[COL_LOAD8]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT18:%.*]] = insertelement <2 x double> undef, double [[TMP20]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT19:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT18]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK17]], <2 x double> [[SPLAT_SPLAT19]], <2 x double> [[TMP19]])
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <2 x double> [[TMP21]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP23]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP22]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    br label [[INNER_LATCH]]
+; CHECK:       inner.latch:
+; CHECK-NEXT:    [[INNER_STEP]] = add i64 [[INNER_IV]], 2
+; CHECK-NEXT:    [[INNER_COND:%.*]] = icmp ne i64 [[INNER_STEP]], 2
+; CHECK-NEXT:    br i1 [[INNER_COND]], label [[INNER_HEADER]], label [[ROWS_LATCH]], !llvm.loop !3
+; CHECK:       rows.latch:
+; CHECK-NEXT:    [[ROWS_STEP]] = add i64 [[ROWS_IV]], 2
+; CHECK-NEXT:    [[ROWS_COND:%.*]] = icmp ne i64 [[ROWS_STEP]], 2
+; CHECK-NEXT:    [[TMP24:%.*]] = mul i64 [[COLS_IV]], 2
+; CHECK-NEXT:    [[TMP25:%.*]] = add i64 [[TMP24]], [[ROWS_IV]]
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast <4 x double>* [[C:%.*]] to double*
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr double, double* [[TMP26]], i64 [[TMP25]]
+; CHECK-NEXT:    [[COL_CAST20:%.*]] = bitcast double* [[TMP27]] to <4 x double>*
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast <4 x double>* [[COL_CAST20]] to double*
+; CHECK-NEXT:    [[VEC_CAST21:%.*]] = bitcast double* [[TMP28]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP17]], <2 x double>* [[VEC_CAST21]], align 8
+; CHECK-NEXT:    [[VEC_GEP22:%.*]] = getelementptr double, double* [[TMP28]], i64 2
+; CHECK-NEXT:    [[VEC_CAST23:%.*]] = bitcast double* [[VEC_GEP22]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP23]], <2 x double>* [[VEC_CAST23]], align 8
+; CHECK-NEXT:    br i1 [[ROWS_COND]], label [[ROWS_HEADER]], label [[COLS_LATCH]]
+; CHECK:       cols.latch:
+; CHECK-NEXT:    [[COLS_STEP]] = add i64 [[COLS_IV]], 2
+; CHECK-NEXT:    [[COLS_COND:%.*]] = icmp ne i64 [[COLS_STEP]], 2
+; CHECK-NEXT:    br i1 [[COLS_COND]], label [[COLS_HEADER]], label [[CONTINUE:%.*]]
+; CHECK:       continue:
 ; CHECK-NEXT:    ret void
 ;
 
+
 entry:
   %a = load <4 x double>, <4 x double>* %A, align 8
   %b = load volatile <4 x double>, <4 x double>* %B, align 8
@@ -142,36 +331,99 @@ entry:
 define void @multiply_store_volatile(<4 x double>* noalias %A, <4 x double>* noalias %B, <4 x double>* noalias %C) {
 ; CHECK-LABEL: @multiply_store_volatile(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x double>* [[A:%.*]] to double*
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr double, double* [[TMP0]], i64 0
-; CHECK-NEXT:    [[COL_CAST:%.*]] = bitcast double* [[TMP1]] to <4 x double>*
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x double>* [[COL_CAST]] to double*
-; CHECK-NEXT:    [[VEC_CAST:%.*]] = bitcast double* [[TMP2]] to <2 x double>*
-; CHECK-NEXT:    load <2 x double>, <2 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP2]], i64 2
+; CHECK-NEXT:    br label [[COLS_HEADER:%.*]]
+; CHECK:       cols.header:
+; CHECK-NEXT:    [[COLS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[COLS_STEP:%.*]], [[COLS_LATCH:%.*]] ]
+; CHECK-NEXT:    br label [[COLS_BODY:%.*]]
+; CHECK:       cols.body:
+; CHECK-NEXT:    br label [[ROWS_HEADER:%.*]]
+; CHECK:       rows.header:
+; CHECK-NEXT:    [[ROWS_IV:%.*]] = phi i64 [ 0, [[COLS_BODY]] ], [ [[ROWS_STEP:%.*]], [[ROWS_LATCH:%.*]] ]
+; CHECK-NEXT:    br label [[ROWS_BODY:%.*]]
+; CHECK:       rows.body:
+; CHECK-NEXT:    br label [[INNER_HEADER:%.*]]
+; CHECK:       inner.header:
+; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, [[ROWS_BODY]] ], [ [[INNER_STEP:%.*]], [[INNER_LATCH:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP17:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP23:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT:    br label [[INNER_BODY:%.*]]
+; CHECK:       inner.body:
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[INNER_IV]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[TMP2]], [[ROWS_IV]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x double>* [[A:%.*]] to double*
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr double, double* [[TMP4]], i64 [[TMP3]]
+; CHECK-NEXT:    [[COL_CAST:%.*]] = bitcast double* [[TMP5]] to <4 x double>*
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x double>* [[COL_CAST]] to double*
+; CHECK-NEXT:    [[VEC_CAST:%.*]] = bitcast double* [[TMP6]] to <2 x double>*
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST]], align 8
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP6]], i64 2
 ; CHECK-NEXT:    [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
-; CHECK-NEXT:    load <2 x double>, <2 x double>* [[VEC_CAST1]], align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x double>* [[B:%.*]] to double*
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr double, double* [[TMP3]], i64 0
-; CHECK-NEXT:    [[COL_CAST3:%.*]] = bitcast double* [[TMP4]] to <4 x double>*
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x double>* [[COL_CAST3]] to double*
-; CHECK-NEXT:    [[VEC_CAST4:%.*]] = bitcast double* [[TMP5]] to <2 x double>*
-; CHECK-NEXT:    load <2 x double>, <2 x double>* [[VEC_CAST4]], align 8
-; CHECK-NEXT:    [[VEC_GEP6:%.*]] = getelementptr double, double* [[TMP5]], i64 2
+; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST1]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[COLS_IV]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[TMP7]], [[INNER_IV]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <4 x double>* [[B:%.*]] to double*
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr double, double* [[TMP9]], i64 [[TMP8]]
+; CHECK-NEXT:    [[COL_CAST3:%.*]] = bitcast double* [[TMP10]] to <4 x double>*
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x double>* [[COL_CAST3]] to double*
+; CHECK-NEXT:    [[VEC_CAST4:%.*]] = bitcast double* [[TMP11]] to <2 x double>*
+; CHECK-NEXT:    [[COL_LOAD5:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST4]], align 8
+; CHECK-NEXT:    [[VEC_GEP6:%.*]] = getelementptr double, double* [[TMP11]], i64 2
 ; CHECK-NEXT:    [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <2 x double>*
-; CHECK-NEXT:    load <2 x double>, <2 x double>* [[VEC_CAST7]], align 8
-
-; CHECK:         [[TMP18:%.*]] = bitcast <4 x double>* [[C:%.*]] to double*
-; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr double, double* [[TMP18]], i64 0
-; CHECK-NEXT:    [[COL_CAST18:%.*]] = bitcast double* [[TMP19]] to <4 x double>*
-; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <4 x double>* [[COL_CAST18]] to double*
-; CHECK-NEXT:    [[VEC_CAST19:%.*]] = bitcast double* [[TMP20]] to <2 x double>*
-; CHECK-NEXT:    store volatile <2 x double> {{.*}}, <2 x double>* [[VEC_CAST19]], align 8
-; CHECK-NEXT:    [[VEC_GEP20:%.*]] = getelementptr double, double* [[TMP20]], i64 2
-; CHECK-NEXT:    [[VEC_CAST21:%.*]] = bitcast double* [[VEC_GEP20]] to <2 x double>*
-; CHECK-NEXT:    store volatile <2 x double> {{.*}}, <2 x double>* [[VEC_CAST21]], align 8
+; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST7]], align 8
+; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[BLOCK9:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x double> [[COL_LOAD5]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <2 x double> undef, double [[TMP12]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK9]], <2 x double> [[SPLAT_SPLAT]], <2 x double> [[BLOCK]])
+; CHECK-NEXT:    [[BLOCK10:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[COL_LOAD5]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT11:%.*]] = insertelement <2 x double> undef, double [[TMP14]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT11]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK10]], <2 x double> [[SPLAT_SPLAT12]], <2 x double> [[TMP13]])
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <2 x double> [[TMP15]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP17]] = shufflevector <2 x double> [[TMP0]], <2 x double> [[TMP16]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[BLOCK13:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[BLOCK14:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x double> [[COL_LOAD8]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT15:%.*]] = insertelement <2 x double> undef, double [[TMP18]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT16:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT15]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK14]], <2 x double> [[SPLAT_SPLAT16]], <2 x double> [[BLOCK13]])
+; CHECK-NEXT:    [[BLOCK17:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <2 x double> [[COL_LOAD8]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT18:%.*]] = insertelement <2 x double> undef, double [[TMP20]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT19:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT18]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK17]], <2 x double> [[SPLAT_SPLAT19]], <2 x double> [[TMP19]])
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <2 x double> [[TMP21]], <2 x double> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP23]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP22]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    br label [[INNER_LATCH]]
+; CHECK:       inner.latch:
+; CHECK-NEXT:    [[INNER_STEP]] = add i64 [[INNER_IV]], 2
+; CHECK-NEXT:    [[INNER_COND:%.*]] = icmp ne i64 [[INNER_STEP]], 2
+; CHECK-NEXT:    br i1 [[INNER_COND]], label [[INNER_HEADER]], label [[ROWS_LATCH]], !llvm.loop !4
+; CHECK:       rows.latch:
+; CHECK-NEXT:    [[ROWS_STEP]] = add i64 [[ROWS_IV]], 2
+; CHECK-NEXT:    [[ROWS_COND:%.*]] = icmp ne i64 [[ROWS_STEP]], 2
+; CHECK-NEXT:    [[TMP24:%.*]] = mul i64 [[COLS_IV]], 2
+; CHECK-NEXT:    [[TMP25:%.*]] = add i64 [[TMP24]], [[ROWS_IV]]
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast <4 x double>* [[C:%.*]] to double*
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr double, double* [[TMP26]], i64 [[TMP25]]
+; CHECK-NEXT:    [[COL_CAST20:%.*]] = bitcast double* [[TMP27]] to <4 x double>*
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast <4 x double>* [[COL_CAST20]] to double*
+; CHECK-NEXT:    [[VEC_CAST21:%.*]] = bitcast double* [[TMP28]] to <2 x double>*
+; CHECK-NEXT:    store volatile <2 x double> [[TMP17]], <2 x double>* [[VEC_CAST21]], align 8
+; CHECK-NEXT:    [[VEC_GEP22:%.*]] = getelementptr double, double* [[TMP28]], i64 2
+; CHECK-NEXT:    [[VEC_CAST23:%.*]] = bitcast double* [[VEC_GEP22]] to <2 x double>*
+; CHECK-NEXT:    store volatile <2 x double> [[TMP23]], <2 x double>* [[VEC_CAST23]], align 8
+; CHECK-NEXT:    br i1 [[ROWS_COND]], label [[ROWS_HEADER]], label [[COLS_LATCH]]
+; CHECK:       cols.latch:
+; CHECK-NEXT:    [[COLS_STEP]] = add i64 [[COLS_IV]], 2
+; CHECK-NEXT:    [[COLS_COND:%.*]] = icmp ne i64 [[COLS_STEP]], 2
+; CHECK-NEXT:    br i1 [[COLS_COND]], label [[COLS_HEADER]], label [[CONTINUE:%.*]]
+; CHECK:       continue:
 ; CHECK-NEXT:    ret void
 ;
+
 entry:
   %a = load <4 x double>, <4 x double>* %A, align 8
   %b = load <4 x double>, <4 x double>* %B, align 8
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused.ll
index 3ec7c4285e75c..7300a5ff77030 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused.ll
@@ -1,10 +1,12 @@
-; RUN: opt -lower-matrix-intrinsics -fuse-matrix-tile-size=2 -matrix-allow-contract -force-fuse-matrix -instcombine -verify-dom-info %s -S | FileCheck %s
+; RUN: opt -lower-matrix-intrinsics -fuse-matrix-use-loops=false -fuse-matrix-tile-size=2 -matrix-allow-contract -force-fuse-matrix -instcombine -verify-dom-info %s -S | FileCheck %s
 
 ; REQUIRES: aarch64-registered-target
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "aarch64-apple-ios"
 
+; Test tiling without generating explicit loops.
+
 define void @multiply(<16 x double> * %A, <16 x double> * %B, <16 x double>* %C) {
 ; CHECK-LABEL: @multiply(
 ; CHECK-NEXT:  entry:
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-minimal.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-minimal.ll
new file mode 100644
index 0000000000000..1271de443dfc3
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-minimal.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-matrix-intrinsics-minimal -fuse-matrix-tile-size=2 -matrix-allow-contract -force-fuse-matrix -instcombine -verify-dom-info %s -S | FileCheck %s
+
+; Test for the minimal version of the matrix lowering pass, which does not
+; require DT or AA. Make sure no tiling is happening, even though it was
+; requested.
+
+; REQUIRES: aarch64-registered-target
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:8:32:64-S128"
+target triple = "aarch64-apple-ios"
+
+define void @multiply(<8 x double> * %A, <8 x double> * %B, <4 x double>* %C) {
+; CHECK-LABEL: @multiply(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[VEC_CAST:%.*]] = bitcast <8 x double>* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST]], align 8
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr <8 x double>, <8 x double>* [[A]], i64 0, i64 2
+; CHECK-NEXT:    [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
+; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST1]], align 8
+; CHECK-NEXT:    [[VEC_GEP3:%.*]] = getelementptr <8 x double>, <8 x double>* [[A]], i64 0, i64 4
+; CHECK-NEXT:    [[VEC_CAST4:%.*]] = bitcast double* [[VEC_GEP3]] to <2 x double>*
+; CHECK-NEXT:    [[COL_LOAD5:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST4]], align 8
+; CHECK-NEXT:    [[VEC_GEP6:%.*]] = getelementptr <8 x double>, <8 x double>* [[A]], i64 0, i64 6
+; CHECK-NEXT:    [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <2 x double>*
+; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST7]], align 8
+; CHECK-NEXT:    [[VEC_CAST9:%.*]] = bitcast <8 x double>* [[B:%.*]] to <4 x double>*
+; CHECK-NEXT:    [[COL_LOAD10:%.*]] = load <4 x double>, <4 x double>* [[VEC_CAST9]], align 8
+; CHECK-NEXT:    [[VEC_GEP11:%.*]] = getelementptr <8 x double>, <8 x double>* [[B]], i64 0, i64 4
+; CHECK-NEXT:    [[VEC_CAST12:%.*]] = bitcast double* [[VEC_GEP11]] to <4 x double>*
+; CHECK-NEXT:    [[COL_LOAD13:%.*]] = load <4 x double>, <4 x double>* [[VEC_CAST12]], align 8
+; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x double> [[COL_LOAD10]], <4 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = fmul <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]]
+; CHECK-NEXT:    [[SPLAT_SPLAT16:%.*]] = shufflevector <4 x double> [[COL_LOAD10]], <4 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD2]], <2 x double> [[SPLAT_SPLAT16]], <2 x double> [[TMP0]])
+; CHECK-NEXT:    [[SPLAT_SPLAT19:%.*]] = shufflevector <4 x double> [[COL_LOAD10]], <4 x double> undef, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD5]], <2 x double> [[SPLAT_SPLAT19]], <2 x double> [[TMP1]])
+; CHECK-NEXT:    [[SPLAT_SPLAT22:%.*]] = shufflevector <4 x double> [[COL_LOAD10]], <4 x double> undef, <2 x i32> <i32 3, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD8]], <2 x double> [[SPLAT_SPLAT22]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    [[SPLAT_SPLAT25:%.*]] = shufflevector <4 x double> [[COL_LOAD13]], <4 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT25]]
+; CHECK-NEXT:    [[SPLAT_SPLAT28:%.*]] = shufflevector <4 x double> [[COL_LOAD13]], <4 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD2]], <2 x double> [[SPLAT_SPLAT28]], <2 x double> [[TMP4]])
+; CHECK-NEXT:    [[SPLAT_SPLAT31:%.*]] = shufflevector <4 x double> [[COL_LOAD13]], <4 x double> undef, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD5]], <2 x double> [[SPLAT_SPLAT31]], <2 x double> [[TMP5]])
+; CHECK-NEXT:    [[SPLAT_SPLAT34:%.*]] = shufflevector <4 x double> [[COL_LOAD13]], <4 x double> undef, <2 x i32> <i32 3, i32 3>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD8]], <2 x double> [[SPLAT_SPLAT34]], <2 x double> [[TMP6]])
+; CHECK-NEXT:    [[VEC_CAST35:%.*]] = bitcast <4 x double>* [[C:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP3]], <2 x double>* [[VEC_CAST35]], align 8
+; CHECK-NEXT:    [[VEC_GEP36:%.*]] = getelementptr <4 x double>, <4 x double>* [[C]], i64 0, i64 2
+; CHECK-NEXT:    [[VEC_CAST37:%.*]] = bitcast double* [[VEC_GEP36]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP7]], <2 x double>* [[VEC_CAST37]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %a = load <8 x double>, <8 x double>* %A, align 8
+  %b = load <8 x double>, <8 x double>* %B, align 8
+
+  %c = call <4 x double> @llvm.matrix.multiply(<8 x double> %a, <8 x double> %b, i32 2, i32 4, i32 2)
+
+  store <4 x double> %c, <4 x double>* %C, align 8
+  ret void
+}
+
+declare <4 x double> @llvm.matrix.multiply(<8 x double>, <8 x double>, i32, i32, i32)
diff --git a/llvm/test/Transforms/OpenMP/hide_mem_transfer_latency.ll b/llvm/test/Transforms/OpenMP/hide_mem_transfer_latency.ll
new file mode 100644
index 0000000000000..daec0e5e78d5a
--- /dev/null
+++ b/llvm/test/Transforms/OpenMP/hide_mem_transfer_latency.ll
@@ -0,0 +1,561 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: -p --function-signature
+; RUN: opt -S -passes=openmpopt < %s | FileCheck %s
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+; FIXME: This struct should be generated after splitting at least one of the runtime calls.
+; %struct.__tgt_async_info = type { i8* }
+%struct.ident_t = type { i32, i32, i32, i32, i8* }
+%struct.__tgt_offload_entry = type { i8*, i8*, i64, i32, i32 }
+
+@.offload_maptypes = private unnamed_addr constant [1 x i64] [i64 35]
+@.__omp_offloading_heavyComputation1.region_id = weak constant i8 0
+@.offload_sizes.1 = private unnamed_addr constant [1 x i64] [i64 8]
+@.offload_maptypes.2 = private unnamed_addr constant [1 x i64] [i64 800]
+
+@.__omp_offloading_heavyComputation2.region_id = weak constant i8 0
+@.offload_maptypes.3 = private unnamed_addr constant [2 x i64] [i64 35, i64 35]
+
+@.__omp_offloading_heavyComputation3.region_id = weak constant i8 0
+@.offload_sizes.2 = private unnamed_addr constant [2 x i64] [i64 4, i64 0]
+@.offload_maptypes.4 = private unnamed_addr constant [2 x i64] [i64 800, i64 544]
+
+@.offload_maptypes.5 = private unnamed_addr constant [1 x i64] [i64 33]
+
+;double heavyComputation1() {
+;  double a = rand() % 777;
+;  double random = rand();
+;
+;  //#pragma omp target data map(a)
+;  void* args[1];
+;  args[0] = &a;
+;  __tgt_target_data_begin(..., args, ...)
+;
+;  #pragma omp target teams
+;  for (int i = 0; i < 1000; ++i) {
+;    a *= i*i / 2;
+;  }
+;
+;  return random + a;
+;}
+define dso_local double @heavyComputation1() {
+; CHECK-LABEL: define {{[^@]+}}@heavyComputation1()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %a = alloca double, align 8
+; CHECK-NEXT:    %.offload_baseptrs = alloca [1 x i8*], align 8
+; CHECK-NEXT:    %.offload_ptrs = alloca [1 x i8*], align 8
+; CHECK-NEXT:    %.offload_baseptrs4 = alloca [1 x i8*], align 8
+; CHECK-NEXT:    %.offload_ptrs5 = alloca [1 x i8*], align 8
+; CHECK-NEXT:    %0 = bitcast double* %a to i8*
+; CHECK-NEXT:    %call = call i32 @rand()
+; CHECK-NEXT:    %rem = srem i32 %call, 777
+; CHECK-NEXT:    %conv = sitofp i32 %rem to double
+; CHECK-NEXT:    store double %conv, double* %a, align 8
+; CHECK-NEXT:    %call1 = call i32 @rand()
+; CHECK-NEXT:    %1 = getelementptr inbounds [1 x i8*], [1 x i8*]* %.offload_baseptrs, i64 0, i64 0
+; CHECK-NEXT:    %2 = bitcast [1 x i8*]* %.offload_baseptrs to double**
+; CHECK-NEXT:    store double* %a, double** %2, align 8
+; CHECK-NEXT:    %3 = getelementptr inbounds [1 x i8*], [1 x i8*]* %.offload_ptrs, i64 0, i64 0
+; CHECK-NEXT:    %4 = bitcast [1 x i8*]* %.offload_ptrs to double**
+; CHECK-NEXT:    store double* %a, double** %4, align 8
+; CHECK-NEXT:    call void @__tgt_target_data_begin(i64 -1, i32 1, i8** nonnull %1, i8** nonnull %3, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @.offload_sizes.1, i64 0, i64 0), i64* getelementptr inbounds ([1 x i64], [1 x i64]* @.offload_maptypes, i64 0, i64 0))
+; CHECK-NEXT:    %5 = bitcast double* %a to i64*
+; CHECK-NEXT:    %6 = load i64, i64* %5, align 8
+; CHECK-NEXT:    %7 = getelementptr inbounds [1 x i8*], [1 x i8*]* %.offload_baseptrs4, i64 0, i64 0
+; CHECK-NEXT:    %8 = bitcast [1 x i8*]* %.offload_baseptrs4 to i64*
+; CHECK-NEXT:    store i64 %6, i64* %8, align 8
+; CHECK-NEXT:    %9 = getelementptr inbounds [1 x i8*], [1 x i8*]* %.offload_ptrs5, i64 0, i64 0
+; CHECK-NEXT:    %10 = bitcast [1 x i8*]* %.offload_ptrs5 to i64*
+; CHECK-NEXT:    store i64 %6, i64* %10, align 8
+; CHECK-NEXT:    %11 = call i32 @__tgt_target_teams(i64 -1, i8* nonnull @.__omp_offloading_heavyComputation1.region_id, i32 1, i8** nonnull %7, i8** nonnull %9, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @.offload_sizes.1, i64 0, i64 0), i64* getelementptr inbounds ([1 x i64], [1 x i64]* @.offload_maptypes.2, i64 0, i64 0), i32 0, i32 0)
+; CHECK-NEXT:    %12 = icmp eq i32 %11, 0
+; CHECK-NEXT:    br i1 %12, label %omp_offload.cont, label %omp_offload.failed
+; CHECK:       omp_offload.failed:
+; CHECK-NEXT:    call void @heavyComputation1FallBack(i64 %6)
+; CHECK-NEXT:    br label %omp_offload.cont
+; CHECK:       omp_offload.cont:
+; CHECK-NEXT:    %conv2 = sitofp i32 %call1 to double
+; CHECK-NEXT:    call void @__tgt_target_data_end(i64 -1, i32 1, i8** nonnull %1, i8** nonnull %3, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @.offload_sizes.1, i64 0, i64 0), i64* getelementptr inbounds ([1 x i64], [1 x i64]* @.offload_maptypes, i64 0, i64 0))
+; CHECK-NEXT:    %13 = load double, double* %a, align 8
+; CHECK-NEXT:    %add = fadd double %13, %conv2
+; CHECK-NEXT:    ret double %add
+;
+entry:
+  %a = alloca double, align 8
+  %.offload_baseptrs = alloca [1 x i8*], align 8
+  %.offload_ptrs = alloca [1 x i8*], align 8
+  %.offload_baseptrs4 = alloca [1 x i8*], align 8
+  %.offload_ptrs5 = alloca [1 x i8*], align 8
+
+  ; FIXME: Should have after splitting the runtime call __tgt_target_data_begin.
+  ; %device_id1 = alloca i64, align 8
+  ; %async_info1 = alloca %struct.__tgt_async_info, align 8
+
+  %0 = bitcast double* %a to i8*
+  %call = call i32 @rand()
+  %rem = srem i32 %call, 777
+  %conv = sitofp i32 %rem to double
+  store double %conv, double* %a, align 8
+
+  ; FIXME: The "isue" should be moved here.
+  %call1 = call i32 @rand()
+
+  ; FIXME: This setup for the runtime call __tgt_target_data_begin should be
+  ;        split into its "issue" and "wait" counterpars and moved upwards
+  ;        and downwards, respectively. The call should be replaced to something
+  ;        like ...
+  ; Issue - this is moved upwards.
+  ; ... setup code ...
+  ; store i64 -1, i64* %device_id1, align 8
+  ; %handle1 = call i8* @__tgt_target_data_begin(i64* dereferenceable(8) %device_id1, i32 1, i8** nonnull %1, i8** nonnull %3, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @.offload_sizes.1, i64 0, i64 0), i64* getelementptr inbounds ([1 x i64], [1 x i64]* @.offload_maptypes, i64 0, i64 0))
+  ; Wait - this is moved downwards.
+  ; %device_id1_copy = load i64, i64* %device_id1, align 8 ; device_id
+  ; %queue1 = getelementptr inbounds %struct.__tgt_async_info, %struct.__tgt_async_info* %async_info1, i32 0, i32 0
+  ; store i8* %handle1, i8** %queue1, align 8
+  ; call void @__tgt_target_data_begin_wait(i64 %device_id1_copy, %struct.__tgt_async_info* dereferenceable(8) %async_info1)
+  %1 = getelementptr inbounds [1 x i8*], [1 x i8*]* %.offload_baseptrs, i64 0, i64 0
+  %2 = bitcast [1 x i8*]* %.offload_baseptrs to double**
+  store double* %a, double** %2, align 8
+  %3 = getelementptr inbounds [1 x i8*], [1 x i8*]* %.offload_ptrs, i64 0, i64 0
+  %4 = bitcast [1 x i8*]* %.offload_ptrs to double**
+  store double* %a, double** %4, align 8
+  call void @__tgt_target_data_begin(i64 -1, i32 1, i8** nonnull %1, i8** nonnull %3, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @.offload_sizes.1, i64 0, i64 0), i64* getelementptr inbounds ([1 x i64], [1 x i64]* @.offload_maptypes, i64 0, i64 0))
+
+  %5 = bitcast double* %a to i64*
+  %6 = load i64, i64* %5, align 8
+  %7 = getelementptr inbounds [1 x i8*], [1 x i8*]* %.offload_baseptrs4, i64 0, i64 0
+  %8 = bitcast [1 x i8*]* %.offload_baseptrs4 to i64*
+  store i64 %6, i64* %8, align 8
+  %9 = getelementptr inbounds [1 x i8*], [1 x i8*]* %.offload_ptrs5, i64 0, i64 0
+  %10 = bitcast [1 x i8*]* %.offload_ptrs5 to i64*
+  store i64 %6, i64* %10, align 8
+
+  ; FIXME: The "wait" should be moved here.
+  %11 = call i32 @__tgt_target_teams(i64 -1, i8* nonnull @.__omp_offloading_heavyComputation1.region_id, i32 1, i8** nonnull %7, i8** nonnull %9, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @.offload_sizes.1, i64 0, i64 0), i64* getelementptr inbounds ([1 x i64], [1 x i64]* @.offload_maptypes.2, i64 0, i64 0), i32 0, i32 0)
+
+  %12 = icmp eq i32 %11, 0
+  br i1 %12, label %omp_offload.cont, label %omp_offload.failed
+
+omp_offload.failed:                               ; preds = %entry
+  call void @heavyComputation1FallBack(i64 %6)
+  br label %omp_offload.cont
+
+omp_offload.cont:                                 ; preds = %entry, %omp_offload.failed
+  %conv2 = sitofp i32 %call1 to double
+  call void @__tgt_target_data_end(i64 -1, i32 1, i8** nonnull %1, i8** nonnull %3, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @.offload_sizes.1, i64 0, i64 0), i64* getelementptr inbounds ([1 x i64], [1 x i64]* @.offload_maptypes, i64 0, i64 0))
+  %13 = load double, double* %a, align 8
+  %add = fadd double %13, %conv2
+  ret double %add
+}
+
+define internal void @heavyComputation1FallBack(i64 %a) {
+entry:
+  ; Fallback for offloading function heavyComputation1.
+  ret void
+}
+
+;int heavyComputation2(double* a, unsigned size) {
+;  int random = rand() % 7;
+;
+;  //#pragma omp target data map(a[0:size], size)
+;  void* args[2];
+;  args[0] = &a;
+;  args[1] = &size;
+;  __tgt_target_data_begin(..., args, ...)
+;
+;  #pragma omp target teams
+;  for (int i = 0; i < size; ++i) {
+;    a[i] = ++a[i] * 3.141624;
+;  }
+;
+;  return random;
+;}
+define dso_local i32 @heavyComputation2(double* %a, i32 %size) {
+; CHECK-LABEL: define {{[^@]+}}@heavyComputation2(double* %a, i32 %size)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %size.addr = alloca i32, align 4
+; CHECK-NEXT:    %.offload_baseptrs = alloca [2 x i8*], align 8
+; CHECK-NEXT:    %.offload_ptrs = alloca [2 x i8*], align 8
+; CHECK-NEXT:    %.offload_sizes = alloca [2 x i64], align 8
+; CHECK-NEXT:    %.offload_baseptrs2 = alloca [2 x i8*], align 8
+; CHECK-NEXT:    %.offload_ptrs3 = alloca [2 x i8*], align 8
+; CHECK-NEXT:    store i32 %size, i32* %size.addr, align 4
+; CHECK-NEXT:    %call = call i32 @rand()
+; CHECK-NEXT:    %conv = zext i32 %size to i64
+; CHECK-NEXT:    %0 = shl nuw nsw i64 %conv, 3
+; CHECK-NEXT:    %1 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_baseptrs, i64 0, i64 0
+; CHECK-NEXT:    %2 = bitcast [2 x i8*]* %.offload_baseptrs to double**
+; CHECK-NEXT:    store double* %a, double** %2, align 8
+; CHECK-NEXT:    %3 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_ptrs, i64 0, i64 0
+; CHECK-NEXT:    %4 = bitcast [2 x i8*]* %.offload_ptrs to double**
+; CHECK-NEXT:    store double* %a, double** %4, align 8
+; CHECK-NEXT:    %5 = getelementptr inbounds [2 x i64], [2 x i64]* %.offload_sizes, i64 0, i64 0
+; CHECK-NEXT:    store i64 %0, i64* %5, align 8
+; CHECK-NEXT:    %6 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_baseptrs, i64 0, i64 1
+; CHECK-NEXT:    %7 = bitcast i8** %6 to i32**
+; CHECK-NEXT:    store i32* %size.addr, i32** %7, align 8
+; CHECK-NEXT:    %8 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_ptrs, i64 0, i64 1
+; CHECK-NEXT:    %9 = bitcast i8** %8 to i32**
+; CHECK-NEXT:    store i32* %size.addr, i32** %9, align 8
+; CHECK-NEXT:    %10 = getelementptr inbounds [2 x i64], [2 x i64]* %.offload_sizes, i64 0, i64 1
+; CHECK-NEXT:    store i64 4, i64* %10, align 8
+; CHECK-NEXT:    call void @__tgt_target_data_begin(i64 -1, i32 2, i8** nonnull %1, i8** nonnull %3, i64* nonnull %5, i64* getelementptr inbounds ([2 x i64], [2 x i64]* @.offload_maptypes.3, i64 0, i64 0))
+; CHECK-NEXT:    %11 = load i32, i32* %size.addr, align 4
+; CHECK-NEXT:    %size.casted = zext i32 %11 to i64
+; CHECK-NEXT:    %12 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_baseptrs2, i64 0, i64 0
+; CHECK-NEXT:    %13 = bitcast [2 x i8*]* %.offload_baseptrs2 to i64*
+; CHECK-NEXT:    store i64 %size.casted, i64* %13, align 8
+; CHECK-NEXT:    %14 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_ptrs3, i64 0, i64 0
+; CHECK-NEXT:    %15 = bitcast [2 x i8*]* %.offload_ptrs3 to i64*
+; CHECK-NEXT:    store i64 %size.casted, i64* %15, align 8
+; CHECK-NEXT:    %16 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_baseptrs2, i64 0, i64 1
+; CHECK-NEXT:    %17 = bitcast i8** %16 to double**
+; CHECK-NEXT:    store double* %a, double** %17, align 8
+; CHECK-NEXT:    %18 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_ptrs3, i64 0, i64 1
+; CHECK-NEXT:    %19 = bitcast i8** %18 to double**
+; CHECK-NEXT:    store double* %a, double** %19, align 8
+; CHECK-NEXT:    %20 = call i32 @__tgt_target_teams(i64 -1, i8* nonnull @.__omp_offloading_heavyComputation2.region_id, i32 2, i8** nonnull %12, i8** nonnull %14, i64* getelementptr inbounds ([2 x i64], [2 x i64]* @.offload_sizes.2, i64 0, i64 0), i64* getelementptr inbounds ([2 x i64], [2 x i64]* @.offload_maptypes.4, i64 0, i64 0), i32 0, i32 0)
+; CHECK-NEXT:    %21 = icmp eq i32 %20, 0
+; CHECK-NEXT:    br i1 %21, label %omp_offload.cont, label %omp_offload.failed
+; CHECK:       omp_offload.failed:
+; CHECK-NEXT:    call void @heavyComputation2FallBack(i64 %size.casted, double* %a)
+; CHECK-NEXT:    br label %omp_offload.cont
+; CHECK:       omp_offload.cont:
+; CHECK-NEXT:    %rem = srem i32 %call, 7
+; CHECK-NEXT:    call void @__tgt_target_data_end(i64 -1, i32 2, i8** nonnull %1, i8** nonnull %3, i64* nonnull %5, i64* getelementptr inbounds ([2 x i64], [2 x i64]* @.offload_maptypes.3, i64 0, i64 0))
+; CHECK-NEXT:    ret i32 %rem
+;
+entry:
+  %size.addr = alloca i32, align 4
+  %.offload_baseptrs = alloca [2 x i8*], align 8
+  %.offload_ptrs = alloca [2 x i8*], align 8
+  %.offload_sizes = alloca [2 x i64], align 8
+  %.offload_baseptrs2 = alloca [2 x i8*], align 8
+  %.offload_ptrs3 = alloca [2 x i8*], align 8
+
+  ; FIXME: Should have after splitting the runtime call __tgt_target_data_begin.
+  ; %device_id1 = alloca i64, align 8
+  ; %async_info1 = alloca %struct.__tgt_async_info, align 8
+
+  store i32 %size, i32* %size.addr, align 4
+  %call = call i32 @rand()
+
+  ; FIXME: This setup for the runtime call __tgt_target_data_begin should be
+  ;        split into its "issue" and "wait" counterpars. Here though, the "issue"
+  ;        cannot be moved upwards because it's not guaranteed that rand()
+  ;        won't modify *a. Nevertheless, the "wait" can be moved downwards.
+  ;        The call should be replaced to something like ...
+  ; Issue - this can't be moved upwards, *a might have aliases.
+  ; ... setup code ...
+  ; store i64 -1, i64* %device_id1, align 8
+  ; %handle1 = call i8* @__tgt_target_data_begin(i64* dereferenceable(8) %device_id1, i32 2, i8** nonnull %1, i8** nonnull %3, i64* nonnull %5, i64* getelementptr inbounds ([2 x i64], [2 x i64]* @.offload_maptypes.3, i64 0, i64 0))
+  ; Wait - this is moved downards.
+  ; %device_id1_copy = load i64, i64* %device_id1, align 8 ; device_id
+  ; %queue1 = getelementptr inbounds %struct.__tgt_async_info, %struct.__tgt_async_info* %async_info1, i32 0, i32 0
+  ; store i8* %handle1, i8** %queue1, align 8
+  ; call void @__tgt_target_data_begin_wait(i64 %device_id1_copy, %struct.__tgt_async_info* dereferenceable(8) %async_info1)
+  %conv = zext i32 %size to i64
+  %0 = shl nuw nsw i64 %conv, 3
+  %1 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_baseptrs, i64 0, i64 0
+  %2 = bitcast [2 x i8*]* %.offload_baseptrs to double**
+  store double* %a, double** %2, align 8
+  %3 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_ptrs, i64 0, i64 0
+  %4 = bitcast [2 x i8*]* %.offload_ptrs to double**
+  store double* %a, double** %4, align 8
+  %5 = getelementptr inbounds [2 x i64], [2 x i64]* %.offload_sizes, i64 0, i64 0
+  store i64 %0, i64* %5, align 8
+  %6 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_baseptrs, i64 0, i64 1
+  %7 = bitcast i8** %6 to i32**
+  store i32* %size.addr, i32** %7, align 8
+  %8 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_ptrs, i64 0, i64 1
+  %9 = bitcast i8** %8 to i32**
+  store i32* %size.addr, i32** %9, align 8
+  %10 = getelementptr inbounds [2 x i64], [2 x i64]* %.offload_sizes, i64 0, i64 1
+  store i64 4, i64* %10, align 8
+  call void @__tgt_target_data_begin(i64 -1, i32 2, i8** nonnull %1, i8** nonnull %3, i64* nonnull %5, i64* getelementptr inbounds ([2 x i64], [2 x i64]* @.offload_maptypes.3, i64 0, i64 0))
+
+  %11 = load i32, i32* %size.addr, align 4
+  %size.casted = zext i32 %11 to i64
+  %12 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_baseptrs2, i64 0, i64 0
+  %13 = bitcast [2 x i8*]* %.offload_baseptrs2 to i64*
+  store i64 %size.casted, i64* %13, align 8
+  %14 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_ptrs3, i64 0, i64 0
+  %15 = bitcast [2 x i8*]* %.offload_ptrs3 to i64*
+  store i64 %size.casted, i64* %15, align 8
+  %16 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_baseptrs2, i64 0, i64 1
+  %17 = bitcast i8** %16 to double**
+  store double* %a, double** %17, align 8
+  %18 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_ptrs3, i64 0, i64 1
+  %19 = bitcast i8** %18 to double**
+  store double* %a, double** %19, align 8
+
+  ; FIXME: The "wait" should be moved here.
+  %20 = call i32 @__tgt_target_teams(i64 -1, i8* nonnull @.__omp_offloading_heavyComputation2.region_id, i32 2, i8** nonnull %12, i8** nonnull %14, i64* getelementptr inbounds ([2 x i64], [2 x i64]* @.offload_sizes.2, i64 0, i64 0), i64* getelementptr inbounds ([2 x i64], [2 x i64]* @.offload_maptypes.4, i64 0, i64 0), i32 0, i32 0)
+
+  %21 = icmp eq i32 %20, 0
+  br i1 %21, label %omp_offload.cont, label %omp_offload.failed
+
+omp_offload.failed:                               ; preds = %entry
+  call void @heavyComputation2FallBack(i64 %size.casted, double* %a)
+  br label %omp_offload.cont
+
+omp_offload.cont:                                 ; preds = %entry, %omp_offload.failed
+  %rem = srem i32 %call, 7
+  call void @__tgt_target_data_end(i64 -1, i32 2, i8** nonnull %1, i8** nonnull %3, i64* nonnull %5, i64* getelementptr inbounds ([2 x i64], [2 x i64]* @.offload_maptypes.3, i64 0, i64 0))
+  ret i32 %rem
+}
+
+define internal void @heavyComputation2FallBack(i64 %size, double* %a) {
+entry:
+  ; Fallback for offloading function heavyComputation2.
+  ret void
+}
+
+;int heavyComputation3(double* restrict a, unsigned size) {
+;  int random = rand() % 7;
+;
+;  //#pragma omp target data map(a[0:size], size)
+;  void* args[2];
+;  args[0] = &a;
+;  args[1] = &size;
+;  __tgt_target_data_begin(..., args, ...)
+;
+;  #pragma omp target teams
+;  for (int i = 0; i < size; ++i) {
+;    a[i] = ++a[i] * 3.141624;
+;  }
+;
+;  return random;
+;}
+define dso_local i32 @heavyComputation3(double* noalias %a, i32 %size) {
+; CHECK-LABEL: define {{[^@]+}}@heavyComputation3(double* noalias %a, i32 %size)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %size.addr = alloca i32, align 4
+; CHECK-NEXT:    %.offload_baseptrs = alloca [2 x i8*], align 8
+; CHECK-NEXT:    %.offload_ptrs = alloca [2 x i8*], align 8
+; CHECK-NEXT:    %.offload_sizes = alloca [2 x i64], align 8
+; CHECK-NEXT:    %.offload_baseptrs2 = alloca [2 x i8*], align 8
+; CHECK-NEXT:    %.offload_ptrs3 = alloca [2 x i8*], align 8
+; CHECK-NEXT:    store i32 %size, i32* %size.addr, align 4
+; CHECK-NEXT:    %call = call i32 @rand()
+; CHECK-NEXT:    %conv = zext i32 %size to i64
+; CHECK-NEXT:    %0 = shl nuw nsw i64 %conv, 3
+; CHECK-NEXT:    %1 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_baseptrs, i64 0, i64 0
+; CHECK-NEXT:    %2 = bitcast [2 x i8*]* %.offload_baseptrs to double**
+; CHECK-NEXT:    store double* %a, double** %2, align 8
+; CHECK-NEXT:    %3 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_ptrs, i64 0, i64 0
+; CHECK-NEXT:    %4 = bitcast [2 x i8*]* %.offload_ptrs to double**
+; CHECK-NEXT:    store double* %a, double** %4, align 8
+; CHECK-NEXT:    %5 = getelementptr inbounds [2 x i64], [2 x i64]* %.offload_sizes, i64 0, i64 0
+; CHECK-NEXT:    store i64 %0, i64* %5, align 8
+; CHECK-NEXT:    %6 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_baseptrs, i64 0, i64 1
+; CHECK-NEXT:    %7 = bitcast i8** %6 to i32**
+; CHECK-NEXT:    store i32* %size.addr, i32** %7, align 8
+; CHECK-NEXT:    %8 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_ptrs, i64 0, i64 1
+; CHECK-NEXT:    %9 = bitcast i8** %8 to i32**
+; CHECK-NEXT:    store i32* %size.addr, i32** %9, align 8
+; CHECK-NEXT:    %10 = getelementptr inbounds [2 x i64], [2 x i64]* %.offload_sizes, i64 0, i64 1
+; CHECK-NEXT:    store i64 4, i64* %10, align 8
+; CHECK-NEXT:    call void @__tgt_target_data_begin(i64 -1, i32 2, i8** nonnull %1, i8** nonnull %3, i64* nonnull %5, i64* getelementptr inbounds ([2 x i64], [2 x i64]* @.offload_maptypes.3, i64 0, i64 0))
+; CHECK-NEXT:    %11 = load i32, i32* %size.addr, align 4
+; CHECK-NEXT:    %size.casted = zext i32 %11 to i64
+; CHECK-NEXT:    %12 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_baseptrs2, i64 0, i64 0
+; CHECK-NEXT:    %13 = bitcast [2 x i8*]* %.offload_baseptrs2 to i64*
+; CHECK-NEXT:    store i64 %size.casted, i64* %13, align 8
+; CHECK-NEXT:    %14 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_ptrs3, i64 0, i64 0
+; CHECK-NEXT:    %15 = bitcast [2 x i8*]* %.offload_ptrs3 to i64*
+; CHECK-NEXT:    store i64 %size.casted, i64* %15, align 8
+; CHECK-NEXT:    %16 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_baseptrs2, i64 0, i64 1
+; CHECK-NEXT:    %17 = bitcast i8** %16 to double**
+; CHECK-NEXT:    store double* %a, double** %17, align 8
+; CHECK-NEXT:    %18 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_ptrs3, i64 0, i64 1
+; CHECK-NEXT:    %19 = bitcast i8** %18 to double**
+; CHECK-NEXT:    store double* %a, double** %19, align 8
+; CHECK-NEXT:    %20 = call i32 @__tgt_target_teams(i64 -1, i8* nonnull @.__omp_offloading_heavyComputation3.region_id, i32 2, i8** nonnull %12, i8** nonnull %14, i64* getelementptr inbounds ([2 x i64], [2 x i64]* @.offload_sizes.2, i64 0, i64 0), i64* getelementptr inbounds ([2 x i64], [2 x i64]* @.offload_maptypes.4, i64 0, i64 0), i32 0, i32 0)
+; CHECK-NEXT:    %21 = icmp eq i32 %20, 0
+; CHECK-NEXT:    br i1 %21, label %omp_offload.cont, label %omp_offload.failed
+; CHECK:       omp_offload.failed:
+; CHECK-NEXT:    call void @heavyComputation3FallBack(i64 %size.casted, double* %a)
+; CHECK-NEXT:    br label %omp_offload.cont
+; CHECK:       omp_offload.cont:
+; CHECK-NEXT:    %rem = srem i32 %call, 7
+; CHECK-NEXT:    call void @__tgt_target_data_end(i64 -1, i32 2, i8** nonnull %1, i8** nonnull %3, i64* nonnull %5, i64* getelementptr inbounds ([2 x i64], [2 x i64]* @.offload_maptypes.3, i64 0, i64 0))
+; CHECK-NEXT:    ret i32 %rem
+;
+entry:
+  %size.addr = alloca i32, align 4
+  %.offload_baseptrs = alloca [2 x i8*], align 8
+  %.offload_ptrs = alloca [2 x i8*], align 8
+  %.offload_sizes = alloca [2 x i64], align 8
+  %.offload_baseptrs2 = alloca [2 x i8*], align 8
+  %.offload_ptrs3 = alloca [2 x i8*], align 8
+
+  ; FIXME: Should have after splitting the runtime call __tgt_target_data_begin.
+  ; %device_id1 = alloca i64, align 8
+  ; %async_info1 = alloca %struct.__tgt_async_info, align 8
+
+  store i32 %size, i32* %size.addr, align 4
+
+  ; FIXME: The "issue" should be moved here.
+  %call = call i32 @rand()
+
+  ; FIXME: This setup for the runtime call __tgt_target_data_begin should be
+  ;        split into its "issue" and "wait" counterpars and moved upwards
+  ;        and downwards, respectively. The call should be replaced to something
+  ;        like ...
+  ; Issue - this is moved upwards.
+  ; ... setup code ...
+  ; store i64 -1, i64* %device_id1, align 8
+  ; %handle1 = call i8* @__tgt_target_data_begin(i64* dereferenceable(8) %device_id1, i32 2, i8** nonnull %1, i8** nonnull %3, i64* nonnull %5, i64* getelementptr inbounds ([2 x i64], [2 x i64]* @.offload_maptypes.3, i64 0, i64 0))
+  ; Wait - this is moved downards.
+  ; %device_id1_copy = load i64, i64* %device_id1, align 8 ; device_id
+  ; %queue1 = getelementptr inbounds %struct.__tgt_async_info, %struct.__tgt_async_info* %async_info1, i32 0, i32 0
+  ; store i8* %handle1, i8** %queue1, align 8
+  ; call void @__tgt_target_data_begin_wait(i64 %device_id1_copy, %struct.__tgt_async_info* dereferenceable(8) %async_info1)
+  %conv = zext i32 %size to i64
+  %0 = shl nuw nsw i64 %conv, 3
+  %1 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_baseptrs, i64 0, i64 0
+  %2 = bitcast [2 x i8*]* %.offload_baseptrs to double**
+  store double* %a, double** %2, align 8
+  %3 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_ptrs, i64 0, i64 0
+  %4 = bitcast [2 x i8*]* %.offload_ptrs to double**
+  store double* %a, double** %4, align 8
+  %5 = getelementptr inbounds [2 x i64], [2 x i64]* %.offload_sizes, i64 0, i64 0
+  store i64 %0, i64* %5, align 8
+  %6 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_baseptrs, i64 0, i64 1
+  %7 = bitcast i8** %6 to i32**
+  store i32* %size.addr, i32** %7, align 8
+  %8 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_ptrs, i64 0, i64 1
+  %9 = bitcast i8** %8 to i32**
+  store i32* %size.addr, i32** %9, align 8
+  %10 = getelementptr inbounds [2 x i64], [2 x i64]* %.offload_sizes, i64 0, i64 1
+  store i64 4, i64* %10, align 8
+  call void @__tgt_target_data_begin(i64 -1, i32 2, i8** nonnull %1, i8** nonnull %3, i64* nonnull %5, i64* getelementptr inbounds ([2 x i64], [2 x i64]* @.offload_maptypes.3, i64 0, i64 0))
+
+  %11 = load i32, i32* %size.addr, align 4
+  %size.casted = zext i32 %11 to i64
+  %12 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_baseptrs2, i64 0, i64 0
+  %13 = bitcast [2 x i8*]* %.offload_baseptrs2 to i64*
+  store i64 %size.casted, i64* %13, align 8
+  %14 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_ptrs3, i64 0, i64 0
+  %15 = bitcast [2 x i8*]* %.offload_ptrs3 to i64*
+  store i64 %size.casted, i64* %15, align 8
+  %16 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_baseptrs2, i64 0, i64 1
+  %17 = bitcast i8** %16 to double**
+  store double* %a, double** %17, align 8
+  %18 = getelementptr inbounds [2 x i8*], [2 x i8*]* %.offload_ptrs3, i64 0, i64 1
+  %19 = bitcast i8** %18 to double**
+  store double* %a, double** %19, align 8
+
+  ; FIXME: The "wait" should be moved here.
+  %20 = call i32 @__tgt_target_teams(i64 -1, i8* nonnull @.__omp_offloading_heavyComputation3.region_id, i32 2, i8** nonnull %12, i8** nonnull %14, i64* getelementptr inbounds ([2 x i64], [2 x i64]* @.offload_sizes.2, i64 0, i64 0), i64* getelementptr inbounds ([2 x i64], [2 x i64]* @.offload_maptypes.4, i64 0, i64 0), i32 0, i32 0)
+
+  %21 = icmp eq i32 %20, 0
+  br i1 %21, label %omp_offload.cont, label %omp_offload.failed
+
+omp_offload.failed:                               ; preds = %entry
+  call void @heavyComputation3FallBack(i64 %size.casted, double* %a)
+  br label %omp_offload.cont
+
+omp_offload.cont:                                 ; preds = %entry, %omp_offload.failed
+  %rem = srem i32 %call, 7
+  call void @__tgt_target_data_end(i64 -1, i32 2, i8** nonnull %1, i8** nonnull %3, i64* nonnull %5, i64* getelementptr inbounds ([2 x i64], [2 x i64]* @.offload_maptypes.3, i64 0, i64 0))
+  ret i32 %rem
+}
+
+define internal void @heavyComputation3FallBack(i64 %size, double* %a) {
+entry:
+  ; Fallback for offloading function heavyComputation3.
+  ret void
+}
+
+;int dataTransferOnly1(double* restrict a, unsigned size) {
+;  // Random computation.
+;  int random = rand();
+;
+;  //#pragma omp target data map(to:a[0:size])
+;  void* args[1];
+;  args[0] = &a;
+;  __tgt_target_data_begin(..., args, ...)
+;
+;  // Random computation.
+;  random %= size;
+;  return random;
+;}
+define dso_local i32 @dataTransferOnly1(double* noalias %a, i32 %size) {
+; CHECK-LABEL: define {{[^@]+}}@dataTransferOnly1(double* noalias %a, i32 %size)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %.offload_baseptrs = alloca [1 x i8*], align 8
+; CHECK-NEXT:    %.offload_ptrs = alloca [1 x i8*], align 8
+; CHECK-NEXT:    %.offload_sizes = alloca [1 x i64], align 8
+; CHECK-NEXT:    %call = call i32 @rand()
+; CHECK-NEXT:    %conv = zext i32 %size to i64
+; CHECK-NEXT:    %0 = shl nuw nsw i64 %conv, 3
+; CHECK-NEXT:    %1 = getelementptr inbounds [1 x i8*], [1 x i8*]* %.offload_baseptrs, i64 0, i64 0
+; CHECK-NEXT:    %2 = bitcast [1 x i8*]* %.offload_baseptrs to double**
+; CHECK-NEXT:    store double* %a, double** %2, align 8
+; CHECK-NEXT:    %3 = getelementptr inbounds [1 x i8*], [1 x i8*]* %.offload_ptrs, i64 0, i64 0
+; CHECK-NEXT:    %4 = bitcast [1 x i8*]* %.offload_ptrs to double**
+; CHECK-NEXT:    store double* %a, double** %4, align 8
+; CHECK-NEXT:    %5 = getelementptr inbounds [1 x i64], [1 x i64]* %.offload_sizes, i64 0, i64 0
+; CHECK-NEXT:    store i64 %0, i64* %5, align 8
+; CHECK-NEXT:    call void @__tgt_target_data_begin(i64 -1, i32 1, i8** nonnull %1, i8** nonnull %3, i64* nonnull %5, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @.offload_maptypes.5, i64 0, i64 0))
+; CHECK-NEXT:    %rem = urem i32 %call, %size
+; CHECK-NEXT:    call void @__tgt_target_data_end(i64 -1, i32 1, i8** nonnull %1, i8** nonnull %3, i64* nonnull %5, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @.offload_maptypes.5, i64 0, i64 0))
+; CHECK-NEXT:    ret i32 %rem
+;
+entry:
+  %.offload_baseptrs = alloca [1 x i8*], align 8
+  %.offload_ptrs = alloca [1 x i8*], align 8
+  %.offload_sizes = alloca [1 x i64], align 8
+
+  ; FIXME: Should have after splitting the runtime call __tgt_target_data_begin.
+  ; %device_id1 = alloca i64, align 8
+  ; %async_info1 = alloca %struct.__tgt_async_info, align 8
+
+  ; FIXME: The "issue" should be moved here.
+  %call = call i32 @rand()
+
+  ; FIXME: This setup for the runtime call __tgt_target_data_begin should be
+  ;        split into its "issue" and "wait" counterpars and moved upwards
+  ;        and downwards, respectively. The call should be replaced to something
+  ;        like ...
+  ; Issue - this is moved upwards.
+  ; ... setup code ...
+  ; store i64 -1, i64* %device_id1, align 8
+  ; %handle1 = call i8* @__tgt_target_data_begin(i64* dereferenceable(8) %device_id1, i32 1, i8** nonnull %1, i8** nonnull %3, i64* nonnull %5, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @.offload_maptypes.5, i64 0, i64 0))
+  ; Wait - this is moved downards.
+  ; %device_id1_copy = load i64, i64* %device_id1, align 8 ; device_id
+  ; %queue1 = getelementptr inbounds %struct.__tgt_async_info, %struct.__tgt_async_info* %async_info1, i32 0, i32 0
+  ; store i8* %handle1, i8** %queue1, align 8
+  ; call void @__tgt_target_data_begin_wait(i64 %device_id1_copy, %struct.__tgt_async_info* dereferenceable(8) %async_info1)
+  %conv = zext i32 %size to i64
+  %0 = shl nuw nsw i64 %conv, 3
+  %1 = getelementptr inbounds [1 x i8*], [1 x i8*]* %.offload_baseptrs, i64 0, i64 0
+  %2 = bitcast [1 x i8*]* %.offload_baseptrs to double**
+  store double* %a, double** %2, align 8
+  %3 = getelementptr inbounds [1 x i8*], [1 x i8*]* %.offload_ptrs, i64 0, i64 0
+  %4 = bitcast [1 x i8*]* %.offload_ptrs to double**
+  store double* %a, double** %4, align 8
+  %5 = getelementptr inbounds [1 x i64], [1 x i64]* %.offload_sizes, i64 0, i64 0
+  store i64 %0, i64* %5, align 8
+  call void @__tgt_target_data_begin(i64 -1, i32 1, i8** nonnull %1, i8** nonnull %3, i64* nonnull %5, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @.offload_maptypes.5, i64 0, i64 0))
+
+  %rem = urem i32 %call, %size
+
+  ; FIXME: The "wait" should be moved here.
+  call void @__tgt_target_data_end(i64 -1, i32 1, i8** nonnull %1, i8** nonnull %3, i64* nonnull %5, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @.offload_maptypes.5, i64 0, i64 0))
+  ret i32 %rem
+}
+
+declare dso_local void @__tgt_target_data_begin(i64, i32, i8**, i8**, i64*, i64*)
+
+declare dso_local i32 @__tgt_target_teams(i64, i8*, i32, i8**, i8**, i64*, i64*, i32, i32)
+
+declare dso_local void @__tgt_target_data_end(i64, i32, i8**, i8**, i64*, i64*)
+
+declare dso_local i32 @rand()
+
+; FIXME: These two function declarations must be generated after splitting the runtime function
+;        __tgt_target_data_begin.
+; declare dso_local i8* @__tgt_target_data_begin_issue(i64* dereferenceable(8), i32, i8**, i8**, i64*, i64*)
+; declare dso_local void @__tgt_target_data_begin_wait(i64, %struct.__tgt_async_info* dereferenceable(8))
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/branch2_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/branch2_entry.proftext
new file mode 100644
index 0000000000000..51de52d824926
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/branch2_entry.proftext
@@ -0,0 +1,9 @@
+# :ir is the flag to indicate this is IR level profile.
+:ir
+:entry_first
+test_br_2
+29667547796
+2
+2
+1
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/criticaledge_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/criticaledge_entry.proftext
new file mode 100644
index 0000000000000..79754261d2f88
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/criticaledge_entry.proftext
@@ -0,0 +1,20 @@
+# :ir is the flag to indicate this is IR level profile.
+:ir
+:entry_first
+test_criticalEdge
+82323253069
+8
+7
+2
+1
+2
+0
+1
+2
+1
+
+<stdin>:bar
+12884901887
+1
+7
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext
index 230f44ba443d7..60c49bf730954 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext
@@ -1,5 +1,6 @@
 # IR level Instrumentation Flag
 :ir
+:entry_first
 hot
 # Func Hash:
 12884901887
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/indirectbr_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/indirectbr_entry.proftext
new file mode 100644
index 0000000000000..7173269d20cc9
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/indirectbr_entry.proftext
@@ -0,0 +1,13 @@
+# IR level Instrumentation Flag
+:ir
+:entry_first
+foo
+# Func Hash:
+47485104005
+# Num Counters:
+4
+# Counter Values:
+202
+20
+5
+63
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/irreducible_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/irreducible_entry.proftext
new file mode 100644
index 0000000000000..d04cfddda896d
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/irreducible_entry.proftext
@@ -0,0 +1,30 @@
+:ir
+:entry_first
+_Z11irreducibleii
+# Func Hash:
+64451410787
+# Num Counters:
+6
+# Counter Values:
+1
+1000
+950
+100
+373
+0
+
+_Z11irreduciblePh
+# Func Hash:
+104649601521
+# Num Counters:
+9
+# Counter Values:
+1
+100
+300
+99
+300
+201
+1
+0
+0
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/landingpad_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/landingpad_entry.proftext
new file mode 100644
index 0000000000000..3175d9c0d630b
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/landingpad_entry.proftext
@@ -0,0 +1,17 @@
+# :ir is the flag to indicate this is IR level profile.
+:ir
+:entry_first
+foo
+59130013419
+4
+5
+1
+2
+0
+
+bar
+24868915205
+2
+3
+2
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/loop1_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/loop1_entry.proftext
new file mode 100644
index 0000000000000..b61634d94689d
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/loop1_entry.proftext
@@ -0,0 +1,9 @@
+# :ir is the flag to indicate this is IR level profile.
+:ir
+:entry_first
+test_simple_for
+34137660316
+2
+4
+96
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/loop2_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/loop2_entry.proftext
new file mode 100644
index 0000000000000..ac8536a52675b
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/loop2_entry.proftext
@@ -0,0 +1,10 @@
+# :ir is the flag to indicate this is IR level profile.
+:ir
+:entry_first
+test_nested_for
+53929068288
+3
+6
+33
+10
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch_entry.proftext
new file mode 100644
index 0000000000000..df7d6cf80466b
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-branch_entry.proftext
@@ -0,0 +1,39 @@
+# IR level Instrumentation Flag
+:ir
+:entry_first
+bar
+# Func Hash:
+29667547796
+# Num Counters:
+2
+# Counter Values:
+2000000
+1600332
+
+baz
+# Func Hash:
+12884901887
+# Num Counters:
+1
+# Counter Values:
+399668
+
+foo
+# Func Hash:
+29212902728
+# Num Counters:
+2
+# Counter Values:
+40803991
+1600332
+
+main
+# Func Hash:
+41605652536
+# Num Counters:
+3
+# Counter Values:
+2000000
+2000
+1
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct_entry.proftext
new file mode 100644
index 0000000000000..4686fec3bee78
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch-correct_entry.proftext
@@ -0,0 +1,17 @@
+# IR level Instrumentation Flag
+:ir
+:entry_first
+main
+# Func Hash:
+74054140268
+# Num Counters:
+7
+# Counter Values:
+1
+0
+0
+20000
+0
+0
+0
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch_entry.proftext
new file mode 100644
index 0000000000000..3da4c10ac19d1
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/misexpect-switch_entry.proftext
@@ -0,0 +1,17 @@
+# IR level Instrumentation Flag
+:ir
+:entry_first
+main
+# Func Hash:
+74054140268
+# Num Counters:
+7
+# Counter Values:
+11889
+3973
+3970
+0
+8111
+1
+0
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/select1.proftext b/llvm/test/Transforms/PGOProfile/Inputs/select1.proftext
index be80acb5d12f7..60663e02ffb49 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/select1.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/select1.proftext
@@ -1,8 +1,9 @@
 :ir
+:entry_first
 test_br_2
 72057623705475732
 3
-4
+5
 1
 1
 
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/select2.proftext b/llvm/test/Transforms/PGOProfile/Inputs/select2.proftext
index 56d78387ec2e8..8f3c562097fae 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/select2.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/select2.proftext
@@ -1,11 +1,12 @@
 # IR level Instrumentation Flag
 :ir
+:entry_first
 foo
 # Func Hash:
 72057628175588252
 # Num Counters:
 3
 # Counter Values:
-800
 3
+800
 300
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/switch_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/switch_entry.proftext
new file mode 100644
index 0000000000000..20df873608c19
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/switch_entry.proftext
@@ -0,0 +1,11 @@
+# :ir is the flag to indicate this is IR level profile.
+:ir
+:entry_first
+test_switch
+46200943743
+4
+10
+5
+2
+3
+
diff --git a/llvm/test/Transforms/PGOProfile/branch2.ll b/llvm/test/Transforms/PGOProfile/branch2.ll
index 4582ab7f725e9..154f86236398e 100644
--- a/llvm/test/Transforms/PGOProfile/branch2.ll
+++ b/llvm/test/Transforms/PGOProfile/branch2.ll
@@ -1,8 +1,14 @@
-; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
-; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -pgo-instr-gen -pgo-instrument-entry=false -S | FileCheck %s --check-prefixes=GEN,NOTENTRY
+; RUN: opt < %s -passes=pgo-instr-gen -pgo-instrument-entry=false -S | FileCheck %s --check-prefixes=GEN,NOTENTRY
 ; RUN: llvm-profdata merge %S/Inputs/branch2.proftext -o %t.profdata
-; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
-; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+
+; RUN: opt < %s -pgo-instr-gen -pgo-instrument-entry=true -S | FileCheck %s --check-prefixes=GEN,ENTRY
+; RUN: opt < %s -passes=pgo-instr-gen -pgo-instrument-entry=true  -S | FileCheck %s --check-prefixes=GEN,ENTRY
+; RUN: llvm-profdata merge %S/Inputs/branch2_entry.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use  -pgo-instrument-entry=true -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use  -pgo-instrument-entry=true -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -13,7 +19,8 @@ target triple = "x86_64-unknown-linux-gnu"
 define i32 @test_br_2(i32 %i) {
 entry:
 ; GEN: entry:
-; GEN-NOT: llvm.instrprof.increment
+; NOTENTRY-NOT: llvm.instrprof.increment
+; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_test_br_2, i32 0, i32 0), i64 29667547796, i32 2, i32 0)
   %cmp = icmp sgt i32 %i, 0
   br i1 %cmp, label %if.then, label %if.else
 ; USE: br i1 %cmp, label %if.then, label %if.else
@@ -22,7 +29,8 @@ entry:
 
 if.then:
 ; GEN: if.then:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_test_br_2, i32 0, i32 0), i64 29667547796, i32 2, i32 0)
+; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_test_br_2, i32 0, i32 0), i64 29667547796, i32 2, i32 0)
+; ENTRY-NOT: llvm.instrprof.increment
   %add = add nsw i32 %i, 2
   br label %if.end
 
diff --git a/llvm/test/Transforms/PGOProfile/counter_promo_exit_catchswitch.ll b/llvm/test/Transforms/PGOProfile/counter_promo_exit_catchswitch.ll
index c49d92965578d..dc7a166e18ada 100644
--- a/llvm/test/Transforms/PGOProfile/counter_promo_exit_catchswitch.ll
+++ b/llvm/test/Transforms/PGOProfile/counter_promo_exit_catchswitch.ll
@@ -2,8 +2,10 @@
 ; compilation for loops that exit to a catchswitch block. In this case, counters
 ; do not get promoted out of the loop body.
 
-; RUN: opt < %s -pgo-instr-gen -instrprof -do-counter-promotion=true -S | FileCheck %s
-; RUN: opt < %s -passes=pgo-instr-gen,instrprof -do-counter-promotion=true -S | FileCheck %s
+; RUN: opt < %s -pgo-instr-gen -instrprof -pgo-instrument-entry=false -do-counter-promotion=true -S | FileCheck %s --check-prefixes=CHECK,NOTENTRY
+; RUN: opt < %s -passes=pgo-instr-gen,instrprof -pgo-instrument-entry=false -do-counter-promotion=true -S | FileCheck %s --check-prefixes=CHECK,NOTENTRY
+; RUN: opt < %s -pgo-instr-gen -instrprof -pgo-instrument-entry=true -do-counter-promotion=true -S | FileCheck %s --check-prefixes=CHECK,ENTRY
+; RUN: opt < %s -passes=pgo-instr-gen,instrprof -pgo-instrument-entry=true -do-counter-promotion=true -S | FileCheck %s --check-prefixes=CHECK,ENTRY
 
 ; Source used to create test:
 ;
@@ -36,9 +38,11 @@ for.cond:                                         ; preds = %for.inc, %entry
 
 for.body:                                         ; preds = %for.cond
 ; CHECK: for.body:
-; CHECK: %pgocount1 = load i64, i64* getelementptr inbounds ([3 x i64], [3 x i64]* @"__profc_?run@@YAXH@Z", i64 0, i64 0)
+; NOTENTRY: %pgocount1 = load i64, i64* getelementptr inbounds ([3 x i64], [3 x i64]* @"__profc_?run@@YAXH@Z", i64 0, i64 0)
+; TENTRY: %pgocount1 = load i64, i64* getelementptr inbounds ([3 x i64], [3 x i64]* @"__profc_?run@@YAXH@Z", i64 0, i64 1)
 ; CHECK: %1 = add i64 %pgocount1, 1
-; CHECK: store i64 %1, i64* getelementptr inbounds ([3 x i64], [3 x i64]* @"__profc_?run@@YAXH@Z", i64 0, i64 0)
+; NOTENTRY: store i64 %1, i64* getelementptr inbounds ([3 x i64], [3 x i64]* @"__profc_?run@@YAXH@Z", i64 0, i64 0)
+; ENTRY: store i64 %1, i64* getelementptr inbounds ([3 x i64], [3 x i64]* @"__profc_?run@@YAXH@Z", i64 0, i64 1)
   %idxprom = zext i32 %i.0 to i64
   %arrayidx = getelementptr inbounds [200 x i8], [200 x i8]* @"?buffer@@3PADA", i64 0, i64 %idxprom
   %0 = load i8, i8* %arrayidx, align 1
@@ -51,9 +55,11 @@ if.end:                                           ; preds = %for.body
 
 for.inc:                                          ; preds = %if.end
 ; CHECK: for.inc:
-; CHECK: %pgocount2 = load i64, i64* getelementptr inbounds ([3 x i64], [3 x i64]* @"__profc_?run@@YAXH@Z", i64 0, i64 1)
+; NOTENTRY: %pgocount2 = load i64, i64* getelementptr inbounds ([3 x i64], [3 x i64]* @"__profc_?run@@YAXH@Z", i64 0, i64 1)
+; ENTRY: %pgocount2 = load i64, i64* getelementptr inbounds ([3 x i64], [3 x i64]* @"__profc_?run@@YAXH@Z", i64 0, i64 2)
 ; CHECK: %3 = add i64 %pgocount2, 1
-; CHECK: store i64 %3, i64* getelementptr inbounds ([3 x i64], [3 x i64]* @"__profc_?run@@YAXH@Z", i64 0, i64 1)
+; NOTENTRY: store i64 %3, i64* getelementptr inbounds ([3 x i64], [3 x i64]* @"__profc_?run@@YAXH@Z", i64 0, i64 1)
+; ENTRY: store i64 %3, i64* getelementptr inbounds ([3 x i64], [3 x i64]* @"__profc_?run@@YAXH@Z", i64 0, i64 2)
   %inc = add nuw nsw i32 %i.0, 1
   br label %for.cond
 
diff --git a/llvm/test/Transforms/PGOProfile/criticaledge.ll b/llvm/test/Transforms/PGOProfile/criticaledge.ll
index 93798b56358c1..49ef155ada38b 100644
--- a/llvm/test/Transforms/PGOProfile/criticaledge.ll
+++ b/llvm/test/Transforms/PGOProfile/criticaledge.ll
@@ -1,8 +1,14 @@
-; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
-; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -pgo-instr-gen -pgo-instrument-entry=false -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -passes=pgo-instr-gen -pgo-instrument-entry=false -S | FileCheck %s --check-prefix=GEN
 ; RUN: llvm-profdata merge %S/Inputs/criticaledge.proftext -o %t.profdata
-; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
-; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+;
+; RUN: opt < %s -pgo-instr-gen -pgo-instrument-entry=true -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -passes=pgo-instr-gen -pgo-instrument-entry=true -S | FileCheck %s --check-prefix=GEN
+; RUN: llvm-profdata merge %S/Inputs/criticaledge_entry.proftext -o %t2.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -S | FileCheck %s --check-prefix=USE
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -14,7 +20,8 @@ target triple = "x86_64-unknown-linux-gnu"
 define i32 @test_criticalEdge(i32 %i, i32 %j) {
 entry:
 ; CHECK: entry:
-; GEN-NOT: call void @llvm.instrprof.increment
+; NOTENTRY-NOT: call void @llvm.instrprof.increment
+; ENTRY:   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 0)
   switch i32 %i, label %sw.default [
     i32 1, label %sw.bb
     i32 2, label %sw.bb1
@@ -28,11 +35,13 @@ entry:
 ; USE-SAME: !prof ![[BW_SWITCH:[0-9]+]]
 
 ; CHECK: entry.sw.bb2_crit_edge1:
-; GEN:   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 1)
+; NOTENTRY:   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 1)
+; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 2)
 ; CHECK:   br label %sw.bb2
 
 ; CHECK: entry.sw.bb2_crit_edge:
-; GEN:   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 0)
+; NOTENTRY:   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 0)
+; TENTRY:   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 1)
 ; CHECK:   br label %sw.bb2
 
 sw.bb:
@@ -57,7 +66,8 @@ sw.bb2:
 
 if.then:
 ; GEN: if.then:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 2)
+; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 2)
+; ENTRY-NOT: call void @llvm.instrprof.increment
   %call4 = call i32 @bar(i32 4)
   br label %return
 
diff --git a/llvm/test/Transforms/PGOProfile/cspgo_profile_summary.ll b/llvm/test/Transforms/PGOProfile/cspgo_profile_summary.ll
index fb5d5d1e78a18..23096d02f00b0 100644
--- a/llvm/test/Transforms/PGOProfile/cspgo_profile_summary.ll
+++ b/llvm/test/Transforms/PGOProfile/cspgo_profile_summary.ll
@@ -1,8 +1,8 @@
 ; Test the profile summary for context sensitive PGO (CSPGO)
 
 ; RUN: llvm-profdata merge %S/Inputs/cspgo.proftext -o %t.profdata
-; RUN: opt < %s -O2 -disable-preinline -pgo-kind=pgo-instr-use-pipeline -profile-file=%t.profdata -S | FileCheck %s --check-prefix=PGOSUMMARY
-; RUN: opt < %s -O2 -disable-preinline -pgo-kind=pgo-instr-use-pipeline -profile-file=%t.profdata -S -cspgo-kind=cspgo-instr-use-pipeline| FileCheck %s --check-prefix=CSPGOSUMMARY
+; RUN: opt < %s -O2 -disable-preinline -pgo-instrument-entry=false -pgo-kind=pgo-instr-use-pipeline -profile-file=%t.profdata -S | FileCheck %s --check-prefix=PGOSUMMARY
+; RUN: opt < %s -O2 -disable-preinline -pgo-instrument-entry=false -pgo-kind=pgo-instr-use-pipeline -profile-file=%t.profdata -S -cspgo-kind=cspgo-instr-use-pipeline| FileCheck %s --check-prefix=CSPGOSUMMARY
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/PGOProfile/indirectbr.ll b/llvm/test/Transforms/PGOProfile/indirectbr.ll
index 8c961adbf6f95..978b1ec5d7606 100644
--- a/llvm/test/Transforms/PGOProfile/indirectbr.ll
+++ b/llvm/test/Transforms/PGOProfile/indirectbr.ll
@@ -1,8 +1,13 @@
 ; RUN: llvm-profdata merge %S/Inputs/indirectbr.proftext -o %t.profdata
-; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
 ; New PM
-; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
-; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | opt -S -analyze -branch-prob | FileCheck %s --check-prefix=BRANCHPROB
+; RUN: opt < %s -passes=pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S | opt -S -analyze -branch-prob | FileCheck %s --check-prefix=BRANCHPROB
+; RUN: llvm-profdata merge %S/Inputs/indirectbr_entry.proftext -o %t2.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -S | FileCheck %s --check-prefix=USE
+; New PM
+; RUN: opt < %s -passes=pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -S | opt -S -analyze -branch-prob | FileCheck %s --check-prefix=BRANCHPROB
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/PGOProfile/irreducible.ll b/llvm/test/Transforms/PGOProfile/irreducible.ll
index 9394b724f7e67..eabcd1a054670 100644
--- a/llvm/test/Transforms/PGOProfile/irreducible.ll
+++ b/llvm/test/Transforms/PGOProfile/irreducible.ll
@@ -1,6 +1,9 @@
 ; RUN: llvm-profdata merge %S/Inputs/irreducible.proftext -o %t.profdata
-; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
-; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: llvm-profdata merge %S/Inputs/irreducible_entry.proftext -o %t2.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -S | FileCheck %s --check-prefix=USE
 
 ; GEN: $__llvm_profile_raw_version = comdat any
 
diff --git a/llvm/test/Transforms/PGOProfile/landingpad.ll b/llvm/test/Transforms/PGOProfile/landingpad.ll
index 932d2299bbf17..998cda2488aaa 100644
--- a/llvm/test/Transforms/PGOProfile/landingpad.ll
+++ b/llvm/test/Transforms/PGOProfile/landingpad.ll
@@ -1,8 +1,14 @@
-; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
-; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -pgo-instr-gen -pgo-instrument-entry=false -S | FileCheck %s --check-prefixes=GEN,NOTENTRY
+; RUN: opt < %s -passes=pgo-instr-gen -pgo-instrument-entry=false -S | FileCheck %s --check-prefixes=GEN,NOTENTRY
 ; RUN: llvm-profdata merge %S/Inputs/landingpad.proftext -o %t.profdata
-; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
-; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+;
+; RUN: opt < %s -pgo-instr-gen -pgo-instrument-entry=true -S | FileCheck %s --check-prefixes=GEN,ENTRY
+; RUN: opt < %s -passes=pgo-instr-gen -pgo-instrument-entry=true -S | FileCheck %s --check-prefixes=GEN,ENTRY
+; RUN: llvm-profdata merge %S/Inputs/landingpad_entry.proftext -o %t2.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -S | FileCheck %s --check-prefix=USE
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -46,7 +52,8 @@ declare void @__cxa_throw(i8*, i8*, i8*)
 define i32 @foo(i32 %i) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
 ; GEN: entry:
-; GEN-NOT: call void @llvm.instrprof.increment
+; NOTENTRY-NOT: call void @llvm.instrprof.increment
+; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 59130013419, i32 4, i32 0)
   %rem = srem i32 %i, 2
   %tobool = icmp ne i32 %rem, 0
   br i1 %tobool, label %if.then, label %if.end
@@ -102,7 +109,8 @@ try.cont:
 
 if.end:
 ; GEN: if.end:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 59130013419, i32 4, i32 0)
+; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 59130013419, i32 4, i32 0)
+; ENTRY-NOT: call void @llvm.instrprof.increment
   %tmp8 = load i32, i32* @val, align 4
   %add = add nsw i32 %tmp8, %i
   store i32 %add, i32* @val, align 4
diff --git a/llvm/test/Transforms/PGOProfile/loop1.ll b/llvm/test/Transforms/PGOProfile/loop1.ll
index dbc29547b3cfd..94c867445fbb3 100644
--- a/llvm/test/Transforms/PGOProfile/loop1.ll
+++ b/llvm/test/Transforms/PGOProfile/loop1.ll
@@ -1,8 +1,14 @@
-; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
-; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -pgo-instr-gen -pgo-instrument-entry=false -S | FileCheck %s --check-prefixes=GEN,NOTENTRY
+; RUN: opt < %s -passes=pgo-instr-gen -pgo-instrument-entry=false -S | FileCheck %s --check-prefixes=GEN,NOTENTRY
 ; RUN: llvm-profdata merge %S/Inputs/loop1.proftext -o %t.profdata
-; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
-; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -pgo-instr-gen -pgo-instrument-entry=true -S | FileCheck %s --check-prefixes=GEN,ENTRY
+; RUN: opt < %s -passes=pgo-instr-gen -pgo-instrument-entry=true -S | FileCheck %s --check-prefixes=GEN,ENTRY
+; RUN: llvm-profdata merge %S/Inputs/loop1_entry.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -13,7 +19,8 @@ target triple = "x86_64-unknown-linux-gnu"
 define i32 @test_simple_for(i32 %n) {
 entry:
 ; GEN: entry:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_simple_for, i32 0, i32 0), i64 34137660316, i32 2, i32 1)
+; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_simple_for, i32 0, i32 0), i64 34137660316, i32 2, i32 1)
+; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_simple_for, i32 0, i32 0), i64 34137660316, i32 2, i32 0)
   br label %for.cond
 
 for.cond:
@@ -35,7 +42,8 @@ for.body:
 
 for.inc:
 ; GEN: for.inc:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_simple_for, i32 0, i32 0), i64 34137660316, i32 2, i32 0)
+; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_simple_for, i32 0, i32 0), i64 34137660316, i32 2, i32 0)
+; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_simple_for, i32 0, i32 0), i64 34137660316, i32 2, i32 1)
   %inc1 = add nsw i32 %i, 1
   br label %for.cond
 
diff --git a/llvm/test/Transforms/PGOProfile/loop2.ll b/llvm/test/Transforms/PGOProfile/loop2.ll
index 1334dd26156b2..90cb2e3e49d98 100644
--- a/llvm/test/Transforms/PGOProfile/loop2.ll
+++ b/llvm/test/Transforms/PGOProfile/loop2.ll
@@ -1,8 +1,13 @@
-; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
-; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -pgo-instr-gen -pgo-instrument-entry=false -S | FileCheck %s --check-prefixes=GEN,NOTENTRY
+; RUN: opt < %s -passes=pgo-instr-gen -pgo-instrument-entry=false -S | FileCheck %s --check-prefixes=GEN,NOTENTRY
 ; RUN: llvm-profdata merge %S/Inputs/loop2.proftext -o %t.profdata
-; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
-; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -pgo-instr-gen -pgo-instrument-entry=true -S | FileCheck %s --check-prefixes=GEN,ENTRY
+; RUN: opt < %s -passes=pgo-instr-gen -pgo-instrument-entry=true -S | FileCheck %s --check-prefixes=GEN,ENTRY
+; RUN: llvm-profdata merge %S/Inputs/loop2_entry.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -13,7 +18,8 @@ target triple = "x86_64-unknown-linux-gnu"
 define i32 @test_nested_for(i32 %r, i32 %s) {
 entry:
 ; GEN: entry:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_nested_for, i32 0, i32 0), i64 53929068288, i32 3, i32 2)
+; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_nested_for, i32 0, i32 0), i64 53929068288, i32 3, i32 2)
+; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_nested_for, i32 0, i32 0), i64 53929068288, i32 3, i32 0)
   br label %for.cond.outer
 
 for.cond.outer:
@@ -49,7 +55,8 @@ for.body.inner:
 
 for.inc.inner:
 ; GEN: for.inc.inner:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_nested_for, i32 0, i32 0), i64 53929068288, i32 3, i32 0)
+; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_nested_for, i32 0, i32 0), i64 53929068288, i32 3, i32 0)
+; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_nested_for, i32 0, i32 0), i64 53929068288, i32 3, i32 1)
   %inc.1 = add nsw i32 %j.0, 1
   br label %for.cond.inner
 
@@ -59,7 +66,8 @@ for.end.inner:
 
 for.inc.outer:
 ; GEN: for.inc.outer:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_nested_for, i32 0, i32 0), i64 53929068288, i32 3, i32 1)
+; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_nested_for, i32 0, i32 0), i64 53929068288, i32 3, i32 1)
+; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_nested_for, i32 0, i32 0), i64 53929068288, i32 3, i32 2)
   %inc.2 = add nsw i32 %i.0, 1
   br label %for.cond.outer
 
diff --git a/llvm/test/Transforms/PGOProfile/memcpy.ll b/llvm/test/Transforms/PGOProfile/memcpy.ll
index e00e1d350871d..6047c95e7c084 100644
--- a/llvm/test/Transforms/PGOProfile/memcpy.ll
+++ b/llvm/test/Transforms/PGOProfile/memcpy.ll
@@ -1,7 +1,5 @@
-; RUN: opt < %s -pgo-instr-gen -instrprof -use-old-memop-value-prof=true -S | FileCheck %s --check-prefix=OLDMEMOPVP
-; RUN: opt < %s -pgo-instr-gen -instrprof -use-old-memop-value-prof=false -S | FileCheck %s --check-prefix=NEWMEMOPVP
-; RUN: opt <%s -passes=pgo-instr-gen,instrprof -use-old-memop-value-prof=true -S | FileCheck %s --check-prefix=OLDMEMOPVP
-; RUN: opt <%s -passes=pgo-instr-gen,instrprof -use-old-memop-value-prof=false -S | FileCheck %s --check-prefix=NEWMEMOPVP
+; RUN: opt < %s -pgo-instr-gen -instrprof -S | FileCheck %s
+; RUN: opt <%s -passes=pgo-instr-gen,instrprof -S | FileCheck %s
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -25,8 +23,7 @@ for.cond1:
 
 for.body3:
   %conv = sext i32 %add to i64
-; OLDMEMOPVP: call void @__llvm_profile_instrument_range(i64 %conv, i8* bitcast ({ i64, i64, i64*, i8*, i8*, i32, [2 x i16] }* @__profd_foo to i8*), i32 0, i64 0, i64 8, i64 8192)
-; NEWMEMOPVP: call void @__llvm_profile_instrument_memop(i64 %conv, i8* bitcast ({ i64, i64, i64*, i8*, i8*, i32, [2 x i16] }* @__profd_foo to i8*), i32 0)
+; CHECK: call void @__llvm_profile_instrument_range(i64 %conv, i8* bitcast ({ i64, i64, i64*, i8*, i8*, i32, [2 x i16] }* @__profd_foo to i8*), i32 0, i64 0, i64 8, i64 8192)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %conv, i1 false)
   %inc = add nsw i32 %j.0, 1
   br label %for.cond1
diff --git a/llvm/test/Transforms/PGOProfile/memop_profile_funclet.ll b/llvm/test/Transforms/PGOProfile/memop_profile_funclet.ll
index 43c85ed25baae..b79431e3128e8 100644
--- a/llvm/test/Transforms/PGOProfile/memop_profile_funclet.ll
+++ b/llvm/test/Transforms/PGOProfile/memop_profile_funclet.ll
@@ -1,10 +1,8 @@
 ; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
-; RUN: opt < %s -pgo-instr-gen -instrprof -use-old-memop-value-prof=true -S | FileCheck %s --check-prefixes=LOWER,LOWEROLDMEMOPVP
-; RUN: opt < %s -pgo-instr-gen -instrprof -use-old-memop-value-prof=false -S | FileCheck %s --check-prefixes=LOWER,LOWERNEWMEMOPVP
+; RUN: opt < %s -pgo-instr-gen -instrprof -S | FileCheck %s --check-prefix=LOWER
 
 ; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
-; RUN: opt < %s -passes=pgo-instr-gen,instrprof -use-old-memop-value-prof=true -S | FileCheck %s --check-prefixes=LOWER,LOWEROLDMEMOPVP
-; RUN: opt < %s -passes=pgo-instr-gen,instrprof -use-old-memop-value-prof=false -S | FileCheck %s --check-prefixes=LOWER,LOWERNEWMEMOPVP
+; RUN: opt < %s -passes=pgo-instr-gen,instrprof -S | FileCheck %s --check-prefix=LOWER
 
 ; This test is to verify that PGO runtime library calls get created with the
 ; appropriate operand bundle funclet information when a memory intrinsic
@@ -65,8 +63,7 @@ try.cont:                                         ; preds = %entry
 ; GEN-SAME: [ "funclet"(token %tmp1) ]
 
 ; LOWER: catch:
-; LOWEROLDMEMOPVP: call void @__llvm_profile_instrument_range(
-; LOWERNEWMEMOPVP: call void @__llvm_profile_instrument_memop(
+; LOWER: call void @__llvm_profile_instrument_range(
 ; LOWER-SAME: [ "funclet"(token %tmp1) ]
 
 declare dso_local void @"?may_throw@@YAXH@Z"(i32)
diff --git a/llvm/test/Transforms/PGOProfile/misexpect-branch-stripped.ll b/llvm/test/Transforms/PGOProfile/misexpect-branch-stripped.ll
index f629543c5a1ca..e21950abc0ec1 100644
--- a/llvm/test/Transforms/PGOProfile/misexpect-branch-stripped.ll
+++ b/llvm/test/Transforms/PGOProfile/misexpect-branch-stripped.ll
@@ -1,20 +1,28 @@
-
 ; RUN: llvm-profdata merge %S/Inputs/misexpect-branch.proftext -o %t.profdata
 
-
-; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pgo-warn-misexpect 2>&1 | FileCheck %s --check-prefix=WARNING
-; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=REMARK
-; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pgo-warn-misexpect -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=BOTH
-; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S -pgo-warn-misexpect 2>&1 | FileCheck %s --check-prefix=WARNING
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S -pgo-warn-misexpect -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=BOTH
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
 
 ; New PM
-; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -S 2>&1 | FileCheck %s --check-prefix=WARNING
-; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=REMARK
-; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=BOTH
-; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -S 2>&1 | FileCheck %s --check-prefix=WARNING
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=BOTH
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
 
+; RUN: llvm-profdata merge %S/Inputs/misexpect-branch_entry.proftext -o %t2.profdata
 
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -S -pgo-warn-misexpect 2>&1 | FileCheck %s --check-prefix=WARNING
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -S -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -S -pgo-warn-misexpect -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=BOTH
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
 
+; New PM
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -pgo-warn-misexpect -S 2>&1 | FileCheck %s --check-prefix=WARNING
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -pgo-warn-misexpect -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=BOTH
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
 ; WARNING-DAG: warning: <unknown>:0:0: 19.98%
 ; WARNING-NOT: remark: <unknown>:0:0: Potential performance regression from use of the llvm.expect intrinsic: Annotation was correct on 19.98% (399668 / 2000000) of profiled executions.
 
diff --git a/llvm/test/Transforms/PGOProfile/misexpect-branch.ll b/llvm/test/Transforms/PGOProfile/misexpect-branch.ll
index df4b53de29475..726d13b0084ac 100644
--- a/llvm/test/Transforms/PGOProfile/misexpect-branch.ll
+++ b/llvm/test/Transforms/PGOProfile/misexpect-branch.ll
@@ -1,19 +1,28 @@
-
 ; RUN: llvm-profdata merge %S/Inputs/misexpect-branch.proftext -o %t.profdata
 
-
-; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pgo-warn-misexpect 2>&1 | FileCheck %s --check-prefix=WARNING
-; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=REMARK
-; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pgo-warn-misexpect -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=BOTH
-; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S -pgo-warn-misexpect 2>&1 | FileCheck %s --check-prefix=WARNING
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S -pgo-warn-misexpect -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=BOTH
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
 
 ; New PM
-; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -S 2>&1 | FileCheck %s --check-prefix=WARNING
-; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=REMARK
-; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=BOTH
-; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -S 2>&1 | FileCheck %s --check-prefix=WARNING
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=BOTH
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
 
+; RUN: llvm-profdata merge %S/Inputs/misexpect-branch_entry.proftext -o %t2.profdata
 
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -S -pgo-warn-misexpect 2>&1 | FileCheck %s --check-prefix=WARNING
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -S -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -S -pgo-warn-misexpect -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=BOTH
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
+
+; New PM
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -pgo-warn-misexpect -S 2>&1 | FileCheck %s --check-prefix=WARNING
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -pgo-warn-misexpect -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=BOTH
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
 
 ; WARNING-DAG: warning: misexpect-branch.c:22:0: 19.98%
 ; WARNING-NOT: remark: misexpect-branch.c:22:0: Potential performance regression from use of the llvm.expect intrinsic: Annotation was correct on 19.98% (399668 / 2000000) of profiled executions.
diff --git a/llvm/test/Transforms/PGOProfile/misexpect-switch-default.ll b/llvm/test/Transforms/PGOProfile/misexpect-switch-default.ll
index e6a9389c884de..1fcac0eabf8df 100644
--- a/llvm/test/Transforms/PGOProfile/misexpect-switch-default.ll
+++ b/llvm/test/Transforms/PGOProfile/misexpect-switch-default.ll
@@ -1,16 +1,15 @@
-
 ; RUN: llvm-profdata merge %S/Inputs/misexpect-switch.proftext -o %t.profdata
 
-; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pgo-warn-misexpect 2>&1 | FileCheck %s --check-prefix=WARNING
-; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=REMARK
-; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pgo-warn-misexpect -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=BOTH
-; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
+; RUN: opt < %s -lower-expect -pgo-instrument-entry=false -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pgo-warn-misexpect 2>&1 | FileCheck %s --check-prefix=WARNING
+; RUN: opt < %s -lower-expect -pgo-instrument-entry=false -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt < %s -lower-expect -pgo-instrument-entry=false -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pgo-warn-misexpect -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=BOTH
+; RUN: opt < %s -lower-expect -pgo-instrument-entry=false -pgo-instr-use -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
 
 ; New PM
-; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -S 2>&1 | FileCheck %s --check-prefix=WARNING
-; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=REMARK
-; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=BOTH
-; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -S 2>&1 | FileCheck %s --check-prefix=WARNING
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=BOTH
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
 
 ; WARNING-DAG: warning: <unknown>:0:0: 0.00%
 ; WARNING-NOT: remark: <unknown>:0:0: Potential performance regression from use of the llvm.expect intrinsic: Annotation was correct on 0.00% (0 / 27943) of profiled executions.
@@ -32,7 +31,6 @@
 ; CHECK-DAG: !{!"misexpect", i64 0, i64 2000, i64 1}
 
 
-
 ; ModuleID = 'misexpect-switch.c'
 source_filename = "misexpect-switch.c"
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/PGOProfile/misexpect-switch.ll b/llvm/test/Transforms/PGOProfile/misexpect-switch.ll
index 7883c23547f89..dd8c7c70ec202 100644
--- a/llvm/test/Transforms/PGOProfile/misexpect-switch.ll
+++ b/llvm/test/Transforms/PGOProfile/misexpect-switch.ll
@@ -1,21 +1,36 @@
-
 ; RUN: llvm-profdata merge %S/Inputs/misexpect-switch.proftext -o %t.profdata
 ; RUN: llvm-profdata merge %S/Inputs/misexpect-switch-correct.proftext -o %t.c.profdata
 
-; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pgo-warn-misexpect 2>&1 | FileCheck %s --check-prefix=WARNING
-; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=REMARK
-; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S -pgo-warn-misexpect -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=BOTH
-; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S -pgo-warn-misexpect 2>&1 | FileCheck %s --check-prefix=WARNING
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S -pgo-warn-misexpect -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=BOTH
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
 
 ; New PM
-; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -S 2>&1 | FileCheck %s --check-prefix=WARNING
-; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=REMARK
-; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=BOTH
-; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -S 2>&1 | FileCheck %s --check-prefix=WARNING
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -pgo-warn-misexpect -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=BOTH
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
+
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.c.profdata -S -pgo-warn-misexpect -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=CORRECT
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=false -pgo-test-profile-file=%t.c.profdata -pgo-warn-misexpect -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=CORRECT
 
-; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-test-profile-file=%t.c.profdata -S -pgo-warn-misexpect -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=CORRECT
-; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-test-profile-file=%t.c.profdata -pgo-warn-misexpect -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=CORRECT
+; RUN: llvm-profdata merge %S/Inputs/misexpect-switch_entry.proftext -o %t2.profdata
+; RUN: llvm-profdata merge %S/Inputs/misexpect-switch-correct_entry.proftext -o %t2.c.profdata
 
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -S -pgo-warn-misexpect 2>&1 | FileCheck %s --check-prefix=WARNING
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -S -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -S -pgo-warn-misexpect -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=BOTH
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
+
+; New PM
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -pgo-warn-misexpect -S 2>&1 | FileCheck %s --check-prefix=WARNING
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -pgo-warn-misexpect -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=BOTH
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=true -pgo-test-profile-file=%t2.profdata -S 2>&1 | FileCheck %s --check-prefix=DISABLED
+
+; RUN: opt < %s -lower-expect -pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t2.c.profdata -S -pgo-warn-misexpect -pass-remarks=misexpect 2>&1 | FileCheck %s --check-prefix=CORRECT
+; RUN: opt < %s -passes="function(lower-expect),pgo-instr-use" -pgo-instrument-entry=true -pgo-test-profile-file=%t2.c.profdata -pgo-warn-misexpect -pass-remarks=misexpect -S 2>&1 | FileCheck %s --check-prefix=CORRECT
 ; WARNING-DAG: warning: misexpect-switch.c:26:5: 0.00%
 ; WARNING-NOT: remark: misexpect-switch.c:26:5: Potential performance regression from use of the llvm.expect intrinsic: Annotation was correct on 0.00% (0 / 8112) of profiled executions.
 
@@ -35,8 +50,6 @@
 ; CORRECT-NOT: remark: {{.*}}
 ; CHECK-DAG: !{!"misexpect", i64 0, i64 2000, i64 1}
 
-
-
 ; ModuleID = 'misexpect-switch.c'
 source_filename = "misexpect-switch.c"
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/PGOProfile/switch.ll b/llvm/test/Transforms/PGOProfile/switch.ll
index 2cefd34d855ca..983470408a48d 100644
--- a/llvm/test/Transforms/PGOProfile/switch.ll
+++ b/llvm/test/Transforms/PGOProfile/switch.ll
@@ -1,8 +1,13 @@
-; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
-; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -pgo-instr-gen -pgo-instrument-entry=false -S | FileCheck %s --check-prefixes=GEN,NOTENTRY
+; RUN: opt < %s -passes=pgo-instr-gen -pgo-instrument-entry=false -S | FileCheck %s --check-prefixes=GEN,NOTENTRY
 ; RUN: llvm-profdata merge %S/Inputs/switch.proftext -o %t.profdata
-; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
-; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-instrument-entry=false -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -pgo-instr-gen -pgo-instrument-entry=true -S | FileCheck %s --check-prefixes=GEN,ENTRY
+; RUN: opt < %s -passes=pgo-instr-gen -pgo-instrument-entry=true -S | FileCheck %s --check-prefixes=GEN,ENTRY
+; RUN: llvm-profdata merge %S/Inputs/switch_entry.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-instrument-entry=true -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -13,7 +18,8 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @test_switch(i32 %i) {
 entry:
 ; GEN: entry:
-; GEN-NOT: call void @llvm.instrprof.increment
+; NOTENTRY-NOT: call void @llvm.instrprof.increment
+; ENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @__profn_test_switch, i32 0, i32 0), i64 46200943743, i32 4, i32 0)
   switch i32 %i, label %sw.default [
     i32 1, label %sw.bb
     i32 2, label %sw.bb1
@@ -30,7 +36,8 @@ sw.bb:
 
 sw.bb1:
 ; GEN: sw.bb1:
-; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @__profn_test_switch, i32 0, i32 0), i64 46200943743, i32 4, i32 0)
+; NOTENTRY: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @__profn_test_switch, i32 0, i32 0), i64 46200943743, i32 4, i32 0)
+; ENTRY-NOT: call void @llvm.instrprof.increment
   br label %sw.epilog
 
 sw.bb2:
diff --git a/llvm/test/Transforms/PGOProfile/thinlto_cspgo_use.ll b/llvm/test/Transforms/PGOProfile/thinlto_cspgo_use.ll
index 1be8de4c0841c..357115dc6501c 100644
--- a/llvm/test/Transforms/PGOProfile/thinlto_cspgo_use.ll
+++ b/llvm/test/Transforms/PGOProfile/thinlto_cspgo_use.ll
@@ -3,7 +3,7 @@
 ; RUN: opt -module-summary %s -o %t1.bc
 ; RUN: opt -module-summary %S/Inputs/thinlto_cspgo_bar_use.ll -o %t2.bc
 ; RUN: llvm-profdata merge %S/Inputs/thinlto_cs.proftext -o %t3.profdata
-; RUN: llvm-lto2 run -lto-cspgo-profile-file=%t3.profdata -save-temps -o %t %t1.bc %t2.bc \
+; RUN: llvm-lto2 run -lto-cspgo-profile-file=%t3.profdata -pgo-instrument-entry=false -save-temps -o %t %t1.bc %t2.bc \
 ; RUN:   -r=%t1.bc,foo,pl \
 ; RUN:   -r=%t1.bc,bar,l \
 ; RUN:   -r=%t1.bc,main,plx \
diff --git a/llvm/test/Transforms/SCCP/indirectbr.ll b/llvm/test/Transforms/SCCP/indirectbr.ll
index f54da3fb5918e..6889282e3874f 100644
--- a/llvm/test/Transforms/SCCP/indirectbr.ll
+++ b/llvm/test/Transforms/SCCP/indirectbr.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -ipsccp < %s | FileCheck %s
 
 declare void @BB0_f()
@@ -5,10 +6,14 @@ declare void @BB1_f()
 
 ; Make sure we can eliminate what is in BB0 as we know that the indirectbr is going to BB1.
 ;
-; CHECK-LABEL: define void @indbrtest1(
-; CHECK-NOT: call void @BB0_f()
-; CHECK: ret void
 define void @indbrtest1() {
+; CHECK-LABEL: @indbrtest1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       BB1:
+; CHECK-NEXT:    call void @BB1_f()
+; CHECK-NEXT:    ret void
+;
 entry:
   indirectbr i8* blockaddress(@indbrtest1, %BB1), [label %BB0, label %BB1]
 BB0:
@@ -23,10 +28,14 @@ BB1:
 ; by looking through the casts. The casts should be folded away when they are visited
 ; before the indirectbr instruction.
 ;
-; CHECK-LABEL: define void @indbrtest2(
-; CHECK-NOT: call void @BB0_f()
-; CHECK: ret void
 define void @indbrtest2() {
+; CHECK-LABEL: @indbrtest2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       BB1:
+; CHECK-NEXT:    call void @BB1_f()
+; CHECK-NEXT:    ret void
+;
 entry:
   %a = ptrtoint i8* blockaddress(@indbrtest2, %BB1) to i64
   %b = inttoptr i64 %a to i8*
@@ -41,11 +50,19 @@ BB1:
 }
 
 ; Make sure we can not eliminate BB0 as we do not know the target of the indirectbr.
-;
-; CHECK-LABEL: define void @indbrtest3(
-; CHECK: call void @BB0_f()
-; CHECK: ret void
+
 define void @indbrtest3(i8** %Q) {
+; CHECK-LABEL: @indbrtest3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[T:%.*]] = load i8*, i8** [[Q:%.*]], align 8
+; CHECK-NEXT:    indirectbr i8* [[T]], [label [[BB0:%.*]], label %BB1]
+; CHECK:       BB0:
+; CHECK-NEXT:    call void @BB0_f()
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       BB1:
+; CHECK-NEXT:    call void @BB1_f()
+; CHECK-NEXT:    ret void
+;
 entry:
   %t = load i8*, i8** %Q
   indirectbr i8* %t, [label %BB0, label %BB1]
@@ -58,25 +75,41 @@ BB1:
 }
 
 ; Make sure we eliminate BB1 as we pick the first successor on undef.
-;
-; CHECK-LABEL: define void @indbrtest4(
-; CHECK: call void @BB0_f()
-; CHECK: ret void
+
 define void @indbrtest4(i8** %Q) {
+; CHECK-LABEL: @indbrtest4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BB0:%.*]]
+; CHECK:       BB0:
+; CHECK-NEXT:    call void @BB0_f()
+; CHECK-NEXT:    ret void
+;
 entry:
   indirectbr i8* undef, [label %BB0, label %BB1]
 BB0:
   call void @BB0_f()
-  br label %BB1
+  ret void
 BB1:
   call void @BB1_f()
   ret void
 }
 
-
-; CHECK-LABEL: define internal i32 @indbrtest5(
-; CHECK: ret i32 undef
 define internal i32 @indbrtest5(i1 %c) {
+; CHECK-LABEL: @indbrtest5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BRANCH_BLOCK:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BRANCH_BLOCK]]
+; CHECK:       branch.block:
+; CHECK-NEXT:    [[ADDR:%.*]] = phi i8* [ blockaddress(@indbrtest5, [[TARGET1:%.*]]), [[BB1]] ], [ blockaddress(@indbrtest5, [[TARGET2:%.*]]), [[BB2]] ]
+; CHECK-NEXT:    indirectbr i8* [[ADDR]], [label [[TARGET1]], label %target2]
+; CHECK:       target1:
+; CHECK-NEXT:    br label [[TARGET2]]
+; CHECK:       target2:
+; CHECK-NEXT:    ret i32 undef
+;
 entry:
   br i1 %c, label %bb1, label %bb2
 
@@ -100,9 +133,34 @@ target2:
 
 
 define i32 @indbrtest5_callee(i1 %c) {
-; CHECK-LABEL: define i32 @indbrtest5_callee(
-; CHECK-NEXT:    %r = call i32 @indbrtest5(i1 %c)
+; CHECK-LABEL: @indbrtest5_callee(
+; CHECK-NEXT:    [[R:%.*]] = call i32 @indbrtest5(i1 [[C:%.*]])
 ; CHECK-NEXT:    ret i32 10
+;
   %r = call i32 @indbrtest5(i1 %c)
   ret i32 %r
 }
+
+define i32 @indbr_duplicate_successors_phi(i1 %c, i32 %x) {
+; CHECK-LABEL: @indbr_duplicate_successors_phi(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[INDBR:%.*]], label [[BB0:%.*]]
+; CHECK:       indbr:
+; CHECK-NEXT:    br label [[BB0]]
+; CHECK:       BB0:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ 0, [[INDBR]] ]
+; CHECK-NEXT:    ret i32 [[PHI]]
+;
+entry:
+  br i1 %c, label %indbr, label %BB0
+
+indbr:
+  indirectbr i8* blockaddress(@indbr_duplicate_successors_phi, %BB0), [label %BB0, label %BB0, label %BB1]
+
+BB0:
+  %phi = phi i32 [ %x, %entry ], [ 0, %indbr ], [ 0, %indbr ]
+  ret i32 %phi
+
+BB1:
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/SCCP/ip-add-range-to-call.ll b/llvm/test/Transforms/SCCP/ip-add-range-to-call.ll
index 6ae234de79a0f..c12e2b5af1f97 100644
--- a/llvm/test/Transforms/SCCP/ip-add-range-to-call.ll
+++ b/llvm/test/Transforms/SCCP/ip-add-range-to-call.ll
@@ -1,10 +1,9 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -ipsccp -S %s | FileCheck %s
 
 ; Test 1.
 ; Both arguments and return value of @callee can be tracked. The inferred range
 ; can be added to call sites.
-define internal i32 @callee(i32 %x) {
+define internal noundef i32 @callee(i32 %x) {
 ; CHECK-LABEL: @callee(
 ; CHECK-NEXT:    ret i32 [[X:%.*]]
 ;
@@ -13,8 +12,8 @@ define internal i32 @callee(i32 %x) {
 
 define i32 @caller1() {
 ; CHECK-LABEL: @caller1(
-; CHECK-NEXT:    [[C1:%.*]] = call i32 @callee(i32 10)
-; CHECK-NEXT:    [[C2:%.*]] = call i32 @callee(i32 20)
+; CHECK-NEXT:    [[C1:%.*]] = call i32 @callee(i32 10), !range [[RANGE_10_21:![0-9]+]]
+; CHECK-NEXT:    [[C2:%.*]] = call i32 @callee(i32 20), !range [[RANGE_10_21]]
 ; CHECK-NEXT:    [[A:%.*]] = add i32 [[C1]], [[C2]]
 ; CHECK-NEXT:    ret i32 [[A]]
 ;
@@ -27,7 +26,7 @@ define i32 @caller1() {
 define i32 @caller2(i32 %x) {
 ; CHECK-LABEL: @caller2(
 ; CHECK-NEXT:    [[X_15:%.*]] = and i32 [[X:%.*]], 15
-; CHECK-NEXT:    [[C:%.*]] = call i32 @callee(i32 [[X_15]])
+; CHECK-NEXT:    [[C:%.*]] = call i32 @callee(i32 [[X_15]]), !range [[RANGE_10_21]]
 ; CHECK-NEXT:    ret i32 [[C]]
 ;
   %x.15 = and i32 %x, 15
@@ -42,7 +41,7 @@ define i32 @caller2(i32 %x) {
 
 declare void @use_cb1(i32 (i32)*)
 
-define internal i32 @callee2(i32 %x) {
+define internal noundef i32 @callee2(i32 %x) {
 ; CHECK-LABEL: @callee2(
 ; CHECK-NEXT:    ret i32 [[X:%.*]]
 ;
@@ -52,7 +51,9 @@ define internal i32 @callee2(i32 %x) {
 define void @caller_cb1() {
 ; CHECK-LABEL: @caller_cb1(
 ; CHECK-NEXT:    [[C1:%.*]] = call i32 @callee2(i32 9)
+; CHECK-NOT:   !range
 ; CHECK-NEXT:    [[C2:%.*]] = call i32 @callee2(i32 10)
+; CHECK-NOT:   !range
 ; CHECK-NEXT:    call void @use_cb1(i32 (i32)* @callee2)
 ; CHECK-NEXT:    ret void
 ;
@@ -69,7 +70,7 @@ define void @caller_cb1() {
 
 declare void @use_cb2(i32 (i32)*)
 
-define internal i32 @callee3(i32 %x) {
+define internal noundef i32 @callee3(i32 %x) {
 ; CHECK-LABEL: @callee3(
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[X:%.*]], 10
 ; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i32 500, i32 600
@@ -82,8 +83,8 @@ define internal i32 @callee3(i32 %x) {
 
 define void @caller_cb2() {
 ; CHECK-LABEL: @caller_cb2(
-; CHECK-NEXT:    [[C1:%.*]] = call i32 @callee3(i32 9)
-; CHECK-NEXT:    [[C2:%.*]] = call i32 @callee3(i32 10)
+; CHECK-NEXT:    [[C1:%.*]] = call i32 @callee3(i32 9), !range [[RANGE_500_601:![0-9]+]]
+; CHECK-NEXT:    [[C2:%.*]] = call i32 @callee3(i32 10), !range [[RANGE_500_601]]
 ; CHECK-NEXT:    call void @use_cb2(i32 (i32)* @callee3)
 ; CHECK-NEXT:    ret void
 ;
@@ -99,7 +100,7 @@ define void @caller_cb2() {
 
 declare void @use_cb3(i32 (i32, i32)*)
 
-define internal i32 @callee4(i32 %x, i32 %y) {
+define internal noundef i32 @callee4(i32 %x, i32 %y) {
 ; CHECK-LABEL: @callee4(
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[X:%.*]], 10
 ; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i32 500, i32 [[Y:%.*]]
@@ -113,7 +114,9 @@ define internal i32 @callee4(i32 %x, i32 %y) {
 define void @caller_cb3() {
 ; CHECK-LABEL: @caller_cb3(
 ; CHECK-NEXT:    [[C1:%.*]] = call i32 @callee4(i32 11, i32 30)
+; CHECK-NOT:   !range
 ; CHECK-NEXT:    [[C2:%.*]] = call i32 @callee4(i32 12, i32 40)
+; CHECK-NOT:   !range
 ; CHECK-NEXT:    call void @use_cb3(i32 (i32, i32)* @callee4)
 ; CHECK-NEXT:    ret void
 ;
@@ -126,7 +129,7 @@ define void @caller_cb3() {
 ; Test 5.
 ; Range for the return value of callee5 includes undef. No range metadata
 ; should be added at call sites.
-define internal i32 @callee5(i32 %x, i32 %y) {
+define internal noundef i32 @callee5(i32 %x, i32 %y) {
 ; CHECK-LABEL: @callee5(
 ; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[X:%.*]], 15
 ; CHECK-NEXT:    br i1 [[C]], label [[BB1:%.*]], label [[BB2:%.*]]
@@ -155,7 +158,9 @@ exit:
 define i32 @caller5() {
 ; CHECK-LABEL: @caller5(
 ; CHECK-NEXT:    [[C1:%.*]] = call i32 @callee5(i32 10, i32 100)
+; CHECK-NOT:   !range
 ; CHECK-NEXT:    [[C2:%.*]] = call i32 @callee5(i32 20, i32 200)
+; CHECK-NOT:   !range
 ; CHECK-NEXT:    [[A:%.*]] = add i32 [[C1]], [[C2]]
 ; CHECK-NEXT:    ret i32 [[A]]
 ;
@@ -164,3 +169,6 @@ define i32 @caller5() {
   %a = add i32 %c1, %c2
   ret i32 %a
 }
+
+; CHECK: [[RANGE_10_21]] = !{i32 0, i32 21}
+; CHECK: [[RANGE_500_601]] = !{i32 500, i32 601}
diff --git a/llvm/test/Transforms/SCCP/ipscp-drop-argmemonly.ll b/llvm/test/Transforms/SCCP/ipscp-drop-argmemonly.ll
new file mode 100644
index 0000000000000..a110476eb4302
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/ipscp-drop-argmemonly.ll
@@ -0,0 +1,165 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-attributes
+; RUN: opt -ipsccp -S %s | FileCheck %s
+
+; Test cases to ensure argmemonly/inaccessiblemem_or_argmemonly attributes are
+; dropped, if a function argument is replaced by a constant.
+;
+; PR46717
+
+@g = internal global i32 0
+
+; Here the pointer argument %arg will be replaced by a constant. We need to
+; drop argmemonly.
+define internal void @ptrarg.1(i32* %arg, i32 %val) argmemonly nounwind {
+; CHECK: Function Attrs: argmemonly nounwind
+; CHECK-LABEL: @ptrarg.1(
+; CHECK-NEXT:    store i32 10, i32* @g, align 4
+; CHECK-NEXT:    ret void
+;
+  store i32 %val, i32* %arg
+  ret void
+}
+
+define i32 @caller.1(i32 %n) {
+; CHECK-LABEL: @caller.1(
+; CHECK-NEXT:    store i32 1, i32* @g, align 4
+; CHECK-NEXT:    tail call void @ptrarg.1(i32* @g, i32 10)
+; CHECK-NEXT:    [[G_VAL:%.*]] = load i32, i32* @g, align 4
+; CHECK-NEXT:    ret i32 [[G_VAL]]
+;
+  store i32 1, i32* @g
+  tail call void @ptrarg.1(i32* @g, i32 10)
+  %g.val = load i32, i32* @g
+  ret i32 %g.val
+}
+
+
+; Here only the non-pointer argument %val is replaced, no need
+; to drop the argmemonly attribute.
+define internal void @ptrarg.2(i32* %arg, i32 %val) argmemonly nounwind {
+; CHECK: Function Attrs: argmemonly nounwind
+; CHECK-LABEL: @ptrarg.2(
+; CHECK-NEXT:    store i32 10, i32* [[ARG:%.*]], align 4
+; CHECK-NEXT:    ret void
+;
+  store i32 %val, i32* %arg
+  ret void
+}
+
+define void @caller.2(i32* %ptr) {
+; CHECK-LABEL: @caller.2(
+; CHECK-NEXT:    tail call void @ptrarg.2(i32* [[PTR:%.*]], i32 10)
+; CHECK-NEXT:    ret void
+;
+  tail call void @ptrarg.2(i32* %ptr, i32 10)
+  ret void
+}
+
+
+; Here the pointer argument %arg will be replaced by a constant. We need to
+; drop inaccessiblemem_or_argmemonly.
+define internal void @ptrarg.3(i32* %arg, i32 %val) inaccessiblemem_or_argmemonly nounwind {
+; CHECK: Function Attrs: inaccessiblemem_or_argmemonly nounwind
+; CHECK-LABEL: @ptrarg.3(
+; CHECK-NEXT:    store i32 10, i32* @g, align 4
+; CHECK-NEXT:    ret void
+;
+  store i32 %val, i32* %arg
+  ret void
+}
+
+define i32 @caller.3(i32 %n) {
+; CHECK-LABEL: @caller.3(
+; CHECK-NEXT:    store i32 1, i32* @g, align 4
+; CHECK-NEXT:    tail call void @ptrarg.3(i32* @g, i32 10)
+; CHECK-NEXT:    [[G_VAL:%.*]] = load i32, i32* @g, align 4
+; CHECK-NEXT:    ret i32 [[G_VAL]]
+;
+  store i32 1, i32* @g
+  tail call void @ptrarg.3(i32* @g, i32 10)
+  %g.val = load i32, i32* @g
+  ret i32 %g.val
+}
+
+
+; Here only the non-pointer argument %val is replaced, no need
+; to drop the inaccessiblemem_or_argmemonly attribute.
+define internal void @ptrarg.4(i32* %arg, i32 %val) inaccessiblemem_or_argmemonly nounwind {
+; CHECK: Function Attrs: inaccessiblemem_or_argmemonly nounwind
+; CHECK-LABEL: @ptrarg.4(
+; CHECK-NEXT:    store i32 10, i32* [[ARG:%.*]], align 4
+; CHECK-NEXT:    ret void
+;
+  store i32 %val, i32* %arg
+  ret void
+}
+
+define void @caller.4(i32* %ptr) {
+; CHECK-LABEL: @caller.4(
+; CHECK-NEXT:    tail call void @ptrarg.4(i32* [[PTR:%.*]], i32 10)
+; CHECK-NEXT:    ret void
+;
+  tail call void @ptrarg.4(i32* %ptr, i32 10)
+  ret void
+}
+
+
+; Here the pointer argument %arg will be replaced by a constant. We need to
+; drop inaccessiblemem_or_argmemonly.
+define internal void @ptrarg.5(i32* %arg, i32 %val) argmemonly inaccessiblemem_or_argmemonly nounwind {
+; CHECK: Function Attrs: argmemonly inaccessiblemem_or_argmemonly nounwind
+; CHECK-LABEL: @ptrarg.5(
+; CHECK-NEXT:    store i32 10, i32* @g, align 4
+; CHECK-NEXT:    ret void
+;
+  store i32 %val, i32* %arg
+  ret void
+}
+
+define i32 @caller.5(i32 %n) {
+; CHECK-LABEL: @caller.5(
+; CHECK-NEXT:    store i32 1, i32* @g, align 4
+; CHECK-NEXT:    tail call void @ptrarg.5(i32* @g, i32 10)
+; CHECK-NEXT:    [[G_VAL:%.*]] = load i32, i32* @g, align 4
+; CHECK-NEXT:    ret i32 [[G_VAL]]
+;
+  store i32 1, i32* @g
+  tail call void @ptrarg.5(i32* @g, i32 10)
+  %g.val = load i32, i32* @g
+  ret i32 %g.val
+}
+
+
+; Make sure callsite attributes are also dropped when a pointer argument is
+; replaced.
+define internal void @ptrarg.6.cs.attributes(i32* %arg, i32 %val) {
+; CHECK-LABEL: @ptrarg.6.cs.attributes(
+; CHECK-NEXT:    unreachable
+;
+  store i32 %val, i32* %arg
+  ret void
+}
+
+define i32 @caller.6.cs.attributes(i32 %n) {
+; CHECK-LABEL: @caller.6.cs.attributes(
+; CHECK-NEXT:    store i32 1, i32* @g, align 4
+; CHECK-NEXT:    tail call void @ptrarg.5(i32* @g, i32 10) [[ARGMEMONLY_INACCESSIBLEMEM_OR_ARGMEMONLY_NOUNWIND:#[0-9]+]]
+; CHECK-NEXT:    tail call void @ptrarg.5(i32* @g, i32 10) [[INACCESSIBLEMEM_OR_ARGMEMONLY_NOUNWIND:#[0-9]+]]
+; CHECK-NEXT:    tail call void @ptrarg.5(i32* @g, i32 10) [[ARGMEMONLY_NOUNWIND:#[0-9]+]]
+; CHECK-NEXT:    tail call void @ptrarg.5(i32* @g, i32 10) [[NOUNWIND:#[0-9]+]]
+; CHECK-NEXT:    [[G_VAL:%.*]] = load i32, i32* @g, align 4
+; CHECK-NEXT:    ret i32 [[G_VAL]]
+;
+  store i32 1, i32* @g
+  tail call void @ptrarg.5(i32* @g, i32 10) argmemonly inaccessiblemem_or_argmemonly nounwind
+  tail call void @ptrarg.5(i32* @g, i32 10) inaccessiblemem_or_argmemonly nounwind
+  tail call void @ptrarg.5(i32* @g, i32 10) argmemonly nounwind
+  tail call void @ptrarg.5(i32* @g, i32 10) nounwind
+  %g.val = load i32, i32* @g
+  ret i32 %g.val
+}
+
+; CHECK-DAG: [[ARGMEMONLY_INACCESSIBLEMEM_OR_ARGMEMONLY_NOUNWIND]] = { argmemonly inaccessiblemem_or_argmemonly nounwind }
+; CHECK-DAG: [[INACCESSIBLEMEM_OR_ARGMEMONLY_NOUNWIND]] = { inaccessiblemem_or_argmemonly nounwind }
+; CHECK-DAG: [[ARGMEMONLY_NOUNWIND]] = { argmemonly nounwind }
+; CHECK-DAG: [[NOUNWIND]] = { nounwind }
diff --git a/llvm/test/Transforms/SCCP/predicateinfo-cond.ll b/llvm/test/Transforms/SCCP/predicateinfo-cond.ll
index d98b3cc76d928..8ed96ec9301f5 100644
--- a/llvm/test/Transforms/SCCP/predicateinfo-cond.ll
+++ b/llvm/test/Transforms/SCCP/predicateinfo-cond.ll
@@ -98,4 +98,32 @@ if3.end:
   ret i32 %phi2
 }
 
+define void @pr46814(i32 %a) {
+; CHECK-LABEL: @pr46814(
+; CHECK-NEXT:    [[C1:%.*]] = icmp uge i32 [[A:%.*]], 10
+; CHECK-NEXT:    [[C2:%.*]] = icmp ult i32 [[A]], 20
+; CHECK-NEXT:    [[C3:%.*]] = and i1 [[C1]], [[C2]]
+; CHECK-NEXT:    br i1 [[C3]], label [[IF_1:%.*]], label [[EXIT:%.*]]
+; CHECK:       if.1:
+; CHECK-NEXT:    br i1 true, label [[IF_2:%.*]], label [[EXIT]]
+; CHECK:       if.2:
+; CHECK-NEXT:    br i1 true, label [[EXIT]], label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+  %c1 = icmp uge i32 %a, 10
+  %c2 = icmp ult i32 %a, 20
+  %c3 = and i1 %c1, %c2
+  br i1 %c3, label %if.1, label %exit
+
+if.1:
+  br i1 %c3, label %if.2, label %exit
+
+if.2:
+  br i1 %c3, label %exit, label %exit
+
+exit:
+  ret void
+}
+
 declare void @llvm.assume(i1)
diff --git a/llvm/test/Transforms/SCCP/switch.ll b/llvm/test/Transforms/SCCP/switch.ll
index 155faa5c6067d..5f607a3afd89d 100644
--- a/llvm/test/Transforms/SCCP/switch.ll
+++ b/llvm/test/Transforms/SCCP/switch.ll
@@ -1,13 +1,271 @@
-; RUN: opt -S -sccp < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -ipsccp < %s | FileCheck %s
 
 ; Make sure we always consider the default edge executable for a switch
 ; with no cases.
 declare void @foo()
 define void @test1() {
-; CHECK-LABEL: define void @test1(
-; CHECK: call void @foo()
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    switch i32 undef, label [[D:%.*]] [
+; CHECK-NEXT:    ]
+; CHECK:       d:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    ret void
+;
   switch i32 undef, label %d []
 d:
   call void @foo()
   ret void
 }
+
+define i32 @test_duplicate_successors_phi(i1 %c, i32 %x) {
+; CHECK-LABEL: @test_duplicate_successors_phi(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[SWITCH:%.*]], label [[END:%.*]]
+; CHECK:       switch:
+; CHECK-NEXT:    switch i32 -1, label [[SWITCH_DEFAULT:%.*]] [
+; CHECK-NEXT:    i32 0, label [[END]]
+; CHECK-NEXT:    i32 1, label [[END]]
+; CHECK-NEXT:    ]
+; CHECK:       switch.default:
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       end:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ 1, [[SWITCH]] ], [ 1, [[SWITCH]] ]
+; CHECK-NEXT:    ret i32 [[PHI]]
+;
+entry:
+  br i1 %c, label %switch, label %end
+
+switch:
+  switch i32 -1, label %switch.default [
+  i32 0, label %end
+  i32 1, label %end
+  ]
+
+switch.default:
+  ret i32 -1
+
+end:
+  %phi = phi i32 [ %x, %entry ], [ 1, %switch ], [ 1, %switch ]
+  ret i32 %phi
+}
+
+define i32 @test_duplicate_successors_phi_2(i1 %c, i32 %x) {
+; CHECK-LABEL: @test_duplicate_successors_phi_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[SWITCH:%.*]], label [[END:%.*]]
+; CHECK:       switch:
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ 1, [[SWITCH]] ]
+; CHECK-NEXT:    ret i32 [[PHI]]
+;
+entry:
+  br i1 %c, label %switch, label %end
+
+switch:
+  switch i32 0, label %switch.default [
+  i32 0, label %end
+  i32 1, label %end
+  ]
+
+switch.default:
+  ret i32 -1
+
+end:
+  %phi = phi i32 [ %x, %entry ], [ 1, %switch ], [ 1, %switch ]
+  ret i32 %phi
+}
+
+define i32 @test_duplicate_successors_phi_3(i1 %c1, i32 %x) {
+; CHECK-LABEL: @test_duplicate_successors_phi_3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C1:%.*]], label [[SWITCH:%.*]], label [[SWITCH_1:%.*]]
+; CHECK:       switch:
+; CHECK-NEXT:    [[C2:%.*]] = icmp ult i32 [[X:%.*]], 3
+; CHECK-NEXT:    call void @llvm.assume(i1 [[C2]])
+; CHECK-NEXT:    switch i32 [[X]], label [[SWITCH_DEFAULT:%.*]] [
+; CHECK-NEXT:    i32 0, label [[SWITCH_DEFAULT]]
+; CHECK-NEXT:    i32 1, label [[SWITCH_0:%.*]]
+; CHECK-NEXT:    i32 2, label [[SWITCH_0]]
+; CHECK-NEXT:    i32 3, label [[SWITCH_1]]
+; CHECK-NEXT:    i32 4, label [[SWITCH_1]]
+; CHECK-NEXT:    ]
+; CHECK:       switch.default:
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       switch.0:
+; CHECK-NEXT:    ret i32 0
+; CHECK:       switch.1:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ [[X]], [[ENTRY:%.*]] ], [ 0, [[SWITCH]] ], [ 0, [[SWITCH]] ]
+; CHECK-NEXT:    ret i32 [[PHI]]
+;
+entry:
+  br i1 %c1, label %switch, label %switch.1
+
+switch:
+  %c2 = icmp ult i32 %x, 3
+  call void @llvm.assume(i1 %c2)
+  switch i32 %x, label %switch.default [
+  i32 0, label %switch.default
+  i32 1, label %switch.0
+  i32 2, label %switch.0
+  i32 3, label %switch.1
+  i32 4, label %switch.1
+  ]
+
+switch.default:
+  ret i32 -1
+
+switch.0:
+  ret i32 0
+
+switch.1:
+  %phi = phi i32 [ %x, %entry ], [ 0, %switch ], [ 0, %switch ]
+  ret i32 %phi
+}
+
+define i32 @test_local_range(i32 %x) {
+; CHECK-LABEL: @test_local_range(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[X:%.*]], 3
+; CHECK-NEXT:    call void @llvm.assume(i1 [[C]])
+; CHECK-NEXT:    switch i32 [[X]], label [[SWITCH_DEFAULT:%.*]] [
+; CHECK-NEXT:    i32 0, label [[SWITCH_0:%.*]]
+; CHECK-NEXT:    i32 1, label [[SWITCH_1:%.*]]
+; CHECK-NEXT:    i32 2, label [[SWITCH_2:%.*]]
+; CHECK-NEXT:    i32 3, label [[SWITCH_3:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       switch.default:
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       switch.0:
+; CHECK-NEXT:    ret i32 0
+; CHECK:       switch.1:
+; CHECK-NEXT:    ret i32 1
+; CHECK:       switch.2:
+; CHECK-NEXT:    ret i32 2
+; CHECK:       switch.3:
+; CHECK-NEXT:    ret i32 3
+;
+  %c = icmp ult i32 %x, 3
+  call void @llvm.assume(i1 %c)
+  switch i32 %x, label %switch.default [
+  i32 0, label %switch.0
+  i32 1, label %switch.1
+  i32 2, label %switch.2
+  i32 3, label %switch.3
+  ]
+
+switch.default:
+  ret i32 -1
+
+switch.0:
+  ret i32 0
+
+switch.1:
+  ret i32 1
+
+switch.2:
+  ret i32 2
+
+switch.3:
+  ret i32 3
+}
+
+define i32 @test_duplicate_successors(i32 %x) {
+; CHECK-LABEL: @test_duplicate_successors(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[X:%.*]], 3
+; CHECK-NEXT:    call void @llvm.assume(i1 [[C]])
+; CHECK-NEXT:    switch i32 [[X]], label [[SWITCH_DEFAULT:%.*]] [
+; CHECK-NEXT:    i32 0, label [[SWITCH_0:%.*]]
+; CHECK-NEXT:    i32 1, label [[SWITCH_0]]
+; CHECK-NEXT:    i32 2, label [[SWITCH_1:%.*]]
+; CHECK-NEXT:    i32 3, label [[SWITCH_1]]
+; CHECK-NEXT:    i32 4, label [[SWITCH_2:%.*]]
+; CHECK-NEXT:    i32 5, label [[SWITCH_2]]
+; CHECK-NEXT:    ]
+; CHECK:       switch.default:
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       switch.0:
+; CHECK-NEXT:    ret i32 0
+; CHECK:       switch.1:
+; CHECK-NEXT:    ret i32 1
+; CHECK:       switch.2:
+; CHECK-NEXT:    ret i32 2
+;
+  %c = icmp ult i32 %x, 3
+  call void @llvm.assume(i1 %c)
+  switch i32 %x, label %switch.default [
+  i32 0, label %switch.0
+  i32 1, label %switch.0
+  i32 2, label %switch.1
+  i32 3, label %switch.1
+  i32 4, label %switch.2
+  i32 5, label %switch.2
+  ]
+
+switch.default:
+  ret i32 -1
+
+switch.0:
+  ret i32 0
+
+switch.1:
+  ret i32 1
+
+switch.2:
+  ret i32 2
+}
+
+define internal i32 @test_ip_range(i32 %x) {
+; CHECK-LABEL: @test_ip_range(
+; CHECK-NEXT:    switch i32 [[X:%.*]], label [[SWITCH_DEFAULT:%.*]] [
+; CHECK-NEXT:    i32 0, label [[SWITCH_0:%.*]]
+; CHECK-NEXT:    i32 1, label [[SWITCH_1:%.*]]
+; CHECK-NEXT:    i32 2, label [[SWITCH_2:%.*]]
+; CHECK-NEXT:    i32 3, label [[SWITCH_3:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       switch.default:
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       switch.0:
+; CHECK-NEXT:    ret i32 0
+; CHECK:       switch.1:
+; CHECK-NEXT:    ret i32 1
+; CHECK:       switch.2:
+; CHECK-NEXT:    ret i32 2
+; CHECK:       switch.3:
+; CHECK-NEXT:    ret i32 3
+;
+  switch i32 %x, label %switch.default [
+  i32 0, label %switch.0
+  i32 1, label %switch.1
+  i32 2, label %switch.2
+  i32 3, label %switch.3
+  ]
+
+switch.default:
+  ret i32 -1
+
+switch.0:
+  ret i32 0
+
+switch.1:
+  ret i32 1
+
+switch.2:
+  ret i32 2
+
+switch.3:
+  ret i32 3
+}
+
+define void @call_test_ip_range() {
+; CHECK-LABEL: @call_test_ip_range(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @test_ip_range(i32 1)
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @test_ip_range(i32 3)
+; CHECK-NEXT:    ret void
+;
+  call i32 @test_ip_range(i32 1)
+  call i32 @test_ip_range(i32 3)
+  ret void
+}
+
+declare void @llvm.assume(i1)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/geps-non-pow-2.ll b/llvm/test/Transforms/SLPVectorizer/X86/geps-non-pow-2.ll
new file mode 100644
index 0000000000000..596543880d365
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/geps-non-pow-2.ll
@@ -0,0 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S -o - -mtriple=x86_64-unknown-linux -mcpu=haswell < %s | FileCheck %s
+@e = dso_local local_unnamed_addr global i32 0, align 4
+@f = dso_local local_unnamed_addr global i32 0, align 4
+
+; Function Attrs: nofree norecurse nounwind uwtable
+define dso_local i32 @g() local_unnamed_addr {
+; CHECK-LABEL: @g(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* @e, align 4
+; CHECK-NEXT:    [[TOBOOL_NOT19:%.*]] = icmp eq i32 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL_NOT19]], label [[WHILE_END:%.*]], label [[WHILE_BODY:%.*]]
+; CHECK:       while.body:
+; CHECK-NEXT:    [[C_022:%.*]] = phi i32* [ [[C_022_BE:%.*]], [[WHILE_BODY_BACKEDGE:%.*]] ], [ undef, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x i32*> [ [[TMP14:%.*]], [[WHILE_BODY_BACKEDGE]] ], [ undef, [[ENTRY]] ]
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[C_022]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint i32* [[C_022]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i32, <2 x i32*> [[TMP1]], <2 x i64> <i64 1, i64 1>
+; CHECK-NEXT:    switch i32 [[TMP3]], label [[WHILE_BODY_BACKEDGE]] [
+; CHECK-NEXT:    i32 2, label [[SW_BB:%.*]]
+; CHECK-NEXT:    i32 4, label [[SW_BB6:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       sw.bb:
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i32*> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint i32* [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i32, <2 x i32*> [[TMP1]], <2 x i64> <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i32*> [[TMP4]], i32 1
+; CHECK-NEXT:    store i32 [[TMP7]], i32* [[TMP9]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds i32, i32* [[C_022]], i64 2
+; CHECK-NEXT:    br label [[WHILE_BODY_BACKEDGE]]
+; CHECK:       sw.bb6:
+; CHECK-NEXT:    [[INCDEC_PTR8:%.*]] = getelementptr inbounds i32, i32* [[C_022]], i64 2
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint i32* [[INCDEC_PTR]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = trunc i64 [[TMP10]] to i32
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i32, <2 x i32*> [[TMP1]], <2 x i64> <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i32*> [[TMP4]], i32 0
+; CHECK-NEXT:    store i32 [[TMP11]], i32* [[TMP13]], align 4
+; CHECK-NEXT:    br label [[WHILE_BODY_BACKEDGE]]
+; CHECK:       while.body.backedge:
+; CHECK-NEXT:    [[C_022_BE]] = phi i32* [ [[INCDEC_PTR]], [[WHILE_BODY]] ], [ [[INCDEC_PTR8]], [[SW_BB6]] ], [ [[INCDEC_PTR5]], [[SW_BB]] ]
+; CHECK-NEXT:    [[TMP14]] = phi <2 x i32*> [ [[TMP4]], [[WHILE_BODY]] ], [ [[TMP12]], [[SW_BB6]] ], [ [[TMP8]], [[SW_BB]] ]
+; CHECK-NEXT:    br label [[WHILE_BODY]]
+; CHECK:       while.end:
+; CHECK-NEXT:    ret i32 undef
+;
+entry:
+  %0 = load i32, i32* @e, align 4
+  %tobool.not19 = icmp eq i32 %0, 0
+  br i1 %tobool.not19, label %while.end, label %while.body
+
+while.body:                                       ; preds = %entry, %while.body.backedge
+  %c.022 = phi i32* [ %c.022.be, %while.body.backedge ], [ undef, %entry ]
+  %b.021 = phi i32* [ %b.021.be, %while.body.backedge ], [ undef, %entry ]
+  %a.020 = phi i32* [ %a.020.be, %while.body.backedge ], [ undef, %entry ]
+  %incdec.ptr = getelementptr inbounds i32, i32* %c.022, i64 1
+  %1 = ptrtoint i32* %c.022 to i64
+  %2 = trunc i64 %1 to i32
+  %incdec.ptr1 = getelementptr inbounds i32, i32* %a.020, i64 1
+  %incdec.ptr2 = getelementptr inbounds i32, i32* %b.021, i64 1
+  switch i32 %2, label %while.body.backedge [
+  i32 2, label %sw.bb
+  i32 4, label %sw.bb6
+  ]
+
+sw.bb:                                            ; preds = %while.body
+  %incdec.ptr3 = getelementptr inbounds i32, i32* %b.021, i64 2
+  %3 = ptrtoint i32* %incdec.ptr2 to i64
+  %4 = trunc i64 %3 to i32
+  %incdec.ptr4 = getelementptr inbounds i32, i32* %a.020, i64 2
+  store i32 %4, i32* %incdec.ptr1, align 4
+  %incdec.ptr5 = getelementptr inbounds i32, i32* %c.022, i64 2
+  br label %while.body.backedge
+
+sw.bb6:                                           ; preds = %while.body
+  %incdec.ptr7 = getelementptr inbounds i32, i32* %a.020, i64 2
+  %incdec.ptr8 = getelementptr inbounds i32, i32* %c.022, i64 2
+  %5 = ptrtoint i32* %incdec.ptr to i64
+  %6 = trunc i64 %5 to i32
+  %incdec.ptr9 = getelementptr inbounds i32, i32* %b.021, i64 2
+  store i32 %6, i32* %incdec.ptr2, align 4
+  br label %while.body.backedge
+
+while.body.backedge:                              ; preds = %sw.bb6, %while.body, %sw.bb
+  %c.022.be = phi i32* [ %incdec.ptr, %while.body ], [ %incdec.ptr8, %sw.bb6 ], [ %incdec.ptr5, %sw.bb ]
+  %b.021.be = phi i32* [ %incdec.ptr2, %while.body ], [ %incdec.ptr9, %sw.bb6 ], [ %incdec.ptr3, %sw.bb ]
+  %a.020.be = phi i32* [ %incdec.ptr1, %while.body ], [ %incdec.ptr7, %sw.bb6 ], [ %incdec.ptr4, %sw.bb ]
+  br label %while.body
+
+while.end:                                        ; preds = %entry
+  ret i32 undef
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr44067.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr44067.ll
new file mode 100644
index 0000000000000..55bee662a613e
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr44067.ll
@@ -0,0 +1,120 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+; See https://reviews.llvm.org/D83779
+
+define <2 x float> @foo({{float, float}}* %A) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast { { float, float } }* [[A:%.*]] to <2 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x float>, <2 x float>* [[TMP0]], align 8
+; CHECK-NEXT:    [[REORDER_SHUFFLE:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul <2 x float> [[REORDER_SHUFFLE]], <float 2.000000e+00, float 2.000000e+00>
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 0
+; CHECK-NEXT:    [[INS1:%.*]] = insertelement <2 x float> undef, float [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x float> [[TMP2]], i32 1
+; CHECK-NEXT:    [[INS0:%.*]] = insertelement <2 x float> [[INS1]], float [[TMP4]], i32 0
+; CHECK-NEXT:    ret <2 x float> [[INS0]]
+;
+entry:
+  %0 = bitcast {{float, float}}* %A to <2 x float>*
+  %1 = load <2 x float>, <2 x float>* %0
+  %L0 = extractelement <2 x float> %1, i32 0
+  %L1 = extractelement <2 x float> %1, i32 1
+  %Mul0 = fmul float %L0, 2.000000e+00
+  %Mul1 = fmul float %L1, 2.000000e+00
+  %Ins1 = insertelement <2 x float> undef, float %Mul1, i32 1
+  %Ins0 = insertelement <2 x float> %Ins1, float %Mul0, i32 0
+  ret <2 x float> %Ins0
+}
+
+
+%Struct1Ty = type { i16, i16 }
+%Struct2Ty = type { %Struct1Ty, %Struct1Ty}
+
+define {%Struct2Ty, %Struct2Ty} @StructOfStructOfStruct(i16 *%Ptr) {
+; CHECK-LABEL: @StructOfStructOfStruct(
+; CHECK-NEXT:    [[GEP0:%.*]] = getelementptr inbounds i16, i16* [[PTR:%.*]], i64 0
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 1
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 2
+; CHECK-NEXT:    [[GEP3:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 3
+; CHECK-NEXT:    [[GEP4:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 4
+; CHECK-NEXT:    [[GEP5:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 5
+; CHECK-NEXT:    [[GEP6:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 6
+; CHECK-NEXT:    [[GEP7:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 7
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16* [[GEP0]] to <8 x i16>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* [[TMP1]], align 2
+; CHECK-NEXT:    [[REORDER_SHUFFLE:%.*]] = shufflevector <8 x i16> [[TMP2]], <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 7, i32 6, i32 1, i32 0, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = add <8 x i16> [[REORDER_SHUFFLE]], <i16 5, i16 6, i16 8, i16 7, i16 2, i16 1, i16 3, i16 4>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i16> [[TMP3]], i32 4
+; CHECK-NEXT:    [[STRUCTIN0:%.*]] = insertvalue [[STRUCT1TY:%.*]] undef, i16 [[TMP4]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i16> [[TMP3]], i32 5
+; CHECK-NEXT:    [[STRUCTIN1:%.*]] = insertvalue [[STRUCT1TY]] %StructIn0, i16 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i16> [[TMP3]], i32 6
+; CHECK-NEXT:    [[STRUCTIN2:%.*]] = insertvalue [[STRUCT1TY]] undef, i16 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i16> [[TMP3]], i32 7
+; CHECK-NEXT:    [[STRUCTIN3:%.*]] = insertvalue [[STRUCT1TY]] %StructIn2, i16 [[TMP7]], 1
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i16> [[TMP3]], i32 0
+; CHECK-NEXT:    [[STRUCTIN4:%.*]] = insertvalue [[STRUCT1TY]] undef, i16 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x i16> [[TMP3]], i32 1
+; CHECK-NEXT:    [[STRUCTIN5:%.*]] = insertvalue [[STRUCT1TY]] %StructIn4, i16 [[TMP9]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i16> [[TMP3]], i32 2
+; CHECK-NEXT:    [[STRUCTIN6:%.*]] = insertvalue [[STRUCT1TY]] undef, i16 [[TMP10]], 1
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i16> [[TMP3]], i32 3
+; CHECK-NEXT:    [[STRUCTIN7:%.*]] = insertvalue [[STRUCT1TY]] %StructIn6, i16 [[TMP11]], 0
+; CHECK-NEXT:    [[STRUCT2IN0:%.*]] = insertvalue [[STRUCT2TY:%.*]] undef, [[STRUCT1TY]] %StructIn1, 0
+; CHECK-NEXT:    [[STRUCT2IN1:%.*]] = insertvalue [[STRUCT2TY]] %Struct2In0, [[STRUCT1TY]] %StructIn3, 1
+; CHECK-NEXT:    [[STRUCT2IN2:%.*]] = insertvalue [[STRUCT2TY]] undef, [[STRUCT1TY]] %StructIn5, 0
+; CHECK-NEXT:    [[STRUCT2IN3:%.*]] = insertvalue [[STRUCT2TY]] %Struct2In2, [[STRUCT1TY]] %StructIn7, 1
+; CHECK-NEXT:    [[RET0:%.*]] = insertvalue { [[STRUCT2TY]], [[STRUCT2TY]] } undef, [[STRUCT2TY]] %Struct2In3, 1
+; CHECK-NEXT:    [[RET1:%.*]] = insertvalue { [[STRUCT2TY]], [[STRUCT2TY]] } [[RET0]], [[STRUCT2TY]] %Struct2In1, 0
+; CHECK-NEXT:    ret { [[STRUCT2TY]], [[STRUCT2TY]] } [[RET1]]
+;
+  %GEP0 = getelementptr inbounds i16, i16* %Ptr, i64 0
+  %L0 = load i16, i16 * %GEP0
+  %GEP1 = getelementptr inbounds i16, i16* %Ptr, i64 1
+  %L1 = load i16, i16 * %GEP1
+  %GEP2 = getelementptr inbounds i16, i16* %Ptr, i64 2
+  %L2 = load i16, i16 * %GEP2
+  %GEP3 = getelementptr inbounds i16, i16* %Ptr, i64 3
+  %L3 = load i16, i16 * %GEP3
+  %GEP4 = getelementptr inbounds i16, i16* %Ptr, i64 4
+  %L4 = load i16, i16 * %GEP4
+  %GEP5 = getelementptr inbounds i16, i16* %Ptr, i64 5
+  %L5 = load i16, i16 * %GEP5
+  %GEP6 = getelementptr inbounds i16, i16* %Ptr, i64 6
+  %L6 = load i16, i16 * %GEP6
+  %GEP7 = getelementptr inbounds i16, i16* %Ptr, i64 7
+  %L7 = load i16, i16 * %GEP7
+
+  %Fadd0 = add i16 %L0, 1
+  %Fadd1 = add i16 %L1, 2
+  %Fadd2 = add i16 %L2, 3
+  %Fadd3 = add i16 %L3, 4
+  %Fadd4 = add i16 %L4, 5
+  %Fadd5 = add i16 %L5, 6
+  %Fadd6 = add i16 %L6, 7
+  %Fadd7 = add i16 %L7, 8
+
+  %StructIn0 = insertvalue %Struct1Ty undef, i16 %Fadd1, 1
+  %StructIn1 = insertvalue %Struct1Ty %StructIn0, i16 %Fadd0, 0
+
+  %StructIn2 = insertvalue %Struct1Ty undef, i16 %Fadd2, 0
+  %StructIn3 = insertvalue %Struct1Ty %StructIn2, i16 %Fadd3, 1
+
+  %StructIn4 = insertvalue %Struct1Ty undef, i16 %Fadd4, 0
+  %StructIn5 = insertvalue %Struct1Ty %StructIn4, i16 %Fadd5, 1
+
+  %StructIn6 = insertvalue %Struct1Ty undef, i16 %Fadd7, 1
+  %StructIn7 = insertvalue %Struct1Ty %StructIn6, i16 %Fadd6, 0
+
+  %Struct2In0 = insertvalue %Struct2Ty undef, %Struct1Ty %StructIn1, 0
+  %Struct2In1 = insertvalue %Struct2Ty %Struct2In0, %Struct1Ty %StructIn3, 1
+
+  %Struct2In2 = insertvalue %Struct2Ty undef, %Struct1Ty %StructIn5, 0
+  %Struct2In3 = insertvalue %Struct2Ty %Struct2In2, %Struct1Ty %StructIn7, 1
+
+  %Ret0 = insertvalue {%Struct2Ty, %Struct2Ty} undef, %Struct2Ty %Struct2In3, 1
+  %Ret1 = insertvalue {%Struct2Ty, %Struct2Ty} %Ret0, %Struct2Ty %Struct2In1, 0
+  ret {%Struct2Ty, %Struct2Ty} %Ret1
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/2008-12-16-DCECond.ll b/llvm/test/Transforms/SimplifyCFG/2008-12-16-DCECond.ll
index 4fc21d9422374..2e356df75ca28 100644
--- a/llvm/test/Transforms/SimplifyCFG/2008-12-16-DCECond.ll
+++ b/llvm/test/Transforms/SimplifyCFG/2008-12-16-DCECond.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -simplifycfg -S | not grep icmp
+; RUN: opt < %s -simplifycfg -S -hoist-common-insts=true | not grep icmp
 ; ModuleID = '/tmp/x.bc'
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i686-pc-linux-gnu"
diff --git a/llvm/test/Transforms/SimplifyCFG/AArch64/prefer-fma.ll b/llvm/test/Transforms/SimplifyCFG/AArch64/prefer-fma.ll
index 7144da2d4bcec..f9a571418b326 100644
--- a/llvm/test/Transforms/SimplifyCFG/AArch64/prefer-fma.ll
+++ b/llvm/test/Transforms/SimplifyCFG/AArch64/prefer-fma.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -mtriple=aarch64-linux-gnu -simplifycfg -enable-unsafe-fp-math -S >%t
+; RUN: opt < %s -mtriple=aarch64-linux-gnu -simplifycfg -hoist-common-insts=true -enable-unsafe-fp-math -S >%t
 ; RUN: FileCheck %s < %t
 ; ModuleID = 't.cc'
 
diff --git a/llvm/test/Transforms/SimplifyCFG/BrUnwind.ll b/llvm/test/Transforms/SimplifyCFG/BrUnwind.ll
index 14853642c09a7..6cd3ad6e4b02a 100644
--- a/llvm/test/Transforms/SimplifyCFG/BrUnwind.ll
+++ b/llvm/test/Transforms/SimplifyCFG/BrUnwind.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -simplifycfg -S | \
+; RUN: opt < %s -simplifycfg -hoist-common-insts=true -S | \
 ; RUN: not grep "br label"
 
 define void @test(i1 %C) {
diff --git a/llvm/test/Transforms/SimplifyCFG/HoistCode.ll b/llvm/test/Transforms/SimplifyCFG/HoistCode.ll
index 575cb4f19eb7b..051d66ee752f4 100644
--- a/llvm/test/Transforms/SimplifyCFG/HoistCode.ll
+++ b/llvm/test/Transforms/SimplifyCFG/HoistCode.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -simplifycfg -S | FileCheck %s
+; RUN: opt < %s -simplifycfg -hoist-common-insts=true -S | FileCheck %s
 
 define void @foo(i1 %C, i32* %P) {
 ; CHECK-LABEL: @foo(
diff --git a/llvm/test/Transforms/SimplifyCFG/PowerPC/prefer-fma.ll b/llvm/test/Transforms/SimplifyCFG/PowerPC/prefer-fma.ll
index 93fe8a2019079..7a6ad85dbd6b8 100644
--- a/llvm/test/Transforms/SimplifyCFG/PowerPC/prefer-fma.ll
+++ b/llvm/test/Transforms/SimplifyCFG/PowerPC/prefer-fma.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=powerpc64le-unknown-linux-gnu -simplifycfg -enable-unsafe-fp-math -S | \
+; RUN: opt < %s -mtriple=powerpc64le-unknown-linux-gnu -simplifycfg -hoist-common-insts=true -enable-unsafe-fp-math -S | \
 ; RUN: FileCheck %s
 
 ; This case is copied from test/Transforms/SimplifyCFG/AArch64/
diff --git a/llvm/test/Transforms/SimplifyCFG/PowerPC/prefer-load-i32.ll b/llvm/test/Transforms/SimplifyCFG/PowerPC/prefer-load-i32.ll
index 943fcba57c654..d85f532aa5c41 100644
--- a/llvm/test/Transforms/SimplifyCFG/PowerPC/prefer-load-i32.ll
+++ b/llvm/test/Transforms/SimplifyCFG/PowerPC/prefer-load-i32.ll
@@ -1,10 +1,9 @@
-; RUN: opt < %s -mtriple=powerpc64le-unknown-linux-gnu -simplifycfg -S | \
-; RUN: FileCheck %s
+; RUN: opt < %s -mtriple=powerpc64le-unknown-linux-gnu -simplifycfg -hoist-common-insts=true -S | FileCheck %s
 
 define float @foo(float* %src, float* %dest, i32 signext %count, i32 signext %cond) {
 ; CHECK-LABEL: @foo(
 ; CHECK-LABEL: entry:
-; CHECK:  %0 = load float, float* %arrayidx, align 4
+; CHECK-NOT:  load float
 entry:
   %cmp = icmp sgt i32 %cond, 10
   %idxprom = sext i32 %count to i64
@@ -12,14 +11,15 @@ entry:
   br i1 %cmp, label %if.then, label %if.else
 
 ; CHECK-LABEL: if.then:
-; CHECK-NOT:   load float
+; CHECK:  %0 = load float, float* %arrayidx, align 4
 if.then:                                          ; preds = %entry
   %0 = load float, float* %arrayidx, align 4
   %res = fmul float %0, 3.000000e+00
   br label %if.end
 
 ; CHECK-LABEL: if.else:
-; CHECK-NOT:   load float
+; CHECK:   %1 = load float, float* %arrayidx, align 4
+; CHECK:   store float %1, float* %arrayidx4, align 4
 if.else:                                          ; preds = %entry
   %1 = load float, float* %arrayidx, align 4
   %idxprom3 = sext i32 %count to i64
diff --git a/llvm/test/Transforms/SimplifyCFG/UncondBranchToReturn.ll b/llvm/test/Transforms/SimplifyCFG/UncondBranchToReturn.ll
index b6d54d32566b5..78bcd345daf98 100644
--- a/llvm/test/Transforms/SimplifyCFG/UncondBranchToReturn.ll
+++ b/llvm/test/Transforms/SimplifyCFG/UncondBranchToReturn.ll
@@ -2,7 +2,7 @@
 ; a PHI node and a return.  Make sure the simplify cfg can straighten out this
 ; important case.  This is basically the most trivial form of tail-duplication.
 
-; RUN: opt < %s -simplifycfg -S | \
+; RUN: opt < %s -simplifycfg -hoist-common-insts=true -S | \
 ; RUN:    not grep "br label"
 
 define i32 @test(i1 %B, i32 %A, i32 %B.upgrd.1) {
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/empty-cleanuppad.ll b/llvm/test/Transforms/SimplifyCFG/X86/empty-cleanuppad.ll
index f9b6e63883394..ce7621908ff01 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/empty-cleanuppad.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/empty-cleanuppad.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -simplifycfg -S | FileCheck %s
+; RUN: opt < %s -simplifycfg -S -hoist-common-insts=true | FileCheck %s
 
 ; ModuleID = 'cppeh-simplify.cpp'
 target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/CodeGen/X86/pr39187-g.ll b/llvm/test/Transforms/SimplifyCFG/X86/pr39187-g.ll
similarity index 98%
rename from llvm/test/CodeGen/X86/pr39187-g.ll
rename to llvm/test/Transforms/SimplifyCFG/X86/pr39187-g.ll
index 2c9dcc5dfff35..1013fab20d1ce 100644
--- a/llvm/test/CodeGen/X86/pr39187-g.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/pr39187-g.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -S -simplifycfg | FileCheck %s
+; RUN: opt < %s -S -simplifycfg -hoist-common-insts=true | FileCheck %s
 
 ; SimplifyCFG can hoist any common code in the 'then' and 'else' blocks to
 ; the 'if' basic block.
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/remove-debug.ll b/llvm/test/Transforms/SimplifyCFG/X86/remove-debug.ll
index 2da7caf157e38..7a69e05267e2f 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/remove-debug.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/remove-debug.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -simplifycfg -S | FileCheck %s
+; RUN: opt < %s -simplifycfg -S -hoist-common-insts=true | FileCheck %s
 
 ; TODO: Track the acutal DebugLoc of the hoisted instruction when no-line
 ; DebugLoc is supported (https://reviews.llvm.org/D24180)
diff --git a/llvm/test/Transforms/SimplifyCFG/common-code-hoisting.ll b/llvm/test/Transforms/SimplifyCFG/common-code-hoisting.ll
new file mode 100644
index 0000000000000..b58017ba7ef0b
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/common-code-hoisting.ll
@@ -0,0 +1,110 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -simplifycfg -hoist-common-insts=1 -S < %s                    | FileCheck %s --check-prefixes=HOIST
+; RUN: opt -simplifycfg -hoist-common-insts=0 -S < %s                    | FileCheck %s --check-prefixes=NOHOIST
+; RUN: opt -simplifycfg                       -S < %s                    | FileCheck %s --check-prefixes=HOIST,DEFAULT
+
+; This example is produced from a very basic C code:
+;
+;   void f0();
+;   void f1();
+;   void f2();
+;
+;   void loop(int width) {
+;       if(width < 1)
+;           return;
+;       for(int i = 0; i < width - 1; ++i) {
+;           f0();
+;           f1();
+;       }
+;       f0();
+;       f2();
+;   }
+
+; We have a choice here. We can either
+; * hoist the f0() call into loop header,
+;   * which potentially makes loop rotation unprofitable since loop header might
+;     have grown above certain threshold, and such unrotated loops will be
+;     ignored by LoopVectorizer, preventing vectorization
+;   * or loop rotation will succeed, resulting in some weird PHIs that will also
+;     harm vectorization
+; * or not hoist f0() call before performing loop rotation,
+;   at the cost of potential code bloat and/or potentially successfully rotating
+;   the loops, vectorizing them at the cost of compile time.
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+declare i1 @gen1()
+
+declare void @f0()
+declare void @f1()
+declare void @f2()
+
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture)
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture)
+
+define void @_Z4loopi(i1 %cmp) {
+; HOIST-LABEL: @_Z4loopi(
+; HOIST-NEXT:  entry:
+; HOIST-NEXT:    br i1 [[CMP:%.*]], label [[RETURN:%.*]], label [[FOR_COND:%.*]]
+; HOIST:       for.cond:
+; HOIST-NEXT:    [[CMP1:%.*]] = call i1 @gen1()
+; HOIST-NEXT:    call void @f0()
+; HOIST-NEXT:    br i1 [[CMP1]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+; HOIST:       for.body:
+; HOIST-NEXT:    call void @f1()
+; HOIST-NEXT:    br label [[FOR_COND]]
+; HOIST:       for.end:
+; HOIST-NEXT:    call void @f2()
+; HOIST-NEXT:    br label [[RETURN]]
+; HOIST:       return:
+; HOIST-NEXT:    ret void
+;
+; NOHOIST-LABEL: @_Z4loopi(
+; NOHOIST-NEXT:  entry:
+; NOHOIST-NEXT:    br i1 [[CMP:%.*]], label [[RETURN:%.*]], label [[FOR_COND:%.*]]
+; NOHOIST:       for.cond:
+; NOHOIST-NEXT:    [[CMP1:%.*]] = call i1 @gen1()
+; NOHOIST-NEXT:    br i1 [[CMP1]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+; NOHOIST:       for.body:
+; NOHOIST-NEXT:    call void @f0()
+; NOHOIST-NEXT:    call void @f1()
+; NOHOIST-NEXT:    br label [[FOR_COND]]
+; NOHOIST:       for.end:
+; NOHOIST-NEXT:    call void @f0()
+; NOHOIST-NEXT:    call void @f2()
+; NOHOIST-NEXT:    br label [[RETURN]]
+; NOHOIST:       return:
+; NOHOIST-NEXT:    ret void
+;
+entry:
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  br label %return
+
+if.end:
+  br label %for.cond
+
+for.cond:
+  %cmp1 = call i1 @gen1()
+  br i1 %cmp1, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+  br label %for.end
+
+for.body:
+  call void @f0()
+  call void @f1()
+  br label %for.inc
+
+for.inc:
+  br label %for.cond
+
+for.end:
+  call void @f0()
+  call void @f2()
+  br label %return
+
+return:
+  ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/convergent.ll b/llvm/test/Transforms/SimplifyCFG/convergent.ll
index be8e8482e94b3..1860f457da6cb 100644
--- a/llvm/test/Transforms/SimplifyCFG/convergent.ll
+++ b/llvm/test/Transforms/SimplifyCFG/convergent.ll
@@ -4,7 +4,6 @@
 
 declare void @foo() convergent
 
-; FIXME: We should not be inserting a PR Phi here.
 define i32 @test_01(i32 %a) {
 ; CHECK-LABEL: @test_01(
 ; CHECK-NEXT:  entry:
@@ -14,9 +13,8 @@ define i32 @test_01(i32 %a) {
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br label [[MERGE]]
 ; CHECK:       merge:
-; CHECK-NEXT:    [[COND_PR:%.*]] = phi i1 [ [[COND]], [[IF_FALSE]] ], [ true, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    call void @foo()
-; CHECK-NEXT:    br i1 [[COND_PR]], label [[EXIT:%.*]], label [[IF_FALSE_2:%.*]]
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT:%.*]], label [[IF_FALSE_2:%.*]]
 ; CHECK:       if.false.2:
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br label [[EXIT]]
diff --git a/llvm/test/Transforms/SimplifyCFG/hoist-common-code.ll b/llvm/test/Transforms/SimplifyCFG/hoist-common-code.ll
index c1ca605cc206c..958cd35a172f9 100644
--- a/llvm/test/Transforms/SimplifyCFG/hoist-common-code.ll
+++ b/llvm/test/Transforms/SimplifyCFG/hoist-common-code.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -simplifycfg -S | not grep br
+; RUN: opt < %s -simplifycfg -S -hoist-common-insts=true | not grep br
 
 declare void @bar(i32)
 
diff --git a/llvm/test/Transforms/SimplifyCFG/hoist-dbgvalue-inlined.ll b/llvm/test/Transforms/SimplifyCFG/hoist-dbgvalue-inlined.ll
index 968c6fd01f4a9..c2fc73f3c7ea4 100644
--- a/llvm/test/Transforms/SimplifyCFG/hoist-dbgvalue-inlined.ll
+++ b/llvm/test/Transforms/SimplifyCFG/hoist-dbgvalue-inlined.ll
@@ -1,4 +1,4 @@
-; RUN: opt -simplifycfg -S < %s | FileCheck %s
+; RUN: opt -simplifycfg -hoist-common-insts=true -S < %s | FileCheck %s
 ; Verify that we don't crash due an invalid !dbg location on the hoisted llvm.dbg.value
 
 define i64 @caller(i64* %ptr, i64 %flag) !dbg !10 {
diff --git a/llvm/test/Transforms/SimplifyCFG/hoist-with-range.ll b/llvm/test/Transforms/SimplifyCFG/hoist-with-range.ll
index 6c12971167a57..82d4538bcbeb8 100644
--- a/llvm/test/Transforms/SimplifyCFG/hoist-with-range.ll
+++ b/llvm/test/Transforms/SimplifyCFG/hoist-with-range.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -simplifycfg -S | FileCheck %s
+; RUN: opt < %s -simplifycfg -hoist-common-insts=true -S | FileCheck %s
 
 define void @foo(i1 %c, i8* %p) {
 ; CHECK: if:
diff --git a/llvm/test/Transforms/SimplifyCFG/pr39807.ll b/llvm/test/Transforms/SimplifyCFG/pr39807.ll
index 8148f79503c84..1214c6fbeafba 100644
--- a/llvm/test/Transforms/SimplifyCFG/pr39807.ll
+++ b/llvm/test/Transforms/SimplifyCFG/pr39807.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -simplifycfg < %s | FileCheck %s
+; RUN: opt -S -simplifycfg -hoist-common-insts=true < %s | FileCheck %s
 
 declare void @personality()
 
diff --git a/llvm/test/Transforms/SimplifyCFG/preserve-load-metadata-2.ll b/llvm/test/Transforms/SimplifyCFG/preserve-load-metadata-2.ll
index 94d3565ce9851..bb3216e5c67f1 100644
--- a/llvm/test/Transforms/SimplifyCFG/preserve-load-metadata-2.ll
+++ b/llvm/test/Transforms/SimplifyCFG/preserve-load-metadata-2.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -simplifycfg -S | FileCheck %s
+; RUN: opt < %s -simplifycfg -hoist-common-insts=true -S | FileCheck %s
 
 declare void @bar(i32*)
 declare void @baz(i32*)
diff --git a/llvm/test/Transforms/SimplifyCFG/preserve-load-metadata-3.ll b/llvm/test/Transforms/SimplifyCFG/preserve-load-metadata-3.ll
index 92bdf6ec5c1a6..2a80e980c8f3f 100644
--- a/llvm/test/Transforms/SimplifyCFG/preserve-load-metadata-3.ll
+++ b/llvm/test/Transforms/SimplifyCFG/preserve-load-metadata-3.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -simplifycfg -S | FileCheck %s
+; RUN: opt < %s -simplifycfg -hoist-common-insts=true -S | FileCheck %s
 
 declare void @bar(i32*)
 declare void @baz(i32*)
diff --git a/llvm/test/Transforms/SimplifyCFG/preserve-load-metadata.ll b/llvm/test/Transforms/SimplifyCFG/preserve-load-metadata.ll
index 89815c843152b..3e87651e7e142 100644
--- a/llvm/test/Transforms/SimplifyCFG/preserve-load-metadata.ll
+++ b/llvm/test/Transforms/SimplifyCFG/preserve-load-metadata.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -simplifycfg -S | FileCheck %s
+; RUN: opt < %s -simplifycfg -hoist-common-insts=true -S | FileCheck %s
 
 declare void @bar(i32*)
 declare void @baz(i32*)
diff --git a/llvm/test/Transforms/VectorCombine/X86/load.ll b/llvm/test/Transforms/VectorCombine/X86/load.ll
new file mode 100644
index 0000000000000..d560f671cfc03
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/X86/load.ll
@@ -0,0 +1,129 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -vector-combine -S -mtriple=x86_64-- -mattr=sse2 | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -vector-combine -S -mtriple=x86_64-- -mattr=avx2 | FileCheck %s --check-prefixes=CHECK,AVX
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define float @matching_fp_scalar(float* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @matching_fp_scalar(
+; CHECK-NEXT:    [[R:%.*]] = load float, float* [[P:%.*]], align 16
+; CHECK-NEXT:    ret float [[R]]
+;
+  %r = load float, float* %p, align 16
+  ret float %r
+}
+
+define float @matching_fp_scalar_volatile(float* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @matching_fp_scalar_volatile(
+; CHECK-NEXT:    [[R:%.*]] = load volatile float, float* [[P:%.*]], align 16
+; CHECK-NEXT:    ret float [[R]]
+;
+  %r = load volatile float, float* %p, align 16
+  ret float %r
+}
+
+define double @larger_fp_scalar(float* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @larger_fp_scalar(
+; CHECK-NEXT:    [[BC:%.*]] = bitcast float* [[P:%.*]] to double*
+; CHECK-NEXT:    [[R:%.*]] = load double, double* [[BC]], align 16
+; CHECK-NEXT:    ret double [[R]]
+;
+  %bc = bitcast float* %p to double*
+  %r = load double, double* %bc, align 16
+  ret double %r
+}
+
+define float @smaller_fp_scalar(double* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @smaller_fp_scalar(
+; CHECK-NEXT:    [[BC:%.*]] = bitcast double* [[P:%.*]] to float*
+; CHECK-NEXT:    [[R:%.*]] = load float, float* [[BC]], align 16
+; CHECK-NEXT:    ret float [[R]]
+;
+  %bc = bitcast double* %p to float*
+  %r = load float, float* %bc, align 16
+  ret float %r
+}
+
+define float @matching_fp_vector(<4 x float>* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @matching_fp_vector(
+; CHECK-NEXT:    [[BC:%.*]] = bitcast <4 x float>* [[P:%.*]] to float*
+; CHECK-NEXT:    [[R:%.*]] = load float, float* [[BC]], align 16
+; CHECK-NEXT:    ret float [[R]]
+;
+  %bc = bitcast <4 x float>* %p to float*
+  %r = load float, float* %bc, align 16
+  ret float %r
+}
+
+define float @matching_fp_vector_gep0(<4 x float>* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @matching_fp_vector_gep0(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* [[P:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[R:%.*]] = load float, float* [[GEP]], align 16
+; CHECK-NEXT:    ret float [[R]]
+;
+  %gep = getelementptr inbounds <4 x float>, <4 x float>* %p, i64 0, i64 0
+  %r = load float, float* %gep, align 16
+  ret float %r
+}
+
+define float @nonmatching_int_vector(<2 x i64>* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @nonmatching_int_vector(
+; CHECK-NEXT:    [[BC:%.*]] = bitcast <2 x i64>* [[P:%.*]] to float*
+; CHECK-NEXT:    [[R:%.*]] = load float, float* [[BC]], align 16
+; CHECK-NEXT:    ret float [[R]]
+;
+  %bc = bitcast <2 x i64>* %p to float*
+  %r = load float, float* %bc, align 16
+  ret float %r
+}
+
+define double @less_aligned(double* align 4 dereferenceable(16) %p) {
+; CHECK-LABEL: @less_aligned(
+; CHECK-NEXT:    [[R:%.*]] = load double, double* [[P:%.*]], align 4
+; CHECK-NEXT:    ret double [[R]]
+;
+  %r = load double, double* %p, align 4
+  ret double %r
+}
+
+define float @matching_fp_scalar_small_deref(float* align 16 dereferenceable(15) %p) {
+; CHECK-LABEL: @matching_fp_scalar_small_deref(
+; CHECK-NEXT:    [[R:%.*]] = load float, float* [[P:%.*]], align 16
+; CHECK-NEXT:    ret float [[R]]
+;
+  %r = load float, float* %p, align 16
+  ret float %r
+}
+
+define i64 @larger_int_scalar(<4 x float>* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @larger_int_scalar(
+; CHECK-NEXT:    [[BC:%.*]] = bitcast <4 x float>* [[P:%.*]] to i64*
+; CHECK-NEXT:    [[R:%.*]] = load i64, i64* [[BC]], align 16
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %bc = bitcast <4 x float>* %p to i64*
+  %r = load i64, i64* %bc, align 16
+  ret i64 %r
+}
+
+define i8 @smaller_int_scalar(<4 x float>* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @smaller_int_scalar(
+; CHECK-NEXT:    [[BC:%.*]] = bitcast <4 x float>* [[P:%.*]] to i8*
+; CHECK-NEXT:    [[R:%.*]] = load i8, i8* [[BC]], align 16
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %bc = bitcast <4 x float>* %p to i8*
+  %r = load i8, i8* %bc, align 16
+  ret i8 %r
+}
+
+define double @larger_fp_scalar_256bit_vec(<8 x float>* align 32 dereferenceable(32) %p) {
+; CHECK-LABEL: @larger_fp_scalar_256bit_vec(
+; CHECK-NEXT:    [[BC:%.*]] = bitcast <8 x float>* [[P:%.*]] to double*
+; CHECK-NEXT:    [[R:%.*]] = load double, double* [[BC]], align 32
+; CHECK-NEXT:    ret double [[R]]
+;
+  %bc = bitcast <8 x float>* %p to double*
+  %r = load double, double* %bc, align 32
+  ret double %r
+}
diff --git a/llvm/test/Verifier/amdgpu-cc.ll b/llvm/test/Verifier/amdgpu-cc.ll
index 68c7f309b6e1f..1cd1b3467413f 100644
--- a/llvm/test/Verifier/amdgpu-cc.ll
+++ b/llvm/test/Verifier/amdgpu-cc.ll
@@ -1,5 +1,7 @@
 ; RUN: not llvm-as < %s 2>&1 | FileCheck %s
 
+target datalayout = "A5"
+
 ; CHECK: Calling convention requires void return type
 ; CHECK-NEXT: i32 ()* @nonvoid_cc_amdgpu_kernel
 define amdgpu_kernel i32 @nonvoid_cc_amdgpu_kernel() {
@@ -13,8 +15,14 @@ define amdgpu_kernel void @varargs_amdgpu_kernel(...) {
 }
 
 ; CHECK: Calling convention does not allow sret
-; CHECK-NEXT: void (i32*)* @sret_cc_amdgpu_kernel
-define amdgpu_kernel void @sret_cc_amdgpu_kernel(i32* sret %ptr) {
+; CHECK-NEXT: void (i32*)* @sret_cc_amdgpu_kernel_as0
+define amdgpu_kernel void @sret_cc_amdgpu_kernel_as0(i32* sret %ptr) {
+  ret void
+}
+
+; CHECK: Calling convention does not allow sret
+; CHECK-NEXT: void (i32 addrspace(5)*)* @sret_cc_amdgpu_kernel
+define amdgpu_kernel void @sret_cc_amdgpu_kernel(i32 addrspace(5)* sret %ptr) {
   ret void
 }
 
@@ -53,3 +61,69 @@ define spir_kernel i32 @nonvoid_cc_spir_kernel() {
 define spir_kernel void @varargs_spir_kernel(...) {
   ret void
 }
+
+; CHECK: Calling convention disallows byval
+; CHECK-NEXT: void (i32 addrspace(5)*)* @byval_cc_amdgpu_kernel
+define amdgpu_kernel void @byval_cc_amdgpu_kernel(i32 addrspace(5)* byval %ptr) {
+  ret void
+}
+
+; CHECK: Calling convention disallows byval
+; CHECK-NEXT: void (i32 addrspace(1)*)* @byval_as1_cc_amdgpu_kernel
+define amdgpu_kernel void @byval_as1_cc_amdgpu_kernel(i32 addrspace(1)* byval %ptr) {
+  ret void
+}
+
+; CHECK: Calling convention disallows byval
+; CHECK-NEXT: void (i32*)* @byval_as0_cc_amdgpu_kernel
+define amdgpu_kernel void @byval_as0_cc_amdgpu_kernel(i32* byval %ptr) {
+  ret void
+}
+
+; CHECK: Calling convention disallows byval
+; CHECK-NEXT: void (i32 addrspace(5)*)* @byval_cc_amdgpu_vs
+define amdgpu_vs void @byval_cc_amdgpu_vs(i32 addrspace(5)* byval %ptr) {
+  ret void
+}
+
+; CHECK: Calling convention disallows byval
+; CHECK-NEXT: void (i32 addrspace(5)*)* @byval_cc_amdgpu_hs
+define amdgpu_hs void @byval_cc_amdgpu_hs(i32 addrspace(5)* byval %ptr) {
+  ret void
+}
+
+; CHECK: Calling convention disallows byval
+; CHECK-NEXT: void (i32 addrspace(5)*)* @byval_cc_amdgpu_gs
+define amdgpu_gs void @byval_cc_amdgpu_gs(i32 addrspace(5)* byval %ptr) {
+  ret void
+}
+
+; CHECK: Calling convention disallows byval
+; CHECK-NEXT: void (i32 addrspace(5)*)* @byval_cc_amdgpu_ps
+define amdgpu_ps void @byval_cc_amdgpu_ps(i32 addrspace(5)* byval %ptr) {
+  ret void
+}
+
+; CHECK: Calling convention disallows byval
+; CHECK-NEXT: void (i32 addrspace(5)*)* @byval_cc_amdgpu_cs
+define amdgpu_cs void @byval_cc_amdgpu_cs(i32 addrspace(5)* byval %ptr) {
+  ret void
+}
+
+; CHECK: Calling convention disallows preallocated
+; CHECK-NEXT: void (i32*)* @preallocated_as0_cc_amdgpu_kernel
+define amdgpu_kernel void @preallocated_as0_cc_amdgpu_kernel(i32* preallocated(i32) %ptr) {
+  ret void
+}
+
+; CHECK: Calling convention disallows inalloca
+; CHECK-NEXT: void (i32*)* @inalloca_as0_cc_amdgpu_kernel
+define amdgpu_kernel void @inalloca_as0_cc_amdgpu_kernel(i32* inalloca %ptr) {
+  ret void
+}
+
+; CHECK: Calling convention disallows stack byref
+; CHECK-NEXT: void (i32 addrspace(5)*)* @byref_as5_cc_amdgpu_kernel
+define amdgpu_kernel void @byref_as5_cc_amdgpu_kernel(i32 addrspace(5)* byref(i32) %ptr) {
+  ret void
+}
diff --git a/llvm/test/Verifier/array_allocated.ll b/llvm/test/Verifier/array_allocated.ll
new file mode 100644
index 0000000000000..38749b84f97a1
--- /dev/null
+++ b/llvm/test/Verifier/array_allocated.ll
@@ -0,0 +1,6 @@
+; RUN: not llvm-as -disable-output <%s 2>&1 | FileCheck %s
+
+!named = !{!0}
+!0 = !DICompositeType(tag: DW_TAG_structure_type, name: "A", size: 64, allocated: !DIExpression(DW_OP_constu, 6789))
+
+; CHECK:  allocated can only appear in array type
diff --git a/llvm/test/Verifier/array_associated.ll b/llvm/test/Verifier/array_associated.ll
new file mode 100644
index 0000000000000..2f0354b05b1bd
--- /dev/null
+++ b/llvm/test/Verifier/array_associated.ll
@@ -0,0 +1,6 @@
+; RUN: not llvm-as -disable-output <%s 2>&1 | FileCheck %s
+
+!named = !{!0}
+!0 = !DICompositeType(tag: DW_TAG_structure_type, name: "A", size: 64, associated: !DIExpression(DW_OP_constu, 6789))
+
+; CHECK:  associated can only appear in array type
diff --git a/llvm/test/Verifier/byref.ll b/llvm/test/Verifier/byref.ll
new file mode 100644
index 0000000000000..5e7d5873a8db3
--- /dev/null
+++ b/llvm/test/Verifier/byref.ll
@@ -0,0 +1,100 @@
+; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
+
+; CHECK: Attribute 'byref' type does not match parameter!
+; CHECK-NEXT: void (i32*)* @byref_mismatched_pointee_type0
+define void @byref_mismatched_pointee_type0(i32* byref(i8)) {
+  ret void
+}
+
+; CHECK: Attribute 'byref' type does not match parameter!
+; CHECK-NEXT: void (i8*)* @byref_mismatched_pointee_type1
+define void @byref_mismatched_pointee_type1(i8* byref(i32)) {
+  ret void
+}
+
+%opaque.ty = type opaque
+
+; CHECK: Attributes 'byval', 'byref', 'inalloca', and 'preallocated' do not support unsized types!
+; CHECK-NEXT: void (%opaque.ty*)* @byref_unsized
+define void @byref_unsized(%opaque.ty* byref(%opaque.ty)) {
+  ret void
+}
+
+; CHECK: Attributes 'byval', 'inalloca', 'preallocated', 'inreg', 'nest', 'byref', and 'sret' are incompatible!
+; CHECK-NEXT: void (i32*)* @byref_byval
+define void @byref_byval(i32* byref(i32) byval(i32)) {
+  ret void
+}
+
+; CHECK: Attributes 'byval', 'inalloca', 'preallocated', 'inreg', 'nest', 'byref', and 'sret' are incompatible!
+; CHECK-NEXT: void (i32*)* @byref_inalloca
+define void @byref_inalloca(i32* byref(i32) inalloca) {
+  ret void
+}
+
+; CHECK: Attributes 'byval', 'inalloca', 'preallocated', 'inreg', 'nest', 'byref', and 'sret' are incompatible!
+; CHECK-NEXT: void (i32*)* @byref_preallocated
+define void @byref_preallocated(i32* byref(i32) preallocated(i32)) {
+  ret void
+}
+
+; CHECK: Attributes 'byval', 'inalloca', 'preallocated', 'inreg', 'nest', 'byref', and 'sret' are incompatible!
+; CHECK-NEXT: void (i32*)* @byref_sret
+define void @byref_sret(i32* byref(i32) sret) {
+  ret void
+}
+
+; CHECK: Attributes 'byval', 'inalloca', 'preallocated', 'inreg', 'nest', 'byref', and 'sret' are incompatible!
+; CHECK-NEXT: void (i32*)* @byref_inreg
+define void @byref_inreg(i32* byref(i32) inreg) {
+  ret void
+}
+
+; CHECK: Attributes 'byval', 'inalloca', 'preallocated', 'inreg', 'nest', 'byref', and 'sret' are incompatible!
+; CHECK-NEXT: void (i32*)* @byref_nest
+define void @byref_nest(i32* byref(i32) nest) {
+  ret void
+}
+
+; CHECK: Wrong types for attribute: inalloca nest noalias nocapture nonnull readnone readonly sret byref(i32) byval(i32) preallocated(i32) dereferenceable(1) dereferenceable_or_null(1)
+; CHECK-NEXT: void (i32)* @byref_non_pointer
+define void @byref_non_pointer(i32 byref(i32)) {
+  ret void
+}
+
+define void @byref_callee([64 x i8]* byref([64 x i8])) {
+  ret void
+}
+
+define void @no_byref_callee(i8*) {
+  ret void
+}
+
+; CHECK: cannot guarantee tail call due to mismatched ABI impacting function attributes
+; CHECK-NEXT: musttail call void @byref_callee([64 x i8]* byref([64 x i8]) %cast)
+; CHECK-NEXT: i8* %ptr
+define void @musttail_byref_caller(i8* %ptr) {
+  %cast = bitcast i8* %ptr to [64 x i8]*
+  musttail call void @byref_callee([64 x i8]* byref([64 x i8]) %cast)
+  ret void
+}
+
+; CHECK: cannot guarantee tail call due to mismatched ABI impacting function attributes
+; CHECK-NEXT: musttail call void @byref_callee([64 x i8]* %ptr)
+; CHECK-NEXT: [64 x i8]* %ptr
+define void @musttail_byref_callee([64 x i8]* byref([64 x i8]) %ptr) {
+  musttail call void @byref_callee([64 x i8]* %ptr)
+  ret void
+}
+
+define void @byref_callee_align32(i8* byref([64 x i8]) align 32) {
+  ret void
+}
+
+; CHECK: cannot guarantee tail call due to mismatched ABI impacting function attributes
+; CHECK-NEXT: musttail call void @byref_callee_align32(i8* byref([64 x i8]) align 32 %ptr)
+; CHECK-NEXT: i8* %ptr
+define void @musttail_byref_caller_mismatched_align(i8* byref([64 x i8]) align 16 %ptr) {
+  musttail call void @byref_callee_align32(i8* byref([64 x i8]) align 32 %ptr)
+  ret void
+}
diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py
index 0a3289fcc4ad4..49bd8ddfb2dc5 100644
--- a/llvm/test/lit.cfg.py
+++ b/llvm/test/lit.cfg.py
@@ -130,6 +130,7 @@ def get_asan_rtlib():
 config.llvm_locstats_used = os.path.exists(llvm_locstats_tool)
 
 tools = [
+    ToolSubst('%extract', FindTool('extract')),
     ToolSubst('%lli', FindTool('lli'), post='.', extra_args=lli_args),
     ToolSubst('%llc_dwarf', FindTool('llc'), extra_args=llc_args),
     ToolSubst('%go', config.go_executable, unresolved='ignore'),
diff --git a/llvm/test/lit.site.cfg.py.in b/llvm/test/lit.site.cfg.py.in
index c04ef04fea601..52f709f817ddd 100644
--- a/llvm/test/lit.site.cfg.py.in
+++ b/llvm/test/lit.site.cfg.py.in
@@ -33,7 +33,7 @@ config.host_cxx = "@HOST_CXX@"
 config.host_ldflags = '@HOST_LDFLAGS@'
 config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
-config.have_zlib = @HAVE_LIBZ@
+config.have_zlib = @LLVM_ENABLE_ZLIB@
 config.have_libxar = @HAVE_LIBXAR@
 config.have_dia_sdk = @LLVM_ENABLE_DIA_SDK@
 config.enable_ffi = @LLVM_ENABLE_FFI@
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/check_attrs.ll b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/check_attrs.ll
new file mode 100644
index 0000000000000..7f529d78d81db
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/check_attrs.ll
@@ -0,0 +1,13 @@
+; RUN: opt -attributor -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
+; RUN: opt -attributor-cgscc -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
+
+%struct.RT = type { i8, [10 x [20 x i32]], i8 }
+%struct.ST = type { i32, double, %struct.RT }
+
+define i32* @foo(%struct.ST* %s) nounwind uwtable readnone optsize ssp {
+entry:
+  %arrayidx = getelementptr inbounds %struct.ST, %struct.ST* %s, i64 1, i32 2, i32 1, i64 5, i64 13
+  ret i32* %arrayidx
+}
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/check_attrs.ll.funcattrs.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/check_attrs.ll.funcattrs.expected
new file mode 100644
index 0000000000000..be84e7824377c
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/check_attrs.ll.funcattrs.expected
@@ -0,0 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-attributes
+; RUN: opt -attributor -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
+; RUN: opt -attributor-cgscc -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
+
+%struct.RT = type { i8, [10 x [20 x i32]], i8 }
+%struct.ST = type { i32, double, %struct.RT }
+
+define i32* @foo(%struct.ST* %s) nounwind uwtable readnone optsize ssp {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind optsize readnone ssp uwtable willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@foo
+; IS__TUNIT____-SAME: (%struct.ST* nofree readnone [[S:%.*]]) #0
+; IS__TUNIT____-NEXT:  entry:
+; IS__TUNIT____-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [[STRUCT_ST:%.*]], %struct.ST* [[S]], i64 1, i32 2, i32 1, i64 5, i64 13
+; IS__TUNIT____-NEXT:    ret i32* [[ARRAYIDX]]
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind optsize readnone ssp uwtable willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@foo
+; IS__CGSCC____-SAME: (%struct.ST* nofree readnone [[S:%.*]]) #0
+; IS__CGSCC____-NEXT:  entry:
+; IS__CGSCC____-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [[STRUCT_ST:%.*]], %struct.ST* [[S]], i64 1, i32 2, i32 1, i64 5, i64 13
+; IS__CGSCC____-NEXT:    ret i32* [[ARRAYIDX]]
+;
+entry:
+  %arrayidx = getelementptr inbounds %struct.ST, %struct.ST* %s, i64 1, i32 2, i32 1, i64 5, i64 13
+  ret i32* %arrayidx
+}
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/check_attrs.ll.plain.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/check_attrs.ll.plain.expected
new file mode 100644
index 0000000000000..af95eefbce2d0
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/check_attrs.ll.plain.expected
@@ -0,0 +1,20 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature
+; RUN: opt -attributor -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
+; RUN: opt -attributor-cgscc -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
+
+%struct.RT = type { i8, [10 x [20 x i32]], i8 }
+%struct.ST = type { i32, double, %struct.RT }
+
+define i32* @foo(%struct.ST* %s) nounwind uwtable readnone optsize ssp {
+; CHECK-LABEL: define {{[^@]+}}@foo
+; CHECK-SAME: (%struct.ST* nofree readnone [[S:%.*]]) #0
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [[STRUCT_ST:%.*]], %struct.ST* [[S]], i64 1, i32 2, i32 1, i64 5, i64 13
+; CHECK-NEXT:    ret i32* [[ARRAYIDX]]
+;
+entry:
+  %arrayidx = getelementptr inbounds %struct.ST, %struct.ST* %s, i64 1, i32 2, i32 1, i64 5, i64 13
+  ret i32* %arrayidx
+}
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/check_attrs.test b/llvm/test/tools/UpdateTestChecks/update_test_checks/check_attrs.test
new file mode 100644
index 0000000000000..b1d4065cd5e70
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/check_attrs.test
@@ -0,0 +1,9 @@
+## check_attrs test checking that update_test_checks.py works correctly
+# RUN: cp -f %S/Inputs/check_attrs.ll %t.ll && %update_test_checks %t.ll --function-signature
+# RUN: diff -u %t.ll %S/Inputs/check_attrs.ll.plain.expected
+## Also try the --check-attributes flag
+# RUN: %update_test_checks %t.ll --check-attributes --function-signature
+# RUN: diff -u %t.ll %S/Inputs/check_attrs.ll.funcattrs.expected
+## Check that running the script again does not change the result:
+# RUN: %update_test_checks %t.ll
+# RUN: diff -u %t.ll %S/Inputs/check_attrs.ll.funcattrs.expected
diff --git a/llvm/test/tools/extract/Inputs/basic-aa.txt b/llvm/test/tools/extract/Inputs/basic-aa.txt
new file mode 100644
index 0000000000000..9eac3fdccbee4
--- /dev/null
+++ b/llvm/test/tools/extract/Inputs/basic-aa.txt
@@ -0,0 +1,6 @@
+
+
+
+aa
+; BB-NOT: {{.}}
+; BB: {{^}}bb{{$}}
diff --git a/llvm/test/tools/extract/Inputs/basic-bb.txt b/llvm/test/tools/extract/Inputs/basic-bb.txt
new file mode 100644
index 0000000000000..de17efab6fb6b
--- /dev/null
+++ b/llvm/test/tools/extract/Inputs/basic-bb.txt
@@ -0,0 +1,10 @@
+
+
+
+
+
+
+
+bb
+
+// CC: // Comments are preserved.
diff --git a/llvm/test/tools/extract/basic.test b/llvm/test/tools/extract/basic.test
new file mode 100644
index 0000000000000..9f9413106cc75
--- /dev/null
+++ b/llvm/test/tools/extract/basic.test
@@ -0,0 +1,32 @@
+# AA-NOT: {{.}}
+# AA: {{^}}aa{{$}}
+#--- aa
+aa
+; BB-NOT: {{.}}
+; BB: {{^}}bb{{$}}
+;--- bb
+bb
+
+// CC: // Comments are preserved.
+//--- cc
+cc
+// Comments are preserved.
+;--- dup
+;--- dup
+
+# RUN: extract aa %s | diff %S/Inputs/basic-aa.txt -
+# RUN: extract bb - < %s | diff %S/Inputs/basic-bb.txt -
+# RUN: extract cc %s -o %t
+# RUN: FileCheck %s --check-prefix=CC < %t
+
+# RUN: not %extract aa 2>&1 | FileCheck %s --check-prefix=NO_INPUT
+
+# NO_INPUT: extract: error: input filename is not specified
+
+# RUN: not %extract dup %s 2>&1 | FileCheck %s --check-prefix=DUP
+
+# DUP: extract: error: {{.*}}.test: ';--- dup' occurs more than once
+
+# RUN: not %extract not_exist %s 2>&1 | FileCheck %s --check-prefix=NOT_EXIST
+
+# NOT_EXIST: extract: error: {{.*}}.test: ';--- not_exist' was not found
diff --git a/llvm/test/tools/extract/help.test b/llvm/test/tools/extract/help.test
new file mode 100644
index 0000000000000..282052869116c
--- /dev/null
+++ b/llvm/test/tools/extract/help.test
@@ -0,0 +1,5 @@
+RUN: extract --help 2>&1 | FileCheck --implicit-check-not='General Options:' %s
+CHECK: OVERVIEW: Split input {{.*}}
+CHECK: Generic Options:
+CHECK: extract Options:
+CHECK:   -o
diff --git a/llvm/test/tools/extract/no-leading-lines.test b/llvm/test/tools/extract/no-leading-lines.test
new file mode 100644
index 0000000000000..f0efff5475afb
--- /dev/null
+++ b/llvm/test/tools/extract/no-leading-lines.test
@@ -0,0 +1,10 @@
+## With --no-leading-lines, don't add leading lines (which is used to preserve line numbers).
+
+# RUN: extract --no-leading-lines input %s -o %t
+# RUN: count 1 < %t
+# RUN: FileCheck %s < %t
+
+# CHECK: input
+
+#--- input
+input
diff --git a/llvm/test/tools/gold/X86/cache.ll b/llvm/test/tools/gold/X86/cache.ll
index 4446aa6d8878d..5ab5563025587 100644
--- a/llvm/test/tools/gold/X86/cache.ll
+++ b/llvm/test/tools/gold/X86/cache.ll
@@ -67,8 +67,7 @@
 ; With save-temps we can confirm that the cached files were copied into temp
 ; files to avoid a race condition with the cached files being pruned, since the
 ; gold plugin-api only accepts native objects passed back as files.
-; RUN: ls %t4.o.o1
-; RUN: ls %t4.o.o2
+; RUN: ls %t4.o.lto.o1 %t4.o.lto.o2
 
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/tools/gold/X86/emit-llvm.ll b/llvm/test/tools/gold/X86/emit-llvm.ll
index 8bab561889ae0..f1f6b22a9c3c9 100644
--- a/llvm/test/tools/gold/X86/emit-llvm.ll
+++ b/llvm/test/tools/gold/X86/emit-llvm.ll
@@ -12,7 +12,7 @@
 ; RUN: llvm-dis %t3.o.0.2.internalize.bc -o - | FileCheck %s
 ; RUN: llvm-dis %t3.o.0.4.opt.bc -o - | FileCheck --check-prefix=OPT %s
 ; RUN: llvm-dis %t3.o.0.4.opt.bc -o - | FileCheck --check-prefix=OPT2 %s
-; RUN: llvm-nm %t3.o.o | FileCheck --check-prefix=NM %s
+; RUN: llvm-nm %t3.o.lto.o | FileCheck --check-prefix=NM %s
 
 ; RUN: rm -f %t4.o
 ; RUN: %gold -plugin %llvmshlibdir/LLVMgold%shlibext \
diff --git a/llvm/test/tools/gold/X86/multiple-sections.ll b/llvm/test/tools/gold/X86/multiple-sections.ll
index facbd8d992ed7..31a89a9d3b484 100644
--- a/llvm/test/tools/gold/X86/multiple-sections.ll
+++ b/llvm/test/tools/gold/X86/multiple-sections.ll
@@ -1,10 +1,8 @@
-; RUN: echo ".text.tin" > %t_order_lto.txt
-; RUN: echo ".text._start" >> %t_order_lto.txt
-; RUN: echo ".text.pat" >> %t_order_lto.txt
-; RUN: llvm-as %s -o %t.o
+; RUN: extract order %s -o %t.order
+; RUN: extract ir %s | llvm-as -o %t.o
 ; RUN: %gold -plugin %llvmshlibdir/LLVMgold%shlibext \
 ; RUN:     -m elf_x86_64 -o %t.exe %t.o \
-; RUN:     --section-ordering-file=%t_order_lto.txt
+; RUN:     --section-ordering-file=%t.order
 ; RUN: llvm-readelf -s %t.exe | FileCheck %s
 
 ; Check that the order of the sections is tin -> _start -> pat.
@@ -13,6 +11,12 @@
 ; CHECK:      00000000004000b0     1 FUNC    LOCAL  DEFAULT    1 tin
 ; CHECK:      00000000004000c0    15 FUNC    GLOBAL DEFAULT    1 _start
 
+;--- order
+.text.tin
+.text._start
+.text.pat
+
+;--- ir
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/tools/gold/X86/parallel.ll b/llvm/test/tools/gold/X86/parallel.ll
index dabb5c4693193..6972efc652a34 100644
--- a/llvm/test/tools/gold/X86/parallel.ll
+++ b/llvm/test/tools/gold/X86/parallel.ll
@@ -1,10 +1,10 @@
 ; RUN: llvm-as -o %t.bc %s
-; RUN: rm -f %t.0.5.precodegen.bc %t.1.5.precodegen.bc %t.o %t.o1
+; RUN: rm -f %t.0.5.precodegen.bc %t.1.5.precodegen.bc %t.lto.o %t.lto.o1
 ; RUN: env LD_PRELOAD=%llvmshlibdir/LLVMgold%shlibext %gold -plugin %llvmshlibdir/LLVMgold%shlibext -u foo -u bar -plugin-opt lto-partitions=2 -plugin-opt save-temps -m elf_x86_64 -o %t %t.bc
 ; RUN: llvm-dis %t.0.5.precodegen.bc -o - | FileCheck --check-prefix=CHECK-BC0 %s
 ; RUN: llvm-dis %t.1.5.precodegen.bc -o - | FileCheck --check-prefix=CHECK-BC1 %s
-; RUN: llvm-nm %t.o | FileCheck --check-prefix=CHECK0 %s
-; RUN: llvm-nm %t.o1 | FileCheck --check-prefix=CHECK1 %s
+; RUN: llvm-nm %t.lto.o | FileCheck --check-prefix=CHECK0 %s
+; RUN: llvm-nm %t.lto.o1 | FileCheck --check-prefix=CHECK1 %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/tools/gold/X86/relax-relocs.ll b/llvm/test/tools/gold/X86/relax-relocs.ll
index f62125c48d1f3..3ad79f73cb84b 100644
--- a/llvm/test/tools/gold/X86/relax-relocs.ll
+++ b/llvm/test/tools/gold/X86/relax-relocs.ll
@@ -2,7 +2,7 @@
 ; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold%shlibext \
 ; RUN:    --plugin-opt=save-temps \
 ; RUN:    -shared %t.o -o %t.so
-; RUN: llvm-readobj -r %t.so.o | FileCheck %s
+; RUN: llvm-readobj -r %t.so.lto.o | FileCheck %s
 
 ; Test that we produce R_X86_64_GOTPCREL instead of R_X86_64_GOTPCRELX
 ; CHECK: R_X86_64_GOTPCREL foo
diff --git a/llvm/test/tools/gold/X86/relocation-model-pic.ll b/llvm/test/tools/gold/X86/relocation-model-pic.ll
index ad7d2981e9efd..98034e49b0a00 100644
--- a/llvm/test/tools/gold/X86/relocation-model-pic.ll
+++ b/llvm/test/tools/gold/X86/relocation-model-pic.ll
@@ -10,44 +10,44 @@
 ; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold%shlibext \
 ; RUN:    --shared \
 ; RUN:    --plugin-opt=save-temps %t.o -o %t-out
-; RUN: llvm-readobj -r %t-out.o | FileCheck %s --check-prefix=PIC
+; RUN: llvm-readobj -r %t-out.lto.o | FileCheck %s --check-prefix=PIC
 
 ; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold%shlibext \
 ; RUN:    --export-dynamic --noinhibit-exec -pie \
 ; RUN:    --plugin-opt=save-temps %t.o -o %t-out
-; RUN: llvm-readobj -r %t-out.o | FileCheck %s --check-prefix=PIC
+; RUN: llvm-readobj -r %t-out.lto.o | FileCheck %s --check-prefix=PIC
 
 ; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold%shlibext \
 ; RUN:    --export-dynamic --noinhibit-exec \
 ; RUN:    --plugin-opt=save-temps %t.o -o %t-out
-; RUN: llvm-readobj -r %t-out.o | FileCheck %s --check-prefix=STATIC
+; RUN: llvm-readobj -r %t-out.lto.o | FileCheck %s --check-prefix=STATIC
 
 ; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold%shlibext \
 ; RUN:    -r \
 ; RUN:    --plugin-opt=save-temps %t.o -o %t-out
-; RUN: llvm-readobj -r %t-out.o | FileCheck %s --check-prefix=STATIC
+; RUN: llvm-readobj -r %t-out.lto.o | FileCheck %s --check-prefix=STATIC
 
 ;; PIC source.
 
 ; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold%shlibext \
 ; RUN:    --shared \
 ; RUN:    --plugin-opt=save-temps %t.pic.o -o %t-out
-; RUN: llvm-readobj -r %t-out.o | FileCheck %s --check-prefix=PIC
+; RUN: llvm-readobj -r %t-out.lto.o | FileCheck %s --check-prefix=PIC
 
 ; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold%shlibext \
 ; RUN:    --export-dynamic --noinhibit-exec -pie \
 ; RUN:    --plugin-opt=save-temps %t.pic.o -o %t-out
-; RUN: llvm-readobj -r %t-out.o | FileCheck %s --check-prefix=PIC
+; RUN: llvm-readobj -r %t-out.lto.o | FileCheck %s --check-prefix=PIC
 
 ; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold%shlibext \
 ; RUN:    --export-dynamic --noinhibit-exec \
 ; RUN:    --plugin-opt=save-temps %t.pic.o -o %t-out
-; RUN: llvm-readobj -r %t-out.o | FileCheck %s --check-prefix=STATIC
+; RUN: llvm-readobj -r %t-out.lto.o | FileCheck %s --check-prefix=STATIC
 
 ; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold%shlibext \
 ; RUN:    -r \
 ; RUN:    --plugin-opt=save-temps %t.pic.o -o %t-out
-; RUN: llvm-readobj -r %t-out.o | FileCheck %s --check-prefix=PIC
+; RUN: llvm-readobj -r %t-out.lto.o | FileCheck %s --check-prefix=PIC
 
 
 ; PIC: R_X86_64_GOTPCREL foo
diff --git a/llvm/test/tools/gold/X86/thinlto.ll b/llvm/test/tools/gold/X86/thinlto.ll
index 6857778b55d5b..3ed07ffd5eed4 100644
--- a/llvm/test/tools/gold/X86/thinlto.ll
+++ b/llvm/test/tools/gold/X86/thinlto.ll
@@ -42,7 +42,7 @@
 ; RUN: llvm-bcanalyzer -dump %t4.index.bc | FileCheck %s --check-prefix=COMBINED
 ; RUN: llvm-nm %t4 | FileCheck %s --check-prefix=NM
 ; Ensure ld does not emit empty combined module in default.
-; RUN: ls %t4.o* | count 2
+; RUN: ls %t4.lto.o* | count 2
 
 ; Check with --no-map-whole-files
 ; RUN: %gold -plugin %llvmshlibdir/LLVMgold%shlibext \
diff --git a/llvm/test/tools/llvm-ifs/func.ifs b/llvm/test/tools/llvm-ifs/func.ifs
index d6d85782b2e37..6a5aede21e25c 100644
--- a/llvm/test/tools/llvm-ifs/func.ifs
+++ b/llvm/test/tools/llvm-ifs/func.ifs
@@ -2,7 +2,7 @@
 # RUN: FileCheck %s --check-prefixes=CHECK-IFS
 
 # RUN: llvm-ifs -action write-bin -o - %s %S/object.ifs | \
-# RUN: llvm-readelf --all | FileCheck %s --check-prefixes=CHECK-ELF
+# RUN: llvm-readelf --all - | FileCheck %s --check-prefixes=CHECK-ELF
 
 # RUN: llvm-ifs -action write-bin -force-format TBD -o - %s %S/object.ifs | \
 # RUN: FileCheck %s --check-prefixes=CHECK-DARWIN-TBD3
diff --git a/llvm/test/tools/llvm-ifs/object-function-size-weak-combo.ifs b/llvm/test/tools/llvm-ifs/object-function-size-weak-combo.ifs
index 769f423f328a0..1eb9f3c7bc1f1 100644
--- a/llvm/test/tools/llvm-ifs/object-function-size-weak-combo.ifs
+++ b/llvm/test/tools/llvm-ifs/object-function-size-weak-combo.ifs
@@ -2,7 +2,7 @@
 # RUN: FileCheck %s --check-prefixes=CHECK-IFS
 
 # RUN: llvm-ifs -action write-bin -o - %s %S/func.ifs %S/object.ifs %S/weak.ifs | \
-# RUN: llvm-readelf --all | FileCheck %s --check-prefixes=CHECK-ELF
+# RUN: llvm-readelf --all - | FileCheck %s --check-prefixes=CHECK-ELF
 
 # CHECK-IFS: --- !experimental-ifs-v2
 # CHECK-IFS-NEXT: IfsVersion: 2.0
diff --git a/llvm/test/tools/llvm-ifs/object.ifs b/llvm/test/tools/llvm-ifs/object.ifs
index c4823c20fce25..fb9685372cadf 100644
--- a/llvm/test/tools/llvm-ifs/object.ifs
+++ b/llvm/test/tools/llvm-ifs/object.ifs
@@ -2,7 +2,7 @@
 # RUN: FileCheck %s --check-prefixes=CHECK-IFS
 
 # RUN: llvm-ifs -action write-bin -o - %s | \
-# RUN: llvm-readelf --all | FileCheck %s --check-prefixes=CHECK-ELF
+# RUN: llvm-readelf --all - | FileCheck %s --check-prefixes=CHECK-ELF
 
 # CHECK-IFS: --- !experimental-ifs-v2
 # CHECK-IFS-NEXT: IfsVersion: 2.0
diff --git a/llvm/test/tools/llvm-libtool-darwin/archive-flattening.test b/llvm/test/tools/llvm-libtool-darwin/archive-flattening.test
new file mode 100644
index 0000000000000..fa3200ec00b70
--- /dev/null
+++ b/llvm/test/tools/llvm-libtool-darwin/archive-flattening.test
@@ -0,0 +1,95 @@
+## This test checks that an archive is flattened correctly.
+
+# RUN: yaml2obj %S/Inputs/input1.yaml -o %t-input1.o
+# RUN: yaml2obj %S/Inputs/input2.yaml -o %t-input2.o
+
+## Input a correct archive:
+# RUN: rm -f %t.correct.ar
+# RUN: llvm-ar cr %t.correct.ar %t-input1.o %t-input2.o
+# RUN: llvm-libtool-darwin -static -o %t.lib %t.correct.ar
+
+## Check that binaries are present:
+# RUN: llvm-ar t %t.lib | \
+# RUN:   FileCheck %s --check-prefix=CHECK-NAMES --implicit-check-not={{.}} -DPREFIX=%basename_t.tmp
+
+# CHECK-NAMES:      [[PREFIX]]-input1.o
+# CHECK-NAMES-NEXT: [[PREFIX]]-input2.o
+
+## Check that symbols are present:
+# RUN: llvm-nm --print-armap %t.lib | \
+# RUN:   FileCheck %s --check-prefix=CHECK-SYMBOLS -DPREFIX=%basename_t.tmp --match-full-lines
+
+# CHECK-SYMBOLS:      Archive map
+# CHECK-SYMBOLS-NEXT: _symbol1 in [[PREFIX]]-input1.o
+# CHECK-SYMBOLS-NEXT: _symbol2 in [[PREFIX]]-input2.o
+# CHECK-SYMBOLS-EMPTY:
+
+## Check that output archive is in Darwin format:
+# RUN: llvm-objdump --macho --archive-headers %t.lib | \
+# RUN:   FileCheck %s --check-prefix=FORMAT -DPREFIX=%basename_t.tmp -DARCHIVE=%t.lib
+
+# FORMAT:      Archive : [[ARCHIVE]]
+# FORMAT-NEXT: __.SYMDEF
+# FORMAT-NEXT: [[PREFIX]]-input1.o
+# FORMAT-NEXT: [[PREFIX]]-input2.o
+# FORMAT-NOT:  {{.}}
+
+## Passing both archive and object file:
+# RUN: llvm-libtool-darwin -static -o %t.lib %t-input2.o %t.correct.ar %t-input1.o
+# RUN: llvm-ar t %t.lib | \
+# RUN:   FileCheck %s --check-prefix=BOTH-NAMES --implicit-check-not={{.}} -DPREFIX=%basename_t.tmp
+# RUN: llvm-nm --print-armap %t.lib | \
+# RUN:   FileCheck %s --check-prefix=BOTH-SYMBOLS -DPREFIX=%basename_t.tmp --match-full-lines
+
+# BOTH-NAMES:      [[PREFIX]]-input2.o
+# BOTH-NAMES-NEXT: [[PREFIX]]-input1.o
+# BOTH-NAMES-NEXT: [[PREFIX]]-input2.o
+# BOTH-NAMES-NEXT: [[PREFIX]]-input1.o
+
+# BOTH-SYMBOLS:      Archive map
+# BOTH-SYMBOLS-NEXT: _symbol2 in [[PREFIX]]-input2.o
+# BOTH-SYMBOLS-NEXT: _symbol1 in [[PREFIX]]-input1.o
+# BOTH-SYMBOLS-NEXT: _symbol2 in [[PREFIX]]-input2.o
+# BOTH-SYMBOLS-NEXT: _symbol1 in [[PREFIX]]-input1.o
+# BOTH-SYMBOLS-EMPTY:
+
+## Cannot read archive:
+# RUN: echo '!<arch>' >  %t-invalid-archive.lib
+# RUN: echo 'invalid' >> %t-invalid-archive.lib
+# RUN: not llvm-libtool-darwin -static -o %t.lib %t-invalid-archive.lib 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=INVALID-ARCHIVE -DARCHIVE=%t-invalid-archive.lib
+
+# INVALID-ARCHIVE: error: '[[ARCHIVE]]': truncated or malformed archive
+
+## Archive member not an object file:
+# RUN: rm -f %t.not-object.ar
+# RUN: touch %t.txt
+# RUN: llvm-ar cr %t.not-object.ar %t.txt
+# RUN: not llvm-libtool-darwin -static -o %t.lib %t.not-object.ar 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=NOT-OBJECT -DARCHIVE=%t.not-object.ar -DFILE=%basename_t.tmp.txt
+
+## Do not recursively flatten archives:
+# RUN: rm -f %t.inner
+# RUN: rm -f %t.outer
+# RUN: llvm-ar cr %t.inner %t-input1.o
+# RUN: llvm-ar cr %t.outer %t.inner
+# RUN: not llvm-libtool-darwin -static -o %t.lib %t.outer 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=NOT-OBJECT -DARCHIVE=%t.outer -DFILE=%basename_t.tmp.inner
+
+# NOT-OBJECT: error: '[[ARCHIVE]]': '[[FILE]]': The file was not recognized as a valid object file
+
+## Archive member not a Mach-O object file:
+# RUN: rm -f %t.not-macho.ar
+# RUN: yaml2obj %s -o %t.elf
+# RUN: llvm-ar cr %t.not-macho.ar %t.elf
+# RUN: not llvm-libtool-darwin -static -o %t.lib %t.not-macho.ar 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=NOT-MACHO -DARCHIVE=%t.not-macho.ar -DFILE=%basename_t.tmp.elf
+
+# NOT-MACHO: error: '[[ARCHIVE]]': '[[FILE]]': format not supported
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_X86_64
diff --git a/llvm/test/tools/llvm-libtool-darwin/basic.test b/llvm/test/tools/llvm-libtool-darwin/basic.test
deleted file mode 100644
index 5cb6fea899bfa..0000000000000
--- a/llvm/test/tools/llvm-libtool-darwin/basic.test
+++ /dev/null
@@ -1,10 +0,0 @@
-## This test checks that main exits normally (error code 0) for correct input/output args.
-
-# RUN: yaml2obj %S/Inputs/input1.yaml -o %t-input1.o
-# RUN: yaml2obj %S/Inputs/input2.yaml -o %t-input2.o
-
-## Pass single input:
-# RUN: llvm-libtool-darwin -o %t.lib %t-input1.o
-
-## Pass multiple inputs:
-# RUN: llvm-libtool-darwin -o %t.lib %t-input1.o %t-input2.o
diff --git a/llvm/test/tools/llvm-libtool-darwin/create-static-lib.test b/llvm/test/tools/llvm-libtool-darwin/create-static-lib.test
new file mode 100644
index 0000000000000..6c96782b57d21
--- /dev/null
+++ b/llvm/test/tools/llvm-libtool-darwin/create-static-lib.test
@@ -0,0 +1,66 @@
+## This test checks that a correct static library is created.
+
+# RUN: yaml2obj %S/Inputs/input1.yaml -o %t-input1.o
+# RUN: yaml2obj %S/Inputs/input2.yaml -o %t-input2.o
+
+# RUN: rm -rf %t.lib
+# RUN: llvm-libtool-darwin -static -o %t.lib %t-input1.o %t-input2.o
+
+## Check that binaries are present:
+# RUN: llvm-ar t %t.lib | \
+# RUN:   FileCheck %s --check-prefix=CHECK-NAMES --implicit-check-not={{.}} -DPREFIX=%basename_t.tmp
+
+# CHECK-NAMES:      [[PREFIX]]-input1.o
+# CHECK-NAMES-NEXT: [[PREFIX]]-input2.o
+
+## Check that symbols are present:
+# RUN: llvm-nm --print-armap %t.lib | \
+# RUN:   FileCheck %s --check-prefix=CHECK-SYMBOLS -DPREFIX=%basename_t.tmp --match-full-lines
+
+# CHECK-SYMBOLS:      Archive map
+# CHECK-SYMBOLS-NEXT: _symbol1 in [[PREFIX]]-input1.o
+# CHECK-SYMBOLS-NEXT: _symbol2 in [[PREFIX]]-input2.o
+# CHECK-SYMBOLS-EMPTY:
+
+## Check that output archive is in Darwin format:
+# RUN: llvm-objdump --macho --archive-headers %t.lib | \
+# RUN:   FileCheck %s --check-prefix=FORMAT -DPREFIX=%basename_t.tmp -DARCHIVE=%t
+
+# FORMAT:      Archive : [[ARCHIVE]]
+# FORMAT-NEXT: __.SYMDEF
+# FORMAT-NEXT: [[PREFIX]]-input1.o
+# FORMAT-NEXT: [[PREFIX]]-input2.o
+# FORMAT-NOT:  {{.}}
+
+## Check that the output file is overwritten:
+# RUN: llvm-libtool-darwin -static -o %t.lib %t-input2.o
+# RUN: llvm-ar t %t.lib | \
+# RUN:   FileCheck %s --check-prefix=OVERWRITE-NAMES --implicit-check-not={{.}} -DPREFIX=%basename_t.tmp
+# RUN: llvm-nm --print-armap %t.lib | \
+# RUN:   FileCheck %s --check-prefix=OVERWRITE-SYMBOLS -DPREFIX=%basename_t.tmp --match-full-lines
+
+# OVERWRITE-NAMES: [[PREFIX]]-input2.o
+
+# OVERWRITE-SYMBOLS:      Archive map
+# OVERWRITE-SYMBOLS-NEXT: _symbol2 in [[PREFIX]]-input2.o
+# OVERWRITE-SYMBOLS-EMPTY:
+
+## Duplicate a binary:
+## cctools' libtool raises a warning in this case.
+## The warning is not yet implemented for llvm-libtool-darwin.
+# RUN: llvm-libtool-darwin -static -o %t.lib %t-input1.o %t-input2.o %t-input1.o 2>&1 | \
+# RUN:   FileCheck %s --allow-empty --implicit-check-not={{.}}
+# RUN: llvm-ar t %t.lib | \
+# RUN:   FileCheck %s --check-prefix=DUPLICATE-NAMES --implicit-check-not={{.}} -DPREFIX=%basename_t.tmp
+# RUN: llvm-nm --print-armap %t.lib | \
+# RUN:   FileCheck %s --check-prefix=DUPLICATE-SYMBOLS -DPREFIX=%basename_t.tmp --match-full-lines
+
+# DUPLICATE-NAMES:      [[PREFIX]]-input1.o
+# DUPLICATE-NAMES-NEXT: [[PREFIX]]-input2.o
+# DUPLICATE-NAMES-NEXT: [[PREFIX]]-input1.o
+
+# DUPLICATE-SYMBOLS:      Archive map
+# DUPLICATE-SYMBOLS-NEXT: _symbol1 in [[PREFIX]]-input1.o
+# DUPLICATE-SYMBOLS-NEXT: _symbol2 in [[PREFIX]]-input2.o
+# DUPLICATE-SYMBOLS-NEXT: _symbol1 in [[PREFIX]]-input1.o
+# DUPLICATE-SYMBOLS-EMPTY:
diff --git a/llvm/test/tools/llvm-libtool-darwin/help-message.test b/llvm/test/tools/llvm-libtool-darwin/help-message.test
index 4face3887ed2b..d8b8c79ba8ec2 100644
--- a/llvm/test/tools/llvm-libtool-darwin/help-message.test
+++ b/llvm/test/tools/llvm-libtool-darwin/help-message.test
@@ -1,10 +1,13 @@
 ## This test checks that the help message is displayed correctly.
 
-# RUN: llvm-libtool-darwin -h | FileCheck --check-prefixes=LIBTOOL-USAGE,CATEG %s --match-full-lines
-# RUN: llvm-libtool-darwin -help | FileCheck --check-prefixes=LIBTOOL-USAGE,CATEG %s --match-full-lines
-# RUN: llvm-libtool-darwin --help | FileCheck --check-prefixes=LIBTOOL-USAGE,CATEG %s --match-full-lines
+# RUN: llvm-libtool-darwin -h | \
+# RUN:   FileCheck --check-prefixes=LIBTOOL-USAGE,CATEG %s --match-full-lines --implicit-check-not="General options:"
+# RUN: llvm-libtool-darwin -help | \
+# RUN:   FileCheck --check-prefixes=LIBTOOL-USAGE,CATEG %s --match-full-lines --implicit-check-not="General options:"
+# RUN: llvm-libtool-darwin --help | \
+# RUN:   FileCheck --check-prefixes=LIBTOOL-USAGE,CATEG %s --match-full-lines --implicit-check-not="General options:"
 # RUN: llvm-libtool-darwin --help-list | \
-# RUN:   FileCheck -check-prefixes=LIBTOOL-USAGE,LIST %s --match-full-lines
+# RUN:   FileCheck -check-prefixes=LIBTOOL-USAGE,LIST %s --match-full-lines --implicit-check-not="safepoint-ir-verifier-print-only"
 
 # RUN: not llvm-libtool-darwin -abcabc 2>&1 | FileCheck --check-prefix=UNKNOWN-ARG %s
 # RUN: not llvm-libtool-darwin --abcabc 2>&1 | FileCheck --check-prefix=UNKNOWN-ARG %s
@@ -19,5 +22,6 @@
 # LIST-NOT: Generic Options:
 # CATEG:    llvm-libtool-darwin Options:
 # LIST-NOT: llvm-libtool-darwin Options:
+# LIST-NOT: General options:
 
 # UNKNOWN-ARG: Unknown command line argument '{{-+}}abcabc'
diff --git a/llvm/test/tools/llvm-libtool-darwin/invalid-input-output-args.test b/llvm/test/tools/llvm-libtool-darwin/invalid-input-output-args.test
index 5b4d2d988ae4c..66ee65b262442 100644
--- a/llvm/test/tools/llvm-libtool-darwin/invalid-input-output-args.test
+++ b/llvm/test/tools/llvm-libtool-darwin/invalid-input-output-args.test
@@ -1,25 +1,52 @@
 ## This test checks that an error is thrown in case of invalid input/output args.
 
 ## Missing input file:
-# RUN: not llvm-libtool-darwin -o %t.lib 2>&1 | \
+# RUN: not llvm-libtool-darwin -static -o %t.lib 2>&1 | \
 # RUN:   FileCheck %s --check-prefix=NO-INPUT
 
 # NO-INPUT: Must specify at least 1 positional argument
 
 ## Missing output file:
-# RUN: not llvm-libtool-darwin %t.input 2>&1 | \
+# RUN: not llvm-libtool-darwin -static %t.input 2>&1 | \
 # RUN:   FileCheck %s --check-prefix=NO-OUTPUT
 
 # NO-OUTPUT: for the -o option: must be specified at least once!
 
 ## Missing argument to -o:
-# RUN: not llvm-libtool-darwin %t.input -o 2>&1 | \
+# RUN: not llvm-libtool-darwin -static %t.input -o 2>&1 | \
 # RUN:   FileCheck %s --check-prefix=MISSING
 
 # MISSING: for the -o option: requires a value!
 
 ## Passing in two output files:
-# RUN: not llvm-libtool-darwin %t.input -o %t.lib1 -o %t.lib2 2>&1 | \
+# RUN: not llvm-libtool-darwin -static %t.input -o %t.lib1 -o %t.lib2 2>&1 | \
 # RUN:   FileCheck %s --check-prefix=DOUBLE-OUTPUT
 
 # DOUBLE-OUTPUT: for the -o option: must occur exactly one time!
+
+## Input file not found:
+# RUN: not llvm-libtool-darwin -static -o %t.lib %t.missing 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=NO-FILE -DFILE=%t.missing
+
+# NO-FILE: error: '[[FILE]]': {{[nN]}}o such file or directory
+
+## Input file is not an object file:
+# RUN: touch %t.invalid
+# RUN: not llvm-libtool-darwin -static -o %t.lib %t.invalid 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=NOT-OBJECT -DFILE=%basename_t.tmp.invalid
+
+# NOT-OBJECT: error: '[[FILE]]': The file was not recognized as a valid object file
+
+## Input file is not a Mach-O object file:
+# RUN: yaml2obj %s -o %t.elf
+# RUN: not llvm-libtool-darwin -static -o %t.lib %t.elf 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=NOT-MACHO -DFILE=%basename_t.tmp.elf
+
+# NOT-MACHO: error: '[[FILE]]': format not supported
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_X86_64
diff --git a/llvm/test/tools/llvm-libtool-darwin/missing-library-type.test b/llvm/test/tools/llvm-libtool-darwin/missing-library-type.test
new file mode 100644
index 0000000000000..45956e787aefb
--- /dev/null
+++ b/llvm/test/tools/llvm-libtool-darwin/missing-library-type.test
@@ -0,0 +1,5 @@
+## Missing library type option:
+# RUN: not llvm-libtool-darwin -o %t.lib %t.input 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=MISSING-OPERATION
+
+# MISSING-OPERATION: Library Type: option: must be specified at least once!
diff --git a/llvm/test/tools/llvm-objcopy/ELF/strip-symbol.test b/llvm/test/tools/llvm-objcopy/ELF/strip-symbol.test
index 78de46cc47b5d..ad71e81eab830 100644
--- a/llvm/test/tools/llvm-objcopy/ELF/strip-symbol.test
+++ b/llvm/test/tools/llvm-objcopy/ELF/strip-symbol.test
@@ -1,19 +1,24 @@
-# RUN: yaml2obj %s -o %t
+# RUN: extract yaml %s | yaml2obj - -o %t
 # RUN: llvm-objcopy --strip-symbol baz -N bar %t %t2
 # RUN: llvm-readobj --symbols --sections %t2 | FileCheck %s
 # RUN: llvm-strip --strip-symbol baz -N bar %t -o %t3
 # RUN: cmp %t2 %t3
 # RUN: llvm-strip --regex --strip-symbol '^b.*' -N bar %t -o %t4
 # RUN: cmp %t3 %t4
-# RUN: echo " bar # bar" > %t-list.txt
-# RUN: echo " baz # baz" >> %t-list.txt
-# RUN: echo " # no symbol" >> %t-list.txt
-# RUN: llvm-objcopy --strip-symbols %t-list.txt %t %t5
+# RUN: extract list1 %s -o %t-list.txt && llvm-objcopy --strip-symbols %t-list.txt %t %t5
 # RUN: cmp %t3 %t5
-# RUN: echo "b.* # bar & baz" > %t-list2.txt
-# RUN: llvm-objcopy --regex --strip-symbols %t-list2.txt %t %t6
+# RUN: extract list2 %s -o %t-list2.txt && llvm-objcopy --regex --strip-symbols %t-list2.txt %t %t6
 # RUN: cmp %t3 %t6
 
+#--- list1
+bar # bar
+baz # baz
+# no symbol
+
+#--- list2
+b.* # bar & baz
+
+#--- yaml
 !ELF
 FileHeader:
   Class:           ELFCLASS64
diff --git a/llvm/test/tools/llvm-profdata/Inputs/header-directives-1.proftext b/llvm/test/tools/llvm-profdata/Inputs/header-directives-1.proftext
new file mode 100644
index 0000000000000..5566523729a11
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/Inputs/header-directives-1.proftext
@@ -0,0 +1,8 @@
+:entry_first
+:ir
+foo
+29667547796
+2
+100
+90
+
diff --git a/llvm/test/tools/llvm-profdata/Inputs/header-directives-2.proftext b/llvm/test/tools/llvm-profdata/Inputs/header-directives-2.proftext
new file mode 100644
index 0000000000000..cadd1e9d1067c
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/Inputs/header-directives-2.proftext
@@ -0,0 +1,8 @@
+:ir
+:not_entry_first
+foo
+29667547796
+2
+100
+90
+
diff --git a/llvm/test/tools/llvm-profdata/Inputs/header-directives-3.proftext b/llvm/test/tools/llvm-profdata/Inputs/header-directives-3.proftext
new file mode 100644
index 0000000000000..5820e147581cf
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/Inputs/header-directives-3.proftext
@@ -0,0 +1,10 @@
+:not_entry_first
+:entry_first
+:fe
+:ir
+foo
+29667547796
+2
+100
+90
+
diff --git a/llvm/test/tools/llvm-profdata/header-directives.test b/llvm/test/tools/llvm-profdata/header-directives.test
new file mode 100644
index 0000000000000..2c75ad81fe64f
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/header-directives.test
@@ -0,0 +1,10 @@
+RUN: llvm-profdata show %p/Inputs/header-directives-1.proftext -o - | FileCheck %s --check-prefixes=ENTRYFIRST,CHECK
+RUN: llvm-profdata show %p/Inputs/header-directives-2.proftext -o - | FileCheck %s --check-prefixes=NOTENTRYFIRST,CHECK
+RUN: llvm-profdata show %p/Inputs/header-directives-3.proftext -o - | FileCheck %s --check-prefixes=ENTRYFIRST,CHECK
+
+ENTRYFIRST: Instrumentation level: IR  entry_first = 1
+NOTENTRYFIRST: Instrumentation level: IR  entry_first = 0
+CHECK: Total functions: 1
+CHECK: Maximum function count: 100
+CHECK: Maximum internal block count: 90
+
diff --git a/llvm/test/tools/llvm-readobj/ELF/AArch64/dwarf-cfi.s b/llvm/test/tools/llvm-readobj/ELF/AArch64/dwarf-cfi.s
index cefc00171544e..d20673f3a40bc 100644
--- a/llvm/test/tools/llvm-readobj/ELF/AArch64/dwarf-cfi.s
+++ b/llvm/test/tools/llvm-readobj/ELF/AArch64/dwarf-cfi.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -triple arm64-linux -filetype obj -o - %s | llvm-readobj -u | FileCheck %s
+# RUN: llvm-mc -triple arm64-linux -filetype obj -o - %s | llvm-readobj -u - | FileCheck %s
 
 # CHECK:      [0x0] CIE length=16
 # CHECK-NEXT:   version: 1
@@ -10,10 +10,9 @@
 # CHECK:        Program:
 # CHECK-NEXT: DW_CFA_def_cfa: reg31 +0
 
-## FIXME Use getEHFrameSection() so that the address is decoded correctly.
 # CHECK:      [0x14] FDE length=16 cie=[0x0]
-# CHECK-NEXT:   initial_location: 0x1c
-# CHECK-NEXT:   address_range: 0x4 (end : 0x20)
+# CHECK-NEXT:   initial_location: 0x0
+# CHECK-NEXT:   address_range: 0x4 (end : 0x4)
 
 # CHECK:        Program:
 # CHECK-NEXT: DW_CFA_nop:
diff --git a/llvm/test/tools/llvm-readobj/ELF/ARM/dwarf-cfi.s b/llvm/test/tools/llvm-readobj/ELF/ARM/dwarf-cfi.s
index 227ac263b4c6b..72b192fe20287 100644
--- a/llvm/test/tools/llvm-readobj/ELF/ARM/dwarf-cfi.s
+++ b/llvm/test/tools/llvm-readobj/ELF/ARM/dwarf-cfi.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -triple arm-linux -filetype obj -o - %s | llvm-readobj -u | FileCheck %s
+# RUN: llvm-mc -triple arm-linux -filetype obj -o - %s | llvm-readobj -u - | FileCheck %s
 
 # CHECK:      [0x0] CIE length=16
 # CHECK-NEXT:   version: 1
@@ -10,10 +10,9 @@
 # CHECK:        Program:
 # CHECK-NEXT: DW_CFA_def_cfa: reg13 +0
 
-## FIXME Use getEHFrameSection() so that the address is decoded correctly.
 # CHECK:      [0x14] FDE length=16 cie=[0x0]
-# CHECK-NEXT:   initial_location: 0x1c
-# CHECK-NEXT:   address_range: 0x4 (end : 0x20)
+# CHECK-NEXT:   initial_location: 0x0
+# CHECK-NEXT:   address_range: 0x4 (end : 0x4)
 
 # CHECK:        Program:
 # CHECK-NEXT: DW_CFA_nop:
diff --git a/llvm/test/tools/llvm-readobj/ELF/broken-group.test b/llvm/test/tools/llvm-readobj/ELF/broken-group.test
deleted file mode 100644
index 23bd76a821ef9..0000000000000
--- a/llvm/test/tools/llvm-readobj/ELF/broken-group.test
+++ /dev/null
@@ -1,80 +0,0 @@
-# RUN: yaml2obj %s -o %t.o
-
-# RUN: llvm-readelf --elf-section-groups \
-# RUN:   %t.o >%tout.log 2>%terr.log
-# RUN: FileCheck %s -check-prefix=GNU-STDERR < %terr.log
-# GNU-STDERR: Error: section [    3] in group section [    4] already in group section [    2]
-# RUN: FileCheck %s -check-prefix=GNU-STDOUT < %tout.log
-# GNU-STDOUT:      COMDAT group section [    2] `.group' [bar] contains 1 sections:
-# GNU-STDOUT-NEXT:   [Index]    Name
-# GNU-STDOUT-NEXT:   [    3]   .foo
-# GNU-STDOUT:      COMDAT group section [    4] `.group1' [zed] contains 1 sections:
-# GNU-STDOUT-NEXT:   [Index]    Name
-
-# RUN: llvm-readobj --elf-section-groups \
-# RUN:   %t.o >%t2out.log 2>%t2err.log %t.o
-# RUN: FileCheck %s -check-prefix=LLVM-STDERR < %t2err.log
-# LLVM-STDERR: Error: .foo (3) in a group .group1 (4) is already in a group .group (2)
-# RUN: FileCheck %s -check-prefix=LLVM-STDOUT < %t2out.log
-# LLVM-STDOUT:      Groups {
-# LLVM-STDOUT-NEXT:   Group {
-# LLVM-STDOUT-NEXT:     Name: .group
-# LLVM-STDOUT-NEXT:     Index: 2
-# LLVM-STDOUT-NEXT:     Link: 5
-# LLVM-STDOUT-NEXT:     Info: 1
-# LLVM-STDOUT-NEXT:     Type: COMDAT
-# LLVM-STDOUT-NEXT:     Signature: bar
-# LLVM-STDOUT-NEXT:     Section(s) in group [
-# LLVM-STDOUT-NEXT:       .foo (3)
-# LLVM-STDOUT-NEXT:     ]
-# LLVM-STDOUT-NEXT:   }
-# LLVM-STDOUT-NEXT:   Group {
-# LLVM-STDOUT-NEXT:     Name: .group1
-# LLVM-STDOUT-NEXT:     Index: 4
-# LLVM-STDOUT-NEXT:     Link: 5
-# LLVM-STDOUT-NEXT:     Info: 2
-# LLVM-STDOUT-NEXT:     Type: COMDAT
-# LLVM-STDOUT-NEXT:     Signature: zed
-# LLVM-STDOUT-NEXT:     Section(s) in group [
-# LLVM-STDOUT-NEXT:     ]
-# LLVM-STDOUT-NEXT:   }
-# LLVM-STDOUT-NEXT: }
-
-!ELF
-FileHeader:      
-  Class:           ELFCLASS64
-  Data:            ELFDATA2LSB
-  Type:            ET_REL
-  Machine:         EM_X86_64
-Sections:        
-  - Name:            .text
-    Type:            SHT_PROGBITS
-    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
-    AddressAlign:    0x0000000000000004
-    Content:         ''
-  - Name:            .group
-    Type:            SHT_GROUP
-    Link:            .symtab
-    AddressAlign:    0x0000000000000004
-    Info:            bar
-    Members:         
-      - SectionOrType:   GRP_COMDAT
-      - SectionOrType:   .foo
-  - Name:            .foo
-    Type:            SHT_PROGBITS
-    Flags:           [ SHF_ALLOC, SHF_EXECINSTR, SHF_GROUP ]
-    AddressAlign:    0x0000000000000001
-    Content:         ''
-  - Name:            .group1
-    Type:            SHT_GROUP
-    Link:            .symtab
-    AddressAlign:    0x0000000000000004
-    Info:            zed
-    Members:         
-      - SectionOrType:   GRP_COMDAT
-      - SectionOrType:   .foo
-Symbols:
-  - Name:            bar
-    Section:         .group
-  - Name:            zed
-    Section:         .group1
diff --git a/llvm/test/tools/llvm-readobj/ELF/dyn-symbols.test b/llvm/test/tools/llvm-readobj/ELF/dyn-symbols.test
index c4ac2f8325fc4..6169871034d5f 100644
--- a/llvm/test/tools/llvm-readobj/ELF/dyn-symbols.test
+++ b/llvm/test/tools/llvm-readobj/ELF/dyn-symbols.test
@@ -83,14 +83,68 @@ ProgramHeaders:
 # RUN: llvm-readelf --dyn-symbols %t1.so > %t.readelf-dyn-symbols
 # RUN: cmp %t.readelf-dyn-symbols %t.readelf-dyn-syms
 
-## Case 3: Check that we are able to dump the dynamic symbol table even when we have no program headers.
+## Case 3.1: Check that we are able to dump the dynamic symbol table even when we have no program headers.
 ## In this case we find the table by it's type (SHT_DYNSYM) and ignore the DT_SYMTAB value.
 # RUN: yaml2obj --docnum=2 %s -o %t2.so
-# RUN: llvm-readobj %t2.so --dyn-symbols | FileCheck %s --check-prefix=NOPHDRS-LLVM
-# RUN: llvm-readelf %t2.so --dyn-symbols | FileCheck %s --check-prefix=NOPHDRS-GNU
+# RUN: llvm-readobj %t2.so --dyn-symbols 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t2.so --implicit-check-not=warning: --check-prefix=NOPHDRS-LLVM
+# RUN: llvm-readelf %t2.so --dyn-symbols 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t2.so --implicit-check-not=warning: -DNAME=.dynsym --check-prefix=NOPHDRS-GNU
+
+# NOPHDRS-LLVM: warning: '[[FILE]]': Unable to parse DT_SYMTAB: virtual address is not in any segment: 0xffff1234
+# NOPHDRS-LLVM:      DynamicSymbols [
+# NOPHDRS-LLVM-NEXT:   Symbol {
+# NOPHDRS-LLVM-NEXT:     Name:  (0)
+# NOPHDRS-LLVM-NEXT:     Value: 0x0
+# NOPHDRS-LLVM-NEXT:     Size: 0
+# NOPHDRS-LLVM-NEXT:     Binding: Local (0x0)
+# NOPHDRS-LLVM-NEXT:     Type: None (0x0)
+# NOPHDRS-LLVM-NEXT:     Other: 0
+# NOPHDRS-LLVM-NEXT:     Section: Undefined (0x0)
+# NOPHDRS-LLVM-NEXT:   }
+# NOPHDRS-LLVM-NEXT:   Symbol {
+# NOPHDRS-LLVM-NEXT:     Name: foo (1)
+# NOPHDRS-LLVM-NEXT:     Value: 0x0
+# NOPHDRS-LLVM-NEXT:     Size: 0
+# NOPHDRS-LLVM-NEXT:     Binding: Local (0x0)
+# NOPHDRS-LLVM-NEXT:     Type: None (0x0)
+# NOPHDRS-LLVM-NEXT:     Other: 0
+# NOPHDRS-LLVM-NEXT:     Section: Undefined (0x0)
+# NOPHDRS-LLVM-NEXT:   }
+# NOPHDRS-LLVM-NEXT: ]
+
+# NOPHDRS-GNU:      warning: '[[FILE]]': Unable to parse DT_SYMTAB: virtual address is not in any segment: 0xffff1234
+# NOPHDRS-NAMEWARN: warning: '[[FILE]]': unable to get the name of SHT_DYNSYM section with index 2: a section [index 2] has an invalid sh_name (0xffffffff) offset which goes past the end of the section name string table
+# NOPHDRS-GNU:      Symbol table '[[NAME]]' contains 2 entries:
+# NOPHDRS-GNU-NEXT:  Num:    Value          Size Type    Bind   Vis       Ndx Name
+# NOPHDRS-GNU-NEXT:    0: 0000000000000000     0 NOTYPE  LOCAL  DEFAULT   UND
+# NOPHDRS-GNU-NEXT:    1: 0000000000000000     0 NOTYPE  LOCAL  DEFAULT   UND foo
 
-# NOPHDRS-LLVM: Name: foo
-# NOPHDRS-GNU:  1: 0000000000000000 0 NOTYPE LOCAL DEFAULT UND foo
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_X86_64
+Sections:
+  - Name:    .dynamic
+    Type:    SHT_DYNAMIC
+    Entries:
+      - Tag:   DT_SYMTAB
+        Value: 0xffff1234
+      - Tag:   DT_NULL
+        Value: 0
+DynamicSymbols:
+  - Name: foo
+
+## Case 3.2: the same as 3.1, but the sh_name field of the SHT_DYNSYM section is invalid.
+##           Check we are still able to dump symbols.
+# RUN: yaml2obj --docnum=3 %s -o %t2.broken.name
+# RUN: llvm-readobj %t2.broken.name --dyn-symbols 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t2.broken.name --check-prefix=NOPHDRS-LLVM --implicit-check-not=warning:
+# RUN: llvm-readelf %t2.broken.name --dyn-symbols 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t2.broken.name -DNAME="<?>" \
+# RUN:     --check-prefixes=NOPHDRS-GNU,NOPHDRS-NAMEWARN --implicit-check-not=warning:
 
 --- !ELF
 FileHeader:
@@ -106,12 +160,15 @@ Sections:
         Value: 0xffff1234
       - Tag:   DT_NULL
         Value: 0
+  - Name:   .dynsym
+    Type:   SHT_DYNSYM
+    ShName: 0xffffffff
 DynamicSymbols:
   - Name: foo
 
 ## Case 4: Check we report a warning when there is no SHT_DYNSYM section and we can't map the DT_SYMTAB value
 ## to an address because of the absence of a corresponding PT_LOAD program header.
-# RUN: yaml2obj --docnum=3 %s -o %t3.so
+# RUN: yaml2obj --docnum=4 %s -o %t3.so
 # RUN: llvm-readobj %t3.so --dyn-symbols 2>&1 | FileCheck %s -DFILE=%t3.so --check-prefixes=NOSHT-DYNSYM,NOSHT-DYNSYM-LLVM
 # RUN: llvm-readelf %t3.so --dyn-symbols 2>&1 | FileCheck %s -DFILE=%t3.so --check-prefix=NOSHT-DYNSYM
 
@@ -140,7 +197,7 @@ DynamicSymbols:
 
 ## Case 5: Check that when we can't map the value of the DT_SYMTAB tag to an address, we report a warning and
 ## use the information in the section header table to locate the dynamic symbol table.
-# RUN: yaml2obj --docnum=4 %s -o %t4.so
+# RUN: yaml2obj --docnum=5 %s -o %t4.so
 # RUN: llvm-readobj %t4.so --dyn-symbols 2>&1 | FileCheck -DFILE=%t4.so %s --check-prefixes=BROKEN-DTSYMTAB,BROKEN-DTSYMTAB-LLVM
 # RUN: llvm-readelf %t4.so --dyn-symbols 2>&1 | FileCheck -DFILE=%t4.so %s --check-prefixes=BROKEN-DTSYMTAB,BROKEN-DTSYMTAB-GNU
 
@@ -172,7 +229,7 @@ ProgramHeaders:
 
 ## Case 6: Check that if we can get the location of the dynamic symbol table using both the DT_SYMTAB value
 ## and the section headers table then we prefer the former and report a warning.
-# RUN: yaml2obj --docnum=5 %s -o %t5.so
+# RUN: yaml2obj --docnum=6 %s -o %t5.so
 # RUN: llvm-readobj %t5.so --dyn-symbols 2>&1 | FileCheck -DFILE=%t5.so %s --check-prefixes=PREFER-DTSYMTAB,PREFER-DTSYMTAB-LLVM
 # RUN: llvm-readelf %t5.so --dyn-symbols 2>&1 | FileCheck -DFILE=%t5.so %s --check-prefixes=PREFER-DTSYMTAB,PREFER-DTSYMTAB-GNU
 
@@ -210,7 +267,7 @@ ProgramHeaders:
 
 ## Case 7: Check how we dump versioned symbols. Use both -V and --dyn-symbols
 ## to check that printed version is consistent.
-# RUN: yaml2obj %s --docnum=6 -o %t6
+# RUN: yaml2obj %s --docnum=7 -o %t6
 # RUN: llvm-readobj -V --dyn-symbols %t6 | FileCheck %s --check-prefix=VERSIONED-LLVM
 # RUN: llvm-readelf -V --dyn-symbols %t6 | FileCheck %s --check-prefix=VERSIONED-GNU
 
@@ -293,15 +350,15 @@ DynamicSymbols:
 
 ## Case 8: Check what we print when:
 ## a) The dynamic symbol table does not exist.
-# RUN: yaml2obj %s --docnum=7 -o %t7
+# RUN: yaml2obj %s --docnum=8 -o %t7
 # RUN: llvm-readobj --dyn-symbols %t7 | FileCheck %s --check-prefix=NO-DYNSYM-LLVM
 # RUN: llvm-readelf --dyn-symbols %t7 | count 0
 ## b) The dynamic symbol table has a size of 0.
-# RUN: yaml2obj %s --docnum=8 -o %t8
+# RUN: yaml2obj %s --docnum=9 -o %t8
 # RUN: llvm-readobj --dyn-symbols %t8 | FileCheck %s --check-prefix=NO-DYNSYM-LLVM
 # RUN: llvm-readelf --dyn-symbols %t8 | count 0
 ## c) The dynamic symbol table only contains the null symbol.
-# RUN: yaml2obj %s --docnum=9 -o %t9
+# RUN: yaml2obj %s --docnum=10 -o %t9
 # RUN: llvm-readobj --dyn-symbols %t9 | FileCheck %s --check-prefix=DYNSYM-EMPTY-LLVM
 # RUN: llvm-readelf --dyn-symbols %t9 | FileCheck %s --check-prefix=DYNSYM-EMPTY-GNU
 
@@ -352,28 +409,28 @@ DynamicSymbols: []
 
 ## Case 9: Check what we print when:
 ## a) The size of the dynamic symbol table is not a multiple of its entry size.
-# RUN: yaml2obj %s --docnum=10 -o %t10
+# RUN: yaml2obj %s --docnum=11 -o %t10
 # RUN: llvm-readobj --dyn-symbols %t10 2>&1 | FileCheck %s -DFILE=%t10 --check-prefix=DYNSYM-SIZE-INVALID1
 # RUN: llvm-readelf --dyn-symbols %t10 2>&1 | FileCheck %s -DFILE=%t10 --check-prefix=DYNSYM-SIZE-INVALID1
 
-# DYNSYM-SIZE-INVALID1: warning: '[[FILE]]': section with index 1 has invalid size (0x1) or entry size (0x10)
+# DYNSYM-SIZE-INVALID1: warning: '[[FILE]]': SHT_DYNSYM section with index 1 has invalid size (0x1) or entry size (0x10)
 
 ## b) The same, but the DT_SYMTAB tag is present. In this case the dynamic tag has priority over the
 ##    information about a location and an entity size of the dynamic symbol table from the section header.
 ##    The code uses sizeof(Elf_Sym) for an entity size, so it can't be incorrect and
 ##    the message printed is a bit shorter.
-# RUN: yaml2obj %s --docnum=11 -o %t11
+# RUN: yaml2obj %s --docnum=12 -o %t11
 # RUN: llvm-readobj --dyn-symbols %t11 2>&1 | FileCheck %s -DFILE=%t11 --check-prefix=DYNSYM-SIZE-INVALID2
 # RUN: llvm-readelf --dyn-symbols %t11 2>&1 | FileCheck %s -DFILE=%t11 --check-prefix=DYNSYM-SIZE-INVALID2
 
-# DYNSYM-SIZE-INVALID2: warning: '[[FILE]]': section with index 2 has invalid size (0x1){{$}}
+# DYNSYM-SIZE-INVALID2: warning: '[[FILE]]': SHT_DYNSYM section with index 2 has invalid size (0x1){{$}}
 
 ## c) In the case when the DT_SYMENT tag is present, we report when it's value does not match the
 #     value of the symbol size for the platform.
-# RUN: yaml2obj %s -D BITS=32 --docnum=12 -o %t12
+# RUN: yaml2obj %s -D BITS=32 --docnum=13 -o %t12
 # RUN: llvm-readobj --dyn-symbols %t12 2>&1 | FileCheck %s -DFILE=%t12 --check-prefix=DYNSYM-SIZE-INVALID3
 # RUN: llvm-readelf --dyn-symbols %t12 2>&1 | FileCheck %s -DFILE=%t12 --check-prefix=DYNSYM-SIZE-INVALID3
-# RUN: yaml2obj %s -D BITS=64 --docnum=12 -o %t13
+# RUN: yaml2obj %s -D BITS=64 --docnum=13 -o %t13
 # RUN: llvm-readobj --dyn-symbols %t13 2>&1 | FileCheck %s -DFILE=%t13 --check-prefix=DYNSYM-SIZE-INVALID4
 # RUN: llvm-readelf --dyn-symbols %t13 2>&1 | FileCheck %s -DFILE=%t13 --check-prefix=DYNSYM-SIZE-INVALID4
 
@@ -433,7 +490,7 @@ Sections:
 ## Check we report a warning when the DT_STRSZ value is broken so that the dynamic string
 ## table goes past the end of the file. Document we stop dumping symbols and report an error.
 
-# RUN: yaml2obj %s --docnum=13 -o %t14
+# RUN: yaml2obj %s --docnum=14 -o %t14
 # RUN: llvm-readobj --dyn-symbols %t14 2>&1 | \
 # RUN:   FileCheck %s -DFILE=%t14 --check-prefix=DYNSTR-INVALID-LLVM
 # RUN: llvm-readelf --dyn-symbols %t14 2>&1 | \
@@ -513,16 +570,16 @@ ProgramHeaders:
       - Section: .dynamic
 
 ## Check we report a warning when the entry size of the dynamic symbol table is zero.
-# RUN: yaml2obj %s --docnum=14 -o %t15
+# RUN: yaml2obj %s --docnum=15 -o %t15
 # RUN: llvm-readobj --dyn-symbols %t15 2>&1 | FileCheck %s -DFILE=%t15 --check-prefix=DYNSYM-ZERO-ENTSIZE-LLVM
 # RUN: llvm-readelf --dyn-symbols %t15 2>&1 | \
 # RUN:   FileCheck %s -DFILE=%t15 --check-prefix=DYNSYM-ZERO-ENTSIZE-GNU --implicit-check-not="Symbol table"
 
 # DYNSYM-ZERO-ENTSIZE-LLVM:      DynamicSymbols [
-# DYNSYM-ZERO-ENTSIZE-LLVM-NEXT:  warning: '[[FILE]]': section with index 1 has invalid size (0x10) or entry size (0x0)
+# DYNSYM-ZERO-ENTSIZE-LLVM-NEXT:  warning: '[[FILE]]': SHT_DYNSYM section with index 1 has invalid size (0x10) or entry size (0x0)
 # DYNSYM-ZERO-ENTSIZE-LLVM-NEXT: ]
 
-# DYNSYM-ZERO-ENTSIZE-GNU:  warning: '[[FILE]]': section with index 1 has invalid size (0x10) or entry size (0x0)
+# DYNSYM-ZERO-ENTSIZE-GNU:  warning: '[[FILE]]': SHT_DYNSYM section with index 1 has invalid size (0x10) or entry size (0x0)
 
 --- !ELF
 FileHeader:
diff --git a/llvm/test/tools/llvm-readobj/ELF/dynamic-malformed.test b/llvm/test/tools/llvm-readobj/ELF/dynamic-malformed.test
index 8b6fc1eca7d2c..4cb0d0b0cc354 100644
--- a/llvm/test/tools/llvm-readobj/ELF/dynamic-malformed.test
+++ b/llvm/test/tools/llvm-readobj/ELF/dynamic-malformed.test
@@ -8,7 +8,7 @@
 # RUN:   | FileCheck %s -DFILE=%t.bad-size --implicit-check-not=warning: --check-prefix WARN-GNU
 
 # WARN: warning: '[[FILE]]': invalid PT_DYNAMIC size (0x4){{$}}
-# WARN: warning: '[[FILE]]': section with index 1 has invalid size (0x4){{$}}
+# WARN: warning: '[[FILE]]': SHT_DYNAMIC section with index 1 has invalid size (0x4){{$}}
 # WARN: warning: '[[FILE]]': no valid dynamic table was found
 # WARN-EMPTY:
 # WARN: File:
@@ -17,7 +17,7 @@
 # WARN: ]
 
 # WARN-GNU: warning: '[[FILE]]': invalid PT_DYNAMIC size (0x4){{$}}
-# WARN-GNU: warning: '[[FILE]]': section with index 1 has invalid size (0x4){{$}}
+# WARN-GNU: warning: '[[FILE]]': SHT_DYNAMIC section with index 1 has invalid size (0x4){{$}}
 # WARN-GNU: warning: '[[FILE]]': no valid dynamic table was found
 # WARN-GNU-NEXT: ELF Header:
 # WARN-GNU:      Symbol table '.symtab' contains 1 entries:
diff --git a/llvm/test/tools/llvm-readobj/ELF/groups.test b/llvm/test/tools/llvm-readobj/ELF/groups.test
index 4e165311ecfb1..ac211893ffe20 100644
--- a/llvm/test/tools/llvm-readobj/ELF/groups.test
+++ b/llvm/test/tools/llvm-readobj/ELF/groups.test
@@ -61,7 +61,7 @@ Sections:
     Info: bar
     Members:
       - SectionOrType: GRP_COMDAT
-      - SectionOrType: .text.bar
+      - SectionOrType: [[TEXTBARNAME=.text.bar]]
       - SectionOrType: .rela.text.bar
   - Name:  .text.foo
     Type:  SHT_PROGBITS
@@ -80,3 +80,47 @@ Symbols:
     Section: .text.foo
   - Name:    bar
     Section: .text.bar
+
+## Check that we report a warning and continue dumping when a section is included
+## in two group sections at the same time.
+
+# RUN: yaml2obj %s -DTEXTBARNAME=.text.foo -o %t.dup.o
+# RUN: llvm-readobj --elf-section-groups %t.dup.o 2>&1 | FileCheck %s -DFILE=%t.dup.o --check-prefix=DUP-LLVM
+# RUN: llvm-readelf --elf-section-groups %t.dup.o 2>&1 | FileCheck %s -DFILE=%t.dup.o --check-prefix=DUP-GNU
+
+# DUP-LLVM:      Group {
+# DUP-LLVM-NEXT:   Name: .group
+# DUP-LLVM-NEXT:   Index: 1
+# DUP-LLVM-NEXT:   Link: 7
+# DUP-LLVM-NEXT:   Info: 1
+# DUP-LLVM-NEXT:   Type: COMDAT
+# DUP-LLVM-NEXT:   Signature: foo
+# DUP-LLVM-NEXT:   Section(s) in group [
+# DUP-LLVM-NEXT:     .text.foo (3)
+# DUP-LLVM-NEXT:     .rela.text.foo (4)
+# DUP-LLVM-NEXT:   ]
+# DUP-LLVM-NEXT: }
+# DUP-LLVM:      Group {
+# DUP-LLVM-NEXT:   Name: .group1
+# DUP-LLVM-NEXT:   Index: 2
+# DUP-LLVM-NEXT:   Link: 7
+# DUP-LLVM-NEXT:   Info: 2
+# DUP-LLVM-NEXT:   Type: COMDAT
+# DUP-LLVM-NEXT:   Signature: bar
+# DUP-LLVM-NEXT:   Section(s) in group [
+# DUP-LLVM-NEXT: warning: '[[FILE]]': section with index 3, included in the group section with index 1, was also found in the group section with index 2
+# DUP-LLVM-NEXT:     .text.foo (3)
+# DUP-LLVM-NEXT:     .rela.text.bar (6)
+# DUP-LLVM-NEXT:   ]
+# DUP-LLVM-NEXT: }
+
+# DUP-GNU:      COMDAT group section [    1] `.group' [foo] contains 2 sections:
+# DUP-GNU-NEXT:  [Index]    Name
+# DUP-GNU-NEXT:  [    3]   .text.foo
+# DUP-GNU-NEXT:  [    4]   .rela.text.foo
+
+# DUP-GNU:      COMDAT group section [    2] `.group1' [bar] contains 2 sections:
+# DUP-GNU-NEXT:    [Index]    Name
+# DUP-GNU-NEXT: warning: '[[FILE]]': section with index 3, included in the group section with index 1, was also found in the group section with index 2
+# DUP-GNU-NEXT:    [    3]   .text.foo
+# DUP-GNU-NEXT:    [    6]   .rela.text.bar
diff --git a/llvm/test/tools/llvm-readobj/ELF/mips-abiflags.test b/llvm/test/tools/llvm-readobj/ELF/mips-abiflags.test
index 1a8affa079843..f4432efb6aeba 100644
--- a/llvm/test/tools/llvm-readobj/ELF/mips-abiflags.test
+++ b/llvm/test/tools/llvm-readobj/ELF/mips-abiflags.test
@@ -312,3 +312,55 @@ Sections:
 
 # FLAG2-ALL-GNU:  FLAGS 2: ffffffff
 # FLAG2-ALL-LLVM: Flags 2: 0xFFFFFFF
+
+## Check what we print when there is no .MIPS.abiflags section in the file.
+# RUN: yaml2obj --docnum=2 %s -o %t.nosection
+## Note: GNU readelf 2.31.1 prints nothing.
+# RUN: llvm-readelf -A %t.nosection 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=NOSEC --implicit-check-not={{.}}
+# RUN: llvm-readobj -A %t.nosection 2>&1 | \
+# RUN:   FileCheck %s --check-prefixes=NOSEC,NOSEC-LLVM --implicit-check-not=warning:
+
+# NOSEC-LLVM: There is no .MIPS.abiflags section in the file.
+# NOSEC:      There is no .MIPS.options section in the file.
+# NOSEC-NEXT: There is no .reginfo section in the file.
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_MIPS
+Sections: []
+
+## Check we report a warning when we are unable to read the content of the .MIPS.abiflags section.
+# RUN: yaml2obj --docnum=3 -DSHOFFSET=0xffffffff %s -o %t.err1
+# RUN: llvm-readelf -A %t.err1 2>&1 | FileCheck %s -DFILE=%t.err1 --check-prefix=CONTENT-ERR
+# RUN: llvm-readobj -A %t.err1 2>&1 | FileCheck %s -DFILE=%t.err1 --check-prefix=CONTENT-ERR
+
+# CONTENT-ERR:      warning: '[[FILE]]': unable to read the .MIPS.abiflags section: section [index 1] has a sh_offset (0xffffffff) + sh_size (0x18) that is greater than the file size (0x240)
+# CONTENT-ERR-NEXT: There is no .MIPS.options section in the file.
+# CONTENT-ERR-NEXT: There is no .reginfo section in the file.
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_MIPS
+Sections:
+  - Name:     .MIPS.abiflags
+    Type:     SHT_MIPS_ABIFLAGS
+    ISA:      MIPS32
+    Offset:   0x100
+    ShOffset: [[SHOFFSET=0x100]]
+    ShSize:   [[SHSIZE=24]]
+
+## Check we report a warning when the .MIPS.abiflags section has an unexpected size.
+# RUN: yaml2obj --docnum=3 -DSHSIZE=23 %s -o %t.err2
+# RUN: llvm-readelf -A %t.err2 2>&1 | FileCheck %s -DFILE=%t.err2 --check-prefix=SIZE-ERR
+# RUN: llvm-readobj -A %t.err2 2>&1 | FileCheck %s -DFILE=%t.err2 --check-prefix=SIZE-ERR
+
+# SIZE-ERR:      warning: '[[FILE]]': unable to read the .MIPS.abiflags section: it has a wrong size (23)
+# SIZE-ERR-NEXT: There is no .MIPS.options section in the file.
+# SIZE-ERR-NEXT: There is no .reginfo section in the file.
diff --git a/llvm/test/tools/llvm-readobj/ELF/mips-plt.test b/llvm/test/tools/llvm-readobj/ELF/mips-plt.test
index 4959d8892ac2b..95b310ba664c1 100644
--- a/llvm/test/tools/llvm-readobj/ELF/mips-plt.test
+++ b/llvm/test/tools/llvm-readobj/ELF/mips-plt.test
@@ -134,9 +134,9 @@ DynamicSymbols: []
 # RUN: yaml2obj --docnum=2 -DVAL2=0x100 -DLINK=0xaaaaaaaa %s -o %t.err6.o
 # RUN: not llvm-readobj -A %t.err6.o 2>&1 | FileCheck %s -DFILE=%t.err6.o -check-prefix ERR6
 
-# ERR6: error: '[[FILE]]': unable to get a symbol table linked to the RELPLT section with index 2: invalid section index: 2863311530
+# ERR6: error: '[[FILE]]': unable to get a symbol table linked to the SHT_PROGBITS section with index 2: invalid section index: 2863311530
 
 # RUN: yaml2obj --docnum=2 -DVAL2=0x100 %s -o %t.err7.o
 # RUN: not llvm-readobj -A %t.err7.o 2>&1 | FileCheck %s -DFILE=%t.err7.o -check-prefix ERR7
 
-# ERR7: error: '[[FILE]]': unable to get a string table for the symbol table with index 1: invalid sh_type for symbol table, expected SHT_SYMTAB or SHT_DYNSYM
+# ERR7: error: '[[FILE]]': unable to get a string table for the SHT_DYNAMIC section with index 1: invalid sh_type for symbol table, expected SHT_SYMTAB or SHT_DYNSYM
diff --git a/llvm/test/tools/llvm-readobj/ELF/symbols.test b/llvm/test/tools/llvm-readobj/ELF/symbols.test
index 492097d55c0fd..69d59311f662b 100644
--- a/llvm/test/tools/llvm-readobj/ELF/symbols.test
+++ b/llvm/test/tools/llvm-readobj/ELF/symbols.test
@@ -195,12 +195,11 @@ Symbols:
 ##         In this case we set the e_shstrndx field to an invalid index so that the .shstrtab section can't be located.
 # RUN: yaml2obj --docnum=2 -DSHSTRTAB=0xff %s -o %t64.err3
 # RUN: llvm-readobj --symbols %t64.err3 2>&1 | \
-# RUN:  FileCheck %s -DFILE=%t64.err3 --check-prefix=SYMTAB-SHSTRTAB-ERR-LLVM
+# RUN:  FileCheck %s -DFILE=%t64.err3 --check-prefix=SYMTAB-SHSTRTAB-ERR-LLVM --implicit-check-not=warning:
 # RUN: llvm-readelf --symbols %t64.err3 2>&1 | \
-# RUN:  FileCheck %s -DFILE=%t64.err3 --check-prefix=SYMTAB-SHSTRTAB-ERR-GNU
+# RUN:  FileCheck %s -DFILE=%t64.err3 --check-prefix=SYMTAB-SHSTRTAB-ERR-GNU --implicit-check-not=warning:
 
 # SYMTAB-SHSTRTAB-ERR-LLVM:      Symbols [
-# SYMTAB-SHSTRTAB-ERR-LLVM-NEXT: warning: '[[FILE]]': unable to get the name of the SHT_SYMTAB section: section header string table index 255 does not exist
 # SYMTAB-SHSTRTAB-ERR-LLVM-NEXT:   Symbol {
 # SYMTAB-SHSTRTAB-ERR-LLVM-NEXT:     Name:  (0)
 # SYMTAB-SHSTRTAB-ERR-LLVM-NEXT:     Value: 0x0
@@ -222,7 +221,7 @@ Symbols:
 # SYMTAB-SHSTRTAB-ERR-LLVM-NEXT:   }
 # SYMTAB-SHSTRTAB-ERR-LLVM-NEXT: ]
 
-# SYMTAB-SHSTRTAB-ERR-GNU:      warning: '[[FILE]]': unable to get the name of the SHT_SYMTAB section: section header string table index 255 does not exist
+# SYMTAB-SHSTRTAB-ERR-GNU:      warning: '[[FILE]]': unable to get the name of SHT_SYMTAB section with index 1: section header string table index 255 does not exist
 # SYMTAB-SHSTRTAB-ERR-GNU:      Symbol table '<?>' contains 2 entries:
 # SYMTAB-SHSTRTAB-ERR-GNU-NEXT:    Num:    Value          Size Type    Bind   Vis       Ndx Name
 # SYMTAB-SHSTRTAB-ERR-GNU-NEXT:      0: 0000000000000000     0 NOTYPE  LOCAL  DEFAULT   UND
diff --git a/llvm/test/tools/llvm-readobj/basic.test b/llvm/test/tools/llvm-readobj/basic.test
index 89962c3231be2..5b317951493bb 100644
--- a/llvm/test/tools/llvm-readobj/basic.test
+++ b/llvm/test/tools/llvm-readobj/basic.test
@@ -4,6 +4,11 @@ RUN: not llvm-readelf %t.blah 2>&1 | FileCheck --check-prefix=ENOENT -DTOOL=read
 
 ENOENT: llvm-[[TOOL]]{{(\.exe)?}}: error: '{{.*}}.blah': {{[Nn]}}o such file or directory
 
+# Test case with no input file.
+RUN: not llvm-readobj 2>&1 | FileCheck %s --check-prefix=NO-FILE
+RUN: not llvm-readelf 2>&1  | FileCheck %s --check-prefix=NO-FILE
+NO-FILE: error: no input files specified
+
 # Test case where input file is too small to be a recognised object file.
 RUN: touch %t.empty
 RUN: not llvm-readobj %t.empty 2>&1 | FileCheck --check-prefix=EMPTY %s
diff --git a/llvm/test/tools/llvm-readobj/wasm/wasm-imports.test b/llvm/test/tools/llvm-readobj/wasm/wasm-imports.test
index a6d8ca7b506da..35639d11ee974 100644
--- a/llvm/test/tools/llvm-readobj/wasm/wasm-imports.test
+++ b/llvm/test/tools/llvm-readobj/wasm/wasm-imports.test
@@ -1,4 +1,4 @@
-# RUN: yaml2obj %s | llvm-readobj --symbols | FileCheck %s
+# RUN: yaml2obj %s | llvm-readobj --symbols - | FileCheck %s
 
 # Test symbols with custom import module and name attributes.
 
diff --git a/llvm/test/tools/llvm-strings/radix.test b/llvm/test/tools/llvm-strings/radix.test
index d23fb3cddc8f8..d9796a937d905 100644
--- a/llvm/test/tools/llvm-strings/radix.test
+++ b/llvm/test/tools/llvm-strings/radix.test
@@ -1,15 +1,18 @@
 ## Show that llvm-strings can handle the -t/--radix switch properly.
 
-RUN: echo one > %t
-RUN: echo two >> %t
-RUN: echo three >> %t
-RUN: echo four >> %t
-RUN: echo five >> %t
-RUN: echo six >> %t
-RUN: echo seven >> %t
-RUN: echo eight >> %t
-RUN: echo nine >> %t
-RUN: echo ten >> %t
+RUN: extract --no-leading-lines input %s -o %t
+#--- input
+one
+two
+three
+four
+five
+six
+seven
+eight
+nine
+ten
+#--- end
 
 RUN: llvm-strings %t | FileCheck %s -check-prefix CHECK-NONE --implicit-check-not={{.}}
 RUN: llvm-strings -t d %t | FileCheck %s -check-prefix CHECK-DEC --strict-whitespace --implicit-check-not={{.}}
diff --git a/llvm/test/tools/llvm-symbolizer/pdb/pdb-native-columns.test b/llvm/test/tools/llvm-symbolizer/pdb/pdb-columns.test
similarity index 90%
rename from llvm/test/tools/llvm-symbolizer/pdb/pdb-native-columns.test
rename to llvm/test/tools/llvm-symbolizer/pdb/pdb-columns.test
index a8ccc33d03574..425d58d4d46f9 100644
--- a/llvm/test/tools/llvm-symbolizer/pdb/pdb-native-columns.test
+++ b/llvm/test/tools/llvm-symbolizer/pdb/pdb-columns.test
@@ -6,6 +6,8 @@ RUN: echo 0x140006C20 >> %t.input
 RUN: echo 0x140006C30 >> %t.input
 RUN: echo 0x140006C40 >> %t.input
 RUN: echo 0x140006C70 >> %t.input
+RUN:    llvm-symbolizer -obj="%p/Inputs/test-columns.exe" < %t.input \
+RUN:    | FileCheck %s
 RUN:    llvm-symbolizer -obj="%p/Inputs/test-columns.exe" -use-native-pdb-reader < %t.input \
 RUN:    | FileCheck %s
 
diff --git a/llvm/test/tools/llvm-symbolizer/pdb/pdb-native.test b/llvm/test/tools/llvm-symbolizer/pdb/pdb-native.test
deleted file mode 100644
index 9356bbc302050..0000000000000
--- a/llvm/test/tools/llvm-symbolizer/pdb/pdb-native.test
+++ /dev/null
@@ -1,45 +0,0 @@
-RUN: echo 0x401380 > %t.input
-RUN: echo 0x401390 >> %t.input
-RUN: echo 0x4013A0 >> %t.input
-RUN: echo 0x4013C0 >> %t.input
-RUN: echo 0x4013D0 >> %t.input
-RUN: echo 0x4013E0 >> %t.input
-RUN: echo 0x4013F0 >> %t.input
-RUN: echo 0x401420 >> %t.input
-RUN: llvm-symbolizer -obj="%p/Inputs/test.exe" -use-native-pdb-reader < %t.input \
-RUN:    | FileCheck %s
-RUN: llvm-symbolizer -obj="%p/Inputs/test.exe" -demangle=false -use-native-pdb-reader < %t.input \
-RUN:    | FileCheck %s --check-prefix=CHECK-NO-DEMANGLE
-
-Subtract ImageBase from all the offsets and run the test again with
---relative-address.
-
-RUN: %python -c 'import sys;print("\n".join([hex(int(x, 16) - 0x400000) for x in sys.stdin]))' < %t.input \
-RUN:    | llvm-symbolizer -obj="%p/Inputs/test.exe" -use-native-pdb-reader -demangle=false --relative-address \
-RUN:    | FileCheck %s --check-prefix=CHECK-NO-DEMANGLE
-
-CHECK: foo(void)
-CHECK-NEXT: test.cpp:10
-CHECK: {{^private_symbol$}}
-CHECK-NEXT: test.cpp:13:0
-CHECK: {{^main}}
-CHECK-NEXT: test.cpp:16:0
-CHECK: {{^foo_cdecl$}}
-CHECK: {{^foo_stdcall$}}
-CHECK: {{^foo_fastcall$}}
-CHECK: {{^foo_vectorcall$}}
-CHECK: NS::Foo::bar(void)
-CHECK-NEXT: test.cpp:6:0
-
-CHECK-NO-DEMANGLE: ?foo@@YAXXZ
-CHECK-NO-DEMANGLE-NEXT: test.cpp:10
-CHECK-NO-DEMANGLE: private_symbol
-CHECK-NO-DEMANGLE-NEXT: test.cpp:13
-CHECK-NO-DEMANGLE: _main
-CHECK-NO-DEMANGLE-NEXT: test.cpp:16
-CHECK-NO-DEMANGLE: _foo_cdecl
-CHECK-NO-DEMANGLE: _foo_stdcall@0
-CHECK-NO-DEMANGLE: @foo_fastcall@0
-CHECK-NO-DEMANGLE: foo_vectorcall@@0
-CHECK-NO-DEMANGLE: ?bar@Foo@NS@@QAEXXZ
-CHECK-NO-DEMANGLE-NEXT: test.cpp:6
diff --git a/llvm/test/tools/llvm-symbolizer/pdb/pdb.test b/llvm/test/tools/llvm-symbolizer/pdb/pdb.test
index 062c0409895f8..df0e2320268d4 100644
--- a/llvm/test/tools/llvm-symbolizer/pdb/pdb.test
+++ b/llvm/test/tools/llvm-symbolizer/pdb/pdb.test
@@ -11,6 +11,12 @@ RUN:    | FileCheck %s
 RUN: llvm-symbolizer -obj="%p/Inputs/test.exe" -demangle=false < %t.input \
 RUN:    | FileCheck %s --check-prefix=CHECK-NO-DEMANGLE
 
+Test with native pdb reader.
+RUN: llvm-symbolizer -use-native-pdb-reader -obj="%p/Inputs/test.exe" < %t.input \
+RUN:    | FileCheck %s
+RUN: llvm-symbolizer -use-native-pdb-reader -obj="%p/Inputs/test.exe" -demangle=false < %t.input \
+RUN:    | FileCheck %s --check-prefix=CHECK-NO-DEMANGLE
+
 Subtract ImageBase from all the offsets and run the test again with
 --relative-address.
 
@@ -18,6 +24,10 @@ RUN: %python -c 'import sys;print("\n".join([hex(int(x, 16) - 0x400000) for x in
 RUN:    | llvm-symbolizer -obj="%p/Inputs/test.exe" -demangle=false --relative-address \
 RUN:    | FileCheck %s --check-prefix=CHECK-NO-DEMANGLE
 
+RUN: %python -c 'import sys;print("\n".join([hex(int(x, 16) - 0x400000) for x in sys.stdin]))' < %t.input \
+RUN:    | llvm-symbolizer -use-native-pdb-reader -obj="%p/Inputs/test.exe" -demangle=false --relative-address \
+RUN:    | FileCheck %s --check-prefix=CHECK-NO-DEMANGLE
+
 CHECK: foo(void)
 CHECK-NEXT: test.cpp:10
 CHECK: {{^private_symbol$}}
diff --git a/llvm/test/tools/obj2yaml/ELF/note-section.yaml b/llvm/test/tools/obj2yaml/ELF/note-section.yaml
new file mode 100644
index 0000000000000..c6e03145ed645
--- /dev/null
+++ b/llvm/test/tools/obj2yaml/ELF/note-section.yaml
@@ -0,0 +1,98 @@
+## Check how obj2yaml dumps SHT_NOTE sections.
+
+## We can dump the Name, Desc and Type fields when the note section is valid.
+
+# RUN: yaml2obj --docnum=1 %s -o %t1
+# RUN: obj2yaml %t1 | FileCheck %s --check-prefix=VALID
+
+# VALID:      - Name: .note.foo
+# VALID-NEXT:   Type: SHT_NOTE
+# VALID-NEXT:   Notes:
+# VALID-NEXT:    - Name: ''
+# VALID-NEXT:      Desc: ''
+# VALID-NEXT:      Type: 0x00
+# VALID-NEXT: - Name: .note.bar
+# VALID-NEXT:   Type: SHT_NOTE
+# VALID-NEXT:   Notes:
+# VALID-NEXT:     - Name: ABC
+# VALID-NEXT:       Desc: '001122'
+# VALID-NEXT:       Type: 0xAABBCCDD
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_X86_64
+Sections:
+  - Name: .note.foo
+    Type: SHT_NOTE
+    Size: 12
+  - Name: .note.bar
+    Type: SHT_NOTE
+    Notes:
+      - Name: ABC
+        Desc: '001122'
+        Type: 0xAABBCCDD
+
+## Check we dump hex digit pairs if the note section is invalid.
+
+# RUN: yaml2obj --docnum=2 %s -o %t2
+# RUN: obj2yaml %t2 | FileCheck %s --check-prefix=INVALID
+
+# INVALID:      - Name:    .note.tooShortNote
+# INVALID-NEXT:   Type:    SHT_NOTE
+# INVALID-NEXT:   Content: '0000000000000000000000'
+# INVALID-NEXT: - Name:    .note.tooLongNameSize
+# INVALID-NEXT:   Type:    SHT_NOTE
+# INVALID-NEXT:   Content: 0100000000000000FF00000000
+# INVALID-NEXT: - Name:    .note.tooLongDescSize
+# INVALID-NEXT:   Type:    SHT_NOTE
+# INVALID-NEXT:   Content: 0000000001000000FF00000000
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_X86_64
+Sections:
+## Content is less than 12 bytes in size.
+## (12 is the size of n_namesz, n_descsz and n_type 4-bytes fields that must always present).
+  - Name: .note.tooShortNote
+    Type: SHT_NOTE
+    Size: 11
+## We can't dump the Name, Desc and Type fields when the
+## content is shorter than the computed size.
+##
+## Too long name size.
+  - Name: .note.tooLongNameSize
+    Type: SHT_NOTE
+    Content: "0100000000000000ff00000000"
+## Too long description size.
+  - Name: .note.tooLongDescSize
+    Type: SHT_NOTE
+    Content: "0000000001000000ff00000000"
+
+## Check we do not crash when dumping a description that has bytes
+## on its bounds that do not fit into signed byte.
+## This is a completely normal case, but we had a crash before.
+
+# RUN: yaml2obj --docnum=3 %s -o %t3
+# RUN: obj2yaml %t3 | FileCheck %s --check-prefix=HEX-DESC
+
+# HEX-DESC: Desc: FE0000FE
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2MSB
+  Type:    ET_DYN
+  Machine: EM_MIPS
+Sections:
+  - Name:  .note.tag
+    Type:  SHT_NOTE
+    Flags: [ SHF_ALLOC ]
+    Notes:
+      - Desc: 'FE0000FE'
+        Type: 0
diff --git a/llvm/test/tools/yaml2obj/ELF/DWARF/debug-info.yaml b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-info.yaml
index 177de3ee816ef..5cd2bca1b4296 100644
--- a/llvm/test/tools/yaml2obj/ELF/DWARF/debug-info.yaml
+++ b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-info.yaml
@@ -735,3 +735,110 @@ DWARF:
       AbbrOffset: 0x1234
       AddrSize:   8
       Entries:    []
+
+## k) Test that yaml2obj is able to emit a correct length for compilation units.
+
+# RUN: yaml2obj --docnum=13 %s -o %t13.o
+# RUN: llvm-readelf --hex-dump=.debug_info %t13.o | \
+# RUN:   FileCheck %s --check-prefix=INFER-LENGTH
+
+#      INFER-LENGTH: Hex dump of section '.debug_info':
+# INFER-LENGTH-NEXT: 0x00000000 37000000 04000000 00000801 00000000 7...............
+##                              ^-------                            unit_length (0x37)
+##                                       ^---                       version (2-byte)
+##                                           ^--------              debug_abbrev_offset (4-byte)
+##                                                    ^-            address_size (1-byte)
+##                                                      ^-          abbrev code (ULEB128) 0x01
+##                                                         ^------- Form: DW_FORM_strp (4-byte)
+# INFER-LENGTH-NEXT: 0x00000010 0c001600 00000000 00001e00 00002011 .............. .
+##                              ^---                                Form: DW_FORM_data2 (2-byte)
+##                                  ^--------                       Form: DW_FORM_strp (4-byte)
+##                                           ^--------              Form: DW_FORM_sec_offset (4-byte)
+##                                                    ^--------     Form: DW_FORM_strp (4-byte)
+##                                                             ^--- Form: DW_FORM_addr (8-byte)
+# INFER-LENGTH-NEXT: 0x00000020 00000000 00003300 00000220 11000000 ......3.... ....
+##                              -------------
+##                                           ^--------              Form: DW_FORM_data4 (4-byte)
+##                                                    ^-            abbrev code (ULEB128) 0x02
+##                                                      ^---------- Form: DW_FORM_addr (8-byte)
+# INFER-LENGTH-NEXT: 0x00000030 00000006 00000038 000000            .......8...
+##                              ------
+##                                    ^--------                     Form: DW_FORM_data4 (4-byte)
+##                                             ^--------            Form: DW_FORM_strp (4-byte)
+
+## The handwritten DIEs should look like:
+
+## 0x0000000b: DW_TAG_compile_unit [1] *
+##               DW_AT_producer [DW_FORM_strp]        ( .debug_str[0x00000000] = "clang version 10.0.0 ")
+##               DW_AT_language [DW_FORM_data2]       (DW_LANG_C99)
+##               DW_AT_name [DW_FORM_strp]            ( .debug_str[0x00000016] = "hello.c")
+##               DW_AT_stmt_list [DW_FORM_sec_offset] (0x00000000)
+##               DW_AT_comp_dir [DW_FORM_strp]        ( .debug_str[0x0000001e] = "/home/v/x/llvm/playground")
+##               DW_AT_low_pc [DW_FORM_addr]          (0x0000000000001120)
+##               DW_AT_high_pc [DW_FORM_data4]        (0x00000033)
+## 
+## 0x0000002a:   DW_TAG_subprogram [2]  
+##                 DW_AT_low_pc [DW_FORM_addr]        (0x0000000000001120)
+##                 DW_AT_high_pc [DW_FORM_data4]      (0x00000006)
+##                 DW_AT_name [DW_FORM_strp]          ( .debug_str[0x00000038] = "main")
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_str:
+    - "clang version 10.0.0 "
+    - "hello.c"
+    - "/home/v/x/llvm/playground"
+    - "main"
+  debug_abbrev:
+    - Code:     1
+      Tag:      DW_TAG_compile_unit
+      Children: DW_CHILDREN_yes
+      Attributes:
+        - Attribute: DW_AT_producer
+          Form:      DW_FORM_strp
+        - Attribute: DW_AT_language
+          Form:      DW_FORM_data2
+        - Attribute: DW_AT_name
+          Form:      DW_FORM_strp
+        - Attribute: DW_AT_stmt_list
+          Form:      DW_FORM_sec_offset
+        - Attribute: DW_AT_comp_dir
+          Form:      DW_FORM_strp
+        - Attribute: DW_AT_low_pc
+          Form:      DW_FORM_addr
+        - Attribute: DW_AT_high_pc
+          Form:      DW_FORM_data4
+    - Code: 2
+      Tag:  DW_TAG_subprogram
+      Children: DW_CHILDREN_no
+      Attributes:
+        - Attribute: DW_AT_low_pc
+          Form:      DW_FORM_addr
+        - Attribute: DW_AT_high_pc
+          Form:      DW_FORM_data4
+        - Attribute: DW_AT_name
+          Form:      DW_FORM_strp
+  debug_info:
+    - Version:    4
+      AbbrOffset: 0x00
+      AddrSize:   0x08
+      Entries:
+        - AbbrCode: 1
+          Values:
+            - Value: 0x00   ## DW_AT_producer [DW_FORM_strp]
+            - Value: 0x0c   ## DW_LANG_C99, DW_AT_language [DW_FORM_data2]
+            - Value: 0x16   ## DW_AT_name [DW_FORM_strp]
+            - Value: 0x00   ## DW_AT_stmt_list [DW_FORM_sec_offset]
+            - Value: 0x1e   ## DW_AT_comp_dir [DW_FORM_strp]
+            - Value: 0x1120 ## DW_AT_low_pc [DW_FORM_addr]
+            - Value: 0x33   ## DW_AT_high_pc [DW_FORM_data4]
+        - AbbrCode: 2
+          Values:
+            - Value: 0x1120 ## DW_AT_low_pc [DW_FORM_addr]
+            - Value: 0x06   ## DW_AT_high_pc [DW_FORM_data4]
+            - Value: 0x38   ## DW_AT_name [DW_FORM_strp]
diff --git a/llvm/test/tools/yaml2obj/ELF/DWARF/debug-rnglists.yaml b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-rnglists.yaml
new file mode 100644
index 0000000000000..80dc9b9fdeb35
--- /dev/null
+++ b/llvm/test/tools/yaml2obj/ELF/DWARF/debug-rnglists.yaml
@@ -0,0 +1,610 @@
+## Test that yaml2obj emits a .debug_rnglists section when requested.
+
+## a) Generate and verify a little endian DWARF32 .debug_rnglists section in a 64-bit object file.
+
+# RUN: yaml2obj --docnum=1 -DENDIAN=ELFDATA2LSB %s -o %t1.le.dwarf32.o
+# RUN: llvm-readobj --sections --section-data %t1.le.dwarf32.o | \
+# RUN:   FileCheck -DSIZE=99 -DADDRALIGN=1 %s --check-prefixes=SHDR,DWARF32-LE
+
+#            SHDR: Index: 1
+#       SHDR-NEXT: Name: .debug_rnglists (1)
+#       SHDR-NEXT: Type: SHT_PROGBITS (0x1)
+#       SHDR-NEXT: Flags [ (0x0)
+#       SHDR-NEXT: ]
+#       SHDR-NEXT: Address: 0x0
+#       SHDR-NEXT: Offset: 0x40
+#       SHDR-NEXT: Size: [[SIZE]]
+#       SHDR-NEXT: Link: 0
+#       SHDR-NEXT: Info: 0
+#       SHDR-NEXT: AddressAlignment: [[ADDRALIGN]]
+#       SHDR-NEXT: EntrySize: 0
+# DWARF32-LE-NEXT: SectionData (
+# DWARF32-LE-NEXT:   0000: 24000000 05000800 02000000 08000000  |$...............|
+##                         ^-------                             unit_length (4-byte)
+##                                  ^---                        version (2-byte)
+##                                      ^-                      address_size (1-byte)
+##                                        ^-                    segment_selector_size (1-byte)
+##                                           ^-------           offset_entry_count (4-byte)
+##                                                    ^-------  offsets[0] (4-byte)
+# DWARF32-LE-NEXT:   0010: 11000000 01B42402 B424B424 0003B424  |......$..$.$...$|
+##                         ^-------                             offsets[1] (4-byte)
+##                                  ^-                          DW_RLE_base_addressx (1-byte)
+##                                    ^---                      operands[0] (ULEB128) 0x1234
+##                                        ^-                    DW_RLE_startx_endx (1-byte)
+##                                           ^---               operands[0] (ULEB128) 0x1234
+##                                               ^---           operands[1] (ULEB128) 0x1234
+##                                                    ^-        DW_RLE_end_of_list (1-byte)
+##                                                      ^-      DW_RLE_startx_length (1-byte)
+##                                                        ^---  operands[0] (ULEB128) 0x1234
+# DWARF32-LE-NEXT:   0020: B42404B4 24B42400 37000000 05000800  |.$..$.$.7.......|
+##                         ^---                                 operands[1] (ULEB128) 0x1234
+##                             ^-                               DW_RLE_offset_pair (1-byte)
+##                               ^----                          operands[0] (ULEB128) 0x1234
+##                                    ^---                      operands[1] (ULEB128) 0x1234
+##                                        ^-                    DW_RLE_end_of_list (1-byte)
+##                                           ^-------           unit_length (4-byte)
+##                                                    ^---      version (2-byte)
+##                                                        ^-    address_size (1-byte)
+##                                                          ^-  segment_selector_size (1-byte)
+# DWARF32-LE-NEXT:   0030: 02000000 08000000 23000000 05341200  |........#....4..|
+##                         ^-------                             offset_entry_count (4-byte)
+##                                  ^-------                    offsets[0] (4-byte)
+##                                           ^-------           offsets[1] (4-byte)
+##                                                    ^-        DW_RLE_base_address (1-byte)
+##                                                      ^-----  operands[0] (8-byte)
+# DWARF32-LE-NEXT:   0040: 00000000 00063412 00000000 00003412  |......4.......4.|
+##                         -----------
+##                                    ^-                        DW_RLE_start_end (1-byte)
+##                                      ^-----------------      operands[0] (8-byte)
+##                                                        ^---  operands[1] (8-byte)
+# DWARF32-LE-NEXT:   0050: 00000000 00000007 34120000 00000000  |........4.......|
+##                         -------------
+##                                      ^-                      DW_RLE_end_of_list (1-byte)
+##                                        ^-                    DW_RLE_start_length (1-byte)
+##                                           ^----------------  operands[0] (8-byte)
+# DWARF32-LE-NEXT:   0060: B42400                               |.$.|
+##                         ^---                                 operands[1] (ULEB128) 0x1234
+##                             ^-                               DW_RLE_end_of_list (1-byte)
+# DWARF32-LE-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    [[ENDIAN]]
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_rnglists:
+    - Lists:
+        - Entries:
+            - Operator: DW_RLE_base_addressx
+              Values:   [ 0x1234 ]
+            - Operator: DW_RLE_startx_endx
+              Values:   [ 0x1234, 0x1234 ]
+            - Operator: DW_RLE_end_of_list
+        - Entries:
+            - Operator: DW_RLE_startx_length
+              Values:   [ 0x1234, 0x1234 ]
+            - Operator: DW_RLE_offset_pair
+              Values:   [ 0x1234, 0x1234 ]
+            - Operator: DW_RLE_end_of_list
+    - Lists:
+        - Entries:
+            - Operator: DW_RLE_base_address
+              Values:   [ 0x1234 ]
+            - Operator: DW_RLE_start_end
+              Values:   [ 0x1234, 0x1234 ]
+            - Operator: DW_RLE_end_of_list
+        - Entries:
+            - Operator: DW_RLE_start_length
+              Values:   [ 0x1234, 0x1234 ]
+            - Operator: DW_RLE_end_of_list
+
+## b) Generate and verify a big endian DWARF32 .debug_rnglists section in a 64-bit object file.
+
+# RUN: yaml2obj --docnum=1 -DENDIAN=ELFDATA2MSB %s -o %t1.be.dwarf32.o
+# RUN: llvm-readobj --sections --section-data %t1.be.dwarf32.o | \
+# RUN:   FileCheck -DSIZE=99 -DADDRALIGN=1 %s --check-prefixes=SHDR,DWARF32-BE
+
+# DWARF32-BE-NEXT: SectionData (
+# DWARF32-BE-NEXT:   0000: 00000024 00050800 00000002 00000008  |...$............|
+##                         ^-------                             unit_length (4-byte)
+##                                  ^---                        version (2-byte)
+##                                      ^-                      address_size (1-byte)
+##                                        ^-                    segment_selector_size (1-byte)
+##                                           ^-------           offset_entry_count (4-byte)
+##                                                    ^-------  offsets[0] (4-byte)
+# DWARF32-BE-NEXT:   0010: 00000011 01B42402 B424B424 0003B424  |......$..$.$...$|
+##                         ^-------                             offsets[1] (4-byte)
+##                                  ^-                          DW_RLE_base_addressx (1-byte)
+##                                    ^---                      operands[0] (ULEB128) 0x1234
+##                                        ^-                    DW_RLE_startx_endx (1-byte)
+##                                           ^---               operands[0] (ULEB128) 0x1234
+##                                               ^---           operands[1] (ULEB128) 0x1234
+##                                                    ^-        DW_RLE_end_of_list (1-byte)
+##                                                      ^-      DW_RLE_startx_length (1-byte)
+##                                                        ^---  operands[0] (ULEB128) 0x1234
+# DWARF32-BE-NEXT:   0020: B42404B4 24B42400 00000037 00050800  |.$..$.$....7....|
+##                         ^---                                 operands[1] (ULEB128) 0x1234
+##                             ^-                               DW_RLE_offset_pair (1-byte)
+##                               ^----                          operands[0] (ULEB128) 0x1234
+##                                    ^---                      operands[1] (ULEB128) 0x1234
+##                                        ^-                    DW_RLE_end_of_list (1-byte)
+##                                           ^-------           unit_length (4-byte)
+##                                                    ^---      version (2-byte)
+##                                                        ^-    address_size (1-byte)
+##                                                          ^-  segment_selector_size (1-byte)
+# DWARF32-BE-NEXT:   0030: 00000002 00000008 00000023 05000000  |...........#....|
+##                         ^-------                             offset_entry_count (4-byte)
+##                                  ^-------                    offsets[0] (4-byte)
+##                                           ^-------           offsets[1] (4-byte)
+##                                                    ^-        DW_RLE_base_address (1-byte)
+##                                                      ^-----  operands[0] (8-byte)
+# DWARF32-BE-NEXT:   0040: 00000012 34060000 00000000 12340000  |....4........4..|
+##                         -----------
+##                                    ^-                        DW_RLE_start_end (1-byte)
+##                                      ^-----------------      operands[0] (8-byte)
+##                                                        ^---  operands[1] (8-byte)
+# DWARF32-BE-NEXT:   0050: 00000000 12340007 00000000 00001234  |.....4.........4|
+##                         -------------
+##                                      ^-                      DW_RLE_end_of_list (1-byte)
+##                                        ^-                    DW_RLE_start_length (1-byte)
+##                                           ^----------------  operands[0] (8-byte)
+# DWARF32-BE-NEXT:   0060: B42400                               |.$.|
+##                         ^---                                 operands[1] (ULEB128) 0x1234
+##                             ^-                               DW_RLE_end_of_list (1-byte)
+# DWARF32-BE-NEXT: )
+
+## c) Generate and verify a little endian DWARF64 .debug_rnglists section in a 64-bit object file.
+
+# RUN: yaml2obj --docnum=2 -DENDIAN=ELFDATA2LSB %s -o %t2.le.dwarf64.o
+# RUN: llvm-readobj --sections --section-data %t2.le.dwarf64.o | \
+# RUN:   FileCheck -DSIZE=38 -DADDRALIGN=1 %s --check-prefixes=SHDR,DWARF64-LE
+
+# DWARF64-LE-NEXT: SectionData (
+# DWARF64-LE-NEXT:   0000: FFFFFFFF 1A000000 00000000 05000800  |................|
+##                         ^-------------------------           unit_length (12-byte)
+##                                                    ^---      version (2-byte)
+##                                                        ^-    address_size (1-byte)
+##                                                          ^-  segment_selector_size (1-byte)
+# DWARF64-LE-NEXT:   0010: 02000000 10000000 00000000 11000000  |................|
+##                         ^-------                             offset_entry_count (4-byte)
+##                                  ^----------------           offsets[0] (8-byte)
+##                                                    ^-------  offsets[1] (8-byte)
+# DWARF64-LE-NEXT:   0020: 00000000 0000                        |......|
+##                         --------
+##                                  ^-                          DW_RLE_end_of_list (1-byte)
+##                                    ^-                        DW_RLE_end_of_list (1-byte)
+# DWARF64-LE-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    [[ENDIAN]]
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_rnglists:
+    - Format:  DWARF64
+      Lists:
+        - Entries:
+            - Operator: DW_RLE_end_of_list
+        - Entries:
+            - Operator: DW_RLE_end_of_list
+
+## d) Generate and verify a big endian DWARF64 .debug_rnglists section in a 64-bit object file.
+
+# RUN: yaml2obj --docnum=2 -DENDIAN=ELFDATA2MSB %s -o %t2.be.dwarf64.o
+# RUN: llvm-readobj --sections --section-data %t2.be.dwarf64.o | \
+# RUN:   FileCheck -DSIZE=38 -DADDRALIGN=1 %s --check-prefixes=SHDR,DWARF64-BE
+
+# DWARF64-BE-NEXT: SectionData (
+# DWARF64-BE-NEXT:   0000: FFFFFFFF 00000000 0000001A 00050800  |................|
+##                         ^-------------------------           unit_length (12-byte)
+##                                                    ^---      version (2-byte)
+##                                                        ^-    address_size (1-byte)
+##                                                          ^-  segment_selector_size (1-byte)
+# DWARF64-BE-NEXT:   0010: 00000002 00000000 00000010 00000000  |................|
+##                         ^-------                             offset_entry_count (4-byte)
+##                                  ^----------------           offsets[0] (8-byte)
+##                                                    ^-------  offsets[1] (8-byte)
+# DWARF64-BE-NEXT:   0020: 00000011 0000                        |......|
+##                         --------
+##                                  ^-                          DW_RLE_end_of_list (1-byte)
+##                                    ^-                        DW_RLE_end_of_list (1-byte)
+# DWARF64-BE-NEXT: )
+
+## e) Test that the length, version, segment_selector_size, address_size, offset_entry_count
+## and offsets fields can be specified manually.
+
+# RUN: yaml2obj --docnum=3 %s -o %t3.o
+# RUN: llvm-readelf --hex-dump=.debug_rnglists %t3.o | \
+# RUN:   FileCheck %s --check-prefix=OVERWRITE
+
+#      OVERWRITE: Hex dump of section '.debug_rnglists':
+# OVERWRITE-NEXT: 0x00000000 34120000 06000303 04000000 01000000 4...............
+##                           ^-------                            unit_length (4-byte)
+##                                    ^---                       version (2-byte)
+##                                        ^-                     address_size (1-byte)
+##                                          ^-                   segment_selector_size (1-byte)
+##                                             ^-------          offset_entry_count (4-byte)
+##                                                      ^------- offsets[0] (4-byte)
+# OVERWRITE-NEXT: 0x00000010 0000                                ..
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_rnglists:
+    - Length:              0x1234
+      Version:             6
+      AddressSize:         3
+      SegmentSelectorSize: 3
+      OffsetEntryCount:    4
+      Offsets:             [ 0x01 ]
+      Lists:
+        - Entries:
+            - Operator: DW_RLE_end_of_list
+        - Entries:
+            - Operator: DW_RLE_end_of_list
+
+## f) Test that the default value of the address_size field in a 32-bit object file is 4.
+
+# RUN: yaml2obj --docnum=4 %s -o %t4.o
+# RUN: llvm-readelf --hex-dump=.debug_rnglists %t4.o | \
+# RUN:   FileCheck %s --check-prefix=ADDRSIZE32
+
+#      ADDRSIZE32: Hex dump of section '.debug_rnglists':
+# ADDRSIZE32-NEXT: 0x00000000 22000000 05000400 01000000 04000000 "...............
+##                                         ^-                     address_size (1-byte)
+# ADDRSIZE32-NEXT: 0x00000010 05341200 00063412 00002143 00000734 .4....4...!C...4
+##                            ^-                                  DW_RLE_base_address (1-byte)
+##                              ^--------                         operands[0] (4-byte)
+##                                       ^-                       DW_RLE_start_end (1-byte)
+##                                         ^--------              operands[0] (4-byte)
+##                                                  ^--------     operands[1] (4-byte)
+##                                                           ^-   DW_RLE_start_length (1-byte)
+##                                                             ^- operands[0] (4-byte)
+# ADDRSIZE32-NEXT: 0x00000020 120000a1 8601                       ......
+##                            ------
+##                                  ^------                       operands[1] (ULEB128) 0x4321
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS32
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_rnglists:
+    - Lists:
+        - Entries:
+            - Operator: DW_RLE_base_address
+              Values:   [ 0x1234 ]
+            - Operator: DW_RLE_start_end
+              Values:   [ 0x1234, 0x4321 ]
+            - Operator: DW_RLE_start_length
+              Values:   [ 0x1234, 0x4321 ]
+
+## g) Test that the address_size field can be specified manually and the size of
+## corresponding operands will be changed accordingly.
+
+# RUN: yaml2obj --docnum=5 %s -o %t5.o
+# RUN: llvm-readelf --hex-dump=.debug_rnglists %t5.o | \
+# RUN:   FileCheck %s --check-prefix=ADDRSIZE
+
+#      ADDRSIZE: Hex dump of section '.debug_rnglists':
+# ADDRSIZE-NEXT: 0x00000000 22000000 05000400 01000000 04000000 "...............
+##                                       ^-                     address_size 0x04
+# ADDRSIZE-NEXT: 0x00000010 05341200 00063412 00002143 00000734 .4....4...!C...4
+##                          ^-                                  DW_RLE_base_address
+##                            ^--------                         operands[0] (4-byte)
+##                                     ^-                       DW_RLE_start_end
+##                                       ^--------              operands[0] (4-byte)
+##                                                ^--------     operands[1] (4-byte)
+##                                                         ^-   DW_RLE_start_length
+##                                                           ^- operands[0] (4-byte)
+# ADDRSIZE-NEXT: 0x00000020 120000b4 2400                       ....$.
+##                          ------
+##                                ^----                         operands[1] (ULEB128)
+##                                     ^-                       DW_RLE_end_of_list
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_rnglists:
+    - AddressSize: 4
+      Lists:
+        - Entries:
+            - Operator: DW_RLE_base_address
+              Values:   [ 0x1234 ]
+            - Operator: DW_RLE_start_end
+              Values:   [ 0x1234, 0x4321 ]
+            - Operator: DW_RLE_start_length
+              Values:   [ 0x1234, 0x1234 ]
+            - Operator: DW_RLE_end_of_list
+
+## h) Test that yaml2obj emits an error message if we try to assign an invalid value to
+## 'AddressSize' when there is an entry whose operands contain address.
+
+# RUN: not yaml2obj -DOPERATOR=base_address -DVALUES=[0x01] --docnum=6 %s 2>&1 | \
+# RUN:   FileCheck -DOPERATOR=base_address %s --check-prefix=INVALID-ADDRSIZE
+
+# RUN: not yaml2obj -DOPERATOR=start_end -DVALUES=[0x01,0x02] --docnum=6 %s 2>&1 | \
+# RUN:   FileCheck -DOPERATOR=start_end %s --check-prefix=INVALID-ADDRSIZE
+
+# RUN: not yaml2obj -DOPERATOR=start_length -DVALUES=[0x01,0x02] --docnum=6 %s 2>&1 | \
+# RUN:   FileCheck -DOPERATOR=start_length %s --check-prefix=INVALID-ADDRSIZE
+
+# INVALID-ADDRSIZE: yaml2obj: error: unable to write address for the operator DW_RLE_[[OPERATOR]]: invalid integer write size: 3
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_rnglists:
+    - AddressSize: 3
+      Lists:
+        - Entries:
+            - Operator: DW_RLE_[[OPERATOR]]
+              Values:   [[VALUES]]
+
+## i) Test that an invalid address_size can be used when there are no address-using operators.
+
+# RUN: yaml2obj --docnum=7 %s -o %t7.o
+# RUN: llvm-readelf --hex-dump=.debug_rnglists %t7.o | \
+# RUN:   FileCheck %s --check-prefix=ADDRSIZE-NOERROR
+
+#      ADDRSIZE-NOERROR: Hex dump of section '.debug_rnglists':
+# ADDRSIZE-NOERROR-NEXT: 0x00000000 0e000000 05000300 01000000 04000000 ................
+##                                               ^-                     address_size (1-byte) 0x03
+# ADDRSIZE-NOERROR-NEXT: 0x00000010 0101                                ..
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_rnglists:
+    - AddressSize: 3
+      Lists:
+        - Entries:
+            - Operator: DW_RLE_base_addressx
+              Values:   [ 0x01 ]
+
+## j) Test that yaml2obj emits an error message if we specify invalid numbers of operands
+## for an operator.
+
+# RUN: not yaml2obj -DOPERATOR=end_of_list -DVALUES=[0x01] --docnum=8 %s 2>&1 | \
+# RUN:   FileCheck -DOPERATOR=end_of_list -DACTUAL=1 -DEXPECTED=0 %s --check-prefix=INVALID-OPERANDS
+
+# RUN: not yaml2obj -DOPERATOR=base_addressx -DVALUES=[] --docnum=8 %s 2>&1 | \
+# RUN:   FileCheck -DOPERATOR=base_addressx -DACTUAL=0 -DEXPECTED=1 %s --check-prefix=INVALID-OPERANDS
+
+# RUN: not yaml2obj -DOPERATOR=startx_endx -DVALUES=[0x01] --docnum=8 %s 2>&1 | \
+# RUN:   FileCheck -DOPERATOR=startx_endx -DACTUAL=1 -DEXPECTED=2 %s --check-prefix=INVALID-OPERANDS
+
+# RUN: not yaml2obj -DOPERATOR=startx_length -DVALUES=[0x01] --docnum=8 %s 2>&1 | \
+# RUN:   FileCheck -DOPERATOR=startx_length -DACTUAL=1 -DEXPECTED=2 %s --check-prefix=INVALID-OPERANDS
+
+# RUN: not yaml2obj -DOPERATOR=offset_pair -DVALUES=[] --docnum=8 %s 2>&1 | \
+# RUN:   FileCheck -DOPERATOR=offset_pair -DACTUAL=0 -DEXPECTED=2 %s --check-prefix=INVALID-OPERANDS
+
+# RUN: not yaml2obj -DOPERATOR=base_address -DVALUES=[0x01,0x02] --docnum=8 %s 2>&1 | \
+# RUN:   FileCheck -DOPERATOR=base_address -DACTUAL=2 -DEXPECTED=1 %s --check-prefix=INVALID-OPERANDS
+
+# RUN: not yaml2obj -DOPERATOR=start_end -DVALUES=[0x01,0x02,0x03] --docnum=8 %s 2>&1 | \
+# RUN:   FileCheck -DOPERATOR=start_end -DACTUAL=3 -DEXPECTED=2 %s --check-prefix=INVALID-OPERANDS
+
+# RUN: not yaml2obj -DOPERATOR=start_length -DVALUES=[0x01] --docnum=8 %s 2>&1 | \
+# RUN:   FileCheck -DOPERATOR=start_length -DACTUAL=1 -DEXPECTED=2 %s --check-prefix=INVALID-OPERANDS
+
+# INVALID-OPERANDS: yaml2obj: error: invalid number ([[ACTUAL]]) of operands for the operator: DW_RLE_[[OPERATOR]], [[EXPECTED]] expected
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_rnglists:
+    - AddressSize: 3
+      Lists:
+        - Entries:
+            - Operator: DW_RLE_[[OPERATOR]]
+              Values:   [[VALUES]]
+
+## k) Test that an empty list is allowed for a range list table.
+
+# RUN: yaml2obj --docnum=9 %s -o %t9.o
+# RUN: llvm-readelf --hex-dump=.debug_rnglists %t9.o | \
+# RUN:   FileCheck %s --check-prefix=EMPTY-LIST
+
+#      EMPTY-LIST: Hex dump of section '.debug_rnglists':
+# EMPTY-LIST-NEXT: 0x00000000 08000000 05000800 00000000          ............
+##                            ^-------                            unit_length (4-byte)
+##                                     ^---                       version (2-byte)
+##                                         ^---                   address_size (1-byte)
+##                                              ^-------          offset_entry_count (4-byte)
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_rnglists:
+    - Lists: []
+
+## l) Generate the .debug_rnglists section from raw section content.
+
+# RUN: yaml2obj --docnum=10 %s -o %t10.o
+# RUN: llvm-readobj --sections --section-data %t10.o | \
+# RUN:   FileCheck %s -DSIZE=3 -DADDRALIGN=0 --check-prefixes=SHDR,ARBITRARY-CONTENT
+
+#      ARBITRARY-CONTENT: SectionData (
+# ARBITRARY-CONTENT-NEXT:   0000: 112233
+# ARBITRARY-CONTENT-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    .debug_rnglists
+    Type:    SHT_PROGBITS
+    Content: "112233"
+
+## m) Generate the .debug_rnglists section when the "Size" is specified.
+
+# RUN: yaml2obj --docnum=11 %s -o %t11.o
+# RUN: llvm-readelf --hex-dump=.debug_rnglists %t11.o | \
+# RUN:   FileCheck %s --check-prefix=SIZE
+
+#       SIZE: Hex dump of section '.debug_rnglists':
+#  SIZE-NEXT: 0x00000000 00000000 00000000 00000000 00000000 ................
+# SIZE-EMPTY:
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .debug_rnglists
+    Type: SHT_PROGBITS
+    Size: 0x10
+
+## n) Test that yaml2obj emits an error message when both the "Size" and the
+## "debug_rnglists" entry are specified at the same time.
+
+# RUN: not yaml2obj --docnum=12 %s 2>&1 | FileCheck %s --check-prefix=ERROR
+
+# ERROR: yaml2obj: error: cannot specify section '.debug_rnglists' contents in the 'DWARF' entry and the 'Content' or 'Size' in the 'Sections' entry at the same time
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .debug_rnglists
+    Type: SHT_PROGBITS
+    Size: 0x10
+DWARF:
+  debug_rnglists:
+    - Lists: []
+
+## o) Test that yaml2obj emits an error message when both the "Content" and the
+## "debug_rnglists" entry are specified at the same time.
+
+# RUN: not yaml2obj --docnum=13 %s 2>&1 | FileCheck %s --check-prefix=ERROR
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    .debug_rnglists
+    Type:    SHT_PROGBITS
+    Content: "00"
+DWARF:
+  debug_rnglists:
+    - Lists: []
+
+## p) Test that all the properties can be overridden by the section header when
+## the "debug_rnglists" entry doesn't exist.
+
+# RUN: yaml2obj --docnum=14 %s -o %t14.o
+# RUN: llvm-readelf --sections %t14.o | FileCheck %s --check-prefix=OVERRIDDEN
+
+#      OVERRIDDEN: [Nr] Name            Type   Address          Off    Size   ES Flg Lk Inf Al
+#      OVERRIDDEN: [ 1] .debug_rnglists STRTAB 0000000000002020 000050 00000c 01   A  2   1  2
+# OVERRIDDEN-NEXT: [ 2] .sec            STRTAB 0000000000000000 00005c 000000 00      0   0  0
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:         .debug_rnglists
+    Type:         SHT_STRTAB  ## SHT_PROGBITS by default.
+    Flags:        [SHF_ALLOC] ## 0 by default.
+    Link:         .sec        ## 0 by default.
+    EntSize:      1           ## 0 by default.
+    Info:         1           ## 0 by default.
+    AddressAlign: 2           ## 0 by default.
+    Address:      0x2020      ## 0x00 by default.
+    Offset:       0x50        ## 0x40 for the first section.
+    Size:         0x0c        ## Set the "Size" so that we can reuse the check tag "OVERRIDDEN".
+  - Name:         .sec        ## Linked by .debug_rnglists.
+    Type:         SHT_STRTAB
+
+## q) Test that all the properties can be overridden by the section header when
+## the "debug_rnglists" entry exists.
+
+# RUN: yaml2obj --docnum=15 %s -o %t15.o
+# RUN: llvm-readelf --sections %t15.o | FileCheck %s --check-prefix=OVERRIDDEN
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:         .debug_rnglists
+    Type:         SHT_STRTAB  ## SHT_PROGBITS by default.
+    Flags:        [SHF_ALLOC] ## 0 by default.
+    Link:         .sec        ## 0 by default.
+    EntSize:      1           ## 0 by default.
+    Info:         1           ## 0 by default.
+    AddressAlign: 2           ## 1 by default.
+    Address:      0x2020      ## 0x00 by default.
+    Offset:       0x50        ## 0x40 for the first section.
+  - Name:         .sec        ## Linked by .debug_rnglists.
+    Type:         SHT_STRTAB
+DWARF:
+  debug_rnglists:
+    - Lists: []
+
+## r) Test that the .debug_rnglists section header is emitted if the "debug_rnglists"
+## entry is empty.
+
+# RUN: yaml2obj --docnum=16 %s -o %t16.o
+# RUN: llvm-readobj --sections --section-data %t16.o | \
+# RUN:   FileCheck -DSIZE=0 -DADDRALIGN=1 %s --check-prefixes=SHDR,EMPTY-CONTENT
+
+# EMPTY-CONTENT-NEXT: SectionData (
+# EMPTY-CONTENT-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+DWARF:
+  debug_rnglists: []
diff --git a/llvm/test/tools/yaml2obj/ELF/note-section.yaml b/llvm/test/tools/yaml2obj/ELF/note-section.yaml
new file mode 100644
index 0000000000000..4ea6fcaef8a41
--- /dev/null
+++ b/llvm/test/tools/yaml2obj/ELF/note-section.yaml
@@ -0,0 +1,317 @@
+## Check how yaml2obj produces SHT_NOTE sections.
+
+## Check we can describe SHT_NOTE using the "Notes" tag. We can define
+## notes using names, descriptions and types.
+## Check we produce a valid name size and description size fields.
+## Check we produce valid paddings.
+
+# RUN: yaml2obj --docnum=1 %s -o %t1
+# RUN: llvm-readobj --sections --section-data %t1 | FileCheck %s --check-prefix=NOTE
+
+# NOTE:      Section {
+# NOTE:        Index: 1
+# NOTE-NEXT:   Name: .note.foo (1)
+# NOTE-NEXT:   Type: SHT_NOTE (0x7)
+# NOTE-NEXT:   Flags [ (0x2)
+# NOTE-NEXT:     SHF_ALLOC (0x2)
+# NOTE-NEXT:   ]
+# NOTE-NEXT:   Address:
+# NOTE-NEXT:   Offset:
+# NOTE-NEXT:   Size: 36
+# NOTE-NEXT:   Link: 0
+# NOTE-NEXT:   Info: 0
+# NOTE-NEXT:   AddressAlignment: 0
+# NOTE-NEXT:   EntrySize: 0
+# NOTE-NEXT:   SectionData (
+## namesz == (0x03000000) == sizeof("AB") + NUL terminator.
+## descsz == (0x00000000) for an empty description.
+## Check we produce a valid 2 bytes zeroes padding after the Name.
+# NOTE-NEXT:     0000: 03000000 00000000 FF000000 41420000  |............AB..|
+## namesz == (0x04000000) == sizeof("ABC") + NUL terminator.
+## descsz == (0x06000000) == sizeof("123456").
+## Check we produce a valid (zero align to 4) 1 byte padding after the Name.
+## Check we produce a valid (zero align to 4) 2 bytes padding after the Desc.
+# NOTE-NEXT:     0010: 04000000 03000000 FE000000 41424300  |............ABC.|
+# NOTE-NEXT:     0020: 12345600                             |.4V.|
+# NOTE-NEXT:   )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_X86_64
+Sections:
+  - Name:  .note.foo
+    Type:  SHT_NOTE
+    Flags: [ SHF_ALLOC ]
+    Notes:
+      - Name: AB
+        Desc: ''
+        Type: 0xFF
+      - Name: ABC
+        Desc: '123456'
+        Type: 254
+
+## Check that for 32-bit little-endian case we produce the same section content.
+
+# RUN: yaml2obj --docnum=2 %s -o %t2
+# RUN: llvm-readobj --sections --section-data %t2 | FileCheck %s --check-prefix=NOTE
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS32
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_386
+Sections:
+  - Name:  .note.foo
+    Type:  SHT_NOTE
+    Flags: [ SHF_ALLOC ]
+    Notes:
+      - Name: AB
+        Desc: ''
+        Type: 0xFF
+      - Name: ABC
+        Desc: '123456'
+        Type: 254
+
+## Check big-endian 32/64 bit cases.
+## Check they produce the same content.
+
+# RUN: yaml2obj --docnum=3 %s -o %t3
+# RUN: llvm-readobj --sections --section-data %t3 | FileCheck %s --check-prefix=NOTE-BE
+# RUN: yaml2obj --docnum=4 %s -o %t4
+# RUN: llvm-readobj --sections --section-data %t4 | FileCheck %s --check-prefix=NOTE-BE
+
+# NOTE-BE:      Name: .note.foo
+# NOTE-BE:      SectionData (
+# NOTE-BE-NEXT:   0000: 00000004 00000003 000000FE 41424300  |
+# NOTE-BE-NEXT:   0010: 12345600                             |
+# NOTE-BE-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS32
+  Data:    ELFDATA2MSB
+  Type:    ET_REL
+  Machine: EM_MIPS
+Sections:
+  - Name: .note.foo
+    Type: SHT_NOTE
+    Notes:
+      - Name: ABC
+        Desc: '123456'
+        Type: 254
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2MSB
+  Type:    ET_REL
+  Machine: EM_MIPS
+Sections:
+  - Name: .note.foo
+    Type: SHT_NOTE
+    Notes:
+      - Name: ABC
+        Desc: '123456'
+        Type: 254
+
+## Check that 'Type' field is mandatory. 
+
+# RUN: not yaml2obj --docnum=5 %s 2>&1 | FileCheck %s --check-prefix=TYPE-REQ
+# TYPE-REQ: error: missing required key 'Type'
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_X86_64
+Sections:
+  - Name: .note.foo
+    Type: SHT_NOTE
+    Notes:
+      - Name: ''
+        Desc: ''
+
+## Check that neither `Name` nor `Desc` are mandatory fields.
+
+# RUN: yaml2obj --docnum=6 %s -o %t6
+# RUN: llvm-readobj --sections --section-data %t6 | FileCheck %s --check-prefix=NAME-DESC
+
+# NAME-DESC:      Name: .note.foo
+# NAME-DESC:      SectionData (
+# NAME-DESC-NEXT:   0000: 00000000 00000000 FF000000 |
+# NAME-DESC-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_X86_64
+Sections:
+  - Name: .note.foo
+    Type: SHT_NOTE
+    Notes:
+      - Type: 0xFF
+
+## Check we can use the "Content" tag to specify any data for SHT_NOTE sections.
+
+# RUN: yaml2obj --docnum=7 %s -o %t7
+# RUN: llvm-readobj --sections --section-data %t7 | FileCheck %s --check-prefix=CONTENT
+
+# CONTENT:      Name: .note.foo
+# CONTENT:      SectionData (
+# CONTENT-NEXT:   0000: 11223344 55 |
+# CONTENT-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_X86_64
+Sections:
+  - Name:    .note.foo
+    Type:    SHT_NOTE
+    Content: "1122334455"
+
+## Either "Content", "Size" or "Notes" must be specifed for SHT_NOTE sections.
+
+# RUN: not yaml2obj --docnum=8 %s 2>&1 | FileCheck %s --check-prefix=NO-TAGS
+
+# NO-TAGS: error: one of "Content", "Size" or "Notes" must be specified
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_X86_64
+Sections:
+  - Name: .note.foo
+    Type: SHT_NOTE
+
+## "Content" and "Notes" cannot be used together to describe the SHT_NOTE section.
+
+# RUN: not yaml2obj --docnum=9 %s 2>&1 | FileCheck %s --check-prefix=CONTENT-NOTES
+
+# CONTENT-NOTES: error: "Notes" cannot be used with "Content" or "Size"
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_X86_64
+Sections:
+  - Name:    .note.foo
+    Type:    SHT_NOTE
+    Content: ""
+    Notes:   []
+
+## "Size" and "Notes" cannot be used together to describe the SHT_NOTE section.
+
+# RUN: not yaml2obj --docnum=10 %s 2>&1 | FileCheck %s --check-prefix=CONTENT-NOTES
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_X86_64
+Sections:
+  - Name:  .note.foo
+    Type:  SHT_NOTE
+    Size:  1
+    Notes: []
+
+## Check we can use only "Size" to create a SHT_NOTE section.
+
+# RUN: yaml2obj --docnum=11 %s -o %t11
+# RUN: llvm-readobj --sections --section-data %t11 | FileCheck %s --check-prefix=SIZE
+
+# SIZE:      Name: .note.foo
+# SIZE:      SectionData (
+# SIZE-NEXT:   0000: 00000000 00000000 00000000 00000000  |
+# SIZE-NEXT:   0010: 00                                   |
+# SIZE-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .note.foo
+    Type: SHT_NOTE
+    Size: 0x11
+
+## Check we can use "Size" and "Content" together to create a SHT_NOTE section.
+
+# RUN: yaml2obj --docnum=12 %s -o %t12
+# RUN: llvm-readobj --sections --section-data %t12 | FileCheck %s --check-prefix=SIZE-CONTENT
+
+# SIZE-CONTENT:      Name: .note.sizegr
+# SIZE-CONTENT:      SectionData (
+# SIZE-CONTENT-NEXT:   0000: 11223300 00 |
+# SIZE-CONTENT-NEXT: )
+
+# SIZE-CONTENT:      Name: .note.sizeeq
+# SIZE-CONTENT:      SectionData (
+# SIZE-CONTENT-NEXT:   0000: 112233 |
+# SIZE-CONTENT-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    .note.sizegr
+    Type:    SHT_NOTE
+    Size:    0x5
+    Content: "112233"
+  - Name:    .note.sizeeq
+    Type:    SHT_NOTE
+    Size:    0x3
+    Content: "112233"
+
+## Check that when "Size" and "Content" are used together, the size
+## must be greater than or equal to the content size.
+
+# RUN: not yaml2obj --docnum=13 %s 2>&1 | FileCheck %s --check-prefix=SIZE-CONTENT-ERR
+# SIZE-CONTENT-ERR: error: "Size" must be greater than or equal to the content size
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    .note
+    Type:    SHT_NOTE
+    Size:    0x1
+    Content: "1122"
+
+## Check we can't use "Size" and "Notes" tags together.
+
+# RUN: not yaml2obj --docnum=14 %s 2>&1 | FileCheck %s --check-prefix=CONTENT-NOTES
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:  .note
+    Type:  SHT_NOTE
+    Size:  0x1
+    Notes: []
diff --git a/llvm/tools/bugpoint/CrashDebugger.cpp b/llvm/tools/bugpoint/CrashDebugger.cpp
index 1a39ff654f056..d06bca9a12587 100644
--- a/llvm/tools/bugpoint/CrashDebugger.cpp
+++ b/llvm/tools/bugpoint/CrashDebugger.cpp
@@ -1229,7 +1229,7 @@ static Error DebugACrash(BugDriver &BD, BugTester TestFn) {
       unsigned NewSize = 0;
       for (std::string &Name : FunctionNames) {
         Function *Fn = BD.getProgram().getFunction(Name);
-        assert(Fn && "Could not find funcion?");
+        assert(Fn && "Could not find function?");
 
         std::vector<Attribute> Attrs;
         for (Attribute A : Fn->getAttributes().getFnAttributes())
diff --git a/llvm/tools/extract/.clang-tidy b/llvm/tools/extract/.clang-tidy
new file mode 100644
index 0000000000000..87ec2ff53af6e
--- /dev/null
+++ b/llvm/tools/extract/.clang-tidy
@@ -0,0 +1,19 @@
+# Almost identical to the top-level .clang-tidy, except that {Member,Parameter,Variable}Case use camelBack.
+Checks: '-*,clang-diagnostic-*,llvm-*,misc-*,-misc-unused-parameters,-misc-non-private-member-variables-in-classes,readability-identifier-naming'
+CheckOptions:
+  - key:             readability-identifier-naming.ClassCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.EnumCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.FunctionCase
+    value:           camelBack
+  - key:             readability-identifier-naming.MemberCase
+    value:           camelBack
+  - key:             readability-identifier-naming.ParameterCase
+    value:           camelBack
+  - key:             readability-identifier-naming.UnionCase
+    value:           CamelCase
+  - key:             readability-identifier-naming.VariableCase
+    value:           camelBack
+  - key:             readability-identifier-naming.IgnoreMainLikeFunctions
+    value:           1
diff --git a/llvm/tools/extract/CMakeLists.txt b/llvm/tools/extract/CMakeLists.txt
new file mode 100644
index 0000000000000..dae1f463f0666
--- /dev/null
+++ b/llvm/tools/extract/CMakeLists.txt
@@ -0,0 +1,7 @@
+set(LLVM_LINK_COMPONENTS
+  Support
+  )
+
+add_llvm_tool(extract
+  extract.cpp
+  )
diff --git a/llvm/tools/extract/extract.cpp b/llvm/tools/extract/extract.cpp
new file mode 100644
index 0000000000000..8ccb539156145
--- /dev/null
+++ b/llvm/tools/extract/extract.cpp
@@ -0,0 +1,113 @@
+//===- extract.cpp - Input splitting utility ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Split input into multipe parts separated by regex '^(.|//)--- ' and extract
+// the specified part.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileOutputBuffer.h"
+#include "llvm/Support/LineIterator.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/WithColor.h"
+#include <string>
+
+using namespace llvm;
+
+static cl::OptionCategory cat("extract Options");
+
+static cl::opt<std::string> part(cl::Positional, cl::desc("part"),
+                                 cl::cat(cat));
+
+static cl::opt<std::string> input(cl::Positional, cl::desc("filename"),
+                                  cl::cat(cat));
+
+static cl::opt<std::string> output("o", cl::desc("Output filename"),
+                                   cl::value_desc("filename"), cl::init("-"),
+                                   cl::cat(cat));
+
+static cl::opt<bool> noLeadingLines("no-leading-lines",
+                                    cl::desc("Don't preserve line numbers"),
+                                    cl::cat(cat));
+
+static StringRef toolName;
+
+LLVM_ATTRIBUTE_NORETURN static void error(StringRef filename,
+                                          const Twine &message) {
+  if (filename.empty())
+    WithColor::error(errs(), toolName) << message << '\n';
+  else
+    WithColor::error(errs(), toolName) << filename << ": " << message << '\n';
+  exit(1);
+}
+
+static void handle(MemoryBuffer &inputBuf, StringRef input) {
+  const char *partBegin = nullptr, *partEnd = nullptr;
+  int numEmptyLines = 0;
+  StringRef separator;
+  for (line_iterator i(inputBuf, /*SkipBlanks=*/false, '\0'); !i.is_at_eof();) {
+    StringRef line = *i++;
+    size_t markerLen = line.startswith("//") ? 6 : 5;
+    if (!(line.size() > markerLen &&
+          line.substr(markerLen - 4).startswith("--- ")))
+      continue;
+    separator = line.substr(0, markerLen);
+    StringRef cur = line.substr(markerLen);
+    if (cur == part) {
+      if (partBegin)
+        error(input, "'" + separator + cur + "' occurs more than once");
+      if (!noLeadingLines)
+        numEmptyLines = i.line_number() - 1;
+      if (i.is_at_eof())
+        break;
+      partBegin = i->data();
+    } else if (partBegin && !partEnd) {
+      partEnd = line.data();
+    }
+  }
+  if (!partBegin)
+    error(input, "'" + separator + part + "' was not found");
+  if (!partEnd)
+    partEnd = inputBuf.getBufferEnd();
+
+  Expected<std::unique_ptr<FileOutputBuffer>> outputBuf =
+      FileOutputBuffer::create(output, numEmptyLines + (partEnd - partBegin));
+  if (!outputBuf)
+    error(input, toString(outputBuf.takeError()));
+  uint8_t *buf = (*outputBuf)->getBufferStart();
+
+  // If --no-leading-lines is not specified, numEmptyLines is 0. Append newlines
+  // so that the extracted part preserves line numbers.
+  std::fill_n(buf, numEmptyLines, '\n');
+  std::copy(partBegin, partEnd, buf + numEmptyLines);
+  if (Error e = (*outputBuf)->commit())
+    error(input, toString(std::move(e)));
+}
+
+int main(int argc, const char **argv) {
+  toolName = sys::path::stem(argv[0]);
+  cl::HideUnrelatedOptions({&cat});
+  cl::ParseCommandLineOptions(
+      argc, argv,
+      "Split input into multiple parts separated by regex '^(.|//)--- ' and "
+      "extract the part specified by '^(.|//)--- <part>'\n",
+      nullptr,
+      /*EnvVar=*/nullptr,
+      /*LongOptionsUseDoubleDash=*/true);
+
+  if (input.empty())
+    error("", "input filename is not specified");
+  ErrorOr<std::unique_ptr<MemoryBuffer>> bufferOrErr =
+      MemoryBuffer::getFileOrSTDIN(input);
+  if (std::error_code ec = bufferOrErr.getError())
+    error(input, ec.message());
+  handle(**bufferOrErr, input);
+}
diff --git a/llvm/tools/gold/gold-plugin.cpp b/llvm/tools/gold/gold-plugin.cpp
index 7654d44ba0089..0124e0a93e8a2 100644
--- a/llvm/tools/gold/gold-plugin.cpp
+++ b/llvm/tools/gold/gold-plugin.cpp
@@ -1050,7 +1050,7 @@ static std::vector<std::pair<SmallString<128>, bool>> runLTO() {
   if (!options::obj_path.empty())
     Filename = options::obj_path;
   else if (options::TheOutputType == options::OT_SAVE_TEMPS)
-    Filename = output_name + ".o";
+    Filename = output_name + ".lto.o";
   else if (options::TheOutputType == options::OT_ASM_ONLY)
     Filename = output_name;
   bool SaveTemps = !Filename.empty();
diff --git a/llvm/tools/llvm-libtool-darwin/CMakeLists.txt b/llvm/tools/llvm-libtool-darwin/CMakeLists.txt
index b2ea63a8da493..c2073990f214d 100644
--- a/llvm/tools/llvm-libtool-darwin/CMakeLists.txt
+++ b/llvm/tools/llvm-libtool-darwin/CMakeLists.txt
@@ -1,4 +1,6 @@
 set(LLVM_LINK_COMPONENTS
+  BinaryFormat
+  Object
   Support
   )
 
diff --git a/llvm/tools/llvm-libtool-darwin/LLVMBuild.txt b/llvm/tools/llvm-libtool-darwin/LLVMBuild.txt
index 3858d2faa778f..3c557a3aaf61f 100644
--- a/llvm/tools/llvm-libtool-darwin/LLVMBuild.txt
+++ b/llvm/tools/llvm-libtool-darwin/LLVMBuild.txt
@@ -17,4 +17,4 @@
 type = Tool
 name = llvm-libtool-darwin
 parent = Tools
-required_libraries = Support
+required_libraries = BinaryFormat Object Support
diff --git a/llvm/tools/llvm-libtool-darwin/llvm-libtool-darwin.cpp b/llvm/tools/llvm-libtool-darwin/llvm-libtool-darwin.cpp
index bc2f9f765f437..bd1d9ac07c699 100644
--- a/llvm/tools/llvm-libtool-darwin/llvm-libtool-darwin.cpp
+++ b/llvm/tools/llvm-libtool-darwin/llvm-libtool-darwin.cpp
@@ -10,11 +10,16 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/BinaryFormat/Magic.h"
+#include "llvm/Object/ArchiveWriter.h"
+#include "llvm/Object/MachO.h"
+#include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/InitLLVM.h"
 #include "llvm/Support/WithColor.h"
 
 using namespace llvm;
+using namespace llvm::object;
 
 cl::OptionCategory LibtoolCategory("llvm-libtool-darwin Options");
 
@@ -27,8 +32,116 @@ static cl::list<std::string> InputFiles(cl::Positional,
                                         cl::OneOrMore,
                                         cl::cat(LibtoolCategory));
 
+enum class Operation { Static };
+
+static cl::opt<Operation> LibraryOperation(
+    cl::desc("Library Type: "),
+    cl::values(
+        clEnumValN(Operation::Static, "static",
+                   "Produce a statically linked library from the input files")),
+    cl::Required, cl::cat(LibtoolCategory));
+
+static Error verifyMachOObject(const NewArchiveMember &Member) {
+  auto MBRef = Member.Buf->getMemBufferRef();
+  Expected<std::unique_ptr<object::ObjectFile>> ObjOrErr =
+      object::ObjectFile::createObjectFile(MBRef);
+
+  // Throw error if not a valid object file.
+  if (!ObjOrErr)
+    return createFileError(Member.MemberName, ObjOrErr.takeError());
+
+  // Throw error if not in Mach-O format.
+  if (!isa<object::MachOObjectFile>(**ObjOrErr))
+    return createStringError(std::errc::invalid_argument,
+                             "'%s': format not supported",
+                             Member.MemberName.data());
+
+  return Error::success();
+}
+
+static Error addChildMember(std::vector<NewArchiveMember> &Members,
+                            const object::Archive::Child &M) {
+  Expected<NewArchiveMember> NMOrErr =
+      NewArchiveMember::getOldMember(M, /*Deterministic=*/true);
+  if (!NMOrErr)
+    return NMOrErr.takeError();
+
+  // Verify that Member is a Mach-O object file.
+  if (Error E = verifyMachOObject(*NMOrErr))
+    return E;
+
+  Members.push_back(std::move(*NMOrErr));
+  return Error::success();
+}
+
+static Error
+addMember(std::vector<NewArchiveMember> &Members, StringRef FileName,
+          std::vector<std::unique_ptr<MemoryBuffer>> &ArchiveBuffers) {
+  Expected<NewArchiveMember> NMOrErr =
+      NewArchiveMember::getFile(FileName, /*Deterministic=*/true);
+  if (!NMOrErr)
+    return createFileError(FileName, NMOrErr.takeError());
+
+  // For regular archives, use the basename of the object path for the member
+  // name.
+  NMOrErr->MemberName = sys::path::filename(NMOrErr->MemberName);
+
+  // Flatten archives.
+  if (identify_magic(NMOrErr->Buf->getBuffer()) == file_magic::archive) {
+    Expected<std::unique_ptr<Archive>> LibOrErr =
+        object::Archive::create(NMOrErr->Buf->getMemBufferRef());
+    if (!LibOrErr)
+      return createFileError(FileName, LibOrErr.takeError());
+    object::Archive &Lib = **LibOrErr;
+
+    Error Err = Error::success();
+    for (const object::Archive::Child &Child : Lib.children(Err))
+      if (Error E = addChildMember(Members, Child))
+        return createFileError(FileName, std::move(E));
+    if (Err)
+      return createFileError(FileName, std::move(Err));
+
+    // Update vector ArchiveBuffers with the MemoryBuffers to transfer
+    // ownership.
+    ArchiveBuffers.push_back(std::move(NMOrErr->Buf));
+    return Error::success();
+  }
+
+  // Verify that Member is a Mach-O object file.
+  if (Error E = verifyMachOObject(*NMOrErr))
+    return E;
+
+  Members.push_back(std::move(*NMOrErr));
+  return Error::success();
+}
+
+static Error createStaticLibrary() {
+  std::vector<NewArchiveMember> NewMembers;
+  std::vector<std::unique_ptr<MemoryBuffer>> ArchiveBuffers;
+  for (StringRef Member : InputFiles)
+    if (Error E = addMember(NewMembers, Member, ArchiveBuffers))
+      return E;
+
+  if (Error E = writeArchive(OutputFile, NewMembers,
+                             /*WriteSymtab=*/true,
+                             /*Kind=*/object::Archive::K_DARWIN,
+                             /*Deterministic=*/true,
+                             /*Thin=*/false))
+    return E;
+  return Error::success();
+}
+
 int main(int Argc, char **Argv) {
   InitLLVM X(Argc, Argv);
   cl::HideUnrelatedOptions({&LibtoolCategory, &ColorCategory});
   cl::ParseCommandLineOptions(Argc, Argv, "llvm-libtool-darwin\n");
+
+  switch (LibraryOperation) {
+  case Operation::Static:
+    if (Error E = createStaticLibrary()) {
+      WithColor::defaultErrorHandler(std::move(E));
+      exit(EXIT_FAILURE);
+    }
+    break;
+  }
 }
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index 843f072a61c32..41f6a4d723eec 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -251,6 +251,7 @@ static void loadInput(const WeightedFile &Input, SymbolRemapper *Remapper,
         Filename);
     return;
   }
+  WC->Writer.setInstrEntryBBEnabled(Reader->instrEntryBBEnabled());
 
   for (auto &I : *Reader) {
     if (Remapper)
@@ -977,8 +978,11 @@ static int showInstrProfile(const std::string &Filename, bool ShowCounts,
   if (TextFormat)
     return 0;
   std::unique_ptr<ProfileSummary> PS(Builder.getSummary());
-  OS << "Instrumentation level: "
-     << (Reader->isIRLevelProfile() ? "IR" : "Front-end") << "\n";
+  bool IsIR = Reader->isIRLevelProfile();
+  OS << "Instrumentation level: " << (IsIR ? "IR" : "Front-end");
+  if (IsIR)
+    OS << "  entry_first = " << Reader->instrEntryBBEnabled();
+  OS << "\n";
   if (ShowAllFunctions || !ShowFunction.empty())
     OS << "Functions shown: " << ShownFunctions << "\n";
   OS << "Total functions: " << PS->getNumFunctions() << "\n";
diff --git a/llvm/tools/llvm-readobj/DwarfCFIEHPrinter.h b/llvm/tools/llvm-readobj/DwarfCFIEHPrinter.h
index 27942224053f1..a82eca080cc9a 100644
--- a/llvm/tools/llvm-readobj/DwarfCFIEHPrinter.h
+++ b/llvm/tools/llvm-readobj/DwarfCFIEHPrinter.h
@@ -13,23 +13,23 @@
 #include "llvm-readobj.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/BinaryFormat/Dwarf.h"
+#include "llvm/DebugInfo/DWARF/DWARFContext.h"
+#include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h"
+#include "llvm/DebugInfo/DWARF/DWARFDebugFrame.h"
 #include "llvm/Object/ELF.h"
-#include "llvm/Object/ELFTypes.h"
 #include "llvm/Object/ELFObjectFile.h"
+#include "llvm/Object/ELFTypes.h"
 #include "llvm/Support/Casting.h"
-#include "llvm/Support/ScopedPrinter.h"
 #include "llvm/Support/Debug.h"
-#include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h"
-#include "llvm/DebugInfo/DWARF/DWARFDebugFrame.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Format.h"
+#include "llvm/Support/ScopedPrinter.h"
 #include "llvm/Support/type_traits.h"
 
 namespace llvm {
 namespace DwarfCFIEH {
 
-template <typename ELFT>
-class PrinterContext {
+template <typename ELFT> class PrinterContext {
   using Elf_Shdr = typename ELFT::Shdr;
   using Elf_Phdr = typename ELFT::Phdr;
 
@@ -156,7 +156,7 @@ void PrinterContext<ELFT>::printEHFrameHdr(const Elf_Phdr *EHFramePHdr) const {
   unsigned NumEntries = 0;
   uint64_t PrevPC = 0;
   while (Offset + 8 <= EHFramePHdr->p_memsz && NumEntries < FDECount) {
-    DictScope D(W, std::string("entry ")  + std::to_string(NumEntries));
+    DictScope D(W, std::string("entry ") + std::to_string(NumEntries));
 
     auto InitialPC = DE.getSigned(&Offset, 4) + EHFrameHdrAddress;
     W.startLine() << format("initial_location: 0x%" PRIx64 "\n", InitialPC);
@@ -186,7 +186,10 @@ void PrinterContext<ELFT>::printEHFrame(const Elf_Shdr *EHFrameShdr) const {
   if (!DataOrErr)
     reportError(DataOrErr.takeError(), ObjF->getFileName());
 
-  DWARFDataExtractor DE(*DataOrErr,
+  // Construct DWARFDataExtractor to handle relocations ("PC Begin" fields).
+  std::unique_ptr<DWARFContext> DICtx = DWARFContext::create(*ObjF, nullptr);
+  DWARFDataExtractor DE(DICtx->getDWARFObj(),
+                        DICtx->getDWARFObj().getEHFrameSection(),
                         ELFT::TargetEndianness == support::endianness::little,
                         ELFT::Is64Bits ? 8 : 4);
   DWARFDebugFrame EHFrame(Triple::ArchType(ObjF->getArch()), /*IsEH=*/true,
@@ -232,7 +235,7 @@ void PrinterContext<ELFT>::printEHFrame(const Elf_Shdr *EHFrameShdr) const {
 
   W.unindent();
 }
-}
-}
+} // namespace DwarfCFIEH
+} // namespace llvm
 
 #endif
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index 15076f1f89337..16270cb0f58c2 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -292,9 +292,9 @@ template <typename ELFT> class ELFDumper : public ObjDumper {
   const Elf_Hash *HashTable = nullptr;
   const Elf_GnuHash *GnuHashTable = nullptr;
   const Elf_Shdr *DotSymtabSec = nullptr;
+  const Elf_Shdr *DotDynsymSec = nullptr;
   const Elf_Shdr *DotCGProfileSec = nullptr;
   const Elf_Shdr *DotAddrsigSec = nullptr;
-  StringRef DynSymtabName;
   ArrayRef<Elf_Word> ShndxTable;
 
   const Elf_Shdr *SymbolVersionSection = nullptr;   // .gnu.version
@@ -309,6 +309,8 @@ template <typename ELFT> class ELFDumper : public ObjDumper {
 
   std::unordered_set<std::string> Warnings;
 
+  std::string describe(const Elf_Shdr &Sec) const;
+
 public:
   Elf_Dyn_Range dynamic_table() const {
     // A valid .dynamic section contains an array of entries terminated
@@ -372,33 +374,42 @@ template <typename ELFT> class ELFDumper : public ObjDumper {
   Expected<std::vector<VerNeed>>
   getVersionDependencies(const Elf_Shdr *Sec) const;
 
+  template <class RelTy>
   Expected<std::pair<const Elf_Sym *, std::string>>
-  getRelocationTarget(const Elf_Shdr *SymTab, const Elf_Rela &R) const;
+  getRelocationTarget(const Elf_Shdr *SymTab, const RelTy &R) const;
 
   std::function<Error(const Twine &Msg)> WarningHandler;
   void reportUniqueWarning(Error Err) const;
 };
 
+template <class ELFT>
+static std::string describe(const ELFFile<ELFT> *Obj,
+                            const typename ELFT::Shdr &Sec) {
+  unsigned SecNdx = &Sec - &cantFail(Obj->sections()).front();
+  return (object::getELFSectionTypeName(Obj->getHeader()->e_machine,
+                                        Sec.sh_type) +
+          " section with index " + Twine(SecNdx))
+      .str();
+}
+
+template <class ELFT>
+std::string ELFDumper<ELFT>::describe(const Elf_Shdr &Sec) const {
+  return ::describe(ObjF->getELFFile(), Sec);
+}
+
 template <class ELFT>
 static Expected<StringRef> getLinkAsStrtab(const ELFFile<ELFT> *Obj,
-                                           const typename ELFT::Shdr *Sec,
-                                           unsigned SecNdx) {
+                                           const typename ELFT::Shdr *Sec) {
   Expected<const typename ELFT::Shdr *> StrTabSecOrErr =
       Obj->getSection(Sec->sh_link);
   if (!StrTabSecOrErr)
-    return createError("invalid section linked to " +
-                       object::getELFSectionTypeName(
-                           Obj->getHeader()->e_machine, Sec->sh_type) +
-                       " section with index " + Twine(SecNdx) + ": " +
-                       toString(StrTabSecOrErr.takeError()));
+    return createError("invalid section linked to " + describe(Obj, *Sec) +
+                       ": " + toString(StrTabSecOrErr.takeError()));
 
   Expected<StringRef> StrTabOrErr = Obj->getStringTable(*StrTabSecOrErr);
   if (!StrTabOrErr)
-    return createError("invalid string table linked to " +
-                       object::getELFSectionTypeName(
-                           Obj->getHeader()->e_machine, Sec->sh_type) +
-                       " section with index " + Twine(SecNdx) + ": " +
-                       toString(StrTabOrErr.takeError()));
+    return createError("invalid string table linked to " + describe(Obj, *Sec) +
+                       ": " + toString(StrTabOrErr.takeError()));
   return *StrTabOrErr;
 }
 
@@ -406,43 +417,33 @@ static Expected<StringRef> getLinkAsStrtab(const ELFFile<ELFT> *Obj,
 template <class ELFT>
 static Expected<std::pair<typename ELFT::SymRange, StringRef>>
 getLinkAsSymtab(const ELFFile<ELFT> *Obj, const typename ELFT::Shdr *Sec,
-                   unsigned SecNdx, unsigned ExpectedType) {
+                unsigned ExpectedType) {
   Expected<const typename ELFT::Shdr *> SymtabOrErr =
       Obj->getSection(Sec->sh_link);
   if (!SymtabOrErr)
-    return createError("invalid section linked to " +
-                       object::getELFSectionTypeName(
-                           Obj->getHeader()->e_machine, Sec->sh_type) +
-                       " section with index " + Twine(SecNdx) + ": " +
-                       toString(SymtabOrErr.takeError()));
+    return createError("invalid section linked to " + describe(Obj, *Sec) +
+                       ": " + toString(SymtabOrErr.takeError()));
 
   if ((*SymtabOrErr)->sh_type != ExpectedType)
     return createError(
-        "invalid section linked to " +
-        object::getELFSectionTypeName(Obj->getHeader()->e_machine,
-                                      Sec->sh_type) +
-        " section with index " + Twine(SecNdx) + ": expected " +
+        "invalid section linked to " + describe(Obj, *Sec) + ": expected " +
         object::getELFSectionTypeName(Obj->getHeader()->e_machine,
                                       ExpectedType) +
         ", but got " +
         object::getELFSectionTypeName(Obj->getHeader()->e_machine,
                                       (*SymtabOrErr)->sh_type));
 
-  Expected<StringRef> StrTabOrErr =
-      getLinkAsStrtab(Obj, *SymtabOrErr, Sec->sh_link);
+  Expected<StringRef> StrTabOrErr = getLinkAsStrtab(Obj, *SymtabOrErr);
   if (!StrTabOrErr)
     return createError(
         "can't get a string table for the symbol table linked to " +
-        object::getELFSectionTypeName(Obj->getHeader()->e_machine,
-                                      Sec->sh_type) +
-        " section with index " + Twine(SecNdx) + ": " +
-        toString(StrTabOrErr.takeError()));
+        describe(Obj, *Sec) + ": " + toString(StrTabOrErr.takeError()));
 
   Expected<typename ELFT::SymRange> SymsOrErr = Obj->symbols(*SymtabOrErr);
   if (!SymsOrErr)
-    return createError(
-        "unable to read symbols from the symbol table with index " +
-        Twine(Sec->sh_link) + ": " + toString(SymsOrErr.takeError()));
+    return createError("unable to read symbols from the " +
+                       describe(Obj, *Sec) + ": " +
+                       toString(SymsOrErr.takeError()));
 
   return std::make_pair(*SymsOrErr, *StrTabOrErr);
 }
@@ -453,21 +454,18 @@ ELFDumper<ELFT>::getVersionTable(const Elf_Shdr *Sec, ArrayRef<Elf_Sym> *SymTab,
                                  StringRef *StrTab) const {
   assert((!SymTab && !StrTab) || (SymTab && StrTab));
   const ELFFile<ELFT> *Obj = ObjF->getELFFile();
-  unsigned SecNdx = Sec - &cantFail(Obj->sections()).front();
 
   if (uintptr_t(Obj->base() + Sec->sh_offset) % sizeof(uint16_t) != 0)
-    return createError("the SHT_GNU_versym section with index " +
-                       Twine(SecNdx) + " is misaligned");
+    return createError("the " + describe(*Sec) + " is misaligned");
 
   Expected<ArrayRef<Elf_Versym>> VersionsOrErr =
       Obj->template getSectionContentsAsArray<Elf_Versym>(Sec);
   if (!VersionsOrErr)
-    return createError(
-        "cannot read content of SHT_GNU_versym section with index " +
-        Twine(SecNdx) + ": " + toString(VersionsOrErr.takeError()));
+    return createError("cannot read content of " + describe(*Sec) + ": " +
+                       toString(VersionsOrErr.takeError()));
 
   Expected<std::pair<ArrayRef<Elf_Sym>, StringRef>> SymTabOrErr =
-      getLinkAsSymtab(Obj, Sec, SecNdx, SHT_DYNSYM);
+      getLinkAsSymtab(Obj, Sec, SHT_DYNSYM);
   if (!SymTabOrErr) {
     reportUniqueWarning(SymTabOrErr.takeError());
     return *VersionsOrErr;
@@ -475,8 +473,8 @@ ELFDumper<ELFT>::getVersionTable(const Elf_Shdr *Sec, ArrayRef<Elf_Sym> *SymTab,
 
   if (SymTabOrErr->first.size() != VersionsOrErr->size())
     reportUniqueWarning(
-        createError("SHT_GNU_versym section with index " + Twine(SecNdx) +
-                    ": the number of entries (" + Twine(VersionsOrErr->size()) +
+        createError(describe(*Sec) + ": the number of entries (" +
+                    Twine(VersionsOrErr->size()) +
                     ") does not match the number of symbols (" +
                     Twine(SymTabOrErr->first.size()) +
                     ") in the symbol table with index " + Twine(Sec->sh_link)));
@@ -490,17 +488,15 @@ template <class ELFT>
 Expected<std::vector<VerDef>>
 ELFDumper<ELFT>::getVersionDefinitions(const Elf_Shdr *Sec) const {
   const ELFFile<ELFT> *Obj = ObjF->getELFFile();
-  unsigned SecNdx = Sec - &cantFail(Obj->sections()).front();
 
-  Expected<StringRef> StrTabOrErr = getLinkAsStrtab(Obj, Sec, SecNdx);
+  Expected<StringRef> StrTabOrErr = getLinkAsStrtab(Obj, Sec);
   if (!StrTabOrErr)
     return StrTabOrErr.takeError();
 
   Expected<ArrayRef<uint8_t>> ContentsOrErr = Obj->getSectionContents(Sec);
   if (!ContentsOrErr)
-    return createError(
-        "cannot read content of SHT_GNU_verdef section with index " +
-        Twine(SecNdx) + ": " + toString(ContentsOrErr.takeError()));
+    return createError("cannot read content of " + describe(*Sec) + ": " +
+                       toString(ContentsOrErr.takeError()));
 
   const uint8_t *Start = ContentsOrErr->data();
   const uint8_t *End = Start + ContentsOrErr->size();
@@ -508,8 +504,7 @@ ELFDumper<ELFT>::getVersionDefinitions(const Elf_Shdr *Sec) const {
   auto ExtractNextAux = [&](const uint8_t *&VerdauxBuf,
                             unsigned VerDefNdx) -> Expected<VerdAux> {
     if (VerdauxBuf + sizeof(Elf_Verdaux) > End)
-      return createError("invalid SHT_GNU_verdef section with index " +
-                         Twine(SecNdx) + ": version definition " +
+      return createError("invalid " + describe(*Sec) + ": version definition " +
                          Twine(VerDefNdx) +
                          " refers to an auxiliary entry that goes past the end "
                          "of the section");
@@ -530,21 +525,19 @@ ELFDumper<ELFT>::getVersionDefinitions(const Elf_Shdr *Sec) const {
   const uint8_t *VerdefBuf = Start;
   for (unsigned I = 1; I <= /*VerDefsNum=*/Sec->sh_info; ++I) {
     if (VerdefBuf + sizeof(Elf_Verdef) > End)
-      return createError("invalid SHT_GNU_verdef section with index " +
-                         Twine(SecNdx) + ": version definition " + Twine(I) +
-                         " goes past the end of the section");
+      return createError("invalid " + describe(*Sec) + ": version definition " +
+                         Twine(I) + " goes past the end of the section");
 
     if (uintptr_t(VerdefBuf) % sizeof(uint32_t) != 0)
       return createError(
-          "invalid SHT_GNU_verdef section with index " + Twine(SecNdx) +
+          "invalid " + describe(*Sec) +
           ": found a misaligned version definition entry at offset 0x" +
           Twine::utohexstr(VerdefBuf - Start));
 
     unsigned Version = *reinterpret_cast<const Elf_Half *>(VerdefBuf);
     if (Version != 1)
-      return createError("unable to dump SHT_GNU_verdef section with index " +
-                         Twine(SecNdx) + ": version " + Twine(Version) +
-                         " is not yet supported");
+      return createError("unable to dump " + describe(*Sec) + ": version " +
+                         Twine(Version) + " is not yet supported");
 
     const Elf_Verdef *D = reinterpret_cast<const Elf_Verdef *>(VerdefBuf);
     VerDef &VD = *Ret.emplace(Ret.end());
@@ -558,8 +551,7 @@ ELFDumper<ELFT>::getVersionDefinitions(const Elf_Shdr *Sec) const {
     const uint8_t *VerdauxBuf = VerdefBuf + D->vd_aux;
     for (unsigned J = 0; J < D->vd_cnt; ++J) {
       if (uintptr_t(VerdauxBuf) % sizeof(uint32_t) != 0)
-        return createError("invalid SHT_GNU_verdef section with index " +
-                           Twine(SecNdx) +
+        return createError("invalid " + describe(*Sec) +
                            ": found a misaligned auxiliary entry at offset 0x" +
                            Twine::utohexstr(VerdauxBuf - Start));
 
@@ -583,10 +575,8 @@ template <class ELFT>
 Expected<std::vector<VerNeed>>
 ELFDumper<ELFT>::getVersionDependencies(const Elf_Shdr *Sec) const {
   const ELFFile<ELFT> *Obj = ObjF->getELFFile();
-  unsigned SecNdx = Sec - &cantFail(Obj->sections()).front();
-
   StringRef StrTab;
-  Expected<StringRef> StrTabOrErr = getLinkAsStrtab(Obj, Sec, SecNdx);
+  Expected<StringRef> StrTabOrErr = getLinkAsStrtab(Obj, Sec);
   if (!StrTabOrErr)
     reportUniqueWarning(StrTabOrErr.takeError());
   else
@@ -594,9 +584,8 @@ ELFDumper<ELFT>::getVersionDependencies(const Elf_Shdr *Sec) const {
 
   Expected<ArrayRef<uint8_t>> ContentsOrErr = Obj->getSectionContents(Sec);
   if (!ContentsOrErr)
-    return createError(
-        "cannot read content of SHT_GNU_verneed section with index " +
-        Twine(SecNdx) + ": " + toString(ContentsOrErr.takeError()));
+    return createError("cannot read content of " + describe(*Sec) + ": " +
+                       toString(ContentsOrErr.takeError()));
 
   const uint8_t *Start = ContentsOrErr->data();
   const uint8_t *End = Start + ContentsOrErr->size();
@@ -605,21 +594,19 @@ ELFDumper<ELFT>::getVersionDependencies(const Elf_Shdr *Sec) const {
   std::vector<VerNeed> Ret;
   for (unsigned I = 1; I <= /*VerneedNum=*/Sec->sh_info; ++I) {
     if (VerneedBuf + sizeof(Elf_Verdef) > End)
-      return createError("invalid SHT_GNU_verneed section with index " +
-                         Twine(SecNdx) + ": version dependency " + Twine(I) +
-                         " goes past the end of the section");
+      return createError("invalid " + describe(*Sec) + ": version dependency " +
+                         Twine(I) + " goes past the end of the section");
 
     if (uintptr_t(VerneedBuf) % sizeof(uint32_t) != 0)
       return createError(
-          "invalid SHT_GNU_verneed section with index " + Twine(SecNdx) +
+          "invalid " + describe(*Sec) +
           ": found a misaligned version dependency entry at offset 0x" +
           Twine::utohexstr(VerneedBuf - Start));
 
     unsigned Version = *reinterpret_cast<const Elf_Half *>(VerneedBuf);
     if (Version != 1)
-      return createError("unable to dump SHT_GNU_verneed section with index " +
-                         Twine(SecNdx) + ": version " + Twine(Version) +
-                         " is not yet supported");
+      return createError("unable to dump " + describe(*Sec) + ": version " +
+                         Twine(Version) + " is not yet supported");
 
     const Elf_Verneed *Verneed =
         reinterpret_cast<const Elf_Verneed *>(VerneedBuf);
@@ -637,15 +624,13 @@ ELFDumper<ELFT>::getVersionDependencies(const Elf_Shdr *Sec) const {
     const uint8_t *VernauxBuf = VerneedBuf + Verneed->vn_aux;
     for (unsigned J = 0; J < Verneed->vn_cnt; ++J) {
       if (uintptr_t(VernauxBuf) % sizeof(uint32_t) != 0)
-        return createError("invalid SHT_GNU_verneed section with index " +
-                           Twine(SecNdx) +
+        return createError("invalid " + describe(*Sec) +
                            ": found a misaligned auxiliary entry at offset 0x" +
                            Twine::utohexstr(VernauxBuf - Start));
 
       if (VernauxBuf + sizeof(Elf_Vernaux) > End)
         return createError(
-            "invalid SHT_GNU_verneed section with index " + Twine(SecNdx) +
-            ": version dependency " + Twine(I) +
+            "invalid " + describe(*Sec) + ": version dependency " + Twine(I) +
             " refers to an auxiliary entry that goes past the end "
             "of the section");
 
@@ -672,19 +657,16 @@ ELFDumper<ELFT>::getVersionDependencies(const Elf_Shdr *Sec) const {
 template <class ELFT>
 void ELFDumper<ELFT>::printSymbolsHelper(bool IsDynamic) const {
   Optional<StringRef> StrTable;
-  StringRef SymtabName;
   size_t Entries = 0;
   Elf_Sym_Range Syms(nullptr, nullptr);
   const ELFFile<ELFT> *Obj = ObjF->getELFFile();
+  const Elf_Shdr *SymtabSec = IsDynamic ? DotDynsymSec : DotSymtabSec;
+
   if (IsDynamic) {
     StrTable = DynamicStringTable;
     Syms = dynamic_symbols();
-    SymtabName = DynSymtabName;
     Entries = Syms.size();
-  } else {
-    if (!DotSymtabSec)
-      return;
-
+  } else if (DotSymtabSec) {
     if (Expected<StringRef> StrTableOrErr =
             Obj->getStringTableForSymtab(*DotSymtabSec))
       StrTable = *StrTableOrErr;
@@ -699,17 +681,6 @@ void ELFDumper<ELFT>::printSymbolsHelper(bool IsDynamic) const {
       reportUniqueWarning(
           createError("unable to read symbols from the SHT_SYMTAB section: " +
                       toString(SymsOrErr.takeError())));
-
-    if (Expected<StringRef> SymtabNameOrErr =
-            Obj->getSectionName(DotSymtabSec)) {
-      SymtabName = *SymtabNameOrErr;
-    } else {
-      reportUniqueWarning(
-          createError("unable to get the name of the SHT_SYMTAB section: " +
-                      toString(SymtabNameOrErr.takeError())));
-      SymtabName = "<?>";
-    }
-
     Entries = DotSymtabSec->getEntityCount();
   }
   if (Syms.begin() == Syms.end())
@@ -721,7 +692,7 @@ void ELFDumper<ELFT>::printSymbolsHelper(bool IsDynamic) const {
                                  return S.st_other & ~0x3;
                                }) != Syms.end();
 
-  ELFDumperStyle->printSymtabMessage(Obj, SymtabName, Entries,
+  ELFDumperStyle->printSymtabMessage(Obj, SymtabSec, Entries,
                                      NonVisibilityBitsUsed);
   for (const auto &Sym : Syms)
     ELFDumperStyle->printSymbol(Obj, &Sym, Syms.begin(), StrTable, IsDynamic,
@@ -752,8 +723,9 @@ template <typename ELFT> class DumpStyle {
   virtual void printDependentLibs(const ELFFile<ELFT> *Obj) = 0;
   virtual void printDynamic(const ELFFile<ELFT> *Obj) {}
   virtual void printDynamicRelocations(const ELFFile<ELFT> *Obj) = 0;
-  virtual void printSymtabMessage(const ELFFile<ELFT> *Obj, StringRef Name,
-                                  size_t Offset, bool NonVisibilityBitsUsed) {}
+  virtual void printSymtabMessage(const ELFFile<ELFT> *Obj,
+                                  const Elf_Shdr *Symtab, size_t Offset,
+                                  bool NonVisibilityBitsUsed) {}
   virtual void printSymbol(const ELFFile<ELFT> *Obj, const Elf_Sym *Symbol,
                            const Elf_Sym *FirstSym,
                            Optional<StringRef> StrTable, bool IsDynamic,
@@ -826,8 +798,8 @@ template <typename ELFT> class GNUStyle : public DumpStyle<ELFT> {
   void printDependentLibs(const ELFFile<ELFT> *Obj) override;
   void printDynamic(const ELFFile<ELFT> *Obj) override;
   void printDynamicRelocations(const ELFO *Obj) override;
-  void printSymtabMessage(const ELFO *Obj, StringRef Name, size_t Offset,
-                          bool NonVisibilityBitsUsed) override;
+  void printSymtabMessage(const ELFO *Obj, const Elf_Shdr *Symtab,
+                          size_t Offset, bool NonVisibilityBitsUsed) override;
   void printProgramHeaders(const ELFO *Obj, bool PrintProgramHeaders,
                            cl::boolOrDefault PrintSectionMapping) override;
   void printVersionSymbolSection(const ELFFile<ELFT> *Obj,
@@ -908,17 +880,20 @@ template <typename ELFT> class GNUStyle : public DumpStyle<ELFT> {
   void printHashedSymbol(const ELFO *Obj, const Elf_Sym *FirstSym, uint32_t Sym,
                          StringRef StrTable, uint32_t Bucket);
   void printRelocHeader(unsigned SType);
+  template <class RelTy>
   void printRelocation(const ELFO *Obj, unsigned SecIndex,
-                       const Elf_Shdr *SymTab, const Elf_Rela &R,
-                       unsigned RelIndex, bool IsRela);
+                       const Elf_Shdr *SymTab, const RelTy &R,
+                       unsigned RelIndex);
+  template <class RelTy>
   void printRelocation(const ELFO *Obj, const Elf_Sym *Sym,
-                       StringRef SymbolName, const Elf_Rela &R, bool IsRela);
+                       StringRef SymbolName, const RelTy &R);
   void printSymbol(const ELFO *Obj, const Elf_Sym *Symbol, const Elf_Sym *First,
                    Optional<StringRef> StrTable, bool IsDynamic,
                    bool NonVisibilityBitsUsed) override;
   std::string getSymbolSectionNdx(const ELFO *Obj, const Elf_Sym *Symbol,
                                   const Elf_Sym *FirstSym);
-  void printDynamicRelocation(const ELFO *Obj, Elf_Rela R, bool IsRela);
+  template <class RelTy>
+  void printDynamicRelocation(const ELFO *Obj, const RelTy &R);
   void printProgramHeaders(const ELFO *Obj);
   void printSectionMapping(const ELFO *Obj);
   void printGNUVersionSectionProlog(const ELFFile<ELFT> *Obj,
@@ -976,9 +951,12 @@ template <typename ELFT> class LLVMStyle : public DumpStyle<ELFT> {
   void printMipsABIFlags(const ELFObjectFile<ELFT> *Obj) override;
 
 private:
-  void printRelocation(const ELFO *Obj, unsigned SecIndex, Elf_Rela Rel,
+  template <class RelTy>
+  void printRelocation(const ELFO *Obj, unsigned SecIndex, const RelTy &Rel,
                        unsigned RelIndex, const Elf_Shdr *SymTab);
-  void printDynamicRelocation(const ELFO *Obj, Elf_Rela Rel);
+  template <class RelTy>
+  void printDynamicRelocation(const ELFO *Obj, const RelTy& Rel);
+
   void printSymbols(const ELFO *Obj);
   void printDynamicSymbols(const ELFO *Obj);
   void printSymbolSection(const Elf_Sym *Symbol, const Elf_Sym *First);
@@ -1095,9 +1073,10 @@ Expected<StringRef> ELFDumper<ELFT>::getSymbolVersion(const Elf_Sym *Sym,
 }
 
 template <typename ELFT>
+template <class RelTy>
 Expected<std::pair<const typename ELFT::Sym *, std::string>>
 ELFDumper<ELFT>::getRelocationTarget(const Elf_Shdr *SymTab,
-                                     const Elf_Rela &R) const {
+                                     const RelTy &R) const {
   const ELFFile<ELFT> *Obj = ObjF->getELFFile();
   Expected<const Elf_Sym *> SymOrErr = Obj->getRelocationSymbol(&R, SymTab);
   if (!SymOrErr)
@@ -1982,9 +1961,7 @@ void ELFDumper<ELFT>::loadDynamicTable(const ELFFile<ELFT> *Obj) {
     FromSec =
         checkDRI({ObjF->getELFFile()->base() + DynamicSec->sh_offset,
                   DynamicSec->sh_size, sizeof(Elf_Dyn), ObjF->getFileName()});
-    FromSec.Context = ("section with index " +
-                       Twine(DynamicSec - &cantFail(Obj->sections()).front()))
-                          .str();
+    FromSec.Context = describe(*DynamicSec);
     FromSec.EntSizePrintName = "";
 
     IsSecTableValid = !FromSec.getAsArrayRef<Elf_Dyn>().empty();
@@ -2067,14 +2044,12 @@ ELFDumper<ELFT>::ELFDumper(const object::ELFObjectFile<ELFT> *ObjF,
         DotSymtabSec = &Sec;
       break;
     case ELF::SHT_DYNSYM:
+      if (!DotDynsymSec)
+        DotDynsymSec = &Sec;
+
       if (!DynSymRegion) {
         DynSymRegion = createDRIFrom(&Sec);
-        DynSymRegion->Context =
-            ("section with index " + Twine(&Sec - &Sections.front())).str();
-        // This is only used (if Elf_Shdr present)for naming section in GNU
-        // style
-        DynSymtabName =
-            unwrapOrError(ObjF->getFileName(), Obj->getSectionName(&Sec));
+        DynSymRegion->Context = describe(Sec);
 
         if (Expected<StringRef> E = Obj->getStringTableForSymtab(Sec))
           DynamicStringTable = *E;
@@ -3112,25 +3087,20 @@ Error MipsGOTParser<ELFT>::findPLT(Elf_Dyn_Range DynTable) {
                        toString(PltContentOrErr.takeError()));
 
   if (Expected<const Elf_Shdr *> PltSymTableOrErr =
-          Obj->getSection(PltRelSec->sh_link)) {
+          Obj->getSection(PltRelSec->sh_link))
     PltSymTable = *PltSymTableOrErr;
-  } else {
-    unsigned SecNdx = PltRelSec - &cantFail(Obj->sections()).front();
-    return createError("unable to get a symbol table linked to the RELPLT "
-                       "section with index " +
-                       Twine(SecNdx) + ": " +
+  else
+    return createError("unable to get a symbol table linked to the " +
+                       describe(Obj, *PltRelSec) + ": " +
                        toString(PltSymTableOrErr.takeError()));
-  }
 
   if (Expected<StringRef> StrTabOrErr =
-          Obj->getStringTableForSymtab(*PltSymTable)) {
+          Obj->getStringTableForSymtab(*PltSymTable))
     PltStrTable = *StrTabOrErr;
-  } else {
-    unsigned SecNdx = PltSymTable - &cantFail(Obj->sections()).front();
-    return createError(
-        "unable to get a string table for the symbol table with index " +
-        Twine(SecNdx) + ": " + toString(StrTabOrErr.takeError()));
-  }
+  else
+    return createError("unable to get a string table for the " +
+                       describe(Obj, *PltSymTable) + ": " +
+                       toString(StrTabOrErr.takeError()));
 
   return Error::success();
 }
@@ -3587,15 +3557,13 @@ template <class ELFT> void GNUStyle<ELFT>::printGroupSections(const ELFO *Obj) {
        << "   [Index]    Name\n";
     for (const GroupMember &GM : G.Members) {
       const GroupSection *MainGroup = Map[GM.Index];
-      if (MainGroup != &G) {
-        OS.flush();
-        errs() << "Error: section [" << format_decimal(GM.Index, 5)
-               << "] in group section [" << format_decimal(G.Index, 5)
-               << "] already in group section ["
-               << format_decimal(MainGroup->Index, 5) << "]";
-        errs().flush();
-        continue;
-      }
+      if (MainGroup != &G)
+        this->reportUniqueWarning(
+            createError("section with index " + Twine(GM.Index) +
+                        ", included in the group section with index " +
+                        Twine(MainGroup->Index) +
+                        ", was also found in the group section with index " +
+                        Twine(G.Index)));
       OS << "   [" << format_decimal(GM.Index, 5) << "]   " << GM.Name << "\n";
     }
   }
@@ -3605,9 +3573,10 @@ template <class ELFT> void GNUStyle<ELFT>::printGroupSections(const ELFO *Obj) {
 }
 
 template <class ELFT>
+template <class RelTy>
 void GNUStyle<ELFT>::printRelocation(const ELFO *Obj, unsigned SecIndex,
-                                     const Elf_Shdr *SymTab, const Elf_Rela &R,
-                                     unsigned RelIndex, bool IsRela) {
+                                     const Elf_Shdr *SymTab, const RelTy &R,
+                                     unsigned RelIndex) {
   Expected<std::pair<const typename ELFT::Sym *, std::string>> Target =
       this->dumper()->getRelocationTarget(SymTab, R);
   if (!Target)
@@ -3615,14 +3584,23 @@ void GNUStyle<ELFT>::printRelocation(const ELFO *Obj, unsigned SecIndex,
         "unable to print relocation " + Twine(RelIndex) + " in section " +
         Twine(SecIndex) + ": " + toString(Target.takeError())));
   else
-    printRelocation(Obj, /*Sym=*/Target->first, /*Name=*/Target->second, R,
-                    IsRela);
+    printRelocation(Obj, /*Sym=*/Target->first, /*Name=*/Target->second, R);
+}
+
+template <class ELFT>
+static Optional<int64_t> getAddend(const typename ELFT::Rela &R) {
+  return (int64_t)R.r_addend;
+}
+
+template <class ELFT>
+static Optional<int64_t> getAddend(const typename ELFT::Rel &) {
+  return None;
 }
 
 template <class ELFT>
+template <class RelTy>
 void GNUStyle<ELFT>::printRelocation(const ELFO *Obj, const Elf_Sym *Sym,
-                                     StringRef SymbolName, const Elf_Rela &R,
-                                     bool IsRela) {
+                                     StringRef SymbolName, const RelTy &R) {
   // First two fields are bit width dependent. The rest of them are fixed width.
   unsigned Bias = ELFT::Is64Bits ? 8 : 0;
   Field Fields[5] = {0, 10 + Bias, 19 + 2 * Bias, 42 + 2 * Bias, 53 + 2 * Bias};
@@ -3643,16 +3621,16 @@ void GNUStyle<ELFT>::printRelocation(const ELFO *Obj, const Elf_Sym *Sym,
     printField(F);
 
   std::string Addend;
-  if (IsRela) {
-    int64_t RelAddend = R.r_addend;
+  if (Optional<int64_t> A = getAddend<ELFT>(R)) {
+    int64_t RelAddend = *A;
     if (!SymbolName.empty()) {
-      if (R.r_addend < 0) {
+      if (RelAddend < 0) {
         Addend = " - ";
         RelAddend = std::abs(RelAddend);
-      } else
+      } else {
         Addend = " + ";
+      }
     }
-
     Addend += to_hexString(RelAddend, false);
   }
   OS << Addend << "\n";
@@ -3698,14 +3676,14 @@ template <class ELFT> void GNUStyle<ELFT>::printRelocations(const ELFO *Obj) {
       AndroidRelas = unwrapOrError(this->FileName, Obj->android_relas(&Sec));
       Entries = AndroidRelas.size();
     }
-    std::vector<Elf_Rela> RelrRelas;
+    std::vector<Elf_Rel> RelrRels;
     if (!opts::RawRelr && (Sec.sh_type == ELF::SHT_RELR ||
                            Sec.sh_type == ELF::SHT_ANDROID_RELR)) {
       // .relr.dyn relative relocation section needs to be unpacked first
       // to get the actual number of entries.
       Elf_Relr_Range Relrs = unwrapOrError(this->FileName, Obj->relrs(&Sec));
-      RelrRelas = unwrapOrError(this->FileName, Obj->decode_relrs(Relrs));
-      Entries = RelrRelas.size();
+      RelrRels = unwrapOrError(this->FileName, Obj->decode_relrs(Relrs));
+      Entries = RelrRels.size();
     }
     uintX_t Offset = Sec.sh_offset;
     OS << "\nRelocation section '" << Name << "' at offset 0x"
@@ -3719,33 +3697,31 @@ template <class ELFT> void GNUStyle<ELFT>::printRelocations(const ELFO *Obj) {
 
     switch (Sec.sh_type) {
     case ELF::SHT_REL:
-      for (const auto &R : unwrapOrError(this->FileName, Obj->rels(&Sec))) {
-        Elf_Rela Rela;
-        Rela.r_offset = R.r_offset;
-        Rela.r_info = R.r_info;
-        Rela.r_addend = 0;
-        printRelocation(Obj, SecNdx, SymTab, Rela, ++RelNdx, false);
-      }
+      for (const Elf_Rel &R : unwrapOrError(this->FileName, Obj->rels(&Sec)))
+        printRelocation(Obj, SecNdx, SymTab, R, ++RelNdx);
       break;
     case ELF::SHT_RELA:
-      for (const auto &R : unwrapOrError(this->FileName, Obj->relas(&Sec)))
-        printRelocation(Obj, SecNdx, SymTab, R, ++RelNdx, true);
+      for (const Elf_Rela &R : unwrapOrError(this->FileName, Obj->relas(&Sec)))
+        printRelocation(Obj, SecNdx, SymTab, R, ++RelNdx);
       break;
     case ELF::SHT_RELR:
     case ELF::SHT_ANDROID_RELR:
       if (opts::RawRelr)
-        for (const auto &R : unwrapOrError(this->FileName, Obj->relrs(&Sec)))
+        for (const Elf_Relr &R :
+             unwrapOrError(this->FileName, Obj->relrs(&Sec)))
           OS << to_string(format_hex_no_prefix(R, ELFT::Is64Bits ? 16 : 8))
              << "\n";
       else
-        for (const auto &R : RelrRelas)
-          printRelocation(Obj, SecNdx, SymTab, R, ++RelNdx, false);
+        for (const Elf_Rel &R : RelrRels)
+          printRelocation(Obj, SecNdx, SymTab, R, ++RelNdx);
       break;
     case ELF::SHT_ANDROID_REL:
+      for (const Elf_Rela &R : AndroidRelas)
+        printRelocation(Obj, SecNdx, SymTab, (const Elf_Rel &)R, ++RelNdx);
+      break;
     case ELF::SHT_ANDROID_RELA:
-      for (const auto &R : AndroidRelas)
-        printRelocation(Obj, SecNdx, SymTab, R, ++RelNdx,
-                        Sec.sh_type == ELF::SHT_ANDROID_RELA);
+      for (const Elf_Rela &R : AndroidRelas)
+        printRelocation(Obj, SecNdx, SymTab, R, ++RelNdx);
       break;
     }
   }
@@ -3767,122 +3743,28 @@ static std::string getSectionTypeOffsetString(unsigned Type) {
   return "0x" + to_hexString(Type) + ": <unknown>";
 }
 
-static std::string getSectionTypeString(unsigned Arch, unsigned Type) {
-  using namespace ELF;
+static std::string getSectionTypeString(unsigned Machine, unsigned Type) {
+  StringRef Name = getELFSectionTypeName(Machine, Type);
 
-  switch (Arch) {
-  case EM_ARM:
-    switch (Type) {
-    case SHT_ARM_EXIDX:
-      return "ARM_EXIDX";
-    case SHT_ARM_PREEMPTMAP:
-      return "ARM_PREEMPTMAP";
-    case SHT_ARM_ATTRIBUTES:
-      return "ARM_ATTRIBUTES";
-    case SHT_ARM_DEBUGOVERLAY:
-      return "ARM_DEBUGOVERLAY";
-    case SHT_ARM_OVERLAYSECTION:
-      return "ARM_OVERLAYSECTION";
-    }
-    break;
-  case EM_X86_64:
-    switch (Type) {
-    case SHT_X86_64_UNWIND:
-      return "X86_64_UNWIND";
-    }
-    break;
-  case EM_MIPS:
-  case EM_MIPS_RS3_LE:
-    switch (Type) {
-    case SHT_MIPS_REGINFO:
-      return "MIPS_REGINFO";
-    case SHT_MIPS_OPTIONS:
-      return "MIPS_OPTIONS";
-    case SHT_MIPS_DWARF:
-      return "MIPS_DWARF";
-    case SHT_MIPS_ABIFLAGS:
-      return "MIPS_ABIFLAGS";
-    }
-    break;
-  case EM_RISCV:
-    switch (Type) {
-    case SHT_RISCV_ATTRIBUTES:
-      return "RISCV_ATTRIBUTES";
-    }
+  // Handle SHT_GNU_* type names.
+  if (Name.startswith("SHT_GNU_")) {
+    if (Name == "SHT_GNU_HASH")
+      return "GNU_HASH";
+    // E.g. SHT_GNU_verneed -> VERNEED.
+    return Name.drop_front(8).upper();
   }
-  switch (Type) {
-  case SHT_NULL:
-    return "NULL";
-  case SHT_PROGBITS:
-    return "PROGBITS";
-  case SHT_SYMTAB:
-    return "SYMTAB";
-  case SHT_STRTAB:
-    return "STRTAB";
-  case SHT_RELA:
-    return "RELA";
-  case SHT_HASH:
-    return "HASH";
-  case SHT_DYNAMIC:
-    return "DYNAMIC";
-  case SHT_NOTE:
-    return "NOTE";
-  case SHT_NOBITS:
-    return "NOBITS";
-  case SHT_REL:
-    return "REL";
-  case SHT_SHLIB:
-    return "SHLIB";
-  case SHT_DYNSYM:
-    return "DYNSYM";
-  case SHT_INIT_ARRAY:
-    return "INIT_ARRAY";
-  case SHT_FINI_ARRAY:
-    return "FINI_ARRAY";
-  case SHT_PREINIT_ARRAY:
-    return "PREINIT_ARRAY";
-  case SHT_GROUP:
-    return "GROUP";
-  case SHT_SYMTAB_SHNDX:
+
+  if (Name == "SHT_SYMTAB_SHNDX")
     return "SYMTAB SECTION INDICES";
-  case SHT_ANDROID_REL:
-    return "ANDROID_REL";
-  case SHT_ANDROID_RELA:
-    return "ANDROID_RELA";
-  case SHT_RELR:
-  case SHT_ANDROID_RELR:
+
+  // The SHT_ANDROID_RELR is special, all other SHT_ANDROID_* types are handled
+  // in the common block below.
+  if (Name == "SHT_ANDROID_RELR")
     return "RELR";
-  case SHT_LLVM_ODRTAB:
-    return "LLVM_ODRTAB";
-  case SHT_LLVM_LINKER_OPTIONS:
-    return "LLVM_LINKER_OPTIONS";
-  case SHT_LLVM_CALL_GRAPH_PROFILE:
-    return "LLVM_CALL_GRAPH_PROFILE";
-  case SHT_LLVM_ADDRSIG:
-    return "LLVM_ADDRSIG";
-  case SHT_LLVM_DEPENDENT_LIBRARIES:
-    return "LLVM_DEPENDENT_LIBRARIES";
-  case SHT_LLVM_SYMPART:
-    return "LLVM_SYMPART";
-  case SHT_LLVM_PART_EHDR:
-    return "LLVM_PART_EHDR";
-  case SHT_LLVM_PART_PHDR:
-    return "LLVM_PART_PHDR";
-  // FIXME: Parse processor specific GNU attributes
-  case SHT_GNU_ATTRIBUTES:
-    return "ATTRIBUTES";
-  case SHT_GNU_HASH:
-    return "GNU_HASH";
-  case SHT_GNU_verdef:
-    return "VERDEF";
-  case SHT_GNU_verneed:
-    return "VERNEED";
-  case SHT_GNU_versym:
-    return "VERSYM";
-  default:
-    return getSectionTypeOffsetString(Type);
-  }
-  return "";
+
+  if (Name.startswith("SHT_"))
+    return Name.drop_front(4).str();
+  return getSectionTypeOffsetString(Type);
 }
 
 static void printSectionDescription(formatted_raw_ostream &OS,
@@ -3967,9 +3849,21 @@ void GNUStyle<ELFT>::printSectionHeaders(const ELFO *Obj) {
 }
 
 template <class ELFT>
-void GNUStyle<ELFT>::printSymtabMessage(const ELFO *Obj, StringRef Name,
+void GNUStyle<ELFT>::printSymtabMessage(const ELFO *Obj, const Elf_Shdr *Symtab,
                                         size_t Entries,
                                         bool NonVisibilityBitsUsed) {
+  StringRef Name;
+  if (Symtab) {
+    if (Expected<StringRef> NameOrErr = Obj->getSectionName(Symtab)) {
+      Name = *NameOrErr;
+    } else {
+      this->reportUniqueWarning(createError("unable to get the name of " +
+                                            describe(Obj, *Symtab) + ": " +
+                                            toString(NameOrErr.takeError())));
+      Name = "<?>";
+    }
+  }
+
   if (!Name.empty())
     OS << "\nSymbol table '" << Name << "'";
   else
@@ -4451,10 +4345,10 @@ template <class ELFT> struct RelSymbol {
   std::string Name;
 };
 
-template <class ELFT>
+template <class ELFT, class RelTy>
 RelSymbol<ELFT> getSymbolForReloc(const ELFFile<ELFT> *Obj, StringRef FileName,
                                   const ELFDumper<ELFT> *Dumper,
-                                  const typename ELFT::Rela &Reloc) {
+                                  const RelTy &Reloc) {
   uint32_t SymIndex = Reloc.getSymbol(Obj->isMips64EL());
   auto WarnAndReturn = [&](const typename ELFT::Sym *Sym,
                            const Twine &Reason) -> RelSymbol<ELFT> {
@@ -4489,10 +4383,10 @@ RelSymbol<ELFT> getSymbolForReloc(const ELFFile<ELFT> *Obj, StringRef FileName,
 } // namespace
 
 template <class ELFT>
-void GNUStyle<ELFT>::printDynamicRelocation(const ELFO *Obj, Elf_Rela R,
-                                            bool IsRela) {
+template <class RelTy>
+void GNUStyle<ELFT>::printDynamicRelocation(const ELFO *Obj, const RelTy &R) {
   RelSymbol<ELFT> S = getSymbolForReloc(Obj, this->FileName, this->dumper(), R);
-  printRelocation(Obj, S.Sym, S.Name, R, IsRela);
+  printRelocation(Obj, S.Sym, S.Name, R);
 }
 
 template <class ELFT>
@@ -4551,7 +4445,7 @@ void GNUStyle<ELFT>::printDynamicRelocations(const ELFO *Obj) {
        << " contains " << DynRelaRegion.Size << " bytes:\n";
     printRelocHeader(ELF::SHT_RELA);
     for (const Elf_Rela &Rela : this->dumper()->dyn_relas())
-      printDynamicRelocation(Obj, Rela, true);
+      printDynamicRelocation(Obj, Rela);
   }
   if (DynRelRegion.Size > 0) {
     OS << "\n'REL' relocation section at offset "
@@ -4560,13 +4454,8 @@ void GNUStyle<ELFT>::printDynamicRelocations(const ELFO *Obj) {
                      1)
        << " contains " << DynRelRegion.Size << " bytes:\n";
     printRelocHeader(ELF::SHT_REL);
-    for (const Elf_Rel &Rel : this->dumper()->dyn_rels()) {
-      Elf_Rela Rela;
-      Rela.r_offset = Rel.r_offset;
-      Rela.r_info = Rel.r_info;
-      Rela.r_addend = 0;
-      printDynamicRelocation(Obj, Rela, false);
-    }
+    for (const Elf_Rel &Rel : this->dumper()->dyn_rels())
+      printDynamicRelocation(Obj, Rel);
   }
   if (DynRelrRegion.Size > 0) {
     OS << "\n'RELR' relocation section at offset "
@@ -4576,11 +4465,10 @@ void GNUStyle<ELFT>::printDynamicRelocations(const ELFO *Obj) {
        << " contains " << DynRelrRegion.Size << " bytes:\n";
     printRelocHeader(ELF::SHT_REL);
     Elf_Relr_Range Relrs = this->dumper()->dyn_relrs();
-    std::vector<Elf_Rela> RelrRelas =
+    std::vector<Elf_Rel> RelrRels =
         unwrapOrError(this->FileName, Obj->decode_relrs(Relrs));
-    for (const Elf_Rela &Rela : RelrRelas) {
-      printDynamicRelocation(Obj, Rela, false);
-    }
+    for (const Elf_Rel &R : RelrRels)
+      printDynamicRelocation(Obj, R);
   }
   if (DynPLTRelRegion.Size) {
     OS << "\n'PLT' relocation section at offset "
@@ -4592,16 +4480,11 @@ void GNUStyle<ELFT>::printDynamicRelocations(const ELFO *Obj) {
     if (DynPLTRelRegion.EntSize == sizeof(Elf_Rela)) {
       printRelocHeader(ELF::SHT_RELA);
       for (const Elf_Rela &Rela : DynPLTRelRegion.getAsArrayRef<Elf_Rela>())
-        printDynamicRelocation(Obj, Rela, true);
+        printDynamicRelocation(Obj, Rela);
     } else {
       printRelocHeader(ELF::SHT_REL);
-      for (const Elf_Rel &Rel : DynPLTRelRegion.getAsArrayRef<Elf_Rel>()) {
-        Elf_Rela Rela;
-        Rela.r_offset = Rel.r_offset;
-        Rela.r_info = Rel.r_info;
-        Rela.r_addend = 0;
-        printDynamicRelocation(Obj, Rela, false);
-      }
+      for (const Elf_Rel &Rel : DynPLTRelRegion.getAsArrayRef<Elf_Rel>())
+        printDynamicRelocation(Obj, Rel);
     }
   }
 }
@@ -4614,21 +4497,16 @@ void GNUStyle<ELFT>::printGNUVersionSectionProlog(
   OS << Label << " section '" << SecName << "' "
      << "contains " << EntriesNum << " entries:\n";
 
-  unsigned SecNdx = Sec - &cantFail(Obj->sections()).front();
   StringRef SymTabName = "<corrupt>";
-
   Expected<const typename ELFT::Shdr *> SymTabOrErr =
       Obj->getSection(Sec->sh_link);
   if (SymTabOrErr)
     SymTabName =
         unwrapOrError(this->FileName, Obj->getSectionName(*SymTabOrErr));
   else
-    this->reportUniqueWarning(
-        createError("invalid section linked to " +
-                    object::getELFSectionTypeName(Obj->getHeader()->e_machine,
-                                                  Sec->sh_type) +
-                    " section with index " + Twine(SecNdx) + ": " +
-                    toString(SymTabOrErr.takeError())));
+    this->reportUniqueWarning(createError("invalid section linked to " +
+                                          describe(Obj, *Sec) + ": " +
+                                          toString(SymTabOrErr.takeError())));
 
   OS << " Addr: " << format_hex_no_prefix(Sec->sh_addr, 16)
      << "  Offset: " << format_hex(Sec->sh_offset, 8)
@@ -4664,13 +4542,10 @@ void GNUStyle<ELFT>::printVersionSymbolSection(const ELFFile<ELFT> *Obj,
     Expected<StringRef> NameOrErr =
         this->dumper()->getSymbolVersionByIndex(Ndx, IsDefault);
     if (!NameOrErr) {
-      if (!NameOrErr) {
-        unsigned SecNdx = Sec - &cantFail(Obj->sections()).front();
+      if (!NameOrErr)
         this->reportUniqueWarning(createError(
-            "unable to get a version for entry " + Twine(I) +
-            " of SHT_GNU_versym section with index " + Twine(SecNdx) + ": " +
-            toString(NameOrErr.takeError())));
-      }
+            "unable to get a version for entry " + Twine(I) + " of " +
+            describe(Obj, *Sec) + ": " + toString(NameOrErr.takeError())));
       Versions.emplace_back("<corrupt>");
       continue;
     }
@@ -6052,20 +5927,35 @@ void GNUStyle<ELFT>::printMipsPLT(const MipsGOTParser<ELFT> &Parser) {
 }
 
 template <class ELFT>
-void GNUStyle<ELFT>::printMipsABIFlags(const ELFObjectFile<ELFT> *ObjF) {
+Expected<const Elf_Mips_ABIFlags<ELFT> *>
+getMipsAbiFlagsSection(const ELFObjectFile<ELFT> *ObjF) {
   const ELFFile<ELFT> *Obj = ObjF->getELFFile();
-  const Elf_Shdr *Shdr =
+  const typename ELFT::Shdr *Shdr =
       findSectionByName(*Obj, ObjF->getFileName(), ".MIPS.abiflags");
   if (!Shdr)
-    return;
+    return nullptr;
 
-  ArrayRef<uint8_t> Sec =
-      unwrapOrError(ObjF->getFileName(), Obj->getSectionContents(Shdr));
-  if (Sec.size() != sizeof(Elf_Mips_ABIFlags<ELFT>))
-    reportError(createError(".MIPS.abiflags section has a wrong size"),
-                ObjF->getFileName());
+  constexpr StringRef ErrPrefix = "unable to read the .MIPS.abiflags section: ";
+  Expected<ArrayRef<uint8_t>> DataOrErr = Obj->getSectionContents(Shdr);
+  if (!DataOrErr)
+    return createError(ErrPrefix + toString(DataOrErr.takeError()));
+
+  if (DataOrErr->size() != sizeof(Elf_Mips_ABIFlags<ELFT>))
+    return createError(ErrPrefix + "it has a wrong size (" +
+        Twine(DataOrErr->size()) + ")");
+  return reinterpret_cast<const Elf_Mips_ABIFlags<ELFT> *>(DataOrErr->data());
+}
 
-  auto *Flags = reinterpret_cast<const Elf_Mips_ABIFlags<ELFT> *>(Sec.data());
+template <class ELFT>
+void GNUStyle<ELFT>::printMipsABIFlags(const ELFObjectFile<ELFT> *ObjF) {
+  const Elf_Mips_ABIFlags<ELFT> *Flags = nullptr;
+  if (Expected<const Elf_Mips_ABIFlags<ELFT> *> SecOrErr =
+          getMipsAbiFlagsSection(ObjF))
+    Flags = *SecOrErr;
+  else
+    this->reportUniqueWarning(SecOrErr.takeError());
+  if (!Flags)
+    return;
 
   OS << "MIPS ABI Flags Version: " << Flags->version << "\n\n";
   OS << "ISA: MIPS" << int(Flags->isa_level);
@@ -6167,15 +6057,13 @@ void LLVMStyle<ELFT>::printGroupSections(const ELFO *Obj) {
     ListScope L(W, "Section(s) in group");
     for (const GroupMember &GM : G.Members) {
       const GroupSection *MainGroup = Map[GM.Index];
-      if (MainGroup != &G) {
-        W.flush();
-        errs() << "Error: " << GM.Name << " (" << GM.Index
-               << ") in a group " + G.Name + " (" << G.Index
-               << ") is already in a group " + MainGroup->Name + " ("
-               << MainGroup->Index << ")\n";
-        errs().flush();
-        continue;
-      }
+      if (MainGroup != &G)
+        this->reportUniqueWarning(
+            createError("section with index " + Twine(GM.Index) +
+                        ", included in the group section with index " +
+                        Twine(MainGroup->Index) +
+                        ", was also found in the group section with index " +
+                        Twine(G.Index)));
       W.startLine() << GM.Name << " (" << GM.Index << ")\n";
     }
   }
@@ -6218,13 +6106,8 @@ void LLVMStyle<ELFT>::printRelocations(const Elf_Shdr *Sec, const ELFO *Obj) {
 
   switch (Sec->sh_type) {
   case ELF::SHT_REL:
-    for (const Elf_Rel &R : unwrapOrError(this->FileName, Obj->rels(Sec))) {
-      Elf_Rela Rela;
-      Rela.r_offset = R.r_offset;
-      Rela.r_info = R.r_info;
-      Rela.r_addend = 0;
-      printRelocation(Obj, SecNdx, Rela, ++RelNdx, SymTab);
-    }
+    for (const Elf_Rel &R : unwrapOrError(this->FileName, Obj->rels(Sec)))
+      printRelocation(Obj, SecNdx, R, ++RelNdx, SymTab);
     break;
   case ELF::SHT_RELA:
     for (const Elf_Rela &R : unwrapOrError(this->FileName, Obj->relas(Sec)))
@@ -6237,9 +6120,9 @@ void LLVMStyle<ELFT>::printRelocations(const Elf_Shdr *Sec, const ELFO *Obj) {
       for (const Elf_Relr &R : Relrs)
         W.startLine() << W.hex(R) << "\n";
     } else {
-      std::vector<Elf_Rela> RelrRelas =
+      std::vector<Elf_Rel> RelrRels =
           unwrapOrError(this->FileName, Obj->decode_relrs(Relrs));
-      for (const Elf_Rela &R : RelrRelas)
+      for (const Elf_Rel &R : RelrRels)
         printRelocation(Obj, SecNdx, R, ++RelNdx, SymTab);
     }
     break;
@@ -6254,8 +6137,9 @@ void LLVMStyle<ELFT>::printRelocations(const Elf_Shdr *Sec, const ELFO *Obj) {
 }
 
 template <class ELFT>
+template <class RelTy>
 void LLVMStyle<ELFT>::printRelocation(const ELFO *Obj, unsigned SecIndex,
-                                      Elf_Rela Rel, unsigned RelIndex,
+                                      const RelTy &Rel, unsigned RelIndex,
                                       const Elf_Shdr *SymTab) {
   Expected<std::pair<const typename ELFT::Sym *, std::string>> Target =
       this->dumper()->getRelocationTarget(SymTab, Rel);
@@ -6270,17 +6154,18 @@ void LLVMStyle<ELFT>::printRelocation(const ELFO *Obj, unsigned SecIndex,
   SmallString<32> RelocName;
   Obj->getRelocationTypeName(Rel.getType(Obj->isMips64EL()), RelocName);
 
+  uintX_t Addend = getAddend<ELFT>(Rel).getValueOr(0);
   if (opts::ExpandRelocs) {
     DictScope Group(W, "Relocation");
     W.printHex("Offset", Rel.r_offset);
     W.printNumber("Type", RelocName, (int)Rel.getType(Obj->isMips64EL()));
     W.printNumber("Symbol", !TargetName.empty() ? TargetName : "-",
                   Rel.getSymbol(Obj->isMips64EL()));
-    W.printHex("Addend", Rel.r_addend);
+    W.printHex("Addend", Addend);
   } else {
     raw_ostream &OS = W.startLine();
     OS << W.hex(Rel.r_offset) << " " << RelocName << " "
-       << (!TargetName.empty() ? TargetName : "-") << " " << W.hex(Rel.r_addend)
+       << (!TargetName.empty() ? TargetName : "-") << " " << W.hex(Addend)
        << "\n";
   }
 }
@@ -6484,54 +6369,47 @@ void LLVMStyle<ELFT>::printDynamicRelocations(const ELFO *Obj) {
       printDynamicRelocation(Obj, Rela);
   }
   if (DynRelRegion.Size > 0) {
-    for (const Elf_Rel &Rel : this->dumper()->dyn_rels()) {
-      Elf_Rela Rela;
-      Rela.r_offset = Rel.r_offset;
-      Rela.r_info = Rel.r_info;
-      Rela.r_addend = 0;
-      printDynamicRelocation(Obj, Rela);
-    }
+    for (const Elf_Rel &Rel : this->dumper()->dyn_rels())
+      printDynamicRelocation(Obj, Rel);
   }
 
   if (DynRelrRegion.Size > 0) {
     Elf_Relr_Range Relrs = this->dumper()->dyn_relrs();
-    std::vector<Elf_Rela> RelrRelas =
+    std::vector<Elf_Rel> RelrRels =
         unwrapOrError(this->FileName, Obj->decode_relrs(Relrs));
-    for (const Elf_Rela &Rela : RelrRelas)
-      printDynamicRelocation(Obj, Rela);
+    for (const Elf_Rel &R : RelrRels)
+      printDynamicRelocation(Obj, R);
   }
   if (DynPLTRelRegion.EntSize == sizeof(Elf_Rela))
     for (const Elf_Rela &Rela : DynPLTRelRegion.getAsArrayRef<Elf_Rela>())
       printDynamicRelocation(Obj, Rela);
   else
-    for (const Elf_Rel &Rel : DynPLTRelRegion.getAsArrayRef<Elf_Rel>()) {
-      Elf_Rela Rela;
-      Rela.r_offset = Rel.r_offset;
-      Rela.r_info = Rel.r_info;
-      Rela.r_addend = 0;
-      printDynamicRelocation(Obj, Rela);
-    }
+    for (const Elf_Rel &Rel : DynPLTRelRegion.getAsArrayRef<Elf_Rel>())
+      printDynamicRelocation(Obj, Rel);
+
   W.unindent();
   W.startLine() << "}\n";
 }
 
 template <class ELFT>
-void LLVMStyle<ELFT>::printDynamicRelocation(const ELFO *Obj, Elf_Rela Rel) {
+ template <class RelTy>
+void LLVMStyle<ELFT>::printDynamicRelocation(const ELFO *Obj, const RelTy& Rel) {
   SmallString<32> RelocName;
   Obj->getRelocationTypeName(Rel.getType(Obj->isMips64EL()), RelocName);
   std::string SymbolName =
       getSymbolForReloc(Obj, this->FileName, this->dumper(), Rel).Name;
 
+  uintX_t Addend = getAddend<ELFT>(Rel).getValueOr(0);
   if (opts::ExpandRelocs) {
     DictScope Group(W, "Relocation");
     W.printHex("Offset", Rel.r_offset);
     W.printNumber("Type", RelocName, (int)Rel.getType(Obj->isMips64EL()));
     W.printString("Symbol", !SymbolName.empty() ? SymbolName : "-");
-    W.printHex("Addend", Rel.r_addend);
+    W.printHex("Addend", Addend);
   } else {
     raw_ostream &OS = W.startLine();
     OS << W.hex(Rel.r_offset) << " " << RelocName << " "
-       << (!SymbolName.empty() ? SymbolName : "-") << " " << W.hex(Rel.r_addend)
+       << (!SymbolName.empty() ? SymbolName : "-") << " " << W.hex(Addend)
        << "\n";
   }
 }
@@ -7044,22 +6922,19 @@ void LLVMStyle<ELFT>::printMipsPLT(const MipsGOTParser<ELFT> &Parser) {
 
 template <class ELFT>
 void LLVMStyle<ELFT>::printMipsABIFlags(const ELFObjectFile<ELFT> *ObjF) {
-  const ELFFile<ELFT> *Obj = ObjF->getELFFile();
-  const Elf_Shdr *Shdr =
-      findSectionByName(*Obj, ObjF->getFileName(), ".MIPS.abiflags");
-  if (!Shdr) {
-    W.startLine() << "There is no .MIPS.abiflags section in the file.\n";
-    return;
-  }
-  ArrayRef<uint8_t> Sec =
-      unwrapOrError(ObjF->getFileName(), Obj->getSectionContents(Shdr));
-  if (Sec.size() != sizeof(Elf_Mips_ABIFlags<ELFT>)) {
-    W.startLine() << "The .MIPS.abiflags section has a wrong size.\n";
+  const Elf_Mips_ABIFlags<ELFT> *Flags;
+  if (Expected<const Elf_Mips_ABIFlags<ELFT> *> SecOrErr =
+          getMipsAbiFlagsSection(ObjF)) {
+    Flags = *SecOrErr;
+    if (!Flags) {
+      W.startLine() << "There is no .MIPS.abiflags section in the file.\n";
+      return;
+    }
+  } else {
+    this->reportUniqueWarning(SecOrErr.takeError());
     return;
   }
 
-  auto *Flags = reinterpret_cast<const Elf_Mips_ABIFlags<ELFT> *>(Sec.data());
-
   raw_ostream &OS = W.getOStream();
   DictScope GS(W, "MIPS ABI Flags");
 
diff --git a/llvm/tools/llvm-readobj/llvm-readobj.cpp b/llvm/tools/llvm-readobj/llvm-readobj.cpp
index b9c6ad2256ae2..6b8a883af01df 100644
--- a/llvm/tools/llvm-readobj/llvm-readobj.cpp
+++ b/llvm/tools/llvm-readobj/llvm-readobj.cpp
@@ -690,6 +690,11 @@ int main(int argc, const char *argv[]) {
 
   cl::ParseCommandLineOptions(argc, argv, "LLVM Object Reader\n");
 
+  // Default to print error if no filename is specified.
+  if (opts::InputFilenames.empty()) {
+    error("no input files specified");
+  }
+
   if (opts::All) {
     opts::FileHeaders = true;
     opts::ProgramHeaders = true;
@@ -714,10 +719,6 @@ int main(int argc, const char *argv[]) {
     opts::SectionHeaders = true;
   }
 
-  // Default to stdin if no filename is specified.
-  if (opts::InputFilenames.empty())
-    opts::InputFilenames.push_back("-");
-
   ScopedPrinter Writer(fouts());
   for (const std::string &I : opts::InputFilenames)
     dumpInput(I, Writer);
diff --git a/llvm/tools/llvm-reduce/deltas/Delta.cpp b/llvm/tools/llvm-reduce/deltas/Delta.cpp
index 3c55adf64db70..c23d70145b69d 100644
--- a/llvm/tools/llvm-reduce/deltas/Delta.cpp
+++ b/llvm/tools/llvm-reduce/deltas/Delta.cpp
@@ -105,26 +105,28 @@ void llvm::runDeltaPass(
     }
   }
 
-  std::vector<Chunk> Chunks = {{1, Targets}};
-  std::set<Chunk> UninterestingChunks;
+  std::vector<Chunk> ChunksStillConsideredInteresting = {{1, Targets}};
   std::unique_ptr<Module> ReducedProgram;
 
-  if (!increaseGranularity(Chunks)) {
-    errs() << "\nAlready at minimum size. Cannot reduce anymore.\n";
-    return;
-  }
-
+  bool FoundAtLeastOneNewUninterestingChunkWithCurrentGranularity;
   do {
-    UninterestingChunks = {};
-    for (int I = Chunks.size() - 1; I >= 0; --I) {
+    FoundAtLeastOneNewUninterestingChunkWithCurrentGranularity = false;
+
+    std::set<Chunk> UninterestingChunks;
+    for (Chunk &ChunkToCheckForUninterestingness :
+         reverse(ChunksStillConsideredInteresting)) {
+      // Take all of ChunksStillConsideredInteresting chunks, except those we've
+      // already deemed uninteresting (UninterestingChunks) but didn't remove
+      // from ChunksStillConsideredInteresting yet, and additionally ignore
+      // ChunkToCheckForUninterestingness chunk.
       std::vector<Chunk> CurrentChunks;
-
-      for (auto C : Chunks)
-        if (!UninterestingChunks.count(C) && C != Chunks[I])
-          CurrentChunks.push_back(C);
-
-      if (CurrentChunks.empty())
-        continue;
+      CurrentChunks.reserve(ChunksStillConsideredInteresting.size() -
+                            UninterestingChunks.size() - 1);
+      copy_if(ChunksStillConsideredInteresting,
+              std::back_inserter(CurrentChunks), [&](const Chunk &C) {
+                return !UninterestingChunks.count(C) &&
+                       C != ChunkToCheckForUninterestingness;
+              });
 
       // Clone module before hacking it up..
       std::unique_ptr<Module> Clone = CloneModule(*Test.getProgram());
@@ -132,28 +134,30 @@ void llvm::runDeltaPass(
       ExtractChunksFromModule(CurrentChunks, Clone.get());
 
       errs() << "Ignoring: ";
-      Chunks[I].print();
-      for (auto C : UninterestingChunks)
+      ChunkToCheckForUninterestingness.print();
+      for (const Chunk &C : UninterestingChunks)
         C.print();
 
-
-
       SmallString<128> CurrentFilepath;
       if (!IsReduced(*Clone, Test, CurrentFilepath)) {
+        // Program became non-reduced, so this chunk appears to be interesting.
         errs() << "\n";
         continue;
       }
 
-      UninterestingChunks.insert(Chunks[I]);
+      FoundAtLeastOneNewUninterestingChunkWithCurrentGranularity = true;
+      UninterestingChunks.insert(ChunkToCheckForUninterestingness);
       ReducedProgram = std::move(Clone);
       errs() << " **** SUCCESS | lines: " << getLines(CurrentFilepath) << "\n";
     }
     // Delete uninteresting chunks
-    erase_if(Chunks, [&UninterestingChunks](const Chunk &C) {
-      return UninterestingChunks.count(C);
-    });
-
-  } while (!UninterestingChunks.empty() || increaseGranularity(Chunks));
+    erase_if(ChunksStillConsideredInteresting,
+             [&UninterestingChunks](const Chunk &C) {
+               return UninterestingChunks.count(C);
+             });
+  } while (!ChunksStillConsideredInteresting.empty() &&
+           (FoundAtLeastOneNewUninterestingChunkWithCurrentGranularity ||
+            increaseGranularity(ChunksStillConsideredInteresting)));
 
   // If we reduced the testcase replace it
   if (ReducedProgram)
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceArguments.cpp b/llvm/tools/llvm-reduce/deltas/ReduceArguments.cpp
index 88c3e326ff97d..e0e1d1c225670 100644
--- a/llvm/tools/llvm-reduce/deltas/ReduceArguments.cpp
+++ b/llvm/tools/llvm-reduce/deltas/ReduceArguments.cpp
@@ -73,7 +73,8 @@ static void extractArgumentsFromModule(std::vector<Chunk> ChunksToKeep,
         continue;
       auto *I = cast<Instruction>(V);
       I->replaceAllUsesWith(UndefValue::get(I->getType()));
-      I->eraseFromParent();
+      if (!I->isTerminator())
+        I->eraseFromParent();
     }
 
     // No arguments to reduce
diff --git a/llvm/tools/llvm-reduce/llvm-reduce.cpp b/llvm/tools/llvm-reduce/llvm-reduce.cpp
index 036d0ee75305e..9dd8aa4a938e7 100644
--- a/llvm/tools/llvm-reduce/llvm-reduce.cpp
+++ b/llvm/tools/llvm-reduce/llvm-reduce.cpp
@@ -28,30 +28,39 @@
 
 using namespace llvm;
 
-static cl::opt<bool> Help("h", cl::desc("Alias for -help"), cl::Hidden);
-static cl::opt<bool> Version("v", cl::desc("Alias for -version"), cl::Hidden);
+static cl::OptionCategory Options("llvm-reduce options");
+
+static cl::opt<bool> Help("h", cl::desc("Alias for -help"), cl::Hidden,
+                          cl::cat(Options));
+static cl::opt<bool> Version("v", cl::desc("Alias for -version"), cl::Hidden,
+                             cl::cat(Options));
 
 static cl::opt<std::string> InputFilename(cl::Positional, cl::Required,
-                                          cl::desc("<input llvm ll/bc file>"));
+                                          cl::desc("<input llvm ll/bc file>"),
+                                          cl::cat(Options));
 
 static cl::opt<std::string>
     TestFilename("test", cl::Required,
-                 cl::desc("Name of the interesting-ness test to be run"));
+                 cl::desc("Name of the interesting-ness test to be run"),
+                 cl::cat(Options));
 
 static cl::list<std::string>
     TestArguments("test-arg", cl::ZeroOrMore,
-                  cl::desc("Arguments passed onto the interesting-ness test"));
+                  cl::desc("Arguments passed onto the interesting-ness test"),
+                  cl::cat(Options));
 
 static cl::opt<std::string>
     OutputFilename("output",
                    cl::desc("Specify the output file. default: reduced.ll"));
 static cl::alias OutputFileAlias("o", cl::desc("Alias for -output"),
-                                 cl::aliasopt(OutputFilename));
+                                 cl::aliasopt(OutputFilename),
+                                 cl::cat(Options));
 
 static cl::opt<bool>
     ReplaceInput("in-place",
                  cl::desc("WARNING: This option will replace your input file "
-                          "with the reduced version!"));
+                          "with the reduced version!"),
+                 cl::cat(Options));
 
 // Parses IR into a Module and verifies it
 static std::unique_ptr<Module> parseInputFile(StringRef Filename,
diff --git a/llvm/tools/obj2yaml/dwarf2yaml.cpp b/llvm/tools/obj2yaml/dwarf2yaml.cpp
index a318a547e32c5..66264a46068a6 100644
--- a/llvm/tools/obj2yaml/dwarf2yaml.cpp
+++ b/llvm/tools/obj2yaml/dwarf2yaml.cpp
@@ -165,13 +165,13 @@ void dumpDebugPubSections(DWARFContext &DCtx, DWARFYAML::Data &Y) {
 void dumpDebugInfo(DWARFContext &DCtx, DWARFYAML::Data &Y) {
   for (const auto &CU : DCtx.compile_units()) {
     DWARFYAML::Unit NewUnit;
-    NewUnit.Format = CU->getFormat();
+    NewUnit.FormParams.Format = CU->getFormat();
     NewUnit.Length = CU->getLength();
-    NewUnit.Version = CU->getVersion();
-    if(NewUnit.Version >= 5)
+    NewUnit.FormParams.Version = CU->getVersion();
+    if (NewUnit.FormParams.Version >= 5)
       NewUnit.Type = (dwarf::UnitType)CU->getUnitType();
     NewUnit.AbbrOffset = CU->getAbbreviations()->getOffset();
-    NewUnit.AddrSize = CU->getAddressByteSize();
+    NewUnit.FormParams.AddrSize = CU->getAddressByteSize();
     for (auto DIE : CU->dies()) {
       DWARFYAML::Entry NewEntry;
       DataExtractor EntryData = CU->getDebugInfoExtractor();
diff --git a/llvm/tools/opt/NewPMDriver.cpp b/llvm/tools/opt/NewPMDriver.cpp
index 47c9cfc65e7ba..acaa57540a1cf 100644
--- a/llvm/tools/opt/NewPMDriver.cpp
+++ b/llvm/tools/opt/NewPMDriver.cpp
@@ -261,7 +261,7 @@ bool llvm::runPassPipeline(StringRef Arg0, Module &M, TargetMachine *TM,
     }
   }
   PassInstrumentationCallbacks PIC;
-  StandardInstrumentations SI;
+  StandardInstrumentations SI(DebugPM);
   SI.registerCallbacks(PIC);
 
   PipelineTuningOptions PTO;
diff --git a/llvm/tools/opt/opt.cpp b/llvm/tools/opt/opt.cpp
index 406684335673d..d3ac413eb1980 100644
--- a/llvm/tools/opt/opt.cpp
+++ b/llvm/tools/opt/opt.cpp
@@ -22,13 +22,11 @@
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/AsmParser/Parser.h"
-#include "llvm/Bitcode/BitcodeWriterPass.h"
 #include "llvm/CodeGen/CommandFlags.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DebugInfo.h"
-#include "llvm/IR/IRPrintingPasses.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/LLVMRemarkStreamer.h"
 #include "llvm/IR/LegacyPassManager.h"
@@ -325,48 +323,6 @@ cl::opt<std::string> CSProfileGenFile(
     cl::desc("Path to the instrumented context sensitive profile."),
     cl::Hidden);
 
-class OptCustomPassManager : public legacy::PassManager {
-  DebugifyStatsMap DIStatsMap;
-
-public:
-  using super = legacy::PassManager;
-
-  void add(Pass *P) override {
-    // Wrap each pass with (-check)-debugify passes if requested, making
-    // exceptions for passes which shouldn't see -debugify instrumentation.
-    bool WrapWithDebugify = DebugifyEach && !P->getAsImmutablePass() &&
-                            !isIRPrintingPass(P) && !isBitcodeWriterPass(P);
-    if (!WrapWithDebugify) {
-      super::add(P);
-      return;
-    }
-
-    // Apply -debugify/-check-debugify before/after each pass and collect
-    // debug info loss statistics.
-    PassKind Kind = P->getPassKind();
-    StringRef Name = P->getPassName();
-
-    // TODO: Implement Debugify for LoopPass.
-    switch (Kind) {
-      case PT_Function:
-        super::add(createDebugifyFunctionPass());
-        super::add(P);
-        super::add(createCheckDebugifyFunctionPass(true, Name, &DIStatsMap));
-        break;
-      case PT_Module:
-        super::add(createDebugifyModulePass());
-        super::add(P);
-        super::add(createCheckDebugifyModulePass(true, Name, &DIStatsMap));
-        break;
-      default:
-        super::add(P);
-        break;
-    }
-  }
-
-  const DebugifyStatsMap &getDebugifyStatsMap() const { return DIStatsMap; }
-};
-
 static inline void addPass(legacy::PassManagerBase &PM, Pass *P) {
   // Add the pass to the pass manager...
   PM.add(P);
@@ -587,6 +543,7 @@ int main(int argc, char **argv) {
   initializeSYCLLowerWGScopeLegacyPassPass(Registry);
   initializeSYCLLowerESIMDLegacyPassPass(Registry);
   initializeESIMDLowerLoadStorePass(Registry);
+  initializeESIMDLowerVecArgLegacyPassPass(Registry);
 
 #ifdef BUILD_EXAMPLES
   initializeExampleIRTransforms(Registry);
@@ -779,8 +736,12 @@ int main(int argc, char **argv) {
   }
 
   // Create a PassManager to hold and optimize the collection of passes we are
-  // about to build.
-  OptCustomPassManager Passes;
+  // about to build. If the -debugify-each option is set, wrap each pass with
+  // the (-check)-debugify passes.
+  DebugifyCustomPassManager Passes;
+  if (DebugifyEach)
+    Passes.enableDebugifyEach();
+
   bool AddOneTimeDebugifyPasses = EnableDebugify && !DebugifyEach;
 
   // Add an appropriate TargetLibraryInfo pass for the module's triple.
diff --git a/llvm/unittests/Analysis/CMakeLists.txt b/llvm/unittests/Analysis/CMakeLists.txt
index b1b396a897ca8..eb97f6289b67a 100644
--- a/llvm/unittests/Analysis/CMakeLists.txt
+++ b/llvm/unittests/Analysis/CMakeLists.txt
@@ -27,7 +27,7 @@ add_llvm_unittest_with_input_files(AnalysisTests
   DivergenceAnalysisTest.cpp
   DomTreeUpdaterTest.cpp
   GlobalsModRefTest.cpp
-  InlineFeaturesAnalysisTest.cpp
+  FunctionPropertiesAnalysisTest.cpp
   IVDescriptorsTest.cpp
   LazyCallGraphTest.cpp
   LoadsTest.cpp
diff --git a/llvm/unittests/Analysis/InlineFeaturesAnalysisTest.cpp b/llvm/unittests/Analysis/FunctionPropertiesAnalysisTest.cpp
similarity index 50%
rename from llvm/unittests/Analysis/InlineFeaturesAnalysisTest.cpp
rename to llvm/unittests/Analysis/FunctionPropertiesAnalysisTest.cpp
index a3df767b76507..7b2a985bba661 100644
--- a/llvm/unittests/Analysis/InlineFeaturesAnalysisTest.cpp
+++ b/llvm/unittests/Analysis/FunctionPropertiesAnalysisTest.cpp
@@ -1,4 +1,4 @@
-//===- InlineFeaturesAnalysisTest.cpp - inline features unit tests --------===//
+//===- FunctionPropertiesAnalysisTest.cpp - Function Properties Unit Tests-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,8 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Analysis/InlineFeaturesAnalysis.h"
+#include "llvm/Analysis/FunctionPropertiesAnalysis.h"
 #include "llvm/AsmParser/Parser.h"
+#include "llvm/IR/Dominators.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
@@ -15,42 +16,49 @@
 #include "gtest/gtest.h"
 
 using namespace llvm;
+namespace {
 
-static std::unique_ptr<Module> parseIR(LLVMContext &C, const char *IR) {
-  SMDiagnostic Err;
-  std::unique_ptr<Module> Mod = parseAssemblyString(IR, Err, C);
-  if (!Mod)
-    Err.print("MLAnalysisTests", errs());
-  return Mod;
-}
+class FunctionPropertiesAnalysisTest : public testing::Test {
+protected:
+  std::unique_ptr<DominatorTree> DT;
+  std::unique_ptr<LoopInfo> LI;
+
+  FunctionPropertiesInfo buildFPI(Function &F) {
+    DT.reset(new DominatorTree(F));
+    LI.reset(new LoopInfo(*DT));
+    return FunctionPropertiesInfo::getFunctionPropertiesInfo(F, *LI);
+  }
 
-TEST(InlineFeaturesTest, BasicTest) {
+  std::unique_ptr<Module> makeLLVMModule(LLVMContext &C, const char *IR) {
+    SMDiagnostic Err;
+    std::unique_ptr<Module> Mod = parseAssemblyString(IR, Err, C);
+    if (!Mod)
+      Err.print("MLAnalysisTests", errs());
+    return Mod;
+  }
+};
+
+TEST_F(FunctionPropertiesAnalysisTest, BasicTest) {
   LLVMContext C;
-  std::unique_ptr<Module> M = parseIR(C,
-                                      R"IR(
+  std::unique_ptr<Module> M = makeLLVMModule(C,
+                                             R"IR(
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-linux-gnu"
-
 declare i32 @f1(i32)
 declare i32 @f2(i32)
-
 define i32 @branches(i32) {
   %cond = icmp slt i32 %0, 3
   br i1 %cond, label %then, label %else
-
 then:
   %ret.1 = call i32 @f1(i32 %0)
   br label %last.block
-
 else:
   %ret.2 = call i32 @f2(i32 %0)
   br label %last.block
-
 last.block:
   %ret = phi i32 [%ret.1, %then], [%ret.2, %else]
   ret i32 %ret
 }
-
 define internal i32 @top() {
   %1 = call i32 @branches(i32 2)
   %2 = call i32 @f1(i32 %1)
@@ -58,20 +66,28 @@ define internal i32 @top() {
 }
 )IR");
 
-  FunctionAnalysisManager FAM;
-  InlineFeaturesAnalysis FA;
-
-  auto BranchesFeatures = FA.run(*M->getFunction("branches"), FAM);
+  Function *BranchesFunction = M->getFunction("branches");
+  FunctionPropertiesInfo BranchesFeatures = buildFPI(*BranchesFunction);
   EXPECT_EQ(BranchesFeatures.BasicBlockCount, 4);
   EXPECT_EQ(BranchesFeatures.BlocksReachedFromConditionalInstruction, 2);
-  EXPECT_EQ(BranchesFeatures.DirectCallsToDefinedFunctions, 0);
   // 2 Users: top is one. The other is added because @branches is not internal,
   // so it may have external callers.
   EXPECT_EQ(BranchesFeatures.Uses, 2);
+  EXPECT_EQ(BranchesFeatures.DirectCallsToDefinedFunctions, 0);
+  EXPECT_EQ(BranchesFeatures.LoadInstCount, 0);
+  EXPECT_EQ(BranchesFeatures.StoreInstCount, 0);
+  EXPECT_EQ(BranchesFeatures.MaxLoopDepth, 0);
+  EXPECT_EQ(BranchesFeatures.TopLevelLoopCount, 0);
 
-  auto TopFeatures = FA.run(*M->getFunction("top"), FAM);
+  Function *TopFunction = M->getFunction("top");
+  FunctionPropertiesInfo TopFeatures = buildFPI(*TopFunction);
   EXPECT_EQ(TopFeatures.BasicBlockCount, 1);
   EXPECT_EQ(TopFeatures.BlocksReachedFromConditionalInstruction, 0);
-  EXPECT_EQ(TopFeatures.DirectCallsToDefinedFunctions, 1);
   EXPECT_EQ(TopFeatures.Uses, 0);
+  EXPECT_EQ(TopFeatures.DirectCallsToDefinedFunctions, 1);
+  EXPECT_EQ(BranchesFeatures.LoadInstCount, 0);
+  EXPECT_EQ(BranchesFeatures.StoreInstCount, 0);
+  EXPECT_EQ(BranchesFeatures.MaxLoopDepth, 0);
+  EXPECT_EQ(BranchesFeatures.TopLevelLoopCount, 0);
 }
+} // end anonymous namespace
\ No newline at end of file
diff --git a/llvm/unittests/CMakeLists.txt b/llvm/unittests/CMakeLists.txt
index d7dbaeaa32fe8..e90f07448c100 100644
--- a/llvm/unittests/CMakeLists.txt
+++ b/llvm/unittests/CMakeLists.txt
@@ -14,6 +14,10 @@ function(add_llvm_target_unittest test_dir_name)
   add_llvm_unittest(${test_dir_name} DISABLE_LLVM_LINK_LLVM_DYLIB ${ARGN})
 endfunction()
 
+if (CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
+  add_compile_options("-Wno-suggest-override")
+endif()
+
 add_subdirectory(ADT)
 add_subdirectory(Analysis)
 add_subdirectory(AsmParser)
diff --git a/llvm/unittests/CodeGen/GlobalISel/GISelUtilsTest.cpp b/llvm/unittests/CodeGen/GlobalISel/GISelUtilsTest.cpp
index cd788bed91ef5..15d3bcb7b2163 100644
--- a/llvm/unittests/CodeGen/GlobalISel/GISelUtilsTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/GISelUtilsTest.cpp
@@ -13,13 +13,19 @@ using namespace llvm;
 
 namespace {
 static const LLT S1 = LLT::scalar(1);
+static const LLT S8 = LLT::scalar(8);
 static const LLT S16 = LLT::scalar(16);
 static const LLT S32 = LLT::scalar(32);
 static const LLT S64 = LLT::scalar(64);
 static const LLT P0 = LLT::pointer(0, 64);
 static const LLT P1 = LLT::pointer(1, 32);
 
+static const LLT V2S8 = LLT::vector(2, 8);
+static const LLT V4S8 = LLT::vector(4, 8);
+static const LLT V8S8 = LLT::vector(8, 8);
+
 static const LLT V2S16 = LLT::vector(2, 16);
+static const LLT V3S16 = LLT::vector(3, 16);
 static const LLT V4S16 = LLT::vector(4, 16);
 
 static const LLT V2S32 = LLT::vector(2, 32);
@@ -27,11 +33,18 @@ static const LLT V3S32 = LLT::vector(3, 32);
 static const LLT V4S32 = LLT::vector(4, 32);
 static const LLT V6S32 = LLT::vector(6, 32);
 
+static const LLT V2S64 = LLT::vector(2, 64);
+static const LLT V3S64 = LLT::vector(3, 64);
+static const LLT V4S64 = LLT::vector(4, 64);
+
 static const LLT V2P0 = LLT::vector(2, P0);
 static const LLT V3P0 = LLT::vector(3, P0);
 static const LLT V4P0 = LLT::vector(4, P0);
 static const LLT V6P0 = LLT::vector(6, P0);
 
+static const LLT V2P1 = LLT::vector(2, P1);
+static const LLT V4P1 = LLT::vector(4, P1);
+
 TEST(GISelUtilsTest, getGCDType) {
   EXPECT_EQ(S1, getGCDType(S1, S1));
   EXPECT_EQ(S32, getGCDType(S32, S32));
@@ -56,7 +69,7 @@ TEST(GISelUtilsTest, getGCDType) {
   EXPECT_EQ(S32, getGCDType(P0, S32));
   EXPECT_EQ(S32, getGCDType(S32, P0));
 
-  EXPECT_EQ(S64, getGCDType(P0, S64));
+  EXPECT_EQ(P0, getGCDType(P0, S64));
   EXPECT_EQ(S64, getGCDType(S64, P0));
 
   EXPECT_EQ(S32, getGCDType(P0, P1));
@@ -64,6 +77,76 @@ TEST(GISelUtilsTest, getGCDType) {
 
   EXPECT_EQ(P0, getGCDType(V3P0, V2P0));
   EXPECT_EQ(P0, getGCDType(V2P0, V3P0));
+
+  EXPECT_EQ(P0, getGCDType(P0, V2P0));
+  EXPECT_EQ(P0, getGCDType(V2P0, P0));
+
+
+  EXPECT_EQ(V2P0, getGCDType(V2P0, V2P0));
+  EXPECT_EQ(P0, getGCDType(V3P0, V2P0));
+  EXPECT_EQ(P0, getGCDType(V2P0, V3P0));
+  EXPECT_EQ(V2P0, getGCDType(V4P0, V2P0));
+
+  EXPECT_EQ(V2P0, getGCDType(V2P0, V4P1));
+  EXPECT_EQ(V4P1, getGCDType(V4P1, V2P0));
+
+  EXPECT_EQ(V2P0, getGCDType(V4P0, V4P1));
+  EXPECT_EQ(V4P1, getGCDType(V4P1, V4P0));
+
+  // Elements have same size, but have different pointeriness, so prefer the
+  // original element type.
+  EXPECT_EQ(V2P0, getGCDType(V2P0, V4S64));
+  EXPECT_EQ(V2S64, getGCDType(V4S64, V2P0));
+
+  EXPECT_EQ(V2S16, getGCDType(V2S16, V4P1));
+  EXPECT_EQ(P1, getGCDType(V4P1, V2S16));
+  EXPECT_EQ(V2P1, getGCDType(V4P1, V4S16));
+  EXPECT_EQ(V4S16, getGCDType(V4S16, V2P1));
+
+  EXPECT_EQ(P0, getGCDType(P0, V2S64));
+  EXPECT_EQ(S64, getGCDType(V2S64, P0));
+
+  EXPECT_EQ(S16, getGCDType(V2S16, V3S16));
+  EXPECT_EQ(S16, getGCDType(V3S16, V2S16));
+  EXPECT_EQ(S16, getGCDType(V3S16, S16));
+  EXPECT_EQ(S16, getGCDType(S16, V3S16));
+
+  EXPECT_EQ(V2S16, getGCDType(V2S16, V2S32));
+  EXPECT_EQ(S32, getGCDType(V2S32, V2S16));
+
+  EXPECT_EQ(V4S8, getGCDType(V4S8, V2S32));
+  EXPECT_EQ(S32, getGCDType(V2S32, V4S8));
+
+  // Test cases where neither element type nicely divides.
+  EXPECT_EQ(LLT::scalar(3), getGCDType(LLT::vector(3, 5), LLT::vector(2, 6)));
+  EXPECT_EQ(LLT::scalar(3), getGCDType(LLT::vector(2, 6), LLT::vector(3, 5)));
+
+  // Have to go smaller than a pointer element.
+  EXPECT_EQ(LLT::scalar(3), getGCDType(LLT::vector(2, LLT::pointer(3, 6)),
+                                       LLT::vector(3, 5)));
+  EXPECT_EQ(LLT::scalar(3), getGCDType(LLT::vector(3, 5),
+                                       LLT::vector(2, LLT::pointer(3, 6))));
+
+  EXPECT_EQ(V4S8, getGCDType(V4S8, S32));
+  EXPECT_EQ(S32, getGCDType(S32, V4S8));
+  EXPECT_EQ(V4S8, getGCDType(V4S8, P1));
+  EXPECT_EQ(P1, getGCDType(P1, V4S8));
+
+  EXPECT_EQ(V2S8, getGCDType(V2S8, V4S16));
+  EXPECT_EQ(S16, getGCDType(V4S16, V2S8));
+
+  EXPECT_EQ(S8, getGCDType(V2S8, LLT::vector(4, 2)));
+  EXPECT_EQ(LLT::vector(4, 2), getGCDType(LLT::vector(4, 2), S8));
+
+
+  EXPECT_EQ(LLT::pointer(4, 8), getGCDType(LLT::vector(2, LLT::pointer(4, 8)),
+                                           LLT::vector(4, 2)));
+
+  EXPECT_EQ(LLT::vector(4, 2), getGCDType(LLT::vector(4, 2),
+                                          LLT::vector(2, LLT::pointer(4, 8))));
+
+  EXPECT_EQ(LLT::scalar(4), getGCDType(LLT::vector(3, 4), S8));
+  EXPECT_EQ(LLT::scalar(4), getGCDType(S8, LLT::vector(3, 4)));
 }
 
 TEST(GISelUtilsTest, getLCMType) {
@@ -76,18 +159,18 @@ TEST(GISelUtilsTest, getLCMType) {
   EXPECT_EQ(S32, getLCMType(S16, S32));
 
   EXPECT_EQ(S64, getLCMType(S64, P0));
-  EXPECT_EQ(S64, getLCMType(P0, S64));
+  EXPECT_EQ(P0, getLCMType(P0, S64));
 
-  EXPECT_EQ(S64, getLCMType(S32, P0));
-  EXPECT_EQ(S64, getLCMType(P0, S32));
+  EXPECT_EQ(P0, getLCMType(S32, P0));
+  EXPECT_EQ(P0, getLCMType(P0, S32));
 
   EXPECT_EQ(S32, getLCMType(S32, P1));
-  EXPECT_EQ(S32, getLCMType(P1, S32));
-  EXPECT_EQ(S64, getLCMType(P0, P0));
-  EXPECT_EQ(S32, getLCMType(P1, P1));
+  EXPECT_EQ(P1, getLCMType(P1, S32));
+  EXPECT_EQ(P0, getLCMType(P0, P0));
+  EXPECT_EQ(P1, getLCMType(P1, P1));
 
-  EXPECT_EQ(S64, getLCMType(P0, P1));
-  EXPECT_EQ(S64, getLCMType(P1, P0));
+  EXPECT_EQ(P0, getLCMType(P0, P1));
+  EXPECT_EQ(P0, getLCMType(P1, P0));
 
   EXPECT_EQ(V2S32, getLCMType(V2S32, V2S32));
   EXPECT_EQ(V2S32, getLCMType(V2S32, S32));
@@ -107,11 +190,55 @@ TEST(GISelUtilsTest, getLCMType) {
   EXPECT_EQ(LLT::vector(12, P0), getLCMType(V4P0, V3P0));
   EXPECT_EQ(LLT::vector(12, P0), getLCMType(V3P0, V4P0));
 
-  // FIXME
-  // EXPECT_EQ(V2S32, getLCMType(V2S32, S64));
+  EXPECT_EQ(LLT::vector(12, S64), getLCMType(V4S64, V3P0));
+  EXPECT_EQ(LLT::vector(12, P0), getLCMType(V3P0, V4S64));
+
+  EXPECT_EQ(LLT::vector(12, P0), getLCMType(V4P0, V3S64));
+  EXPECT_EQ(LLT::vector(12, S64), getLCMType(V3S64, V4P0));
+
+  EXPECT_EQ(V2P0, getLCMType(V2P0, S32));
+  EXPECT_EQ(V4S32, getLCMType(S32, V2P0));
+  EXPECT_EQ(V2P0, getLCMType(V2P0, S64));
+  EXPECT_EQ(V2S64, getLCMType(S64, V2P0));
+
+
+  EXPECT_EQ(V2P0, getLCMType(V2P0, V2P1));
+  EXPECT_EQ(V4P1, getLCMType(V2P1, V2P0));
+
+  EXPECT_EQ(V2P0, getLCMType(V2P0, V4P1));
+  EXPECT_EQ(V4P1, getLCMType(V4P1, V2P0));
+
+
+  EXPECT_EQ(V2S32, getLCMType(V2S32, S64));
+  EXPECT_EQ(S64, getLCMType(S64, V2S32));
+
+  EXPECT_EQ(V4S16, getLCMType(V4S16, V2S32));
+  EXPECT_EQ(V2S32, getLCMType(V2S32, V4S16));
+
+  EXPECT_EQ(V2S32, getLCMType(V2S32, V4S8));
+  EXPECT_EQ(V8S8, getLCMType(V4S8, V2S32));
+
+  EXPECT_EQ(V2S16, getLCMType(V2S16, V4S8));
+  EXPECT_EQ(V4S8, getLCMType(V4S8, V2S16));
+
+  EXPECT_EQ(LLT::vector(6, S16), getLCMType(V3S16, V4S8));
+  EXPECT_EQ(LLT::vector(12, S8), getLCMType(V4S8, V3S16));
+  EXPECT_EQ(V4S16, getLCMType(V4S16, V4S8));
+  EXPECT_EQ(V8S8, getLCMType(V4S8, V4S16));
+
+  EXPECT_EQ(LLT::vector(6, 4), getLCMType(LLT::vector(3, 4), S8));
+  EXPECT_EQ(LLT::vector(3, 8), getLCMType(S8, LLT::vector(3, 4)));
+
+  EXPECT_EQ(LLT::vector(6, 4),
+            getLCMType(LLT::vector(3, 4), LLT::pointer(4, 8)));
+  EXPECT_EQ(LLT::vector(3, LLT::pointer(4, 8)),
+            getLCMType(LLT::pointer(4, 8), LLT::vector(3, 4)));
+
+  EXPECT_EQ(V2S64, getLCMType(V2S64, P0));
+  EXPECT_EQ(V2P0, getLCMType(P0, V2S64));
 
-  // FIXME
-  //EXPECT_EQ(S64, getLCMType(S64, V2S32));
+  EXPECT_EQ(V2S64, getLCMType(V2S64, P1));
+  EXPECT_EQ(V4P1, getLCMType(P1, V2S64));
 }
 
 }
diff --git a/llvm/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp b/llvm/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp
index 6825981de0c3c..6e5215659e9da 100644
--- a/llvm/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp
+++ b/llvm/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp
@@ -1368,8 +1368,7 @@ TEST(DWARFDebugInfo, TestEmptyChildren) {
                          "    Children:        DW_CHILDREN_yes\n"
                          "    Attributes:\n"
                          "debug_info:\n"
-                         "  - Length:          0\n"
-                         "    Version:         4\n"
+                         "  - Version:         4\n"
                          "    AbbrOffset:      0\n"
                          "    AddrSize:        8\n"
                          "    Entries:\n"
@@ -1378,7 +1377,7 @@ TEST(DWARFDebugInfo, TestEmptyChildren) {
                          "      - AbbrCode:        0x00000000\n"
                          "        Values:\n";
 
-  auto ErrOrSections = DWARFYAML::emitDebugSections(StringRef(yamldata), true);
+  auto ErrOrSections = DWARFYAML::emitDebugSections(StringRef(yamldata));
   ASSERT_TRUE((bool)ErrOrSections);
   std::unique_ptr<DWARFContext> DwarfContext =
       DWARFContext::create(*ErrOrSections, 8);
diff --git a/llvm/unittests/DebugInfo/DWARF/DWARFDieTest.cpp b/llvm/unittests/DebugInfo/DWARF/DWARFDieTest.cpp
index 983f0624939a2..9fb6a48157cfb 100644
--- a/llvm/unittests/DebugInfo/DWARF/DWARFDieTest.cpp
+++ b/llvm/unittests/DebugInfo/DWARF/DWARFDieTest.cpp
@@ -34,8 +34,7 @@ TEST(DWARFDie, getLocations) {
           - Attribute:       DW_AT_call_data_location
             Form:            DW_FORM_sec_offset
     debug_info:
-      - Length:          0
-        Version:         5
+      - Version:         5
         UnitType:        DW_UT_compile
         AbbrOffset:      0
         AddrSize:        4
@@ -49,7 +48,7 @@ TEST(DWARFDie, getLocations) {
               - Value:           25
   )";
   Expected<StringMap<std::unique_ptr<MemoryBuffer>>> Sections =
-      DWARFYAML::emitDebugSections(StringRef(yamldata), /*ApplyFixups=*/true,
+      DWARFYAML::emitDebugSections(StringRef(yamldata),
                                    /*IsLittleEndian=*/true);
   ASSERT_THAT_EXPECTED(Sections, Succeeded());
   std::vector<uint8_t> Loclists{
diff --git a/llvm/unittests/ExecutionEngine/Orc/LegacyCompileOnDemandLayerTest.cpp b/llvm/unittests/ExecutionEngine/Orc/LegacyCompileOnDemandLayerTest.cpp
index a13d8bdeeeb38..543c164e67c4e 100644
--- a/llvm/unittests/ExecutionEngine/Orc/LegacyCompileOnDemandLayerTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/LegacyCompileOnDemandLayerTest.cpp
@@ -16,10 +16,8 @@ using namespace llvm::orc;
 namespace {
 
 class DummyTrampolinePool : public orc::TrampolinePool {
-public:
-  Expected<JITTargetAddress> getTrampoline() override {
-    llvm_unreachable("Unimplemented");
-  }
+protected:
+  Error grow() override { llvm_unreachable("Unimplemented"); }
 };
 
 class DummyCallbackManager : public JITCompileCallbackManager {
diff --git a/llvm/unittests/IR/DebugTypeODRUniquingTest.cpp b/llvm/unittests/IR/DebugTypeODRUniquingTest.cpp
index a809b52f27233..af857c1c7aff7 100644
--- a/llvm/unittests/IR/DebugTypeODRUniquingTest.cpp
+++ b/llvm/unittests/IR/DebugTypeODRUniquingTest.cpp
@@ -30,7 +30,7 @@ TEST(DebugTypeODRUniquingTest, getODRType) {
   EXPECT_FALSE(DICompositeType::getODRType(
       Context, UUID, dwarf::DW_TAG_class_type, nullptr, nullptr, 0, nullptr,
       nullptr, 0, 0, 0, DINode::FlagZero, nullptr, 0, nullptr, nullptr, nullptr,
-      nullptr));
+      nullptr, nullptr, nullptr));
 
   // Enable the mapping.  There still shouldn't be a type.
   Context.enableDebugTypeODRUniquing();
@@ -40,7 +40,7 @@ TEST(DebugTypeODRUniquingTest, getODRType) {
   auto &CT = *DICompositeType::getODRType(
       Context, UUID, dwarf::DW_TAG_class_type, nullptr, nullptr, 0, nullptr,
       nullptr, 0, 0, 0, DINode::FlagZero, nullptr, 0, nullptr, nullptr, nullptr,
-      nullptr);
+      nullptr, nullptr, nullptr);
   EXPECT_EQ(UUID.getString(), CT.getIdentifier());
 
   // Check that we get it back, even if we change a field.
@@ -48,12 +48,12 @@ TEST(DebugTypeODRUniquingTest, getODRType) {
   EXPECT_EQ(&CT, DICompositeType::getODRType(
                      Context, UUID, dwarf::DW_TAG_class_type, nullptr, nullptr,
                      0, nullptr, nullptr, 0, 0, 0, DINode::FlagZero, nullptr, 0,
-                     nullptr, nullptr, nullptr, nullptr));
+                     nullptr, nullptr, nullptr, nullptr, nullptr, nullptr));
   EXPECT_EQ(&CT, DICompositeType::getODRType(
                      Context, UUID, dwarf::DW_TAG_class_type,
                      MDString::get(Context, "name"), nullptr, 0, nullptr,
                      nullptr, 0, 0, 0, DINode::FlagZero, nullptr, 0, nullptr,
-                     nullptr, nullptr, nullptr));
+                     nullptr, nullptr, nullptr, nullptr, nullptr));
 
   // Check that it's discarded with the type map.
   Context.disableDebugTypeODRUniquing();
@@ -73,34 +73,36 @@ TEST(DebugTypeODRUniquingTest, buildODRType) {
   auto &CT = *DICompositeType::buildODRType(
       Context, UUID, dwarf::DW_TAG_class_type, nullptr, nullptr, 0, nullptr,
       nullptr, 0, 0, 0, DINode::FlagFwdDecl, nullptr, 0, nullptr, nullptr,
-      nullptr, nullptr);
+      nullptr, nullptr, nullptr, nullptr);
   EXPECT_EQ(&CT, DICompositeType::getODRTypeIfExists(Context, UUID));
   EXPECT_EQ(dwarf::DW_TAG_class_type, CT.getTag());
 
   // Update with another forward decl.  This should be a no-op.
-  EXPECT_EQ(&CT, DICompositeType::buildODRType(
-                     Context, UUID, dwarf::DW_TAG_structure_type, nullptr,
-                     nullptr, 0, nullptr, nullptr, 0, 0, 0, DINode::FlagFwdDecl,
-                     nullptr, 0, nullptr, nullptr, nullptr, nullptr));
+  EXPECT_EQ(&CT,
+            DICompositeType::buildODRType(
+                Context, UUID, dwarf::DW_TAG_structure_type, nullptr, nullptr,
+                0, nullptr, nullptr, 0, 0, 0, DINode::FlagFwdDecl, nullptr, 0,
+                nullptr, nullptr, nullptr, nullptr, nullptr, nullptr));
   EXPECT_EQ(dwarf::DW_TAG_class_type, CT.getTag());
 
   // Update with a definition.  This time we should see a change.
-  EXPECT_EQ(&CT, DICompositeType::buildODRType(
-                     Context, UUID, dwarf::DW_TAG_structure_type, nullptr,
-                     nullptr, 0, nullptr, nullptr, 0, 0, 0, DINode::FlagZero,
-                     nullptr, 0, nullptr, nullptr, nullptr, nullptr));
+  EXPECT_EQ(&CT,
+            DICompositeType::buildODRType(
+                Context, UUID, dwarf::DW_TAG_structure_type, nullptr, nullptr,
+                0, nullptr, nullptr, 0, 0, 0, DINode::FlagZero, nullptr, 0,
+                nullptr, nullptr, nullptr, nullptr, nullptr, nullptr));
   EXPECT_EQ(dwarf::DW_TAG_structure_type, CT.getTag());
 
   // Further updates should be ignored.
   EXPECT_EQ(&CT, DICompositeType::buildODRType(
                      Context, UUID, dwarf::DW_TAG_class_type, nullptr, nullptr,
                      0, nullptr, nullptr, 0, 0, 0, DINode::FlagFwdDecl, nullptr,
-                     0, nullptr, nullptr, nullptr, nullptr));
+                     0, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr));
   EXPECT_EQ(dwarf::DW_TAG_structure_type, CT.getTag());
   EXPECT_EQ(&CT, DICompositeType::buildODRType(
                      Context, UUID, dwarf::DW_TAG_class_type, nullptr, nullptr,
                      0, nullptr, nullptr, 0, 0, 0, DINode::FlagZero, nullptr, 0,
-                     nullptr, nullptr, nullptr, nullptr));
+                     nullptr, nullptr, nullptr, nullptr, nullptr, nullptr));
   EXPECT_EQ(dwarf::DW_TAG_structure_type, CT.getTag());
 }
 
@@ -112,7 +114,8 @@ TEST(DebugTypeODRUniquingTest, buildODRTypeFields) {
   MDString &UUID = *MDString::get(Context, "UUID");
   auto &CT = *DICompositeType::buildODRType(
       Context, UUID, 0, nullptr, nullptr, 0, nullptr, nullptr, 0, 0, 0,
-      DINode::FlagFwdDecl, nullptr, 0, nullptr, nullptr, nullptr, nullptr);
+      DINode::FlagFwdDecl, nullptr, 0, nullptr, nullptr, nullptr, nullptr,
+      nullptr, nullptr);
 
 // Create macros for running through all the fields except Identifier and Flags.
 #define FOR_EACH_MDFIELD()                                                     \
@@ -141,11 +144,12 @@ TEST(DebugTypeODRUniquingTest, buildODRTypeFields) {
 #undef DO_FOR_FIELD
 
   // Replace all the fields with new values that are distinct from each other.
-  EXPECT_EQ(&CT, DICompositeType::buildODRType(
-                     Context, UUID, Tag, Name, File, Line, Scope, BaseType,
-                     SizeInBits, AlignInBits, OffsetInBits,
-                     DINode::FlagArtificial, Elements, RuntimeLang,
-                     VTableHolder, TemplateParams, nullptr, nullptr));
+  EXPECT_EQ(&CT,
+            DICompositeType::buildODRType(
+                Context, UUID, Tag, Name, File, Line, Scope, BaseType,
+                SizeInBits, AlignInBits, OffsetInBits, DINode::FlagArtificial,
+                Elements, RuntimeLang, VTableHolder, TemplateParams, nullptr,
+                nullptr, nullptr, nullptr));
 
   // Confirm that all the right fields got updated.
 #define DO_FOR_FIELD(X) EXPECT_EQ(X, CT.getRaw##X());
diff --git a/llvm/unittests/ProfileData/CMakeLists.txt b/llvm/unittests/ProfileData/CMakeLists.txt
index 00a0079e675a8..366ed5482bf2c 100644
--- a/llvm/unittests/ProfileData/CMakeLists.txt
+++ b/llvm/unittests/ProfileData/CMakeLists.txt
@@ -7,7 +7,6 @@ set(LLVM_LINK_COMPONENTS
 
 add_llvm_unittest(ProfileDataTests
   CoverageMappingTest.cpp
-  InstrProfDataTest.cpp
   InstrProfTest.cpp
   SampleProfTest.cpp
   )
diff --git a/llvm/unittests/ProfileData/InstrProfDataTest.cpp b/llvm/unittests/ProfileData/InstrProfDataTest.cpp
deleted file mode 100644
index af1a3de0657c2..0000000000000
--- a/llvm/unittests/ProfileData/InstrProfDataTest.cpp
+++ /dev/null
@@ -1,68 +0,0 @@
-//===- unittest/ProfileData/InstProfDataTest.cpp ----------------------------=//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "gtest/gtest.h"
-
-#include <cstdint>
-
-#define INSTR_PROF_VALUE_PROF_MEMOP_API
-#include "llvm/ProfileData/InstrProfData.inc"
-
-namespace {
-
-TEST(InstrProfDataTest, MapValueToRangeRepValue) {
-  EXPECT_EQ(0ULL, InstrProfGetRangeRepValue(0));
-  EXPECT_EQ(1ULL, InstrProfGetRangeRepValue(1));
-  EXPECT_EQ(2ULL, InstrProfGetRangeRepValue(2));
-  EXPECT_EQ(3ULL, InstrProfGetRangeRepValue(3));
-  EXPECT_EQ(4ULL, InstrProfGetRangeRepValue(4));
-  EXPECT_EQ(5ULL, InstrProfGetRangeRepValue(5));
-  EXPECT_EQ(6ULL, InstrProfGetRangeRepValue(6));
-  EXPECT_EQ(7ULL, InstrProfGetRangeRepValue(7));
-  EXPECT_EQ(8ULL, InstrProfGetRangeRepValue(8));
-  EXPECT_EQ(9ULL, InstrProfGetRangeRepValue(9));
-  EXPECT_EQ(16ULL, InstrProfGetRangeRepValue(16));
-  EXPECT_EQ(17ULL, InstrProfGetRangeRepValue(30));
-  EXPECT_EQ(32ULL, InstrProfGetRangeRepValue(32));
-  EXPECT_EQ(33ULL, InstrProfGetRangeRepValue(54));
-  EXPECT_EQ(64ULL, InstrProfGetRangeRepValue(64));
-  EXPECT_EQ(65ULL, InstrProfGetRangeRepValue(127));
-  EXPECT_EQ(128ULL, InstrProfGetRangeRepValue(128));
-  EXPECT_EQ(129ULL, InstrProfGetRangeRepValue(200));
-  EXPECT_EQ(256ULL, InstrProfGetRangeRepValue(256));
-  EXPECT_EQ(257ULL, InstrProfGetRangeRepValue(397));
-  EXPECT_EQ(512ULL, InstrProfGetRangeRepValue(512));
-  EXPECT_EQ(513ULL, InstrProfGetRangeRepValue(2832048023));
-}
-
-TEST(InstrProfDataTest, IsInOneValueRange) {
-  EXPECT_EQ(true, InstrProfIsSingleValRange(0));
-  EXPECT_EQ(true, InstrProfIsSingleValRange(1));
-  EXPECT_EQ(true, InstrProfIsSingleValRange(2));
-  EXPECT_EQ(true, InstrProfIsSingleValRange(3));
-  EXPECT_EQ(true, InstrProfIsSingleValRange(4));
-  EXPECT_EQ(true, InstrProfIsSingleValRange(5));
-  EXPECT_EQ(true, InstrProfIsSingleValRange(6));
-  EXPECT_EQ(true, InstrProfIsSingleValRange(7));
-  EXPECT_EQ(true, InstrProfIsSingleValRange(8));
-  EXPECT_EQ(false, InstrProfIsSingleValRange(9));
-  EXPECT_EQ(true, InstrProfIsSingleValRange(16));
-  EXPECT_EQ(false, InstrProfIsSingleValRange(30));
-  EXPECT_EQ(true, InstrProfIsSingleValRange(32));
-  EXPECT_EQ(false, InstrProfIsSingleValRange(54));
-  EXPECT_EQ(true, InstrProfIsSingleValRange(64));
-  EXPECT_EQ(false, InstrProfIsSingleValRange(127));
-  EXPECT_EQ(true, InstrProfIsSingleValRange(128));
-  EXPECT_EQ(false, InstrProfIsSingleValRange(200));
-  EXPECT_EQ(true, InstrProfIsSingleValRange(256));
-  EXPECT_EQ(false, InstrProfIsSingleValRange(397));
-  EXPECT_EQ(true, InstrProfIsSingleValRange(512));
-  EXPECT_EQ(false, InstrProfIsSingleValRange(2832048023344));
-}
-
-} // end anonymous namespace
diff --git a/llvm/unittests/Support/CommandLineTest.cpp b/llvm/unittests/Support/CommandLineTest.cpp
index e1b706e2a78ee..e8c2cef18e367 100644
--- a/llvm/unittests/Support/CommandLineTest.cpp
+++ b/llvm/unittests/Support/CommandLineTest.cpp
@@ -763,6 +763,18 @@ TEST(CommandLineTest, DefaultOptions) {
 TEST(CommandLineTest, ArgumentLimit) {
   std::string args(32 * 4096, 'a');
   EXPECT_FALSE(llvm::sys::commandLineFitsWithinSystemLimits("cl", args.data()));
+  std::string args2(256, 'a');
+  EXPECT_TRUE(llvm::sys::commandLineFitsWithinSystemLimits("cl", args2.data()));
+  if (Triple(sys::getProcessTriple()).isOSWindows()) {
+    // We use 32000 as a limit for command line length. Program name ('cl'),
+    // separating spaces and termination null character occupy 5 symbols.
+    std::string long_arg(32000 - 5, 'b');
+    EXPECT_TRUE(
+        llvm::sys::commandLineFitsWithinSystemLimits("cl", long_arg.data()));
+    long_arg += 'b';
+    EXPECT_FALSE(
+        llvm::sys::commandLineFitsWithinSystemLimits("cl", long_arg.data()));
+  }
 }
 
 TEST(CommandLineTest, ResponseFileWindows) {
diff --git a/llvm/unittests/Support/CompressionTest.cpp b/llvm/unittests/Support/CompressionTest.cpp
index cc7be431b62bc..51723898e950d 100644
--- a/llvm/unittests/Support/CompressionTest.cpp
+++ b/llvm/unittests/Support/CompressionTest.cpp
@@ -21,7 +21,7 @@ using namespace llvm;
 
 namespace {
 
-#if LLVM_ENABLE_ZLIB == 1 && HAVE_LIBZ
+#if LLVM_ENABLE_ZLIB
 
 void TestZlibCompression(StringRef Input, int Level) {
   SmallString<32> Compressed;
diff --git a/llvm/unittests/Support/FormatVariadicTest.cpp b/llvm/unittests/Support/FormatVariadicTest.cpp
index e3754fbd1bf02..37946165e3651 100644
--- a/llvm/unittests/Support/FormatVariadicTest.cpp
+++ b/llvm/unittests/Support/FormatVariadicTest.cpp
@@ -60,6 +60,18 @@ TEST(FormatVariadicTest, EscapedBrace) {
   ASSERT_EQ(1u, Replacements.size());
   EXPECT_EQ("{{{", Replacements[0].Spec);
   EXPECT_EQ(ReplacementType::Literal, Replacements[0].Type);
+
+  // } does not require doubling up.
+  Replacements = formatv_object_base::parseFormatString("}");
+  ASSERT_EQ(1u, Replacements.size());
+  EXPECT_EQ("}", Replacements[0].Spec);
+  EXPECT_EQ(ReplacementType::Literal, Replacements[0].Type);
+
+  // } does not require doubling up.
+  Replacements = formatv_object_base::parseFormatString("}}}");
+  ASSERT_EQ(1u, Replacements.size());
+  EXPECT_EQ("}}}", Replacements[0].Spec);
+  EXPECT_EQ(ReplacementType::Literal, Replacements[0].Type);
 }
 
 TEST(FormatVariadicTest, ValidReplacementSequence) {
diff --git a/llvm/unittests/Target/ARM/MachineInstrTest.cpp b/llvm/unittests/Target/ARM/MachineInstrTest.cpp
index 3e5403f209887..792a15dcbfafd 100644
--- a/llvm/unittests/Target/ARM/MachineInstrTest.cpp
+++ b/llvm/unittests/Target/ARM/MachineInstrTest.cpp
@@ -1103,8 +1103,8 @@ TEST(MachineInstr, HasSideEffects) {
 
   for (unsigned Op = 0; Op < ARM::INSTRUCTION_LIST_END; ++Op) {
     const MCInstrDesc &Desc = TII->get(Op);
-    if ((Desc.TSFlags & ARMII::DomainMask) != ARMII::DomainMVE &&
-        (Desc.TSFlags & ARMII::DomainMask) != ARMII::DomainVFP)
+    if ((Desc.TSFlags &
+         (ARMII::DomainMVE | ARMII::DomainVFP | ARMII::DomainNEONA8)) == 0)
       continue;
     if (UnpredictableOpcodes.count(Op))
       continue;
diff --git a/llvm/unittests/Transforms/Utils/CodeMoverUtilsTest.cpp b/llvm/unittests/Transforms/Utils/CodeMoverUtilsTest.cpp
index f9f1b235d0d08..c7e63d4a38fc3 100644
--- a/llvm/unittests/Transforms/Utils/CodeMoverUtilsTest.cpp
+++ b/llvm/unittests/Transforms/Utils/CodeMoverUtilsTest.cpp
@@ -514,8 +514,8 @@ TEST(CodeMoverUtils, IsSafeToMoveTest1) {
 
         // Moving instruction to non control flow equivalent places are not
         // supported.
-        EXPECT_FALSE(isSafeToMoveBefore(*SI_A5, *Entry->getTerminator(), DT,
-                                        &PDT, &DI));
+        EXPECT_FALSE(
+            isSafeToMoveBefore(*SI_A5, *Entry->getTerminator(), DT, &PDT, &DI));
 
         // Moving PHINode is not supported.
         EXPECT_FALSE(isSafeToMoveBefore(PN, *PN.getNextNode()->getNextNode(),
@@ -652,3 +652,205 @@ TEST(CodeMoverUtils, IsSafeToMoveTest4) {
         EXPECT_FALSE(isSafeToMoveBefore(*SubInst, *AddInst, DT, &PDT, &DI));
       });
 }
+
+TEST(CodeMoverUtils, IsSafeToMoveTest5) {
+  LLVMContext C;
+
+  std::unique_ptr<Module> M =
+      parseIR(C, R"(define void @dependence(i32* noalias %A, i32* noalias %B){
+entry:
+    store i32 0, i32* %A, align 4 ; storeA0
+    store i32 2, i32* %A, align 4 ; storeA1
+    %tmp0 = load i32, i32* %A, align 4 ; loadA0
+    store i32 1, i32* %B, align 4 ; storeB0
+    %tmp1 = load i32, i32* %A, align 4 ; loadA1
+    store i32 2, i32* %A, align 4 ; storeA2 
+    store i32 4, i32* %B, align 4 ; StoreB1 
+    %tmp2 = load i32, i32* %A, align 4 ; loadA2 
+    %tmp3 = load i32, i32* %A, align 4 ; loadA3 
+    %tmp4 = load i32, i32* %B, align 4 ; loadB2
+    %tmp5 = load i32, i32* %B, align 4 ; loadB3
+    ret void
+})");
+
+  run(*M, "dependence",
+      [&](Function &F, DominatorTree &DT, PostDominatorTree &PDT,
+          DependenceInfo &DI) {
+        Instruction *LoadA0 = getInstructionByName(F, "tmp0");
+        Instruction *LoadA1 = getInstructionByName(F, "tmp1");
+        Instruction *LoadA2 = getInstructionByName(F, "tmp2");
+        Instruction *LoadA3 = getInstructionByName(F, "tmp3");
+        Instruction *LoadB2 = getInstructionByName(F, "tmp4");
+        Instruction *LoadB3 = getInstructionByName(F, "tmp5");
+        Instruction *StoreA1 = LoadA0->getPrevNode();
+        Instruction *StoreA0 = StoreA1->getPrevNode();
+        Instruction *StoreB0 = LoadA0->getNextNode();
+        Instruction *StoreB1 = LoadA2->getPrevNode();
+        Instruction *StoreA2 = StoreB1->getPrevNode();
+
+        // Input forward dependency
+        EXPECT_TRUE(isSafeToMoveBefore(*LoadA2, *LoadB2, DT, &PDT, &DI));
+        // Input backward dependency
+        EXPECT_TRUE(isSafeToMoveBefore(*LoadA3, *LoadA2, DT, &PDT, &DI));
+
+        // Output forward dependency
+        EXPECT_FALSE(isSafeToMoveBefore(*StoreA0, *LoadA0, DT, &PDT, &DI));
+        // Output backward dependency
+        EXPECT_FALSE(isSafeToMoveBefore(*StoreA1, *StoreA0, DT, &PDT, &DI));
+
+        // Flow forward dependency
+        EXPECT_FALSE(isSafeToMoveBefore(*StoreA1, *StoreB0, DT, &PDT, &DI));
+        // Flow backward dependency
+        EXPECT_FALSE(isSafeToMoveBefore(*LoadA0, *StoreA1, DT, &PDT, &DI));
+
+        // Anti forward dependency
+        EXPECT_FALSE(isSafeToMoveBefore(*LoadA1, *StoreB1, DT, &PDT, &DI));
+        // Anti backward dependency
+        EXPECT_FALSE(isSafeToMoveBefore(*StoreA2, *LoadA1, DT, &PDT, &DI));
+
+        // No input backward dependency
+        EXPECT_TRUE(isSafeToMoveBefore(*LoadB2, *LoadA3, DT, &PDT, &DI));
+        // No input forward dependency
+        EXPECT_TRUE(isSafeToMoveBefore(*LoadA3, *LoadB3, DT, &PDT, &DI));
+
+        // No Output forward dependency
+        EXPECT_TRUE(isSafeToMoveBefore(*StoreA2, *LoadA2, DT, &PDT, &DI));
+        // No Output backward dependency
+        EXPECT_TRUE(isSafeToMoveBefore(*StoreB1, *StoreA2, DT, &PDT, &DI));
+
+        // No flow forward dependency
+        EXPECT_TRUE(isSafeToMoveBefore(*StoreB0, *StoreA2, DT, &PDT, &DI));
+        // No flow backward dependency
+        EXPECT_TRUE(isSafeToMoveBefore(*LoadA1, *StoreB0, DT, &PDT, &DI));
+
+        // No anti backward dependency
+        EXPECT_TRUE(isSafeToMoveBefore(*StoreB0, *LoadA0, DT, &PDT, &DI));
+        // No anti forward dependency
+        EXPECT_TRUE(isSafeToMoveBefore(*LoadA0, *LoadA1, DT, &PDT, &DI));
+      });
+}
+
+TEST(CodeMoverUtils, IsSafeToMoveTest6) {
+  LLVMContext C;
+
+  std::unique_ptr<Module> M = parseIR(
+      C, R"(define void @dependence(i1 %cond, i32* noalias %A, i32* noalias %B){
+   entry:
+        br i1 %cond, label %bb0, label %bb1
+   bb0:
+        br label %bb1
+    bb1:
+        store i32 0, i32* %A, align 4 ; storeA0
+        br i1 %cond, label %bb2, label %bb3
+    bb2:
+        br label %bb3
+    bb3:
+        store i32 2, i32* %A, align 4 ; storeA1
+        br i1 %cond, label %bb4, label %bb5
+    bb4:
+        br label %bb5
+    bb5:
+        %tmp0 = load i32, i32* %A, align 4 ; loadA0
+        br i1 %cond, label %bb6, label %bb7
+    bb6:
+        br label %bb7
+    bb7:
+        store i32 1, i32* %B, align 4 ; storeB0
+        br i1 %cond, label %bb8, label %bb9
+    bb8:
+        br label %bb9
+    bb9:
+        %tmp1 = load i32, i32* %A, align 4 ; loadA1
+        br i1 %cond, label %bb10, label %bb11
+    bb10:
+        br label %bb11
+    bb11:
+        store i32 2, i32* %A, align 4 ; storeA2
+        br i1 %cond, label %bb12, label %bb13
+    bb12:
+        br label %bb13
+    bb13:
+        store i32 4, i32* %B, align 4 ; StoreB1
+        br i1 %cond, label %bb14, label %bb15
+    bb14:
+        br label %bb15
+    bb15:
+        %tmp2 = load i32, i32* %A, align 4 ; loadA2
+        br i1 %cond, label %bb16, label %bb17
+    bb16:
+        br label %bb17
+    bb17:
+        %tmp3 = load i32, i32* %A, align 4 ; loadA3
+        br i1 %cond, label %bb18, label %bb19
+    bb18:
+        br label %bb19
+    bb19:
+        %tmp4 = load i32, i32* %B, align 4 ; loadB2
+        br i1 %cond, label %bb20, label %bb21
+    bb20:
+       br label %bb21
+    bb21:
+       %tmp5 = load i32, i32* %B, align 4 ; loadB3
+       ret void
+   })");
+  run(*M, "dependence",
+      [&](Function &F, DominatorTree &DT, PostDominatorTree &PDT,
+          DependenceInfo &DI) {
+        BasicBlock *BB1 = getBasicBlockByName(F, "bb1");
+        BasicBlock *BB3 = getBasicBlockByName(F, "bb3");
+        BasicBlock *BB7 = getBasicBlockByName(F, "bb7");
+        BasicBlock *BB11 = getBasicBlockByName(F, "bb11");
+        BasicBlock *BB13 = getBasicBlockByName(F, "bb13");
+        Instruction *LoadA0 = getInstructionByName(F, "tmp0");
+        Instruction *LoadA1 = getInstructionByName(F, "tmp1");
+        Instruction *LoadA2 = getInstructionByName(F, "tmp2");
+        Instruction *LoadA3 = getInstructionByName(F, "tmp3");
+        Instruction *LoadB2 = getInstructionByName(F, "tmp4");
+        Instruction *LoadB3 = getInstructionByName(F, "tmp5");
+        Instruction &StoreA1 = BB3->front();
+        Instruction &StoreA0 = BB1->front();
+        Instruction &StoreB0 = BB7->front();
+        Instruction &StoreB1 = BB13->front();
+        Instruction &StoreA2 = BB11->front();
+
+        // Input forward dependency
+        EXPECT_TRUE(isSafeToMoveBefore(*LoadA2, *LoadB2, DT, &PDT, &DI));
+        // Input backward dependency
+        EXPECT_TRUE(isSafeToMoveBefore(*LoadA3, *LoadA2, DT, &PDT, &DI));
+
+        // Output forward dependency
+        EXPECT_FALSE(isSafeToMoveBefore(StoreA0, *LoadA0, DT, &PDT, &DI));
+        // Output backward dependency
+        EXPECT_FALSE(isSafeToMoveBefore(StoreA1, StoreA0, DT, &PDT, &DI));
+
+        // Flow forward dependency
+        EXPECT_FALSE(isSafeToMoveBefore(StoreA1, StoreB0, DT, &PDT, &DI));
+        // Flow backward dependency
+        EXPECT_FALSE(isSafeToMoveBefore(*LoadA0, StoreA1, DT, &PDT, &DI));
+
+        // Anti forward dependency
+        EXPECT_FALSE(isSafeToMoveBefore(*LoadA1, StoreB1, DT, &PDT, &DI));
+        // Anti backward dependency
+        EXPECT_FALSE(isSafeToMoveBefore(StoreA2, *LoadA1, DT, &PDT, &DI));
+
+        // No input backward dependency
+        EXPECT_TRUE(isSafeToMoveBefore(*LoadB2, *LoadA3, DT, &PDT, &DI));
+        // No input forward dependency
+        EXPECT_TRUE(isSafeToMoveBefore(*LoadA3, *LoadB3, DT, &PDT, &DI));
+
+        // No Output forward dependency
+        EXPECT_TRUE(isSafeToMoveBefore(StoreA2, *LoadA2, DT, &PDT, &DI));
+        // No Output backward dependency
+        EXPECT_TRUE(isSafeToMoveBefore(StoreB1, StoreA2, DT, &PDT, &DI));
+
+        // No flow forward dependency
+        EXPECT_TRUE(isSafeToMoveBefore(StoreB0, StoreA2, DT, &PDT, &DI));
+        // No flow backward dependency
+        EXPECT_TRUE(isSafeToMoveBefore(*LoadA1, StoreB0, DT, &PDT, &DI));
+
+        // No anti backward dependency
+        EXPECT_TRUE(isSafeToMoveBefore(StoreB0, *LoadA0, DT, &PDT, &DI));
+        // No anti forward dependency
+        EXPECT_TRUE(isSafeToMoveBefore(*LoadA0, *LoadA1, DT, &PDT, &DI));
+      });
+}
diff --git a/llvm/utils/TableGen/CodeGenDAGPatterns.cpp b/llvm/utils/TableGen/CodeGenDAGPatterns.cpp
index 6fdc116721f33..d8588c4dc6eda 100644
--- a/llvm/utils/TableGen/CodeGenDAGPatterns.cpp
+++ b/llvm/utils/TableGen/CodeGenDAGPatterns.cpp
@@ -3589,6 +3589,9 @@ static bool hasNullFragReference(DagInit *DI) {
   if (Operator->getName() == "null_frag") return true;
   // If any of the arguments reference the null fragment, return true.
   for (unsigned i = 0, e = DI->getNumArgs(); i != e; ++i) {
+    if (auto Arg = dyn_cast<DefInit>(DI->getArg(i)))
+      if (Arg->getDef()->getName() == "null_frag")
+        return true;
     DagInit *Arg = dyn_cast<DagInit>(DI->getArg(i));
     if (Arg && hasNullFragReference(Arg))
       return true;
diff --git a/llvm/utils/TableGen/CodeGenDAGPatterns.h b/llvm/utils/TableGen/CodeGenDAGPatterns.h
index 5d8d6d346cd26..7a0da531469e8 100644
--- a/llvm/utils/TableGen/CodeGenDAGPatterns.h
+++ b/llvm/utils/TableGen/CodeGenDAGPatterns.h
@@ -14,7 +14,6 @@
 #ifndef LLVM_UTILS_TABLEGEN_CODEGENDAGPATTERNS_H
 #define LLVM_UTILS_TABLEGEN_CODEGENDAGPATTERNS_H
 
-#include "CodeGenHwModes.h"
 #include "CodeGenIntrinsics.h"
 #include "CodeGenTarget.h"
 #include "SDNodeProperties.h"
@@ -42,7 +41,6 @@ class SDNodeInfo;
 class TreePattern;
 class TreePatternNode;
 class CodeGenDAGPatterns;
-class ComplexPattern;
 
 /// Shared pointer for TreePatternNode.
 using TreePatternNodePtr = std::shared_ptr<TreePatternNode>;
diff --git a/llvm/utils/TableGen/SubtargetFeatureInfo.cpp b/llvm/utils/TableGen/SubtargetFeatureInfo.cpp
index 3821f47574649..975a014205d79 100644
--- a/llvm/utils/TableGen/SubtargetFeatureInfo.cpp
+++ b/llvm/utils/TableGen/SubtargetFeatureInfo.cpp
@@ -7,11 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "SubtargetFeatureInfo.h"
-
 #include "Types.h"
 #include "llvm/Config/llvm-config.h"
+#include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
-
 #include <map>
 
 using namespace llvm;
diff --git a/llvm/utils/TableGen/SubtargetFeatureInfo.h b/llvm/utils/TableGen/SubtargetFeatureInfo.h
index d72f8b93461fa..8c8a4487934cd 100644
--- a/llvm/utils/TableGen/SubtargetFeatureInfo.h
+++ b/llvm/utils/TableGen/SubtargetFeatureInfo.h
@@ -9,17 +9,12 @@
 #ifndef LLVM_UTIL_TABLEGEN_SUBTARGETFEATUREINFO_H
 #define LLVM_UTIL_TABLEGEN_SUBTARGETFEATUREINFO_H
 
-#include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
-
 #include <map>
 #include <string>
 #include <vector>
 
 namespace llvm {
-class Record;
-class RecordKeeper;
-
 struct SubtargetFeatureInfo;
 using SubtargetFeatureInfoMap = std::map<Record *, SubtargetFeatureInfo, LessRecordByID>;
 
diff --git a/llvm/utils/UpdateTestChecks/asm.py b/llvm/utils/UpdateTestChecks/asm.py
index 998d8b1f57736..cfcc8deb7c8c1 100644
--- a/llvm/utils/UpdateTestChecks/asm.py
+++ b/llvm/utils/UpdateTestChecks/asm.py
@@ -364,7 +364,7 @@ def build_function_body_dictionary_for_triple(args, raw_tool_output, triple, pre
   scrubber, function_re = handler
   common.build_function_body_dictionary(
           function_re, scrubber, [args], raw_tool_output, prefixes,
-          func_dict, args.verbose, False)
+          func_dict, args.verbose, False, False)
 
 ##### Generator of assembly CHECK lines
 
diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py
index 7b4a9b3e6d165..17f738601f61b 100644
--- a/llvm/utils/UpdateTestChecks/common.py
+++ b/llvm/utils/UpdateTestChecks/common.py
@@ -142,7 +142,7 @@ def invoke_tool(exe, cmd_args, ir):
 UTC_ADVERT = 'NOTE: Assertions have been autogenerated by '
 
 OPT_FUNCTION_RE = re.compile(
-    r'^\s*define\s+(?:internal\s+)?[^@]*@(?P<func>[\w.-]+?)\s*'
+    r'^(\s*;\s*Function\sAttrs:\s(?P<attrs>[\w\s]+?))?\s*define\s+(?:internal\s+)?[^@]*@(?P<func>[\w.-]+?)\s*'
     r'(?P<args_and_sig>\((\)|(.*?[\w.-]+?)\))[^{]*)\{\n(?P<body>.*?)^\}$',
     flags=(re.M | re.S))
 
@@ -218,11 +218,12 @@ def do_scrub(body, scrubber, scrubber_args, extra):
 
 # Build up a dictionary of all the function bodies.
 class function_body(object):
-  def __init__(self, string, extra, args_and_sig):
+  def __init__(self, string, extra, args_and_sig, attrs):
     self.scrub = string
     self.extrascrub = extra
     self.args_and_sig = args_and_sig
-  def is_same_except_arg_names(self, extrascrub, args_and_sig):
+    self.attrs = attrs
+  def is_same_except_arg_names(self, extrascrub, args_and_sig, attrs):
     arg_names = set()
     def drop_arg_names(match):
         arg_names.add(match.group(2))
@@ -231,6 +232,8 @@ def repl_arg_names(match):
         if match.group(2) in arg_names:
             return match.group(1) + match.group(3)
         return match.group(1) + match.group(2) + match.group(3)
+    if self.attrs != attrs:
+      return False
     ans0 = IR_VALUE_RE.sub(drop_arg_names, self.args_and_sig)
     ans1 = IR_VALUE_RE.sub(drop_arg_names, args_and_sig)
     if ans0 != ans1:
@@ -244,12 +247,13 @@ def repl_arg_names(match):
   def __str__(self):
     return self.scrub
 
-def build_function_body_dictionary(function_re, scrubber, scrubber_args, raw_tool_output, prefixes, func_dict, verbose, record_args):
+def build_function_body_dictionary(function_re, scrubber, scrubber_args, raw_tool_output, prefixes, func_dict, verbose, record_args, check_attributes):
   for m in function_re.finditer(raw_tool_output):
     if not m:
       continue
     func = m.group('func')
     body = m.group('body')
+    attrs = m.group('attrs') if check_attributes else ''
     # Determine if we print arguments, the opening brace, or nothing after the function name
     if record_args and 'args_and_sig' in m.groupdict():
         args_and_sig = scrub_body(m.group('args_and_sig').strip())
@@ -271,19 +275,20 @@ def build_function_body_dictionary(function_re, scrubber, scrubber_args, raw_too
       for l in scrubbed_body.splitlines():
         print('  ' + l, file=sys.stderr)
     for prefix in prefixes:
-      if func in func_dict[prefix] and (str(func_dict[prefix][func]) != scrubbed_body or (func_dict[prefix][func] and func_dict[prefix][func].args_and_sig != args_and_sig)):
-        if func_dict[prefix][func] and func_dict[prefix][func].is_same_except_arg_names(scrubbed_extra, args_and_sig):
-          func_dict[prefix][func].scrub = scrubbed_extra
-          func_dict[prefix][func].args_and_sig = args_and_sig
-          continue
-        else:
-          if prefix == prefixes[-1]:
-            warn('Found conflicting asm under the same prefix: %r!' % (prefix,))
-          else:
-            func_dict[prefix][func] = None
+      if func in func_dict[prefix]:
+        if str(func_dict[prefix][func]) != scrubbed_body or (func_dict[prefix][func] and (func_dict[prefix][func].args_and_sig != args_and_sig or func_dict[prefix][func].attrs != attrs)):
+          if func_dict[prefix][func] and func_dict[prefix][func].is_same_except_arg_names(scrubbed_extra, args_and_sig, attrs):
+            func_dict[prefix][func].scrub = scrubbed_extra
+            func_dict[prefix][func].args_and_sig = args_and_sig
             continue
+          else:
+            if prefix == prefixes[-1]:
+              warn('Found conflicting asm under the same prefix: %r!' % (prefix,))
+            else:
+              func_dict[prefix][func] = None
+              continue
 
-      func_dict[prefix][func] = function_body(scrubbed_body, scrubbed_extra, args_and_sig)
+      func_dict[prefix][func] = function_body(scrubbed_body, scrubbed_extra, args_and_sig, attrs)
 
 ##### Generator of LLVM IR CHECK lines
 
@@ -383,6 +388,10 @@ def add_checks(output_lines, comment_marker, prefix_list, func_dict, func_name,
 
       vars_seen = set()
       printed_prefixes.append(checkprefix)
+      attrs = str(func_dict[checkprefix][func_name].attrs)
+      attrs = '' if attrs == 'None' else attrs
+      if attrs:
+        output_lines.append('%s %s: Function Attrs: %s' % (comment_marker, checkprefix, attrs))
       args_and_sig = str(func_dict[checkprefix][func_name].args_and_sig)
       args_and_sig = genericize_check_lines([args_and_sig], is_analyze, vars_seen)[0]
       if '[[' in args_and_sig:
diff --git a/llvm/utils/benchmark/CMakeLists.txt b/llvm/utils/benchmark/CMakeLists.txt
index 38bc8c6bc9560..3a80b906d5cc7 100644
--- a/llvm/utils/benchmark/CMakeLists.txt
+++ b/llvm/utils/benchmark/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required (VERSION 2.8.12)
+cmake_minimum_required(VERSION 3.13.4)
 
 # Tell cmake 3.0+ that it's safe to clear the PROJECT_VERSION variable in the
 # call to project() below.
@@ -156,6 +156,10 @@ else()
     add_cxx_compiler_flag(-fno-exceptions)
   endif()
 
+  if (CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
+    add_cxx_compiler_flag(-Wno-suggest-override)
+  endif()
+
   if (HAVE_CXX_FLAG_FSTRICT_ALIASING)
     if (NOT CMAKE_CXX_COMPILER_ID STREQUAL "Intel") #ICC17u2: Many false positives for Wstrict-aliasing
       add_cxx_compiler_flag(-Wstrict-aliasing)
diff --git a/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
index ca24b4b403783..97007eecd1eb9 100644
--- a/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
@@ -37,6 +37,7 @@ write_cmake_config("lit_common_configured") {
     "LLVM_LIBCXX_USED=0",
 
     "GOLD_EXECUTABLE=ld",
+    "GNU_LD_EXECUTABLE=ld",
     "COMPILER_RT_RESOLVED_TEST_COMPILER=" +
         rebase_path("$root_build_dir/bin/clang"),
     "COMPILER_RT_TEST_COMPILER_ID=Clang",
diff --git a/llvm/utils/gn/secondary/lld/test/BUILD.gn b/llvm/utils/gn/secondary/lld/test/BUILD.gn
index 96a6b07d39c1b..dac50890d4ac9 100644
--- a/llvm/utils/gn/secondary/lld/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/lld/test/BUILD.gn
@@ -78,6 +78,7 @@ group("test") {
     ":lit_unit_site_cfg",
     "//lld/tools/lld:symlinks",
     "//lld/unittests",
+    "//llvm/tools/extract",
     "//llvm/tools/llc",
     "//llvm/tools/llvm-ar:symlinks",
     "//llvm/tools/llvm-as",
diff --git a/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn b/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
index 096812db893a5..81da62eff1743 100644
--- a/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
@@ -124,6 +124,7 @@ write_cmake_config("config") {
     "RETSIGTYPE=void",
     "LLVM_GISEL_COV_ENABLED=",
     "LLVM_GISEL_COV_PREFIX=",
+    "LLVM_HAVE_TF_AOT=",
     "LLVM_WITH_Z3=",
 
     # FIXME: Set to 1 on mac once the 10.14 SDK is in common use.
diff --git a/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn
index c13dc723ecd1b..a1ed1d25d381a 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn
@@ -41,11 +41,13 @@ static_library("Analysis") {
     "DemandedBits.cpp",
     "DependenceAnalysis.cpp",
     "DependenceGraphBuilder.cpp",
+    "DevelopmentModeInlineAdvisor.cpp",
     "DivergenceAnalysis.cpp",
     "DomPrinter.cpp",
     "DomTreeUpdater.cpp",
     "DominanceFrontier.cpp",
     "EHPersonalities.cpp",
+    "FunctionPropertiesAnalysis.cpp",
     "GlobalsModRef.cpp",
     "GuardUtils.cpp",
     "HeatUtils.cpp",
@@ -54,7 +56,6 @@ static_library("Analysis") {
     "IndirectCallPromotionAnalysis.cpp",
     "InlineAdvisor.cpp",
     "InlineCost.cpp",
-    "InlineFeaturesAnalysis.cpp",
     "InlineSizeEstimatorAnalysis.cpp",
     "InstCount.cpp",
     "InstructionPrecedenceTracking.cpp",
@@ -75,6 +76,7 @@ static_library("Analysis") {
     "LoopNestAnalysis.cpp",
     "LoopPass.cpp",
     "LoopUnrollAnalyzer.cpp",
+    "MLInlineAdvisor.cpp",
     "MemDepPrinter.cpp",
     "MemDerefPrinter.cpp",
     "MemoryBuiltins.cpp",
@@ -97,6 +99,7 @@ static_library("Analysis") {
     "RegionInfo.cpp",
     "RegionPass.cpp",
     "RegionPrinter.cpp",
+    "ReleaseModeModelRunner.cpp",
     "ScalarEvolution.cpp",
     "ScalarEvolutionAliasAnalysis.cpp",
     "ScalarEvolutionDivision.cpp",
@@ -106,6 +109,7 @@ static_library("Analysis") {
     "StackSafetyAnalysis.cpp",
     "SyncDependenceAnalysis.cpp",
     "SyntheticCountsUtils.cpp",
+    "TFUtils.cpp",
     "TargetLibraryInfo.cpp",
     "TargetTransformInfo.cpp",
     "Trace.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn
index b83fe92d74166..5620da139909f 100644
--- a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn
@@ -37,6 +37,7 @@ static_library("Orc") {
     "RTDyldObjectLinkingLayer.cpp",
     "SpeculateAnalyses.cpp",
     "Speculation.cpp",
+    "TPCDynamicLibrarySearchGenerator.cpp",
     "TPCIndirectionUtils.cpp",
     "TargetProcessControl.cpp",
     "ThreadSafeModule.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/ObjectYAML/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ObjectYAML/BUILD.gn
index fbb4187ab4a41..5c9391db0341c 100644
--- a/llvm/utils/gn/secondary/llvm/lib/ObjectYAML/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/ObjectYAML/BUILD.gn
@@ -14,7 +14,6 @@ static_library("ObjectYAML") {
     "CodeViewYAMLTypeHashing.cpp",
     "CodeViewYAMLTypes.cpp",
     "DWARFEmitter.cpp",
-    "DWARFVisitor.cpp",
     "DWARFYAML.cpp",
     "ELFEmitter.cpp",
     "ELFYAML.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
index 8f554673f752d..e1c725d03f790 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
@@ -63,6 +63,11 @@ tablegen("AMDGPUGenRegisterBank") {
   td_file = "AMDGPU.td"
 }
 
+tablegen("InstCombineTables") {
+  visibility = [ ":LLVMAMDGPUCodeGen" ]
+  args = [ "-gen-searchable-tables" ]
+}
+
 tablegen("R600GenCallingConv") {
   visibility = [ ":LLVMAMDGPUCodeGen" ]
   args = [ "-gen-callingconv" ]
@@ -92,6 +97,7 @@ static_library("LLVMAMDGPUCodeGen") {
     ":AMDGPUGenPreLegalizeGICombiner",
     ":AMDGPUGenRegBankGICombiner",
     ":AMDGPUGenRegisterBank",
+    ":InstCombineTables",
     ":R600GenCallingConv",
     ":R600GenDAGISel",
     ":R600GenDFAPacketizer",
@@ -131,6 +137,7 @@ static_library("LLVMAMDGPUCodeGen") {
     "AMDGPUISelDAGToDAG.cpp",
     "AMDGPUISelLowering.cpp",
     "AMDGPUInline.cpp",
+    "AMDGPUInstCombineIntrinsic.cpp",
     "AMDGPUInstrInfo.cpp",
     "AMDGPUInstructionSelector.cpp",
     "AMDGPULegalizerInfo.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/X86/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/X86/BUILD.gn
index 87d58156d0672..80098c996f870 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/X86/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/X86/BUILD.gn
@@ -100,6 +100,7 @@ static_library("LLVMX86CodeGen") {
     "X86IndirectThunks.cpp",
     "X86InsertPrefetch.cpp",
     "X86InsertWait.cpp",
+    "X86InstCombineIntrinsic.cpp",
     "X86InstrFMA3Info.cpp",
     "X86InstrFoldTables.cpp",
     "X86InstrInfo.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/InstCombine/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/InstCombine/BUILD.gn
index 6e929a912d910..e2d131981560c 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Transforms/InstCombine/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/InstCombine/BUILD.gn
@@ -1,14 +1,8 @@
 import("//llvm/utils/TableGen/tablegen.gni")
 
-tablegen("InstCombineTables") {
-  visibility = [ ":InstCombine" ]
-  args = [ "-gen-searchable-tables" ]
-}
-
 static_library("InstCombine") {
   output_name = "LLVMInstCombine"
   deps = [
-    ":InstCombineTables",
     "//llvm/lib/Analysis",
     "//llvm/lib/IR",
     "//llvm/lib/Support",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn
index 8f4ef8d6fc34d..5fae2e450b481 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn
@@ -53,6 +53,7 @@ static_library("Utils") {
     "LowerInvoke.cpp",
     "LowerMemIntrinsics.cpp",
     "LowerSwitch.cpp",
+    "MatrixUtils.cpp",
     "Mem2Reg.cpp",
     "MetaRenamer.cpp",
     "MisExpect.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/test/BUILD.gn b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
index 550793839183f..3e72ef3c3a446 100644
--- a/llvm/utils/gn/secondary/llvm/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
@@ -203,6 +203,7 @@ group("test") {
     "//llvm/lib/Testing/Support",
     "//llvm/tools/bugpoint",
     "//llvm/tools/dsymutil",
+    "//llvm/tools/extract",
     "//llvm/tools/llc",
     "//llvm/tools/lli",
     "//llvm/tools/lli/ChildTarget:lli-child-target",
diff --git a/llvm/utils/gn/secondary/llvm/tools/extract/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/extract/BUILD.gn
new file mode 100644
index 0000000000000..f4553476f1e8b
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/tools/extract/BUILD.gn
@@ -0,0 +1,4 @@
+executable("extract") {
+  deps = [ "//llvm/lib/Support" ]
+  sources = [ "extract.cpp" ]
+}
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/X86/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/X86/BUILD.gn
index c93827d8cbd7d..42bce295c9a7e 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/X86/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/X86/BUILD.gn
@@ -14,9 +14,6 @@ static_library("X86") {
     # depend on this Target/X86-internal build target.
     "//llvm/lib/Target/X86/MCTargetDesc",
   ]
-  sources = [
-    # Make `gn format` not collapse this, for sync_source_lists_from_cmake.py.
-    "Target.cpp",
-  ]
+  sources = [ "Target.cpp" ]
   include_dirs = [ "//llvm/lib/Target/X86" ]
 }
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn
index b0dcd497d844e..c4bed481e051b 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn
@@ -22,9 +22,9 @@ unittest("AnalysisTests") {
     "DDGTest.cpp",
     "DivergenceAnalysisTest.cpp",
     "DomTreeUpdaterTest.cpp",
+    "FunctionPropertiesAnalysisTest.cpp",
     "GlobalsModRefTest.cpp",
     "IVDescriptorsTest.cpp",
-    "InlineFeaturesAnalysisTest.cpp",
     "LazyCallGraphTest.cpp",
     "LoadsTest.cpp",
     "LoopInfoTest.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn
index 3818b0caad04b..e933b510003cd 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn
@@ -9,7 +9,6 @@ unittest("ProfileDataTests") {
   ]
   sources = [
     "CoverageMappingTest.cpp",
-    "InstrProfDataTest.cpp",
     "InstrProfTest.cpp",
     "SampleProfTest.cpp",
   ]
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index a71cec2382ae6..d80f0aeee8ce9 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -1171,7 +1171,7 @@ def processLine(ln):
             # short-lived, since the set of substitutions is fairly small, and
             # since thrashing has such bad consequences, not bounding the cache
             # seems reasonable.
-            ln = _caching_re_compile(a).sub(b, ln)
+            ln = _caching_re_compile(a).sub(str(b), ln)
 
         # Strip the trailing newline and any extra whitespace.
         return ln.strip()
diff --git a/llvm/utils/lit/lit/reports.py b/llvm/utils/lit/lit/reports.py
index c23b4c7140900..9631d5fe3b358 100755
--- a/llvm/utils/lit/lit/reports.py
+++ b/llvm/utils/lit/lit/reports.py
@@ -74,14 +74,13 @@ def __init__(self, output_file):
         self.skipped_codes = {lit.Test.EXCLUDED,
                               lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
 
-    # TODO(yln): elapsed unused, put it somewhere?
     def write_results(self, tests, elapsed):
         tests.sort(key=by_suite_and_test_path)
         tests_by_suite = itertools.groupby(tests, lambda t: t.suite)
 
         with open(self.output_file, 'w') as file:
             file.write('<?xml version="1.0" encoding="UTF-8"?>\n')
-            file.write('<testsuites>\n')
+            file.write('<testsuites time="{time:.2f}">\n'.format(time=elapsed))
             for suite, test_iter in tests_by_suite:
                 self._write_testsuite(file, suite, list(test_iter))
             file.write('</testsuites>\n')
diff --git a/llvm/utils/lit/tests/shtest-format.py b/llvm/utils/lit/tests/shtest-format.py
index 86742e39ac4be..aa74a2caa8a2a 100644
--- a/llvm/utils/lit/tests/shtest-format.py
+++ b/llvm/utils/lit/tests/shtest-format.py
@@ -86,7 +86,7 @@
 
 
 # XUNIT: <?xml version="1.0" encoding="UTF-8"?>
-# XUNIT-NEXT: <testsuites>
+# XUNIT-NEXT: <testsuites time="{{[0-9.]+}}">
 # XUNIT-NEXT: <testsuite name="shtest-format" tests="21" failures="7" skipped="4">
 
 # XUNIT: <testcase classname="shtest-format.external_shell" name="fail.txt" time="{{[0-9]+\.[0-9]+}}">
diff --git a/llvm/utils/lit/tests/xunit-output.py b/llvm/utils/lit/tests/xunit-output.py
index 32a35f82a9433..81d8525f33b46 100644
--- a/llvm/utils/lit/tests/xunit-output.py
+++ b/llvm/utils/lit/tests/xunit-output.py
@@ -8,7 +8,7 @@
 # RUN: FileCheck < %t.xunit.xml %s
 
 # CHECK:      <?xml version="1.0" encoding="UTF-8"?>
-# CHECK-NEXT: <testsuites>
+# CHECK-NEXT: <testsuites time="{{[0-9.]+}}">
 # CHECK-NEXT: <testsuite name="test-data" tests="5" failures="1" skipped="3">
 # CHECK-NEXT: <testcase classname="test-data.test-data" name="bad&amp;name.ini" time="{{[0-1]\.[0-9]+}}">
 # CHECK-NEXT:   <failure><![CDATA[& < > ]]]]><![CDATA[> &"]]></failure>
diff --git a/llvm/utils/release/test-release.sh b/llvm/utils/release/test-release.sh
index 5351ae1064f9e..6bb6f42b08058 100755
--- a/llvm/utils/release/test-release.sh
+++ b/llvm/utils/release/test-release.sh
@@ -502,12 +502,18 @@ fi
 # Setup the test-suite.  Do this early so we can catch failures before
 # we do the full 3 stage build.
 if [ $do_test_suite = "yes" ]; then
+  venv=virtualenv
+  if ! type -P 'virtualenv' > /dev/null 2>&1 ; then
+    check_program_exists 'python3'
+    venv="python3 -m venv"
+  fi
+
   SandboxDir="$BuildDir/sandbox"
   Lit=$SandboxDir/bin/lit
   TestSuiteBuildDir="$BuildDir/test-suite-build"
   TestSuiteSrcDir="$BuildDir/llvm-test-suite"
 
-  virtualenv $SandboxDir
+  ${venv} $SandboxDir
   $SandboxDir/bin/python $BuildDir/llvm-project/llvm/utils/lit/setup.py install
   mkdir -p $TestSuiteBuildDir
 fi
diff --git a/llvm/utils/unittest/CMakeLists.txt b/llvm/utils/unittest/CMakeLists.txt
index 5aad048f2c35d..36761a60d9f72 100644
--- a/llvm/utils/unittest/CMakeLists.txt
+++ b/llvm/utils/unittest/CMakeLists.txt
@@ -43,6 +43,9 @@ endif()
 if(CXX_SUPPORTS_COVERED_SWITCH_DEFAULT_FLAG)
   add_definitions("-Wno-covered-switch-default")
 endif()
+if(CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
+  add_definitions("-Wno-suggest-override")
+endif()
 
 set(LLVM_REQUIRES_RTTI 1)
 add_definitions( -DGTEST_HAS_RTTI=0 )
diff --git a/llvm/utils/update_analyze_test_checks.py b/llvm/utils/update_analyze_test_checks.py
index 2f1a842976428..f34a4b4c0541d 100755
--- a/llvm/utils/update_analyze_test_checks.py
+++ b/llvm/utils/update_analyze_test_checks.py
@@ -122,7 +122,7 @@ def main():
       for raw_tool_output in re.split(r'Printing analysis ', raw_tool_outputs):
         common.build_function_body_dictionary(
           common.ANALYZE_FUNCTION_RE, common.scrub_body, [],
-          raw_tool_output, prefixes, func_dict, args.verbose, False)
+          raw_tool_output, prefixes, func_dict, args.verbose, False, False)
 
     is_in_function = False
     is_in_function_start = False
diff --git a/llvm/utils/update_cc_test_checks.py b/llvm/utils/update_cc_test_checks.py
index 5851f19e3d2de..fa7b4fa73b98c 100755
--- a/llvm/utils/update_cc_test_checks.py
+++ b/llvm/utils/update_cc_test_checks.py
@@ -131,6 +131,8 @@ def config():
       help='Use more regex for x86 matching to reduce diffs between various subtargets')
   parser.add_argument('--function-signature', action='store_true',
                       help='Keep function signature information around for the check line')
+  parser.add_argument('--check-attributes', action='store_true',
+                      help='Check "Function Attributes" for functions')
   parser.add_argument('tests', nargs='+')
   args = common.parse_commandline_args(parser)
 
@@ -189,7 +191,7 @@ def get_function_body(args, filename, clang_args, extra_commands, prefixes, trip
   if '-emit-llvm' in clang_args:
     common.build_function_body_dictionary(
             common.OPT_FUNCTION_RE, common.scrub_body, [],
-            raw_tool_output, prefixes, func_dict, args.verbose, args.function_signature)
+            raw_tool_output, prefixes, func_dict, args.verbose, args.function_signature, args.check_attributes)
   else:
     print('The clang command line should include -emit-llvm as asm tests '
           'are discouraged in Clang testsuite.', file=sys.stderr)
diff --git a/llvm/utils/update_test_checks.py b/llvm/utils/update_test_checks.py
index be15ae9e26852..660b109c323a9 100755
--- a/llvm/utils/update_test_checks.py
+++ b/llvm/utils/update_test_checks.py
@@ -52,6 +52,8 @@ def main():
                       help='Keep function signature information around for the check line')
   parser.add_argument('--scrub-attributes', action='store_true',
                       help='Remove attribute annotations (#0) from the end of check line')
+  parser.add_argument('--check-attributes', action='store_true',
+                      help='Check "Function Attributes" for functions')
   parser.add_argument('tests', nargs='+')
   initial_args = common.parse_commandline_args(parser)
 
@@ -111,7 +113,7 @@ def main():
       common.build_function_body_dictionary(
               common.OPT_FUNCTION_RE, common.scrub_body, [],
               raw_tool_output, prefixes, func_dict, ti.args.verbose,
-              ti.args.function_signature)
+              ti.args.function_signature, ti.args.check_attributes)
 
     is_in_function = False
     is_in_function_start = False
diff --git a/mlir/docs/Dialects/LLVM.md b/mlir/docs/Dialects/LLVM.md
index e70944d997dff..57c32a3fc74b0 100644
--- a/mlir/docs/Dialects/LLVM.md
+++ b/mlir/docs/Dialects/LLVM.md
@@ -322,7 +322,7 @@ that model LLVM IR constants. These operations do not correspond to LLVM IR
 instructions and are therefore prefixed with `llvm.mlir`.
 
 Inline constants can be created by `llvm.mlir.constant`, which currently
-supports integer, float, string or elements attributes (constant sturcts are not
+supports integer, float, string or elements attributes (constant structs are not
 currently supported). LLVM IR constant expressions are expected to be
 constructed as sequences of regular operations on SSA values produced by
 `llvm.mlir.constant`. Additionally, MLIR provides semantically-charged
diff --git a/mlir/docs/Dialects/SPIR-V.md b/mlir/docs/Dialects/SPIR-V.md
index eb9d18b3b7e26..021a0622d927a 100644
--- a/mlir/docs/Dialects/SPIR-V.md
+++ b/mlir/docs/Dialects/SPIR-V.md
@@ -1158,6 +1158,71 @@ The [Code organization](#code-organization) section gives an overview of how
 SPIR-V related functionalities are implemented in MLIR. This section gives more
 concrete steps on how to contribute.
 
+## Rationale
+
+## Lowering `memref`s to `!spv.array<..>` and `!spv.rtarray<..>`.
+
+The LLVM dialect lowers `memref` types to a `MemrefDescriptor`:
+
+```
+struct MemrefDescriptor {
+  void *allocated_ptr; // Pointer to the base allocation.
+  void *aligned_ptr;   // Pointer within base allocation which is aligned to
+                       // the value set in the memref.
+  size_t offset;       // Offset from aligned_ptr from where to get values
+                       // corresponding to the memref.
+  size_t shape[rank];  // Shape of the memref.
+  size_t stride[rank]; // Strides used while accessing elements of the memref.
+};
+```
+
+In SPIR-V dialect, we chose not to use a `MemrefDescriptor`. Instead a `memref`
+is lowered directly to a `!spv.ptr<!spv.array<nelts x elem_type>>` when the
+`memref` is statically shaped, and `!spv.ptr<!spv.rtarray<elem_type>>` when the
+`memref` is dynamically shaped. The rationale behind this choice is described
+below.
+
+1.  Inputs/output buffers to a SPIR-V kernel are specified using
+    [`OpVariable`][SpirvOpVariable] inside
+    [interface storage classes][VulkanShaderInterfaceStorageClass] (e.g.,
+    Uniform, StorageBuffer, etc.), while kernel private variables reside in
+    non-interface storage classes (e.g., Function, Workgroup, etc.). By default,
+    Vulkan-flavored SPIR-V requires logical addressing mode: one cannot
+    load/store pointers from/to variables and cannot perform pointer arithmetic.
+    Expressing a struct like `MemrefDescriptor` in interface storage class
+    requires special addressing mode
+    ([PhysicalStorageBuffer][VulkanExtensionPhysicalStorageBuffer]) and
+    manipulating such a struct in non-interface storage classes requires special
+    capabilities ([VariablePointers][VulkanExtensionVariablePointers]).
+    Requiring these two extensions together will significantly limit the
+    Vulkan-capable device we can target; basically ruling out mobile support..
+
+1.  An alternative to having one level of indirection (as is the case with
+    `MemrefDescriptor`s), is to embed the `!spv.array` or `!spv.rtarray`
+    directly in the `MemrefDescriptor`, Having such a descriptor at the ABI
+    boundary implies that the first few bytes of the input/output buffers would
+    need to be reserved for shape/stride information. This adds an unnecessary
+    burden on the host side.
+
+1.  A more performant approach would be to have the data be an `OpVariable`,
+    with the shape and strides passed using a separate `OpVariable`. This has
+    further advantages:
+
+    *   All the dynamic shape/stride information of the `memref` can be combined
+        into a single descriptor. Descriptors are
+        [limited resources on many Vulkan hardware][VulkanGPUInfoMaxPerStageDescriptorStorageBuffers].
+        So combining them would help make the generated code more portable
+        across devices.
+    *   If the shape/stride information is small enough, they could be accessed
+        using [PushConstants][VulkanPushConstants] that are faster to access and
+        avoid buffer allocation overheads. These would be unnecessary if all
+        shapes are static. In the dynamic shape cases, a few parameters are
+        typically enough to compute the shape of all `memref`s used/referenced
+        within the kernel making the use of PushConstants possible.
+    *   The shape/stride information (typically) needs to be update less
+        frequently than the data stored in the buffers. They could be part of
+        different descriptor sets.
+
 ### Automated development flow
 
 One of the goals of SPIR-V dialect development is to leverage both the SPIR-V
@@ -1303,6 +1368,7 @@ dialect.
 [SpirvLogicalLayout]: https://www.khronos.org/registry/spir-v/specs/unified1/SPIRV.html#_a_id_logicallayout_a_logical_layout_of_a_module
 [SpirvGrammar]: https://raw.githubusercontent.com/KhronosGroup/SPIRV-Headers/master/include/spirv/unified1/spirv.core.grammar.json
 [SpirvShaderValidation]: https://www.khronos.org/registry/spir-v/specs/unified1/SPIRV.html#_a_id_shadervalidation_a_validation_rules_for_shader_a_href_capability_capabilities_a
+[SpirvOpVariable]: https://www.khronos.org/registry/spir-v/specs/unified1/SPIRV.html#OpVariable
 [GlslStd450]: https://www.khronos.org/registry/spir-v/specs/1.0/GLSL.std.450.html
 [ArrayType]: https://www.khronos.org/registry/spir-v/specs/unified1/SPIRV.html#OpTypeArray
 [ImageType]: https://www.khronos.org/registry/spir-v/specs/unified1/SPIRV.html#OpTypeImage
@@ -1346,6 +1412,11 @@ dialect.
 [GitHubLoweringTracking]: https://github.com/tensorflow/mlir/issues/303
 [GenSpirvUtilsPy]: https://github.com/llvm/llvm-project/blob/master/mlir/utils/spirv/gen_spirv_dialect.py
 [CustomTypeAttrTutorial]: ../Tutorials/DefiningAttributesAndTypes.md
+[VulkanExtensionPhysicalStorageBuffer]: https://github.com/KhronosGroup/SPIRV-Registry/blob/master/extensions/KHR/SPV_KHR_physical_storage_buffer.html
+[VulkanExtensionVariablePointers]: https://github.com/KhronosGroup/SPIRV-Registry/blob/master/extensions/KHR/SPV_KHR_variable_pointers.html
 [VulkanSpirv]: https://renderdoc.org/vkspec_chunked/chap40.html#spirvenv
 [VulkanShaderInterface]: https://renderdoc.org/vkspec_chunked/chap14.html#interfaces-resources
+[VulkanShaderInterfaceStorageClass]: https://renderdoc.org/vkspec_chunked/chap15.html#interfaces
 [VulkanResourceLimits]: https://renderdoc.org/vkspec_chunked/chap36.html#limits
+[VulkanGPUInfoMaxPerStageDescriptorStorageBuffers]: https://vulkan.gpuinfo.org/displaydevicelimit.php?name=maxPerStageDescriptorStorageBuffers&platform=android
+[VulkanPushConstants]: https://www.khronos.org/registry/vulkan/specs/1.2-extensions/man/html/vkCmdPushConstants.html
diff --git a/mlir/docs/LangRef.md b/mlir/docs/LangRef.md
index 17736ce065285..eb0530d82546f 100644
--- a/mlir/docs/LangRef.md
+++ b/mlir/docs/LangRef.md
@@ -359,7 +359,7 @@ operations.
 ### Functions
 
 An MLIR Function is an operation with a name containing a single [SSACFG
-region)[#control-flow-and-ssacfg-regions].  Operations within this region
+region](#control-flow-and-ssacfg-regions).  Operations within this region
 cannot implicitly capture values defined outside of the function,
 i.e. Functions are [IsolatedFromAbove](Traits.md#isolatedfromabove).  All
 external references must use function arguments or attributes that establish a
@@ -980,9 +980,10 @@ Example:
 
 ```mlir
 // With static ranks, we need a function for each possible argument type
-%A = alloc() : memref<16x32xf32> %B = alloc() :
-memref<16x32x64xf32> call @helper_2D(%A) : (memref<16x32xf32>)->() call
-@helper_3D(%B) : (memref<16x32x64xf32>)->()
+%A = alloc() : memref<16x32xf32>
+%B = alloc() : memref<16x32x64xf32>
+call @helper_2D(%A) : (memref<16x32xf32>)->()
+call @helper_3D(%B) : (memref<16x32x64xf32>)->()
 
 // With unknown rank, the functions can be unified under one unranked type
 %A = alloc() : memref<16x32xf32>
diff --git a/mlir/docs/SPIRVToLLVMDialectConversion.md b/mlir/docs/SPIRVToLLVMDialectConversion.md
new file mode 100644
index 0000000000000..1f76741bb2ed8
--- /dev/null
+++ b/mlir/docs/SPIRVToLLVMDialectConversion.md
@@ -0,0 +1,567 @@
+# SPIR-V Dialect to LLVM Dialect conversion manual
+
+This manual describes the conversion from [SPIR-V Dialect](Dialects/SPIR-V.md)
+to [LLVM Dialect](Dialects/LLVM.md). It assumes familiarity with both, and
+describes the design choices behind the modelling of SPIR-V concepts in LLVM
+Dialect. The conversion is an ongoing work, and is expected to grow as more
+features are implemented.
+
+Conversion can be performed by invoking an appropriate conversion pass:
+
+```shell
+mlir-opt -convert-spirv-to-llvm <filename.mlir>
+```
+
+This pass performs type and operation conversions for SPIR-V operations as
+described in this document.
+
+[TOC]
+
+## Type Conversion
+
+This section describes how SPIR-V Dialect types are mapped to LLVM Dialect.
+
+### Scalar types
+
+SPIR-V Dialect                       | LLVM Dialect
+:----------------------------------: | :----------------------------------:
+`i<bitwidth>`                        | `!llvm.i<bitwidth>`
+`si<bitwidth>`                       | `!llvm.i<bitwidth>`
+`ui<bitwidth>`                       | `!llvm.i<bitwidth>`
+`f16`                                | `!llvm.half`
+`f32`                                | `!llvm.float`
+`f64`                                | `!llvm.double`
+
+### Vector types
+
+SPIR-V Dialect                       | LLVM Dialect
+:----------------------------------: | :----------------------------------:
+`vector<<count> x <scalar-type>>`    | `!llvm<"<<count> x <scalar-type>>">`
+
+### Pointer types
+
+A SPIR-V pointer also takes a Storage Class. At the moment, conversion does
+**not** take it into account.
+
+SPIR-V Dialect                                | LLVM Dialect
+:-------------------------------------------: | :-------------------------:
+`!spv.ptr< <element-type>, <storage-class> >` | `!llvm.element-type*`
+
+### Array types
+
+SPIR-V distinguishes between array type and run-time array type, the length of
+which is not known at compile time. In LLVM, it is possible to index beyond the
+end of the array. Therfore, runtime array can be implemented as a zero length
+array type.
+
+Moreover, SPIR-V supports the notion of array stride. Note that this is **not**
+supported by type conversion at the moment.
+
+SPIR-V Dialect                        | LLVM Dialect
+:-----------------------------------: | :-----------------------------------:
+`!spv.array<<count> x <element-type>>`| `!llvm<"[<count> x <element-type>]">`
+`!spv.rtarray< <element-type> >`      | `!llvm<"[0 x <element-type>]">`
+
+### Struct types
+
+Members of SPIR-V struct types may have decorations and offset information.
+Currently, there is **no** support of member decorations conversion for structs.
+For more information see section on [Decorations](#Decorations-conversion). 
+
+Usually we expect that each struct member has a natural size and alignment.
+However, there are cases (*e.g.* in graphics) where one would place struct 
+members explicitly at particular offsets. This case is **not** supported
+at the moment. Hence, we adhere to the following mapping:
+
+*   Structs with no offset are modelled as LLVM packed structures.
+
+*   Structs with natural offset (*i.e.* offset that equals to cumulative size of
+    the previous struct elements or is a natural alignment) are mapped to
+    naturally padded structs. Nested structs with offset and structs with offset
+    containing arrays are **not** supported.
+
+*   Structs with unnatural offset (*i.e.* offset that is not equal to cumulative
+    size of the previous struct elements) are **not** supported. In this case,
+    offsets can be emulated with padding fields (*e.g.* integers). However, such
+    a design would require index recalculation in the conversion of ops that
+    involve memmory addressing.
+
+Examples of SPIR-V struct conversion are:
+```mlir
+!spv.struct<i8, i32>          =>  !llvm<"<{ i8, i32> }>">
+!spv.struct<i8 [0], i32 [4]>  =>  !llvm<"{ i8, i32> }">
+
+// error
+!spv.struct<i8 [0], i32 [8]>
+```
+
+### Not implemented types
+
+The rest of the types not mentioned explicitly above are not supported by the
+conversion. This includes `ImageType` and `MatrixType`.
+
+## Operation Conversion
+
+This section describes how SPIR-V Dialect operations are converted to LLVM
+Dialect. It lists already working conversion patterns, as well as those that are
+an ongoing work. 
+
+There are also multiple ops for which there is no clear mapping in LLVM.
+Conversion for those have to be discussed within the community on the 
+case-by-case basis.
+
+### Arithmetic ops
+
+SPIR-V arithmetic ops mostly have a direct equivalent in LLVM Dialect. Such
+exceptions as `spv.SMod` and `spv.FMod` are rare.
+
+SPIR-V Dialect op                     | LLVM Dialect op
+:-----------------------------------: | :-----------------------------------:
+`spv.FAdd`                            | `llvm.fadd`
+`spv.FDiv`                            | `llvm.fdiv`
+`spv.FNegate`                         | `llvm.fneg`
+`spv.FMul`                            | `llvm.fmul`
+`spv.FRem`                            | `llvm.frem`
+`spv.FSub`                            | `llvm.fsub`
+`spv.IAdd`                            | `llvm.add`
+`spv.IMul`                            | `llvm.mul`
+`spv.ISub`                            | `llvm.sub`
+`spv.SDiv`                            | `llvm.sdiv`
+`spv.SRem`                            | `llvm.srem`
+`spv.UDiv`                            | `llvm.udiv`
+`spv.UMod`                            | `llvm.urem`
+
+### Bitwise ops
+
+SPIR-V has a range of bit ops that are mapped to LLVM dialect ops, intrinsics or
+may have a specific conversion pattern.
+
+#### Direct conversion
+
+As with arithmetic ops, most of bitwise ops have a semantically equivalent op in
+LLVM:
+
+SPIR-V Dialect op                     | LLVM Dialect op
+:-----------------------------------: | :-----------------------------------:
+`spv.BitwiseAnd`                      | `llvm.and`
+`spv.BitwiseOr`                       | `llvm.or`
+`spv.BitwiseXor`                      | `llvm.xor`
+
+Also, some of bitwise ops can be modelled with LLVM intrinsics:
+
+SPIR-V Dialect op                     | LLVM Dialect intrinsic
+:-----------------------------------: | :-----------------------------------:
+`spv.BitCount`                        | `llvm.intr.ctpop`
+`spv.BitReverse`                      | `llvm.intr.bitreverse`
+
+#### `spv.Not`
+
+`spv.Not` is modelled with a `xor` operation with a mask with all bits set.
+
+```mlir
+                            %mask = llvm.mlir.constant(-1 : i32) : !llvm.i32
+%0 = spv.Not %op : i32  =>  %0  = llvm.xor %op, %mask : !llvm.i32
+```
+
+#### Bitfield ops
+
+SPIR-V dialect has three bitfield ops: `spv.BitFieldInsert`,
+`spv.BitFieldSExtract` and `spv.BitFieldUExtract`. This section will first
+outline the general design of conversion patterns for this ops, and then
+describe each of them.
+
+All of these ops take `base`, `offset` and `count` (`insert` for 
+`spv.BitFieldInsert`) as arguments. There are two important things
+to note:
+
+*   `offset` and `count` are always scalar. This means that we can have the
+    following case:
+
+    ```mlir
+    %0 = spv.BitFieldSExtract %base, %offset, %count : vector<2xi32>, i8, i8
+    ```
+
+    To be able to proceed with conversion algorithms described below, all
+    operands have to be of the same type and bitwidth. This requires
+    broadcasting of `offset` and `count` to vectors, for example for the case
+    above it gives:
+
+    ```mlir
+    // Broadcasting offset
+    %offset0 = llvm.mlir.undef : !llvm<"<2 x i8>">
+    %zero = llvm.mlir.constant(0 : i32) : !llvm.i32
+    %offset1 = llvm.insertelement %offset, %offset0[%zero : !llvm.i32] : !llvm<"<2 x i8>">
+    %one = llvm.mlir.constant(1 : i32) : !llvm.i32
+    %vec_offset = llvm.insertelement  %offset, %offset1[%one : !llvm.i32] : !llvm<"<2 x i8>">
+
+    // Broadcasting count
+    // ...
+    ```
+
+*   `offset` and `count` may have different bitwidths from `base`. In this case,
+    both of these operands have to be zero extended (since they are treated as
+    unsigned by the specification) or truncated. For the above example it would
+    be:
+
+    ```mlir
+    // Zero extending offest after broadcasting
+    %res_offset = llvm.zext %vec_offset: !llvm<"<2 x i8>"> to !llvm<"<2 x i32>">
+    ```
+
+    Also, note that if the bitwidth of `offset` or `count` is greater than the
+    bitwidth of `base`, truncation is still permitted. This is because the ops have a
+    defined behaviour with `offset` and `count` being less than the size of
+    `base`. It creates a natural upper bound on what values `offset` and `count`
+    can take, which is 64. This can be expressed in less than 8 bits.
+
+Now, having these two cases in mind, we can proceed with conversion for the ops
+and their operands.
+
+##### `spv.BitFieldInsert`
+
+This operation is implemented as a series of LLVM Dialect operations. First step
+would be to create a mask with bits set outside
+[`offset`, `offset` + `count` - 1]. Then, unchanged bits are extracted from
+`base` that are outside of [`offset`, `offset` + `count` - 1]. The result is
+`or`ed with shifted `insert`.
+
+```mlir
+// Create mask
+// %minus_one = llvm.mlir.constant(-1 : i32) : !llvm.i32
+// %t0        = llvm.shl %minus_one, %count : !llvm.i32
+// %t1        = llvm.xor %t0, %minus_one : !llvm.i32
+// %t2        = llvm.shl %t1, %offset : !llvm.i32
+// %mask      = llvm.xor %t2, %minus_one : !llvm.i32
+
+// Extract unchanged bits from the Base
+// %new_base  = llvm.and %base, %mask : !llvm.i32
+
+// Insert new bits
+// %sh_insert = llvm.shl %insert, %offset : !llvm.i32
+// %res       = llvm.or %new_base, %sh_insert : !llvm.i32
+%res = spv.BitFieldInsert %base, %insert, %offset, %count : i32, i32, i32
+```
+
+##### `spv.BitFieldSExtract`
+
+To implement `spv.BitFieldSExtract`, `base` is shifted left by [sizeof(`base`) -
+(`count` + `offset`)], so that the bit at `offset` + `count` - 1 is the most
+significant bit. After, the result is shifted right, filling the bits with the
+sign bit.
+
+```mlir
+// Calculate the amount to shift left.
+// %size    = llvm.mlir.constant(32 : i32) : !llvm.i32
+// %t0      = llvm.add %count, %offset : !llvm.i32
+// %t1      = llvm.sub %size, %t0 : !llvm.i32
+
+// Shift left and then right to extract the bits
+// %sh_left = llvm.shl %base, %t1 : !llvm.i32
+// %t2      = llvm.add %offset, %t1 : !llvm.i32
+// %res     = llvm.ashr %sh_left, %t2 : !llvm.i32
+%res = spv.BitFieldSExtract %base, %offset, %count : i32, i32, i32
+```
+
+##### `spv.BitFieldUExtract`
+
+For this op a similar pattern as for `spv.BitFieldInsert` is used. First, a mask
+with bits set at [0, `count` - 1] is created. Then `base` is shifted by `offset`
+and the mask is applied.
+
+```mlir
+// Create a mask
+// %minus_one = llvm.mlir.constant(-1 : i32) : !llvm.i32
+// %t0        = llvm.shl %minus_one, %count : !llvm.i32
+// mask       = llvm.xor  %t0, %minus_one : !llvm.i32
+
+// Shift Base and apply mask
+// %sh_base   = llvm.lshr %base, %offset : !llvm.i32
+// %res       = llvm.and %sh_base, %mask : !llvm.i32
+%res = spv.BitFieldUExtract %base, %offset, %count : i32, i32, i32
+```
+
+### Cast ops
+
+#### Direct conversions
+
+SPIR-V Dialect op                     | LLVM Dialect op
+:-----------------------------------: | :-----------------------------------:
+`spv.ConvertFToS`                     | `llvm.fptosi`
+`spv.ConvertFToU`                     | `llvm.fptoui`
+`spv.ConvertSToF`                     | `llvm.sitofp`
+`spv.ConvertUToF`                     | `llvm.uitofp`
+
+#### spv.Bitcast
+This operation has a direct counterpart in LLVM: `llvm.bitcast`. It is treated
+separately since it also supports pointer to pointer bit pattern-preserving type
+conversion, apart from regular scalar or vector of numerical type.
+
+#### Special cases
+Special cases include `spv.FConvert`, `spv.SConvert` and `spv.UConvert`. These
+operations are either a truncate or extend. Let's denote the operand component
+width as A, and result component width as R. Then, the following mappings are
+used:
+
+##### `spv.FConvert` 
+Case            | LLVM Dialect op
+:-------------: | :-----------------------------------:
+A < R           | `llvm.fpext`
+A > R           | `llvm.fptrunc`
+
+##### `spv.SConvert` 
+Case            | LLVM Dialect op
+:-------------: | :-----------------------------------:
+A < R           | `llvm.sext`
+A > R           | `llvm.trunc`
+
+##### `spv.UConvert` 
+Case            | LLVM Dialect op
+:-------------: | :-----------------------------------:
+A < R           | `llvm.zext`
+A > R           | `llvm.trunc`
+
+The case when A = R is not possible, based on SPIR-V Dialect specification:
+> The component width cannot equal the component width in Result Type.
+
+### Comparison ops
+
+SPIR-V comparison ops are mapped to LLVM `icmp` and `fcmp` operations.
+
+SPIR-V Dialect op                     | LLVM Dialect op
+:-----------------------------------: | :-----------------------------------:
+`spv.IEqual`                          | `llvm.icmp "eq"`
+`spv.INotEqual`                       | `llvm.icmp "ne"`
+`spv.FOrdEqual`                       | `llvm.fcmp "oeq"`
+`spv.FOrdGreaterThan`                 | `llvm.fcmp "ogt"`
+`spv.FOrdGreaterThanEqual`            | `llvm.fcmp "oge"`
+`spv.FOrdLessThan`                    | `llvm.fcmp "olt"`
+`spv.FOrdLessThanEqual`               | `llvm.fcmp "ole"`
+`spv.FOrdNotEqual`                    | `llvm.fcmp "one"`
+`spv.FUnordEqual`                     | `llvm.fcmp "ueq"`
+`spv.FUnordGreaterThan`               | `llvm.fcmp "ugt"`
+`spv.FUnordGreaterThanEqual`          | `llvm.fcmp "uge"`
+`spv.FUnordLessThan`                  | `llvm.fcmp "ult"`
+`spv.FUnordLessThanEqual`             | `llvm.fcmp "ule"`
+`spv.FUnordNotEqual`                  | `llvm.fcmp "une"`
+`spv.SGreaterThan`                    | `llvm.icmp "sgt"`
+`spv.SGreaterThanEqual`               | `llvm.icmp "sge"`
+`spv.SLessThan`                       | `llvm.icmp "slt"`
+`spv.SLessThanEqual`                  | `llvm.icmp "sle"`
+`spv.UGreaterThan`                    | `llvm.icmp "ugt"`
+`spv.UGreaterThanEqual`               | `llvm.icmp "uge"`
+`spv.ULessThan`                       | `llvm.icmp "ult"`
+`spv.ULessThanEqual`                  | `llvm.icmp "ule"`
+
+### Logical ops
+
+Logical ops follow a similar pattern as bitwise ops, with the difference that
+they operate on `i1` or vector of `i1` values. The following mapping is used to
+emulate SPIR-V ops behaviour:
+
+SPIR-V Dialect op                     | LLVM Dialect op
+:-----------------------------------: | :-----------------------------------:
+`spv.LogicalAnd`                      | `llvm.and`
+`spv.LogicalOr`                       | `llvm.or`
+`spv.LogicalEqual`                    | `llvm.icmp "eq"`
+`spv.LogicalNotEqual`                 | `llvm.icmp "ne"`
+
+`spv.LogicalNot` has the same conversion pattern as bitwise `spv.Not`. It is
+modelled with `xor` operation with a mask with all bits set.
+
+```mlir
+                                  %mask = llvm.mlir.constant(-1 : i1) : !llvm.i1
+%0 = spv.LogicalNot %op : i1  =>  %0    = llvm.xor %op, %mask : !llvm.i1
+```
+
+### Miscellaneous ops with direct conversions
+
+There are multiple SPIR-V ops that do not fit in a particular group but can be
+converted directly to LLVM dialect. Their conversion is addressed in this
+section.
+
+SPIR-V Dialect op                     | LLVM Dialect op
+:-----------------------------------: | :-----------------------------------:
+`spv.Select`                          | `llvm.select`
+`spv.Undef`                           | `llvm.mlir.undef`
+
+### Shift ops
+
+Shift operates on two operands: `shift` and `base`.
+
+In SPIR-V dialect, `shift` and `base` may have different bit width. On the
+contrary, in LLVM Dialect both `base` and `shift` have to be of the same
+bitwidth. This leads to the following conversions:
+
+*   if `base` has the same bitwidth as `shift`, the conversion is
+    straightforward.
+
+*   if `base` has a greater bit width than `shift`, shift is sign or zero
+    extended first. Then the extended value is passed to the shift.
+
+*   otherwise, the conversion is considered to be illegal.
+
+```mlir
+// Shift without extension
+%res0 = spv.ShiftRightArithmetic %0, %2 : i32, i32  =>  %res0 = llvm.ashr %0, %2 : !llvm.i32
+
+// Shift with extension
+                                                        %ext  = llvm.sext %1 : !llvm.i16 to !llvm.i32
+%res1 = spv.ShiftRightArithmetic %0, %1 : i32, i16  =>  %res1 = llvm.ashr %0, %ext: !llvm.i32
+```
+
+### `spv.constant`
+
+At the moment `spv.constant` conversion supports scalar and vector constants
+**only**.
+
+#### Mapping
+
+`spv.constant` is mapped to `llvm.mlir.constant`. This is a straightforward
+conversion pattern with a special case when the argument is signed or unsigned.
+
+#### Special case
+
+SPIR-V constant can be a signed or unsigned integer. Since LLVM Dialect does not
+have signedness semantics, this case should be handled separately.
+
+The conversion casts constant value attribute to a signless integer or a vector
+of signless integers. This is correct because in SPIR-V, like in LLVM, how to
+interpret an integer number is also dictated by the opcode. However, in reality
+hardware implementation might show unexpected behavior. Therefore, it is better
+to handle it case-by-case, given that the purpose of the conversion is not to
+cover all possible corner cases.
+
+```mlir
+// %0 = llvm.mlir.constant(0 : i8) : !llvm.i8
+%0 = spv.constant  0 : i8
+
+// %1 = llvm.mlir.constant(dense<[2, 3, 4]> : vector<3xi32>) : !llvm<"<3 x i32>">
+%1 = spv.constant dense<[2, 3, 4]> : vector<3xui32>
+```
+
+### Not implemented ops
+
+There is no support of the following ops:
+
+*   All Atomic ops
+*   All matrix ops
+*   All GLSL ops
+*   All GroupNonUniform ops
+*   spv.AccessChain
+*   spv._address_of
+*   spv.Branch
+*   spv.BranchConditional
+*   spv.CompositeConstruct
+*   spv.CompositeExtract
+*   spv.CompositeInsert
+*   spv.ControlBarrier
+*   spv.CopyMemory
+*   spv.EntryPoint
+*   spv.ExecutionMode
+*   spv.FMod
+*   spv.globalVariable
+*   spv.Load
+*   spv.loop
+*   spv.MemoryBarrier
+*   spv._merge
+*   spv._reference_of
+*   spv.selection
+*   spv.SMod
+*   spv.specConstant
+*   spv.Store
+*   spv.SubgroupBallotKHR
+*   spv.Variable
+*   spv.Unreachable
+
+## Control flow conversion
+
+**Note: these conversions have not been implemented yet**
+
+## Decorations conversion
+
+**Note: these conversions have not been implemented yet**
+
+## GLSL extended instruction set
+
+**Note: these conversions have not been implemented yet**
+
+This section describes how SPIR-V ops from GLSL extended instructions set are
+mapped to LLVM Dialect.
+
+### Direct conversions
+
+SPIR-V Dialect op                     | LLVM Dialect op
+:-----------------------------------: | :-----------------------------------:
+`spv.GLSL.Ceil`                       | `llvm.intr.ceil`
+`spv.GLSL.Cos`                        | `llvm.intr.cos`
+`spv.GLSL.Exp`                        | `llvm.intr.exp`
+`spv.GLSL.FAbs`                       | `llvm.intr.fabs`
+`spv.GLSL.Floor`                      | `llvm.intr.floor`
+`spv.GLSL.FMax`                       | `llvm.intr.maxnum`
+`spv.GLSL.FMin`                       | `llvm.intr.minnum`
+`spv.GLSL.Log`                        | `llvm.intr.log`
+`spv.GLSL.Sin`                        | `llvm.intr.sin`
+`spv.GLSL.Sqrt`                       | `llvm.intr.sqrt`
+
+### Special cases
+
+TODO: add more patterns for special cases.
+
+`spv.Tan` is mapped to:
+```mlir
+                               %sin = "llvm.intr.sin"(%arg) : (!llvm.float) -> !llvm.float
+                               %cos = "llvm.intr.cos"(%arg) : (!llvm.float) -> !llvm.float
+%res = spv.Tan %arg : f32  =>  %res = fdiv %sin, %cos : !llvm.float
+```
+
+## Function conversion and related ops
+
+This section describes the conversion of function-related operations from SPIR-V
+to LLVM dialect.
+
+### `spv.func`
+This op declares or defines a SPIR-V function and it is converted to `llvm.func`.
+This conversion handles signarture conversion, and function control attributes
+remapping to LLVM dialect function [`passthrough` attribute](Dialects/LLVM.md#Attribute-pass-through).
+
+The following mapping is used to map [SPIR-V function control](SPIRVFunctionAttributes) to
+[LLVM function attributes](LLVMFunctionAttributes):
+
+SPIR-V Function Control Attributes    | LLVM Function Attributes
+:-----------------------------------: | :-----------------------------------:
+None                                  | No function attributes passed
+Inline                                | `alwaysinline`
+DontInline                            | `noinline`
+Pure                                  | `readonly`
+Const                                 | `readnone`
+
+### `spv.FunctionCall`
+
+`spv.FunctionCall` maps to `llvm.call`. For example:
+
+```mlir
+%0 = spv.FunctionCall @foo() : () -> i32  =>  %0 = llvm.call @foo() : () -> !llvm.float
+spv.FunctionCall @bar(%0) : (i32) -> ()   =>  llvm.call @bar(%0) : (!llvm.float) -> ()
+```
+
+### `spv.Return` and `spv.ReturnValue`
+
+In LLVM IR, functions may return either 1 or 0 value. Hence, we map both ops to
+`llvm.return` with or without a return value.
+
+## Module ops
+
+Module in SPIR-V has one region that contains one block. It is defined via
+`spv.module` op that also takes a range of attributes:
+
+*   Addressing model
+*   Memory model
+*   Version-Capability-Extension attribute
+
+`spv.module` is converted into `ModuleOp`. This plays a role of enclosing scope
+to LLVM ops. At the moment, SPIR-V module attributes are ignored.
+
+`spv._module_end` is mapped to an equivalent terminator `ModuleTerminatorOp`.
+
+[LLVMFunctionAttributes]: https://llvm.org/docs/LangRef.html#function-attributes
+[SPIRVFunctionAttributes]: https://www.khronos.org/registry/spir-v/specs/unified1/SPIRV.html#_a_id_function_control_a_function_control
diff --git a/mlir/examples/standalone/CMakeLists.txt b/mlir/examples/standalone/CMakeLists.txt
index 9f30f70f949fd..59d3c693546f4 100644
--- a/mlir/examples/standalone/CMakeLists.txt
+++ b/mlir/examples/standalone/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.10)
+cmake_minimum_required(VERSION 3.13.4)
 
 if(POLICY CMP0068)
   cmake_policy(SET CMP0068 NEW)
@@ -28,6 +28,11 @@ set(MLIR_BINARY_DIR ${CMAKE_BINARY_DIR})
 
 list(APPEND CMAKE_MODULE_PATH "${MLIR_CMAKE_DIR}")
 list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_DIR}")
+
+if(LLVM_ENABLE_ZLIB)
+  find_package(ZLIB)
+endif()
+
 include(TableGen)
 include(AddLLVM)
 include(AddMLIR)
diff --git a/mlir/include/mlir/Dialect/Affine/Utils.h b/mlir/include/mlir/Dialect/Affine/Utils.h
index 19df93f760f5e..327c4df7f7749 100644
--- a/mlir/include/mlir/Dialect/Affine/Utils.h
+++ b/mlir/include/mlir/Dialect/Affine/Utils.h
@@ -13,12 +13,15 @@
 #ifndef MLIR_DIALECT_AFFINE_UTILS_H
 #define MLIR_DIALECT_AFFINE_UTILS_H
 
+#include "mlir/Support/LLVM.h"
+
 namespace mlir {
 
 class AffineForOp;
 class AffineIfOp;
 class AffineParallelOp;
 struct LogicalResult;
+class Operation;
 
 /// Replaces parallel affine.for op with 1-d affine.parallel op.
 /// mlir::isLoopParallel detect the parallel affine.for ops.
@@ -31,6 +34,15 @@ void affineParallelize(AffineForOp forOp);
 /// significant code expansion in some cases.
 LogicalResult hoistAffineIfOp(AffineIfOp ifOp, bool *folded = nullptr);
 
+/// Vectorizes affine loops in 'loops' using the n-D vectorization factors in
+/// 'vectorSizes'. By default, each vectorization factor is applied
+/// inner-to-outer to the loops of each loop nest. 'fastestVaryingPattern' can
+/// be optionally used to provide a different loop vectorization order.
+void vectorizeAffineLoops(
+    Operation *parentOp,
+    llvm::DenseSet<Operation *, DenseMapInfo<Operation *>> &loops,
+    ArrayRef<int64_t> vectorSizes, ArrayRef<int64_t> fastestVaryingPattern);
+
 } // namespace mlir
 
 #endif // MLIR_DIALECT_AFFINE_UTILS_H
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index ce0b3de82d2c9..5322e243427a9 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -514,21 +514,29 @@ def LLVM_CondBrOp : LLVM_TerminatorOp<"cond_br",
      NoSideEffect]> {
   let arguments = (ins LLVMI1:$condition,
                    Variadic<LLVM_Type>:$trueDestOperands,
-                   Variadic<LLVM_Type>:$falseDestOperands);
+                   Variadic<LLVM_Type>:$falseDestOperands,
+                   OptionalAttr<ElementsAttr>:$branch_weights);
   let successors = (successor AnySuccessor:$trueDest, AnySuccessor:$falseDest);
   let assemblyFormat = [{
-    $condition `,`
+    $condition ( `weights` `(` $branch_weights^ `)` )? `,`
     $trueDest (`(` $trueDestOperands^ `:` type($trueDestOperands) `)`)? `,`
     $falseDest (`(` $falseDestOperands^ `:` type($falseDestOperands) `)`)?
     attr-dict
   }];
 
   let builders = [OpBuilder<
-    "OpBuilder &builder, OperationState &result, Value condition,"
-    "Block *trueDest, ValueRange trueOperands,"
-    "Block *falseDest, ValueRange falseOperands", [{
-      build(builder, result, condition, trueOperands, falseOperands, trueDest,
-            falseDest);
+     "OpBuilder &builder, OperationState &result, Value condition,"
+     "Block *trueDest, ValueRange trueOperands,"
+     "Block *falseDest, ValueRange falseOperands,"
+     "Optional<std::pair<uint32_t, uint32_t>> weights = {}", [{
+        ElementsAttr weightsAttr;
+        if (weights) {
+          weightsAttr =
+              builder.getI32VectorAttr({static_cast<int32_t>(weights->first),
+                                       static_cast<int32_t>(weights->second)});
+        }
+        build(builder, result, condition, trueOperands, falseOperands, weightsAttr,
+              trueDest, falseDest);
   }]>, OpBuilder<
     "OpBuilder &builder, OperationState &result, Value condition,"
     "Block *trueDest, Block *falseDest, ValueRange falseOperands = {}", [{
@@ -991,6 +999,35 @@ def LLVM_MaskedStoreOp
   let assemblyFormat = "$value `,` $data `,` $mask attr-dict `:` "
     "type($value) `,` type($mask) `into` type($data)";
 }
+
+/// Create a call to Masked Gather intrinsic.
+def LLVM_masked_gather
+    : LLVM_OneResultOp<"intr.masked.gather">,
+      Arguments<(ins LLVM_Type:$ptrs, LLVM_Type:$mask,
+                 Variadic<LLVM_Type>:$pass_thru, I32Attr:$alignment)> {
+  string llvmBuilder = [{
+    $res = $pass_thru.empty() ? builder.CreateMaskedGather(
+      $ptrs, llvm::Align($alignment.getZExtValue()), $mask) :
+      builder.CreateMaskedGather(
+        $ptrs, llvm::Align($alignment.getZExtValue()), $mask, $pass_thru[0]);
+  }];
+  let assemblyFormat =
+    "operands attr-dict `:` functional-type(operands, results)";
+}
+
+/// Create a call to Masked Scatter intrinsic.
+def LLVM_masked_scatter
+    : LLVM_ZeroResultOp<"intr.masked.scatter">,
+      Arguments<(ins LLVM_Type:$value, LLVM_Type:$ptrs, LLVM_Type:$mask,
+                 I32Attr:$alignment)> {
+  string llvmBuilder = [{
+    builder.CreateMaskedScatter(
+      $value, $ptrs, llvm::Align($alignment.getZExtValue()), $mask);
+  }];
+  let assemblyFormat = "$value `,` $ptrs `,` $mask attr-dict `:` "
+    "type($value) `,` type($mask) `into` type($ptrs)";
+}
+
 //
 // Atomic operations.
 //
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
index 7d259fde05e72..81f911a37cea2 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
@@ -485,7 +485,9 @@ class GenericOpBase<string mnemonic> : LinalgStructuredBase_Op<mnemonic,
                    AffineMapArrayAttr:$indexing_maps,
                    ArrayAttr:$iterator_types,
                    OptionalAttr<StrAttr>:$doc,
-                   OptionalAttr<StrAttr>:$library_call);
+                   OptionalAttr<StrAttr>:$library_call,
+                   Confined<OptionalAttr<I64Attr>,
+                     [IntMinValue<0>]>:$symbol_source);
   let results = (outs Variadic<AnyRankedTensor>:$output_tensors);
   let regions = (region AnyRegion:$region);
   let extraClassDeclaration = [{
@@ -493,7 +495,7 @@ class GenericOpBase<string mnemonic> : LinalgStructuredBase_Op<mnemonic,
       return SmallVector<StringRef, 8>{
         getArgsInAttrName(), getArgsOutAttrName(), getDocAttrName(),
         getIndexingMapsAttrName(), getLibraryCallAttrName(),
-        getIteratorTypesAttrName()
+        getIteratorTypesAttrName(), getSymbolSourceAttrName()
       };
     }
 
@@ -508,12 +510,18 @@ class GenericOpBase<string mnemonic> : LinalgStructuredBase_Op<mnemonic,
     llvm::Optional<SmallVector<StringRef, 8>> referenceIterators() {
       llvm_unreachable(
         "No such thing as reference iterator types for a generic op.");
-     }
+    }
 
     llvm::Optional<SmallVector<AffineMap, 8>> referenceIndexingMaps() {
       llvm_unreachable(
         "No such thing as reference indexing maps for a generic op.");
-     }
+    }
+
+    llvm::Optional<unsigned> getSymbolSource() {
+      auto ss = symbol_source();
+      return ss.hasValue() ?
+        llvm::Optional<unsigned>(ss.getValue().getLimitedValue()) : llvm::None;
+    }
   }];
 
   let printer = [{ return ::print(p, *this); }];
@@ -549,6 +557,10 @@ def GenericOp : GenericOpBase<"generic"> {
         Each element of the list represents and iterator of one of the following
         types:
           parallel, reduction, window
+      - symbol_source: index of the operand whose dimensions will be propagated
+        as symbols to the indexing maps. When specified the number of symbols
+        in each of the indexing maps has to be either 0 or the rank of the
+        specified operand.
 
     Example:
     Defining a #matmul_trait attribute in MLIR can be done as follows:
@@ -630,6 +642,35 @@ def GenericOp : GenericOpBase<"generic"> {
     escape naturally. Still, transformations and rewrites that take advantage of
     tensor SSA values are expected to be useful and will be added in the near
     future.
+
+    Example of 1D convolution with symbols:
+    ```mlir
+    #conv_1d_accesses = [
+      affine_map<(m, n)[dimN] -> (m + n - dimN floordiv 2)>, // in
+      affine_map<(m, n)[dimN] -> (n)>, // filter
+      affine_map<(m, n)[dimN] -> (m)> // out
+    ]
+
+    #conv_1d_trait = {
+      doc = "O(m) += I(m + n - size(n) floordiv 2) * K(n)",
+      indexing_maps = #conv_1d_accesses,
+      library_call  = "linalg_conv_1d",
+      iterator_types = ["parallel", "parallel"],
+      symbol_source = 1
+    }
+
+    linalg.generic #conv_1d_trait %in, %filter, %out {
+      ^bb0(%a: f32, %b: f32, %c: f32) :
+        %d = mulf %a, %b : f32
+        %e = addf %c, %d : f32
+        linalg.yield %e : f32
+    } : memref<?xf32>,
+        memref<?xf32>,
+        memref<?xf32>
+    ```
+    where symbol s0 will be substituted with `dim %filter, %c0` i.e. the first
+    and only dimension of the second operand as specified by the symbol_source
+    attribute.
   }];
 
   let builders = [
diff --git a/mlir/include/mlir/Dialect/Linalg/Utils/Utils.h b/mlir/include/mlir/Dialect/Linalg/Utils/Utils.h
index ab85cebee1782..0dbf863c2f69f 100644
--- a/mlir/include/mlir/Dialect/Linalg/Utils/Utils.h
+++ b/mlir/include/mlir/Dialect/Linalg/Utils/Utils.h
@@ -101,11 +101,32 @@ template <typename ConcreteOp>
 SmallVector<Value, 8> getViewSizes(OpBuilder &builder, ConcreteOp linalgOp) {
   auto loc = linalgOp.getLoc();
   SmallVector<Value, 8> res;
+  SmallVector<unsigned, 4> ranks;
   for (auto v : linalgOp.getInputsAndOutputBuffers()) {
     MemRefType t = v.getType().template cast<MemRefType>();
+    ranks.push_back(t.getRank());
     for (unsigned i = 0; i < t.getRank(); ++i)
       res.push_back(builder.create<DimOp>(loc, v, i));
   }
+
+  auto attr = linalgOp.template getAttrOfType<IntegerAttr>("symbol_source");
+  if (attr) {
+    // Find the correct position for inserting values for symbols.
+    unsigned numSymb = ranks[attr.getInt()], symbolsPos = 0;
+    for (unsigned idx = 0; idx < attr.getInt(); idx++)
+      symbolsPos += ranks[idx];
+
+    // Append the end of the value list that corresponds to the
+    // values mapping to symbols. Since inside concatinated map symbols are
+    // repeated we have to repeat the sizes as well.
+
+    // Reserve is mandatory to avoid a potential undefined behavior with
+    // pushing back to smallvector from itself.
+    res.reserve(res.size() + ranks.size() * numSymb);
+    for (unsigned idx = 0, s = ranks.size(); idx < s; ++idx)
+      for (unsigned idx2 = 0; idx2 < numSymb; ++idx2)
+        res.push_back(res[symbolsPos + idx2]);
+  }
   return res;
 }
 
@@ -114,7 +135,7 @@ SmallVector<Value, 8> getViewSizes(OpBuilder &builder, ConcreteOp linalgOp) {
 /// `createAndFold` builder method. If `folder` is null, the regular `create`
 /// method is called.
 SmallVector<Value, 4> applyMapToValues(OpBuilder &b, Location loc,
-                                       AffineMap map, ArrayRef<Value> values,
+                                       AffineMap map, ValueRange values,
                                        OperationFolder *folder = nullptr);
 
 /// Returns all the operands of `linalgOp` that are not views.
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVOps.td b/mlir/include/mlir/Dialect/SPIRV/SPIRVOps.td
index e20bde5904bd3..04ebdcb24435a 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVOps.td
@@ -490,11 +490,11 @@ def SPV_VariableOp : SPV_Op<"Variable", []> {
     Result Type must be an OpTypePointer. Its Type operand is the type of
     object in memory.
 
-    Storage Class is the Storage Class of the memory holding the object. It
-    cannot be Generic. It must be the same as the Storage Class operand of
-    the Result Type.
+    Storage Class is the Storage Class of the memory holding the object.
+    Since the op is used to model function-level variables, the storage class
+    must be the `Function` Storage Class.
 
-    Initializer is optional.  If Initializer is present, it will be the
+    Initializer is optional. If Initializer is present, it will be the
     initial value of the variable’s memory content. Initializer must be an
     <id> from a constant instruction or a global (module scope) OpVariable
     instruction. Initializer must have the same type as the type pointed to
@@ -504,14 +504,10 @@ def SPV_VariableOp : SPV_Op<"Variable", []> {
 
     ```
     variable-op ::= ssa-id `=` `spv.Variable` (`init(` ssa-use `)`)?
-                    (`bind(` integer-literal, integer-literal `)`)?
-                    (`built_in(` string-literal `)`)?
                     attribute-dict? `:` spirv-pointer-type
     ```
 
-    where `init` specifies initializer and `bind` specifies the
-    descriptor set and binding number. `built_in` specifies SPIR-V
-    BuiltIn decoration associated with the op.
+    where `init` specifies initializer.
 
     #### Example:
 
@@ -519,9 +515,7 @@ def SPV_VariableOp : SPV_Op<"Variable", []> {
     %0 = spv.constant ...
 
     %1 = spv.Variable : !spv.ptr<f32, Function>
-    %2 = spv.Variable init(%0): !spv.ptr<f32, Private>
-    %3 = spv.Variable init(%0) bind(1, 2): !spv.ptr<f32, Uniform>
-    %3 = spv.Variable built_in("GlobalInvocationID") : !spv.ptr<vector<3xi32>, Uniform>
+    %2 = spv.Variable init(%0): !spv.ptr<f32, Function>
     ```
   }];
 
diff --git a/mlir/include/mlir/Dialect/Shape/IR/ShapeBase.td b/mlir/include/mlir/Dialect/Shape/IR/ShapeBase.td
index ef20b5a9813d9..4c00c1f3c8f7c 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/ShapeBase.td
+++ b/mlir/include/mlir/Dialect/Shape/IR/ShapeBase.td
@@ -104,13 +104,24 @@ def Shape_ValueShapeType : DialectType<ShapeDialect,
   }];
 }
 
+def Shape_ExtentTensorType :
+    1DTensorOf<[Index]>,
+    BuildableType<"::mlir::RankedTensorType::get({ShapedType::kDynamicSize}, "
+                  "$_builder.getType<::mlir::IndexType>())"> {
+  let typeDescription = [{
+    The extent tensor is a tensor of rank one with arbitrarily many index
+    elements. Like `!shape.shape`, it is used to represent shapes with the
+    difference that it is guaranteed to be error-free.
+  }];
+}
+
 def Shape_ShapeOrSizeType : AnyTypeOf<[Shape_SizeType, Shape_ShapeType],
   "shape or size">;
 
-def Shape_ExtentTensorType : 1DTensorOf<[Index]>;
-
 def Shape_ShapeOrExtentTensorType : AnyTypeOf<[Shape_ShapeType,
                                                Shape_ExtentTensorType],
                                               "shape or extent tensor">;
 
+def Shape_SizeOrIndexType : AnyTypeOf<[Shape_SizeType, Index], "size or index">;
+
 #endif // SHAPE_BASE_TD
diff --git a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
index 090b4c6f4abbc..32d6ebafff321 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
+++ b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
@@ -96,18 +96,20 @@ def Shape_BroadcastOp : Shape_Op<"broadcast", [Commutative]> {
 }
 
 def Shape_ConstShapeOp : Shape_Op<"const_shape", [ConstantLike, NoSideEffect]> {
-  let summary = "Creates a constant of !shape.shape type";
+  let summary = "Creates a constant shape or extent tensor";
   let description = [{
-    Creates a !shape.shape with rank given by the length of `shape` and with
-    dimension sizes given by the values of `shape`.
+    Creates a constant shape or extent tensor. The individual extents are given
+    as the `shape` attribute. The number of these values equals the shape's
+    rank.
 
     ```mlir
-    %0 = shape.const_shape []
-    %1 = shape.const_shape [1, 2, 3]
+    %0 = shape.const_shape [] : !shape.shape
+    %1 = shape.const_shape [1, 2, 3] : !shape.shape
+    %2 = shape.const_shape [4, 5, 6] : tensor<?xindex>
     ```
   }];
   let arguments = (ins IndexElementsAttr:$shape);
-  let results = (outs Shape_ShapeType:$result);
+  let results = (outs Shape_ShapeOrExtentTensorType:$result);
 
   // TODO: Move this to main so that all shape ops implement these.
   let printer = [{ return ::print(p, *this); }];
@@ -154,6 +156,7 @@ def Shape_ShapeEqOp : Shape_Op<"shape_eq", [Commutative, NoSideEffect]> {
   let results = (outs I1:$result);
 
   let assemblyFormat = "$lhs `,` $rhs attr-dict `:` type($lhs) `,` type($rhs)";
+  let hasFolder = 1;
 }
 
 def Shape_FromExtentsOp : Shape_Op<"from_extents", [NoSideEffect]> {
@@ -194,16 +197,17 @@ def Shape_FromExtentTensorOp : Shape_Op<"from_extent_tensor", [NoSideEffect]> {
 def Shape_RankOp : Shape_Op<"rank", [NoSideEffect]> {
   let summary = "Gets the rank of a shape";
   let description = [{
-    Returns the rank of the shape, i.e. the number of extents.
+    Returns the rank of the shape or extent tensor, i.e. the number of extents.
   }];
 
-  let arguments = (ins Shape_ShapeType:$shape);
-  let results = (outs Shape_SizeType:$rank);
+  let arguments = (ins Shape_ShapeOrExtentTensorType:$shape);
+  let results = (outs Shape_SizeOrIndexType:$rank);
 
-  let assemblyFormat = "attr-dict $shape";
+  let assemblyFormat = "$shape `:` type($shape) `->` type($rank) attr-dict";
 
   let hasFolder = 1;
   let hasCanonicalizer = 1;
+  let verifier = [{ return ::verify(*this); }];
 }
 
 def Shape_ToExtentTensorOp : Shape_Op<"to_extent_tensor", [NoSideEffect]> {
@@ -225,17 +229,16 @@ def Shape_ToExtentTensorOp : Shape_Op<"to_extent_tensor", [NoSideEffect]> {
 }
 
 def Shape_GetExtentOp : Shape_Op<"get_extent", [NoSideEffect]> {
-  let summary = "Gets the specified extent from a shape";
+  let summary = "Gets the specified extent from a shape or extent tensor";
   let description = [{
-    Gets the extent indexed by `dim` from `shape`.
-    If the shape is an error, it returns an error size.
+    Gets the extent indexed by `dim` from the `shape` operand. If the shape is
+    an error then it returns an error size.
   }];
-  let arguments = (ins
-    Shape_ShapeType:$shape,
-    Shape_SizeType:$dim
-  );
-  let results = (outs Shape_SizeType:$extent);
-  let assemblyFormat = "$shape `,` $dim attr-dict";
+  let arguments = (ins Shape_ShapeOrExtentTensorType:$shape,
+                       Shape_SizeOrIndexType:$dim);
+  let results = (outs Shape_SizeOrIndexType:$extent);
+  let assemblyFormat = "$shape `,` $dim `:` type($shape) `,` type($dim) `->` "
+                       "type($extent) attr-dict";
 
   let builders = [
     // Builder that allows passing a constant dimension as a simple integer.
@@ -249,6 +252,7 @@ def Shape_GetExtentOp : Shape_Op<"get_extent", [NoSideEffect]> {
   }];
 
   let hasFolder = 1;
+  let verifier = [{ return ::verify(*this); }];
 }
 
 def Shape_IndexToSizeOp : Shape_Op<"index_to_size", [NoSideEffect]> {
@@ -390,11 +394,17 @@ def Shape_ReduceOp : Shape_Op<"reduce",
 def Shape_ShapeOfOp : Shape_Op<"shape_of", [NoSideEffect]> {
   let summary = "Returns shape of a value or shaped type operand";
 
+  let description = [{
+    The operation takes a value or a shaped operand as an argument and it
+    returns a shape or extent tensor.
+  }];
+
   let arguments = (ins AnyTypeOf<[AnyShaped, Shape_ValueShapeType]>:$arg);
-  let results = (outs Shape_ShapeType:$result);
+  let results = (outs Shape_ShapeOrExtentTensorType:$result);
 
-  let assemblyFormat = "$arg `:` type($arg) attr-dict";
+  let assemblyFormat = "$arg `:` type($arg) `->` type($result) attr-dict";
 
+  let verifier = [{ return ::verify(*this); }];
   let hasFolder = 1;
 }
 
@@ -501,11 +511,14 @@ def Shape_ConcatOp : Shape_Op<"concat", []> {
 //===----------------------------------------------------------------------===//
 
 // TODO: Move the code below and witnesses to a different file.
-def Shape_AnyOp : Shape_Op<"any", [Commutative, NoSideEffect]> {
+def Shape_AnyOp : Shape_Op<"any", [Commutative,
+                                   NoSideEffect,
+                                   SameOperandsAndResultType]> {
   let summary = "Return any combination of the input shapes";
   let description = [{
-    This operation takes multiple input shapes and returns some combination of
-    their dimensions. This can be best seen with examples below.
+    This operation takes multiple input shapes or extent tensors and returns
+    some combination of their dimensions. This can be best seen with examples
+    below.
 
     The result is undefined, but still side-effect free, in cases where the
     inputs have differing ranks or differ in extents of shared dimensions.
@@ -517,11 +530,10 @@ def Shape_AnyOp : Shape_Op<"any", [Commutative, NoSideEffect]> {
     ```
   }];
 
-  let arguments = (ins Variadic<Shape_ShapeType>:$inputs);
-  let results = (outs Shape_ShapeType:$result);
-
-  let assemblyFormat = "$inputs attr-dict";
+  let arguments = (ins Variadic<Shape_ShapeOrExtentTensorType>:$inputs);
+  let results = (outs Shape_ShapeOrExtentTensorType:$result);
 
+  let assemblyFormat = "$inputs `:` type($result)  attr-dict";
   let hasFolder = 1;
 }
 
@@ -609,8 +621,9 @@ def Shape_AssumingYieldOp : Shape_Op<"assuming_yield",
 def Shape_CstrBroadcastableOp : Shape_Op<"cstr_broadcastable", [Commutative]> {
   let summary = "Determines if 2 shapes can be successfully broadcasted";
   let description = [{
-    Given 2 input shapes, return a witness specifying if they are broadcastable.
-    This broadcastable follows the same logic as what shape.broadcast documents.
+    Given two input shapes or extent tensors, return a witness specifying if
+    they are broadcastable. This broadcastable follows the same logic as what
+    shape.broadcast documents.
 
     "cstr" operations represent runtime assertions.
 
@@ -621,10 +634,11 @@ def Shape_CstrBroadcastableOp : Shape_Op<"cstr_broadcastable", [Commutative]> {
     ```
   }];
 
-  let arguments = (ins Shape_ShapeType:$lhs, Shape_ShapeType:$rhs);
+  let arguments = (ins Shape_ShapeOrExtentTensorType:$lhs,
+                       Shape_ShapeOrExtentTensorType:$rhs);
   let results = (outs Shape_WitnessType:$result);
 
-  let assemblyFormat = "$lhs `,` $rhs attr-dict";
+  let assemblyFormat = "$lhs `,` $rhs `:` type($lhs) `,` type($rhs) attr-dict";
 
   let hasCanonicalizer = 1;
   let hasFolder = 1;
diff --git a/mlir/include/mlir/Dialect/StandardOps/EDSC/Intrinsics.h b/mlir/include/mlir/Dialect/StandardOps/EDSC/Intrinsics.h
index f654020a53385..b0d649ca5ee26 100644
--- a/mlir/include/mlir/Dialect/StandardOps/EDSC/Intrinsics.h
+++ b/mlir/include/mlir/Dialect/StandardOps/EDSC/Intrinsics.h
@@ -29,6 +29,8 @@ using std_divis = ValueBuilder<SignedDivIOp>;
 using std_diviu = ValueBuilder<UnsignedDivIOp>;
 using std_dim = ValueBuilder<DimOp>;
 using std_extract_element = ValueBuilder<ExtractElementOp>;
+using std_fpext = ValueBuilder<FPExtOp>;
+using std_fptrunc = ValueBuilder<FPTruncOp>;
 using std_im = ValueBuilder<ImOp>;
 using std_index_cast = ValueBuilder<IndexCastOp>;
 using std_muli = ValueBuilder<MulIOp>;
diff --git a/mlir/include/mlir/Dialect/StandardOps/IR/Ops.h b/mlir/include/mlir/Dialect/StandardOps/IR/Ops.h
index 0f24d74dcac28..2500343c0af38 100644
--- a/mlir/include/mlir/Dialect/StandardOps/IR/Ops.h
+++ b/mlir/include/mlir/Dialect/StandardOps/IR/Ops.h
@@ -21,7 +21,7 @@
 #include "mlir/Interfaces/CallInterfaces.h"
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
-#include "mlir/Interfaces/VectorUnrollInterface.h"
+#include "mlir/Interfaces/VectorInterfaces.h"
 #include "mlir/Interfaces/ViewLikeInterface.h"
 
 // Pull in all enum type definitions and utility function declarations.
diff --git a/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td b/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td
index 702b912d3103c..78307b8974763 100644
--- a/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td
+++ b/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td
@@ -18,7 +18,7 @@ include "mlir/IR/OpAsmInterface.td"
 include "mlir/Interfaces/CallInterfaces.td"
 include "mlir/Interfaces/ControlFlowInterfaces.td"
 include "mlir/Interfaces/SideEffectInterfaces.td"
-include "mlir/Interfaces/VectorUnrollInterface.td"
+include "mlir/Interfaces/VectorInterfaces.td"
 include "mlir/Interfaces/ViewLikeInterface.td"
 
 def StandardOps_Dialect : Dialect {
diff --git a/mlir/include/mlir/Dialect/Utils/StructuredOpsUtils.h b/mlir/include/mlir/Dialect/Utils/StructuredOpsUtils.h
index 168e877e50561..41d6142519365 100644
--- a/mlir/include/mlir/Dialect/Utils/StructuredOpsUtils.h
+++ b/mlir/include/mlir/Dialect/Utils/StructuredOpsUtils.h
@@ -46,6 +46,10 @@ inline bool isColumnMajorMatmul(ArrayAttr indexingMaps) {
   return indexingMaps == maps;
 }
 
+/// Attribute name for the IntegerAttr which encodes the index of operand
+/// whose dimensions will be propagated as symbols to the indexing maps
+constexpr StringRef getSymbolSourceAttrName() { return "symbol_source"; }
+
 /// Attribute name for the AffineArrayAttr which encodes the relationship
 /// between a structured op iterators' and its operands.
 constexpr StringRef getIndexingMapsAttrName() { return "indexing_maps"; }
diff --git a/mlir/include/mlir/Dialect/Vector/VectorOps.h b/mlir/include/mlir/Dialect/Vector/VectorOps.h
index 0f6aa66e926c0..edf9557df389b 100644
--- a/mlir/include/mlir/Dialect/Vector/VectorOps.h
+++ b/mlir/include/mlir/Dialect/Vector/VectorOps.h
@@ -19,7 +19,7 @@
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/StandardTypes.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
-#include "mlir/Interfaces/VectorUnrollInterface.h"
+#include "mlir/Interfaces/VectorInterfaces.h"
 
 namespace mlir {
 class MLIRContext;
diff --git a/mlir/include/mlir/Dialect/Vector/VectorOps.td b/mlir/include/mlir/Dialect/Vector/VectorOps.td
index 8880c288b648e..b49cc4a62a502 100644
--- a/mlir/include/mlir/Dialect/Vector/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/VectorOps.td
@@ -15,7 +15,7 @@
 
 include "mlir/Dialect/Affine/IR/AffineOpsBase.td"
 include "mlir/Interfaces/SideEffectInterfaces.td"
-include "mlir/Interfaces/VectorUnrollInterface.td"
+include "mlir/Interfaces/VectorInterfaces.td"
 
 def Vector_Dialect : Dialect {
   let name = "vector";
@@ -905,34 +905,9 @@ def Vector_ExtractStridedSliceOp :
   let assemblyFormat = "$vector attr-dict `:` type($vector) `to` type(results)";
 }
 
-def Vector_TransferOpUtils {
-  code extraTransferDeclaration = [{
-    static StringRef getMaskedAttrName() { return "masked"; }
-    static StringRef getPermutationMapAttrName() { return "permutation_map"; }
-    bool isMaskedDim(unsigned dim) {
-      return !masked() ||
-        masked()->cast<ArrayAttr>()[dim].cast<BoolAttr>().getValue();
-    }
-    MemRefType getMemRefType() {
-      return memref().getType().cast<MemRefType>();
-    }
-    VectorType getVectorType() {
-      return vector().getType().cast<VectorType>();
-    }
-    // Number of dimensions that participate in the permutation map.
-    unsigned getTransferRank() {
-      return permutation_map().getNumResults();
-    }
-    // Number of leading dimensions that do not participate in the permutation
-    // map.
-    unsigned getLeadingMemRefRank() {
-      return getMemRefType().getRank() - permutation_map().getNumResults();
-    }
-  }];
-}
-
 def Vector_TransferReadOp :
   Vector_Op<"transfer_read", [
+      DeclareOpInterfaceMethods<VectorTransferOpInterface>,
       DeclareOpInterfaceMethods<VectorUnrollOpInterface, ["getShapeForUnroll"]>
     ]>,
     Arguments<(ins AnyMemRef:$memref, Variadic<Index>:$indices,
@@ -1090,23 +1065,12 @@ def Vector_TransferReadOp :
               "ArrayRef<bool> maybeMasked = {}">
   ];
 
-  let extraClassDeclaration = Vector_TransferOpUtils.extraTransferDeclaration #
-  [{
-    /// Build the default minor identity map suitable for a vector transfer.
-    /// This also handles the case memref<... x vector<...>> -> vector<...> in
-    /// which the rank of the identity map must take the vector element type
-    /// into account.
-    static AffineMap getTransferMinorIdentityMap(
-      MemRefType memRefType, VectorType vectorType) {
-        return impl::getTransferMinorIdentityMap(memRefType, vectorType);
-      }
-  }];
-
   let hasFolder = 1;
 }
 
 def Vector_TransferWriteOp :
   Vector_Op<"transfer_write", [
+      DeclareOpInterfaceMethods<VectorTransferOpInterface>,
       DeclareOpInterfaceMethods<VectorUnrollOpInterface, ["getShapeForUnroll"]>
     ]>,
     Arguments<(ins AnyVector:$vector, AnyMemRef:$memref,
@@ -1183,19 +1147,122 @@ def Vector_TransferWriteOp :
               "Value memref, ValueRange indices, AffineMap permutationMap">,
   ];
 
-  let extraClassDeclaration = Vector_TransferOpUtils.extraTransferDeclaration #
-  [{
-    /// Build the default minor identity map suitable for a vector transfer.
-    /// This also handles the case memref<... x vector<...>> -> vector<...> in
-    /// which the rank of the identity map must take the vector element type
-    /// into account.
-    static AffineMap getTransferMinorIdentityMap(
-      MemRefType memRefType, VectorType vectorType) {
-        return impl::getTransferMinorIdentityMap(memRefType, vectorType);
-      }
+  let hasFolder = 1;
+}
+
+def Vector_GatherOp :
+  Vector_Op<"gather">,
+    Arguments<(ins AnyMemRef:$base,
+               VectorOfRankAndType<[1], [AnyInteger]>:$indices,
+               VectorOfRankAndType<[1], [I1]>:$mask,
+               Variadic<VectorOfRank<[1]>>:$pass_thru)>,
+    Results<(outs VectorOfRank<[1]>:$result)> {
+
+  let summary = "gathers elements from memory into a vector as defined by an index vector";
+
+  let description = [{
+    The gather operation gathers elements from memory into a 1-D vector as
+    defined by a base and a 1-D index vector, but only if the corresponding
+    bit is set in a 1-D mask vector. Otherwise, the element is taken from a
+    1-D pass-through vector, if provided, or left undefined. Informally the
+    semantics are:
+    ```
+    if (!defined(pass_thru)) pass_thru = [undef, .., undef]
+    result[0] := mask[0] ? MEM[base + index[0]] : pass_thru[0]
+    result[1] := mask[1] ? MEM[base + index[1]] : pass_thru[1]
+    etc.
+    ```
+    The vector dialect leaves out-of-bounds behavior undefined.
+
+    The gather operation can be used directly where applicable, or can be used
+    during progressively lowering to bring other memory operations closer to
+    hardware ISA support for a gather. The semantics of the operation closely
+    correspond to those of the `llvm.masked.gather`
+    [intrinsic](https://llvm.org/docs/LangRef.html#llvm-masked-gather-intrinsics).
+
+    Example:
+
+    ```mlir
+    %g = vector.gather %base, %indices, %mask, %pass_thru
+        : (memref<?xf32>, vector<16xi32>, vector<16xi1>, vector<16xf32>) -> vector<16xf32>
+    ```
+
   }];
+  let extraClassDeclaration = [{
+    MemRefType getMemRefType() {
+      return base().getType().cast<MemRefType>();
+    }
+    VectorType getIndicesVectorType() {
+      return indices().getType().cast<VectorType>();
+    }
+    VectorType getMaskVectorType() {
+      return mask().getType().cast<VectorType>();
+    }
+    VectorType getPassThruVectorType() {
+      return (llvm::size(pass_thru()) == 0)
+        ? VectorType()
+        : (*pass_thru().begin()).getType().cast<VectorType>();
+    }
+    VectorType getResultVectorType() {
+      return result().getType().cast<VectorType>();
+    }
+  }];
+  let assemblyFormat = "operands attr-dict `:` functional-type(operands, results)";
+}
 
-  let hasFolder = 1;
+def Vector_ScatterOp :
+  Vector_Op<"scatter">,
+    Arguments<(ins AnyMemRef:$base,
+               VectorOfRankAndType<[1], [AnyInteger]>:$indices,
+               VectorOfRankAndType<[1], [I1]>:$mask,
+               VectorOfRank<[1]>:$value)> {
+
+  let summary = "scatters elements from a vector into memory as defined by an index vector";
+
+  let description = [{
+    The scatter operation scatters elements from a 1-D vector into memory as
+    defined by a base and a 1-D index vector, but only if the corresponding
+    bit in a 1-D mask vector is set. Otherwise, no action is taken for that
+    element. Informally the semantics are:
+    ```
+    if (mask[0]) MEM[base + index[0]] = value[0]
+    if (mask[1]) MEM[base + index[1]] = value[1]
+    etc.
+    ```
+    The vector dialect leaves out-of-bounds and repeated index behavior
+    undefined. Underlying implementations may enforce strict sequential
+    semantics for the latter, though.
+    TODO: enforce the latter always?
+
+    The scatter operation can be used directly where applicable, or can be used
+    during progressively lowering to bring other memory operations closer to
+    hardware ISA support for a scatter. The semantics of the operation closely
+    correspond to those of the `llvm.masked.scatter`
+    [intrinsic](https://llvm.org/docs/LangRef.html#llvm-masked-scatter-intrinsics).
+
+    Example:
+
+    ```mlir
+    vector.scatter %base, %indices, %mask, %value
+        : vector<16xi32>, vector<16xi1>, vector<16xf32> into memref<?xf32>
+    ```
+  }];
+  let extraClassDeclaration = [{
+    MemRefType getMemRefType() {
+      return base().getType().cast<MemRefType>();
+    }
+    VectorType getIndicesVectorType() {
+      return indices().getType().cast<VectorType>();
+    }
+    VectorType getMaskVectorType() {
+      return mask().getType().cast<VectorType>();
+    }
+    VectorType getValueVectorType() {
+      return value().getType().cast<VectorType>();
+    }
+  }];
+  let assemblyFormat = "$base `,` $indices `,` $mask `,` $value attr-dict `:` "
+    "type($indices) `,` type($mask) `,` type($value) `into` type($base)";
 }
 
 def Vector_ShapeCastOp :
diff --git a/mlir/include/mlir/Dialect/Vector/VectorUtils.h b/mlir/include/mlir/Dialect/Vector/VectorUtils.h
index 19f7f9538307d..448004db32fa5 100644
--- a/mlir/include/mlir/Dialect/Vector/VectorUtils.h
+++ b/mlir/include/mlir/Dialect/Vector/VectorUtils.h
@@ -153,6 +153,12 @@ AffineMap
 makePermutationMap(Operation *op, ArrayRef<Value> indices,
                    const DenseMap<Operation *, unsigned> &loopToVectorDim);
 
+/// Build the default minor identity map suitable for a vector transfer. This
+/// also handles the case memref<... x vector<...>> -> vector<...> in which the
+/// rank of the identity map must take the vector element type into account.
+AffineMap getTransferMinorIdentityMap(MemRefType memRefType,
+                                      VectorType vectorType);
+
 namespace matcher {
 
 /// Matches vector.transfer_read, vector.transfer_write and ops that return a
diff --git a/mlir/include/mlir/IR/AffineExpr.h b/mlir/include/mlir/IR/AffineExpr.h
index 2302abd8554d4..2df16ee2bfc9d 100644
--- a/mlir/include/mlir/IR/AffineExpr.h
+++ b/mlir/include/mlir/IR/AffineExpr.h
@@ -118,6 +118,10 @@ class AffineExpr {
   AffineExpr replaceDimsAndSymbols(ArrayRef<AffineExpr> dimReplacements,
                                    ArrayRef<AffineExpr> symReplacements) const;
 
+  /// Replace symbols[0 .. numDims - 1] by
+  /// symbols[shift .. shift + numDims - 1].
+  AffineExpr shiftSymbols(unsigned numSymbols, unsigned shift) const;
+
   AffineExpr operator+(int64_t v) const;
   AffineExpr operator+(AffineExpr other) const;
   AffineExpr operator-() const;
diff --git a/mlir/include/mlir/Interfaces/CMakeLists.txt b/mlir/include/mlir/Interfaces/CMakeLists.txt
index 0de2b5a8688bc..65e19f3eec1bf 100644
--- a/mlir/include/mlir/Interfaces/CMakeLists.txt
+++ b/mlir/include/mlir/Interfaces/CMakeLists.txt
@@ -5,6 +5,6 @@ add_mlir_interface(DerivedAttributeOpInterface)
 add_mlir_interface(InferTypeOpInterface)
 add_mlir_interface(LoopLikeInterface)
 add_mlir_interface(SideEffectInterfaces)
-add_mlir_interface(VectorUnrollInterface)
+add_mlir_interface(VectorInterfaces)
 add_mlir_interface(ViewLikeInterface)
 
diff --git a/mlir/include/mlir/Interfaces/VectorUnrollInterface.h b/mlir/include/mlir/Interfaces/VectorInterfaces.h
similarity index 59%
rename from mlir/include/mlir/Interfaces/VectorUnrollInterface.h
rename to mlir/include/mlir/Interfaces/VectorInterfaces.h
index a68cc3411533b..2134969e4020d 100644
--- a/mlir/include/mlir/Interfaces/VectorUnrollInterface.h
+++ b/mlir/include/mlir/Interfaces/VectorInterfaces.h
@@ -1,4 +1,4 @@
-//===- VectorUnrollInterface.h - Vector unrolling interface ---------------===//
+//===- VectorInterfaces.h - Vector interfaces -----------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,18 +6,18 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file implements the operation interface for vector ops that can be
-// unrolled.
+// This file implements the operation interfaces for vector ops.
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef MLIR_INTERFACES_VECTORUNROLLINTERFACE_H
-#define MLIR_INTERFACES_VECTORUNROLLINTERFACE_H
+#ifndef MLIR_INTERFACES_VECTORINTERFACES_H
+#define MLIR_INTERFACES_VECTORINTERFACES_H
 
+#include "mlir/IR/AffineMap.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/StandardTypes.h"
 
 /// Include the generated interface declarations.
-#include "mlir/Interfaces/VectorUnrollInterface.h.inc"
+#include "mlir/Interfaces/VectorInterfaces.h.inc"
 
-#endif // MLIR_INTERFACES_VECTORUNROLLINTERFACE_H
+#endif // MLIR_INTERFACES_VECTORINTERFACES_H
diff --git a/mlir/include/mlir/Interfaces/VectorInterfaces.td b/mlir/include/mlir/Interfaces/VectorInterfaces.td
new file mode 100644
index 0000000000000..aefbb7d471172
--- /dev/null
+++ b/mlir/include/mlir/Interfaces/VectorInterfaces.td
@@ -0,0 +1,194 @@
+//===- VectorInterfaces.td - Vector interfaces -------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines the interface for operations on vectors.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_INTERFACES_VECTORINTERFACES
+#define MLIR_INTERFACES_VECTORINTERFACES
+
+include "mlir/IR/OpBase.td"
+
+def VectorUnrollOpInterface : OpInterface<"VectorUnrollOpInterface"> {
+  let description = [{
+    Encodes properties of an operation on vectors that can be unrolled.
+  }];
+  let cppNamespace = "::mlir";
+
+  let methods = [
+    InterfaceMethod<
+      /*desc=*/[{
+        Return the shape ratio of unrolling to the target vector shape
+        `targetShape`. Return `None` if the op cannot be unrolled to the target
+        vector shape.
+      }],
+      /*retTy=*/"Optional<SmallVector<int64_t, 4>>",
+      /*methodName=*/"getShapeForUnroll",
+      /*args=*/(ins),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        assert($_op.getOperation()->getNumResults() == 1);
+        auto vt = $_op.getResult().getType().
+          template dyn_cast<VectorType>();
+        if (!vt)
+          return None;
+        SmallVector<int64_t, 4> res(vt.getShape().begin(), vt.getShape().end());
+        return res;
+      }]
+    >,
+  ];
+}
+
+def VectorTransferOpInterface : OpInterface<"VectorTransferOpInterface"> {
+  let description = [{
+    Encodes properties of an operation on vectors that can be unrolled.
+  }];
+  let cppNamespace = "::mlir";
+
+  let methods = [
+    StaticInterfaceMethod<
+      /*desc=*/"Return the `masked` attribute name.",
+      /*retTy=*/"StringRef",
+      /*methodName=*/"getMaskedAttrName",
+      /*args=*/(ins),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/ [{ return "masked"; }]
+    >,
+    StaticInterfaceMethod<
+      /*desc=*/"Return the `permutation_map` attribute name.",
+      /*retTy=*/"StringRef",
+      /*methodName=*/"getPermutationMapAttrName",
+      /*args=*/(ins),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/ [{ return "permutation_map"; }]
+    >,
+    InterfaceMethod<
+      /*desc=*/[{
+      Return `false` when the `masked` attribute at dimension
+      `dim` is set to `false`. Return `true` otherwise.}],
+      /*retTy=*/"bool",
+      /*methodName=*/"isMaskedDim",
+      /*args=*/(ins "unsigned":$dim),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        return !$_op.masked() ||
+          $_op.masked()->template cast<ArrayAttr>()[dim]
+                        .template cast<BoolAttr>().getValue();
+      }]
+    >,
+    InterfaceMethod<
+      /*desc=*/"Return the memref operand.",
+      /*retTy=*/"Value",
+      /*methodName=*/"memref",
+      /*args=*/(ins),
+      /*methodBody=*/"return $_op.memref();"
+      /*defaultImplementation=*/
+    >,
+    InterfaceMethod<
+      /*desc=*/"Return the vector operand or result.",
+      /*retTy=*/"Value",
+      /*methodName=*/"vector",
+      /*args=*/(ins),
+      /*methodBody=*/"return $_op.vector();"
+      /*defaultImplementation=*/
+    >,
+    InterfaceMethod<
+      /*desc=*/"Return the indices operands.",
+      /*retTy=*/"ValueRange",
+      /*methodName=*/"indices",
+      /*args=*/(ins),
+      /*methodBody=*/"return $_op.indices();"
+      /*defaultImplementation=*/
+    >,
+    InterfaceMethod<
+      /*desc=*/"Return the permutation map.",
+      /*retTy=*/"AffineMap",
+      /*methodName=*/"permutation_map",
+      /*args=*/(ins),
+      /*methodBody=*/"return $_op.permutation_map();"
+      /*defaultImplementation=*/
+    >,
+    InterfaceMethod<
+      /*desc=*/"Return the `masked` boolean ArrayAttr.",
+      /*retTy=*/"Optional<ArrayAttr>",
+      /*methodName=*/"masked",
+      /*args=*/(ins),
+      /*methodBody=*/"return $_op.masked();"
+      /*defaultImplementation=*/
+    >,
+    InterfaceMethod<
+      /*desc=*/"Return the MemRefType.",
+      /*retTy=*/"MemRefType",
+      /*methodName=*/"getMemRefType",
+      /*args=*/(ins),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/
+        "return $_op.memref().getType().template cast<MemRefType>();"
+    >,
+    InterfaceMethod<
+      /*desc=*/"Return the VectorType.",
+      /*retTy=*/"VectorType",
+      /*methodName=*/"getVectorType",
+      /*args=*/(ins),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/
+        "return $_op.vector().getType().template cast<VectorType>();"
+    >,
+    InterfaceMethod<
+      /*desc=*/[{ Return the number of dimensions that participate in the
+                  permutation map.}],
+      /*retTy=*/"unsigned",
+      /*methodName=*/"getTransferRank",
+      /*args=*/(ins),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/
+        "return $_op.permutation_map().getNumResults();"
+    >,
+    InterfaceMethod<
+      /*desc=*/[{ Return the number of leading memref dimensions that do not
+                  participate in the permutation map.}],
+      /*retTy=*/"unsigned",
+      /*methodName=*/"getLeadingMemRefRank",
+      /*args=*/(ins),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/
+        "return $_op.getMemRefType().getRank() - $_op.getTransferRank();"
+    >,
+    InterfaceMethod<
+      /*desc=*/[{
+      Helper function to account for the fact that `permutationMap` results and
+      `op.indices` sizes may not match and may not be aligned. The first
+      `getLeadingMemRefRank()` indices may just be indexed and not transferred
+      from/into the vector.
+      For example:
+      ```
+         vector.transfer %0[%i, %j, %k, %c0] :
+           memref<?x?x?x?xf32>, vector<2x4xf32>
+      ```
+      with `permutation_map = (d0, d1, d2, d3) -> (d2, d3)`.
+      Provide a zip function to coiterate on 2 running indices: `resultIdx` and
+      `indicesIdx` which accounts for this misalignment.
+      }],
+      /*retTy=*/"void",
+      /*methodName=*/"zipResultAndIndexing",
+      /*args=*/(ins "llvm::function_ref<void(int64_t, int64_t)>":$fun),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        for (int64_t resultIdx = 0,
+                   indicesIdx = $_op.getLeadingMemRefRank(),
+                   eResult = $_op.getTransferRank();
+           resultIdx < eResult;
+           ++resultIdx, ++indicesIdx)
+        fun(resultIdx, indicesIdx);
+      }]
+    >,
+  ];
+}
+
+#endif // MLIR_INTERFACES_VECTORINTERFACES
diff --git a/mlir/include/mlir/Interfaces/VectorUnrollInterface.td b/mlir/include/mlir/Interfaces/VectorUnrollInterface.td
deleted file mode 100644
index 166780b20e772..0000000000000
--- a/mlir/include/mlir/Interfaces/VectorUnrollInterface.td
+++ /dev/null
@@ -1,46 +0,0 @@
-//===- VectorUnrollInterface.td - VectorUnroll interface ---*- tablegen -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Defines the interface for operations on vectors that can be unrolled.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef MLIR_INTERFACES_VECTORUNROLLINTERFACE
-#define MLIR_INTERFACES_VECTORUNROLLINTERFACE
-
-include "mlir/IR/OpBase.td"
-
-def VectorUnrollOpInterface : OpInterface<"VectorUnrollOpInterface"> {
-  let description = [{
-    Encodes properties of an operation on vectors that can be unrolled.
-  }];
-  let cppNamespace = "::mlir";
-
-  let methods = [
-    InterfaceMethod<[{
-        Returns the shape ratio of unrolling to the target vector shape
-        `targetShape`. Returns `None` if the op cannot be unrolled to the target
-        vector shape.
-      }],
-      "Optional<SmallVector<int64_t, 4>>",
-      "getShapeForUnroll",
-      (ins),
-      /*methodBody=*/[{}],
-      [{
-        auto vt = this->getOperation()->getResult(0).getType().
-          template dyn_cast<VectorType>();
-        if (!vt)
-          return None;
-        SmallVector<int64_t, 4> res(vt.getShape().begin(), vt.getShape().end());
-        return res;
-      }]
-    >,
-  ];
-}
-
-#endif // MLIR_INTERFACES_VECTORUNROLLINTERFACE
diff --git a/mlir/include/mlir/Transforms/DialectConversion.h b/mlir/include/mlir/Transforms/DialectConversion.h
index 26b7ce6ea6c35..8bffb9649d1f8 100644
--- a/mlir/include/mlir/Transforms/DialectConversion.h
+++ b/mlir/include/mlir/Transforms/DialectConversion.h
@@ -113,20 +113,40 @@ class TypeConverter {
 
   /// Register a materialization function, which must be convertible to the
   /// following form:
-  ///   `Optional<Value>(PatternRewriter &, T, ValueRange, Location)`,
+  ///   `Optional<Value>(OpBuilder &, T, ValueRange, Location)`,
   /// where `T` is any subclass of `Type`. This function is responsible for
-  /// creating an operation, using the PatternRewriter and Location provided,
-  /// that "casts" a range of values into a single value of the given type `T`.
-  /// It must return a Value of the converted type on success, an `llvm::None`
-  /// if it failed but other materialization can be attempted, and `nullptr` on
+  /// creating an operation, using the OpBuilder and Location provided, that
+  /// "casts" a range of values into a single value of the given type `T`. It
+  /// must return a Value of the converted type on success, an `llvm::None` if
+  /// it failed but other materialization can be attempted, and `nullptr` on
   /// unrecoverable failure. It will only be called for (sub)types of `T`.
   /// Materialization functions must be provided when a type conversion
   /// results in more than one type, or if a type conversion may persist after
   /// the conversion has finished.
+  ///
+  /// This method registers a materialization that will be called when
+  /// converting an illegal block argument type, to a legal type.
   template <typename FnT,
             typename T = typename llvm::function_traits<FnT>::template arg_t<1>>
-  void addMaterialization(FnT &&callback) {
-    registerMaterialization(
+  void addArgumentMaterialization(FnT &&callback) {
+    argumentMaterializations.emplace_back(
+        wrapMaterialization<T>(std::forward<FnT>(callback)));
+  }
+  /// This method registers a materialization that will be called when
+  /// converting a legal type to an illegal source type. This is used when
+  /// conversions to an illegal type must persist beyond the main conversion.
+  template <typename FnT,
+            typename T = typename llvm::function_traits<FnT>::template arg_t<1>>
+  void addSourceMaterialization(FnT &&callback) {
+    sourceMaterializations.emplace_back(
+        wrapMaterialization<T>(std::forward<FnT>(callback)));
+  }
+  /// This method registers a materialization that will be called when
+  /// converting type from an illegal, or source, type to a legal type.
+  template <typename FnT,
+            typename T = typename llvm::function_traits<FnT>::template arg_t<1>>
+  void addTargetMaterialization(FnT &&callback) {
+    targetMaterializations.emplace_back(
         wrapMaterialization<T>(std::forward<FnT>(callback)));
   }
 
@@ -182,9 +202,24 @@ class TypeConverter {
   Optional<SignatureConversion> convertBlockSignature(Block *block);
 
   /// Materialize a conversion from a set of types into one result type by
-  /// generating a cast operation of some kind.
-  Value materializeConversion(PatternRewriter &rewriter, Location loc,
-                              Type resultType, ValueRange inputs);
+  /// generating a cast sequence of some kind. See the respective
+  /// `add*Materialization` for more information on the context for these
+  /// methods.
+  Value materializeArgumentConversion(OpBuilder &builder, Location loc,
+                                      Type resultType, ValueRange inputs) {
+    return materializeConversion(argumentMaterializations, builder, loc,
+                                 resultType, inputs);
+  }
+  Value materializeSourceConversion(OpBuilder &builder, Location loc,
+                                    Type resultType, ValueRange inputs) {
+    return materializeConversion(sourceMaterializations, builder, loc,
+                                 resultType, inputs);
+  }
+  Value materializeTargetConversion(OpBuilder &builder, Location loc,
+                                    Type resultType, ValueRange inputs) {
+    return materializeConversion(targetMaterializations, builder, loc,
+                                 resultType, inputs);
+  }
 
 private:
   /// The signature of the callback used to convert a type. If the new set of
@@ -193,8 +228,15 @@ class TypeConverter {
   using ConversionCallbackFn =
       std::function<Optional<LogicalResult>(Type, SmallVectorImpl<Type> &)>;
 
-  using MaterializationCallbackFn = std::function<Optional<Value>(
-      PatternRewriter &, Type, ValueRange, Location)>;
+  /// The signature of the callback used to materialize a conversion.
+  using MaterializationCallbackFn =
+      std::function<Optional<Value>(OpBuilder &, Type, ValueRange, Location)>;
+
+  /// Attempt to materialize a conversion using one of the provided
+  /// materialization functions.
+  Value materializeConversion(
+      MutableArrayRef<MaterializationCallbackFn> materializations,
+      OpBuilder &builder, Location loc, Type resultType, ValueRange inputs);
 
   /// Generate a wrapper for the given callback. This allows for accepting
   /// different callback forms, that all compose into a single version.
@@ -240,24 +282,21 @@ class TypeConverter {
   template <typename T, typename FnT>
   MaterializationCallbackFn wrapMaterialization(FnT &&callback) {
     return [callback = std::forward<FnT>(callback)](
-               PatternRewriter &rewriter, Type resultType, ValueRange inputs,
+               OpBuilder &builder, Type resultType, ValueRange inputs,
                Location loc) -> Optional<Value> {
       if (T derivedType = resultType.dyn_cast<T>())
-        return callback(rewriter, derivedType, inputs, loc);
+        return callback(builder, derivedType, inputs, loc);
       return llvm::None;
     };
   }
 
-  /// Register a materialization.
-  void registerMaterialization(MaterializationCallbackFn &&callback) {
-    materializations.emplace_back(std::move(callback));
-  }
-
   /// The set of registered conversion functions.
   SmallVector<ConversionCallbackFn, 4> conversions;
 
   /// The list of registered materialization functions.
-  SmallVector<MaterializationCallbackFn, 2> materializations;
+  SmallVector<MaterializationCallbackFn, 2> argumentMaterializations;
+  SmallVector<MaterializationCallbackFn, 2> sourceMaterializations;
+  SmallVector<MaterializationCallbackFn, 2> targetMaterializations;
 
   /// A set of cached conversions to avoid recomputing in the common case.
   /// Direct 1-1 conversions are the most common, so this cache stores the
@@ -325,7 +364,7 @@ class ConversionPattern : public RewritePattern {
 
 protected:
   /// An optional type converter for use by this pattern.
-  TypeConverter *typeConverter;
+  TypeConverter *typeConverter = nullptr;
 
 private:
   using RewritePattern::rewrite;
diff --git a/mlir/integration_test/Dialect/Vector/CPU/test-gather.mlir b/mlir/integration_test/Dialect/Vector/CPU/test-gather.mlir
new file mode 100644
index 0000000000000..5ed8f3ee38f8d
--- /dev/null
+++ b/mlir/integration_test/Dialect/Vector/CPU/test-gather.mlir
@@ -0,0 +1,97 @@
+// RUN: mlir-opt %s -convert-scf-to-std -convert-vector-to-llvm -convert-std-to-llvm | \
+// RUN: mlir-cpu-runner -e entry -entry-point-result=void \
+// RUN:   -shared-libs=%mlir_integration_test_dir/libmlir_c_runner_utils%shlibext | \
+// RUN: FileCheck %s
+
+func @gather8(%base: memref<?xf32>,
+              %indices: vector<8xi32>, %mask: vector<8xi1>) -> vector<8xf32> {
+  %g = vector.gather %base, %indices, %mask
+    : (memref<?xf32>, vector<8xi32>, vector<8xi1>) -> vector<8xf32>
+  return %g : vector<8xf32>
+}
+
+func @gather_pass_thru8(%base: memref<?xf32>,
+                        %indices: vector<8xi32>, %mask: vector<8xi1>,
+                        %pass_thru: vector<8xf32>) -> vector<8xf32> {
+  %g = vector.gather %base, %indices, %mask, %pass_thru
+    : (memref<?xf32>, vector<8xi32>, vector<8xi1>, vector<8xf32>) -> vector<8xf32>
+  return %g : vector<8xf32>
+}
+
+func @entry() {
+  // Set up memory.
+  %c0 = constant 0: index
+  %c1 = constant 1: index
+  %c10 = constant 10: index
+  %A = alloc(%c10) : memref<?xf32>
+  scf.for %i = %c0 to %c10 step %c1 {
+    %i32 = index_cast %i : index to i32
+    %fi = sitofp %i32 : i32 to f32
+    store %fi, %A[%i] : memref<?xf32>
+  }
+
+  // Set up idx vector.
+  %i0 = constant 0: i32
+  %i1 = constant 1: i32
+  %i2 = constant 2: i32
+  %i3 = constant 3: i32
+  %i4 = constant 4: i32
+  %i5 = constant 5: i32
+  %i6 = constant 6: i32
+  %i9 = constant 9: i32
+  %0 = vector.broadcast %i0 : i32 to vector<8xi32>
+  %1 = vector.insert %i6, %0[1] : i32 into vector<8xi32>
+  %2 = vector.insert %i1, %1[2] : i32 into vector<8xi32>
+  %3 = vector.insert %i3, %2[3] : i32 into vector<8xi32>
+  %4 = vector.insert %i5, %3[4] : i32 into vector<8xi32>
+  %5 = vector.insert %i4, %4[5] : i32 into vector<8xi32>
+  %6 = vector.insert %i9, %5[6] : i32 into vector<8xi32>
+  %idx = vector.insert %i2, %6[7] : i32 into vector<8xi32>
+
+  // Set up pass thru vector.
+  %u = constant -7.0: f32
+  %pass = vector.broadcast %u : f32 to vector<8xf32>
+
+  // Set up masks.
+  %t = constant 1: i1
+  %none = vector.constant_mask [0] : vector<8xi1>
+  %all = vector.constant_mask [8] : vector<8xi1>
+  %some = vector.constant_mask [4] : vector<8xi1>
+  %more = vector.insert %t, %some[7] : i1 into vector<8xi1>
+
+  //
+  // Gather tests.
+  //
+
+  %g1 = call @gather8(%A, %idx, %all)
+    : (memref<?xf32>, vector<8xi32>, vector<8xi1>)
+    -> (vector<8xf32>)
+  vector.print %g1 : vector<8xf32>
+  // CHECK: ( 0, 6, 1, 3, 5, 4, 9, 2 )
+
+  %g2 = call @gather_pass_thru8(%A, %idx, %none, %pass)
+    : (memref<?xf32>, vector<8xi32>, vector<8xi1>, vector<8xf32>)
+    -> (vector<8xf32>)
+  vector.print %g2 : vector<8xf32>
+  // CHECK: ( -7, -7, -7, -7, -7, -7, -7, -7 )
+
+  %g3 = call @gather_pass_thru8(%A, %idx, %some, %pass)
+    : (memref<?xf32>, vector<8xi32>, vector<8xi1>, vector<8xf32>)
+    -> (vector<8xf32>)
+  vector.print %g3 : vector<8xf32>
+  // CHECK: ( 0, 6, 1, 3, -7, -7, -7, -7 )
+
+  %g4 = call @gather_pass_thru8(%A, %idx, %more, %pass)
+    : (memref<?xf32>, vector<8xi32>, vector<8xi1>, vector<8xf32>)
+    -> (vector<8xf32>)
+  vector.print %g4 : vector<8xf32>
+  // CHECK: ( 0, 6, 1, 3, -7, -7, -7, 2 )
+
+  %g5 = call @gather_pass_thru8(%A, %idx, %all, %pass)
+    : (memref<?xf32>, vector<8xi32>, vector<8xi1>, vector<8xf32>)
+    -> (vector<8xf32>)
+  vector.print %g5 : vector<8xf32>
+  // CHECK: ( 0, 6, 1, 3, 5, 4, 9, 2 )
+
+  return
+}
diff --git a/mlir/integration_test/Dialect/Vector/CPU/test-scatter.mlir b/mlir/integration_test/Dialect/Vector/CPU/test-scatter.mlir
new file mode 100644
index 0000000000000..6dd0cf1695525
--- /dev/null
+++ b/mlir/integration_test/Dialect/Vector/CPU/test-scatter.mlir
@@ -0,0 +1,135 @@
+// RUN: mlir-opt %s -convert-scf-to-std -convert-vector-to-llvm -convert-std-to-llvm | \
+// RUN: mlir-cpu-runner -e entry -entry-point-result=void \
+// RUN:   -shared-libs=%mlir_integration_test_dir/libmlir_c_runner_utils%shlibext | \
+// RUN: FileCheck %s
+
+func @scatter8(%base: memref<?xf32>,
+               %indices: vector<8xi32>,
+               %mask: vector<8xi1>, %value: vector<8xf32>) {
+  vector.scatter %base, %indices, %mask, %value
+    : vector<8xi32>, vector<8xi1>, vector<8xf32> into memref<?xf32>
+  return
+}
+
+func @printmem(%A: memref<?xf32>) {
+  %f = constant 0.0: f32
+  %0 = vector.broadcast %f : f32 to vector<8xf32>
+  %1 = constant 0: index
+  %2 = load %A[%1] : memref<?xf32>
+  %3 = vector.insert %2, %0[0] : f32 into vector<8xf32>
+  %4 = constant 1: index
+  %5 = load %A[%4] : memref<?xf32>
+  %6 = vector.insert %5, %3[1] : f32 into vector<8xf32>
+  %7 = constant 2: index
+  %8 = load %A[%7] : memref<?xf32>
+  %9 = vector.insert %8, %6[2] : f32 into vector<8xf32>
+  %10 = constant 3: index
+  %11 = load %A[%10] : memref<?xf32>
+  %12 = vector.insert %11, %9[3] : f32 into vector<8xf32>
+  %13 = constant 4: index
+  %14 = load %A[%13] : memref<?xf32>
+  %15 = vector.insert %14, %12[4] : f32 into vector<8xf32>
+  %16 = constant 5: index
+  %17 = load %A[%16] : memref<?xf32>
+  %18 = vector.insert %17, %15[5] : f32 into vector<8xf32>
+  %19 = constant 6: index
+  %20 = load %A[%19] : memref<?xf32>
+  %21 = vector.insert %20, %18[6] : f32 into vector<8xf32>
+  %22 = constant 7: index
+  %23 = load %A[%22] : memref<?xf32>
+  %24 = vector.insert %23, %21[7] : f32 into vector<8xf32>
+  vector.print %24 : vector<8xf32>
+  return
+}
+
+func @entry() {
+  // Set up memory.
+  %c0 = constant 0: index
+  %c1 = constant 1: index
+  %c8 = constant 8: index
+  %A = alloc(%c8) : memref<?xf32>
+  scf.for %i = %c0 to %c8 step %c1 {
+    %i32 = index_cast %i : index to i32
+    %fi = sitofp %i32 : i32 to f32
+    store %fi, %A[%i] : memref<?xf32>
+  }
+
+  // Set up idx vector.
+  %i0 = constant 0: i32
+  %i1 = constant 1: i32
+  %i2 = constant 2: i32
+  %i3 = constant 3: i32
+  %i4 = constant 4: i32
+  %i5 = constant 5: i32
+  %i6 = constant 6: i32
+  %i7 = constant 7: i32
+  %0 = vector.broadcast %i7 : i32 to vector<8xi32>
+  %1 = vector.insert %i0, %0[1] : i32 into vector<8xi32>
+  %2 = vector.insert %i1, %1[2] : i32 into vector<8xi32>
+  %3 = vector.insert %i6, %2[3] : i32 into vector<8xi32>
+  %4 = vector.insert %i2, %3[4] : i32 into vector<8xi32>
+  %5 = vector.insert %i4, %4[5] : i32 into vector<8xi32>
+  %6 = vector.insert %i5, %5[6] : i32 into vector<8xi32>
+  %idx = vector.insert %i3, %6[7] : i32 into vector<8xi32>
+
+  // Set up value vector.
+  %f0 = constant 0.0: f32
+  %f1 = constant 1.0: f32
+  %f2 = constant 2.0: f32
+  %f3 = constant 3.0: f32
+  %f4 = constant 4.0: f32
+  %f5 = constant 5.0: f32
+  %f6 = constant 6.0: f32
+  %f7 = constant 7.0: f32
+  %7 = vector.broadcast %f0 : f32 to vector<8xf32>
+  %8 = vector.insert %f1, %7[1] : f32 into vector<8xf32>
+  %9 = vector.insert %f2, %8[2] : f32 into vector<8xf32>
+  %10 = vector.insert %f3, %9[3] : f32 into vector<8xf32>
+  %11 = vector.insert %f4, %10[4] : f32 into vector<8xf32>
+  %12 = vector.insert %f5, %11[5] : f32 into vector<8xf32>
+  %13 = vector.insert %f6, %12[6] : f32 into vector<8xf32>
+  %val = vector.insert %f7, %13[7] : f32 into vector<8xf32>
+
+  // Set up masks.
+  %t = constant 1: i1
+  %none = vector.constant_mask [0] : vector<8xi1>
+  %some = vector.constant_mask [4] : vector<8xi1>
+  %more = vector.insert %t, %some[7] : i1 into vector<8xi1>
+  %all = vector.constant_mask [8] : vector<8xi1>
+
+  //
+  // Scatter tests.
+  //
+
+  vector.print %idx : vector<8xi32>
+  // CHECK: ( 7, 0, 1, 6, 2, 4, 5, 3 )
+
+  call @printmem(%A) : (memref<?xf32>) -> ()
+  // CHECK: ( 0, 1, 2, 3, 4, 5, 6, 7 )
+
+  call @scatter8(%A, %idx, %none, %val)
+    : (memref<?xf32>, vector<8xi32>, vector<8xi1>, vector<8xf32>) -> ()
+
+  call @printmem(%A) : (memref<?xf32>) -> ()
+  // CHECK: ( 0, 1, 2, 3, 4, 5, 6, 7 )
+
+  call @scatter8(%A, %idx, %some, %val)
+    : (memref<?xf32>, vector<8xi32>, vector<8xi1>, vector<8xf32>) -> ()
+
+  call @printmem(%A) : (memref<?xf32>) -> ()
+  // CHECK: ( 1, 2, 2, 3, 4, 5, 3, 0 )
+
+  call @scatter8(%A, %idx, %more, %val)
+    : (memref<?xf32>, vector<8xi32>, vector<8xi1>, vector<8xf32>) -> ()
+
+  call @printmem(%A) : (memref<?xf32>) -> ()
+  // CHECK: ( 1, 2, 2, 7, 4, 5, 3, 0 )
+
+  call @scatter8(%A, %idx, %all, %val)
+    : (memref<?xf32>, vector<8xi32>, vector<8xi1>, vector<8xf32>) -> ()
+
+  call @printmem(%A) : (memref<?xf32>) -> ()
+  // CHECK: ( 1, 2, 4, 7, 5, 6, 3, 0 )
+
+  return
+}
diff --git a/mlir/lib/Analysis/Presburger/Simplex.cpp b/mlir/lib/Analysis/Presburger/Simplex.cpp
index 825346478819b..cd13f80fb9b91 100644
--- a/mlir/lib/Analysis/Presburger/Simplex.cpp
+++ b/mlir/lib/Analysis/Presburger/Simplex.cpp
@@ -674,7 +674,7 @@ class GBRSimplex {
       } else {
         if (simplex.con[i + 1].orientation == Orientation::Row) {
           unsigned ineqRow = simplex.con[i + 1].pos;
-          // Since it is an equality, the the sample value must be zero.
+          // Since it is an equality, the sample value must be zero.
           assert(simplex.tableau(ineqRow, 1) == 0 &&
                  "Equality's sample value must be zero.");
           for (unsigned col = 2; col < simplex.nCol; ++col) {
diff --git a/mlir/lib/Conversion/GPUToVulkan/ConvertLaunchFuncToVulkanCalls.cpp b/mlir/lib/Conversion/GPUToVulkan/ConvertLaunchFuncToVulkanCalls.cpp
index 10394b795daa3..e6527e0ca42bd 100644
--- a/mlir/lib/Conversion/GPUToVulkan/ConvertLaunchFuncToVulkanCalls.cpp
+++ b/mlir/lib/Conversion/GPUToVulkan/ConvertLaunchFuncToVulkanCalls.cpp
@@ -101,7 +101,7 @@ class VulkanLaunchFuncToVulkanCallsPass
   LLVM::LLVMType getInt32Type() { return llvmInt32Type; }
   LLVM::LLVMType getInt64Type() { return llvmInt64Type; }
 
-  /// Creates a LLVM global for the given `name`.
+  /// Creates an LLVM global for the given `name`.
   Value createEntryPointNameConstant(StringRef name, Location loc,
                                      OpBuilder &builder);
 
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
index b0702912c0fd0..15aaa7947a497 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
@@ -408,6 +408,39 @@ class BitFieldUExtractPattern
   }
 };
 
+class BranchConversionPattern : public SPIRVToLLVMConversion<spirv::BranchOp> {
+public:
+  using SPIRVToLLVMConversion<spirv::BranchOp>::SPIRVToLLVMConversion;
+
+  LogicalResult
+  matchAndRewrite(spirv::BranchOp branchOp, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    rewriter.replaceOpWithNewOp<LLVM::BrOp>(branchOp, operands,
+                                            branchOp.getTarget());
+    return success();
+  }
+};
+
+class BranchConditionalConversionPattern
+    : public SPIRVToLLVMConversion<spirv::BranchConditionalOp> {
+public:
+  using SPIRVToLLVMConversion<
+      spirv::BranchConditionalOp>::SPIRVToLLVMConversion;
+
+  LogicalResult
+  matchAndRewrite(spirv::BranchConditionalOp op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    // There is no support of branch weights in LLVM dialect at the moment.
+    if (auto weights = op.branch_weights())
+      return failure();
+
+    rewriter.replaceOpWithNewOp<LLVM::CondBrOp>(
+        op, op.condition(), op.getTrueBlock(), op.getTrueBlockArguments(),
+        op.getFalseBlock(), op.getFalseBlockArguments());
+    return success();
+  }
+};
+
 /// Converts SPIR-V operations that have straightforward LLVM equivalent
 /// into LLVM dialect operations.
 template <typename SPIRVOp, typename LLVMOp>
@@ -580,6 +613,84 @@ class ReturnValuePattern : public SPIRVToLLVMConversion<spirv::ReturnValueOp> {
   }
 };
 
+class MergePattern : public SPIRVToLLVMConversion<spirv::MergeOp> {
+public:
+  using SPIRVToLLVMConversion<spirv::MergeOp>::SPIRVToLLVMConversion;
+
+  LogicalResult
+  matchAndRewrite(spirv::MergeOp op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    rewriter.eraseOp(op);
+    return success();
+  }
+};
+
+/// Converts `spv.selection` with `spv.BranchConditional` in its header block.
+/// All blocks within selection should be reachable for conversion to succeed.
+class SelectionPattern : public SPIRVToLLVMConversion<spirv::SelectionOp> {
+public:
+  using SPIRVToLLVMConversion<spirv::SelectionOp>::SPIRVToLLVMConversion;
+
+  LogicalResult
+  matchAndRewrite(spirv::SelectionOp op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    // There is no support for `Flatten` or `DontFlatten` selection control at
+    // the moment. This are just compiler hints and can be performed during the
+    // optimization passes.
+    if (op.selection_control() != spirv::SelectionControl::None)
+      return failure();
+
+    // `spv.selection` should have at least two blocks: one selection header
+    // block and one merge block. If no blocks are present, or control flow
+    // branches straight to merge block (two blocks are present), the op is
+    // redundant and it is erased.
+    if (op.body().getBlocks().size() <= 2) {
+      rewriter.eraseOp(op);
+      return success();
+    }
+
+    Location loc = op.getLoc();
+
+    // Split the current block after `spv.selection`. The remaing ops will be
+    // used in `continueBlock`.
+    auto *currentBlock = rewriter.getInsertionBlock();
+    rewriter.setInsertionPointAfter(op);
+    auto position = rewriter.getInsertionPoint();
+    auto *continueBlock = rewriter.splitBlock(currentBlock, position);
+
+    // Extract conditional branch information from the header block. By SPIR-V
+    // dialect spec, it should contain `spv.BranchConditional` or `spv.Switch`
+    // op. Note that `spv.Switch op` is not supported at the moment in the
+    // SPIR-V dialect. Remove this block when finished.
+    auto *headerBlock = op.getHeaderBlock();
+    assert(headerBlock->getOperations().size() == 1);
+    auto condBrOp = dyn_cast<spirv::BranchConditionalOp>(
+        headerBlock->getOperations().front());
+    if (!condBrOp)
+      return failure();
+    rewriter.eraseBlock(headerBlock);
+
+    // Branch from merge block to continue block.
+    auto *mergeBlock = op.getMergeBlock();
+    Operation *terminator = mergeBlock->getTerminator();
+    ValueRange terminatorOperands = terminator->getOperands();
+    rewriter.setInsertionPointToEnd(mergeBlock);
+    rewriter.create<LLVM::BrOp>(loc, terminatorOperands, continueBlock);
+
+    // Link current block to `true` and `false` blocks within the selection.
+    Block *trueBlock = condBrOp.getTrueBlock();
+    Block *falseBlock = condBrOp.getFalseBlock();
+    rewriter.setInsertionPointToEnd(currentBlock);
+    rewriter.create<LLVM::CondBrOp>(loc, condBrOp.condition(), trueBlock,
+                                    condBrOp.trueTargetOperands(), falseBlock,
+                                    condBrOp.falseTargetOperands());
+
+    rewriter.inlineRegionBefore(op.body(), continueBlock);
+    rewriter.replaceOp(op, continueBlock->getArguments());
+    return success();
+  }
+};
+
 /// Converts SPIR-V shift ops to LLVM shift ops. Since LLVM dialect
 /// puts a restriction on `Shift` and `Base` to have the same bit width,
 /// `Shift` is zero or sign extended to match this specification. Cases when
@@ -808,6 +919,10 @@ void mlir::populateSPIRVToLLVMConversionPatterns(
       // Constant op
       ConstantScalarAndVectorPattern,
 
+      // Control Flow ops
+      BranchConversionPattern, BranchConditionalConversionPattern,
+      SelectionPattern, MergePattern,
+
       // Function Call op
       FunctionCallPattern,
 
diff --git a/mlir/lib/Conversion/ShapeToSCF/ShapeToSCF.cpp b/mlir/lib/Conversion/ShapeToSCF/ShapeToSCF.cpp
index 7986aaaa68163..824db2685a777 100644
--- a/mlir/lib/Conversion/ShapeToSCF/ShapeToSCF.cpp
+++ b/mlir/lib/Conversion/ShapeToSCF/ShapeToSCF.cpp
@@ -172,39 +172,37 @@ LogicalResult
 ShapeOfOpConverter::matchAndRewrite(ShapeOfOp op, ArrayRef<Value> operands,
                                     ConversionPatternRewriter &rewriter) const {
   ShapeOfOp::Adaptor transformed(operands);
-  auto tensorVal = transformed.arg();
-  auto tensorTy = tensorVal.getType();
+  Value arg = transformed.arg();
+  Type argTy = arg.getType();
 
   // For ranked tensors `shape_of` lowers to `std` and the pattern can be
   // found in the corresponding pass.
-  if (tensorTy.isa<RankedTensorType>())
+  if (argTy.isa<RankedTensorType>())
     return failure();
 
   // Allocate stack memory.
   auto loc = op.getLoc();
-  auto rankVal = rewriter.create<mlir::RankOp>(loc, tensorVal);
-  auto i64Ty = rewriter.getI64Type();
-  auto memTy = MemRefType::get({ShapedType::kDynamicSize}, i64Ty);
-  auto memVal = rewriter.create<AllocaOp>(loc, memTy, ValueRange({rankVal}));
+  Value rank = rewriter.create<mlir::RankOp>(loc, arg);
+  Type i64Ty = rewriter.getI64Type();
+  Type memTy = MemRefType::get({ShapedType::kDynamicSize}, i64Ty);
+  Value mem = rewriter.create<AllocaOp>(loc, memTy, ValueRange{rank});
 
   // Copy shape extents to stack-allocated memory.
-  auto zeroVal = rewriter.create<ConstantIndexOp>(loc, 0);
-  auto oneVal = rewriter.create<ConstantIndexOp>(loc, 1);
+  Value zero = rewriter.create<ConstantIndexOp>(loc, 0);
+  Value one = rewriter.create<ConstantIndexOp>(loc, 1);
   rewriter.create<scf::ForOp>(
-      loc, zeroVal, rankVal, oneVal, llvm::None,
-      [&](OpBuilder &b, Location loc, Value iVal, ValueRange args) {
-        auto dimVal = rewriter.create<DimOp>(loc, tensorVal, iVal);
-        auto dimIntVal = rewriter.create<IndexCastOp>(loc, dimVal, i64Ty);
-        rewriter.create<StoreOp>(loc, dimIntVal, memVal, ValueRange{iVal});
+      loc, zero, rank, one, llvm::None,
+      [&](OpBuilder &b, Location loc, Value iv, ValueRange args) {
+        Value dim = rewriter.create<DimOp>(loc, arg, iv);
+        Value dimInt = rewriter.create<IndexCastOp>(loc, dim, i64Ty);
+        rewriter.create<StoreOp>(loc, dimInt, mem, ValueRange{iv});
         rewriter.create<scf::YieldOp>(loc);
       });
 
   // Load extents to tensor value.
-  auto shapeIntVal = rewriter.create<TensorLoadOp>(loc, memVal);
-  auto indexTy = rewriter.getIndexType();
-  auto shapeTy = RankedTensorType::get({ShapedType::kDynamicSize}, indexTy);
-  rewriter.replaceOpWithNewOp<IndexCastOp>(op.getOperation(), shapeIntVal,
-                                           shapeTy);
+  Value extentTensorInt = rewriter.create<TensorLoadOp>(loc, mem);
+  rewriter.replaceOpWithNewOp<IndexCastOp>(op.getOperation(), extentTensorInt,
+                                           op.getType());
   return success();
 }
 
diff --git a/mlir/lib/Conversion/ShapeToStandard/CMakeLists.txt b/mlir/lib/Conversion/ShapeToStandard/CMakeLists.txt
index 4d97af2f235cc..8750c331859e0 100644
--- a/mlir/lib/Conversion/ShapeToStandard/CMakeLists.txt
+++ b/mlir/lib/Conversion/ShapeToStandard/CMakeLists.txt
@@ -1,7 +1,3 @@
-set(LLVM_TARGET_DEFINITIONS ShapeToStandardPatterns.td)
-mlir_tablegen(ShapeToStandardPatterns.inc -gen-rewriters)
-add_public_tablegen_target(ShapeToStandardPatternsIncGen)
-
 add_mlir_conversion_library(MLIRShapeToStandard
   ShapeToStandard.cpp
 
@@ -10,7 +6,6 @@ add_mlir_conversion_library(MLIRShapeToStandard
 
   DEPENDS
   MLIRConversionPassIncGen
-  ShapeToStandardPatternsIncGen
 
   LINK_COMPONENTS
   Core
diff --git a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
index f82019989e705..f239d1cfb4f0b 100644
--- a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
+++ b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
@@ -17,28 +17,30 @@
 using namespace mlir;
 using namespace mlir::shape;
 
-namespace {
-
-/// Generated conversion patterns.
-#include "ShapeToStandardPatterns.inc"
-
 /// Conversion patterns.
+namespace {
 class AnyOpConversion : public OpConversionPattern<AnyOp> {
 public:
   using OpConversionPattern<AnyOp>::OpConversionPattern;
 
   LogicalResult
   matchAndRewrite(AnyOp op, ArrayRef<Value> operands,
-                  ConversionPatternRewriter &rewriter) const override {
-    AnyOp::Adaptor transformed(operands);
-
-    // Replace `any` with its first operand.
-    // Any operand would be a valid substitution.
-    rewriter.replaceOp(op, {transformed.inputs().front()});
-    return success();
-  }
+                  ConversionPatternRewriter &rewriter) const override;
 };
+} // namespace
 
+LogicalResult
+AnyOpConversion::matchAndRewrite(AnyOp op, ArrayRef<Value> operands,
+                                 ConversionPatternRewriter &rewriter) const {
+  AnyOp::Adaptor transformed(operands);
+
+  // Replace `any` with its first operand.
+  // Any operand would be a valid substitution.
+  rewriter.replaceOp(op, {transformed.inputs().front()});
+  return success();
+}
+
+namespace {
 template <typename SrcOpTy, typename DstOpTy>
 class BinaryOpConversion : public OpConversionPattern<SrcOpTy> {
 public:
@@ -48,101 +50,107 @@ class BinaryOpConversion : public OpConversionPattern<SrcOpTy> {
   matchAndRewrite(SrcOpTy op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
     typename SrcOpTy::Adaptor adaptor(operands);
-    rewriter.replaceOpWithNewOp<DstOpTy>(op.getOperation(), adaptor.lhs(),
-                                         adaptor.rhs());
+    rewriter.replaceOpWithNewOp<DstOpTy>(op, adaptor.lhs(), adaptor.rhs());
     return success();
   }
 };
+} // namespace
 
+namespace {
 class ShapeOfOpConversion : public OpConversionPattern<ShapeOfOp> {
 public:
   using OpConversionPattern<ShapeOfOp>::OpConversionPattern;
 
   LogicalResult
   matchAndRewrite(ShapeOfOp op, ArrayRef<Value> operands,
-                  ConversionPatternRewriter &rewriter) const override {
-    ShapeOfOp::Adaptor transformed(operands);
-    auto loc = op.getLoc();
-    auto tensorVal = transformed.arg();
-    auto tensorTy = tensorVal.getType();
-
-    // For unranked tensors `shape_of` lowers to `scf` and the pattern can be
-    // found in the corresponding pass.
-    if (tensorTy.isa<UnrankedTensorType>())
-      return failure();
-
-    // Build values for individual dimensions.
-    SmallVector<Value, 8> dimValues;
-    auto rankedTensorTy = tensorTy.cast<RankedTensorType>();
-    int64_t rank = rankedTensorTy.getRank();
-    for (int64_t i = 0; i < rank; i++) {
-      if (rankedTensorTy.isDynamicDim(i)) {
-        auto dimVal = rewriter.create<DimOp>(loc, tensorVal, i);
-        dimValues.push_back(dimVal);
-      } else {
-        int64_t dim = rankedTensorTy.getDimSize(i);
-        auto dimVal = rewriter.create<ConstantIndexOp>(loc, dim);
-        dimValues.push_back(dimVal);
-      }
-    }
-
-    // Materialize shape as ranked tensor.
-    rewriter.replaceOpWithNewOp<TensorFromElementsOp>(op.getOperation(),
-                                                      dimValues);
-    return success();
-  }
+                  ConversionPatternRewriter &rewriter) const override;
 };
+} // namespace
 
-class ConstSizeOpConverter : public OpConversionPattern<ConstSizeOp> {
-public:
-  using OpConversionPattern<ConstSizeOp>::OpConversionPattern;
-
-  LogicalResult
-  matchAndRewrite(ConstSizeOp op, ArrayRef<Value> operands,
-                  ConversionPatternRewriter &rewriter) const override {
-    rewriter.replaceOpWithNewOp<ConstantIndexOp>(op.getOperation(),
-                                                 op.value().getSExtValue());
-    return success();
+LogicalResult ShapeOfOpConversion::matchAndRewrite(
+    ShapeOfOp op, ArrayRef<Value> operands,
+    ConversionPatternRewriter &rewriter) const {
+  ShapeOfOp::Adaptor transformed(operands);
+  auto loc = op.getLoc();
+  auto tensorVal = transformed.arg();
+  auto tensorTy = tensorVal.getType();
+
+  // For unranked tensors `shape_of` lowers to `scf` and the pattern can be
+  // found in the corresponding pass.
+  if (tensorTy.isa<UnrankedTensorType>())
+    return failure();
+
+  // Build values for individual dimensions.
+  SmallVector<Value, 8> dimValues;
+  auto rankedTensorTy = tensorTy.cast<RankedTensorType>();
+  int64_t rank = rankedTensorTy.getRank();
+  for (int64_t i = 0; i < rank; i++) {
+    if (rankedTensorTy.isDynamicDim(i)) {
+      auto dimVal = rewriter.create<DimOp>(loc, tensorVal, i);
+      dimValues.push_back(dimVal);
+    } else {
+      int64_t dim = rankedTensorTy.getDimSize(i);
+      auto dimVal = rewriter.create<ConstantIndexOp>(loc, dim);
+      dimValues.push_back(dimVal);
+    }
   }
-};
 
+  // Materialize extent tensor.
+  Value staticExtentTensor =
+      rewriter.create<TensorFromElementsOp>(loc, dimValues);
+  rewriter.replaceOpWithNewOp<TensorCastOp>(op, staticExtentTensor,
+                                            op.getType());
+  return success();
+}
+
+namespace {
 class GetExtentOpConverter : public OpConversionPattern<GetExtentOp> {
   using OpConversionPattern<GetExtentOp>::OpConversionPattern;
 
   LogicalResult
   matchAndRewrite(GetExtentOp op, ArrayRef<Value> operands,
-                  ConversionPatternRewriter &rewriter) const override {
-    GetExtentOp::Adaptor transformed(operands);
-
-    // Derive shape extent directly from shape origin if possible.
-    // This circumvents the necessity to materialize the shape in memory.
-    if (auto shapeOfOp = op.shape().getDefiningOp<ShapeOfOp>()) {
-      rewriter.replaceOpWithNewOp<DimOp>(op, shapeOfOp.arg(),
-                                         transformed.dim());
-      return success();
-    }
+                  ConversionPatternRewriter &rewriter) const override;
+};
+} // namespace
 
-    rewriter.replaceOpWithNewOp<ExtractElementOp>(
-        op, rewriter.getIndexType(), transformed.shape(),
-        ValueRange{transformed.dim()});
+LogicalResult GetExtentOpConverter::matchAndRewrite(
+    GetExtentOp op, ArrayRef<Value> operands,
+    ConversionPatternRewriter &rewriter) const {
+  GetExtentOp::Adaptor transformed(operands);
+
+  // Derive shape extent directly from shape origin if possible.
+  // This circumvents the necessity to materialize the shape in memory.
+  if (auto shapeOfOp = op.shape().getDefiningOp<ShapeOfOp>()) {
+    rewriter.replaceOpWithNewOp<DimOp>(op, shapeOfOp.arg(), transformed.dim());
     return success();
   }
-};
 
+  rewriter.replaceOpWithNewOp<ExtractElementOp>(op, rewriter.getIndexType(),
+                                                transformed.shape(),
+                                                ValueRange{transformed.dim()});
+  return success();
+}
+
+namespace {
 class RankOpConverter : public OpConversionPattern<shape::RankOp> {
 public:
   using OpConversionPattern<shape::RankOp>::OpConversionPattern;
 
   LogicalResult
   matchAndRewrite(shape::RankOp op, ArrayRef<Value> operands,
-                  ConversionPatternRewriter &rewriter) const override {
-    shape::RankOp::Adaptor transformed(operands);
-    rewriter.replaceOpWithNewOp<DimOp>(op.getOperation(), transformed.shape(),
-                                       0);
-    return success();
-  }
+                  ConversionPatternRewriter &rewriter) const override;
 };
+} // namespace
+
+LogicalResult
+RankOpConverter::matchAndRewrite(shape::RankOp op, ArrayRef<Value> operands,
+                                 ConversionPatternRewriter &rewriter) const {
+  shape::RankOp::Adaptor transformed(operands);
+  rewriter.replaceOpWithNewOp<DimOp>(op, transformed.shape(), 0);
+  return success();
+}
 
+namespace {
 /// Type conversions.
 class ShapeTypeConverter : public TypeConverter {
 public:
@@ -159,48 +167,49 @@ class ShapeTypeConverter : public TypeConverter {
     });
   }
 };
+} // namespace
 
+namespace {
 /// Conversion pass.
 class ConvertShapeToStandardPass
     : public ConvertShapeToStandardBase<ConvertShapeToStandardPass> {
 
-  void runOnOperation() override {
-    // Setup type conversion.
-    MLIRContext &ctx = getContext();
-    ShapeTypeConverter typeConverter(&ctx);
-
-    // Setup target legality.
-    ConversionTarget target(ctx);
-    target.addLegalDialect<scf::SCFDialect, StandardOpsDialect>();
-    target.addLegalOp<ModuleOp, ModuleTerminatorOp, ReturnOp>();
-    target.addDynamicallyLegalOp<FuncOp>([&](FuncOp op) {
-      return typeConverter.isSignatureLegal(op.getType()) &&
-             typeConverter.isLegal(&op.getBody());
-    });
-
-    // Setup conversion patterns.
-    OwningRewritePatternList patterns;
-    populateShapeToStandardConversionPatterns(patterns, &ctx);
-    populateFuncOpTypeConversionPattern(patterns, &ctx, typeConverter);
-
-    // Apply conversion.
-    auto module = getOperation();
-    if (failed(applyFullConversion(module, target, patterns)))
-      signalPassFailure();
-  }
+  void runOnOperation() override;
 };
-
 } // namespace
 
+void ConvertShapeToStandardPass::runOnOperation() {
+  // Setup type conversion.
+  MLIRContext &ctx = getContext();
+  ShapeTypeConverter typeConverter(&ctx);
+
+  // Setup target legality.
+  ConversionTarget target(ctx);
+  target.addLegalDialect<scf::SCFDialect, StandardOpsDialect>();
+  target.addLegalOp<ModuleOp, ModuleTerminatorOp, ReturnOp>();
+  target.addDynamicallyLegalOp<FuncOp>([&](FuncOp op) {
+    return typeConverter.isSignatureLegal(op.getType()) &&
+           typeConverter.isLegal(&op.getBody());
+  });
+
+  // Setup conversion patterns.
+  OwningRewritePatternList patterns;
+  populateShapeToStandardConversionPatterns(patterns, &ctx);
+  populateFuncOpTypeConversionPattern(patterns, &ctx, typeConverter);
+
+  // Apply conversion.
+  auto module = getOperation();
+  if (failed(applyFullConversion(module, target, patterns)))
+    signalPassFailure();
+}
+
 void mlir::populateShapeToStandardConversionPatterns(
     OwningRewritePatternList &patterns, MLIRContext *ctx) {
-  populateWithGenerated(ctx, &patterns);
   // clang-format off
   patterns.insert<
       AnyOpConversion,
       BinaryOpConversion<AddOp, AddIOp>,
       BinaryOpConversion<MulOp, MulIOp>,
-      ConstSizeOpConverter,
       GetExtentOpConverter,
       RankOpConverter,
       ShapeOfOpConversion>(ctx);
diff --git a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandardPatterns.td b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandardPatterns.td
deleted file mode 100644
index 30bed6d7fb65e..0000000000000
--- a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandardPatterns.td
+++ /dev/null
@@ -1,20 +0,0 @@
-include "mlir/Dialect/Shape/IR/ShapeOps.td"
-include "mlir/Dialect/StandardOps/IR/Ops.td"
-
-// Convert `from_extent_tensor` and `to_extent_tensor` to no-ops as shapes will
-// be represented as extent tensors.
-def FromExtentTensorOpConversion : Pat<
-    (Shape_FromExtentTensorOp $input),
-    (replaceWithValue $input)>;
-def ToExtentTensorOpConversion : Pat<
-    (Shape_ToExtentTensorOp $input),
-    (replaceWithValue $input)>;
-
-// Convert `index_to_size` and `size_to_index` to no-ops as sizes will be
-// represented as indices.
-def IndexToSizeOpConversion : Pat<
-    (Shape_IndexToSizeOp $arg),
-    (replaceWithValue $arg)>;
-def SizeToIndexOpConversion : Pat<
-    (Shape_SizeToIndexOp $arg),
-    (replaceWithValue $arg)>;
diff --git a/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp b/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
index 91a4867ad3075..1e6fa6a8754ba 100644
--- a/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
+++ b/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
@@ -150,19 +150,42 @@ LLVMTypeConverter::LLVMTypeConverter(MLIRContext *ctx,
   // Materialization for memrefs creates descriptor structs from individual
   // values constituting them, when descriptors are used, i.e. more than one
   // value represents a memref.
-  addMaterialization([&](PatternRewriter &rewriter,
-                         UnrankedMemRefType resultType, ValueRange inputs,
-                         Location loc) -> Optional<Value> {
+  addArgumentMaterialization(
+      [&](OpBuilder &builder, UnrankedMemRefType resultType, ValueRange inputs,
+          Location loc) -> Optional<Value> {
+        if (inputs.size() == 1)
+          return llvm::None;
+        return UnrankedMemRefDescriptor::pack(builder, loc, *this, resultType,
+                                              inputs);
+      });
+  addArgumentMaterialization([&](OpBuilder &builder, MemRefType resultType,
+                                 ValueRange inputs,
+                                 Location loc) -> Optional<Value> {
     if (inputs.size() == 1)
       return llvm::None;
-    return UnrankedMemRefDescriptor::pack(rewriter, loc, *this, resultType,
-                                          inputs);
+    return MemRefDescriptor::pack(builder, loc, *this, resultType, inputs);
   });
-  addMaterialization([&](PatternRewriter &rewriter, MemRefType resultType,
-                         ValueRange inputs, Location loc) -> Optional<Value> {
-    if (inputs.size() == 1)
+  // Add generic source and target materializations to handle cases where
+  // non-LLVM types persist after an LLVM conversion.
+  addSourceMaterialization([&](OpBuilder &builder, Type resultType,
+                               ValueRange inputs,
+                               Location loc) -> Optional<Value> {
+    if (inputs.size() != 1)
+      return llvm::None;
+    // FIXME: These should check LLVM::DialectCastOp can actually be constructed
+    // from the input and result.
+    return builder.create<LLVM::DialectCastOp>(loc, resultType, inputs[0])
+        .getResult();
+  });
+  addTargetMaterialization([&](OpBuilder &builder, Type resultType,
+                               ValueRange inputs,
+                               Location loc) -> Optional<Value> {
+    if (inputs.size() != 1)
       return llvm::None;
-    return MemRefDescriptor::pack(rewriter, loc, *this, resultType, inputs);
+    // FIXME: These should check LLVM::DialectCastOp can actually be constructed
+    // from the input and result.
+    return builder.create<LLVM::DialectCastOp>(loc, resultType, inputs[0])
+        .getResult();
   });
 }
 
@@ -1399,6 +1422,50 @@ using UnsignedShiftRightOpLowering =
     OneToOneConvertToLLVMPattern<UnsignedShiftRightOp, LLVM::LShrOp>;
 using XOrOpLowering = VectorConvertToLLVMPattern<XOrOp, LLVM::XOrOp>;
 
+/// Lower `std.assert`. The default lowering calls the `abort` function if the
+/// assertion is violated and has no effect otherwise. The failure message is
+/// ignored by the default lowering but should be propagated by any custom
+/// lowering.
+struct AssertOpLowering : public ConvertOpToLLVMPattern<AssertOp> {
+  using ConvertOpToLLVMPattern<AssertOp>::ConvertOpToLLVMPattern;
+
+  LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto loc = op->getLoc();
+    AssertOp::Adaptor transformed(operands);
+
+    // Insert the `abort` declaration if necessary.
+    auto module = op->getParentOfType<ModuleOp>();
+    auto abortFunc = module.lookupSymbol<LLVM::LLVMFuncOp>("abort");
+    if (!abortFunc) {
+      OpBuilder::InsertionGuard guard(rewriter);
+      rewriter.setInsertionPointToStart(module.getBody());
+      auto abortFuncTy =
+          LLVM::LLVMType::getFunctionTy(getVoidType(), {}, /*isVarArg=*/false);
+      abortFunc = rewriter.create<LLVM::LLVMFuncOp>(rewriter.getUnknownLoc(),
+                                                    "abort", abortFuncTy);
+    }
+
+    // Split block at `assert` operation.
+    Block *opBlock = rewriter.getInsertionBlock();
+    auto opPosition = rewriter.getInsertionPoint();
+    Block *continuationBlock = rewriter.splitBlock(opBlock, opPosition);
+
+    // Generate IR to call `abort`.
+    Block *failureBlock = rewriter.createBlock(opBlock->getParent());
+    rewriter.create<LLVM::CallOp>(loc, abortFunc, llvm::None);
+    rewriter.create<LLVM::UnreachableOp>(loc);
+
+    // Generate assertion test.
+    rewriter.setInsertionPointToEnd(opBlock);
+    rewriter.replaceOpWithNewOp<LLVM::CondBrOp>(
+        op, transformed.arg(), continuationBlock, failureBlock);
+
+    return success();
+  }
+};
+
 // Lowerings for operations on complex numbers.
 
 struct CreateComplexOpLowering
@@ -3146,6 +3213,7 @@ void mlir::populateStdToLLVMNonMemoryConversionPatterns(
       AddIOpLowering,
       AllocaOpLowering,
       AndOpLowering,
+      AssertOpLowering,
       AtomicRMWOpLowering,
       BranchOpLowering,
       CallIndirectOpLowering,
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index a59f02681c54e..5dbc8394b03a5 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -34,14 +34,6 @@
 using namespace mlir;
 using namespace mlir::vector;
 
-template <typename T>
-static LLVM::LLVMType getPtrToElementType(T containerType,
-                                          LLVMTypeConverter &typeConverter) {
-  return typeConverter.convertType(containerType.getElementType())
-      .template cast<LLVM::LLVMType>()
-      .getPointerTo();
-}
-
 // Helper to reduce vector type by one rank at front.
 static VectorType reducedVectorTypeFront(VectorType tp) {
   assert((tp.getRank() > 1) && "unlowerable vector type");
@@ -124,11 +116,12 @@ static SmallVector<int64_t, 4> getI64SubArray(ArrayAttr arrayAttr,
   return res;
 }
 
-template <typename TransferOp>
-LogicalResult getVectorTransferAlignment(LLVMTypeConverter &typeConverter,
-                                         TransferOp xferOp, unsigned &align) {
+// Helper that returns data layout alignment of an operation with memref.
+template <typename T>
+LogicalResult getMemRefAlignment(LLVMTypeConverter &typeConverter, T op,
+                                 unsigned &align) {
   Type elementTy =
-      typeConverter.convertType(xferOp.getMemRefType().getElementType());
+      typeConverter.convertType(op.getMemRefType().getElementType());
   if (!elementTy)
     return failure();
 
@@ -138,13 +131,39 @@ LogicalResult getVectorTransferAlignment(LLVMTypeConverter &typeConverter,
   return success();
 }
 
+// Helper that returns vector of pointers given a base and an index vector.
+LogicalResult getIndexedPtrs(ConversionPatternRewriter &rewriter,
+                             LLVMTypeConverter &typeConverter, Location loc,
+                             Value memref, Value indices, MemRefType memRefType,
+                             VectorType vType, Type iType, Value &ptrs) {
+  // Inspect stride and offset structure.
+  //
+  // TODO: flat memory only for now, generalize
+  //
+  int64_t offset;
+  SmallVector<int64_t, 4> strides;
+  auto successStrides = getStridesAndOffset(memRefType, strides, offset);
+  if (failed(successStrides) || strides.size() != 1 || strides[0] != 1 ||
+      offset != 0 || memRefType.getMemorySpace() != 0)
+    return failure();
+
+  // Create a vector of pointers from base and indices.
+  MemRefDescriptor memRefDescriptor(memref);
+  Value base = memRefDescriptor.alignedPtr(rewriter, loc);
+  int64_t size = vType.getDimSize(0);
+  auto pType = memRefDescriptor.getElementType();
+  auto ptrsType = LLVM::LLVMType::getVectorTy(pType, size);
+  ptrs = rewriter.create<LLVM::GEPOp>(loc, ptrsType, base, indices);
+  return success();
+}
+
 static LogicalResult
 replaceTransferOpWithLoadOrStore(ConversionPatternRewriter &rewriter,
                                  LLVMTypeConverter &typeConverter, Location loc,
                                  TransferReadOp xferOp,
                                  ArrayRef<Value> operands, Value dataPtr) {
   unsigned align;
-  if (failed(getVectorTransferAlignment(typeConverter, xferOp, align)))
+  if (failed(getMemRefAlignment(typeConverter, xferOp, align)))
     return failure();
   rewriter.replaceOpWithNewOp<LLVM::LoadOp>(xferOp, dataPtr, align);
   return success();
@@ -165,7 +184,7 @@ replaceTransferOpWithMasked(ConversionPatternRewriter &rewriter,
     return failure();
 
   unsigned align;
-  if (failed(getVectorTransferAlignment(typeConverter, xferOp, align)))
+  if (failed(getMemRefAlignment(typeConverter, xferOp, align)))
     return failure();
 
   rewriter.replaceOpWithNewOp<LLVM::MaskedLoadOp>(
@@ -180,7 +199,7 @@ replaceTransferOpWithLoadOrStore(ConversionPatternRewriter &rewriter,
                                  TransferWriteOp xferOp,
                                  ArrayRef<Value> operands, Value dataPtr) {
   unsigned align;
-  if (failed(getVectorTransferAlignment(typeConverter, xferOp, align)))
+  if (failed(getMemRefAlignment(typeConverter, xferOp, align)))
     return failure();
   auto adaptor = TransferWriteOpAdaptor(operands);
   rewriter.replaceOpWithNewOp<LLVM::StoreOp>(xferOp, adaptor.vector(), dataPtr,
@@ -194,7 +213,7 @@ replaceTransferOpWithMasked(ConversionPatternRewriter &rewriter,
                             TransferWriteOp xferOp, ArrayRef<Value> operands,
                             Value dataPtr, Value mask) {
   unsigned align;
-  if (failed(getVectorTransferAlignment(typeConverter, xferOp, align)))
+  if (failed(getMemRefAlignment(typeConverter, xferOp, align)))
     return failure();
 
   auto adaptor = TransferWriteOpAdaptor(operands);
@@ -259,6 +278,83 @@ class VectorFlatTransposeOpConversion : public ConvertToLLVMPattern {
   }
 };
 
+/// Conversion pattern for a vector.gather.
+class VectorGatherOpConversion : public ConvertToLLVMPattern {
+public:
+  explicit VectorGatherOpConversion(MLIRContext *context,
+                                    LLVMTypeConverter &typeConverter)
+      : ConvertToLLVMPattern(vector::GatherOp::getOperationName(), context,
+                             typeConverter) {}
+
+  LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto loc = op->getLoc();
+    auto gather = cast<vector::GatherOp>(op);
+    auto adaptor = vector::GatherOpAdaptor(operands);
+
+    // Resolve alignment.
+    unsigned align;
+    if (failed(getMemRefAlignment(typeConverter, gather, align)))
+      return failure();
+
+    // Get index ptrs.
+    VectorType vType = gather.getResultVectorType();
+    Type iType = gather.getIndicesVectorType().getElementType();
+    Value ptrs;
+    if (failed(getIndexedPtrs(rewriter, typeConverter, loc, adaptor.base(),
+                              adaptor.indices(), gather.getMemRefType(), vType,
+                              iType, ptrs)))
+      return failure();
+
+    // Replace with the gather intrinsic.
+    ValueRange v = (llvm::size(adaptor.pass_thru()) == 0) ? ValueRange({})
+                                                          : adaptor.pass_thru();
+    rewriter.replaceOpWithNewOp<LLVM::masked_gather>(
+        gather, typeConverter.convertType(vType), ptrs, adaptor.mask(), v,
+        rewriter.getI32IntegerAttr(align));
+    return success();
+  }
+};
+
+/// Conversion pattern for a vector.scatter.
+class VectorScatterOpConversion : public ConvertToLLVMPattern {
+public:
+  explicit VectorScatterOpConversion(MLIRContext *context,
+                                     LLVMTypeConverter &typeConverter)
+      : ConvertToLLVMPattern(vector::ScatterOp::getOperationName(), context,
+                             typeConverter) {}
+
+  LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto loc = op->getLoc();
+    auto scatter = cast<vector::ScatterOp>(op);
+    auto adaptor = vector::ScatterOpAdaptor(operands);
+
+    // Resolve alignment.
+    unsigned align;
+    if (failed(getMemRefAlignment(typeConverter, scatter, align)))
+      return failure();
+
+    // Get index ptrs.
+    VectorType vType = scatter.getValueVectorType();
+    Type iType = scatter.getIndicesVectorType().getElementType();
+    Value ptrs;
+    if (failed(getIndexedPtrs(rewriter, typeConverter, loc, adaptor.base(),
+                              adaptor.indices(), scatter.getMemRefType(), vType,
+                              iType, ptrs)))
+      return failure();
+
+    // Replace with the scatter intrinsic.
+    rewriter.replaceOpWithNewOp<LLVM::masked_scatter>(
+        scatter, adaptor.value(), ptrs, adaptor.mask(),
+        rewriter.getI32IntegerAttr(align));
+    return success();
+  }
+};
+
+/// Conversion pattern for all vector reductions.
 class VectorReductionOpConversion : public ConvertToLLVMPattern {
 public:
   explicit VectorReductionOpConversion(MLIRContext *context,
@@ -1173,7 +1269,9 @@ void mlir::populateVectorToLLVMConversionPatterns(
               VectorPrintOpConversion,
               VectorTransferConversion<TransferReadOp>,
               VectorTransferConversion<TransferWriteOp>,
-              VectorTypeCastOpConversion>(ctx, converter);
+              VectorTypeCastOpConversion,
+              VectorGatherOpConversion,
+              VectorScatterOpConversion>(ctx, converter);
   // clang-format on
 }
 
diff --git a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
index d0529668b2ee4..ea368c9eb14e0 100644
--- a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
+++ b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
@@ -249,8 +249,8 @@ LogicalResult NDTransferOpHelper<TransferReadOp>::doReplace() {
       indexing.append(majorIvsPlusOffsets.begin(), majorIvsPlusOffsets.end());
       indexing.append(minorOffsets.begin(), minorOffsets.end());
       Value memref = xferOp.memref();
-      auto map = TransferReadOp::getTransferMinorIdentityMap(
-          xferOp.getMemRefType(), minorVectorType);
+      auto map =
+          getTransferMinorIdentityMap(xferOp.getMemRefType(), minorVectorType);
       ArrayAttr masked;
       if (!xferOp.isMaskedDim(xferOp.getVectorType().getRank() - 1)) {
         OpBuilder &b = ScopedContext::getBuilderRef();
@@ -353,8 +353,8 @@ LogicalResult NDTransferOpHelper<TransferWriteOp>::doReplace() {
         result = vector_extract(xferOp.vector(), majorIvs);
       else
         result = std_load(alloc, majorIvs);
-      auto map = TransferWriteOp::getTransferMinorIdentityMap(
-          xferOp.getMemRefType(), minorVectorType);
+      auto map =
+          getTransferMinorIdentityMap(xferOp.getMemRefType(), minorVectorType);
       ArrayAttr masked;
       if (!xferOp.isMaskedDim(xferOp.getVectorType().getRank() - 1)) {
         OpBuilder &b = ScopedContext::getBuilderRef();
diff --git a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
index 1638502508e3d..748530f033585 100644
--- a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
@@ -18,6 +18,7 @@
 #include "mlir/Analysis/Utils.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Affine/Passes.h"
+#include "mlir/Dialect/Affine/Utils.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/Dialect/Vector/VectorOps.h"
 #include "mlir/Dialect/Vector/VectorUtils.h"
@@ -1198,25 +1199,38 @@ void Vectorize::runOnFunction() {
     return signalPassFailure();
   }
 
-  // Thread-safe RAII local context, BumpPtrAllocator freed on exit.
-  NestedPatternContext mlContext;
-
   DenseSet<Operation *> parallelLoops;
   f.walk([&parallelLoops](AffineForOp loop) {
     if (isLoopParallel(loop))
       parallelLoops.insert(loop);
   });
 
+  vectorizeAffineLoops(f, parallelLoops, vectorSizes, fastestVaryingPattern);
+}
+
+namespace mlir {
+
+/// Vectorizes affine loops in 'loops' using the n-D vectorization factors in
+/// 'vectorSizes'. By default, each vectorization factor is applied
+/// inner-to-outer to the loops of each loop nest. 'fastestVaryingPattern' can
+/// be optionally used to provide a different loop vectorization order.
+void vectorizeAffineLoops(Operation *parentOp, DenseSet<Operation *> &loops,
+                          ArrayRef<int64_t> vectorSizes,
+                          ArrayRef<int64_t> fastestVaryingPattern) {
+  // Thread-safe RAII local context, BumpPtrAllocator freed on exit.
+  NestedPatternContext mlContext;
+
   for (auto &pat :
-       makePatterns(parallelLoops, vectorSizes.size(), fastestVaryingPattern)) {
+       makePatterns(loops, vectorSizes.size(), fastestVaryingPattern)) {
     LLVM_DEBUG(dbgs() << "\n******************************************");
     LLVM_DEBUG(dbgs() << "\n******************************************");
-    LLVM_DEBUG(dbgs() << "\n[early-vect] new pattern on Function\n");
-    LLVM_DEBUG(f.print(dbgs()));
+    LLVM_DEBUG(dbgs() << "\n[early-vect] new pattern on parent op\n");
+    LLVM_DEBUG(parentOp->print(dbgs()));
+
     unsigned patternDepth = pat.getDepth();
 
     SmallVector<NestedMatch, 8> matches;
-    pat.match(f, &matches);
+    pat.match(parentOp, &matches);
     // Iterate over all the top-level matches and vectorize eagerly.
     // This automatically prunes intersecting matches.
     for (auto m : matches) {
@@ -1239,9 +1253,11 @@ void Vectorize::runOnFunction() {
 }
 
 std::unique_ptr<OperationPass<FuncOp>>
-mlir::createSuperVectorizePass(ArrayRef<int64_t> virtualVectorSize) {
+createSuperVectorizePass(ArrayRef<int64_t> virtualVectorSize) {
   return std::make_unique<Vectorize>(virtualVectorSize);
 }
-std::unique_ptr<OperationPass<FuncOp>> mlir::createSuperVectorizePass() {
+std::unique_ptr<OperationPass<FuncOp>> createSuperVectorizePass() {
   return std::make_unique<Vectorize>();
 }
+
+} // namespace mlir
diff --git a/mlir/lib/Dialect/Linalg/EDSC/Builders.cpp b/mlir/lib/Dialect/Linalg/EDSC/Builders.cpp
index b9ec01d3ec791..44eb5e723075b 100644
--- a/mlir/lib/Dialect/Linalg/EDSC/Builders.cpp
+++ b/mlir/lib/Dialect/Linalg/EDSC/Builders.cpp
@@ -69,7 +69,8 @@ Operation *mlir::edsc::makeGenericLinalgOp(
               builder.getAffineMapArrayAttr(maps),
               builder.getStrArrayAttr(iteratorStrTypes),
               StringAttr() /*doc*/,
-              StringAttr() /*library_call*/
+              StringAttr() /*library_call*/,
+              IntegerAttr() /*symbol_source*/
               /* TODO: other attributes in op */
               )
           .getOperation();
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index 528e856fe5bb2..4c68f0265677c 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -79,7 +79,8 @@ void GenericOp::build(
         builder.getI64IntegerAttr(argsOut),
         builder.getAffineMapArrayAttr(indexingMaps),
         builder.getStrArrayAttr(iteratorTypes),
-        /*doc=*/nullptr, /*library_call=*/nullptr);
+        /*doc=*/nullptr, /*library_call=*/nullptr,
+        /*symbol_source=*/nullptr);
   if (!bodyBuild)
     return;
 
@@ -103,7 +104,8 @@ void IndexedGenericOp::build(
         builder.getI64IntegerAttr(argsOut),
         builder.getAffineMapArrayAttr(indexingMaps),
         builder.getStrArrayAttr(iteratorTypes),
-        /*doc=*/nullptr, /*library_call=*/nullptr);
+        /*doc=*/nullptr, /*library_call=*/nullptr,
+        /*symbol_source=*/nullptr);
   if (!bodyBuild)
     return;
 
@@ -257,6 +259,15 @@ static LogicalResult verifyGenericOp(GenericOpType op) {
   if (failed(BlockArgsVerifier<GenericOpType>::verify(op, region.front())))
     return failure();
 
+  auto attr = op.template getAttrOfType<IntegerAttr>("symbol_source");
+  int64_t targetRank = 0;
+  if (attr) {
+    unsigned index = attr.getInt();
+    if (index >= op.getNumOperands())
+      return op.emitOpError("symbol_source index out of range");
+    targetRank = op.getShapedType(index).getRank();
+  }
+
   SmallVector<AffineMap, 4> indexingMaps;
   indexingMaps.reserve(op.indexing_maps().size());
   for (auto en : llvm::enumerate(op.indexing_maps())) {
@@ -266,9 +277,9 @@ static LogicalResult verifyGenericOp(GenericOpType op) {
     auto view = (idx < nInputViews) ? op.getInputShapedType(idx)
                                     : op.getOutputShapedType(idx - nInputViews);
 
-    if (m.getNumSymbols() != 0)
-      return op.emitOpError("expected indexing_map #")
-             << idx << " to have no symbols";
+    if (m.getNumSymbols() != targetRank)
+      return op.emitOpError("expected the number of symbols in indexing_map #")
+             << idx << " to match target rank";
 
     if (m.getNumDims() != nLoops)
       return op.emitOpError("expected indexing_map #")
@@ -281,8 +292,8 @@ static LogicalResult verifyGenericOp(GenericOpType op) {
   }
 
   auto concatMap = concatAffineMaps(indexingMaps);
-  auto aggregateMap = inversePermutation(concatMap);
-  if (!aggregateMap)
+  // TODO: Bound inference for maps with symbols
+  if (!concatMap.getNumSymbols() && !inversePermutation(concatMap))
     return op.emitOpError("expected the concatenation of maps in indexing_map "
                           "to be invertible");
 
@@ -984,6 +995,8 @@ static LogicalResult verify(ConvOp op) {
     return op.emitOpError("expects memref elemental types to match");
   if (oType.getRank() != iType.getRank() || oType.getRank() != fType.getRank())
     return op.emitOpError("expects memref ranks to match");
+  if (oType.getRank() <= 2)
+    return op.emitOpError("expects memref ranks to be greater than 2");
   if (auto strides = op.strides()) {
     if (failed(
             verifyStrideOrDilation(op, strides->getValue(), /*isStride=*/true)))
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
index e08c43d48ba04..65fd197a6661a 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
@@ -319,7 +319,8 @@ struct ReplaceUnitExtentTensors : public OpRewritePattern<GenericOp> {
         genericOp.args_out(), rewriter.getAffineMapArrayAttr(newIndexingMaps),
         genericOp.iterator_types(),
         /*doc = */ nullptr,
-        /*library_call = */ nullptr);
+        /*library_call = */ nullptr,
+        /*symbol_source = */ nullptr);
     rewriter.inlineRegionBefore(genericOp.region(), replacementOp.region(),
                                 replacementOp.region().begin());
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
index d67126c21f3e5..82dfa75fc1f4b 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
@@ -510,7 +510,8 @@ struct FuseGenericOpsOnTensors {
                   rewriter.getArrayAttr(fusedIndexMaps),
                   consumer.iterator_types(),
                   /*doc=*/nullptr,
-                  /*library_call=*/nullptr)
+                  /*library_call=*/nullptr,
+                  /*symbol_source=*/nullptr)
               .getOperation();
     } else {
       fusedOp =
@@ -524,7 +525,8 @@ struct FuseGenericOpsOnTensors {
                   rewriter.getArrayAttr(fusedIndexMaps),
                   consumer.iterator_types(),
                   /*doc=*/nullptr,
-                  /*library_call=*/nullptr)
+                  /*library_call=*/nullptr,
+                  /*symbol_source=*/nullptr)
               .getOperation();
     }
 
@@ -787,7 +789,8 @@ template <typename LinalgOpTy> struct FuseTensorReshapeOpAsProducer {
         rewriter.getI64IntegerAttr(consumer.getNumResults()),
         rewriter.getArrayAttr(indexMapAttrs), consumer.iterator_types(),
         /*doc=*/nullptr,
-        /*library_call=*/nullptr);
+        /*library_call=*/nullptr,
+        /*symbol_source=*/nullptr);
     auto &fusedRegion = fusedOp.region();
     rewriter.cloneRegionBefore(consumer.region(), fusedRegion,
                                fusedRegion.begin());
@@ -843,7 +846,8 @@ template <typename LinalgOpTy> struct FuseTensorReshapeOpAsConsumer {
         rewriter.getI64IntegerAttr(1), rewriter.getArrayAttr(indexMapAttrs),
         producer.iterator_types(),
         /*doc=*/nullptr,
-        /*library_call=*/nullptr);
+        /*library_call=*/nullptr,
+        /*symbol_source=*/nullptr);
     auto &fusedRegion = fusedOp.region();
     rewriter.cloneRegionBefore(producer.region(), fusedRegion,
                                fusedRegion.begin());
@@ -893,7 +897,8 @@ template <typename LinalgOpTy> struct FuseConstantOpAsProducer {
         rewriter.getAffineMapArrayAttr(fusedIndexMaps),
         consumer.iterator_types(),
         /*doc=*/nullptr,
-        /*library_call=*/nullptr);
+        /*library_call=*/nullptr,
+        /*symbol_source=*/nullptr);
 
     // Map the block argument corresponding to the replaced argument with the
     // scalar constant.
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
index cb7540b46cf84..180fe069b681c 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
@@ -82,8 +82,8 @@ void mlir::linalg::hoistViewAllocOps(FuncOp func) {
 
 /// Return true if we can prove that the transfer operations access dijoint
 /// memory.
-template <typename TransferTypeA, typename TransferTypeB>
-static bool isDisjoint(TransferTypeA transferA, TransferTypeB transferB) {
+static bool isDisjoint(VectorTransferOpInterface transferA,
+                       VectorTransferOpInterface transferB) {
   if (transferA.memref() != transferB.memref())
     return false;
   // For simplicity only look at transfer of same type.
@@ -91,8 +91,8 @@ static bool isDisjoint(TransferTypeA transferA, TransferTypeB transferB) {
     return false;
   unsigned rankOffset = transferA.getLeadingMemRefRank();
   for (unsigned i = 0, e = transferA.indices().size(); i < e; i++) {
-    auto indexA = transferA.indices()[i].template getDefiningOp<ConstantOp>();
-    auto indexB = transferB.indices()[i].template getDefiningOp<ConstantOp>();
+    auto indexA = transferA.indices()[i].getDefiningOp<ConstantOp>();
+    auto indexB = transferB.indices()[i].getDefiningOp<ConstantOp>();
     // If any of the indices are dynamic we cannot prove anything.
     if (!indexA || !indexB)
       continue;
@@ -100,15 +100,15 @@ static bool isDisjoint(TransferTypeA transferA, TransferTypeB transferB) {
     if (i < rankOffset) {
       // For dimension used as index if we can prove that index are different we
       // know we are accessing disjoint slices.
-      if (indexA.getValue().template cast<IntegerAttr>().getInt() !=
-          indexB.getValue().template cast<IntegerAttr>().getInt())
+      if (indexA.getValue().cast<IntegerAttr>().getInt() !=
+          indexB.getValue().cast<IntegerAttr>().getInt())
         return true;
     } else {
       // For this dimension, we slice a part of the memref we need to make sure
       // the intervals accessed don't overlap.
       int64_t distance =
-          std::abs(indexA.getValue().template cast<IntegerAttr>().getInt() -
-                   indexB.getValue().template cast<IntegerAttr>().getInt());
+          std::abs(indexA.getValue().cast<IntegerAttr>().getInt() -
+                   indexB.getValue().cast<IntegerAttr>().getInt());
       if (distance >= transferA.getVectorType().getDimSize(i - rankOffset))
         return true;
     }
@@ -185,11 +185,17 @@ void mlir::linalg::hoistRedundantVectorTransfers(FuncOp func) {
           continue;
         if (auto transferWriteUse =
                 dyn_cast<vector::TransferWriteOp>(use.getOwner())) {
-          if (!isDisjoint(transferWrite, transferWriteUse))
+          if (!isDisjoint(
+                  cast<VectorTransferOpInterface>(transferWrite.getOperation()),
+                  cast<VectorTransferOpInterface>(
+                      transferWriteUse.getOperation())))
             return WalkResult::advance();
         } else if (auto transferReadUse =
                        dyn_cast<vector::TransferReadOp>(use.getOwner())) {
-          if (!isDisjoint(transferWrite, transferReadUse))
+          if (!isDisjoint(
+                  cast<VectorTransferOpInterface>(transferWrite.getOperation()),
+                  cast<VectorTransferOpInterface>(
+                      transferReadUse.getOperation())))
             return WalkResult::advance();
         } else {
           // Unknown use, we cannot prove that it doesn't alias with the
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
index 6a1d00fe620c6..7fb1018fc5881 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
@@ -36,13 +36,13 @@ static SmallVector<Value, 8> makeCanonicalAffineApplies(OpBuilder &b,
                                                         ArrayRef<Value> vals) {
   if (map.isEmpty())
     return {};
-  assert(map.getNumSymbols() == 0);
+
   assert(map.getNumInputs() == vals.size());
   SmallVector<Value, 8> res;
   res.reserve(map.getNumResults());
   auto dims = map.getNumDims();
   for (auto e : map.getResults()) {
-    auto exprMap = AffineMap::get(dims, 0, e);
+    auto exprMap = AffineMap::get(dims, map.getNumSymbols(), e);
     SmallVector<Value, 4> operands(vals.begin(), vals.end());
     canonicalizeMapAndOperands(&exprMap, &operands);
     res.push_back(affine_apply(exprMap, operands));
@@ -58,20 +58,74 @@ static SmallVector<Value, 4> permuteIvs(ArrayRef<Value> ivs,
                      : SmallVector<Value, 4>(ivs.begin(), ivs.end());
 }
 
-// Creates a number of ranges equal to the number of results in `map`.
-// The returned ranges correspond to the loop ranges, in the proper order, for
-// which new loops will be created.
-static SmallVector<SubViewOp::Range, 4>
-emitLoopRanges(OpBuilder &b, Location loc, AffineMap map,
-               ArrayRef<Value> allViewSizes) {
-  // Apply `map` to get view sizes in loop order.
-  auto sizes = applyMapToValues(b, loc, map, allViewSizes);
-  // Create a new range with the applied tile sizes.
-  ScopedContext scope(b, loc);
-  SmallVector<SubViewOp::Range, 4> res;
-  for (unsigned idx = 0, e = map.getNumResults(); idx < e; ++idx) {
-    res.push_back(SubViewOp::Range{std_constant_index(0), sizes[idx],
-                                   std_constant_index(1)});
+/// Creates a number of ranges equal to the number of dimensions in the `map`.
+/// The returned ranges correspond to the loop ranges, in the proper order, for
+/// which new loops will be created.
+/// The function supports only maps that are invertible and have results of type
+/// DimExpr or (DimExpr + DimExpr - SymbolExpr floordiv ConstExpr).
+/// It expects a non-inverted, concatenated map and last values in
+/// allViewSizes will be applied to the symbols in the map if it contains any.
+static SmallVector<SubViewOp::Range, 4> emitLoopRanges(OpBuilder &b,
+                                                       Location loc,
+                                                       AffineMap map,
+                                                       ValueRange viewSizes) {
+  unsigned numDims = map.getNumDims(), numRes = map.getNumResults();
+  unsigned numSym = map.getNumSymbols();
+  assert(viewSizes.size() == numRes + numSym &&
+         "viewSizes must contain sizes of all views and values for symbols");
+  SmallVector<SubViewOp::Range, 4> res(numDims);
+  for (unsigned idx = 0; idx < numRes; ++idx) {
+    auto result = map.getResult(idx);
+    if (auto d = result.dyn_cast<AffineDimExpr>()) {
+      if (res[d.getPosition()].offset)
+        continue;
+      res[d.getPosition()] = SubViewOp::Range{
+          std_constant_index(0), viewSizes[idx], std_constant_index(1)};
+    }
+
+    // If the access pattern is of form (m, n)[s] -> (m + n - s floordiv 2),
+    // then the bounds are:
+    //   (s floordiv 2) <= m <= (size(m) + s floordiv 2 - s + 1).
+    // where size(n) is applied to the symbol s.
+    // This is done statically now.
+    if (auto binOp = result.dyn_cast<AffineBinaryOpExpr>()) {
+      auto lhs = binOp.getLHS().dyn_cast<AffineBinaryOpExpr>();
+      auto rhs = binOp.getRHS().dyn_cast<AffineBinaryOpExpr>();
+      if (!lhs || !rhs || binOp.getKind() != AffineExprKind::Add ||
+          lhs.getKind() != AffineExprKind::Add ||
+          rhs.getKind() != mlir::AffineExprKind::Mul)
+        continue;
+
+      auto m = lhs.getLHS().dyn_cast<AffineDimExpr>();
+      auto n = lhs.getRHS().dyn_cast<AffineDimExpr>();
+      auto fDiv = rhs.getLHS().dyn_cast<AffineBinaryOpExpr>();
+      auto minusOne = rhs.getRHS().dyn_cast<AffineConstantExpr>();
+      if (!m || !n || !fDiv || !minusOne ||
+          fDiv.getKind() != AffineExprKind::FloorDiv ||
+          fDiv.getLHS().getKind() != AffineExprKind::SymbolId ||
+          fDiv.getRHS().getKind() != AffineExprKind::Constant)
+        continue;
+
+      auto s = fDiv.getLHS().dyn_cast<AffineSymbolExpr>();
+      if (minusOne.getValue() != -1)
+        continue;
+
+      int mPos = m.getPosition();
+      AffineExpr one = getAffineConstantExpr(1, s.getContext());
+      AffineExpr sizeOfM = getAffineSymbolExpr(numSym, s.getContext());
+      // Construction of upper bound (size(m) + s floordiv 2 - s + 1).
+      AffineExpr upperOffsetExpr = sizeOfM + fDiv + one - s;
+      AffineMap fromMap = AffineMap::get(numDims, numSym + 1, fDiv);
+      AffineMap toMap = AffineMap::get(numDims, numSym + 1, upperOffsetExpr);
+      SmallVector<Value, 8> values(viewSizes.begin(),
+                                   viewSizes.begin() + numDims);
+      values.insert(values.end(), viewSizes.begin() + numRes, viewSizes.end());
+      values.push_back(viewSizes[mPos]);
+      // Construction of the lower bound (s floordiv 2).
+      Value from = applyMapToValues(b, loc, fromMap, values).front();
+      Value to = applyMapToValues(b, loc, toMap, values).front();
+      res[mPos] = SubViewOp::Range{from, to, std_constant_index(1)};
+    }
   }
   return res;
 }
@@ -165,19 +219,29 @@ void emitScalarImplementation(ArrayRef<Value> allIvs,
   SmallVector<Value, 4> indexedValues;
   indexedValues.reserve(nInputs + nOutputs);
 
+  auto attr = linalgOp.template getAttrOfType<IntegerAttr>("symbol_source");
+  auto allIvsPlusDims = SmallVector<Value, 4>(allIvs.begin(), allIvs.end());
+  if (attr) {
+    auto operand = linalgOp.getOperand(attr.getInt());
+    auto shapedType = operand.getType().template cast<ShapedType>();
+    allIvsPlusDims.reserve(allIvs.size() + shapedType.getRank());
+    for (unsigned idx = 0, e = shapedType.getRank(); idx < e; ++idx)
+      allIvsPlusDims.push_back(b.create<DimOp>(loc, operand, idx));
+  }
+
   // TODO: Avoid the loads if the corresponding argument of the
   // region has no uses.
   // 1.a. Emit load from input views.
   for (unsigned i = 0; i < nInputs; ++i) {
     auto indexing = makeCanonicalAffineApplies(
-        b, loc, linalgOp.getInputIndexingMap(i), allIvs);
+        b, loc, linalgOp.getInputIndexingMap(i), allIvsPlusDims);
     // Passing through IndexedValueType emits the proper load operation.
     indexedValues.push_back(IndexedValueType(linalgOp.getInput(i))(indexing));
   }
   // 1.b. Emit load from output views.
   for (unsigned i = 0; i < nOutputs; ++i) {
     auto indexing = makeCanonicalAffineApplies(
-        b, loc, linalgOp.getOutputIndexingMap(i), allIvs);
+        b, loc, linalgOp.getOutputIndexingMap(i), allIvsPlusDims);
     // Passing through IndexedValueType emits the proper load operation.
     indexedValues.push_back(
         IndexedValueType(linalgOp.getOutputBuffer(i))(indexing));
@@ -190,7 +254,7 @@ void emitScalarImplementation(ArrayRef<Value> allIvs,
   SmallVector<Value, 8> outputBuffers;
   for (unsigned i = 0; i < nOutputs; ++i) {
     indexing.push_back(makeCanonicalAffineApplies(
-        b, loc, linalgOp.getOutputIndexingMap(i), allIvs));
+        b, loc, linalgOp.getOutputIndexingMap(i), allIvsPlusDims));
     outputBuffers.push_back(linalgOp.getOutputBuffer(i));
   }
   inlineRegionAndEmitStore<IndexedValueType>(linalgOp, indexedValues, indexing,
@@ -457,18 +521,11 @@ Optional<LinalgLoops> linalgOpToLoopsImpl(Operation *op, OpBuilder &builder) {
       linalgOp.indexing_maps().template getAsRange<AffineMapAttr>();
   auto maps = llvm::to_vector<8>(
       llvm::map_range(mapsRange, [](AffineMapAttr a) { return a.getValue(); }));
-  AffineMap invertedMap = inversePermutation(concatAffineMaps(maps));
-  if (!invertedMap)
-    return {};
-  if (invertedMap.isEmpty()) {
-    emitScalarImplementation<IndexedValueTy>({}, linalgOp);
-    return LinalgLoops();
-  }
-
+  SmallVector<Value, 8> sizes = getViewSizes(builder, linalgOp);
+  AffineMap map = concatAffineMaps(maps);
+  auto loopRanges = emitLoopRanges(scope.getBuilderRef(), scope.getLocation(),
+                                   map, getViewSizes(builder, linalgOp));
   SmallVector<Value, 4> allIvs;
-  auto loopRanges =
-      emitLoopRanges(scope.getBuilderRef(), scope.getLocation(), invertedMap,
-                     getViewSizes(builder, linalgOp));
   GenerateLoopNest<LoopTy>::doit(
       loopRanges, linalgOp.iterator_types().getValue(), [&](ValueRange ivs) {
         allIvs.append(ivs.begin(), ivs.end());
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TensorsToBuffers.cpp b/mlir/lib/Dialect/Linalg/Transforms/TensorsToBuffers.cpp
index afd94cc06c6ef..04c1fbd5d5655 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TensorsToBuffers.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TensorsToBuffers.cpp
@@ -65,7 +65,8 @@ class GenericOpConverter
     auto linalgOp = rewriter.create<linalg::GenericOp>(
         loc, llvm::None, newArgs, rewriter.getI64IntegerAttr(operands.size()),
         rewriter.getI64IntegerAttr(results.size()), op.indexing_maps(),
-        op.iterator_types(), op.docAttr(), op.library_callAttr());
+        op.iterator_types(), op.docAttr(), op.library_callAttr(),
+        op.symbol_sourceAttr());
 
     // Create a new block in the region of the new Generic Op.
     Block &oldBlock = op.getRegion().front();
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
index 6ffc181fed67a..2c327276610ae 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
@@ -223,10 +223,10 @@ static bool isTiled(AffineMap map, ArrayRef<Value> tileSizes) {
 }
 
 static SmallVector<Value, 4> makeTiledViews(OpBuilder &b, Location loc,
-                                            LinalgOp linalgOp,
+                                            LinalgOp linalgOp, AffineMap map,
                                             ArrayRef<Value> ivs,
                                             ArrayRef<Value> tileSizes,
-                                            ArrayRef<Value> viewSizes) {
+                                            ArrayRef<Value> allViewSizes) {
   assert(linalgOp.hasBufferSemantics() &&
          "expected linalg op with buffer semantics");
   assert(ivs.size() == static_cast<size_t>(llvm::count_if(
@@ -236,6 +236,7 @@ static SmallVector<Value, 4> makeTiledViews(OpBuilder &b, Location loc,
 
   using namespace edsc::op;
 
+  auto viewSizes = applyMapToValues(b, loc, map, allViewSizes);
   // Construct (potentially temporary) mins and maxes on which to apply maps
   // that define tile subviews.
   SmallVector<Value, 8> lbs, subViewSizes;
@@ -356,7 +357,7 @@ Optional<TiledLinalgOp> static tileLinalgOpImpl(
     return llvm::None;
 
   // 2. Build the tiled loop ranges.
-  auto viewSizes = getViewSizes(b, op);
+  auto allViewSizes = getViewSizes(b, op);
   // The flattened loopToOperandRangesMaps is expected to be an invertible
   // permutation map (asserted in the inverse calculation).
   auto mapsRange = op.indexing_maps().getAsRange<AffineMapAttr>();
@@ -369,7 +370,7 @@ Optional<TiledLinalgOp> static tileLinalgOpImpl(
   SmallVector<SubViewOp::Range, 4> loopRanges;
   LoopIndexToRangeIndexMap loopIndexToRangeIndex;
   std::tie(loopRanges, loopIndexToRangeIndex) = makeTiledLoopRanges(
-      b, scope.getLocation(), viewSizesToLoopsMap, viewSizes, tileSizes);
+      b, scope.getLocation(), viewSizesToLoopsMap, allViewSizes, tileSizes);
   if (!options.interchangeVector.empty())
     applyPermutationToVector(loopRanges, options.interchangeVector);
 
@@ -395,7 +396,8 @@ Optional<TiledLinalgOp> static tileLinalgOpImpl(
         if (!options.interchangeVector.empty())
           ivValues = applyMapToValues(b, loc, invPermutationMap, ivValues);
 
-        auto views = makeTiledViews(b, loc, op, ivValues, tileSizes, viewSizes);
+        auto views = makeTiledViews(b, loc, op, viewSizesToLoopsMap, ivValues,
+                                    tileSizes, allViewSizes);
         auto operands = getAssumedNonViewOperands(op);
         views.append(operands.begin(), operands.end());
         res = op.clone(b, loc, views);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index d923ea1bea76e..23d89c21e6e0a 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -52,9 +52,17 @@ static bool hasMultiplyAddBody(Region &r) {
   auto pattern2 = m_Op<YieldOp>(m_Op<AddFOp>(c, m_Op<MulFOp>(a, b)));
   auto pattern3 = m_Op<YieldOp>(m_Op<AddFOp>(m_Op<MulFOp>(b, a), c));
   auto pattern4 = m_Op<YieldOp>(m_Op<AddFOp>(c, m_Op<MulFOp>(b, a)));
+  auto pattern5 = m_Op<YieldOp>(m_Op<AddIOp>(m_Op<MulIOp>(a, b), c));
+  auto pattern6 = m_Op<YieldOp>(m_Op<AddIOp>(c, m_Op<MulIOp>(a, b)));
+  auto pattern7 = m_Op<YieldOp>(m_Op<AddIOp>(m_Op<MulIOp>(b, a), c));
+  auto pattern8 = m_Op<YieldOp>(m_Op<AddIOp>(c, m_Op<MulIOp>(b, a)));
   return pattern1.match(&r.front().back()) ||
          pattern2.match(&r.front().back()) ||
-         pattern3.match(&r.front().back()) || pattern4.match(&r.front().back());
+         pattern3.match(&r.front().back()) ||
+         pattern4.match(&r.front().back()) ||
+         pattern5.match(&r.front().back()) ||
+         pattern6.match(&r.front().back()) ||
+         pattern7.match(&r.front().back()) || pattern8.match(&r.front().back());
 }
 
 // TODO: Should be Tablegen'd from a single source that generates the op itself.
@@ -88,7 +96,7 @@ LogicalResult mlir::linalg::vectorizeLinalgOpPrecondition(Operation *op) {
     if (!outputTensorType.cast<ShapedType>().hasStaticShape())
       return failure();
 
-  if (isa<linalg::FillOp>(op))
+  if (isa<linalg::FillOp, linalg::CopyOp>(op))
     return success();
 
   return isContraction(op);
@@ -111,12 +119,6 @@ void mlir::linalg::vectorizeLinalgOp(OpBuilder &builder, Operation *op) {
     return;
   }
 
-  assert(succeeded(isContraction(op)) && "Expected contraction");
-
-  // Vectorize other ops as vector contraction.
-  // TODO: interface.
-  LLVM_DEBUG(dbgs() << dbgPref
-                    << "Rewrite linalg op as vector.contract: " << *op);
   // In the case of 0-D memrefs, return null and special case to scalar load or
   // store later.
   auto extractVectorTypeFromScalarView = [](Value v) {
@@ -125,6 +127,49 @@ void mlir::linalg::vectorizeLinalgOp(OpBuilder &builder, Operation *op) {
                ? VectorType()
                : VectorType::get(mt.getShape(), mt.getElementType());
   };
+
+  if (auto copyOp = dyn_cast<linalg::CopyOp>(op)) {
+    // Vectorize copy as a vector.transfer_read+vector.transfer_write.
+    LLVM_DEBUG(dbgs() << dbgPref
+                      << "Rewrite linalg.copy as vector.transfer_read + "
+                         "vector.transfer_write: "
+                      << *op);
+    Value zero = std_constant_index(0);
+    Value viewInput = copyOp.input();
+    Value viewOutput = copyOp.output();
+    Value vector;
+    if (VectorType inputType = extractVectorTypeFromScalarView(viewInput)) {
+      SmallVector<Value, 4> indicesInput(inputType.getRank(), zero);
+      if (copyOp.inputPermutation())
+        vector = vector_transfer_read(
+            extractVectorTypeFromScalarView(viewInput), viewInput, indicesInput,
+            copyOp.inputPermutation().getValue());
+      else
+        vector =
+            vector_transfer_read(extractVectorTypeFromScalarView(viewInput),
+                                 viewInput, indicesInput);
+    } else {
+      vector = std_load(viewInput).value;
+    }
+    if (VectorType outputType = extractVectorTypeFromScalarView(viewOutput)) {
+      SmallVector<Value, 4> indicesOutput(outputType.getRank(), zero);
+      if (copyOp.outputPermutation())
+        vector_transfer_write(vector, viewOutput, indicesOutput,
+                              copyOp.outputPermutation().getValue());
+      else
+        vector_transfer_write(vector, viewOutput, indicesOutput);
+    } else {
+      std_store(vector, viewOutput);
+    }
+    return;
+  }
+
+  assert(succeeded(isContraction(op)) && "Expected contraction");
+
+  // Vectorize other ops as vector contraction.
+  // TODO: interface.
+  LLVM_DEBUG(dbgs() << dbgPref
+                    << "Rewrite linalg op as vector.contract: " << *op);
   auto linalgOp = cast<linalg::LinalgOp>(op);
   Value viewA = linalgOp.getInput(0);
   Value viewB = linalgOp.getInput(1);
diff --git a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
index a9d5e2028c227..968f5b1d82c50 100644
--- a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
@@ -57,7 +57,7 @@ RegionMatcher::matchAsScalarBinaryOp(GenericOp op) {
 
 static Value emitOrFoldComposedAffineApply(OpBuilder &b, Location loc,
                                            AffineMap map,
-                                           ArrayRef<Value> operandsRef,
+                                           ValueRange operandsRef,
                                            OperationFolder *folder) {
   SmallVector<Value, 4> operands(operandsRef.begin(), operandsRef.end());
   fullyComposeAffineMapAndOperands(&map, &operands);
@@ -68,16 +68,16 @@ static Value emitOrFoldComposedAffineApply(OpBuilder &b, Location loc,
 
 SmallVector<Value, 4> mlir::linalg::applyMapToValues(OpBuilder &b, Location loc,
                                                      AffineMap map,
-                                                     ArrayRef<Value> values,
+                                                     ValueRange values,
                                                      OperationFolder *folder) {
   SmallVector<Value, 4> res;
   res.reserve(map.getNumResults());
-  unsigned numDims = map.getNumDims();
+  unsigned numDims = map.getNumDims(), numSym = map.getNumSymbols();
   // For each `expr` in `map`, applies the `expr` to the values extracted from
   // ranges. If the resulting application can be folded into a Value, the
   // folding occurs eagerly. Otherwise, an affine.apply operation is emitted.
   for (auto expr : map.getResults()) {
-    AffineMap map = AffineMap::get(numDims, 0, expr);
+    AffineMap map = AffineMap::get(numDims, numSym, expr);
     res.push_back(emitOrFoldComposedAffineApply(b, loc, map, values, folder));
   }
   return res;
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
index be1d271413901..aa376993ae718 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
@@ -222,6 +222,16 @@ void LowerABIAttributesPass::runOnOperation() {
   spirv::TargetEnv targetEnv(spirv::lookupTargetEnv(module));
 
   SPIRVTypeConverter typeConverter(targetEnv);
+
+  // Insert a bitcast in the case of a pointer type change.
+  typeConverter.addSourceMaterialization([](OpBuilder &builder,
+                                            spirv::PointerType type,
+                                            ValueRange inputs, Location loc) {
+    if (inputs.size() != 1 || !inputs[0].getType().isa<spirv::PointerType>())
+      return Value();
+    return builder.create<spirv::BitcastOp>(loc, type, inputs[0]).getResult();
+  });
+
   OwningRewritePatternList patterns;
   patterns.insert<ProcessInterfaceVarABI>(context, typeConverter);
 
diff --git a/mlir/lib/Dialect/Shape/IR/Shape.cpp b/mlir/lib/Dialect/Shape/IR/Shape.cpp
index b983968b124dc..3bdc5cc39a7bd 100644
--- a/mlir/lib/Dialect/Shape/IR/Shape.cpp
+++ b/mlir/lib/Dialect/Shape/IR/Shape.cpp
@@ -8,6 +8,7 @@
 
 #include "mlir/Dialect/Shape/IR/Shape.h"
 
+#include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/Dialect/Traits.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/DialectImplementation.h"
@@ -23,6 +24,10 @@ namespace {
 #include "ShapeCanonicalization.inc"
 }
 
+static RankedTensorType getExtentTensorType(MLIRContext *ctx) {
+  return RankedTensorType::get({ShapedType::kDynamicSize}, IndexType::get(ctx));
+}
+
 ShapeDialect::ShapeDialect(MLIRContext *context)
     : Dialect(getDialectNamespace(), context) {
   addOperations<
@@ -40,13 +45,16 @@ ShapeDialect::ShapeDialect(MLIRContext *context)
 Operation *ShapeDialect::materializeConstant(OpBuilder &builder,
                                              Attribute value, Type type,
                                              Location loc) {
-  if (auto shapeType = type.dyn_cast<ShapeType>())
+  if (type.isa<ShapeType>() ||
+      type == getExtentTensorType(builder.getContext()))
     return builder.create<ConstShapeOp>(loc, type,
                                         value.cast<DenseIntElementsAttr>());
-  if (auto sizeType = type.dyn_cast<SizeType>())
+  if (type.isa<SizeType>())
     return builder.create<ConstSizeOp>(loc, type, value.cast<IntegerAttr>());
-  if (auto witnessType = type.dyn_cast<WitnessType>())
+  if (type.isa<WitnessType>())
     return builder.create<ConstWitnessOp>(loc, type, value.cast<BoolAttr>());
+  if (type.isa<IndexType>())
+    return builder.create<ConstantOp>(loc, type, value);
   return nullptr;
 }
 
@@ -290,7 +298,8 @@ static void print(OpAsmPrinter &p, ConstShapeOp &op) {
   p << "[";
   interleaveComma(op.shape().getValues<int64_t>(), p,
                   [&](int64_t i) { p << i; });
-  p << "]";
+  p << "] : ";
+  p.printType(op.getType());
 }
 
 static ParseResult parseConstShapeOp(OpAsmParser &parser,
@@ -316,8 +325,10 @@ static ParseResult parseConstShapeOp(OpAsmParser &parser,
   }
   Builder &builder = parser.getBuilder();
   result.addAttribute("shape", builder.getIndexTensorAttr(ints));
-
-  result.types.push_back(ShapeType::get(builder.getContext()));
+  Type resultTy;
+  if (parser.parseColonType(resultTy))
+    return failure();
+  result.types.push_back(resultTy);
   return success();
 }
 
@@ -475,6 +486,20 @@ void ConstSizeOp::getAsmResultNames(
 
 OpFoldResult ConstWitnessOp::fold(ArrayRef<Attribute>) { return passingAttr(); }
 
+//===----------------------------------------------------------------------===//
+// ShapeEqOp
+//===----------------------------------------------------------------------===//
+
+OpFoldResult ShapeEqOp::fold(ArrayRef<Attribute> operands) {
+  auto lhs = operands[0].dyn_cast_or_null<DenseIntElementsAttr>();
+  if (lhs == nullptr)
+    return {};
+  auto rhs = operands[1].dyn_cast_or_null<DenseIntElementsAttr>();
+  if (rhs == nullptr)
+    return {};
+  return BoolAttr::get(lhs == rhs, getContext());
+}
+
 //===----------------------------------------------------------------------===//
 // IndexToSizeOp
 //===----------------------------------------------------------------------===//
@@ -510,10 +535,30 @@ OpFoldResult FromExtentsOp::fold(ArrayRef<Attribute> operands) {
 // GetExtentOp
 //===----------------------------------------------------------------------===//
 
+static LogicalResult verify(GetExtentOp op) {
+  Type shapeTy = op.shape().getType();
+  Type dimTy = op.dim().getType();
+  Type extentTy = op.extent().getType();
+  bool errorPropagationPossible =
+      shapeTy.isa<ShapeType>() || dimTy.isa<SizeType>();
+  if (errorPropagationPossible) {
+    if (!extentTy.isa<SizeType>())
+      op.emitError()
+          << "if at least one of the operands can hold error values then the "
+             "result must be of type `size` to propagate them";
+  } else {
+    if (extentTy.isa<SizeType>())
+      op.emitError() << "if none of the operands can hold error values then "
+                        "the result must be of type `index`";
+  }
+  return success();
+}
+
 Optional<int64_t> GetExtentOp::getConstantDim() {
-  if (auto constSizeOp = dim().getDefiningOp<ConstSizeOp>()) {
+  if (auto constSizeOp = dim().getDefiningOp<ConstSizeOp>())
     return constSizeOp.value().getLimitedValue();
-  }
+  if (auto constantOp = dim().getDefiningOp<ConstantOp>())
+    return constantOp.value().cast<IntegerAttr>().getInt();
   return llvm::None;
 }
 
@@ -533,15 +578,31 @@ void GetExtentOp::build(OpBuilder &builder, OperationState &result, Value shape,
                         int64_t dim) {
   auto loc = result.location;
   auto dimAttr = builder.getIndexAttr(dim);
-  Value dimValue = builder.create<ConstSizeOp>(loc, dimAttr);
-  build(builder, result, shape, dimValue);
+  if (shape.getType().isa<ShapeType>()) {
+    Value dim = builder.create<ConstSizeOp>(loc, dimAttr);
+    build(builder, result, builder.getType<SizeType>(), shape, dim);
+  } else {
+    Value dim =
+        builder.create<ConstantOp>(loc, builder.getIndexType(), dimAttr);
+    build(builder, result, builder.getIndexType(), shape, dim);
+  }
 }
 
 //===----------------------------------------------------------------------===//
 // RankOp
 //===----------------------------------------------------------------------===//
 
-OpFoldResult RankOp::fold(ArrayRef<Attribute> operands) {
+static LogicalResult verify(shape::RankOp op) {
+  Type argTy = op.shape().getType();
+  Type resultTy = op.rank().getType();
+  if (argTy.isa<ShapeType>() && !resultTy.isa<SizeType>())
+    return op.emitOpError()
+           << "if operand is of type `shape` then the result must be of type "
+              "`size` to propagate potential errors";
+  return success();
+}
+
+OpFoldResult shape::RankOp::fold(ArrayRef<Attribute> operands) {
   auto shape = operands[0].dyn_cast_or_null<DenseIntElementsAttr>();
   if (!shape)
     return {};
@@ -565,10 +626,11 @@ OpFoldResult RankOp::fold(ArrayRef<Attribute> operands) {
 /// %rank = shape.const_size 3
 
 namespace {
-struct RankShapeOfCanonicalizationPattern : public OpRewritePattern<RankOp> {
-  using OpRewritePattern<RankOp>::OpRewritePattern;
+struct RankShapeOfCanonicalizationPattern
+    : public OpRewritePattern<shape::RankOp> {
+  using OpRewritePattern<shape::RankOp>::OpRewritePattern;
 
-  LogicalResult matchAndRewrite(RankOp op,
+  LogicalResult matchAndRewrite(shape::RankOp op,
                                 PatternRewriter &rewriter) const override {
     auto shapeOfOp = op.shape().getDefiningOp<ShapeOfOp>();
     if (!shapeOfOp)
@@ -577,15 +639,18 @@ struct RankShapeOfCanonicalizationPattern : public OpRewritePattern<RankOp> {
         shapeOfOp.arg().getType().dyn_cast<RankedTensorType>();
     if (!rankedTensorType)
       return failure();
+    assert(op.getType().isa<IndexType>() &&
+           "expected `rank(shape_of( ... )]` based on a shaped argument to "
+           "yield an index type");
     int64_t rank = rankedTensorType.getRank();
-    rewriter.replaceOpWithNewOp<ConstSizeOp>(op.getOperation(), rank);
+    rewriter.replaceOpWithNewOp<ConstantIndexOp>(op.getOperation(), rank);
     return success();
   }
 };
 } // namespace
 
-void RankOp::getCanonicalizationPatterns(OwningRewritePatternList &patterns,
-                                         MLIRContext *context) {
+void shape::RankOp::getCanonicalizationPatterns(
+    OwningRewritePatternList &patterns, MLIRContext *context) {
   patterns.insert<RankShapeOfCanonicalizationPattern>(context);
 }
 
@@ -619,6 +684,23 @@ OpFoldResult ShapeOfOp::fold(ArrayRef<Attribute>) {
   return builder.getIndexTensorAttr(type.getShape());
 }
 
+static LogicalResult verify(ShapeOfOp op) {
+  Type argTy = op.arg().getType();
+  Type resultTy = op.result().getType();
+  if (argTy.isa<ValueShapeType>()) {
+    if (!resultTy.isa<ShapeType>())
+      return op.emitOpError()
+             << "if operand is of type `value_shape` then the result must be "
+                "of type `shape` to propagate potential error shapes";
+  } else {
+    assert(argTy.isa<ShapedType>());
+    if (resultTy != getExtentTensorType(op.getContext()))
+      return op.emitOpError() << "if operand is a shaped type then the result "
+                                 "must be an extent tensor";
+  }
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // SizeToIndexOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/StandardOps/CMakeLists.txt b/mlir/lib/Dialect/StandardOps/CMakeLists.txt
index 7d61aea3116ed..06284f5d1daa2 100644
--- a/mlir/lib/Dialect/StandardOps/CMakeLists.txt
+++ b/mlir/lib/Dialect/StandardOps/CMakeLists.txt
@@ -15,7 +15,7 @@ add_mlir_dialect_library(MLIRStandardOps
   MLIREDSC
   MLIRIR
   MLIRSideEffectInterfaces
-  MLIRVectorUnrollInterface
+  MLIRVectorInterfaces
   MLIRViewLikeInterface
   )
 
diff --git a/mlir/lib/Dialect/Vector/CMakeLists.txt b/mlir/lib/Dialect/Vector/CMakeLists.txt
index 69a329917228d..d6ba987e6622f 100644
--- a/mlir/lib/Dialect/Vector/CMakeLists.txt
+++ b/mlir/lib/Dialect/Vector/CMakeLists.txt
@@ -19,5 +19,5 @@ add_mlir_dialect_library(MLIRVector
   MLIRSCF
   MLIRLoopAnalysis
   MLIRSideEffectInterfaces
-  MLIRVectorUnrollInterface
+  MLIRVectorInterfaces
   )
diff --git a/mlir/lib/Dialect/Vector/VectorOps.cpp b/mlir/lib/Dialect/Vector/VectorOps.cpp
index 5e01fa26f32e9..d16c7c3d6fdbe 100644
--- a/mlir/lib/Dialect/Vector/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/VectorOps.cpp
@@ -1466,22 +1466,6 @@ void ExtractStridedSliceOp::getCanonicalizationPatterns(
 // TransferReadOp
 //===----------------------------------------------------------------------===//
 
-/// Build the default minor identity map suitable for a vector transfer. This
-/// also handles the case memref<... x vector<...>> -> vector<...> in which the
-/// rank of the identity map must take the vector element type into account.
-AffineMap
-mlir::vector::impl::getTransferMinorIdentityMap(MemRefType memRefType,
-                                                VectorType vectorType) {
-  int64_t elementVectorRank = 0;
-  VectorType elementVectorType =
-      memRefType.getElementType().dyn_cast<VectorType>();
-  if (elementVectorType)
-    elementVectorRank += elementVectorType.getRank();
-  return AffineMap::getMinorIdentityMap(
-      memRefType.getRank(), vectorType.getRank() - elementVectorRank,
-      memRefType.getContext());
-}
-
 template <typename EmitFun>
 static LogicalResult verifyPermutationMap(AffineMap permutationMap,
                                           EmitFun emitOpError) {
@@ -1600,11 +1584,10 @@ void TransferReadOp::build(OpBuilder &builder, OperationState &result,
   build(builder, result, vectorType, memref, indices, permMap, maybeMasked);
 }
 
-template <typename TransferOp>
-static void printTransferAttrs(OpAsmPrinter &p, TransferOp op) {
+static void printTransferAttrs(OpAsmPrinter &p, VectorTransferOpInterface op) {
   SmallVector<StringRef, 2> elidedAttrs;
-  if (op.permutation_map() == TransferOp::getTransferMinorIdentityMap(
-                                  op.getMemRefType(), op.getVectorType()))
+  if (op.permutation_map() ==
+      getTransferMinorIdentityMap(op.getMemRefType(), op.getVectorType()))
     elidedAttrs.push_back(op.getPermutationMapAttrName());
   bool elideMasked = true;
   if (auto maybeMasked = op.masked()) {
@@ -1623,7 +1606,7 @@ static void printTransferAttrs(OpAsmPrinter &p, TransferOp op) {
 static void print(OpAsmPrinter &p, TransferReadOp op) {
   p << op.getOperationName() << " " << op.memref() << "[" << op.indices()
     << "], " << op.padding();
-  printTransferAttrs(p, op);
+  printTransferAttrs(p, cast<VectorTransferOpInterface>(op.getOperation()));
   p << " : " << op.getMemRefType() << ", " << op.getVectorType();
 }
 
@@ -1653,8 +1636,7 @@ static ParseResult parseTransferReadOp(OpAsmParser &parser,
   auto permutationAttrName = TransferReadOp::getPermutationMapAttrName();
   auto attr = result.attributes.get(permutationAttrName);
   if (!attr) {
-    auto permMap =
-        TransferReadOp::getTransferMinorIdentityMap(memRefType, vectorType);
+    auto permMap = getTransferMinorIdentityMap(memRefType, vectorType);
     result.attributes.set(permutationAttrName, AffineMapAttr::get(permMap));
   }
   return failure(
@@ -1733,6 +1715,7 @@ static bool isInBounds(TransferOp op, int64_t resultIdx, int64_t indicesIdx) {
 
   int64_t memrefSize = op.getMemRefType().getDimSize(indicesIdx);
   int64_t vectorSize = op.getVectorType().getDimSize(resultIdx);
+
   return cstOp.getValue() + vectorSize <= memrefSize;
 }
 
@@ -1744,23 +1727,11 @@ static LogicalResult foldTransferMaskAttribute(TransferOp op) {
   bool changed = false;
   SmallVector<bool, 4> isMasked;
   isMasked.reserve(op.getTransferRank());
-  // `permutationMap` results and `op.indices` sizes may not match and may not
-  // be aligned. The first `indicesIdx` may just be indexed and not transferred
-  // from/into the vector.
-  // For example:
-  //  vector.transfer %0[%i, %j, %k, %c0] : memref<?x?x?x?xf32>, vector<2x4xf32>
-  // with `permutation_map = (d0, d1, d2, d3) -> (d2, d3)`.
-  // The `permutationMap` results and `op.indices` are however aligned when
-  // iterating in reverse until we exhaust `permutationMap` results.
-  // As a consequence we iterate with 2 running indices: `resultIdx` and
-  // `indicesIdx`, until `resultIdx` reaches 0.
-  for (int64_t resultIdx = permutationMap.getNumResults() - 1,
-               indicesIdx = op.indices().size() - 1;
-       resultIdx >= 0; --resultIdx, --indicesIdx) {
+  op.zipResultAndIndexing([&](int64_t resultIdx, int64_t indicesIdx) {
     // Already marked unmasked, nothing to see here.
     if (!op.isMaskedDim(resultIdx)) {
       isMasked.push_back(false);
-      continue;
+      return;
     }
     // Currently masked, check whether we can statically determine it is
     // inBounds.
@@ -1768,12 +1739,11 @@ static LogicalResult foldTransferMaskAttribute(TransferOp op) {
     isMasked.push_back(!inBounds);
     // We commit the pattern if it is "more inbounds".
     changed |= inBounds;
-  }
+  });
   if (!changed)
     return failure();
   // OpBuilder is only used as a helper to build an I64ArrayAttr.
   OpBuilder b(op.getContext());
-  std::reverse(isMasked.begin(), isMasked.end());
   op.setAttr(TransferOp::getMaskedAttrName(), b.getBoolArrayAttr(isMasked));
   return success();
 }
@@ -1842,8 +1812,7 @@ static ParseResult parseTransferWriteOp(OpAsmParser &parser,
   auto permutationAttrName = TransferWriteOp::getPermutationMapAttrName();
   auto attr = result.attributes.get(permutationAttrName);
   if (!attr) {
-    auto permMap =
-        TransferWriteOp::getTransferMinorIdentityMap(memRefType, vectorType);
+    auto permMap = getTransferMinorIdentityMap(memRefType, vectorType);
     result.attributes.set(permutationAttrName, AffineMapAttr::get(permMap));
   }
   return failure(
@@ -1855,7 +1824,7 @@ static ParseResult parseTransferWriteOp(OpAsmParser &parser,
 static void print(OpAsmPrinter &p, TransferWriteOp op) {
   p << op.getOperationName() << " " << op.vector() << ", " << op.memref() << "["
     << op.indices() << "]";
-  printTransferAttrs(p, op);
+  printTransferAttrs(p, cast<VectorTransferOpInterface>(op.getOperation()));
   p << " : " << op.getVectorType() << ", " << op.getMemRefType();
 }
 
@@ -1889,6 +1858,49 @@ Optional<SmallVector<int64_t, 4>> TransferWriteOp::getShapeForUnroll() {
   return SmallVector<int64_t, 4>{s.begin(), s.end()};
 }
 
+//===----------------------------------------------------------------------===//
+// GatherOp
+//===----------------------------------------------------------------------===//
+
+static LogicalResult verify(GatherOp op) {
+  VectorType indicesVType = op.getIndicesVectorType();
+  VectorType maskVType = op.getMaskVectorType();
+  VectorType resVType = op.getResultVectorType();
+
+  if (resVType.getElementType() != op.getMemRefType().getElementType())
+    return op.emitOpError("base and result element type should match");
+
+  if (resVType.getDimSize(0) != indicesVType.getDimSize(0))
+    return op.emitOpError("expected result dim to match indices dim");
+  if (resVType.getDimSize(0) != maskVType.getDimSize(0))
+    return op.emitOpError("expected result dim to match mask dim");
+  if (llvm::size(op.pass_thru()) != 0) {
+    VectorType passVType = op.getPassThruVectorType();
+    if (resVType != passVType)
+      return op.emitOpError("expected pass_thru of same type as result type");
+  }
+  return success();
+}
+
+//===----------------------------------------------------------------------===//
+// ScatterOp
+//===----------------------------------------------------------------------===//
+
+static LogicalResult verify(ScatterOp op) {
+  VectorType indicesVType = op.getIndicesVectorType();
+  VectorType maskVType = op.getMaskVectorType();
+  VectorType valueVType = op.getValueVectorType();
+
+  if (valueVType.getElementType() != op.getMemRefType().getElementType())
+    return op.emitOpError("base and value element type should match");
+
+  if (valueVType.getDimSize(0) != indicesVType.getDimSize(0))
+    return op.emitOpError("expected value dim to match indices dim");
+  if (valueVType.getDimSize(0) != maskVType.getDimSize(0))
+    return op.emitOpError("expected value dim to match mask dim");
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // ShapeCastOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Vector/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/VectorTransforms.cpp
index a63862c1a4fef..ab93ef406024e 100644
--- a/mlir/lib/Dialect/Vector/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/VectorTransforms.cpp
@@ -30,7 +30,7 @@
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/IR/Types.h"
-#include "mlir/Interfaces/VectorUnrollInterface.h"
+#include "mlir/Interfaces/VectorInterfaces.h"
 
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
diff --git a/mlir/lib/Dialect/Vector/VectorUtils.cpp b/mlir/lib/Dialect/Vector/VectorUtils.cpp
index d5beaefc5eac9..75ebb2f7d959a 100644
--- a/mlir/lib/Dialect/Vector/VectorUtils.cpp
+++ b/mlir/lib/Dialect/Vector/VectorUtils.cpp
@@ -243,6 +243,18 @@ AffineMap mlir::makePermutationMap(
   return ::makePermutationMap(indices, enclosingLoopToVectorDim);
 }
 
+AffineMap mlir::getTransferMinorIdentityMap(MemRefType memRefType,
+                                            VectorType vectorType) {
+  int64_t elementVectorRank = 0;
+  VectorType elementVectorType =
+      memRefType.getElementType().dyn_cast<VectorType>();
+  if (elementVectorType)
+    elementVectorRank += elementVectorType.getRank();
+  return AffineMap::getMinorIdentityMap(
+      memRefType.getRank(), vectorType.getRank() - elementVectorRank,
+      memRefType.getContext());
+}
+
 bool matcher::operatesOnSuperVectorsOf(Operation &op,
                                        VectorType subVectorType) {
   // First, extract the vector type and distinguish between:
@@ -257,11 +269,8 @@ bool matcher::operatesOnSuperVectorsOf(Operation &op,
   bool mustDivide = false;
   (void)mustDivide;
   VectorType superVectorType;
-  if (auto read = dyn_cast<vector::TransferReadOp>(op)) {
-    superVectorType = read.getVectorType();
-    mustDivide = true;
-  } else if (auto write = dyn_cast<vector::TransferWriteOp>(op)) {
-    superVectorType = write.getVectorType();
+  if (auto transfer = dyn_cast<VectorTransferOpInterface>(op)) {
+    superVectorType = transfer.getVectorType();
     mustDivide = true;
   } else if (op.getNumResults() == 0) {
     if (!isa<ReturnOp>(op)) {
diff --git a/mlir/lib/IR/AffineExpr.cpp b/mlir/lib/IR/AffineExpr.cpp
index e0c4b6b208f7a..5ba9737a5245b 100644
--- a/mlir/lib/IR/AffineExpr.cpp
+++ b/mlir/lib/IR/AffineExpr.cpp
@@ -93,6 +93,14 @@ AffineExpr::replaceDimsAndSymbols(ArrayRef<AffineExpr> dimReplacements,
   llvm_unreachable("Unknown AffineExpr");
 }
 
+/// Replace symbols[0 .. numDims - 1] by symbols[shift .. shift + numDims - 1].
+AffineExpr AffineExpr::shiftSymbols(unsigned numSymbols, unsigned shift) const {
+  SmallVector<AffineExpr, 4> symbols;
+  for (unsigned idx = 0; idx < numSymbols; ++idx)
+    symbols.push_back(getAffineSymbolExpr(idx + shift, getContext()));
+  return replaceDimsAndSymbols({}, symbols);
+}
+
 /// Returns true if this expression is made out of only symbols and
 /// constants (no dimensional identifiers).
 bool AffineExpr::isSymbolicOrConstant() const {
diff --git a/mlir/lib/IR/AffineMap.cpp b/mlir/lib/IR/AffineMap.cpp
index 42dbef273b6d1..ba76976a17c1d 100644
--- a/mlir/lib/IR/AffineMap.cpp
+++ b/mlir/lib/IR/AffineMap.cpp
@@ -434,18 +434,19 @@ AffineMap mlir::inversePermutation(AffineMap map) {
 }
 
 AffineMap mlir::concatAffineMaps(ArrayRef<AffineMap> maps) {
-  unsigned numResults = 0;
+  unsigned numResults = 0, numDims = 0, numSymbols = 0;
   for (auto m : maps)
     numResults += m.getNumResults();
-  unsigned numDims = 0;
   SmallVector<AffineExpr, 8> results;
   results.reserve(numResults);
   for (auto m : maps) {
-    assert(m.getNumSymbols() == 0 && "expected map without symbols");
-    results.append(m.getResults().begin(), m.getResults().end());
+    for (auto res : m.getResults())
+      results.push_back(res.shiftSymbols(m.getNumSymbols(), numSymbols));
+
+    numSymbols += m.getNumSymbols();
     numDims = std::max(m.getNumDims(), numDims);
   }
-  return AffineMap::get(numDims, /*numSymbols=*/0, results,
+  return AffineMap::get(numDims, numSymbols, results,
                         maps.front().getContext());
 }
 
diff --git a/mlir/lib/IR/Value.cpp b/mlir/lib/IR/Value.cpp
index 6467a7f2295b3..776b32a73d58f 100644
--- a/mlir/lib/IR/Value.cpp
+++ b/mlir/lib/IR/Value.cpp
@@ -77,7 +77,11 @@ Operation *Value::getDefiningOp() const {
 Location Value::getLoc() const {
   if (auto *op = getDefiningOp())
     return op->getLoc();
-  return UnknownLoc::get(getContext());
+
+  // Use the location of the parent operation if this is a block argument.
+  // TODO: Should we just add locations to block arguments?
+  Operation *parentOp = cast<BlockArgument>().getOwner()->getParentOp();
+  return parentOp ? parentOp->getLoc() : UnknownLoc::get(getContext());
 }
 
 /// Return the Region in which this Value is defined.
diff --git a/mlir/lib/Interfaces/CMakeLists.txt b/mlir/lib/Interfaces/CMakeLists.txt
index b8498e224f250..0a8f75b6f7d97 100644
--- a/mlir/lib/Interfaces/CMakeLists.txt
+++ b/mlir/lib/Interfaces/CMakeLists.txt
@@ -6,7 +6,7 @@ set(LLVM_OPTIONAL_SOURCES
   InferTypeOpInterface.cpp
   LoopLikeInterface.cpp
   SideEffectInterfaces.cpp
-  VectorUnrollInterface.cpp
+  VectorInterfaces.cpp
   ViewLikeInterface.cpp
   )
 
@@ -33,6 +33,6 @@ add_mlir_interface_library(DerivedAttributeOpInterface)
 add_mlir_interface_library(InferTypeOpInterface)
 add_mlir_interface_library(LoopLikeInterface)
 add_mlir_interface_library(SideEffectInterfaces)
-add_mlir_interface_library(VectorUnrollInterface)
+add_mlir_interface_library(VectorInterfaces)
 add_mlir_interface_library(ViewLikeInterface)
 
diff --git a/mlir/lib/Interfaces/VectorUnrollInterface.cpp b/mlir/lib/Interfaces/VectorInterfaces.cpp
similarity index 74%
rename from mlir/lib/Interfaces/VectorUnrollInterface.cpp
rename to mlir/lib/Interfaces/VectorInterfaces.cpp
index 6d3d432a70618..0f16b885ca2fc 100644
--- a/mlir/lib/Interfaces/VectorUnrollInterface.cpp
+++ b/mlir/lib/Interfaces/VectorInterfaces.cpp
@@ -1,4 +1,4 @@
-//===- VectorUnrollInterface.cpp - Unrollable vector operations -*- C++ -*-===//
+//===- VectorInterfaces.cpp - Unrollable vector operations -*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Interfaces/VectorUnrollInterface.h"
+#include "mlir/Interfaces/VectorInterfaces.h"
 
 using namespace mlir;
 
@@ -15,4 +15,4 @@ using namespace mlir;
 //===----------------------------------------------------------------------===//
 
 /// Include the definitions of the VectorUntoll interfaces.
-#include "mlir/Interfaces/VectorUnrollInterface.cpp.inc"
+#include "mlir/Interfaces/VectorInterfaces.cpp.inc"
diff --git a/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp b/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp
index c6a58a8dc5a80..9754c614efdfb 100644
--- a/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp
+++ b/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp
@@ -846,8 +846,9 @@ LogicalResult Importer::processFunction(llvm::Function *f) {
     return failure();
 
   b.setInsertionPoint(module.getBody(), getFuncInsertPt());
-  LLVMFuncOp fop = b.create<LLVMFuncOp>(UnknownLoc::get(context), f->getName(),
-                                        functionType);
+  LLVMFuncOp fop =
+      b.create<LLVMFuncOp>(UnknownLoc::get(context), f->getName(), functionType,
+                           convertLinkageFromLLVM(f->getLinkage()));
 
   if (FlatSymbolRefAttr personality = getPersonalityAsAttr(f))
     fop.setAttr(b.getIdentifier("personality"), personality);
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index 0defea6bbbb95..3a70dd3932e92 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -30,6 +30,7 @@
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/Cloning.h"
@@ -594,9 +595,22 @@ LogicalResult ModuleTranslation::convertOperation(Operation &opInst,
     return success();
   }
   if (auto condbrOp = dyn_cast<LLVM::CondBrOp>(opInst)) {
+    auto weights = condbrOp.branch_weights();
+    llvm::MDNode *branchWeights = nullptr;
+    if (weights) {
+      // Map weight attributes to LLVM metadata.
+      auto trueWeight =
+          weights.getValue().getValue(0).cast<IntegerAttr>().getInt();
+      auto falseWeight =
+          weights.getValue().getValue(1).cast<IntegerAttr>().getInt();
+      branchWeights =
+          llvm::MDBuilder(llvmModule->getContext())
+              .createBranchWeights(static_cast<uint32_t>(trueWeight),
+                                   static_cast<uint32_t>(falseWeight));
+    }
     builder.CreateCondBr(valueMapping.lookup(condbrOp.getOperand(0)),
                          blockMapping[condbrOp.getSuccessor(0)],
-                         blockMapping[condbrOp.getSuccessor(1)]);
+                         blockMapping[condbrOp.getSuccessor(1)], branchWeights);
     return success();
   }
 
@@ -884,6 +898,7 @@ LogicalResult ModuleTranslation::convertFunctionSignatures() {
         function.getName(),
         cast<llvm::FunctionType>(function.getType().getUnderlyingType()));
     llvm::Function *llvmFunc = cast<llvm::Function>(llvmFuncCst.getCallee());
+    llvmFunc->setLinkage(convertLinkageToLLVM(function.linkage()));
     functionMapping[function.getName()] = llvmFunc;
 
     // Forward the pass-through attributes to LLVM.
diff --git a/mlir/lib/Transforms/DialectConversion.cpp b/mlir/lib/Transforms/DialectConversion.cpp
index b9ed64f573f22..9778958a45885 100644
--- a/mlir/lib/Transforms/DialectConversion.cpp
+++ b/mlir/lib/Transforms/DialectConversion.cpp
@@ -17,6 +17,7 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/SaveAndRestore.h"
 #include "llvm/Support/ScopedPrinter.h"
 
 using namespace mlir;
@@ -106,8 +107,15 @@ namespace {
 /// functionality, i.e. we will traverse if the mapped value also has a mapping.
 struct ConversionValueMapping {
   /// Lookup a mapped value within the map. If a mapping for the provided value
-  /// does not exist then return the provided value.
-  Value lookupOrDefault(Value from) const;
+  /// does not exist then return the provided value. If `desiredType` is
+  /// non-null, returns the most recently mapped value with that type. If an
+  /// operand of that type does not exist, defaults to normal behavior.
+  Value lookupOrDefault(Value from, Type desiredType = nullptr) const;
+
+  /// Lookup a mapped value within the map, or return null if a mapping does not
+  /// exist. If a mapping exists, this follows the same behavior of
+  /// `lookupOrDefault`.
+  Value lookupOrNull(Value from) const;
 
   /// Map a value to the one provided.
   void map(Value oldVal, Value newVal) { mapping.map(oldVal, newVal); }
@@ -121,14 +129,36 @@ struct ConversionValueMapping {
 };
 } // end anonymous namespace
 
-/// Lookup a mapped value within the map. If a mapping for the provided value
-/// does not exist then return the provided value.
-Value ConversionValueMapping::lookupOrDefault(Value from) const {
-  // If this value had a valid mapping, unmap that value as well in the case
-  // that it was also replaced.
-  while (auto mappedValue = mapping.lookupOrNull(from))
+Value ConversionValueMapping::lookupOrDefault(Value from,
+                                              Type desiredType) const {
+  // If there was no desired type, simply find the leaf value.
+  if (!desiredType) {
+    // If this value had a valid mapping, unmap that value as well in the case
+    // that it was also replaced.
+    while (auto mappedValue = mapping.lookupOrNull(from))
+      from = mappedValue;
+    return from;
+  }
+
+  // Otherwise, try to find the deepest value that has the desired type.
+  Value desiredValue;
+  do {
+    if (from.getType() == desiredType)
+      desiredValue = from;
+
+    Value mappedValue = mapping.lookupOrNull(from);
+    if (!mappedValue)
+      break;
     from = mappedValue;
-  return from;
+  } while (true);
+
+  // If the desired value was found use it, otherwise default to the leaf value.
+  return desiredValue ? desiredValue : from;
+}
+
+Value ConversionValueMapping::lookupOrNull(Value from) const {
+  Value result = lookupOrDefault(from);
+  return result == from ? nullptr : result;
 }
 
 //===----------------------------------------------------------------------===//
@@ -209,10 +239,17 @@ struct ArgConverter {
   /// its original state.
   void discardRewrites(Block *block);
 
-  /// Fully replace uses of the old arguments with the new, materializing cast
-  /// operations as necessary.
+  /// Fully replace uses of the old arguments with the new.
   void applyRewrites(ConversionValueMapping &mapping);
 
+  /// Materialize any necessary conversions for converted arguments that have
+  /// live users, using the provided `findLiveUser` to search for a user that
+  /// survives the conversion process.
+  LogicalResult
+  materializeLiveConversions(ConversionValueMapping &mapping,
+                             OpBuilder &builder,
+                             function_ref<Operation *(Value)> findLiveUser);
+
   //===--------------------------------------------------------------------===//
   // Conversion
   //===--------------------------------------------------------------------===//
@@ -307,7 +344,6 @@ void ArgConverter::discardRewrites(Block *block) {
 
 void ArgConverter::applyRewrites(ConversionValueMapping &mapping) {
   for (auto &info : conversionInfo) {
-    Block *newBlock = info.first;
     ConvertedBlockInfo &blockInfo = info.second;
     Block *origBlock = blockInfo.origBlock;
 
@@ -318,24 +354,8 @@ void ArgConverter::applyRewrites(ConversionValueMapping &mapping) {
 
       // Handle the case of a 1->0 value mapping.
       if (!argInfo) {
-        // If a replacement value was given for this argument, use that to
-        // replace all uses.
-        auto argReplacementValue = mapping.lookupOrDefault(origArg);
-        if (argReplacementValue != origArg) {
-          origArg.replaceAllUsesWith(argReplacementValue);
-          continue;
-        }
-        // If there are any dangling uses then replace the argument with one
-        // generated by the type converter. This is necessary as the cast must
-        // persist in the IR after conversion.
-        if (!origArg.use_empty()) {
-          rewriter.setInsertionPointToStart(newBlock);
-          Value newArg = blockInfo.converter->materializeConversion(
-              rewriter, origArg.getLoc(), origArg.getType(), llvm::None);
-          assert(newArg &&
-                 "Couldn't materialize a block argument after 1->0 conversion");
+        if (Value newArg = mapping.lookupOrNull(origArg))
           origArg.replaceAllUsesWith(newArg);
-        }
         continue;
       }
 
@@ -355,6 +375,59 @@ void ArgConverter::applyRewrites(ConversionValueMapping &mapping) {
   }
 }
 
+LogicalResult ArgConverter::materializeLiveConversions(
+    ConversionValueMapping &mapping, OpBuilder &builder,
+    function_ref<Operation *(Value)> findLiveUser) {
+  for (auto &info : conversionInfo) {
+    Block *newBlock = info.first;
+    ConvertedBlockInfo &blockInfo = info.second;
+    Block *origBlock = blockInfo.origBlock;
+
+    // Process the remapping for each of the original arguments.
+    for (unsigned i = 0, e = origBlock->getNumArguments(); i != e; ++i) {
+      // FIXME: We should run the below checks even if the type conversion was
+      // 1->N, but a lot of existing lowering rely on the block argument being
+      // blindly replaced. Those usages should be updated, and this if should be
+      // removed.
+      if (blockInfo.argInfo[i])
+        continue;
+
+      // If the type of this argument changed and the argument is still live, we
+      // need to materialize a conversion.
+      BlockArgument origArg = origBlock->getArgument(i);
+      auto argReplacementValue = mapping.lookupOrDefault(origArg);
+      bool isDroppedArg = argReplacementValue == origArg;
+      if (argReplacementValue.getType() == origArg.getType() && !isDroppedArg)
+        continue;
+      Operation *liveUser = findLiveUser(origArg);
+      if (!liveUser)
+        continue;
+
+      if (OpResult result = argReplacementValue.dyn_cast<OpResult>())
+        rewriter.setInsertionPointAfter(result.getOwner());
+      else
+        rewriter.setInsertionPointToStart(newBlock);
+      Value newArg = blockInfo.converter->materializeSourceConversion(
+          rewriter, origArg.getLoc(), origArg.getType(),
+          isDroppedArg ? ValueRange() : ValueRange(argReplacementValue));
+      if (!newArg) {
+        InFlightDiagnostic diag =
+            emitError(origArg.getLoc())
+            << "failed to materialize conversion for block argument #" << i
+            << " that remained live after conversion, type was "
+            << origArg.getType();
+        if (!isDroppedArg)
+          diag << ", with target type " << argReplacementValue.getType();
+        diag.attachNote(liveUser->getLoc())
+            << "see existing live user here: " << *liveUser;
+        return failure();
+      }
+      mapping.map(origArg, newArg);
+    }
+  }
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // Conversion
 
@@ -417,8 +490,8 @@ Block *ArgConverter::applySignatureConversion(
     // to pack the new values. For 1->1 mappings, if there is no materialization
     // provided, use the argument directly instead.
     auto replArgs = newArgs.slice(inputMap->inputNo, inputMap->size);
-    Value newArg = converter.materializeConversion(rewriter, origArg.getLoc(),
-                                                   origArg.getType(), replArgs);
+    Value newArg = converter.materializeArgumentConversion(
+        rewriter, origArg.getLoc(), origArg.getType(), replArgs);
     if (!newArg) {
       assert(replArgs.size() == 1 &&
              "couldn't materialize the result of 1->N conversion");
@@ -516,13 +589,15 @@ class OperationTransactionState {
   SmallVector<Block *, 2> successors;
 };
 
-/// This class represents one requested operation replacement via 'replaceOp'.
+/// This class represents one requested operation replacement via 'replaceOp' or
+/// 'eraseOp`.
 struct OpReplacement {
   OpReplacement() = default;
-  OpReplacement(ValueRange newValues)
-      : newValues(newValues.begin(), newValues.end()) {}
+  OpReplacement(TypeConverter *converter) : converter(converter) {}
 
-  SmallVector<Value, 2> newValues;
+  /// An optional type converter that can be used to materialize conversions
+  /// between the new and old values if necessary.
+  TypeConverter *converter = nullptr;
 };
 
 /// The kind of the block action performed during the rewrite.  Actions can be
@@ -611,9 +686,14 @@ struct ConversionPatternRewriterImpl {
   /// "numActionsToKeep" actions remains.
   void undoBlockActions(unsigned numActionsToKeep = 0);
 
-  /// Remap the given operands to those with potentially different types.
-  void remapValues(Operation::operand_range operands,
-                   SmallVectorImpl<Value> &remapped);
+  /// Remap the given operands to those with potentially different types. The
+  /// provided type converter is used to ensure that the remapped types are
+  /// legal. Returns success if the operands could be remapped, failure
+  /// otherwise.
+  LogicalResult remapValues(Location loc, PatternRewriter &rewriter,
+                            TypeConverter *converter,
+                            Operation::operand_range operands,
+                            SmallVectorImpl<Value> &remapped);
 
   /// Returns true if the given operation is ignored, and does not need to be
   /// converted.
@@ -666,6 +746,11 @@ struct ConversionPatternRewriterImpl {
   void notifyRegionWasClonedBefore(iterator_range<Region::iterator> &blocks,
                                    Location origRegionLoc);
 
+  /// Notifies that a pattern match failed for the given reason.
+  LogicalResult
+  notifyMatchFailure(Location loc,
+                     function_ref<void(Diagnostic &)> reasonCallback);
+
   //===--------------------------------------------------------------------===//
   // State
   //===--------------------------------------------------------------------===//
@@ -712,6 +797,10 @@ struct ConversionPatternRewriterImpl {
   /// explicitly provided.
   TypeConverter defaultTypeConverter;
 
+  /// The current conversion pattern that is being rewritten, or nullptr if
+  /// called from outside of a conversion pattern rewrite.
+  const ConversionPattern *currentConversionPattern = nullptr;
+
 #ifndef NDEBUG
   /// A set of operations that have pending updates. This tracking isn't
   /// strictly necessary, and is thus only active during debug builds for extra
@@ -759,11 +848,9 @@ void ConversionPatternRewriterImpl::discardRewrites() {
 void ConversionPatternRewriterImpl::applyRewrites() {
   // Apply all of the rewrites replacements requested during conversion.
   for (auto &repl : replacements) {
-    for (unsigned i = 0, e = repl.second.newValues.size(); i != e; ++i) {
-      if (auto newValue = repl.second.newValues[i])
-        repl.first->getResult(i).replaceAllUsesWith(
-            mapping.lookupOrDefault(newValue));
-    }
+    for (OpResult result : repl.first->getResults())
+      if (Value newValue = mapping.lookupOrNull(result))
+        result.replaceAllUsesWith(newValue);
 
     // If this operation defines any regions, drop any pending argument
     // rewrites.
@@ -905,11 +992,61 @@ void ConversionPatternRewriterImpl::undoBlockActions(
   blockActions.resize(numActionsToKeep);
 }
 
-void ConversionPatternRewriterImpl::remapValues(
+LogicalResult ConversionPatternRewriterImpl::remapValues(
+    Location loc, PatternRewriter &rewriter, TypeConverter *converter,
     Operation::operand_range operands, SmallVectorImpl<Value> &remapped) {
   remapped.reserve(llvm::size(operands));
-  for (Value operand : operands)
-    remapped.push_back(mapping.lookupOrDefault(operand));
+
+  SmallVector<Type, 1> legalTypes;
+  for (auto it : llvm::enumerate(operands)) {
+    Value operand = it.value();
+    Type origType = operand.getType();
+
+    // If a converter was provided, get the desired legal types for this
+    // operand.
+    Type desiredType;
+    if (converter) {
+      // If there is no legal conversion, fail to match this pattern.
+      legalTypes.clear();
+      if (failed(converter->convertType(origType, legalTypes))) {
+        return notifyMatchFailure(loc, [=](Diagnostic &diag) {
+          diag << "unable to convert type for operand #" << it.index()
+               << ", type was " << origType;
+        });
+      }
+      // TODO: There currently isn't any mechanism to do 1->N type conversion
+      // via the PatternRewriter replacement API, so for now we just ignore it.
+      if (legalTypes.size() == 1)
+        desiredType = legalTypes.front();
+    } else {
+      // TODO: What we should do here is just set `desiredType` to `origType`
+      // and then handle the necessary type conversions after the conversion
+      // process has finished. Unfortunately a lot of patterns currently rely on
+      // receiving the new operands even if the types change, so we keep the
+      // original behavior here for now until all of the patterns relying on
+      // this get updated.
+    }
+    Value newOperand = mapping.lookupOrDefault(operand, desiredType);
+
+    // Handle the case where the conversion was 1->1 and the new operand type
+    // isn't legal.
+    Type newOperandType = newOperand.getType();
+    if (converter && desiredType && newOperandType != desiredType) {
+      // Attempt to materialize a conversion for this new value.
+      newOperand = converter->materializeTargetConversion(
+          rewriter, loc, desiredType, newOperand);
+      if (!newOperand) {
+        return notifyMatchFailure(loc, [=](Diagnostic &diag) {
+          diag << "unable to materialize a conversion for "
+                  "operand #"
+               << it.index() << ", from " << newOperandType << " to "
+               << desiredType;
+        });
+      }
+    }
+    remapped.push_back(newOperand);
+  }
+  return success();
 }
 
 bool ConversionPatternRewriterImpl::isOpIgnored(Operation *op) const {
@@ -987,16 +1124,22 @@ void ConversionPatternRewriterImpl::notifyOpReplaced(Operation *op,
   Value newValue, result;
   for (auto it : llvm::zip(newValues, op->getResults())) {
     std::tie(newValue, result) = it;
-    if (!newValue)
+    if (!newValue) {
       resultChanged = true;
-    else
-      mapping.map(result, newValue);
+      continue;
+    }
+    // Remap, and check for any result type changes.
+    mapping.map(result, newValue);
+    resultChanged |= (newValue.getType() != result.getType());
   }
   if (resultChanged)
     operationsWithChangedResults.push_back(replacements.size());
 
   // Record the requested operation replacement.
-  replacements.insert(std::make_pair(op, OpReplacement(newValues)));
+  TypeConverter *converter = nullptr;
+  if (currentConversionPattern)
+    converter = currentConversionPattern->getTypeConverter();
+  replacements.insert(std::make_pair(op, OpReplacement(converter)));
 
   // Mark this operation as recursively ignored so that we don't need to
   // convert any nested operations.
@@ -1041,6 +1184,16 @@ void ConversionPatternRewriterImpl::notifyRegionWasClonedBefore(
   assert(succeeded(result) && "expected region to have no unreachable blocks");
 }
 
+LogicalResult ConversionPatternRewriterImpl::notifyMatchFailure(
+    Location loc, function_ref<void(Diagnostic &)> reasonCallback) {
+  LLVM_DEBUG({
+    Diagnostic diag(loc, DiagnosticSeverity::Remark);
+    reasonCallback(diag);
+    logger.startLine() << "** Failure : " << diag.str() << "\n";
+  });
+  return failure();
+}
+
 //===----------------------------------------------------------------------===//
 // ConversionPatternRewriter
 //===----------------------------------------------------------------------===//
@@ -1200,12 +1353,7 @@ void ConversionPatternRewriter::cancelRootUpdate(Operation *op) {
 /// PatternRewriter hook for notifying match failure reasons.
 LogicalResult ConversionPatternRewriter::notifyMatchFailure(
     Operation *op, function_ref<void(Diagnostic &)> reasonCallback) {
-  LLVM_DEBUG({
-    Diagnostic diag(op->getLoc(), DiagnosticSeverity::Remark);
-    reasonCallback(diag);
-    impl->logger.startLine() << "** Failure : " << diag.str() << "\n";
-  });
-  return failure();
+  return impl->notifyMatchFailure(op->getLoc(), reasonCallback);
 }
 
 /// Return a reference to the internal implementation.
@@ -1221,9 +1369,22 @@ detail::ConversionPatternRewriterImpl &ConversionPatternRewriter::getImpl() {
 LogicalResult
 ConversionPattern::matchAndRewrite(Operation *op,
                                    PatternRewriter &rewriter) const {
-  SmallVector<Value, 4> operands;
   auto &dialectRewriter = static_cast<ConversionPatternRewriter &>(rewriter);
-  dialectRewriter.getImpl().remapValues(op->getOperands(), operands);
+  auto &rewriterImpl = dialectRewriter.getImpl();
+
+  // Track the current conversion pattern in the rewriter.
+  assert(!rewriterImpl.currentConversionPattern &&
+         "already inside of a pattern rewrite");
+  llvm::SaveAndRestore<const ConversionPattern *> currentPatternGuard(
+      rewriterImpl.currentConversionPattern, this);
+
+  // Remap the operands of the operation.
+  SmallVector<Value, 4> operands;
+  if (failed(rewriterImpl.remapValues(op->getLoc(), rewriter,
+                                      getTypeConverter(), op->getOperands(),
+                                      operands))) {
+    return failure();
+  }
   return matchAndRewrite(op, operands, dialectRewriter);
 }
 
@@ -1878,6 +2039,24 @@ struct OperationConverter {
   /// remaining artifacts and complete the conversion.
   LogicalResult finalize(ConversionPatternRewriter &rewriter);
 
+  /// Legalize the types of converted block arguments.
+  LogicalResult
+  legalizeConvertedArgumentTypes(ConversionPatternRewriter &rewriter,
+                                 ConversionPatternRewriterImpl &rewriterImpl);
+
+  /// Legalize an operation result that was marked as "erased".
+  LogicalResult
+  legalizeErasedResult(Operation *op, OpResult result,
+                       ConversionPatternRewriterImpl &rewriterImpl);
+
+  /// Legalize an operation result that was replaced with a value of a different
+  /// type.
+  LogicalResult
+  legalizeChangedResultType(Operation *op, OpResult result, Value newValue,
+                            TypeConverter *replConverter,
+                            ConversionPatternRewriter &rewriter,
+                            ConversionPatternRewriterImpl &rewriterImpl);
+
   /// The legalizer to use when converting operations.
   OperationLegalizer opLegalizer;
 
@@ -1961,33 +2140,145 @@ LogicalResult OperationConverter::convertOperations(ArrayRef<Operation *> ops) {
 LogicalResult
 OperationConverter::finalize(ConversionPatternRewriter &rewriter) {
   ConversionPatternRewriterImpl &rewriterImpl = rewriter.getImpl();
-  auto isOpDead = [&](Operation *op) { return rewriterImpl.isOpIgnored(op); };
 
-  // Process the operations with changed results.
-  for (unsigned replIdx : rewriterImpl.operationsWithChangedResults) {
+  // Legalize converted block arguments.
+  if (failed(legalizeConvertedArgumentTypes(rewriter, rewriterImpl)))
+    return failure();
+
+  // Process requested operation replacements.
+  for (unsigned i = 0, e = rewriterImpl.operationsWithChangedResults.size();
+       i != e; ++i) {
+    unsigned replIdx = rewriterImpl.operationsWithChangedResults[i];
     auto &repl = *(rewriterImpl.replacements.begin() + replIdx);
-    for (auto it : llvm::zip(repl.first->getResults(), repl.second.newValues)) {
-      Value result = std::get<0>(it), newValue = std::get<1>(it);
+    for (OpResult result : repl.first->getResults()) {
+      Value newValue = rewriterImpl.mapping.lookupOrNull(result);
 
       // If the operation result was replaced with null, all of the uses of this
       // value should be replaced.
-      if (newValue)
+      if (!newValue) {
+        if (failed(legalizeErasedResult(repl.first, result, rewriterImpl)))
+          return failure();
+        continue;
+      }
+
+      // Otherwise, check to see if the type of the result changed.
+      if (result.getType() == newValue.getType())
         continue;
 
-      auto liveUserIt = llvm::find_if_not(result.getUsers(), isOpDead);
-      if (liveUserIt != result.user_end()) {
-        InFlightDiagnostic diag = repl.first->emitError()
-                                  << "failed to legalize operation '"
-                                  << repl.first->getName()
-                                  << "' marked as erased";
-        diag.attachNote(liveUserIt->getLoc())
-            << "found live user of result #"
-            << result.cast<OpResult>().getResultNumber() << ": " << *liveUserIt;
+      // Legalize this result.
+      rewriter.setInsertionPoint(repl.first);
+      if (failed(legalizeChangedResultType(repl.first, result, newValue,
+                                           repl.second.converter, rewriter,
+                                           rewriterImpl)))
         return failure();
-      }
+
+      // Update the end iterator for this loop in the case it was updated
+      // when legalizing generated conversion operations.
+      e = rewriterImpl.operationsWithChangedResults.size();
+    }
+  }
+  return success();
+}
+
+LogicalResult OperationConverter::legalizeConvertedArgumentTypes(
+    ConversionPatternRewriter &rewriter,
+    ConversionPatternRewriterImpl &rewriterImpl) {
+  // Functor used to check if all users of a value will be dead after
+  // conversion.
+  auto findLiveUser = [&](Value val) {
+    auto liveUserIt = llvm::find_if_not(val.getUsers(), [&](Operation *user) {
+      return rewriterImpl.isOpIgnored(user);
+    });
+    return liveUserIt == val.user_end() ? nullptr : *liveUserIt;
+  };
+
+  // Materialize any necessary conversions for converted block arguments that
+  // are still live.
+  size_t numCreatedOps = rewriterImpl.createdOps.size();
+  if (failed(rewriterImpl.argConverter.materializeLiveConversions(
+          rewriterImpl.mapping, rewriter, findLiveUser)))
+    return failure();
+
+  // Legalize any newly created operations during argument materialization.
+  for (int i : llvm::seq<int>(numCreatedOps, rewriterImpl.createdOps.size())) {
+    if (failed(opLegalizer.legalize(rewriterImpl.createdOps[i], rewriter))) {
+      return rewriterImpl.createdOps[i]->emitError()
+             << "failed to legalize conversion operation generated for block "
+                "argument that remained live after conversion";
+    }
+  }
+  return success();
+}
+
+LogicalResult OperationConverter::legalizeErasedResult(
+    Operation *op, OpResult result,
+    ConversionPatternRewriterImpl &rewriterImpl) {
+  // If the operation result was replaced with null, all of the uses of this
+  // value should be replaced.
+  auto liveUserIt = llvm::find_if_not(result.getUsers(), [&](Operation *user) {
+    return rewriterImpl.isOpIgnored(user);
+  });
+  if (liveUserIt != result.user_end()) {
+    InFlightDiagnostic diag = op->emitError("failed to legalize operation '")
+                              << op->getName() << "' marked as erased";
+    diag.attachNote(liveUserIt->getLoc())
+        << "found live user of result #" << result.getResultNumber() << ": "
+        << *liveUserIt;
+    return failure();
+  }
+  return success();
+}
+
+LogicalResult OperationConverter::legalizeChangedResultType(
+    Operation *op, OpResult result, Value newValue,
+    TypeConverter *replConverter, ConversionPatternRewriter &rewriter,
+    ConversionPatternRewriterImpl &rewriterImpl) {
+  // Walk the users of this value to see if there are any live users that
+  // weren't replaced during conversion.
+  auto liveUserIt = llvm::find_if_not(result.getUsers(), [&](Operation *user) {
+    return rewriterImpl.isOpIgnored(user);
+  });
+  if (liveUserIt == result.user_end())
+    return success();
+
+  // If the replacement has a type converter, attempt to materialize a
+  // conversion back to the original type.
+  if (!replConverter) {
+    // TODO: We should emit an error here, similarly to the case where the
+    // result is replaced with null. Unfortunately a lot of existing
+    // patterns rely on this behavior, so until those patterns are updated
+    // we keep the legacy behavior here of just forwarding the new value.
+    return success();
+  }
+
+  // Track the number of created operations so that new ones can be legalized.
+  size_t numCreatedOps = rewriterImpl.createdOps.size();
+
+  // Materialize a conversion for this live result value.
+  Type resultType = result.getType();
+  Value convertedValue = replConverter->materializeSourceConversion(
+      rewriter, op->getLoc(), resultType, newValue);
+  if (!convertedValue) {
+    InFlightDiagnostic diag = op->emitError()
+                              << "failed to materialize conversion for result #"
+                              << result.getResultNumber() << " of operation '"
+                              << op->getName()
+                              << "' that remained live after conversion";
+    diag.attachNote(liveUserIt->getLoc())
+        << "see existing live user here: " << *liveUserIt;
+    return failure();
+  }
+
+  // Legalize all of the newly created conversion operations.
+  for (int i : llvm::seq<int>(numCreatedOps, rewriterImpl.createdOps.size())) {
+    if (failed(opLegalizer.legalize(rewriterImpl.createdOps[i], rewriter))) {
+      return op->emitError("failed to legalize conversion operation generated ")
+             << "for result #" << result.getResultNumber() << " of operation '"
+             << op->getName() << "' that remained live after conversion";
     }
   }
 
+  rewriterImpl.mapping.map(result, convertedValue);
   return success();
 }
 
@@ -2136,11 +2427,11 @@ LogicalResult TypeConverter::convertSignatureArgs(TypeRange types,
   return success();
 }
 
-Value TypeConverter::materializeConversion(PatternRewriter &rewriter,
-                                           Location loc, Type resultType,
-                                           ValueRange inputs) {
+Value TypeConverter::materializeConversion(
+    MutableArrayRef<MaterializationCallbackFn> materializations,
+    OpBuilder &builder, Location loc, Type resultType, ValueRange inputs) {
   for (MaterializationCallbackFn &fn : llvm::reverse(materializations))
-    if (Optional<Value> result = fn(rewriter, resultType, inputs, loc))
+    if (Optional<Value> result = fn(builder, resultType, inputs, loc))
       return result.getValue();
   return nullptr;
 }
diff --git a/mlir/lib/Transforms/Utils/Utils.cpp b/mlir/lib/Transforms/Utils/Utils.cpp
index 5d6d58fc6939d..35853c17232bd 100644
--- a/mlir/lib/Transforms/Utils/Utils.cpp
+++ b/mlir/lib/Transforms/Utils/Utils.cpp
@@ -445,7 +445,9 @@ LogicalResult mlir::normalizeMemRef(AllocOp allocOp) {
       MemRefType::Builder(memrefType)
           .setShape(newShape)
           .setAffineMaps(b.getMultiDimIdentityMap(newRank));
-  auto newAlloc = b.create<AllocOp>(allocOp.getLoc(), newMemRefType);
+
+  auto newAlloc = b.create<AllocOp>(allocOp.getLoc(), newMemRefType, llvm::None,
+                                    allocOp.alignmentAttr());
 
   // Replace all uses of the old memref.
   if (failed(replaceAllMemRefUsesWith(oldMemRef, /*newMemRef=*/newAlloc,
diff --git a/mlir/test/Conversion/SPIRVToLLVM/control-flow-ops-to-llvm.mlir b/mlir/test/Conversion/SPIRVToLLVM/control-flow-ops-to-llvm.mlir
new file mode 100644
index 0000000000000..0a2f6d681769d
--- /dev/null
+++ b/mlir/test/Conversion/SPIRVToLLVM/control-flow-ops-to-llvm.mlir
@@ -0,0 +1,172 @@
+// RUN: mlir-opt -convert-spirv-to-llvm -split-input-file -verify-diagnostics %s | FileCheck %s
+
+//===----------------------------------------------------------------------===//
+// spv.Branch
+//===----------------------------------------------------------------------===//
+
+spv.module Logical GLSL450 {
+  spv.func @branch_without_arguments() -> () "None" {
+	  // CHECK: llvm.br ^bb1
+    spv.Branch ^label
+  // CHECK: ^bb1
+  ^label:
+    spv.Return
+  }
+
+  spv.func @branch_with_arguments() -> () "None" {
+    %0 = spv.constant 0 : i32
+    %1 = spv.constant true
+    // CHECK: llvm.br ^bb1(%{{.*}}, %{{.*}} : !llvm.i32, !llvm.i1)
+    spv.Branch ^label(%0, %1: i32, i1)
+  // CHECK: ^bb1(%{{.*}}: !llvm.i32, %{{.*}}: !llvm.i1)
+  ^label(%arg0: i32, %arg1: i1):
+    spv.Return
+  }
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// spv.BranchConditional
+//===----------------------------------------------------------------------===//
+
+spv.module Logical GLSL450 {
+  spv.func @cond_branch_without_arguments() -> () "None" {
+    // CHECK: %[[COND:.*]] = llvm.mlir.constant(true) : !llvm.i1
+    %cond = spv.constant true
+    // CHECK: lvm.cond_br %[[COND]], ^bb1, ^bb2
+    spv.BranchConditional %cond, ^true, ^false
+    // CHECK: ^bb1:
+  ^true:
+    spv.Return
+    // CHECK: ^bb2:
+  ^false:
+    spv.Return
+  }
+
+  spv.func @cond_branch_with_arguments_nested() -> () "None" {
+    // CHECK: %[[COND1:.*]] = llvm.mlir.constant(true) : !llvm.i1
+    %cond = spv.constant true
+    %0 = spv.constant 0 : i32
+    // CHECK: %[[COND2:.*]] = llvm.mlir.constant(false) : !llvm.i1
+    %false = spv.constant false
+    // CHECK: llvm.cond_br %[[COND1]], ^bb1(%{{.*}}, %[[COND2]] : !llvm.i32, !llvm.i1), ^bb2
+    spv.BranchConditional %cond, ^outer_true(%0, %false: i32, i1), ^outer_false
+  // CHECK: ^bb1(%{{.*}}: !llvm.i32, %[[COND:.*]]: !llvm.i1):
+  ^outer_true(%arg0: i32, %arg1: i1):
+    // CHECK: llvm.cond_br %[[COND]], ^bb3, ^bb4(%{{.*}}, %{{.*}} : !llvm.i32, !llvm.i32)
+    spv.BranchConditional %arg1, ^inner_true, ^inner_false(%arg0, %arg0: i32, i32)
+  // CHECK: ^bb2:
+  ^outer_false:
+    spv.Return
+  // CHECK: ^bb3:
+  ^inner_true:
+    spv.Return
+  // CHECK: ^bb4(%{{.*}}: !llvm.i32, %{{.*}}: !llvm.i32):
+  ^inner_false(%arg3: i32, %arg4: i32):
+    spv.Return
+  }
+}
+
+// -----
+
+spv.module Logical GLSL450 {
+  spv.func @cond_branch_with_weights(%cond: i1) -> () "None" {
+    // expected-error@+1 {{failed to legalize operation 'spv.BranchConditional' that was explicitly marked illegal}}
+    spv.BranchConditional %cond [1, 2], ^true, ^false
+  ^true:
+    spv.Return
+  ^false:
+    spv.Return
+  }
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// spv.selection
+//===----------------------------------------------------------------------===//
+
+spv.module Logical GLSL450 {
+  spv.func @selection_empty() -> () "None" {
+    // CHECK: llvm.return
+    spv.selection {
+    }
+    spv.Return
+  }
+
+  spv.func @selection_with_merge_block_only() -> () "None" {
+    %cond = spv.constant true
+    // CHECK: llvm.return
+    spv.selection {
+      spv.BranchConditional %cond, ^merge, ^merge
+    ^merge:
+      spv._merge
+    }
+    spv.Return
+  }
+
+  spv.func @selection_with_true_block_only() -> () "None" {
+    // CHECK: %[[COND:.*]] = llvm.mlir.constant(true) : !llvm.i1
+    %cond = spv.constant true
+    // CHECK: llvm.cond_br %[[COND]], ^bb1, ^bb2
+    spv.selection {
+      spv.BranchConditional %cond, ^true, ^merge
+    // CHECK: ^bb1:
+    ^true:
+    // CHECK: llvm.br ^bb2
+      spv.Branch ^merge
+    // CHECK: ^bb2:
+    ^merge:
+      // CHECK: llvm.br ^bb3
+      spv._merge
+    }
+    // CHECK: ^bb3:
+    // CHECK-NEXT: llvm.return
+    spv.Return
+  }
+
+  spv.func @selection_with_both_true_and_false_block() -> () "None" {
+    // CHECK: %[[COND:.*]] = llvm.mlir.constant(true) : !llvm.i1
+    %cond = spv.constant true
+    // CHECK: llvm.cond_br %[[COND]], ^bb1, ^bb2
+    spv.selection {
+      spv.BranchConditional %cond, ^true, ^false
+    // CHECK: ^bb1:
+    ^true:
+    // CHECK: llvm.br ^bb3
+      spv.Branch ^merge
+    // CHECK: ^bb2:
+    ^false:
+    // CHECK: llvm.br ^bb3
+      spv.Branch ^merge
+    // CHECK: ^bb3:
+    ^merge:
+      // CHECK: llvm.br ^bb4
+      spv._merge
+    }
+    // CHECK: ^bb4:
+    // CHECK-NEXT: llvm.return
+    spv.Return
+  }
+
+  spv.func @selection_with_early_return(%arg0: i1) -> i32 "None" {
+    // CHECK: %[[ZERO:.*]] = llvm.mlir.constant(0 : i32) : !llvm.i32
+    %0 = spv.constant 0 : i32
+    // CHECK: llvm.cond_br %{{.*}}, ^bb1(%[[ZERO]] : !llvm.i32), ^bb2
+    spv.selection {
+      spv.BranchConditional %arg0, ^true(%0 : i32), ^merge
+    // CHECK: ^bb1(%[[ARG:.*]]: !llvm.i32):
+    ^true(%arg1: i32):
+      // CHECK: llvm.return %[[ARG]] : !llvm.i32
+      spv.ReturnValue %arg1 : i32
+    // CHECK: ^bb2:
+    ^merge:
+      // CHECK: llvm.br ^bb3
+      spv._merge
+    }
+    // CHECK: ^bb3:
+    %one = spv.constant 1 : i32
+    spv.ReturnValue %one : i32
+  }
+}
diff --git a/mlir/test/Conversion/ShapeToSCF/shape-to-scf.mlir b/mlir/test/Conversion/ShapeToSCF/shape-to-scf.mlir
index 2e5a45c4cc11e..441b2e92cc3d9 100644
--- a/mlir/test/Conversion/ShapeToSCF/shape-to-scf.mlir
+++ b/mlir/test/Conversion/ShapeToSCF/shape-to-scf.mlir
@@ -39,7 +39,7 @@ func @shape_of_unranked(%arg : tensor<*xf32>) {
   // CHECK:     }
   // CHECK-DAG: %[[SHAPE_INT:.*]] = tensor_load %[[SHAPE_MEM]] : memref<?xi64>
   // CHECK-DAG: %[[SHAPE:.*]] = index_cast %[[SHAPE_INT]] : tensor<?xi64> to tensor<?xindex>
-  %shape = shape.shape_of %arg : tensor<*xf32>
+  %shape = shape.shape_of %arg : tensor<*xf32> -> tensor<?xindex>
   return
 }
 
diff --git a/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir b/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
index 22206637adbf4..908acabe5345d 100644
--- a/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
+++ b/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
@@ -10,28 +10,6 @@ func @size_id(%size : !shape.size) -> !shape.size {
 
 // -----
 
-// Lower `size_to_index` conversion to no-op.
-// CHECK-LABEL: @size_to_index
-// CHECK-SAME: (%[[SIZE:.*]]: index) -> index
-func @size_to_index(%size : !shape.size) -> index {
-  // CHECK-NEXT: return %[[SIZE]] : index
-  %index = shape.size_to_index %size
-  return %index : index
-}
-
-// -----
-
-// Lower `index_to_size` conversion to no-op.
-// CHECK-LABEL: @index_to_size
-// CHECK-SAME: (%[[INDEX:.*]]: index) -> index
-func @index_to_size(%index : index) -> !shape.size {
-  // CHECK-NEXT: return %[[INDEX]] : index
-  %size = shape.index_to_size %index
-  return %size : !shape.size
-}
-
-// -----
-
 // Convert `shape` to `tensor<?xindex>` type.
 // CHECK-LABEL: @shape_id
 // CHECK-SAME: (%[[SHAPE:.*]]: tensor<?xindex>)
@@ -42,29 +20,6 @@ func @shape_id(%shape : !shape.shape) -> !shape.shape {
 
 // -----
 
-// Lower `to_extent_tensor` operation to no-op.
-// CHECK-LABEL: @to_extent_tensor
-// CHECK-SAME: (%[[SHAPE:.*]]: tensor<?xindex>) -> tensor<?xindex>
-func @to_extent_tensor(%shape : !shape.shape) -> tensor<?xindex> {
-  // CHECK-NEXT: return %[[SHAPE]] : tensor<?xindex>
-  %tensor = "shape.to_extent_tensor"(%shape) : (!shape.shape) -> tensor<?xindex>
-  return %tensor : tensor<?xindex>
-}
-
-// -----
-
-// Lower `from_extent_tensor` operation to no-op.
-// CHECK-LABEL: @from_extent_tensor
-// CHECK-SAME: (%[[TENSOR:.*]]: tensor<?xindex>) -> tensor<?xindex>
-func @from_extent_tensor(%tensor : tensor<?xindex>) -> !shape.shape {
-  // CHECK-NEXT: return %[[TENSOR]] : tensor<?xindex>
-  %shape = "shape.from_extent_tensor"(%tensor)
-      : (tensor<?xindex>) -> !shape.shape
-  return %shape : !shape.shape
-}
-
-// -----
-
 // Lower binary ops.
 // CHECK-LABEL: @binary_ops
 // CHECK-SAME: (%[[LHS:.*]]: index, %[[RHS:.*]]: index)
@@ -78,16 +33,6 @@ func @binary_ops(%lhs : !shape.size, %rhs : !shape.size) {
 
 // -----
 
-// Convert `const_size` to `constant` op.
-// CHECK-LABEL: @size_const
-func @size_const() -> !shape.size {
-  %c1 = shape.const_size 1
-  return %c1 : !shape.size
-}
-// CHECK: %[[C1:.*]] = constant 1 : index
-// CHECK: return %[[C1]] : index
-// -----
-
 // Lower `shape_of` for statically shaped tensor.
 // CHECK-LABEL: @shape_of_stat
 // CHECK-SAME: (%[[ARG:.*]]: tensor<1x2x3xf32>)
@@ -95,8 +40,9 @@ func @shape_of_stat(%arg : tensor<1x2x3xf32>) {
   // CHECK-DAG: %[[C1:.*]] = constant 1 : index
   // CHECK-DAG: %[[C2:.*]] = constant 2 : index
   // CHECK-DAG: %[[C3:.*]] = constant 3 : index
-  // CHECK-DAG: %[[SHAPE:.*]] = tensor_from_elements(%[[C1]], %[[C2]], %[[C3]]) : tensor<3xindex>
-  %shape = shape.shape_of %arg : tensor<1x2x3xf32>
+  // CHECK-DAG: %[[SHAPE_UNCASTED:.*]] = tensor_from_elements(%[[C1]], %[[C2]], %[[C3]]) : tensor<3xindex>
+  // CHECK-DAG: %[[SHAPE:.*]] = tensor_cast %[[SHAPE_UNCASTED]] : tensor<3xindex> to tensor<?xindex>
+  %shape = shape.shape_of %arg : tensor<1x2x3xf32> -> tensor<?xindex>
   return
 }
 
@@ -110,8 +56,9 @@ func @shape_of_dyn(%arg : tensor<1x5x?xf32>) {
   // CHECK-DAG: %[[C5:.*]] = constant 5 : index
   // CHECK-DAG: %[[C2:.*]] = constant 2 : index
   // CHECK-DAG: %[[DYN_DIM:.*]] = dim %[[ARG]], %[[C2]] : tensor<1x5x?xf32>
-  // CHECK-DAG: %[[SHAPE:.*]] = tensor_from_elements(%[[C1]], %[[C5]], %[[DYN_DIM]]) : tensor<3xindex>
-  %shape = shape.shape_of %arg : tensor<1x5x?xf32>
+  // CHECK-DAG: %[[SHAPE_UNCASTED:.*]] = tensor_from_elements(%[[C1]], %[[C5]], %[[DYN_DIM]]) : tensor<3xindex>
+  // CHECK-DAG: %[[SHAPE:.*]] = tensor_cast %[[SHAPE_UNCASTED]] : tensor<3xindex> to tensor<?xindex>
+  %shape = shape.shape_of %arg : tensor<1x5x?xf32> -> tensor<?xindex>
   return
 }
 
@@ -120,12 +67,12 @@ func @shape_of_dyn(%arg : tensor<1x5x?xf32>) {
 // Convert `rank` to `dim` of the first dimension.
 // CHECK-LABEL: @rank
 // CHECK-SAME: (%[[SHAPE:.*]]: tensor<?xindex>) -> index
-func @rank(%shape : !shape.shape) -> !shape.size {
-  // CHECK-DAG: %[[C0:.*]] = constant 0 : index
-  // CHECK-DAG: %[[RESULT:.*]] = dim %[[SHAPE]], %[[C0]]
-  // CHECK-DAG: return %[[RESULT]] : index
-  %rank = shape.rank %shape
-  return %rank : !shape.size
+func @rank(%shape : tensor<?xindex>) -> index {
+  // CHECK: %[[C0:.*]] = constant 0 : index
+  // CHECK: %[[RESULT:.*]] = dim %[[SHAPE]], %[[C0]]
+  // CHECK: return %[[RESULT]] : index
+  %rank = shape.rank %shape : tensor<?xindex> -> index
+  return %rank : index
 }
 
 // -----
@@ -134,28 +81,25 @@ func @rank(%shape : !shape.shape) -> !shape.size {
 // `shape_of` operation.
 // CHECK-LABEL: @get_extent_shape_of
 // CHECK-SAME:  (%[[ARG:.*]]: tensor<2x3xf32>, %[[IDX:.*]]: index) -> index
-func @get_extent_shape_of(%arg : tensor<2x3xf32>, %idx : !shape.size)
-    -> !shape.size {
+func @get_extent_shape_of(%arg : tensor<2x3xf32>, %idx : index) -> index {
   // CHECK: %[[RESULT:.*]] = dim %[[ARG]], %[[IDX]] : tensor<2x3xf32>
   // CHECK: return %[[RESULT]] : index
-  %shape = shape.shape_of %arg : tensor<2x3xf32>
-  %result = shape.get_extent %shape, %idx
-  return %result : !shape.size
+  %shape = shape.shape_of %arg : tensor<2x3xf32> -> tensor<?xindex>
+  %result = shape.get_extent %shape, %idx : tensor<?xindex>, index -> index
+  return %result : index
 }
 
 // -----
 
-// Express `get_extent` as `std.extract_element` when it relies directly on the
-// outcome of a `from_extent_tensor` operation.
+// Express `get_extent` as `std.extract_element`.
 // CHECK-LABEL: @get_extent_from_extent_tensor
 // CHECK-SAME: (%[[EXTENTS:.*]]: tensor<?xindex>, %[[IDX:.*]]: index) -> index
-func @get_extent_from_extent_tensor(%extents : tensor<?xindex>,
-                                    %idx : !shape.size) -> !shape.size {
+func @get_extent_from_extent_tensor(%extents : tensor<?xindex>, %idx : index)
+    -> index {
   // CHECK: %[[RESULT:.*]] = extract_element %[[EXTENTS]][%[[IDX]]] : tensor<?xindex>
   // CHECK: return %[[RESULT]] : index
-  %shape = shape.from_extent_tensor %extents : tensor<?xindex>
-  %result = shape.get_extent %shape, %idx
-  return %result : !shape.size
+  %result = shape.get_extent %extents, %idx : tensor<?xindex>, index -> index
+  return %result : index
 }
 
 // -----
@@ -163,11 +107,12 @@ func @get_extent_from_extent_tensor(%extents : tensor<?xindex>,
 // Lower `any` to its first operand.
 // CHECK-LABEL: @any_of_three
 // CHECK-SAME:  (%[[A:.*]]: tensor<?xindex>, %[[B:.*]]: tensor<?xindex>, %[[C:.*]]: tensor<?xindex>) -> tensor<?xindex>
-func @any_of_three(%a : !shape.shape, %b : !shape.shape, %c : !shape.shape)
-    -> !shape.shape {
+func @any_of_three(%a : tensor<?xindex>,
+                   %b : tensor<?xindex>,
+                   %c : tensor<?xindex>) -> tensor<?xindex> {
   // CHECK: return %[[A]] : tensor<?xindex>
-  %result = shape.any %a, %b, %c
-  return %result : !shape.shape
+  %result = shape.any %a, %b, %c : tensor<?xindex>
+  return %result : tensor<?xindex>
 }
 
 // -----
@@ -175,9 +120,9 @@ func @any_of_three(%a : !shape.shape, %b : !shape.shape, %c : !shape.shape)
 // Lower `any` to its first operand.
 // CHECK-LABEL: @any_of_one
 // CHECK-SAME:  (%[[A:.*]]: tensor<?xindex>) -> tensor<?xindex>
-func @any_of_one(%a : !shape.shape) -> !shape.shape {
+func @any_of_one(%a : tensor<?xindex>) -> tensor<?xindex> {
   // CHECK: return %[[A]] : tensor<?xindex>
-  %result = shape.any %a
-  return %result : !shape.shape
+  %result = shape.any %a : tensor<?xindex>
+  return %result : tensor<?xindex>
 }
 
diff --git a/mlir/test/Conversion/StandardToLLVM/standard-to-llvm.mlir b/mlir/test/Conversion/StandardToLLVM/standard-to-llvm.mlir
index b8ebdfbf35f1c..a6006023a5092 100644
--- a/mlir/test/Conversion/StandardToLLVM/standard-to-llvm.mlir
+++ b/mlir/test/Conversion/StandardToLLVM/standard-to-llvm.mlir
@@ -78,12 +78,18 @@ func @rsqrt_multidim_vector(%arg0 : vector<4x3xf32>) {
 
 // -----
 
-// This should not crash. The first operation cannot be converted, so the
-// second should not match. This attempts to convert `return` to `llvm.return`
-// and complains about non-LLVM types.
-func @unknown_source() -> i32 {
-  %0 = "foo"() : () -> i32
-  %1 = addi %0, %0 : i32
-  // expected-error@+1 {{must be LLVM dialect type}}
-  return %1 : i32
+// Lowers `assert` to a function call to `abort` if the assertion is violated.
+// CHECK: llvm.func @abort()
+// CHECK-LABEL: @assert_test_function
+// CHECK-SAME:  (%[[ARG:.*]]: !llvm.i1)
+func @assert_test_function(%arg : i1) {
+  // CHECK: llvm.cond_br %[[ARG]], ^[[CONTINUATION_BLOCK:.*]], ^[[FAILURE_BLOCK:.*]]
+  // CHECK: ^[[CONTINUATION_BLOCK]]:
+  // CHECK: llvm.return
+  // CHECK: ^[[FAILURE_BLOCK]]:
+  // CHECK: llvm.call @abort() : () -> ()
+  // CHECK: llvm.unreachable
+  assert %arg, "Computer says no"
+  return
 }
+
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
index 874cb5cca1410..c5259a17fcef0 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
@@ -969,3 +969,23 @@ func @flat_transpose(%arg0: vector<16xf32>) -> vector<16xf32> {
 // CHECK-SAME:      {columns = 4 : i32, rows = 4 : i32} :
 // CHECK-SAME:      !llvm<"<16 x float>"> into !llvm<"<16 x float>">
 // CHECK:       llvm.return %[[T]] : !llvm<"<16 x float>">
+
+func @gather_op(%arg0: memref<?xf32>, %arg1: vector<3xi32>, %arg2: vector<3xi1>, %arg3: vector<3xf32>) -> vector<3xf32> {
+  %0 = vector.gather %arg0, %arg1, %arg2, %arg3 : (memref<?xf32>, vector<3xi32>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
+  return %0 : vector<3xf32>
+}
+
+// CHECK-LABEL: func @gather_op
+// CHECK: %[[P:.*]] = llvm.getelementptr {{.*}}[%{{.*}}] : (!llvm<"float*">, !llvm<"<3 x i32>">) -> !llvm<"<3 x float*>">
+// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (!llvm<"<3 x float*>">, !llvm<"<3 x i1>">, !llvm<"<3 x float>">) -> !llvm<"<3 x float>">
+// CHECK: llvm.return %[[G]] : !llvm<"<3 x float>">
+
+func @scatter_op(%arg0: memref<?xf32>, %arg1: vector<3xi32>, %arg2: vector<3xi1>, %arg3: vector<3xf32>) {
+  vector.scatter %arg0, %arg1, %arg2, %arg3 : vector<3xi32>, vector<3xi1>, vector<3xf32> into memref<?xf32>
+  return
+}
+
+// CHECK-LABEL: func @scatter_op
+// CHECK: %[[P:.*]] = llvm.getelementptr {{.*}}[%{{.*}}] : (!llvm<"float*">, !llvm<"<3 x i32>">) -> !llvm<"<3 x float*>">
+// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : !llvm<"<3 x float>">, !llvm<"<3 x i1>"> into !llvm<"<3 x float*>">
+// CHECK: llvm.return
diff --git a/mlir/test/Dialect/Linalg/invalid.mlir b/mlir/test/Dialect/Linalg/invalid.mlir
index 585dc36dcaa85..79d61d8d78921 100644
--- a/mlir/test/Dialect/Linalg/invalid.mlir
+++ b/mlir/test/Dialect/Linalg/invalid.mlir
@@ -106,7 +106,7 @@ func @generic_mismatched_num_returns(%arg0: memref<f32>) {
 // -----
 
 func @generic_symbol_in_map(%arg0: memref<i32>) {
-  // expected-error @+1 {{op expected indexing_map #0 to have no symbols}}
+  // expected-error @+1 {{expected the number of symbols in indexing_map #0 to match target rank}}
   linalg.generic {
     args_in = 0,
     args_out = 1,
@@ -120,6 +120,22 @@ func @generic_symbol_in_map(%arg0: memref<i32>) {
 
 // -----
 
+func @generic_symbol_source_out_of_range(%arg0: memref<i32>) {
+  // expected-error @+1 {{symbol_source index out of range}}
+  linalg.generic {
+    args_in = 0,
+    args_out = 1,
+    indexing_maps =  [ affine_map<()[N] -> (0)> ],
+    iterator_types = ["parallel"],
+    symbol_source = 1
+  } %arg0 {
+    ^bb(%i : i32):
+    linalg.yield %i : i32
+  }: memref<i32>
+}
+
+// -----
+
 func @generic_wrong_dim_in_map(%arg0: memref<1xi32>) {
   // expected-error @+1 {{op expected indexing_map #0 to have 1 dim(s) to match the number of loops}}
   linalg.generic {
@@ -412,6 +428,13 @@ func @generic_result_0_element_type(%arg0: memref<?xf32>) {
 
 // -----
 
+func @conv_rank_limit(%arg0: memref<?xf32>, %arg1: memref<?xf32>, %arg2: memref<?xf32>) {
+  // expected-error @+1 {{expects memref ranks to be greater than 2}}
+  linalg.conv(%arg0, %arg1, %arg2) : memref<?xf32>, memref<?xf32>, memref<?xf32>
+}
+
+// -----
+
 // expected-error @+1 {{unknown Linalg type}}
 !invalid_type = type !linalg.unknown
 
diff --git a/mlir/test/Dialect/Linalg/loops.mlir b/mlir/test/Dialect/Linalg/loops.mlir
index f03129c4d8beb..30bb90bdd43a1 100644
--- a/mlir/test/Dialect/Linalg/loops.mlir
+++ b/mlir/test/Dialect/Linalg/loops.mlir
@@ -14,6 +14,9 @@
 // CHECKLOOP-DAG: #[[$stride2Dilation1:.*]] = affine_map<(d0, d1) -> (d0 * 2 + d1)>
 // CHECKLOOP-DAG: #[[$stride2Dilation4:.*]] = affine_map<(d0, d1) -> (d0 * 2 + d1 * 4)>
 // CHECKLOOP-DAG: #[[$stride3Dilation5:.*]] = affine_map<(d0, d1) -> (d0 * 3 + d1 * 5)>
+// CHECKLOOP-DAG: #[[$convLowerBound:.*]] = affine_map<()[s0] -> (s0 floordiv 2)>
+// CHECKLOOP-DAG: #[[$convUpperBound:.*]] = affine_map<()[s0, s1] -> (s1 + s0 floordiv 2 - s0 + 1)>
+// CHECKLOOP-DAG: #[[$convMap:.*]] = affine_map<(d0, d1)[s0] -> (d0 + d1 - s0 floordiv 2)>
 
 // CHECKPARALLEL-DAG: #[[$strided1D:.*]] = affine_map<(d0)[s0] -> (d0 + s0)>
 // CHECKPARALLEL-DAG: #[[$strided2D:.*]] = affine_map<(d0, d1)[s0, s1] -> (d0 * s1 + s0 + d1)>
@@ -25,6 +28,9 @@
 // CHECKPARALLEL-DAG: #[[$stride2Dilation1:.*]] = affine_map<(d0, d1) -> (d0 * 2 + d1)>
 // CHECKPARALLEL-DAG: #[[$stride2Dilation4:.*]] = affine_map<(d0, d1) -> (d0 * 2 + d1 * 4)>
 // CHECKPARALLEL-DAG: #[[$stride3Dilation5:.*]] = affine_map<(d0, d1) -> (d0 * 3 + d1 * 5)>
+// CHECKPARALLEL-DAG: #[[$convLowerBound:.*]] = affine_map<()[s0] -> (s0 floordiv 2)>
+// CHECKPARALLEL-DAG: #[[$convUpperBound:.*]] = affine_map<()[s0, s1] -> (s1 + s0 floordiv 2 - s0 + 1)>
+// CHECKPARALLEL-DAG: #[[$convMap:.*]] = affine_map<(d0, d1)[s0] -> (d0 + d1 - s0 floordiv 2)>
 
 
 func @matmul(%arg0: memref<?xi8>, %M: index, %N: index, %K: index) {
@@ -910,3 +916,371 @@ func @named_batch_matmul(%A: memref<?x?x?xf32>, %B: memref<?x?x?xf32>, %C: memre
 //       CHECKPARALLEL:       %[[inc:.*]] = mulf %[[va]], %[[vb]] : f32
 //       CHECKPARALLEL:       %[[res:.*]] = addf %[[vc]], %[[inc]] : f32
 //       CHECKPARALLEL:       store %[[res]], %[[mC]][%[[b]], %[[m]], %[[n]]] : memref<?x?x?xf32>
+
+#conv_1d_accesses = [
+  affine_map<(m, n)[s0] -> (m + n - s0 floordiv 2)>, // in
+  affine_map<(m, n)[s0] -> (n)>, // filter
+  affine_map<(m, n)[s0] -> (m)> // out
+]
+
+#conv_1d_trait = {
+  args_in = 2,
+  args_out = 1,
+  doc = "C(m) += A(m) * B(n)",
+  indexing_maps = #conv_1d_accesses,
+  library_call  = "linalg_conv_1d",
+  n_views = [2, 1],
+  iterator_types = ["parallel", "parallel"],
+  symbol_source = 1
+}
+
+func @conv1d(%in : memref<?xf32>, %filter : memref<?xf32>, %out :  memref<?xf32>) -> () {
+  linalg.generic #conv_1d_trait %in, %filter, %out {
+    ^bb0(%a: f32, %b: f32, %c: f32) :
+      %d = mulf %a, %b : f32
+      %e = addf %c, %d : f32
+      linalg.yield %e : f32
+  } : memref<?xf32>,
+      memref<?xf32>,
+      memref<?xf32>
+  return
+}
+
+// CHECKLOOP-LABEL: @conv1d
+//  CHECKLOOP-SAME: %[[arg0:[a-zA-Z0-9]+]]: memref<?xf32>
+//  CHECKLOOP-SAME: %[[arg1:[a-zA-Z0-9]+]]: memref<?xf32>
+//  CHECKLOOP-SAME: %[[arg2:[a-zA-Z0-9]+]]: memref<?xf32>
+//       CHECKLOOP: %[[c0:.*]] = constant 0 : index
+//       CHECKLOOP: %[[dim0:.*]] = dim %[[arg0]], %[[c0]] : memref<?xf32>
+//       CHECKLOOP: %[[dim1:.*]] = dim %[[arg1]], %[[c0]] : memref<?xf32>
+//       CHECKLOOP: %[[lowerBound:.*]] = affine.apply #[[$convLowerBound]]()[%[[dim1]]]
+//       CHECKLOOP: %[[upperBound:.*]] = affine.apply #[[$convUpperBound]]()[%[[dim1]], %[[dim0]]]
+//       CHECKLOOP: scf.for %[[b:.*]] = %[[lowerBound]] to %[[upperBound]] step %{{.*}} {
+//       CHECKLOOP:   scf.for %[[m:.*]] = %{{.*}} to %[[dim1]] step %{{.*}} {
+//       CHECKLOOP:     %[[dim2:.*]] = dim %[[arg1]], %[[c0]] : memref<?xf32>
+//       CHECKLOOP:     %[[aff:.*]] = affine.apply #[[$convMap]](%{{.*}}, %{{.*}})[%[[dim2]]]
+//       CHECKLOOP:     %[[va:.*]] = load %[[arg0]][%[[aff]]] : memref<?xf32>
+//       CHECKLOOP:     %[[vb:.*]] = load %[[arg1]][%[[m]]] : memref<?xf32>
+//       CHECKLOOP:     %[[vc:.*]] = load %[[arg2]][%[[b]]] : memref<?xf32>
+//       CHECKLOOP:     %[[inc:.*]] = mulf %[[va]], %[[vb]] : f32
+//       CHECKLOOP:     %[[res:.*]] = addf %[[vc]], %[[inc]] : f32
+//       CHECKLOOP:     store %[[res]], %[[arg2]][%[[b]]] : memref<?xf32>
+
+// CHECKPARALLEL-LABEL: @conv1d
+//  CHECKPARALLEL-SAME: %[[arg0:[a-zA-Z0-9]+]]: memref<?xf32>
+//  CHECKPARALLEL-SAME: %[[arg1:[a-zA-Z0-9]+]]: memref<?xf32>
+//  CHECKPARALLEL-SAME: %[[arg2:[a-zA-Z0-9]+]]: memref<?xf32>
+//       CHECKPARALLEL: %[[c0:.*]] = constant 0 : index
+//       CHECKPARALLEL: %[[dim0:.*]] = dim %[[arg0]], %[[c0]] : memref<?xf32>
+//       CHECKPARALLEL: %[[dim1:.*]] = dim %[[arg1]], %[[c0]] : memref<?xf32>
+//       CHECKPARALLEL: %[[lowerBound:.*]] = affine.apply #[[$convLowerBound]]()[%[[dim1]]]
+//       CHECKPARALLEL: %[[upperBound:.*]] = affine.apply #[[$convUpperBound]]()[%[[dim1]], %[[dim0]]]
+//       CHECKPARALLEL: scf.parallel (%[[b:.*]], %[[m:.*]]) = (%[[lowerBound]], %{{.*}}) to (%[[upperBound]], %[[dim1]]) step ({{.*}}) {
+//       CHECKPARALLEL:   %[[dim2:.*]] = dim %[[arg1]], %[[c0]] : memref<?xf32>
+//       CHECKPARALLEL:   %[[aff:.*]] = affine.apply #[[$convMap]](%{{.*}}, %{{.*}})[%[[dim2]]]
+//       CHECKPARALLEL:   %[[va:.*]] = load %[[arg0]][%[[aff]]] : memref<?xf32>
+//       CHECKPARALLEL:   %[[vb:.*]] = load %[[arg1]][%[[m]]] : memref<?xf32>
+//       CHECKPARALLEL:   %[[vc:.*]] = load %[[arg2]][%[[b]]] : memref<?xf32>
+//       CHECKPARALLEL:   %[[inc:.*]] = mulf %[[va]], %[[vb]] : f32
+//       CHECKPARALLEL:   %[[res:.*]] = addf %[[vc]], %[[inc]] : f32
+//       CHECKPARALLEL:   store %[[res]], %[[arg2]][%[[b]]] : memref<?xf32>
+
+#conv_2d_accesses = [
+  affine_map<(m, n, m1, n1)[s0, s1] -> (m + m1 - s0 floordiv 2, n + n1 - s1 floordiv 2)>, // in
+  affine_map<(m, n, m1, n1)[s0, s1] -> (m1, n1)>, // filter
+  affine_map<(m, n, m1, n1)[s0, s1] -> (m, n)> // out
+]
+
+#conv_2d_trait = {
+  args_in = 2,
+  args_out = 1,
+  doc = "C(m,n) += A(m,n) * B(m1,n1)",
+  indexing_maps = #conv_2d_accesses,
+  library_call  = "linalg_conv_2d",
+  n_views = [2, 1],
+  iterator_types = ["parallel", "parallel", "parallel", "parallel"],
+  symbol_source = 1
+}
+
+func @conv2d(%in : memref<?x?xf32>, %filter : memref<?x?xf32>, %out :  memref<?x?xf32>) -> () {
+  linalg.generic #conv_2d_trait %in, %filter, %out {
+    ^bb0(%a: f32, %b: f32, %c: f32) :
+      %d = mulf %a, %b : f32
+      %e = addf %c, %d : f32
+      linalg.yield %e : f32
+  } : memref<?x?xf32>,
+      memref<?x?xf32>,
+      memref<?x?xf32>
+  return
+}
+
+// CHECKLOOP-LABEL: @conv2d
+//  CHECKLOOP-SAME: %[[arg0:[a-zA-Z0-9]+]]: memref<?x?xf32>
+//  CHECKLOOP-SAME: %[[arg1:[a-zA-Z0-9]+]]: memref<?x?xf32>
+//  CHECKLOOP-SAME: %[[arg2:[a-zA-Z0-9]+]]: memref<?x?xf32>
+//       CHECKLOOP: %[[c0:.*]] = constant 0 : index
+//       CHECKLOOP: %[[c1:.*]] = constant 1 : index
+//       CHECKLOOP: %[[dim0:.*]] = dim %[[arg0]], %[[c0]] : memref<?x?xf32>
+//       CHECKLOOP: %[[dim1:.*]] = dim %[[arg0]], %[[c1]] : memref<?x?xf32>
+//       CHECKLOOP: %[[dim2:.*]] = dim %[[arg1]], %[[c0]] : memref<?x?xf32>
+//       CHECKLOOP: %[[dim3:.*]] = dim %[[arg1]], %[[c1]] : memref<?x?xf32>
+//       CHECKLOOP: %[[lowerBound1:.*]] = affine.apply #[[$convLowerBound]]()[%[[dim2]]]
+//       CHECKLOOP: %[[upperBound1:.*]] = affine.apply #[[$convUpperBound]]()[%[[dim2]], %[[dim0]]]
+//       CHECKLOOP: %[[lowerBound2:.*]] = affine.apply #[[$convLowerBound]]()[%[[dim3]]]
+//       CHECKLOOP: %[[upperBound2:.*]] = affine.apply #[[$convUpperBound]]()[%[[dim3]], %[[dim1]]]
+//       CHECKLOOP: scf.for %[[i0:.*]] = %[[lowerBound1]] to %[[upperBound1]] step %{{.*}} {
+//       CHECKLOOP:   scf.for %[[i1:.*]] = %[[lowerBound2]] to %[[upperBound2]] step %{{.*}} {
+//       CHECKLOOP:     scf.for %[[i2:.*]] = %{{.*}} to %[[dim2]] step %{{.*}} {
+//       CHECKLOOP:       scf.for %[[i3:.*]] = %{{.*}} to %[[dim3]] step %{{.*}} {
+//       CHECKLOOP:         %[[dim4:.*]] = dim %[[arg1]], %[[c0]] : memref<?x?xf32>
+//       CHECKLOOP:         %[[dim5:.*]] = dim %[[arg1]], %[[c1]] : memref<?x?xf32>
+//       CHECKLOOP:         %[[aff1:.*]] = affine.apply #[[$convMap]](%{{.*}}, %{{.*}})[%[[dim4]]]
+//       CHECKLOOP:         %[[aff2:.*]] = affine.apply #[[$convMap]](%{{.*}}, %{{.*}})[%[[dim5]]]
+//       CHECKLOOP:         %[[va:.*]] = load %[[arg0]][%[[aff1]], %[[aff2]]] : memref<?x?xf32>
+//       CHECKLOOP:         %[[vb:.*]] = load %[[arg1]][%[[i2]], %[[i3]]] : memref<?x?xf32>
+//       CHECKLOOP:         %[[vc:.*]] = load %[[arg2]][%[[i0]], %[[i1]]] : memref<?x?xf32>
+//       CHECKLOOP:         %[[inc:.*]] = mulf %[[va]], %[[vb]] : f32
+//       CHECKLOOP:         %[[res:.*]] = addf %[[vc]], %[[inc]] : f32
+//       CHECKLOOP:         store %[[res]], %[[arg2]][%[[i0]], %[[i1]]] : memref<?x?xf32>
+
+// CHECKPARALLEL-LABEL: @conv2d
+//  CHECKPARALLEL-SAME: %[[arg0:[a-zA-Z0-9]+]]: memref<?x?xf32>
+//  CHECKPARALLEL-SAME: %[[arg1:[a-zA-Z0-9]+]]: memref<?x?xf32>
+//  CHECKPARALLEL-SAME: %[[arg2:[a-zA-Z0-9]+]]: memref<?x?xf32>
+//       CHECKPARALLEL: %[[c0:.*]] = constant 0 : index
+//       CHECKPARALLEL: %[[c1:.*]] = constant 1 : index
+//       CHECKPARALLEL: %[[dim0:.*]] = dim %[[arg0]], %[[c0]] : memref<?x?xf32>
+//       CHECKPARALLEL: %[[dim1:.*]] = dim %[[arg0]], %[[c1]] : memref<?x?xf32>
+//       CHECKPARALLEL: %[[dim2:.*]] = dim %[[arg1]], %[[c0]] : memref<?x?xf32>
+//       CHECKPARALLEL: %[[dim3:.*]] = dim %[[arg1]], %[[c1]] : memref<?x?xf32>
+//       CHECKPARALLEL: %[[lowerBound1:.*]] = affine.apply #[[$convLowerBound]]()[%[[dim2]]]
+//       CHECKPARALLEL: %[[upperBound1:.*]] = affine.apply #[[$convUpperBound]]()[%[[dim2]], %[[dim0]]]
+//       CHECKPARALLEL: %[[lowerBound2:.*]] = affine.apply #[[$convLowerBound]]()[%[[dim3]]]
+//       CHECKPARALLEL: %[[upperBound2:.*]] = affine.apply #[[$convUpperBound]]()[%[[dim3]], %[[dim1]]]
+//       CHECKPARALLEL: scf.parallel (%[[i0:.*]], %[[i1:.*]], %[[i2:.*]], %[[i3:.*]]) = (%[[lowerBound1]], %[[lowerBound2]], %{{.*}}, %{{.*}}) to (%[[upperBound1]], %[[upperBound2]], %[[dim2]], %[[dim3]]) step ({{.*}}) {
+//       CHECKPARALLEL:   %[[dim4:.*]] = dim %[[arg1]], %[[c0]] : memref<?x?xf32>
+//       CHECKPARALLEL:   %[[dim5:.*]] = dim %[[arg1]], %[[c1]] : memref<?x?xf32>
+//       CHECKPARALLEL:   %[[aff1:.*]] = affine.apply #[[$convMap]](%{{.*}}, %{{.*}})[%[[dim4]]]
+//       CHECKPARALLEL:   %[[aff2:.*]] = affine.apply #[[$convMap]](%{{.*}}, %{{.*}})[%[[dim5]]]
+//       CHECKPARALLEL:   %[[va:.*]] = load %[[arg0]][%[[aff1]], %[[aff2]]] : memref<?x?xf32>
+//       CHECKPARALLEL:   %[[vb:.*]] = load %[[arg1]][%[[i2]], %[[i3]]] : memref<?x?xf32>
+//       CHECKPARALLEL:   %[[vc:.*]] = load %[[arg2]][%[[i0]], %[[i1]]] : memref<?x?xf32>
+//       CHECKPARALLEL:   %[[inc:.*]] = mulf %[[va]], %[[vb]] : f32
+//       CHECKPARALLEL:   %[[res:.*]] = addf %[[vc]], %[[inc]] : f32
+//       CHECKPARALLEL:   store %[[res]], %[[arg2]][%[[i0]], %[[i1]]] : memref<?x?xf32>
+
+#conv_3d_accesses = [
+  affine_map<(m, n, k, m1, n1, k1)[s0, s1, s2] -> (m + m1 - s0 floordiv 2, n + n1 - s1 floordiv 2, k + k1 - s2 floordiv 2)>, // in
+  affine_map<(m, n, k, m1, n1, k1)[s0, s1, s2] -> (m1, n1, k1)>, // filter
+  affine_map<(m, n, k, m1, n1, k1)[s0, s1, s2] -> (m, n, k)> // out
+]
+
+#conv_3d_trait = {
+  args_in = 2,
+  args_out = 1,
+  doc = "C(m,n,k) += A(m,n,k) * B(m1,n1,k1)",
+  indexing_maps = #conv_3d_accesses,
+  library_call  = "linalg_conv_3d",
+  n_views = [2, 1],
+  iterator_types = ["parallel", "parallel", "parallel", "parallel", "parallel", "parallel"],
+  symbol_source = 1
+}
+
+func @conv3d(%in : memref<?x?x?xf32>, %filter : memref<?x?x?xf32>, %out :  memref<?x?x?xf32>) -> () {
+  linalg.generic #conv_3d_trait %in, %filter, %out {
+    ^bb0(%a: f32, %b: f32, %c: f32) :
+      %d = mulf %a, %b : f32
+      %e = addf %c, %d : f32
+      linalg.yield %e : f32
+  } : memref<?x?x?xf32>,
+      memref<?x?x?xf32>,
+      memref<?x?x?xf32>
+  return
+}
+
+// CHECKLOOP-LABEL: @conv3d
+//  CHECKLOOP-SAME: %[[arg0:[a-zA-Z0-9]+]]: memref<?x?x?xf32>
+//  CHECKLOOP-SAME: %[[arg1:[a-zA-Z0-9]+]]: memref<?x?x?xf32>
+//  CHECKLOOP-SAME: %[[arg2:[a-zA-Z0-9]+]]: memref<?x?x?xf32>
+//       CHECKLOOP: %[[c0:.*]] = constant 0 : index
+//       CHECKLOOP: %[[c1:.*]] = constant 1 : index
+//       CHECKLOOP: %[[c2:.*]] = constant 2 : index
+//       CHECKLOOP: %[[dim0:.*]] = dim %[[arg0]], %[[c0]] : memref<?x?x?xf32>
+//       CHECKLOOP: %[[dim1:.*]] = dim %[[arg0]], %[[c1]] : memref<?x?x?xf32>
+//       CHECKLOOP: %[[dim2:.*]] = dim %[[arg0]], %[[c2]] : memref<?x?x?xf32>
+//       CHECKLOOP: %[[dim3:.*]] = dim %[[arg1]], %[[c0]] : memref<?x?x?xf32>
+//       CHECKLOOP: %[[dim4:.*]] = dim %[[arg1]], %[[c1]] : memref<?x?x?xf32>
+//       CHECKLOOP: %[[dim5:.*]] = dim %[[arg1]], %[[c2]] : memref<?x?x?xf32>
+//       CHECKLOOP: %[[lowerBound1:.*]] = affine.apply #[[$convLowerBound]]()[%[[dim3]]]
+//       CHECKLOOP: %[[upperBound1:.*]] = affine.apply #[[$convUpperBound]]()[%[[dim3]], %[[dim0]]]
+//       CHECKLOOP: %[[lowerBound2:.*]] = affine.apply #[[$convLowerBound]]()[%[[dim4]]]
+//       CHECKLOOP: %[[upperBound2:.*]] = affine.apply #[[$convUpperBound]]()[%[[dim4]], %[[dim1]]]
+//       CHECKLOOP: %[[lowerBound3:.*]] = affine.apply #[[$convLowerBound]]()[%[[dim5]]]
+//       CHECKLOOP: %[[upperBound3:.*]] = affine.apply #[[$convUpperBound]]()[%[[dim5]], %[[dim2]]]
+//       CHECKLOOP: scf.for %[[i0:.*]] = %[[lowerBound1]] to %[[upperBound1]] step %{{.*}} {
+//       CHECKLOOP:   scf.for %[[i1:.*]] = %[[lowerBound2]] to %[[upperBound2]] step %{{.*}} {
+//       CHECKLOOP:     scf.for %[[i2:.*]] = %[[lowerBound3]] to %[[upperBound3]] step %{{.*}} {
+//       CHECKLOOP:       scf.for %[[i3:.*]] = %{{.*}} to %[[dim3]] step %{{.*}} {
+//       CHECKLOOP:         scf.for %[[i4:.*]] = %{{.*}} to %[[dim4]] step %{{.*}} {
+//       CHECKLOOP:           scf.for %[[i5:.*]] = %{{.*}} to %[[dim5]] step %{{.*}} {
+//       CHECKLOOP:             %[[dim6:.*]] = dim %[[arg1]], %[[c0]] : memref<?x?x?xf32>
+//       CHECKLOOP:             %[[dim7:.*]] = dim %[[arg1]], %[[c1]] : memref<?x?x?xf32>
+//       CHECKLOOP:             %[[dim8:.*]] = dim %[[arg1]], %[[c2]] : memref<?x?x?xf32>
+//       CHECKLOOP:             %[[aff1:.*]] = affine.apply #[[$convMap]](%{{.*}}, %{{.*}})[%[[dim6]]]
+//       CHECKLOOP:             %[[aff2:.*]] = affine.apply #[[$convMap]](%{{.*}}, %{{.*}})[%[[dim7]]]
+//       CHECKLOOP:             %[[aff3:.*]] = affine.apply #[[$convMap]](%{{.*}}, %{{.*}})[%[[dim8]]]
+//       CHECKLOOP:             %[[va:.*]] = load %[[arg0]][%[[aff1]], %[[aff2]], %[[aff3]]] : memref<?x?x?xf32>
+//       CHECKLOOP:             %[[vb:.*]] = load %[[arg1]][%[[i3]], %[[i4]], %[[i5]]] : memref<?x?x?xf32>
+//       CHECKLOOP:             %[[vc:.*]] = load %[[arg2]][%[[i0]], %[[i1]], %[[i2]]] : memref<?x?x?xf32>
+//       CHECKLOOP:             %[[inc:.*]] = mulf %[[va]], %[[vb]] : f32
+//       CHECKLOOP:             %[[res:.*]] = addf %[[vc]], %[[inc]] : f32
+//       CHECKLOOP:             store %[[res]], %[[arg2]][%[[i0]], %[[i1]], %[[i2]]] : memref<?x?x?xf32>
+
+// CHECKPARALLEL-LABEL: @conv3d
+//  CHECKPARALLEL-SAME: %[[arg0:[a-zA-Z0-9]+]]: memref<?x?x?xf32>
+//  CHECKPARALLEL-SAME: %[[arg1:[a-zA-Z0-9]+]]: memref<?x?x?xf32>
+//  CHECKPARALLEL-SAME: %[[arg2:[a-zA-Z0-9]+]]: memref<?x?x?xf32>
+//       CHECKPARALLEL: %[[c0:.*]] = constant 0 : index
+//       CHECKPARALLEL: %[[c1:.*]] = constant 1 : index
+//       CHECKPARALLEL: %[[c2:.*]] = constant 2 : index
+//       CHECKPARALLEL: %[[dim0:.*]] = dim %[[arg0]], %[[c0]] : memref<?x?x?xf32>
+//       CHECKPARALLEL: %[[dim1:.*]] = dim %[[arg0]], %[[c1]] : memref<?x?x?xf32>
+//       CHECKPARALLEL: %[[dim2:.*]] = dim %[[arg0]], %[[c2]] : memref<?x?x?xf32>
+//       CHECKPARALLEL: %[[dim3:.*]] = dim %[[arg1]], %[[c0]] : memref<?x?x?xf32>
+//       CHECKPARALLEL: %[[dim4:.*]] = dim %[[arg1]], %[[c1]] : memref<?x?x?xf32>
+//       CHECKPARALLEL: %[[dim5:.*]] = dim %[[arg1]], %[[c2]] : memref<?x?x?xf32>
+//       CHECKPARALLEL: %[[lowerBound1:.*]] = affine.apply #[[$convLowerBound]]()[%[[dim3]]]
+//       CHECKPARALLEL: %[[upperBound1:.*]] = affine.apply #[[$convUpperBound]]()[%[[dim3]], %[[dim0]]]
+//       CHECKPARALLEL: %[[lowerBound2:.*]] = affine.apply #[[$convLowerBound]]()[%[[dim4]]]
+//       CHECKPARALLEL: %[[upperBound2:.*]] = affine.apply #[[$convUpperBound]]()[%[[dim4]], %[[dim1]]]
+//       CHECKPARALLEL: %[[lowerBound3:.*]] = affine.apply #[[$convLowerBound]]()[%[[dim5]]]
+//       CHECKPARALLEL: %[[upperBound3:.*]] = affine.apply #[[$convUpperBound]]()[%[[dim5]], %[[dim2]]]
+//       CHECKPARALLEL: scf.parallel (%[[i0:.*]], %[[i1:.*]], %[[i2:.*]], %[[i3:.*]], %[[i4:.*]], %[[i5:.*]]) = (%[[lowerBound1]], %[[lowerBound2]], %[[lowerBound3]], %{{.*}}, %{{.*}}, %{{.*}}) to (%[[upperBound1]], %[[upperBound2]], %[[upperBound3]], %[[dim3]], %[[dim4]], %[[dim5]]) step ({{.*}}) {
+//       CHECKPARALLEL:   %[[dim6:.*]] = dim %[[arg1]], %[[c0]] : memref<?x?x?xf32>
+//       CHECKPARALLEL:   %[[dim7:.*]] = dim %[[arg1]], %[[c1]] : memref<?x?x?xf32>
+//       CHECKPARALLEL:   %[[dim8:.*]] = dim %[[arg1]], %[[c2]] : memref<?x?x?xf32>
+//       CHECKPARALLEL:   %[[aff1:.*]] = affine.apply #[[$convMap]](%{{.*}}, %{{.*}})[%[[dim6]]]
+//       CHECKPARALLEL:   %[[aff2:.*]] = affine.apply #[[$convMap]](%{{.*}}, %{{.*}})[%[[dim7]]]
+//       CHECKPARALLEL:   %[[aff3:.*]] = affine.apply #[[$convMap]](%{{.*}}, %{{.*}})[%[[dim8]]]
+//       CHECKPARALLEL:   %[[va:.*]] = load %[[arg0]][%[[aff1]], %[[aff2]], %[[aff3]]] : memref<?x?x?xf32>
+//       CHECKPARALLEL:   %[[vb:.*]] = load %[[arg1]][%[[i3]], %[[i4]], %[[i5]]] : memref<?x?x?xf32>
+//       CHECKPARALLEL:   %[[vc:.*]] = load %[[arg2]][%[[i0]], %[[i1]], %[[i2]]] : memref<?x?x?xf32>
+//       CHECKPARALLEL:   %[[inc:.*]] = mulf %[[va]], %[[vb]] : f32
+//       CHECKPARALLEL:   %[[res:.*]] = addf %[[vc]], %[[inc]] : f32
+//       CHECKPARALLEL:   store %[[res]], %[[arg2]][%[[i0]], %[[i1]], %[[i2]]] : memref<?x?x?xf32>
+
+#conv_4d_accesses = [
+  affine_map<(m, n, k, l, m1, n1, k1, l1)[s0, s1, s2, s3] -> (m + m1 - s0 floordiv 2, n + n1 - s1 floordiv 2, k + k1 - s2 floordiv 2, l + l1 - s3 floordiv 2)>, // in
+  affine_map<(m, n, k, l, m1, n1, k1, l1)[s0, s1, s2, s3] -> (m1, n1, k1, l1)>, // filter
+  affine_map<(m, n, k, l, m1, n1, k1, l1)[s0, s1, s2, s3] -> (m, n, k, l)> // out
+]
+
+#conv_4d_trait = {
+  args_in = 2,
+  args_out = 1,
+  doc = "C(m,n,k,l) += A(m,n,k,l) * B(m1,n1,k1,l1)",
+  indexing_maps = #conv_4d_accesses,
+  library_call  = "linalg_conv_4d",
+  n_views = [2, 1],
+  iterator_types = ["parallel", "parallel", "parallel", "parallel", "parallel", "parallel", "parallel", "parallel"],
+  symbol_source = 1
+}
+
+func @conv4d(%in : memref<?x?x?x?xf32>, %filter : memref<?x?x?x?xf32>, %out :  memref<?x?x?x?xf32>) -> () {
+  linalg.generic #conv_4d_trait %in, %filter, %out {
+    ^bb0(%a: f32, %b: f32, %c: f32) :
+      %d = mulf %a, %b : f32
+      %e = addf %c, %d : f32
+      linalg.yield %e : f32
+  } : memref<?x?x?x?xf32>,
+      memref<?x?x?x?xf32>,
+      memref<?x?x?x?xf32>
+  return
+}
+
+// CHECKLOOP-LABEL: @conv4d
+//  CHECKLOOP-SAME: %[[arg0:[a-zA-Z0-9]+]]: memref<?x?x?x?xf32>
+//  CHECKLOOP-SAME: %[[arg1:[a-zA-Z0-9]+]]: memref<?x?x?x?xf32>
+//  CHECKLOOP-SAME: %[[arg2:[a-zA-Z0-9]+]]: memref<?x?x?x?xf32>
+//       CHECKLOOP: %[[c0:.*]] = constant 0 : index
+//       CHECKLOOP: %[[c1:.*]] = constant 1 : index
+//       CHECKLOOP: %[[c2:.*]] = constant 2 : index
+//       CHECKLOOP: %[[c3:.*]] = constant 3 : index
+//       CHECKLOOP: %[[dim0:.*]] = dim %[[arg0]], %[[c0]] : memref<?x?x?x?xf32>
+//       CHECKLOOP: %[[dim1:.*]] = dim %[[arg0]], %[[c1]] : memref<?x?x?x?xf32>
+//       CHECKLOOP: %[[dim2:.*]] = dim %[[arg0]], %[[c2]] : memref<?x?x?x?xf32>
+//       CHECKLOOP: %[[dim3:.*]] = dim %[[arg0]], %[[c3]] : memref<?x?x?x?xf32>
+//       CHECKLOOP: %[[dim4:.*]] = dim %[[arg1]], %[[c0]] : memref<?x?x?x?xf32>
+//       CHECKLOOP: %[[dim5:.*]] = dim %[[arg1]], %[[c1]] : memref<?x?x?x?xf32>
+//       CHECKLOOP: %[[dim6:.*]] = dim %[[arg1]], %[[c2]] : memref<?x?x?x?xf32>
+//       CHECKLOOP: %[[dim7:.*]] = dim %[[arg1]], %[[c3]] : memref<?x?x?x?xf32>
+//       CHECKLOOP: %[[lowerBound1:.*]] = affine.apply #[[$convLowerBound]]()[%[[dim4]]]
+//       CHECKLOOP: %[[upperBound1:.*]] = affine.apply #[[$convUpperBound]]()[%[[dim4]], %[[dim0]]]
+//       CHECKLOOP: %[[lowerBound2:.*]] = affine.apply #[[$convLowerBound]]()[%[[dim5]]]
+//       CHECKLOOP: %[[upperBound2:.*]] = affine.apply #[[$convUpperBound]]()[%[[dim5]], %[[dim1]]]
+//       CHECKLOOP: %[[lowerBound3:.*]] = affine.apply #[[$convLowerBound]]()[%[[dim6]]]
+//       CHECKLOOP: %[[upperBound3:.*]] = affine.apply #[[$convUpperBound]]()[%[[dim6]], %[[dim2]]]
+//       CHECKLOOP: %[[lowerBound4:.*]] = affine.apply #[[$convLowerBound]]()[%[[dim7]]]
+//       CHECKLOOP: %[[upperBound4:.*]] = affine.apply #[[$convUpperBound]]()[%[[dim7]], %[[dim3]]]
+//       CHECKLOOP: scf.for %[[i0:.*]] = %[[lowerBound1]] to %[[upperBound1]] step %{{.*}} {
+//       CHECKLOOP:   scf.for %[[i1:.*]] = %[[lowerBound2]] to %[[upperBound2]] step %{{.*}} {
+//       CHECKLOOP:     scf.for %[[i2:.*]] = %[[lowerBound3]] to %[[upperBound3]] step %{{.*}} {
+//       CHECKLOOP:       scf.for %[[i3:.*]] = %[[lowerBound4]] to %[[upperBound4]] step %{{.*}} {
+//       CHECKLOOP:         scf.for %[[i4:.*]] = %{{.*}} to %[[dim4]] step %{{.*}} {
+//       CHECKLOOP:           scf.for %[[i5:.*]] = %{{.*}} to %[[dim5]] step %{{.*}} {
+//       CHECKLOOP:             scf.for %[[i6:.*]] = %{{.*}} to %[[dim6]] step %{{.*}} {
+//       CHECKLOOP:               scf.for %[[i7:.*]] = %{{.*}} to %[[dim7]] step %{{.*}} {
+//       CHECKLOOP:                 %[[dim8:.*]] = dim %[[arg1]], %[[c0]] : memref<?x?x?x?xf32>
+//       CHECKLOOP:                 %[[dim9:.*]] = dim %[[arg1]], %[[c1]] : memref<?x?x?x?xf32>
+//       CHECKLOOP:                 %[[dim10:.*]] = dim %[[arg1]], %[[c2]] : memref<?x?x?x?xf32>
+//       CHECKLOOP:                 %[[dim11:.*]] = dim %[[arg1]], %[[c3]] : memref<?x?x?x?xf32>
+//       CHECKLOOP:                 %[[aff1:.*]] = affine.apply #[[$convMap]](%{{.*}}, %{{.*}})[%[[dim8]]]
+//       CHECKLOOP:                 %[[aff2:.*]] = affine.apply #[[$convMap]](%{{.*}}, %{{.*}})[%[[dim9]]]
+//       CHECKLOOP:                 %[[aff3:.*]] = affine.apply #[[$convMap]](%{{.*}}, %{{.*}})[%[[dim10]]]
+//       CHECKLOOP:                 %[[aff4:.*]] = affine.apply #[[$convMap]](%{{.*}}, %{{.*}})[%[[dim11]]]
+//       CHECKLOOP:                 %[[va:.*]] = load %[[arg0]][%[[aff1]], %[[aff2]], %[[aff3]], %[[aff4]]] : memref<?x?x?x?xf32>
+//       CHECKLOOP:                 %[[vb:.*]] = load %[[arg1]][%[[i4]], %[[i5]], %[[i6]], %[[i7]]] : memref<?x?x?x?xf32>
+//       CHECKLOOP:                 %[[vc:.*]] = load %[[arg2]][%[[i0]], %[[i1]], %[[i2]], %[[i3]]] : memref<?x?x?x?xf32>
+//       CHECKLOOP:                 %[[inc:.*]] = mulf %[[va]], %[[vb]] : f32
+//       CHECKLOOP:                 %[[res:.*]] = addf %[[vc]], %[[inc]] : f32
+//       CHECKLOOP:                 store %[[res]], %[[arg2]][%[[i0]], %[[i1]], %[[i2]], %[[i3]]] : memref<?x?x?x?xf32>
+
+// CHECKPARALLEL-LABEL: @conv4d
+//  CHECKPARALLEL-SAME: %[[arg0:[a-zA-Z0-9]+]]: memref<?x?x?x?xf32>
+//  CHECKPARALLEL-SAME: %[[arg1:[a-zA-Z0-9]+]]: memref<?x?x?x?xf32>
+//  CHECKPARALLEL-SAME: %[[arg2:[a-zA-Z0-9]+]]: memref<?x?x?x?xf32>
+//       CHECKPARALLEL: %[[c0:.*]] = constant 0 : index
+//       CHECKPARALLEL: %[[c1:.*]] = constant 1 : index
+//       CHECKPARALLEL: %[[c2:.*]] = constant 2 : index
+//       CHECKPARALLEL: %[[c3:.*]] = constant 3 : index
+//       CHECKPARALLEL: %[[dim0:.*]] = dim %[[arg0]], %[[c0]] : memref<?x?x?x?xf32>
+//       CHECKPARALLEL: %[[dim1:.*]] = dim %[[arg0]], %[[c1]] : memref<?x?x?x?xf32>
+//       CHECKPARALLEL: %[[dim2:.*]] = dim %[[arg0]], %[[c2]] : memref<?x?x?x?xf32>
+//       CHECKPARALLEL: %[[dim3:.*]] = dim %[[arg0]], %[[c3]] : memref<?x?x?x?xf32>
+//       CHECKPARALLEL: %[[dim4:.*]] = dim %[[arg1]], %[[c0]] : memref<?x?x?x?xf32>
+//       CHECKPARALLEL: %[[dim5:.*]] = dim %[[arg1]], %[[c1]] : memref<?x?x?x?xf32>
+//       CHECKPARALLEL: %[[dim6:.*]] = dim %[[arg1]], %[[c2]] : memref<?x?x?x?xf32>
+//       CHECKPARALLEL: %[[dim7:.*]] = dim %[[arg1]], %[[c3]] : memref<?x?x?x?xf32>
+//       CHECKPARALLEL: %[[lowerBound1:.*]] = affine.apply #[[$convLowerBound]]()[%[[dim4]]]
+//       CHECKPARALLEL: %[[upperBound1:.*]] = affine.apply #[[$convUpperBound]]()[%[[dim4]], %[[dim0]]]
+//       CHECKPARALLEL: %[[lowerBound2:.*]] = affine.apply #[[$convLowerBound]]()[%[[dim5]]]
+//       CHECKPARALLEL: %[[upperBound2:.*]] = affine.apply #[[$convUpperBound]]()[%[[dim5]], %[[dim1]]]
+//       CHECKPARALLEL: %[[lowerBound3:.*]] = affine.apply #[[$convLowerBound]]()[%[[dim6]]]
+//       CHECKPARALLEL: %[[upperBound3:.*]] = affine.apply #[[$convUpperBound]]()[%[[dim6]], %[[dim2]]]
+//       CHECKPARALLEL: %[[lowerBound4:.*]] = affine.apply #[[$convLowerBound]]()[%[[dim7]]]
+//       CHECKPARALLEL: %[[upperBound4:.*]] = affine.apply #[[$convUpperBound]]()[%[[dim7]], %[[dim3]]]
+//       CHECKPARALLEL: scf.parallel (%[[i0:.*]], %[[i1:.*]], %[[i2:.*]], %[[i3:.*]], %[[i4:.*]], %[[i5:.*]], %[[i6:.*]], %[[i7:.*]]) = (%[[lowerBound1]], %[[lowerBound2]], %[[lowerBound3]], %[[lowerBound4]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) to (%[[upperBound1]], %[[upperBound2]], %[[upperBound3]], %[[upperBound4]], %[[dim4]], %[[dim5]], %[[dim6]], %[[dim7]]) step ({{.*}}) {
+//       CHECKPARALLEL:   %[[dim8:.*]] = dim %[[arg1]], %[[c0]] : memref<?x?x?x?xf32>
+//       CHECKPARALLEL:   %[[dim9:.*]] = dim %[[arg1]], %[[c1]] : memref<?x?x?x?xf32>
+//       CHECKPARALLEL:   %[[dim10:.*]] = dim %[[arg1]], %[[c2]] : memref<?x?x?x?xf32>
+//       CHECKPARALLEL:   %[[dim11:.*]] = dim %[[arg1]], %[[c3]] : memref<?x?x?x?xf32>
+//       CHECKPARALLEL:   %[[aff1:.*]] = affine.apply #[[$convMap]](%{{.*}}, %{{.*}})[%[[dim8]]]
+//       CHECKPARALLEL:   %[[aff2:.*]] = affine.apply #[[$convMap]](%{{.*}}, %{{.*}})[%[[dim9]]]
+//       CHECKPARALLEL:   %[[aff3:.*]] = affine.apply #[[$convMap]](%{{.*}}, %{{.*}})[%[[dim10]]]
+//       CHECKPARALLEL:   %[[aff4:.*]] = affine.apply #[[$convMap]](%{{.*}}, %{{.*}})[%[[dim11]]]
+//       CHECKPARALLEL:   %[[va:.*]] = load %[[arg0]][%[[aff1]], %[[aff2]], %[[aff3]], %[[aff4]]] : memref<?x?x?x?xf32>
+//       CHECKPARALLEL:   %[[vb:.*]] = load %[[arg1]][%[[i4]], %[[i5]], %[[i6]], %[[i7]]] : memref<?x?x?x?xf32>
+//       CHECKPARALLEL:   %[[vc:.*]] = load %[[arg2]][%[[i0]], %[[i1]], %[[i2]], %[[i3]]] : memref<?x?x?x?xf32>
+//       CHECKPARALLEL:   %[[inc:.*]] = mulf %[[va]], %[[vb]] : f32
+//       CHECKPARALLEL:   %[[res:.*]] = addf %[[vc]], %[[inc]] : f32
+//       CHECKPARALLEL:   store %[[res]], %[[arg2]][%[[i0]], %[[i1]], %[[i2]], %[[i3]]] : memref<?x?x?x?xf32>
diff --git a/mlir/test/Dialect/Linalg/tile_conv.mlir b/mlir/test/Dialect/Linalg/tile_conv.mlir
index 1bbb8b60382ef..a08a2f1e585c6 100644
--- a/mlir/test/Dialect/Linalg/tile_conv.mlir
+++ b/mlir/test/Dialect/Linalg/tile_conv.mlir
@@ -9,36 +9,38 @@ func @conv(%arg0: memref<?x?x?x?xf32, offset: ?, strides: [?, ?, ?, 1]>, %arg1:
   linalg.conv(%arg0, %arg1, %arg2) {dilations = [10, 20], strides = [30, 40]} : memref<?x?x?x?xf32, offset: ?, strides: [?, ?, ?, 1]>, memref<?x?x?x?xf32, offset: ?, strides: [?, ?, ?, 1]>, memref<?x?x?x?xf32, offset: ?, strides: [?, ?, ?, 1]>
   return
 }
-// TILE-23004-LABEL: func @conv(
-//       TILE-23004:   %{{.*}}: memref<?x?x?x?xf32, #[[$strided4D]]>, %{{.*}}: memref<?x?x?x?xf32, #[[$strided4D]]>, %{{.*}}: memref<?x?x?x?xf32, #[[$strided4D]]>) {
-//       TILE-23004-DAG: %[[C0:.*]] = constant 0 : index
-//       TILE-23004-DAG: %[[C2:.*]] = constant 2 : index
-//       TILE-23004-DAG: %[[C3:.*]] = constant 3 : index
-//       TILE-23004-DAG: %[[C4:.*]] = constant 4 : index
-//       TILE-23004:   %[[Q:.*]] = dim %{{.*}}, %c2 : memref<?x?x?x?xf32, #[[$strided4D]]>
-//       TILE-23004:   %[[B:.*]] = dim %{{.*}}, %c0 : memref<?x?x?x?xf32, #[[$strided4D]]>
-//       TILE-23004:   %[[PaddedInput0:.*]] = dim %{{.*}}, %c1 : memref<?x?x?x?xf32, #[[$strided4D]]>
-//       TILE-23004:   %[[X0:.*]] = dim %{{.*}}, %c1 : memref<?x?x?x?xf32, #[[$strided4D]]>
+//       TILE-23004: func @conv(
+//  TILE-23004-SAME:   %[[ARG0:[a-zA-Z0-9_]*]]: memref<?x?x?x?xf32, #[[$strided4D]]>
+//  TILE-23004-SAME:   %[[ARG1:[a-zA-Z0-9_]*]]: memref<?x?x?x?xf32, #[[$strided4D]]>
+//  TILE-23004-SAME:   %[[ARG2:[a-zA-Z0-9_]*]]: memref<?x?x?x?xf32, #[[$strided4D]]>)
+//   TILE-23004-DAG:   %[[C0:.*]] = constant 0 : index
+//   TILE-23004-DAG:   %[[C2:.*]] = constant 2 : index
+//   TILE-23004-DAG:   %[[C3:.*]] = constant 3 : index
+//   TILE-23004-DAG:   %[[C4:.*]] = constant 4 : index
+//       TILE-23004:   %[[Z0:.*]] = dim %[[ARG0]], %c0 : memref<?x?x?x?xf32, #[[$strided4D]]>
+//       TILE-23004:   %[[Q:.*]] = dim %[[ARG0]], %c2 : memref<?x?x?x?xf32, #[[$strided4D]]>
+//       TILE-23004:   %[[B:.*]] = dim %[[ARG1]], %c0 : memref<?x?x?x?xf32, #[[$strided4D]]>
+//       TILE-23004:   %[[X0:.*]] = dim %[[ARG2]], %c1 : memref<?x?x?x?xf32, #[[$strided4D]]>
 //       TILE-23004:   scf.for %[[ivI:.*]] = %{{.*}} to %[[B]] step %{{.*}} {
 //       TILE-23004:     scf.for %[[ivJ:.*]] = %{{.*}} to %[[X0]] step %{{.*}} {
 //       TILE-23004:       scf.for %[[ivK:.*]] = %{{.*}} to %[[Q]] step %{{.*}} {
-//       TILE-23004:         %[[Z0:.*]] = dim %{{.*}}, %c0 : memref<?x?x?x?xf32, #[[$strided4D]]>
-//       TILE-23004:         %[[Z1:.*]] = dim %{{.*}}, %c1 : memref<?x?x?x?xf32, #[[$strided4D]]>
-//       TILE-23004:         %[[Z2:.*]] = dim %{{.*}}, %c2 : memref<?x?x?x?xf32, #[[$strided4D]]>
+//       TILE-23004:         %[[Z0_1:.*]] = dim %[[ARG0]], %c0 : memref<?x?x?x?xf32, #[[$strided4D]]>
+//       TILE-23004:         %[[Z1:.*]] = dim %[[ARG0]], %c1 : memref<?x?x?x?xf32, #[[$strided4D]]>
+//       TILE-23004:         %[[Z2:.*]] = dim %[[ARG0]], %c2 : memref<?x?x?x?xf32, #[[$strided4D]]>
 //       TILE-23004:         %[[szK:.*]] = affine.min #[[$bound_map_4]](%[[ivK]])[%[[Z2]]]
-//       TILE-23004:         %[[K:.*]] = dim %{{.*}}, %c3 : memref<?x?x?x?xf32, #[[$strided4D]]>
-//       TILE-23004:         %[[FilterView:.*]] = subview %{{.*}}[0, 0, %[[ivK]], 0] [%[[Z0]], %[[Z1]], %[[szK]], %[[K]]] [1, 1, 1, 1] : memref<?x?x?x?xf32, #[[$strided4D]]> to memref<?x?x?x?xf32, #[[$strided4D]]>
+//       TILE-23004:         %[[K:.*]] = dim %[[ARG0]], %c3 : memref<?x?x?x?xf32, #[[$strided4D]]>
+//       TILE-23004:         %[[FilterView:.*]] = subview %{{.*}}[0, 0, %[[ivK]], 0] [%[[Z0_1]], %[[Z1]], %[[szK]], %[[K]]] [1, 1, 1, 1] : memref<?x?x?x?xf32, #[[$strided4D]]> to memref<?x?x?x?xf32, #[[$strided4D]]>
 //
 //       TILE-23004:         %[[J1:.*]] = affine.apply #[[$D0x30pS0x10]](%[[ivJ]])
-//       TILE-23004:         %[[PaddedInput0b:.*]] = dim %{{.*}}, %c1 : memref<?x?x?x?xf32, #[[$strided4D]]>
-//       TILE-23004:         %[[I1pStep:.*]] = affine.min #[[$S0x10p90D0x30pS1]](%[[ivJ]])[%[[PaddedInput0]], %[[PaddedInput0b]]]
-//       TILE-23004:         %[[SZ2:.*]] = dim %{{.*}}, %c2 : memref<?x?x?x?xf32, #[[$strided4D]]>
-//       TILE-23004:         %[[dim3:.*]] = dim %{{.*}}, %c3
+//       TILE-23004:         %[[PaddedInput0b:.*]] = dim %[[ARG1]], %c1 : memref<?x?x?x?xf32, #[[$strided4D]]>
+//       TILE-23004:         %[[I1pStep:.*]] = affine.min #[[$S0x10p90D0x30pS1]](%[[ivJ]])[%[[Z0]], %[[PaddedInput0b]]]
+//       TILE-23004:         %[[SZ2:.*]] = dim %[[ARG1]], %c2 : memref<?x?x?x?xf32, #[[$strided4D]]>
+//       TILE-23004:         %[[dim3:.*]] = dim %[[ARG1]], %c3
 //       TILE-23004:         %[[sz3:.*]] = affine.min #[[$bound_map_4]](%[[ivK]])[%[[dim3]]]
 //       TILE-23004:         %[[InputView:.*]] = subview %{{.*}}[%[[ivI]], %[[J1]], 0, %[[ivK]]] [%{{.*}}, %{{.*}}, %[[SZ2]], %[[sz3]]] [1, 1, 1, 1] : memref<?x?x?x?xf32, #[[$strided4D]]> to memref<?x?x?x?xf32, #[[$strided4D]]>
 //
-//       TILE-23004:         %[[X0:.*]] = dim %{{.*}}, %c2 : memref<?x?x?x?xf32, #[[$strided4D]]>
-//       TILE-23004:         %[[X1:.*]] = dim %{{.*}}, %c3 : memref<?x?x?x?xf32, #[[$strided4D]]>
+//       TILE-23004:         %[[X0:.*]] = dim %[[ARG2]], %c2 : memref<?x?x?x?xf32, #[[$strided4D]]>
+//       TILE-23004:         %[[X1:.*]] = dim %[[ARG2]], %c3 : memref<?x?x?x?xf32, #[[$strided4D]]>
 //       TILE-23004:         %[[OutputView:.*]] = subview %{{.*}}[%[[ivI]], %[[ivJ]], 0, 0] [%{{.*}}, %{{.*}}, %[[X0]], %[[X1]]] [1, 1, 1, 1] : memref<?x?x?x?xf32, #[[$strided4D]]> to memref<?x?x?x?xf32, #[[$strided4D]]>
 //
 //       TILE-23004:         linalg.conv(%[[FilterView]], %[[InputView]], %[[OutputView]]) {dilations = [10, 20], strides = [30, 40]} : memref<?x?x?x?xf32, #[[$strided4D]]>, memref<?x?x?x?xf32, #[[$strided4D]]>, memref<?x?x?x?xf32, #[[$strided4D]]>
diff --git a/mlir/test/Dialect/Linalg/tile_simple_conv.mlir b/mlir/test/Dialect/Linalg/tile_simple_conv.mlir
new file mode 100644
index 0000000000000..f854f7570fef3
--- /dev/null
+++ b/mlir/test/Dialect/Linalg/tile_simple_conv.mlir
@@ -0,0 +1,49 @@
+// RUN: mlir-opt %s -linalg-tile="linalg-tile-sizes=2,3,4" | FileCheck %s
+
+//  CHECK-DAG: #[[MAP0:.*]] = affine_map<(d0)[s0] -> (2, -d0 + s0)>
+//  CHECK-DAG: #[[MAP1:.*]] = affine_map<(d0)[s0, s1] -> (s0 + 3, -d0 + s1)>
+//  CHECK-DAG: #[[MAP2:.*]] = affine_map<(d0)[s0, s1] -> (s0 + 4, -d0 + s1)>
+//  CHECK-DAG: #[[MAP4:.*]] = affine_map<(d0)[s0] -> (3, -d0 + s0)>
+//  CHECK-DAG: #[[MAP5:.*]] = affine_map<(d0)[s0] -> (4, -d0 + s0)>
+
+func @conv(%arg0 : memref<?x?x?x?xf32>, %arg1 : memref<?x?x?x?xf32>, %arg2 : memref<?x?x?x?xf32>) {
+  linalg.conv(%arg0, %arg1, %arg2) : memref<?x?x?x?xf32>, memref<?x?x?x?xf32>, memref<?x?x?x?xf32>
+  return
+}
+
+//       CHECK: func @conv
+//  CHECK-SAME:   %[[ARG0:[a-zA-Z0-9_]*]]: memref<?x?x?x?xf32>
+//  CHECK-SAME:   %[[ARG1:[a-zA-Z0-9_]*]]: memref<?x?x?x?xf32>
+//  CHECK-SAME:   %[[ARG2:[a-zA-Z0-9_]*]]: memref<?x?x?x?xf32>
+//   CHECK-DAG:   %[[C0:.*]] = constant 0 : index
+//   CHECK-DAG:   %[[C1:.*]] = constant 1 : index
+//   CHECK-DAG:   %[[C2:.*]] = constant 2 : index
+//   CHECK-DAG:   %[[C3:.*]] = constant 3 : index
+//   CHECK-DAG:   %[[C4:.*]] = constant 4 : index
+//       CHECK:   %[[T0:.*]] = dim %[[ARG0]], %[[C0]]
+//       CHECK:   %[[T1:.*]] = dim %[[ARG0]], %[[C1]]
+//       CHECK:   %[[T2:.*]] = dim %[[ARG1]], %[[C0]]
+//       CHECK:   %[[T3:.*]] = dim %[[ARG2]], %[[C1]]
+//       CHECK:   %[[T4:.*]] = dim %[[ARG2]], %[[C2]]
+//       CHECK:   scf.for %[[ARG3:.*]] = %[[C0]] to %[[T2]] step %[[C2]]
+//       CHECK:     scf.for %[[ARG4:.*]] = %[[C0]] to %[[T3]] step %[[C3]]
+//       CHECK:       scf.for %[[ARG5:.*]] = %[[C0]] to %[[T4]] step %[[C4]]
+//       CHECK:         %[[T5:.*]] = dim %[[ARG1]], %[[C0]]
+//       CHECK:         %[[T6:.*]] = affine.min #[[MAP0]](%[[ARG3]])[%[[T5]]]
+//       CHECK:         %[[T7:.*]] = dim %[[ARG1]], %[[C1]]
+//       CHECK:         %[[T8:.*]] = affine.min #[[MAP1]](%[[ARG4]])[%[[T0]], %[[T7]]]
+//       CHECK:         %[[T9:.*]] = dim %[[ARG1]], %[[C2]]
+//       CHECK:         %[[T10:.*]] = affine.min #[[MAP2]](%[[ARG5]])[%[[T1]], %[[T9]]]
+//       CHECK:         %[[T11:.*]] = dim %[[ARG1]], %[[C3]]
+//       CHECK:         %[[SV1:.*]] = subview %[[ARG1]][%[[ARG3]], %[[ARG4]], %[[ARG5]], 0]
+//  CHECK-SAME:                                        [%[[T6]], %[[T8]], %[[T10]], %[[T11]]]
+//       CHECK:         %[[T13:.*]] = dim %[[ARG2]], %[[C0]]
+//       CHECK:         %[[T14:.*]] = affine.min #[[MAP0]](%[[ARG3]])[%[[T13]]]
+//       CHECK:         %[[T15:.*]] = dim %[[ARG2]], %[[C1]]
+//       CHECK:         %[[T16:.*]] = affine.min #[[MAP4]](%[[ARG4]])[%[[T15]]]
+//       CHECK:         %[[T17:.*]] = dim %[[ARG2]], %[[C2]]
+//       CHECK:         %[[T18:.*]] = affine.min #[[MAP5]](%[[ARG5]])[%[[T17]]]
+//       CHECK:         %[[T19:.*]] = dim %[[ARG2]], %[[C3]]
+//       CHECK:         %[[SV2:.*]] = subview %[[ARG2]][%[[ARG3]], %[[ARG4]], %[[ARG5]], 0]
+//  CHECK-SAME:                                        [%[[T14]], %[[T16]], %[[T18]], %[[T19]]]
+//       CHECK:         linalg.conv(%[[ARG0]], %[[SV1]], %[[SV2]])
\ No newline at end of file
diff --git a/mlir/test/Dialect/Linalg/transform-patterns.mlir b/mlir/test/Dialect/Linalg/transform-patterns.mlir
index 9eedc31ef43a8..3f7d16497253d 100644
--- a/mlir/test/Dialect/Linalg/transform-patterns.mlir
+++ b/mlir/test/Dialect/Linalg/transform-patterns.mlir
@@ -118,6 +118,23 @@ func @vectorization_test(%A: memref<8x16xf32>, %B: memref<16x32xf32>,
 //       CHECK: vector.contract {indexing_maps = [#[[$mk]], #[[$kn]], #[[$mn]]], iterator_types = ["parallel", "parallel", "reduction"]} %{{.*}}, %{{.*}}, %{{.*}} : vector<8x16xf32>, vector<16x32xf32> into vector<8x32xf32>
 //       CHECK: vector.transfer_write %{{.*}}, %{{.*}} : vector<8x32xf32>, memref<8x32xf32>
 
+func @vectorization_test_integer(%A: memref<8x16xi32>, %B: memref<16x32xi32>,
+                                 %C: memref<8x32xi32>) {
+  linalg.generic #matmul_trait %A, %B, %C {
+    ^bb(%a: i32, %b: i32, %c: i32) :
+      %d = muli %a, %b: i32
+      %e = addi %c, %d: i32
+      linalg.yield %e : i32
+  } : memref<8x16xi32>, memref<16x32xi32>, memref<8x32xi32>
+  return
+}
+// CHECK-LABEL: func @vectorization_test_integer
+//       CHECK: vector.transfer_read %{{.*}} : memref<8x16xi32>, vector<8x16xi32>
+//       CHECK: vector.transfer_read %{{.*}} : memref<16x32xi32>, vector<16x32xi32>
+//       CHECK: vector.transfer_read %{{.*}} : memref<8x32xi32>, vector<8x32xi32>
+//       CHECK: vector.contract {indexing_maps = [#[[$mk]], #[[$kn]], #[[$mn]]], iterator_types = ["parallel", "parallel", "reduction"]} %{{.*}}, %{{.*}}, %{{.*}} : vector<8x16xi32>, vector<16x32xi32> into vector<8x32xi32>
+//       CHECK: vector.transfer_write %{{.*}}, %{{.*}} : vector<8x32xi32>, memref<8x32xi32>
+
 func @vectorization_test_2(%A: memref<8x16xf32>, %B: memref<16x32xf32>,
                          %C: memref<8x32xf32>) {
   linalg.matmul %A, %B, %C { __internal_linalg_transform__ = "VECTORIZE"} :
@@ -135,6 +152,23 @@ func @test_vectorize_fill(%A : memref<8x16xf32>, %arg0 : f32) {
 // CHECK-LABEL: func @test_vectorize_fill
 //       CHECK: vector.broadcast {{.*}} : f32 to vector<8x16xf32>
 
+func @test_vectorize_copy(%A : memref<8x16xf32>, %B : memref<8x16xf32>) {
+  linalg.copy(%A, %B) { __internal_linalg_transform__ = "VECTORIZE"} :  memref<8x16xf32>, memref<8x16xf32>
+  return
+}
+// CHECK-LABEL: func @test_vectorize_copy
+//       CHECK: %[[V:.*]] = vector.transfer_read {{.*}} : memref<8x16xf32>, vector<8x16xf32>
+//       CHECK: vector.transfer_write %[[V]], {{.*}} : vector<8x16xf32>, memref<8x16xf32>
+
+func @test_vectorize_copy_scalar(%A : memref<f32>, %B : memref<f32>) {
+  linalg.copy(%A, %B) { __internal_linalg_transform__ = "VECTORIZE"} :  memref<f32>, memref<f32>
+  return
+}
+// CHECK-LABEL: func @test_vectorize_copy_scalar
+//       CHECK: %[[V:.*]] = load {{.*}} : memref<f32>
+//       CHECK: store %[[V]], {{.*}} : memref<f32>
+
+
 #matmul_accesses = [
   affine_map<(m, n, k) -> (m, k)>,
   affine_map<(m, n, k) -> (k, n)>,
diff --git a/mlir/test/Dialect/SPIRV/Transforms/abi-load-store.mlir b/mlir/test/Dialect/SPIRV/Transforms/abi-load-store.mlir
index 4e4bf06e6f737..3d37f35b1c466 100644
--- a/mlir/test/Dialect/SPIRV/Transforms/abi-load-store.mlir
+++ b/mlir/test/Dialect/SPIRV/Transforms/abi-load-store.mlir
@@ -57,9 +57,12 @@ spv.module Logical GLSL450 {
     // CHECK: [[CONST3:%.*]] = spv.constant 0 : i32
     // CHECK: [[ARG3PTR:%.*]] = spv.AccessChain [[ADDRESSARG3]]{{\[}}[[CONST3]]
     // CHECK: [[ARG3:%.*]] = spv.Load "StorageBuffer" [[ARG3PTR]]
-    // CHECK: [[ARG2:%.*]] = spv._address_of [[VAR2]]
-    // CHECK: [[ARG1:%.*]] = spv._address_of [[VAR1]]
-    // CHECK: [[ARG0:%.*]] = spv._address_of [[VAR0]]
+    // CHECK: [[ADDRESSARG2:%.*]] = spv._address_of [[VAR2]]
+    // CHECK: [[ARG2:%.*]] = spv.Bitcast [[ADDRESSARG2]]
+    // CHECK: [[ADDRESSARG1:%.*]] = spv._address_of [[VAR1]]
+    // CHECK: [[ARG1:%.*]] = spv.Bitcast [[ADDRESSARG1]]
+    // CHECK: [[ADDRESSARG0:%.*]] = spv._address_of [[VAR0]]
+    // CHECK: [[ARG0:%.*]] = spv.Bitcast [[ADDRESSARG0]]
     %0 = spv._address_of @__builtin_var_WorkgroupId__ : !spv.ptr<vector<3xi32>, Input>
     %1 = spv.Load "Input" %0 : vector<3xi32>
     %2 = spv.CompositeExtract %1[0 : i32] : vector<3xi32>
diff --git a/mlir/test/Dialect/Shape/canonicalize.mlir b/mlir/test/Dialect/Shape/canonicalize.mlir
index 4e320f303b186..b4dca5e3c2bf5 100644
--- a/mlir/test/Dialect/Shape/canonicalize.mlir
+++ b/mlir/test/Dialect/Shape/canonicalize.mlir
@@ -1,10 +1,10 @@
-// RUN: mlir-opt -split-input-file -allow-unregistered-dialect -canonicalize <%s | FileCheck %s
+// RUN: mlir-opt -split-input-file -allow-unregistered-dialect -canonicalize %s | FileCheck %s
 
 // CHECK-LABEL: func @f
-func @f(%arg0: tensor<2x3x4xf32>) -> !shape.shape {
-  // CHECK: shape.const_shape [2, 3, 4]
-  %0 = "shape.shape_of"(%arg0) : (tensor<2x3x4xf32>) -> !shape.shape
-  return %0 : !shape.shape
+func @f(%arg0: tensor<2x3x4xf32>) -> tensor<?xindex> {
+  // CHECK: shape.const_shape [2, 3, 4] : tensor<?xindex>
+  %0 = shape.shape_of %arg0 : tensor<2x3x4xf32> -> tensor<?xindex>
+  return %0 : tensor<?xindex>
 }
 
 // -----
@@ -12,10 +12,10 @@ func @f(%arg0: tensor<2x3x4xf32>) -> !shape.shape {
 // Basic case.
 // CHECK-LABEL: func @f
 func @f() -> (!shape.shape, !shape.shape) {
-  // CHECK: shape.const_shape [2, 3]
-  // CHECK: shape.const_shape [4, 5]
+  // CHECK: shape.const_shape [2, 3] : !shape.shape
+  // CHECK: shape.const_shape [4, 5] : !shape.shape
   %c2 = constant 2 : i32
-  %0 = shape.const_shape [2, 3, 4, 5]
+  %0 = shape.const_shape [2, 3, 4, 5] : !shape.shape
   %head, %tail = "shape.split_at"(%0, %c2) : (!shape.shape, i32) -> (!shape.shape, !shape.shape)
   return %head, %tail : !shape.shape, !shape.shape
 
@@ -26,10 +26,10 @@ func @f() -> (!shape.shape, !shape.shape) {
 // Negative split point.
 // CHECK-LABEL: func @f
 func @f() -> (!shape.shape, !shape.shape) {
-  // CHECK: shape.const_shape [2, 3, 4]
-  // CHECK: shape.const_shape [5]
+  // CHECK: shape.const_shape [2, 3, 4] : !shape.shape
+  // CHECK: shape.const_shape [5] : !shape.shape
   %c-1 = constant -1 : i32
-  %0 = shape.const_shape [2, 3, 4, 5]
+  %0 = shape.const_shape [2, 3, 4, 5] : !shape.shape
   %head, %tail = "shape.split_at"(%0, %c-1) : (!shape.shape, i32) -> (!shape.shape, !shape.shape)
   return %head, %tail : !shape.shape, !shape.shape
 }
@@ -41,7 +41,7 @@ func @f() -> (!shape.shape, !shape.shape) {
 func @f() -> (!shape.shape, !shape.shape) {
   // CHECK: shape.split_at
   %c5 = constant 5 : i32
-  %0 = shape.const_shape [2, 3, 4, 5]
+  %0 = shape.const_shape [2, 3, 4, 5] : !shape.shape
   %head, %tail = "shape.split_at"(%0, %c5) : (!shape.shape, i32) -> (!shape.shape, !shape.shape)
   return %head, %tail : !shape.shape, !shape.shape
 }
@@ -51,9 +51,9 @@ func @f() -> (!shape.shape, !shape.shape) {
 // Basic case.
 // CHECK-LABEL: func @f
 func @f() -> !shape.shape {
-  // CHECK: shape.const_shape [7, 2]
-  %0 = shape.const_shape [1, 2]
-  %1 = shape.const_shape [7, 1]
+  // CHECK: shape.const_shape [7, 2] : !shape.shape
+  %0 = shape.const_shape [1, 2] : !shape.shape
+  %1 = shape.const_shape [7, 1] : !shape.shape
   %2 = shape.broadcast %0, %1
   return %2 : !shape.shape
 }
@@ -64,7 +64,7 @@ func @f() -> !shape.shape {
 // CHECK-LABEL: func @f
 func @f(%arg0 : !shape.shape) -> !shape.shape {
   // CHECK: return %arg0
-  %0 = shape.const_shape []
+  %0 = shape.const_shape [] : !shape.shape
   %1 = shape.broadcast %arg0, %0
   return %1 : !shape.shape
 }
@@ -75,7 +75,7 @@ func @f(%arg0 : !shape.shape) -> !shape.shape {
 // CHECK-LABEL: func @f
 func @f(%arg0 : !shape.shape) -> !shape.shape {
   // CHECK: return %arg0
-  %0 = shape.const_shape []
+  %0 = shape.const_shape [] : !shape.shape
   %1 = shape.broadcast %0, %arg0
   return %1 : !shape.shape
 }
@@ -85,10 +85,10 @@ func @f(%arg0 : !shape.shape) -> !shape.shape {
 // Lhs is a scalar and rhs is constant.
 // CHECK-LABEL: func @f
 func @f() -> !shape.shape {
-  // CHECK: %[[CST:.*]] = shape.const_shape [1, 2, 3]
+  // CHECK: %[[CST:.*]] = shape.const_shape [1, 2, 3] : !shape.shape
   // CHECK: return %[[CST]]
-  %0 = shape.const_shape []
-  %1 = shape.const_shape [1, 2, 3]
+  %0 = shape.const_shape [] : !shape.shape
+  %1 = shape.const_shape [1, 2, 3] : !shape.shape
   %2 = shape.broadcast %0, %1
   return %2 : !shape.shape
 }
@@ -99,8 +99,8 @@ func @f() -> !shape.shape {
 // CHECK-LABEL: func @f
 func @f() -> !shape.shape {
   // CHECK: shape.broadcast
-  %0 = shape.const_shape [2]
-  %1 = shape.const_shape [7]
+  %0 = shape.const_shape [2] : !shape.shape
+  %1 = shape.const_shape [7] : !shape.shape
   %2 = shape.broadcast %0, %1
   return %2 : !shape.shape
 }
@@ -110,9 +110,9 @@ func @f() -> !shape.shape {
 // Basic case.
 // CHECK-LABEL: func @f
 func @f() -> !shape.shape {
-  // CHECK: shape.const_shape [0, 1, 2, 3]
-  %lhs = shape.const_shape [0, 1]
-  %rhs = shape.const_shape [2, 3]
+  // CHECK: shape.const_shape [0, 1, 2, 3] : !shape.shape
+  %lhs = shape.const_shape [0, 1] : !shape.shape
+  %rhs = shape.const_shape [2, 3] : !shape.shape
   %0 = shape.concat %lhs, %rhs
   return %0 : !shape.shape
 }
@@ -123,7 +123,7 @@ func @f() -> !shape.shape {
 // CHECK-LABEL: func @f
 func @f() -> tensor<2xindex> {
   // CHECK: constant dense<[0, 1]> : tensor<2xindex>
-  %cs = shape.const_shape [0, 1]
+  %cs = shape.const_shape [0, 1] : !shape.shape
   %0 = shape.to_extent_tensor %cs : tensor<2xindex>
   return %0 : tensor<2xindex>
 }
@@ -133,7 +133,7 @@ func @f() -> tensor<2xindex> {
 // Basic case.
 // CHECK-LABEL: func @f()
 func @f() -> !shape.shape {
-  // CHECK: shape.const_shape [3, 5, 11]
+  // CHECK: shape.const_shape [3, 5, 11] : !shape.shape
   %e0 = constant 3 : index
   %e1 = constant 5 : index
   %e2 = constant 11 : index
@@ -215,7 +215,7 @@ func @nonfoldable_index_to_size(%ci : index) -> !shape.size {
 // CHECK-LABEL: func @num_elements
 func @num_elements() -> !shape.size {
   // CHECK-NOT: shape.const_shape
-  %shape = shape.const_shape [4, 5, 6]
+  %shape = shape.const_shape [4, 5, 6] : !shape.shape
   // CHECK-NOT: shape.num_elements
   %num_elements = shape.num_elements %shape
   // CHECK: %[[NUM:.*]] = shape.const_size 120
@@ -235,13 +235,49 @@ func @nonfoldable_num_elements(%shape : !shape.shape) -> !shape.size {
 
 // -----
 
+// Basic folding.
+// CHECK-LABEL: func @basic
+func @basic() -> index {
+  // CHECK: constant 2 : index
+  %0 = shape.const_shape [0, 1, 2] : tensor<?xindex>
+  %c2 = constant 2 : index
+  %1 = shape.get_extent %0, %c2 : tensor<?xindex>, index -> index
+  return %1 : index
+}
+
+// -----
+
+// Should not fold.
+// CHECK-LABEL: func @out_of_bounds
+func @out_of_bounds() -> index {
+  // CHECK: shape.const_shape
+  // CHECK: shape.get_extent
+  %0 = shape.const_shape [0, 1, 2] : tensor<?xindex>
+  %c3 = constant 3 : index
+  %1 = shape.get_extent %0, %c3 : tensor<?xindex>, index -> index
+  return %1 : index
+}
+
+// -----
+
+// Should not fold.
+// CHECK-LABEL: func @not_const
+func @not_const(%arg0: tensor<?xindex>) -> index {
+  // CHECK: shape.get_extent
+  %c3 = constant 3 : index
+  %0 = shape.get_extent %arg0, %c3 : tensor<?xindex>, index -> index
+  return %0 : index
+}
+
+// -----
+
 // Basic folding.
 // CHECK-LABEL: func @basic
 func @basic() -> !shape.size {
   // CHECK: shape.const_size 2
-  %0 = shape.const_shape [0, 1, 2]
+  %0 = shape.const_shape [0, 1, 2] : !shape.shape
   %c2 = shape.const_size 2
-  %1 = shape.get_extent %0, %c2
+  %1 = shape.get_extent %0, %c2 : !shape.shape, !shape.size -> !shape.size
   return %1 : !shape.size
 }
 
@@ -252,9 +288,9 @@ func @basic() -> !shape.size {
 func @out_of_bounds() -> !shape.size {
   // CHECK: shape.const_shape
   // CHECK: shape.get_extent
-  %0 = shape.const_shape [0, 1, 2]
+  %0 = shape.const_shape [0, 1, 2] : !shape.shape
   %c3 = shape.const_size 3
-  %1 = shape.get_extent %0, %c3
+  %1 = shape.get_extent %0, %c3 : !shape.shape, !shape.size -> !shape.size
   return %1 : !shape.size
 }
 
@@ -262,14 +298,13 @@ func @out_of_bounds() -> !shape.size {
 
 // Should not fold.
 // CHECK-LABEL: func @not_const
-func @not_const(%arg0: !shape.shape) -> !shape.size {
+func @not_const(%arg0 : !shape.shape) -> !shape.size {
   // CHECK: shape.get_extent
   %c3 = shape.const_size 3
-  %0 = shape.get_extent %arg0, %c3
+  %0 = shape.get_extent %arg0, %c3 : !shape.shape, !shape.size -> !shape.size
   return %0 : !shape.size
 }
 
-
 // -----
 // cstr_eq with non-constant but known equal shapes can be removed.
 // CHECK-LABEL: func @f
@@ -289,9 +324,9 @@ func @f() {
   // CHECK-NEXT: shape.const_witness true
   // CHECK-NEXT: consume.witness
   // CHECK-NEXT: return
-  %cs0 = shape.const_shape [0, 1]
-  %cs1 = shape.const_shape [0, 1]
-  %cs2 = shape.const_shape [0, 1]
+  %cs0 = shape.const_shape [0, 1] : !shape.shape
+  %cs1 = shape.const_shape [0, 1] : !shape.shape
+  %cs2 = shape.const_shape [0, 1] : !shape.shape
   %0 = shape.cstr_eq %cs0, %cs1, %cs2
   "consume.witness"(%0) : (!shape.witness) -> ()
   return
@@ -306,8 +341,8 @@ func @f() {
   // CHECK-NEXT: shape.cstr_eq
   // CHECK-NEXT: consume.witness
   // CHECK-NEXT: return
-  %cs0 = shape.const_shape [0, 1]
-  %cs1 = shape.const_shape [3, 1]
+  %cs0 = shape.const_shape [0, 1] : !shape.shape
+  %cs1 = shape.const_shape [3, 1] : !shape.shape
   %0 = shape.cstr_eq %cs0, %cs1
   "consume.witness"(%0) : (!shape.witness) -> ()
   return
@@ -364,14 +399,25 @@ func @f() {
 
 // any can be replaced with a constant input if it has one.
 // CHECK-LABEL: func @f
-func @f(%arg0 : !shape.shape) -> !shape.shape {
+func @f(%arg : !shape.shape) -> !shape.shape {
   // CHECK-NEXT: %[[CS:.*]] = shape.const_shape
   // CHECK-NEXT: return %[[CS]]
-  %0 = shape.const_shape [2, 3, 4]
-  %1 = shape.any %0, %arg0
+  %0 = shape.const_shape [2, 3, 4] : !shape.shape
+  %1 = shape.any %0, %arg : !shape.shape
   return %1 : !shape.shape
 }
 
+// -----
+
+// any can be replaced with a constant input if it has one.
+// CHECK-LABEL: func @f
+func @f(%arg : tensor<?xindex>) -> tensor<?xindex> {
+  // CHECK-NEXT: %[[CS:.*]] = shape.const_shape [2, 3, 4] : tensor<?xindex>
+  // CHECK-NEXT: return %[[CS]] : tensor<?xindex>
+  %0 = shape.const_shape [2, 3, 4] : tensor<?xindex>
+  %1 = shape.any %0, %arg : tensor<?xindex>
+  return %1 : tensor<?xindex>
+}
 
 // -----
 
@@ -380,7 +426,7 @@ func @f(%arg0 : !shape.shape) -> !shape.shape {
 func @f(%arg0 : !shape.shape, %arg1 : !shape.shape) -> !shape.shape {
   // CHECK-NEXT: %[[CS:.*]] = shape.any
   // CHECK-NEXT: return %[[CS]]
-  %1 = shape.any %arg0, %arg1
+  %1 = shape.any %arg0, %arg1 : !shape.shape
   return %1 : !shape.shape
 }
 
@@ -429,9 +475,9 @@ func @f() {
   // CHECK-NEXT: shape.const_witness true
   // CHECK-NEXT: consume.witness
   // CHECK-NEXT: return
-  %cs0 = shape.const_shape [3, 1]
-  %cs1 = shape.const_shape [1, 5]
-  %0 = shape.cstr_broadcastable %cs0, %cs1
+  %cs0 = shape.const_shape [3, 1] : !shape.shape
+  %cs1 = shape.const_shape [1, 5] : !shape.shape
+  %0 = shape.cstr_broadcastable %cs0, %cs1 : !shape.shape, !shape.shape
   "consume.witness"(%0) : (!shape.witness) -> ()
   return
 }
@@ -445,9 +491,9 @@ func @static_non_broadcastable() {
   // CHECK-NEXT: shape.cstr_broadcastable
   // CHECK-NEXT: consume.witness
   // CHECK-NEXT: return
-  %cs0 = shape.const_shape [1, 3]
-  %cs1 = shape.const_shape [1, 5]
-  %0 = shape.cstr_broadcastable %cs0, %cs1
+  %cs0 = shape.const_shape [1, 3] : !shape.shape
+  %cs1 = shape.const_shape [1, 5] : !shape.shape
+  %0 = shape.cstr_broadcastable %cs0, %cs1 : !shape.shape, !shape.shape
   "consume.witness"(%0) : (!shape.witness) -> ()
   return
 }
@@ -460,8 +506,8 @@ func @f(%arg0 : !shape.shape) {
   // CHECK-NEXT: shape.cstr_broadcastable
   // CHECK-NEXT: consume.witness
   // CHECK-NEXT: return
-  %cs0 = shape.const_shape [1,3]
-  %0 = shape.cstr_broadcastable %arg0, %cs0
+  %cs0 = shape.const_shape [1, 3] : !shape.shape
+  %0 = shape.cstr_broadcastable %arg0, %cs0 : !shape.shape, !shape.shape
   "consume.witness"(%0) : (!shape.witness) -> ()
   return
 }
@@ -473,7 +519,20 @@ func @f(%arg0 : !shape.shape) {
   // CHECK-NEXT: shape.const_witness true
   // CHECK-NEXT: consume.witness
   // CHECK-NEXT: return
-  %0 = shape.cstr_broadcastable %arg0, %arg0
+  %0 = shape.cstr_broadcastable %arg0, %arg0 : !shape.shape, !shape.shape
+  "consume.witness"(%0) : (!shape.witness) -> ()
+  return
+}
+
+// -----
+
+// Broadcastable canonicalization also works on extent tensors.
+// CHECK-LABEL: func @broadcastable_on_extent_tensors
+func @broadcastable_on_extent_tensors(%arg : tensor<?xindex>) {
+  // CHECK-NEXT: shape.const_witness true
+  // CHECK-NEXT: consume.witness
+  // CHECK-NEXT: return
+  %0 = shape.cstr_broadcastable %arg, %arg : tensor<?xindex>, tensor<?xindex>
   "consume.witness"(%0) : (!shape.witness) -> ()
   return
 }
@@ -483,10 +542,10 @@ func @f(%arg0 : !shape.shape) {
 // Fold `rank` based on constant shape.
 // CHECK-LABEL: @fold_rank
 func @fold_rank() -> !shape.size {
-  // CHECK-DAG: %[[RESULT:.*]] = shape.const_size 5
-  // CHECK-DAG: return %[[RESULT]] : !shape.size
-  %shape = shape.const_shape [3, 4, 5, 6, 7]
-  %rank = shape.rank %shape
+  // CHECK: %[[RESULT:.*]] = shape.const_size 5
+  // CHECK: return %[[RESULT]] : !shape.size
+  %shape = shape.const_shape [3, 4, 5, 6, 7] : !shape.shape
+  %rank = shape.rank %shape : !shape.shape -> !shape.size
   return %rank : !shape.size
 }
 
@@ -496,38 +555,64 @@ func @fold_rank() -> !shape.size {
 // CHECK-LABEL: @dont_fold_rank
 // CHECK-SAME: (%[[SHAPE:.*]]: !shape.shape) -> !shape.size
 func @dont_fold_rank(%shape : !shape.shape) -> !shape.size {
-  // CHECK-DAG: %[[RESULT:.*]] = shape.rank %[[SHAPE]]
-  // CHECK-DAG: return %[[RESULT]] : !shape.size
-  %rank = shape.rank %shape
+  // CHECK: %[[RESULT:.*]] = shape.rank %[[SHAPE]]
+  // CHECK: return %[[RESULT]] : !shape.size
+  %rank = shape.rank %shape : !shape.shape -> !shape.size
   return %rank : !shape.size
 }
 
 // -----
 
+// Fold `rank` based on constant extent tensor.
+// CHECK-LABEL: @fold_rank
+func @fold_rank() -> index {
+  // CHECK: %[[RESULT:.*]] = constant 5 : index
+  // CHECK: return %[[RESULT]] : index
+  %shape = shape.const_shape [3, 4, 5, 6, 7] : tensor<?xindex>
+  %rank = shape.rank %shape : tensor<?xindex> -> index
+  return %rank : index
+}
+
+// -----
+
+// Do not fold `rank` for non-constant extent tensors.
+// CHECK-LABEL: @dont_fold_rank
+// CHECK-SAME: (%[[SHAPE:.*]]: tensor<?xindex>) -> index
+func @dont_fold_rank(%shape : tensor<?xindex>) -> index {
+  // CHECK: %[[RESULT:.*]] = shape.rank %[[SHAPE]] : tensor<?xindex> -> index
+  // CHECK: return %[[RESULT]] : index
+  %rank = shape.rank %shape : tensor<?xindex> -> index
+  return %rank : index
+}
+
+// -----
+
 // Canonicalize `rank` when shape is derived from ranked tensor.
 // CHECK-LABEL: @canonicalize_rank
-func @canonicalize_rank(%arg : tensor<1x2x?xf32>) -> !shape.size {
-// CHECK-DAG: %[[RESULT:.*]] = shape.const_size 3
-// CHECK-DAG: return %[[RESULT]] : !shape.size
-%shape = shape.shape_of %arg : tensor<1x2x?xf32>
-%rank = shape.rank %shape
-return %rank : !shape.size
+func @canonicalize_rank(%arg : tensor<1x2x?xf32>) -> index {
+  // CHECK: %[[RESULT:.*]] = constant 3 : index
+  // CHECK: return %[[RESULT]] : index
+  %shape = shape.shape_of %arg : tensor<1x2x?xf32> -> tensor<?xindex>
+  %rank = shape.rank %shape : tensor<?xindex> -> index
+  return %rank : index
 }
 
 // -----
 
 // Do not canonicalize `rank` when shape is derived from unranked tensor.
 // CHECK-LABEL: @dont_canonicalize_rank
-// CHECK-SAME: (%[[ARG:.*]]: tensor<*xf32>) -> !shape.size
-func @dont_canonicalize_rank(%arg : tensor<*xf32>) -> !shape.size {
-// CHECK-DAG: %[[SHAPE:.*]] = shape.shape_of %[[ARG]] : tensor<*xf32>
-// CHECK-DAG: %[[SIZE:.*]] = shape.rank %[[SHAPE]]
-// CHECK-DAG: return %[[SIZE]] : !shape.size
-%shape = shape.shape_of %arg : tensor<*xf32>
-%rank = shape.rank %shape
-return %rank : !shape.size
+// CHECK-SAME: (%[[ARG:.*]]: tensor<*xf32>) -> index
+func @dont_canonicalize_rank(%arg : tensor<*xf32>) -> index {
+  // CHECK: %[[SHAPE:.*]] = shape.shape_of %[[ARG]] : tensor<*xf32> -> tensor<?xindex>
+  // CHECK: %[[SIZE:.*]] = shape.rank %[[SHAPE]]
+  // CHECK: return %[[SIZE]] : index
+  %shape = shape.shape_of %arg : tensor<*xf32> -> tensor<?xindex>
+  %rank = shape.rank %shape : tensor<?xindex> -> index
+  return %rank : index
 }
 
+// -----
+
 // Canonicalize redundant conversion from `index` to `size` and back.
 // CHECK-LABEL: @index_to_size_to_index
 // CHECK-SAME: (%[[IDX:.*]]: index) -> index
@@ -558,9 +643,9 @@ func @cstr_broadcastable_scalar(%arg0 : tensor<?xf32>) {
   // CHECK-NEXT: shape.const_witness true
   // CHECK-NEXT: consume.witness
   // CHECK-NEXT: return
-  %0 = shape.const_shape []
-  %1 = shape.shape_of %arg0 : tensor<?xf32>
-  %2 = shape.cstr_broadcastable %0, %1
+  %0 = shape.const_shape [] : !shape.shape
+  %1 = shape.shape_of %arg0 : tensor<?xf32> -> tensor<?xindex>
+  %2 = shape.cstr_broadcastable %0, %1 : !shape.shape, tensor<?xindex>
   "consume.witness"(%2) : (!shape.witness) -> ()
   return
 }
@@ -575,9 +660,9 @@ func @cstr_broadcastable_unknown(%arg0 : tensor<?xf32>, %arg1 : tensor<?xf32>) {
   // CHECK-NEXT: shape.cstr_broadcastable
   // CHECK-NEXT: consume.witness
   // CHECK-NEXT: return
-  %0 = shape.shape_of %arg0 : tensor<?xf32>
-  %1 = shape.shape_of %arg1 : tensor<?xf32>
-  %2 = shape.cstr_broadcastable %0, %1
+  %0 = shape.shape_of %arg0 : tensor<?xf32> -> tensor<?xindex>
+  %1 = shape.shape_of %arg1 : tensor<?xf32> -> tensor<?xindex>
+  %2 = shape.cstr_broadcastable %0, %1 : tensor<?xindex>, tensor<?xindex>
   "consume.witness"(%2) : (!shape.witness) -> ()
   return
 }
@@ -590,9 +675,62 @@ func @cstr_broadcastable_scalar_unranked(%arg0 : tensor<*xf32>, %arg1 : tensor<i
   // CHECK-NEXT: shape.const_witness true
   // CHECK-NEXT: consume.witness
   // CHECK-NEXT: return
-  %0 = shape.shape_of %arg1 : tensor<index>
-  %1 = shape.shape_of %arg0 : tensor<*xf32>
-  %2 = shape.cstr_broadcastable %0, %1
+  %0 = shape.shape_of %arg1 : tensor<index> -> tensor<?xindex>
+  %1 = shape.shape_of %arg0 : tensor<*xf32> -> tensor<?xindex>
+  %2 = shape.cstr_broadcastable %0, %1 : tensor<?xindex>, tensor<?xindex>
   "consume.witness"(%2) : (!shape.witness) -> ()
   return
 }
+
+// -----
+
+// Fold `shape_eq` for equal and constant shapes.
+// CHECK-LABEL: @shape_eq_fold_1
+func @shape_eq_fold_1() -> i1 {
+  // CHECK: %[[RESULT:.*]] = constant true
+  // CHECK: return %[[RESULT]] : i1
+  %a = shape.const_shape [1, 2, 3] : !shape.shape
+  %b = shape.const_shape [1, 2, 3] : tensor<?xindex>
+  %result = shape.shape_eq %a, %b : !shape.shape, tensor<?xindex>
+  return %result : i1
+}
+
+// -----
+
+// Fold `shape_eq` for different but constant shapes of same length.
+// CHECK-LABEL: @shape_eq_fold_0
+func @shape_eq_fold_0() -> i1 {
+  // CHECK: %[[RESULT:.*]] = constant false
+  // CHECK: return %[[RESULT]] : i1
+  %a = shape.const_shape [1, 2, 3] : tensor<?xindex>
+  %b = shape.const_shape [4, 5, 6] : tensor<?xindex>
+  %result = shape.shape_eq %a, %b : tensor<?xindex>, tensor<?xindex>
+  return %result : i1
+}
+
+// -----
+
+// Fold `shape_eq` for different but constant shapes of different length.
+// CHECK-LABEL: @shape_eq_fold_0
+func @shape_eq_fold_0() -> i1 {
+  // CHECK: %[[RESULT:.*]] = constant false
+  // CHECK: return %[[RESULT]] : i1
+  %a = shape.const_shape [1, 2, 3, 4, 5, 6] : !shape.shape
+  %b = shape.const_shape [1, 2, 3] : !shape.shape
+  %result = shape.shape_eq %a, %b : !shape.shape, !shape.shape
+  return %result : i1
+}
+
+// -----
+
+// Do not fold `shape_eq` for non-constant shapes.
+// CHECK-LABEL: @shape_eq_do_not_fold
+// CHECK-SAME: (%[[A:.*]]: !shape.shape) -> i1
+func @shape_eq_do_not_fold(%a : !shape.shape) -> i1 {
+  // CHECK: %[[B:.*]] = shape.const_shape [4, 5, 6]
+  // CHECK: %[[RESULT:.*]] = shape.shape_eq %[[A]], %[[B]] : !shape.shape, !shape.shape
+  // CHECK: return %[[RESULT]] : i1
+  %b = shape.const_shape [4, 5, 6] : !shape.shape
+  %result = shape.shape_eq %a, %b : !shape.shape, !shape.shape
+  return %result : i1
+}
diff --git a/mlir/test/Dialect/Shape/invalid.mlir b/mlir/test/Dialect/Shape/invalid.mlir
index 3aca3677c143f..d7e9e40ed3f2e 100644
--- a/mlir/test/Dialect/Shape/invalid.mlir
+++ b/mlir/test/Dialect/Shape/invalid.mlir
@@ -78,3 +78,45 @@ func @assuming_all_op_too_few_operands() {
   %w0 = shape.assuming_all
   return
 }
+
+// -----
+
+func @shape_of(%value_arg : !shape.value_shape,
+               %shaped_arg : tensor<?x3x4xf32>) {
+  // expected-error@+1 {{if operand is of type `value_shape` then the result must be of type `shape` to propagate potential error shapes}}
+  %0 = shape.shape_of %value_arg : !shape.value_shape -> tensor<?xindex>
+}
+
+// -----
+
+func @shape_of(%value_arg : !shape.value_shape,
+               %shaped_arg : tensor<?x3x4xf32>) {
+  // expected-error@+1 {{if operand is a shaped type then the result must be an extent tensor}}
+  %1 = shape.shape_of %shaped_arg : tensor<?x3x4xf32> -> !shape.shape
+}
+
+// -----
+
+func @rank(%arg : !shape.shape) {
+  // expected-error@+1 {{if operand is of type `shape` then the result must be of type `size` to propagate potential errors}}
+  %0 = shape.rank %arg : !shape.shape -> index
+}
+
+// -----
+
+func @get_extent_error_free(%arg : tensor<?xindex>) -> !shape.size {
+  %c0 = constant 0 : index
+  // expected-error@+1 {{if none of the operands can hold error values then the result must be of type `index`}}
+  %result = shape.get_extent %arg, %c0 : tensor<?xindex>, index -> !shape.size
+  return %result : !shape.size
+}
+
+// -----
+
+func @get_extent_error_possible(%arg : tensor<?xindex>) -> index {
+  %c0 = shape.const_size 0
+  // expected-error@+1 {{if at least one of the operands can hold error values then the result must be of type `size` to propagate them}}
+  %result = shape.get_extent %arg, %c0 : tensor<?xindex>, !shape.size -> index
+  return %result : index
+}
+
diff --git a/mlir/test/Dialect/Shape/ops.mlir b/mlir/test/Dialect/Shape/ops.mlir
index c6f52519ad2e8..b6b839251a884 100644
--- a/mlir/test/Dialect/Shape/ops.mlir
+++ b/mlir/test/Dialect/Shape/ops.mlir
@@ -1,4 +1,3 @@
-// RUN: mlir-opt -split-input-file %s | mlir-opt | FileCheck %s
 // Verify the printed output can be parsed.
 // RUN: mlir-opt %s | mlir-opt | FileCheck %s
 // Verify the generic form can be parsed.
@@ -33,71 +32,84 @@ func @test_shape_num_elements_unknown() {
   return
 }
 
+func @const_shape() {
+  %0 = shape.const_shape [1, 2, 3] : !shape.shape
+  %1 = shape.const_shape [4, 5, 6] : tensor<?xindex>
+  return
+}
+
 func @test_shape_num_elements_fixed() {
-  %0 = shape.const_shape [1, 57, 92]
+  %0 = shape.const_shape [1, 57, 92] : !shape.shape
   %1 = call @shape_num_elements(%0) : (!shape.shape) -> (!shape.size)
   %3 = "shape.print"(%1) : (!shape.size) -> !shape.size
   return
 }
 
 func @test_broadcast_fixed() {
-  %0 = shape.const_shape [10, 1, 57, 92]
-  %1 = shape.const_shape [4, 57, 92]
+  %0 = shape.const_shape [10, 1, 57, 92] : !shape.shape
+  %1 = shape.const_shape [4, 57, 92] : !shape.shape
   %2 = shape.broadcast %0, %1
   %3 = "shape.print"(%2) : (!shape.shape) -> !shape.shape
   return
 }
 
 func @test_shape_any_fixed() {
-  %0 = shape.const_shape [4, 57, 92]
-  %1 = shape.const_shape [4, 57, 92]
+  %0 = shape.const_shape [4, 57, 92] : !shape.shape
+  %1 = shape.const_shape [4, 57, 92] : !shape.shape
   %2 = "shape.join"(%0, %1) : (!shape.shape, !shape.shape) -> !shape.shape
   %3 = "shape.print"(%2) : (!shape.shape) -> !shape.shape
   return
 }
 
 func @test_shape_any_unknown() {
-  %0 = shape.const_shape [4, -1, 92]
-  %1 = shape.const_shape [-1, 57, 92]
+  %0 = shape.const_shape [4, -1, 92] : !shape.shape
+  %1 = shape.const_shape [-1, 57, 92] : !shape.shape
   %2 = "shape.join"(%0, %1) : (!shape.shape, !shape.shape) -> !shape.shape
   %3 = "shape.print"(%2) : (!shape.shape) -> !shape.shape
   return
 }
 
 func @test_shape_any_fixed_mismatch() {
-  %0 = shape.const_shape [4, 57, 92]
-  %1 = shape.const_shape [2, 57, 92]
+  %0 = shape.const_shape [4, 57, 92] : !shape.shape
+  %1 = shape.const_shape [2, 57, 92] : !shape.shape
   %2 = "shape.join"(%0, %1) : (!shape.shape, !shape.shape) -> !shape.shape
   %3 = "shape.print"(%2) : (!shape.shape) -> !shape.shape
   return
 }
 
 func @test_parse_const_shape() {
-  %0 = shape.const_shape []
-  %1 = shape.const_shape [1, 2, 3]
+  %0 = shape.const_shape [] : !shape.shape
+  %1 = shape.const_shape [1, 2, 3] : !shape.shape
+  %2 = shape.const_shape [1, 2, 3] : tensor<?xindex>
   return
 }
 
-func @test_shape_of(%arg0: tensor<?xf32>) -> !shape.shape {
-  %0 = shape.shape_of %arg0 : tensor<?xf32>
-  return %0 : !shape.shape
+func @test_shape_of(%arg0: tensor<?xf32>) -> tensor<?xindex> {
+  %0 = shape.shape_of %arg0 : tensor<?xf32> -> tensor<?xindex>
+  return %0 : tensor<?xindex>
 }
 
 func @test_constraints() {
-  %0 = shape.const_shape []
-  %1 = shape.const_shape [1, 2, 3]
-  %w0 = shape.cstr_broadcastable %0, %1
+  %0 = shape.const_shape [] : !shape.shape
+  %1 = shape.const_shape [1, 2, 3] : !shape.shape
+  %w0 = shape.cstr_broadcastable %0, %1 : !shape.shape, !shape.shape
   %w1 = shape.cstr_eq %0, %1
   %w2 = shape.const_witness true
   %w3 = shape.const_witness false
   %w4 = shape.assuming_all %w0, %w1, %w2, %w3
   shape.assuming %w4 -> !shape.shape {
-    %2 = shape.any %0, %1
+    %2 = shape.any %0, %1 : !shape.shape
     shape.assuming_yield %2 : !shape.shape
   }
   return
 }
 
+func @broadcastable_on_extent_tensors(%lhs : tensor<?xindex>,
+                                      %rhs : tensor<?xindex>) {
+  %w0 = shape.cstr_broadcastable %lhs, %rhs : tensor<?xindex>, tensor<?xindex>
+  return
+}
+
 func @test_mul(%lhs: !shape.size, %rhs: !shape.size) -> !shape.size {
   %product = shape.mul %lhs, %rhs
   return %product: !shape.size
@@ -124,10 +136,16 @@ func @test_from_extent_tensor(%arg: tensor<?xindex>) -> !shape.shape {
 }
 
 func @rank(%shape : !shape.shape) -> !shape.size {
-  %rank = shape.rank %shape
+  %rank = shape.rank %shape : !shape.shape -> !shape.size
   return %rank : !shape.size
 }
 
+func @rank_on_extent_tensor(%shape : tensor<?xindex>) -> index {
+  %rank = shape.rank %shape : tensor<?xindex> -> index
+  return %rank : index
+}
+
+
 func @shape_eq_on_shapes(%a : !shape.shape, %b : !shape.shape) -> i1 {
   %result = shape.shape_eq %a, %b : !shape.shape, !shape.shape
   return %result : i1
@@ -142,3 +160,33 @@ func @shape_eq_on_mixed(%a : tensor<?xindex>, %b : !shape.shape) -> i1 {
   %result = shape.shape_eq %a, %b : tensor<?xindex>, !shape.shape
   return %result : i1
 }
+
+func @get_extent_on_shape(%arg : !shape.shape) -> !shape.size {
+  %c0 = shape.const_size 0
+  %result = shape.get_extent %arg, %c0 :
+      !shape.shape, !shape.size -> !shape.size
+  return %result : !shape.size
+}
+
+func @get_extent_on_extent_tensor(%arg : tensor<?xindex>) -> index {
+  %c0 = constant 0 : index
+  %result = shape.get_extent %arg, %c0 : tensor<?xindex>, index -> index
+  return %result : index
+}
+
+func @get_extent_on_mixed_operands(%arg : tensor<?xindex>) -> !shape.size {
+  %c0 = shape.const_size 0
+  %result = shape.get_extent %arg, %c0 : tensor<?xindex>, !shape.size -> !shape.size
+  return %result : !shape.size
+}
+
+func @any() {
+  %0 = shape.const_shape [1, 2, 3] : !shape.shape
+  %1 = shape.const_shape [4, 5, 6] : !shape.shape
+  %2 = shape.any %0, %1 : !shape.shape
+  %3 = shape.const_shape [1, 2, 3] : tensor<?xindex>
+  %4 = shape.const_shape [4, 5, 6] : tensor<?xindex>
+  %5 = shape.any %3, %4 : tensor<?xindex>
+  return
+}
+
diff --git a/mlir/test/Dialect/Shape/remove-shape-constraints.mlir b/mlir/test/Dialect/Shape/remove-shape-constraints.mlir
index 69887c6994f4d..31bb7bd562554 100644
--- a/mlir/test/Dialect/Shape/remove-shape-constraints.mlir
+++ b/mlir/test/Dialect/Shape/remove-shape-constraints.mlir
@@ -11,7 +11,7 @@ func @f(%arg0 : !shape.shape, %arg1 : !shape.shape) -> index {
   // REPLACE: shape.assuming %[[WITNESS]]
   // CANON-NEXT: test.source
   // CANON-NEXT: return
-  %0 = shape.cstr_broadcastable %arg0, %arg1
+  %0 = shape.cstr_broadcastable %arg0, %arg1 : !shape.shape, !shape.shape
   %1 = shape.assuming %0 -> index {
     %2 = "test.source"() : () -> (index)
     shape.assuming_yield %2 : index
@@ -45,7 +45,7 @@ func @f(%arg0 : !shape.shape, %arg1 : !shape.shape) -> index {
 func @f(%arg0 : !shape.shape, %arg1 : !shape.shape) -> index {
   // CANON-NEXT: test.source
   // CANON-NEXT: return
-  %0 = shape.cstr_broadcastable %arg0, %arg1
+  %0 = shape.cstr_broadcastable %arg0, %arg1 : !shape.shape, !shape.shape
   %1 = shape.cstr_eq %arg0, %arg1
   %2 = shape.assuming_all %0, %1
   %3 = shape.assuming %0 -> index {
diff --git a/mlir/test/Dialect/Vector/invalid.mlir b/mlir/test/Dialect/Vector/invalid.mlir
index 916403800fe15..ea354f51645af 100644
--- a/mlir/test/Dialect/Vector/invalid.mlir
+++ b/mlir/test/Dialect/Vector/invalid.mlir
@@ -1177,3 +1177,66 @@ func @type_cast_layout(%arg0: memref<4x3xf32, affine_map<(d0, d1)[s0, s1, s2] ->
   // expected-error@+1 {{expects operand to be a memref with no layout}}
   %0 = vector.type_cast %arg0: memref<4x3xf32, affine_map<(d0, d1)[s0, s1, s2] -> (d0 * s0 + d1 * s1 + s2)>> to memref<vector<4x3xf32>>
 }
+
+// -----
+
+func @gather_base_type_mismatch(%base: memref<?xf64>, %indices: vector<16xi32>, %mask: vector<16xi1>) {
+  // expected-error@+1 {{'vector.gather' op base and result element type should match}}
+  %0 = vector.gather %base, %indices, %mask : (memref<?xf64>, vector<16xi32>, vector<16xi1>) -> vector<16xf32>
+}
+
+// -----
+
+func @gather_rank_mismatch(%base: memref<?xf32>, %indices: vector<16xi32>, %mask: vector<16xi1>) {
+  // expected-error@+1 {{'vector.gather' op result #0 must be  of ranks 1, but got 'vector<2x16xf32>'}}
+  %0 = vector.gather %base, %indices, %mask : (memref<?xf32>, vector<16xi32>, vector<16xi1>) -> vector<2x16xf32>
+}
+
+// -----
+
+func @gather_dim_indices_mismatch(%base: memref<?xf32>, %indices: vector<17xi32>, %mask: vector<16xi1>) {
+  // expected-error@+1 {{'vector.gather' op expected result dim to match indices dim}}
+  %0 = vector.gather %base, %indices, %mask : (memref<?xf32>, vector<17xi32>, vector<16xi1>) -> vector<16xf32>
+}
+
+// -----
+
+func @gather_dim_mask_mismatch(%base: memref<?xf32>, %indices: vector<16xi32>, %mask: vector<17xi1>) {
+  // expected-error@+1 {{'vector.gather' op expected result dim to match mask dim}}
+  %0 = vector.gather %base, %indices, %mask : (memref<?xf32>, vector<16xi32>, vector<17xi1>) -> vector<16xf32>
+}
+
+// -----
+
+func @gather_pass_thru_type_mismatch(%base: memref<?xf32>, %indices: vector<16xi32>, %mask: vector<16xi1>, %pass_thru: vector<16xf64>) {
+  // expected-error@+1 {{'vector.gather' op expected pass_thru of same type as result type}}
+  %0 = vector.gather %base, %indices, %mask, %pass_thru : (memref<?xf32>, vector<16xi32>, vector<16xi1>, vector<16xf64>) -> vector<16xf32>
+}
+
+// -----
+
+func @scatter_base_type_mismatch(%base: memref<?xf64>, %indices: vector<16xi32>, %mask: vector<16xi1>, %value: vector<16xf32>) {
+  // expected-error@+1 {{'vector.scatter' op base and value element type should match}}
+  vector.scatter %base, %indices, %mask, %value : vector<16xi32>, vector<16xi1>, vector<16xf32> into memref<?xf64>
+}
+
+// -----
+
+func @scatter_rank_mismatch(%base: memref<?xf32>, %indices: vector<16xi32>, %mask: vector<16xi1>, %value: vector<2x16xf32>) {
+  // expected-error@+1 {{'vector.scatter' op operand #3 must be  of ranks 1, but got 'vector<2x16xf32>'}}
+  vector.scatter %base, %indices, %mask, %value : vector<16xi32>, vector<16xi1>, vector<2x16xf32> into memref<?xf32>
+}
+
+// -----
+
+func @scatter_dim_indices_mismatch(%base: memref<?xf32>, %indices: vector<17xi32>, %mask: vector<16xi1>, %value: vector<16xf32>) {
+  // expected-error@+1 {{'vector.scatter' op expected value dim to match indices dim}}
+  vector.scatter %base, %indices, %mask, %value : vector<17xi32>, vector<16xi1>, vector<16xf32> into memref<?xf32>
+}
+
+// -----
+
+func @scatter_dim_mask_mismatch(%base: memref<?xf32>, %indices: vector<16xi32>, %mask: vector<17xi1>, %value: vector<16xf32>) {
+  // expected-error@+1 {{'vector.scatter' op expected value dim to match mask dim}}
+  vector.scatter %base, %indices, %mask, %value : vector<16xi32>, vector<17xi1>, vector<16xf32> into memref<?xf32>
+}
diff --git a/mlir/test/Dialect/Vector/ops.mlir b/mlir/test/Dialect/Vector/ops.mlir
index 4ea72864ea94e..0bf4ed8f84c7c 100644
--- a/mlir/test/Dialect/Vector/ops.mlir
+++ b/mlir/test/Dialect/Vector/ops.mlir
@@ -368,3 +368,14 @@ func @flat_transpose_int(%arg0: vector<16xi32>) -> vector<16xi32> {
   // CHECK: return %[[X]] : vector<16xi32>
   return %0 : vector<16xi32>
 }
+
+// CHECK-LABEL: @gather_and_scatter
+func @gather_and_scatter(%base: memref<?xf32>, %indices: vector<16xi32>, %mask: vector<16xi1>) {
+  // CHECK: %[[X:.*]] = vector.gather %{{.*}}, %{{.*}}, %{{.*}} : (memref<?xf32>, vector<16xi32>, vector<16xi1>) -> vector<16xf32>
+  %0 = vector.gather %base, %indices, %mask : (memref<?xf32>, vector<16xi32>, vector<16xi1>) -> vector<16xf32>
+  // CHECK: %[[Y:.*]] = vector.gather %{{.*}}, %{{.*}}, %{{.*}}, %[[X]] : (memref<?xf32>, vector<16xi32>, vector<16xi1>, vector<16xf32>) -> vector<16xf32>
+  %1 = vector.gather %base, %indices, %mask, %0 : (memref<?xf32>, vector<16xi32>, vector<16xi1>, vector<16xf32>) -> vector<16xf32>
+  // CHECK: vector.scatter %{{.*}}, %{{.*}}, %{{.*}}, %[[Y]] : vector<16xi32>, vector<16xi1>, vector<16xf32> into memref<?xf32>
+  vector.scatter %base, %indices, %mask, %1 : vector<16xi32>, vector<16xi1>, vector<16xf32> into memref<?xf32>
+  return
+}
diff --git a/mlir/test/EDSC/builder-api-test.cpp b/mlir/test/EDSC/builder-api-test.cpp
index 73f7adeeaf711..3fcfcf24ef8fe 100644
--- a/mlir/test/EDSC/builder-api-test.cpp
+++ b/mlir/test/EDSC/builder-api-test.cpp
@@ -457,6 +457,44 @@ TEST_FUNC(diviu_op_i32) {
   f.erase();
 }
 
+TEST_FUNC(fpext_f32_f64) {
+  using namespace edsc::op;
+  auto f = makeFunction("fpext", {}, {});
+
+  OpBuilder builder(f.getBody());
+  ScopedContext scope(builder, f.getLoc());
+  auto f32Type = builder.getF32Type();
+  auto f64Type = builder.getF64Type();
+  std_fpext(std_constant_float(llvm::APFloat(10.0f), f32Type), f64Type);
+
+  // clang-format off
+  // CHECK-LABEL: @fpext
+  //       CHECK: {{.*}} = constant 1.0
+  //  CHECK-NEXT: {{.*}} = fpext
+  // clang-format on
+  f.print(llvm::outs());
+  f.erase();
+}
+
+TEST_FUNC(fptrunc_f32_bf16) {
+  using namespace edsc::op;
+  auto f = makeFunction("fptrunc", {}, {});
+
+  OpBuilder builder(f.getBody());
+  ScopedContext scope(builder, f.getLoc());
+  auto f32Type = builder.getF32Type();
+  auto bf16Type = builder.getBF16Type();
+  std_fptrunc(std_constant_float(llvm::APFloat(10.0f), f32Type), bf16Type);
+
+  // clang-format off
+  // CHECK-LABEL: @fptrunc
+  //       CHECK: {{.*}} = constant 1.0
+  //  CHECK-NEXT: {{.*}} = fptrunc
+  // clang-format on
+  f.print(llvm::outs());
+  f.erase();
+}
+
 TEST_FUNC(select_op_i32) {
   using namespace edsc::op;
   auto i32Type = IntegerType::get(32, &globalContext());
diff --git a/mlir/test/Target/import.ll b/mlir/test/Target/import.ll
index 4f4e822eb78ef..24b4a0b392b0f 100644
--- a/mlir/test/Target/import.ll
+++ b/mlir/test/Target/import.ll
@@ -64,11 +64,20 @@
 ; CHECK: llvm.mlir.global internal constant @nested_array_vector(dense<[{{\[}}[1, 2], [3, 4]]]> : vector<1x2x2xi32>) : !llvm<"[1 x [2 x <2 x i32>]]">
 @nested_array_vector = internal constant [1 x [2 x <2 x i32>]] [[2 x <2 x i32>] [<2 x i32> <i32 1, i32 2>, <2 x i32> <i32 3, i32 4>]]
 
+;
+; Linkage on functions.
+;
+
+; CHECK: llvm.func internal @func_internal
+define internal void @func_internal() {
+  ret void
+}
+
 ; CHECK: llvm.func @fe(!llvm.i32) -> !llvm.float
 declare float @fe(i32)
 
 ; FIXME: function attributes.
-; CHECK-LABEL: llvm.func @f1(%arg0: !llvm.i64) -> !llvm.i32 {
+; CHECK-LABEL: llvm.func internal @f1(%arg0: !llvm.i64) -> !llvm.i32 {
 ; CHECK-DAG: %[[c2:[0-9]+]] = llvm.mlir.constant(2 : i32) : !llvm.i32
 ; CHECK-DAG: %[[c42:[0-9]+]] = llvm.mlir.constant(42 : i32) : !llvm.i32
 ; CHECK-DAG: %[[c1:[0-9]+]] = llvm.mlir.constant(true) : !llvm.i1
diff --git a/mlir/test/Target/llvmir-intrinsics.mlir b/mlir/test/Target/llvmir-intrinsics.mlir
index 1595edffb45b9..79b7edb3c7010 100644
--- a/mlir/test/Target/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/llvmir-intrinsics.mlir
@@ -206,6 +206,20 @@ llvm.func @masked_intrinsics(%A: !llvm<"<7 x float>*">, %mask: !llvm<"<7 x i1>">
   llvm.return
 }
 
+// CHECK-LABEL: @masked_gather_scatter_intrinsics
+llvm.func @masked_gather_scatter_intrinsics(%M: !llvm<"<7 x float*>">, %mask: !llvm<"<7 x i1>">) {
+  // CHECK: call <7 x float> @llvm.masked.gather.v7f32.v7p0f32(<7 x float*> %{{.*}}, i32 1, <7 x i1> %{{.*}}, <7 x float> undef)
+  %a = llvm.intr.masked.gather %M, %mask { alignment = 1: i32} :
+      (!llvm<"<7 x float*>">, !llvm<"<7 x i1>">) -> !llvm<"<7 x float>">
+  // CHECK: call <7 x float> @llvm.masked.gather.v7f32.v7p0f32(<7 x float*> %{{.*}}, i32 1, <7 x i1> %{{.*}}, <7 x float> %{{.*}})
+  %b = llvm.intr.masked.gather %M, %mask, %a { alignment = 1: i32} :
+      (!llvm<"<7 x float*>">, !llvm<"<7 x i1>">, !llvm<"<7 x float>">) -> !llvm<"<7 x float>">
+  // CHECK: call void @llvm.masked.scatter.v7f32.v7p0f32(<7 x float> %{{.*}}, <7 x float*> %{{.*}}, i32 1, <7 x i1> %{{.*}})
+  llvm.intr.masked.scatter %b, %M, %mask { alignment = 1: i32} :
+      !llvm<"<7 x float>">, !llvm<"<7 x i1>"> into !llvm<"<7 x float*>">
+  llvm.return
+}
+
 // CHECK-LABEL: @memcpy_test
 llvm.func @memcpy_test(%arg0: !llvm.i32, %arg1: !llvm.i1, %arg2: !llvm<"i8*">, %arg3: !llvm<"i8*">) {
   // CHECK: call void @llvm.memcpy.p0i8.p0i8.i32(i8* %{{.*}}, i8* %{{.*}}, i32 %{{.*}}, i1 %{{.*}})
diff --git a/mlir/test/Target/llvmir.mlir b/mlir/test/Target/llvmir.mlir
index 05b8ef10588ac..954b5b1345412 100644
--- a/mlir/test/Target/llvmir.mlir
+++ b/mlir/test/Target/llvmir.mlir
@@ -61,7 +61,8 @@ llvm.mlir.global external @external() : !llvm.i32
 
 
 //
-// Declarations of the allocation functions to be linked against.
+// Declarations of the allocation functions to be linked against. These are
+// inserted before other functions in the module.
 //
 
 // CHECK: declare i8* @malloc(i64)
@@ -390,6 +391,17 @@ llvm.func @more_imperfectly_nested_loops() {
   llvm.return
 }
 
+
+//
+// Check that linkage is translated for functions. No need to check all linkage
+// flags since the logic is the same as for globals.
+//
+
+// CHECK: define internal void @func_internal
+llvm.func internal @func_internal() {
+  llvm.return
+}
+
 //
 // MemRef type conversion, allocation and communication with functions.
 //
@@ -1240,3 +1252,17 @@ llvm.mlir.global internal constant @taker_of_address() : !llvm<"void()*"> {
   %0 = llvm.mlir.addressof @address_taken : !llvm<"void()*">
   llvm.return %0 : !llvm<"void()*">
 }
+
+// -----
+
+// Check that branch weight attributes are exported properly as metadata.
+llvm.func @cond_br_weights(%cond : !llvm.i1, %arg0 : !llvm.i32,  %arg1 : !llvm.i32) -> !llvm.i32 {
+  // CHECK: !prof ![[NODE:[0-9]+]]
+  llvm.cond_br %cond weights(dense<[5, 10]> : vector<2xi32>), ^bb1, ^bb2
+^bb1:  // pred: ^bb0
+  llvm.return %arg0 : !llvm.i32
+^bb2:  // pred: ^bb0
+  llvm.return %arg1 : !llvm.i32
+}
+
+// CHECK: ![[NODE]] = !{!"branch_weights", i32 5, i32 10}
diff --git a/mlir/test/Transforms/memref-normalize.mlir b/mlir/test/Transforms/memref-normalize.mlir
index 6254899b1fe8e..375bd3ef0e6c6 100644
--- a/mlir/test/Transforms/memref-normalize.mlir
+++ b/mlir/test/Transforms/memref-normalize.mlir
@@ -143,3 +143,10 @@ func @semi_affine_layout_map(%s0: index, %s1: index) {
   }
   return
 }
+
+// CHECK-LABEL: func @alignment
+func @alignment() {
+  %A = alloc() {alignment = 32 : i64}: memref<64x128x256xf32, affine_map<(d0, d1, d2) -> (d2, d0, d1)>>
+  // CHECK-NEXT: alloc() {alignment = 32 : i64} : memref<256x64x128xf32>
+  return
+}
diff --git a/mlir/test/Transforms/test-legalize-type-conversion.mlir b/mlir/test/Transforms/test-legalize-type-conversion.mlir
new file mode 100644
index 0000000000000..c56b3c8ca1e2d
--- /dev/null
+++ b/mlir/test/Transforms/test-legalize-type-conversion.mlir
@@ -0,0 +1,64 @@
+// RUN: mlir-opt %s -test-legalize-type-conversion -allow-unregistered-dialect -split-input-file -verify-diagnostics | FileCheck %s
+
+// expected-error@below {{failed to materialize conversion for block argument #0 that remained live after conversion, type was 'i16'}}
+func @test_invalid_arg_materialization(%arg0: i16) {
+  // expected-note@below {{see existing live user here}}
+  "foo.return"(%arg0) : (i16) -> ()
+}
+
+// -----
+
+// expected-error@below {{failed to legalize conversion operation generated for block argument}}
+func @test_invalid_arg_illegal_materialization(%arg0: i32) {
+  "foo.return"(%arg0) : (i32) -> ()
+}
+
+// -----
+
+// CHECK-LABEL: func @test_valid_arg_materialization
+func @test_valid_arg_materialization(%arg0: i64) {
+  // CHECK: %[[ARG:.*]] = "test.type_producer"
+  // CHECK: "foo.return"(%[[ARG]]) : (i64)
+
+  "foo.return"(%arg0) : (i64) -> ()
+}
+
+// -----
+
+func @test_invalid_result_materialization() {
+  // expected-error@below {{failed to materialize conversion for result #0 of operation 'test.type_producer' that remained live after conversion}}
+  %result = "test.type_producer"() : () -> f16
+
+  // expected-note@below {{see existing live user here}}
+  "foo.return"(%result) : (f16) -> ()
+}
+
+// -----
+
+func @test_invalid_result_materialization() {
+  // expected-error@below {{failed to materialize conversion for result #0 of operation 'test.type_producer' that remained live after conversion}}
+  %result = "test.type_producer"() : () -> f16
+
+  // expected-note@below {{see existing live user here}}
+  "foo.return"(%result) : (f16) -> ()
+}
+
+// -----
+
+func @test_invalid_result_legalization() {
+  // expected-error@below {{failed to legalize conversion operation generated for result #0 of operation 'test.type_producer' that remained live after conversion}}
+  %result = "test.type_producer"() : () -> i16
+  "foo.return"(%result) : (i16) -> ()
+}
+
+// -----
+
+// CHECK-LABEL: func @test_valid_result_legalization
+func @test_valid_result_legalization() {
+  // CHECK: %[[RESULT:.*]] = "test.type_producer"() : () -> f64
+  // CHECK: %[[CAST:.*]] = "test.cast"(%[[RESULT]]) : (f64) -> f32
+  // CHECK: "foo.return"(%[[CAST]]) : (f32)
+
+  %result = "test.type_producer"() : () -> f32
+  "foo.return"(%result) : (f32) -> ()
+}
diff --git a/mlir/test/lib/Dialect/Test/TestPatterns.cpp b/mlir/test/lib/Dialect/Test/TestPatterns.cpp
index 255b1c152a365..5bc947fc8c916 100644
--- a/mlir/test/lib/Dialect/Test/TestPatterns.cpp
+++ b/mlir/test/lib/Dialect/Test/TestPatterns.cpp
@@ -485,8 +485,9 @@ struct TestTypeConverter : public TypeConverter {
   using TypeConverter::TypeConverter;
   TestTypeConverter() {
     addConversion(convertType);
-    addMaterialization(materializeCast);
-    addMaterialization(materializeOneToOneCast);
+    addArgumentMaterialization(materializeCast);
+    addArgumentMaterialization(materializeOneToOneCast);
+    addSourceMaterialization(materializeCast);
   }
 
   static LogicalResult convertType(Type t, SmallVectorImpl<Type> &results) {
@@ -519,21 +520,20 @@ struct TestTypeConverter : public TypeConverter {
 
   /// Hook for materializing a conversion. This is necessary because we generate
   /// 1->N type mappings.
-  static Optional<Value> materializeCast(PatternRewriter &rewriter,
-                                         Type resultType, ValueRange inputs,
-                                         Location loc) {
+  static Optional<Value> materializeCast(OpBuilder &builder, Type resultType,
+                                         ValueRange inputs, Location loc) {
     if (inputs.size() == 1)
       return inputs[0];
-    return rewriter.create<TestCastOp>(loc, resultType, inputs).getResult();
+    return builder.create<TestCastOp>(loc, resultType, inputs).getResult();
   }
 
   /// Materialize the cast for one-to-one conversion from i64 to f64.
-  static Optional<Value> materializeOneToOneCast(PatternRewriter &rewriter,
+  static Optional<Value> materializeOneToOneCast(OpBuilder &builder,
                                                  IntegerType resultType,
                                                  ValueRange inputs,
                                                  Location loc) {
     if (resultType.getWidth() == 42 && inputs.size() == 1)
-      return rewriter.create<TestCastOp>(loc, resultType, inputs).getResult();
+      return builder.create<TestCastOp>(loc, resultType, inputs).getResult();
     return llvm::None;
   }
 };
@@ -742,6 +742,102 @@ struct TestUnknownRootOpDriver
 };
 } // end anonymous namespace
 
+//===----------------------------------------------------------------------===//
+// Test type conversions
+//===----------------------------------------------------------------------===//
+
+namespace {
+struct TestTypeConversionProducer
+    : public OpConversionPattern<TestTypeProducerOp> {
+  using OpConversionPattern<TestTypeProducerOp>::OpConversionPattern;
+  LogicalResult
+  matchAndRewrite(TestTypeProducerOp op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const final {
+    Type resultType = op.getType();
+    if (resultType.isa<FloatType>())
+      resultType = rewriter.getF64Type();
+    else if (resultType.isInteger(16))
+      resultType = rewriter.getIntegerType(64);
+    else
+      return failure();
+
+    rewriter.replaceOpWithNewOp<TestTypeProducerOp>(op, resultType);
+    return success();
+  }
+};
+
+struct TestTypeConversionDriver
+    : public PassWrapper<TestTypeConversionDriver, OperationPass<ModuleOp>> {
+  void runOnOperation() override {
+    // Initialize the type converter.
+    TypeConverter converter;
+
+    /// Add the legal set of type conversions.
+    converter.addConversion([](Type type) -> Type {
+      // Treat F64 as legal.
+      if (type.isF64())
+        return type;
+      // Allow converting BF16/F16/F32 to F64.
+      if (type.isBF16() || type.isF16() || type.isF32())
+        return FloatType::getF64(type.getContext());
+      // Otherwise, the type is illegal.
+      return nullptr;
+    });
+    converter.addConversion([](IntegerType type, SmallVectorImpl<Type> &) {
+      // Drop all integer types.
+      return success();
+    });
+
+    /// Add the legal set of type materializations.
+    converter.addSourceMaterialization([](OpBuilder &builder, Type resultType,
+                                          ValueRange inputs,
+                                          Location loc) -> Value {
+      // Allow casting from F64 back to F32.
+      if (!resultType.isF16() && inputs.size() == 1 &&
+          inputs[0].getType().isF64())
+        return builder.create<TestCastOp>(loc, resultType, inputs).getResult();
+      // Allow producing an i32 or i64 from nothing.
+      if ((resultType.isInteger(32) || resultType.isInteger(64)) &&
+          inputs.empty())
+        return builder.create<TestTypeProducerOp>(loc, resultType);
+      // Allow producing an i64 from an integer.
+      if (resultType.isa<IntegerType>() && inputs.size() == 1 &&
+          inputs[0].getType().isa<IntegerType>())
+        return builder.create<TestCastOp>(loc, resultType, inputs).getResult();
+      // Otherwise, fail.
+      return nullptr;
+    });
+
+    // Initialize the conversion target.
+    mlir::ConversionTarget target(getContext());
+    target.addDynamicallyLegalOp<TestTypeProducerOp>([](TestTypeProducerOp op) {
+      return op.getType().isF64() || op.getType().isInteger(64);
+    });
+    target.addDynamicallyLegalOp<FuncOp>([&](FuncOp op) {
+      return converter.isSignatureLegal(op.getType()) &&
+             converter.isLegal(&op.getBody());
+    });
+    target.addDynamicallyLegalOp<TestCastOp>([&](TestCastOp op) {
+      // Allow casts from F64 to F32.
+      return (*op.operand_type_begin()).isF64() && op.getType().isF32();
+    });
+
+    // Initialize the set of rewrite patterns.
+    OwningRewritePatternList patterns;
+    patterns.insert<TestTypeConversionProducer>(converter, &getContext());
+    mlir::populateFuncOpTypeConversionPattern(patterns, &getContext(),
+                                              converter);
+
+    if (failed(applyPartialConversion(getOperation(), target, patterns)))
+      signalPassFailure();
+  }
+};
+} // end anonymous namespace
+
+//===----------------------------------------------------------------------===//
+// PassRegistration
+//===----------------------------------------------------------------------===//
+
 namespace mlir {
 void registerPatternsTestPass() {
   PassRegistration<TestReturnTypeDriver>("test-return-type",
@@ -766,5 +862,9 @@ void registerPatternsTestPass() {
   PassRegistration<TestUnknownRootOpDriver>(
       "test-legalize-unknown-root-patterns",
       "Test public remapped value mechanism in ConversionPatternRewriter");
+
+  PassRegistration<TestTypeConversionDriver>(
+      "test-legalize-type-conversion",
+      "Test various type conversion functionalities in DialectConversion");
 }
 } // namespace mlir
diff --git a/mlir/test/lib/Transforms/TestBufferPlacement.cpp b/mlir/test/lib/Transforms/TestBufferPlacement.cpp
index 2fbdfe989d050..5ad441aa15c3e 100644
--- a/mlir/test/lib/Transforms/TestBufferPlacement.cpp
+++ b/mlir/test/lib/Transforms/TestBufferPlacement.cpp
@@ -77,7 +77,8 @@ struct TestBufferPlacementPreparationPass
       auto linalgOp = rewriter.create<linalg::GenericOp>(
           loc, llvm::None, newArgs, rewriter.getI64IntegerAttr(operands.size()),
           rewriter.getI64IntegerAttr(results.size()), op.indexing_maps(),
-          op.iterator_types(), op.docAttr(), op.library_callAttr());
+          op.iterator_types(), op.docAttr(), op.library_callAttr(),
+          op.symbol_sourceAttr());
 
       // Create a new block in the region of the new Generic Op.
       Block &oldBlock = op.getRegion().front();
diff --git a/mlir/test/lib/Transforms/TestLinalgTransforms.cpp b/mlir/test/lib/Transforms/TestLinalgTransforms.cpp
index 4fb378c5ab8a5..e356eb72fa429 100644
--- a/mlir/test/lib/Transforms/TestLinalgTransforms.cpp
+++ b/mlir/test/lib/Transforms/TestLinalgTransforms.cpp
@@ -144,6 +144,7 @@ static void applyPatterns(FuncOp funcOp) {
   //===--------------------------------------------------------------------===//
   patterns.insert<LinalgVectorizationPattern<MatmulOp>,
                   LinalgVectorizationPattern<FillOp>,
+                  LinalgVectorizationPattern<CopyOp>,
                   LinalgVectorizationPattern<GenericOp>>(
       ctx, LinalgMarker(Identifier::get("VECTORIZE", ctx)));
 
diff --git a/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp b/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp
index 4049c42a62967..9edbad8fdd540 100644
--- a/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp
+++ b/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp
@@ -102,7 +102,7 @@ static bool isOperandName(const tblgen::Operator &op, StringRef name) {
 }
 
 // Emit to `os` the operator-name driven check and the call to LLVM IRBuilder
-// for one definition of a LLVM IR Dialect operation.  Return true on success.
+// for one definition of an LLVM IR Dialect operation.  Return true on success.
 static bool emitOneBuilder(const Record &record, raw_ostream &os) {
   auto op = tblgen::Operator(record);
 
diff --git a/mlir/unittests/CMakeLists.txt b/mlir/unittests/CMakeLists.txt
index 851092c5b56a4..1dc07413a8850 100644
--- a/mlir/unittests/CMakeLists.txt
+++ b/mlir/unittests/CMakeLists.txt
@@ -5,6 +5,10 @@ function(add_mlir_unittest test_dirname)
   add_unittest(MLIRUnitTests ${test_dirname} ${ARGN})
 endfunction()
 
+if (CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
+  add_compile_options("-Wno-suggest-override")
+endif()
+
 add_subdirectory(Analysis)
 add_subdirectory(Dialect)
 add_subdirectory(IR)
diff --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt
index 6f0d9e7280026..f18688f08c457 100644
--- a/openmp/CMakeLists.txt
+++ b/openmp/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.13.4)
 
 # Add cmake directory to search for custom cmake functions.
 set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH})
diff --git a/openmp/cmake/DetectTestCompiler/CMakeLists.txt b/openmp/cmake/DetectTestCompiler/CMakeLists.txt
index c230fc8d4cf1a..7fa32a90972ba 100644
--- a/openmp/cmake/DetectTestCompiler/CMakeLists.txt
+++ b/openmp/cmake/DetectTestCompiler/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8)
+cmake_minimum_required(VERSION 3.13.4)
 project(DetectTestCompiler C CXX)
 
 include(CheckCCompilerFlag)
diff --git a/openmp/libomptarget/include/omptarget.h b/openmp/libomptarget/include/omptarget.h
index 95d7158969f34..0751816cd9d9e 100644
--- a/openmp/libomptarget/include/omptarget.h
+++ b/openmp/libomptarget/include/omptarget.h
@@ -49,6 +49,8 @@ enum tgt_map_type {
   OMP_TGT_MAPTYPE_IMPLICIT        = 0x200,
   // copy data to device
   OMP_TGT_MAPTYPE_CLOSE           = 0x400,
+  // runtime error if not already allocated
+  OMP_TGT_MAPTYPE_PRESENT         = 0x1000,
   // member of struct, member given by [16 MSBs] - 1
   OMP_TGT_MAPTYPE_MEMBER_OF       = 0xffff000000000000
 };
@@ -259,14 +261,6 @@ void __kmpc_push_target_tripcount(int64_t device_id, uint64_t loop_tripcount);
 }
 #endif
 
-#ifdef OMPTARGET_DEBUG
-#include <stdio.h>
-#define DEBUGP(prefix, ...)                                                    \
-  {                                                                            \
-    fprintf(stderr, "%s --> ", prefix);                                        \
-    fprintf(stderr, __VA_ARGS__);                                              \
-  }
-
 #ifndef __STDC_FORMAT_MACROS
 #define __STDC_FORMAT_MACROS
 #endif
@@ -293,6 +287,14 @@ void __kmpc_push_target_tripcount(int64_t device_id, uint64_t loop_tripcount);
  *                               // 16 digits for 64bit
  *   (uintptr_t) ptr);
  */
+
+#ifdef OMPTARGET_DEBUG
+#include <stdio.h>
+#define DEBUGP(prefix, ...)                                                    \
+  {                                                                            \
+    fprintf(stderr, "%s --> ", prefix);                                        \
+    fprintf(stderr, __VA_ARGS__);                                              \
+  }
 #else
 #define DEBUGP(prefix, ...)                                                    \
   {}
diff --git a/openmp/libomptarget/src/device.cpp b/openmp/libomptarget/src/device.cpp
index 575354305dda0..867083fde4892 100644
--- a/openmp/libomptarget/src/device.cpp
+++ b/openmp/libomptarget/src/device.cpp
@@ -160,8 +160,10 @@ LookupResult DeviceTy::lookupMapping(void *HstPtrBegin, int64_t Size) {
 // If NULL is returned, then either data allocation failed or the user tried
 // to do an illegal mapping.
 void *DeviceTy::getOrAllocTgtPtr(void *HstPtrBegin, void *HstPtrBase,
-    int64_t Size, bool &IsNew, bool &IsHostPtr, bool IsImplicit,
-    bool UpdateRefCount, bool HasCloseModifier) {
+                                 int64_t Size, bool &IsNew, bool &IsHostPtr,
+                                 bool IsImplicit, bool UpdateRefCount,
+                                 bool HasCloseModifier,
+                                 bool HasPresentModifier) {
   void *rc = NULL;
   IsHostPtr = false;
   IsNew = false;
@@ -190,31 +192,40 @@ void *DeviceTy::getOrAllocTgtPtr(void *HstPtrBegin, void *HstPtrBase,
   } else if ((lr.Flags.ExtendsBefore || lr.Flags.ExtendsAfter) && !IsImplicit) {
     // Explicit extension of mapped data - not allowed.
     DP("Explicit extension of mapping is not allowed.\n");
-  } else if (Size) {
-    // If unified shared memory is active, implicitly mapped variables that are not
-    // privatized use host address. Any explicitly mapped variables also use
-    // host address where correctness is not impeded. In all other cases
-    // maps are respected.
-    // In addition to the mapping rules above, the close map
-    // modifier forces the mapping of the variable to the device.
-    if (RTLs->RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY &&
-        !HasCloseModifier) {
+  } else if (RTLs->RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY &&
+             !HasCloseModifier) {
+    // If unified shared memory is active, implicitly mapped variables that are
+    // not privatized use host address. Any explicitly mapped variables also use
+    // host address where correctness is not impeded. In all other cases maps
+    // are respected.
+    // In addition to the mapping rules above, the close map modifier forces the
+    // mapping of the variable to the device.
+    if (Size) {
       DP("Return HstPtrBegin " DPxMOD " Size=%ld RefCount=%s\n",
-         DPxPTR((uintptr_t)HstPtrBegin), Size, (UpdateRefCount ? " updated" : ""));
+         DPxPTR((uintptr_t)HstPtrBegin), Size,
+         (UpdateRefCount ? " updated" : ""));
       IsHostPtr = true;
       rc = HstPtrBegin;
-    } else {
-      // If it is not contained and Size > 0 we should create a new entry for it.
-      IsNew = true;
-      uintptr_t tp = (uintptr_t)RTL->data_alloc(RTLDeviceID, Size, HstPtrBegin);
-      DP("Creating new map entry: HstBase=" DPxMOD ", HstBegin=" DPxMOD ", "
-         "HstEnd=" DPxMOD ", TgtBegin=" DPxMOD "\n", DPxPTR(HstPtrBase),
-         DPxPTR(HstPtrBegin), DPxPTR((uintptr_t)HstPtrBegin + Size), DPxPTR(tp));
-      HostDataToTargetMap.emplace(
-          HostDataToTargetTy((uintptr_t)HstPtrBase, (uintptr_t)HstPtrBegin,
-                             (uintptr_t)HstPtrBegin + Size, tp));
-      rc = (void *)tp;
     }
+  } else if (HasPresentModifier) {
+    DP("Mapping required by 'present' map type modifier does not exist for "
+       "HstPtrBegin=" DPxMOD ", Size=%ld\n",
+       DPxPTR(HstPtrBegin), Size);
+    MESSAGE("device mapping required by 'present' map type modifier does not "
+            "exist for host address " DPxMOD " (%ld bytes)",
+            DPxPTR(HstPtrBegin), Size);
+  } else if (Size) {
+    // If it is not contained and Size > 0, we should create a new entry for it.
+    IsNew = true;
+    uintptr_t tp = (uintptr_t)RTL->data_alloc(RTLDeviceID, Size, HstPtrBegin);
+    DP("Creating new map entry: HstBase=" DPxMOD ", HstBegin=" DPxMOD ", "
+       "HstEnd=" DPxMOD ", TgtBegin=" DPxMOD "\n",
+       DPxPTR(HstPtrBase), DPxPTR(HstPtrBegin),
+       DPxPTR((uintptr_t)HstPtrBegin + Size), DPxPTR(tp));
+    HostDataToTargetMap.emplace(
+        HostDataToTargetTy((uintptr_t)HstPtrBase, (uintptr_t)HstPtrBegin,
+                           (uintptr_t)HstPtrBegin + Size, tp));
+    rc = (void *)tp;
   }
 
   DataMapMtx.unlock();
diff --git a/openmp/libomptarget/src/device.h b/openmp/libomptarget/src/device.h
index 309785acb4721..ebec76c3cf62a 100644
--- a/openmp/libomptarget/src/device.h
+++ b/openmp/libomptarget/src/device.h
@@ -177,8 +177,9 @@ struct DeviceTy {
   uint64_t getMapEntryRefCnt(void *HstPtrBegin);
   LookupResult lookupMapping(void *HstPtrBegin, int64_t Size);
   void *getOrAllocTgtPtr(void *HstPtrBegin, void *HstPtrBase, int64_t Size,
-      bool &IsNew, bool &IsHostPtr, bool IsImplicit, bool UpdateRefCount = true,
-      bool HasCloseModifier = false);
+                         bool &IsNew, bool &IsHostPtr, bool IsImplicit,
+                         bool UpdateRefCount, bool HasCloseModifier,
+                         bool HasPresentModifier);
   void *getTgtPtrBegin(void *HstPtrBegin, int64_t Size);
   void *getTgtPtrBegin(void *HstPtrBegin, int64_t Size, bool &IsLast,
       bool UpdateRefCount, bool &IsHostPtr);
diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp
index 6b4549be6ae11..47971b9c0a00a 100644
--- a/openmp/libomptarget/src/omptarget.cpp
+++ b/openmp/libomptarget/src/omptarget.cpp
@@ -308,6 +308,7 @@ int target_data_begin(DeviceTy &Device, int32_t arg_num, void **args_base,
     // Force the creation of a device side copy of the data when:
     // a close map modifier was associated with a map that contained a to.
     bool HasCloseModifier = arg_types[i] & OMP_TGT_MAPTYPE_CLOSE;
+    bool HasPresentModifier = arg_types[i] & OMP_TGT_MAPTYPE_PRESENT;
     // UpdateRef is based on MEMBER_OF instead of TARGET_PARAM because if we
     // have reached this point via __tgt_target_data_begin and not __tgt_target
     // then no argument is marked as TARGET_PARAM ("omp target data map" is not
@@ -316,13 +317,26 @@ int target_data_begin(DeviceTy &Device, int32_t arg_num, void **args_base,
     bool UpdateRef = !(arg_types[i] & OMP_TGT_MAPTYPE_MEMBER_OF);
     if (arg_types[i] & OMP_TGT_MAPTYPE_PTR_AND_OBJ) {
       DP("Has a pointer entry: \n");
-      // base is address of pointer.
-      Pointer_TgtPtrBegin = Device.getOrAllocTgtPtr(HstPtrBase, HstPtrBase,
-          sizeof(void *), Pointer_IsNew, IsHostPtr, IsImplicit, UpdateRef,
-          HasCloseModifier);
+      // Base is address of pointer.
+      //
+      // Usually, the pointer is already allocated by this time.  For example:
+      //
+      //   #pragma omp target map(s.p[0:N])
+      //
+      // The map entry for s comes first, and the PTR_AND_OBJ entry comes
+      // afterward, so the pointer is already allocated by the time the
+      // PTR_AND_OBJ entry is handled below, and Pointer_TgtPtrBegin is thus
+      // non-null.  However, "declare target link" can produce a PTR_AND_OBJ
+      // entry for a global that might not already be allocated by the time the
+      // PTR_AND_OBJ entry is handled below, and so the allocation might fail
+      // when HasPresentModifier.
+      Pointer_TgtPtrBegin = Device.getOrAllocTgtPtr(
+          HstPtrBase, HstPtrBase, sizeof(void *), Pointer_IsNew, IsHostPtr,
+          IsImplicit, UpdateRef, HasCloseModifier, HasPresentModifier);
       if (!Pointer_TgtPtrBegin) {
-        DP("Call to getOrAllocTgtPtr returned null pointer (device failure or "
-            "illegal mapping).\n");
+        DP("Call to getOrAllocTgtPtr returned null pointer (%s).\n",
+           HasPresentModifier ? "'present' map type modifier"
+                              : "device failure or illegal mapping");
         return OFFLOAD_FAIL;
       }
       DP("There are %zu bytes allocated at target address " DPxMOD " - is%s new"
@@ -334,13 +348,15 @@ int target_data_begin(DeviceTy &Device, int32_t arg_num, void **args_base,
       UpdateRef = true; // subsequently update ref count of pointee
     }
 
-    void *TgtPtrBegin = Device.getOrAllocTgtPtr(HstPtrBegin, HstPtrBase,
-        data_size, IsNew, IsHostPtr, IsImplicit, UpdateRef, HasCloseModifier);
-    if (!TgtPtrBegin && data_size) {
-      // If data_size==0, then the argument could be a zero-length pointer to
-      // NULL, so getOrAlloc() returning NULL is not an error.
-      DP("Call to getOrAllocTgtPtr returned null pointer (device failure or "
-          "illegal mapping).\n");
+    void *TgtPtrBegin = Device.getOrAllocTgtPtr(
+        HstPtrBegin, HstPtrBase, data_size, IsNew, IsHostPtr, IsImplicit,
+        UpdateRef, HasCloseModifier, HasPresentModifier);
+    // If data_size==0, then the argument could be a zero-length pointer to
+    // NULL, so getOrAlloc() returning NULL is not an error.
+    if (!TgtPtrBegin && (data_size || HasPresentModifier)) {
+      DP("Call to getOrAllocTgtPtr returned null pointer (%s).\n",
+         HasPresentModifier ? "'present' map type modifier"
+                            : "device failure or illegal mapping");
       return OFFLOAD_FAIL;
     }
     DP("There are %" PRId64 " bytes allocated at target address " DPxMOD
@@ -459,13 +475,27 @@ int target_data_end(DeviceTy &Device, int32_t arg_num, void **args_base,
         (arg_types[i] & OMP_TGT_MAPTYPE_PTR_AND_OBJ);
     bool ForceDelete = arg_types[i] & OMP_TGT_MAPTYPE_DELETE;
     bool HasCloseModifier = arg_types[i] & OMP_TGT_MAPTYPE_CLOSE;
+    bool HasPresentModifier = arg_types[i] & OMP_TGT_MAPTYPE_PRESENT;
 
     // If PTR_AND_OBJ, HstPtrBegin is address of pointee
     void *TgtPtrBegin = Device.getTgtPtrBegin(HstPtrBegin, data_size, IsLast,
         UpdateRef, IsHostPtr);
-    DP("There are %" PRId64 " bytes allocated at target address " DPxMOD
-        " - is%s last\n", data_size, DPxPTR(TgtPtrBegin),
-        (IsLast ? "" : " not"));
+    if (!TgtPtrBegin && (data_size || HasPresentModifier)) {
+      DP("Mapping does not exist (%s)\n",
+         (HasPresentModifier ? "'present' map type modifier" : "ignored"));
+      if (HasPresentModifier) {
+        // FIXME: This should not be an error on exit from "omp target data",
+        // but it should be an error upon entering an "omp target exit data".
+        MESSAGE("device mapping required by 'present' map type modifier does "
+                "not exist for host address " DPxMOD " (%ld bytes)",
+                DPxPTR(HstPtrBegin), data_size);
+        return OFFLOAD_FAIL;
+      }
+    } else {
+      DP("There are %" PRId64 " bytes allocated at target address " DPxMOD
+         " - is%s last\n",
+         data_size, DPxPTR(TgtPtrBegin), (IsLast ? "" : " not"));
+    }
 
     bool DelEntry = IsLast || ForceDelete;
 
diff --git a/openmp/libomptarget/src/private.h b/openmp/libomptarget/src/private.h
index cb20d8cdc7907..7772175cdca2e 100644
--- a/openmp/libomptarget/src/private.h
+++ b/openmp/libomptarget/src/private.h
@@ -80,9 +80,19 @@ typedef int (*TargetDataFuncPtrTy)(DeviceTy &, int32_t, void **, void **,
     int64_t *, int64_t *, void **, __tgt_async_info *);
 
 ////////////////////////////////////////////////////////////////////////////////
-// implementation for fatal messages
+// implementation for messages
 ////////////////////////////////////////////////////////////////////////////////
 
+#define MESSAGE0(_str)                                                         \
+  do {                                                                         \
+    fprintf(stderr, "Libomptarget message: %s\n", _str);                       \
+  } while (0)
+
+#define MESSAGE(_str, ...)                                                     \
+  do {                                                                         \
+    fprintf(stderr, "Libomptarget message: " _str "\n", __VA_ARGS__);          \
+  } while (0)
+
 #define FATAL_MESSAGE0(_num, _str)                                    \
   do {                                                                \
     fprintf(stderr, "Libomptarget fatal error %d: %s\n", _num, _str); \
diff --git a/openmp/libomptarget/test/lit.cfg b/openmp/libomptarget/test/lit.cfg
index 3cc0b83d27ba2..3dbe8789d5162 100644
--- a/openmp/libomptarget/test/lit.cfg
+++ b/openmp/libomptarget/test/lit.cfg
@@ -9,6 +9,14 @@ if 'PYLINT_IMPORT' in os.environ:
     config = object()
     lit_config = object()
 
+# Use the CUDA device as suggested by the env
+if 'CUDA_VISIBLE_DEVICES' in os.environ:
+    config.environment['CUDA_VISIBLE_DEVICES'] = os.environ['CUDA_VISIBLE_DEVICES']
+
+# Allow running the tests with omptarget debug output
+if 'LIBOMPTARGET_DEBUG' in os.environ:
+    config.environment['LIBOMPTARGET_DEBUG'] = os.environ['LIBOMPTARGET_DEBUG']
+
 def append_dynamic_library_path(name, value, sep):
     if name in config.environment:
         config.environment[name] = value + sep + config.environment[name]
diff --git a/openmp/libomptarget/test/mapping/present/target.c b/openmp/libomptarget/test/mapping/present/target.c
new file mode 100644
index 0000000000000..1d61dc06baa42
--- /dev/null
+++ b/openmp/libomptarget/test/mapping/present/target.c
@@ -0,0 +1,42 @@
+// RUN: %libomptarget-compile-aarch64-unknown-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-fail-aarch64-unknown-linux-gnu 2>&1 \
+// RUN: | %fcheck-aarch64-unknown-linux-gnu
+
+// RUN: %libomptarget-compile-powerpc64-ibm-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-fail-powerpc64-ibm-linux-gnu 2>&1 \
+// RUN: | %fcheck-powerpc64-ibm-linux-gnu
+
+// RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-fail-powerpc64le-ibm-linux-gnu 2>&1 \
+// RUN: | %fcheck-powerpc64le-ibm-linux-gnu
+
+// RUN: %libomptarget-compile-x86_64-pc-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-fail-x86_64-pc-linux-gnu 2>&1 \
+// RUN: | %fcheck-x86_64-pc-linux-gnu
+
+#include <stdio.h>
+
+int main() {
+  int i;
+
+  // CHECK: addr=0x[[#%x,HOST_ADDR:]], size=[[#%u,SIZE:]]
+  fprintf(stderr, "addr=%p, size=%ld\n", &i, sizeof i);
+
+  // CHECK-NOT: Libomptarget
+#pragma omp target data map(alloc: i)
+#pragma omp target map(present, alloc: i)
+  ;
+
+  // CHECK: i is present
+  fprintf(stderr, "i is present\n");
+
+  // CHECK: Libomptarget message: device mapping required by 'present' map type modifier does not exist for host address 0x{{0*}}[[#HOST_ADDR]] ([[#SIZE]] bytes)
+  // CHECK: Libomptarget fatal error 1: failure of target construct while offloading is mandatory
+#pragma omp target map(present, alloc: i)
+  ;
+
+  // CHECK-NOT: i is present
+  fprintf(stderr, "i is present\n");
+
+  return 0;
+}
diff --git a/openmp/libomptarget/test/mapping/present/target_data.c b/openmp/libomptarget/test/mapping/present/target_data.c
new file mode 100644
index 0000000000000..fd3107d87fefc
--- /dev/null
+++ b/openmp/libomptarget/test/mapping/present/target_data.c
@@ -0,0 +1,42 @@
+// RUN: %libomptarget-compile-aarch64-unknown-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-fail-aarch64-unknown-linux-gnu 2>&1 \
+// RUN: | %fcheck-aarch64-unknown-linux-gnu
+
+// RUN: %libomptarget-compile-powerpc64-ibm-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-fail-powerpc64-ibm-linux-gnu 2>&1 \
+// RUN: | %fcheck-powerpc64-ibm-linux-gnu
+
+// RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-fail-powerpc64le-ibm-linux-gnu 2>&1 \
+// RUN: | %fcheck-powerpc64le-ibm-linux-gnu
+
+// RUN: %libomptarget-compile-x86_64-pc-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-fail-x86_64-pc-linux-gnu 2>&1 \
+// RUN: | %fcheck-x86_64-pc-linux-gnu
+
+#include <stdio.h>
+
+int main() {
+  int i;
+
+  // CHECK: addr=0x[[#%x,HOST_ADDR:]], size=[[#%u,SIZE:]]
+  fprintf(stderr, "addr=%p, size=%ld\n", &i, sizeof i);
+
+  // CHECK-NOT: Libomptarget
+#pragma omp target data map(alloc: i)
+#pragma omp target data map(present, alloc: i)
+  ;
+
+  // CHECK: i is present
+  fprintf(stderr, "i is present\n");
+
+  // CHECK: Libomptarget message: device mapping required by 'present' map type modifier does not exist for host address 0x{{0*}}[[#HOST_ADDR]] ([[#SIZE]] bytes)
+  // CHECK: Libomptarget fatal error 1: failure of target construct while offloading is mandatory
+#pragma omp target data map(present, alloc: i)
+  ;
+
+  // CHECK-NOT: i is present
+  fprintf(stderr, "i is present\n");
+
+  return 0;
+}
diff --git a/openmp/libomptarget/test/mapping/present/target_enter_data.c b/openmp/libomptarget/test/mapping/present/target_enter_data.c
new file mode 100644
index 0000000000000..d96e7a4140da9
--- /dev/null
+++ b/openmp/libomptarget/test/mapping/present/target_enter_data.c
@@ -0,0 +1,41 @@
+// RUN: %libomptarget-compile-aarch64-unknown-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-fail-aarch64-unknown-linux-gnu 2>&1 \
+// RUN: | %fcheck-aarch64-unknown-linux-gnu
+
+// RUN: %libomptarget-compile-powerpc64-ibm-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-fail-powerpc64-ibm-linux-gnu 2>&1 \
+// RUN: | %fcheck-powerpc64-ibm-linux-gnu
+
+// RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-fail-powerpc64le-ibm-linux-gnu 2>&1 \
+// RUN: | %fcheck-powerpc64le-ibm-linux-gnu
+
+// RUN: %libomptarget-compile-x86_64-pc-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-fail-x86_64-pc-linux-gnu 2>&1 \
+// RUN: | %fcheck-x86_64-pc-linux-gnu
+
+#include <stdio.h>
+
+int main() {
+  int i;
+
+  // CHECK: addr=0x[[#%x,HOST_ADDR:]], size=[[#%u,SIZE:]]
+  fprintf(stderr, "addr=%p, size=%ld\n", &i, sizeof i);
+
+  // CHECK-NOT: Libomptarget
+#pragma omp target enter data map(alloc: i)
+#pragma omp target enter data map(present, alloc: i)
+#pragma omp target exit data map(delete: i)
+
+  // CHECK: i is present
+  fprintf(stderr, "i is present\n");
+
+  // CHECK: Libomptarget message: device mapping required by 'present' map type modifier does not exist for host address 0x{{0*}}[[#HOST_ADDR]] ([[#SIZE]] bytes)
+  // CHECK: Libomptarget fatal error 1: failure of target construct while offloading is mandatory
+#pragma omp target enter data map(present, alloc: i)
+
+  // CHECK-NOT: i is present
+  fprintf(stderr, "i is present\n");
+
+  return 0;
+}
diff --git a/openmp/libomptarget/test/mapping/present/target_exit_data.c b/openmp/libomptarget/test/mapping/present/target_exit_data.c
new file mode 100644
index 0000000000000..86b7ad89ce3ad
--- /dev/null
+++ b/openmp/libomptarget/test/mapping/present/target_exit_data.c
@@ -0,0 +1,40 @@
+// RUN: %libomptarget-compile-aarch64-unknown-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-fail-aarch64-unknown-linux-gnu 2>&1 \
+// RUN: | %fcheck-aarch64-unknown-linux-gnu
+
+// RUN: %libomptarget-compile-powerpc64-ibm-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-fail-powerpc64-ibm-linux-gnu 2>&1 \
+// RUN: | %fcheck-powerpc64-ibm-linux-gnu
+
+// RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-fail-powerpc64le-ibm-linux-gnu 2>&1 \
+// RUN: | %fcheck-powerpc64le-ibm-linux-gnu
+
+// RUN: %libomptarget-compile-x86_64-pc-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-fail-x86_64-pc-linux-gnu 2>&1 \
+// RUN: | %fcheck-x86_64-pc-linux-gnu
+
+#include <stdio.h>
+
+int main() {
+  int i;
+
+  // CHECK: addr=0x[[#%x,HOST_ADDR:]], size=[[#%u,SIZE:]]
+  fprintf(stderr, "addr=%p, size=%ld\n", &i, sizeof i);
+
+  // CHECK-NOT: Libomptarget
+#pragma omp target enter data map(alloc: i)
+#pragma omp target exit data map(present, release: i)
+
+  // CHECK: i is present
+  fprintf(stderr, "i is present\n");
+
+  // CHECK: Libomptarget message: device mapping required by 'present' map type modifier does not exist for host address 0x{{0*}}[[#HOST_ADDR]] ([[#SIZE]] bytes)
+  // CHECK: Libomptarget fatal error 1: failure of target construct while offloading is mandatory
+#pragma omp target exit data map(present, release: i)
+
+  // CHECK-NOT: i is present
+  fprintf(stderr, "i is present\n");
+
+  return 0;
+}
diff --git a/openmp/libomptarget/test/mapping/present/unified_shared_memory.c b/openmp/libomptarget/test/mapping/present/unified_shared_memory.c
new file mode 100644
index 0000000000000..22d87460179ca
--- /dev/null
+++ b/openmp/libomptarget/test/mapping/present/unified_shared_memory.c
@@ -0,0 +1,41 @@
+// RUN: %libomptarget-compile-aarch64-unknown-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-aarch64-unknown-linux-gnu 2>&1 \
+// RUN: | %fcheck-aarch64-unknown-linux-gnu
+
+// RUN: %libomptarget-compile-powerpc64-ibm-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-powerpc64-ibm-linux-gnu 2>&1 \
+// RUN: | %fcheck-powerpc64-ibm-linux-gnu
+
+// RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-powerpc64le-ibm-linux-gnu 2>&1 \
+// RUN: | %fcheck-powerpc64le-ibm-linux-gnu
+
+// RUN: %libomptarget-compile-x86_64-pc-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-x86_64-pc-linux-gnu 2>&1 \
+// RUN: | %fcheck-x86_64-pc-linux-gnu
+
+#include <stdio.h>
+
+// The runtime considers unified shared memory to be always present.
+#pragma omp requires unified_shared_memory
+
+int main() {
+  int i;
+
+  // CHECK-NOT: Libomptarget
+#pragma omp target data map(alloc: i)
+#pragma omp target map(present, alloc: i)
+  ;
+
+  // CHECK: i is present
+  fprintf(stderr, "i is present\n");
+
+  // CHECK-NOT: Libomptarget
+#pragma omp target map(present, alloc: i)
+  ;
+
+  // CHECK: is present
+  fprintf(stderr, "i is present\n");
+
+  return 0;
+}
diff --git a/openmp/libomptarget/test/mapping/present/zero_length_array_section.c b/openmp/libomptarget/test/mapping/present/zero_length_array_section.c
new file mode 100644
index 0000000000000..5488888e14018
--- /dev/null
+++ b/openmp/libomptarget/test/mapping/present/zero_length_array_section.c
@@ -0,0 +1,45 @@
+// RUN: %libomptarget-compile-aarch64-unknown-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-fail-aarch64-unknown-linux-gnu 2>&1 \
+// RUN: | %fcheck-aarch64-unknown-linux-gnu
+
+// RUN: %libomptarget-compile-powerpc64-ibm-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-fail-powerpc64-ibm-linux-gnu 2>&1 \
+// RUN: | %fcheck-powerpc64-ibm-linux-gnu
+
+// RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-fail-powerpc64le-ibm-linux-gnu 2>&1 \
+// RUN: | %fcheck-powerpc64le-ibm-linux-gnu
+
+// RUN: %libomptarget-compile-x86_64-pc-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-fail-x86_64-pc-linux-gnu 2>&1 \
+// RUN: | %fcheck-x86_64-pc-linux-gnu
+
+#include <stdio.h>
+
+int main() {
+  int arr[5];
+
+  // CHECK: addr=0x[[#%x,HOST_ADDR:]]
+  fprintf(stderr, "addr=%p\n", arr);
+
+  // CHECK-NOT: Libomptarget
+#pragma omp target data map(alloc: arr[0:5])
+#pragma omp target map(present, alloc: arr[0:0])
+  ;
+
+  // CHECK: arr is present
+  fprintf(stderr, "arr is present\n");
+
+  // arr[0:0] doesn't create an actual mapping in the first directive.
+  //
+  // CHECK: Libomptarget message: device mapping required by 'present' map type modifier does not exist for host address 0x{{0*}}[[#HOST_ADDR]] (0 bytes)
+  // CHECK: Libomptarget fatal error 1: failure of target construct while offloading is mandatory
+#pragma omp target data map(alloc: arr[0:0])
+#pragma omp target map(present, alloc: arr[0:0])
+  ;
+
+  // CHECK-NOT: arr is present
+  fprintf(stderr, "arr is present\n");
+
+  return 0;
+}
diff --git a/openmp/libomptarget/test/mapping/present/zero_length_array_section_exit.c b/openmp/libomptarget/test/mapping/present/zero_length_array_section_exit.c
new file mode 100644
index 0000000000000..bedc6a2729c56
--- /dev/null
+++ b/openmp/libomptarget/test/mapping/present/zero_length_array_section_exit.c
@@ -0,0 +1,43 @@
+// RUN: %libomptarget-compile-aarch64-unknown-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-fail-aarch64-unknown-linux-gnu 2>&1 \
+// RUN: | %fcheck-aarch64-unknown-linux-gnu
+
+// RUN: %libomptarget-compile-powerpc64-ibm-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-fail-powerpc64-ibm-linux-gnu 2>&1 \
+// RUN: | %fcheck-powerpc64-ibm-linux-gnu
+
+// RUN: %libomptarget-compile-powerpc64le-ibm-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-fail-powerpc64le-ibm-linux-gnu 2>&1 \
+// RUN: | %fcheck-powerpc64le-ibm-linux-gnu
+
+// RUN: %libomptarget-compile-x86_64-pc-linux-gnu -fopenmp-version=51
+// RUN: %libomptarget-run-fail-x86_64-pc-linux-gnu 2>&1 \
+// RUN: | %fcheck-x86_64-pc-linux-gnu
+
+#include <stdio.h>
+
+int main() {
+  int arr[5];
+
+  // CHECK: addr=0x[[#%x,HOST_ADDR:]]
+  fprintf(stderr, "addr=%p\n", arr);
+
+  // CHECK-NOT: Libomptarget
+#pragma omp target enter data map(alloc: arr[0:5])
+#pragma omp target exit data map(present, release: arr[0:0])
+
+  // CHECK: arr is present
+  fprintf(stderr, "arr is present\n");
+
+  // arr[0:0] doesn't create an actual mapping in the first directive.
+  //
+  // CHECK: Libomptarget message: device mapping required by 'present' map type modifier does not exist for host address 0x{{0*}}[[#HOST_ADDR]] (0 bytes)
+  // CHECK: Libomptarget fatal error 1: failure of target construct while offloading is mandatory
+#pragma omp target enter data map(alloc: arr[0:0])
+#pragma omp target exit data map(present, release: arr[0:0])
+
+  // CHECK-NOT: arr is present
+  fprintf(stderr, "arr is present\n");
+
+  return 0;
+}
diff --git a/openmp/runtime/cmake/LibompCheckLinkerFlag.cmake b/openmp/runtime/cmake/LibompCheckLinkerFlag.cmake
index 2b6cda676a62c..e601e53f11237 100644
--- a/openmp/runtime/cmake/LibompCheckLinkerFlag.cmake
+++ b/openmp/runtime/cmake/LibompCheckLinkerFlag.cmake
@@ -17,7 +17,7 @@ function(libomp_check_linker_flag flag boolean)
   set(library_source
     "int foo(int a) { return a*a; }")
   set(cmake_source
-    "cmake_minimum_required(VERSION 2.8)
+    "cmake_minimum_required(VERSION 3.13.4)
      project(foo C)
      set(CMAKE_SHARED_LINKER_FLAGS \"${flag}\")
      add_library(foo SHARED src_to_link.c)")
diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index 5f9b7c895619b..933573bc810cb 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -136,6 +136,10 @@ typedef unsigned int kmp_hwloc_depth_t;
 #include "ompt-internal.h"
 #endif
 
+#ifndef UNLIKELY
+#define UNLIKELY(x) (x)
+#endif
+
 // Affinity format function
 #include "kmp_str.h"
 
@@ -3078,6 +3082,11 @@ static inline kmp_team_t *__kmp_team_from_gtid(int gtid) {
   return __kmp_threads[gtid]->th.th_team;
 }
 
+static inline void __kmp_assert_valid_gtid(kmp_int32 gtid) {
+  if (UNLIKELY(gtid < 0 || gtid >= __kmp_threads_capacity))
+    KMP_FATAL(ThreadIdentInvalid);
+}
+
 /* ------------------------------------------------------------------------- */
 
 extern kmp_global_t __kmp_global; /* global status */
diff --git a/openmp/runtime/src/kmp_csupport.cpp b/openmp/runtime/src/kmp_csupport.cpp
index 9cfa64d6ff9e2..b5c641cc7273c 100644
--- a/openmp/runtime/src/kmp_csupport.cpp
+++ b/openmp/runtime/src/kmp_csupport.cpp
@@ -231,13 +231,12 @@ void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
                              kmp_int32 num_threads) {
   KA_TRACE(20, ("__kmpc_push_num_threads: enter T#%d num_threads=%d\n",
                 global_tid, num_threads));
-
+  __kmp_assert_valid_gtid(global_tid);
   __kmp_push_num_threads(loc, global_tid, num_threads);
 }
 
 void __kmpc_pop_num_threads(ident_t *loc, kmp_int32 global_tid) {
   KA_TRACE(20, ("__kmpc_pop_num_threads: enter\n"));
-
   /* the num_threads are automatically popped */
 }
 
@@ -245,7 +244,7 @@ void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
                            kmp_int32 proc_bind) {
   KA_TRACE(20, ("__kmpc_push_proc_bind: enter T#%d proc_bind=%d\n", global_tid,
                 proc_bind));
-
+  __kmp_assert_valid_gtid(global_tid);
   __kmp_push_proc_bind(loc, global_tid, (kmp_proc_bind_t)proc_bind);
 }
 
@@ -353,7 +352,7 @@ void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid,
   KA_TRACE(20,
            ("__kmpc_push_num_teams: enter T#%d num_teams=%d num_threads=%d\n",
             global_tid, num_teams, num_threads));
-
+  __kmp_assert_valid_gtid(global_tid);
   __kmp_push_num_teams(loc, global_tid, num_teams, num_threads);
 }
 
@@ -474,9 +473,10 @@ conditional parallel region, like this,
 when the condition is false.
 */
 void __kmpc_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
-// The implementation is now in kmp_runtime.cpp so that it can share static
-// functions with kmp_fork_call since the tasks to be done are similar in
-// each case.
+  // The implementation is now in kmp_runtime.cpp so that it can share static
+  // functions with kmp_fork_call since the tasks to be done are similar in
+  // each case.
+  __kmp_assert_valid_gtid(global_tid);
 #if OMPT_SUPPORT
   OMPT_STORE_RETURN_ADDRESS(global_tid);
 #endif
@@ -504,6 +504,7 @@ void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
     return;
 
   // Not autopar code
+  __kmp_assert_valid_gtid(global_tid);
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
 
@@ -713,6 +714,7 @@ Execute a barrier.
 void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid) {
   KMP_COUNT_BLOCK(OMP_BARRIER);
   KC_TRACE(10, ("__kmpc_barrier: called T#%d\n", global_tid));
+  __kmp_assert_valid_gtid(global_tid);
 
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
@@ -762,6 +764,7 @@ kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid) {
   int status = 0;
 
   KC_TRACE(10, ("__kmpc_master: called T#%d\n", global_tid));
+  __kmp_assert_valid_gtid(global_tid);
 
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
@@ -816,7 +819,7 @@ thread that executes the <tt>master</tt> region.
 */
 void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid) {
   KC_TRACE(10, ("__kmpc_end_master: called T#%d\n", global_tid));
-
+  __kmp_assert_valid_gtid(global_tid);
   KMP_DEBUG_ASSERT(KMP_MASTER_GTID(global_tid));
   KMP_POP_PARTITIONED_TIMER();
 
@@ -833,9 +836,6 @@ void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid) {
 #endif
 
   if (__kmp_env_consistency_check) {
-    if (global_tid < 0)
-      KMP_WARNING(ThreadIdentInvalid);
-
     if (KMP_MASTER_GTID(global_tid))
       __kmp_pop_sync(global_tid, ct_master, loc);
   }
@@ -854,6 +854,7 @@ void __kmpc_ordered(ident_t *loc, kmp_int32 gtid) {
   KMP_DEBUG_ASSERT(__kmp_init_serial);
 
   KC_TRACE(10, ("__kmpc_ordered: called T#%d\n", gtid));
+  __kmp_assert_valid_gtid(gtid);
 
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
@@ -925,6 +926,7 @@ void __kmpc_end_ordered(ident_t *loc, kmp_int32 gtid) {
   kmp_info_t *th;
 
   KC_TRACE(10, ("__kmpc_end_ordered: called T#%d\n", gtid));
+  __kmp_assert_valid_gtid(gtid);
 
 #if USE_ITT_BUILD
   __kmp_itt_ordered_end(gtid);
@@ -1147,7 +1149,7 @@ static kmp_user_lock_p __kmp_get_critical_section_ptr(kmp_critical_name *crit,
 /*!
 @ingroup WORK_SHARING
 @param loc  source location information.
-@param global_tid  global thread number .
+@param global_tid  global thread number.
 @param crit identity of the critical section. This could be a pointer to a lock
 associated with the critical section, or some other suitably unique value.
 
@@ -1170,6 +1172,7 @@ void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
   kmp_user_lock_p lck;
 
   KC_TRACE(10, ("__kmpc_critical: called T#%d\n", global_tid));
+  __kmp_assert_valid_gtid(global_tid);
 
   // TODO: add THR_OVHD_STATE
 
@@ -1392,6 +1395,7 @@ void __kmpc_critical_with_hint(ident_t *loc, kmp_int32 global_tid,
 #endif
 
   KC_TRACE(10, ("__kmpc_critical: called T#%d\n", global_tid));
+  __kmp_assert_valid_gtid(global_tid);
 
   kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)crit;
   // Check if it is initialized.
@@ -1607,8 +1611,8 @@ this function.
 */
 kmp_int32 __kmpc_barrier_master(ident_t *loc, kmp_int32 global_tid) {
   int status;
-
   KC_TRACE(10, ("__kmpc_barrier_master: called T#%d\n", global_tid));
+  __kmp_assert_valid_gtid(global_tid);
 
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
@@ -1651,7 +1655,7 @@ still be waiting at the barrier and this call releases them.
 */
 void __kmpc_end_barrier_master(ident_t *loc, kmp_int32 global_tid) {
   KC_TRACE(10, ("__kmpc_end_barrier_master: called T#%d\n", global_tid));
-
+  __kmp_assert_valid_gtid(global_tid);
   __kmp_end_split_barrier(bs_plain_barrier, global_tid);
 }
 
@@ -1667,8 +1671,8 @@ There is no equivalent "end" function, since the
 */
 kmp_int32 __kmpc_barrier_master_nowait(ident_t *loc, kmp_int32 global_tid) {
   kmp_int32 ret;
-
   KC_TRACE(10, ("__kmpc_barrier_master_nowait: called T#%d\n", global_tid));
+  __kmp_assert_valid_gtid(global_tid);
 
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
@@ -1706,14 +1710,9 @@ kmp_int32 __kmpc_barrier_master_nowait(ident_t *loc, kmp_int32 global_tid) {
   if (__kmp_env_consistency_check) {
     /*  there's no __kmpc_end_master called; so the (stats) */
     /*  actions of __kmpc_end_master are done here          */
-
-    if (global_tid < 0) {
-      KMP_WARNING(ThreadIdentInvalid);
-    }
     if (ret) {
       /* only one thread should do the pop since only */
       /* one did the push (see __kmpc_master())       */
-
       __kmp_pop_sync(global_tid, ct_master, loc);
     }
   }
@@ -1734,6 +1733,7 @@ should introduce an explicit barrier if it is required.
 */
 
 kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid) {
+  __kmp_assert_valid_gtid(global_tid);
   kmp_int32 rc = __kmp_enter_single(global_tid, loc, TRUE);
 
   if (rc) {
@@ -1786,6 +1786,7 @@ only be called by the thread that executed the block of code protected
 by the `single` construct.
 */
 void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid) {
+  __kmp_assert_valid_gtid(global_tid);
   __kmp_exit_single(global_tid);
   KMP_POP_PARTITIONED_TIMER();
 
@@ -2065,8 +2066,8 @@ void __kmpc_copyprivate(ident_t *loc, kmp_int32 gtid, size_t cpy_size,
                         void *cpy_data, void (*cpy_func)(void *, void *),
                         kmp_int32 didit) {
   void **data_ptr;
-
   KC_TRACE(10, ("__kmpc_copyprivate: called T#%d\n", gtid));
+  __kmp_assert_valid_gtid(gtid);
 
   KMP_MB();
 
@@ -3382,6 +3383,7 @@ __kmpc_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars,
   kmp_team_t *team;
   int teams_swapped = 0, task_state;
   KA_TRACE(10, ("__kmpc_reduce_nowait() enter: called T#%d\n", global_tid));
+  __kmp_assert_valid_gtid(global_tid);
 
   // why do we need this initialization here at all?
   // Reduction clause can not be used as a stand-alone directive.
@@ -3535,6 +3537,7 @@ void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
   PACKED_REDUCTION_METHOD_T packed_reduction_method;
 
   KA_TRACE(10, ("__kmpc_end_reduce_nowait() enter: called T#%d\n", global_tid));
+  __kmp_assert_valid_gtid(global_tid);
 
   packed_reduction_method = __KMP_GET_REDUCTION_METHOD(global_tid);
 
@@ -3609,6 +3612,7 @@ kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars,
   int teams_swapped = 0, task_state;
 
   KA_TRACE(10, ("__kmpc_reduce() enter: called T#%d\n", global_tid));
+  __kmp_assert_valid_gtid(global_tid);
 
   // why do we need this initialization here at all?
   // Reduction clause can not be a stand-alone directive.
@@ -3727,6 +3731,7 @@ void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
   int teams_swapped = 0, task_state;
 
   KA_TRACE(10, ("__kmpc_end_reduce() enter: called T#%d\n", global_tid));
+  __kmp_assert_valid_gtid(global_tid);
 
   th = __kmp_thread_from_gtid(global_tid);
   teams_swapped = __kmp_swap_teams_for_teams_reduction(th, &team, &task_state);
@@ -3883,6 +3888,7 @@ e.g. for(i=2;i<9;i+=2) lo=2, up=8, st=2.
 */
 void __kmpc_doacross_init(ident_t *loc, int gtid, int num_dims,
                           const struct kmp_dim *dims) {
+  __kmp_assert_valid_gtid(gtid);
   int j, idx;
   kmp_int64 last, trace_count;
   kmp_info_t *th = __kmp_threads[gtid];
@@ -4002,6 +4008,7 @@ void __kmpc_doacross_init(ident_t *loc, int gtid, int num_dims,
 }
 
 void __kmpc_doacross_wait(ident_t *loc, int gtid, const kmp_int64 *vec) {
+  __kmp_assert_valid_gtid(gtid);
   kmp_int32 shft, num_dims, i;
   kmp_uint32 flag;
   kmp_int64 iter_number; // iteration number of "collapsed" loop nest
@@ -4112,6 +4119,7 @@ void __kmpc_doacross_wait(ident_t *loc, int gtid, const kmp_int64 *vec) {
 }
 
 void __kmpc_doacross_post(ident_t *loc, int gtid, const kmp_int64 *vec) {
+  __kmp_assert_valid_gtid(gtid);
   kmp_int32 shft, num_dims, i;
   kmp_uint32 flag;
   kmp_int64 iter_number; // iteration number of "collapsed" loop nest
@@ -4183,6 +4191,7 @@ void __kmpc_doacross_post(ident_t *loc, int gtid, const kmp_int64 *vec) {
 }
 
 void __kmpc_doacross_fini(ident_t *loc, int gtid) {
+  __kmp_assert_valid_gtid(gtid);
   kmp_int32 num_done;
   kmp_info_t *th = __kmp_threads[gtid];
   kmp_team_t *team = th->th.th_team;
diff --git a/openmp/runtime/src/kmp_dispatch.cpp b/openmp/runtime/src/kmp_dispatch.cpp
index 9d7b81733eba0..ca14bbc954aff 100644
--- a/openmp/runtime/src/kmp_dispatch.cpp
+++ b/openmp/runtime/src/kmp_dispatch.cpp
@@ -773,6 +773,7 @@ __kmp_dispatch_init(ident_t *loc, int gtid, enum sched_type schedule, T lb,
                    sizeof(dispatch_private_info));
   KMP_BUILD_ASSERT(sizeof(dispatch_shared_info_template<UT>) ==
                    sizeof(dispatch_shared_info));
+  __kmp_assert_valid_gtid(gtid);
 
   if (!TCR_4(__kmp_init_parallel))
     __kmp_parallel_initialize();
@@ -997,6 +998,7 @@ __kmp_dispatch_init(ident_t *loc, int gtid, enum sched_type schedule, T lb,
 template <typename UT>
 static void __kmp_dispatch_finish(int gtid, ident_t *loc) {
   typedef typename traits_t<UT>::signed_t ST;
+  __kmp_assert_valid_gtid(gtid);
   kmp_info_t *th = __kmp_threads[gtid];
 
   KD_TRACE(100, ("__kmp_dispatch_finish: T#%d called\n", gtid));
@@ -1060,6 +1062,7 @@ static void __kmp_dispatch_finish(int gtid, ident_t *loc) {
 template <typename UT>
 static void __kmp_dispatch_finish_chunk(int gtid, ident_t *loc) {
   typedef typename traits_t<UT>::signed_t ST;
+  __kmp_assert_valid_gtid(gtid);
   kmp_info_t *th = __kmp_threads[gtid];
 
   KD_TRACE(100, ("__kmp_dispatch_finish_chunk: T#%d called\n", gtid));
@@ -1900,6 +1903,7 @@ static int __kmp_dispatch_next(ident_t *loc, int gtid, kmp_int32 *p_last,
 
   int status;
   dispatch_private_info_template<T> *pr;
+  __kmp_assert_valid_gtid(gtid);
   kmp_info_t *th = __kmp_threads[gtid];
   kmp_team_t *team = th->th.th_team;
 
@@ -2192,6 +2196,7 @@ static void __kmp_dist_get_bounds(ident_t *loc, kmp_int32 gtid,
       __kmp_error_construct(kmp_i18n_msg_CnsLoopIncrIllegal, ct_pdo, loc);
     }
   }
+  __kmp_assert_valid_gtid(gtid);
   th = __kmp_threads[gtid];
   team = th->th.th_team;
   KMP_DEBUG_ASSERT(th->th.th_teams_microtask); // we are in the teams construct
diff --git a/openmp/runtime/src/kmp_error.cpp b/openmp/runtime/src/kmp_error.cpp
index b30b26e3ab2b4..7fc0ce17a05cc 100644
--- a/openmp/runtime/src/kmp_error.cpp
+++ b/openmp/runtime/src/kmp_error.cpp
@@ -415,9 +415,6 @@ void __kmp_pop_sync(int gtid, enum cons_type ct, ident_t const *ident) {
     __kmp_error_construct2(kmp_i18n_msg_CnsExpectedEnd, ct, ident,
                            &p->stack_data[tos]);
   }
-  if (gtid < 0) {
-    __kmp_check_null_func();
-  }
   KE_TRACE(100, (POP_MSG(p)));
   p->s_top = p->stack_data[tos].prev;
   p->stack_data[tos].type = ct_none;
diff --git a/openmp/runtime/src/kmp_itt.inl b/openmp/runtime/src/kmp_itt.inl
index 04f00285ba354..29a05cb86a3f5 100644
--- a/openmp/runtime/src/kmp_itt.inl
+++ b/openmp/runtime/src/kmp_itt.inl
@@ -230,8 +230,9 @@ LINKAGE void __kmp_itt_frame_submit(int gtid, __itt_timestamp begin,
       // Check if team size was changed. Then create new region domain for this
       // location
       unsigned int frm = (loc->reserved_2 & 0x0000FFFF) - 1;
-      if ((frm < KMP_MAX_FRAME_DOMAINS) &&
-          (__kmp_itt_region_team_size[frm] != team_size)) {
+      if (frm >= KMP_MAX_FRAME_DOMAINS)
+        return; // something's gone wrong, returning
+      if (__kmp_itt_region_team_size[frm] != team_size) {
         char *buff = NULL;
         kmp_str_loc_t str_loc = __kmp_str_loc_init(loc->psource, 1);
         buff = __kmp_str_format("%s$omp$parallel:%d@%s:%d:%d", str_loc.func,
diff --git a/openmp/runtime/src/kmp_runtime.cpp b/openmp/runtime/src/kmp_runtime.cpp
index 00e0c7db4536f..305ec4051cdbb 100644
--- a/openmp/runtime/src/kmp_runtime.cpp
+++ b/openmp/runtime/src/kmp_runtime.cpp
@@ -4950,12 +4950,15 @@ __kmp_allocate_team(kmp_root_t *root, int new_nproc, int max_nproc,
     }
     hot_teams = master->th.th_hot_teams;
     if (level < __kmp_hot_teams_max_level && hot_teams &&
-        hot_teams[level]
-            .hot_team) { // hot team has already been allocated for given level
+        hot_teams[level].hot_team) {
+      // hot team has already been allocated for given level
       use_hot_team = 1;
     } else {
       use_hot_team = 0;
     }
+  } else {
+    // check we won't access uninitialized hot_teams, just in case
+    KMP_DEBUG_ASSERT(new_nproc == 1);
   }
 #endif
   // Optimization to use a "hot" team
diff --git a/openmp/runtime/src/kmp_sched.cpp b/openmp/runtime/src/kmp_sched.cpp
index 28d0ffe0fb9dd..2d8f644c88b27 100644
--- a/openmp/runtime/src/kmp_sched.cpp
+++ b/openmp/runtime/src/kmp_sched.cpp
@@ -61,6 +61,12 @@ char const *traits_t<long>::spec = "ld";
 #define KMP_STATS_LOOP_END(stat) /* Nothing */
 #endif
 
+static ident_t loc_stub = {0, KMP_IDENT_KMPC, 0, 0, ";unknown;unknown;0;0;;"};
+static inline void check_loc(ident_t *&loc) {
+  if (loc == NULL)
+    loc = &loc_stub; // may need to report location info to ittnotify
+}
+
 template <typename T>
 static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid,
                                   kmp_int32 schedtype, kmp_int32 *plastiter,
@@ -85,6 +91,7 @@ static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid,
   kmp_uint32 nth;
   UT trip_count;
   kmp_team_t *team;
+  __kmp_assert_valid_gtid(gtid);
   kmp_info_t *th = __kmp_threads[gtid];
 
 #if OMPT_SUPPORT && OMPT_OPTIONAL
@@ -381,6 +388,7 @@ static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid,
       __kmp_forkjoin_frames_mode == 3 && th->th.th_teams_microtask == NULL &&
       team->t.t_active_level == 1) {
     kmp_uint64 cur_chunk = chunk;
+    check_loc(loc);
     // Calculate chunk in case it was not specified; it is specified for
     // kmp_sch_static_chunked
     if (schedtype == kmp_sch_static) {
@@ -438,6 +446,7 @@ static void __kmp_dist_for_static_init(ident_t *loc, kmp_int32 gtid,
 
   KMP_DEBUG_ASSERT(plastiter && plower && pupper && pupperDist && pstride);
   KE_TRACE(10, ("__kmpc_dist_for_static_init called (%d)\n", gtid));
+  __kmp_assert_valid_gtid(gtid);
 #ifdef KMP_DEBUG
   {
     char *buff;
@@ -681,6 +690,7 @@ static void __kmp_team_static_init(ident_t *loc, kmp_int32 gtid,
 
   KMP_DEBUG_ASSERT(p_last && p_lb && p_ub && p_st);
   KE_TRACE(10, ("__kmp_team_static_init called (%d)\n", gtid));
+  __kmp_assert_valid_gtid(gtid);
 #ifdef KMP_DEBUG
   {
     char *buff;
diff --git a/openmp/runtime/src/kmp_taskdeps.cpp b/openmp/runtime/src/kmp_taskdeps.cpp
index a654951f5b3b2..9a81196879a8f 100644
--- a/openmp/runtime/src/kmp_taskdeps.cpp
+++ b/openmp/runtime/src/kmp_taskdeps.cpp
@@ -514,7 +514,7 @@ kmp_int32 __kmpc_omp_task_with_deps(ident_t *loc_ref, kmp_int32 gtid,
   kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
   KA_TRACE(10, ("__kmpc_omp_task_with_deps(enter): T#%d loc=%p task=%p\n", gtid,
                 loc_ref, new_taskdata));
-
+  __kmp_assert_valid_gtid(gtid);
   kmp_info_t *thread = __kmp_threads[gtid];
   kmp_taskdata_t *current_task = thread->th.th_current_task;
 
@@ -677,7 +677,7 @@ void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32 gtid, kmp_int32 ndeps,
                   gtid, loc_ref));
     return;
   }
-
+  __kmp_assert_valid_gtid(gtid);
   kmp_info_t *thread = __kmp_threads[gtid];
   kmp_taskdata_t *current_task = thread->th.th_current_task;
 
diff --git a/openmp/runtime/src/kmp_tasking.cpp b/openmp/runtime/src/kmp_tasking.cpp
index c5a3744ad27ba..3dfc3c4030d4b 100644
--- a/openmp/runtime/src/kmp_tasking.cpp
+++ b/openmp/runtime/src/kmp_tasking.cpp
@@ -984,6 +984,7 @@ static void __kmpc_omp_task_complete_if0_template(ident_t *loc_ref,
                                                   kmp_task_t *task) {
   KA_TRACE(10, ("__kmpc_omp_task_complete_if0(enter): T#%d loc=%p task=%p\n",
                 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task)));
+  __kmp_assert_valid_gtid(gtid);
   // this routine will provide task to resume
   __kmp_task_finish<ompt>(gtid, task, NULL);
 
@@ -1381,10 +1382,9 @@ kmp_task_t *__kmpc_omp_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
                                   kmp_routine_entry_t task_entry) {
   kmp_task_t *retval;
   kmp_tasking_flags_t *input_flags = (kmp_tasking_flags_t *)&flags;
-
+  __kmp_assert_valid_gtid(gtid);
   input_flags->native = FALSE;
 // __kmp_task_alloc() sets up all other runtime flags
-
   KA_TRACE(10, ("__kmpc_omp_task_alloc(enter): T#%d loc=%p, flags=(%s %s %s) "
                 "sizeof_task=%ld sizeof_shared=%ld entry=%p\n",
                 gtid, loc_ref, input_flags->tiedness ? "tied  " : "untied",
@@ -1716,6 +1716,7 @@ kmp_int32 __kmpc_omp_task(ident_t *loc_ref, kmp_int32 gtid,
 #endif
   KA_TRACE(10, ("__kmpc_omp_task(enter): T#%d loc=%p task=%p\n", gtid, loc_ref,
                 new_taskdata));
+  __kmp_assert_valid_gtid(gtid);
 
 #if OMPT_SUPPORT
   kmp_taskdata_t *parent = NULL;
@@ -1824,6 +1825,7 @@ static kmp_int32 __kmpc_omp_taskwait_template(ident_t *loc_ref, kmp_int32 gtid,
   KMP_SET_THREAD_STATE_BLOCK(TASKWAIT);
 
   KA_TRACE(10, ("__kmpc_omp_taskwait(enter): T#%d loc=%p\n", gtid, loc_ref));
+  __kmp_assert_valid_gtid(gtid);
 
   if (__kmp_tasking_mode != tskm_immediate_exec) {
     thread = __kmp_threads[gtid];
@@ -1953,6 +1955,7 @@ kmp_int32 __kmpc_omp_taskyield(ident_t *loc_ref, kmp_int32 gtid, int end_part) {
 
   KA_TRACE(10, ("__kmpc_omp_taskyield(enter): T#%d loc=%p end_part = %d\n",
                 gtid, loc_ref, end_part));
+  __kmp_assert_valid_gtid(gtid);
 
   if (__kmp_tasking_mode != tskm_immediate_exec && __kmp_init_parallel) {
     thread = __kmp_threads[gtid];
@@ -2108,6 +2111,7 @@ void __kmp_call_init<kmp_taskred_input_t>(kmp_taskred_data_t &item,
 
 template <typename T>
 void *__kmp_task_reduction_init(int gtid, int num, T *data) {
+  __kmp_assert_valid_gtid(gtid);
   kmp_info_t *thread = __kmp_threads[gtid];
   kmp_taskgroup_t *tg = thread->th.th_current_task->td_taskgroup;
   kmp_int32 nth = thread->th.th_team_nproc;
@@ -2223,6 +2227,7 @@ void __kmp_task_reduction_init_copy(kmp_info_t *thr, int num, T *data,
 Get thread-specific location of data item
 */
 void *__kmpc_task_reduction_get_th_data(int gtid, void *tskgrp, void *data) {
+  __kmp_assert_valid_gtid(gtid);
   kmp_info_t *thread = __kmp_threads[gtid];
   kmp_int32 nth = thread->th.th_team_nproc;
   if (nth == 1)
@@ -2328,6 +2333,7 @@ static void __kmp_task_reduction_clean(kmp_info_t *th, kmp_taskgroup_t *tg) {
 template <typename T>
 void *__kmp_task_reduction_modifier_init(ident_t *loc, int gtid, int is_ws,
                                          int num, T *data) {
+  __kmp_assert_valid_gtid(gtid);
   kmp_info_t *thr = __kmp_threads[gtid];
   kmp_int32 nth = thr->th.th_team_nproc;
   __kmpc_taskgroup(loc, gtid); // form new taskgroup first
@@ -2423,6 +2429,7 @@ void __kmpc_task_reduction_modifier_fini(ident_t *loc, int gtid, int is_ws) {
 
 // __kmpc_taskgroup: Start a new taskgroup
 void __kmpc_taskgroup(ident_t *loc, int gtid) {
+  __kmp_assert_valid_gtid(gtid);
   kmp_info_t *thread = __kmp_threads[gtid];
   kmp_taskdata_t *taskdata = thread->th.th_current_task;
   kmp_taskgroup_t *tg_new =
@@ -2455,6 +2462,7 @@ void __kmpc_taskgroup(ident_t *loc, int gtid) {
 // __kmpc_end_taskgroup: Wait until all tasks generated by the current task
 //                       and its descendants are complete
 void __kmpc_end_taskgroup(ident_t *loc, int gtid) {
+  __kmp_assert_valid_gtid(gtid);
   kmp_info_t *thread = __kmp_threads[gtid];
   kmp_taskdata_t *taskdata = thread->th.th_current_task;
   kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup;
@@ -3807,7 +3815,7 @@ void __kmpc_proxy_task_completed(kmp_int32 gtid, kmp_task_t *ptask) {
   KA_TRACE(
       10, ("__kmp_proxy_task_completed(enter): T#%d proxy task %p completing\n",
            gtid, taskdata));
-
+  __kmp_assert_valid_gtid(gtid);
   KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY);
 
   __kmp_first_top_half_finish_proxy(taskdata);
@@ -4439,7 +4447,7 @@ void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
                      int sched, kmp_uint64 grainsize, void *task_dup) {
   kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
   KMP_DEBUG_ASSERT(task != NULL);
-
+  __kmp_assert_valid_gtid(gtid);
   if (nogroup == 0) {
 #if OMPT_SUPPORT && OMPT_OPTIONAL
     OMPT_STORE_RETURN_ADDRESS(gtid);
diff --git a/openmp/runtime/src/kmp_threadprivate.cpp b/openmp/runtime/src/kmp_threadprivate.cpp
index 87bfff39e70e0..a0256d6b9fc2f 100644
--- a/openmp/runtime/src/kmp_threadprivate.cpp
+++ b/openmp/runtime/src/kmp_threadprivate.cpp
@@ -244,9 +244,8 @@ void __kmp_common_destroy_gtid(int gtid) {
 
         d_tn = __kmp_find_shared_task_common(&__kmp_threadprivate_d_table, gtid,
                                              tn->gbl_addr);
-
-        KMP_DEBUG_ASSERT(d_tn);
-
+        if (d_tn == NULL)
+          continue;
         if (d_tn->is_vec) {
           if (d_tn->dt.dtorv != 0) {
             (void)(*d_tn->dt.dtorv)(tn->par_addr, d_tn->vec_len);
diff --git a/openmp/runtime/test/env/kmp_set_dispatch_buf.c b/openmp/runtime/test/env/kmp_set_dispatch_buf.c
index 61cb9d05bdb87..264510d555350 100644
--- a/openmp/runtime/test/env/kmp_set_dispatch_buf.c
+++ b/openmp/runtime/test/env/kmp_set_dispatch_buf.c
@@ -9,7 +9,7 @@
 // RUN: env KMP_DISP_NUM_BUFFERS=3 %libomp-run
 // RUN: env KMP_DISP_NUM_BUFFERS=4 %libomp-run
 // RUN: env KMP_DISP_NUM_BUFFERS=7 %libomp-run
-// UNSUPPORTED: clang-11
+// UNSUPPORTED: clang-11, clang-12
 #include <stdio.h>
 #include <omp.h>
 #include <stdlib.h>
diff --git a/openmp/runtime/test/ompt/tasks/task_early_fulfill.c b/openmp/runtime/test/ompt/tasks/task_early_fulfill.c
index f1d07a1503c84..e1324e6af681a 100644
--- a/openmp/runtime/test/ompt/tasks/task_early_fulfill.c
+++ b/openmp/runtime/test/ompt/tasks/task_early_fulfill.c
@@ -1,5 +1,6 @@
 // RUN: %libomp-compile -fopenmp-version=50 && env OMP_NUM_THREADS='3' \
 // RUN:    %libomp-run | %sort-threads | FileCheck %s
+// REQUIRES: ompt
 
 // Checked gcc 10.1 still does not support detach clause on task construct.
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7, gcc-8, gcc-9, gcc-10
diff --git a/openmp/runtime/test/ompt/tasks/task_late_fulfill.c b/openmp/runtime/test/ompt/tasks/task_late_fulfill.c
index 4824f3358cfdb..13a2a54a60df7 100644
--- a/openmp/runtime/test/ompt/tasks/task_late_fulfill.c
+++ b/openmp/runtime/test/ompt/tasks/task_late_fulfill.c
@@ -1,5 +1,6 @@
 // RUN: %libomp-compile -fopenmp-version=50 && env OMP_NUM_THREADS='3' \
 // RUN:    %libomp-run | %sort-threads | FileCheck %s
+// REQUIRES: ompt
 
 // Checked gcc 10.1 still does not support detach clause on task construct.
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7, gcc-8, gcc-9, gcc-10
diff --git a/openmp/runtime/test/tasking/omp_task_red_taskloop.c b/openmp/runtime/test/tasking/omp_task_red_taskloop.c
index 35987f6bc0538..a56e9cebed964 100644
--- a/openmp/runtime/test/tasking/omp_task_red_taskloop.c
+++ b/openmp/runtime/test/tasking/omp_task_red_taskloop.c
@@ -3,6 +3,9 @@
 // Parsing error until gcc8:
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7, gcc-8
 
+// Parsing error until clang11:
+// UNSUPPORTED: clang-10, clang-9, clang-8, clang-7
+
 // Missing GOMP_taskgroup_reduction_(un)register in LLVM/OpenMP
 // Should be removed once the functions are implemented
 // XFAIL: gcc-9, gcc-10
diff --git a/openmp/runtime/test/worksharing/for/kmp_set_dispatch_buf.c b/openmp/runtime/test/worksharing/for/kmp_set_dispatch_buf.c
index 9c3fd8b59d208..244acdc97a9a8 100644
--- a/openmp/runtime/test/worksharing/for/kmp_set_dispatch_buf.c
+++ b/openmp/runtime/test/worksharing/for/kmp_set_dispatch_buf.c
@@ -3,7 +3,7 @@
 // RUN: %libomp-run 1 && %libomp-run 2 && %libomp-run 5
 // RUN: %libomp-compile -DMY_SCHEDULE=guided && %libomp-run 7
 // RUN: %libomp-run 1 && %libomp-run 2 && %libomp-run 5
-// UNSUPPORTED: clang-11
+// UNSUPPORTED: clang-11, clang-12
 #include <stdio.h>
 #include <omp.h>
 #include <stdlib.h>
diff --git a/parallel-libs/CMakeLists.txt b/parallel-libs/CMakeLists.txt
index c1fcf45cd7123..35ca8bc1bce10 100644
--- a/parallel-libs/CMakeLists.txt
+++ b/parallel-libs/CMakeLists.txt
@@ -1 +1 @@
-cmake_minimum_required(VERSION 3.1)
+cmake_minimum_required(VERSION 3.13.4)
diff --git a/parallel-libs/acxxel/CMakeLists.txt b/parallel-libs/acxxel/CMakeLists.txt
index 547dd62d2fbf5..6391c731903bf 100644
--- a/parallel-libs/acxxel/CMakeLists.txt
+++ b/parallel-libs/acxxel/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.1)
+cmake_minimum_required(VERSION 3.13.4)
 
 option(ACXXEL_ENABLE_UNIT_TESTS "enable acxxel unit tests" ON)
 option(ACXXEL_ENABLE_MULTI_DEVICE_UNIT_TESTS "enable acxxel multi-device unit tests" OFF)
@@ -36,6 +36,9 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 # Add warning flags.
 set(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} -Wall -Wextra")
+if (CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
+    add_compile_options("-Wno-suggest-override")
+endif()
 
 add_library(
     acxxel
diff --git a/polly/CMakeLists.txt b/polly/CMakeLists.txt
index 9939097f743ee..2e94512b2cffb 100644
--- a/polly/CMakeLists.txt
+++ b/polly/CMakeLists.txt
@@ -1,7 +1,7 @@
 # Check if this is a in tree build.
 if (NOT DEFINED LLVM_MAIN_SRC_DIR)
   project(Polly)
-  cmake_minimum_required(VERSION 3.4.3)
+  cmake_minimum_required(VERSION 3.13.4)
 
   # Where is LLVM installed?
   find_package(LLVM CONFIG REQUIRED)
diff --git a/polly/include/polly/ScopDetection.h b/polly/include/polly/ScopDetection.h
index e3c435064ca72..ead8079d2e2ee 100644
--- a/polly/include/polly/ScopDetection.h
+++ b/polly/include/polly/ScopDetection.h
@@ -514,7 +514,7 @@ class ScopDetection {
   /// @param Args Argument list that gets passed to the constructor of RR.
   template <class RR, typename... Args>
   inline bool invalid(DetectionContext &Context, bool Assert,
-                      Args &&... Arguments) const;
+                      Args &&...Arguments) const;
 
 public:
   ScopDetection(Function &F, const DominatorTree &DT, ScalarEvolution &SE,
diff --git a/polly/unittests/CMakeLists.txt b/polly/unittests/CMakeLists.txt
index fac70383de948..1a6881fde3fd0 100644
--- a/polly/unittests/CMakeLists.txt
+++ b/polly/unittests/CMakeLists.txt
@@ -19,6 +19,10 @@ function(add_polly_unittest test_name)
   target_link_libraries(${test_name} PRIVATE Polly)
 endfunction()
 
+if (CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
+  add_compile_options("-Wno-suggest-override")
+endif()
+
 add_subdirectory(Isl)
 add_subdirectory(Flatten)
 add_subdirectory(DeLICM)
diff --git a/pstl/CMakeLists.txt b/pstl/CMakeLists.txt
index 8e6e1354707b0..8bea8843589bb 100644
--- a/pstl/CMakeLists.txt
+++ b/pstl/CMakeLists.txt
@@ -5,7 +5,7 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 #===----------------------------------------------------------------------===##
-cmake_minimum_required(VERSION 3.4.3)
+cmake_minimum_required(VERSION 3.13.4)
 
 set(PARALLELSTL_VERSION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/include/pstl/internal/pstl_config.h")
 file(STRINGS "${PARALLELSTL_VERSION_FILE}" PARALLELSTL_VERSION_SOURCE REGEX "#define _PSTL_VERSION .*$")
diff --git a/sycl/CMakeLists.txt b/sycl/CMakeLists.txt
index 231917e47a9e3..1b7ae47f7b7fb 100644
--- a/sycl/CMakeLists.txt
+++ b/sycl/CMakeLists.txt
@@ -14,7 +14,7 @@ include(AddSYCLExecutable)
 set(SYCL_MAJOR_VERSION 2)
 set(SYCL_MINOR_VERSION 1)
 set(SYCL_PATCH_VERSION 0)
-set(SYCL_DEV_ABI_VERSION 0)
+set(SYCL_DEV_ABI_VERSION 4)
 if (SYCL_ADD_DEV_VERSION_POSTFIX)
   set(SYCL_VERSION_POSTFIX "-${SYCL_DEV_ABI_VERSION}")
 endif()
@@ -248,7 +248,7 @@ add_subdirectory( source )
 # Auxilliary extras for SYCL headers/library
 if (NOT WIN32)
   install(FILES
-      "${CMAKE_CURRENT_SOURCE_DIR}/xmethods/libsycl.so-gdb.py"
+      "${CMAKE_CURRENT_SOURCE_DIR}/gdb/libsycl.so-gdb.py"
       RENAME "libsycl.so.${SYCL_VERSION_STRING}-gdb.py"
       DESTINATION "lib${LLVM_LIBDIR_SUFFIX}/"
       COMPONENT sycl-headers-extras)
@@ -326,7 +326,7 @@ set( SYCL_TOOLCHAIN_DEPLOY_COMPONENTS
      sycl-headers-extras
      sycl
      pi_opencl
-     pi_level0
+     pi_level_zero
      libsycldevice
 )
 if(OpenCL_INSTALL_KHRONOS_ICD_LOADER AND TARGET ocl-icd)
diff --git a/sycl/ReleaseNotes.md b/sycl/ReleaseNotes.md
index 70d30cf3cfd5b..7c6963777fb30 100644
--- a/sycl/ReleaseNotes.md
+++ b/sycl/ReleaseNotes.md
@@ -62,7 +62,7 @@ Release notes for the commit range ba404be..24726df
   - Added a cache for PI plugins, so subsequent calls for `sycl::device`
     creation should be cheaper [03dd60d]
   - A SYCL program will be aborted now if program linking is requested when
-    using L0 plugin. This is done because L0 doesn't support program linking
+    using Level Zero plugin. This is done because L0 doesn't support program linking
     [d4a5b71]
   - Added a diagnostic on attempt to use `sycl::program::set_spec_constant` when
     the program is already in compiled or linked state [e2e3d3d]
diff --git a/sycl/doc/EnvironmentVariables.md b/sycl/doc/EnvironmentVariables.md
index 3d8d5cb41f741..b353135ecb9db 100644
--- a/sycl/doc/EnvironmentVariables.md
+++ b/sycl/doc/EnvironmentVariables.md
@@ -1,6 +1,6 @@
 # Environment Variables
 
-This document describes environment variables that are having effect on DPC++ 
+This document describes environment variables that are having effect on DPC++
 compiler and runtime.
 
 ## Controlling DPC++ RT
@@ -12,7 +12,7 @@ subject to change. Do not rely on these variables in production code.
 | Environment variable | Values | Description |
 | -------------------- | ------ | ----------- |
 | SYCL_PI_TRACE | Described [below](#sycl_pi_trace-options)  | Enable specified level of tracing for PI. |
-| SYCL_BE | PI_OPENCL, PI_LEVEL0, PI_CUDA | Force SYCL RT to consider only devices of the specified backend during the device selection. |
+| SYCL_BE | PI_OPENCL, PI_LEVEL_ZERO, PI_CUDA | Force SYCL RT to consider only devices of the specified backend during the device selection. |
 | SYCL_DEVICE_TYPE | One of: CPU, GPU, ACC, HOST | Force SYCL to use the specified device type. If unset, default selection rules are applied. If set to any unlisted value, this control has no effect. If the requested device type is not found, a `cl::sycl::runtime_error` exception is thrown. If a non-default device selector is used, a device must satisfy both the selector and this control to be chosen. This control only has effect on devices created with a selector. |
 | SYCL_PROGRAM_COMPILE_OPTIONS | String of valid OpenCL compile options | Override compile options for all programs. |
 | SYCL_PROGRAM_LINK_OPTIONS | String of valid OpenCL link options | Override link options for all programs. |
diff --git a/sycl/doc/GetStartedGuide.md b/sycl/doc/GetStartedGuide.md
index 16064b4e93c6b..66312abd105b2 100644
--- a/sycl/doc/GetStartedGuide.md
+++ b/sycl/doc/GetStartedGuide.md
@@ -27,7 +27,7 @@ and a wide range of compute accelerators such as GPU and FPGA.
 * `ninja` -
 [Download](https://github.com/ninja-build/ninja/wiki/Pre-built-Ninja-packages)
 * C++ compiler
-  * Linux: `GCC` version 5.1.0 or later (including libstdc++) -
+  * Linux: `GCC` version 7.1.0 or later (including libstdc++) -
     [Download](https://gcc.gnu.org/install/)
   * Windows: `Visual Studio` version 15.7 preview 4 or later -
     [Download](https://visualstudio.microsoft.com/downloads/)
@@ -166,37 +166,37 @@ To run DPC++ applications on Level Zero devices, Level Zero implementation(s)
 must be present in the system. You can find the link to the Level Zero spec in
 the following section [Find More](#find-more).
 
-The Level Zero RT for `GPU`, OpenCL RT for `GPU`, OpenCL RT for `CPU` and TBB runtime
-which are needed to run DPC++ application on Intel `GPU` or Intel `CPU` devices can be
-downloaded using links in
+The Level Zero RT for `GPU`, OpenCL RT for `GPU`, OpenCL RT for `CPU`, FPGA
+emulation RT and TBB runtime which are needed to run DPC++ application
+on Intel `GPU` or Intel `CPU` devices can be downloaded using links in
 [the dependency configuration file](../../buildbot/dependency.conf)
 and installed following the instructions below. The same versions are used in
 PR testing.
 
-Intel OpenCL RT for `CPU` devices can be switched into OpenCL runtime for
-Intel FPGA Emulation. The following parameter should be set in `cl.cfg` file
-(available in directory containing CPU runtime for OpenCL) or environment
-variable with the same name. The following value should be set to switch
-OpenCL device mode:
-
-```bash
-CL_CONFIG_DEVICES = fpga-emu
-```
-
 **Linux**:
 
-1) Extract the archive. For example, for the archive
-`oclcpu_rt_<cpu_version>.tar.gz` you would run the following commands
+1) Extract the archive. For example, for the archives
+`oclcpuexp_<cpu_version>.tar.gz` and `fpgaemu_<fpga_version>.tar.gz` you would
+run the following commands
 
     ```bash
+    # Extract OpenCL FPGA emulation RT
+    mkdir -p /opt/intel/oclfpgaemu_<fpga_version>
+    cd /opt/intel/oclfpgaemu_<fpga_version>
+    tar zxvf fpgaemu_<fpga_version>.tar.gz
+    # Extract OpenCL CPU RT
     mkdir -p /opt/intel/oclcpuexp_<cpu_version>
     cd /opt/intel/oclcpuexp_<cpu_version>
     tar -zxvf oclcpu_rt_<cpu_version>.tar.gz
     ```
 
-2) Create ICD file pointing to the new runtime
+2) Create ICD file pointing to the new runtime (requires root access)
 
     ```bash
+    # OpenCL FPGA emulation RT
+    echo  /opt/intel/oclfpgaemu_<fpga_version>/x64/libintelocl_emu.so >
+      /etc/OpenCL/vendors/intel_fpgaemu.icd
+    # OpenCL CPU RT
     echo /opt/intel/oclcpuexp_<cpu_version>/x64/libintelocl.so >
       /etc/OpenCL/vendors/intel_expcpu.icd
     ```
@@ -213,6 +213,16 @@ CL_CONFIG_DEVICES = fpga-emu
 folder:
 
     ```bash
+    # OpenCL FPGA emulation RT
+    ln -s /opt/intel/tbb_<tbb_version>/tbb/lib/intel64/gcc4.8/libtbb.so
+      /opt/intel/oclfpgaemu_<fpga_version>/x64
+    ln -s /opt/intel/tbb_<tbb_version>/tbb/lib/intel64/gcc4.8/libtbbmalloc.so
+      /opt/intel/oclfpgaemu_<fpga_version>/x64
+    ln -s /opt/intel/tbb_<tbb_version>/tbb/lib/intel64/gcc4.8/libtbb.so.2
+      /opt/intel/oclfpgaemu_<fpga_version>/x64
+    ln -s /opt/intel/tbb_<tbb_version>/tbb/lib/intel64/gcc4.8/libtbbmalloc.so.2
+      /opt/intel/oclfpgaemu_<fpga_version>/x64
+    # OpenCL CPU RT
     ln -s /opt/intel/tbb_<tbb_version>/tbb/lib/intel64/gcc4.8/libtbb.so
       /opt/intel/oclcpuexp_<cpu_version>/x64
     ln -s /opt/intel/tbb_<tbb_version>/tbb/lib/intel64/gcc4.8/libtbbmalloc.so
@@ -223,10 +233,12 @@ folder:
       /opt/intel/oclcpuexp_<cpu_version>/x64
     ```
 
-5) Configure library paths
+5) Configure library paths (requires root access)
 
     ```bash
-    echo /opt/intel/oclcpuexp_<cpu_version>/x64 >
+    echo /opt/intel/oclfpgaemu_<fpga_version>/x64 >
+      /etc/ld.so.conf.d/libintelopenclexp.conf
+    echo /opt/intel/oclcpuexp_<cpu_version>/x64 >>
       /etc/ld.so.conf.d/libintelopenclexp.conf
     ldconfig -f /etc/ld.so.conf.d/libintelopenclexp.conf
     ```
@@ -239,7 +251,8 @@ OpenCL runtime for Intel `GPU` installer may re-write some important
 files or settings and make existing OpenCL runtime for Intel `CPU` runtime
 not working properly.
 
-2) Extract the archive with OpenCL runtime for Intel `CPU` using links in
+2) Extract the archive with OpenCL runtime for Intel `CPU` and/or for Intel
+`FPGA` emulation using links in
 [the dependency configuration file](../../buildbot/dependency.conf).  For
 example, to `c:\oclcpu_rt_<cpu_version>`.
 
@@ -257,6 +270,11 @@ extracted files are in `c:\oclcpu_rt_<cpu_version>\` folder, then type the
 command:
 
     ```bash
+    # Install OpenCL FPGA emulation RT
+    # Answer N to clean previous OCL_ICD_FILENAMES configuration
+    c:\oclfpga_rt_<fpga_version>\install.bat c:\tbb_<tbb_version>\tbb\bin\intel64\vc14
+    # Install OpenCL CPU RT
+    # Answer Y to setup CPU RT side-bi-side with FPGA RT
     c:\oclcpu_rt_<cpu_version>\install.bat c:\tbb_<tbb_version>\tbb\bin\intel64\vc14
     ```
 
diff --git a/sycl/doc/extensions/ExplicitSIMD/dpcpp-explicit-simd.md b/sycl/doc/extensions/ExplicitSIMD/dpcpp-explicit-simd.md
new file mode 100644
index 0000000000000..501d19b172e16
--- /dev/null
+++ b/sycl/doc/extensions/ExplicitSIMD/dpcpp-explicit-simd.md
@@ -0,0 +1,561 @@
+# Explicit SIMD Programming Extension for DPC++
+
+Note:
+_This document describes current design and API for the Intel Explicit SIMD
+extension to DPC++. This is initial experimental version to try out functionality
+and performance. We are going to work with the community on incrementally improving
+the APIs to bring them closer to std C++ and SYCL in the next several months._
+
+## Introduction
+
+The main motivation for introducing the Explicit SIMD Programming (ESP) DPC++
+extension is enabling low-level efficient programming for Intel Gen
+architectures. More specifically, Explicit SIMD provides the following
+additional features:
+- Manual vectorization of device code using the `simd` class mapped to Gen's
+  general register file. This allows to write efficient code not relying on
+  further widening by the compiler, as with traditional SPMD programming.
+- Low-level APIs efficiently mapped to Gen architecture, such as block reads.
+
+## Explicit SIMD execution model
+
+Explicit SIMD execution model is basically an equivalent of the base SYCL
+execution model with subgroup size restricted to 1. Which means each subgroup
+maps to a single hardware thread. All standard SYCL APIs continue to work,
+including `sycl::intel::sub_group` ones, which become either a no-op or
+trivial. E.g. a barrier becomes just a memory fence for a compiler, collectives
+just return the value in the single work-item. Another consequences of the unit
+subgroup size is guaranteed independent forward progress between work-items on
+some Gen architecture generations.
+
+## Explicit SIMD extension APIs
+
+Explicit SIMD APIs can be used only in code to be executed on Intel Gen
+architecture devices and the host device for now. Attempt to run such code on
+other devices will result in error. 
+
+Kernels and `SYCL_EXTERNAL` functions using ESP must be explicitly marked with
+the `[[intel::sycl_explicit_simd]]` attribute. Subgroup size query within such
+functions will always return `1`.
+
+*Functor kernel*
+```cpp
+using AccTy = sycl::accessor<int, 1, sycl::access::mode::read_write,
+                                 sycl::access::target::global_buffer>;
+class Functor {
+public:
+  Functor(int X, AccTy &Acc) : X(X), Acc(Acc) {}
+
+  void operator()() [[intel::sycl_explicit_simd]] { Acc[0] += X; }
+
+private:
+  int X;
+  AccTy Acc;
+};
+```
+
+*Lambda kernel and function*
+```cpp
+SYCL_EXTERNAL
+void sycl_device_f(sycl::global_ptr<int> ptr, sycl::intel::gpu::simd<float, 8> X) [[intel::sycl_explicit_simd]] {
+  sycl::intel::gpu::flat_block_write(*ptr.get(), X);
+}
+...
+  Q.submit([&](sycl::handler &Cgh) {
+    auto Acc1 = Buf1.get_access<sycl::access::mode::read>(Cgh);
+    auto Acc2 = Buf2.get_access<sycl::access::mode::read_write>(Cgh);
+
+    Cgh.single_task<class KernelID>([=] () [[intel::sycl_explicit_simd]] {
+      sycl::intel::gpu::simd<float, 8> Val = sycl::intel::gpu::flat_block_read(Acc1.get_pointer());
+      sycl_device_f(Acc2, Val);
+    });
+  });
+}
+```
+
+## Implementation restrictions
+Current ESP implementation does not support using certain standard SYCL features
+inside explicit SIMD kernels and functions. Most of them will be eventually
+dropped. What's not supported today:
+- Mixing `[[intel::sycl_explicit_simd]]` kernels with SYCL kernels in a single source
+- Local accessors. Local memory is allocated and accessed via explicit
+device-side API
+- 2D and 3D accessors
+- Constant accessors
+- `sycl::accessor::get_pointer()`. All memory accesses through an accessor are
+done via explicit APIs; e.g. `sycl::intel::gpu::block_store(acc, offset)`
+- Few others (to be documented)
+
+
+## Core Explicit SIMD programming APIs
+
+The DPC++ Explicit SIMD library defines the following classes to enhance the
+expressiveness for explicit SIMD data-parallel programming while enabling
+efficient mapping to SIMD vector operations on Intel GPU architectures.
+
+### SIMD vector class
+
+The `sycl::intel::gpu::simd` class is a vector templated on some element type.
+The element type must be vectorizable type. The set of vectorizable types is the
+set of fundamental SYCL arithmetic types (C++ arithmetic types or `half` type)
+excluding `bool`. The length of the vector is the second template parameter.
+
+ESIMD compiler back-end does the best it can to map each `simd` class object to a consecutive block
+of registers in the general register file (GRF).
+
+Every specialization of ```sycl::intel::gpu::simd``` shall be a complete type. The term
+simd type refers to all supported specialization of the simd class template.
+To access the i-th individual data element in a simd vector, Explicit SIMD supports the
+standard subscript operator ```[]```, which returns by value.
+
+For simd type object, Explicit SIMD supports the following simd vector operations:
+- Unary operators: ++ (*pre-/post-increment*), -- (*pre-/post-decrement*)
+- Binary operators: +, -, *, /, &, |, ^
+- Compound assignments: +=, -=, *=, /=, &=, |=, ^=
+- Compare operators: >, >=, <, <=, ==, !=
+
+These are all element-wise operations, which apply a specified operation to the
+elements of one or more simd objects and should follow the standard C++ rules for
+the corresponding scalar data element computation. Each such application is
+unsequenced with respect to the others.
+
+To reference a subset of the elements in simd vector object, Explicit SIMD provides ```select```
+function, which returns a `simd` or `simd_view` object (*described below*) representing
+the selected sub-vector starting from the i-th element. The number of selected
+elements is specified by the template parameter **Size**, while the distance
+between two adjacent elements is specified by the template parameter **Stride**.
+
+```cpp
+  simd<int, 8> a;
+  simd<int, 4> b;
+  // ...
+  b = a.select<4, 2>(1);  // size=4, stride=2, offset=1 (elements a(1),
+                          // a(3), a(5) and a(7) are copied to b)
+```
+
+<p align="center">
+<img src="images/VectorOdd.svg" title="1D select example" width="600" height="60"/>
+</p>
+
+```cpp
+a.select<4, 2>(0) = b;  // selected elements of a are replaced
+                        // with elements of b (all elements of b are
+                        // copied to elements a(0), a(2), a(4), a(6))
+```
+
+<p align="center">
+<img src="images/VectorEven.svg" title="1D select example" width="600" height="60"/>
+</p>
+
+
+Gen ISA provides powerful register region addressing modes to facilitate cross-lane
+SIMD vector operation. To exploit this feature Explicit SIMD provides ```replicate``` function
+to allow programmer to implement any native Gen ISA region in the following forms:
+- ```replicate<REP>()```: replicate a simd vector object **REP** times and return a new simd
+vector of **REP** * Width, where Width specifies the original vector size.
+- ```replicate<REP, W>(uint16_t i)```: replicate **W** consecutive elements starting at the
+i-th element from the simd vector object **REP** times, and return a new simd vector of **REP** * **W** length.
+- ```replicate<REP, VS, W>(uint16_t i)```: replicate **REP** blocks of **W** consecutive
+elements starting at the i-th from the simd vector object with each block strided by **VS**
+elements, and return a new vector of **REP** * **W** length. Selected blocks of **W**
+elements will overlap if **VS** < **W**.
+- ```replicate<REP, VS, W, HS>(uint16_t i=0 )```: replicate **REP** blocks of **W** sequential
+elements with a stride of **HS** starting at the i-th element from the simd vector object with
+each block strided by **VS** elements, and return a new vector of **REP** * **W** length.
+Selected blocks of **W** elements will overlap if **VS** < **W**.
+
+To avoid explicit type cast and the resulting move instructions for large vectors, Explicit SIMD allows
+programmer to reinterpret the fundamental data element type of a simd vector object and change
+its shape to 1D or 2D object through the ```format``` function:
+- ```format<EltTy>( )```: returns a reference to the calling simd object interpreted as a new
+simd vector with the size determined by the template **EltTy** parameter.
+- ```format<EltTy, Height, Width>( )```: returns a reference to the calling simd object interpreted
+as a new 2D simd_view object with the shape determined by the template parameters **Height** and**Width**. The size of the new 2D block must not exceed the size of the original object.
+
+```cpp
+  simd<int, 16> v1;
+   // ...
+  auto v2 = v1.format<short>;
+  // v2 is a reference to the location of v1
+  // interpreted as a vector of 32 shorts.
+  // ...
+  auto m1 = v1.format<int, 4, 4>;
+  // m1 is a reference to the location of v1
+  // interpreted as a matrix 4x4 of ints.
+  // ...
+  auto m2 = v1.format<char, 4, 16>( );
+  // m2 is a reference to the location of v1
+  // interpreted as a matrix 4x16 of chars.
+```
+
+To model predicated move, Explicit SIMD provides the following merge functions:
+- ```merge(value_type Val, mask_type Mask)```: this merge operation takes one source operand **Val** and a mask **Mask** defined as unsigned short vector of the same length. The semantic is that if the LSB of an element of **Mask** is set, then the corresponding data element of **Val** is copied to the corresponding position in the calling simd object.
+
+```cpp
+  simd<int, 4>   m, src;
+  simd<unsigned short, 4> mask;
+  // ...
+  m.merge(src, mask);
+  // m           src         mask      --->  m
+  // 2 2 2 2     4 4 4 4     1 1 0 1         4 4 2 4
+```
+
+- ```merge(value_type Val1, value_type Val2, mask_type Mask)```: this merge operation takes two source operands **Val1** and **Val2** as well as a simd mask. The semantic is that if the LSB of an element of **Mask** is set, then the corresponding data element of **Val1** is copied to the corresponding position in the calling simd object. Otherwise the corresponding data element of **Val2** is copied to the corresponding position in the calling simd object.
+
+```cpp
+  simd<int,4>   m, src1, src2;
+  simd<unsigned short, 4> mask;
+  // ...
+  m.merge(src1, src2, mask);
+  // m           src1        src2         mask        --->  m
+  // 2 2 2 2     4 4 4 4     3 3 3 3      1 1 0 1           4 4 3 4
+```
+### `simd_view` class
+
+The ```sycl::intel::gpu::simd_view``` represents a "window" into existing simd object,
+through which a part of the original object can be read or modified. This is a
+syntactic convenience feature to reduce verbosity when accessing sub-regions of
+simd objects. **RegionTy** describes the window shape and can be 1D or 2D,
+**BaseTy** is the original simd object type, which can be a ```simd_view```
+itself.
+
+```simd_view``` allows to model hierarchical "views" of the parent ```simd```
+object's parts, read/modify its elements through the views. Views can be of
+different shapes and dimensions as illustrated below (`auto` resolves to a
+`simd_view` instantiation):
+
+<p align="center">
+<img src="images/simd_view.svg" title="1D select example" width="800" height="300"/>
+</p>
+
+```sycl::intel::gpu::simd_view``` class supports all the element-wise operations and
+other utility functions defined for ```sycl::intel::gpu::simd``` class. It also
+provides region accessors and more generic operations tailored for 2D regions,
+such as row/column operators and 2D select/replicate/format/merge operations.
+
+```cpp
+  simd<float, 32> v1;
+  auto m1 = v1.format<float, 4, 8>();
+  simd<float, 4> v2;
+  auto m2 = v2.format<float, 2, 2>();
+
+  // ...
+  m2 = m1.select<2, 2, 2, 4>(1, 2);  // v_size = 2, v_stride = 2,
+                                     // h_size = 2, h_stride = 4,
+                                     // v_offset = 1, h_offset = 2.
+```
+<p align="center">
+<img src="images/Matrix_2_2_2_4__1_2.svg" title="2D select example" width="700" height="200"/>
+</p>
+
+```cpp
+  m1.select<4, 1, 4, 2>(0, 0) = 0.0f; // selected elements of m1
+                                      // are replaced with 0.0f
+```
+
+<p align="center">
+<img src="images/Matrix_4_1_4_2__0_0.svg" title="2D select example" width="700" height="200"/>
+</p>
+
+### Reduction functions
+
+Explicit SIMD provides the following reduction functions for simd objects.
+Compiler will produce optimal code sequence on the target device to apply the
+specified operation to all scalar elements in the input simd vector. Note that
+the order of element-wise operations is not guaranteed and the correctness of
+result should not depend on a particular computation order.
+- ```reduce(simd<T1, SZ> v, std::plus<>())```: returns a scalar value of type
+**T0** that's equal to the sum of all data elements of type **T1** in simd vector
+**v** of length **SZ**.
+- ```reduce(simd<T1, SZ> v, std::multiplies<>())```: returns a scalar value of
+type **T0** that's equal to the product of all data elements of type **T1** in simd
+vector **v** of length **SZ**.
+- ```hmax(simd<T1, SZ> v)```: returns a scalar value of type **T0**
+that's equal to the maximum value of all data elements of type **T1** in simd
+vector **v** of length **SZ**.
+- ```hmin(simd<T1, SZ> v)```: returns a scalar value of type **T0**
+that's equal to the minimum value of all data elements of type **T1** in simd
+vector **v** of length **SZ**.
+
+```cpp
+template <typename T0, typename T1, int SZ, typename BinaryOperation>
+T0 reduce(simd<T1, SZ> v, BinaryOperation op);
+
+template <typename T0, typename T1, int SZ> T0 hmax(simd<T1, SZ> v);
+template <typename T0, typename T1, int SZ> T0 hmin(simd<T1, SZ> v);
+```
+
+### Memory access APIs
+
+Currently the variety of memory objects supported by the Explicit SIMD extension
+implementation is limited to the following:
+- USM pointers (aka 'flat address')
+- 1D global accessors
+- 2D image accessors
+
+The implementation further limits the set of memory access operations
+which a kernel can perform through those memory objects. Basically, all of them
+are special intrinsic APIs described below rather than standard SYCL accessor or
+pointer operations. Examples of unsupported features include
+`accessor::get_pointer()`, accessor's `operator []`, C/C++ dereference of an
+USM pointer, local accessors, 2D and 3D accessors.
+Those are temporary restrictions, to be dropped in future.
+
+See auto-generated documentation for the complete list of APIs here. (TBD)
+
+#### USM pointer-based memory access
+##### Flat-address gather/scatter
+perform scattered read/write for the given starting pointer **p** and
+**offsets**.
+
+```cpp
+template <typename T, int n, int ElemsPerAddr = 1,
+	  CacheHint L1H = CacheHint::Default,
+          CacheHint L3H = CacheHint::Default>
+typename std::enable_if<((n == 16 || n == 32) &&
+    (ElemsPerAddr == 1 || ElemsPerAddr == 2 || ElemsPerAddr == 4)),
+    simd<T, n*ElemsPerAddr>>::type
+flat_load(T *p, simd<uint32_t, n> offsets, simd<uint16_t, n> pred = 1);
+
+template <typename T, int n, int ElemsPerAddr = 1,
+	  CacheHint L1H = CacheHint::Default,
+          CacheHint L3H = CacheHint::Default>
+typename std::enable_if<((n == 16 || n == 32) &&
+    (ElemsPerAddr == 1 || ElemsPerAddr == 2 || ElemsPerAddr == 4)),
+    void>::type
+flat_store(T *p, simd<T, n*ElemsPerAddr> vals, simd<uint32_t, n> offsets,
+           simd<uint16_t, n> pred = 1);
+```
+##### Flat-address block load/store
+read or write a consecutive block of data for the memory location specified by
+**addr**.
+
+```cpp
+template <typename T, int n, CacheHint L1H = CacheHint::Default,
+          CacheHint L3H = CacheHint::Default>
+simd<T, n> flat_block_load(const T *const addr);
+
+template <typename T, int n, CacheHint L1H = CacheHint::Default,
+          CacheHint L3H = CacheHint::Default>
+void flat_block_store(T *p, simd<T, n> vals);
+```
+
+##### Flat-address load4/store4
+perform scattered read/write for the given starting pointer **p** and
+**offsets**. Up to 4 data elements may be accessed at each address depending on
+the enabled channel **Mask**.
+
+```cpp
+template <typename T, int n, ChannelMaskType Mask,
+          CacheHint L1H = CacheHint::Default,
+          CacheHint L3H = CacheHint::Default>
+    typename std::enable_if<(n == 16 || n == 32),
+                            simd<T, n * NumChannels(Mask)>>::type
+    flat_load4(T *p, simd<uint32_t, n> offsets, simd<uint16_t, n> pred = 1);
+
+template <typename T, int n, ChannelMaskType Mask,
+          CacheHint L1H = CacheHint::Default,
+          CacheHint L3H = CacheHint::Default>
+typename std::enable_if<(n == 16 || n == 32), void>::type
+    flat_store4(T *p, simd<T, n * NumChannels(Mask)> vals,
+            simd<uint32_t, n> offsets, simd<uint16_t, n> pred = 1);
+```
+
+##### Flat-address atomic inc/dec
+perform atomic memory access operation with zero source operand.
+
+```cpp
+template <CmAtomicOpType Op, typename T, int n,
+          CacheHint L1H = CacheHint::Default,
+          CacheHint L3H = CacheHint::Default>
+    typename std::enable_if<check_atomic<Op, T, n, 0>(), simd<T, n>>::type
+    flat_atomic(T *p, simd<unsigned, n> offset, simd<ushort, n> pred);
+```
+
+- ```Flat-address atomic add/sub/min/max/etc.```: perform atomic memory
+access operation with one source operand.
+
+```cpp
+template <CmAtomicOpType Op, typename T, int n,
+          CacheHint L1H = CacheHint::Default,
+          CacheHint L3H = CacheHint::Default>
+    typename std::enable_if<check_atomic<Op, T, n, 1>(), simd<T, n>>::type
+    flat_atomic(T *p, simd<unsigned, n> offset, simd<T, n> src0,
+                simd<ushort, n> pred);
+```
+
+- ```Flat-address atomic CMPXCHG```: perform atomic memory access operation
+with two source operands.
+
+```cpp
+
+template <CmAtomicOpType Op, typename T, int n,
+          CacheHint L1H = CacheHint::Default,
+          CacheHint L3H = CacheHint::Default>
+    typename std::enable_if<check_atomic<Op, T, n, 2>(), simd<T, n>>::type
+    flat_atomic(T *p, simd<unsigned, n> offset, simd<T, n> src0,
+                simd<T, n> src1, simd<ushort, n> pred);
+```
+
+#### Accessor-based memory access
+
+Examples:
+##### 1D surface block store.
+T - element type, n - vector length, acc - SYCL global buffer accessor,
+offset - byte offset in the buffer, vals - vector value to store
+```cpp
+template <typename T, int n, typename AccessorTy>
+void block_store(AccessorTy acc, uint32_t offset, simd<T, n> vals);
+```
+##### 2D media block load.
+T - element type, m and n - block dimensions, acc - SYCL image2D accessor,
+x and y - image coordinates
+```cpp
+template <typename T, int m, int n, typename AccessorTy, unsigned plane = 0>
+simd<T, m * n> media_block_load(AccessorTy acc, unsigned x, unsigned y);
+```
+
+#### Local address space allocation and access
+Examples:
+
+##### SLM scatter.
+T - element type, n - vector length (16 or 32), vals - vector value to store,
+offsets - byte offsets in the local memory, pred - store mask
+```cpp
+template <typename T, int n>
+typename std::enable_if<(n == 16 || n == 32), void>::type
+slm_store(simd<T, n> vals, simd<uint32_t, n> offsets, simd<uint16_t, n> pred = 1);
+```
+<br>
+
+### Private Global Variables.
+
+Explicit SIMD extension supports "private global" variables - file scope
+variables in private address space (similar to thread-local variables on host).
+These variables have 1 copy per work-item (which maps to a single SIMD thread in
+ESP) and are visible to all functions in the translation unit. Conceptually they
+map to SPIRV variable with private storage class. Private globals can be bound
+to a specific byte offset within the GRF. To mark a file scope variable as
+private global, the `INTEL_GPU_PRIVATE` attribute is used,
+`INTEL_GPU_REGISTER` is used to bind it the register file:
+
+```cpp
+INTEL_GPU_PRIVATE INTEL_GPU_REGISTER(32) simd<int, 16> vc;
+```
+<br>
+
+## Examples
+### Vector addition (USM)
+```cpp
+#include <iostream>
+#include <CL/sycl.hpp>
+#include <sycl_esimd.hpp>
+
+using namespace sycl;
+
+int main(void) {
+  constexpr unsigned Size = 128;
+  constexpr unsigned VL = 32;
+  constexpr unsigned GroupSize = 2;
+
+  queue q;
+  auto dev = q.get_device();
+  std::cout << "Running on " << dev.get_info<info::device::name>() << "\n";
+  auto ctxt = q.get_context();
+  float* A = static_cast<float*>(malloc_shared(Size*sizeof(float), dev, ctxt));
+  float* B = static_cast<float*>(malloc_shared(Size*sizeof(float), dev, ctxt));
+  float* C = static_cast<float*>(malloc_shared(Size*sizeof(float), dev, ctxt));
+
+  for (auo i = 0; i != Size; i++) {
+    A[i] = B[i] = i;
+  }
+
+  // iteration space
+  nd_range<1> Range(range<1>(Size/VL), range<1>(GroupSize));
+
+  auto e = q.submit([&](handler &cgh) {
+    cgh.parallel_for<class Test>(
+      Range, [=](nd_item<1> i) [[intel::sycl_explicit_simd]] {
+
+      auto offset = i.get_global_id(0) * VL;
+      sycl::intel::gpu<float, VL> va = sycl::intel::gpu::flat_block_load<float, VL>(A + offset);
+      sycl::intel::gpu<float, VL> vb = sycl::intel::gpu::flat_block_load<float, VL>(B + offset);
+      sycl::intel::gpu<float, VL> vc = va + vb;
+      sycl::intel::gpu::flat_block_store<float, VL>(C + offset, vc);
+    });
+  });
+  e.wait();
+  return 0;
+}
+```
+
+### Open questions
+- Vectorization controls (e.g. enforcing vector length generated by the
+  compiler) for vector/matrix operations.
+- Enabling loop vectorizer on inner loops in a Explicit SIMD kernel or function.
+
+### TODOs
+
+- Design interoperability with SPMD context - e.g. invocation of ESIMD functions
+  from a standard SYCL code
+- Generate sycl::intel::gpu API documentation from sources
+- Section covering 2D use cases
+- A bridge from `std::simd` to `sycl::intel::gpu::simd`
+- Describe `simd_view` class restrictions
+- Support intermixing SYCL and ESIMD kernels in the same source.
+- Support OpenCL and L0 interop for ESIMD kernels
+- Consider auto-inclusion of sycl_explicit_simd.hpp under -fsycl-explicit-simd option
+- Add example showing the mapping between an ND-range and the number of
+thread-groups and EU threads, and showing the usage of explicit SIMD together
+with work-group barriers and SLM.
+
+### Discussion topics
+1. Inter-relation with `std::simd`.
+
+- @rolandschulz suggests __not__ to imeplement `std::simd` for Intel GPU based on
+`sycl::intel::gpu`, but directly on top of clang vectors.
+> Implementing it on top of `gpu::simd` is the strategy which requires a reimplementation
+> of `std::simd` features missing from `gpu::simd`. On the other hand there is an existing
+> implementation directly on top of gcc/clang vector: https://github.com/VcDevel/std-simd.
+> We should explore how we can reuse most of that implementation also for the GPU.
+
+- @mattkretz suggests to consider making `std::simd` good enough for GPU programming
+ instead of creating a highly specific extension.
+> A SIMD type that doesn't allow you to reach full performance is a failure
+> (or hopefully just buggy and needs to be fixed). Performance is the reason why we use it.
+
+...
+
+> I guess what I'd like to discuss is: If you deviate from the Parallelism TS 2, why?
+> Is it to simplify your implementation, is it because of missing functionality, is it
+> because of performance, or? This is important feedback to the C++ committee when merging
+> the TS into the IS is discussed.
+
+
+2. Invocation API
+`Cgh.single_task<class KernelID>([=] () EXPLICIT_SIMD`
+- @Ruyk suggests to use different handler API to invoke ESIMD kernels rather
+  than using attributes to mark lambda/functor type:
+> Well we could have a single "parallel_for" for everything but sometimes is
+> better to distinguish entry points, specially if the programming model is very
+> different. That macro in the lambda is easy to miss, specially if the kernel
+> lambda is defined elsewhere.
+
+...
+
+> One of my concerns is that you could have the kernel separated from the
+> single_task point of entry, and you may not even know what kind of kernel is
+> that you are dispatching, just a SYCL kernel. Without a way for a C++
+> developer to identify the usage of this different API inside, they will be
+> reduced to random runtime errors if the wrong kernel is used in the wrong device.
+
+- @keryell suggests to use kernel properties and avoid macros:
+https://www.youtube.com/watch?v=Fp8DuVWesT4 and https://gitlab.khronos.org/sycl/Specification/issues/296
+
+- @kbobrovs mentioned that there was another suggestion from @rolandschulz to
+  use some extra "executor" template parameter, which would be set to "ESIMD"
+  for ESIMD kernels.
+
+3. Enabling this extension for other architectures, such as x86, with extracting
+  and clearly marking generic and target-dependent API portions
+
diff --git a/sycl/doc/extensions/ExplicitSIMD/images/Matrix_2_2_2_4__1_2.svg b/sycl/doc/extensions/ExplicitSIMD/images/Matrix_2_2_2_4__1_2.svg
new file mode 100644
index 0000000000000..1e526eba415ba
--- /dev/null
+++ b/sycl/doc/extensions/ExplicitSIMD/images/Matrix_2_2_2_4__1_2.svg
@@ -0,0 +1,374 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   sodipodi:docname="CMMatrix_2_2_2_4__1_2.svg"
+   inkscape:version="1.0rc1 (09960d6f05, 2020-04-09)"
+   id="svg8"
+   version="1.1"
+   viewBox="0 0 276.2329 71.202034"
+   height="71.202034mm"
+   width="276.23288mm">
+  <defs
+     id="defs2">
+    <rect
+       id="rect1123"
+       height="27.796032"
+       width="201.25395"
+       y="102.8446"
+       x="-47.093624" />
+    <rect
+       id="rect1117"
+       height="25.177505"
+       width="109.10014"
+       y="26.672779"
+       x="-99.74572" />
+  </defs>
+  <sodipodi:namedview
+     inkscape:window-maximized="1"
+     inkscape:window-y="-9"
+     inkscape:window-x="69"
+     inkscape:window-height="1411"
+     inkscape:window-width="2482"
+     lock-margins="true"
+     fit-margin-bottom="5"
+     fit-margin-right="5"
+     fit-margin-left="5"
+     fit-margin-top="5"
+     showgrid="false"
+     inkscape:document-rotation="0"
+     inkscape:current-layer="layer1"
+     inkscape:document-units="mm"
+     inkscape:cy="242.02806"
+     inkscape:cx="875.92901"
+     inkscape:zoom="0.98994949"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0.0"
+     borderopacity="1.0"
+     bordercolor="#666666"
+     pagecolor="#ffffff"
+     id="base" />
+  <metadata
+     id="metadata5">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     transform="translate(135.59491,-1.0149053)"
+     id="layer1"
+     inkscape:groupmode="layer"
+     inkscape:label="Layer 1">
+    <rect
+       id="rect833-4"
+       width="15.768902"
+       height="15.234363"
+       x="40.892223"
+       y="51.850288"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       id="rect833-6"
+       width="15.768902"
+       height="15.234363"
+       x="40.892246"
+       y="6.1471977"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="21.381563"
+       x="103.96786"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-4-9" />
+    <rect
+       id="rect833-3"
+       width="15.768902"
+       height="15.234363"
+       x="25.123344"
+       y="6.1471968"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="51.850288"
+       x="103.96786"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-4-3" />
+    <rect
+       id="rect833-5"
+       width="15.768902"
+       height="15.234363"
+       x="9.3544416"
+       y="6.1471968"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="21.381565"
+       x="40.892242"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-4-6" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="6.1471977"
+       x="56.661148"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-6-8" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="6.1472006"
+       x="103.96786"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-6-4" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="6.1471996"
+       x="88.198959"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-3-2" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="6.1471996"
+       x="72.430054"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-5-3" />
+    <rect
+       id="rect833-6-8-3"
+       width="15.768902"
+       height="15.234363"
+       x="119.73676"
+       y="6.1472006"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="21.381559"
+       x="25.123331"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-3-4" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="21.381559"
+       x="9.3544283"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-5-1" />
+    <rect
+       id="rect833-6-8-0"
+       width="15.768902"
+       height="15.234363"
+       x="56.661133"
+       y="21.381563"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       id="rect833-3-2-7"
+       width="15.768902"
+       height="15.234363"
+       x="88.198952"
+       y="21.381563"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       id="rect833-5-3-0"
+       width="15.768902"
+       height="15.234363"
+       x="72.430038"
+       y="21.381563"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="21.381563"
+       x="119.73676"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-6-8-3-6" />
+    <rect
+       id="rect833-6-9-4"
+       width="15.768902"
+       height="15.234363"
+       x="40.892242"
+       y="36.615925"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       id="rect833-3-4-7"
+       width="15.768902"
+       height="15.234363"
+       x="25.123337"
+       y="36.615921"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       id="rect833-5-1-3"
+       width="15.768902"
+       height="15.234363"
+       x="9.3544312"
+       y="36.615921"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="36.615925"
+       x="56.66114"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-6-8-0-8" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="36.615925"
+       x="103.96786"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-6-4-6-5" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="36.615925"
+       x="88.198967"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-3-2-7-6" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="36.615925"
+       x="72.430046"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-5-3-0-3" />
+    <rect
+       id="rect833-6-8-3-6-2"
+       width="15.768902"
+       height="15.234363"
+       x="119.73678"
+       y="36.615925"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="51.850285"
+       x="25.123322"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-3-4-7-3" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="51.850285"
+       x="9.3544159"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-5-1-3-3" />
+    <rect
+       id="rect833-6-8-0-8-3"
+       width="15.768902"
+       height="15.234363"
+       x="56.661125"
+       y="51.850288"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       id="rect833-3-2-7-6-9"
+       width="15.768902"
+       height="15.234363"
+       x="88.198952"
+       y="51.850288"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       id="rect833-5-3-0-3-4"
+       width="15.768902"
+       height="15.234363"
+       x="72.430031"
+       y="51.850288"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="51.850288"
+       x="119.73676"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-6-8-3-6-2-6" />
+    <text
+       style="font-style:normal;font-weight:normal;font-size:10.5833px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;white-space:pre;shape-inside:url(#rect1117);fill:#000000;fill-opacity:1;stroke:none;"
+       id="text1115"
+       xml:space="preserve" />
+    <text
+       transform="translate(-84.457171,-72.430042)"
+       style="font-style:normal;font-weight:normal;font-size:10.5833px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;white-space:pre;shape-inside:url(#rect1123);fill:#000000;fill-opacity:1;stroke:none;"
+       id="text1121"
+       xml:space="preserve"><tspan
+         sodipodi:role="line"
+         x="-47.09375"
+         y="112.49922"><tspan>m1.select&lt;2,2,2,4&gt;(1, 2)</tspan></tspan></text>
+  </g>
+</svg>
diff --git a/sycl/doc/extensions/ExplicitSIMD/images/Matrix_4_1_4_2__0_0.svg b/sycl/doc/extensions/ExplicitSIMD/images/Matrix_4_1_4_2__0_0.svg
new file mode 100644
index 0000000000000..3adb83cf1abb6
--- /dev/null
+++ b/sycl/doc/extensions/ExplicitSIMD/images/Matrix_4_1_4_2__0_0.svg
@@ -0,0 +1,384 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   sodipodi:docname="CMMatrix_4_1_4_2__0_0.svg"
+   inkscape:version="1.0rc1 (09960d6f05, 2020-04-09)"
+   id="svg8"
+   version="1.1"
+   viewBox="0 0 276.23287 71.202049"
+   height="71.202049mm"
+   width="276.23285mm">
+  <defs
+     id="defs2">
+    <rect
+       id="rect2415"
+       height="100.49334"
+       width="309.49811"
+       y="93.222893"
+       x="170.4287" />
+    <rect
+       id="rect1123"
+       height="27.796032"
+       width="201.25395"
+       y="102.8446"
+       x="-47.093624" />
+    <rect
+       id="rect1117"
+       height="25.177505"
+       width="109.10014"
+       y="26.672779"
+       x="-99.74572" />
+  </defs>
+  <sodipodi:namedview
+     inkscape:window-maximized="1"
+     inkscape:window-y="-9"
+     inkscape:window-x="69"
+     inkscape:window-height="1411"
+     inkscape:window-width="2482"
+     lock-margins="true"
+     fit-margin-bottom="5"
+     fit-margin-right="5"
+     fit-margin-left="5"
+     fit-margin-top="5"
+     showgrid="false"
+     inkscape:document-rotation="0"
+     inkscape:current-layer="layer1"
+     inkscape:document-units="mm"
+     inkscape:cy="401.63215"
+     inkscape:cx="875.92901"
+     inkscape:zoom="0.98994949"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0.0"
+     borderopacity="1.0"
+     bordercolor="#666666"
+     pagecolor="#ffffff"
+     id="base" />
+  <metadata
+     id="metadata5">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     transform="translate(135.59491,-1.5494455)"
+     id="layer1"
+     inkscape:groupmode="layer"
+     inkscape:label="Layer 1">
+    <rect
+       id="rect833-4"
+       width="15.768902"
+       height="15.234363"
+       x="40.892242"
+       y="6.6817369"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       id="rect833-3"
+       width="15.768902"
+       height="15.234363"
+       x="56.661144"
+       y="6.6817369"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       id="rect833-5"
+       width="15.768902"
+       height="15.234363"
+       x="25.123341"
+       y="6.6817446"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="6.6817446"
+       x="9.3544378"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-4-6" />
+    <text
+       style="font-style:normal;font-weight:normal;font-size:10.5833px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;white-space:pre;shape-inside:url(#rect1117);fill:#000000;fill-opacity:1;stroke:none;"
+       id="text1115"
+       xml:space="preserve" />
+    <text
+       transform="translate(-84.457171,-72.430042)"
+       style="font-style:normal;font-weight:normal;font-size:10.5833px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;white-space:pre;shape-inside:url(#rect1123);fill:#000000;fill-opacity:1;stroke:none;"
+       id="text1121"
+       xml:space="preserve"><tspan
+         sodipodi:role="line"
+         x="-47.09375"
+         y="112.49922"><tspan>m1.select&lt;4,1,4,2&gt;(0, 0)</tspan></tspan></text>
+    <rect
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="6.6817369"
+       x="103.96785"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-4-5" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="6.6817369"
+       x="119.73675"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-3-1" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="6.6817369"
+       x="88.198944"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-5-0" />
+    <rect
+       id="rect833-4-6-3"
+       width="15.768902"
+       height="15.234363"
+       x="72.430046"
+       y="6.6817369"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="21.916096"
+       x="40.892242"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-4-56" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="21.916096"
+       x="56.66114"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-3-7" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="21.916103"
+       x="25.123339"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-5-08" />
+    <rect
+       id="rect833-4-6-5"
+       width="15.768902"
+       height="15.234363"
+       x="9.3544407"
+       y="21.916103"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       id="rect833-4-5-0"
+       width="15.768902"
+       height="15.234363"
+       x="103.96784"
+       y="21.916096"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       id="rect833-3-1-1"
+       width="15.768902"
+       height="15.234363"
+       x="119.73675"
+       y="21.916096"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       id="rect833-5-0-2"
+       width="15.768902"
+       height="15.234363"
+       x="88.198944"
+       y="21.916096"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="21.916096"
+       x="72.430046"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-4-6-3-7" />
+    <rect
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="37.150455"
+       x="40.892242"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-4-7" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="37.150455"
+       x="56.66114"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-3-11" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="37.150455"
+       x="25.123341"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-5-6" />
+    <rect
+       id="rect833-4-6-37"
+       width="15.768902"
+       height="15.234363"
+       x="9.3544426"
+       y="37.150455"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       id="rect833-4-5-1"
+       width="15.768902"
+       height="15.234363"
+       x="103.96784"
+       y="37.150455"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       id="rect833-3-1-3"
+       width="15.768902"
+       height="15.234363"
+       x="119.73675"
+       y="37.150455"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       id="rect833-5-0-8"
+       width="15.768902"
+       height="15.234363"
+       x="88.198944"
+       y="37.150455"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="37.150455"
+       x="72.430046"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-4-6-3-8" />
+    <rect
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="52.384823"
+       x="40.892227"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-4-59" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="52.384823"
+       x="56.661133"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-3-46" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="52.384838"
+       x="25.123322"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-5-9" />
+    <rect
+       id="rect833-4-6-1"
+       width="15.768902"
+       height="15.234363"
+       x="9.3544235"
+       y="52.384838"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       id="rect833-4-5-12"
+       width="15.768902"
+       height="15.234363"
+       x="103.96783"
+       y="52.384823"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       id="rect833-3-1-6"
+       width="15.768902"
+       height="15.234363"
+       x="119.73674"
+       y="52.384823"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       id="rect833-5-0-80"
+       width="15.768902"
+       height="15.234363"
+       x="88.198929"
+       y="52.384823"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="52.384823"
+       x="72.430038"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-4-6-3-9" />
+    <text
+       style="font-style:normal;font-weight:normal;font-size:10.5833px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;white-space:pre;shape-inside:url(#rect2415);fill:#000000;fill-opacity:1;stroke:none;"
+       id="text2413"
+       xml:space="preserve" />
+  </g>
+</svg>
diff --git a/sycl/doc/extensions/ExplicitSIMD/images/VectorEven.svg b/sycl/doc/extensions/ExplicitSIMD/images/VectorEven.svg
new file mode 100644
index 0000000000000..c9ef9eb84c2c1
--- /dev/null
+++ b/sycl/doc/extensions/ExplicitSIMD/images/VectorEven.svg
@@ -0,0 +1,148 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="232.28477mm"
+   height="25.498947mm"
+   viewBox="0 0 232.28479 25.498947"
+   version="1.1"
+   id="svg8"
+   inkscape:version="1.0rc1 (09960d6f05, 2020-04-09)"
+   sodipodi:docname="CMVectorOdd.svg">
+  <defs
+     id="defs2">
+    <rect
+       id="rect2309"
+       height="17.372519"
+       width="133.90202"
+       y="7.4293742"
+       x="-108.03106" />
+  </defs>
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="0.98994949"
+     inkscape:cx="685.47331"
+     inkscape:cy="441.36183"
+     inkscape:document-units="mm"
+     inkscape:current-layer="layer1"
+     inkscape:document-rotation="0"
+     showgrid="false"
+     fit-margin-top="5"
+     fit-margin-left="5"
+     fit-margin-right="5"
+     fit-margin-bottom="5"
+     lock-margins="true"
+     inkscape:window-width="2482"
+     inkscape:window-height="1411"
+     inkscape:window-x="69"
+     inkscape:window-y="-9"
+     inkscape:window-maximized="1" />
+  <metadata
+     id="metadata5">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(59.84172,-2.3512519)">
+    <rect
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="7.4835443"
+       x="41.159512"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-4" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="7.4835443"
+       x="120.00404"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-6" />
+    <rect
+       id="rect833-4-9"
+       width="15.768902"
+       height="15.234363"
+       x="135.77295"
+       y="7.4835443"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="7.4835443"
+       x="88.466225"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-3" />
+    <rect
+       id="rect833-4-3"
+       width="15.768902"
+       height="15.234363"
+       x="104.23514"
+       y="7.4835443"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="7.4835443"
+       x="56.928417"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-5" />
+    <rect
+       id="rect833-4-6"
+       width="15.768902"
+       height="15.234363"
+       x="72.697319"
+       y="7.4835443"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       id="rect833-6-8"
+       width="15.768902"
+       height="15.234363"
+       x="151.54185"
+       y="7.4835434"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <text
+       transform="translate(52.652097,0.53453906)"
+       style="font-style:normal;font-weight:normal;font-size:10.5833px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;white-space:pre;shape-inside:url(#rect2309);fill:#000000;fill-opacity:1;stroke:none;"
+       id="text2307"
+       xml:space="preserve"><tspan
+         sodipodi:role="line"
+         x="-108.03125"
+         y="17.085153"><tspan>a.select&lt;4,2&gt;(0)</tspan></tspan></text>
+  </g>
+</svg>
diff --git a/sycl/doc/extensions/ExplicitSIMD/images/VectorOdd.svg b/sycl/doc/extensions/ExplicitSIMD/images/VectorOdd.svg
new file mode 100644
index 0000000000000..4c803d26a520b
--- /dev/null
+++ b/sycl/doc/extensions/ExplicitSIMD/images/VectorOdd.svg
@@ -0,0 +1,148 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="233.62173mm"
+   height="25.498945mm"
+   viewBox="0 0 233.62175 25.498945"
+   version="1.1"
+   id="svg8"
+   inkscape:version="1.0rc1 (09960d6f05, 2020-04-09)"
+   sodipodi:docname="CMVectorEven.svg">
+  <defs
+     id="defs2">
+    <rect
+       id="rect1716"
+       height="15.234363"
+       width="140.3165"
+       y="8.7657213"
+       x="-108.56559" />
+  </defs>
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="0.98994949"
+     inkscape:cx="603.6151"
+     inkscape:cy="573.03849"
+     inkscape:document-units="mm"
+     inkscape:current-layer="layer1"
+     inkscape:document-rotation="0"
+     showgrid="false"
+     fit-margin-top="5"
+     fit-margin-left="5"
+     fit-margin-right="5"
+     fit-margin-bottom="5"
+     lock-margins="true"
+     inkscape:window-width="2482"
+     inkscape:window-height="1411"
+     inkscape:window-x="69"
+     inkscape:window-y="-9"
+     inkscape:window-maximized="1" />
+  <metadata
+     id="metadata5">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(61.178684,-2.6185225)">
+    <rect
+       id="rect833"
+       width="15.768902"
+       height="15.234363"
+       x="41.159512"
+       y="7.750814"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="7.750814"
+       x="56.928413"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-4" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="7.750814"
+       x="135.77295"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-6" />
+    <rect
+       id="rect833-4-9"
+       width="15.768902"
+       height="15.234363"
+       x="151.54185"
+       y="7.750814"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="7.750814"
+       x="104.23513"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-3" />
+    <rect
+       id="rect833-4-3"
+       width="15.768902"
+       height="15.234363"
+       x="120.00404"
+       y="7.750814"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.264583;stroke-opacity:1"
+       ry="1.1999595"
+       rx="1.6933334"
+       y="7.750814"
+       x="72.697319"
+       height="15.234363"
+       width="15.768902"
+       id="rect833-5" />
+    <rect
+       id="rect833-4-6"
+       width="15.768902"
+       height="15.234363"
+       x="88.466225"
+       y="7.750814"
+       rx="1.6933334"
+       ry="1.1999595"
+       style="fill:#c50000;fill-opacity:1;stroke:#000000;stroke-width:0.264583;stroke-opacity:1" />
+    <text
+       transform="translate(51.850289,-0.53453906)"
+       style="font-style:normal;font-weight:normal;font-size:10.5833px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;white-space:pre;shape-inside:url(#rect1716);fill:#000000;fill-opacity:1;stroke:none;"
+       id="text1714"
+       xml:space="preserve"><tspan
+         sodipodi:role="line"
+         x="-108.56641"
+         y="18.42109"><tspan>a.select&lt;4,2&gt;(1)</tspan></tspan></text>
+  </g>
+</svg>
diff --git a/sycl/doc/extensions/ExplicitSIMD/images/simd_view.svg b/sycl/doc/extensions/ExplicitSIMD/images/simd_view.svg
new file mode 100644
index 0000000000000..ee9af6e5764f9
--- /dev/null
+++ b/sycl/doc/extensions/ExplicitSIMD/images/simd_view.svg
@@ -0,0 +1,1856 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   sodipodi:docname="simd_view.svg"
+   inkscape:version="1.0rc1 (09960d6f05, 2020-04-09)"
+   id="svg8"
+   version="1.1"
+   viewBox="0 0 262.13336 97.751869"
+   height="97.751869mm"
+   width="262.13336mm">
+  <defs
+     id="defs2">
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath849">
+      <path
+         d="M 0,1.2207e-4 H 960 V 540.00012 H 0 Z"
+         clip-rule="evenodd"
+         id="path847"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath929">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path927"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath941">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path939"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath953">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path951"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath965">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path963"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath977">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path975"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath989">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path987"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1001">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path999"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1013">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1011"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1025">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1023"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1037">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1035"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1049">
+      <path
+         d="M 261.96,14.76 H 283.8 V 98.88 H 261.96 Z"
+         clip-rule="evenodd"
+         id="path1047"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1061">
+      <path
+         d="m 283.68,14.76 h 21.84 v 84.12 h -21.84 z"
+         clip-rule="evenodd"
+         id="path1059"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1073">
+      <path
+         d="m 305.4,14.76 h 21.96 V 98.88 H 305.4 Z"
+         clip-rule="evenodd"
+         id="path1071"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1085">
+      <path
+         d="m 327.24,14.76 h 21.84 v 84.12 h -21.84 z"
+         clip-rule="evenodd"
+         id="path1083"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1097">
+      <path
+         d="M 348.96,14.76 H 370.8 V 98.88 H 348.96 Z"
+         clip-rule="evenodd"
+         id="path1095"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1109">
+      <path
+         d="m 370.68,14.76 h 21.96 v 84.12 h -21.96 z"
+         clip-rule="evenodd"
+         id="path1107"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1125">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1123"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1141">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1139"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1157">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1155"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1173">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1171"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1189">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1187"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1205">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1203"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1221">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1219"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1237">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1235"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1269">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1267"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1285">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1283"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1301">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1299"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1317">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1315"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1337">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1335"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1349">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1347"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1361">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1359"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1373">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1371"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1385">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1383"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1397">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1395"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1409">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1407"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1421">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1419"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1437">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1435"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1453">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1451"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1469">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1467"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1485">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1483"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1501">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1499"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1517">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1515"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1533">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1531"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1549">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1547"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1561">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1559"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1573">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1571"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1585">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1583"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1603">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1601"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1619">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1617"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1635">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1633"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1651">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1649"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1663">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1661"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1677">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1675"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+    <clipPath
+       clipPathUnits="userSpaceOnUse"
+       id="clipPath1689">
+      <path
+         d="M 1.4305e-5,0 H 960.00001 V 540 H 1.4305e-5 Z"
+         clip-rule="evenodd"
+         id="path1687"
+         inkscape:connector-curvature="0" />
+    </clipPath>
+  </defs>
+  <sodipodi:namedview
+     inkscape:window-maximized="1"
+     inkscape:window-y="-9"
+     inkscape:window-x="69"
+     inkscape:window-height="1411"
+     inkscape:window-width="2482"
+     fit-margin-bottom="5"
+     fit-margin-left="5"
+     fit-margin-right="5"
+     fit-margin-top="5"
+     showgrid="false"
+     inkscape:document-rotation="0"
+     inkscape:current-layer="layer1"
+     inkscape:document-units="mm"
+     inkscape:cy="84.459695"
+     inkscape:cx="662.07333"
+     inkscape:zoom="0.98994949"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0.0"
+     borderopacity="1.0"
+     bordercolor="#666666"
+     pagecolor="#ffffff"
+     id="base" />
+  <metadata
+     id="metadata5">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     transform="translate(198.87728,14.097069)"
+     id="layer1"
+     inkscape:groupmode="layer"
+     inkscape:label="Layer 1">
+    <path
+       inkscape:connector-curvature="0"
+       id="path853"
+       style="fill:#d9d9d9;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.352778"
+       d="m -193.70089,70.340591 h 7.67362 v -6.739113 h -7.67362 z" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path855"
+       style="fill:#d9d9d9;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.352778"
+       d="m -186.02727,70.340591 h 7.67362 v -6.739113 h -7.67362 z" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path857"
+       style="fill:#d9d9d9;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.352778"
+       d="m -178.35365,70.340591 h 7.67362 v -6.739113 h -7.67362 z" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path859"
+       style="fill:#d9d9d9;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.352778"
+       d="m -170.68144,70.340591 h 7.67362 v -6.739113 h -7.67362 z" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path861"
+       style="fill:#d9d9d9;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.352778"
+       d="m -163.005,70.340591 h 7.67363 v -6.739113 h -7.67363 z" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path863"
+       style="fill:#d9d9d9;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.352778"
+       d="m -155.33208,70.340591 h 7.67362 v -6.739113 h -7.67362 z" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path865"
+       style="fill:#d9d9d9;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.352778"
+       d="m -147.65916,70.340591 h 7.67362 v -6.739113 h -7.67362 z" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path867"
+       style="fill:#d9d9d9;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.352778"
+       d="m -139.98625,70.340591 h 7.67363 v -6.739113 h -7.67363 z" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path869"
+       style="fill:#d9d9d9;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.352778"
+       d="m -132.31333,70.340591 h 7.67362 v -6.739113 h -7.67362 z" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path871"
+       style="fill:#d9d9d9;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.352778"
+       d="m -124.64041,70.340591 h 7.67362 v -6.739113 h -7.67362 z" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path873"
+       style="fill:#d9d9d9;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.352778"
+       d="m -116.96397,70.340591 h 7.67362 v -6.739113 h -7.67362 z" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path875"
+       style="fill:#d9d9d9;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.352778"
+       d="m -109.29105,70.340591 h 7.67362 v -6.739113 h -7.67362 z" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path877"
+       style="fill:#d9d9d9;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.352778"
+       d="m -101.61814,70.340591 h 7.673626 v -6.739113 h -7.673626 z" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path879"
+       style="fill:#d9d9d9;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.352778"
+       d="m -93.945219,70.340591 h 7.673622 v -6.739113 h -7.673622 z" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path881"
+       style="fill:#d9d9d9;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.352778"
+       d="m -86.272303,70.340591 h 7.673622 v -6.739113 h -7.673622 z" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path883"
+       style="fill:#d9d9d9;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.352778"
+       d="m -78.599386,70.340591 h 7.673622 v -6.739113 h -7.673622 z" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path885"
+       style="fill:none;stroke:#000000;stroke-width:0.352778;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:10;stroke-dasharray:none;stroke-opacity:1"
+       d="M -186.02727,63.425089 V 70.51698" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path887"
+       style="fill:none;stroke:#000000;stroke-width:0.352778;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:10;stroke-dasharray:none;stroke-opacity:1"
+       d="M -178.35365,63.425089 V 70.51698" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path889"
+       style="fill:none;stroke:#000000;stroke-width:0.352778;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:10;stroke-dasharray:none;stroke-opacity:1"
+       d="M -170.68144,63.425089 V 70.51698" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path891"
+       style="fill:none;stroke:#000000;stroke-width:0.352778;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:10;stroke-dasharray:none;stroke-opacity:1"
+       d="M -163.005,63.425089 V 70.51698" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path893"
+       style="fill:none;stroke:#000000;stroke-width:0.352778;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:10;stroke-dasharray:none;stroke-opacity:1"
+       d="M -155.33208,63.425089 V 70.51698" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path895"
+       style="fill:none;stroke:#000000;stroke-width:0.352778;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:10;stroke-dasharray:none;stroke-opacity:1"
+       d="M -147.65916,63.425089 V 70.51698" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path897"
+       style="fill:none;stroke:#000000;stroke-width:0.352778;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:10;stroke-dasharray:none;stroke-opacity:1"
+       d="M -139.98625,63.425089 V 70.51698" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path899"
+       style="fill:none;stroke:#000000;stroke-width:0.352778;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:10;stroke-dasharray:none;stroke-opacity:1"
+       d="M -132.31333,63.425089 V 70.51698" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path901"
+       style="fill:none;stroke:#000000;stroke-width:0.352778;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:10;stroke-dasharray:none;stroke-opacity:1"
+       d="M -124.64041,63.425089 V 70.51698" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path903"
+       style="fill:none;stroke:#000000;stroke-width:0.352778;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:10;stroke-dasharray:none;stroke-opacity:1"
+       d="M -116.96397,63.425089 V 70.51698" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path905"
+       style="fill:none;stroke:#000000;stroke-width:0.352778;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:10;stroke-dasharray:none;stroke-opacity:1"
+       d="M -109.29105,63.425089 V 70.51698" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path907"
+       style="fill:none;stroke:#000000;stroke-width:0.352778;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:10;stroke-dasharray:none;stroke-opacity:1"
+       d="M -101.61814,63.425089 V 70.51698" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path909"
+       style="fill:none;stroke:#000000;stroke-width:0.352778;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:10;stroke-dasharray:none;stroke-opacity:1"
+       d="M -93.945219,63.425089 V 70.51698" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path911"
+       style="fill:none;stroke:#000000;stroke-width:0.352778;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:10;stroke-dasharray:none;stroke-opacity:1"
+       d="M -86.272303,63.425089 V 70.51698" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path913"
+       style="fill:none;stroke:#000000;stroke-width:0.352778;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:10;stroke-dasharray:none;stroke-opacity:1"
+       d="M -78.599386,63.425089 V 70.51698" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path915"
+       style="fill:none;stroke:#000000;stroke-width:0.352778;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:10;stroke-dasharray:none;stroke-opacity:1"
+       d="M -193.70089,63.425089 V 70.51698" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path917"
+       style="fill:none;stroke:#000000;stroke-width:0.352778;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:10;stroke-dasharray:none;stroke-opacity:1"
+       d="M -70.922942,63.425089 V 70.51698" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path919"
+       style="fill:none;stroke:#000000;stroke-width:0.352778;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:10;stroke-dasharray:none;stroke-opacity:1"
+       d="M -193.87728,63.601478 H -70.746553" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path921"
+       style="fill:none;stroke:#000000;stroke-width:0.352778;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:10;stroke-dasharray:none;stroke-opacity:1"
+       d="M -193.87728,70.340591 H -70.746553" />
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g923">
+      <g
+         clip-path="url(#clipPath929)"
+         id="g925">
+        <text
+           id="text933"
+           style="font-variant:normal;font-weight:bold;font-size:9.984px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,51.696,53.424)"><tspan
+             id="tspan931"
+             y="0"
+             x="0">0</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g935">
+      <g
+         clip-path="url(#clipPath941)"
+         id="g937">
+        <text
+           id="text945"
+           style="font-variant:normal;font-weight:bold;font-size:9.984px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,73.464,53.424)"><tspan
+             id="tspan943"
+             y="0"
+             x="0">1</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g947">
+      <g
+         clip-path="url(#clipPath953)"
+         id="g949">
+        <text
+           id="text957"
+           style="font-variant:normal;font-weight:bold;font-size:9.984px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,95.208,53.424)"><tspan
+             id="tspan955"
+             y="0"
+             x="0">2</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g959">
+      <g
+         clip-path="url(#clipPath965)"
+         id="g961">
+        <text
+           id="text969"
+           style="font-variant:normal;font-weight:bold;font-size:9.984px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,116.98,53.424)"><tspan
+             id="tspan967"
+             y="0"
+             x="0">3</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g971">
+      <g
+         clip-path="url(#clipPath977)"
+         id="g973">
+        <text
+           id="text981"
+           style="font-variant:normal;font-weight:bold;font-size:9.984px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,138.72,53.424)"><tspan
+             id="tspan979"
+             y="0"
+             x="0">4</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g983">
+      <g
+         clip-path="url(#clipPath989)"
+         id="g985">
+        <text
+           id="text993"
+           style="font-variant:normal;font-weight:bold;font-size:9.984px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,160.49,53.424)"><tspan
+             id="tspan991"
+             y="0"
+             x="0">5</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g995">
+      <g
+         clip-path="url(#clipPath1001)"
+         id="g997">
+        <text
+           id="text1005"
+           style="font-variant:normal;font-weight:bold;font-size:9.984px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,182.23,53.424)"><tspan
+             id="tspan1003"
+             y="0"
+             x="0">6</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1007">
+      <g
+         clip-path="url(#clipPath1013)"
+         id="g1009">
+        <text
+           id="text1017"
+           style="font-variant:normal;font-weight:bold;font-size:9.984px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,204,53.424)"><tspan
+             id="tspan1015"
+             y="0"
+             x="0">7</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1019">
+      <g
+         clip-path="url(#clipPath1025)"
+         id="g1021">
+        <text
+           id="text1029"
+           style="font-variant:normal;font-weight:bold;font-size:9.984px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,225.74,53.424)"><tspan
+             id="tspan1027"
+             y="0"
+             x="0">8</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1031">
+      <g
+         clip-path="url(#clipPath1037)"
+         id="g1033">
+        <text
+           id="text1041"
+           style="font-variant:normal;font-weight:bold;font-size:9.984px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,247.49,53.424)"><tspan
+             id="tspan1039"
+             y="0"
+             x="0">9</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1043">
+      <g
+         clip-path="url(#clipPath1049)"
+         id="g1045">
+        <text
+           id="text1053"
+           style="font-variant:normal;font-weight:bold;font-size:9.984px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,269.26,53.424)"><tspan
+             id="tspan1051"
+             sodipodi:role="line"
+             y="0"
+             x="0 5.039988">10</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1055">
+      <g
+         clip-path="url(#clipPath1061)"
+         id="g1057">
+        <text
+           id="text1065"
+           style="font-variant:normal;font-weight:bold;font-size:9.984px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,291,53.424)"><tspan
+             id="tspan1063"
+             sodipodi:role="line"
+             y="0"
+             x="0 5.039988">11</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1067">
+      <g
+         clip-path="url(#clipPath1073)"
+         id="g1069">
+        <text
+           id="text1077"
+           style="font-variant:normal;font-weight:bold;font-size:9.984px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,312.77,53.424)"><tspan
+             id="tspan1075"
+             sodipodi:role="line"
+             y="0"
+             x="0 5.039988">12</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1079">
+      <g
+         clip-path="url(#clipPath1085)"
+         id="g1081">
+        <text
+           id="text1089"
+           style="font-variant:normal;font-weight:bold;font-size:9.984px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,334.51,53.424)"><tspan
+             id="tspan1087"
+             sodipodi:role="line"
+             y="0"
+             x="0 5.039988">13</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1091">
+      <g
+         clip-path="url(#clipPath1097)"
+         id="g1093">
+        <text
+           id="text1101"
+           style="font-variant:normal;font-weight:bold;font-size:9.984px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,356.28,53.424)"><tspan
+             id="tspan1099"
+             sodipodi:role="line"
+             y="0"
+             x="0 5.039988">14</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1103">
+      <g
+         clip-path="url(#clipPath1109)"
+         id="g1105">
+        <text
+           id="text1113"
+           style="font-variant:normal;font-weight:bold;font-size:9.984px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,378.02,53.424)"><tspan
+             id="tspan1111"
+             sodipodi:role="line"
+             y="0"
+             x="0 5.039988">15</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1115">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1117"
+         style="fill:none;stroke:#2f528f;stroke-width:0.96;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:8;stroke-dasharray:3.84, 2.88;stroke-dashoffset:0;stroke-opacity:1"
+         d="m 141,123.36 h 18 v 17.4 h -18 z" />
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1119">
+      <g
+         clip-path="url(#clipPath1125)"
+         id="g1121">
+        <text
+           id="text1129"
+           style="font-variant:normal;font-weight:normal;font-size:11.04px;font-family:Calibri;-inkscape-font-specification:Calibri;writing-mode:lr-tb;fill:#afabab;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,147.17,128.26)"><tspan
+             id="tspan1127"
+             y="0"
+             x="0">0</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1131">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1133"
+         style="fill:none;stroke:#2f528f;stroke-width:0.96;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:8;stroke-dasharray:3.84, 2.88;stroke-dashoffset:0;stroke-opacity:1"
+         d="m 159.96,123.36 h 18 v 17.4 h -18 z" />
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1135">
+      <g
+         clip-path="url(#clipPath1141)"
+         id="g1137">
+        <text
+           id="text1145"
+           style="font-variant:normal;font-weight:normal;font-size:11.04px;font-family:Calibri;-inkscape-font-specification:Calibri;writing-mode:lr-tb;fill:#afabab;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,166.13,128.26)"><tspan
+             id="tspan1143"
+             y="0"
+             x="0">2</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1147">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1149"
+         style="fill:none;stroke:#2f528f;stroke-width:0.96;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:8;stroke-dasharray:3.84, 2.88;stroke-dashoffset:0;stroke-opacity:1"
+         d="m 178.92,123.36 h 18 v 17.4 h -18 z" />
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1151">
+      <g
+         clip-path="url(#clipPath1157)"
+         id="g1153">
+        <text
+           id="text1161"
+           style="font-variant:normal;font-weight:normal;font-size:11.04px;font-family:Calibri;-inkscape-font-specification:Calibri;writing-mode:lr-tb;fill:#afabab;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,185.09,128.26)"><tspan
+             id="tspan1159"
+             y="0"
+             x="0">4</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1163">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1165"
+         style="fill:none;stroke:#2f528f;stroke-width:0.96;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:8;stroke-dasharray:3.84, 2.88;stroke-dashoffset:0;stroke-opacity:1"
+         d="m 197.88,123.36 h 18 v 17.4 h -18 z" />
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1167">
+      <g
+         clip-path="url(#clipPath1173)"
+         id="g1169">
+        <text
+           id="text1177"
+           style="font-variant:normal;font-weight:normal;font-size:11.04px;font-family:Calibri;-inkscape-font-specification:Calibri;writing-mode:lr-tb;fill:#afabab;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,204.05,128.26)"><tspan
+             id="tspan1175"
+             y="0"
+             x="0">6</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1179">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1181"
+         style="fill:none;stroke:#2f528f;stroke-width:0.96;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:8;stroke-dasharray:3.84, 2.88;stroke-dashoffset:0;stroke-opacity:1"
+         d="m 216.84,123.36 h 18 v 17.4 h -18 z" />
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1183">
+      <g
+         clip-path="url(#clipPath1189)"
+         id="g1185">
+        <text
+           id="text1193"
+           style="font-variant:normal;font-weight:normal;font-size:11.04px;font-family:Calibri;-inkscape-font-specification:Calibri;writing-mode:lr-tb;fill:#afabab;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,223.01,128.26)"><tspan
+             id="tspan1191"
+             y="0"
+             x="0">8</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1195">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1197"
+         style="fill:none;stroke:#2f528f;stroke-width:0.96;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:8;stroke-dasharray:3.84, 2.88;stroke-dashoffset:0;stroke-opacity:1"
+         d="m 235.8,123.36 h 18 v 17.4 h -18 z" />
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1199">
+      <g
+         clip-path="url(#clipPath1205)"
+         id="g1201">
+        <text
+           id="text1209"
+           style="font-variant:normal;font-weight:normal;font-size:11.04px;font-family:Calibri;-inkscape-font-specification:Calibri;writing-mode:lr-tb;fill:#afabab;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,239.21,128.26)"><tspan
+             id="tspan1207"
+             sodipodi:role="line"
+             y="0"
+             x="0 5.6399798">10</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1211">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1213"
+         style="fill:none;stroke:#2f528f;stroke-width:0.96;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:8;stroke-dasharray:3.84, 2.88;stroke-dashoffset:0;stroke-opacity:1"
+         d="m 254.76,123.36 h 18 v 17.4 h -18 z" />
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1215">
+      <g
+         clip-path="url(#clipPath1221)"
+         id="g1217">
+        <text
+           id="text1225"
+           style="font-variant:normal;font-weight:normal;font-size:11.04px;font-family:Calibri;-inkscape-font-specification:Calibri;writing-mode:lr-tb;fill:#afabab;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,258.17,128.26)"><tspan
+             id="tspan1223"
+             sodipodi:role="line"
+             y="0"
+             x="0 5.6399798">12</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1227">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1229"
+         style="fill:none;stroke:#2f528f;stroke-width:0.96;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:8;stroke-dasharray:3.84, 2.88;stroke-dashoffset:0;stroke-opacity:1"
+         d="m 273.72,123.36 h 18 v 17.4 h -18 z" />
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1231">
+      <g
+         clip-path="url(#clipPath1237)"
+         id="g1233">
+        <text
+           id="text1241"
+           style="font-variant:normal;font-weight:normal;font-size:11.04px;font-family:Calibri;-inkscape-font-specification:Calibri;writing-mode:lr-tb;fill:#afabab;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,277.13,128.26)"><tspan
+             id="tspan1239"
+             sodipodi:role="line"
+             y="0"
+             x="0 5.6399798">14</tspan></text>
+      </g>
+    </g>
+    <path
+       inkscape:connector-curvature="0"
+       id="path1243"
+       style="fill:#4472c4;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.352778"
+       d="m -156.57033,43.264192 -33.98343,19.319169 0.17462,0.306917 33.9852,-19.322697 z m -34.11255,18.378311 -1.31692,1.96603 2.36326,-0.125942 z" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path1245"
+       style="fill:#4472c4;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.352778"
+       d="m -149.91694,43.278303 -24.26406,19.438055 0.21873,0.275167 24.26758,-19.438055 z m -24.54028,18.528947 -0.99025,2.149475 2.31422,-0.497417 z" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path1247"
+       style="fill:#4472c4;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.352778"
+       d="m -143.22827,43.302997 -15.44814,18.829867 0.27516,0.224014 15.44462,-18.828103 z m -15.90323,17.998017 -0.52563,2.307519 2.159,-0.9652 z" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path1249"
+       style="fill:#4472c4;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.352778"
+       d="m -136.57136,43.352386 -7.48594,18.899364 0.32808,0.130175 7.48595,-18.902539 z m -8.17739,18.246725 0.20462,2.357614 1.76388,-1.577975 z" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path1251"
+       style="fill:#4472c4;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.352778"
+       d="m -129.55108,43.401775 1.43228,18.434755 -0.35278,0.02716 -1.43228,-18.433697 z m 2.28247,18.014597 -0.889,2.192161 -1.22061,-2.028119 z" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path1253"
+       style="fill:#4472c4;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.352778"
+       d="m -122.88005,43.334747 9.398,18.619611 -0.3175,0.159103 -9.398,-18.616436 z m 10.02594,17.907353 0.007,2.366433 -1.89795,-1.412522 z" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path1255"
+       style="fill:#4472c4;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.352778"
+       d="m -116.22314,43.292414 18.196282,18.922647 -0.254,0.244475 -18.196282,-18.920177 z m 18.587865,18.05693 0.705556,2.259189 -2.229556,-0.791986 z" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path1257"
+       style="fill:#4472c4;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.352778"
+       d="m -109.55916,43.271247 26.592385,18.842567 -0.204611,0.287866 -26.592384,-18.841155 z m 26.814635,17.918994 1.114778,2.087034 -2.338917,-0.360186 z" />
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1259">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1261"
+         style="fill:none;stroke:#2f528f;stroke-width:0.96;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:8;stroke-dasharray:3.84, 2.88;stroke-dashoffset:0;stroke-opacity:1"
+         d="m 182.52,199.2 h 18 v 17.4 h -18 z" />
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1263">
+      <g
+         clip-path="url(#clipPath1269)"
+         id="g1265">
+        <text
+           id="text1273"
+           style="font-variant:normal;font-weight:normal;font-size:11.04px;font-family:Calibri;-inkscape-font-specification:Calibri;writing-mode:lr-tb;fill:#afabab;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,188.66,204.12)"><tspan
+             id="tspan1271"
+             y="0"
+             x="0">0</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1275">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1277"
+         style="fill:none;stroke:#2f528f;stroke-width:0.96;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:8;stroke-dasharray:3.84, 2.88;stroke-dashoffset:0;stroke-opacity:1"
+         d="m 201.48,199.2 h 18 v 17.4 h -18 z" />
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1279">
+      <g
+         clip-path="url(#clipPath1285)"
+         id="g1281">
+        <text
+           id="text1289"
+           style="font-variant:normal;font-weight:normal;font-size:11.04px;font-family:Calibri;-inkscape-font-specification:Calibri;writing-mode:lr-tb;fill:#afabab;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,207.62,204.12)"><tspan
+             id="tspan1287"
+             y="0"
+             x="0">2</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1291">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1293"
+         style="fill:none;stroke:#2f528f;stroke-width:0.96;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:8;stroke-dasharray:3.84, 2.88;stroke-dashoffset:0;stroke-opacity:1"
+         d="m 220.44,199.2 h 18 v 17.4 h -18 z" />
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1295">
+      <g
+         clip-path="url(#clipPath1301)"
+         id="g1297">
+        <text
+           id="text1305"
+           style="font-variant:normal;font-weight:normal;font-size:11.04px;font-family:Calibri;-inkscape-font-specification:Calibri;writing-mode:lr-tb;fill:#afabab;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,226.58,204.12)"><tspan
+             id="tspan1303"
+             y="0"
+             x="0">4</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1307">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1309"
+         style="fill:none;stroke:#2f528f;stroke-width:0.96;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:8;stroke-dasharray:3.84, 2.88;stroke-dashoffset:0;stroke-opacity:1"
+         d="m 239.4,199.2 h 18 v 17.4 h -18 z" />
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1311">
+      <g
+         clip-path="url(#clipPath1317)"
+         id="g1313">
+        <text
+           id="text1321"
+           style="font-variant:normal;font-weight:normal;font-size:11.04px;font-family:Calibri;-inkscape-font-specification:Calibri;writing-mode:lr-tb;fill:#afabab;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,245.57,204.12)"><tspan
+             id="tspan1319"
+             y="0"
+             x="0">6</tspan></text>
+      </g>
+    </g>
+    <path
+       inkscape:connector-curvature="0"
+       id="path1323"
+       style="fill:#4472c4;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.352778"
+       d="m -141.9865,16.558915 -13.61722,19.194638 0.28575,0.204611 13.62075,-19.194638 z m -14.13227,18.39736 -0.36336,2.338917 2.08844,-1.114778 z" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path1325"
+       style="fill:#4472c4;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.352778"
+       d="m -135.29783,16.558915 -13.61722,19.194638 0.28575,0.204611 13.62075,-19.194638 z m -14.13228,18.39736 -0.36336,2.338917 2.08845,-1.114778 z" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path1327"
+       style="fill:#4472c4;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.352778"
+       d="m -128.60916,16.558915 -13.61722,19.194638 0.28575,0.204611 13.62074,-19.194638 z m -14.13228,18.39736 -0.36336,2.338917 2.08844,-1.114778 z" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path1329"
+       style="fill:#4472c4;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.352778"
+       d="m -121.9205,16.558915 -13.61722,19.194638 0.28575,0.204611 13.62075,-19.194638 z m -14.13227,18.39736 -0.36336,2.338917 2.08844,-1.114778 z" />
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1331">
+      <g
+         clip-path="url(#clipPath1337)"
+         id="g1333">
+        <text
+           id="text1341"
+           style="font-variant:normal;font-weight:bold;font-size:18px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#000000;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,94.488,231.07)"><tspan
+             id="tspan1339"
+             sodipodi:role="line"
+             y="0"
+             x="0 8.8920002 18.558001 24.714001 34.397999 38.268002 42.695999 52.380001 65.844002 69.804001 78.767998">auto low =</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1343">
+      <g
+         clip-path="url(#clipPath1349)"
+         id="g1345">
+        <text
+           id="text1353"
+           style="font-variant:normal;font-weight:bold;font-size:18px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#000000;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,177.41,231.07)"><tspan
+             id="tspan1351"
+             sodipodi:role="line"
+             y="0"
+             x="0 9 17.406 26.514 36.216 40.896 48.077999 57.132 61.416 70.416 77.940002">even.select</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1355">
+      <g
+         clip-path="url(#clipPath1361)"
+         id="g1357">
+        <text
+           id="text1365"
+           style="font-variant:normal;font-weight:bold;font-size:18px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#000000;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,261.67,231.07)"><tspan
+             id="tspan1363"
+             sodipodi:role="line"
+             y="0"
+             x="0 8.9639997 18.09 22.68 26.747999 35.874001 44.838001 50.507999 59.633999 65.25">&lt;4, 1&gt;(0);</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1367">
+      <g
+         clip-path="url(#clipPath1373)"
+         id="g1369">
+        <text
+           id="text1377"
+           style="font-variant:normal;font-weight:bold;font-size:18.024px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#000000;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,43.392,26.04)"><tspan
+             id="tspan1375"
+             sodipodi:role="line"
+             y="0"
+             x="0 7.191576 11.62548 26.278992">simd</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1379">
+      <g
+         clip-path="url(#clipPath1385)"
+         id="g1381">
+        <text
+           id="text1389"
+           style="font-variant:normal;font-weight:bold;font-size:18.024px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#000000;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,79.392,26.04)"><tspan
+             id="tspan1387"
+             sodipodi:role="line"
+             y="0"
+             x="0 8.9759521 13.409856 22.926529 29.180857 33.831047 37.562016 46.700184 55.838352 64.814301 68.977844 77.503197 81.576622 90.552574 94.625999 107.47711">&lt;int, 16&gt; v = …;</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1391">
+      <g
+         clip-path="url(#clipPath1397)"
+         id="g1393">
+        <text
+           id="text1401"
+           style="font-variant:normal;font-weight:bold;font-size:18px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#000000;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,91.848,158.33)"><tspan
+             id="tspan1399"
+             sodipodi:role="line"
+             y="0"
+             x="0 8.8920002 18.558001 24.714001 34.397999 38.268002 47.268002 55.674 64.781998 74.484001 78.318001 87.281998">auto even =</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1403">
+      <g
+         clip-path="url(#clipPath1409)"
+         id="g1405">
+        <text
+           id="text1413"
+           style="font-variant:normal;font-weight:bold;font-size:18px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#000000;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,183.29,158.33)"><tspan
+             id="tspan1411"
+             sodipodi:role="line"
+             y="0"
+             x="0 7.1999998 12.006 19.188 28.296 32.723999 41.832001 49.265999">v.select</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1415">
+      <g
+         clip-path="url(#clipPath1421)"
+         id="g1417">
+        <text
+           id="text1425"
+           style="font-variant:normal;font-weight:bold;font-size:18px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#000000;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,238.85,158.33)"><tspan
+             id="tspan1423"
+             sodipodi:role="line"
+             y="0"
+             x="0 8.9639997 18.09 22.788 26.514 35.639999 44.604 50.220001 59.346001 65.015999">&lt;8, 2&gt;(0);</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1427">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1429"
+         style="fill:none;stroke:#2f528f;stroke-width:0.96;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:8;stroke-dasharray:3.84, 2.88;stroke-dashoffset:0;stroke-opacity:1"
+         d="m 382.68,134.28 h 18 v 17.4 h -18 z" />
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1431">
+      <g
+         clip-path="url(#clipPath1437)"
+         id="g1433">
+        <text
+           id="text1441"
+           style="font-variant:normal;font-weight:normal;font-size:11.04px;font-family:Calibri;-inkscape-font-specification:Calibri;writing-mode:lr-tb;fill:#afabab;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,388.87,139.18)"><tspan
+             id="tspan1439"
+             y="0"
+             x="0">0</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1443">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1445"
+         style="fill:none;stroke:#2f528f;stroke-width:0.96;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:8;stroke-dasharray:3.84, 2.88;stroke-dashoffset:0;stroke-opacity:1"
+         d="m 401.64,134.28 h 18 v 17.4 h -18 z" />
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1447">
+      <g
+         clip-path="url(#clipPath1453)"
+         id="g1449">
+        <text
+           id="text1457"
+           style="font-variant:normal;font-weight:normal;font-size:11.04px;font-family:Calibri;-inkscape-font-specification:Calibri;writing-mode:lr-tb;fill:#afabab;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,407.86,139.18)"><tspan
+             id="tspan1455"
+             y="0"
+             x="0">2</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1459">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1461"
+         style="fill:none;stroke:#2f528f;stroke-width:0.96;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:8;stroke-dasharray:3.84, 2.88;stroke-dashoffset:0;stroke-opacity:1"
+         d="m 420.6,134.28 h 18 v 17.4 h -18 z" />
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1463">
+      <g
+         clip-path="url(#clipPath1469)"
+         id="g1465">
+        <text
+           id="text1473"
+           style="font-variant:normal;font-weight:normal;font-size:11.04px;font-family:Calibri;-inkscape-font-specification:Calibri;writing-mode:lr-tb;fill:#afabab;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,426.82,139.18)"><tspan
+             id="tspan1471"
+             y="0"
+             x="0">4</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1475">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1477"
+         style="fill:none;stroke:#2f528f;stroke-width:0.96;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:8;stroke-dasharray:3.84, 2.88;stroke-dashoffset:0;stroke-opacity:1"
+         d="m 439.56,134.28 h 18.12 v 17.4 h -18.12 z" />
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1479">
+      <g
+         clip-path="url(#clipPath1485)"
+         id="g1481">
+        <text
+           id="text1489"
+           style="font-variant:normal;font-weight:normal;font-size:11.04px;font-family:Calibri;-inkscape-font-specification:Calibri;writing-mode:lr-tb;fill:#afabab;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,445.78,139.18)"><tspan
+             id="tspan1487"
+             y="0"
+             x="0">6</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1491">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1493"
+         style="fill:none;stroke:#2f528f;stroke-width:0.96;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:8;stroke-dasharray:3.84, 2.88;stroke-dashoffset:0;stroke-opacity:1"
+         d="m 382.68,116.64 h 18 v 17.28 h -18 z" />
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1495">
+      <g
+         clip-path="url(#clipPath1501)"
+         id="g1497">
+        <text
+           id="text1505"
+           style="font-variant:normal;font-weight:normal;font-size:11.04px;font-family:Calibri;-inkscape-font-specification:Calibri;writing-mode:lr-tb;fill:#afabab;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,388.87,121.49)"><tspan
+             id="tspan1503"
+             y="0"
+             x="0">8</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1507">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1509"
+         style="fill:none;stroke:#2f528f;stroke-width:0.96;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:8;stroke-dasharray:3.84, 2.88;stroke-dashoffset:0;stroke-opacity:1"
+         d="m 401.64,116.64 h 18 v 17.28 h -18 z" />
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1511">
+      <g
+         clip-path="url(#clipPath1517)"
+         id="g1513">
+        <text
+           id="text1521"
+           style="font-variant:normal;font-weight:normal;font-size:11.04px;font-family:Calibri;-inkscape-font-specification:Calibri;writing-mode:lr-tb;fill:#afabab;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,405.07,121.49)"><tspan
+             id="tspan1519"
+             sodipodi:role="line"
+             y="0"
+             x="0 5.6399798">10</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1523">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1525"
+         style="fill:none;stroke:#2f528f;stroke-width:0.96;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:8;stroke-dasharray:3.84, 2.88;stroke-dashoffset:0;stroke-opacity:1"
+         d="m 420.6,116.64 h 18 v 17.28 h -18 z" />
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1527">
+      <g
+         clip-path="url(#clipPath1533)"
+         id="g1529">
+        <text
+           id="text1537"
+           style="font-variant:normal;font-weight:normal;font-size:11.04px;font-family:Calibri;-inkscape-font-specification:Calibri;writing-mode:lr-tb;fill:#afabab;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,424.06,121.49)"><tspan
+             id="tspan1535"
+             sodipodi:role="line"
+             y="0"
+             x="0 5.6399798">12</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1539">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1541"
+         style="fill:none;stroke:#2f528f;stroke-width:0.96;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:8;stroke-dasharray:3.84, 2.88;stroke-dashoffset:0;stroke-opacity:1"
+         d="m 439.56,116.64 h 18.12 v 17.28 h -18.12 z" />
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1543">
+      <g
+         clip-path="url(#clipPath1549)"
+         id="g1545">
+        <text
+           id="text1553"
+           style="font-variant:normal;font-weight:normal;font-size:11.04px;font-family:Calibri;-inkscape-font-specification:Calibri;writing-mode:lr-tb;fill:#afabab;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,443.02,121.49)"><tspan
+             id="tspan1551"
+             sodipodi:role="line"
+             y="0"
+             x="0 5.6399798">14</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1555">
+      <g
+         clip-path="url(#clipPath1561)"
+         id="g1557">
+        <text
+           id="text1565"
+           style="font-variant:normal;font-weight:bold;font-size:18px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#000000;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,355.85,166.68)"><tspan
+             id="tspan1563"
+             sodipodi:role="line"
+             y="0"
+             x="0 8.8920002 18.558001 24.714001 34.397999 38.268002 47.268002 55.674 64.781998 74.484001 83.610001 94.949997 98.730003 107.694">auto even2D =</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1567">
+      <g
+         clip-path="url(#clipPath1573)"
+         id="g1569">
+        <text
+           id="text1577"
+           style="font-variant:normal;font-weight:bold;font-size:18px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#000000;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,467.81,166.68)"><tspan
+             id="tspan1575"
+             sodipodi:role="line"
+             y="0"
+             x="0 9 17.406 26.514 36.216 40.535999 45.936001 55.619999 62.009998 76.643997 85.410004">even.format</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1579">
+      <g
+         clip-path="url(#clipPath1585)"
+         id="g1581">
+        <text
+           id="text1589"
+           style="font-variant:normal;font-weight:bold;font-size:18px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#000000;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,559.51,166.68)"><tspan
+             id="tspan1587"
+             sodipodi:role="line"
+             y="0"
+             x="0 8.9639997 13.32 22.806 29.052 33.695999 37.439999 46.566002 51.209999 55.278 64.403999 73.421997 79.038002 84.653999">&lt;int, 2, 4&gt;();</tspan></text>
+      </g>
+    </g>
+    <path
+       inkscape:connector-curvature="0"
+       id="path1591"
+       style="fill:#4472c4;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.352778"
+       d="m -76.70497,39.598831 -26.03147,0.529166 0.007,0.352778 26.031476,-0.529166 z m -25.69633,-0.359834 -2.0955,1.100667 2.13783,1.016 z" />
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1593">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1595"
+         style="fill:none;stroke:#2f528f;stroke-width:0.96;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:8;stroke-dasharray:3.84, 2.88;stroke-dashoffset:0;stroke-opacity:1"
+         d="M 518.88,201.48 H 537 v 17.4 h -18.12 z" />
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1597">
+      <g
+         clip-path="url(#clipPath1603)"
+         id="g1599">
+        <text
+           id="text1607"
+           style="font-variant:normal;font-weight:normal;font-size:11.04px;font-family:Calibri;-inkscape-font-specification:Calibri;writing-mode:lr-tb;fill:#afabab;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,525.12,206.4)"><tspan
+             id="tspan1605"
+             y="0"
+             x="0">8</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1609">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1611"
+         style="fill:none;stroke:#2f528f;stroke-width:0.96;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:8;stroke-dasharray:3.84, 2.88;stroke-dashoffset:0;stroke-opacity:1"
+         d="m 537.84,201.48 h 18.12 v 17.4 h -18.12 z" />
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1613">
+      <g
+         clip-path="url(#clipPath1619)"
+         id="g1615">
+        <text
+           id="text1623"
+           style="font-variant:normal;font-weight:normal;font-size:11.04px;font-family:Calibri;-inkscape-font-specification:Calibri;writing-mode:lr-tb;fill:#afabab;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,541.32,206.4)"><tspan
+             id="tspan1621"
+             sodipodi:role="line"
+             y="0"
+             x="0 5.6399798">10</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1625">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1627"
+         style="fill:none;stroke:#2f528f;stroke-width:0.96;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:8;stroke-dasharray:3.84, 2.88;stroke-dashoffset:0;stroke-opacity:1"
+         d="m 556.8,201.48 h 18.12 v 17.4 H 556.8 Z" />
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1629">
+      <g
+         clip-path="url(#clipPath1635)"
+         id="g1631">
+        <text
+           id="text1639"
+           style="font-variant:normal;font-weight:normal;font-size:11.04px;font-family:Calibri;-inkscape-font-specification:Calibri;writing-mode:lr-tb;fill:#afabab;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,560.28,206.4)"><tspan
+             id="tspan1637"
+             sodipodi:role="line"
+             y="0"
+             x="0 5.6399798">12</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1641">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1643"
+         style="fill:none;stroke:#2f528f;stroke-width:0.96;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:8;stroke-dasharray:3.84, 2.88;stroke-dashoffset:0;stroke-opacity:1"
+         d="m 575.76,201.48 h 18.12 v 17.4 h -18.12 z" />
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1645">
+      <g
+         clip-path="url(#clipPath1651)"
+         id="g1647">
+        <text
+           id="text1655"
+           style="font-variant:normal;font-weight:normal;font-size:11.04px;font-family:Calibri;-inkscape-font-specification:Calibri;writing-mode:lr-tb;fill:#afabab;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,579.24,206.4)"><tspan
+             id="tspan1653"
+             sodipodi:role="line"
+             y="0"
+             x="0 5.6399798">14</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1657">
+      <g
+         clip-path="url(#clipPath1663)"
+         id="g1659">
+        <text
+           id="text1667"
+           style="font-variant:normal;font-weight:bold;font-size:18px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#000000;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,455.86,233.69)"><tspan
+             id="tspan1665"
+             sodipodi:role="line"
+             y="0"
+             x="0 8.8920002 18.558001 24.714001 34.397999 38.268002 47.970001 52.397999 60.93 70.632004 74.466003 83.43 87.641998 96.641998 105.048 114.156 123.858 132.98399 143.892 148.698 155.772 164.772 169.2 178.254 185.65199 191.89799 200.862 209.98801 214.668 223.79401 228.34801 237.474 242.118 251.244 255.924 264.888 270.54001 279.66599 284.22 293.34601 298.96201">auto high = even2D.select&lt;1,1,4,1,&gt;(1,0);</tspan></text>
+      </g>
+    </g>
+    <path
+       inkscape:connector-curvature="0"
+       id="path1669"
+       style="fill:#4472c4;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:0.352778"
+       d="m -26.46236,15.747526 -20.510499,25.512888 0.275167,0.22225 20.510499,-25.516416 z m -20.976166,24.68386 -0.500944,2.314223 2.151944,-0.987778 z" />
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1671">
+      <g
+         clip-path="url(#clipPath1677)"
+         id="g1673">
+        <text
+           id="text1681"
+           style="font-variant:normal;font-weight:bold;font-size:18px;font-family:Calibri;-inkscape-font-specification:Calibri-Bold;writing-mode:lr-tb;fill:#000000;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,455.86,259.7)"><tspan
+             id="tspan1679"
+             sodipodi:role="line"
+             y="0"
+             x="0 8.8920002 18.558001 24.714001 34.397999 38.268002 47.970001 52.397999 60.93 70.632004 74.466003 83.43 87.641998 96.641998 105.048 114.156 123.858 132.98399 143.892 148.698 154.81799 164.412 177.82201 183.366 192.492 198.108 203.076">auto high = even2D.row(1);</tspan></text>
+      </g>
+    </g>
+    <g
+       transform="matrix(0.35277777,0,0,-0.35277777,-209.3988,86.934552)"
+       id="g1683">
+      <g
+         clip-path="url(#clipPath1689)"
+         id="g1685">
+        <text
+           id="text1693"
+           style="font-style:italic;font-variant:normal;font-weight:normal;font-size:18px;font-family:Calibri;-inkscape-font-specification:Calibri-Italic;writing-mode:lr-tb;fill:#000000;fill-opacity:1;fill-rule:nonzero;stroke:none"
+           transform="matrix(1,0,0,-1,667.08,259.7)"><tspan
+             id="tspan1691"
+             sodipodi:role="line"
+             y="0"
+             x="0 9.2399998">or</tspan></text>
+      </g>
+    </g>
+  </g>
+</svg>
diff --git a/sycl/doc/extensions/ExtendedAtomics/SYCL_INTEL_extended_atomics.asciidoc b/sycl/doc/extensions/ExtendedAtomics/SYCL_INTEL_extended_atomics.asciidoc
index 05aaac1fb0b6b..576027701e533 100644
--- a/sycl/doc/extensions/ExtendedAtomics/SYCL_INTEL_extended_atomics.asciidoc
+++ b/sycl/doc/extensions/ExtendedAtomics/SYCL_INTEL_extended_atomics.asciidoc
@@ -331,6 +331,35 @@ The effects of a call to +atomic_fence+ depend on the value of the +order+ param
 - `acq_rel`: Both an acquire fence and a release fence
 - `seq_cst`: A sequentially consistent acquire and release fence
 
+==== Atomic Accessor
+
+The +atomic_accessor+ class replaces accessors created with +access::mode::atomic+.  All operators of +atomic_accessor+ which provide access to an element of the underlying memory do so by wrapping the element in an +atomic_ref+.  In order to guarantee that all accesses to the underlying memory are atomic, an +atomic_accessor+ does not allow direct access to the memory via +get_pointer()+.
+
+All other member functions are as defined in the +accessor+ class.
+
+|===
+| Member Function | Description
+
+| `operator atomic_ref<DataT, DefaultOrder, DefaultScope, access::address_space::global_space>() const`
+| Available only when: `Dimensions == 0`.  Returns an `atomic_ref` associated with the element stored in the underlying `buffer` or work-group local memory.
+
+| `atomic_ref<DataT, DefaultOrder, DefaultScope, access::address_space::global_space> operator[](id<Dimensions> index) const`
+| Available only when: `Dimensions > 0`.  Returns an `atomic_ref` associated with the element stored at the specified _index_ in the underlying `buffer` or work-group local memory.
+
+| `atomic_ref<DataT, DefaultOrder, DefaultScope, access::address_space::global_space> operator[](size_t index) const`
+| Available only when: `Dimensions == 1`.  Returns an `atomic_ref` associated with the element stored at the specified _index_ in the underlying `buffer` or work-group local memory.
+
+| `global_ptr<DataT> get_pointer() const = delete`
+| Direct access to the underlying `buffer` or work-group local memory is not permitted.
+
+|===
+
+To simplify the construction of an +atomic_accessor+, tag objects of type +order_tag_t+ and +scope_tag_t+ may optionally be passed to the constructor.  These tag objects enable the `DefaultOrder` and `DefaultScope` template arguments to be deduced via CTAD, as shown in the example below:
+[source,c++]
+----
+auto acc = atomic_accessor(buf, h, relaxed_order, device_scope);
+----
+
 ==== Sample  Header
 
 [source,c++]
@@ -338,6 +367,7 @@ The effects of a call to +atomic_fence+ depend on the value of the +order+ param
 namespace cl {
 namespace sycl {
 namespace intel {
+
 enum class memory_order : /* unspecified */ {
   relaxed, acquire, release, acq_rel, seq_cst
 };
@@ -356,6 +386,24 @@ inline constexpr memory_scope memory_scope_work_group = memory_scope::work_group
 inline constexpr memory_scope memory_scope_device = memory_scope::device;
 inline constexpr memory_scope memory_scope_system = memory_scope::system;
 
+template <memory_order> struct order_tag_t {
+  explicit order_tag_t() = default;
+};
+inline constexpr order_tag_t<memory_order::relaxed> relaxed_order{};
+inline constexpr order_tag_t<memory_order::acquire> acquire_order{};
+inline constexpr order_tag_t<memory_order::release> release_order{};
+inline constexpr order_tag_t<memory_order::acq_rel> acq_rel_order{};
+inline constexpr order_tag_t<memory_order::seq_cst> seq_cst_order{};
+
+template <memory_scope> struct scope_tag_t {
+  explicit scope_tag_t() = default;
+};
+inline constexpr scope_tag_t<memory_scope::work_item> work_item_scope{};
+inline constexpr scope_tag_t<memory_scope::sub_group> sub_group_scope{};
+inline constexpr scope_tag_t<memory_scope::work_group> work_group_scope{};
+inline constexpr scope_tag_t<memory_scope::device> device_scope{};
+inline constexpr scope_tag_t<memory_scope::system> system_scope{};
+
 // Exposition only
 template <memory_order ReadModifyWriteOrder>
 struct memory_order_traits;
@@ -578,6 +626,12 @@ class atomic_ref<T*, DefaultOrder, DefaultScope, Space> {
 
 void atomic_fence(memory_order order, memory_scope scope):
 
+template <typename DataT, int Dimensions,
+          memory_order DefaultOrder, memory_scope DefaultScope,
+          access::target AccessTarget = access::target::global_buffer,
+          access::placeholder IsPlaceholder = access::placeholder::false_t>
+class atomic_accessor;
+
 } // namespace intel
 } // namespace sycl
 } // namespace cl
@@ -605,6 +659,7 @@ None.
 |3|2020-04-09|John Pennycook|*Add atomic_fence*
 |4|2020-04-24|John Pennycook|*Add memory scope*
 |5|2020-04-29|John Pennycook|*Fix ambiguous overloads of compare_exchange and typo in fetch_sub*
+|6|2020-07-08|John Pennycook|*Add atomic_accessor*
 |========================================
 
 //************************************************************************
diff --git a/sycl/doc/extensions/IntelFPGA/FPGALsu.md b/sycl/doc/extensions/IntelFPGA/FPGALsu.md
new file mode 100644
index 0000000000000..7a412128d9afc
--- /dev/null
+++ b/sycl/doc/extensions/IntelFPGA/FPGALsu.md
@@ -0,0 +1,116 @@
+
+# FPGA lsu
+
+The Intel FPGA `lsu` class is implemented in `CL/sycl/intel/fpga_lsu.hpp` which
+is included in `CL/sycl/intel/fpga_extensions.hpp`.
+
+The class `cl::sycl::intel::lsu` allows users to explicitly request that the
+implementation of a global memory access is configured in a certain way. The
+class has two member functions, `load()` and `store()` which allow loading from
+and storing to a `global_ptr`, respectively, and is templated on the following
+4 optional paremeters:
+
+1.  **`cl::sycl::intel::burst_coalesce<B>`, where `B` is a boolean**: request,
+to the extent possible, that a dynamic burst coalescer be implemented when
+`load` or `store` are called. The default value of this parameter is `false`.
+2. **`cl::sycl::intel::cache<N>`, where `N` is an integer greater or equal to
+0**: request, to the extent possible, that a read-only cache of the specified
+size in bytes be implemented when when `load` is called. It is not allowed to
+use that parameter for `store`. The default value of this parameter is `0`.
+3. **`cl::sycl::intel::statically_coalesce<N>`, where `B` is a boolean**:
+request, to the extent possible, that `load` or `store` accesses, is allowed to
+be statically coalesced with other memory accesses at compile time. The default
+value of this parameter is `true`.
+4. **`cl::sycl::intel::prefetch<B>`, where `N` is a boolean**: request, to the
+extent possible, that a prefetcher be implemented when `load` is called. It is
+not allowed to use that parameter for `store`. The default value of this
+parameter is `false`.
+
+Currently, not every combination of parameters is allowed due to limitations in
+the backend. The following rules apply:
+1. For `store`, `cl::sycl::intel::cache` must be `0` and
+`cl::sycl::intel::prefetch` must be `false`.
+2. For `load`, if `cl::sycl::intel::cache` is set to a value greater than `0`,
+then `cl::sycl::intel::burst_coalesce` must be set to `true`.
+3. For `load`, exactly one of `cl::sycl::intel::prefetch` and
+`cl::sycl::intel::burst_coalesce` is allowed to be `true`.
+4. For `load`, exactly one of `cl::sycl::intel::prefetch` and
+`cl::sycl::intel::cache` is allowed to be `true`.
+
+## Implementation
+
+The implementation relies on the Clang built-in `__builtin_intel_fpga_mem` when
+parsing the SYCL device code. The built-in uses the LLVM `ptr.annotation`
+intrinsic under the hood to annotate the pointer that is being accessed.
+```c++
+template <class... mem_access_params> class lsu final {
+public:
+  lsu() = delete;
+
+  template <typename T> static T &load(sycl::global_ptr<T> Ptr) {
+    check_load();
+#if defined(__SYCL_DEVICE_ONLY__) && __has_builtin(__builtin_intel_fpga_mem)
+    return *__builtin_intel_fpga_mem((T *)Ptr,
+                                     _burst_coalesce | _cache |
+                                         _dont_statically_coalesce | _prefetch,
+                                     _cache_val);
+#else
+    return *Ptr;
+#endif
+  }
+
+  template <typename T> static void store(sycl::global_ptr<T> Ptr, T Val) {
+    check_store();
+#if defined(__SYCL_DEVICE_ONLY__) && __has_builtin(__builtin_intel_fpga_mem)
+    *__builtin_intel_fpga_mem((T *)Ptr,
+                              _burst_coalesce | _cache |
+                                  _dont_statically_coalesce | _prefetch,
+                              _cache_val) = Val;
+#else
+    *Ptr = Val;
+#endif
+  }
+  ...
+}
+```
+
+## Usage
+
+```c++
+#include <CL/sycl/intel/fpga_extensions.hpp>
+...
+cl::sycl::buffer<int, 1> output_buffer(output_data, 1);
+cl::sycl::buffer<int, 1> input_buffer(input_data, 1);
+
+Queue.submit([&](cl::sycl::handler &cgh) {
+  auto output_accessor = output_buffer.get_access<cl::sycl::access::mode::write>(cgh);
+  auto input_accessor = input_buffer.get_access<cl::sycl::access::mode::read>(cgh);
+
+  cgh.single_task<class kernel>([=] {
+    auto input_ptr = input_accessor.get_pointer();
+    auto output_ptr = output_accessor.get_pointer();
+
+    using PrefetchingLSU =
+        cl::sycl::intel::lsu<cl::sycl::intel::prefetch<true>,
+                             cl::sycl::intel::statically_coalesce<false>>;
+
+    using BurstCoalescedLSU =
+        cl::sycl::intel::lsu<cl::sycl::intel::burst_coalesce<false>,
+                             cl::sycl::intel::statically_coalesce<false>>;
+
+    using CachingLSU =
+        cl::sycl::intel::lsu<cl::sycl::intel::burst_coalesce<true>,
+                             cl::sycl::intel::cache<1024>,
+                             cl::sycl::intel::statically_coalesce<true>>;
+
+    using PipelinedLSU = cl::sycl::intel::lsu<>;
+
+    int X = PrefetchingLSU::load(input_ptr); // int X = input_ptr[0]
+    int Y = CachingLSU::load(input_ptr + 1); // int Y = input_ptr[1]
+
+    BurstCoalescedLSU::store(output_ptr, X); // output_ptr[0] = X
+    PipelinedLSU::store(output_ptr + 1, Y);  // output_ptr[1] = Y
+  });
+});
+...
+```
diff --git a/sycl/doc/extensions/ParallelForSimpification/SYCL_INTEL_parallel_for_simplification.asciidoc b/sycl/doc/extensions/ParallelForSimplification/SYCL_INTEL_parallel_for_simplification.asciidoc
similarity index 100%
rename from sycl/doc/extensions/ParallelForSimpification/SYCL_INTEL_parallel_for_simplification.asciidoc
rename to sycl/doc/extensions/ParallelForSimplification/SYCL_INTEL_parallel_for_simplification.asciidoc
diff --git a/sycl/doc/extensions/README.md b/sycl/doc/extensions/README.md
index d9d107cb7a6f6..30ff1c7c905bd 100644
--- a/sycl/doc/extensions/README.md
+++ b/sycl/doc/extensions/README.md
@@ -13,7 +13,7 @@ DPC++ extensions status:
 | [SYCL_INTEL_deduction_guides](deduction_guides/SYCL_INTEL_deduction_guides.asciidoc)                                        | Supported                                 | |
 | [SYCL_INTEL_device_specific_kernel_queries](DeviceSpecificKernelQueries/SYCL_INTEL_device_specific_kernel_queries.asciidoc) | Proposal                                  | |
 | [SYCL_INTEL_enqueue_barrier](EnqueueBarrier/enqueue_barrier.asciidoc)                                                       | Supported(OpenCL, Level Zero)             | |
-| [SYCL_INTEL_extended_atomics](ExtendedAtomics/SYCL_INTEL_extended_atomics.asciidoc)                                         | Partially supported(OpenCL: CPU, GPU)     | Not supported: pointer types |
+| [SYCL_INTEL_extended_atomics](ExtendedAtomics/SYCL_INTEL_extended_atomics.asciidoc)                                         | Supported(OpenCL: CPU, GPU)               | |
 | [SYCL_INTEL_group_algorithms](GroupAlgorithms/SYCL_INTEL_group_algorithms.asciidoc)                                         | Supported(OpenCL)                         | |
 | [SYCL_INTEL_group_mask](./GroupMask/SYCL_INTEL_group_mask.asciidoc)                                                         | Proposal                                  | |
 | [FPGA selector](IntelFPGA/FPGASelector.md)                                                                                  | Supported                                 | |
diff --git a/sycl/doc/extensions/SubGroup/SYCL_INTEL_sub_group.asciidoc b/sycl/doc/extensions/SubGroup/SYCL_INTEL_sub_group.asciidoc
index 7b36b219062e3..c459184970de9 100755
--- a/sycl/doc/extensions/SubGroup/SYCL_INTEL_sub_group.asciidoc
+++ b/sycl/doc/extensions/SubGroup/SYCL_INTEL_sub_group.asciidoc
@@ -68,7 +68,7 @@ Providing a generic group abstraction encapsulating the shared functionality of
 
 === Attributes
 
-The +[[intel::reqd_sub_group_size(n)]]+ attribute indicates that the kernel must be compiled and executed with a sub-group of size _n_.  The value of _n_ must be set to a sub-group size that is both supported by the device and compatible with all language features used by the kernel, or device compilation will fail.  The set of valid sub-group sizes can be queried as described below.
+The +[[intel::reqd_sub_group_size(n)]]+ attribute indicates that the kernel must be compiled and executed with a sub-group of size _n_.  The value of _n_ must be a compile-time integral constant expression.  The value of _n_ must be set to a sub-group size that is both supported by the device and compatible with all language features used by the kernel, or device compilation will fail.  The set of valid sub-group sizes can be queried as described below.
 
 In addition to device functions, the required sub-group size attribute may also be specified in the definition of a named functor object, as in the example below:
 
@@ -314,6 +314,7 @@ Yes, the four shuffles in this extension are a defining feature of sub-groups.
 |4|2020-04-21|John Pennycook|*Restore missing barrier function*
 |5|2020-04-21|John Pennycook|*Restore sub-group shuffles as member functions*
 |6|2020-04-22|John Pennycook|*Align with SYCL_INTEL_device_specific_kernel_queries*
+|7|2020-07-13|John Pennycook|*Clarify that reqd_sub_group_size must be a compile-time constant*
 |========================================
 
 //************************************************************************
diff --git a/sycl/doc/extensions/UsePinnedMemoryProperty/UsePinnedMemoryPropery.adoc b/sycl/doc/extensions/UsePinnedMemoryProperty/UsePinnedMemoryPropery.adoc
new file mode 100644
index 0000000000000..8c97dbdd88759
--- /dev/null
+++ b/sycl/doc/extensions/UsePinnedMemoryProperty/UsePinnedMemoryPropery.adoc
@@ -0,0 +1,22 @@
+= SYCL Proposals: Use Pinned Host Memory Property
+Vlad Romanov <vlad.romanov@intel.com>
+v0.1
+:source-highlighter: pygments
+:icons: font
+== Introduction
+This document describes an extension that introduces a +sycl::ext::oneapi::property::buffer::use_pinned_host_memory+ property for the `sycl::buffer`.  Some SYCL backends can accelerate copies between host and device by allocating pinned memory.  The property can be passed to the `sycl::buffer` constructor in order to enable such an allocation.
+
+== Name Strings
+
++SYCL_INTEL_use_pinned_host_memory+
+
+== Use Pinned Host Memory Property
+
+.Proposed Buffer Property
+[cols="^50,50",options="header"]
+|===
+
+|Property |Description
+|`syc::ext::oneapi::property::buffer::use_pinned_host_memory`
+| The `use_pinned_host_memory` property adds the requirement that the SYCL runtime must allocate host pinned memory for the `sycl::buffer`. The property cannot be used with the `sycl::buffer` constructors that take hostData parameter, an invalid_object_error SYCL exception must be thrown in this case.
+|===
diff --git a/sycl/xmethods/libsycl.so-gdb.py b/sycl/gdb/libsycl.so-gdb.py
similarity index 56%
rename from sycl/xmethods/libsycl.so-gdb.py
rename to sycl/gdb/libsycl.so-gdb.py
index 393c33a858acb..4e1c4bacb7fb5 100644
--- a/sycl/xmethods/libsycl.so-gdb.py
+++ b/sycl/gdb/libsycl.so-gdb.py
@@ -2,9 +2,14 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+import gdb
 import gdb.xmethod
+import gdb.printing
+import itertools
 import re
 
+### XMethod implementations ###
+
 """
 Generalized base class for buffer index calculation
 """
@@ -134,3 +139,86 @@ def match(self, class_type, method_name):
 
 
 gdb.xmethod.register_xmethod_matcher(None, AccessorOpIndexMatcher(), replace=True)
+
+### Pretty-printer implementations ###
+
+"""
+Print an object deriving from cl::sycl::detail::array
+"""
+class SyclArrayPrinter:
+    class ElementIterator:
+        def __init__(self, data, size):
+            self.data = data
+            self.size = size
+            self.count = 0
+
+        def __iter__(self):
+            return self
+
+        def __next__(self):
+            if self.count == self.size:
+                raise StopIteration
+            count = self.count
+            self.count = self.count + 1
+            try:
+                elt = self.data[count]
+            except:
+                elt = "<error reading variable>"
+            return ('[%d]' % count, elt)
+
+    def __init__(self, value):
+        if value.type.code == gdb.TYPE_CODE_REF:
+            if hasattr(gdb.Value,"referenced_value"):
+                value = value.referenced_value()
+
+        self.value = value
+        self.type = value.type.unqualified().strip_typedefs()
+        self.dimensions = self.type.template_argument(0)
+
+    def children(self):
+        try:
+            return self.ElementIterator(self.value['common_array'], self.dimensions)
+        except:
+            # There is no way to return an error from this method. Return an
+            # empty iterable to make GDB happy and rely on to_string method
+            # to take care of formatting.
+            return [ ]
+
+    def to_string(self):
+        try:
+            # Check if accessing array value will succeed and resort to
+            # error message otherwise. Individual array element access failures
+            # will be caught by iterator itself.
+            _ = self.value['common_array']
+            return self.type.tag
+        except:
+            return "<error reading variable>"
+
+    def display_hint(self):
+        return 'array'
+
+"""
+Print a cl::sycl::buffer
+"""
+class SyclBufferPrinter:
+    def __init__(self, value):
+        self.value = value
+        self.type = value.type.unqualified().strip_typedefs()
+        self.elt_type = value.type.template_argument(0)
+        self.dimensions = value.type.template_argument(1)
+        self.typeregex = re.compile('^([a-zA-Z0-9_:]+)(<.*>)?$')
+
+    def to_string(self):
+        match = self.typeregex.match(self.type.tag)
+        if not match:
+            return "<error parsing type>"
+        return ('%s<%s, %s> = {impl=%s}'
+                % (match.group(1), self.elt_type, self.dimensions,
+                   self.value['impl'].address))
+
+sycl_printer = gdb.printing.RegexpCollectionPrettyPrinter("SYCL")
+sycl_printer.add_printer("cl::sycl::id",     '^cl::sycl::id<.*$',     SyclArrayPrinter)
+sycl_printer.add_printer("cl::sycl::range",  '^cl::sycl::range<.*$',  SyclArrayPrinter)
+sycl_printer.add_printer("cl::sycl::buffer", '^cl::sycl::buffer<.*$', SyclBufferPrinter)
+gdb.printing.register_pretty_printer(None, sycl_printer, True)
+
diff --git a/sycl/include/CL/sycl/accessor.hpp b/sycl/include/CL/sycl/accessor.hpp
index 7b26a13f475e7..18084a8a2d2b2 100755
--- a/sycl/include/CL/sycl/accessor.hpp
+++ b/sycl/include/CL/sycl/accessor.hpp
@@ -260,6 +260,7 @@ class accessor_common {
       AccessMode == access::mode::read_write;
 
   using RefType = detail::const_if_const_AS<AS, DataT> &;
+  using ConstRefType = const DataT &;
   using PtrType = detail::const_if_const_AS<AS, DataT> *;
 
   using AccType =
@@ -299,7 +300,7 @@ class accessor_common {
 
     template <int CurDims = SubDims,
               typename = detail::enable_if_t<CurDims == 1 && IsAccessReadOnly>>
-    DataT operator[](size_t Index) const {
+    ConstRefType operator[](size_t Index) const {
       MIDs[Dims - SubDims] = Index;
       return MAccessor[MIDs];
     }
@@ -767,6 +768,7 @@ class accessor :
   using ConcreteASPtrType = typename detail::PtrValueType<DataT, AS>::type *;
 
   using RefType = detail::const_if_const_AS<AS, DataT> &;
+  using ConstRefType = const DataT &;
   using PtrType = detail::const_if_const_AS<AS, DataT> *;
 
   template <int Dims = Dimensions> size_t getLinearIndex(id<Dims> Id) const {
@@ -1199,9 +1201,9 @@ class accessor :
     return *(getQualifiedPtr() + LinearIndex);
   }
 
-  template <int Dims = Dimensions,
-            typename = detail::enable_if_t<(Dims > 0) && IsAccessReadOnly>>
-  DataT operator[](id<Dimensions> Index) const {
+  template <int Dims = Dimensions>
+  typename detail::enable_if_t<(Dims > 0) && IsAccessReadOnly, ConstRefType>
+  operator[](id<Dimensions> Index) const {
     const size_t LinearIndex = getLinearIndex(Index);
     return getQualifiedPtr()[LinearIndex];
   }
@@ -1357,7 +1359,7 @@ class accessor<DataT, Dimensions, AccessMode, access::target::local,
 #endif
     public detail::accessor_common<DataT, Dimensions, AccessMode,
                                    access::target::local, IsPlaceholder> {
-
+protected:
   constexpr static int AdjustedDim = Dimensions == 0 ? 1 : Dimensions;
 
   using AccessorCommonT =
@@ -1393,7 +1395,7 @@ class accessor<DataT, Dimensions, AccessMode, access::target::local,
   accessor()
       : impl(detail::InitializedVal<AdjustedDim, range>::template get<0>()) {}
 
-private:
+protected:
   ConcreteASPtrType getQualifiedPtr() const { return MData; }
 
   ConcreteASPtrType MData;
diff --git a/sycl/include/CL/sycl/backend/cuda.hpp b/sycl/include/CL/sycl/backend/cuda.hpp
index 5235131e62401..0bc3fa6f4b0aa 100644
--- a/sycl/include/CL/sycl/backend/cuda.hpp
+++ b/sycl/include/CL/sycl/backend/cuda.hpp
@@ -50,5 +50,12 @@ struct interop<backend::cuda, accessor<DataT, Dimensions, AccessMode,
   using type = CUdeviceptr;
 };
 
+template <typename DataT, int Dimensions, access::mode AccessMode>
+struct interop<backend::cuda, accessor<DataT, Dimensions, AccessMode,
+                                       access::target::constant_buffer,
+                                       access::placeholder::false_t>> {
+  using type = CUdeviceptr;
+};
+
 } // namespace sycl
 } // namespace cl
diff --git a/sycl/include/CL/sycl/backend/level_zero.hpp b/sycl/include/CL/sycl/backend/level_zero.hpp
index 54a1a04d5f996..82de4eb4c6a78 100644
--- a/sycl/include/CL/sycl/backend/level_zero.hpp
+++ b/sycl/include/CL/sycl/backend/level_zero.hpp
@@ -14,30 +14,37 @@
 __SYCL_INLINE_NAMESPACE(cl) {
 namespace sycl {
 
-template <> struct interop<backend::level0, platform> {
+template <> struct interop<backend::level_zero, platform> {
   using type = ze_driver_handle_t;
 };
 
-template <> struct interop<backend::level0, device> {
+template <> struct interop<backend::level_zero, device> {
   using type = ze_device_handle_t;
 };
 
-template <> struct interop<backend::level0, queue> {
+template <> struct interop<backend::level_zero, queue> {
   using type = ze_command_queue_handle_t;
 };
 
-template <> struct interop<backend::level0, program> {
+template <> struct interop<backend::level_zero, program> {
   using type = ze_module_handle_t;
 };
 
 template <typename DataT, int Dimensions, access::mode AccessMode>
-struct interop<backend::level0, accessor<DataT, Dimensions, AccessMode,
-                                         access::target::global_buffer,
-                                         access::placeholder::false_t>> {
+struct interop<backend::level_zero, accessor<DataT, Dimensions, AccessMode,
+                                             access::target::global_buffer,
+                                             access::placeholder::false_t>> {
   using type = char *;
 };
 
-namespace level0 {
+template <typename DataT, int Dimensions, access::mode AccessMode>
+struct interop<backend::level_zero, accessor<DataT, Dimensions, AccessMode,
+                                             access::target::constant_buffer,
+                                             access::placeholder::false_t>> {
+  using type = char *;
+};
+
+namespace level_zero {
 
 // Implementation of various "make" functions resides in libsycl.so
 platform make_platform(pi_native_handle NativeHandle);
@@ -48,7 +55,7 @@ queue make_queue(const context &Context, pi_native_handle InteropHandle);
 // Construction of SYCL platform.
 template <typename T, typename std::enable_if<
                           std::is_same<T, platform>::value>::type * = nullptr>
-T make(typename interop<backend::level0, T>::type Interop) {
+T make(typename interop<backend::level_zero, T>::type Interop) {
   return make_platform(reinterpret_cast<pi_native_handle>(Interop));
 }
 
@@ -56,7 +63,7 @@ T make(typename interop<backend::level0, T>::type Interop) {
 template <typename T, typename std::enable_if<
                           std::is_same<T, device>::value>::type * = nullptr>
 T make(const platform &Platform,
-       typename interop<backend::level0, T>::type Interop) {
+       typename interop<backend::level_zero, T>::type Interop) {
   return make_device(Platform, reinterpret_cast<pi_native_handle>(Interop));
 }
 
@@ -64,7 +71,7 @@ T make(const platform &Platform,
 template <typename T, typename std::enable_if<
                           std::is_same<T, program>::value>::type * = nullptr>
 T make(const context &Context,
-       typename interop<backend::level0, T>::type Interop) {
+       typename interop<backend::level_zero, T>::type Interop) {
   return make_program(Context, reinterpret_cast<pi_native_handle>(Interop));
 }
 
@@ -72,10 +79,10 @@ T make(const context &Context,
 template <typename T, typename std::enable_if<
                           std::is_same<T, queue>::value>::type * = nullptr>
 T make(const context &Context,
-       typename interop<backend::level0, T>::type Interop) {
+       typename interop<backend::level_zero, T>::type Interop) {
   return make_queue(Context, reinterpret_cast<pi_native_handle>(Interop));
 }
 
-} // namespace level0
+} // namespace level_zero
 } // namespace sycl
 } // __SYCL_INLINE_NAMESPACE(cl)
diff --git a/sycl/include/CL/sycl/backend/opencl.hpp b/sycl/include/CL/sycl/backend/opencl.hpp
index 90fecce1f5e4e..f0f8ddc86d053 100644
--- a/sycl/include/CL/sycl/backend/opencl.hpp
+++ b/sycl/include/CL/sycl/backend/opencl.hpp
@@ -43,6 +43,13 @@ struct interop<backend::opencl, accessor<DataT, Dimensions, AccessMode,
   using type = cl_mem;
 };
 
+template <typename DataT, int Dimensions, access::mode AccessMode>
+struct interop<backend::opencl, accessor<DataT, Dimensions, AccessMode,
+                                         access::target::constant_buffer,
+                                         access::placeholder::false_t>> {
+  using type = cl_mem;
+};
+
 namespace opencl {
 
 // Implementation of various "make" functions resides in SYCL RT because
diff --git a/sycl/include/CL/sycl/backend_types.hpp b/sycl/include/CL/sycl/backend_types.hpp
index 362c6d9d9b5b9..655bbf89d8d39 100644
--- a/sycl/include/CL/sycl/backend_types.hpp
+++ b/sycl/include/CL/sycl/backend_types.hpp
@@ -18,7 +18,7 @@
 __SYCL_INLINE_NAMESPACE(cl) {
 namespace sycl {
 
-enum class backend : char { host, opencl, level0, cuda };
+enum class backend : char { host, opencl, level_zero, cuda };
 
 template <backend name, typename SYCLObjectT> struct interop;
 
@@ -30,7 +30,7 @@ inline std::ostream &operator<<(std::ostream &Out, backend be) {
   case backend::opencl:
     Out << std::string("opencl");
     break;
-  case backend::level0:
+  case backend::level_zero:
     Out << std::string("level-zero");
     break;
   case backend::cuda:
diff --git a/sycl/include/CL/sycl/buffer.hpp b/sycl/include/CL/sycl/buffer.hpp
index 2299ac7c3d483..7d6c830ba3cc5 100755
--- a/sycl/include/CL/sycl/buffer.hpp
+++ b/sycl/include/CL/sycl/buffer.hpp
@@ -170,8 +170,8 @@ class buffer {
          const property_list &propList = {})
       : Range(range<1>(container.size())) {
     impl = std::make_shared<detail::buffer_impl>(
-        container.data(), container.data() + container.size(),
-        get_count() * sizeof(T), detail::getNextPowerOfTwo(sizeof(T)), propList,
+        container.data(), get_count() * sizeof(T),
+        detail::getNextPowerOfTwo(sizeof(T)), propList,
         make_unique_ptr<detail::SYCLMemObjAllocatorHolder<AllocatorT>>(
             allocator));
   }
diff --git a/sycl/include/CL/sycl/detail/buffer_impl.hpp b/sycl/include/CL/sycl/detail/buffer_impl.hpp
index d66692a5ea688..0ff488cf97363 100644
--- a/sycl/include/CL/sycl/detail/buffer_impl.hpp
+++ b/sycl/include/CL/sycl/detail/buffer_impl.hpp
@@ -45,12 +45,25 @@ class __SYCL_EXPORT buffer_impl final : public SYCLMemObjT {
 public:
   buffer_impl(size_t SizeInBytes, size_t, const property_list &Props,
               unique_ptr_class<SYCLMemObjAllocator> Allocator)
-      : BaseT(SizeInBytes, Props, std::move(Allocator)) {}
+      : BaseT(SizeInBytes, Props, std::move(Allocator)) {
+
+    if (Props.has_property<sycl::property::buffer::use_host_ptr>())
+      throw sycl::invalid_object_error(
+          "The use_host_ptr property requires host pointer to be provided",
+          PI_INVALID_OPERATION);
+  }
 
   buffer_impl(void *HostData, size_t SizeInBytes, size_t RequiredAlign,
               const property_list &Props,
               unique_ptr_class<SYCLMemObjAllocator> Allocator)
       : BaseT(SizeInBytes, Props, std::move(Allocator)) {
+
+    if (Props.has_property<
+            sycl::ext::oneapi::property::buffer::use_pinned_host_memory>())
+      throw sycl::invalid_object_error(
+          "The use_pinned_host_memory cannot be used with host pointer",
+          PI_INVALID_OPERATION);
+
     BaseT::handleHostData(HostData, RequiredAlign);
   }
 
@@ -58,6 +71,13 @@ class __SYCL_EXPORT buffer_impl final : public SYCLMemObjT {
               const property_list &Props,
               unique_ptr_class<SYCLMemObjAllocator> Allocator)
       : BaseT(SizeInBytes, Props, std::move(Allocator)) {
+
+    if (Props.has_property<
+            sycl::ext::oneapi::property::buffer::use_pinned_host_memory>())
+      throw sycl::invalid_object_error(
+          "The use_pinned_host_memory cannot be used with host pointer",
+          PI_INVALID_OPERATION);
+
     BaseT::handleHostData(HostData, RequiredAlign);
   }
 
@@ -66,6 +86,13 @@ class __SYCL_EXPORT buffer_impl final : public SYCLMemObjT {
               size_t RequiredAlign, const property_list &Props,
               unique_ptr_class<SYCLMemObjAllocator> Allocator)
       : BaseT(SizeInBytes, Props, std::move(Allocator)) {
+
+    if (Props.has_property<
+            sycl::ext::oneapi::property::buffer::use_pinned_host_memory>())
+      throw sycl::invalid_object_error(
+          "The use_pinned_host_memory cannot be used with host pointer",
+          PI_INVALID_OPERATION);
+
     BaseT::handleHostData(HostData, RequiredAlign);
   }
 
@@ -79,6 +106,13 @@ class __SYCL_EXPORT buffer_impl final : public SYCLMemObjT {
               const property_list &Props,
               unique_ptr_class<SYCLMemObjAllocator> Allocator)
       : BaseT(SizeInBytes, Props, std::move(Allocator)) {
+
+    if (Props.has_property<sycl::property::buffer::use_host_ptr>())
+      throw sycl::invalid_object_error(
+          "Buffer constructor from a pair of iterator values cannot have the "
+          "use_host_ptr property.",
+          PI_INVALID_OPERATION);
+
     BaseT::handleHostData(First, Last, RequiredAlign);
   }
 
@@ -92,6 +126,13 @@ class __SYCL_EXPORT buffer_impl final : public SYCLMemObjT {
               const property_list &Props,
               unique_ptr_class<SYCLMemObjAllocator> Allocator)
       : BaseT(SizeInBytes, Props, std::move(Allocator)) {
+
+    if (Props.has_property<sycl::property::buffer::use_host_ptr>())
+      throw sycl::invalid_object_error(
+          "Buffer constructor from a pair of iterator values cannot have the "
+          "use_host_ptr property.",
+          PI_INVALID_OPERATION);
+
     BaseT::handleHostData(First, Last, RequiredAlign);
   }
 
diff --git a/sycl/include/CL/sycl/detail/memory_manager.hpp b/sycl/include/CL/sycl/detail/memory_manager.hpp
index 711d9a0ac0347..8f17f0f7d95f1 100644
--- a/sycl/include/CL/sycl/detail/memory_manager.hpp
+++ b/sycl/include/CL/sycl/detail/memory_manager.hpp
@@ -12,6 +12,7 @@
 #include <CL/sycl/detail/cl.h>
 #include <CL/sycl/detail/export.hpp>
 #include <CL/sycl/detail/sycl_mem_obj_i.hpp>
+#include <CL/sycl/property_list.hpp>
 #include <CL/sycl/range.hpp>
 
 #include <memory>
@@ -71,6 +72,7 @@ class __SYCL_EXPORT MemoryManager {
                                  bool HostPtrReadOnly, size_t Size,
                                  const EventImplPtr &InteropEvent,
                                  const ContextImplPtr &InteropContext,
+                                 const sycl::property_list &PropsList,
                                  RT::PiEvent &OutEventToWait);
 
   // Allocates images in specified context taking into account situations such
@@ -80,7 +82,8 @@ class __SYCL_EXPORT MemoryManager {
       ContextImplPtr TargetContext, SYCLMemObjI *MemObj, void *UserPtr,
       bool HostPtrReadOnly, size_t Size, const RT::PiMemImageDesc &Desc,
       const RT::PiMemImageFormat &Format, const EventImplPtr &InteropEvent,
-      const ContextImplPtr &InteropContext, RT::PiEvent &OutEventToWait);
+      const ContextImplPtr &InteropContext,
+      const sycl::property_list &PropsList, RT::PiEvent &OutEventToWait);
 
   // Releases memory object(buffer or image). TargetContext should be device
   // one(not host).
@@ -88,21 +91,25 @@ class __SYCL_EXPORT MemoryManager {
                             void *MemAllocation, void *UserPtr);
 
   static void *allocateHostMemory(SYCLMemObjI *MemObj, void *UserPtr,
-                                  bool HostPtrReadOnly, size_t Size);
+                                  bool HostPtrReadOnly, size_t Size,
+                                  const sycl::property_list &PropsList);
 
   static void *allocateInteropMemObject(ContextImplPtr TargetContext,
                                         void *UserPtr,
                                         const EventImplPtr &InteropEvent,
                                         const ContextImplPtr &InteropContext,
+                                        const sycl::property_list &PropsList,
                                         RT::PiEvent &OutEventToWait);
 
   static void *allocateImageObject(ContextImplPtr TargetContext, void *UserPtr,
                                    bool HostPtrReadOnly,
                                    const RT::PiMemImageDesc &Desc,
-                                   const RT::PiMemImageFormat &Format);
+                                   const RT::PiMemImageFormat &Format,
+                                   const sycl::property_list &PropsList);
 
   static void *allocateBufferObject(ContextImplPtr TargetContext, void *UserPtr,
-                                    bool HostPtrReadOnly, const size_t Size);
+                                    bool HostPtrReadOnly, const size_t Size,
+                                    const sycl::property_list &PropsList);
 
   // Copies memory between: host and device, host and host,
   // device and device if memory objects bound to the one context.
diff --git a/sycl/include/CL/sycl/detail/pi.hpp b/sycl/include/CL/sycl/detail/pi.hpp
index 5301dd288502a..c106c5cba35bd 100644
--- a/sycl/include/CL/sycl/detail/pi.hpp
+++ b/sycl/include/CL/sycl/detail/pi.hpp
@@ -57,11 +57,11 @@ bool trace(TraceLevel level);
 
 #ifdef SYCL_RT_OS_WINDOWS
 #define OPENCL_PLUGIN_NAME "pi_opencl.dll"
-#define LEVEL0_PLUGIN_NAME "pi_level0.dll"
+#define LEVEL_ZERO_PLUGIN_NAME "pi_level_zero.dll"
 #define CUDA_PLUGIN_NAME "pi_cuda.dll"
 #else
 #define OPENCL_PLUGIN_NAME "libpi_opencl.so"
-#define LEVEL0_PLUGIN_NAME "libpi_level0.so"
+#define LEVEL_ZERO_PLUGIN_NAME "libpi_level_zero.so"
 #define CUDA_PLUGIN_NAME "libpi_cuda.so"
 #endif
 
diff --git a/sycl/include/CL/sycl/detail/stream_impl.hpp b/sycl/include/CL/sycl/detail/stream_impl.hpp
deleted file mode 100644
index f31990e627a55..0000000000000
--- a/sycl/include/CL/sycl/detail/stream_impl.hpp
+++ /dev/null
@@ -1,677 +0,0 @@
-//==----------------- stream_impl.hpp - SYCL standard header file ----------==//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#pragma once
-
-#include <CL/sycl/accessor.hpp>
-#include <CL/sycl/builtins.hpp>
-#include <CL/sycl/detail/array.hpp>
-#include <CL/sycl/detail/export.hpp>
-#include <CL/sycl/device_selector.hpp>
-#include <CL/sycl/queue.hpp>
-
-__SYCL_INLINE_NAMESPACE(cl) {
-namespace sycl {
-
-namespace detail {
-
-using FmtFlags = unsigned int;
-
-// Mapping from stream_manipulator to FmtFlags. Each manipulator corresponds
-// to the bit in FmtFlags.
-static constexpr FmtFlags Dec = 0x0001;
-static constexpr FmtFlags Hex = 0x0002;
-static constexpr FmtFlags Oct = 0x0004;
-static constexpr FmtFlags ShowBase = 0x0008;
-static constexpr FmtFlags ShowPos = 0x0010;
-static constexpr FmtFlags Fixed = 0x0020;
-static constexpr FmtFlags Scientific = 0x0040;
-
-// Bitmask made of the combination of the base flags. Base flags are mutually
-// exclusive, this mask is used to clean base field before setting the new
-// base flag.
-static constexpr FmtFlags BaseField = Dec | Hex | Oct;
-
-// Bitmask made of the combination of the floating point value format flags.
-// Thease flags are mutually exclusive, this mask is used to clean float field
-// before setting the new float flag.
-static constexpr FmtFlags FloatField = Scientific | Fixed;
-
-constexpr size_t MAX_FLOATING_POINT_DIGITS = 24;
-constexpr size_t MAX_INTEGRAL_DIGITS = 23;
-constexpr const char *VEC_ELEMENT_DELIMITER = ", ";
-constexpr char VEC_OPEN_BRACE = '{';
-constexpr char VEC_CLOSE_BRACE = '}';
-
-constexpr size_t MAX_DIMENSIONS = 3;
-
-// Space for integrals (up to 3), comma and space between the
-// integrals and enclosing braces.
-constexpr size_t MAX_ARRAY_SIZE =
-    MAX_INTEGRAL_DIGITS * MAX_DIMENSIONS + 2 * (MAX_DIMENSIONS - 1) + 2;
-
-template <class F, class T = void>
-using EnableIfFP = typename std::enable_if<std::is_same<F, float>::value ||
-                                               std::is_same<F, double>::value ||
-                                               std::is_same<F, half>::value,
-                                           T>::type;
-
-template <typename> struct IsSwizzleOp : std::false_type {};
-
-template <typename VecT, typename OperationLeftT, typename OperationRightT,
-          template <typename> class OperationCurrentT, int... Indexes>
-struct IsSwizzleOp<cl::sycl::detail::SwizzleOp<
-    VecT, OperationLeftT, OperationRightT, OperationCurrentT, Indexes...>>
-    : std::true_type {
-  using T = typename VecT::element_type;
-  using Type = typename cl::sycl::vec<T, (sizeof...(Indexes))>;
-};
-
-template <typename T>
-using EnableIfSwizzleVec =
-    typename std::enable_if<IsSwizzleOp<T>::value,
-                            typename IsSwizzleOp<T>::Type>::type;
-
-class __SYCL_EXPORT stream_impl {
-public:
-  using GlobalBufAccessorT =
-      accessor<char, 1, cl::sycl::access::mode::read_write,
-               cl::sycl::access::target::global_buffer,
-               cl::sycl::access::placeholder::false_t>;
-
-  using GlobalOffsetAccessorT =
-      accessor<unsigned, 1, cl::sycl::access::mode::atomic,
-               cl::sycl::access::target::global_buffer,
-               cl::sycl::access::placeholder::false_t>;
-
-  stream_impl(size_t BufferSize, size_t MaxStatementSize, handler &CGH);
-
-  // Method to provide an access to the global stream buffer
-  GlobalBufAccessorT accessGlobalBuf(handler &CGH) {
-    return Buf.get_access<cl::sycl::access::mode::read_write>(
-        CGH, range<1>(BufferSize_), id<1>(OffsetSize));
-  }
-
-  // Method to provide an accessor to the global flush buffer
-  GlobalBufAccessorT accessGlobalFlushBuf(handler &CGH) {
-    return FlushBuf.get_access<cl::sycl::access::mode::read_write>(
-        CGH, range<1>(MaxStatementSize_), id<1>(0));
-  }
-
-  // Method to provide an atomic access to the offset in the global stream
-  // buffer
-  GlobalOffsetAccessorT accessGlobalOffset(handler &CGH) {
-    auto OffsetSubBuf = buffer<char, 1>(Buf, id<1>(0), range<1>(OffsetSize));
-    auto ReinterpretedBuf = OffsetSubBuf.reinterpret<unsigned, 1>(range<1>(1));
-    return ReinterpretedBuf.get_access<cl::sycl::access::mode::atomic>(
-        CGH, range<1>(1), id<1>(0));
-  }
-
-  // Copy stream buffer to the host and print the contents
-  void flush();
-
-  size_t get_size() const;
-
-  size_t get_max_statement_size() const;
-
-private:
-  // Size of the stream buffer
-  size_t BufferSize_;
-
-  // Maximum number of symbols which could be streamed from the beginning of a
-  // statement till the semicolon
-  unsigned MaxStatementSize_;
-
-  // Size of the variable which is used as an offset in the stream buffer.
-  // Additinonal memory is allocated in the beginning of the stream buffer for
-  // this variable.
-  static const size_t OffsetSize = sizeof(unsigned);
-
-  // Vector on the host side which is used to initialize the stream buffer
-  std::vector<char> Data;
-
-  // Stream buffer
-  buffer<char, 1> Buf;
-
-  // Global flush buffer
-  buffer<char, 1> FlushBuf;
-};
-
-template <typename T>
-inline typename std::make_unsigned<T>::type getAbsVal(const T Val,
-                                                      const int Base) {
-  return ((Base == 10) && (Val < 0)) ? -Val : Val;
-}
-
-inline char digitToChar(const int Digit) {
-  if (Digit < 10) {
-    return '0' + Digit;
-  } else {
-    return 'a' + Digit - 10;
-  }
-}
-
-template <typename T>
-inline typename std::enable_if<std::is_integral<T>::value, unsigned>::type
-integralToBase(T Val, int Base, char *Digits) {
-  unsigned NumDigits = 0;
-
-  do {
-    Digits[NumDigits++] = digitToChar(Val % Base);
-    Val /= Base;
-  } while (Val);
-
-  return NumDigits;
-}
-
-template <typename T>
-EnableIfFP<T, unsigned> floatingPointToDecStr(T AbsVal, char *Digits,
-                                              int Precision, bool IsSci) {
-  int Exp = 0;
-
-  // For the case that the value is larger than 10.0
-  while (AbsVal >= 10.0) {
-    ++Exp;
-    AbsVal /= 10.0;
-  }
-  // For the case that the value is less than 1.0
-  while (AbsVal > 0.0 && AbsVal < 1.0) {
-    --Exp;
-    AbsVal *= 10.0;
-  }
-
-  auto IntegralPart = static_cast<int>(AbsVal);
-  auto FractionPart = AbsVal - IntegralPart;
-
-  int FractionDigits[MAX_FLOATING_POINT_DIGITS] = {0};
-
-  // Exponent
-  int P = Precision > 0 ? Precision : 4;
-  size_t FractionLength = Exp + P;
-
-  // After normalization integral part contains 1 symbol, also there could be
-  // '.', 'e', sign of the exponent and sign of the number, overall 5 symbols.
-  // So, clamp fraction length if required according to maximum size of the
-  // buffer for floating point number.
-  if (FractionLength > MAX_FLOATING_POINT_DIGITS - 5)
-    FractionLength = MAX_FLOATING_POINT_DIGITS - 5;
-
-  for (unsigned I = 0; I < FractionLength; ++I) {
-    FractionPart *= 10.0;
-    FractionDigits[I] = static_cast<int>(FractionPart);
-    FractionPart -= static_cast<int>(FractionPart);
-  }
-
-  int Carry = FractionPart > static_cast<T>(0.5) ? 1 : 0;
-
-  // Propagate the Carry
-  for (int I = FractionLength - 1; I >= 0 && Carry; --I) {
-    auto Digit = FractionDigits[I] + Carry;
-    FractionDigits[I] = Digit % 10;
-    Carry = Digit / 10;
-  }
-
-  // Carry from the fraction part is propagated to integral part
-  IntegralPart += Carry;
-  if (IntegralPart == 10) {
-    IntegralPart = 1;
-    ++Exp;
-  }
-
-  unsigned Offset = 0;
-
-  // Assemble the final string correspondingly
-  if (IsSci) { // scientific mode
-    // Append the integral part
-    Digits[Offset++] = digitToChar(IntegralPart);
-    Digits[Offset++] = '.';
-
-    // Append all fraction
-    for (unsigned I = 0; I < FractionLength; ++I)
-      Digits[Offset++] = digitToChar(FractionDigits[I]);
-
-    // Exponent part
-    Digits[Offset++] = 'e';
-    Digits[Offset++] = Exp >= 0 ? '+' : '-';
-    Digits[Offset++] = digitToChar(abs(Exp) / 10);
-    Digits[Offset++] = digitToChar(abs(Exp) % 10);
-  } else { // normal mode
-    if (Exp < 0) {
-      Digits[Offset++] = '0';
-      Digits[Offset++] = '.';
-      while (++Exp)
-        Digits[Offset++] = '0';
-
-      // Append the integral part
-      Digits[Offset++] = digitToChar(IntegralPart);
-
-      // Append all fraction
-      for (unsigned I = 0; I < FractionLength; ++I)
-        Digits[Offset++] = digitToChar(FractionDigits[I]);
-    } else {
-      // Append the integral part
-      Digits[Offset++] = digitToChar(IntegralPart);
-      unsigned I = 0;
-      // Append the integral part first
-      for (; I < FractionLength && Exp--; ++I)
-        Digits[Offset++] = digitToChar(FractionDigits[I]);
-
-      // Put the dot
-      Digits[Offset++] = '.';
-
-      // Append the rest of fraction part, or the real fraction part
-      for (; I < FractionLength; ++I)
-        Digits[Offset++] = digitToChar(FractionDigits[I]);
-    }
-    // The normal mode requires no tailing zero digit, then we need to first
-    // find the first non-zero digit
-    while (Digits[Offset - 1] == '0')
-      Offset--;
-
-    // If dot is the last digit, it should be stripped off as well
-    if (Digits[Offset - 1] == '.')
-      Offset--;
-  }
-  return Offset;
-}
-
-// Helper method to update offset in the global buffer atomically according to
-// the provided size of the data in the flush buffer. Return true if offset is
-// updated and false in case of overflow.
-inline bool updateOffset(stream_impl::GlobalOffsetAccessorT &GlobalOffset,
-                         stream_impl::GlobalBufAccessorT &GlobalBuf,
-                         unsigned Size, unsigned &Cur) {
-  unsigned New;
-  Cur = GlobalOffset[0].load();
-  do {
-    if (GlobalBuf.get_range().size() - Cur < Size)
-      // Overflow
-      return false;
-    New = Cur + Size;
-  } while (!GlobalOffset[0].compare_exchange_strong(Cur, New));
-  return true;
-}
-
-inline void flushBuffer(stream_impl::GlobalOffsetAccessorT &GlobalOffset,
-                        stream_impl::GlobalBufAccessorT &GlobalBuf,
-                        stream_impl::GlobalBufAccessorT &GlobalFlushBuf,
-                        unsigned &WIOffset, unsigned &Offset) {
-
-  unsigned Cur = 0;
-  if (!updateOffset(GlobalOffset, GlobalBuf, Offset, Cur))
-    return;
-
-  for (unsigned I = WIOffset; I < WIOffset + Offset; I++) {
-    GlobalBuf[Cur++] = GlobalFlushBuf[I];
-  }
-  // Reset the offset in the flush buffer
-  Offset = 0;
-}
-
-inline void write(stream_impl::GlobalBufAccessorT &GlobalFlushBuf,
-                  size_t FlushBufferSize, unsigned WIOffset, unsigned &Offset,
-                  const char *Str, unsigned Len, unsigned Padding = 0) {
-  if ((FlushBufferSize - Offset < Len + Padding) ||
-      (WIOffset + Offset + Len + Padding > GlobalFlushBuf.get_count()))
-    // TODO: flush here
-    return;
-
-  // Write padding
-  for (size_t I = 0; I < Padding; ++I, ++Offset)
-    GlobalFlushBuf[WIOffset + Offset] = ' ';
-
-  for (size_t I = 0; I < Len; ++I, ++Offset) {
-    GlobalFlushBuf[WIOffset + Offset] = Str[I];
-  }
-}
-
-inline void reverseBuf(char *Buf, unsigned Len) {
-  int I = Len - 1;
-  int J = 0;
-  while (I > J) {
-    int Temp = Buf[I];
-    Buf[I] = Buf[J];
-    Buf[J] = Temp;
-    I--;
-    J++;
-  }
-}
-
-inline unsigned append(char *Dst, const char *Src) {
-  unsigned Len = 0;
-  for (; Src[Len] != '\0'; ++Len)
-    ;
-
-  for (unsigned I = 0; I < Len; ++I)
-    Dst[I] = Src[I];
-  return Len;
-}
-
-template <typename T>
-inline typename std::enable_if<std::is_same<T, half>::value, unsigned>::type
-checkForInfNan(char *Buf, T Val) {
-  if (Val != Val)
-    return append(Buf, "nan");
-
-  // Extract the sign from the bits
-  const uint16_t Sign = reinterpret_cast<uint16_t &>(Val) & 0x8000;
-  // Extract the exponent from the bits
-  const uint16_t Exp16 = (reinterpret_cast<uint16_t &>(Val) & 0x7c00) >> 10;
-
-  if (Exp16 == 0x1f) {
-    if (Sign)
-      return append(Buf, "-inf");
-    return append(Buf, "inf");
-  }
-  return 0;
-}
-
-template <typename T>
-inline typename std::enable_if<std::is_same<T, float>::value ||
-                                   std::is_same<T, double>::value,
-                               unsigned>::type
-checkForInfNan(char *Buf, T Val) {
-  if (isnan(Val))
-    return append(Buf, "nan");
-  if (isinf(Val)) {
-    if (signbit(Val))
-      return append(Buf, "-inf");
-    return append(Buf, "inf");
-  }
-  return 0;
-}
-
-// Returns number of symbols written to the buffer
-template <typename T>
-inline EnableIfFP<T, unsigned>
-ScalarToStr(const T &Val, char *Buf, unsigned Flags, int, int Precision = -1) {
-  unsigned Offset = checkForInfNan(Buf, Val);
-  if (Offset)
-    return Offset;
-
-  T Neg = -Val;
-  auto AbsVal = Val < 0 ? Neg : Val;
-
-  if (Val < 0) {
-    Buf[Offset++] = '-';
-  } else if (Flags & ShowPos) {
-    Buf[Offset++] = '+';
-  }
-
-  bool IsSci = false;
-  if (Flags & detail::Scientific)
-    IsSci = true;
-
-  // TODO: manipulators for floating-point output - hexfloat, fixed
-  Offset += floatingPointToDecStr(AbsVal, Buf + Offset, Precision, IsSci);
-
-  return Offset;
-}
-
-// Returns number of symbols written to the buffer
-template <typename T>
-inline typename std::enable_if<std::is_integral<T>::value, unsigned>::type
-ScalarToStr(const T &Val, char *Buf, unsigned Flags, int, int Precision = -1) {
-  (void)Precision;
-  int Base = 10;
-
-  // append base manipulator
-  switch (Flags & BaseField) {
-  case Dec:
-    Base = 10;
-    break;
-  case Hex:
-    Base = 16;
-    break;
-  case Oct:
-    Base = 8;
-    break;
-  default:
-    // default value is 10
-    break;
-  }
-
-  unsigned Offset = 0;
-
-  // write '+' to the stream if the base is 10 and the value is non-negative
-  // or write '-' to stream if base is 10 and the value is negative
-  if (Base == 10) {
-    if ((Flags & ShowPos) && Val >= 0)
-      Buf[Offset++] = '+';
-    else if (Val < 0)
-      Buf[Offset++] = '-';
-  }
-
-  // write 0 or 0x to the stream if base is not 10 and the manipulator is set
-  if (Base != 10 && (Flags & ShowBase)) {
-    Buf[Offset++] = '0';
-    if (Base == 16)
-      Buf[Offset++] = 'x';
-  }
-
-  auto AbsVal = getAbsVal(Val, Base);
-
-  const unsigned NumBuf = integralToBase(AbsVal, Base, Buf + Offset);
-
-  reverseBuf(Buf + Offset, NumBuf);
-  return Offset + NumBuf;
-}
-
-template <typename T>
-inline typename std::enable_if<std::is_integral<T>::value>::type
-writeIntegral(stream_impl::GlobalBufAccessorT &GlobalFlushBuf,
-              size_t FlushBufferSize, unsigned WIOffset, unsigned &Offset,
-              unsigned Flags, int Width, const T &Val) {
-  char Digits[MAX_INTEGRAL_DIGITS] = {0};
-  unsigned Len = ScalarToStr(Val, Digits, Flags, Width);
-  write(GlobalFlushBuf, FlushBufferSize, WIOffset, Offset, Digits, Len,
-        (Width > 0 && static_cast<unsigned>(Width) > Len)
-            ? static_cast<unsigned>(Width) - Len
-            : 0);
-}
-
-template <typename T>
-inline EnableIfFP<T>
-writeFloatingPoint(stream_impl::GlobalBufAccessorT &GlobalFlushBuf,
-                   size_t FlushBufferSize, unsigned WIOffset, unsigned &Offset,
-                   unsigned Flags, int Width, int Precision, const T &Val) {
-  char Digits[MAX_FLOATING_POINT_DIGITS] = {0};
-  unsigned Len = ScalarToStr(Val, Digits, Flags, Width, Precision);
-  write(GlobalFlushBuf, FlushBufferSize, WIOffset, Offset, Digits, Len,
-        (Width > 0 && static_cast<unsigned>(Width) > Len)
-            ? static_cast<unsigned>(Width) - Len
-            : 0);
-}
-
-template <typename T, int VecLength>
-typename std::enable_if<(VecLength == 1), unsigned>::type
-VecToStr(const vec<T, VecLength> &Vec, char *VecStr, unsigned Flags, int Width,
-         int Precision) {
-  return ScalarToStr(static_cast<T>(Vec.x()), VecStr, Flags, Width, Precision);
-}
-
-template <typename T, int VecLength>
-typename std::enable_if<(VecLength == 2 || VecLength == 4 || VecLength == 8 ||
-                         VecLength == 16),
-                        unsigned>::type
-VecToStr(const vec<T, VecLength> &Vec, char *VecStr, unsigned Flags, int Width,
-         int Precision) {
-  unsigned Len =
-      VecToStr<T, VecLength / 2>(Vec.lo(), VecStr, Flags, Width, Precision);
-  Len += append(VecStr + Len, VEC_ELEMENT_DELIMITER);
-  Len += VecToStr<T, VecLength / 2>(Vec.hi(), VecStr + Len, Flags, Width,
-                                    Precision);
-  return Len;
-}
-
-template <typename T, int VecLength>
-typename std::enable_if<(VecLength == 3), unsigned>::type
-VecToStr(const vec<T, VecLength> &Vec, char *VecStr, unsigned Flags, int Width,
-         int Precision) {
-  unsigned Len = VecToStr<T, 2>(Vec.lo(), VecStr, Flags, Width, Precision);
-  Len += append(VecStr + Len, VEC_ELEMENT_DELIMITER);
-  Len += VecToStr<T, 1>(Vec.z(), VecStr + Len, Flags, Width, Precision);
-  return Len;
-}
-
-template <typename T, int VecLength>
-inline void writeVec(stream_impl::GlobalBufAccessorT &GlobalFlushBuf,
-                     size_t FlushBufferSize, unsigned WIOffset,
-                     unsigned &Offset, unsigned Flags, int Width, int Precision,
-                     const vec<T, VecLength> &Vec) {
-  // Reserve space for vector elements and delimiters
-  constexpr size_t MAX_VEC_SIZE =
-      MAX_FLOATING_POINT_DIGITS * VecLength + (VecLength - 1) * 2;
-  char VecStr[MAX_VEC_SIZE] = {0};
-  unsigned Len = VecToStr<T, VecLength>(Vec, VecStr, Flags, Width, Precision);
-  write(GlobalFlushBuf, FlushBufferSize, WIOffset, Offset, VecStr, Len,
-        (Width > 0 && Width > Len) ? Width - Len : 0);
-}
-
-template <int ArrayLength>
-inline unsigned ArrayToStr(char *Buf, const array<ArrayLength> &Arr) {
-  unsigned Len = 0;
-  Buf[Len++] = VEC_OPEN_BRACE;
-
-  for (int I = 0; I < ArrayLength; ++I) {
-    Len += ScalarToStr(Arr[I], Buf + Len, 0 /* No flags */, -1, -1);
-    if (I != ArrayLength - 1)
-      Len += append(Buf + Len, VEC_ELEMENT_DELIMITER);
-  }
-
-  Buf[Len++] = VEC_CLOSE_BRACE;
-
-  return Len;
-}
-
-template <int ArrayLength>
-inline void writeArray(stream_impl::GlobalBufAccessorT &GlobalFlushBuf,
-                       size_t FlushBufferSize, unsigned WIOffset,
-                       unsigned &Offset, const array<ArrayLength> &Arr) {
-  char Buf[MAX_ARRAY_SIZE];
-  unsigned Len = ArrayToStr(Buf, Arr);
-  write(GlobalFlushBuf, FlushBufferSize, WIOffset, Offset, Buf, Len);
-}
-
-template <int Dimensions>
-inline void writeItem(stream_impl::GlobalBufAccessorT &GlobalFlushBuf,
-                      size_t FlushBufferSize, unsigned WIOffset,
-                      unsigned &Offset, const item<Dimensions> &Item) {
-  // Reserve space for 3 arrays and additional place (40 symbols) for printing
-  // the text
-  char Buf[3 * MAX_ARRAY_SIZE + 40];
-  unsigned Len = 0;
-  Len += append(Buf, "item(");
-  Len += append(Buf + Len, "range: ");
-  Len += ArrayToStr(Buf + Len, Item.get_range());
-  Len += append(Buf + Len, ", id: ");
-  Len += ArrayToStr(Buf + Len, Item.get_id());
-  Len += append(Buf + Len, ", offset: ");
-  Len += ArrayToStr(Buf + Len, Item.get_offset());
-  Buf[Len++] = ')';
-  write(GlobalFlushBuf, FlushBufferSize, WIOffset, Offset, Buf, Len);
-}
-
-template <int Dimensions>
-inline void writeNDRange(stream_impl::GlobalBufAccessorT &GlobalFlushBuf,
-                         size_t FlushBufferSize, unsigned WIOffset,
-                         unsigned &Offset,
-                         const nd_range<Dimensions> &ND_Range) {
-  // Reserve space for 3 arrays and additional place (50 symbols) for printing
-  // the text
-  char Buf[3 * MAX_ARRAY_SIZE + 50];
-  unsigned Len = 0;
-  Len += append(Buf, "nd_range(");
-  Len += append(Buf + Len, "global_range: ");
-  Len += ArrayToStr(Buf + Len, ND_Range.get_global_range());
-  Len += append(Buf + Len, ", local_range: ");
-  Len += ArrayToStr(Buf + Len, ND_Range.get_local_range());
-  Len += append(Buf + Len, ", offset: ");
-  Len += ArrayToStr(Buf + Len, ND_Range.get_offset());
-  Buf[Len++] = ')';
-  write(GlobalFlushBuf, FlushBufferSize, WIOffset, Offset, Buf, Len);
-}
-
-template <int Dimensions>
-inline void writeNDItem(stream_impl::GlobalBufAccessorT &GlobalFlushBuf,
-                        size_t FlushBufferSize, unsigned WIOffset,
-                        unsigned &Offset, const nd_item<Dimensions> &ND_Item) {
-  // Reserve space for 2 arrays and additional place (40 symbols) for printing
-  // the text
-  char Buf[2 * MAX_ARRAY_SIZE + 40];
-  unsigned Len = 0;
-  Len += append(Buf, "nd_item(");
-  Len += append(Buf + Len, "global_id: ");
-  Len += ArrayToStr(Buf + Len, ND_Item.get_global_id());
-  Len += append(Buf + Len, ", local_id: ");
-  Len += ArrayToStr(Buf + Len, ND_Item.get_local_id());
-  Buf[Len++] = ')';
-  write(GlobalFlushBuf, FlushBufferSize, WIOffset, Offset, Buf, Len);
-}
-
-template <int Dimensions>
-inline void writeGroup(stream_impl::GlobalBufAccessorT &GlobalFlushBuf,
-                       size_t FlushBufferSize, unsigned WIOffset,
-                       unsigned &Offset, const group<Dimensions> &Group) {
-  // Reserve space for 4 arrays and additional place (60 symbols) for printing
-  // the text
-  char Buf[4 * MAX_ARRAY_SIZE + 60];
-  unsigned Len = 0;
-  Len += append(Buf, "group(");
-  Len += append(Buf + Len, "id: ");
-  Len += ArrayToStr(Buf + Len, Group.get_id());
-  Len += append(Buf + Len, ", global_range: ");
-  Len += ArrayToStr(Buf + Len, Group.get_global_range());
-  Len += append(Buf + Len, ", local_range: ");
-  Len += ArrayToStr(Buf + Len, Group.get_local_range());
-  Len += append(Buf + Len, ", group_range: ");
-  Len += ArrayToStr(Buf + Len, Group.get_group_range());
-  Buf[Len++] = ')';
-  write(GlobalFlushBuf, FlushBufferSize, WIOffset, Offset, Buf, Len);
-}
-
-// Space for 2 arrays and additional place (20 symbols) for printing
-// the text
-constexpr size_t MAX_ITEM_SIZE = 2 * MAX_ARRAY_SIZE + 20;
-
-template <int Dimensions>
-inline unsigned ItemToStr(char *Buf, const item<Dimensions, false> &Item) {
-  unsigned Len = 0;
-  Len += append(Buf, "item(");
-  for (int I = 0; I < 2; ++I) {
-    Len += append(Buf + Len, I == 0 ? "range: " : ", id: ");
-    Len += ArrayToStr(Buf + Len, I == 0 ? Item.get_range() : Item.get_id());
-  }
-  Buf[Len++] = ')';
-  return Len;
-}
-
-template <int Dimensions>
-inline void writeHItem(stream_impl::GlobalBufAccessorT &GlobalFlushBuf,
-                       size_t FlushBufferSize, unsigned WIOffset,
-                       unsigned &Offset, const h_item<Dimensions> &HItem) {
-  // Reserve space for 3 items and additional place (60 symbols) for printing
-  // the text
-  char Buf[3 * MAX_ITEM_SIZE + 60];
-  unsigned Len = 0;
-  Len += append(Buf, "h_item(");
-  for (int I = 0; I < 3; ++I) {
-    Len += append(Buf + Len, I == 0 ? "\n  global "
-                                    : I == 1 ? "\n  logical local "
-                                             : "\n  physical local ");
-    Len += ItemToStr(Buf + Len, I == 0 ? HItem.get_global()
-                                       : I == 1 ? HItem.get_logical_local()
-                                                : HItem.get_physical_local());
-  }
-  Len += append(Buf + Len, "\n)");
-  write(GlobalFlushBuf, FlushBufferSize, WIOffset, Offset, Buf, Len);
-}
-
-} // namespace detail
-} // namespace sycl
-} // __SYCL_INLINE_NAMESPACE(cl)
diff --git a/sycl/include/CL/sycl/exception.hpp b/sycl/include/CL/sycl/exception.hpp
index 038ddeb94fc73..0004bc311a4cd 100644
--- a/sycl/include/CL/sycl/exception.hpp
+++ b/sycl/include/CL/sycl/exception.hpp
@@ -30,7 +30,7 @@ class __SYCL_EXPORT exception : public std::exception {
 public:
   exception() = default;
 
-  const char *what() const noexcept final;
+  const char *what() const noexcept final override;
 
   bool has_context() const;
 
diff --git a/sycl/include/CL/sycl/handler.hpp b/sycl/include/CL/sycl/handler.hpp
index 0a40efb33bd2e..b4504489662c0 100644
--- a/sycl/include/CL/sycl/handler.hpp
+++ b/sycl/include/CL/sycl/handler.hpp
@@ -179,6 +179,7 @@ template <typename T, class BinaryOperation, int Dims, bool IsUSM,
 class reduction_impl;
 
 using cl::sycl::detail::enable_if_t;
+using cl::sycl::detail::queue_impl;
 
 template <typename KernelName, typename KernelType, int Dims, class Reduction,
           typename OutputT>
@@ -191,10 +192,14 @@ enable_if_t<!Reduction::has_fast_atomics>
 reduCGFunc(handler &CGH, KernelType KernelFunc, const nd_range<Dims> &Range,
            Reduction &Redu);
 
-template <typename KernelName, typename KernelType, int Dims, class Reduction>
-enable_if_t<!Reduction::has_fast_atomics>
-reduAuxCGFunc(handler &CGH, const nd_range<Dims> &Range, size_t NWorkItems,
+template <typename KernelName, typename KernelType, class Reduction>
+enable_if_t<!Reduction::has_fast_atomics, size_t>
+reduAuxCGFunc(handler &CGH, size_t NWorkItems, size_t MaxWGSize,
               Reduction &Redu);
+
+__SYCL_EXPORT size_t reduGetMaxWGSize(shared_ptr_class<queue_impl> Queue,
+                                      size_t LocalMemBytesPerWorkItem);
+
 } // namespace detail
 } // namespace intel
 
@@ -504,8 +509,8 @@ class __SYCL_EXPORT handler {
 
   template <typename T, int Dim, access::mode Mode, access::target Target,
             access::placeholder IsPH>
-  detail::enable_if_t<Dim == 0 && Mode == access::mode::atomic, T>
-  readFromFirstAccElement(accessor<T, Dim, Mode, Target, IsPH> Src) const {
+  static detail::enable_if_t<Dim == 0 && Mode == access::mode::atomic, T>
+  readFromFirstAccElement(accessor<T, Dim, Mode, Target, IsPH> Src) {
 #ifdef __ENABLE_USM_ADDR_SPACE__
     atomic<T, access::address_space::global_device_space> AtomicSrc = Src;
 #else
@@ -516,23 +521,23 @@ class __SYCL_EXPORT handler {
 
   template <typename T, int Dim, access::mode Mode, access::target Target,
             access::placeholder IsPH>
-  detail::enable_if_t<(Dim > 0) && Mode == access::mode::atomic, T>
-  readFromFirstAccElement(accessor<T, Dim, Mode, Target, IsPH> Src) const {
+  static detail::enable_if_t<(Dim > 0) && Mode == access::mode::atomic, T>
+  readFromFirstAccElement(accessor<T, Dim, Mode, Target, IsPH> Src) {
     id<Dim> Id = getDelinearizedIndex(Src.get_range(), 0);
     return Src[Id].load();
   }
 
   template <typename T, int Dim, access::mode Mode, access::target Target,
             access::placeholder IsPH>
-  detail::enable_if_t<Mode != access::mode::atomic, T>
-  readFromFirstAccElement(accessor<T, Dim, Mode, Target, IsPH> Src) const {
+  static detail::enable_if_t<Mode != access::mode::atomic, T>
+  readFromFirstAccElement(accessor<T, Dim, Mode, Target, IsPH> Src) {
     return *(Src.get_pointer());
   }
 
   template <typename T, int Dim, access::mode Mode, access::target Target,
             access::placeholder IsPH>
-  detail::enable_if_t<Dim == 0 && Mode == access::mode::atomic, void>
-  writeToFirstAccElement(accessor<T, Dim, Mode, Target, IsPH> Dst, T V) const {
+  static detail::enable_if_t<Dim == 0 && Mode == access::mode::atomic, void>
+  writeToFirstAccElement(accessor<T, Dim, Mode, Target, IsPH> Dst, T V) {
 #ifdef __ENABLE_USM_ADDR_SPACE__
     atomic<T, access::address_space::global_device_space> AtomicDst = Dst;
 #else
@@ -543,16 +548,16 @@ class __SYCL_EXPORT handler {
 
   template <typename T, int Dim, access::mode Mode, access::target Target,
             access::placeholder IsPH>
-  detail::enable_if_t<(Dim > 0) && Mode == access::mode::atomic, void>
-  writeToFirstAccElement(accessor<T, Dim, Mode, Target, IsPH> Dst, T V) const {
+  static detail::enable_if_t<(Dim > 0) && Mode == access::mode::atomic, void>
+  writeToFirstAccElement(accessor<T, Dim, Mode, Target, IsPH> Dst, T V) {
     id<Dim> Id = getDelinearizedIndex(Dst.get_range(), 0);
     Dst[Id].store(V);
   }
 
   template <typename T, int Dim, access::mode Mode, access::target Target,
             access::placeholder IsPH>
-  detail::enable_if_t<Mode != access::mode::atomic, void>
-  writeToFirstAccElement(accessor<T, Dim, Mode, Target, IsPH> Dst, T V) const {
+  static detail::enable_if_t<Mode != access::mode::atomic, void>
+  writeToFirstAccElement(accessor<T, Dim, Mode, Target, IsPH> Dst, T V) {
     *(Dst.get_pointer()) = V;
   }
 
@@ -721,6 +726,7 @@ class __SYCL_EXPORT handler {
     MNDRDesc.set(std::move(NumWorkItems));
     MCGType = detail::CG::KERNEL;
     extractArgsAndReqs();
+    MKernelName = getKernelName();
   }
 
 #ifdef __SYCL_DEVICE_ONLY__
@@ -791,13 +797,42 @@ class __SYCL_EXPORT handler {
     }
   }
 
+  template <typename T>
+  using remove_cv_ref_t =
+      typename std::remove_cv<detail::remove_reference_t<T>>::type;
+
+  template <typename U, typename T>
+  using is_same_type = std::is_same<remove_cv_ref_t<U>, remove_cv_ref_t<T>>;
+
+  template <typename T> struct ShouldEnableSetArg {
+    static constexpr bool value =
+        std::is_trivially_copyable<detail::remove_reference_t<T>>::value
+#if CL_SYCL_LANGUAGE_VERSION && CL_SYCL_LANGUAGE_VERSION <= 121
+            && std::is_standard_layout<detail::remove_reference_t<T>>::value
+#endif
+        || is_same_type<sampler, T>::value // Sampler
+        || (!is_same_type<cl_mem, T>::value &&
+            std::is_pointer<remove_cv_ref_t<T>>::value) // USM
+        || is_same_type<cl_mem, T>::value;              // Interop
+  };
+
   /// Sets argument for OpenCL interoperability kernels.
   ///
   /// Registers Arg passed as argument # ArgIndex.
   ///
   /// \param ArgIndex is a positional number of argument to be set.
   /// \param Arg is an argument value to be set.
-  template <typename T> void set_arg(int ArgIndex, T &&Arg) {
+  template <typename T>
+  typename std::enable_if<ShouldEnableSetArg<T>::value, void>::type
+  set_arg(int ArgIndex, T &&Arg) {
+    setArgHelper(ArgIndex, std::move(Arg));
+  }
+
+  template <typename DataT, int Dims, access::mode AccessMode,
+            access::target AccessTarget, access::placeholder IsPlaceholder>
+  void
+  set_arg(int ArgIndex,
+          accessor<DataT, Dims, AccessMode, AccessTarget, IsPlaceholder> Arg) {
     setArgHelper(ArgIndex, std::move(Arg));
   }
 
@@ -1036,8 +1071,6 @@ class __SYCL_EXPORT handler {
             int Dims, typename Reduction>
   detail::enable_if_t<!Reduction::has_fast_atomics>
   parallel_for(nd_range<Dims> Range, Reduction Redu, KernelType KernelFunc) {
-    size_t NWorkGroups = Range.get_group_range().size();
-
     // This parallel_for() is lowered to the following sequence:
     // 1) Call a kernel that a) call user's lambda function and b) performs
     //    one iteration of reduction, storing the partial reductions/sums
@@ -1051,42 +1084,51 @@ class __SYCL_EXPORT handler {
     // 2) Call an aux kernel (if necessary, i.e. if N2 > 1) as many times as
     //    necessary to reduce all partial sums into one final sum.
 
+    // Before running the kernels, check that device has enough local memory
+    // to hold local arrays that may be required for the reduction algorithm.
+    // TODO: If the work-group-size is limited by the local memory, then
+    // a special version of the main kernel may be created. The one that would
+    // not use local accessors, which means it would not do the reduction in
+    // the main kernel, but simply generate Range.get_global_range.size() number
+    // of partial sums, leaving the reduction work to the additional/aux
+    // kernels.
+    constexpr bool HFR = Reduction::has_fast_reduce;
+    size_t OneElemSize = HFR ? 0 : sizeof(typename Reduction::result_type);
+    // TODO: currently the maximal work group size is determined for the given
+    // queue/device, while it may be safer to use queries to the kernel compiled
+    // for the device.
+    size_t MaxWGSize = intel::detail::reduGetMaxWGSize(MQueue, OneElemSize);
+    if (Range.get_local_range().size() > MaxWGSize)
+      throw sycl::runtime_error("The implementation handling parallel_for with"
+                                " reduction requires smaller work group size.",
+                                PI_INVALID_WORK_GROUP_SIZE);
+
     // 1. Call the kernel that includes user's lambda function.
     intel::detail::reduCGFunc<KernelName>(*this, KernelFunc, Range, Redu);
     shared_ptr_class<detail::queue_impl> QueueCopy = MQueue;
     this->finalize();
 
-    // 2. Run the additional aux kernel as many times as needed to reduce
+    // 2. Run the additional kernel as many times as needed to reduce
     // all partial sums into one scalar.
 
-    // TODO: user's nd_range and the work-group size specified there must
-    // be honored only for the main kernel that calls user's lambda functions.
-    // There is no need in using the same work-group size in these additional
-    // kernels. Thus, the better strategy here is to make the work-group size
-    // as big as possible to converge/reduce the partial sums into the last
-    // sum faster.
-    size_t WGSize = Range.get_local_range().size();
-    size_t NWorkItems = NWorkGroups;
+    // TODO: Create a special slow/sequential version of the kernel that would
+    // handle the reduction instead of reporting an assert below.
+    if (MaxWGSize <= 1)
+      throw sycl::runtime_error("The implementation handling parallel_for with "
+                                "reduction requires the maximal work group "
+                                "size to be greater than 1 to converge. "
+                                "The maximal work group size depends on the "
+                                "device and the size of the objects passed to "
+                                "the reduction.",
+                                PI_INVALID_WORK_GROUP_SIZE);
+    size_t NWorkItems = Range.get_group_range().size();
     while (NWorkItems > 1) {
-      WGSize = std::min(WGSize, NWorkItems);
-      NWorkGroups = NWorkItems / WGSize;
-      // The last group may be not fully loaded. Still register it as a group.
-      if ((NWorkItems % WGSize) != 0)
-        ++NWorkGroups;
-      nd_range<1> Range(range<1>(WGSize * NWorkGroups), range<1>(WGSize));
-
       handler AuxHandler(QueueCopy, MIsHost);
       AuxHandler.saveCodeLoc(MCodeLoc);
 
-      // The last kernel DOES write to user's accessor passed to reduction.
-      // Associate it with handler manually.
-      if (NWorkGroups == 1 && !Reduction::is_usm)
-        Redu.associateWithHandler(AuxHandler);
-      intel::detail::reduAuxCGFunc<KernelName, KernelType>(AuxHandler, Range,
-                                                           NWorkItems, Redu);
+      NWorkItems = intel::detail::reduAuxCGFunc<KernelName, KernelType>(
+          AuxHandler, NWorkItems, MaxWGSize, Redu);
       MLastEvent = AuxHandler.finalize();
-
-      NWorkItems = NWorkGroups;
     } // end while (NWorkItems > 1)
   }
 
@@ -1173,6 +1215,7 @@ class __SYCL_EXPORT handler {
     MKernel = detail::getSyclObjImpl(std::move(Kernel));
     MCGType = detail::CG::KERNEL;
     extractArgsAndReqs();
+    MKernelName = getKernelName();
   }
 
   void parallel_for(range<1> NumWorkItems, kernel Kernel) {
@@ -1206,6 +1249,7 @@ class __SYCL_EXPORT handler {
     MNDRDesc.set(std::move(NumWorkItems), std::move(WorkItemOffset));
     MCGType = detail::CG::KERNEL;
     extractArgsAndReqs();
+    MKernelName = getKernelName();
   }
 
   /// Defines and invokes a SYCL kernel function for the specified range and
@@ -1226,6 +1270,7 @@ class __SYCL_EXPORT handler {
     MNDRDesc.set(std::move(NDRange));
     MCGType = detail::CG::KERNEL;
     extractArgsAndReqs();
+    MKernelName = getKernelName();
   }
 
   /// Defines and invokes a SYCL kernel function.
@@ -1248,9 +1293,10 @@ class __SYCL_EXPORT handler {
     MNDRDesc.set(range<1>{1});
     MKernel = detail::getSyclObjImpl(std::move(Kernel));
     MCGType = detail::CG::KERNEL;
-    if (!MIsHost && !lambdaAndKernelHaveEqualName<NameT>())
+    if (!MIsHost && !lambdaAndKernelHaveEqualName<NameT>()) {
       extractArgsAndReqs();
-    else
+      MKernelName = getKernelName();
+    } else
       StoreLambda<NameT, KernelType, /*Dims*/ 0, void>(std::move(KernelFunc));
 #endif
   }
@@ -1288,9 +1334,10 @@ class __SYCL_EXPORT handler {
     MNDRDesc.set(std::move(NumWorkItems));
     MKernel = detail::getSyclObjImpl(std::move(Kernel));
     MCGType = detail::CG::KERNEL;
-    if (!MIsHost && !lambdaAndKernelHaveEqualName<NameT>())
+    if (!MIsHost && !lambdaAndKernelHaveEqualName<NameT>()) {
       extractArgsAndReqs();
-    else
+      MKernelName = getKernelName();
+    } else
       StoreLambda<NameT, KernelType, Dims, LambdaArgType>(
           std::move(KernelFunc));
 #endif
@@ -1324,9 +1371,10 @@ class __SYCL_EXPORT handler {
     MNDRDesc.set(std::move(NumWorkItems), std::move(WorkItemOffset));
     MKernel = detail::getSyclObjImpl(std::move(Kernel));
     MCGType = detail::CG::KERNEL;
-    if (!MIsHost && !lambdaAndKernelHaveEqualName<NameT>())
+    if (!MIsHost && !lambdaAndKernelHaveEqualName<NameT>()) {
       extractArgsAndReqs();
-    else
+      MKernelName = getKernelName();
+    } else
       StoreLambda<NameT, KernelType, Dims, LambdaArgType>(
           std::move(KernelFunc));
 #endif
@@ -1361,9 +1409,10 @@ class __SYCL_EXPORT handler {
     MNDRDesc.set(std::move(NDRange));
     MKernel = detail::getSyclObjImpl(std::move(Kernel));
     MCGType = detail::CG::KERNEL;
-    if (!MIsHost && !lambdaAndKernelHaveEqualName<NameT>())
+    if (!MIsHost && !lambdaAndKernelHaveEqualName<NameT>()) {
       extractArgsAndReqs();
-    else
+      MKernelName = getKernelName();
+    } else
       StoreLambda<NameT, KernelType, Dims, LambdaArgType>(
           std::move(KernelFunc));
 #endif
@@ -1779,7 +1828,7 @@ class __SYCL_EXPORT handler {
   unique_ptr_class<detail::HostKernelBase> MHostKernel;
   /// Storage for lambda/function when using HostTask
   unique_ptr_class<detail::HostTask> MHostTask;
-  detail::OSModuleHandle MOSModuleHandle;
+  detail::OSModuleHandle MOSModuleHandle = detail::OSUtil::ExeModuleHandle;
   // Storage for a lambda or function when using InteropTasks
   std::unique_ptr<detail::InteropTask> MInteropTask;
   /// The list of events that order this operation.
diff --git a/sycl/include/CL/sycl/intel/atomic.hpp b/sycl/include/CL/sycl/intel/atomic.hpp
index bbc49ecc210d9..ecb32b76a6976 100644
--- a/sycl/include/CL/sycl/intel/atomic.hpp
+++ b/sycl/include/CL/sycl/intel/atomic.hpp
@@ -8,6 +8,7 @@
 
 #pragma once
 
+#include <CL/sycl/intel/atomic_accessor.hpp>
 #include <CL/sycl/intel/atomic_enums.hpp>
 #include <CL/sycl/intel/atomic_fence.hpp>
 #include <CL/sycl/intel/atomic_ref.hpp>
diff --git a/sycl/include/CL/sycl/intel/atomic_accessor.hpp b/sycl/include/CL/sycl/intel/atomic_accessor.hpp
new file mode 100644
index 0000000000000..524e2d4a3ab80
--- /dev/null
+++ b/sycl/include/CL/sycl/intel/atomic_accessor.hpp
@@ -0,0 +1,128 @@
+//==--- atomic_accessor.hpp - SYCL_INTEL_extended_atomics atomic_accessor --==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include <CL/sycl/access/access.hpp>
+#include <CL/sycl/intel/atomic_enums.hpp>
+#include <CL/sycl/intel/atomic_ref.hpp>
+
+__SYCL_INLINE_NAMESPACE(cl) {
+namespace sycl {
+namespace intel {
+
+#if __cplusplus > 201402L
+
+template <memory_order> struct order_tag_t {
+  explicit order_tag_t() = default;
+};
+inline constexpr order_tag_t<memory_order::relaxed> relaxed_order{};
+inline constexpr order_tag_t<memory_order::acquire> acquire_order{};
+inline constexpr order_tag_t<memory_order::release> release_order{};
+inline constexpr order_tag_t<memory_order::acq_rel> acq_rel_order{};
+inline constexpr order_tag_t<memory_order::seq_cst> seq_cst_order{};
+
+template <memory_scope> struct scope_tag_t {
+  explicit scope_tag_t() = default;
+};
+inline constexpr scope_tag_t<memory_scope::work_item> work_item_scope{};
+inline constexpr scope_tag_t<memory_scope::sub_group> sub_group_scope{};
+inline constexpr scope_tag_t<memory_scope::work_group> work_group_scope{};
+inline constexpr scope_tag_t<memory_scope::device> device_scope{};
+inline constexpr scope_tag_t<memory_scope::system> system_scope{};
+
+#endif
+
+template <typename DataT, int Dimensions, memory_order DefaultOrder,
+          memory_scope DefaultScope,
+          access::target AccessTarget = access::target::global_buffer,
+          access::placeholder IsPlaceholder = access::placeholder::false_t>
+class atomic_accessor
+    : public accessor<DataT, Dimensions, access::mode::read_write, AccessTarget,
+                      IsPlaceholder> {
+
+  using AccessorT = accessor<DataT, Dimensions, access::mode::read_write,
+                             AccessTarget, IsPlaceholder>;
+
+private:
+  using AccessorT::getLinearIndex;
+  using AccessorT::getQualifiedPtr;
+
+  // Prevent non-atomic access to atomic accessor
+  multi_ptr<DataT, AccessorT::AS> get_pointer() const = delete;
+
+protected:
+  using AccessorT::AdjustedDim;
+
+public:
+  using value_type = DataT;
+  using reference =
+      atomic_ref<DataT, DefaultOrder, DefaultScope, AccessorT::AS>;
+
+  using AccessorT::AccessorT;
+
+#if __cplusplus > 201402L
+
+  template <typename T = DataT, int Dims = Dimensions, typename AllocatorT,
+            memory_order Order, memory_scope Scope>
+  atomic_accessor(buffer<T, Dims, AllocatorT> &BufferRef, order_tag_t<Order>,
+                  scope_tag_t<Scope>, const property_list &PropertyList = {})
+      : atomic_accessor(BufferRef, PropertyList) {}
+
+  template <typename T = DataT, int Dims = Dimensions, typename AllocatorT,
+            memory_order Order, memory_scope Scope>
+  atomic_accessor(buffer<T, Dims, AllocatorT> &BufferRef,
+                  handler &CommandGroupHandler, order_tag_t<Order>,
+                  scope_tag_t<Scope>, const property_list &PropertyList = {})
+      : atomic_accessor(BufferRef, CommandGroupHandler, PropertyList) {}
+
+#endif
+
+  // Override subscript operators and conversions to wrap in an atomic_ref
+  template <int Dims = Dimensions>
+  operator typename detail::enable_if_t<Dims == 0, reference>() const {
+    const size_t LinearIndex = getLinearIndex(id<AdjustedDim>());
+    return reference(getQualifiedPtr()[LinearIndex]);
+  }
+
+  template <int Dims = Dimensions>
+  typename detail::enable_if_t<(Dims > 0), reference>
+  operator[](id<Dimensions> Index) const {
+    const size_t LinearIndex = getLinearIndex(Index);
+    return reference(getQualifiedPtr()[LinearIndex]);
+  }
+
+  template <int Dims = Dimensions>
+  typename detail::enable_if_t<Dims == 1, reference>
+  operator[](size_t Index) const {
+    const size_t LinearIndex = getLinearIndex(id<AdjustedDim>(Index));
+    return reference(getQualifiedPtr()[LinearIndex]);
+  }
+};
+
+#if __cplusplus > 201402L
+
+template <typename DataT, int Dimensions, typename AllocatorT,
+          memory_order Order, memory_scope Scope>
+atomic_accessor(buffer<DataT, Dimensions, AllocatorT>, order_tag_t<Order>,
+                scope_tag_t<Scope>, property_list = {})
+    ->atomic_accessor<DataT, Dimensions, Order, Scope, target::global_buffer,
+                      access::placeholder::true_t>;
+
+template <typename DataT, int Dimensions, typename AllocatorT,
+          memory_order Order, memory_scope Scope>
+atomic_accessor(buffer<DataT, Dimensions, AllocatorT>, handler,
+                order_tag_t<Order>, scope_tag_t<Scope>, property_list = {})
+    ->atomic_accessor<DataT, Dimensions, Order, Scope, target::global_buffer,
+                      access::placeholder::false_t>;
+
+#endif
+
+} // namespace intel
+} // namespace sycl
+} // __SYCL_INLINE_NAMESPACE(cl)
diff --git a/sycl/include/CL/sycl/intel/atomic_ref.hpp b/sycl/include/CL/sycl/intel/atomic_ref.hpp
index f6e8d4ff68616..1616727f919b8 100644
--- a/sycl/include/CL/sycl/intel/atomic_ref.hpp
+++ b/sycl/include/CL/sycl/intel/atomic_ref.hpp
@@ -135,8 +135,6 @@ class atomic_ref_base {
   static_assert(!(std::is_same<T, short>::value ||
                   std::is_same<T, unsigned short>::value),
                 "intel::atomic_ref does not support short type");
-  static_assert(!std::is_pointer<T>::value,
-                "intel::atomic_ref does not yet support pointer types");
   static_assert(detail::IsValidAtomicAddressSpace<AddressSpace>::value,
                 "Invalid atomic address_space.  Valid address spaces are: "
                 "global_space, local_space, global_device_space");
@@ -508,12 +506,138 @@ class atomic_ref_impl<
 };
 
 // Partial specialization for pointer types
+// Arithmetic is emulated because target's representation of T* is unknown
+// TODO: Find a way to use intptr_t or uintptr_t atomics instead
 template <typename T, memory_order DefaultOrder, memory_scope DefaultScope,
           access::address_space AddressSpace>
-class atomic_ref_impl<T *, DefaultOrder, DefaultScope, AddressSpace,
-                      typename detail::enable_if_t<std::is_pointer<T>::value>>
-    : public atomic_ref_base<T *, DefaultOrder, DefaultScope, AddressSpace> {
-  // TODO: Implement partial specialization for pointer types
+class atomic_ref_impl<T *, DefaultOrder, DefaultScope, AddressSpace>
+    : public atomic_ref_base<uintptr_t, DefaultOrder, DefaultScope,
+                             AddressSpace> {
+
+private:
+  using base_type =
+      atomic_ref_base<uintptr_t, DefaultOrder, DefaultScope, AddressSpace>;
+
+public:
+  using value_type = T *;
+  using difference_type = ptrdiff_t;
+  static constexpr size_t required_alignment = sizeof(T *);
+  static constexpr bool is_always_lock_free =
+      detail::IsValidAtomicType<T>::value;
+  static constexpr memory_order default_read_order =
+      detail::memory_order_traits<DefaultOrder>::read_order;
+  static constexpr memory_order default_write_order =
+      detail::memory_order_traits<DefaultOrder>::write_order;
+  static constexpr memory_order default_read_modify_write_order = DefaultOrder;
+  static constexpr memory_scope default_scope = DefaultScope;
+
+  using base_type::is_lock_free;
+
+  atomic_ref_impl(T *&ref) : base_type(reinterpret_cast<uintptr_t &>(ref)) {}
+
+  void store(T *operand, memory_order order = default_write_order,
+             memory_scope scope = default_scope) const noexcept {
+    base_type::store(reinterpret_cast<uintptr_t>(operand), order, scope);
+  }
+
+  T *operator=(T *desired) const noexcept {
+    store(desired);
+    return desired;
+  }
+
+  T *load(memory_order order = default_read_order,
+          memory_scope scope = default_scope) const noexcept {
+    return reinterpret_cast<T *>(base_type::load(order, scope));
+  }
+
+  operator T *() const noexcept { return load(); }
+
+  T *exchange(T *operand, memory_order order = default_read_modify_write_order,
+              memory_scope scope = default_scope) const noexcept {
+    return reinterpret_cast<T *>(base_type::exchange(
+        reinterpret_cast<uintptr_t>(operand), order, scope));
+  }
+
+  T *fetch_add(difference_type operand,
+               memory_order order = default_read_modify_write_order,
+               memory_scope scope = default_scope) const noexcept {
+    // TODO: Find a way to avoid compare_exchange here
+    auto load_order = detail::getLoadOrder(order);
+    T *expected = load(load_order, scope);
+    T *desired;
+    do {
+      desired = expected + operand;
+    } while (!compare_exchange_weak(expected, desired, order, scope));
+    return expected;
+  }
+
+  T *operator+=(difference_type operand) const noexcept {
+    return fetch_add(operand) + operand;
+  }
+
+  T *operator++(int) const noexcept { return fetch_add(difference_type(1)); }
+
+  T *operator++() const noexcept {
+    return fetch_add(difference_type(1)) + difference_type(1);
+  }
+
+  T *fetch_sub(difference_type operand,
+               memory_order order = default_read_modify_write_order,
+               memory_scope scope = default_scope) const noexcept {
+    // TODO: Find a way to avoid compare_exchange here
+    auto load_order = detail::getLoadOrder(order);
+    T *expected = load(load_order, scope);
+    T *desired;
+    do {
+      desired = expected - operand;
+    } while (!compare_exchange_weak(expected, desired, order, scope));
+    return expected;
+  }
+
+  T *operator-=(difference_type operand) const noexcept {
+    return fetch_sub(operand) - operand;
+  }
+
+  T *operator--(int) const noexcept { return fetch_sub(difference_type(1)); }
+
+  T *operator--() const noexcept {
+    return fetch_sub(difference_type(1)) - difference_type(1);
+  }
+
+  bool
+  compare_exchange_strong(T *&expected, T *desired, memory_order success,
+                          memory_order failure,
+                          memory_scope scope = default_scope) const noexcept {
+    return base_type::compare_exchange_strong(
+        reinterpret_cast<uintptr_t &>(expected),
+        reinterpret_cast<uintptr_t>(desired), success, failure, scope);
+  }
+
+  bool
+  compare_exchange_strong(T *&expected, T *desired,
+                          memory_order order = default_read_modify_write_order,
+                          memory_scope scope = default_scope) const noexcept {
+    return compare_exchange_strong(expected, desired, order, order, scope);
+  }
+
+  bool
+  compare_exchange_weak(T *&expected, T *desired, memory_order success,
+                        memory_order failure,
+                        memory_scope scope = default_scope) const noexcept {
+    return base_type::compare_exchange_weak(
+        reinterpret_cast<uintptr_t &>(expected),
+        reinterpret_cast<uintptr_t>(desired), success, failure, scope);
+  }
+
+  bool
+  compare_exchange_weak(T *&expected, T *desired,
+                        memory_order order = default_read_modify_write_order,
+                        memory_scope scope = default_scope) const noexcept {
+    return compare_exchange_weak(expected, desired, order, order, scope);
+  }
+
+private:
+  using base_type::ptr;
 };
 
 } // namespace detail
diff --git a/sycl/include/CL/sycl/intel/esimd/detail/esimd_util.hpp b/sycl/include/CL/sycl/intel/esimd/detail/esimd_util.hpp
index 54d769db75b4b..4bd68905e069b 100755
--- a/sycl/include/CL/sycl/intel/esimd/detail/esimd_util.hpp
+++ b/sycl/include/CL/sycl/intel/esimd/detail/esimd_util.hpp
@@ -17,10 +17,42 @@ namespace __esimd {
 /// Constant in number of bytes.
 enum { BYTE = 1, WORD = 2, DWORD = 4, QWORD = 8, OWORD = 16, GRF = 32 };
 
-/// Compute the next power of 2 at compile time.
-static ESIMD_INLINE constexpr unsigned int getNextPowerOf2(unsigned int n,
-                                                           unsigned int k = 1) {
-  return (k >= n) ? k : getNextPowerOf2(n, k * 2);
+/// Compute next power of 2 of a constexpr with guaranteed compile-time
+/// evaluation.
+template <unsigned int N, unsigned int K, bool K_gt_eq_N> struct NextPowerOf2;
+
+template <unsigned int N, unsigned int K> struct NextPowerOf2<N, K, true> {
+  static constexpr unsigned int get() { return K; }
+};
+
+template <unsigned int N, unsigned int K> struct NextPowerOf2<N, K, false> {
+  static constexpr unsigned int get() {
+    return NextPowerOf2<N, K * 2, K * 2 >= N>::get();
+  }
+};
+
+template <unsigned int N> constexpr unsigned int getNextPowerOf2() {
+  return NextPowerOf2<N, 1, (1 >= N)>::get();
+}
+
+template <> constexpr unsigned int getNextPowerOf2<0>() { return 0; }
+
+/// Compute binary logarithm of a constexpr with guaranteed compile-time
+/// evaluation.
+template <unsigned int N, bool N_gt_1> struct Log2;
+
+template <unsigned int N> struct Log2<N, false> {
+  static constexpr unsigned int get() { return 0; }
+};
+
+template <unsigned int N> struct Log2<N, true> {
+  static constexpr unsigned int get() {
+    return 1 + Log2<(N >> 1), ((N >> 1) > 1)>::get();
+  }
+};
+
+template <unsigned int N> constexpr unsigned int log2() {
+  return Log2<N, (N > 1)>::get();
 }
 
 /// Check if a given 32 bit positive integer is a power of 2 at compile time.
@@ -33,10 +65,6 @@ static ESIMD_INLINE constexpr bool isPowerOf2(unsigned int n,
   return (n & (n - 1)) == 0 && n <= limit;
 }
 
-static ESIMD_INLINE constexpr unsigned log2(unsigned n) {
-  return (n > 1) ? 1 + log2(n >> 1) : 0;
-}
-
 } // namespace __esimd
 
 __SYCL_INLINE_NAMESPACE(cl) {
diff --git a/sycl/include/CL/sycl/intel/esimd/esimd_math.hpp b/sycl/include/CL/sycl/intel/esimd/esimd_math.hpp
index 4eb145a315124..cd5b962dd60a2 100644
--- a/sycl/include/CL/sycl/intel/esimd/esimd_math.hpp
+++ b/sycl/include/CL/sycl/intel/esimd/esimd_math.hpp
@@ -1884,7 +1884,7 @@ T1 esimd_reduce(simd<T1, SZ> v) {
   if constexpr (isPowerOf2) {
     return esimd_reduce_single<T0, T1, SZ, OpType>(v);
   } else {
-    constexpr unsigned N1 = 1u << __esimd::log2(SZ);
+    constexpr unsigned N1 = 1u << __esimd::log2<SZ>();
     constexpr unsigned N2 = SZ - N1;
 
     simd<T1, N1> v1 = v.template select<N1, 1>(0);
diff --git a/sycl/include/CL/sycl/intel/esimd/esimd_memory.hpp b/sycl/include/CL/sycl/intel/esimd/esimd_memory.hpp
index 84d175e981595..77035cb16c9c3 100644
--- a/sycl/include/CL/sycl/intel/esimd/esimd_memory.hpp
+++ b/sycl/include/CL/sycl/intel/esimd/esimd_memory.hpp
@@ -567,7 +567,7 @@ media_block_load(AccessorTy acc, unsigned x, unsigned y) {
   static_assert(plane <= 3u, "valid plane index is in range [0, 3]");
 #if defined(__SYCL_DEVICE_ONLY__) && defined(__SYCL_EXPLICIT_SIMD__)
   constexpr unsigned int RoundedWidth =
-      Width < 4 ? 4 : __esimd::getNextPowerOf2(Width);
+      Width < 4 ? 4 : __esimd::getNextPowerOf2<Width>();
 
   if constexpr (Width < RoundedWidth) {
     constexpr unsigned int n1 = RoundedWidth / sizeof(T);
@@ -617,7 +617,7 @@ media_block_store(AccessorTy acc, unsigned x, unsigned y, simd<T, m * n> vals) {
   static_assert(plane <= 3u, "valid plane index is in range [0, 3]");
 #if defined(__SYCL_DEVICE_ONLY__) && defined(__SYCL_EXPLICIT_SIMD__)
   constexpr unsigned int RoundedWidth =
-      Width < 4 ? 4 : __esimd::getNextPowerOf2(Width);
+      Width < 4 ? 4 : __esimd::getNextPowerOf2<Width>();
   constexpr unsigned int n1 = RoundedWidth / sizeof(T);
 
   if constexpr (Width < RoundedWidth) {
diff --git a/sycl/include/CL/sycl/intel/fpga_extensions.hpp b/sycl/include/CL/sycl/intel/fpga_extensions.hpp
index a9fca1e6139d2..7140421fe5189 100644
--- a/sycl/include/CL/sycl/intel/fpga_extensions.hpp
+++ b/sycl/include/CL/sycl/intel/fpga_extensions.hpp
@@ -8,5 +8,6 @@
 
 #pragma once
 #include <CL/sycl/intel/fpga_device_selector.hpp>
+#include <CL/sycl/intel/fpga_lsu.hpp>
 #include <CL/sycl/intel/fpga_reg.hpp>
 #include <CL/sycl/intel/pipes.hpp>
diff --git a/sycl/include/CL/sycl/intel/fpga_lsu.hpp b/sycl/include/CL/sycl/intel/fpga_lsu.hpp
new file mode 100644
index 0000000000000..5f8d37f802e76
--- /dev/null
+++ b/sycl/include/CL/sycl/intel/fpga_lsu.hpp
@@ -0,0 +1,113 @@
+//==-------------- fpga_lsu.hpp --- SYCL FPGA Reg Extensions ---------------==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#pragma once
+
+#include "fpga_utils.hpp"
+#include <CL/sycl/detail/defines.hpp>
+#include <CL/sycl/pointers.hpp>
+
+__SYCL_INLINE_NAMESPACE(cl) {
+namespace sycl {
+namespace intel {
+constexpr uint8_t BURST_COALESCE = 0x1;
+constexpr uint8_t CACHE = 0x2;
+constexpr uint8_t STATICALLY_COALESCE = 0x4;
+constexpr uint8_t PREFETCH = 0x8;
+
+template <int32_t N> struct burst_coalesce_impl {
+  static constexpr int32_t value = N;
+  static constexpr int32_t default_value = 0;
+};
+
+template <int32_t N> struct cache {
+  static constexpr int32_t value = N;
+  static constexpr int32_t default_value = 0;
+};
+
+template <int32_t N> struct prefetch_impl {
+  static constexpr int32_t value = N;
+  static constexpr int32_t default_value = 0;
+};
+
+template <int32_t N> struct statically_coalesce_impl {
+  static constexpr int32_t value = N;
+  static constexpr int32_t default_value = 1;
+};
+
+template <bool B> using burst_coalesce = burst_coalesce_impl<B>;
+template <bool B> using prefetch = prefetch_impl<B>;
+template <bool B> using statically_coalesce = statically_coalesce_impl<B>;
+
+template <class... mem_access_params> class lsu final {
+public:
+  lsu() = delete;
+
+  template <typename T> static T &load(sycl::global_ptr<T> Ptr) {
+    check_load();
+#if defined(__SYCL_DEVICE_ONLY__) && __has_builtin(__builtin_intel_fpga_mem)
+    return *__builtin_intel_fpga_mem((T *)Ptr,
+                                     _burst_coalesce | _cache |
+                                         _dont_statically_coalesce | _prefetch,
+                                     _cache_val);
+#else
+    return *Ptr;
+#endif
+  }
+
+  template <typename T> static void store(sycl::global_ptr<T> Ptr, T Val) {
+    check_store();
+#if defined(__SYCL_DEVICE_ONLY__) && __has_builtin(__builtin_intel_fpga_mem)
+    *__builtin_intel_fpga_mem((T *)Ptr,
+                              _burst_coalesce | _cache |
+                                  _dont_statically_coalesce | _prefetch,
+                              _cache_val) = Val;
+#else
+    *Ptr = Val;
+#endif
+  }
+
+private:
+  static constexpr int32_t _burst_coalesce_val =
+      GetValue<burst_coalesce_impl, mem_access_params...>::value;
+  static constexpr uint8_t _burst_coalesce =
+      _burst_coalesce_val == 1 ? BURST_COALESCE : 0;
+
+  static constexpr int32_t _cache_val =
+      GetValue<cache, mem_access_params...>::value;
+  static constexpr uint8_t _cache = (_cache_val > 0) ? CACHE : 0;
+
+  static constexpr int32_t _statically_coalesce_val =
+      GetValue<statically_coalesce_impl, mem_access_params...>::value;
+  static constexpr uint8_t _dont_statically_coalesce =
+      _statically_coalesce_val == 0 ? STATICALLY_COALESCE : 0;
+
+  static constexpr int32_t _prefetch_val =
+      GetValue<prefetch_impl, mem_access_params...>::value;
+  static constexpr uint8_t _prefetch = _prefetch_val ? PREFETCH : 0;
+
+  static_assert(_cache_val >= 0, "cache size parameter must be non-negative");
+
+  static void check_load() {
+    static_assert(_cache == 0 || _burst_coalesce == BURST_COALESCE,
+                  "unable to implement a cache without a burst coalescer");
+    static_assert(_prefetch == 0 || _burst_coalesce == 0,
+                  "unable to implement a prefetcher and a burst coalescer "
+                  "simulataneously");
+    static_assert(
+        _prefetch == 0 || _cache == 0,
+        "unable to implement a prefetcher and a cache simulataneously");
+  }
+  static void check_store() {
+    static_assert(_cache == 0, "unable to implement a store LSU with a cache.");
+    static_assert(_prefetch == 0,
+                  "unable to implement a store LSU with a prefetcher.");
+  }
+};
+} // namespace intel
+} // namespace sycl
+} // __SYCL_INLINE_NAMESPACE(cl)
diff --git a/sycl/include/CL/sycl/intel/fpga_utils.hpp b/sycl/include/CL/sycl/intel/fpga_utils.hpp
new file mode 100644
index 0000000000000..be9bf1a6fc5af
--- /dev/null
+++ b/sycl/include/CL/sycl/intel/fpga_utils.hpp
@@ -0,0 +1,33 @@
+//==------------- fpga_utils.hpp --- SYCL FPGA Reg Extensions --------------==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include <CL/sycl/detail/defines.hpp>
+#include <CL/sycl/stl.hpp>
+
+__SYCL_INLINE_NAMESPACE(cl) {
+namespace sycl {
+namespace intel {
+
+template <template <int32_t> class Type, class T>
+struct MatchType : std::is_same<Type<T::value>, T> {};
+
+template <template <int32_t> class Type, class... T> struct GetValue {
+  static constexpr auto value = Type<0>::default_value;
+};
+
+template <template <int32_t> class Type, class T1, class... T>
+struct GetValue<Type, T1, T...> {
+  static constexpr auto value =
+      std::conditional<MatchType<Type, T1>::value, T1,
+                       GetValue<Type, T...>>::type::value;
+};
+} // namespace intel
+} // namespace sycl
+} // __SYCL_INLINE_NAMESPACE(cl)
diff --git a/sycl/include/CL/sycl/intel/group_algorithm.hpp b/sycl/include/CL/sycl/intel/group_algorithm.hpp
index e49ed1592e4da..932a53ba07675 100644
--- a/sycl/include/CL/sycl/intel/group_algorithm.hpp
+++ b/sycl/include/CL/sycl/intel/group_algorithm.hpp
@@ -91,6 +91,17 @@ template <typename T, typename V> struct identity<T, intel::maximum<V>> {
   static constexpr T value = std::numeric_limits<T>::lowest();
 };
 
+template <typename T>
+using native_op_list =
+    type_list<intel::plus<T>, intel::bit_or<T>, intel::bit_xor<T>,
+              intel::bit_and<T>, intel::maximum<T>, intel::minimum<T>>;
+
+template <typename T, typename BinaryOperation> struct is_native_op {
+  static constexpr bool value =
+      is_contained<BinaryOperation, native_op_list<T>>::value ||
+      is_contained<BinaryOperation, native_op_list<void>>::value;
+};
+
 template <typename Group, typename Ptr, class Function>
 Function for_each(Group g, Ptr first, Ptr last, Function f) {
 #ifdef __SYCL_DEVICE_ONLY__
@@ -114,6 +125,7 @@ Function for_each(Group g, Ptr first, Ptr last, Function f) {
 
 namespace intel {
 
+// EnableIf shorthands for algorithms that depend only on type
 template <typename T>
 using EnableIfIsScalarArithmetic = cl::sycl::detail::enable_if_t<
     cl::sycl::detail::is_scalar_arithmetic<T>::value, T>;
@@ -126,6 +138,28 @@ template <typename Ptr, typename T>
 using EnableIfIsPointer =
     cl::sycl::detail::enable_if_t<cl::sycl::detail::is_pointer<Ptr>::value, T>;
 
+// EnableIf shorthands for algorithms that depend on type and an operator
+template <typename T, typename BinaryOperation>
+using EnableIfIsScalarArithmeticNativeOp = cl::sycl::detail::enable_if_t<
+    cl::sycl::detail::is_scalar_arithmetic<T>::value &&
+        cl::sycl::detail::is_native_op<T, BinaryOperation>::value,
+    T>;
+
+template <typename T, typename BinaryOperation>
+using EnableIfIsVectorArithmeticNativeOp = cl::sycl::detail::enable_if_t<
+    cl::sycl::detail::is_vector_arithmetic<T>::value &&
+        cl::sycl::detail::is_native_op<T, BinaryOperation>::value,
+    T>;
+
+// TODO: Lift TriviallyCopyable restriction eventually
+template <typename T, typename BinaryOperation>
+using EnableIfIsNonNativeOp = cl::sycl::detail::enable_if_t<
+    (!cl::sycl::detail::is_scalar_arithmetic<T>::value &&
+     !cl::sycl::detail::is_vector_arithmetic<T>::value &&
+     std::is_trivially_copyable<T>::value) ||
+        !cl::sycl::detail::is_native_op<T, BinaryOperation>::value,
+    T>;
+
 template <typename Group> bool all_of(Group, bool pred) {
   static_assert(sycl::detail::is_generic_group<Group>::value,
                 "Group algorithms only support the sycl::group and "
@@ -363,7 +397,8 @@ EnableIfIsVectorArithmetic<T> broadcast(Group g, T x) {
 }
 
 template <typename Group, typename T, class BinaryOperation>
-EnableIfIsScalarArithmetic<T> reduce(Group, T x, BinaryOperation binary_op) {
+EnableIfIsScalarArithmeticNativeOp<T, BinaryOperation>
+reduce(Group, T x, BinaryOperation binary_op) {
   static_assert(sycl::detail::is_generic_group<Group>::value,
                 "Group algorithms only support the sycl::group and "
                 "intel::sub_group class.");
@@ -384,7 +419,8 @@ EnableIfIsScalarArithmetic<T> reduce(Group, T x, BinaryOperation binary_op) {
 }
 
 template <typename Group, typename T, class BinaryOperation>
-EnableIfIsVectorArithmetic<T> reduce(Group g, T x, BinaryOperation binary_op) {
+EnableIfIsVectorArithmeticNativeOp<T, BinaryOperation>
+reduce(Group g, T x, BinaryOperation binary_op) {
   static_assert(sycl::detail::is_generic_group<Group>::value,
                 "Group algorithms only support the sycl::group and "
                 "intel::sub_group class.");
@@ -402,9 +438,25 @@ EnableIfIsVectorArithmetic<T> reduce(Group g, T x, BinaryOperation binary_op) {
   return result;
 }
 
+template <typename Group, typename T, class BinaryOperation>
+EnableIfIsNonNativeOp<T, BinaryOperation> reduce(Group g, T x,
+                                                 BinaryOperation op) {
+  static_assert(sycl::detail::is_sub_group<Group>::value,
+                "reduce algorithm with user-defined types and operators"
+                "only supports intel::sub_group class.");
+  T result = x;
+  for (int mask = 1; mask < g.get_max_local_range()[0]; mask *= 2) {
+    T tmp = g.shuffle_xor(result, id<1>(mask));
+    if ((g.get_local_id()[0] ^ mask) < g.get_local_range()[0]) {
+      result = op(result, tmp);
+    }
+  }
+  return g.shuffle(result, 0);
+}
+
 template <typename Group, typename V, typename T, class BinaryOperation>
-EnableIfIsScalarArithmetic<T> reduce(Group g, V x, T init,
-                                     BinaryOperation binary_op) {
+EnableIfIsScalarArithmeticNativeOp<T, BinaryOperation>
+reduce(Group g, V x, T init, BinaryOperation binary_op) {
   static_assert(sycl::detail::is_generic_group<Group>::value,
                 "Group algorithms only support the sycl::group and "
                 "intel::sub_group class.");
@@ -424,8 +476,8 @@ EnableIfIsScalarArithmetic<T> reduce(Group g, V x, T init,
 }
 
 template <typename Group, typename V, typename T, class BinaryOperation>
-EnableIfIsVectorArithmetic<T> reduce(Group g, V x, T init,
-                                     BinaryOperation binary_op) {
+EnableIfIsVectorArithmeticNativeOp<T, BinaryOperation>
+reduce(Group g, V x, T init, BinaryOperation binary_op) {
   static_assert(sycl::detail::is_generic_group<Group>::value,
                 "Group algorithms only support the sycl::group and "
                 "intel::sub_group class.");
@@ -449,6 +501,22 @@ EnableIfIsVectorArithmetic<T> reduce(Group g, V x, T init,
 #endif
 }
 
+template <typename Group, typename V, typename T, class BinaryOperation>
+EnableIfIsNonNativeOp<T, BinaryOperation> reduce(Group g, V x, T init,
+                                                 BinaryOperation op) {
+  static_assert(sycl::detail::is_sub_group<Group>::value,
+                "reduce algorithm with user-defined types and operators"
+                "only supports intel::sub_group class.");
+  T result = x;
+  for (int mask = 1; mask < g.get_max_local_range()[0]; mask *= 2) {
+    T tmp = g.shuffle_xor(result, id<1>(mask));
+    if ((g.get_local_id()[0] ^ mask) < g.get_local_range()[0]) {
+      result = op(result, tmp);
+    }
+  }
+  return g.shuffle(op(init, result), 0);
+}
+
 template <typename Group, typename Ptr, class BinaryOperation>
 EnableIfIsPointer<Ptr, typename Ptr::element_type>
 reduce(Group g, Ptr first, Ptr last, BinaryOperation binary_op) {
@@ -509,8 +577,8 @@ EnableIfIsPointer<Ptr, T> reduce(Group g, Ptr first, Ptr last, T init,
 }
 
 template <typename Group, typename T, class BinaryOperation>
-EnableIfIsScalarArithmetic<T> exclusive_scan(Group, T x,
-                                             BinaryOperation binary_op) {
+EnableIfIsScalarArithmeticNativeOp<T, BinaryOperation>
+exclusive_scan(Group, T x, BinaryOperation binary_op) {
   static_assert(sycl::detail::is_generic_group<Group>::value,
                 "Group algorithms only support the sycl::group and "
                 "intel::sub_group class.");
@@ -530,8 +598,8 @@ EnableIfIsScalarArithmetic<T> exclusive_scan(Group, T x,
 }
 
 template <typename Group, typename T, class BinaryOperation>
-EnableIfIsVectorArithmetic<T> exclusive_scan(Group g, T x,
-                                             BinaryOperation binary_op) {
+EnableIfIsVectorArithmeticNativeOp<T, BinaryOperation>
+exclusive_scan(Group g, T x, BinaryOperation binary_op) {
   static_assert(sycl::detail::is_generic_group<Group>::value,
                 "Group algorithms only support the sycl::group and "
                 "intel::sub_group class.");
@@ -550,8 +618,8 @@ EnableIfIsVectorArithmetic<T> exclusive_scan(Group g, T x,
 }
 
 template <typename Group, typename V, typename T, class BinaryOperation>
-EnableIfIsVectorArithmetic<T> exclusive_scan(Group g, V x, T init,
-                                             BinaryOperation binary_op) {
+EnableIfIsVectorArithmeticNativeOp<T, BinaryOperation>
+exclusive_scan(Group g, V x, T init, BinaryOperation binary_op) {
   static_assert(sycl::detail::is_generic_group<Group>::value,
                 "Group algorithms only support the sycl::group and "
                 "intel::sub_group class.");
@@ -570,8 +638,8 @@ EnableIfIsVectorArithmetic<T> exclusive_scan(Group g, V x, T init,
 }
 
 template <typename Group, typename V, typename T, class BinaryOperation>
-EnableIfIsScalarArithmetic<T> exclusive_scan(Group g, V x, T init,
-                                             BinaryOperation binary_op) {
+EnableIfIsScalarArithmeticNativeOp<T, BinaryOperation>
+exclusive_scan(Group g, V x, T init, BinaryOperation binary_op) {
   static_assert(sycl::detail::is_generic_group<Group>::value,
                 "Group algorithms only support the sycl::group and "
                 "intel::sub_group class.");
@@ -663,8 +731,8 @@ EnableIfIsPointer<InPtr, OutPtr> exclusive_scan(Group g, InPtr first,
 }
 
 template <typename Group, typename T, class BinaryOperation>
-EnableIfIsVectorArithmetic<T> inclusive_scan(Group g, T x,
-                                             BinaryOperation binary_op) {
+EnableIfIsVectorArithmeticNativeOp<T, BinaryOperation>
+inclusive_scan(Group g, T x, BinaryOperation binary_op) {
   static_assert(sycl::detail::is_generic_group<Group>::value,
                 "Group algorithms only support the sycl::group and "
                 "intel::sub_group class.");
@@ -683,8 +751,8 @@ EnableIfIsVectorArithmetic<T> inclusive_scan(Group g, T x,
 }
 
 template <typename Group, typename T, class BinaryOperation>
-EnableIfIsScalarArithmetic<T> inclusive_scan(Group, T x,
-                                             BinaryOperation binary_op) {
+EnableIfIsScalarArithmeticNativeOp<T, BinaryOperation>
+inclusive_scan(Group, T x, BinaryOperation binary_op) {
   static_assert(sycl::detail::is_generic_group<Group>::value,
                 "Group algorithms only support the sycl::group and "
                 "intel::sub_group class.");
@@ -704,7 +772,7 @@ EnableIfIsScalarArithmetic<T> inclusive_scan(Group, T x,
 }
 
 template <typename Group, typename V, class BinaryOperation, typename T>
-EnableIfIsScalarArithmetic<T>
+EnableIfIsScalarArithmeticNativeOp<T, BinaryOperation>
 inclusive_scan(Group g, V x, BinaryOperation binary_op, T init) {
   static_assert(sycl::detail::is_generic_group<Group>::value,
                 "Group algorithms only support the sycl::group and "
@@ -727,7 +795,7 @@ inclusive_scan(Group g, V x, BinaryOperation binary_op, T init) {
 }
 
 template <typename Group, typename V, class BinaryOperation, typename T>
-EnableIfIsVectorArithmetic<T>
+EnableIfIsVectorArithmeticNativeOp<T, BinaryOperation>
 inclusive_scan(Group g, V x, BinaryOperation binary_op, T init) {
   static_assert(sycl::detail::is_generic_group<Group>::value,
                 "Group algorithms only support the sycl::group and "
diff --git a/sycl/include/CL/sycl/intel/reduction.hpp b/sycl/include/CL/sycl/intel/reduction.hpp
index 01b44cb429d6b..26adff47778e9 100644
--- a/sycl/include/CL/sycl/intel/reduction.hpp
+++ b/sycl/include/CL/sycl/intel/reduction.hpp
@@ -9,6 +9,7 @@
 #pragma once
 
 #include <CL/sycl/accessor.hpp>
+#include <CL/sycl/handler.hpp>
 #include <CL/sycl/intel/group_algorithm.hpp>
 
 __SYCL_INLINE_NAMESPACE(cl) {
@@ -17,6 +18,11 @@ namespace intel {
 
 namespace detail {
 
+__SYCL_EXPORT size_t reduGetMaxWGSize(shared_ptr_class<queue_impl> Queue,
+                                      size_t LocalMemBytesPerWorkItem);
+__SYCL_EXPORT size_t reduComputeWGSize(size_t NWorkItems, size_t MaxWGSize,
+                                       size_t &NWorkGroups);
+
 using cl::sycl::detail::bool_constant;
 using cl::sycl::detail::enable_if_t;
 using cl::sycl::detail::is_geninteger16bit;
@@ -141,11 +147,9 @@ using IsKnownIdentityOp =
 template <typename T, class BinaryOperation, typename Subst = void>
 class reducer {
 public:
-  reducer(const T &Identity) : MValue(Identity), MIdentity(Identity) {}
-  void combine(const T &Partial) {
-    BinaryOperation BOp;
-    MValue = BOp(MValue, Partial);
-  }
+  reducer(const T &Identity, BinaryOperation BOp)
+      : MValue(Identity), MIdentity(Identity), MBinaryOp(BOp) {}
+  void combine(const T &Partial) { MValue = MBinaryOp(MValue, Partial); }
 
   T getIdentity() const { return MIdentity; }
 
@@ -153,6 +157,7 @@ class reducer {
 
 private:
   const T MIdentity;
+  BinaryOperation MBinaryOp;
 };
 
 /// Specialization of the generic class 'reducer'. It is used for reductions
@@ -177,7 +182,7 @@ class reducer<T, BinaryOperation,
               enable_if_t<IsKnownIdentityOp<T, BinaryOperation>::value>> {
 public:
   reducer() : MValue(getIdentity()) {}
-  reducer(const T &) : MValue(getIdentity()) {}
+  reducer(const T &, BinaryOperation) : MValue(getIdentity()) {}
 
   void combine(const T &Partial) {
     BinaryOperation BOp;
@@ -399,7 +404,7 @@ class reduction_impl {
   template <
       typename _T = T, class _BinaryOperation = BinaryOperation,
       enable_if_t<IsKnownIdentityOp<_T, _BinaryOperation>::value> * = nullptr>
-  reduction_impl(accessor_type &Acc, const T &Identity)
+  reduction_impl(accessor_type &Acc, const T &Identity, BinaryOperation)
       : MAcc(shared_ptr_class<accessor_type>(shared_ptr_class<accessor_type>{},
                                              &Acc)),
         MIdentity(getIdentity()) {
@@ -425,10 +430,10 @@ class reduction_impl {
   template <
       typename _T = T, class _BinaryOperation = BinaryOperation,
       enable_if_t<!IsKnownIdentityOp<_T, _BinaryOperation>::value> * = nullptr>
-  reduction_impl(accessor_type &Acc, const T &Identity)
+  reduction_impl(accessor_type &Acc, const T &Identity, BinaryOperation BOp)
       : MAcc(shared_ptr_class<accessor_type>(shared_ptr_class<accessor_type>{},
                                              &Acc)),
-        MIdentity(Identity) {
+        MIdentity(Identity), MBinaryOp(BOp) {
     assert(Acc.get_count() == 1 &&
            "Only scalar/1-element reductions are supported now.");
   }
@@ -450,7 +455,7 @@ class reduction_impl {
   template <
       typename _T = T, class _BinaryOperation = BinaryOperation,
       enable_if_t<IsKnownIdentityOp<_T, _BinaryOperation>::value> * = nullptr>
-  reduction_impl(T *VarPtr, const T &Identity)
+  reduction_impl(T *VarPtr, const T &Identity, BinaryOperation)
       : MIdentity(Identity), MUSMPointer(VarPtr) {
     // For now the implementation ignores the identity value given by user
     // when the implementation knows the identity.
@@ -472,8 +477,8 @@ class reduction_impl {
   template <
       typename _T = T, class _BinaryOperation = BinaryOperation,
       enable_if_t<!IsKnownIdentityOp<_T, _BinaryOperation>::value> * = nullptr>
-  reduction_impl(T *VarPtr, const T &Identity)
-      : MIdentity(Identity), MUSMPointer(VarPtr) {}
+  reduction_impl(T *VarPtr, const T &Identity, BinaryOperation BOp)
+      : MIdentity(Identity), MUSMPointer(VarPtr), MBinaryOp(BOp) {}
 
   /// Associates reduction accessor with the given handler and saves reduction
   /// buffer so that it is alive until the command group finishes the work.
@@ -557,6 +562,9 @@ class reduction_impl {
     return OutPtr;
   }
 
+  /// Returns the binary operation associated with the reduction.
+  BinaryOperation getBinaryOperation() const { return MBinaryOp; }
+
 private:
   /// Identity of the BinaryOperation.
   /// The result of BinaryOperation(X, MIdentity) is equal to X for any X.
@@ -570,6 +578,8 @@ class reduction_impl {
   /// USM pointer referencing the memory to where the result of the reduction
   /// must be written. Applicable/used only for USM reductions.
   T *MUSMPointer = nullptr;
+
+  BinaryOperation MBinaryOp;
 };
 
 /// These are the forward declaration for the classes that help to create
@@ -788,9 +798,10 @@ reduCGFuncImpl(handler &CGH, KernelType KernelFunc, const nd_range<Dims> &Range,
   typename Reduction::result_type ReduIdentity = Redu.getIdentity();
   using Name = typename get_reduction_main_kernel_name_t<
       KernelName, KernelType, Reduction::is_usm, UniformPow2WG, OutputT>::name;
+  auto BOp = Redu.getBinaryOperation();
   CGH.parallel_for<Name>(Range, [=](nd_item<Dims> NDIt) {
     // Call user's functions. Reducer.MValue gets initialized there.
-    typename Reduction::reducer_type Reducer(ReduIdentity);
+    typename Reduction::reducer_type Reducer(ReduIdentity, BOp);
     KernelFunc(NDIt, Reducer);
 
     size_t WGSize = NDIt.get_local_range().size();
@@ -805,7 +816,6 @@ reduCGFuncImpl(handler &CGH, KernelType KernelFunc, const nd_range<Dims> &Range,
     // Tree-reduction: reduce the local array LocalReds[:] to LocalReds[0]
     // LocalReds[WGSize] accumulates last/odd elements when the step
     // of tree-reduction loop is not even.
-    typename Reduction::binary_operation BOp;
     size_t PrevStep = WGSize;
     for (size_t CurStep = PrevStep >> 1; CurStep > 0; CurStep >>= 1) {
       if (LID < CurStep)
@@ -867,19 +877,19 @@ reduCGFunc(handler &CGH, KernelType KernelFunc, const nd_range<Dims> &Range,
 /// of work-groups. At the end of each work-groups the partial sum is written
 /// to a global buffer.
 ///
-/// Briefly: aux kernel, intel:reduce(), reproducible results,FP + ADD/MIN/MAX
-template <typename KernelName, typename KernelType, int Dims, class Reduction,
-          bool UniformWG, typename InputT, typename OutputT>
+/// Briefly: aux kernel, intel:reduce(), reproducible results, FP + ADD/MIN/MAX
+template <typename KernelName, typename KernelType, bool UniformWG,
+          class Reduction, typename InputT, typename OutputT>
 enable_if_t<Reduction::has_fast_reduce && !Reduction::has_fast_atomics>
-reduAuxCGFuncImpl(handler &CGH, const nd_range<Dims> &Range, size_t NWorkItems,
-                  Reduction &, InputT In, OutputT Out) {
-  size_t NWorkGroups = Range.get_group_range().size();
-  bool IsUpdateOfUserVar =
-      Reduction::accessor_mode == access::mode::read_write && NWorkGroups == 1;
-
+reduAuxCGFuncImpl(handler &CGH, size_t NWorkItems, size_t NWorkGroups,
+                  size_t WGSize, Reduction &, InputT In, OutputT Out) {
   using Name = typename get_reduction_aux_kernel_name_t<
       KernelName, KernelType, Reduction::is_usm, UniformWG, OutputT>::name;
-  CGH.parallel_for<Name>(Range, [=](nd_item<Dims> NDIt) {
+
+  bool IsUpdateOfUserVar =
+      Reduction::accessor_mode == access::mode::read_write && NWorkGroups == 1;
+  nd_range<1> Range{range<1>(NWorkItems), range<1>(WGSize)};
+  CGH.parallel_for<Name>(Range, [=](nd_item<1> NDIt) {
     typename Reduction::binary_operation BOp;
     size_t WGID = NDIt.get_group_linear_id();
     size_t GID = NDIt.get_global_linear_id();
@@ -903,14 +913,11 @@ reduAuxCGFuncImpl(handler &CGH, const nd_range<Dims> &Range, size_t NWorkItems,
 /// to a global buffer.
 ///
 /// Briefly: aux kernel, tree-reduction, CUSTOM types/ops.
-template <typename KernelName, typename KernelType, int Dims, class Reduction,
-          bool UniformPow2WG, typename InputT, typename OutputT>
+template <typename KernelName, typename KernelType, bool UniformPow2WG,
+          class Reduction, typename InputT, typename OutputT>
 enable_if_t<!Reduction::has_fast_reduce && !Reduction::has_fast_atomics>
-reduAuxCGFuncImpl(handler &CGH, const nd_range<Dims> &Range, size_t NWorkItems,
-                  Reduction &Redu, InputT In, OutputT Out) {
-  size_t WGSize = Range.get_local_range().size();
-  size_t NWorkGroups = Range.get_group_range().size();
-
+reduAuxCGFuncImpl(handler &CGH, size_t NWorkItems, size_t NWorkGroups,
+                  size_t WGSize, Reduction &Redu, InputT In, OutputT Out) {
   bool IsUpdateOfUserVar =
       Reduction::accessor_mode == access::mode::read_write && NWorkGroups == 1;
 
@@ -922,9 +929,11 @@ reduAuxCGFuncImpl(handler &CGH, const nd_range<Dims> &Range, size_t NWorkItems,
   auto LocalReds = Redu.getReadWriteLocalAcc(NumLocalElements, CGH);
 
   auto ReduIdentity = Redu.getIdentity();
+  auto BOp = Redu.getBinaryOperation();
   using Name = typename get_reduction_aux_kernel_name_t<
       KernelName, KernelType, Reduction::is_usm, UniformPow2WG, OutputT>::name;
-  CGH.parallel_for<Name>(Range, [=](nd_item<Dims> NDIt) {
+  nd_range<1> Range{range<1>(NWorkItems), range<1>(WGSize)};
+  CGH.parallel_for<Name>(Range, [=](nd_item<1> NDIt) {
     size_t WGSize = NDIt.get_local_range().size();
     size_t LID = NDIt.get_local_linear_id();
     size_t GID = NDIt.get_global_linear_id();
@@ -939,7 +948,6 @@ reduAuxCGFuncImpl(handler &CGH, const nd_range<Dims> &Range, size_t NWorkItems,
     // Tree-reduction: reduce the local array LocalReds[:] to LocalReds[0]
     // LocalReds[WGSize] accumulates last/odd elements when the step
     // of tree-reduction loop is not even.
-    typename Reduction::binary_operation BOp;
     size_t PrevStep = WGSize;
     for (size_t CurStep = PrevStep >> 1; CurStep > 0; CurStep >>= 1) {
       if (LID < CurStep)
@@ -962,12 +970,22 @@ reduAuxCGFuncImpl(handler &CGH, const nd_range<Dims> &Range, size_t NWorkItems,
   });
 }
 
-template <typename KernelName, typename KernelType, int Dims, class Reduction>
-enable_if_t<!Reduction::has_fast_atomics>
-reduAuxCGFunc(handler &CGH, const nd_range<Dims> &Range, size_t NWorkItems,
+/// Implements a command group function that enqueues a kernel that does one
+/// iteration of reduction of elements in each of work-groups.
+/// At the end of each work-group the partial sum is written to a global buffer.
+/// The function returns the number of the newly generated partial sums.
+template <typename KernelName, typename KernelType, class Reduction>
+enable_if_t<!Reduction::has_fast_atomics, size_t>
+reduAuxCGFunc(handler &CGH, size_t NWorkItems, size_t MaxWGSize,
               Reduction &Redu) {
-  size_t WGSize = Range.get_local_range().size();
-  size_t NWorkGroups = Range.get_group_range().size();
+
+  size_t NWorkGroups;
+  size_t WGSize = reduComputeWGSize(NWorkItems, MaxWGSize, NWorkGroups);
+
+  // The last kernel DOES write to user's accessor passed to reduction.
+  // Associate it with handler manually.
+  if (NWorkGroups == 1 && !Reduction::is_usm)
+    Redu.associateWithHandler(CGH);
 
   // The last work-group may be not fully loaded with work, or the work group
   // size may be not power of two. Those two cases considered inefficient
@@ -981,20 +999,21 @@ reduAuxCGFunc(handler &CGH, const nd_range<Dims> &Range, size_t NWorkItems,
   auto In = Redu.getReadAccToPreviousPartialReds(CGH);
   if (Reduction::is_usm && NWorkGroups == 1) {
     if (HasUniformWG)
-      reduAuxCGFuncImpl<KernelName, KernelType, Dims, Reduction, true>(
-          CGH, Range, NWorkItems, Redu, In, Redu.getUSMPointer());
+      reduAuxCGFuncImpl<KernelName, KernelType, true>(
+          CGH, NWorkItems, NWorkGroups, WGSize, Redu, In, Redu.getUSMPointer());
     else
-      reduAuxCGFuncImpl<KernelName, KernelType, Dims, Reduction, false>(
-          CGH, Range, NWorkItems, Redu, In, Redu.getUSMPointer());
+      reduAuxCGFuncImpl<KernelName, KernelType, false>(
+          CGH, NWorkItems, NWorkGroups, WGSize, Redu, In, Redu.getUSMPointer());
   } else {
     auto Out = Redu.getWriteAccForPartialReds(NWorkGroups, CGH);
     if (HasUniformWG)
-      reduAuxCGFuncImpl<KernelName, KernelType, Dims, Reduction, true>(
-          CGH, Range, NWorkItems, Redu, In, Out);
+      reduAuxCGFuncImpl<KernelName, KernelType, true>(
+          CGH, NWorkItems, NWorkGroups, WGSize, Redu, In, Out);
     else
-      reduAuxCGFuncImpl<KernelName, KernelType, Dims, Reduction, false>(
-          CGH, Range, NWorkItems, Redu, In, Out);
+      reduAuxCGFuncImpl<KernelName, KernelType, false>(
+          CGH, NWorkItems, NWorkGroups, WGSize, Redu, In, Out);
   }
+  return NWorkGroups;
 }
 
 } // namespace detail
@@ -1007,10 +1026,10 @@ template <typename T, class BinaryOperation, int Dims, access::mode AccMode,
           access::placeholder IsPH>
 detail::reduction_impl<T, BinaryOperation, Dims, false, AccMode, IsPH>
 reduction(accessor<T, Dims, AccMode, access::target::global_buffer, IsPH> &Acc,
-          const T &Identity, BinaryOperation) {
+          const T &Identity, BinaryOperation BOp) {
   // The Combiner argument was needed only to define the BinaryOperation param.
   return detail::reduction_impl<T, BinaryOperation, Dims, false, AccMode, IsPH>(
-      Acc, Identity);
+      Acc, Identity, BOp);
 }
 
 /// Creates and returns an object implementing the reduction functionality.
@@ -1035,9 +1054,10 @@ reduction(accessor<T, Dims, AccMode, access::target::global_buffer, IsPH> &Acc,
 /// \param Identity, and the binary operation used in the reduction.
 template <typename T, class BinaryOperation>
 detail::reduction_impl<T, BinaryOperation, 0, true, access::mode::read_write>
-reduction(T *VarPtr, const T &Identity, BinaryOperation) {
+reduction(T *VarPtr, const T &Identity, BinaryOperation BOp) {
   return detail::reduction_impl<T, BinaryOperation, 0, true,
-                                access::mode::read_write>(VarPtr, Identity);
+                                access::mode::read_write>(VarPtr, Identity,
+                                                          BOp);
 }
 
 /// Creates and returns an object implementing the reduction functionality.
diff --git a/sycl/include/CL/sycl/interop_handle.hpp b/sycl/include/CL/sycl/interop_handle.hpp
index 800ab85b31eab..025d5ab0a8dcc 100644
--- a/sycl/include/CL/sycl/interop_handle.hpp
+++ b/sycl/include/CL/sycl/interop_handle.hpp
@@ -45,7 +45,8 @@ class interop_handle {
   template <backend BackendName = backend::opencl, typename DataT, int Dims,
             access::mode Mode, access::target Target, access::placeholder IsPlh>
   typename std::enable_if<
-      Target != access::target::host_buffer,
+      Target == access::target::global_buffer ||
+          Target == access::target::constant_buffer,
       typename interop<BackendName,
                        accessor<DataT, Dims, Mode, Target, IsPlh>>::type>::type
   get_native_mem(const accessor<DataT, Dims, Mode, Target, IsPlh> &Acc) const {
@@ -63,12 +64,14 @@ class interop_handle {
   template <backend BackendName = backend::opencl, typename DataT, int Dims,
             access::mode Mode, access::target Target, access::placeholder IsPlh>
   typename std::enable_if<
-      Target == access::target::host_buffer,
-      typename interop<BackendName,
-                       accessor<DataT, Dims, Mode, Target, IsPlh>>::type>::type
+      !(Target == access::target::global_buffer ||
+        Target == access::target::constant_buffer),
+      typename interop<BackendName, accessor<DataT, Dims, Mode,
+                                             access::target::global_buffer,
+                                             IsPlh>>::type>::type
   get_native_mem(const accessor<DataT, Dims, Mode, Target, IsPlh> &) const {
-    throw invalid_object_error("Getting memory object out of host accessor is "
-                               "not allowed",
+    throw invalid_object_error("Getting memory object out of accessor for "
+                               "specified target is not allowed",
                                PI_INVALID_MEM_OBJECT);
   }
 
diff --git a/sycl/include/CL/sycl/property_list.hpp b/sycl/include/CL/sycl/property_list.hpp
index 90aea2d60164a..b727bf05671c2 100644
--- a/sycl/include/CL/sycl/property_list.hpp
+++ b/sycl/include/CL/sycl/property_list.hpp
@@ -47,6 +47,11 @@ class noinit;
 
 namespace detail {
 
+// Will be aliased in the sycl::ext::oneapi::property namespace
+namespace buffer_ {
+class use_pinned_host_memory;
+}
+
 // List of all properties' IDs.
 enum PropKind {
   // Buffer properties
@@ -66,6 +71,8 @@ enum PropKind {
   // Accessor
   NoInit,
 
+  BufferUsePinnedHostMemory,
+
   PropKindSize
 };
 
@@ -148,6 +155,8 @@ RegisterProp(PropKind::ImageContextBound, image::context_bound);
 RegisterProp(PropKind::BufferUseHostPtr, buffer::use_host_ptr);
 RegisterProp(PropKind::BufferUseMutex, buffer::use_mutex);
 RegisterProp(PropKind::BufferContextBound, buffer::context_bound);
+RegisterProp(PropKind::BufferUsePinnedHostMemory,
+             buffer_::use_pinned_host_memory);
 
 // Queue
 RegisterProp(PropKind::QueueEnableProfiling, queue::enable_profiling);
@@ -212,8 +221,16 @@ class context_bound
 public:
   context_bound(cl::sycl::context Context) : ContextBoundBase(Context) {}
 };
+
 } // namespace buffer
 
+namespace detail {
+namespace buffer_ {
+class use_pinned_host_memory
+    : public detail::Prop<detail::PropKind::BufferUsePinnedHostMemory> {};
+} // namespace buffer_
+} // namespace detail
+
 namespace queue {
 class enable_profiling
     : public detail::Prop<detail::PropKind::QueueEnableProfiling> {};
@@ -225,6 +242,17 @@ class noinit : public detail::Prop<detail::PropKind::NoInit> {};
 
 } // namespace property
 
+namespace ext {
+namespace oneapi {
+namespace property {
+namespace buffer {
+using use_pinned_host_memory =
+    sycl::property::detail::buffer_::use_pinned_host_memory;
+} // namespace buffer
+} // namespace property
+} // namespace oneapi
+} // namespace ext
+
 #if __cplusplus > 201402L
 
 inline constexpr property::noinit noinit;
@@ -280,17 +308,18 @@ class property_list {
   }
 
   template <typename propertyT> propertyT get_property() const {
-    static_assert((int)(propertyT::getKind()) <=
-                      property::detail::PropKind::PropKindSize,
-                  "Invalid option passed.");
-    const auto &PropHolder = std::get<(int)(propertyT::getKind())>(m_PropsList);
-    if (PropHolder.isInitialized()) {
-      return PropHolder.getProp();
+    if (!has_property<propertyT>()) {
+      throw sycl::invalid_object_error();
     }
-    throw invalid_object_error();
+    const auto &PropHolder =
+        std::get<static_cast<int>(propertyT::getKind())>(m_PropsList);
+    return PropHolder.getProp();
   }
 
   template <typename propertyT> bool has_property() const {
+    if (static_cast<int>(propertyT::getKind()) >
+        property::detail::PropKind::PropKindSize)
+      return false;
     return std::get<(int)(propertyT::getKind())>(m_PropsList).isInitialized();
   }
 
@@ -299,7 +328,7 @@ class property_list {
 
   template <typename... propertyTN, class PropT>
   void ctorHelper(PropT &Prop, propertyTN... props) {
-    std::get<(int)(PropT::getKind())>(m_PropsList).setProp(Prop);
+    std::get<static_cast<int>(PropT::getKind())>(m_PropsList).setProp(Prop);
     ctorHelper(props...);
   }
 
diff --git a/sycl/include/CL/sycl/sampler.hpp b/sycl/include/CL/sycl/sampler.hpp
index 5843b0a47bc33..edbd484a32b2f 100644
--- a/sycl/include/CL/sycl/sampler.hpp
+++ b/sycl/include/CL/sycl/sampler.hpp
@@ -12,7 +12,6 @@
 #include <CL/sycl/access/access.hpp>
 #include <CL/sycl/detail/common.hpp>
 #include <CL/sycl/detail/export.hpp>
-#include <CL/sycl/detail/sampler_impl.hpp>
 
 __SYCL_INLINE_NAMESPACE(cl) {
 namespace sycl {
@@ -40,6 +39,23 @@ template <typename DataT, int Dimensions, access::mode AccessMode,
 class image_accessor;
 }
 
+namespace detail {
+#ifdef __SYCL_DEVICE_ONLY__
+class __SYCL_EXPORT sampler_impl {
+public:
+  sampler_impl() = default;
+
+  sampler_impl(__ocl_sampler_t Sampler) : m_Sampler(Sampler) {}
+
+  ~sampler_impl() = default;
+
+  __ocl_sampler_t m_Sampler;
+};
+#else
+class sampler_impl;
+#endif
+} // namespace detail
+
 /// Encapsulates a configuration for sampling an image accessor.
 ///
 /// \sa sycl_api_acc
diff --git a/sycl/include/CL/sycl/stream.hpp b/sycl/include/CL/sycl/stream.hpp
index 58f038980545d..20cdef064b2ef 100644
--- a/sycl/include/CL/sycl/stream.hpp
+++ b/sycl/include/CL/sycl/stream.hpp
@@ -8,12 +8,613 @@
 
 #pragma once
 
+#include <CL/sycl/builtins.hpp>
+#include <CL/sycl/detail/defines.hpp>
 #include <CL/sycl/detail/export.hpp>
-#include <CL/sycl/detail/stream_impl.hpp>
+#include <CL/sycl/handler.hpp>
 
 __SYCL_INLINE_NAMESPACE(cl) {
 namespace sycl {
 
+namespace detail {
+
+using FmtFlags = unsigned int;
+
+// Mapping from stream_manipulator to FmtFlags. Each manipulator corresponds
+// to the bit in FmtFlags.
+static constexpr FmtFlags Dec = 0x0001;
+static constexpr FmtFlags Hex = 0x0002;
+static constexpr FmtFlags Oct = 0x0004;
+static constexpr FmtFlags ShowBase = 0x0008;
+static constexpr FmtFlags ShowPos = 0x0010;
+static constexpr FmtFlags Fixed = 0x0020;
+static constexpr FmtFlags Scientific = 0x0040;
+
+// Bitmask made of the combination of the base flags. Base flags are mutually
+// exclusive, this mask is used to clean base field before setting the new
+// base flag.
+static constexpr FmtFlags BaseField = Dec | Hex | Oct;
+
+// Bitmask made of the combination of the floating point value format flags.
+// Thease flags are mutually exclusive, this mask is used to clean float field
+// before setting the new float flag.
+static constexpr FmtFlags FloatField = Scientific | Fixed;
+
+constexpr size_t MAX_FLOATING_POINT_DIGITS = 24;
+constexpr size_t MAX_INTEGRAL_DIGITS = 23;
+constexpr const char *VEC_ELEMENT_DELIMITER = ", ";
+constexpr char VEC_OPEN_BRACE = '{';
+constexpr char VEC_CLOSE_BRACE = '}';
+
+constexpr size_t MAX_DIMENSIONS = 3;
+
+// Space for integrals (up to 3), comma and space between the
+// integrals and enclosing braces.
+constexpr size_t MAX_ARRAY_SIZE =
+    MAX_INTEGRAL_DIGITS * MAX_DIMENSIONS + 2 * (MAX_DIMENSIONS - 1) + 2;
+
+template <class F, class T = void>
+using EnableIfFP = typename std::enable_if<std::is_same<F, float>::value ||
+                                               std::is_same<F, double>::value ||
+                                               std::is_same<F, half>::value,
+                                           T>::type;
+
+using GlobalBufAccessorT = accessor<char, 1, cl::sycl::access::mode::read_write,
+                                    cl::sycl::access::target::global_buffer,
+                                    cl::sycl::access::placeholder::false_t>;
+
+using GlobalOffsetAccessorT =
+    accessor<unsigned, 1, cl::sycl::access::mode::atomic,
+             cl::sycl::access::target::global_buffer,
+             cl::sycl::access::placeholder::false_t>;
+
+inline void write(GlobalBufAccessorT &GlobalFlushBuf, size_t FlushBufferSize,
+                  unsigned WIOffset, unsigned &Offset, const char *Str,
+                  unsigned Len, unsigned Padding = 0) {
+  if ((FlushBufferSize - Offset < Len + Padding) ||
+      (WIOffset + Offset + Len + Padding > GlobalFlushBuf.get_count()))
+    // TODO: flush here
+    return;
+
+  // Write padding
+  for (size_t I = 0; I < Padding; ++I, ++Offset)
+    GlobalFlushBuf[WIOffset + Offset] = ' ';
+
+  for (size_t I = 0; I < Len; ++I, ++Offset) {
+    GlobalFlushBuf[WIOffset + Offset] = Str[I];
+  }
+}
+
+inline void reverseBuf(char *Buf, unsigned Len) {
+  int I = Len - 1;
+  int J = 0;
+  while (I > J) {
+    int Temp = Buf[I];
+    Buf[I] = Buf[J];
+    Buf[J] = Temp;
+    I--;
+    J++;
+  }
+}
+
+template <typename T>
+inline typename std::make_unsigned<T>::type getAbsVal(const T Val,
+                                                      const int Base) {
+  return ((Base == 10) && (Val < 0)) ? -Val : Val;
+}
+
+inline char digitToChar(const int Digit) {
+  if (Digit < 10) {
+    return '0' + Digit;
+  } else {
+    return 'a' + Digit - 10;
+  }
+}
+
+template <typename T>
+inline typename std::enable_if<std::is_integral<T>::value, unsigned>::type
+integralToBase(T Val, int Base, char *Digits) {
+  unsigned NumDigits = 0;
+
+  do {
+    Digits[NumDigits++] = digitToChar(Val % Base);
+    Val /= Base;
+  } while (Val);
+
+  return NumDigits;
+}
+
+// Returns number of symbols written to the buffer
+template <typename T>
+inline typename std::enable_if<std::is_integral<T>::value, unsigned>::type
+ScalarToStr(const T &Val, char *Buf, unsigned Flags, int, int Precision = -1) {
+  (void)Precision;
+  int Base = 10;
+
+  // append base manipulator
+  switch (Flags & BaseField) {
+  case Dec:
+    Base = 10;
+    break;
+  case Hex:
+    Base = 16;
+    break;
+  case Oct:
+    Base = 8;
+    break;
+  default:
+    // default value is 10
+    break;
+  }
+
+  unsigned Offset = 0;
+
+  // write '+' to the stream if the base is 10 and the value is non-negative
+  // or write '-' to stream if base is 10 and the value is negative
+  if (Base == 10) {
+    if ((Flags & ShowPos) && Val >= 0)
+      Buf[Offset++] = '+';
+    else if (Val < 0)
+      Buf[Offset++] = '-';
+  }
+
+  // write 0 or 0x to the stream if base is not 10 and the manipulator is set
+  if (Base != 10 && (Flags & ShowBase)) {
+    Buf[Offset++] = '0';
+    if (Base == 16)
+      Buf[Offset++] = 'x';
+  }
+
+  auto AbsVal = getAbsVal(Val, Base);
+
+  const unsigned NumBuf = integralToBase(AbsVal, Base, Buf + Offset);
+
+  reverseBuf(Buf + Offset, NumBuf);
+  return Offset + NumBuf;
+}
+
+inline unsigned append(char *Dst, const char *Src) {
+  unsigned Len = 0;
+  for (; Src[Len] != '\0'; ++Len)
+    ;
+
+  for (unsigned I = 0; I < Len; ++I)
+    Dst[I] = Src[I];
+  return Len;
+}
+
+template <typename T>
+inline typename std::enable_if<std::is_same<T, float>::value ||
+                                   std::is_same<T, double>::value,
+                               unsigned>::type
+checkForInfNan(char *Buf, T Val) {
+  if (isnan(Val))
+    return append(Buf, "nan");
+  if (isinf(Val)) {
+    if (signbit(Val))
+      return append(Buf, "-inf");
+    return append(Buf, "inf");
+  }
+  return 0;
+}
+
+template <typename T>
+inline typename std::enable_if<std::is_same<T, half>::value, unsigned>::type
+checkForInfNan(char *Buf, T Val) {
+  if (Val != Val)
+    return append(Buf, "nan");
+
+  // Extract the sign from the bits
+  const uint16_t Sign = reinterpret_cast<uint16_t &>(Val) & 0x8000;
+  // Extract the exponent from the bits
+  const uint16_t Exp16 = (reinterpret_cast<uint16_t &>(Val) & 0x7c00) >> 10;
+
+  if (Exp16 == 0x1f) {
+    if (Sign)
+      return append(Buf, "-inf");
+    return append(Buf, "inf");
+  }
+  return 0;
+}
+
+template <typename T>
+EnableIfFP<T, unsigned> floatingPointToDecStr(T AbsVal, char *Digits,
+                                              int Precision, bool IsSci) {
+  int Exp = 0;
+
+  // For the case that the value is larger than 10.0
+  while (AbsVal >= 10.0) {
+    ++Exp;
+    AbsVal /= 10.0;
+  }
+  // For the case that the value is less than 1.0
+  while (AbsVal > 0.0 && AbsVal < 1.0) {
+    --Exp;
+    AbsVal *= 10.0;
+  }
+
+  auto IntegralPart = static_cast<int>(AbsVal);
+  auto FractionPart = AbsVal - IntegralPart;
+
+  int FractionDigits[MAX_FLOATING_POINT_DIGITS] = {0};
+
+  // Exponent
+  int P = Precision > 0 ? Precision : 4;
+  size_t FractionLength = Exp + P;
+
+  // After normalization integral part contains 1 symbol, also there could be
+  // '.', 'e', sign of the exponent and sign of the number, overall 5 symbols.
+  // So, clamp fraction length if required according to maximum size of the
+  // buffer for floating point number.
+  if (FractionLength > MAX_FLOATING_POINT_DIGITS - 5)
+    FractionLength = MAX_FLOATING_POINT_DIGITS - 5;
+
+  for (unsigned I = 0; I < FractionLength; ++I) {
+    FractionPart *= 10.0;
+    FractionDigits[I] = static_cast<int>(FractionPart);
+    FractionPart -= static_cast<int>(FractionPart);
+  }
+
+  int Carry = FractionPart > static_cast<T>(0.5) ? 1 : 0;
+
+  // Propagate the Carry
+  for (int I = FractionLength - 1; I >= 0 && Carry; --I) {
+    auto Digit = FractionDigits[I] + Carry;
+    FractionDigits[I] = Digit % 10;
+    Carry = Digit / 10;
+  }
+
+  // Carry from the fraction part is propagated to integral part
+  IntegralPart += Carry;
+  if (IntegralPart == 10) {
+    IntegralPart = 1;
+    ++Exp;
+  }
+
+  unsigned Offset = 0;
+
+  // Assemble the final string correspondingly
+  if (IsSci) { // scientific mode
+    // Append the integral part
+    Digits[Offset++] = digitToChar(IntegralPart);
+    Digits[Offset++] = '.';
+
+    // Append all fraction
+    for (unsigned I = 0; I < FractionLength; ++I)
+      Digits[Offset++] = digitToChar(FractionDigits[I]);
+
+    // Exponent part
+    Digits[Offset++] = 'e';
+    Digits[Offset++] = Exp >= 0 ? '+' : '-';
+    Digits[Offset++] = digitToChar(abs(Exp) / 10);
+    Digits[Offset++] = digitToChar(abs(Exp) % 10);
+  } else { // normal mode
+    if (Exp < 0) {
+      Digits[Offset++] = '0';
+      Digits[Offset++] = '.';
+      while (++Exp)
+        Digits[Offset++] = '0';
+
+      // Append the integral part
+      Digits[Offset++] = digitToChar(IntegralPart);
+
+      // Append all fraction
+      for (unsigned I = 0; I < FractionLength; ++I)
+        Digits[Offset++] = digitToChar(FractionDigits[I]);
+    } else {
+      // Append the integral part
+      Digits[Offset++] = digitToChar(IntegralPart);
+      unsigned I = 0;
+      // Append the integral part first
+      for (; I < FractionLength && Exp--; ++I)
+        Digits[Offset++] = digitToChar(FractionDigits[I]);
+
+      // Put the dot
+      Digits[Offset++] = '.';
+
+      // Append the rest of fraction part, or the real fraction part
+      for (; I < FractionLength; ++I)
+        Digits[Offset++] = digitToChar(FractionDigits[I]);
+    }
+    // The normal mode requires no tailing zero digit, then we need to first
+    // find the first non-zero digit
+    while (Digits[Offset - 1] == '0')
+      Offset--;
+
+    // If dot is the last digit, it should be stripped off as well
+    if (Digits[Offset - 1] == '.')
+      Offset--;
+  }
+  return Offset;
+}
+
+// Returns number of symbols written to the buffer
+template <typename T>
+inline EnableIfFP<T, unsigned>
+ScalarToStr(const T &Val, char *Buf, unsigned Flags, int, int Precision = -1) {
+  unsigned Offset = checkForInfNan(Buf, Val);
+  if (Offset)
+    return Offset;
+
+  T Neg = -Val;
+  auto AbsVal = Val < 0 ? Neg : Val;
+
+  if (Val < 0) {
+    Buf[Offset++] = '-';
+  } else if (Flags & ShowPos) {
+    Buf[Offset++] = '+';
+  }
+
+  bool IsSci = false;
+  if (Flags & detail::Scientific)
+    IsSci = true;
+
+  // TODO: manipulators for floating-point output - hexfloat, fixed
+  Offset += floatingPointToDecStr(AbsVal, Buf + Offset, Precision, IsSci);
+
+  return Offset;
+}
+
+template <typename T>
+inline typename std::enable_if<std::is_integral<T>::value>::type
+writeIntegral(GlobalBufAccessorT &GlobalFlushBuf, size_t FlushBufferSize,
+              unsigned WIOffset, unsigned &Offset, unsigned Flags, int Width,
+              const T &Val) {
+  char Digits[MAX_INTEGRAL_DIGITS] = {0};
+  unsigned Len = ScalarToStr(Val, Digits, Flags, Width);
+  write(GlobalFlushBuf, FlushBufferSize, WIOffset, Offset, Digits, Len,
+        (Width > 0 && static_cast<unsigned>(Width) > Len)
+            ? static_cast<unsigned>(Width) - Len
+            : 0);
+}
+
+template <typename T>
+inline EnableIfFP<T>
+writeFloatingPoint(GlobalBufAccessorT &GlobalFlushBuf, size_t FlushBufferSize,
+                   unsigned WIOffset, unsigned &Offset, unsigned Flags,
+                   int Width, int Precision, const T &Val) {
+  char Digits[MAX_FLOATING_POINT_DIGITS] = {0};
+  unsigned Len = ScalarToStr(Val, Digits, Flags, Width, Precision);
+  write(GlobalFlushBuf, FlushBufferSize, WIOffset, Offset, Digits, Len,
+        (Width > 0 && static_cast<unsigned>(Width) > Len)
+            ? static_cast<unsigned>(Width) - Len
+            : 0);
+}
+
+// Helper method to update offset in the global buffer atomically according to
+// the provided size of the data in the flush buffer. Return true if offset is
+// updated and false in case of overflow.
+inline bool updateOffset(GlobalOffsetAccessorT &GlobalOffset,
+                         GlobalBufAccessorT &GlobalBuf, unsigned Size,
+                         unsigned &Cur) {
+  unsigned New;
+  Cur = GlobalOffset[0].load();
+  do {
+    if (GlobalBuf.get_range().size() - Cur < Size)
+      // Overflow
+      return false;
+    New = Cur + Size;
+  } while (!GlobalOffset[0].compare_exchange_strong(Cur, New));
+  return true;
+}
+
+inline void flushBuffer(GlobalOffsetAccessorT &GlobalOffset,
+                        GlobalBufAccessorT &GlobalBuf,
+                        GlobalBufAccessorT &GlobalFlushBuf, unsigned &WIOffset,
+                        unsigned &Offset) {
+
+  unsigned Cur = 0;
+  if (!updateOffset(GlobalOffset, GlobalBuf, Offset, Cur))
+    return;
+
+  for (unsigned I = WIOffset; I < WIOffset + Offset; I++) {
+    GlobalBuf[Cur++] = GlobalFlushBuf[I];
+  }
+  // Reset the offset in the flush buffer
+  Offset = 0;
+}
+
+template <typename T, int VecLength>
+typename std::enable_if<(VecLength == 1), unsigned>::type
+VecToStr(const vec<T, VecLength> &Vec, char *VecStr, unsigned Flags, int Width,
+         int Precision) {
+  return ScalarToStr(static_cast<T>(Vec.x()), VecStr, Flags, Width, Precision);
+}
+
+template <typename T, int VecLength>
+typename std::enable_if<(VecLength == 2 || VecLength == 4 || VecLength == 8 ||
+                         VecLength == 16),
+                        unsigned>::type
+VecToStr(const vec<T, VecLength> &Vec, char *VecStr, unsigned Flags, int Width,
+         int Precision) {
+  unsigned Len =
+      VecToStr<T, VecLength / 2>(Vec.lo(), VecStr, Flags, Width, Precision);
+  Len += append(VecStr + Len, VEC_ELEMENT_DELIMITER);
+  Len += VecToStr<T, VecLength / 2>(Vec.hi(), VecStr + Len, Flags, Width,
+                                    Precision);
+  return Len;
+}
+
+template <typename T, int VecLength>
+typename std::enable_if<(VecLength == 3), unsigned>::type
+VecToStr(const vec<T, VecLength> &Vec, char *VecStr, unsigned Flags, int Width,
+         int Precision) {
+  unsigned Len = VecToStr<T, 2>(Vec.lo(), VecStr, Flags, Width, Precision);
+  Len += append(VecStr + Len, VEC_ELEMENT_DELIMITER);
+  Len += VecToStr<T, 1>(Vec.z(), VecStr + Len, Flags, Width, Precision);
+  return Len;
+}
+
+template <typename T, int VecLength>
+inline void writeVec(GlobalBufAccessorT &GlobalFlushBuf, size_t FlushBufferSize,
+                     unsigned WIOffset, unsigned &Offset, unsigned Flags,
+                     int Width, int Precision, const vec<T, VecLength> &Vec) {
+  // Reserve space for vector elements and delimiters
+  constexpr size_t MAX_VEC_SIZE =
+      MAX_FLOATING_POINT_DIGITS * VecLength + (VecLength - 1) * 2;
+  char VecStr[MAX_VEC_SIZE] = {0};
+  unsigned Len = VecToStr<T, VecLength>(Vec, VecStr, Flags, Width, Precision);
+  write(GlobalFlushBuf, FlushBufferSize, WIOffset, Offset, VecStr, Len,
+        (Width > 0 && Width > Len) ? Width - Len : 0);
+}
+
+template <int ArrayLength>
+inline unsigned ArrayToStr(char *Buf, const array<ArrayLength> &Arr) {
+  unsigned Len = 0;
+  Buf[Len++] = VEC_OPEN_BRACE;
+
+  for (int I = 0; I < ArrayLength; ++I) {
+    Len += ScalarToStr(Arr[I], Buf + Len, 0 /* No flags */, -1, -1);
+    if (I != ArrayLength - 1)
+      Len += append(Buf + Len, VEC_ELEMENT_DELIMITER);
+  }
+
+  Buf[Len++] = VEC_CLOSE_BRACE;
+
+  return Len;
+}
+
+template <int ArrayLength>
+inline void writeArray(GlobalBufAccessorT &GlobalFlushBuf,
+                       size_t FlushBufferSize, unsigned WIOffset,
+                       unsigned &Offset, const array<ArrayLength> &Arr) {
+  char Buf[MAX_ARRAY_SIZE];
+  unsigned Len = ArrayToStr(Buf, Arr);
+  write(GlobalFlushBuf, FlushBufferSize, WIOffset, Offset, Buf, Len);
+}
+
+template <int Dimensions>
+inline void writeItem(GlobalBufAccessorT &GlobalFlushBuf,
+                      size_t FlushBufferSize, unsigned WIOffset,
+                      unsigned &Offset, const item<Dimensions> &Item) {
+  // Reserve space for 3 arrays and additional place (40 symbols) for printing
+  // the text
+  char Buf[3 * MAX_ARRAY_SIZE + 40];
+  unsigned Len = 0;
+  Len += append(Buf, "item(");
+  Len += append(Buf + Len, "range: ");
+  Len += ArrayToStr(Buf + Len, Item.get_range());
+  Len += append(Buf + Len, ", id: ");
+  Len += ArrayToStr(Buf + Len, Item.get_id());
+  Len += append(Buf + Len, ", offset: ");
+  Len += ArrayToStr(Buf + Len, Item.get_offset());
+  Buf[Len++] = ')';
+  write(GlobalFlushBuf, FlushBufferSize, WIOffset, Offset, Buf, Len);
+}
+
+template <int Dimensions>
+inline void writeNDRange(GlobalBufAccessorT &GlobalFlushBuf,
+                         size_t FlushBufferSize, unsigned WIOffset,
+                         unsigned &Offset,
+                         const nd_range<Dimensions> &ND_Range) {
+  // Reserve space for 3 arrays and additional place (50 symbols) for printing
+  // the text
+  char Buf[3 * MAX_ARRAY_SIZE + 50];
+  unsigned Len = 0;
+  Len += append(Buf, "nd_range(");
+  Len += append(Buf + Len, "global_range: ");
+  Len += ArrayToStr(Buf + Len, ND_Range.get_global_range());
+  Len += append(Buf + Len, ", local_range: ");
+  Len += ArrayToStr(Buf + Len, ND_Range.get_local_range());
+  Len += append(Buf + Len, ", offset: ");
+  Len += ArrayToStr(Buf + Len, ND_Range.get_offset());
+  Buf[Len++] = ')';
+  write(GlobalFlushBuf, FlushBufferSize, WIOffset, Offset, Buf, Len);
+}
+
+template <int Dimensions>
+inline void writeNDItem(GlobalBufAccessorT &GlobalFlushBuf,
+                        size_t FlushBufferSize, unsigned WIOffset,
+                        unsigned &Offset, const nd_item<Dimensions> &ND_Item) {
+  // Reserve space for 2 arrays and additional place (40 symbols) for printing
+  // the text
+  char Buf[2 * MAX_ARRAY_SIZE + 40];
+  unsigned Len = 0;
+  Len += append(Buf, "nd_item(");
+  Len += append(Buf + Len, "global_id: ");
+  Len += ArrayToStr(Buf + Len, ND_Item.get_global_id());
+  Len += append(Buf + Len, ", local_id: ");
+  Len += ArrayToStr(Buf + Len, ND_Item.get_local_id());
+  Buf[Len++] = ')';
+  write(GlobalFlushBuf, FlushBufferSize, WIOffset, Offset, Buf, Len);
+}
+
+template <int Dimensions>
+inline void writeGroup(GlobalBufAccessorT &GlobalFlushBuf,
+                       size_t FlushBufferSize, unsigned WIOffset,
+                       unsigned &Offset, const group<Dimensions> &Group) {
+  // Reserve space for 4 arrays and additional place (60 symbols) for printing
+  // the text
+  char Buf[4 * MAX_ARRAY_SIZE + 60];
+  unsigned Len = 0;
+  Len += append(Buf, "group(");
+  Len += append(Buf + Len, "id: ");
+  Len += ArrayToStr(Buf + Len, Group.get_id());
+  Len += append(Buf + Len, ", global_range: ");
+  Len += ArrayToStr(Buf + Len, Group.get_global_range());
+  Len += append(Buf + Len, ", local_range: ");
+  Len += ArrayToStr(Buf + Len, Group.get_local_range());
+  Len += append(Buf + Len, ", group_range: ");
+  Len += ArrayToStr(Buf + Len, Group.get_group_range());
+  Buf[Len++] = ')';
+  write(GlobalFlushBuf, FlushBufferSize, WIOffset, Offset, Buf, Len);
+}
+
+// Space for 2 arrays and additional place (20 symbols) for printing
+// the text
+constexpr size_t MAX_ITEM_SIZE = 2 * MAX_ARRAY_SIZE + 20;
+
+template <int Dimensions>
+inline unsigned ItemToStr(char *Buf, const item<Dimensions, false> &Item) {
+  unsigned Len = 0;
+  Len += append(Buf, "item(");
+  for (int I = 0; I < 2; ++I) {
+    Len += append(Buf + Len, I == 0 ? "range: " : ", id: ");
+    Len += ArrayToStr(Buf + Len, I == 0 ? Item.get_range() : Item.get_id());
+  }
+  Buf[Len++] = ')';
+  return Len;
+}
+
+template <int Dimensions>
+inline void writeHItem(GlobalBufAccessorT &GlobalFlushBuf,
+                       size_t FlushBufferSize, unsigned WIOffset,
+                       unsigned &Offset, const h_item<Dimensions> &HItem) {
+  // Reserve space for 3 items and additional place (60 symbols) for printing
+  // the text
+  char Buf[3 * MAX_ITEM_SIZE + 60];
+  unsigned Len = 0;
+  Len += append(Buf, "h_item(");
+  for (int I = 0; I < 3; ++I) {
+    Len += append(Buf + Len, I == 0 ? "\n  global "
+                                    : I == 1 ? "\n  logical local "
+                                             : "\n  physical local ");
+    Len += ItemToStr(Buf + Len, I == 0 ? HItem.get_global()
+                                       : I == 1 ? HItem.get_logical_local()
+                                                : HItem.get_physical_local());
+  }
+  Len += append(Buf + Len, "\n)");
+  write(GlobalFlushBuf, FlushBufferSize, WIOffset, Offset, Buf, Len);
+}
+
+template <typename> struct IsSwizzleOp : std::false_type {};
+
+template <typename VecT, typename OperationLeftT, typename OperationRightT,
+          template <typename> class OperationCurrentT, int... Indexes>
+struct IsSwizzleOp<cl::sycl::detail::SwizzleOp<
+    VecT, OperationLeftT, OperationRightT, OperationCurrentT, Indexes...>>
+    : std::true_type {
+  using T = typename VecT::element_type;
+  using Type = typename cl::sycl::vec<T, (sizeof...(Indexes))>;
+};
+
+template <typename T>
+using EnableIfSwizzleVec =
+    typename std::enable_if<IsSwizzleOp<T>::value,
+                            typename IsSwizzleOp<T>::Type>::type;
+
+} // namespace detail
+
 enum class stream_manipulator {
   dec,
   hex,
@@ -123,16 +724,16 @@ class __SYCL_EXPORT stream {
 
   // Accessor to the global stream buffer. Global buffer contains all output
   // from the kernel.
-  mutable detail::stream_impl::GlobalBufAccessorT GlobalBuf;
+  mutable detail::GlobalBufAccessorT GlobalBuf;
 
   // Atomic accessor to the global offset variable. It represents an offset in
   // the global stream buffer. Since work items will flush data to global buffer
   // in parallel we need atomic access to this offset.
-  mutable detail::stream_impl::GlobalOffsetAccessorT GlobalOffset;
+  mutable detail::GlobalOffsetAccessorT GlobalOffset;
 
   // Accessor to the flush buffer. Each work item writes its
   // output to a designated section of the flush buffer.
-  mutable detail::stream_impl::GlobalBufAccessorT GlobalFlushBuf;
+  mutable detail::GlobalBufAccessorT GlobalFlushBuf;
 
   // Offset of the WI's flush buffer in the pool.
   mutable unsigned WIOffset = 0;
diff --git a/sycl/include/CL/sycl/usm/usm_allocator.hpp b/sycl/include/CL/sycl/usm/usm_allocator.hpp
index f821175205c03..3270fa1d6d67e 100644
--- a/sycl/include/CL/sycl/usm/usm_allocator.hpp
+++ b/sycl/include/CL/sycl/usm/usm_allocator.hpp
@@ -26,112 +26,45 @@ __SYCL_EXPORT void *aligned_alloc(size_t alignment, size_t size,
                                   usm::alloc kind);
 __SYCL_EXPORT void free(void *ptr, const context &ctxt);
 
-template <typename T, usm::alloc AllocKind, size_t Alignment = 0>
+template <typename T, usm::alloc AllocKind, size_t Alignment = alignof(T)>
 class usm_allocator {
 public:
   using value_type = T;
-  using pointer = T *;
-  using const_pointer = const T *;
-  using reference = T &;
-  using const_reference = const T &;
+  using propagate_on_container_copy_assignment = std::true_type;
+  using propagate_on_container_move_assignment = std::true_type;
+  using propagate_on_container_swap = std::true_type;
 
 public:
   template <typename U> struct rebind {
     typedef usm_allocator<U, AllocKind, Alignment> other;
   };
 
-  usm_allocator() = delete;
-  usm_allocator(const context &Ctxt, const device &Dev)
+  static_assert(
+      AllocKind != usm::alloc::device,
+      "usm_allocator does not support AllocKind == usm::alloc::device");
+
+  usm_allocator() noexcept = delete;
+  usm_allocator(const context &Ctxt, const device &Dev) noexcept
       : MContext(Ctxt), MDevice(Dev) {}
-  usm_allocator(const queue &Q)
+  usm_allocator(const queue &Q) noexcept
       : MContext(Q.get_context()), MDevice(Q.get_device()) {}
-  usm_allocator(const usm_allocator &Other)
-      : MContext(Other.MContext), MDevice(Other.MDevice) {}
-
-  /// Constructs an object on memory pointed by Ptr.
-  ///
-  /// Note: AllocKind == alloc::device is not allowed.
-  ///
-  /// \param Ptr is a pointer to memory that will be used to construct the
-  /// object.
-  /// \param Val is a value to initialize the newly constructed object.
-  template <
-      usm::alloc AllocT = AllocKind,
-      typename std::enable_if<AllocT != usm::alloc::device, int>::type = 0>
-  void construct(pointer Ptr, const_reference Val) {
-    new (Ptr) value_type(Val);
-  }
-
-  template <
-      usm::alloc AllocT = AllocKind,
-      typename std::enable_if<AllocT == usm::alloc::device, int>::type = 0>
-  void construct(pointer, const_reference) {
-    throw feature_not_supported(
-        "Device pointers do not support construct on host",
-        PI_INVALID_OPERATION);
-  }
+  usm_allocator(const usm_allocator &) noexcept = default;
+  usm_allocator(usm_allocator &&) noexcept = default;
+  usm_allocator &operator=(const usm_allocator &) = delete;
+  usm_allocator &operator=(usm_allocator &&) = default;
 
-  /// Destroys an object.
-  ///
-  /// Note:: AllocKind == alloc::device is not allowed
-  ///
-  /// \param Ptr is a pointer to memory where the object resides.
-  template <
-      usm::alloc AllocT = AllocKind,
-      typename std::enable_if<AllocT != usm::alloc::device, int>::type = 0>
-  void destroy(pointer Ptr) {
-    Ptr->~value_type();
-  }
-
-  template <
-      usm::alloc AllocT = AllocKind,
-      typename std::enable_if<AllocT == usm::alloc::device, int>::type = 0>
-  void destroy(pointer) {
-    // This method must be a NOP for device pointers.
-  }
-
-  /// Note:: AllocKind == alloc::device is not allowed.
-  ///
-  /// \param Val is a reference to object.
-  /// \return an address of the object referenced by Val.
-  template <
-      usm::alloc AllocT = AllocKind,
-      typename std::enable_if<AllocT != usm::alloc::device, int>::type = 0>
-  pointer address(reference Val) const {
-    return &Val;
-  }
-
-  template <
-      usm::alloc AllocT = AllocKind,
-      typename std::enable_if<AllocT == usm::alloc::device, int>::type = 0>
-  pointer address(reference) const {
-    throw feature_not_supported(
-        "Device pointers do not support address on host", PI_INVALID_OPERATION);
-  }
-
-  template <
-      usm::alloc AllocT = AllocKind,
-      typename std::enable_if<AllocT != usm::alloc::device, int>::type = 0>
-  const_pointer address(const_reference Val) const {
-    return &Val;
-  }
-
-  template <
-      usm::alloc AllocT = AllocKind,
-      typename std::enable_if<AllocT == usm::alloc::device, int>::type = 0>
-  const_pointer address(const_reference) const {
-    throw feature_not_supported(
-        "Device pointers do not support address on host", PI_INVALID_OPERATION);
-  }
+  template <class U>
+  usm_allocator(const usm_allocator<U, AllocKind, Alignment> &Other) noexcept
+      : MContext(Other.MContext), MDevice(Other.MDevice) {}
 
   /// Allocates memory.
   ///
   /// \param NumberOfElements is a count of elements to allocate memory for.
-  pointer allocate(size_t NumberOfElements) {
+  T *allocate(size_t NumberOfElements) {
 
-    auto Result = reinterpret_cast<pointer>(
+    auto Result = reinterpret_cast<T *>(
         aligned_alloc(getAlignment(), NumberOfElements * sizeof(value_type),
-                                 MDevice, MContext, AllocKind));
+                      MDevice, MContext, AllocKind));
     if (!Result) {
       throw memory_allocation_error();
     }
@@ -142,24 +75,32 @@ class usm_allocator {
   ///
   /// \param Ptr is a pointer to memory being deallocated.
   /// \param Size is a number of elements previously passed to allocate.
-  void deallocate(pointer Ptr, size_t) {
+  void deallocate(T *Ptr, size_t) {
     if (Ptr) {
       free(Ptr, MContext);
     }
   }
 
-private:
-  constexpr size_t getAlignment() const {
-    /*
-      // This form might be preferable if the underlying implementation
-      // doesn't do the right thing when given 0 for alignment
-    return ((Alignment == 0)
-            ? alignof(value_type)
-            : Alignment);
-    */
-    return Alignment;
+  template <class U, usm::alloc AllocKindU, size_t AlignmentU>
+  friend bool operator==(const usm_allocator<T, AllocKind, Alignment> &One,
+                         const usm_allocator<U, AllocKindU, AlignmentU> &Two) {
+    return ((AllocKind == AllocKindU) && (One.MContext == Two.MContext) &&
+            (One.MDevice == Two.MDevice));
+  }
+
+  template <class U, usm::alloc AllocKindU, size_t AlignmentU>
+  friend bool operator!=(const usm_allocator<T, AllocKind, Alignment> &One,
+                         const usm_allocator<U, AllocKindU, AlignmentU> &Two) {
+    return !((AllocKind == AllocKindU) && (One.MContext == Two.MContext) &&
+             (One.MDevice == Two.MDevice));
   }
 
+private:
+  constexpr size_t getAlignment() const { return Alignment; }
+
+  template <class U, usm::alloc AllocKindU, size_t AlignmentU>
+  friend class usm_allocator;
+
   const context MContext;
   const device MDevice;
 };
diff --git a/sycl/plugins/level_zero/CMakeLists.txt b/sycl/plugins/level_zero/CMakeLists.txt
index bc75ae3ecee9c..a90dfc8ac841f 100755
--- a/sycl/plugins/level_zero/CMakeLists.txt
+++ b/sycl/plugins/level_zero/CMakeLists.txt
@@ -1,14 +1,14 @@
-# PI Level0 plugin library
+# PI Level Zero plugin library
 
 if(MSVC)
-  set(L0_LOADER
+  set(LEVEL_ZERO_LOADER
       "${LLVM_LIBRARY_OUTPUT_INTDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}ze_loader${CMAKE_STATIC_LIBRARY_SUFFIX}")
 else()
-  set(L0_LOADER
+  set(LEVEL_ZERO_LOADER
       "${LLVM_LIBRARY_OUTPUT_INTDIR}/${CMAKE_SHARED_LIBRARY_PREFIX}ze_loader${CMAKE_SHARED_LIBRARY_SUFFIX}")
 endif()
 
-if (NOT DEFINED L0_LIBRARY OR NOT DEFINED L0_INCLUDE_DIR)
+if (NOT DEFINED LEVEL_ZERO_LIBRARY OR NOT DEFINED LEVEL_ZERO_INCLUDE_DIR)
   message(STATUS "Download Level Zero loader and headers from github.com")
   if (CMAKE_C_COMPILER)
       list(APPEND AUX_CMAKE_FLAGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER})
@@ -16,18 +16,18 @@ if (NOT DEFINED L0_LIBRARY OR NOT DEFINED L0_INCLUDE_DIR)
   if (CMAKE_CXX_COMPILER)
       list(APPEND AUX_CMAKE_FLAGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER})
   endif()
-  file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/l0_loader_build)
-  set(L0_LOADER_SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/Level0/l0_loader")
-  if (NOT DEFINED SYCL_EP_L0_LOADER_SKIP_AUTO_UPDATE)
-    set(SYCL_EP_L0_LOADER_SKIP_AUTO_UPDATE ${SYCL_EXTERNAL_PROJECTS_SKIP_AUTO_UPDATE})
+  file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/level_zero_loader_build)
+  set(LEVEL_ZERO_LOADER_SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/level_zero/level_zero_loader")
+  if (NOT DEFINED SYCL_EP_LEVEL_ZERO_LOADER_SKIP_AUTO_UPDATE)
+    set(SYCL_EP_LEVEL_ZERO_LOADER_SKIP_AUTO_UPDATE ${SYCL_EXTERNAL_PROJECTS_SKIP_AUTO_UPDATE})
   endif()
-  ExternalProject_Add(l0-loader
+  ExternalProject_Add(level-zero-loader
     GIT_REPOSITORY    https://github.com/oneapi-src/level-zero.git
-    GIT_TAG           origin/master
-    UPDATE_DISCONNECTED ${SYCL_EP_L0_LOADER_SKIP_AUTO_UPDATE}
-    SOURCE_DIR        ${L0_LOADER_SOURCE_DIR}
-    BINARY_DIR        "${CMAKE_CURRENT_BINARY_DIR}/l0_loader_build"
-    INSTALL_DIR       "${CMAKE_CURRENT_BINARY_DIR}/l0_loader_install"
+    GIT_TAG           v0.91.21
+    UPDATE_DISCONNECTED ${SYCL_EP_LEVEL_ZERO_LOADER_SKIP_AUTO_UPDATE}
+    SOURCE_DIR        ${LEVEL_ZERO_LOADER_SOURCE_DIR}
+    BINARY_DIR        "${CMAKE_CURRENT_BINARY_DIR}/level_zero_loader_build"
+    INSTALL_DIR       "${CMAKE_CURRENT_BINARY_DIR}/level_zero_loader_install"
     CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
                -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM}
                -DOpenCL_INCLUDE_DIR=${OpenCL_INCLUDE_DIRS}
@@ -36,50 +36,50 @@ if (NOT DEFINED L0_LIBRARY OR NOT DEFINED L0_INCLUDE_DIR)
                ${AUX_CMAKE_FLAGS}
     STEP_TARGETS      configure,build,install
     DEPENDS           ocl-headers
-    BUILD_BYPRODUCTS ${L0_LOADER}
+    BUILD_BYPRODUCTS ${LEVEL_ZERO_LOADER}
   )
-  ExternalProject_Add_Step(l0-loader llvminstall
+  ExternalProject_Add_Step(level-zero-loader llvminstall
     COMMAND ${CMAKE_COMMAND} -E copy_directory <INSTALL_DIR>/ ${LLVM_BINARY_DIR}
-    COMMENT "Installing l0-loader into the LLVM binary directory"
+    COMMENT "Installing level-zero-loader into the LLVM binary directory"
     DEPENDEES install
   )
 
-  install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/l0_loader_install/"
+  install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/level_zero_loader_install/"
     DESTINATION "."
-    COMPONENT l0-loader
+    COMPONENT level-zero-loader
   )
 
-  list(APPEND SYCL_TOOLCHAIN_DEPLOY_COMPONENTS l0-loader)
+  list(APPEND SYCL_TOOLCHAIN_DEPLOY_COMPONENTS level-zero-loader)
 else()
-  include_directories("${L0_INCLUDE_DIR}")
-  file(GLOB L0_LIBRARY_SRC "${L0_LIBRARY}*")
-  file(COPY ${L0_LIBRARY_SRC} DESTINATION ${LLVM_LIBRARY_OUTPUT_INTDIR})
-  add_custom_target(l0-loader DEPENDS ${L0_LIBRARY} COMMENT "Copying Level Zero Loader ...")
+  include_directories("${LEVEL_ZERO_INCLUDE_DIR}")
+  file(GLOB LEVEL_ZERO_LIBRARY_SRC "${LEVEL_ZERO_LIBRARY}*")
+  file(COPY ${LEVEL_ZERO_LIBRARY_SRC} DESTINATION ${LLVM_LIBRARY_OUTPUT_INTDIR})
+  add_custom_target(level-zero-loader DEPENDS ${LEVEL_ZERO_LIBRARY} COMMENT "Copying Level Zero Loader ...")
 endif()
 
-add_library (L0Loader-Headers INTERFACE)
-add_library (L0Loader::Headers ALIAS L0Loader-Headers)
-target_include_directories(L0Loader-Headers
-  INTERFACE "${L0_INCLUDE_DIR}"
+add_library (LevelZeroLoader-Headers INTERFACE)
+add_library (LevelZeroLoader::Headers ALIAS LevelZeroLoader-Headers)
+target_include_directories(LevelZeroLoader-Headers
+  INTERFACE "${LEVEL_ZERO_INCLUDE_DIR}"
 )
 
 include_directories("${sycl_inc_dir}")
 include_directories(${OPENCL_INCLUDE})
 
-add_library(pi_level0 SHARED
+add_library(pi_level_zero SHARED
   "${sycl_inc_dir}/CL/sycl/detail/pi.h"
-  "${CMAKE_CURRENT_SOURCE_DIR}/pi_level0.cpp"
-  "${CMAKE_CURRENT_SOURCE_DIR}/pi_level0.hpp"
+  "${CMAKE_CURRENT_SOURCE_DIR}/pi_level_zero.cpp"
+  "${CMAKE_CURRENT_SOURCE_DIR}/pi_level_zero.hpp"
 )
 
 if (MSVC)
   # by defining __SYCL_BUILD_SYCL_DLL, we can use __declspec(dllexport)
   # which are individually tagged for all pi* symbols in pi.h
-  target_compile_definitions(pi_level0 PRIVATE __SYCL_BUILD_SYCL_DLL)
+  target_compile_definitions(pi_level_zero PRIVATE __SYCL_BUILD_SYCL_DLL)
 else()
   # we set the visibility of all symbols 'hidden' by default.
   # In pi.h file, we set exported symbols with visibility==default individually
-  target_compile_options(pi_level0 PUBLIC -fvisibility=hidden)
+  target_compile_options(pi_level_zero PUBLIC -fvisibility=hidden)
 
   # This script file is used to allow exporting pi* symbols only.
   # All other symbols are regarded as local (hidden)
@@ -87,23 +87,23 @@ else()
 
   # Filter symbols based on the scope defined in the script file,
   # and export pi* function symbols in the library.
-  target_link_libraries( pi_level0
+  target_link_libraries( pi_level_zero
     PRIVATE "-Wl,--version-script=${linker_script}"
   )
 endif()
 
-if (TARGET l0-loader)
-  add_dependencies(pi_level0 l0-loader)
+if (TARGET level-zero-loader)
+  add_dependencies(pi_level_zero level-zero-loader)
 endif()
-  add_dependencies(sycl-toolchain pi_level0)
+  add_dependencies(sycl-toolchain pi_level_zero)
 
-  target_link_libraries(pi_level0 PRIVATE "${L0_LOADER}")
+  target_link_libraries(pi_level_zero PRIVATE "${LEVEL_ZERO_LOADER}")
 if (UNIX)
-  target_link_libraries(pi_level0 PRIVATE pthread)
+  target_link_libraries(pi_level_zero PRIVATE pthread)
 endif()
 
-add_common_options(pi_level0)
+add_common_options(pi_level_zero)
 
-install(TARGETS pi_level0
-    LIBRARY DESTINATION "lib" COMPONENT pi_level0
-    RUNTIME DESTINATION "bin" COMPONENT pi_level0)
+install(TARGETS pi_level_zero
+    LIBRARY DESTINATION "lib" COMPONENT pi_level_zero
+    RUNTIME DESTINATION "bin" COMPONENT pi_level_zero)
diff --git a/sycl/plugins/level_zero/pi_level0.cpp b/sycl/plugins/level_zero/pi_level_zero.cpp
similarity index 96%
rename from sycl/plugins/level_zero/pi_level0.cpp
rename to sycl/plugins/level_zero/pi_level_zero.cpp
index 0fdf4e272e1d2..a77719a2220ae 100644
--- a/sycl/plugins/level_zero/pi_level0.cpp
+++ b/sycl/plugins/level_zero/pi_level_zero.cpp
@@ -1,17 +1,17 @@
-//===----------- pi_level0.cpp - Level Zero Plugin--------------------------==//
+//===-------- pi_level_zero.cpp - Level Zero Plugin --------------------==//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-//===----------------------------------------------------------------------===//
+//===------------------------------------------------------------------===//
 
-/// \file pi_level0.cpp
+/// \file pi_level_zero.cpp
 /// Implementation of Level Zero Plugin.
 ///
-/// \ingroup sycl_pi_level0
+/// \ingroup sycl_pi_level_zero
 
-#include "pi_level0.hpp"
+#include "pi_level_zero.hpp"
 #include <cstdarg>
 #include <cstdio>
 #include <cstring>
@@ -26,8 +26,8 @@
 
 namespace {
 
-// Controls L0 calls serialization to w/a L0 driver being not MT ready.
-// Recognized values (can be used as a bit mask):
+// Controls Level Zero calls serialization to w/a Level Zero driver being not MT
+// ready. Recognized values (can be used as a bit mask):
 enum {
   ZeSerializeNone =
       0, // no locking or blocking (except when SYCL RT requested blocking)
@@ -37,10 +37,10 @@ enum {
 };
 static pi_uint32 ZeSerialize = 0;
 
-// This class encapsulates actions taken along with a call to L0 API.
+// This class encapsulates actions taken along with a call to Level Zero API.
 class ZeCall {
 private:
-  // The global mutex that is used for total serialization of L0 calls.
+  // The global mutex that is used for total serialization of Level Zero calls.
   static std::mutex GlobalLock;
 
 public:
@@ -61,7 +61,7 @@ class ZeCall {
 };
 std::mutex ZeCall::GlobalLock;
 
-// Controls L0 calls tracing in zePrint.
+// Controls Level Zero calls tracing in zePrint.
 static bool ZeDebug = false;
 
 static void zePrint(const char *Format, ...) {
@@ -233,12 +233,12 @@ _pi_context::decrementAliveEventsInPool(ze_event_pool_handle_t ZePool) {
   return ZE_RESULT_SUCCESS;
 }
 
-// Some opencl extensions we know are supported by all Level0 devices.
+// Some opencl extensions we know are supported by all Level Zero devices.
 constexpr char ZE_SUPPORTED_EXTENSIONS[] =
     "cl_khr_il_program cl_khr_subgroups cl_intel_subgroups "
     "cl_intel_subgroups_short cl_intel_required_subgroup_size ";
 
-// Map L0 runtime error code to PI error code
+// Map Level Zero runtime error code to PI error code
 static pi_result mapError(ze_result_t ZeResult) {
   // TODO: these mapping need to be clarified and synced with the PI API return
   // values, which is TBD.
@@ -373,7 +373,7 @@ pi_result _pi_device::initialize() {
 // Crate a new command list to be used in a PI call
 pi_result
 _pi_device::createCommandList(ze_command_list_handle_t *ZeCommandList) {
-  // Create the command list, because in L0 commands are added to
+  // Create the command list, because in Level Zero commands are added to
   // the command lists, and later are then added to the command queue.
   //
   // TODO: Figure out how to lower the overhead of creating a new list
@@ -448,7 +448,7 @@ pi_result piPlatformsGet(pi_uint32 NumEntries, pi_platform *Platforms,
     return PI_INVALID_VALUE;
   }
 
-  // This is a good time to initialize L0.
+  // This is a good time to initialize Level Zero.
   // TODO: We can still safely recover if something goes wrong during the init.
   // Implement handling segfault using sigaction.
   // TODO: We should not call zeInit multiples times ever, so
@@ -467,7 +467,7 @@ pi_result piPlatformsGet(pi_uint32 NumEntries, pi_platform *Platforms,
     return mapError(ZeResult);
   }
 
-  // L0 does not have concept of Platforms, but L0 driver is the
+  // Level Zero does not have concept of Platforms, but Level Zero driver is the
   // closest match.
   if (Platforms && NumEntries > 0) {
     uint32_t ZeDriverCount = 0;
@@ -533,10 +533,10 @@ pi_result piPlatformGetInfo(pi_platform Platform, pi_platform_info ParamName,
 
   switch (ParamName) {
   case PI_PLATFORM_INFO_NAME:
-    // TODO: Query L0 driver when relevant info is added there.
+    // TODO: Query Level Zero driver when relevant info is added there.
     return ReturnValue("Intel(R) Level-Zero");
   case PI_PLATFORM_INFO_VENDOR:
-    // TODO: Query L0 driver when relevant info is added there.
+    // TODO: Query Level Zero driver when relevant info is added there.
     return ReturnValue("Intel(R) Corporation");
   case PI_PLATFORM_INFO_EXTENSIONS:
     // Convention adopted from OpenCL:
@@ -547,7 +547,7 @@ pi_result piPlatformGetInfo(pi_platform Platform, pi_platform_info ParamName,
     //
     // TODO: Check the common extensions supported by all connected devices and
     // return them. For now, hardcoding some extensions we know are supported by
-    // all Level0 devices.
+    // all Level Zero devices.
     return ReturnValue(ZE_SUPPORTED_EXTENSIONS);
   case PI_PLATFORM_INFO_PROFILE:
     // TODO: figure out what this means and how is this used
@@ -575,7 +575,7 @@ pi_result piextPlatformGetNativeHandle(pi_platform Platform,
   assert(NativeHandle);
 
   auto ZeDriver = pi_cast<ze_driver_handle_t *>(NativeHandle);
-  // Extract the L0 driver handle from the given PI platform
+  // Extract the Level Zero driver handle from the given PI platform
   *ZeDriver = Platform->ZeDriver;
   return PI_SUCCESS;
 }
@@ -585,7 +585,7 @@ pi_result piextPlatformCreateWithNativeHandle(pi_native_handle NativeHandle,
   assert(NativeHandle);
   assert(Platform);
 
-  // Create PI platform from the given L0 driver handle.
+  // Create PI platform from the given Level Zero driver handle.
   auto ZeDriver = pi_cast<ze_driver_handle_t>(NativeHandle);
   *Platform = new _pi_platform(ZeDriver);
   return PI_SUCCESS;
@@ -598,7 +598,7 @@ pi_result piDevicesGet(pi_platform Platform, pi_device_type DeviceType,
   assert(Platform);
   ze_driver_handle_t ZeDriver = Platform->ZeDriver;
 
-  // Get number of devices supporting L0
+  // Get number of devices supporting Level Zero
   uint32_t ZeDeviceCount = 0;
   const bool AskingForGPU = (DeviceType & PI_DEVICE_TYPE_GPU);
   const bool AskingForDefault = (DeviceType == PI_DEVICE_TYPE_DEFAULT);
@@ -718,7 +718,7 @@ pi_result piDeviceGetInfo(pi_device Device, pi_device_info ParamName,
     return ReturnValue(PI_DEVICE_TYPE_GPU);
   }
   case PI_DEVICE_INFO_PARENT_DEVICE:
-    // TODO: all L0 devices are parent ?
+    // TODO: all Level Zero devices are parent ?
     return ReturnValue(pi_device{0});
   case PI_DEVICE_INFO_PLATFORM:
     return ReturnValue(Device->Platform);
@@ -729,8 +729,8 @@ pi_result piDeviceGetInfo(pi_device Device, pi_device_info ParamName,
     //     "Returns a space separated list of extension names (the extension
     // names themselves do not contain any spaces) supported by the device."
     //
-    // TODO: Use proper mechanism to get this information from Level0 after
-    // it is added to Level0.
+    // TODO: Use proper mechanism to get this information from Level Zero after
+    // it is added to Level Zero.
     // Hardcoding the few we know are supported by the current hardware.
     //
     //
@@ -754,7 +754,8 @@ pi_result piDeviceGetInfo(pi_device Device, pi_device_info ParamName,
     // cl_khr_3d_image_writes - Extension to enable writes to 3D image memory
     //   objects.
     //
-    // Hardcoding some extensions we know are supported by all Level0 devices.
+    // Hardcoding some extensions we know are supported by all Level Zero
+    // devices.
     SupportedExtensions += (ZE_SUPPORTED_EXTENSIONS);
     if (ZeDeviceKernelProperties.fp16Supported)
       SupportedExtensions += ("cl_khr_fp16 ");
@@ -784,7 +785,7 @@ pi_result piDeviceGetInfo(pi_device Device, pi_device_info ParamName,
     return ReturnValue(pi_uint32{MaxComputeUnits});
   }
   case PI_DEVICE_INFO_MAX_WORK_ITEM_DIMENSIONS:
-    // L0 spec defines only three dimensions
+    // Level Zero spec defines only three dimensions
     return ReturnValue(pi_uint32{3});
   case PI_DEVICE_INFO_MAX_WORK_GROUP_SIZE:
     return ReturnValue(
@@ -845,7 +846,7 @@ pi_result piDeviceGetInfo(pi_device Device, pi_device_info ParamName,
     return ReturnValue(pi_uint32{Device->RefCount});
   case PI_DEVICE_INFO_PARTITION_PROPERTIES: {
     // It is debatable if SYCL sub-device and partitioning APIs sufficient to
-    // expose Level0 sub-devices?  We start with support of
+    // expose Level Zero sub-devices?  We start with support of
     // "partition_by_affinity_domain" and "numa" but if that doesn't seem to
     // be a good fit we could look at adding a more descriptive partitioning
     // type.
@@ -913,7 +914,7 @@ pi_result piDeviceGetInfo(pi_device Device, pi_device_info ParamName,
   case PI_DEVICE_INFO_MEM_BASE_ADDR_ALIGN:
     // SYCL/OpenCL spec is vague on what this means exactly, but seems to
     // be for "alignment requirement (in bits) for sub-buffer offsets."
-    // An OpenCL implementation returns 8*128, but L0 can do just 8,
+    // An OpenCL implementation returns 8*128, but Level Zero can do just 8,
     // meaning unaligned access for values of types larger than 8 bits.
     return ReturnValue(pi_uint32{8});
   case PI_DEVICE_INFO_MAX_SAMPLERS:
@@ -995,24 +996,24 @@ pi_result piDeviceGetInfo(pi_device Device, pi_device_info ParamName,
     return ReturnValue(pi_uint64{DoubleFPValue});
   }
   case PI_DEVICE_INFO_IMAGE2D_MAX_WIDTH:
-    // Until L0 provides needed info, hardcode default minimum values required
-    // by the SYCL specification.
+    // Until Level Zero provides needed info, hardcode default minimum values
+    // required by the SYCL specification.
     return ReturnValue(size_t{8192});
   case PI_DEVICE_INFO_IMAGE2D_MAX_HEIGHT:
-    // Until L0 provides needed info, hardcode default minimum values required
-    // by the SYCL specification.
+    // Until Level Zero provides needed info, hardcode default minimum values
+    // required by the SYCL specification.
     return ReturnValue(size_t{8192});
   case PI_DEVICE_INFO_IMAGE3D_MAX_WIDTH:
-    // Until L0 provides needed info, hardcode default minimum values required
-    // by the SYCL specification.
+    // Until Level Zero provides needed info, hardcode default minimum values
+    // required by the SYCL specification.
     return ReturnValue(size_t{2048});
   case PI_DEVICE_INFO_IMAGE3D_MAX_HEIGHT:
-    // Until L0 provides needed info, hardcode default minimum values required
-    // by the SYCL specification.
+    // Until Level Zero provides needed info, hardcode default minimum values
+    // required by the SYCL specification.
     return ReturnValue(size_t{2048});
   case PI_DEVICE_INFO_IMAGE3D_MAX_DEPTH:
-    // Until L0 provides needed info, hardcode default minimum values required
-    // by the SYCL specification.
+    // Until Level Zero provides needed info, hardcode default minimum values
+    // required by the SYCL specification.
     return ReturnValue(size_t{2048});
   case PI_DEVICE_INFO_IMAGE_MAX_BUFFER_SIZE:
     return ReturnValue(size_t{ZeDeviceImageProperties.maxImageBufferSize});
@@ -1106,7 +1107,7 @@ pi_result piDevicePartition(pi_device Device,
                             const pi_device_partition_property *Properties,
                             pi_uint32 NumDevices, pi_device *OutDevices,
                             pi_uint32 *OutNumDevices) {
-  // Other partitioning ways are not supported by L0
+  // Other partitioning ways are not supported by Level Zero
   if (Properties[0] != PI_DEVICE_PARTITION_BY_AFFINITY_DOMAIN ||
       Properties[1] != PI_DEVICE_AFFINITY_DOMAIN_NEXT_PARTITIONABLE) {
     return PI_INVALID_VALUE;
@@ -1139,7 +1140,7 @@ pi_result piDevicePartition(pi_device Device,
     auto ZeSubdevices = new ze_device_handle_t[Count];
     ZE_CALL(zeDeviceGetSubDevices(Device->ZeDevice, &Count, ZeSubdevices));
 
-    // Wrap the L0 sub-devices into PI sub-devices, and write them out.
+    // Wrap the Level Zero sub-devices into PI sub-devices, and write them out.
     for (uint32_t I = 0; I < Count; ++I) {
       OutDevices[I] = new _pi_device(ZeSubdevices[I], Device->Platform,
                                      true /* isSubDevice */);
@@ -1183,7 +1184,7 @@ pi_result piextDeviceGetNativeHandle(pi_device Device,
   assert(NativeHandle);
 
   auto ZeDevice = pi_cast<ze_device_handle_t *>(NativeHandle);
-  // Extract the L0 module handle from the given PI device
+  // Extract the Level Zero module handle from the given PI device
   *ZeDevice = Device->ZeDevice;
   return PI_SUCCESS;
 }
@@ -1195,7 +1196,7 @@ pi_result piextDeviceCreateWithNativeHandle(pi_native_handle NativeHandle,
   assert(Device);
   assert(Platform);
 
-  // Create PI device from the given L0 device handle.
+  // Create PI device from the given Level Zero device handle.
   auto ZeDevice = pi_cast<ze_device_handle_t>(NativeHandle);
   *Device = new _pi_device(ZeDevice, Platform);
   return (*Device)->initialize();
@@ -1208,7 +1209,7 @@ pi_result piContextCreate(const pi_context_properties *Properties,
                                             void *UserData),
                           void *UserData, pi_context *RetContext) {
 
-  // L0 does not have notion of contexts.
+  // Level Zero does not have notion of contexts.
   // Return the device handle (only single device is allowed) as a context
   // handle.
   if (NumDevices != 1) {
@@ -1387,7 +1388,7 @@ pi_result piextQueueGetNativeHandle(pi_queue Queue,
   assert(NativeHandle);
 
   auto ZeQueue = pi_cast<ze_command_queue_handle_t *>(NativeHandle);
-  // Extract the L0 queue handle from the given PI queue
+  // Extract the Level Zero queue handle from the given PI queue
   *ZeQueue = Queue->ZeCommandQueue;
   return PI_SUCCESS;
 }
@@ -1437,9 +1438,9 @@ pi_result piMemBufferCreate(pi_context Context, pi_mem_flags Flags, size_t Size,
   auto HostPtrOrNull =
       (Flags & PI_MEM_FLAGS_HOST_PTR_USE) ? pi_cast<char *>(HostPtr) : nullptr;
   try {
-    *RetMem = new _pi_buffer(Context->Device->Platform,
-                             pi_cast<char *>(Ptr) /* L0 Memory Handle */,
-                             HostPtrOrNull);
+    *RetMem = new _pi_buffer(
+        Context->Device->Platform,
+        pi_cast<char *>(Ptr) /* Level Zero Memory Handle */, HostPtrOrNull);
   } catch (const std::bad_alloc &) {
     return PI_OUT_OF_HOST_MEMORY;
   } catch (...) {
@@ -1660,8 +1661,8 @@ pi_result piProgramCreate(pi_context Context, const void *IL, size_t Length,
   assert(Context);
   assert(Program);
 
-  // NOTE: the L0 module creation is also building the program, so we are
-  // deferring it until the program is ready to be built in piProgramBuild
+  // NOTE: the Level Zero module creation is also building the program, so we
+  // are deferring it until the program is ready to be built in piProgramBuild
   // and piProgramCompile. Also it is only then we know the build options.
   //
   ze_module_desc_t ZeModuleDesc = {};
@@ -1726,7 +1727,7 @@ pi_result piclProgramCreateWithSource(pi_context Context, pi_uint32 Count,
                                       const size_t *Lengths,
                                       pi_program *RetProgram) {
 
-  zePrint("piclProgramCreateWithSource: not supported in L0\n");
+  zePrint("piclProgramCreateWithSource: not supported in Level Zero\n");
   return PI_INVALID_OPERATION;
 }
 
@@ -1740,7 +1741,7 @@ pi_result piProgramGetInfo(pi_program Program, pi_program_info ParamName,
   case PI_PROGRAM_INFO_REFERENCE_COUNT:
     return ReturnValue(pi_uint32{Program->RefCount});
   case PI_PROGRAM_INFO_NUM_DEVICES:
-    // L0 Module is always for a single device.
+    // Level Zero Module is always for a single device.
     return ReturnValue(pi_uint32{1});
   case PI_PROGRAM_INFO_DEVICES:
     return ReturnValue(Program->Context->Device);
@@ -1764,7 +1765,7 @@ pi_result piProgramGetInfo(pi_program Program, pi_program_info ParamName,
   case PI_PROGRAM_INFO_KERNEL_NAMES:
     try {
       // There are extra allocations/copying here dictated by the difference
-      // in L0 and PI interfaces.
+      // in Level Zero and PI interfaces.
       uint32_t Count = 0;
       ZE_CALL(zeModuleGetKernelNames(Program->ZeModule, &Count, nullptr));
       char **PNames = new char *[Count];
@@ -1795,7 +1796,9 @@ pi_result piProgramLink(pi_context Context, pi_uint32 NumDevices,
                         const pi_program *InputPrograms,
                         void (*PFnNotify)(pi_program Program, void *UserData),
                         void *UserData, pi_program *RetProgram) {
-  // TODO: L0 does not [yet] support linking so dummy implementation here.
+
+  // TODO: Level Zero does not [yet] support linking so dummy implementation
+  // here.
   assert(NumInputPrograms == 1 && InputPrograms);
   assert(RetProgram);
   *RetProgram = InputPrograms[0];
@@ -1812,7 +1815,7 @@ pi_result piProgramCompile(
   assert(NumInputHeaders == 0);
   assert(!InputHeaders);
 
-  // There is no support foe linking yet in L0 so "compile" actually
+  // There is no support for linking yet in Level Zero so "compile" actually
   // does the "build".
   return piProgramBuild(Program, NumDevices, DeviceList, Options, PFnNotify,
                         UserData);
@@ -1858,6 +1861,7 @@ pi_result piProgramBuild(pi_program Program, pi_uint32 NumDevices,
   ze_device_handle_t ZeDevice = Program->Context->Device->ZeDevice;
   ZE_CALL(zeModuleCreate(ZeDevice, &Program->ZeModuleDesc, &Program->ZeModule,
                          &Program->ZeBuildLog));
+
   return PI_SUCCESS;
 }
 
@@ -1868,14 +1872,14 @@ pi_result piProgramGetBuildInfo(pi_program Program, pi_device Device,
 
   ReturnHelper ReturnValue(ParamValueSize, ParamValue, ParamValueSizeRet);
   if (ParamName == CL_PROGRAM_BINARY_TYPE) {
-    // TODO: is this the only supported binary type in L0?
+    // TODO: is this the only supported binary type in Level Zero?
     // We should probably return CL_PROGRAM_BINARY_TYPE_NONE if asked
     // before the program was compiled.
     return ReturnValue(
         cl_program_binary_type{CL_PROGRAM_BINARY_TYPE_EXECUTABLE});
   }
   if (ParamName == CL_PROGRAM_BUILD_OPTIONS) {
-    // TODO: how to get module build options out of L0?
+    // TODO: how to get module build options out of Level Zero?
     // For the programs that we compiled we can remember the options
     // passed with piProgramCompile/piProgramBuild, but what can we
     // return for programs that were built outside and registered
@@ -1909,7 +1913,7 @@ pi_result piProgramRelease(pi_program Program) {
     delete[] Program->ZeModuleDesc.pInputModule;
     if (Program->ZeBuildLog)
       zeModuleBuildLogDestroy(Program->ZeBuildLog);
-    // TODO: call zeModuleDestroy for non-interop L0 modules
+    // TODO: call zeModuleDestroy for non-interop Level Zero modules
     delete Program;
   }
   return PI_SUCCESS;
@@ -1921,7 +1925,7 @@ pi_result piextProgramGetNativeHandle(pi_program Program,
   assert(NativeHandle);
 
   auto ZeModule = pi_cast<ze_module_handle_t *>(NativeHandle);
-  // Extract the L0 module handle from the given PI program
+  // Extract the Level Zero module handle from the given PI program
   *ZeModule = Program->ZeModule;
   return PI_SUCCESS;
 }
@@ -1935,9 +1939,9 @@ pi_result piextProgramCreateWithNativeHandle(pi_native_handle NativeHandle,
 
   auto ZeModule = pi_cast<ze_module_handle_t>(NativeHandle);
 
-  // Create PI program from the given L0 module handle.
+  // Create PI program from the given Level Zero module handle.
   //
-  // TODO: We don't have the real L0 module descriptor with
+  // TODO: We don't have the real Level Zero module descriptor with
   // which it was created, but that's only needed for zeModuleCreate,
   // which we don't expect to be called on the interop program.
   //
@@ -2050,9 +2054,9 @@ pi_result piKernelGetInfo(pi_kernel Kernel, pi_kernel_info ParamName,
   case PI_KERNEL_INFO_PROGRAM:
     return ReturnValue(pi_program{Kernel->Program});
   case PI_KERNEL_INFO_FUNCTION_NAME:
-    // TODO: Replace with the line in the comment once bug in the L0 driver will
-    // be fixed. Problem is that currently L0 driver truncates name of the
-    // returned kernel if it is longer than 256 symbols.
+    // TODO: Replace with the line in the comment once bug in the Level Zero
+    // driver will be fixed. Problem is that currently Level Zero driver
+    // truncates name of the returned kernel if it is longer than 256 symbols.
     //
     // return ReturnValue(ZeKernelProperties.name);
     return ReturnValue(Kernel->KernelName.c_str());
@@ -2258,6 +2262,8 @@ piEnqueueKernelLaunch(pi_queue Queue, pi_kernel Kernel, pi_uint32 WorkDim,
 
   ze_event_handle_t *ZeEventWaitList =
       _pi_event::createZeEventList(NumEventsInWaitList, EventWaitList);
+  if (!ZeEventWaitList)
+    return PI_OUT_OF_HOST_MEMORY;
 
   // Add the command to the command list
   ZE_CALL(zeCommandListAppendLaunchKernel(
@@ -2374,7 +2380,7 @@ pi_result piEventGetProfilingInfo(pi_event Event, pi_profiling_info ParamName,
   }
   case PI_PROFILING_INFO_COMMAND_QUEUED:
   case PI_PROFILING_INFO_COMMAND_SUBMIT:
-    // TODO: Support these when L0 supported is added.
+    // TODO: Support these when Level Zero supported is added.
     return ReturnValue(uint64_t{0});
   default:
     zePrint("piEventGetProfilingInfo: not supported ParamName\n");
@@ -2708,6 +2714,8 @@ enqueueMemCopyHelper(pi_command_type CommandType, pi_queue Queue, void *Dst,
 
   ze_event_handle_t *ZeEventWaitList =
       _pi_event::createZeEventList(NumEventsInWaitList, EventWaitList);
+  if (!ZeEventWaitList)
+    return PI_OUT_OF_HOST_MEMORY;
 
   ZE_CALL(zeCommandListAppendWaitOnEvents(ZeCommandList, NumEventsInWaitList,
                                           ZeEventWaitList));
@@ -2766,6 +2774,8 @@ static pi_result enqueueMemCopyRectHelper(
 
   ze_event_handle_t *ZeEventWaitList =
       _pi_event::createZeEventList(NumEventsInWaitList, EventWaitList);
+  if (!ZeEventWaitList)
+    return PI_OUT_OF_HOST_MEMORY;
 
   ZE_CALL(zeCommandListAppendWaitOnEvents(ZeCommandList, NumEventsInWaitList,
                                           ZeEventWaitList));
@@ -2926,6 +2936,8 @@ enqueueMemFillHelper(pi_command_type CommandType, pi_queue Queue, void *Ptr,
 
   ze_event_handle_t *ZeEventWaitList =
       _pi_event::createZeEventList(NumEventsInWaitList, EventWaitList);
+  if (!ZeEventWaitList)
+    return PI_OUT_OF_HOST_MEMORY;
 
   ZE_CALL(zeCommandListAppendWaitOnEvents(ZeCommandList, NumEventsInWaitList,
                                           ZeEventWaitList));
@@ -3008,8 +3020,8 @@ piEnqueueMemBufferMap(pi_queue Queue, pi_mem Buffer, pi_bool BlockingMap,
   ZE_CALL(zeCommandListAppendWaitOnEvents(ZeCommandList, NumEventsInWaitList,
                                           ZeEventWaitList));
 
-  // TODO: L0 is missing the memory "mapping" capabilities, so we are left
-  // to doing new memory allocation and a copy (read).
+  // TODO: Level Zero is missing the memory "mapping" capabilities, so we are
+  // left to doing new memory allocation and a copy (read).
   //
   // TODO: check if the input buffer is already allocated in shared
   // memory and thus is accessible from the host as is. Can we get SYCL RT
@@ -3079,8 +3091,8 @@ pi_result piEnqueueMemUnmap(pi_queue Queue, pi_mem MemObj, void *MappedPtr,
   ZE_CALL(zeCommandListAppendWaitOnEvents(ZeCommandList, NumEventsInWaitList,
                                           ZeEventWaitList));
 
-  // TODO: L0 is missing the memory "mapping" capabilities, so we are left
-  // to doing copy (write back to the device).
+  // TODO: Level Zero is missing the memory "mapping" capabilities, so we are
+  // left to doing copy (write back to the device).
   //
   // NOTE: Keep this in sync with the implementation of
   // piEnqueueMemBufferMap/piEnqueueMemImageMap.
@@ -3196,7 +3208,7 @@ enqueueMemImageCommandHelper(pi_command_type CommandType, pi_queue Queue,
     const ze_image_region_t ZeSrcRegion =
         getImageRegionHelper(SrcMem, SrcOrigin, Region);
 
-    // TODO: L0 does not support row_pitch/slice_pitch for images yet.
+    // TODO: Level Zero does not support row_pitch/slice_pitch for images yet.
     // Check that SYCL RT did not want pitch larger than default.
 #ifndef NDEBUG
     assert(SrcMem->isImage());
@@ -3221,7 +3233,7 @@ enqueueMemImageCommandHelper(pi_command_type CommandType, pi_queue Queue,
     const ze_image_region_t ZeDstRegion =
         getImageRegionHelper(DstMem, DstOrigin, Region);
 
-    // TODO: L0 does not support row_pitch/slice_pitch for images yet.
+    // TODO: Level Zero does not support row_pitch/slice_pitch for images yet.
     // Check that SYCL RT did not want pitch larger than default.
 #ifndef NDEBUG
     assert(DstMem->isImage());
@@ -3352,7 +3364,7 @@ pi_result piMemBufferPartition(pi_mem Buffer, pi_mem_flags Flags,
     *RetMem =
         new _pi_buffer(Buffer->Platform,
                        pi_cast<char *>(Buffer->getZeHandle()) +
-                           Region->origin /* L0 memory handle */,
+                           Region->origin /* Level Zero memory handle */,
                        nullptr /* Host pointer */, Buffer /* Parent buffer */,
                        Region->origin /* Sub-buffer origin */,
                        Region->size /*Sub-buffer size*/);
@@ -3398,7 +3410,7 @@ pi_result piextUSMHostAlloc(void **ResultPtr, pi_context Context,
 
   ze_host_mem_alloc_desc_t ZeDesc = {};
   ZeDesc.flags = ZE_HOST_MEM_ALLOC_FLAG_DEFAULT;
-  // TODO: translate PI properties to L0 flags
+  // TODO: translate PI properties to Level Zero flags
   ZE_CALL(zeDriverAllocHostMem(Context->Device->Platform->ZeDriver, &ZeDesc,
                                Size, Alignment, ResultPtr));
 
@@ -3415,7 +3427,7 @@ pi_result piextUSMDeviceAlloc(void **ResultPtr, pi_context Context,
   // Check that incorrect bits are not set in the properties.
   assert(!Properties || (Properties && !(*Properties & ~PI_MEM_ALLOC_FLAGS)));
 
-  // TODO: translate PI properties to L0 flags
+  // TODO: translate PI properties to Level Zero flags
   ze_device_mem_alloc_desc_t ZeDesc = {};
   ZeDesc.flags = ZE_DEVICE_MEM_ALLOC_FLAG_DEFAULT;
   ZeDesc.ordinal = 0;
@@ -3435,7 +3447,7 @@ pi_result piextUSMSharedAlloc(void **ResultPtr, pi_context Context,
   // Check that incorrect bits are not set in the properties.
   assert(!Properties || (Properties && !(*Properties & ~PI_MEM_ALLOC_FLAGS)));
 
-  // TODO: translate PI properties to L0 flags
+  // TODO: translate PI properties to Level Zero flags
   ze_host_mem_alloc_desc_t ZeHostDesc = {};
   ZeHostDesc.flags = ZE_HOST_MEM_ALLOC_FLAG_DEFAULT;
   ze_device_mem_alloc_desc_t ZeDevDesc = {};
@@ -3547,7 +3559,7 @@ pi_result piextUSMEnqueuePrefetch(pi_queue Queue, const void *Ptr, size_t Size,
   // TODO: figure out how to translate "flags"
   ZE_CALL(zeCommandListAppendMemoryPrefetch(ZeCommandList, Ptr, Size));
 
-  // TODO: L0 does not have a completion "event" with the prefetch API,
+  // TODO: Level Zero does not have a completion "event" with the prefetch API,
   // so manually add command to signal our event.
   ZE_CALL(zeCommandListAppendSignalEvent(ZeCommandList, ZeEvent));
 
@@ -3630,7 +3642,7 @@ pi_result piextUSMEnqueueMemAdvise(pi_queue Queue, const void *Ptr,
   ZE_CALL(zeCommandListAppendMemAdvise(
       ZeCommandList, Queue->Context->Device->ZeDevice, Ptr, Length, ZeAdvice));
 
-  // TODO: L0 does not have a completion "event" with the advise API,
+  // TODO: Level Zero does not have a completion "event" with the advise API,
   // so manually add command to signal our event.
   ZE_CALL(zeCommandListAppendSignalEvent(ZeCommandList, ZeEvent));
 
@@ -3691,7 +3703,7 @@ pi_result piextUSMGetMemAllocInfo(pi_context Context, const void *Ptr,
   }
   case PI_MEM_ALLOC_DEVICE: {
     // TODO: this wants pi_device, but we didn't remember it, and cannot
-    // deduct from the L0 device.
+    // deduct from the Level Zero device.
     die("piextUSMGetMemAllocInfo: PI_MEM_ALLOC_DEVICE not implemented");
     break;
   }
@@ -3752,7 +3764,8 @@ pi_result piextProgramSetSpecializationConstant(pi_program Prog,
   // Pass SpecValue pointer. Spec constant value is retrieved
   // by Level-Zero when creating the modul
   //
-  // NOTE: SpecSize is unused in L0, the size is known from SPIR-V by SpecID.
+  // NOTE: SpecSize is unused in Level Zero, the size is known from SPIR-V by
+  // SpecID.
   Prog->ZeSpecConstants[SpecID] = reinterpret_cast<uint64_t>(SpecValue);
 
   return PI_SUCCESS;
diff --git a/sycl/plugins/level_zero/pi_level0.hpp b/sycl/plugins/level_zero/pi_level_zero.hpp
old mode 100755
new mode 100644
similarity index 86%
rename from sycl/plugins/level_zero/pi_level0.hpp
rename to sycl/plugins/level_zero/pi_level_zero.hpp
index faf39d759c385..a3db143a55a48
--- a/sycl/plugins/level_zero/pi_level0.hpp
+++ b/sycl/plugins/level_zero/pi_level_zero.hpp
@@ -1,22 +1,22 @@
-//===---------- pi_level0.hpp - Level Zero Plugin -------------------------===//
+//===------- pi_level_zero.hpp - Level Zero Plugin -------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-//===----------------------------------------------------------------------===//
+//===-----------------------------------------------------------------===//
 
-/// \defgroup sycl_pi_level0 Level Zero Plugin
+/// \defgroup sycl_pi_level_zero Level Zero Plugin
 /// \ingroup sycl_pi
 
-/// \file pi_level0.hpp
+/// \file pi_level_zero.hpp
 /// Declarations for Level Zero Plugin. It is the interface between the
 /// device-agnostic SYCL runtime layer and underlying Level Zero runtime.
 ///
-/// \ingroup sycl_pi_level0
+/// \ingroup sycl_pi_level_zero
 
-#ifndef PI_LEVEL0_HPP
-#define PI_LEVEL0_HPP
+#ifndef PI_LEVEL_ZERO_HPP
+#define PI_LEVEL_ZERO_HPP
 
 #include <CL/sycl/detail/pi.h>
 #include <atomic>
@@ -51,18 +51,19 @@ template <> uint32_t pi_cast(uint64_t Value) {
 struct _pi_object {
   _pi_object() : RefCount{1} {}
 
-  // L0 doesn't do the reference counting, so we have to do.
+  // Level Zero doesn't do the reference counting, so we have to do.
   // Must be atomic to prevent data race when incrementing/decrementing.
   std::atomic<pi_uint32> RefCount;
 };
 
-// Define the types that are opaque in pi.h in a manner suitabale for L0 plugin
+// Define the types that are opaque in pi.h in a manner suitabale for Level Zero
+// plugin
 
 struct _pi_platform {
   _pi_platform(ze_driver_handle_t Driver) : ZeDriver{Driver} {}
 
-  // L0 lacks the notion of a platform, but there is a driver, which is a
-  // pretty good fit to keep here.
+  // Level Zero lacks the notion of a platform, but there is a driver, which is
+  // a pretty good fit to keep here.
   ze_driver_handle_t ZeDriver;
 
   // Cache versions info from zeDriverGetProperties.
@@ -83,14 +84,14 @@ struct _pi_device : _pi_object {
   // Initialize the entire PI device.
   pi_result initialize();
 
-  // L0 device handle.
+  // Level Zero device handle.
   ze_device_handle_t ZeDevice;
 
   // PI platform to which this device belongs.
   pi_platform Platform;
 
-  // Immediate L0 command list for this device, to be used for initializations.
-  // To be created as:
+  // Immediate Level Zero command list for this device, to be used for
+  // initializations. To be created as:
   // - Immediate command list: So any command appended to it is immediately
   //   offloaded to the device.
   // - Synchronous: So implicit synchronization is made inside the level-zero
@@ -117,7 +118,7 @@ struct _pi_context : _pi_object {
       : Device{Device}, ZeEventPool{nullptr}, NumEventsAvailableInEventPool{},
         NumEventsLiveInEventPool{} {}
 
-  // L0 does not have notion of contexts.
+  // Level Zero does not have notion of contexts.
   // Keep the device here (must be exactly one) to return it when PI context
   // is queried for devices.
   pi_device Device;
@@ -164,7 +165,7 @@ struct _pi_queue : _pi_object {
   _pi_queue(ze_command_queue_handle_t Queue, pi_context Context)
       : ZeCommandQueue{Queue}, Context{Context} {}
 
-  // L0 command queue handle.
+  // Level Zero command queue handle.
   ze_command_queue_handle_t ZeCommandQueue;
 
   // Keeps the PI context to which this queue belongs.
@@ -197,10 +198,10 @@ struct _pi_mem : _pi_object {
 
   // Interface of the _pi_mem object
 
-  // Get the L0 handle of the current memory object
+  // Get the Level Zero handle of the current memory object
   virtual void *getZeHandle() = 0;
 
-  // Get a pointer to the L0 handle of the current memory object
+  // Get a pointer to the Level Zero handle of the current memory object
   virtual void *getZeHandlePtr() = 0;
 
   // Method to get type of the derived object (image or buffer)
@@ -241,7 +242,7 @@ struct _pi_buffer final : _pi_mem {
 
   bool isSubBuffer() const { return SubBuffer.Parent != nullptr; }
 
-  // L0 memory handle is really just a naked pointer.
+  // Level Zero memory handle is really just a naked pointer.
   // It is just convenient to have it char * to simplify offset arithmetics.
   char *ZeMem;
 
@@ -268,7 +269,7 @@ struct _pi_image final : _pi_mem {
   ze_image_desc_t ZeImageDesc;
 #endif // !NDEBUG
 
-  // L0 image handle.
+  // Level Zero image handle.
   ze_image_handle_t ZeImage;
 };
 
@@ -278,14 +279,14 @@ struct _pi_event : _pi_object {
       : ZeEvent{ZeEvent}, ZeEventPool{ZeEventPool}, ZeCommandList{nullptr},
         CommandType{CommandType}, Context{Context}, CommandData{nullptr} {}
 
-  // L0 event handle.
+  // Level Zero event handle.
   ze_event_handle_t ZeEvent;
-  // L0 event pool handle.
+  // Level Zero event pool handle.
   ze_event_pool_handle_t ZeEventPool;
 
-  // L0 command list where the command signaling this event was appended to.
-  // This is currently used to remember/destroy the command list after
-  // all commands in it are completed, i.e. this event signaled.
+  // Level Zero command list where the command signaling this event was appended
+  // to. This is currently used to remember/destroy the command list after all
+  // commands in it are completed, i.e. this event signaled.
   ze_command_list_handle_t ZeCommandList;
 
   // Keeps the command-queue and command associated with the event.
@@ -300,7 +301,7 @@ struct _pi_event : _pi_object {
   // Opaque data to hold any data needed for CommandType.
   void *CommandData;
 
-  // Methods for translating PI events list into L0 events list
+  // Methods for translating PI events list into Level Zero events list
   static ze_event_handle_t *createZeEventList(pi_uint32, const pi_event *);
   static void deleteZeEventList(ze_event_handle_t *);
 };
@@ -311,16 +312,16 @@ struct _pi_program : _pi_object {
       : ZeModuleDesc(ModuleDesc), ZeModule{Module},
         ZeBuildLog{nullptr}, Context{Context} {}
 
-  // L0 module descriptor.
+  // Level Zero module descriptor.
   ze_module_desc_t ZeModuleDesc;
 
-  // L0 module handle.
+  // Level Zero module handle.
   ze_module_handle_t ZeModule;
-  // L0 module specialization constants
+  // Level Zero module specialization constants
   std::mutex ZeSpecConstantsMutex;
   std::unordered_map<uint32_t, uint64_t> ZeSpecConstants;
 
-  // L0 build log.
+  // Level Zero build log.
   ze_module_build_log_handle_t ZeBuildLog;
 
   // Keep the context of the program.
@@ -332,21 +333,21 @@ struct _pi_kernel : _pi_object {
              const char *KernelName)
       : ZeKernel{Kernel}, Program{Program}, KernelName(KernelName) {}
 
-  // L0 function handle.
+  // Level Zero function handle.
   ze_kernel_handle_t ZeKernel;
 
   // Keep the program of the kernel.
   pi_program Program;
 
-  // TODO: remove when bug in the L0 runtime will be fixed.
+  // TODO: remove when bug in the Level Zero runtime will be fixed.
   std::string KernelName;
 };
 
 struct _pi_sampler : _pi_object {
   _pi_sampler(ze_sampler_handle_t Sampler) : ZeSampler{Sampler} {}
 
-  // L0 sampler handle.
+  // Level Zero sampler handle.
   ze_sampler_handle_t ZeSampler;
 };
 
-#endif // PI_LEVEL0_HPP
+#endif // PI_LEVEL_ZERO_HPP
diff --git a/sycl/source/CMakeLists.txt b/sycl/source/CMakeLists.txt
index 5c202d369e2d3..3460e11bae70b 100644
--- a/sycl/source/CMakeLists.txt
+++ b/sycl/source/CMakeLists.txt
@@ -125,6 +125,7 @@ set(SYCL_SOURCES
     "detail/queue_impl.cpp"
     "detail/os_util.cpp"
     "detail/platform_util.cpp"
+    "detail/reduction.cpp"
     "detail/sampler_impl.cpp"
     "detail/stream_impl.cpp"
     "detail/scheduler/commands.cpp"
diff --git a/sycl/source/backend/level_zero.cpp b/sycl/source/backend/level_zero.cpp
index 2e62223c0301e..6d7c7a347e89d 100644
--- a/sycl/source/backend/level_zero.cpp
+++ b/sycl/source/backend/level_zero.cpp
@@ -14,13 +14,13 @@
 
 __SYCL_INLINE_NAMESPACE(cl) {
 namespace sycl {
-namespace level0 {
+namespace level_zero {
 using namespace detail;
 
 //----------------------------------------------------------------------------
-// Implementation of level0::make<platform>
+// Implementation of level_zero::make<platform>
 __SYCL_EXPORT platform make_platform(pi_native_handle NativeHandle) {
-  const auto &Plugin = pi::getPlugin<backend::level0>();
+  const auto &Plugin = pi::getPlugin<backend::level_zero>();
   // Create PI platform first.
   pi::PiPlatform PiPlatform;
   Plugin.call<PiApiKind::piextPlatformCreateWithNativeHandle>(NativeHandle,
@@ -32,10 +32,10 @@ __SYCL_EXPORT platform make_platform(pi_native_handle NativeHandle) {
 }
 
 //----------------------------------------------------------------------------
-// Implementation of level0::make<device>
+// Implementation of level_zero::make<device>
 __SYCL_EXPORT device make_device(const platform &Platform,
                                  pi_native_handle NativeHandle) {
-  const auto &Plugin = pi::getPlugin<backend::level0>();
+  const auto &Plugin = pi::getPlugin<backend::level_zero>();
   const auto &PlatformImpl = getSyclObjImpl(Platform);
   // Create PI device first.
   pi::PiDevice PiDevice;
@@ -47,7 +47,7 @@ __SYCL_EXPORT device make_device(const platform &Platform,
 }
 
 //----------------------------------------------------------------------------
-// Implementation of level0::make<program>
+// Implementation of level_zero::make<program>
 __SYCL_EXPORT program make_program(const context &Context,
                                    pi_native_handle NativeHandle) {
   // Construct the SYCL program from native program.
@@ -58,10 +58,10 @@ __SYCL_EXPORT program make_program(const context &Context,
 }
 
 //----------------------------------------------------------------------------
-// Implementation of level0::make<queue>
+// Implementation of level_zero::make<queue>
 __SYCL_EXPORT queue make_queue(const context &Context,
                                pi_native_handle NativeHandle) {
-  const auto &Plugin = pi::getPlugin<backend::level0>();
+  const auto &Plugin = pi::getPlugin<backend::level_zero>();
   const auto &ContextImpl = getSyclObjImpl(Context);
   // Create PI queue first.
   pi::PiQueue PiQueue;
@@ -72,6 +72,6 @@ __SYCL_EXPORT queue make_queue(const context &Context,
       PiQueue, ContextImpl, ContextImpl->get_async_handler()));
 }
 
-} // namespace level0
+} // namespace level_zero
 } // namespace sycl
 } // __SYCL_INLINE_NAMESPACE(cl)
diff --git a/sycl/source/detail/buffer_impl.cpp b/sycl/source/detail/buffer_impl.cpp
index a9aa0e4194f9e..26e189f1abffd 100644
--- a/sycl/source/detail/buffer_impl.cpp
+++ b/sycl/source/detail/buffer_impl.cpp
@@ -29,7 +29,7 @@ void *buffer_impl::allocateMem(ContextImplPtr Context, bool InitFromUserData,
 
   return MemoryManager::allocateMemBuffer(
       std::move(Context), this, UserPtr, BaseT::MHostPtrReadOnly,
-      BaseT::getSize(), BaseT::MInteropEvent, BaseT::MInteropContext,
+      BaseT::getSize(), BaseT::MInteropEvent, BaseT::MInteropContext, MProps,
       OutEventToWait);
 }
 } // namespace detail
diff --git a/sycl/include/CL/sycl/detail/circular_buffer.hpp b/sycl/source/detail/circular_buffer.hpp
similarity index 100%
rename from sycl/include/CL/sycl/detail/circular_buffer.hpp
rename to sycl/source/detail/circular_buffer.hpp
diff --git a/sycl/source/detail/config.hpp b/sycl/source/detail/config.hpp
index 1559f40e5ad86..ac6fe8fbcbd2b 100644
--- a/sycl/source/detail/config.hpp
+++ b/sycl/source/detail/config.hpp
@@ -15,6 +15,7 @@
 #include <algorithm>
 #include <array>
 #include <cstdlib>
+#include <string>
 #include <utility>
 
 __SYCL_INLINE_NAMESPACE(cl) {
@@ -118,9 +119,10 @@ template <> class SYCLConfig<SYCL_BE> {
       return BackendPtr;
 
     const char *ValStr = BaseT::getRawValue();
-    const std::array<std::pair<std::string, backend>, 3> SyclBeMap = {
+    const std::array<std::pair<std::string, backend>, 4> SyclBeMap = {
         {{"PI_OPENCL", backend::opencl},
-         {"PI_LEVEL0", backend::level0},
+         {"PI_LEVEL_ZERO", backend::level_zero},
+         {"PI_LEVEL0", backend::level_zero}, // for backward compatibility
          {"PI_CUDA", backend::cuda}}};
     if (ValStr) {
       auto It = std::find_if(
@@ -130,7 +132,7 @@ template <> class SYCLConfig<SYCL_BE> {
           });
       if (It == SyclBeMap.end())
         pi::die("Invalid backend. "
-                "Valid values are PI_OPENCL/PI_LEVEL0/PI_CUDA");
+                "Valid values are PI_OPENCL/PI_LEVEL_ZERO/PI_CUDA");
       static backend Backend = It->second;
       BackendPtr = &Backend;
     }
@@ -161,6 +163,6 @@ template <> class SYCLConfig<SYCL_PI_TRACE> {
   }
 };
 
-} // __SYCL_INLINE_NAMESPACE(cl)
-} // namespace sycl
 } // namespace detail
+} // namespace sycl
+} // __SYCL_INLINE_NAMESPACE(cl)
diff --git a/sycl/source/detail/image_impl.cpp b/sycl/source/detail/image_impl.cpp
index 9d7ccf51e2f79..747f3b193fd85 100644
--- a/sycl/source/detail/image_impl.cpp
+++ b/sycl/source/detail/image_impl.cpp
@@ -325,7 +325,7 @@ void *image_impl<Dimensions>::allocateMem(ContextImplPtr Context,
   return MemoryManager::allocateMemImage(
       std::move(Context), this, UserPtr, BaseT::MHostPtrReadOnly,
       BaseT::getSize(), Desc, Format, BaseT::MInteropEvent,
-      BaseT::MInteropContext, OutEventToWait);
+      BaseT::MInteropContext, MProps, OutEventToWait);
 }
 
 template <int Dimensions>
diff --git a/sycl/source/detail/kernel_impl.hpp b/sycl/source/detail/kernel_impl.hpp
index bab3d4749508d..7761f168670c4 100644
--- a/sycl/source/detail/kernel_impl.hpp
+++ b/sycl/source/detail/kernel_impl.hpp
@@ -62,6 +62,15 @@ class kernel_impl {
   /// \param ProgramImpl is a valid instance of program_impl
   kernel_impl(ContextImplPtr Context, ProgramImplPtr ProgramImpl);
 
+  // This section means the object is non-movable and non-copyable
+  // There is no need of move and copy constructors in kernel_impl.
+  // If they need to be added, piKernelRetain method for MKernel
+  // should be present.
+  kernel_impl(const kernel_impl &) = delete;
+  kernel_impl(kernel_impl &&) = delete;
+  kernel_impl &operator=(const kernel_impl &) = delete;
+  kernel_impl &operator=(kernel_impl &&) = delete;
+
   ~kernel_impl();
 
   /// Gets a valid OpenCL kernel handle
diff --git a/sycl/source/detail/kernel_program_cache.hpp b/sycl/source/detail/kernel_program_cache.hpp
index 24bc38ae7e241..83261ce661142 100644
--- a/sycl/source/detail/kernel_program_cache.hpp
+++ b/sycl/source/detail/kernel_program_cache.hpp
@@ -42,11 +42,25 @@ class KernelProgramCache {
   /// The pointer is not null if and only if the entity is usable.
   /// State of the entity is provided by the user of cache instance.
   /// Currently there is only a single user - ProgramManager class.
-  template<typename T> struct BuildResult {
+  template <typename T> struct BuildResult {
     std::atomic<T *> Ptr;
     std::atomic<int> State;
     BuildError Error;
 
+    /// Condition variable to signal that build result is ready.
+    /// A per-object (i.e. kernel or program) condition variable is employed
+    /// instead of global one in order to eliminate the following deadlock.
+    /// A thread T1 awaiting for build result BR1 to be ready may be awakened by
+    /// another thread (due to use of global condition variable), which made
+    /// build result BR2 ready. Meanwhile, a thread which made build result BR1
+    /// ready notifies everyone via a global condition variable and T1 will skip
+    /// this notification as it's not in condition_variable::wait()'s wait cycle
+    /// now. Now T1 goes to sleep again and will wait until either a spurious
+    /// wake-up or another thread will wake it up.
+    std::condition_variable MBuildCV;
+    /// A mutex to be employed along with MBuildCV.
+    std::mutex MBuildResultMutex;
+
     BuildResult(T* P, int S) : Ptr{P}, State{S}, Error{"", 0} {}
   };
 
@@ -59,14 +73,8 @@ class KernelProgramCache {
 
   using PiKernelT = std::remove_pointer<RT::PiKernel>::type;
 
-  struct BuildResultKernel : public BuildResult<PiKernelT> {
-    std::mutex MKernelMutex;
-
-    BuildResultKernel(PiKernelT *P, int S) : BuildResult(P, S) {}
-  };
-
   using PiKernelPtrT = std::atomic<PiKernelT *>;
-  using KernelWithBuildStateT = BuildResultKernel;
+  using KernelWithBuildStateT = BuildResult<PiKernelT>;
   using KernelByNameT = std::map<string_class, KernelWithBuildStateT>;
   using KernelCacheT = std::map<RT::PiProgram, KernelByNameT>;
 
@@ -82,21 +90,21 @@ class KernelProgramCache {
     return {MKernelsPerProgramCache, MKernelsPerProgramCacheMutex};
   }
 
-  template <class Predicate> void waitUntilBuilt(Predicate Pred) const {
-    std::unique_lock<std::mutex> Lock(MBuildCVMutex);
+  template <typename T, class Predicate>
+  void waitUntilBuilt(BuildResult<T> &BR, Predicate Pred) const {
+    std::unique_lock<std::mutex> Lock(BR.MBuildResultMutex);
 
-    MBuildCV.wait(Lock, Pred);
+    BR.MBuildCV.wait(Lock, Pred);
   }
 
-  void notifyAllBuild() const { MBuildCV.notify_all(); }
+  template <typename T> void notifyAllBuild(BuildResult<T> &BR) const {
+    BR.MBuildCV.notify_all();
+  }
 
 private:
   std::mutex MProgramCacheMutex;
   std::mutex MKernelsPerProgramCacheMutex;
 
-  mutable std::condition_variable MBuildCV;
-  mutable std::mutex MBuildCVMutex;
-
   ProgramCacheT MCachedPrograms;
   KernelCacheT MKernelsPerProgramCache;
   ContextPtr MParentContext;
diff --git a/sycl/source/detail/memory_manager.cpp b/sycl/source/detail/memory_manager.cpp
index fc0d79b4c05cf..f80073fbddace 100644
--- a/sycl/source/detail/memory_manager.cpp
+++ b/sycl/source/detail/memory_manager.cpp
@@ -106,7 +106,8 @@ void *MemoryManager::wrapIntoImageBuffer(ContextImplPtr TargetContext,
 }
 
 void *MemoryManager::allocateHostMemory(SYCLMemObjI *MemObj, void *UserPtr,
-                                        bool HostPtrReadOnly, size_t Size) {
+                                        bool HostPtrReadOnly, size_t Size,
+                                        const sycl::property_list &) {
   // Can return user pointer directly if it points to writable memory.
   if (UserPtr && HostPtrReadOnly == false)
     return UserPtr;
@@ -123,7 +124,7 @@ void *MemoryManager::allocateHostMemory(SYCLMemObjI *MemObj, void *UserPtr,
 void *MemoryManager::allocateInteropMemObject(
     ContextImplPtr TargetContext, void *UserPtr,
     const EventImplPtr &InteropEvent, const ContextImplPtr &InteropContext,
-    RT::PiEvent &OutEventToWait) {
+    const sycl::property_list &, RT::PiEvent &OutEventToWait) {
   // If memory object is created with interop c'tor.
   // Return cl_mem as is if contexts match.
   if (TargetContext == InteropContext) {
@@ -144,7 +145,8 @@ void *MemoryManager::allocateInteropMemObject(
 void *MemoryManager::allocateImageObject(ContextImplPtr TargetContext,
                                          void *UserPtr, bool HostPtrReadOnly,
                                          const RT::PiMemImageDesc &Desc,
-                                         const RT::PiMemImageFormat &Format) {
+                                         const RT::PiMemImageFormat &Format,
+                                         const sycl::property_list &) {
   // Create read_write mem object by default to handle arbitrary uses.
   RT::PiMemFlags CreationFlags = PI_MEM_FLAGS_ACCESS_RW;
   if (UserPtr)
@@ -159,16 +161,20 @@ void *MemoryManager::allocateImageObject(ContextImplPtr TargetContext,
   return NewMem;
 }
 
-void *MemoryManager::allocateBufferObject(ContextImplPtr TargetContext,
-                                          void *UserPtr, bool HostPtrReadOnly,
-                                          const size_t Size) {
+void *
+MemoryManager::allocateBufferObject(ContextImplPtr TargetContext, void *UserPtr,
+                                    bool HostPtrReadOnly, const size_t Size,
+                                    const sycl::property_list &PropsList) {
   // Create read_write mem object by default to handle arbitrary uses.
   RT::PiMemFlags CreationFlags = PI_MEM_FLAGS_ACCESS_RW;
   if (UserPtr)
     CreationFlags |= HostPtrReadOnly ? PI_MEM_FLAGS_HOST_PTR_COPY
                                      : PI_MEM_FLAGS_HOST_PTR_USE;
+  else if (PropsList.has_property<
+               sycl::ext::oneapi::property::buffer::use_pinned_host_memory>())
+    CreationFlags |= PI_MEM_FLAGS_HOST_PTR_ALLOC;
 
-  RT::PiMem NewMem;
+  RT::PiMem NewMem = nullptr;
   const detail::plugin &Plugin = TargetContext->getPlugin();
   Plugin.call<PiApiKind::piMemBufferCreate>(
       TargetContext->getHandleRef(), CreationFlags, Size, UserPtr, &NewMem);
@@ -180,27 +186,32 @@ void *MemoryManager::allocateMemBuffer(ContextImplPtr TargetContext,
                                        bool HostPtrReadOnly, size_t Size,
                                        const EventImplPtr &InteropEvent,
                                        const ContextImplPtr &InteropContext,
+                                       const sycl::property_list &PropsList,
                                        RT::PiEvent &OutEventToWait) {
   if (TargetContext->is_host())
-    return allocateHostMemory(MemObj, UserPtr, HostPtrReadOnly, Size);
+    return allocateHostMemory(MemObj, UserPtr, HostPtrReadOnly, Size,
+                              PropsList);
   if (UserPtr && InteropContext)
     return allocateInteropMemObject(TargetContext, UserPtr, InteropEvent,
-                                    InteropContext, OutEventToWait);
-  return allocateBufferObject(TargetContext, UserPtr, HostPtrReadOnly, Size);
+                                    InteropContext, PropsList, OutEventToWait);
+  return allocateBufferObject(TargetContext, UserPtr, HostPtrReadOnly, Size,
+                              PropsList);
 }
 
 void *MemoryManager::allocateMemImage(
     ContextImplPtr TargetContext, SYCLMemObjI *MemObj, void *UserPtr,
     bool HostPtrReadOnly, size_t Size, const RT::PiMemImageDesc &Desc,
     const RT::PiMemImageFormat &Format, const EventImplPtr &InteropEvent,
-    const ContextImplPtr &InteropContext, RT::PiEvent &OutEventToWait) {
+    const ContextImplPtr &InteropContext, const sycl::property_list &PropsList,
+    RT::PiEvent &OutEventToWait) {
   if (TargetContext->is_host())
-    return allocateHostMemory(MemObj, UserPtr, HostPtrReadOnly, Size);
+    return allocateHostMemory(MemObj, UserPtr, HostPtrReadOnly, Size,
+                              PropsList);
   if (UserPtr && InteropContext)
     return allocateInteropMemObject(TargetContext, UserPtr, InteropEvent,
-                                    InteropContext, OutEventToWait);
+                                    InteropContext, PropsList, OutEventToWait);
   return allocateImageObject(TargetContext, UserPtr, HostPtrReadOnly, Desc,
-                             Format);
+                             Format, PropsList);
 }
 
 void *MemoryManager::allocateMemSubBuffer(ContextImplPtr TargetContext,
diff --git a/sycl/source/detail/pi.cpp b/sycl/source/detail/pi.cpp
index 58df2a3f7ed9f..68b9f3cf59a27 100644
--- a/sycl/source/detail/pi.cpp
+++ b/sycl/source/detail/pi.cpp
@@ -215,7 +215,7 @@ bool findPlugins(vector_class<std::pair<std::string, backend>> &PluginNames) {
   // env only.
   //
   PluginNames.emplace_back(OPENCL_PLUGIN_NAME, backend::opencl);
-  PluginNames.emplace_back(LEVEL0_PLUGIN_NAME, backend::level0);
+  PluginNames.emplace_back(LEVEL_ZERO_PLUGIN_NAME, backend::level_zero);
   PluginNames.emplace_back(CUDA_PLUGIN_NAME, backend::cuda);
   return true;
 }
@@ -319,11 +319,11 @@ static void initializePlugins(vector_class<plugin> *Plugins) {
                PluginNames[I].first.find("cuda") != std::string::npos) {
       // Use the CUDA plugin as the GlobalPlugin
       GlobalPlugin = std::make_shared<plugin>(PluginInformation, backend::cuda);
-    } else if (InteropBE == backend::level0 &&
-               PluginNames[I].first.find("level0") != std::string::npos) {
-      // Use the LEVEL0 plugin as the GlobalPlugin
+    } else if (InteropBE == backend::level_zero &&
+               PluginNames[I].first.find("level_zero") != std::string::npos) {
+      // Use the LEVEL_ZERO plugin as the GlobalPlugin
       GlobalPlugin =
-          std::make_shared<plugin>(PluginInformation, backend::level0);
+          std::make_shared<plugin>(PluginInformation, backend::level_zero);
     }
     Plugins->emplace_back(plugin(PluginInformation, PluginNames[I].second));
     if (trace(TraceLevel::PI_TRACE_BASIC))
@@ -395,7 +395,7 @@ template <backend BE> const plugin &getPlugin() {
 }
 
 template const plugin &getPlugin<backend::opencl>();
-template const plugin &getPlugin<backend::level0>();
+template const plugin &getPlugin<backend::level_zero>();
 
 // Report error and no return (keeps compiler from printing warnings).
 // TODO: Probably change that to throw a catchable exception,
diff --git a/sycl/source/detail/program_impl.cpp b/sycl/source/detail/program_impl.cpp
index ff90729b80367..a73064e455067 100644
--- a/sycl/source/detail/program_impl.cpp
+++ b/sycl/source/detail/program_impl.cpp
@@ -9,9 +9,9 @@
 #include <CL/sycl/detail/common.hpp>
 #include <CL/sycl/detail/kernel_desc.hpp>
 #include <CL/sycl/detail/pi.h>
-#include <CL/sycl/detail/spec_constant_impl.hpp>
 #include <CL/sycl/kernel.hpp>
 #include <detail/program_impl.hpp>
+#include <detail/spec_constant_impl.hpp>
 
 #include <algorithm>
 #include <fstream>
@@ -399,7 +399,7 @@ bool program_impl::has_cl_kernel(const string_class &KernelName) const {
 }
 
 RT::PiKernel program_impl::get_pi_kernel(const string_class &KernelName) const {
-  RT::PiKernel Kernel;
+  RT::PiKernel Kernel = nullptr;
 
   if (is_cacheable()) {
     std::tie(Kernel, std::ignore) =
@@ -416,6 +416,11 @@ RT::PiKernel program_impl::get_pi_kernel(const string_class &KernelName) const {
           Err);
     }
     Plugin.checkPiResult(Err);
+
+    // Some PI Plugins (like OpenCL) require this call to enable USM
+    // For others, PI will turn this into a NOP.
+    Plugin.call<PiApiKind::piKernelSetExecInfo>(Kernel, PI_USM_INDIRECT_ACCESS,
+                                                sizeof(pi_bool), &PI_TRUE);
   }
 
   return Kernel;
diff --git a/sycl/source/detail/program_impl.hpp b/sycl/source/detail/program_impl.hpp
index 2f58a01b8252c..b4153eb151339 100644
--- a/sycl/source/detail/program_impl.hpp
+++ b/sycl/source/detail/program_impl.hpp
@@ -10,12 +10,12 @@
 #include <CL/sycl/context.hpp>
 #include <CL/sycl/detail/common_info.hpp>
 #include <CL/sycl/detail/kernel_desc.hpp>
-#include <CL/sycl/detail/spec_constant_impl.hpp>
 #include <CL/sycl/device.hpp>
 #include <CL/sycl/program.hpp>
 #include <CL/sycl/stl.hpp>
 #include <detail/kernel_impl.hpp>
 #include <detail/program_manager/program_manager.hpp>
+#include <detail/spec_constant_impl.hpp>
 
 #include <algorithm>
 #include <cassert>
diff --git a/sycl/source/detail/program_manager/program_manager.cpp b/sycl/source/detail/program_manager/program_manager.cpp
index 78838d0178ee6..a4d7f162c1ab6 100644
--- a/sycl/source/detail/program_manager/program_manager.cpp
+++ b/sycl/source/detail/program_manager/program_manager.cpp
@@ -10,7 +10,6 @@
 #include <CL/sycl/context.hpp>
 #include <CL/sycl/detail/common.hpp>
 #include <CL/sycl/detail/os_util.hpp>
-#include <CL/sycl/detail/spec_constant_impl.hpp>
 #include <CL/sycl/detail/type_traits.hpp>
 #include <CL/sycl/detail/util.hpp>
 #include <CL/sycl/device.hpp>
@@ -21,6 +20,7 @@
 #include <detail/device_impl.hpp>
 #include <detail/program_impl.hpp>
 #include <detail/program_manager/program_manager.hpp>
+#include <detail/spec_constant_impl.hpp>
 
 #include <algorithm>
 #include <cassert>
@@ -125,7 +125,7 @@ RetT *waitUntilBuilt(KernelProgramCache &Cache,
                      KernelProgramCache::BuildResult<RetT> *BuildResult) {
   // any thread which will find nullptr in cache will wait until the pointer
   // is not null anymore
-  Cache.waitUntilBuilt([BuildResult]() {
+  Cache.waitUntilBuilt(*BuildResult, [BuildResult]() {
     int State = BuildResult->State.load();
 
     return State == BS_Done || State == BS_Failed;
@@ -212,7 +212,7 @@ getOrBuild(KernelProgramCache &KPCache, KeyT &&CacheKey, AcquireFT &&Acquire,
 
     BuildResult->State.store(BS_Done);
 
-    KPCache.notifyAllBuild();
+    KPCache.notifyAllBuild(*BuildResult);
 
     return BuildResult;
   } catch (const exception &Ex) {
@@ -221,13 +221,13 @@ getOrBuild(KernelProgramCache &KPCache, KeyT &&CacheKey, AcquireFT &&Acquire,
 
     BuildResult->State.store(BS_Failed);
 
-    KPCache.notifyAllBuild();
+    KPCache.notifyAllBuild(*BuildResult);
 
     std::rethrow_exception(std::current_exception());
   } catch (...) {
     BuildResult->State.store(BS_Failed);
 
-    KPCache.notifyAllBuild();
+    KPCache.notifyAllBuild(*BuildResult);
 
     std::rethrow_exception(std::current_exception());
   }
@@ -442,13 +442,18 @@ ProgramManager::getOrCreateKernel(OSModuleHandle M, const context &Context,
     Plugin.call<PiApiKind::piKernelCreate>(Program, KernelName.c_str(),
                                            &Result);
 
+    // Some PI Plugins (like OpenCL) require this call to enable USM
+    // For others, PI will turn this into a NOP.
+    Plugin.call<PiApiKind::piKernelSetExecInfo>(Result, PI_USM_INDIRECT_ACCESS,
+                                                sizeof(pi_bool), &PI_TRUE);
+
     return Result;
   };
 
-  auto BuildResult = static_cast<KernelProgramCache::BuildResultKernel *>(
-      getOrBuild<PiKernelT, invalid_object_error>(Cache, KernelName, AcquireF,
-                                                  GetF, BuildF));
-  return std::make_pair(BuildResult->Ptr.load(), &(BuildResult->MKernelMutex));
+  auto BuildResult = getOrBuild<PiKernelT, invalid_object_error>(
+      Cache, KernelName, AcquireF, GetF, BuildF);
+  return std::make_pair(BuildResult->Ptr.load(),
+                        &(BuildResult->MBuildResultMutex));
 }
 
 RT::PiProgram
@@ -783,19 +788,19 @@ ProgramManager::build(ProgramPtr Program, const ContextImplPtr Context,
     LinkOpts = LinkOptions.c_str();
   }
 
-  // L0 plugin doesn't support piProgramCompile/piProgramLink commands, program
-  // is built during piProgramCreate.
+  // Level-Zero plugin doesn't support piProgramCompile/piProgramLink commands,
+  // program is built during piProgramCreate.
   // TODO: remove this check as soon as piProgramCompile/piProgramLink will be
-  // implemented in L0 plugin.
-  if (Context->getPlugin().getBackend() == backend::level0) {
+  // implemented in Level-Zero plugin.
+  if (Context->getPlugin().getBackend() == backend::level_zero) {
     LinkDeviceLibs = false;
   }
 
   // TODO: this is a temporary workaround for GPU tests for ESIMD compiler.
   // We do not link with other device libraries, because it may fail
   // due to unrecognized SPIRV format of those libraries.
-  if (std::string(LinkOpts).find(std::string("-cmc")) != std::string::npos ||
-      std::string(LinkOpts).find(std::string("-vc-codegen")) !=
+  if (std::string(CompileOpts).find(std::string("-cmc")) != std::string::npos ||
+      std::string(CompileOpts).find(std::string("-vc-codegen")) !=
           std::string::npos)
     LinkDeviceLibs = false;
 
diff --git a/sycl/source/detail/program_manager/program_manager.hpp b/sycl/source/detail/program_manager/program_manager.hpp
index 952bf5ee6157b..9246d295cdc66 100644
--- a/sycl/source/detail/program_manager/program_manager.hpp
+++ b/sycl/source/detail/program_manager/program_manager.hpp
@@ -12,9 +12,9 @@
 #include <CL/sycl/detail/export.hpp>
 #include <CL/sycl/detail/os_util.hpp>
 #include <CL/sycl/detail/pi.hpp>
-#include <CL/sycl/detail/spec_constant_impl.hpp>
 #include <CL/sycl/detail/util.hpp>
 #include <CL/sycl/stl.hpp>
+#include <detail/spec_constant_impl.hpp>
 
 #include <cstdint>
 #include <map>
diff --git a/sycl/source/detail/queue_impl.cpp b/sycl/source/detail/queue_impl.cpp
index db2ceae4edda0..1ae5f1a68809a 100644
--- a/sycl/source/detail/queue_impl.cpp
+++ b/sycl/source/detail/queue_impl.cpp
@@ -192,11 +192,18 @@ void queue_impl::wait(const detail::code_location &CodeLoc) {
   TelemetryEvent = instrumentationProlog(CodeLoc, Name, StreamID, IId);
 #endif
 
-  std::lock_guard<mutex_class> Guard(MMutex);
-  for (std::weak_ptr<event_impl> &EventImplWeakPtr : MEvents) {
-    if (std::shared_ptr<event_impl> EventImplPtr = EventImplWeakPtr.lock())
-      EventImplPtr->wait(EventImplPtr);
+  std::vector<std::shared_ptr<event_impl>> Events;
+  {
+    std::lock_guard<mutex_class> Guard(MMutex);
+    for (std::weak_ptr<event_impl> &EventImplWeakPtr : MEvents)
+      if (std::shared_ptr<event_impl> EventImplPtr = EventImplWeakPtr.lock())
+        Events.push_back(EventImplPtr);
+  }
+
+  for (std::shared_ptr<event_impl> &Event : Events) {
+    Event->wait(Event);
   }
+
   for (event &Event : MUSMEvents) {
     Event.wait();
   }
diff --git a/sycl/source/detail/reduction.cpp b/sycl/source/detail/reduction.cpp
new file mode 100644
index 0000000000000..71e16724ce3b2
--- /dev/null
+++ b/sycl/source/detail/reduction.cpp
@@ -0,0 +1,67 @@
+//==---------------- reduction.cpp - SYCL reduction ------------*- C++ -*---==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <CL/sycl/intel/reduction.hpp>
+#include <detail/queue_impl.hpp>
+
+__SYCL_INLINE_NAMESPACE(cl) {
+namespace sycl {
+namespace intel {
+namespace detail {
+
+// TODO: The algorithm of choosing the work-group size is definitely
+// imperfect now and can be improved.
+__SYCL_EXPORT size_t reduComputeWGSize(size_t NWorkItems, size_t MaxWGSize,
+                                       size_t &NWorkGroups) {
+  size_t WGSize = MaxWGSize;
+  if (NWorkItems <= WGSize) {
+    NWorkGroups = 1;
+    WGSize = NWorkItems;
+  } else {
+    NWorkGroups = NWorkItems / WGSize;
+    size_t Rem = NWorkItems % WGSize;
+    if (Rem != 0) {
+      // Let's say MaxWGSize = 128 and NWorkItems is (128+32).
+      // It seems better to have 5 groups 32 work-items each than 2 groups with
+      // 128 work-items in the 1st group and 32 work-items in the 2nd group.
+      size_t NWorkGroupsAlt = NWorkItems / Rem;
+      size_t RemAlt = NWorkItems % Rem;
+      if (RemAlt == 0 && NWorkGroupsAlt <= MaxWGSize) {
+        NWorkGroups = NWorkGroupsAlt;
+        WGSize = Rem;
+      }
+    } else {
+      NWorkGroups++;
+    }
+  }
+  return WGSize;
+}
+
+__SYCL_EXPORT size_t
+reduGetMaxWGSize(shared_ptr_class<sycl::detail::queue_impl> Queue,
+                 size_t LocalMemBytesPerWorkItem) {
+  device Dev = Queue->get_device();
+  size_t WGSize = Dev.get_info<info::device::max_work_group_size>();
+  if (LocalMemBytesPerWorkItem != 0) {
+    size_t MemSize = Dev.get_info<info::device::local_mem_size>();
+    size_t WGSizePerMem = MemSize / LocalMemBytesPerWorkItem;
+
+    // If the work group size is not pow of two, then an additional element
+    // in local memory is needed for the reduction algorithm and thus the real
+    // work-group size requirement per available memory is stricter.
+    if ((WGSize & (WGSize - 1)) == 0)
+      WGSizePerMem--;
+    WGSize = (std::min)(WGSizePerMem, WGSize);
+  }
+  return WGSize;
+}
+
+} // namespace detail
+} // namespace intel
+} // namespace sycl
+} // __SYCL_INLINE_NAMESPACE(cl)
diff --git a/sycl/source/detail/sampler_impl.cpp b/sycl/source/detail/sampler_impl.cpp
index 1e875839060e1..6ef03918f2862 100644
--- a/sycl/source/detail/sampler_impl.cpp
+++ b/sycl/source/detail/sampler_impl.cpp
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <CL/sycl/detail/sampler_impl.hpp>
 #include <detail/context_impl.hpp>
+#include <detail/sampler_impl.hpp>
 
 __SYCL_INLINE_NAMESPACE(cl) {
 namespace sycl {
diff --git a/sycl/include/CL/sycl/detail/sampler_impl.hpp b/sycl/source/detail/sampler_impl.hpp
similarity index 86%
rename from sycl/include/CL/sycl/detail/sampler_impl.hpp
rename to sycl/source/detail/sampler_impl.hpp
index 1f3b013d78768..271942d39ad0b 100644
--- a/sycl/include/CL/sycl/detail/sampler_impl.hpp
+++ b/sycl/source/detail/sampler_impl.hpp
@@ -24,11 +24,6 @@ enum class coordinate_normalization_mode : unsigned int;
 namespace detail {
 class __SYCL_EXPORT sampler_impl {
 public:
-#ifdef __SYCL_DEVICE_ONLY__
-  __ocl_sampler_t m_Sampler;
-  sampler_impl(__ocl_sampler_t Sampler) : m_Sampler(Sampler) {}
-  sampler_impl() = default;
-#else
   std::unordered_map<context, RT::PiSampler> m_contextToSampler;
 
 private:
@@ -49,13 +44,8 @@ class __SYCL_EXPORT sampler_impl {
   coordinate_normalization_mode get_coordinate_normalization_mode() const;
 
   RT::PiSampler getOrCreateSampler(const context &Context);
-#endif
 
-#ifdef __SYCL_DEVICE_ONLY__
-  ~sampler_impl() = default;
-#else
   ~sampler_impl();
-#endif
 };
 
 } // namespace detail
diff --git a/sycl/source/detail/scheduler/commands.cpp b/sycl/source/detail/scheduler/commands.cpp
index a8804fb7958ee..f21d39d8efc3d 100644
--- a/sycl/source/detail/scheduler/commands.cpp
+++ b/sycl/source/detail/scheduler/commands.cpp
@@ -14,7 +14,6 @@
 #include <CL/sycl/detail/cl.h>
 #include <CL/sycl/detail/kernel_desc.hpp>
 #include <CL/sycl/detail/memory_manager.hpp>
-#include <CL/sycl/detail/stream_impl.hpp>
 #include <CL/sycl/sampler.hpp>
 #include <detail/context_impl.hpp>
 #include <detail/event_impl.hpp>
@@ -22,8 +21,10 @@
 #include <detail/kernel_info.hpp>
 #include <detail/program_manager/program_manager.hpp>
 #include <detail/queue_impl.hpp>
+#include <detail/sampler_impl.hpp>
 #include <detail/scheduler/commands.hpp>
 #include <detail/scheduler/scheduler.hpp>
+#include <detail/stream_impl.hpp>
 
 #include <cassert>
 #include <string>
@@ -200,15 +201,19 @@ class DispatchHostTask {
 
     CGHostTask &HostTask = static_cast<CGHostTask &>(MThisCmd->getCG());
 
-    // we're ready to call the user-defined lambda now
-    if (HostTask.MHostTask->isInteropTask()) {
-      interop_handle IH{MReqToMem, HostTask.MQueue,
-                        getSyclObjImpl(HostTask.MQueue->get_device()),
-                        HostTask.MQueue->getContextImplPtr()};
-
-      HostTask.MHostTask->call(IH);
-    } else
-      HostTask.MHostTask->call();
+    try {
+      // we're ready to call the user-defined lambda now
+      if (HostTask.MHostTask->isInteropTask()) {
+        interop_handle IH{MReqToMem, HostTask.MQueue,
+                          getSyclObjImpl(HostTask.MQueue->get_device()),
+                          HostTask.MQueue->getContextImplPtr()};
+
+        HostTask.MHostTask->call(IH);
+      } else
+        HostTask.MHostTask->call();
+    } catch (...) {
+      HostTask.MQueue->reportAsyncException(std::current_exception());
+    }
 
     HostTask.MHostTask.reset();
 
@@ -1686,11 +1691,6 @@ pi_result ExecCGCommand::SetKernelParamsAndLaunch(
   adjustNDRangePerKernel(NDRDesc, Kernel,
                          *(detail::getSyclObjImpl(MQueue->get_device())));
 
-  // Some PI Plugins (like OpenCL) require this call to enable USM
-  // For others, PI will turn this into a NOP.
-  Plugin.call<PiApiKind::piKernelSetExecInfo>(Kernel, PI_USM_INDIRECT_ACCESS,
-                                              sizeof(pi_bool), &PI_TRUE);
-
   // Remember this information before the range dimensions are reversed
   const bool HasLocalSize = (NDRDesc.LocalSize[0] != 0);
 
diff --git a/sycl/source/detail/scheduler/commands.hpp b/sycl/source/detail/scheduler/commands.hpp
index 6c2330fed7061..04d85d5206739 100644
--- a/sycl/source/detail/scheduler/commands.hpp
+++ b/sycl/source/detail/scheduler/commands.hpp
@@ -219,8 +219,15 @@ class Command {
   bool MIsBlockable = false;
   /// Counts the number of memory objects this command is a leaf for.
   unsigned MLeafCounter = 0;
-  /// Used for marking the node as visited during graph traversal.
-  bool MVisited = false;
+
+  struct Marks {
+    /// Used for marking the node as visited during graph traversal.
+    bool MVisited = false;
+    /// Used for marking the node for deletion during cleanup.
+    bool MToBeDeleted = false;
+  };
+  /// Used for marking the node during graph traversal.
+  Marks MMarks;
 
   enum class BlockReason : int { HostAccessor = 0, HostTask };
 
@@ -272,15 +279,15 @@ class EmptyCommand : public Command {
 public:
   EmptyCommand(QueueImplPtr Queue);
 
-  void printDot(std::ostream &Stream) const final;
-  const Requirement *getRequirement() const final { return &MRequirements[0]; }
+  void printDot(std::ostream &Stream) const final override;
+  const Requirement *getRequirement() const final override { return &MRequirements[0]; }
   void addRequirement(Command *DepCmd, AllocaCommandBase *AllocaCmd,
                       const Requirement *Req);
 
-  void emitInstrumentationData();
+  void emitInstrumentationData() override;
 
 private:
-  cl_int enqueueImp() final;
+  cl_int enqueueImp() final override;
 
   // Employing deque here as it allows to push_back/emplace_back without
   // invalidation of pointer or reference to stored data item regardless of
@@ -294,11 +301,11 @@ class ReleaseCommand : public Command {
 public:
   ReleaseCommand(QueueImplPtr Queue, AllocaCommandBase *AllocaCmd);
 
-  void printDot(std::ostream &Stream) const final;
-  void emitInstrumentationData();
+  void printDot(std::ostream &Stream) const final override;
+  void emitInstrumentationData() override;
 
 private:
-  cl_int enqueueImp() final;
+  cl_int enqueueImp() final override;
 
   /// Command which allocates memory release command should dealocate.
   AllocaCommandBase *MAllocaCmd = nullptr;
@@ -316,9 +323,9 @@ class AllocaCommandBase : public Command {
 
   virtual void *getMemAllocation() const = 0;
 
-  const Requirement *getRequirement() const final { return &MRequirement; }
+  const Requirement *getRequirement() const final override { return &MRequirement; }
 
-  void emitInstrumentationData();
+  void emitInstrumentationData() override;
 
   void *MMemAllocation = nullptr;
 
@@ -355,12 +362,12 @@ class AllocaCommand : public AllocaCommandBase {
                 bool InitFromUserData = true,
                 AllocaCommandBase *LinkedAllocaCmd = nullptr);
 
-  void *getMemAllocation() const final { return MMemAllocation; }
-  void printDot(std::ostream &Stream) const final;
-  void emitInstrumentationData();
+  void *getMemAllocation() const final override { return MMemAllocation; }
+  void printDot(std::ostream &Stream) const final override;
+  void emitInstrumentationData() override;
 
 private:
-  cl_int enqueueImp() final;
+  cl_int enqueueImp() final override;
 
   /// The flag indicates that alloca should try to reuse pointer provided by
   /// the user during memory object construction.
@@ -373,13 +380,13 @@ class AllocaSubBufCommand : public AllocaCommandBase {
   AllocaSubBufCommand(QueueImplPtr Queue, Requirement Req,
                       AllocaCommandBase *ParentAlloca);
 
-  void *getMemAllocation() const final;
-  void printDot(std::ostream &Stream) const final;
+  void *getMemAllocation() const final override;
+  void printDot(std::ostream &Stream) const final override;
   AllocaCommandBase *getParentAlloca() { return MParentAlloca; }
-  void emitInstrumentationData();
+  void emitInstrumentationData() override;
 
 private:
-  cl_int enqueueImp() final;
+  cl_int enqueueImp() final override;
 
   AllocaCommandBase *MParentAlloca = nullptr;
 };
@@ -390,12 +397,12 @@ class MapMemObject : public Command {
   MapMemObject(AllocaCommandBase *SrcAllocaCmd, Requirement Req, void **DstPtr,
                QueueImplPtr Queue, access::mode MapMode);
 
-  void printDot(std::ostream &Stream) const final;
-  const Requirement *getRequirement() const final { return &MSrcReq; }
-  void emitInstrumentationData();
+  void printDot(std::ostream &Stream) const final override;
+  const Requirement *getRequirement() const final override { return &MSrcReq; }
+  void emitInstrumentationData() override;
 
 private:
-  cl_int enqueueImp() final;
+  cl_int enqueueImp() final override;
 
   AllocaCommandBase *MSrcAllocaCmd = nullptr;
   Requirement MSrcReq;
@@ -409,12 +416,12 @@ class UnMapMemObject : public Command {
   UnMapMemObject(AllocaCommandBase *DstAllocaCmd, Requirement Req,
                  void **SrcPtr, QueueImplPtr Queue);
 
-  void printDot(std::ostream &Stream) const final;
-  const Requirement *getRequirement() const final { return &MDstReq; }
-  void emitInstrumentationData();
+  void printDot(std::ostream &Stream) const final override;
+  const Requirement *getRequirement() const final override { return &MDstReq; }
+  void emitInstrumentationData() override;
 
 private:
-  cl_int enqueueImp() final;
+  cl_int enqueueImp() final override;
 
   AllocaCommandBase *MDstAllocaCmd = nullptr;
   Requirement MDstReq;
@@ -429,13 +436,13 @@ class MemCpyCommand : public Command {
                 Requirement DstReq, AllocaCommandBase *DstAllocaCmd,
                 QueueImplPtr SrcQueue, QueueImplPtr DstQueue);
 
-  void printDot(std::ostream &Stream) const final;
-  const Requirement *getRequirement() const final { return &MDstReq; }
-  void emitInstrumentationData() final;
-  ContextImplPtr getContext() const final;
+  void printDot(std::ostream &Stream) const final override;
+  const Requirement *getRequirement() const final override { return &MDstReq; }
+  void emitInstrumentationData() final override;
+  ContextImplPtr getContext() const final override;
 
 private:
-  cl_int enqueueImp() final;
+  cl_int enqueueImp() final override;
 
   QueueImplPtr MSrcQueue;
   Requirement MSrcReq;
@@ -452,13 +459,13 @@ class MemCpyCommandHost : public Command {
                     Requirement DstReq, void **DstPtr, QueueImplPtr SrcQueue,
                     QueueImplPtr DstQueue);
 
-  void printDot(std::ostream &Stream) const final;
-  const Requirement *getRequirement() const final { return &MDstReq; }
-  void emitInstrumentationData() final;
-  ContextImplPtr getContext() const final;
+  void printDot(std::ostream &Stream) const final override;
+  const Requirement *getRequirement() const final override { return &MDstReq; }
+  void emitInstrumentationData() final override;
+  ContextImplPtr getContext() const final override;
 
 private:
-  cl_int enqueueImp() final;
+  cl_int enqueueImp() final override;
 
   QueueImplPtr MSrcQueue;
   Requirement MSrcReq;
@@ -475,8 +482,8 @@ class ExecCGCommand : public Command {
 
   void flushStreams();
 
-  void printDot(std::ostream &Stream) const final;
-  void emitInstrumentationData() final;
+  void printDot(std::ostream &Stream) const final override;
+  void emitInstrumentationData() final override;
 
   detail::CG &getCG() const { return *MCommandGroup; }
 
@@ -487,7 +494,7 @@ class ExecCGCommand : public Command {
   EmptyCommand *MEmptyCmd = nullptr;
 
 private:
-  cl_int enqueueImp() final;
+  cl_int enqueueImp() final override;
 
   AllocaCommandBase *getAllocaForReq(Requirement *Req);
 
@@ -506,12 +513,12 @@ class UpdateHostRequirementCommand : public Command {
   UpdateHostRequirementCommand(QueueImplPtr Queue, Requirement Req,
                                AllocaCommandBase *SrcAllocaCmd, void **DstPtr);
 
-  void printDot(std::ostream &Stream) const final;
-  const Requirement *getRequirement() const final { return &MDstReq; }
-  void emitInstrumentationData() final;
+  void printDot(std::ostream &Stream) const final override;
+  const Requirement *getRequirement() const final override { return &MDstReq; }
+  void emitInstrumentationData() final override;
 
 private:
-  cl_int enqueueImp() final;
+  cl_int enqueueImp() final override;
 
   AllocaCommandBase *MSrcAllocaCmd = nullptr;
   Requirement MDstReq;
diff --git a/sycl/source/detail/scheduler/graph_builder.cpp b/sycl/source/detail/scheduler/graph_builder.cpp
index 8f4da1cb46a16..1532ab3da8666 100644
--- a/sycl/source/detail/scheduler/graph_builder.cpp
+++ b/sycl/source/detail/scheduler/graph_builder.cpp
@@ -93,16 +93,27 @@ Scheduler::GraphBuilder::GraphBuilder() {
 }
 
 static bool markNodeAsVisited(Command *Cmd, std::vector<Command *> &Visited) {
-  if (Cmd->MVisited)
+  assert(Cmd && "Cmd can't be nullptr");
+  if (Cmd->MMarks.MVisited)
     return false;
-  Cmd->MVisited = true;
+  Cmd->MMarks.MVisited = true;
   Visited.push_back(Cmd);
   return true;
 }
 
 static void unmarkVisitedNodes(std::vector<Command *> &Visited) {
   for (Command *Cmd : Visited)
-    Cmd->MVisited = false;
+    Cmd->MMarks.MVisited = false;
+}
+
+static void handleVisitedNodes(std::vector<Command *> &Visited) {
+  for (Command *Cmd : Visited) {
+    if (Cmd->MMarks.MToBeDeleted) {
+      Cmd->getEvent()->setCommand(nullptr);
+      delete Cmd;
+    } else
+      Cmd->MMarks.MVisited = false;
+  }
 }
 
 static void printDotRecursive(std::fstream &Stream,
@@ -825,7 +836,6 @@ void Scheduler::GraphBuilder::cleanupCommandsForRecord(MemObjRecord *Record) {
 
   std::queue<Command *> ToVisit;
   std::vector<Command *> Visited;
-  std::vector<Command *> CmdsToDelete;
   // First, mark all allocas for deletion and their direct users for traversal
   // Dependencies of the users will be cleaned up during the traversal
   for (Command *AllocaCmd : AllocaCommands) {
@@ -839,7 +849,7 @@ void Scheduler::GraphBuilder::cleanupCommandsForRecord(MemObjRecord *Record) {
       else
         markNodeAsVisited(UserCmd, Visited);
 
-    CmdsToDelete.push_back(AllocaCmd);
+    AllocaCmd->MMarks.MToBeDeleted = true;
     // These commands will be deleted later, clear users now to avoid
     // updating them during edge removal
     AllocaCmd->MUsers.clear();
@@ -851,7 +861,7 @@ void Scheduler::GraphBuilder::cleanupCommandsForRecord(MemObjRecord *Record) {
     AllocaCommandBase *LinkedCmd = AllocaCmd->MLinkedAllocaCmd;
 
     if (LinkedCmd) {
-      assert(LinkedCmd->MVisited);
+      assert(LinkedCmd->MMarks.MVisited);
 
       for (DepDesc &Dep : AllocaCmd->MDeps)
         if (Dep.MDepCommand)
@@ -896,17 +906,12 @@ void Scheduler::GraphBuilder::cleanupCommandsForRecord(MemObjRecord *Record) {
     // If all dependencies have been removed this way, mark the command for
     // deletion
     if (Cmd->MDeps.empty()) {
-      CmdsToDelete.push_back(Cmd);
+      Cmd->MMarks.MToBeDeleted = true;
       Cmd->MUsers.clear();
     }
   }
 
-  unmarkVisitedNodes(Visited);
-
-  for (Command *Cmd : CmdsToDelete) {
-    Cmd->getEvent()->setCommand(nullptr);
-    delete Cmd;
-  }
+  handleVisitedNodes(Visited);
 }
 
 void Scheduler::GraphBuilder::cleanupFinishedCommands(Command *FinishedCmd) {
@@ -948,12 +953,10 @@ void Scheduler::GraphBuilder::cleanupFinishedCommands(Command *FinishedCmd) {
       Command *DepCmd = Dep.MDepCommand;
       DepCmd->MUsers.erase(Cmd);
     }
-    Cmd->getEvent()->setCommand(nullptr);
 
-    Visited.pop_back();
-    delete Cmd;
+    Cmd->MMarks.MToBeDeleted = true;
   }
-  unmarkVisitedNodes(Visited);
+  handleVisitedNodes(Visited);
 }
 
 void Scheduler::GraphBuilder::removeRecordForMemObj(SYCLMemObjI *MemObject) {
diff --git a/sycl/source/detail/scheduler/scheduler.hpp b/sycl/source/detail/scheduler/scheduler.hpp
index b7d580e5d1823..1550cfd628d47 100644
--- a/sycl/source/detail/scheduler/scheduler.hpp
+++ b/sycl/source/detail/scheduler/scheduler.hpp
@@ -9,8 +9,8 @@
 #pragma once
 
 #include <CL/sycl/detail/cg.hpp>
-#include <CL/sycl/detail/circular_buffer.hpp>
 #include <CL/sycl/detail/sycl_mem_obj_i.hpp>
+#include <detail/circular_buffer.hpp>
 #include <detail/scheduler/commands.hpp>
 
 #include <cstddef>
diff --git a/sycl/source/detail/spec_constant_impl.cpp b/sycl/source/detail/spec_constant_impl.cpp
index b196aa8c91684..f61db6fe4a0bc 100644
--- a/sycl/source/detail/spec_constant_impl.cpp
+++ b/sycl/source/detail/spec_constant_impl.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <CL/sycl/detail/spec_constant_impl.hpp>
+#include <detail/spec_constant_impl.hpp>
 
 #include <CL/sycl/detail/defines.hpp>
 #include <CL/sycl/detail/pi.h>
diff --git a/sycl/include/CL/sycl/detail/spec_constant_impl.hpp b/sycl/source/detail/spec_constant_impl.hpp
similarity index 100%
rename from sycl/include/CL/sycl/detail/spec_constant_impl.hpp
rename to sycl/source/detail/spec_constant_impl.hpp
diff --git a/sycl/source/detail/stream_impl.cpp b/sycl/source/detail/stream_impl.cpp
index 76247c239728a..db79b83fd1408 100644
--- a/sycl/source/detail/stream_impl.cpp
+++ b/sycl/source/detail/stream_impl.cpp
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <CL/sycl/detail/stream_impl.hpp>
 #include <detail/scheduler/scheduler.hpp>
+#include <detail/stream_impl.hpp>
 
 #include <cstdio>
 
diff --git a/sycl/source/detail/stream_impl.hpp b/sycl/source/detail/stream_impl.hpp
new file mode 100644
index 0000000000000..9091441ce4a4d
--- /dev/null
+++ b/sycl/source/detail/stream_impl.hpp
@@ -0,0 +1,81 @@
+//==----------------- stream_impl.hpp - SYCL standard header file ----------==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include <CL/sycl/accessor.hpp>
+#include <CL/sycl/buffer.hpp>
+#include <CL/sycl/detail/export.hpp>
+#include <CL/sycl/handler.hpp>
+#include <CL/sycl/range.hpp>
+#include <CL/sycl/stream.hpp>
+
+#include <vector>
+
+__SYCL_INLINE_NAMESPACE(cl) {
+namespace sycl {
+namespace detail {
+
+class __SYCL_EXPORT stream_impl {
+public:
+  stream_impl(size_t BufferSize, size_t MaxStatementSize, handler &CGH);
+
+  // Method to provide an access to the global stream buffer
+  GlobalBufAccessorT accessGlobalBuf(handler &CGH) {
+    return Buf.get_access<cl::sycl::access::mode::read_write>(
+        CGH, range<1>(BufferSize_), id<1>(OffsetSize));
+  }
+
+  // Method to provide an accessor to the global flush buffer
+  GlobalBufAccessorT accessGlobalFlushBuf(handler &CGH) {
+    return FlushBuf.get_access<cl::sycl::access::mode::read_write>(
+        CGH, range<1>(MaxStatementSize_), id<1>(0));
+  }
+
+  // Method to provide an atomic access to the offset in the global stream
+  // buffer
+  GlobalOffsetAccessorT accessGlobalOffset(handler &CGH) {
+    auto OffsetSubBuf = buffer<char, 1>(Buf, id<1>(0), range<1>(OffsetSize));
+    auto ReinterpretedBuf = OffsetSubBuf.reinterpret<unsigned, 1>(range<1>(1));
+    return ReinterpretedBuf.get_access<cl::sycl::access::mode::atomic>(
+        CGH, range<1>(1), id<1>(0));
+  }
+
+  // Copy stream buffer to the host and print the contents
+  void flush();
+
+  size_t get_size() const;
+
+  size_t get_max_statement_size() const;
+
+private:
+  // Size of the stream buffer
+  size_t BufferSize_;
+
+  // Maximum number of symbols which could be streamed from the beginning of a
+  // statement till the semicolon
+  unsigned MaxStatementSize_;
+
+  // Size of the variable which is used as an offset in the stream buffer.
+  // Additinonal memory is allocated in the beginning of the stream buffer for
+  // this variable.
+  static const size_t OffsetSize = sizeof(unsigned);
+
+  // Vector on the host side which is used to initialize the stream buffer
+  std::vector<char> Data;
+
+  // Stream buffer
+  buffer<char, 1> Buf;
+
+  // Global flush buffer
+  buffer<char, 1> FlushBuf;
+};
+
+} // namespace detail
+} // namespace sycl
+} // __SYCL_INLINE_NAMESPACE(cl)
diff --git a/sycl/source/device_selector.cpp b/sycl/source/device_selector.cpp
index ce6011439f4bf..831ae5f124bfa 100644
--- a/sycl/source/device_selector.cpp
+++ b/sycl/source/device_selector.cpp
@@ -19,13 +19,13 @@ __SYCL_INLINE_NAMESPACE(cl) {
 namespace sycl {
 
 // Utility function to check if device is of the preferred backend.
-// Currently preference is given to the level0 backend.
+// Currently preference is given to the level_zero backend.
 static bool isDeviceOfPreferredSyclBe(const device &Device) {
   if (Device.is_host())
     return false;
 
   return detail::getSyclObjImpl(Device)->getPlugin().getBackend() ==
-         backend::level0;
+         backend::level_zero;
 }
 
 device device_selector::select_device() const {
@@ -37,14 +37,14 @@ device device_selector::select_device() const {
     int dev_score = (*this)(dev);
 
     if (detail::pi::trace(detail::pi::TraceLevel::PI_TRACE_ALL)) {
-      string_class PlatformVersion = dev.get_info<info::device::platform>()
-                                         .get_info<info::platform::version>();
+      string_class PlatformName = dev.get_info<info::device::platform>()
+                                      .get_info<info::platform::name>();
       string_class DeviceName = dev.get_info<info::device::name>();
       std::cout << "SYCL_PI_TRACE[all]: "
-                << "select_device(): -> score = " << score
-                << ((score < 0) ? "(REJECTED)" : " ") << std::endl
+                << "select_device(): -> score = " << dev_score
+                << ((dev_score < 0) ? " (REJECTED)" : "") << std::endl
                 << "SYCL_PI_TRACE[all]: "
-                << "  platform: " << PlatformVersion << std::endl
+                << "  platform: " << PlatformName << std::endl
                 << "SYCL_PI_TRACE[all]: "
                 << "  device: " << DeviceName << std::endl;
     }
@@ -67,13 +67,13 @@ device device_selector::select_device() const {
 
   if (res != nullptr) {
     if (detail::pi::trace(detail::pi::TraceLevel::PI_TRACE_BASIC)) {
-      string_class PlatformVersion = res->get_info<info::device::platform>()
-                                         .get_info<info::platform::version>();
+      string_class PlatformName = res->get_info<info::device::platform>()
+                                      .get_info<info::platform::name>();
       string_class DeviceName = res->get_info<info::device::name>();
       std::cout << "SYCL_PI_TRACE[all]: "
                 << "Selected device ->" << std::endl
                 << "SYCL_PI_TRACE[all]: "
-                << "  platform: " << PlatformVersion << std::endl
+                << "  platform: " << PlatformName << std::endl
                 << "SYCL_PI_TRACE[all]: "
                 << "  device: " << DeviceName << std::endl;
     }
diff --git a/sycl/source/sampler.cpp b/sycl/source/sampler.cpp
index 8a5668905e3c5..2adca5c4fee55 100644
--- a/sycl/source/sampler.cpp
+++ b/sycl/source/sampler.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <CL/sycl/sampler.hpp>
+#include <detail/sampler_impl.hpp>
 
 __SYCL_INLINE_NAMESPACE(cl) {
 namespace sycl {
diff --git a/sycl/source/stream.cpp b/sycl/source/stream.cpp
index d0378cbb3eb6c..88b332051c5db 100644
--- a/sycl/source/stream.cpp
+++ b/sycl/source/stream.cpp
@@ -8,6 +8,7 @@
 
 #include <CL/sycl/stream.hpp>
 #include <detail/queue_impl.hpp>
+#include <detail/stream_impl.hpp>
 
 __SYCL_INLINE_NAMESPACE(cl) {
 namespace sycl {
diff --git a/sycl/test/.clang-format b/sycl/test/.clang-format
index 4799b66f3e9a6..0fb3ce609e376 100644
--- a/sycl/test/.clang-format
+++ b/sycl/test/.clang-format
@@ -1,2 +1,2 @@
 BasedOnStyle: LLVM
-ColumnLimit: 0
+CommentPragmas: "(RUN|FAIL|REQUIRES|UNSUPPORTED|CHECK) *:|expected-"
diff --git a/sycl/test/CMakeLists.txt b/sycl/test/CMakeLists.txt
index 4010c13d3b219..882de8263cb36 100644
--- a/sycl/test/CMakeLists.txt
+++ b/sycl/test/CMakeLists.txt
@@ -81,17 +81,17 @@ add_lit_testsuite(check-sycl-opencl "Running the SYCL regression tests for OpenC
   )
 set_target_properties(check-sycl-opencl PROPERTIES FOLDER "SYCL tests")
 
-add_lit_testsuite(check-sycl-level0 "Running the SYCL regression tests for Level Zero"
+add_lit_testsuite(check-sycl-level-zero "Running the SYCL regression tests for Level Zero"
   ${CMAKE_CURRENT_BINARY_DIR}
   ARGS ${RT_TEST_ARGS}
-  PARAMS "SYCL_BE=PI_LEVEL0"
+  PARAMS "SYCL_BE=PI_LEVEL_ZERO"
   DEPENDS ${SYCL_TEST_DEPS}
   EXCLUDE_FROM_CHECK_ALL
   )
-set_target_properties(check-sycl-level0 PROPERTIES FOLDER "SYCL tests")
+set_target_properties(check-sycl-level-zero PROPERTIES FOLDER "SYCL tests")
 
 add_custom_target(check-sycl)
-add_dependencies(check-sycl check-sycl-opencl check-sycl-level0)
+add_dependencies(check-sycl check-sycl-opencl check-sycl-level-zero)
 set_target_properties(check-sycl PROPERTIES FOLDER "SYCL tests")
 
 if(SYCL_BUILD_PI_CUDA)
diff --git a/sycl/test/abi/pi_level0_symbol_check.dump b/sycl/test/abi/pi_level_zero_symbol_check.dump
similarity index 98%
rename from sycl/test/abi/pi_level0_symbol_check.dump
rename to sycl/test/abi/pi_level_zero_symbol_check.dump
index f6bb19aa16a5b..7c8c74b8cc77d 100644
--- a/sycl/test/abi/pi_level0_symbol_check.dump
+++ b/sycl/test/abi/pi_level_zero_symbol_check.dump
@@ -3,7 +3,7 @@
 # DO NOT EDIT IT MANUALLY. Refer to sycl/docs/ABIPolicyGuide.md for more info.
 ################################################################################
 
-# RUN: env LLVM_BIN_PATH=%llvm_build_bin_dir python %sycl_tools_src_dir/abi_check.py --mode check_symbols --reference %s %sycl_libs_dir/libpi_level0.so
+# RUN: env LLVM_BIN_PATH=%llvm_build_bin_dir python %sycl_tools_src_dir/abi_check.py --mode check_symbols --reference %s %sycl_libs_dir/libpi_level_zero.so
 # REQUIRES: linux
 
 piContextCreate
diff --git a/sycl/test/abi/sycl_symbols_linux.dump b/sycl/test/abi/sycl_symbols_linux.dump
index ce0f108c77243..0b336464462d3 100644
--- a/sycl/test/abi/sycl_symbols_linux.dump
+++ b/sycl/test/abi/sycl_symbols_linux.dump
@@ -3589,6 +3589,10 @@ _ZN2cl10__host_std9u_sub_satEhh
 _ZN2cl10__host_std9u_sub_satEjj
 _ZN2cl10__host_std9u_sub_satEmm
 _ZN2cl10__host_std9u_sub_satEtt
+_ZN2cl4sycl10level_zero10make_queueERKNS0_7contextEm
+_ZN2cl4sycl10level_zero11make_deviceERKNS0_8platformEm
+_ZN2cl4sycl10level_zero12make_programERKNS0_7contextEm
+_ZN2cl4sycl10level_zero13make_platformEm
 _ZN2cl4sycl11malloc_hostEmRKNS0_5queueE
 _ZN2cl4sycl11malloc_hostEmRKNS0_7contextE
 _ZN2cl4sycl13aligned_allocEmmRKNS0_5queueENS0_3usm5allocE
@@ -3621,6 +3625,8 @@ _ZN2cl4sycl5eventC1Ev
 _ZN2cl4sycl5eventC2EP9_cl_eventRKNS0_7contextE
 _ZN2cl4sycl5eventC2ESt10shared_ptrINS0_6detail10event_implEE
 _ZN2cl4sycl5eventC2Ev
+_ZN2cl4sycl5intel6detail16reduGetMaxWGSizeESt10shared_ptrINS0_6detail10queue_implEEm
+_ZN2cl4sycl5intel6detail17reduComputeWGSizeEmmRm
 _ZN2cl4sycl5queue10mem_adviseEPKvm14_pi_mem_advice
 _ZN2cl4sycl5queue10wait_proxyERKNS0_6detail13code_locationE
 _ZN2cl4sycl5queue11submit_implESt8functionIFvRNS0_7handlerEEERKNS0_6detail13code_locationE
@@ -3726,15 +3732,15 @@ _ZN2cl4sycl6detail12sampler_implD2Ev
 _ZN2cl4sycl6detail12split_stringERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEc
 _ZN2cl4sycl6detail13MemoryManager12prefetch_usmEPvSt10shared_ptrINS1_10queue_implEEmSt6vectorIP9_pi_eventSaIS9_EERS9_
 _ZN2cl4sycl6detail13MemoryManager13releaseMemObjESt10shared_ptrINS1_12context_implEEPNS1_11SYCLMemObjIEPvS8_
-_ZN2cl4sycl6detail13MemoryManager16allocateMemImageESt10shared_ptrINS1_12context_implEEPNS1_11SYCLMemObjIEPvbmRK14_pi_image_descRK16_pi_image_formatRKS3_INS1_10event_implEERKS5_RP9_pi_event
-_ZN2cl4sycl6detail13MemoryManager17allocateMemBufferESt10shared_ptrINS1_12context_implEEPNS1_11SYCLMemObjIEPvbmRKS3_INS1_10event_implEERKS5_RP9_pi_event
-_ZN2cl4sycl6detail13MemoryManager18allocateHostMemoryEPNS1_11SYCLMemObjIEPvbm
+_ZN2cl4sycl6detail13MemoryManager16allocateMemImageESt10shared_ptrINS1_12context_implEEPNS1_11SYCLMemObjIEPvbmRK14_pi_image_descRK16_pi_image_formatRKS3_INS1_10event_implEERKS5_RKNS0_13property_listERP9_pi_event
+_ZN2cl4sycl6detail13MemoryManager17allocateMemBufferESt10shared_ptrINS1_12context_implEEPNS1_11SYCLMemObjIEPvbmRKS3_INS1_10event_implEERKS5_RKNS0_13property_listERP9_pi_event
+_ZN2cl4sycl6detail13MemoryManager18allocateHostMemoryEPNS1_11SYCLMemObjIEPvbmRKNS0_13property_listE
 _ZN2cl4sycl6detail13MemoryManager18releaseImageBufferESt10shared_ptrINS1_12context_implEEPv
-_ZN2cl4sycl6detail13MemoryManager19allocateImageObjectESt10shared_ptrINS1_12context_implEEPvbRK14_pi_image_descRK16_pi_image_format
+_ZN2cl4sycl6detail13MemoryManager19allocateImageObjectESt10shared_ptrINS1_12context_implEEPvbRK14_pi_image_descRK16_pi_image_formatRKNS0_13property_listE
 _ZN2cl4sycl6detail13MemoryManager19wrapIntoImageBufferESt10shared_ptrINS1_12context_implEEPvPNS1_11SYCLMemObjIE
-_ZN2cl4sycl6detail13MemoryManager20allocateBufferObjectESt10shared_ptrINS1_12context_implEEPvbm
+_ZN2cl4sycl6detail13MemoryManager20allocateBufferObjectESt10shared_ptrINS1_12context_implEEPvbmRKNS0_13property_listE
 _ZN2cl4sycl6detail13MemoryManager20allocateMemSubBufferESt10shared_ptrINS1_12context_implEEPvmmNS0_5rangeILi3EEESt6vectorIS3_INS1_10event_implEESaISB_EERP9_pi_event
-_ZN2cl4sycl6detail13MemoryManager24allocateInteropMemObjectESt10shared_ptrINS1_12context_implEEPvRKS3_INS1_10event_implEERKS5_RP9_pi_event
+_ZN2cl4sycl6detail13MemoryManager24allocateInteropMemObjectESt10shared_ptrINS1_12context_implEEPvRKS3_INS1_10event_implEERKS5_RKNS0_13property_listERP9_pi_event
 _ZN2cl4sycl6detail13MemoryManager3mapEPNS1_11SYCLMemObjIEPvSt10shared_ptrINS1_10queue_implEENS0_6access4modeEjNS0_5rangeILi3EEESC_NS0_2idILi3EEEjSt6vectorIP9_pi_eventSaISH_EERSH_
 _ZN2cl4sycl6detail13MemoryManager4copyEPNS1_11SYCLMemObjIEPvSt10shared_ptrINS1_10queue_implEEjNS0_5rangeILi3EEESA_NS0_2idILi3EEEjS5_S8_jSA_SA_SC_jSt6vectorIP9_pi_eventSaISF_EERSF_
 _ZN2cl4sycl6detail13MemoryManager4fillEPNS1_11SYCLMemObjIEPvSt10shared_ptrINS1_10queue_implEEmPKcjNS0_5rangeILi3EEESC_NS0_2idILi3EEEjSt6vectorIP9_pi_eventSaISH_EERSH_
@@ -3789,10 +3795,6 @@ _ZN2cl4sycl6kernelC1EP10_cl_kernelRKNS0_7contextE
 _ZN2cl4sycl6kernelC1ESt10shared_ptrINS0_6detail11kernel_implEE
 _ZN2cl4sycl6kernelC2EP10_cl_kernelRKNS0_7contextE
 _ZN2cl4sycl6kernelC2ESt10shared_ptrINS0_6detail11kernel_implEE
-_ZN2cl4sycl6level010make_queueERKNS0_7contextEm
-_ZN2cl4sycl6level011make_deviceERKNS0_8platformEm
-_ZN2cl4sycl6level012make_programERKNS0_7contextEm
-_ZN2cl4sycl6level013make_platformEm
 _ZN2cl4sycl6mallocEmRKNS0_5queueENS0_3usm5allocE
 _ZN2cl4sycl6mallocEmRKNS0_6deviceERKNS0_7contextENS0_3usm5allocE
 _ZN2cl4sycl6opencl10make_queueERKNS0_7contextEm
diff --git a/sycl/test/abi/symbol_size.cpp b/sycl/test/abi/symbol_size.cpp
index e49fa77521fdd..40270190afc4f 100644
--- a/sycl/test/abi/symbol_size.cpp
+++ b/sycl/test/abi/symbol_size.cpp
@@ -5,6 +5,9 @@
 
 #include <CL/sycl/accessor.hpp>
 #include <CL/sycl/buffer.hpp>
+#include <CL/sycl/detail/accessor_impl.hpp>
+#include <CL/sycl/detail/buffer_impl.hpp>
+#include <CL/sycl/detail/image_impl.hpp>
 #include <CL/sycl/device.hpp>
 #include <CL/sycl/device_event.hpp>
 #include <CL/sycl/device_selector.hpp>
@@ -33,8 +36,14 @@ void check_size() {
 
 int main() {
   using accessor_t = accessor<int, 1, access::mode::read,
-                              access::target::global_buffer, access::placeholder::true_t>;
+                              access::target::global_buffer,
+                              access::placeholder::true_t>;
   check_size<accessor_t, 32>();
+  check_size<detail::AccessorImplDevice<1>, 24>();
+  check_size<detail::LocalAccessorBaseDevice<1>, 24>();
+  check_size<detail::AccessorImplHost, 128>();
+  check_size<detail::AccessorBaseHost, 16>();
+  check_size<detail::LocalAccessorImplHost, 56>();
   check_size<buffer<int>, 40>();
   check_size<context, 16>();
   check_size<cpu_selector, 8>();
@@ -45,16 +54,19 @@ int main() {
   check_size<gpu_selector, 8>();
 #ifdef _MSC_VER
   check_size<handler, 552>();
+  check_size<detail::buffer_impl, 288>();
+  check_size<detail::image_impl<1>, 344>();
 #else
   check_size<handler, 560>();
+  check_size<detail::buffer_impl, 256>();
+  check_size<detail::image_impl<1>, 312>();
 #endif
   check_size<image<1>, 16>();
   check_size<kernel, 16>();
   check_size<platform, 16>();
 #ifdef __SYCL_DEVICE_ONLY__
   check_size<private_memory<int, 1>, 4>();
-#else
-  check_size<private_memory<int, 1>, 8>();
+  check_size<detail::sampler_impl, 8>();
 #endif
   check_size<program, 16>();
   check_size<range<1>, 8>();
diff --git a/sycl/test/atomic_ref/accessor.cpp b/sycl/test/atomic_ref/accessor.cpp
new file mode 100644
index 0000000000000..86067e2c74906
--- /dev/null
+++ b/sycl/test/atomic_ref/accessor.cpp
@@ -0,0 +1,110 @@
+// RUN: %clangxx -fsycl -fsycl-unnamed-lambda -fsycl-targets=%sycl_triple %s -o %t.out
+// RUN: env SYCL_DEVICE_TYPE=HOST %t.out
+// RUN: %CPU_RUN_PLACEHOLDER %t.out
+// RUN: %GPU_RUN_PLACEHOLDER %t.out
+
+#include <CL/sycl.hpp>
+#include <algorithm>
+#include <cassert>
+#include <iostream>
+#include <numeric>
+#include <vector>
+using namespace sycl;
+using namespace sycl::intel;
+
+// Equivalent to add_test from add.cpp
+// Uses atomic_accessor instead of atomic_ref
+template <typename T> void accessor_test(queue q, size_t N) {
+  T sum = 0;
+  std::vector<T> output(N, 0);
+  {
+    buffer<T> sum_buf(&sum, 1);
+    buffer<T> output_buf(output.data(), output.size());
+
+    q.submit([&](handler &cgh) {
+#if __cplusplus > 201402L
+      static_assert(
+          std::is_same<decltype(atomic_accessor(sum_buf, cgh, relaxed_order,
+                                                device_scope)),
+                       atomic_accessor<T, 1, intel::memory_order::relaxed,
+                                       intel::memory_scope::device>>::value,
+          "atomic_accessor type incorrectly deduced");
+#endif
+      auto sum = atomic_accessor<T, 1, intel::memory_order::relaxed,
+                                 intel::memory_scope::device>(sum_buf, cgh);
+      auto out =
+          output_buf.template get_access<access::mode::discard_write>(cgh);
+      cgh.parallel_for(range<1>(N), [=](item<1> it) {
+        int gid = it.get_id(0);
+        static_assert(
+            std::is_same<
+                decltype(sum[0]),
+                atomic_ref<T, intel::memory_order::relaxed,
+                           intel::memory_scope::device,
+                           access::address_space::global_space>>::value,
+            "atomic_accessor returns incorrect atomic_ref");
+        out[gid] = sum[0].fetch_add(T(1));
+      });
+    });
+  }
+
+  // All work-items increment by 1, so final value should be equal to N
+  assert(sum == N);
+
+  // Intermediate values should be unique
+  std::sort(output.begin(), output.end());
+  assert(std::unique(output.begin(), output.end()) == output.end());
+
+  // Fetch returns original value: will be in [0, N-1]
+  auto min_e = output[0];
+  auto max_e = output[output.size() - 1];
+  assert(min_e == 0 && max_e == N - 1);
+}
+
+// Simplified form of accessor_test for local memory
+template <typename T>
+void local_accessor_test(queue q, size_t N, size_t L = 8) {
+  assert(N % L == 0);
+  std::vector<T> output(N / L, 0);
+  {
+    buffer<T> output_buf(output.data(), output.size());
+    q.submit([&](handler &cgh) {
+      auto sum =
+          atomic_accessor<T, 1, intel::memory_order::relaxed,
+                          intel::memory_scope::device, access::target::local>(
+              1, cgh);
+      auto out = output_buf.template get_access<access::mode::read_write>(cgh);
+      cgh.parallel_for(nd_range<1>(N, L), [=](nd_item<1> it) {
+        int grp = it.get_group(0);
+        sum[0].store(0);
+        it.barrier();
+        static_assert(
+            std::is_same<decltype(sum[0]),
+                         atomic_ref<T, intel::memory_order::relaxed,
+                                    intel::memory_scope::device,
+                                    access::address_space::local_space>>::value,
+            "local atomic_accessor returns incorrect atomic_ref");
+        T result = sum[0].fetch_add(T(1));
+        if (result == it.get_local_range(0) - 1) {
+          out[grp] = result;
+        }
+      });
+    });
+  }
+
+  // All work-items increment by 1, and last in the group writes out old value
+  // All values should be L-1
+  assert(std::all_of(output.begin(), output.end(),
+                     [=](T x) { return x == L - 1; }));
+}
+
+int main() {
+  queue q;
+  constexpr int N = 32;
+  accessor_test<int>(q, N);
+  // TODO: Enable local accessor test for host when barrier is supported
+  if (!q.get_device().is_host()) {
+    local_accessor_test<int>(q, N);
+  }
+  std::cout << "Test passed." << std::endl;
+}
diff --git a/sycl/test/atomic_ref/add.cpp b/sycl/test/atomic_ref/add.cpp
index b152166e4f966..cfe943d176299 100644
--- a/sycl/test/atomic_ref/add.cpp
+++ b/sycl/test/atomic_ref/add.cpp
@@ -12,11 +12,11 @@
 using namespace sycl;
 using namespace sycl::intel;
 
-template <typename T>
+template <typename T, typename Difference = T>
 void add_fetch_test(queue q, size_t N) {
   T sum = 0;
   std::vector<T> output(N);
-  std::fill(output.begin(), output.end(), 0);
+  std::fill(output.begin(), output.end(), T(0));
   {
     buffer<T> sum_buf(&sum, 1);
     buffer<T> output_buf(output.data(), output.size());
@@ -27,29 +27,29 @@ void add_fetch_test(queue q, size_t N) {
       cgh.parallel_for(range<1>(N), [=](item<1> it) {
         int gid = it.get_id(0);
         auto atm = atomic_ref<T, intel::memory_order::relaxed, intel::memory_scope::device, access::address_space::global_space>(sum[0]);
-        out[gid] = atm.fetch_add(T(1));
+        out[gid] = atm.fetch_add(Difference(1));
       });
     });
   }
 
   // All work-items increment by 1, so final value should be equal to N
-  assert(sum == N);
+  assert(sum == T(N));
 
   // Fetch returns original value: will be in [0, N-1]
   auto min_e = std::min_element(output.begin(), output.end());
   auto max_e = std::max_element(output.begin(), output.end());
-  assert(*min_e == 0 && *max_e == N - 1);
+  assert(*min_e == T(0) && *max_e == T(N - 1));
 
   // Intermediate values should be unique
   std::sort(output.begin(), output.end());
   assert(std::unique(output.begin(), output.end()) == output.end());
 }
 
-template <typename T>
+template <typename T, typename Difference = T>
 void add_plus_equal_test(queue q, size_t N) {
   T sum = 0;
   std::vector<T> output(N);
-  std::fill(output.begin(), output.end(), 0);
+  std::fill(output.begin(), output.end(), T(0));
   {
     buffer<T> sum_buf(&sum, 1);
     buffer<T> output_buf(output.data(), output.size());
@@ -60,29 +60,29 @@ void add_plus_equal_test(queue q, size_t N) {
       cgh.parallel_for(range<1>(N), [=](item<1> it) {
         int gid = it.get_id(0);
         auto atm = atomic_ref<T, intel::memory_order::relaxed, intel::memory_scope::device, access::address_space::global_space>(sum[0]);
-        out[gid] = atm += T(1);
+        out[gid] = atm += Difference(1);
       });
     });
   }
 
   // All work-items increment by 1, so final value should be equal to N
-  assert(sum == N);
+  assert(sum == T(N));
 
   // += returns updated value: will be in [1, N]
   auto min_e = std::min_element(output.begin(), output.end());
   auto max_e = std::max_element(output.begin(), output.end());
-  assert(*min_e == 1 && *max_e == N);
+  assert(*min_e == T(1) && *max_e == T(N));
 
   // Intermediate values should be unique
   std::sort(output.begin(), output.end());
   assert(std::unique(output.begin(), output.end()) == output.end());
 }
 
-template <typename T>
+template <typename T, typename Difference = T>
 void add_pre_inc_test(queue q, size_t N) {
   T sum = 0;
   std::vector<T> output(N);
-  std::fill(output.begin(), output.end(), 0);
+  std::fill(output.begin(), output.end(), T(0));
   {
     buffer<T> sum_buf(&sum, 1);
     buffer<T> output_buf(output.data(), output.size());
@@ -99,23 +99,23 @@ void add_pre_inc_test(queue q, size_t N) {
   }
 
   // All work-items increment by 1, so final value should be equal to N
-  assert(sum == N);
+  assert(sum == T(N));
 
   // Pre-increment returns updated value: will be in [1, N]
   auto min_e = std::min_element(output.begin(), output.end());
   auto max_e = std::max_element(output.begin(), output.end());
-  assert(*min_e == 1 && *max_e == N);
+  assert(*min_e == T(1) && *max_e == T(N));
 
   // Intermediate values should be unique
   std::sort(output.begin(), output.end());
   assert(std::unique(output.begin(), output.end()) == output.end());
 }
 
-template <typename T>
+template <typename T, typename Difference = T>
 void add_post_inc_test(queue q, size_t N) {
   T sum = 0;
   std::vector<T> output(N);
-  std::fill(output.begin(), output.end(), 0);
+  std::fill(output.begin(), output.end(), T(0));
   {
     buffer<T> sum_buf(&sum, 1);
     buffer<T> output_buf(output.data(), output.size());
@@ -132,24 +132,24 @@ void add_post_inc_test(queue q, size_t N) {
   }
 
   // All work-items increment by 1, so final value should be equal to N
-  assert(sum == N);
+  assert(sum == T(N));
 
   // Post-increment returns original value: will be in [0, N-1]
   auto min_e = std::min_element(output.begin(), output.end());
   auto max_e = std::max_element(output.begin(), output.end());
-  assert(*min_e == 0 && *max_e == N - 1);
+  assert(*min_e == T(0) && *max_e == T(N - 1));
 
   // Intermediate values should be unique
   std::sort(output.begin(), output.end());
   assert(std::unique(output.begin(), output.end()) == output.end());
 }
 
-template <typename T>
+template <typename T, typename Difference = T>
 void add_test(queue q, size_t N) {
-  add_fetch_test<T>(q, N);
-  add_plus_equal_test<T>(q, N);
-  add_pre_inc_test<T>(q, N);
-  add_post_inc_test<T>(q, N);
+  add_fetch_test<T, Difference>(q, N);
+  add_plus_equal_test<T, Difference>(q, N);
+  add_pre_inc_test<T, Difference>(q, N);
+  add_post_inc_test<T, Difference>(q, N);
 }
 
 // Floating-point types do not support pre- or post-increment
@@ -173,8 +173,6 @@ int main() {
   }
 
   constexpr int N = 32;
-
-  // TODO: Enable missing tests when supported
   add_test<int>(q, N);
   add_test<unsigned int>(q, N);
   add_test<long>(q, N);
@@ -183,7 +181,7 @@ int main() {
   add_test<unsigned long long>(q, N);
   add_test<float>(q, N);
   add_test<double>(q, N);
-  //add_test<char*>(q, N);
+  add_test<char *, ptrdiff_t>(q, N);
 
   std::cout << "Test passed." << std::endl;
 }
diff --git a/sycl/test/atomic_ref/compare_exchange.cpp b/sycl/test/atomic_ref/compare_exchange.cpp
index 8f563fccb65fd..11c2caa6ef3c4 100644
--- a/sycl/test/atomic_ref/compare_exchange.cpp
+++ b/sycl/test/atomic_ref/compare_exchange.cpp
@@ -16,10 +16,10 @@ class compare_exchange_kernel;
 
 template <typename T>
 void compare_exchange_test(queue q, size_t N) {
-  const T initial = std::numeric_limits<T>::max();
+  const T initial = T(N);
   T compare_exchange = initial;
   std::vector<T> output(N);
-  std::fill(output.begin(), output.end(), 0);
+  std::fill(output.begin(), output.end(), T(0));
   {
     buffer<T> compare_exchange_buf(&compare_exchange, 1);
     buffer<T> output_buf(output.data(), output.size());
@@ -27,15 +27,16 @@ void compare_exchange_test(queue q, size_t N) {
     q.submit([&](handler &cgh) {
       auto exc = compare_exchange_buf.template get_access<access::mode::read_write>(cgh);
       auto out = output_buf.template get_access<access::mode::discard_write>(cgh);
-      cgh.parallel_for<compare_exchange_kernel<T>>(range<1>(N), [=](item<1> it) {
-        int gid = it.get_id(0);
+      cgh.parallel_for<compare_exchange_kernel<T>>(range<1>(N), [=](item<1>
+                                                                        it) {
+        size_t gid = it.get_id(0);
         auto atm = atomic_ref<T, intel::memory_order::relaxed, intel::memory_scope::device, access::address_space::global_space>(exc[0]);
-        T result = initial;
+        T result = T(N); // Avoid copying pointer
         bool success = atm.compare_exchange_strong(result, (T)gid);
         if (success) {
           out[gid] = result;
         } else {
-          out[gid] = gid;
+          out[gid] = T(gid);
         }
       });
     });
@@ -45,7 +46,7 @@ void compare_exchange_test(queue q, size_t N) {
   assert(std::count(output.begin(), output.end(), initial) == 1);
 
   // All other values should be the index itself or the sentinel value
-  for (int i = 0; i < N; ++i) {
+  for (size_t i = 0; i < N; ++i) {
     assert(output[i] == T(i) || output[i] == initial);
   }
 }
@@ -59,8 +60,6 @@ int main() {
   }
 
   constexpr int N = 32;
-
-  // TODO: Enable missing tests when supported
   compare_exchange_test<int>(q, N);
   compare_exchange_test<unsigned int>(q, N);
   compare_exchange_test<long>(q, N);
@@ -69,7 +68,7 @@ int main() {
   compare_exchange_test<unsigned long long>(q, N);
   compare_exchange_test<float>(q, N);
   compare_exchange_test<double>(q, N);
-  //compare_exchange_test<char*>(q, N);
+  compare_exchange_test<char *>(q, N);
 
   std::cout << "Test passed." << std::endl;
 }
diff --git a/sycl/test/atomic_ref/exchange.cpp b/sycl/test/atomic_ref/exchange.cpp
index 2ce1292cfdd55..b4445928ea075 100644
--- a/sycl/test/atomic_ref/exchange.cpp
+++ b/sycl/test/atomic_ref/exchange.cpp
@@ -16,10 +16,10 @@ class exchange_kernel;
 
 template <typename T>
 void exchange_test(queue q, size_t N) {
-  const T initial = std::numeric_limits<T>::max();
+  const T initial = T(N);
   T exchange = initial;
   std::vector<T> output(N);
-  std::fill(output.begin(), output.end(), 0);
+  std::fill(output.begin(), output.end(), T(0));
   {
     buffer<T> exchange_buf(&exchange, 1);
     buffer<T> output_buf(output.data(), output.size());
@@ -28,9 +28,9 @@ void exchange_test(queue q, size_t N) {
       auto exc = exchange_buf.template get_access<access::mode::read_write>(cgh);
       auto out = output_buf.template get_access<access::mode::discard_write>(cgh);
       cgh.parallel_for<exchange_kernel<T>>(range<1>(N), [=](item<1> it) {
-        int gid = it.get_id(0);
+        size_t gid = it.get_id(0);
         auto atm = atomic_ref<T, intel::memory_order::relaxed, intel::memory_scope::device, access::address_space::global_space>(exc[0]);
-        out[gid] = atm.exchange(gid);
+        out[gid] = atm.exchange(T(gid));
       });
     });
   }
@@ -52,8 +52,6 @@ int main() {
   }
 
   constexpr int N = 32;
-
-  // TODO: Enable missing tests when supported
   exchange_test<int>(q, N);
   exchange_test<unsigned int>(q, N);
   exchange_test<long>(q, N);
@@ -62,7 +60,7 @@ int main() {
   exchange_test<unsigned long long>(q, N);
   exchange_test<float>(q, N);
   exchange_test<double>(q, N);
-  //exchange_test<char*>(q, N);
+  exchange_test<char *>(q, N);
 
   std::cout << "Test passed." << std::endl;
 }
diff --git a/sycl/test/atomic_ref/load.cpp b/sycl/test/atomic_ref/load.cpp
index 274191b9a5ac3..30ae13e16e65e 100644
--- a/sycl/test/atomic_ref/load.cpp
+++ b/sycl/test/atomic_ref/load.cpp
@@ -16,10 +16,10 @@ class load_kernel;
 
 template <typename T>
 void load_test(queue q, size_t N) {
-  T initial = 42;
+  T initial = T(42);
   T load = initial;
   std::vector<T> output(N);
-  std::fill(output.begin(), output.end(), 0);
+  std::fill(output.begin(), output.end(), T(0));
   {
     buffer<T> load_buf(&load, 1);
     buffer<T> output_buf(output.data(), output.size());
@@ -28,7 +28,7 @@ void load_test(queue q, size_t N) {
       auto ld = load_buf.template get_access<access::mode::read_write>(cgh);
       auto out = output_buf.template get_access<access::mode::discard_write>(cgh);
       cgh.parallel_for<load_kernel<T>>(range<1>(N), [=](item<1> it) {
-        int gid = it.get_id(0);
+        size_t gid = it.get_id(0);
         auto atm = atomic_ref<T, intel::memory_order::relaxed, intel::memory_scope::device, access::address_space::global_space>(ld[0]);
         out[gid] = atm.load();
       });
@@ -49,8 +49,6 @@ int main() {
   }
 
   constexpr int N = 32;
-
-  // TODO: Enable missing tests when supported
   load_test<int>(q, N);
   load_test<unsigned int>(q, N);
   load_test<long>(q, N);
@@ -59,7 +57,7 @@ int main() {
   load_test<unsigned long long>(q, N);
   load_test<float>(q, N);
   load_test<double>(q, N);
-  //load_test<char*>(q, N);
+  load_test<char *>(q, N);
 
   std::cout << "Test passed." << std::endl;
 }
diff --git a/sycl/test/atomic_ref/max.cpp b/sycl/test/atomic_ref/max.cpp
index c8bccf1c28067..0c95653b8219b 100644
--- a/sycl/test/atomic_ref/max.cpp
+++ b/sycl/test/atomic_ref/max.cpp
@@ -57,8 +57,6 @@ int main() {
   }
 
   constexpr int N = 32;
-
-  // TODO: Enable missing tests when supported
   max_test<int>(q, N);
   max_test<unsigned int>(q, N);
   max_test<long>(q, N);
@@ -67,7 +65,6 @@ int main() {
   max_test<unsigned long long>(q, N);
   max_test<float>(q, N);
   max_test<double>(q, N);
-  //max_test<char*>(q, N);
 
   std::cout << "Test passed." << std::endl;
 }
diff --git a/sycl/test/atomic_ref/min.cpp b/sycl/test/atomic_ref/min.cpp
index 8313c4931136c..6a0e32ca14bb5 100644
--- a/sycl/test/atomic_ref/min.cpp
+++ b/sycl/test/atomic_ref/min.cpp
@@ -55,8 +55,6 @@ int main() {
   }
 
   constexpr int N = 32;
-
-  // TODO: Enable missing tests when supported
   min_test<int>(q, N);
   min_test<unsigned int>(q, N);
   min_test<long>(q, N);
@@ -65,7 +63,6 @@ int main() {
   min_test<unsigned long long>(q, N);
   min_test<float>(q, N);
   min_test<double>(q, N);
-  //min_test<char*>(q, N);
 
   std::cout << "Test passed." << std::endl;
 }
diff --git a/sycl/test/atomic_ref/store.cpp b/sycl/test/atomic_ref/store.cpp
index eebdba5ced095..db076ee994a3d 100644
--- a/sycl/test/atomic_ref/store.cpp
+++ b/sycl/test/atomic_ref/store.cpp
@@ -16,14 +16,14 @@ class store_kernel;
 
 template <typename T>
 void store_test(queue q, size_t N) {
-  T initial = std::numeric_limits<T>::max();
+  T initial = T(N);
   T store = initial;
   {
     buffer<T> store_buf(&store, 1);
     q.submit([&](handler &cgh) {
       auto st = store_buf.template get_access<access::mode::read_write>(cgh);
       cgh.parallel_for<store_kernel<T>>(range<1>(N), [=](item<1> it) {
-        int gid = it.get_id(0);
+        size_t gid = it.get_id(0);
         auto atm = atomic_ref<T, intel::memory_order::relaxed, intel::memory_scope::device, access::address_space::global_space>(st[0]);
         atm.store(T(gid));
       });
@@ -45,8 +45,6 @@ int main() {
   }
 
   constexpr int N = 32;
-
-  // TODO: Enable missing tests when supported
   store_test<int>(q, N);
   store_test<unsigned int>(q, N);
   store_test<long>(q, N);
@@ -55,7 +53,7 @@ int main() {
   store_test<unsigned long long>(q, N);
   store_test<float>(q, N);
   store_test<double>(q, N);
-  //store_test<char*>(q, N);
+  store_test<char *>(q, N);
 
   std::cout << "Test passed." << std::endl;
 }
diff --git a/sycl/test/atomic_ref/sub.cpp b/sycl/test/atomic_ref/sub.cpp
index 52e338048e7be..10ed75d21da25 100644
--- a/sycl/test/atomic_ref/sub.cpp
+++ b/sycl/test/atomic_ref/sub.cpp
@@ -12,11 +12,11 @@
 using namespace sycl;
 using namespace sycl::intel;
 
-template <typename T>
+template <typename T, typename Difference = T>
 void sub_fetch_test(queue q, size_t N) {
-  T val = N;
+  T val = T(N);
   std::vector<T> output(N);
-  std::fill(output.begin(), output.end(), 0);
+  std::fill(output.begin(), output.end(), T(0));
   {
     buffer<T> val_buf(&val, 1);
     buffer<T> output_buf(output.data(), output.size());
@@ -27,29 +27,29 @@ void sub_fetch_test(queue q, size_t N) {
       cgh.parallel_for(range<1>(N), [=](item<1> it) {
         int gid = it.get_id(0);
         auto atm = atomic_ref<T, intel::memory_order::relaxed, intel::memory_scope::device, access::address_space::global_space>(val[0]);
-        out[gid] = atm.fetch_sub(T(1));
+        out[gid] = atm.fetch_sub(Difference(1));
       });
     });
   }
 
   // All work-items decrement by 1, so final value should be equal to 0
-  assert(val == 0);
+  assert(val == T(0));
 
   // Fetch returns original value: will be in [1, N]
   auto min_e = std::min_element(output.begin(), output.end());
   auto max_e = std::max_element(output.begin(), output.end());
-  assert(*min_e == 1 && *max_e == N);
+  assert(*min_e == T(1) && *max_e == T(N));
 
   // Intermediate values should be unique
   std::sort(output.begin(), output.end());
   assert(std::unique(output.begin(), output.end()) == output.end());
 }
 
-template <typename T>
+template <typename T, typename Difference = T>
 void sub_plus_equal_test(queue q, size_t N) {
-  T val = N;
+  T val = T(N);
   std::vector<T> output(N);
-  std::fill(output.begin(), output.end(), 0);
+  std::fill(output.begin(), output.end(), T(0));
   {
     buffer<T> val_buf(&val, 1);
     buffer<T> output_buf(output.data(), output.size());
@@ -60,29 +60,29 @@ void sub_plus_equal_test(queue q, size_t N) {
       cgh.parallel_for(range<1>(N), [=](item<1> it) {
         int gid = it.get_id(0);
         auto atm = atomic_ref<T, intel::memory_order::relaxed, intel::memory_scope::device, access::address_space::global_space>(val[0]);
-        out[gid] = atm -= T(1);
+        out[gid] = atm -= Difference(1);
       });
     });
   }
 
   // All work-items decrement by 1, so final value should be equal to 0
-  assert(val == 0);
+  assert(val == T(0));
 
   // -= returns updated value: will be in [0, N-1]
   auto min_e = std::min_element(output.begin(), output.end());
   auto max_e = std::max_element(output.begin(), output.end());
-  assert(*min_e == 0 && *max_e == N - 1);
+  assert(*min_e == T(0) && *max_e == T(N - 1));
 
   // Intermediate values should be unique
   std::sort(output.begin(), output.end());
   assert(std::unique(output.begin(), output.end()) == output.end());
 }
 
-template <typename T>
+template <typename T, typename Difference = T>
 void sub_pre_dec_test(queue q, size_t N) {
-  T val = N;
+  T val = T(N);
   std::vector<T> output(N);
-  std::fill(output.begin(), output.end(), 0);
+  std::fill(output.begin(), output.end(), T(0));
   {
     buffer<T> val_buf(&val, 1);
     buffer<T> output_buf(output.data(), output.size());
@@ -99,23 +99,23 @@ void sub_pre_dec_test(queue q, size_t N) {
   }
 
   // All work-items decrement by 1, so final value should be equal to 0
-  assert(val == 0);
+  assert(val == T(0));
 
   // Pre-decrement returns updated value: will be in [0, N-1]
   auto min_e = std::min_element(output.begin(), output.end());
   auto max_e = std::max_element(output.begin(), output.end());
-  assert(*min_e == 0 && *max_e == N - 1);
+  assert(*min_e == T(0) && *max_e == T(N - 1));
 
   // Intermediate values should be unique
   std::sort(output.begin(), output.end());
   assert(std::unique(output.begin(), output.end()) == output.end());
 }
 
-template <typename T>
+template <typename T, typename Difference = T>
 void sub_post_dec_test(queue q, size_t N) {
-  T val = N;
+  T val = T(N);
   std::vector<T> output(N);
-  std::fill(output.begin(), output.end(), 0);
+  std::fill(output.begin(), output.end(), T(0));
   {
     buffer<T> val_buf(&val, 1);
     buffer<T> output_buf(output.data(), output.size());
@@ -132,24 +132,24 @@ void sub_post_dec_test(queue q, size_t N) {
   }
 
   // All work-items decrement by 1, so final value should be equal to 0
-  assert(val == 0);
+  assert(val == T(0));
 
   // Post-decrement returns original value: will be in [1, N]
   auto min_e = std::min_element(output.begin(), output.end());
   auto max_e = std::max_element(output.begin(), output.end());
-  assert(*min_e == 1 && *max_e == N);
+  assert(*min_e == T(1) && *max_e == T(N));
 
   // Intermediate values should be unique
   std::sort(output.begin(), output.end());
   assert(std::unique(output.begin(), output.end()) == output.end());
 }
 
-template <typename T>
+template <typename T, typename Difference = T>
 void sub_test(queue q, size_t N) {
-  sub_fetch_test<T>(q, N);
-  sub_plus_equal_test<T>(q, N);
-  sub_pre_dec_test<T>(q, N);
-  sub_post_dec_test<T>(q, N);
+  sub_fetch_test<T, Difference>(q, N);
+  sub_plus_equal_test<T, Difference>(q, N);
+  sub_pre_dec_test<T, Difference>(q, N);
+  sub_post_dec_test<T, Difference>(q, N);
 }
 
 // Floating-point types do not support pre- or post-decrement
@@ -173,8 +173,6 @@ int main() {
   }
 
   constexpr int N = 32;
-
-  // TODO: Enable missing tests when supported
   sub_test<int>(q, N);
   sub_test<unsigned int>(q, N);
   sub_test<long>(q, N);
@@ -183,7 +181,7 @@ int main() {
   sub_test<unsigned long long>(q, N);
   sub_test<float>(q, N);
   sub_test<double>(q, N);
-  //sub_test<char*>(q, N);
+  sub_test<char *, ptrdiff_t>(q, N);
 
   std::cout << "Test passed." << std::endl;
 }
diff --git a/sycl/test/basic_tests/accessor/addrspace_exposure.cpp b/sycl/test/basic_tests/accessor/addrspace_exposure.cpp
index ce73bf0296c10..ebc2540ee44d9 100644
--- a/sycl/test/basic_tests/accessor/addrspace_exposure.cpp
+++ b/sycl/test/basic_tests/accessor/addrspace_exposure.cpp
@@ -33,9 +33,9 @@ int main() {
     Cgh.single_task<class test>([=]() {
       static_assert(std::is_same<decltype(GlobalRWAcc[0]), int &>::value,
                     "Incorrect type from global read-write accessor");
-      static_assert(std::is_same<decltype(GlobalRAcc[0]), int>::value,
+      static_assert(std::is_same<decltype(GlobalRAcc[0]), const int &>::value,
                     "Incorrect type from global read accessor");
-      static_assert(std::is_same<decltype(ConstantAcc[0]), int>::value,
+      static_assert(std::is_same<decltype(ConstantAcc[0]), const int &>::value,
                     "Incorrect type from constant accessor");
       static_assert(std::is_same<decltype(LocalAcc[0]), int &>::value,
                     "Incorrect type from local accessor");
diff --git a/sycl/test/basic_tests/buffer/buffer.cpp b/sycl/test/basic_tests/buffer/buffer.cpp
index 6df0ab06ce1a3..2722afadb2369 100644
--- a/sycl/test/basic_tests/buffer/buffer.cpp
+++ b/sycl/test/basic_tests/buffer/buffer.cpp
@@ -183,10 +183,10 @@ int main() {
   }
   {
     int data[10] = {0};
-    std::shared_ptr<int> result(new int[10]());
+    std::shared_ptr<int[]> result(new int[10]());
     {
       buffer<int, 1> Buffer(data, range<1>(10));
-      std::weak_ptr<int> resultWeak = result;
+      std::weak_ptr<int[]> resultWeak = result;
       Buffer.set_final_data(resultWeak);
       queue myQueue;
       myQueue.submit([&](handler &cgh) {
@@ -200,7 +200,7 @@ int main() {
 
   {
     int data[10] = {0};
-    std::shared_ptr<int> result(new int[10]());
+    std::shared_ptr<int[]> result(new int[10]());
     {
       buffer<int, 1> Buffer(data, range<1>(10));
       Buffer.set_final_data(result);
@@ -284,11 +284,11 @@ int main() {
   }
   {
     int data[10] = {0};
-    std::shared_ptr<int> result(new int[10]());
+    std::shared_ptr<int[]> result(new int[10]());
     {
       buffer<int, 1> Buffer(data, range<1>(10),
                             {property::buffer::use_host_ptr()});
-      std::weak_ptr<int> resultWeak = result;
+      std::weak_ptr<int[]> resultWeak = result;
       Buffer.set_final_data(resultWeak);
       queue myQueue;
       myQueue.submit([&](handler &cgh) {
diff --git a/sycl/test/basic_tests/buffer/buffer_container.cpp b/sycl/test/basic_tests/buffer/buffer_container.cpp
new file mode 100644
index 0000000000000..b2e5efa2f43fd
--- /dev/null
+++ b/sycl/test/basic_tests/buffer/buffer_container.cpp
@@ -0,0 +1,30 @@
+// RUN: %clangxx %s -std=c++17 -o %t1.out -lsycl -I %sycl_include
+// RUN: env SYCL_DEVICE_TYPE=HOST %t1.out
+// RUN: %clangxx -std=c++17 -fsycl -fsycl-targets=%sycl_triple %s -o %t2.out
+// RUN: env SYCL_DEVICE_TYPE=HOST %t2.out
+// RUN: %CPU_RUN_PLACEHOLDER %t2.out
+// RUN: %GPU_RUN_PLACEHOLDER %t2.out
+// RUN: %ACC_RUN_PLACEHOLDER %t2.out
+
+#include <CL/sycl.hpp>
+
+using namespace cl::sycl;
+
+int main() {
+  // buffer created from contiguous container copies back
+  {
+    constexpr int num = 10;
+    std::vector<int> out_data(num, -1);
+    {
+      buffer A(out_data);
+      queue Q;
+      Q.submit([&](handler &h) {
+        auto out = A.get_access<access::mode::write>(h);
+        h.parallel_for<class containerBuffer>(A.get_range(),
+                                              [out](id<1> i) { out[i] = 1; });
+      });
+    } //~buffer
+    for (int i = 0; i < num; i++)
+      assert(out_data[i] == 1);
+  }
+}
\ No newline at end of file
diff --git a/sycl/test/basic_tests/buffer/buffer_full_copy.cpp b/sycl/test/basic_tests/buffer/buffer_full_copy.cpp
index 79e69067682d5..2743557f6f971 100644
--- a/sycl/test/basic_tests/buffer/buffer_full_copy.cpp
+++ b/sycl/test/basic_tests/buffer/buffer_full_copy.cpp
@@ -6,7 +6,7 @@
 // RUN: %GPU_RUN_PLACEHOLDER %t2.out
 // RUN: %ACC_RUN_PLACEHOLDER %t2.out
 
-// XFAIL: level0
+// XFAIL: level_zero
 
 //==------------- buffer_full_copy.cpp - SYCL buffer basic test ------------==//
 //
diff --git a/sycl/test/basic_tests/buffer/reinterpret.cpp b/sycl/test/basic_tests/buffer/reinterpret.cpp
index 7fd000f165131..7288d9bfb2c97 100644
--- a/sycl/test/basic_tests/buffer/reinterpret.cpp
+++ b/sycl/test/basic_tests/buffer/reinterpret.cpp
@@ -3,7 +3,7 @@
 // RUN: %GPU_RUN_PLACEHOLDER %t.out
 // RUN: %ACC_RUN_PLACEHOLDER %t.out
 //
-// XFAIL: level0
+// XFAIL: level_zero
 
 //==---------- reinterpret.cpp --- SYCL buffer reinterpret basic test ------==//
 //
diff --git a/sycl/test/basic_tests/esimd/vadd.cpp b/sycl/test/basic_tests/esimd/vadd.cpp
new file mode 100644
index 0000000000000..481c22a2718d9
--- /dev/null
+++ b/sycl/test/basic_tests/esimd/vadd.cpp
@@ -0,0 +1,105 @@
+// TODO ESIMD enable host device under -fsycl
+// RUN: %clangxx -I %sycl_include %s -o %t.out -lsycl
+// RUN: env SYCL_DEVICE_TYPE=HOST %t.out
+
+#include <CL/sycl.hpp>
+#include <CL/sycl/intel/esimd.hpp>
+#include <iostream>
+
+using namespace cl::sycl;
+
+class ESIMDSelector : public device_selector {
+  // Require GPU device unless HOST is requested in SYCL_DEVICE_TYPE env
+  virtual int operator()(const device &device) const {
+    if (const char *dev_type = getenv("SYCL_DEVICE_TYPE")) {
+      if (!strcmp(dev_type, "GPU"))
+        return device.is_gpu() ? 1000 : -1;
+      if (!strcmp(dev_type, "HOST"))
+        return device.is_host() ? 1000 : -1;
+      std::cerr << "Supported 'SYCL_DEVICE_TYPE' env var values are 'GPU' and "
+                   "'HOST', '"
+                << dev_type << "' is not.\n";
+      return -1;
+    }
+    // If "SYCL_DEVICE_TYPE" not defined, only allow gpu device
+    return device.is_gpu() ? 1000 : -1;
+  }
+};
+
+auto exception_handler = [](exception_list l) {
+  for (auto ep : l) {
+    try {
+      std::rethrow_exception(ep);
+    } catch (cl::sycl::exception &e0) {
+      std::cout << "sycl::exception: " << e0.what() << std::endl;
+    } catch (std::exception &e) {
+      std::cout << "std::exception: " << e.what() << std::endl;
+    } catch (...) {
+      std::cout << "generic exception\n";
+    }
+  }
+};
+
+int main(void) {
+  constexpr unsigned Size = 256;
+  constexpr unsigned VL = 32;
+  constexpr unsigned GroupSize = 2;
+
+  int A[Size];
+  int B[Size];
+  int C[Size] = {};
+
+  for (unsigned i = 0; i < Size; ++i) {
+    A[i] = B[i] = i;
+  }
+
+  {
+    cl::sycl::buffer<int, 1> bufA(A, Size);
+    cl::sycl::buffer<int, 1> bufB(B, Size);
+    cl::sycl::buffer<int, 1> bufC(C, Size);
+
+    // We need that many task groups
+    cl::sycl::range<1> GroupRange{Size / VL};
+
+    // We need that many tasks in each group
+    cl::sycl::range<1> TaskRange{GroupSize};
+
+    cl::sycl::nd_range<1> Range{GroupRange, TaskRange};
+
+    queue q(ESIMDSelector{}, exception_handler);
+    q.submit([&](cl::sycl::handler &cgh) {
+      auto accA = bufA.get_access<cl::sycl::access::mode::read>(cgh);
+      auto accB = bufB.get_access<cl::sycl::access::mode::read>(cgh);
+      auto accC = bufC.get_access<cl::sycl::access::mode::write>(cgh);
+
+      cgh.parallel_for<class Test>(
+          Range, [=](nd_item<1> ndi) SYCL_ESIMD_KERNEL {
+            using namespace sycl::intel::gpu;
+            auto pA = accA.get_pointer().get();
+            auto pB = accB.get_pointer().get();
+            auto pC = accC.get_pointer().get();
+
+            int i = ndi.get_global_id(0);
+            constexpr int ESIZE = sizeof(int);
+            simd<uint32_t, VL> offsets(0, ESIZE);
+
+            simd<int, VL> va = gather<int, VL>(pA + i * VL, offsets);
+            simd<int, VL> vb = block_load<int, VL>(pB + i * VL);
+            simd<int, VL> vc = va + vb;
+
+            block_store<int, VL>(pC + i * VL, vc);
+          });
+    });
+
+    for (unsigned i = 0; i < Size; ++i) {
+      if (A[i] + B[i] != C[i]) {
+        std::cout << "failed at index " << i << ", " << C[i] << " != " << A[i]
+                  << " + " << B[i] << "\n";
+        return 1;
+      }
+    }
+  }
+
+  std::cout << "Passed\n";
+  return 0;
+}
diff --git a/sycl/test/basic_tests/get_backend.cpp b/sycl/test/basic_tests/get_backend.cpp
index de0738dd88341..a312304e0741b 100644
--- a/sycl/test/basic_tests/get_backend.cpp
+++ b/sycl/test/basic_tests/get_backend.cpp
@@ -3,7 +3,6 @@
 //
 //==----------------- get_backend.cpp ------------------------==//
 // This is a test of get_backend().
-// Also prints handy info about the system.
 // Do not set SYCL_BE.  We do not want the preferred backend.
 //==----------------------------------------------------------==//
 
@@ -16,7 +15,7 @@ using namespace cl::sycl;
 bool check(backend be) {
   switch (be) {
   case backend::opencl:
-  case backend::level0:
+  case backend::level_zero:
   case backend::cuda:
   case backend::host:
     return true;
diff --git a/sycl/test/basic_tests/handler/handler_set_args.cpp b/sycl/test/basic_tests/handler/handler_set_args.cpp
new file mode 100644
index 0000000000000..6e15ecc6fa836
--- /dev/null
+++ b/sycl/test/basic_tests/handler/handler_set_args.cpp
@@ -0,0 +1,230 @@
+// RUN: %clangxx -fsycl -fsycl-targets=%sycl_triple %s -o %t.out
+// RUN: %CPU_RUN_PLACEHOLDER %t.out
+// RUN: %GPU_RUN_PLACEHOLDER %t.out
+// RUN: %ACC_RUN_PLACEHOLDER %t.out
+//==--------------- handler_set_args.cpp -------------------==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include <CL/sycl.hpp>
+#include <cassert>
+
+constexpr bool UseOffset = true;
+constexpr bool NoOffset = false;
+const cl::sycl::range<1> Range = 1;
+
+using AccessorT = cl::sycl::accessor<int, 1, cl::sycl::access::mode::read_write,
+                                     cl::sycl::access::target::global_buffer>;
+
+struct SingleTaskFunctor {
+  SingleTaskFunctor(AccessorT acc) : MAcc(acc) {}
+
+  void operator()() const { MAcc[0] = 10; }
+
+  AccessorT MAcc;
+};
+
+template <bool useOffset> struct ParallelForRangeIdFunctor {
+  ParallelForRangeIdFunctor(AccessorT acc) : MAcc(acc) {}
+
+  void operator()(cl::sycl::id<1> id) const { MAcc[0] = 10; }
+
+  AccessorT MAcc;
+};
+
+template <bool useOffset> struct ParallelForRangeItemFunctor {
+  ParallelForRangeItemFunctor(AccessorT acc) : MAcc(acc) {}
+
+  void operator()(cl::sycl::item<1> item) const { MAcc[0] = 10; }
+
+  AccessorT MAcc;
+};
+
+struct ParallelForNdRangeFunctor {
+  ParallelForNdRangeFunctor(AccessorT acc) : MAcc(acc) {}
+
+  void operator()(cl::sycl::nd_item<1> ndItem) const { MAcc[0] = 10; }
+
+  AccessorT MAcc;
+};
+
+template <class kernel_name>
+cl::sycl::kernel getPrebuiltKernel(cl::sycl::queue &queue) {
+  cl::sycl::program program(queue.get_context());
+  program.build_with_kernel_type<kernel_name>();
+  return program.get_kernel<kernel_name>();
+}
+
+template <class kernel_wrapper>
+void checkApiCall(cl::sycl::queue &queue, kernel_wrapper &&kernelWrapper) {
+  int result = 0;
+  {
+    auto buf = cl::sycl::buffer<int, 1>(&result, Range);
+    queue.submit([&](cl::sycl::handler &cgh) {
+      auto acc = buf.get_access<cl::sycl::access::mode::read_write>(cgh);
+      kernelWrapper(cgh, acc);
+    });
+  }
+  assert(result == 10);
+}
+
+int main() {
+  cl::sycl::queue Queue;
+  const cl::sycl::id<1> Offset(0);
+  const cl::sycl::nd_range<1> NdRange(Range, Range);
+
+  checkApiCall(Queue, [&](cl::sycl::handler &cgh, AccessorT acc) {
+    cgh.single_task(SingleTaskFunctor(acc));
+  });
+
+  checkApiCall(Queue, [&](cl::sycl::handler &cgh, AccessorT acc) {
+    cgh.parallel_for(Range, ParallelForRangeIdFunctor<NoOffset>(acc));
+  });
+
+  checkApiCall(Queue, [&](cl::sycl::handler &cgh, AccessorT acc) {
+    cgh.parallel_for(Range, Offset, ParallelForRangeIdFunctor<UseOffset>(acc));
+  });
+
+  checkApiCall(Queue, [&](cl::sycl::handler &cgh, AccessorT acc) {
+    cgh.parallel_for(Range, ParallelForRangeItemFunctor<NoOffset>(acc));
+  });
+
+  checkApiCall(Queue, [&](cl::sycl::handler &cgh, AccessorT acc) {
+    cgh.parallel_for(Range, Offset,
+                     ParallelForRangeItemFunctor<UseOffset>(acc));
+  });
+
+  checkApiCall(Queue, [&](cl::sycl::handler &cgh, AccessorT acc) {
+    cgh.parallel_for(NdRange, ParallelForNdRangeFunctor(acc));
+  });
+
+  {
+    auto preBuiltKernel = getPrebuiltKernel<SingleTaskFunctor>(Queue);
+
+    checkApiCall(Queue, [&](cl::sycl::handler &cgh, AccessorT acc) {
+      cgh.set_args(acc);
+      cgh.single_task(preBuiltKernel);
+    });
+  }
+
+  {
+    auto preBuiltKernel =
+        getPrebuiltKernel<ParallelForRangeIdFunctor<NoOffset>>(Queue);
+
+    checkApiCall(Queue, [&](cl::sycl::handler &cgh, AccessorT acc) {
+      cgh.set_args(acc);
+      cgh.parallel_for(Range, preBuiltKernel);
+    });
+  }
+
+  {
+    auto preBuiltKernel =
+        getPrebuiltKernel<ParallelForRangeIdFunctor<UseOffset>>(Queue);
+
+    checkApiCall(Queue, [&](cl::sycl::handler &cgh, AccessorT acc) {
+      cgh.set_args(acc);
+      cgh.parallel_for(Range, Offset, preBuiltKernel);
+    });
+  }
+
+  {
+    auto preBuiltKernel =
+        getPrebuiltKernel<ParallelForRangeItemFunctor<NoOffset>>(Queue);
+
+    checkApiCall(Queue, [&](cl::sycl::handler &cgh, AccessorT acc) {
+      cgh.set_args(acc);
+      cgh.parallel_for(Range, preBuiltKernel);
+    });
+  }
+
+  {
+    auto preBuiltKernel =
+        getPrebuiltKernel<ParallelForRangeItemFunctor<UseOffset>>(Queue);
+
+    checkApiCall(Queue, [&](cl::sycl::handler &cgh, AccessorT acc) {
+      cgh.set_args(acc);
+      cgh.parallel_for(Range, Offset, preBuiltKernel);
+    });
+  }
+
+  {
+    auto preBuiltKernel = getPrebuiltKernel<ParallelForNdRangeFunctor>(Queue);
+
+    checkApiCall(Queue, [&](cl::sycl::handler &cgh, AccessorT acc) {
+      cgh.set_args(acc);
+      cgh.parallel_for(NdRange, preBuiltKernel);
+    });
+  }
+
+  {
+    auto preBuiltKernel = getPrebuiltKernel<SingleTaskFunctor>(Queue);
+
+    checkApiCall(Queue, [&](cl::sycl::handler &cgh, AccessorT acc) {
+      cgh.set_args(acc);
+      cgh.single_task<class OtherKernelName1>(preBuiltKernel,
+                                              [=]() { acc[0] = 10; });
+    });
+  }
+
+  {
+    auto preBuiltKernel =
+        getPrebuiltKernel<ParallelForRangeIdFunctor<NoOffset>>(Queue);
+
+    checkApiCall(Queue, [&](cl::sycl::handler &cgh, AccessorT acc) {
+      cgh.set_args(acc);
+      cgh.parallel_for<class OtherKernelName2>(
+          preBuiltKernel, Range, [=](cl::sycl::id<1> id) { acc[0] = 10; });
+    });
+  }
+
+  {
+    auto preBuiltKernel =
+        getPrebuiltKernel<ParallelForRangeIdFunctor<UseOffset>>(Queue);
+
+    checkApiCall(Queue, [&](cl::sycl::handler &cgh, AccessorT acc) {
+      cgh.set_args(acc);
+      cgh.parallel_for<class OtherKernelName3>(
+          preBuiltKernel, Range, Offset,
+          [=](cl::sycl::id<1> id) { acc[0] = 10; });
+    });
+  }
+
+  {
+    auto preBuiltKernel =
+        getPrebuiltKernel<ParallelForRangeItemFunctor<NoOffset>>(Queue);
+
+    checkApiCall(Queue, [&](cl::sycl::handler &cgh, AccessorT acc) {
+      cgh.set_args(acc);
+      cgh.parallel_for<class OtherKernelName4>(
+          preBuiltKernel, Range, [=](cl::sycl::item<1> item) { acc[0] = 10; });
+    });
+  }
+
+  {
+    auto preBuiltKernel =
+        getPrebuiltKernel<ParallelForRangeItemFunctor<UseOffset>>(Queue);
+
+    checkApiCall(Queue, [&](cl::sycl::handler &cgh, AccessorT acc) {
+      cgh.set_args(acc);
+      cgh.parallel_for<class OtherKernelName5>(
+          preBuiltKernel, Range, Offset,
+          [=](cl::sycl::item<1> item) { acc[0] = 10; });
+    });
+  }
+
+  {
+    auto preBuiltKernel = getPrebuiltKernel<ParallelForNdRangeFunctor>(Queue);
+
+    checkApiCall(Queue, [&](cl::sycl::handler &cgh, AccessorT acc) {
+      cgh.set_args(acc);
+      cgh.parallel_for<class OtherKernelName6>(
+          preBuiltKernel, NdRange,
+          [=](cl::sycl::nd_item<1> ndItem) { acc[0] = 10; });
+    });
+  }
+
+  return 0;
+}
diff --git a/sycl/test/basic_tests/image_accessor_readwrite.cpp b/sycl/test/basic_tests/image_accessor_readwrite.cpp
index 13c0b175bd1fc..0451ced85dc55 100644
--- a/sycl/test/basic_tests/image_accessor_readwrite.cpp
+++ b/sycl/test/basic_tests/image_accessor_readwrite.cpp
@@ -6,7 +6,7 @@
 // RUN: %CPU_RUN_PLACEHOLDER %t.out
 // RUN: %GPU_RUN_PLACEHOLDER %t.out
 //
-// XFAIL: windows && level0
+// XFAIL: windows && level_zero
 
 //==--------------------image_accessor_readwrite.cpp ----------------------==//
 //==----------image_accessor read without sampler & write API test---------==//
diff --git a/sycl/test/basic_tests/image_accessor_readwrite_half.cpp b/sycl/test/basic_tests/image_accessor_readwrite_half.cpp
index 6d5c1960655e5..931780c6c36a6 100644
--- a/sycl/test/basic_tests/image_accessor_readwrite_half.cpp
+++ b/sycl/test/basic_tests/image_accessor_readwrite_half.cpp
@@ -6,7 +6,7 @@
 // RUN: %CPU_RUN_PLACEHOLDER %t.out
 // RUN: %GPU_RUN_PLACEHOLDER %t.out
 //
-// XFAIL: windows && level0
+// XFAIL: windows && level_zero
 
 //==--------------------image_accessor_readwrite_half.cpp -------------------==//
 //==-image_accessor read (without sampler)& write API test for half datatype-==//
diff --git a/sycl/test/basic_tests/kernel_info.cpp b/sycl/test/basic_tests/kernel_info.cpp
index 193d89a4c4bd8..68a642b476e27 100644
--- a/sycl/test/basic_tests/kernel_info.cpp
+++ b/sycl/test/basic_tests/kernel_info.cpp
@@ -3,8 +3,8 @@
 // RUN: %GPU_RUN_PLACEHOLDER %t.out
 // RUN: %ACC_RUN_PLACEHOLDER %t.out
 //
-// Fail is flaky for level0, enable when fixed.
-// UNSUPPORTED: level0
+// Fail is flaky for level_zero, enable when fixed.
+// UNSUPPORTED: level_zero
 
 //==--- kernel_info.cpp - SYCL kernel info test ----------------------------==//
 //
diff --git a/sycl/test/basic_tests/parallel_for_indexers.cpp b/sycl/test/basic_tests/parallel_for_indexers.cpp
index 3d20ec3d66903..a4ec46b42bc9a 100644
--- a/sycl/test/basic_tests/parallel_for_indexers.cpp
+++ b/sycl/test/basic_tests/parallel_for_indexers.cpp
@@ -8,7 +8,7 @@
 
 // TODO: Unexpected result
 // TODO: _indexers.cpp:37: int main(): Assertion `id == -1' failed.
-// XFAIL: level0
+// XFAIL: level_zero
 
 #include <CL/sycl.hpp>
 
diff --git a/sycl/test/basic_tests/parallel_for_range.cpp b/sycl/test/basic_tests/parallel_for_range.cpp
index 3031d3d30f388..4ec9b23158239 100644
--- a/sycl/test/basic_tests/parallel_for_range.cpp
+++ b/sycl/test/basic_tests/parallel_for_range.cpp
@@ -1,4 +1,4 @@
-// XFAIL: cuda || level0
+// XFAIL: cuda || level_zero
 // CUDA exposes broken hierarchical parallelism.
 
 // RUN: %clangxx -fsycl -fsycl-targets=%sycl_triple  %s -o %t.out
diff --git a/sycl/test/basic_tests/set_arg_error.cpp b/sycl/test/basic_tests/set_arg_error.cpp
new file mode 100644
index 0000000000000..52df555902b44
--- /dev/null
+++ b/sycl/test/basic_tests/set_arg_error.cpp
@@ -0,0 +1,55 @@
+// RUN: %clangxx -fsycl -Xclang -verify %s -I %sycl_include -Xclang -verify-ignore-unexpected=note,warning -fsyntax-only
+
+#include <CL/sycl.hpp>
+
+struct TriviallyCopyable {
+  int a;
+  int b;
+};
+
+struct NonTriviallyCopyable {
+  NonTriviallyCopyable() = default;
+  NonTriviallyCopyable(NonTriviallyCopyable const &) {}
+  int a;
+  int b;
+};
+
+struct NonStdLayout {
+  int a;
+
+private:
+  int b;
+};
+
+int main() {
+  constexpr size_t size = 1;
+  cl::sycl::buffer<int> buf(size);
+  cl::sycl::queue q;
+  q.submit([&](cl::sycl::handler &h) {
+    auto global_acc = buf.get_access<cl::sycl::access::mode::write>(h);
+    cl::sycl::sampler samp(cl::sycl::coordinate_normalization_mode::normalized,
+                           cl::sycl::addressing_mode::clamp,
+                           cl::sycl::filtering_mode::nearest);
+    cl::sycl::accessor<int, 1, cl::sycl::access::mode::read_write,
+                       cl::sycl::access::target::local>
+        local_acc({size}, h);
+    TriviallyCopyable tc{1, 2};
+    NonTriviallyCopyable ntc;
+    h.set_arg(0, local_acc);
+    h.set_arg(1, global_acc);
+    h.set_arg(2, samp);
+    h.set_arg(3, tc);
+    h.set_arg(4, TriviallyCopyable{});
+    h.set_arg( // expected-error {{no matching member function for call to 'set_arg'}}
+        5, ntc);
+    h.set_arg( // expected-error {{no matching member function for call to 'set_arg'}}
+        4, NonTriviallyCopyable{});
+#if CL_SYCL_LANGUAGE_VERSION && CL_SYCL_LANGUAGE_VERSION <= 121
+    NonStdLayout nstd;
+    h.set_arg( // expected-error {{no matching member function for call to 'set_arg'}}
+        6, nstd);
+    h.set_arg( // expected-error {{no matching member function for call to 'set_arg'}}
+        7, NonStdLayout{});
+#endif
+  });
+}
diff --git a/sycl/test/basic_tests/use_pinned_host_memory.cpp b/sycl/test/basic_tests/use_pinned_host_memory.cpp
new file mode 100644
index 0000000000000..a596093313cdf
--- /dev/null
+++ b/sycl/test/basic_tests/use_pinned_host_memory.cpp
@@ -0,0 +1,45 @@
+// RUN: %clangxx -fsycl -fsycl-targets=%sycl_triple %s -o %t.out
+// RUN: env SYCL_PI_TRACE=2 %CPU_RUN_PLACEHOLDER %t.out 2>&1 %CPU_CHECK_PLACEHOLDER
+
+#include <CL/sycl.hpp>
+
+#include <string>
+
+int main() {
+  const sycl::range<1> N{1};
+  sycl::buffer<int, 1> Buf(
+      N, {sycl::ext::oneapi::property::buffer::use_pinned_host_memory()});
+  if (!Buf.has_property<
+          sycl::ext::oneapi::property::buffer::use_pinned_host_memory>()) {
+    std::cerr << "Buffer should have the use_pinned_host_memory property"
+              << std::endl;
+    return 1;
+  }
+
+  sycl::queue Q;
+  Q.submit([&](sycl::handler &CGH) {
+    auto Acc = Buf.get_access<sycl::access::mode::read_write>(CGH);
+    CGH.single_task<class init_a>([=]() {});
+  });
+
+  try {
+    int Data = 0;
+    sycl::buffer<int, 1> Buf(
+        &Data, N,
+        {sycl::ext::oneapi::property::buffer::use_pinned_host_memory()});
+    // Expected that exception is thrown
+    return 1;
+  } catch (sycl::invalid_object_error &E) {
+    if (std::string(E.what()).find(
+            "The use_pinned_host_memory cannot be used with host pointer") ==
+        std::string::npos) {
+      return 1;
+    }
+
+    return 0;
+  }
+}
+
+// CHECK:---> piMemBufferCreate
+// CHECK-NEXT: {{.*}} : {{.*}}
+// CHECK-NEXT: {{.*}} : 17
diff --git a/sycl/test/fpga_tests/ap_fixed.cpp b/sycl/test/check_device_code/ap_fixed.cpp
similarity index 100%
rename from sycl/test/fpga_tests/ap_fixed.cpp
rename to sycl/test/check_device_code/ap_fixed.cpp
diff --git a/sycl/test/fpga_tests/fpga_ihs_float.cpp b/sycl/test/check_device_code/fpga_ihs_float.cpp
similarity index 65%
rename from sycl/test/fpga_tests/fpga_ihs_float.cpp
rename to sycl/test/check_device_code/fpga_ihs_float.cpp
index 47ff0a3b8b3b5..f452fc2517c35 100644
--- a/sycl/test/fpga_tests/fpga_ihs_float.cpp
+++ b/sycl/test/check_device_code/fpga_ihs_float.cpp
@@ -6,7 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// RUN: %clangxx -I %sycl_include -S -emit-llvm -fsycl-device-only %s -o - | FileCheck %s
+// RUN: %clangxx -I %sycl_include -S -emit-llvm -fsycl -fsycl-device-only %s -o - | FileCheck %s
+// RUN: %clangxx -I %sycl_include -S -emit-llvm -fsycl -fsycl-device-only %s -Xclang -fsycl-enable-optimizations -o - | FileCheck %s
 
 #include "CL/__spirv/spirv_ops.hpp"
 
@@ -18,7 +19,7 @@ void ap_float_cast() {
   ap_int<1 + Eout + Mout> float_cast_res =
       __spirv_ArbitraryFloatCastINTEL<1 + EA + MA, 1 + Eout + Mout>(
           A, MA, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func i40 @_Z{{[0-9]+}}__spirv_ArbitraryFloatCastINTEL{{.*}}(i40 %{{[0-9]+}}, i32 28, i32 30, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func i40 @_Z{{[0-9]+}}__spirv_ArbitraryFloatCastINTEL{{.*}}(i40 {{[%a-z0-9.]+}}, i32 28, i32 30, i32 0, i32 2, i32 1)
 }
 
 template <int WA, int Eout, int Mout>
@@ -27,7 +28,7 @@ void ap_float_cast_from_int() {
   ap_int<1 + Eout + Mout> cast_from_int_res =
       __spirv_ArbitraryFloatCastFromIntINTEL<WA, 1 + Eout + Mout>(
           A, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func signext i25 @_Z{{[0-9]+}}__spirv_ArbitraryFloatCastFromIntINTEL{{.*}}(i43 %{{[0-9]+}}, i32 16, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func signext i25 @_Z{{[0-9]+}}__spirv_ArbitraryFloatCastFromIntINTEL{{.*}}(i43 {{[%a-z0-9.]+}}, i32 16, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int Wout>
@@ -36,7 +37,7 @@ void ap_float_cast_to_int() {
   ap_int<Wout> cast_to_int_res =
       __spirv_ArbitraryFloatCastToIntINTEL<1 + EA + MA, Wout>(
           A, MA, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func signext i30 @_Z{{[0-9]+}}__spirv_ArbitraryFloatCastToIntINTEL{{.*}}(i23 signext %{{[0-9]+}}, i32 15, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func signext i30 @_Z{{[0-9]+}}__spirv_ArbitraryFloatCastToIntINTEL{{.*}}(i23 signext {{[%a-z0-9.]+}}, i32 15, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int EB, int MB, int Eout, int Mout>
@@ -46,8 +47,8 @@ void ap_float_add() {
   ap_int<1 + Eout + Mout> add_res =
       __spirv_ArbitraryFloatAddINTEL<1 + EA + MA, 1 + EB + MB, 1 + Eout + Mout>(
           A, MA, B, MB, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func signext i14 @_Z{{[0-9]+}}__spirv_ArbitraryFloatAddINTEL{{.*}}(i13 signext %{{[0-9]+}}, i32 7, i15 signext %{{[0-9]+}}, i32 8, i32 9, i32 0, i32 2, i32 1)
-  // CHECK: %{{.*}} = call spir_func signext i13 @_Z{{[0-9]+}}__spirv_ArbitraryFloatAddINTEL{{.*}}(i15 signext %{{[0-9]+}}, i32 8, i14 signext %{{[0-9]+}}, i32 9, i32 7, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func signext i14 @_Z{{[0-9]+}}__spirv_ArbitraryFloatAddINTEL{{.*}}(i13 signext {{[%a-z0-9.]+}}, i32 7, i15 signext {{[%a-z0-9.]+}}, i32 8, i32 9, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func signext i13 @_Z{{[0-9]+}}__spirv_ArbitraryFloatAddINTEL{{.*}}(i15 signext {{[%a-z0-9.]+}}, i32 8, i14 signext {{[%a-z0-9.]+}}, i32 9, i32 7, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int EB, int MB, int Eout, int Mout>
@@ -57,7 +58,7 @@ void ap_float_sub() {
   ap_int<1 + Eout + Mout> sub_res =
       __spirv_ArbitraryFloatSubINTEL<1 + EA + MA, 1 + EB + MB, 1 + Eout + Mout>(
           A, MA, B, MB, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func signext i13 @_Z{{[0-9]+}}__spirv_ArbitraryFloatSubINTEL{{.*}}(i9 signext %{{[0-9]+}}, i32 4, i11 signext %{{[0-9]+}}, i32 5, i32 6, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func signext i13 @_Z{{[0-9]+}}__spirv_ArbitraryFloatSubINTEL{{.*}}(i9 signext {{[%a-z0-9.]+}}, i32 4, i11 signext {{[%a-z0-9.]+}}, i32 5, i32 6, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int EB, int MB, int Eout, int Mout>
@@ -67,7 +68,7 @@ void ap_float_mul() {
   ap_int<1 + Eout + Mout> mul_res =
       __spirv_ArbitraryFloatMulINTEL<1 + EA + MA, 1 + EB + MB, 1 + Eout + Mout>(
           A, MA, B, MB, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func i51 @_Z{{[0-9]+}}__spirv_ArbitraryFloatMulINTEL{{.*}}(i51 %{{[0-9]+}}, i32 34, i51 %{{[0-9]+}}, i32 34, i32 34, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func i51 @_Z{{[0-9]+}}__spirv_ArbitraryFloatMulINTEL{{.*}}(i51 {{[%a-z0-9.]+}}, i32 34, i51 {{[%a-z0-9.]+}}, i32 34, i32 34, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int EB, int MB, int Eout, int Mout>
@@ -77,7 +78,7 @@ void ap_float_div() {
   ap_int<1 + Eout + Mout> div_res =
       __spirv_ArbitraryFloatDivINTEL<1 + EA + MA, 1 + EB + MB, 1 + Eout + Mout>(
           A, MA, B, MB, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func signext i18 @_Z{{[0-9]+}}__spirv_ArbitraryFloatDivINTEL{{.*}}(i16 signext %{{[0-9]+}}, i32 11, i16 signext %{{[0-9]+}}, i32 11, i32 12, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func signext i18 @_Z{{[0-9]+}}__spirv_ArbitraryFloatDivINTEL{{.*}}(i16 signext {{[%a-z0-9.]+}}, i32 11, i16 signext {{[%a-z0-9.]+}}, i32 11, i32 12, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int EB, int MB>
@@ -85,7 +86,7 @@ void ap_float_gt() {
   ap_int<1 + EA + MA> A;
   ap_int<1 + EB + MB> B;
   bool gt_res = __spirv_ArbitraryFloatGTINTEL<1 + EA + MA, 1 + EB + MB>(A, MA, B, MB);
-  // CHECK: %{{.*}} = call spir_func zeroext i1 @_Z{{[0-9]+}}__spirv_ArbitraryFloatGTINTEL{{.*}}(i63 %{{[0-9]+}}, i32 42, i63 %{{[0-9]+}}, i32 41)
+  // CHECK: call spir_func zeroext i1 @_Z{{[0-9]+}}__spirv_ArbitraryFloatGTINTEL{{.*}}(i63 {{[%a-z0-9.]+}}, i32 42, i63 {{[%a-z0-9.]+}}, i32 41)
 }
 
 template <int EA, int MA, int EB, int MB>
@@ -93,7 +94,7 @@ void ap_float_ge() {
   ap_int<1 + EA + MA> A;
   ap_int<1 + EB + MB> B;
   bool ge_res = __spirv_ArbitraryFloatGEINTEL<1 + EA + MA, 1 + EB + MB>(A, MA, B, MB);
-  // CHECK: %{{.*}} = call spir_func zeroext i1 @_Z{{[0-9]+}}__spirv_ArbitraryFloatGEINTEL{{.*}}(i47 %{{[0-9]+}}, i32 27, i47 %{{[0-9]+}}, i32 27)
+  // CHECK: call spir_func zeroext i1 @_Z{{[0-9]+}}__spirv_ArbitraryFloatGEINTEL{{.*}}(i47 {{[%a-z0-9.]+}}, i32 27, i47 {{[%a-z0-9.]+}}, i32 27)
 }
 
 template <int EA, int MA, int EB, int MB>
@@ -101,7 +102,7 @@ void ap_float_lt() {
   ap_int<1 + EA + MA> A;
   ap_int<1 + EB + MB> B;
   bool lt_res = __spirv_ArbitraryFloatLTINTEL<1 + EA + MA, 1 + EB + MB>(A, MA, B, MB);
-  // CHECK: %{{.*}} = call spir_func zeroext i1 @_Z{{[0-9]+}}__spirv_ArbitraryFloatLTINTEL{{.*}}(i5 signext %{{[0-9]+}}, i32 2, i7 signext %{{[0-9]+}}, i32 3)
+  // CHECK: call spir_func zeroext i1 @_Z{{[0-9]+}}__spirv_ArbitraryFloatLTINTEL{{.*}}(i5 signext {{[%a-z0-9.]+}}, i32 2, i7 signext {{[%a-z0-9.]+}}, i32 3)
 }
 
 template <int EA, int MA, int EB, int MB>
@@ -109,7 +110,7 @@ void ap_float_le() {
   ap_int<1 + EA + MA> A;
   ap_int<1 + EB + MB> B;
   bool le_res = __spirv_ArbitraryFloatLEINTEL<1 + EA + MA, 1 + EB + MB>(A, MA, B, MB);
-  // CHECK: %{{.*}} = call spir_func zeroext i1 @_Z{{[0-9]+}}__spirv_ArbitraryFloatLEINTEL{{.*}}(i55 %{{[0-9]+}}, i32 27, i55 %{{[0-9]+}}, i32 28)
+  // CHECK: call spir_func zeroext i1 @_Z{{[0-9]+}}__spirv_ArbitraryFloatLEINTEL{{.*}}(i55 {{[%a-z0-9.]+}}, i32 27, i55 {{[%a-z0-9.]+}}, i32 28)
 }
 
 template <int EA, int MA, int EB, int MB>
@@ -117,7 +118,7 @@ void ap_float_eq() {
   ap_int<1 + EA + MA> A;
   ap_int<1 + EB + MB> B;
   bool eq_res = __spirv_ArbitraryFloatEQINTEL<1 + EA + MA, 1 + EB + MB>(A, MA, B, MB);
-  // CHECK: %{{.*}} = call spir_func zeroext i1 @_Z{{[0-9]+}}__spirv_ArbitraryFloatEQINTEL{{.*}}(i20 signext %{{[0-9]+}}, i32 12, i15 signext %{{[0-9]+}}, i32 7)
+  // CHECK: call spir_func zeroext i1 @_Z{{[0-9]+}}__spirv_ArbitraryFloatEQINTEL{{.*}}(i20 signext {{[%a-z0-9.]+}}, i32 12, i15 signext {{[%a-z0-9.]+}}, i32 7)
 }
 
 template <int EA, int MA, int Eout, int Mout>
@@ -126,7 +127,7 @@ void ap_float_recip() {
   ap_int<1 + Eout + Mout> recip_res =
       __spirv_ArbitraryFloatRecipINTEL<1 + EA + MA, 1 + Eout + Mout>(
           A, MA, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func i39 @_Z{{[0-9]+}}__spirv_ArbitraryFloatRecipINTEL{{.*}}(i39 %{{[0-9]+}}, i32 29, i32 29, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func i39 @_Z{{[0-9]+}}__spirv_ArbitraryFloatRecipINTEL{{.*}}(i39 {{[%a-z0-9.]+}}, i32 29, i32 29, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int Eout, int Mout>
@@ -135,7 +136,7 @@ void ap_float_rsqrt() {
   ap_int<1 + Eout + Mout> rsqrt_res =
       __spirv_ArbitraryFloatRSqrtINTEL<1 + EA + MA, 1 + Eout + Mout>(
           A, MA, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func i34 @_Z{{[0-9]+}}__spirv_ArbitraryFloatRSqrtINTEL{{.*}}(i32 %{{[0-9]+}}, i32 19, i32 20, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func i34 @_Z{{[0-9]+}}__spirv_ArbitraryFloatRSqrtINTEL{{.*}}(i32 {{[%a-z0-9.]+}}, i32 19, i32 20, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int Eout, int Mout>
@@ -144,7 +145,7 @@ void ap_float_cbrt() {
   ap_int<1 + Eout + Mout> cbrt_res =
       __spirv_ArbitraryFloatCbrtINTEL<1 + EA + MA, 1 + Eout + Mout>(
           A, MA, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func signext i2 @_Z{{[0-9]+}}__spirv_ArbitraryFloatCbrtINTEL{{.*}}(i2 signext %{{[0-9]+}}, i32 1, i32 1, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func signext i2 @_Z{{[0-9]+}}__spirv_ArbitraryFloatCbrtINTEL{{.*}}(i2 signext {{[%a-z0-9.]+}}, i32 1, i32 1, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int EB, int MB, int Eout, int Mout>
@@ -154,7 +155,7 @@ void ap_float_hypot() {
   ap_int<1 + Eout + Mout> hypot_res =
       __spirv_ArbitraryFloatHypotINTEL<1 + EA + MA, 1 + EB + MB, 1 + Eout + Mout>(
           A, MA, B, MB, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func i42 @_Z{{[0-9]+}}__spirv_ArbitraryFloatHypotINTEL{{.*}}(i41 %{{[0-9]+}}, i32 20, i43 %{{[0-9]+}}, i32 21, i32 22, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func i42 @_Z{{[0-9]+}}__spirv_ArbitraryFloatHypotINTEL{{.*}}(i41 {{[%a-z0-9.]+}}, i32 20, i43 {{[%a-z0-9.]+}}, i32 21, i32 22, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int Eout, int Mout>
@@ -163,7 +164,7 @@ void ap_float_sqrt() {
   ap_int<1 + Eout + Mout> sqrt_res =
       __spirv_ArbitraryFloatSqrtINTEL<1 + EA + MA, 1 + Eout + Mout>(
           A, MA, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func signext i17 @_Z{{[0-9]+}}__spirv_ArbitraryFloatSqrtINTEL{{.*}}(i15 signext %{{[0-9]+}}, i32 7, i32 8, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func signext i17 @_Z{{[0-9]+}}__spirv_ArbitraryFloatSqrtINTEL{{.*}}(i15 signext {{[%a-z0-9.]+}}, i32 7, i32 8, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int Eout, int Mout>
@@ -172,7 +173,7 @@ void ap_float_log() {
   ap_int<1 + Eout + Mout> log_res =
       __spirv_ArbitraryFloatLogINTEL<1 + EA + MA, 1 + Eout + Mout>(
           A, MA, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func i50 @_Z{{[0-9]+}}__spirv_ArbitraryFloatLogINTEL{{.*}}(i50 %{{[0-9]+}}, i32 19, i32 30, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func i50 @_Z{{[0-9]+}}__spirv_ArbitraryFloatLogINTEL{{.*}}(i50 {{[%a-z0-9.]+}}, i32 19, i32 30, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int Eout, int Mout>
@@ -181,7 +182,7 @@ void ap_float_log2() {
   ap_int<1 + Eout + Mout> log2_res =
       __spirv_ArbitraryFloatLog2INTEL<1 + EA + MA, 1 + Eout + Mout>(
           A, MA, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func i38 @_Z{{[0-9]+}}__spirv_ArbitraryFloatLog2INTEL{{.*}}(i38 %{{[0-9]+}}, i32 20, i32 19, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func i38 @_Z{{[0-9]+}}__spirv_ArbitraryFloatLog2INTEL{{.*}}(i38 {{[%a-z0-9.]+}}, i32 20, i32 19, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int Eout, int Mout>
@@ -190,7 +191,7 @@ void ap_float_log10() {
   ap_int<1 + Eout + Mout> log10_res =
       __spirv_ArbitraryFloatLog10INTEL<1 + EA + MA, 1 + Eout + Mout>(
           A, MA, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func signext i10 @_Z{{[0-9]+}}__spirv_ArbitraryFloatLog10INTEL{{.*}}(i8 signext %{{[0-9]+}}, i32 3, i32 5, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func signext i10 @_Z{{[0-9]+}}__spirv_ArbitraryFloatLog10INTEL{{.*}}(i8 signext {{[%a-z0-9.]+}}, i32 3, i32 5, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int Eout, int Mout>
@@ -199,7 +200,7 @@ void ap_float_log1p() {
   ap_int<1 + Eout + Mout> log1p_res =
       __spirv_ArbitraryFloatLog1pINTEL<1 + EA + MA, 1 + Eout + Mout>(
           A, MA, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func i49 @_Z{{[0-9]+}}__spirv_ArbitraryFloatLog1pINTEL{{.*}}(i48 %{{[0-9]+}}, i32 30, i32 30, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func i49 @_Z{{[0-9]+}}__spirv_ArbitraryFloatLog1pINTEL{{.*}}(i48 {{[%a-z0-9.]+}}, i32 30, i32 30, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int Eout, int Mout>
@@ -208,7 +209,7 @@ void ap_float_exp() {
   ap_int<1 + Eout + Mout> exp_res =
       __spirv_ArbitraryFloatExpINTEL<1 + EA + MA, 1 + Eout + Mout>(
           A, MA, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func i42 @_Z{{[0-9]+}}__spirv_ArbitraryFloatExpINTEL{{.*}}(i42 %{{[0-9]+}}, i32 25, i32 25, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func i42 @_Z{{[0-9]+}}__spirv_ArbitraryFloatExpINTEL{{.*}}(i42 {{[%a-z0-9.]+}}, i32 25, i32 25, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int Eout, int Mout>
@@ -217,7 +218,7 @@ void ap_float_exp2() {
   ap_int<1 + Eout + Mout> exp2_res =
       __spirv_ArbitraryFloatExp2INTEL<1 + EA + MA, 1 + Eout + Mout>(
           A, MA, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func signext i5 @_Z{{[0-9]+}}__spirv_ArbitraryFloatExp2INTEL{{.*}}(i3 signext %{{[0-9]+}}, i32 1, i32 2, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func signext i5 @_Z{{[0-9]+}}__spirv_ArbitraryFloatExp2INTEL{{.*}}(i3 signext {{[%a-z0-9.]+}}, i32 1, i32 2, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int Eout, int Mout>
@@ -226,7 +227,7 @@ void ap_float_exp10() {
   ap_int<1 + Eout + Mout> exp10_res =
       __spirv_ArbitraryFloatExp10INTEL<1 + EA + MA, 1 + Eout + Mout>(
           A, MA, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func signext i25 @_Z{{[0-9]+}}__spirv_ArbitraryFloatExp10INTEL{{.*}}(i25 signext %{{[0-9]+}}, i32 16, i32 16, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func signext i25 @_Z{{[0-9]+}}__spirv_ArbitraryFloatExp10INTEL{{.*}}(i25 signext {{[%a-z0-9.]+}}, i32 16, i32 16, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int Eout, int Mout>
@@ -235,7 +236,7 @@ void ap_float_expm1() {
   ap_int<1 + Eout + Mout> expm1_res =
       __spirv_ArbitraryFloatExpm1INTEL<1 + EA + MA, 1 + Eout + Mout>(
           A, MA, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func i62 @_Z{{[0-9]+}}__spirv_ArbitraryFloatExpm1INTEL{{.*}}(i64 %{{[0-9]+}}, i32 42, i32 41, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func i62 @_Z{{[0-9]+}}__spirv_ArbitraryFloatExpm1INTEL{{.*}}(i64 {{[%a-z0-9.]+}}, i32 42, i32 41, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int Eout, int Mout>
@@ -244,7 +245,7 @@ void ap_float_sin() {
   ap_int<1 + Eout + Mout> sin_res =
       __spirv_ArbitraryFloatSinINTEL<1 + EA + MA, 1 + Eout + Mout>(
           A, MA, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func i34 @_Z{{[0-9]+}}__spirv_ArbitraryFloatSinINTEL{{.*}}(i30 signext %{{[0-9]+}}, i32 15, i32 17, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func i34 @_Z{{[0-9]+}}__spirv_ArbitraryFloatSinINTEL{{.*}}(i30 signext {{[%a-z0-9.]+}}, i32 15, i32 17, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int Eout, int Mout>
@@ -253,7 +254,7 @@ void ap_float_cos() {
   ap_int<1 + Eout + Mout> cos_res =
       __spirv_ArbitraryFloatCosINTEL<1 + EA + MA, 1 + Eout + Mout>(
           A, MA, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func signext i4 @_Z{{[0-9]+}}__spirv_ArbitraryFloatCosINTEL{{.*}}(i4 signext %{{[0-9]+}}, i32 2, i32 1, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func signext i4 @_Z{{[0-9]+}}__spirv_ArbitraryFloatCosINTEL{{.*}}(i4 signext {{[%a-z0-9.]+}}, i32 2, i32 1, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int Eout, int Mout>
@@ -262,7 +263,7 @@ void ap_float_sincos() {
   ap_int<2 * (1 + Eout + Mout)> sincos_res =
       __spirv_ArbitraryFloatSinCosINTEL<1 + EA + MA, 1 + Eout + Mout>(
           A, MA, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func i62 @_Z{{[0-9]+}}__spirv_ArbitraryFloatSinCosINTEL{{.*}}(i27 signext %{{[0-9]+}}, i32 18, i32 20, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func i62 @_Z{{[0-9]+}}__spirv_ArbitraryFloatSinCosINTEL{{.*}}(i27 signext {{[%a-z0-9.]+}}, i32 18, i32 20, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int Eout, int Mout>
@@ -271,7 +272,7 @@ void ap_float_sinpi() {
   ap_int<1 + Eout + Mout> sinpi_res =
       __spirv_ArbitraryFloatSinPiINTEL<1 + EA + MA, 1 + Eout + Mout>(
           A, MA, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func signext i13 @_Z{{[0-9]+}}__spirv_ArbitraryFloatSinPiINTEL{{.*}}(i10 signext %{{[0-9]+}}, i32 6, i32 6, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func signext i13 @_Z{{[0-9]+}}__spirv_ArbitraryFloatSinPiINTEL{{.*}}(i10 signext {{[%a-z0-9.]+}}, i32 6, i32 6, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int Eout, int Mout>
@@ -280,7 +281,7 @@ void ap_float_cospi() {
   ap_int<1 + Eout + Mout> cospi_res =
       __spirv_ArbitraryFloatCosPiINTEL<1 + EA + MA, 1 + Eout + Mout>(
           A, MA, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func i59 @_Z{{[0-9]+}}__spirv_ArbitraryFloatCosPiINTEL{{.*}}(i59 %{{[0-9]+}}, i32 40, i32 40, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func i59 @_Z{{[0-9]+}}__spirv_ArbitraryFloatCosPiINTEL{{.*}}(i59 {{[%a-z0-9.]+}}, i32 40, i32 40, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int Eout, int Mout>
@@ -289,7 +290,7 @@ void ap_float_sincospi() {
   ap_int<2 * (1 + Eout + Mout)> sincos_res =
       __spirv_ArbitraryFloatSinCosPiINTEL<1 + EA + MA, 1 + Eout + Mout>(
           A, MA, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func i64 @_Z{{[0-9]+}}__spirv_ArbitraryFloatSinCosPiINTEL{{.*}}(i30 signext %{{[0-9]+}}, i32 20, i32 20, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func i64 @_Z{{[0-9]+}}__spirv_ArbitraryFloatSinCosPiINTEL{{.*}}(i30 signext {{[%a-z0-9.]+}}, i32 20, i32 20, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int Eout, int Mout>
@@ -298,7 +299,7 @@ void ap_float_asin() {
   ap_int<1 + Eout + Mout> asin_res =
       __spirv_ArbitraryFloatASinINTEL<1 + EA + MA, 1 + Eout + Mout>(
           A, MA, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func signext i11 @_Z{{[0-9]+}}__spirv_ArbitraryFloatASinINTEL{{.*}}(i7 signext %{{[0-9]+}}, i32 4, i32 8, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func signext i11 @_Z{{[0-9]+}}__spirv_ArbitraryFloatASinINTEL{{.*}}(i7 signext {{[%a-z0-9.]+}}, i32 4, i32 8, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int Eout, int Mout>
@@ -307,7 +308,7 @@ void ap_float_asinpi() {
   ap_int<1 + Eout + Mout> asinpi_res =
       __spirv_ArbitraryFloatASinPiINTEL<1 + EA + MA, 1 + Eout + Mout>(
           A, MA, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func i35 @_Z{{[0-9]+}}__spirv_ArbitraryFloatASinPiINTEL{{.*}}(i35 %{{[0-9]+}}, i32 23, i32 23, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func i35 @_Z{{[0-9]+}}__spirv_ArbitraryFloatASinPiINTEL{{.*}}(i35 {{[%a-z0-9.]+}}, i32 23, i32 23, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int Eout, int Mout>
@@ -316,7 +317,7 @@ void ap_float_acos() {
   ap_int<1 + Eout + Mout> acos_res =
       __spirv_ArbitraryFloatACosINTEL<1 + EA + MA, 1 + Eout + Mout>(
           A, MA, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func signext i14 @_Z{{[0-9]+}}__spirv_ArbitraryFloatACosINTEL{{.*}}(i14 signext %{{[0-9]+}}, i32 9, i32 10, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func signext i14 @_Z{{[0-9]+}}__spirv_ArbitraryFloatACosINTEL{{.*}}(i14 signext {{[%a-z0-9.]+}}, i32 9, i32 10, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int Eout, int Mout>
@@ -325,7 +326,7 @@ void ap_float_acospi() {
   ap_int<1 + Eout + Mout> acospi_res =
       __spirv_ArbitraryFloatACosPiINTEL<1 + EA + MA, 1 + Eout + Mout>(
           A, MA, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func signext i8 @_Z{{[0-9]+}}__spirv_ArbitraryFloatACosPiINTEL{{.*}}(i8 signext %{{[0-9]+}}, i32 5, i32 4, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func signext i8 @_Z{{[0-9]+}}__spirv_ArbitraryFloatACosPiINTEL{{.*}}(i8 signext {{[%a-z0-9.]+}}, i32 5, i32 4, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int Eout, int Mout>
@@ -334,7 +335,7 @@ void ap_float_atan() {
   ap_int<1 + Eout + Mout> atan_res =
       __spirv_ArbitraryFloatATanINTEL<1 + EA + MA, 1 + Eout + Mout>(
           A, MA, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func i44 @_Z{{[0-9]+}}__spirv_ArbitraryFloatATanINTEL{{.*}}(i44 %{{[0-9]+}}, i32 31, i32 31, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func i44 @_Z{{[0-9]+}}__spirv_ArbitraryFloatATanINTEL{{.*}}(i44 {{[%a-z0-9.]+}}, i32 31, i32 31, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int Eout, int Mout>
@@ -343,7 +344,7 @@ void ap_float_atapin() {
   ap_int<1 + Eout + Mout> atanpi_res =
       __spirv_ArbitraryFloatATanPiINTEL<1 + EA + MA, 1 + Eout + Mout>(
           A, MA, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func i34 @_Z{{[0-9]+}}__spirv_ArbitraryFloatATanPiINTEL{{.*}}(i40 %{{[0-9]+}}, i32 38, i32 32, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func i34 @_Z{{[0-9]+}}__spirv_ArbitraryFloatATanPiINTEL{{.*}}(i40 {{[%a-z0-9.]+}}, i32 38, i32 32, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int EB, int MB, int Eout, int Mout>
@@ -353,7 +354,7 @@ void ap_float_atan2() {
   ap_int<1 + Eout + Mout> atan2_res =
       __spirv_ArbitraryFloatATan2INTEL<1 + EA + MA, 1 + EB + MB, 1 + Eout + Mout>(
           A, MA, B, MB, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func signext i27 @_Z{{[0-9]+}}__spirv_ArbitraryFloatATan2INTEL{{.*}}(i24 signext %{{[0-9]+}}, i32 16, i25 signext %{{[0-9]+}}, i32 17, i32 18, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func signext i27 @_Z{{[0-9]+}}__spirv_ArbitraryFloatATan2INTEL{{.*}}(i24 signext {{[%a-z0-9.]+}}, i32 16, i25 signext {{[%a-z0-9.]+}}, i32 17, i32 18, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int EB, int MB, int Eout, int Mout>
@@ -363,7 +364,7 @@ void ap_float_pow() {
   ap_int<1 + Eout + Mout> pow_res =
       __spirv_ArbitraryFloatPowINTEL<1 + EA + MA, 1 + EB + MB, 1 + Eout + Mout>(
           A, MA, B, MB, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func signext i21 @_Z{{[0-9]+}}__spirv_ArbitraryFloatPowINTEL{{.*}}(i17 signext %{{[0-9]+}}, i32 8, i19 signext %{{[0-9]+}}, i32 9, i32 10, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func signext i21 @_Z{{[0-9]+}}__spirv_ArbitraryFloatPowINTEL{{.*}}(i17 signext {{[%a-z0-9.]+}}, i32 8, i19 signext {{[%a-z0-9.]+}}, i32 9, i32 10, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int EB, int MB, int Eout, int Mout>
@@ -373,7 +374,7 @@ void ap_float_powr() {
   ap_int<1 + Eout + Mout> powr_res =
       __spirv_ArbitraryFloatPowRINTEL<1 + EA + MA, 1 + EB + MB, 1 + Eout + Mout>(
           A, MA, B, MB, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func i56 @_Z{{[0-9]+}}__spirv_ArbitraryFloatPowRINTEL{{.*}}(i54 %{{[0-9]+}}, i32 35, i55 %{{[0-9]+}}, i32 35, i32 35, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func i56 @_Z{{[0-9]+}}__spirv_ArbitraryFloatPowRINTEL{{.*}}(i54 {{[%a-z0-9.]+}}, i32 35, i55 {{[%a-z0-9.]+}}, i32 35, i32 35, i32 0, i32 2, i32 1)
 }
 
 template <int EA, int MA, int WB, int Eout, int Mout>
@@ -383,7 +384,7 @@ void ap_float_pown() {
   ap_int<1 + Eout + Mout> pown_res =
       __spirv_ArbitraryFloatPowNINTEL<1 + EA + MA, WB, 1 + Eout + Mout>(
           A, MA, B, Mout, Subnorm, RndMode, RndAcc);
-  // CHECK: %{{.*}} = call spir_func signext i15 @_Z{{[0-9]+}}__spirv_ArbitraryFloatPowNINTEL{{.*}}(i12 signext %{{[0-9]+}}, i32 7, i10 signext %{{[0-9]+}}, i32 9, i32 0, i32 2, i32 1)
+  // CHECK: call spir_func signext i15 @_Z{{[0-9]+}}__spirv_ArbitraryFloatPowNINTEL{{.*}}(i12 signext {{[%a-z0-9.]+}}, i32 7, i10 signext {{[%a-z0-9.]+}}, i32 9, i32 0, i32 2, i32 1)
 }
 
 template <typename name, typename Func>
diff --git a/sycl/test/esimd/glob.cpp b/sycl/test/esimd/glob.cpp
new file mode 100644
index 0000000000000..ae752ad5801ea
--- /dev/null
+++ b/sycl/test/esimd/glob.cpp
@@ -0,0 +1,28 @@
+// RUN: %clangxx -fsycl -fsycl-explicit-simd -c -fsycl-device-only -Xclang -emit-llvm %s -o - | \
+// RUN:  FileCheck %s
+
+// This test checks that globals with register attribute are allowed in ESIMD
+// mode, can be accessed in functions and correct LLVM IR is generated
+// (including translation of the register attribute)
+
+#include <CL/sycl.hpp>
+#include <CL/sycl/intel/esimd.hpp>
+#include <iostream>
+
+using namespace cl::sycl;
+using namespace sycl::intel::gpu;
+
+constexpr unsigned VL = 16;
+
+ESIMD_PRIVATE ESIMD_REGISTER(17) simd<int, VL> vc;
+// CHECK-DAG: @vc = {{.+}} <16 x i32> zeroinitializer, align 64 #0
+// CHECK-DAG: attributes #0 = { {{.*}}"VCByteOffset"="17" "VCGlobalVariable" "VCVolatile"{{.*}} }
+
+ESIMD_PRIVATE ESIMD_REGISTER(17 + VL) simd<int, VL> vc1;
+// CHECK-DAG: @vc1 = {{.+}} <16 x i32> zeroinitializer, align 64 #1
+// CHECK-DAG: attributes #1 = { {{.*}}"VCByteOffset"="33" "VCGlobalVariable" "VCVolatile"{{.*}} }
+
+SYCL_EXTERNAL ESIMD_NOINLINE void init_vc(int x) {
+  vc1 = vc + 1;
+  vc = x;
+}
diff --git a/sycl/test/esimd/hw_compile.cpp b/sycl/test/esimd/hw_compile.cpp
new file mode 100644
index 0000000000000..96ffe30fda89c
--- /dev/null
+++ b/sycl/test/esimd/hw_compile.cpp
@@ -0,0 +1,39 @@
+// Basic ESIMD test which checks that ESIMD invocation syntax can get compiled.
+// RUN: %clangxx -fsycl -fsycl-explicit-simd -fsycl-device-only -c %s -o %t.bc
+
+#include <CL/sycl.hpp>
+#include <CL/sycl/intel/esimd.hpp>
+#include <iostream>
+
+int main(void) {
+  constexpr unsigned Size = 4;
+  int A[Size] = {1, 2, 3, 4};
+  int B[Size] = {1, 2, 3, 4};
+  int C[Size];
+
+  {
+    cl::sycl::range<1> UnitRange{1};
+    cl::sycl::buffer<int, 1> bufA(A, UnitRange);
+    cl::sycl::buffer<int, 1> bufB(B, UnitRange);
+    cl::sycl::buffer<int, 1> bufC(C, UnitRange);
+
+    cl::sycl::queue().submit([&](cl::sycl::handler &cgh) {
+      auto accA = bufA.get_access<cl::sycl::access::mode::read>(cgh);
+      auto accB = bufB.get_access<cl::sycl::access::mode::read>(cgh);
+      auto accC = bufC.get_access<cl::sycl::access::mode::write>(cgh);
+
+      cgh.parallel_for<class Test>(UnitRange * UnitRange,
+                                   [=](sycl::id<1> i) SYCL_ESIMD_KERNEL {
+                                     // those operations below would normally be
+                                     // represented as a single vector operation
+                                     // through ESIMD vector
+                                     accC[i + 0] = accA[i + 0] + accB[i + 0];
+                                     accC[i + 1] = accA[i + 1] + accB[i + 1];
+                                     accC[i + 2] = accA[i + 2] + accB[i + 2];
+                                     accC[i + 3] = accA[i + 3] + accB[i + 3];
+                                   });
+    });
+  }
+
+  return 0;
+}
diff --git a/sycl/test/esimd/intrins_trans.cpp b/sycl/test/esimd/intrins_trans.cpp
new file mode 100644
index 0000000000000..8bdd4981b78e9
--- /dev/null
+++ b/sycl/test/esimd/intrins_trans.cpp
@@ -0,0 +1,117 @@
+// RUN: %clangxx -O0 -fsycl -fsycl-explicit-simd -fsycl-device-only -Xclang -emit-llvm %s -o - | \
+// RUN:   FileCheck %s
+
+// Checks ESIMD intrinsic translation.
+// NOTE: must be run in -O0, as optimizer optimizes away some of the code
+
+#include <CL/sycl.hpp>
+#include <CL/sycl/detail/image_ocl_types.hpp>
+#include <CL/sycl/intel/esimd.hpp>
+
+using namespace sycl::intel::gpu;
+
+ESIMD_PRIVATE vector_type_t<int, 32> vc;
+ESIMD_PRIVATE ESIMD_REGISTER(192) simd<int, 16> vg;
+
+SYCL_ESIMD_FUNCTION SYCL_EXTERNAL simd<float, 16> foo();
+
+class EsimdFunctor {
+public:
+  void operator()() __attribute__((sycl_explicit_simd)) { foo(); }
+};
+
+template <typename name, typename Func>
+__attribute__((sycl_kernel)) void kernel(Func kernelFunc) {
+  kernelFunc();
+}
+
+void bar() {
+  EsimdFunctor esimdf;
+  kernel<class kernel_esimd>(esimdf);
+}
+
+SYCL_ESIMD_FUNCTION SYCL_EXTERNAL simd<float, 16> foo() {
+  // CHECK-LABEL: @_Z3foov
+  constexpr int VL = 32;
+  uint32_t *ptr = 0;
+
+  int x = 0, y = 0, z = 0;
+
+  simd<uint32_t, VL> v1(0, x + z);
+  simd<uint64_t, VL> offsets(0, y);
+  simd<uintptr_t, VL> v_addr(reinterpret_cast<uintptr_t>(ptr));
+  simd<ushort, VL> pred;
+  v_addr += offsets;
+
+  __esimd_flat_atomic0<EsimdAtomicOpType::ATOMIC_INC, uint32_t, VL>(
+      v_addr.data(), pred.data());
+  // CHECK: %{{[0-9a-zA-Z_.]+}} = call <32 x i32> @llvm.genx.svm.atomic.inc.v32i32.v32i1.v32i64(<32 x i1> %{{[0-9a-zA-Z_.]+}}, <32 x i64> %{{[0-9a-zA-Z_.]+}}, <32 x i32> undef)
+
+  __esimd_flat_atomic1<EsimdAtomicOpType::ATOMIC_ADD, uint32_t, VL>(
+      v_addr.data(), v1, pred.data());
+  // CHECK: %{{[0-9a-zA-Z_.]+}} = call <32 x i32> @llvm.genx.svm.atomic.add.v32i32.v32i1.v32i64(<32 x i1> %{{[0-9a-zA-Z_.]+}}, <32 x i64> %{{[0-9a-zA-Z_.]+}}, <32 x i32> %{{[0-9a-zA-Z_.]+}}, <32 x i32> undef)
+  __esimd_flat_atomic2<EsimdAtomicOpType::ATOMIC_CMPXCHG, uint32_t, VL>(
+      v_addr.data(), v1, v1, pred.data());
+  // CHECK: %{{[0-9a-zA-Z_.]+}} = call <32 x i32> @llvm.genx.svm.atomic.cmpxchg.v32i32.v32i1.v32i64(<32 x i1> %{{[0-9a-zA-Z_.]+}}, <32 x i64> %{{[0-9a-zA-Z_.]+}}, <32 x i32> %{{[0-9a-zA-Z_.]+}}, <32 x i32> %{{[0-9a-zA-Z_.]+}}, <32 x i32> undef)
+
+  uintptr_t addr = reinterpret_cast<uintptr_t>(ptr);
+  simd<uint32_t, VL> v00 =
+      __esimd_flat_block_read_unaligned<uint32_t, VL>(addr);
+  // CHECK: %{{[0-9a-zA-Z_.]+}} = call <32 x i32> @llvm.genx.svm.block.ld.unaligned.v32i32(i64 %{{[0-9a-zA-Z_.]+}})
+  __esimd_flat_block_write<uint32_t, VL>(addr, v00.data());
+  // CHECK: call void @llvm.genx.svm.block.st.v32i32(i64 %{{[0-9a-zA-Z_.]+}}, <32 x i32> %{{[0-9a-zA-Z_.]+}})
+
+  simd<uint32_t, VL> v01 =
+      __esimd_flat_read<uint32_t, VL>(v_addr.data(), 0, pred.data());
+  // CHECK: %{{[0-9a-zA-Z_.]+}} = call <32 x i32> @llvm.genx.svm.gather.v32i32.v32i1.v32i64(<32 x i1> %{{[0-9a-zA-Z_.]+}}, i32 0, <32 x i64> %{{[0-9a-zA-Z_.]+}}, <32 x i32> undef)
+
+  __esimd_flat_write<uint32_t, VL>(v_addr.data(), v01.data(), 0, pred.data());
+  // CHECK: call void @llvm.genx.svm.scatter.v32i1.v32i64.v32i32(<32 x i1> %{{[0-9a-zA-Z_.]+}}, i32 0, <32 x i64> %{{[0-9a-zA-Z_.]+}}, <32 x i32> %{{[0-9a-zA-Z_.]+}})
+
+  simd<short, 16> mina(0, 1);
+  simd<short, 16> minc(5);
+  minc = __esimd_smin<short, 16>(mina.data(), minc.data());
+  // CHECK:  %{{[0-9a-zA-Z_.]+}} = call <16 x i16> @llvm.genx.smin.v16i16.v16i16(<16 x i16> %{{[0-9a-zA-Z_.]+}}, <16 x i16> %{{[0-9a-zA-Z_.]+}})
+
+  simd<float, 1> diva(2.f);
+  simd<float, 1> divb(1.f);
+  diva = __esimd_div_ieee<1>(diva.data(), divb.data());
+  // CHECK:  %{{[0-9a-zA-Z_.]+}} = call <1 x float> @llvm.genx.ieee.div.v1f32(<1 x float>  %{{[0-9a-zA-Z_.]+}}, <1 x float>  %{{[0-9a-zA-Z_.]+}})
+
+  simd<float, 16> a(0.1f);
+  simd<float, 8> b = __esimd_rdregion<float, 16, 8, 0, 8, 1>(a.data(), 0);
+  // CHECK: %{{[0-9a-zA-Z_.]+}} = call <8 x float> @llvm.genx.rdregionf.v8f32.v16f32.i16(<16 x float> %{{[0-9a-zA-Z_.]+}}, i32 0, i32 8, i32 1, i16 0, i32 0)
+
+  simd<float, 16> c(0.0f);
+
+  using PH = cl::sycl::access::placeholder;
+
+  cl::sycl::accessor<cl::sycl::cl_int4, 2, cl::sycl::access::mode::read,
+                     cl::sycl::access::target::image, PH::false_t>
+      pA;
+  cl::sycl::accessor<cl::sycl::cl_int4, 2, cl::sycl::access::mode::write,
+                     cl::sycl::access::target::image, PH::false_t>
+      pB;
+
+  auto d = __esimd_wrregion<float, 16 /*ret size*/, 8 /*write size*/,
+                            0 /*vstride*/, 8 /*row width*/, 1 /*hstride*/>(
+      c.data() /*dst*/, b.data() /*src*/, 0 /*offset*/);
+  // CHECK: %{{[0-9a-zA-Z_.]+}} = call <16 x float> @llvm.genx.wrregionf.v16f32.v8f32.i16.v8i1(<16 x float> %{{[0-9a-zA-Z_.]+}}, <8 x float> %{{[0-9a-zA-Z_.]+}}, i32 0, i32 8, i32 1, i16 0, i32 0, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+
+  simd<int, 32> va;
+  va = media_block_load<int, 4, 8>(pA, x, y);
+  // CHECK: %[[SI0:[0-9a-zA-Z_.]+]] = ptrtoint %opencl.image2d_ro_t addrspace(1)* %{{[0-9a-zA-Z_.]+}} to i32
+  // CHECK: %{{[0-9a-zA-Z_.]+}} = call <32 x i32> @llvm.genx.media.ld.v32i32(i32 0, i32 %[[SI0]], i32 0, i32 32, i32 %{{[0-9a-zA-Z_.]+}}, i32 %{{[0-9a-zA-Z_.]+}})
+
+  simd<int, 32> vb = va + 1;
+  media_block_store<int, 4, 8>(pB, x, y, vb);
+  // CHECK: %[[SI2:[0-9a-zA-Z_.]+]] = ptrtoint %opencl.image2d_wo_t addrspace(1)* %{{[0-9a-zA-Z_.]+}} to i32
+  // CHECK: call void @llvm.genx.media.st.v32i32(i32 0, i32 %[[SI2]], i32 0, i32 32, i32 %{{[0-9a-zA-Z_.]+}}, i32 %{{[0-9a-zA-Z_.]+}}, <32 x i32> %{{[0-9a-zA-Z_.]+}})
+
+  auto ee = __esimd_vload<int, 16>((vector_type_t<int, 16> *)(&vg));
+  // CHECK: %{{[0-9a-zA-Z_.]+}} = call <16 x i32> @llvm.genx.vload.v16i32.p4v16i32(<16 x i32> addrspace(4)* {{.*}})
+  __esimd_vstore<int, 32>(&vc, va.data());
+  // CHECK: store <32 x i32>  %{{[0-9a-zA-Z_.]+}}, <32 x i32> addrspace(4)* {{.*}}
+
+  return d;
+}
diff --git a/sycl/test/esimd/spirv_intrins_trans.cpp b/sycl/test/esimd/spirv_intrins_trans.cpp
new file mode 100644
index 0000000000000..3327b0c13e117
--- /dev/null
+++ b/sycl/test/esimd/spirv_intrins_trans.cpp
@@ -0,0 +1,205 @@
+// RUN: %clangxx -fsycl -fsycl-explicit-simd -fsycl-device-only -O0 -S -emit-llvm -x c++ %s -o - | FileCheck %s
+// This test checks that all SPIRV intrinsics are correctly
+// translated into GenX counterparts (implemented in LowerCM.cpp)
+
+#include <CL/sycl.hpp>
+#include <CL/sycl/intel/esimd.hpp>
+
+SYCL_EXTERNAL size_t __spirv_GlobalInvocationId_x();
+SYCL_EXTERNAL size_t __spirv_GlobalInvocationId_y();
+SYCL_EXTERNAL size_t __spirv_GlobalInvocationId_z();
+
+SYCL_EXTERNAL size_t __spirv_GlobalSize_x();
+SYCL_EXTERNAL size_t __spirv_GlobalSize_y();
+SYCL_EXTERNAL size_t __spirv_GlobalSize_z();
+
+SYCL_EXTERNAL size_t __spirv_GlobalOffset_x();
+SYCL_EXTERNAL size_t __spirv_GlobalOffset_y();
+SYCL_EXTERNAL size_t __spirv_GlobalOffset_z();
+
+SYCL_EXTERNAL size_t __spirv_NumWorkgroups_x();
+SYCL_EXTERNAL size_t __spirv_NumWorkgroups_y();
+SYCL_EXTERNAL size_t __spirv_NumWorkgroups_z();
+
+SYCL_EXTERNAL size_t __spirv_WorkgroupSize_x();
+SYCL_EXTERNAL size_t __spirv_WorkgroupSize_y();
+SYCL_EXTERNAL size_t __spirv_WorkgroupSize_z();
+
+SYCL_EXTERNAL size_t __spirv_WorkgroupId_x();
+SYCL_EXTERNAL size_t __spirv_WorkgroupId_y();
+SYCL_EXTERNAL size_t __spirv_WorkgroupId_z();
+
+SYCL_EXTERNAL size_t __spirv_LocalInvocationId_x();
+SYCL_EXTERNAL size_t __spirv_LocalInvocationId_y();
+SYCL_EXTERNAL size_t __spirv_LocalInvocationId_z();
+
+template <typename name, typename Func>
+__attribute__((sycl_kernel)) void kernel(Func kernelFunc) {
+  kernelFunc();
+}
+
+size_t caller() {
+
+  size_t DoNotOpt;
+  cl::sycl::buffer<size_t, 1> buf(&DoNotOpt, 1);
+  cl::sycl::queue().submit([&](cl::sycl::handler &cgh) {
+    auto DoNotOptimize = buf.get_access<cl::sycl::access::mode::write>(cgh);
+
+    kernel<class kernel_GlobalInvocationId_x>([=]() SYCL_ESIMD_KERNEL {
+      *DoNotOptimize.get_pointer() = __spirv_GlobalInvocationId_x();
+    });
+    // CHECK-LABEL: @{{.*}}kernel_GlobalInvocationId_x
+    // CHECK: [[CALL_ESIMD1:%.*]] = call <3 x i32> @llvm.genx.local.id.v3i32()
+    // CHECK: {{.*}} extractelement <3 x i32> [[CALL_ESIMD1]], i32 0
+    // CHECK: [[CALL_ESIMD2:%.*]] = call <3 x i32> @llvm.genx.local.size.v3i32()
+    // CHECK: {{.*}} extractelement <3 x i32> [[CALL_ESIMD2]], i32 0
+    // CHECK: {{.*}} call i32 @llvm.genx.group.id.x()
+
+    kernel<class kernel_GlobalInvocationId_y>([=]() SYCL_ESIMD_KERNEL {
+      *DoNotOptimize.get_pointer() = __spirv_GlobalInvocationId_y();
+    });
+    // CHECK-LABEL: @{{.*}}kernel_GlobalInvocationId_y
+    // CHECK: [[CALL_ESIMD1:%.*]] = call <3 x i32> @llvm.genx.local.id.v3i32()
+    // CHECK: {{.*}} extractelement <3 x i32> [[CALL_ESIMD1]], i32 1
+    // CHECK: [[CALL_ESIMD2:%.*]] = call <3 x i32> @llvm.genx.local.size.v3i32()
+    // CHECK: {{.*}} extractelement <3 x i32> [[CALL_ESIMD2]], i32 1
+    // CHECK: {{.*}} call i32 @llvm.genx.group.id.y()
+
+    kernel<class kernel_GlobalInvocationId_z>([=]() SYCL_ESIMD_KERNEL {
+      *DoNotOptimize.get_pointer() = __spirv_GlobalInvocationId_z();
+    });
+    // CHECK-LABEL: @{{.*}}kernel_GlobalInvocationId_z
+    // CHECK: [[CALL_ESIMD1:%.*]] = call <3 x i32> @llvm.genx.local.id.v3i32()
+    // CHECK: {{.*}} extractelement <3 x i32> [[CALL_ESIMD1]], i32 2
+    // CHECK: [[CALL_ESIMD2:%.*]] = call <3 x i32> @llvm.genx.local.size.v3i32()
+    // CHECK: {{.*}} extractelement <3 x i32> [[CALL_ESIMD2]], i32 2
+    // CHECK: {{.*}} call i32 @llvm.genx.group.id.z()
+
+    kernel<class kernel_GlobalSize_x>([=]() SYCL_ESIMD_KERNEL {
+      *DoNotOptimize.get_pointer() = __spirv_GlobalSize_x();
+    });
+    // CHECK-LABEL: @{{.*}}kernel_GlobalSize_x
+    // CHECK: [[CALL_ESIMD1:%.*]] = call <3 x i32> @llvm.genx.local.size.v3i32()
+    // CHECK: {{.*}} extractelement <3 x i32> [[CALL_ESIMD1]], i32 0
+    // CHECK: [[CALL_ESIMD2:%.*]] = call <3 x i32> @llvm.genx.group.count.v3i32()
+    // CHECK: {{.*}} extractelement <3 x i32> [[CALL_ESIMD2]], i32 0
+
+    kernel<class kernel_GlobalSize_y>([=]() SYCL_ESIMD_KERNEL {
+      *DoNotOptimize.get_pointer() = __spirv_GlobalSize_y();
+    });
+    // CHECK-LABEL: @{{.*}}kernel_GlobalSize_y
+    // CHECK: [[CALL_ESIMD1:%.*]] = call <3 x i32> @llvm.genx.local.size.v3i32()
+    // CHECK: {{.*}} extractelement <3 x i32> [[CALL_ESIMD1]], i32 1
+    // CHECK: [[CALL_ESIMD2:%.*]] = call <3 x i32> @llvm.genx.group.count.v3i32()
+    // CHECK: {{.*}} extractelement <3 x i32> [[CALL_ESIMD2]], i32 1
+
+    kernel<class kernel_GlobalSize_z>([=]() SYCL_ESIMD_KERNEL {
+      *DoNotOptimize.get_pointer() = __spirv_GlobalSize_z();
+    });
+    // CHECK-LABEL: @{{.*}}kernel_GlobalSize_z
+    // CHECK: [[CALL_ESIMD1:%.*]] = call <3 x i32> @llvm.genx.local.size.v3i32()
+    // CHECK: {{.*}} extractelement <3 x i32> [[CALL_ESIMD1]], i32 2
+    // CHECK: [[CALL_ESIMD2:%.*]] = call <3 x i32> @llvm.genx.group.count.v3i32()
+    // CHECK: {{.*}} extractelement <3 x i32> [[CALL_ESIMD2]], i32 2
+
+    kernel<class kernel_GlobalOffset_x>([=]() SYCL_ESIMD_KERNEL {
+      *DoNotOptimize.get_pointer() = __spirv_GlobalOffset_x();
+    });
+    // CHECK-LABEL: @{{.*}}kernel_GlobalOffset_x
+    // CHECK: store i64 0
+
+    kernel<class kernel_GlobalOffset_y>([=]() SYCL_ESIMD_KERNEL {
+      *DoNotOptimize.get_pointer() = __spirv_GlobalOffset_y();
+    });
+    // CHECK-LABEL: @{{.*}}kernel_GlobalOffset_y
+    // CHECK: store i64 0
+
+    kernel<class kernel_GlobalOffset_z>([=]() SYCL_ESIMD_KERNEL {
+      *DoNotOptimize.get_pointer() = __spirv_GlobalOffset_z();
+    });
+    // CHECK-LABEL: @{{.*}}kernel_GlobalOffset_z
+    // CHECK: store i64 0
+
+    kernel<class kernel_NumWorkgroups_x>([=]() SYCL_ESIMD_KERNEL {
+      *DoNotOptimize.get_pointer() = __spirv_NumWorkgroups_x();
+    });
+    // CHECK-LABEL: @{{.*}}kernel_NumWorkgroups_x
+    // CHECK: [[CALL_ESIMD:%.*]] = call <3 x i32> @llvm.genx.group.count.v3i32()
+    // CHECK: {{.*}} extractelement <3 x i32> [[CALL_ESIMD]], i32 0
+
+    kernel<class kernel_NumWorkgroups_y>([=]() SYCL_ESIMD_KERNEL {
+      *DoNotOptimize.get_pointer() = __spirv_NumWorkgroups_y();
+    });
+    // CHECK-LABEL: @{{.*}}kernel_NumWorkgroups_y
+    // CHECK: [[CALL_ESIMD:%.*]] = call <3 x i32> @llvm.genx.group.count.v3i32()
+    // CHECK: {{.*}} extractelement <3 x i32> [[CALL_ESIMD]], i32 1
+
+    kernel<class kernel_NumWorkgroups_z>([=]() SYCL_ESIMD_KERNEL {
+      *DoNotOptimize.get_pointer() = __spirv_NumWorkgroups_z();
+    });
+    // CHECK-LABEL: @{{.*}}kernel_NumWorkgroups_z
+    // CHECK: [[CALL_ESIMD:%.*]] = call <3 x i32> @llvm.genx.group.count.v3i32()
+    // CHECK: {{.*}} extractelement <3 x i32> [[CALL_ESIMD]], i32 2
+
+    kernel<class kernel_WorkgroupSize_x>([=]() SYCL_ESIMD_KERNEL {
+      *DoNotOptimize.get_pointer() = __spirv_WorkgroupSize_x();
+    });
+    // CHECK-LABEL: @{{.*}}kernel_WorkgroupSize_x
+    // CHECK: [[CALL_ESIMD:%.*]] = call <3 x i32> @llvm.genx.local.size.v3i32()
+    // CHECK: {{.*}} extractelement <3 x i32> [[CALL_ESIMD]], i32 0
+
+    kernel<class kernel_WorkgroupSize_y>([=]() SYCL_ESIMD_KERNEL {
+      *DoNotOptimize.get_pointer() = __spirv_WorkgroupSize_y();
+    });
+    // CHECK-LABEL: @{{.*}}kernel_WorkgroupSize_y
+    // CHECK: [[CALL_ESIMD:%.*]] = call <3 x i32> @llvm.genx.local.size.v3i32()
+    // CHECK: {{.*}} extractelement <3 x i32> [[CALL_ESIMD]], i32 1
+
+    kernel<class kernel_WorkgroupSize_z>([=]() SYCL_ESIMD_KERNEL {
+      *DoNotOptimize.get_pointer() = __spirv_WorkgroupSize_z();
+    });
+    // CHECK-LABEL: @{{.*}}kernel_WorkgroupSize_z
+    // CHECK: [[CALL_ESIMD:%.*]] = call <3 x i32> @llvm.genx.local.size.v3i32()
+    // CHECK: {{.*}} extractelement <3 x i32> [[CALL_ESIMD]], i32 2
+
+    kernel<class kernel_WorkgroupId_x>([=]() SYCL_ESIMD_KERNEL {
+      *DoNotOptimize.get_pointer() = __spirv_WorkgroupId_x();
+    });
+    // CHECK-LABEL: @{{.*}}kernel_WorkgroupId_x
+    // CHECK: {{.*}} call i32 @llvm.genx.group.id.x()
+
+    kernel<class kernel_WorkgroupId_y>([=]() SYCL_ESIMD_KERNEL {
+      *DoNotOptimize.get_pointer() = __spirv_WorkgroupId_y();
+    });
+    // CHECK-LABEL: @{{.*}}kernel_WorkgroupId_y
+    // CHECK: {{.*}} call i32 @llvm.genx.group.id.y()
+
+    kernel<class kernel_WorkgroupId_z>([=]() SYCL_ESIMD_KERNEL {
+      *DoNotOptimize.get_pointer() = __spirv_WorkgroupId_z();
+    });
+    // CHECK-LABEL: @{{.*}}kernel_WorkgroupId_z
+    // CHECK: {{.*}} call i32 @llvm.genx.group.id.z()
+
+    kernel<class kernel_LocalInvocationId_x>([=]() SYCL_ESIMD_KERNEL {
+      *DoNotOptimize.get_pointer() = __spirv_LocalInvocationId_x();
+    });
+    // CHECK-LABEL: @{{.*}}kernel_LocalInvocationId_x
+    // CHECK: [[CALL_ESIMD:%.*]] = call <3 x i32> @llvm.genx.local.id.v3i32()
+    // CHECK: {{.*}} extractelement <3 x i32> [[CALL_ESIMD]], i32 0
+
+    kernel<class kernel_LocalInvocationId_y>([=]() SYCL_ESIMD_KERNEL {
+      *DoNotOptimize.get_pointer() = __spirv_LocalInvocationId_y();
+    });
+    // CHECK-LABEL: @{{.*}}kernel_LocalInvocationId_y
+    // CHECK: [[CALL_ESIMD:%.*]] = call <3 x i32> @llvm.genx.local.id.v3i32()
+    // CHECK: {{.*}} extractelement <3 x i32> [[CALL_ESIMD]], i32 1
+
+    kernel<class kernel_LocalInvocationId_z>([=]() SYCL_ESIMD_KERNEL {
+      *DoNotOptimize.get_pointer() = __spirv_LocalInvocationId_z();
+    });
+    // CHECK-LABEL: @{{.*}}kernel_LocalInvocationId_z
+    // CHECK: [[CALL_ESIMD:%.*]] = call <3 x i32> @llvm.genx.local.id.v3i32()
+    // CHECK: {{.*}} extractelement <3 x i32> [[CALL_ESIMD]], i32 2
+  });
+  return DoNotOpt;
+}
diff --git a/sycl/test/fpga_tests/fpga_lsu.cpp b/sycl/test/fpga_tests/fpga_lsu.cpp
new file mode 100644
index 0000000000000..65b35e09dbd69
--- /dev/null
+++ b/sycl/test/fpga_tests/fpga_lsu.cpp
@@ -0,0 +1,80 @@
+// RUN: %clangxx -fsycl %s -o %t.out
+// RUNx: %ACC_RUN_PLACEHOLDER %t.out
+//==----------------- fpga_lsu.cpp - SYCL FPGA LSU test --------------------==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include <CL/sycl.hpp>
+#include <CL/sycl/intel/fpga_extensions.hpp>
+
+// TODO: run is disabled, since no support added in FPGA backend yet. Check
+// implementation correctness from CXX and SYCL languages perspective.
+
+int test_lsu(cl::sycl::queue Queue) {
+  int output_data[2];
+  for (size_t i = 0; i < 2; i++) {
+    output_data[i] = -1;
+  }
+
+  int input_data[2];
+  for (size_t i = 0; i < 2; i++) {
+    input_data[i] = i + 1;
+  }
+
+  {
+    cl::sycl::buffer<int, 1> output_buffer(output_data, 1);
+    cl::sycl::buffer<int, 1> input_buffer(input_data, 1);
+
+    Queue.submit([&](cl::sycl::handler &cgh) {
+      auto output_accessor =
+          output_buffer.get_access<cl::sycl::access::mode::write>(cgh);
+      auto input_accessor =
+          input_buffer.get_access<cl::sycl::access::mode::read>(cgh);
+
+      cgh.single_task<class kernel>([=] {
+        auto input_ptr = input_accessor.get_pointer();
+        auto output_ptr = output_accessor.get_pointer();
+
+        using PrefetchingLSU =
+            cl::sycl::intel::lsu<cl::sycl::intel::prefetch<true>,
+                                 cl::sycl::intel::statically_coalesce<false>>;
+
+        using BurstCoalescedLSU =
+            cl::sycl::intel::lsu<cl::sycl::intel::burst_coalesce<true>,
+                                 cl::sycl::intel::statically_coalesce<false>>;
+
+        using CachingLSU =
+            cl::sycl::intel::lsu<cl::sycl::intel::burst_coalesce<true>,
+                                 cl::sycl::intel::cache<1024>,
+                                 cl::sycl::intel::statically_coalesce<false>>;
+
+        using PipelinedLSU = cl::sycl::intel::lsu<>;
+
+        int X = PrefetchingLSU::load(input_ptr); // int X = input_ptr[0]
+        int Y = CachingLSU::load(input_ptr + 1); // int Y = input_ptr[1]
+
+        BurstCoalescedLSU::store(output_ptr, X); // output_ptr[0] = X
+        PipelinedLSU::store(output_ptr + 1, Y);  // output_ptr[1] = Y
+      });
+    });
+  }
+
+  for (int i = 0; i < 2; i++) {
+    if (output_data[i] != input_data[i]) {
+      std::cout << "Unexpected read from output_data: " << output_data[i]
+                << ", v.s. expected " << input_data[i] << std::endl;
+
+      return -1;
+    }
+  }
+  return 0;
+}
+
+int main() {
+  cl::sycl::queue Queue{cl::sycl::intel::fpga_emulator_selector{}};
+
+  return test_lsu(Queue);
+}
diff --git a/sycl/test/function-pointers/pass-fp-through-buffer.cpp b/sycl/test/function-pointers/pass-fp-through-buffer.cpp
index aa2bc85ec9874..67ecea5509a5f 100644
--- a/sycl/test/function-pointers/pass-fp-through-buffer.cpp
+++ b/sycl/test/function-pointers/pass-fp-through-buffer.cpp
@@ -11,6 +11,7 @@
 
 #include <CL/sycl.hpp>
 
+#include <algorithm>
 #include <iostream>
 #include <vector>
 
diff --git a/sycl/test/xmethods/accessors-device.cpp b/sycl/test/gdb/accessors-device.cpp
similarity index 100%
rename from sycl/test/xmethods/accessors-device.cpp
rename to sycl/test/gdb/accessors-device.cpp
diff --git a/sycl/test/xmethods/accessors.cpp b/sycl/test/gdb/accessors.cpp
similarity index 100%
rename from sycl/test/xmethods/accessors.cpp
rename to sycl/test/gdb/accessors.cpp
diff --git a/sycl/test/gdb/printers.cpp b/sycl/test/gdb/printers.cpp
new file mode 100644
index 0000000000000..1d7898d5686f9
--- /dev/null
+++ b/sycl/test/gdb/printers.cpp
@@ -0,0 +1,19 @@
+// RUN: %clangxx -c -fno-color-diagnostics -I %sycl_include -Xclang -ast-dump %s | FileCheck %s
+// UNSUPPORTED: windows
+#include <CL/sycl/buffer.hpp>
+#include <CL/sycl/detail/array.hpp>
+
+typedef cl::sycl::id<1> dummy_id;
+typedef cl::sycl::buffer<int> dummy_buffer;
+
+// array must have common_array field
+
+// CHECK: CXXRecordDecl {{.*}} class array definition
+// CHECK-NOT: CXXRecordDecl {{.*}} definition
+// CHECK: FieldDecl {{.*}} referenced common_array
+
+// buffer must have impl field
+
+// CHECK: CXXRecordDecl {{.*}} class buffer definition
+// CHECK-NOT: CXXRecordDecl {{.*}} definition
+// CHECK: FieldDecl {{.*}} referenced impl
diff --git a/sycl/test/host-interop-task/host-task-dependency.cpp b/sycl/test/host-interop-task/host-task-dependency.cpp
index 60a1e60883d71..2bbc059a43e44 100644
--- a/sycl/test/host-interop-task/host-task-dependency.cpp
+++ b/sycl/test/host-interop-task/host-task-dependency.cpp
@@ -4,7 +4,7 @@
 // RUN: %ACC_RUN_PLACEHOLDER SYCL_PI_TRACE=-1 %t.out 2>&1 %ACC_CHECK_PLACEHOLDER
 //
 // TODO: Behaviour is unstable for level zero on Windows. Enable when fixed.
-// UNSUPPORTED: windows && level0
+// UNSUPPORTED: windows && level_zero
 
 #include <atomic>
 #include <condition_variable>
diff --git a/sycl/test/host-interop-task/host-task-two-queues.cpp b/sycl/test/host-interop-task/host-task-two-queues.cpp
index 5157b83b60092..7644d6bcfcd53 100644
--- a/sycl/test/host-interop-task/host-task-two-queues.cpp
+++ b/sycl/test/host-interop-task/host-task-two-queues.cpp
@@ -4,7 +4,7 @@
 // RUN: %ACC_RUN_PLACEHOLDER %t.out
 //
 // TODO: Flaky fail on Level Zero that is why mark as unsupported temporarily.
-// UNSUPPORTED: level0
+// UNSUPPORTED: level_zero
 
 #include <CL/sycl.hpp>
 #include <vector>
diff --git a/sycl/test/host-interop-task/interop-task.cpp b/sycl/test/host-interop-task/interop-task.cpp
index 6db65398e06bb..ea4f08eee4a83 100644
--- a/sycl/test/host-interop-task/interop-task.cpp
+++ b/sycl/test/host-interop-task/interop-task.cpp
@@ -2,8 +2,9 @@
 // RUN: %CPU_RUN_PLACEHOLDER %t.out
 // RUN: %GPU_RUN_PLACEHOLDER %t.out
 // RUN: %ACC_RUN_PLACEHOLDER %t.out
-// USUPPORTED: level0, cuda
+// UNSUPPORTED: level_zero, cuda
 // REQUIRES: opencl
+// REQUIRES: TEMPORARY_DISABLED
 
 #include <CL/sycl.hpp>
 #include <CL/sycl/backend/opencl.hpp>
@@ -219,11 +220,41 @@ void test5() {
   }
 }
 
+// The test checks that an exception which is thrown from host_task body
+// is reported as asynchronous.
+void test6() {
+  queue Queue([](sycl::exception_list ExceptionList) {
+    if (ExceptionList.size() != 1) {
+      std::cerr << "Should be one exception in exception list" << std::endl;
+      std::abort();
+    }
+    std::rethrow_exception(*ExceptionList.begin());
+  });
+
+  try {
+    size_t size = 1;
+    buffer<int, 1> Buf{size};
+    Queue.submit([&](sycl::handler &CGH) {
+      auto acc = Buf.get_access<mode::write, target::host_buffer>(CGH);
+      CGH.codeplay_host_task(
+          [=](interop_handle IH) { (void)IH.get_native_mem(acc); });
+    });
+    Queue.wait_and_throw();
+    assert(!"Expected exception was not caught");
+  } catch (sycl::exception &ExpectedException) {
+    assert(std::string(ExpectedException.what())
+                   .find("memory object out of accessor for specified target "
+                         "is not allowed") != std::string::npos &&
+           "Unexpected error was caught!");
+  }
+}
+
 int main() {
   test1();
   test2();
   test3();
   test4();
   test5();
+  test6();
   return 0;
 }
diff --git a/sycl/test/lit.cfg.py b/sycl/test/lit.cfg.py
index 545ebb995f60d..f2a58749a2a93 100644
--- a/sycl/test/lit.cfg.py
+++ b/sycl/test/lit.cfg.py
@@ -81,7 +81,7 @@
 
 def getDeviceCount(device_type):
     is_cuda = False;
-    is_level0 = False;
+    is_level_zero = False;
     process = subprocess.Popen([get_device_count_by_type_path, device_type, backend],
         stdout=subprocess.PIPE)
     (output, err) = process.communicate()
@@ -106,12 +106,12 @@ def getDeviceCount(device_type):
         if re.match(r".*cuda", result[1]):
             is_cuda = True;
         if re.match(r".*level zero", result[1]):
-            is_level0 = True;
+            is_level_zero = True;
 
     if err:
         lit_config.warning("getDeviceCount {TYPE} {BACKEND} stderr:{ERR}".format(
             TYPE=device_type, BACKEND=backend, ERR=err))
-    return [value,is_cuda,is_level0]
+    return [value,is_cuda,is_level_zero]
 
 # Every SYCL implementation provides a host implementation.
 config.available_features.add('host')
@@ -149,8 +149,8 @@ def getDeviceCount(device_type):
 gpu_check_on_linux_substitute = ""
 
 cuda = False
-level0 = False
-[gpu_count, cuda, level0] = getDeviceCount("gpu")
+level_zero = False
+[gpu_count, cuda, level_zero] = getDeviceCount("gpu")
 
 if gpu_count > 0:
     found_at_least_one_device = True
@@ -160,8 +160,8 @@ def getDeviceCount(device_type):
     config.available_features.add('gpu')
     if cuda:
        config.available_features.add('cuda')
-    elif level0:
-       config.available_features.add('level0')
+    elif level_zero:
+       config.available_features.add('level_zero')
 
     if platform.system() == "Linux":
         gpu_run_on_linux_substitute = "env SYCL_DEVICE_TYPE=GPU SYCL_BE={SYCL_BE} ".format(SYCL_BE=backend)
@@ -188,7 +188,7 @@ def getDeviceCount(device_type):
 config.substitutions.append( ('%ACC_CHECK_PLACEHOLDER',  acc_check_substitute) )
 
 # LIT testing either supports OpenCL or CUDA or Level Zero.
-if not cuda and not level0 and found_at_least_one_device:
+if not cuda and not level_zero and found_at_least_one_device:
     config.available_features.add('opencl')
 
 if cuda:
diff --git a/sycl/test/plugins/sycl-ls-gpu-default.cpp b/sycl/test/plugins/sycl-ls-gpu-default.cpp
index a07933f2ccff8..a57fc2a39a027 100755
--- a/sycl/test/plugins/sycl-ls-gpu-default.cpp
+++ b/sycl/test/plugins/sycl-ls-gpu-default.cpp
@@ -1,4 +1,4 @@
-// REQUIRES: gpu, level0
+// REQUIRES: gpu, level_zero
 
 // RUN: sycl-ls --verbose >%t.default.out
 // RUN: FileCheck %s --check-prefixes=CHECK-GPU-BUILTIN,CHECK-GPU-CUSTOM --input-file %t.default.out
diff --git a/sycl/test/reduction/reduction_ctor.cpp b/sycl/test/reduction/reduction_ctor.cpp
index 7f8e8e9726e59..cbf1b5907cfc2 100644
--- a/sycl/test/reduction/reduction_ctor.cpp
+++ b/sycl/test/reduction/reduction_ctor.cpp
@@ -23,13 +23,12 @@ void test_reducer(Reduction &Redu, T A, T B) {
          "Wrong result of binary operation.");
 }
 
-template <typename T, typename Reduction>
-void test_reducer(Reduction &Redu, T Identity, T A, T B) {
-  typename Reduction::reducer_type Reducer(Identity);
+template <typename T, typename Reduction, typename BinaryOperation>
+void test_reducer(Reduction &Redu, T Identity, BinaryOperation BOp, T A, T B) {
+  typename Reduction::reducer_type Reducer(Identity, BOp);
   Reducer.combine(A);
   Reducer.combine(B);
 
-  typename Reduction::binary_operation BOp;
   T ExpectedValue = BOp(A, B);
   assert(ExpectedValue == Reducer.MValue &&
          "Wrong result of binary operation.");
@@ -40,35 +39,8 @@ class Known;
 template <typename T, int Dim, class BinaryOperation>
 class Unknown;
 
-template <typename T>
-struct Point {
-  Point() : X(0), Y(0) {}
-  Point(T X, T Y) : X(X), Y(Y) {}
-  Point(T V) : X(V), Y(V) {}
-  bool operator==(const Point &P) const {
-    return P.X == X && P.Y == Y;
-  }
-  T X;
-  T Y;
-};
-
-template <typename T>
-bool operator==(const Point<T> &A, const Point<T> &B) {
-  return A.X == B.X && A.Y == B.Y;
-}
-
-template <class T>
-struct PointPlus {
-  using P = Point<T>;
-  P operator()(const P &A, const P &B) const {
-    return P(A.X + B.X, A.Y + B.Y);
-  }
-};
-
 template <typename T, int Dim, class BinaryOperation>
-void testKnown(T Identity, T A, T B) {
-
-  BinaryOperation BOp;
+void testKnown(T Identity, BinaryOperation BOp, T A, T B) {
   buffer<T, 1> ReduBuf(1);
 
   queue Q;
@@ -81,17 +53,15 @@ void testKnown(T Identity, T A, T B) {
     assert(Redu.getIdentity() == Identity &&
            "Failed getIdentity() check().");
     test_reducer(Redu, A, B);
-    test_reducer(Redu, Identity, A, B);
+    test_reducer(Redu, Identity, BOp, A, B);
 
     // Command group must have at least one task in it. Use an empty one.
     CGH.single_task<Known<T, Dim, BinaryOperation>>([=]() {});
   });
 }
 
-template <typename T, int Dim, class BinaryOperation>
-void testUnknown(T Identity, T A, T B) {
-
-  BinaryOperation BOp;
+template <typename T, int Dim, typename KernelName, class BinaryOperation>
+void testUnknown(T Identity, BinaryOperation BOp, T A, T B) {
   buffer<T, 1> ReduBuf(1);
   queue Q;
   Q.submit([&](handler &CGH) {
@@ -102,38 +72,46 @@ void testUnknown(T Identity, T A, T B) {
     auto Redu = intel::reduction(ReduAcc, Identity, BOp);
     assert(Redu.getIdentity() == Identity &&
            "Failed getIdentity() check().");
-    test_reducer(Redu, Identity, A, B);
+    test_reducer(Redu, Identity, BOp, A, B);
 
     // Command group must have at least one task in it. Use an empty one.
-    CGH.single_task<Unknown<T, Dim, BinaryOperation>>([=]() {});
+    CGH.single_task<KernelName>([=]() {});
   });
 }
 
 template <typename T, class BinaryOperation>
-void testBoth(T Identity, T A, T B) {
-  testKnown<T, 0, BinaryOperation>(Identity, A, B);
-  testKnown<T, 1, BinaryOperation>(Identity, A, B);
-  testUnknown<T, 0, BinaryOperation>(Identity, A, B);
-  testUnknown<T, 1, BinaryOperation>(Identity, A, B);
+void testBoth(T Identity, BinaryOperation BOp, T A, T B) {
+  testKnown<T, 0>(Identity, BOp, A, B);
+  testKnown<T, 1>(Identity, BOp, A, B);
+  testUnknown<T, 0, Unknown<T, 0, BinaryOperation>>(Identity, BOp, A, B);
+  testUnknown<T, 1, Unknown<T, 1, BinaryOperation>>(Identity, BOp, A, B);
 }
 
 int main() {
-  // testKnown does not pass identity to reduction ctor.
-  testBoth<int, intel::plus<int>>(0, 1, 7);
-  testBoth<int, std::multiplies<int>>(1, 1, 7);
-  testBoth<int, intel::bit_or<int>>(0, 1, 8);
-  testBoth<int, intel::bit_xor<int>>(0, 7, 3);
-  testBoth<int, intel::bit_and<int>>(~0, 7, 3);
-  testBoth<int, intel::minimum<int>>((std::numeric_limits<int>::max)(), 7, 3);
-  testBoth<int, intel::maximum<int>>((std::numeric_limits<int>::min)(), 7, 3);
-
-  testBoth<float, intel::plus<float>>(0, 1, 7);
-  testBoth<float, std::multiplies<float>>(1, 1, 7);
-  testBoth<float, intel::minimum<float>>(getMaximumFPValue<float>(), 7, 3);
-  testBoth<float, intel::maximum<float>>(getMinimumFPValue<float>(), 7, 3);
-
-  testUnknown<Point<float>, 0, PointPlus<float>>(Point<float>(0), Point<float>(1), Point<float>(7));
-  testUnknown<Point<float>, 1, PointPlus<float>>(Point<float>(0), Point<float>(1), Point<float>(7));
+  testBoth<int>(0, intel::plus<int>(), 1, 7);
+  testBoth<int>(1, std::multiplies<int>(), 1, 7);
+  testBoth<int>(0, intel::bit_or<int>(), 1, 8);
+  testBoth<int>(0, intel::bit_xor<int>(), 7, 3);
+  testBoth<int>(~0, intel::bit_and<int>(), 7, 3);
+  testBoth<int>((std::numeric_limits<int>::max)(), intel::minimum<int>(), 7, 3);
+  testBoth<int>((std::numeric_limits<int>::min)(), intel::maximum<int>(), 7, 3);
+
+  testBoth<float>(0, intel::plus<float>(), 1, 7);
+  testBoth<float>(1, std::multiplies<float>(), 1, 7);
+  testBoth<float>(getMaximumFPValue<float>(), intel::minimum<float>(), 7, 3);
+  testBoth<float>(getMinimumFPValue<float>(), intel::maximum<float>(), 7, 3);
+
+  testUnknown<CustomVec<float>, 0,
+              Unknown<CustomVec<float>, 0, CustomVecPlus<float>>>(
+      CustomVec<float>(0), CustomVecPlus<float>(), CustomVec<float>(1),
+      CustomVec<float>(7));
+  testUnknown<CustomVec<float>, 1,
+              Unknown<CustomVec<float>, 1, CustomVecPlus<float>>>(
+      CustomVec<float>(0), CustomVecPlus<float>(), CustomVec<float>(1),
+      CustomVec<float>(7));
+
+  testUnknown<int, 0, class BitOrName>(
+      0, [](auto a, auto b) { return a | b; }, 1, 8);
 
   std::cout << "Test passed\n";
   return 0;
diff --git a/sycl/test/reduction/reduction_nd_lambda.cpp b/sycl/test/reduction/reduction_nd_lambda.cpp
new file mode 100644
index 0000000000000..3d5cf21658995
--- /dev/null
+++ b/sycl/test/reduction/reduction_nd_lambda.cpp
@@ -0,0 +1,72 @@
+// UNSUPPORTED: cuda
+// Reductions use work-group builtins (e.g. intel::reduce()) not yet supported
+// by CUDA.
+//
+// RUN: %clangxx -fsycl -fsycl-targets=%sycl_triple %s -o %t.out
+// RUNx: env SYCL_DEVICE_TYPE=HOST %t.out
+// RUN: %CPU_RUN_PLACEHOLDER %t.out
+// RUN: %GPU_RUN_PLACEHOLDER %t.out
+// RUN: %ACC_RUN_PLACEHOLDER %t.out
+
+// This test performs basic checks of parallel_for(nd_range, reduction, lambda)
+
+#include "reduction_utils.hpp"
+#include <CL/sycl.hpp>
+#include <cassert>
+
+using namespace cl::sycl;
+
+template <class KernelName, typename T, class BinaryOperation>
+void test(T Identity, BinaryOperation BOp, size_t WGSize, size_t NWItems) {
+  buffer<T, 1> InBuf(NWItems);
+  buffer<T, 1> OutBuf(1);
+
+  // Initialize.
+  T CorrectOut;
+  initInputData(InBuf, CorrectOut, Identity, BOp, NWItems);
+
+  // Compute.
+  queue Q;
+  Q.submit([&](handler &CGH) {
+    auto In = InBuf.template get_access<access::mode::read>(CGH);
+    auto Out = OutBuf.template get_access<access::mode::discard_write>(CGH);
+    auto Redu = intel::reduction(Out, Identity, BOp);
+
+    range<1> GlobalRange(NWItems);
+    range<1> LocalRange(WGSize);
+    nd_range<1> NDRange(GlobalRange, LocalRange);
+    CGH.parallel_for<KernelName>(NDRange, Redu,
+                                 [=](nd_item<1> NDIt, auto &Sum) {
+                                   Sum.combine(In[NDIt.get_global_linear_id()]);
+                                 });
+  });
+
+  // Check correctness.
+  auto Out = OutBuf.template get_access<access::mode::read>();
+  T ComputedOut = *(Out.get_pointer());
+  if (ComputedOut != CorrectOut) {
+    std::cout << "NWItems = " << NWItems << ", WGSize = " << WGSize << "\n";
+    std::cout << "Computed value: " << ComputedOut
+              << ", Expected value: " << CorrectOut << "\n";
+    assert(0 && "Wrong value.");
+  }
+}
+
+int main() {
+  test<class AddTestName, int>(
+      0, [](auto x, auto y) { return (x + y); }, 8, 32);
+  test<class MulTestName, int>(
+      0, [](auto x, auto y) { return (x * y); }, 8, 32);
+
+  // Check with CUSTOM type.
+  test<class CustomAddTestname, CustomVec<long long>>(
+      CustomVec<long long>(0),
+      [](auto x, auto y) {
+        CustomVecPlus<long long> BOp;
+        return BOp(x, y);
+      },
+      4, 64);
+
+  std::cout << "Test passed\n";
+  return 0;
+}
diff --git a/sycl/test/reduction/reduction_nd_s1_rw.cpp b/sycl/test/reduction/reduction_nd_s1_rw.cpp
index e59ed8c4785a5..d283fbe3cebe0 100644
--- a/sycl/test/reduction/reduction_nd_s1_rw.cpp
+++ b/sycl/test/reduction/reduction_nd_s1_rw.cpp
@@ -87,7 +87,7 @@ int main() {
 
   // Check with various types.
   test<float, 1, std::multiplies<float>>(1, 8, 256);
-  test<float, 1, intel::minimum<float>>(getMaximumFPValue<float>(), 8, 256);
+  test<float, 1, intel::minimum<float>>(getMaximumFPValue<float>(), 1, 16);
   test<float, 1, intel::maximum<float>>(getMinimumFPValue<float>(), 8, 256);
 
   // Check with CUSTOM type.
diff --git a/sycl/test/reduction/reduction_transparent.cpp b/sycl/test/reduction/reduction_transparent.cpp
index ee44ecd37998c..fd527f8f4e0ef 100644
--- a/sycl/test/reduction/reduction_transparent.cpp
+++ b/sycl/test/reduction/reduction_transparent.cpp
@@ -113,6 +113,7 @@ int main() {
   test<float, 0, intel::maximum<>>(getMinimumFPValue<float>(), 7, 7 * 5);
   test<signed char, 0, intel::plus<>>(0, 7, 49);
   test<unsigned char, 1, std::multiplies<>>(1, 4, 16);
+  test<unsigned short, 0, intel::plus<>>(0, 1, 512 + 32);
 #endif // __cplusplus >= 201402L
 
   std::cout << "Test passed\n";
diff --git a/sycl/test/regression/esimd-util-compiler-eval.cpp b/sycl/test/regression/esimd-util-compiler-eval.cpp
new file mode 100644
index 0000000000000..4d494f1b5e78e
--- /dev/null
+++ b/sycl/test/regression/esimd-util-compiler-eval.cpp
@@ -0,0 +1,15 @@
+// RUN: %clangxx -fsycl -fsycl-explicit-simd -fsycl-device-only -c %s
+// This test checks compile-time evaluation of functions from esimd_util.hpp
+
+#include "CL/sycl.hpp"
+#include "CL/sycl/intel/esimd/esimd.hpp"
+
+static_assert(__esimd::getNextPowerOf2<0>() == 0, "");
+static_assert(__esimd::getNextPowerOf2<1>() == 1, "");
+static_assert(__esimd::getNextPowerOf2<7>() == 8, "");
+static_assert(__esimd::getNextPowerOf2<1024>() == 1024, "");
+
+static_assert(__esimd::log2<0>() == 0, "");
+static_assert(__esimd::log2<1>() == 0, "");
+static_assert(__esimd::log2<7>() == 2, "");
+static_assert(__esimd::log2<1024 * 1024>() == 20, "");
diff --git a/sycl/test/regression/fsycl-save-temps.cpp b/sycl/test/regression/fsycl-save-temps.cpp
index ce9e653af2608..7f89fa54feeca 100644
--- a/sycl/test/regression/fsycl-save-temps.cpp
+++ b/sycl/test/regression/fsycl-save-temps.cpp
@@ -22,6 +22,6 @@ int main() {
 }
 
 // TODO: Address a Windows-specific issue with integration header filenames
-// XFAIL: system-windows && !level0
+// XFAIL: system-windows && !level_zero
 // TODO: fail is flaky on Windows for Level Zero. Enable when fixed.
-// UNSUPPORTED: system-windows && level0
+// UNSUPPORTED: system-windows && level_zero
diff --git a/sycl/test/regression/image_access.cpp b/sycl/test/regression/image_access.cpp
index f32d0a33f6b91..e779fb5b4111b 100644
--- a/sycl/test/regression/image_access.cpp
+++ b/sycl/test/regression/image_access.cpp
@@ -5,9 +5,9 @@
 // TODO: For now PI checks are skipped for ACC device. To decide if it's good.
 // RUN: env %ACC_RUN_PLACEHOLDER %t.out
 //
-// UNSUPPORTED: cuda || windows && level0
+// UNSUPPORTED: cuda || windows && level_zero
 // CUDA cannot support OpenCL spec conform images.
-// TODO: test hangs on level0, enable when fixed.
+// TODO: test hangs on level_zero, enable when fixed.
 
 //==-------------- image_access.cpp - SYCL image accessors test  -----------==//
 //
diff --git a/sycl/test/regression/memcpy-in-vec-as.cpp b/sycl/test/regression/memcpy-in-vec-as.cpp
index d53921234ffc1..b63b8643f3822 100644
--- a/sycl/test/regression/memcpy-in-vec-as.cpp
+++ b/sycl/test/regression/memcpy-in-vec-as.cpp
@@ -13,13 +13,17 @@ int main() {
                                                  cl::sycl::range<1>(1));
     cl::sycl::queue Queue;
     Queue.submit([&](cl::sycl::handler &cgh) {
-      auto In = InBuf.get_access<cl::sycl::access::mode::write>(cgh);
-      auto Out = OutBuf.get_access<cl::sycl::access::mode::read>(cgh);
+      auto In = InBuf.get_access<cl::sycl::access::mode::read>(cgh);
+      auto Out = OutBuf.get_access<cl::sycl::access::mode::write>(cgh);
       cgh.single_task<class as_op>(
           [=]() { Out[0] = In[0].as<res_vec_type>(); });
     });
   }
-  std::cout << res.s0() << " " << res.s1() << " " << res.s2() << " " << res.s3()
-            << std::endl;
+
+  if (res.s0() != 513 || res.s1() != 1027 || res.s2() != 1541 || res.s3() != 2055) {
+    std::cerr << "Incorrect result" << std::endl;
+    return 1;
+  }
+
   return 0;
 }
diff --git a/sycl/test/regression/static-buffer-dtor.cpp b/sycl/test/regression/static-buffer-dtor.cpp
index 5899420c20454..c541c180e7d73 100644
--- a/sycl/test/regression/static-buffer-dtor.cpp
+++ b/sycl/test/regression/static-buffer-dtor.cpp
@@ -14,7 +14,7 @@
 // RUN: %GPU_RUN_PLACEHOLDER %t.out
 // RUN: %ACC_RUN_PLACEHOLDER %t.out
 //
-// XFAIL: linux && level0
+// XFAIL: linux && level_zero
 
 #include <CL/sycl.hpp>
 
diff --git a/sycl/test/spec_const/spec_const_hw.cpp b/sycl/test/spec_const/spec_const_hw.cpp
index 6e051910ce680..442121353bb73 100644
--- a/sycl/test/spec_const/spec_const_hw.cpp
+++ b/sycl/test/spec_const/spec_const_hw.cpp
@@ -1,12 +1,9 @@
-// RUN: %clangxx -fsycl %s -o %t.out
+// UNSUPPORTED: cuda || level_zero
+//
+// RUN: %clangxx -fsycl -fsycl-targets=%sycl_triple %s -o %t.out
 // RUN: env SYCL_DEVICE_TYPE=HOST %t.out
 // RUN: %CPU_RUN_PLACEHOLDER %t.out
 // RUN: %GPU_RUN_PLACEHOLDER %t.out
-// RUN: %ACC_RUN_PLACEHOLDER %t.out
-// TODO: re-enable after CI drivers are updated to newer which support spec
-// constants:
-// XFAIL: linux && opencl
-// UNSUPPORTED: cuda || level0
 //
 //==----------- spec_const_hw.cpp ------------------------------------------==//
 //
diff --git a/sycl/test/spec_const/spec_const_hw_accelerator.cpp b/sycl/test/spec_const/spec_const_hw_accelerator.cpp
new file mode 100644
index 0000000000000..bd7df40ed378a
--- /dev/null
+++ b/sycl/test/spec_const/spec_const_hw_accelerator.cpp
@@ -0,0 +1,17 @@
+//==----------- spec_const_hw_accelerator.cpp ------------------------------==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// The test checks that the specialization constant feature works correctly -
+// tool chain processes them correctly and runtime can correctly execute the
+// program.
+
+// TODO: re-enable after CI drivers are updated to newer which support spec
+// constants:
+// XFAIL: linux && opencl && accelerator
+// UNSUPPORTED: cuda || level_zero
+#include "spec_const_hw.cpp"
+// RUN: %ACC_RUN_PLACEHOLDER %t.out
diff --git a/sycl/test/spec_const/spec_const_neg.cpp b/sycl/test/spec_const/spec_const_neg.cpp
index 18fb8ed5d9d0c..7312e29ab40e1 100644
--- a/sycl/test/spec_const/spec_const_neg.cpp
+++ b/sycl/test/spec_const/spec_const_neg.cpp
@@ -3,7 +3,7 @@
 // RUN: %CPU_RUN_PLACEHOLDER %t.out
 // RUN: %GPU_RUN_PLACEHOLDER %t.out
 // Specialization constants are not supported on FPGA h/w and emulator.
-// UNSUPPORTED: cuda || level0
+// UNSUPPORTED: cuda || level_zero
 //
 //==----------- spec_const_hw.cpp ------------------------------------------==//
 //
diff --git a/sycl/test/spec_const/spec_const_redefine.cpp b/sycl/test/spec_const/spec_const_redefine.cpp
index ac65587905bf0..6883ce5c9d7d6 100644
--- a/sycl/test/spec_const/spec_const_redefine.cpp
+++ b/sycl/test/spec_const/spec_const_redefine.cpp
@@ -1,12 +1,9 @@
-// RUN: %clangxx -fsycl %s -o %t.out
+// UNSUPPORTED: cuda || level_zero
+//
+// RUN: %clangxx -fsycl -fsycl-targets=%sycl_triple %s -o %t.out
 // RUN: env SYCL_DEVICE_TYPE=HOST %t.out
 // RUN: env SYCL_PI_TRACE=2 %CPU_RUN_PLACEHOLDER %t.out 2>&1 %CPU_CHECK_PLACEHOLDER
 // RUN: env SYCL_PI_TRACE=2 %GPU_RUN_PLACEHOLDER %t.out 2>&1 %GPU_CHECK_PLACEHOLDER
-// RUN: env SYCL_PI_TRACE=2 %ACC_RUN_PLACEHOLDER %t.out 2>&1 %ACC_CHECK_PLACEHOLDER
-// TODO: re-enable after CI drivers are updated to newer which support spec
-// constants:
-// XFAIL: linux && opencl
-// UNSUPPORTED: cuda || level0
 //
 //==----------- spec_const_redefine.cpp ------------------------------------==//
 //
diff --git a/sycl/test/spec_const/spec_const_redefine_accelerator.cpp b/sycl/test/spec_const/spec_const_redefine_accelerator.cpp
new file mode 100644
index 0000000000000..52293f7a953e7
--- /dev/null
+++ b/sycl/test/spec_const/spec_const_redefine_accelerator.cpp
@@ -0,0 +1,19 @@
+//==----------- spec_const_redefine.cpp ------------------------------------==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// The test checks that:
+// - a specialization constant can be redifined and correct new value is used
+//   after redefinition.
+// - the program is JITted only once per a unique set of specialization
+//   constants values.
+
+// TODO: re-enable after CI drivers are updated to newer which support spec
+// constants:
+// XFAIL: linux && opencl && accelerator
+// UNSUPPORTED: cuda || level_zero
+#include "spec_const_redefine_accelerator.cpp"
+// RUN: env SYCL_PI_TRACE=2 %ACC_RUN_PLACEHOLDER %t.out 2>&1 %ACC_CHECK_PLACEHOLDER
diff --git a/sycl/test/sub_group/generic_reduce.cpp b/sycl/test/sub_group/generic_reduce.cpp
new file mode 100644
index 0000000000000..cfeea7f459b69
--- /dev/null
+++ b/sycl/test/sub_group/generic_reduce.cpp
@@ -0,0 +1,95 @@
+// UNSUPPORTED: cuda
+// CUDA compilation and runtime do not yet support sub-groups.
+//
+// RUN: %clangxx -fsycl -fsycl-unnamed-lambda -std=c++14 %s -o %t.out
+// RUN: %clangxx -fsycl -fsycl-unnamed-lambda -fsycl-targets=%sycl_triple -std=c++14 -D SG_GPU %s -o %t_gpu.out
+// RUN: env SYCL_DEVICE_TYPE=HOST %t.out
+// RUN: %CPU_RUN_PLACEHOLDER %t.out
+// RUN: %GPU_RUN_PLACEHOLDER %t_gpu.out
+// RUN: %ACC_RUN_PLACEHOLDER %t.out
+
+#include "helper.hpp"
+#include <CL/sycl.hpp>
+#include <complex>
+
+using namespace cl::sycl;
+
+template <typename T, class BinaryOperation>
+void check_op(queue &Queue, T init, BinaryOperation op, bool skip_init = false,
+              size_t G = 240, size_t L = 60) {
+  try {
+    nd_range<1> NdRange(G, L);
+    buffer<T> buf(G);
+    buffer<size_t> sgsizebuf(1);
+    Queue.submit([&](handler &cgh) {
+      auto sgsizeacc = sgsizebuf.get_access<access::mode::read_write>(cgh);
+      auto acc = buf.template get_access<access::mode::read_write>(cgh);
+      cgh.parallel_for(
+          NdRange, [=](nd_item<1> NdItem) {
+            intel::sub_group sg = NdItem.get_sub_group();
+            if (skip_init) {
+              acc[NdItem.get_global_id(0)] =
+                  reduce(sg, T(NdItem.get_global_id(0)), op);
+            } else {
+              acc[NdItem.get_global_id(0)] =
+                  reduce(sg, T(NdItem.get_global_id(0)), init, op);
+            }
+            if (NdItem.get_global_id(0) == 0)
+              sgsizeacc[0] = sg.get_max_local_range()[0];
+          });
+    });
+    auto acc = buf.template get_access<access::mode::read_write>();
+    auto sgsizeacc = sgsizebuf.get_access<access::mode::read_write>();
+    size_t sg_size = sgsizeacc[0];
+    int WGid = -1, SGid = 0;
+    T result = init;
+    for (int j = 0; j < G; j++) {
+      if (j % L % sg_size == 0) {
+        SGid++;
+        result = init;
+        for (int i = j; (i % L && i % L % sg_size) || (i == j); i++) {
+          result = op(result, T(i));
+        }
+      }
+      if (j % L == 0) {
+        WGid++;
+        SGid = 0;
+      }
+      std::string name =
+          std::string("reduce_") + typeid(BinaryOperation).name();
+      exit_if_not_equal(acc[j], result, name.c_str());
+    }
+  } catch (exception e) {
+    std::cout << "SYCL exception caught: " << e.what();
+    exit(1);
+  }
+}
+
+int main() {
+  queue Queue;
+  if (!Queue.get_device().has_extension("cl_intel_subgroups")) {
+    std::cout << "Skipping test\n";
+    return 0;
+  }
+
+  size_t G = 240;
+  size_t L = 60;
+
+  // Test user-defined type
+  // Use complex as a proxy for this
+  using UDT = std::complex<float>;
+  check_op<UDT>(Queue, UDT(L, L), intel::plus<UDT>(), false, G, L);
+  check_op<UDT>(Queue, UDT(0, 0), intel::plus<UDT>(), true, G, L);
+
+  // Test user-defined operator
+  auto UDOp = [=](const auto &lhs, const auto &rhs) { return lhs + rhs; };
+  check_op<int>(Queue, int(L), UDOp, false, G, L);
+  check_op<int>(Queue, int(0), UDOp, true, G, L);
+
+  // Test both user-defined type and operator
+  check_op<UDT>(Queue, UDT(L, L), UDOp, false, G, L);
+  check_op<UDT>(Queue, UDT(0, 0), UDOp, true, G, L);
+
+  std::cout << "Test passed." << std::endl;
+  return 0;
+}
diff --git a/sycl/test/usm/allocator_equal.cpp b/sycl/test/usm/allocator_equal.cpp
new file mode 100644
index 0000000000000..e02040fe16c28
--- /dev/null
+++ b/sycl/test/usm/allocator_equal.cpp
@@ -0,0 +1,55 @@
+// piextUSM*Alloc functions for CUDA are not behaving as described in
+// https://github.com/intel/llvm/blob/sycl/sycl/doc/extensions/USM/USM.adoc
+// https://github.com/intel/llvm/blob/sycl/sycl/doc/extensions/USM/cl_intel_unified_shared_memory.asciidoc
+//
+// RUN: %clangxx -fsycl -fsycl-targets=%sycl_triple %s -o %t1.out
+// RUN: env SYCL_DEVICE_TYPE=HOST %t1.out
+// RUN: %CPU_RUN_PLACEHOLDER %t1.out
+// RUN: %GPU_RUN_PLACEHOLDER %t1.out
+
+//==---------- allocator_equal.cpp - Allocator Equality test ---------------==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <CL/sycl.hpp>
+
+#include <cassert>
+
+using namespace cl::sycl;
+
+int main() {
+  queue q;
+  auto dev = q.get_device();
+  auto ctxt = q.get_context();
+
+  queue q2;
+  auto dev2 = q2.get_device();
+  auto ctxt2 = q2.get_context();
+
+  // Test allocator equality
+  if (dev.get_info<info::device::usm_host_allocations>()) {
+    usm_allocator<int, usm::alloc::host> alloc1(ctxt, dev);
+    usm_allocator<int, usm::alloc::host> alloc2(q);
+
+    assert((alloc1 == alloc2) && "Allocators should be equal.");
+
+    usm_allocator<int, usm::alloc::host, 8> alloc3(ctxt, dev);
+    usm_allocator<int, usm::alloc::host, 16> alloc4(q);
+
+    assert((alloc1 == alloc2) && "Allocators should be equal.");
+  }
+
+  if (dev.get_info<info::device::usm_shared_allocations>() &&
+      dev.get_info<info::device::usm_host_allocations>()) {
+    usm_allocator<int, usm::alloc::shared> alloc1(ctxt, dev);
+    usm_allocator<int, usm::alloc::host> alloc2(ctxt, dev);
+
+    assert((alloc1 != alloc2) && "Allocators should NOT be equal.");
+  }
+
+  return 0;
+}
diff --git a/sycl/test/usm/allocator_shared.cpp b/sycl/test/usm/allocator_shared.cpp
new file mode 100644
index 0000000000000..2dbc0fe57bd6f
--- /dev/null
+++ b/sycl/test/usm/allocator_shared.cpp
@@ -0,0 +1,49 @@
+// RUN: %clangxx -fsycl -fsycl-targets=%sycl_triple %s -o %t1.out
+// RUN: env SYCL_DEVICE_TYPE=HOST %t1.out
+// RUN: %CPU_RUN_PLACEHOLDER %t1.out
+// RUN: %GPU_RUN_PLACEHOLDER %t1.out
+
+//==-------- allocator_shared.cpp - Allocate Shared test -------------------==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <CL/sycl.hpp>
+
+#include <cassert>
+#include <memory>
+
+using namespace cl::sycl;
+
+int main() {
+  queue q;
+  auto dev = q.get_device();
+  auto ctxt = q.get_context();
+
+  // Test ability to create a shared pointer.
+  if (dev.get_info<info::device::usm_host_allocations>()) {
+    usm_allocator<int, usm::alloc::host> alloc(ctxt, dev);
+    auto ptr1 = std::allocate_shared<int>(alloc);
+
+    // Test construction
+    auto ptr2 = std::allocate_shared<int>(alloc, 42);
+    assert((*ptr2 == 42) && "Host construct passed.");
+  }
+
+  if (dev.get_info<info::device::usm_shared_allocations>()) {
+    usm_allocator<int, usm::alloc::shared> alloc(ctxt, dev);
+    auto ptr1 = std::allocate_shared<int>(alloc);
+
+    // Test construction
+    auto ptr2 = std::allocate_shared<int>(alloc, 42);
+    assert((*ptr2 == 42) && "Shared construct passed.");
+  }
+
+  // Device allocations are not supported due to how allocated_shared is
+  // written.
+
+  return 0;
+}
diff --git a/sycl/test/usm/allocator_vector.cpp b/sycl/test/usm/allocator_vector.cpp
index 265c071e1cf0e..8f0dcaa62055a 100644
--- a/sycl/test/usm/allocator_vector.cpp
+++ b/sycl/test/usm/allocator_vector.cpp
@@ -1,8 +1,3 @@
-// XFAIL: cuda || level0
-// piextUSM*Alloc functions for CUDA are not behaving as described in
-// https://github.com/intel/llvm/blob/sycl/sycl/doc/extensions/USM/USM.adoc
-// https://github.com/intel/llvm/blob/sycl/sycl/doc/extensions/USM/cl_intel_unified_shared_memory.asciidoc
-//
 // RUN: %clangxx -fsycl -fsycl-targets=%sycl_triple %s -o %t1.out
 // RUN: env SYCL_DEVICE_TYPE=HOST %t1.out
 // RUN: %CPU_RUN_PLACEHOLDER %t1.out
@@ -88,43 +83,5 @@ int main() {
       return -1;
   }
 
-  if (dev.get_info<info::device::usm_device_allocations>()) {
-    usm_allocator<int, usm::alloc::device> alloc(ctxt, dev);
-
-    std::vector<int, decltype(alloc)> vec(alloc);
-    vec.resize(N);
-
-    int *res = &vec[0];
-    int *vals = &vec[0];
-
-    auto e0 = q.submit([=](handler &h) {
-      h.single_task<class baz_init>([=]() {
-        res[0] = 0;
-        for (int i = 0; i < N; i++) {
-          vals[i] = i;
-        }
-      });
-    });
-
-    auto e1 = q.submit([=](handler &h) {
-      h.depends_on(e0);
-      h.single_task<class baz>([=]() {
-        for (int i = 1; i < N; i++) {
-          res[0] += vals[i];
-        }
-      });
-    });
-
-    e1.wait();
-
-    int answer = (N * (N - 1)) / 2;
-    int result;
-    q.memcpy(&result, res, sizeof(int));
-    q.wait();
-
-    if (result != answer)
-      return -1;
-  }
-
   return 0;
 }
diff --git a/sycl/test/usm/allocator_vector_fail.cpp b/sycl/test/usm/allocator_vector_fail.cpp
deleted file mode 100644
index 8c0e176eaa411..0000000000000
--- a/sycl/test/usm/allocator_vector_fail.cpp
+++ /dev/null
@@ -1,48 +0,0 @@
-// XFAIL: cuda
-// piextUSM*Alloc functions for CUDA are not behaving as described in
-// https://github.com/intel/llvm/blob/sycl/sycl/doc/extensions/USM/USM.adoc
-// https://github.com/intel/llvm/blob/sycl/sycl/doc/extensions/USM/cl_intel_unified_shared_memory.asciidoc
-//
-// RUN: %clangxx -fsycl -fsycl-targets=%sycl_triple %s -o %t1.out
-// RUN: env SYCL_DEVICE_TYPE=HOST %t1.out
-// RUN: %CPU_RUN_PLACEHOLDER %t1.out
-// RUN: %GPU_RUN_PLACEHOLDER %t1.out
-
-//==-- allocator_vector_fail.cpp - Device Memory Allocator fail test -------==//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include <CL/sycl.hpp>
-
-#include <vector>
-
-using namespace cl::sycl;
-
-const int N = 8;
-
-class foo;
-int main() {
-  queue q;
-  auto dev = q.get_device();
-  auto ctxt = q.get_context();
-
-  if (dev.get_info<info::device::usm_device_allocations>()) {
-    try {
-      usm_allocator<int, usm::alloc::device> alloc(ctxt, dev);
-      std::vector<int, decltype(alloc)> vec(alloc);
-
-      // This statement should throw an exception since
-      // device pointers may not be accessed on the host.
-      vec.assign(N, 42);
-    } catch (feature_not_supported) {
-      return 0;
-    }
-
-    return -1;
-  }
-  return 0;
-}
diff --git a/sycl/test/usm/allocatorll.cpp b/sycl/test/usm/allocatorll.cpp
deleted file mode 100644
index 323dc0d75a1c1..0000000000000
--- a/sycl/test/usm/allocatorll.cpp
+++ /dev/null
@@ -1,88 +0,0 @@
-// XFAIL: cuda
-// piextUSM*Alloc functions for CUDA are not behaving as described in
-// https://github.com/intel/llvm/blob/sycl/sycl/doc/extensions/USM/USM.adoc
-// https://github.com/intel/llvm/blob/sycl/sycl/doc/extensions/USM/cl_intel_unified_shared_memory.asciidoc
-//
-// RUN: %clangxx -fsycl -fsycl-targets=%sycl_triple %s -o %t1.out
-// RUN: env SYCL_DEVICE_TYPE=HOST %t1.out
-// RUN: %CPU_RUN_PLACEHOLDER %t1.out
-// RUN: %GPU_RUN_PLACEHOLDER %t1.out
-
-//==---- allocatorll.cpp - Device Memory Linked List Allocator test --------==//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include <CL/sycl.hpp>
-
-using namespace cl::sycl;
-
-int numNodes = 4;
-
-struct Node {
-  Node() : pNext(nullptr), Num(0xDEADBEEF) {}
-
-  Node *pNext;
-  uint32_t Num;
-};
-
-class foo;
-int main() {
-  queue q;
-  auto dev = q.get_device();
-  auto ctxt = q.get_context();
-
-  if (!dev.get_info<info::device::usm_device_allocations>())
-    return 0;
-
-  usm_allocator<Node, usm::alloc::device> alloc(ctxt, dev);
-  Node h_cur;
-
-  Node *d_head = alloc.allocate(1);
-  Node *d_cur = d_head;
-
-  for (int i = 0; i < numNodes; i++) {
-    h_cur.Num = i * 2;
-
-    if (i != (numNodes - 1)) {
-      h_cur.pNext = alloc.allocate(1);
-    } else {
-      h_cur.pNext = nullptr;
-    }
-
-    event e0 = q.memcpy(d_cur, &h_cur, sizeof(Node));
-    e0.wait();
-
-    d_cur = h_cur.pNext;
-  }
-
-  auto e1 = q.submit([=](handler &cgh) {
-    cgh.single_task<class foo>([=]() {
-      Node *pHead = d_head;
-      while (pHead) {
-        pHead->Num = pHead->Num * 2 + 1;
-        pHead = pHead->pNext;
-      }
-    });
-  });
-
-  e1.wait();
-
-  d_cur = d_head;
-  for (int i = 0; i < numNodes; i++) {
-    event c = q.memcpy(&h_cur, d_cur, sizeof(Node));
-    c.wait();
-    alloc.deallocate(d_cur, 1);
-
-    const int want = i * 4 + 1;
-    if (h_cur.Num != want) {
-      return -2;
-    }
-    d_cur = h_cur.pNext;
-  }
-
-  return 0;
-}
diff --git a/sycl/tools/CMakeLists.txt b/sycl/tools/CMakeLists.txt
index 1e2843e0b9385..67b0c7329a10b 100644
--- a/sycl/tools/CMakeLists.txt
+++ b/sycl/tools/CMakeLists.txt
@@ -6,22 +6,22 @@ add_subdirectory(sycl-ls)
 
 # TODO: move each tool in its own sub-directory
 add_executable(get_device_count_by_type get_device_count_by_type.cpp)
-add_dependencies(get_device_count_by_type ocl-headers ocl-icd l0-loader)
+add_dependencies(get_device_count_by_type ocl-headers ocl-icd level-zero-loader)
 
 if(MSVC)
-  set(L0_LIBRARY
+  set(LEVEL_ZERO_LIBRARY
     "${LLVM_LIBRARY_OUTPUT_INTDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}ze_loader${CMAKE_STATIC_LIBRARY_SUFFIX}")
 else()
-  set(L0_LIBRARY
+  set(LEVEL_ZERO_LIBRARY
     "${LLVM_LIBRARY_OUTPUT_INTDIR}/${CMAKE_SHARED_LIBRARY_PREFIX}ze_loader${CMAKE_SHARED_LIBRARY_SUFFIX}")
 endif()
 
 target_link_libraries(get_device_count_by_type
   PRIVATE
     OpenCL::Headers
-    L0Loader::Headers
+    LevelZeroLoader::Headers
     ${OpenCL_LIBRARIES}
-    ${L0_LIBRARY}
+    ${LEVEL_ZERO_LIBRARY}
     $<$<BOOL:${SYCL_BUILD_PI_CUDA}>:cudadrv>
 )
 target_compile_definitions(get_device_count_by_type
diff --git a/sycl/tools/abi_check.py b/sycl/tools/abi_check.py
index 2493a158fbb98..f044ef195d212 100644
--- a/sycl/tools/abi_check.py
+++ b/sycl/tools/abi_check.py
@@ -91,11 +91,16 @@ def check_symbols(ref_path, target_path):
     correct_return = True
     if missing_symbols:
       correct_return = False
+      print(("There are missing symbols in the new library. It is a breaking "
+      "change. Refer to sycl/doc/ABIPolicyGuide.md for further instructions. "
+      "Do not forget to update ABI version according to the policy."))
       print('The following symbols are missing from the new object file:\n')
       print("\n".join(missing_symbols))
 
     if new_symbols:
       correct_return = False
+      print(("There are new symbols in the new library. It is a non-breaking "
+      "change. Refer to sycl/doc/ABIPolicyGuide.md for further instructions."))
       print('The following symbols are new to the object file:\n')
       print("\n".join(new_symbols))
 
diff --git a/sycl/tools/get_device_count_by_type.cpp b/sycl/tools/get_device_count_by_type.cpp
index 26a9a0a4ed6ad..abf70ce98c357 100644
--- a/sycl/tools/get_device_count_by_type.cpp
+++ b/sycl/tools/get_device_count_by_type.cpp
@@ -32,7 +32,7 @@ static const std::string help =
     "   Help\n"
     "   Example: ./get_device_count_by_type cpu opencl\n"
     "   Supported device types: cpu/gpu/accelerator/default/all\n"
-    "   Supported backends: PI_CUDA/PI_OPENCL/PI_LEVEL0 \n"
+    "   Supported backends: PI_CUDA/PI_OPENCL/PI_LEVEL_ZERO \n"
     "   Output format: <number_of_devices>:<additional_Information>";
 
 // Return the string with all characters translated to lower case.
@@ -259,7 +259,7 @@ int main(int argc, char *argv[]) {
 
   if (backend == "opencl" || backend == "pi_opencl") {
     querySuccess = queryOpenCL(deviceType, deviceCount, msg);
-  } else if (backend == "level0" || backend == "pi_level0") {
+  } else if (backend == "level_zero" || backend == "pi_level_zero") {
     querySuccess = queryLevelZero(deviceType, deviceCount, msg);
   } else if (backend == "cuda" || backend == "pi_cuda") {
     querySuccess = queryCUDA(deviceType, deviceCount, msg);
diff --git a/sycl/tools/install.bat b/sycl/tools/install.bat
index f770d47d3583a..6b1af03030117 100755
--- a/sycl/tools/install.bat
+++ b/sycl/tools/install.bat
@@ -1,5 +1,5 @@
 @echo off
-
+setlocal EnableDelayedExpansion
 set OCL_RT_DIR=%~dp0
 
 echo ###
@@ -7,6 +7,32 @@ echo ### 1. Save and update OpenCL.dll available in the system
 echo ###
 set TMP_FILE=%TEMP%\install.bat.tmp
 
+set OCL_RT_ENTRY_LIB=%OCL_RT_DIR%intelocl64.dll
+IF NOT EXIST %OCL_RT_ENTRY_LIB% (
+   set OCL_RT_ENTRY_LIB=%OCL_RT_DIR%intelocl64_emu.dll
+)
+
+IF "%OCL_ICD_FILENAMES%" == "" (
+  set EXTENDEXISTING=N
+) else (
+  echo OCL_ICD_FILENAMES is present and contains %OCL_ICD_FILENAMES%
+  :USERINPUT
+  set /P "EXTENDEXISTING=Should the OpenCL RT extend existing configuration (Y/N): "
+)
+IF "%EXTENDEXISTING%" == "N" (
+  echo Clean up previous configuration
+  set OCL_ICD_FILENAMES=%OCL_RT_ENTRY_LIB%
+) else (
+  IF "%EXTENDEXISTING%" == "Y" (
+
+    set OCL_ICD_FILENAMES=%OCL_ICD_FILENAMES%;%OCL_RT_ENTRY_LIB%
+    echo Extend previous configuration to %OCL_ICD_FILENAMES%;%OCL_RT_ENTRY_LIB%
+  ) else (
+    echo WARNING: Incorrect input %EXTENDEXISTING%. Only Y and N are allowed.
+    goto USERINPUT
+  )
+)
+
 
 set SYSTEM_OCL_ICD_LOADER=C:\Windows\System32\OpenCL.dll
 set NEW_OCL_ICD_LOADER=%OCL_RT_DIR%\OpenCL.dll
@@ -73,11 +99,13 @@ IF %NEED_OPENCL_UPGRADE% == True (
   echo System OpenCL.dll is already new, no need to upgrade it.
 )
 
+
+
 echo.
 echo ###
 echo ### 3. Set the environment variable OCL_ICD_FILENAMES to %OCL_ICD_FILENAMES%
 echo ###
-REG ADD "HKLM\SYSTEM\CurrentControlSet\Control\Session Manager\Environment" /v OCL_ICD_FILENAMES /d "%OCL_ICD_FILENAMES%"
+REG ADD "HKLM\SYSTEM\CurrentControlSet\Control\Session Manager\Environment" /f /v OCL_ICD_FILENAMES /d "%OCL_ICD_FILENAMES%"
 IF ERRORLEVEL 1 (
   echo !!! Cannot set the environment variable OCL_ICD_FILENAMES
   set INSTALL_ERRORS=1
@@ -137,7 +165,7 @@ IF %INSTALL_ERRORS% == 1 (
   echo See recommendations printed above and perform the following actions manually:
   echo   1. Save %SYSTEM_OCL_ICD_LOADER% to %SYSTEM_OCL_ICD_LOADER%.%SYSTEM_OPENCL_VER%
   echo   2. Copy %NEW_OCL_ICD_LOADER% to %SYSTEM_OCL_ICD_LOADER%
-  echo   3. Add/set the environment variable OCL_ICD_FILENAMES to %OCL_RT_DIR%intelocl64.dll
+  echo   3. Add/set the environment variable OCL_ICD_FILENAMES to %OCL_RT_ENTRY_LIB%
   echo   4. Copy TBB libraries or create symbolic links in %OCL_RT_DIR%tbb.
   echo   5. Add/set the environment variable PATH to %OCL_RT_DIR%tbb
   echo Or try running this batch file as Administrator.
@@ -147,5 +175,5 @@ IF %INSTALL_ERRORS% == 1 (
 echo.
 
 endlocal& ^
-set OCL_ICD_FILENAMES=%OCL_RT_DIR%intelocl64.dll
+set OCL_ICD_FILENAMES=%OCL_ICD_FILENAMES%
 set "PATH=%PATH%;%OCL_RT_DIR%\tbb"
diff --git a/sycl/unittests/misc/CMakeLists.txt b/sycl/unittests/misc/CMakeLists.txt
index 6587f9edea163..169a1572b2d09 100644
--- a/sycl/unittests/misc/CMakeLists.txt
+++ b/sycl/unittests/misc/CMakeLists.txt
@@ -2,4 +2,5 @@ set(sycl_lib_dir $<TARGET_FILE_DIR:sycl>)
 add_definitions(-DSYCL_LIB_DIR="${sycl_lib_dir}")
 add_sycl_unittest(MiscTests SHARED
   OsUtils.cpp
+  CircularBuffer.cpp
 )
diff --git a/sycl/test/basic_tests/circular_buffer.cpp b/sycl/unittests/misc/CircularBuffer.cpp
similarity index 51%
rename from sycl/test/basic_tests/circular_buffer.cpp
rename to sycl/unittests/misc/CircularBuffer.cpp
index d3ab0c3a77530..3ffc666fa0c8f 100644
--- a/sycl/test/basic_tests/circular_buffer.cpp
+++ b/sycl/unittests/misc/CircularBuffer.cpp
@@ -1,7 +1,14 @@
-// RUN: %clangxx -fsycl %s -o %t.out
-// RUN: %t.out
+//==---- CircularBuffer.cpp ------------------------------------------------==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
 
-#include <CL/sycl/detail/circular_buffer.hpp>
+#include <gtest/gtest.h>
+
+#include <detail/circular_buffer.hpp>
 
 #include <algorithm>
 #include <cassert>
@@ -11,21 +18,21 @@
 // This test contains basic checks for cl::sycl::detail::CircularBuffer
 void checkEquality(const cl::sycl::detail::CircularBuffer<int> &CB,
                    const std::vector<int> &V) {
-  assert(std::equal(CB.begin(), CB.end(), V.begin()));
+  ASSERT_TRUE(std::equal(CB.begin(), CB.end(), V.begin()));
 }
 
-int main() {
+TEST(CircularBufferTest, CircularBufferTest) {
   const std::size_t Capacity = 6;
   cl::sycl::detail::CircularBuffer<int> CB{Capacity};
-  assert(CB.capacity() == Capacity);
-  assert(CB.empty());
+  ASSERT_TRUE(CB.capacity() == Capacity);
+  ASSERT_TRUE(CB.empty());
 
-  int nextValue = 0;
+  size_t nextValue = 0;
   for (; nextValue < Capacity; ++nextValue) {
-    assert(CB.size() == nextValue);
+    ASSERT_TRUE(CB.size() == nextValue);
     CB.push_back(nextValue);
   }
-  assert(CB.full() && CB.size() == CB.capacity());
+  ASSERT_TRUE(CB.full() && CB.size() == CB.capacity());
   checkEquality(CB, {0, 1, 2, 3, 4, 5});
 
   CB.push_back(nextValue++);
@@ -33,8 +40,8 @@ int main() {
   CB.push_front(nextValue++);
   checkEquality(CB, {7, 1, 2, 3, 4, 5});
 
-  assert(CB.front() == 7);
-  assert(CB.back() == 5);
+  ASSERT_TRUE(CB.front() == 7);
+  ASSERT_TRUE(CB.back() == 5);
 
   CB.erase(CB.begin() + 2);
   checkEquality(CB, {7, 1, 3, 4, 5});
diff --git a/sycl/unittests/pi/BackendString.hpp b/sycl/unittests/pi/BackendString.hpp
index cea0eee8b8338..7f051f5ab6790 100644
--- a/sycl/unittests/pi/BackendString.hpp
+++ b/sycl/unittests/pi/BackendString.hpp
@@ -15,7 +15,7 @@ inline const char *GetBackendString(cl::sycl::backend backend) {
     PI_BACKEND_STR(cuda);
     PI_BACKEND_STR(host);
     PI_BACKEND_STR(opencl);
-    PI_BACKEND_STR(level0);
+    PI_BACKEND_STR(level_zero);
 #undef PI_BACKEND_STR
   default:
     return "Unknown Plugin";
diff --git a/sycl/unittests/program/KernelRelease.cpp b/sycl/unittests/program/KernelRelease.cpp
index 5b94dedbb3970..766333aa908e0 100644
--- a/sycl/unittests/program/KernelRelease.cpp
+++ b/sycl/unittests/program/KernelRelease.cpp
@@ -68,6 +68,13 @@ pi_result redefinedKernelGetInfo(pi_kernel kernel, pi_kernel_info param_name,
   return PI_SUCCESS;
 }
 
+pi_result redefinedKernelSetExecInfo(pi_kernel kernel,
+                                     pi_kernel_exec_info param_name,
+                                     size_t param_value_size,
+                                     const void *param_value) {
+  return PI_SUCCESS;
+}
+
 TEST(KernelReleaseTest, GetKernelRelease) {
   platform Plt{default_selector()};
   if (Plt.is_host()) {
@@ -85,6 +92,8 @@ TEST(KernelReleaseTest, GetKernelRelease) {
   Mock.redefine<detail::PiApiKind::piKernelRetain>(redefinedKernelRetain);
   Mock.redefine<detail::PiApiKind::piKernelRelease>(redefinedKernelRelease);
   Mock.redefine<detail::PiApiKind::piKernelGetInfo>(redefinedKernelGetInfo);
+  Mock.redefine<detail::PiApiKind::piKernelSetExecInfo>(
+      redefinedKernelSetExecInfo);
 
   context Ctx{Plt};
   TestContext.reset(new TestCtx(Ctx));